Fix syntax highlighting, process spawning, extensions, terminals (#22)

* Fix syntax highlighting, process spawning, extensions, terminals

* Replace colons in toISOString

* Move pathSets included in task

packages/ide/src/fill/os.ts

@@ -33,6 +33,16 @@ class OS {
 		this._tmpdir = data.tmpDirectory;
 	}
 
+	public platform(): NodeJS.Platform {
+		if (navigator.appVersion.indexOf("Win") != -1) {
+			return "win32";
+		}
+		if (navigator.appVersion.indexOf("Mac") != -1) {
+			return "darwin";
+		}
+		return "linux";
+	}
+
 }
 
 export = new OS();

packages/protocol/src/browser/client.ts

@@ -161,7 +161,7 @@ export class Client {
 	 * @param options Options to execute for the command
 	 */
 	public spawn(command: string, args: string[] = [], options?: SpawnOptions): ChildProcess {
-		return this.doSpawn(command, args, options, false);
+		return this.doSpawn(command, args, options, false, false);
 	}
 
 	/**
@@ -272,6 +272,7 @@ export class Client {
 				tmpDirectory: init.getTmpDirectory(),
 				workingDirectory: init.getWorkingDirectory(),
 				os: opSys,
+				shell: init.getShell(),
 			};
 			this.initDataEmitter.emit(this._initData);
 		} else if (message.hasEvalDone()) {
@@ -316,7 +317,14 @@ export class Client {
 			if (!s) {
 				return;
 			}
-			s.pid = message.getIdentifySession()!.getPid();
+			const pid = message.getIdentifySession()!.getPid();
+			if (typeof pid !== "undefined") {
+				s.pid = pid;
+			}
+			const title = message.getIdentifySession()!.getTitle();
+			if (typeof title !== "undefined") {
+				s.title = title;
+			}
 		} else if (message.hasConnectionEstablished()) {
 			const c = this.connections.get(message.getConnectionEstablished()!.getId());
 			if (!c) {
@@ -347,6 +355,7 @@ export class Client {
 		} else if (message.hasSharedProcessActive()) {
 			this.sharedProcessActiveEmitter.emit({
 				socketPath: message.getSharedProcessActive()!.getSocketPath(),
+				logPath: message.getSharedProcessActive()!.getLogPath(),
 			});
 		} else if (message.hasServerEstablished()) {
 			const s = this.servers.get(message.getServerEstablished()!.getId());

packages/protocol/src/browser/command.ts

@@ -26,6 +26,7 @@ export interface ChildProcess {
 
 	readonly killed?: boolean;
 	readonly pid: number | undefined;
+	readonly title?: string;
 
 	kill(signal?: string): void;
 
@@ -45,6 +46,7 @@ export class ServerProcess extends events.EventEmitter implements ChildProcess {
 	public readonly stderr = new stream.Readable({ read: (): boolean => true });
 
 	private _pid: number | undefined;
+	private _title: string | undefined;
 	private _killed: boolean = false;
 	private _connected: boolean = false;
 
@@ -69,6 +71,14 @@ export class ServerProcess extends events.EventEmitter implements ChildProcess {
 		this._connected = true;
 	}
 
+	public get title(): string | undefined {
+		return this._title;
+	}
+
+	public set title(title: string | undefined) {
+		this._title = title;
+	}
+
 	public get connected(): boolean {
 		return this._connected;
 	}

packages/protocol/src/browser/modules/fs.ts

@@ -113,7 +113,7 @@ export class FS {
 	}
 
 	public createWriteStream = (): void => {
-		throw new Error("not implemented");
+		throw new Error("createWriteStream not implemented");
 	}
 
 	public exists = (path: fs.PathLike, callback: (exists: boolean) => void): void => {

packages/protocol/src/common/connection.ts

@@ -20,8 +20,10 @@ export interface InitData {
 	readonly workingDirectory: string;
 	readonly homeDirectory: string;
 	readonly tmpDirectory: string;
+	readonly shell: string;
 }
 
 export interface ISharedProcessData {
 	readonly socketPath: string;
+	readonly logPath: string;
 }

packages/protocol/src/node/command.ts

@@ -35,6 +35,7 @@ export const handleNewSession = (connection: SendableConnection, newSession: New
 	]);
 
 	let process: Process;
+	let processTitle: string | undefined;
 
 	const env: { [key: string]: string } = {};
 	newSession.getEnvMap().forEach((value, key) => {
@@ -42,12 +43,31 @@ export const handleNewSession = (connection: SendableConnection, newSession: New
 	});
 	if (newSession.getTtyDimensions()) {
 		// Spawn with node-pty
-		process = nodePty.spawn(newSession.getCommand(), newSession.getArgsList(), {
+		const ptyProc = nodePty.spawn(newSession.getCommand(), newSession.getArgsList(), {
 			cols: newSession.getTtyDimensions()!.getWidth(),
 			rows: newSession.getTtyDimensions()!.getHeight(),
 			cwd: newSession.getCwd(),
 			env,
 		});
+
+		const timer = setInterval(() => {
+			if (ptyProc.process !== processTitle) {
+				processTitle = ptyProc.process;
+				const id = new IdentifySessionMessage();
+				id.setId(newSession.getId());
+				id.setTitle(processTitle);
+				const sm = new ServerMessage();
+				sm.setIdentifySession(id);
+				connection.send(sm.serializeBinary());
+			}
+		}, 200);
+		
+		ptyProc.on("exit", () => {
+			clearTimeout(timer);
+		});
+
+		process = ptyProc;
+		processTitle = ptyProc.process;
 	} else {
 		const options = {
 			cwd: newSession.getCwd(),
@@ -129,6 +149,9 @@ export const handleNewSession = (connection: SendableConnection, newSession: New
 	const id = new IdentifySessionMessage();
 	id.setId(newSession.getId());
 	id.setPid(process.pid);
+	if (processTitle) {
+		id.setTitle(processTitle);
+	}
 	const sm = new ServerMessage();
 	sm.setIdentifySession(id);
 	connection.send(sm.serializeBinary());

packages/protocol/src/node/server.ts

@@ -86,6 +86,9 @@ export class Server {
 				throw new Error(`unrecognized platform "${platform}"`);
 		}
 		initMsg.setOperatingSystem(operatingSystem);
+		if (process.env.SHELL) {
+			initMsg.setShell(process.env.SHELL);
+		}
 		const srvMsg = new ServerMessage();
 		srvMsg.setInit(initMsg);
 		connection.send(srvMsg.serializeBinary());

packages/protocol/src/proto/client.proto

@@ -60,4 +60,5 @@ message WorkingInitMessage {
 		Mac = 2;
 	}
 	OperatingSystem operating_system = 5;
+	string shell = 6;
 }

packages/protocol/src/proto/client_pb.d.ts

@@ -253,6 +253,9 @@ export class WorkingInitMessage extends jspb.Message {
   getOperatingSystem(): WorkingInitMessage.OperatingSystem;
   setOperatingSystem(value: WorkingInitMessage.OperatingSystem): void;
 
+  getShell(): string;
+  setShell(value: string): void;
+
   serializeBinary(): Uint8Array;
   toObject(includeInstance?: boolean): WorkingInitMessage.AsObject;
   static toObject(includeInstance: boolean, msg: WorkingInitMessage): WorkingInitMessage.AsObject;
@@ -270,6 +273,7 @@ export namespace WorkingInitMessage {
     dataDirectory: string,
     workingDirectory: string,
     operatingSystem: WorkingInitMessage.OperatingSystem,
+    shell: string,
   }
 
   export enum OperatingSystem {

packages/protocol/src/proto/client_pb.js

@@ -1593,7 +1593,8 @@ proto.WorkingInitMessage.toObject = function(includeInstance, msg) {
     tmpDirectory: msg.getTmpDirectory(),
     dataDirectory: msg.getDataDirectory(),
     workingDirectory: msg.getWorkingDirectory(),
-    operatingSystem: msg.getOperatingSystem()
+    operatingSystem: msg.getOperatingSystem(),
+    shell: msg.getShell()
   };
 
   if (includeInstance) {
@@ -1650,6 +1651,10 @@ proto.WorkingInitMessage.deserializeBinaryFromReader = function(msg, reader) {
       var value = /** @type {!proto.WorkingInitMessage.OperatingSystem} */ (reader.readEnum());
       msg.setOperatingSystem(value);
       break;
+    case 6:
+      var value = /** @type {string} */ (reader.readString());
+      msg.setShell(value);
+      break;
     default:
       reader.skipField();
       break;
@@ -1723,6 +1728,13 @@ proto.WorkingInitMessage.prototype.serializeBinaryToWriter = function (writer) {
       f
     );
   }
+  f = this.getShell();
+  if (f.length > 0) {
+    writer.writeString(
+      6,
+      f
+    );
+  }
 };
 
 
@@ -1810,6 +1822,21 @@ proto.WorkingInitMessage.prototype.setOperatingSystem = function(value) {
 };
 
 
+/**
+ * optional string shell = 6;
+ * @return {string}
+ */
+proto.WorkingInitMessage.prototype.getShell = function() {
+  return /** @type {string} */ (jspb.Message.getFieldProto3(this, 6, ""));
+};
+
+
+/** @param {string} value  */
+proto.WorkingInitMessage.prototype.setShell = function(value) {
+  jspb.Message.setField(this, 6, value);
+};
+
+
 /**
  * @enum {number}
  */

packages/protocol/src/proto/command.proto

@@ -35,10 +35,12 @@ message SessionDoneMessage {
 	int64 exit_status = 2;
 }
 
-// Identifies a session with a PID.
+// Identifies a session with a PID and a title.
+// Can be sent multiple times when title changes.
 message IdentifySessionMessage {
 	uint64 id = 1;
 	uint64 pid = 2;
+	string title = 3;
 }
 
 // Writes data to a session.

packages/protocol/src/proto/command_pb.d.ts

@@ -118,6 +118,9 @@ export class IdentifySessionMessage extends jspb.Message {
   getPid(): number;
   setPid(value: number): void;
 
+  getTitle(): string;
+  setTitle(value: string): void;
+
   serializeBinary(): Uint8Array;
   toObject(includeInstance?: boolean): IdentifySessionMessage.AsObject;
   static toObject(includeInstance: boolean, msg: IdentifySessionMessage): IdentifySessionMessage.AsObject;
@@ -132,6 +135,7 @@ export namespace IdentifySessionMessage {
   export type AsObject = {
     id: number,
     pid: number,
+    title: string,
   }
 }
 

packages/protocol/src/proto/command_pb.js

@@ -867,7 +867,8 @@ proto.IdentifySessionMessage.prototype.toObject = function(opt_includeInstance)
 proto.IdentifySessionMessage.toObject = function(includeInstance, msg) {
   var f, obj = {
     id: msg.getId(),
-    pid: msg.getPid()
+    pid: msg.getPid(),
+    title: msg.getTitle()
   };
 
   if (includeInstance) {
@@ -912,6 +913,10 @@ proto.IdentifySessionMessage.deserializeBinaryFromReader = function(msg, reader)
       var value = /** @type {number} */ (reader.readUint64());
       msg.setPid(value);
       break;
+    case 3:
+      var value = /** @type {string} */ (reader.readString());
+      msg.setTitle(value);
+      break;
     default:
       reader.skipField();
       break;
@@ -964,6 +969,13 @@ proto.IdentifySessionMessage.prototype.serializeBinaryToWriter = function (write
       f
     );
   }
+  f = this.getTitle();
+  if (f.length > 0) {
+    writer.writeString(
+      3,
+      f
+    );
+  }
 };
 
 
@@ -1006,6 +1018,21 @@ proto.IdentifySessionMessage.prototype.setPid = function(value) {
 };
 
 
+/**
+ * optional string title = 3;
+ * @return {string}
+ */
+proto.IdentifySessionMessage.prototype.getTitle = function() {
+  return /** @type {string} */ (jspb.Message.getFieldProto3(this, 3, ""));
+};
+
+
+/** @param {string} value  */
+proto.IdentifySessionMessage.prototype.setTitle = function(value) {
+  jspb.Message.setField(this, 3, value);
+};
+
+
 
 /**
  * Generated by JsPbCodeGenerator.

packages/protocol/src/proto/vscode.proto

@@ -3,4 +3,5 @@ syntax = "proto3";
 // Sent when a shared process becomes active
 message SharedProcessActiveMessage {
 	string socket_path = 1;
+	string log_path = 2;
 }

packages/protocol/src/proto/vscode_pb.d.ts

@@ -7,6 +7,9 @@ export class SharedProcessActiveMessage extends jspb.Message {
   getSocketPath(): string;
   setSocketPath(value: string): void;
 
+  getLogPath(): string;
+  setLogPath(value: string): void;
+
   serializeBinary(): Uint8Array;
   toObject(includeInstance?: boolean): SharedProcessActiveMessage.AsObject;
   static toObject(includeInstance: boolean, msg: SharedProcessActiveMessage): SharedProcessActiveMessage.AsObject;
@@ -20,6 +23,7 @@ export class SharedProcessActiveMessage extends jspb.Message {
 export namespace SharedProcessActiveMessage {
   export type AsObject = {
     socketPath: string,
+    logPath: string,
   }
 }
 

packages/protocol/src/proto/vscode_pb.js

@@ -56,7 +56,8 @@ proto.SharedProcessActiveMessage.prototype.toObject = function(opt_includeInstan
  */
 proto.SharedProcessActiveMessage.toObject = function(includeInstance, msg) {
   var f, obj = {
-    socketPath: msg.getSocketPath()
+    socketPath: msg.getSocketPath(),
+    logPath: msg.getLogPath()
   };
 
   if (includeInstance) {
@@ -97,6 +98,10 @@ proto.SharedProcessActiveMessage.deserializeBinaryFromReader = function(msg, rea
       var value = /** @type {string} */ (reader.readString());
       msg.setSocketPath(value);
       break;
+    case 2:
+      var value = /** @type {string} */ (reader.readString());
+      msg.setLogPath(value);
+      break;
     default:
       reader.skipField();
       break;
@@ -142,6 +147,13 @@ proto.SharedProcessActiveMessage.prototype.serializeBinaryToWriter = function (w
       f
     );
   }
+  f = this.getLogPath();
+  if (f.length > 0) {
+    writer.writeString(
+      2,
+      f
+    );
+  }
 };
 
 
@@ -169,4 +181,19 @@ proto.SharedProcessActiveMessage.prototype.setSocketPath = function(value) {
 };
 
 
+/**
+ * optional string log_path = 2;
+ * @return {string}
+ */
+proto.SharedProcessActiveMessage.prototype.getLogPath = function() {
+  return /** @type {string} */ (jspb.Message.getFieldProto3(this, 2, ""));
+};
+
+
+/** @param {string} value  */
+proto.SharedProcessActiveMessage.prototype.setLogPath = function(value) {
+  jspb.Message.setField(this, 2, value);
+};
+
+
 goog.object.extend(exports, proto);

packages/server/package.json

@@ -21,7 +21,8 @@
 		"nexe": "^2.0.0-rc.34",
 		"node-pty": "^0.8.1",
 		"spdlog": "^0.7.2",
-		"ws": "^6.1.2"
+		"ws": "^6.1.2",
+		"xhr2": "^0.1.4"
 	},
 	"devDependencies": {
 		"@types/express": "^4.16.0",

packages/server/src/cli.ts

@@ -68,15 +68,24 @@ export class Entry extends Command {
 		const dataDir = flags["data-dir"] || path.join(os.homedir(), ".vscode-online");
 		const workingDir = args["workdir"];
 
+		if (process.env.BUILD_DIR && process.env.BUILD_DIR.startsWith(workingDir)) {
+			logger.error("Cannot run binary inside of BUILD_DIR", field("build_dir", process.env.BUILD_DIR), field("cwd", process.cwd()));
+			process.exit(1);
+		}
+
+		const logDir = path.join(dataDir, "logs", new Date().toISOString().replace(/[-:.TZ]/g, ""));
+		process.env.VSCODE_LOGS = logDir;
+
 		logger.info("\u001B[1mvscode-remote v1.0.0");
 		// TODO: fill in appropriate doc url
 		logger.info("Additional documentation: https://coder.com/docs");
-		logger.info("Initializing", field("data-dir", dataDir), field("working-dir", workingDir));
+		logger.info("Initializing", field("data-dir", dataDir), field("working-dir", workingDir), field("log-dir", logDir));
 		const sharedProcess = new SharedProcess(dataDir);
 		logger.info("Starting shared process...", field("socket", sharedProcess.socketPath));
 		const sendSharedProcessReady = (socket: WebSocket): void => {
 			const active = new SharedProcessActiveMessage();
 			active.setSocketPath(sharedProcess.socketPath);
+			active.setLogPath(logDir);
 			const serverMessage = new ServerMessage();
 			serverMessage.setSharedProcessActive(active);
 			socket.send(serverMessage.serializeBinary());

packages/server/src/server.ts

@@ -36,6 +36,9 @@ export const createApp = (registerMiddleware?: (app: express.Application) => voi
 			},
 			close: (): void => ws.close(),
 			send: (data): void => {
+				if (ws.readyState !== ws.OPEN) {
+					return;
+				}
 				try {
 					ws.send(data);
 				} catch (error) {
@@ -64,15 +67,15 @@ export const createApp = (registerMiddleware?: (app: express.Application) => voi
 		} : undefined);
 	});
 
-	app.use(express.static(path.join(__dirname, "../build/web")));
+	app.use(express.static(path.join(process.env.BUILD_DIR || path.join(__dirname, ".."), "build/web")));
 
 	app.get("/resource/:url(*)", async (req, res) => {
 		try {
 			const fullPath = `/${req.params.url}`;
-			const relative = path.relative(options!.dataDirectory, fullPath);
-			if (relative.startsWith("..")) {
-				return res.status(403).end();
-			}
+			// const relative = path.relative(options!.dataDirectory, fullPath);
+			// if (relative.startsWith("..")) {
+			// 	return res.status(403).end();
+			// }
 			const exists = await util.promisify(fs.exists)(fullPath);
 			if (!exists) {
 				res.status(404).end();

packages/server/src/vscode/bootstrapFork.ts

@@ -6,6 +6,10 @@ import * as path from "path";
 export const requireModule = (modulePath: string): void => {
 	process.env.AMD_ENTRYPOINT = modulePath;
 
+	const xml = require("xhr2");
+
+	(<any>global).XMLHttpRequest = xml.XMLHttpRequest;
+
 	// Always do this so we can see console.logs.
 	// process.env.VSCODE_ALLOW_IO = "true";
 

packages/server/src/vscode/sharedProcess.ts

@@ -68,6 +68,7 @@ export class SharedProcess {
 		let resolved: boolean = false;
 		this.activeProcess = forkModule("vs/code/electron-browser/sharedProcess/sharedProcessMain", {
 			VSCODE_ALLOW_IO: "true",
+			VSCODE_LOGS: process.env.VSCODE_LOGS,
 		});
 		this.activeProcess.on("exit", (err) => {
 			if (this._state !== SharedProcessState.Stopped) {

packages/server/yarn.lock

@@ -3689,6 +3689,11 @@ ws@^6.1.2:
   dependencies:
     async-limiter "~1.0.0"
 
+xhr2@^0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/xhr2/-/xhr2-0.1.4.tgz#7f87658847716db5026323812f818cadab387a5f"
+  integrity sha1-f4dliEdxbbUCYyOBL4GMras4el8=
+
 xtend@^4.0.0:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"

packages/vscode/package.json

@@ -7,7 +7,11 @@
   },
   "dependencies": {
     "iconv-lite": "^0.4.24",
+    "onigasm": "^2.2.1",
     "spdlog": "^0.7.2",
     "string-replace-loader": "^2.1.1"
+  },
+  "devDependencies": {
+    "vscode-textmate": "^4.0.1"
   }
 }

packages/vscode/src/client.ts

@@ -3,6 +3,7 @@ import * as paths from "./fill/paths";
 import "./fill/storageDatabase";
 import "./fill/windowsService";
 import "./fill/environmentService";
+import "./fill/vscodeTextmate";
 import "./fill/dom";
 import "./vscode.scss";
 
@@ -22,18 +23,21 @@ export class Client extends IDEClient {
 
 	public readonly protocolPromise: Promise<Protocol>;
 	public protoResolve: ((protocol: Protocol) => void) | undefined;
+	private readonly pathSets: Promise<void>;
 
 	public constructor() {
 		super();
 		this.protocolPromise = new Promise((resolve): void => {
 			this.protoResolve = resolve;
 		});
-		this.sharedProcessData.then((data) => {
+		this.pathSets = this.sharedProcessData.then((data) => {
 			paths._paths.socketPath = data.socketPath;
+			process.env.VSCODE_LOGS = data.logPath;
 		});
 		this.initData.then((data) => {
 			paths._paths.appData = data.dataDirectory;
 			paths._paths.defaultUserData = data.dataDirectory;
+			process.env.SHELL = data.shell;
 		});
 	}
 
@@ -53,7 +57,7 @@ export class Client extends IDEClient {
 				nodeCachedDataDir: data.tmpDirectory,
 				perfEntries: [],
 				_: [],
-				folderUri: URI.file(data.dataDirectory),
+				folderUri: URI.file(data.workingDirectory),
 			});
 
 			// TODO: Set notification service for retrying.
@@ -92,7 +96,7 @@ export class Client extends IDEClient {
 			// 	bounded.set(enabled);
 			// });
 			this.clipboard.initialize();
-		}, this.initData);
+		}, this.initData, this.pathSets);
 	}
 
 	protected createUriFactory(): IURIFactory {

packages/vscode/src/fill/node-pty.ts

@@ -1,6 +1,7 @@
-import * as cp from "child_process";
+import { client } from "@coder/ide/src/fill/client";
 import { EventEmitter } from "events";
 import * as nodePty from "node-pty";
+import { ChildProcess } from "@coder/protocol/src/browser/command";
 
 type nodePtyType = typeof nodePty;
 
@@ -10,38 +11,32 @@ type nodePtyType = typeof nodePty;
 class Pty implements nodePty.IPty {
 
 	private readonly emitter: EventEmitter;
+	private readonly cp: ChildProcess;
 
 	public constructor(file: string, args: string[] | string, options: nodePty.IPtyForkOptions) {
 		this.emitter = new EventEmitter();
-		const session = wush.execute({
-			command: `${file} ${Array.isArray(args) ? args.join(" ") : args}`,
-			directory: options.cwd,
-			environment: {
-				...(options.env || {}),
-				TERM: "xterm-color",
+		this.cp = client.spawn(file, Array.isArray(args) ? args : [args], {
+			...options,
+			tty: {
+				columns: options.cols || 100,
+				rows: options.rows || 100,
 			},
-			size: options && options.cols && options.rows ? {
-				columns: options.cols,
-				rows: options.rows,
-			} : {
-					columns: 100,
-					rows: 100,
-				},
 		});
-		this.on("write", (data) => session.sendStdin(data));
-		this.on("kill", (exitCode) => session.close());
-		this.on("resize", (columns, rows) => session.setSize({ columns, rows }));
-		session.onStdout((data) => this.emitter.emit("data", data));
-		session.onStderr((data) => this.emitter.emit("data", data));
-		session.onDone((exitCode) => this.emitter.emit("exit", exitCode));
+		this.on("write", (d) => this.cp.send(d));
+		this.on("kill", (exitCode) => this.cp.kill(exitCode));
+		this.on("resize", (cols, rows) => this.cp.resize!({ columns: cols, rows }));
+
+		this.cp.stdout.on("data", (data) => this.emitter.emit("data", data));
+		this.cp.stderr.on("data", (data) => this.emitter.emit("data", data));
+		this.cp.on("exit", (code) => this.emitter.emit("exit", code));
 	}
 
 	public get pid(): number {
-		return 1;
+		return this.cp.pid!;
 	}
 
 	public get process(): string {
-		return "unknown";
+		return this.cp.title!;
 	}
 
 	public on(event: string, listener: (...args) => void): void {
@@ -70,4 +65,4 @@ const ptyType: nodePtyType = {
 
 };
 
-exports = ptyType;
+module.exports = ptyType;

packages/vscode/src/fill/vscodeTextmate.ts

@@ -0,0 +1,35 @@
+import * as vscodeTextmate from "../../../../lib/vscode/node_modules/vscode-textmate";
+
+const target = vscodeTextmate as typeof vscodeTextmate;
+
+target.Registry = class Registry extends vscodeTextmate.Registry {
+	public constructor(opts: vscodeTextmate.RegistryOptions) {
+		super({
+			...opts,
+			getOnigLib: (): Promise<vscodeTextmate.IOnigLib> => {
+				return new Promise<vscodeTextmate.IOnigLib>((res, rej) => {
+					const onigasm = require('onigasm');
+					const wasmUrl = require('!!file-loader!onigasm/lib/onigasm.wasm');
+					return fetch(wasmUrl).then(resp => resp.arrayBuffer()).then(buffer => {
+						return onigasm.loadWASM(buffer);
+					}).then(() => {
+						res({
+							createOnigScanner: function (patterns) { return new onigasm.OnigScanner(patterns); },
+							createOnigString: function (s) { return new onigasm.OnigString(s); }
+						})
+					}).catch(reason => rej(reason));
+				});
+			},
+		});
+	}
+}
+
+enum StandardTokenType {
+	Other = 0,
+    Comment = 1,
+    String = 2,
+    RegEx = 4,
+};
+
+// Any needed here to override const
+(<any>target).StandardTokenType = StandardTokenType;

packages/vscode/webpack.config.bootstrap.js

@@ -43,9 +43,6 @@ module.exports = (env) => {
 						},
 					],
 				},
-			}, {
-				test: /\.wasm$/,
-				type: "javascript/auto",
 			}, {
 				// Ignore a bunch of file types we don't have loaders for. Also ignore
 				// test directories, some files with invalid JSON, and files we don't

packages/vscode/yarn.lock

@@ -70,6 +70,14 @@ loader-utils@^1.1.0:
     emojis-list "^2.0.0"
     json5 "^1.0.1"
 
+lru-cache@^4.1.1:
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
+  integrity sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==
+  dependencies:
+    pseudomap "^1.0.2"
+    yallist "^2.1.2"
+
 minimist@0.0.8:
   version "0.0.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
@@ -87,11 +95,30 @@ mkdirp@^0.5.1:
   dependencies:
     minimist "0.0.8"
 
-nan@^2.8.0:
+nan@^2.10.0, nan@^2.8.0:
   version "2.12.1"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.12.1.tgz#7b1aa193e9aa86057e3c7bbd0ac448e770925552"
   integrity sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==
 
+onigasm@^2.2.1:
+  version "2.2.1"
+  resolved "https://registry.yarnpkg.com/onigasm/-/onigasm-2.2.1.tgz#d56da809d63d3bb25510e8b8e447ffe98e56bebb"
+  integrity sha512-pa361CpVfsWOk0MQ1jLuJ1GvEJMHEHgZmaBpOIfBbvbp2crkDHacXB6mA4vgEfO7fL0OEMUSuZjX0Q9yTx6jTg==
+  dependencies:
+    lru-cache "^4.1.1"
+
+oniguruma@^7.0.0:
+  version "7.0.2"
+  resolved "https://registry.yarnpkg.com/oniguruma/-/oniguruma-7.0.2.tgz#a5c922cf7066da1dbcc60f6385a90437a83f8d0b"
+  integrity sha512-zCsdNxTrrB4yVPMxhcIODGv1p4NVBu9WvsWnIGhMpu5djO4MQWXrC7YKjtza+OyoRqqgy27CqYWa1h5e2DDbig==
+  dependencies:
+    nan "^2.10.0"
+
+pseudomap@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
+  integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM=
+
 punycode@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
@@ -133,3 +160,15 @@ uri-js@^4.2.2:
   integrity sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==
   dependencies:
     punycode "^2.1.0"
+
+vscode-textmate@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-4.0.1.tgz#6c36f28e9059ce12bc34907f7a33ea43166b26a8"
+  integrity sha512-gHTXTj04TUgbjB8y7pkVwxOiuCuD6aU5gnFzIByQuqdgFpe/bJaaEIS4geGjbjWbd1XJh6zG1EthLfpNaXEqUw==
+  dependencies:
+    oniguruma "^7.0.0"
+
+yallist@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
+  integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=

packages/web/webpack.common.config.js

@@ -58,6 +58,7 @@ module.exports = merge({
 			"dns": path.join(fills, "empty.ts"),
 			"console": path.join(fills, "empty.ts"),
 			"readline": path.join(fills, "empty.ts"),
+			"oniguruma": path.join(fills, "empty.ts"),
 
 			"crypto": "crypto-browserify",
 			"http": "http-browserify",

packages/web/webpack.dev.config.js

@@ -1,6 +1,7 @@
 const path = require("path");
 const webpack = require("webpack");
 const merge = require("webpack-merge");
+const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
 
 module.exports = merge(require("./webpack.common.config.js"), {
 	devtool: "cheap-module-eval-source-map",
@@ -13,5 +14,6 @@ module.exports = merge(require("./webpack.common.config.js"), {
 	],
 	plugins: [
 		new webpack.HotModuleReplacementPlugin(),
+		// new BundleAnalyzerPlugin(),
 	]
 });

scripts/webpack.general.config.js

@@ -77,7 +77,7 @@ module.exports = (options = {}) => ({
 	plugins: [
 		new HappyPack({
 			id: "ts",
-			threads: 2,
+			threads: 10,
 			loaders: [{
 				path: "ts-loader",
 				query: {