cubetiq/code-server
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor evaluations (#285)

Browse Source 
* Replace evaluations with proxies and messages

* Return proxies synchronously

Otherwise events can be lost.

* Ensure events cannot be missed

* Refactor remaining fills

* Use more up-to-date version of util

For callbackify.

* Wait for dispose to come back before removing

This prevents issues with the "done" event not always being the last
event fired. For example a socket might close and then end, but only
if the caller called end.

* Remove old node-pty tests

* Fix emitting events twice on duplex streams

* Preserve environment when spawning processes

* Throw a better error if the proxy doesn't exist

* Remove rimraf dependency from ide

* Update net.Server.listening

* Use exit event instead of killed

Doesn't look like killed is even a thing.

* Add response timeout to server

* Fix trash

* Require node-pty & spdlog after they get unpackaged

This fixes an error when running in the binary.

* Fix errors in down emitter preventing reconnecting

* Fix disposing proxies when nothing listens to "error" event

* Refactor event tests to use jest.fn()

* Reject proxy call when disconnected

Otherwise it'll wait for the timeout which is a waste of time since we
already know the connection is dead.

* Use nbin for binary packaging

* Remove additional module requires

* Attempt to remove require for local bootstrap-fork

* Externalize fsevents
This commit is contained in:
Asher
2019-03-26 13:01:25 -05:00
committed by Kyle Carberry
parent d16c6aeb30
commit dc2253e718
75 changed files with 5866 additions and 6181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
packages/protocol/src/node/evaluate.ts
View File

@@ -1,157 +0,0 @@
import { fork as cpFork } from "child_process";
import { EventEmitter } from "events";
import * as vm from "vm";
import { logger, field } from "@coder/logger";
import { NewEvalMessage, EvalFailedMessage, EvalDoneMessage, ServerMessage, EvalEventMessage } from "../proto";
import { SendableConnection } from "../common/connection";
import { ServerActiveEvalHelper, EvalHelper, ForkProvider, Modules } from "../common/helpers";
import { stringify, parse } from "../common/util";
export interface ActiveEvaluation {
onEvent(msg: EvalEventMessage): void;
dispose(): void;
}
declare var __non_webpack_require__: typeof require;
export const evaluate = (connection: SendableConnection, message: NewEvalMessage, onDispose: () => void, fork?: ForkProvider): ActiveEvaluation | void => {
/**
* Send the response and call onDispose.
*/
// tslint:disable-next-line no-any
const sendResp = (resp: any): void => {
logger.trace(() => [
"resolve",
field("id", message.getId()),
field("response", stringify(resp)),
]);
const evalDone = new EvalDoneMessage();
evalDone.setId(message.getId());
evalDone.setResponse(stringify(resp));
const serverMsg = new ServerMessage();
serverMsg.setEvalDone(evalDone);
connection.send(serverMsg.serializeBinary());
onDispose();
};
/**
* Send an exception and call onDispose.
*/
const sendException = (error: Error): void => {
logger.trace(() => [
"reject",
field("id", message.getId()),
field("response", stringify(error, true)),
]);
const evalFailed = new EvalFailedMessage();
evalFailed.setId(message.getId());
evalFailed.setResponse(stringify(error, true));
const serverMsg = new ServerMessage();
serverMsg.setEvalFailed(evalFailed);
connection.send(serverMsg.serializeBinary());
onDispose();
};
const modules: Modules = {
spdlog: require("spdlog"),
pty: require("node-pty-prebuilt"),
trash: require("trash"),
};
let eventEmitter = message.getActive() ? new EventEmitter(): undefined;
const sandbox = {
helper: eventEmitter ? new ServerActiveEvalHelper(modules, {
removeAllListeners: (event?: string): void => {
eventEmitter!.removeAllListeners(event);
},
// tslint:disable no-any
on: (event: string, cb: (...args: any[]) => void): void => {
eventEmitter!.on(event, (...args: any[]) => {
logger.trace(() => [
`${event}`,
field("id", message.getId()),
field("args", args.map((a) => stringify(a))),
]);
cb(...args);
});
},
emit: (event: string, ...args: any[]): void => {
logger.trace(() => [
`emit ${event}`,
field("id", message.getId()),
field("args", args.map((a) => stringify(a))),
]);
const eventMsg = new EvalEventMessage();
eventMsg.setEvent(event);
eventMsg.setArgsList(args.map((a) => stringify(a)));
eventMsg.setId(message.getId());
const serverMsg = new ServerMessage();
serverMsg.setEvalEvent(eventMsg);
connection.send(serverMsg.serializeBinary());
},
// tslint:enable no-any
}, fork || cpFork) : new EvalHelper(modules),
_Buffer: Buffer,
// When the client is ran from Webpack, it will replace
// __non_webpack_require__ with require, which we then need to provide to
// the sandbox. Since the server might also be using Webpack, we need to set
// it to the non-Webpack version when that's the case. Then we need to also
// provide __non_webpack_require__ for when the client doesn't run through
// Webpack meaning it doesn't get replaced with require (Jest for example).
require: typeof __non_webpack_require__ !== "undefined" ? __non_webpack_require__ : require,
__non_webpack_require__: typeof __non_webpack_require__ !== "undefined" ? __non_webpack_require__ : require,
setTimeout,
setInterval,
clearTimeout,
process: {
env: process.env,
},
args: message.getArgsList().map(parse),
};
let value: any; // tslint:disable-line no-any
try {
const code = `(${message.getFunction()})(helper, ...args);`;
value = vm.runInNewContext(code, sandbox, {
// If the code takes longer than this to return, it is killed and throws.
timeout: message.getTimeout() || 15000,
});
} catch (ex) {
sendException(ex);
}
// An evaluation completes when the value it returns resolves. An active
// evaluation completes when it is disposed. Active evaluations are required
// to return disposers so we can know both when it has ended (so we can clean
// up on our end) and how to force end it (for example when the client
// disconnects).
// tslint:disable-next-line no-any
const promise = !eventEmitter ? value as Promise<any> : new Promise((resolve): void => {
value.onDidDispose(resolve);
});
if (promise && promise.then) {
promise.then(sendResp).catch(sendException);
} else {
sendResp(value);
}
return eventEmitter ? {
onEvent: (eventMsg: EvalEventMessage): void => {
eventEmitter!.emit(eventMsg.getEvent(), ...eventMsg.getArgsList().map(parse));
},
dispose: (): void => {
if (eventEmitter) {
if (value && value.dispose) {
value.dispose();
}
eventEmitter.removeAllListeners();
eventEmitter = undefined;
}
},
} : undefined;
};

103
packages/protocol/src/node/modules/child_process.ts Normal file
View File

@@ -0,0 +1,103 @@
import * as cp from "child_process";
import { ServerProxy } from "../../common/proxy";
import { preserveEnv } from "../../common/util";
import { WritableProxy, ReadableProxy } from "./stream";
export type ForkProvider = (modulePath: string, args?: string[], options?: cp.ForkOptions) => cp.ChildProcess;
export class ChildProcessProxy implements ServerProxy {
public constructor(private readonly process: cp.ChildProcess) {}
public async kill(signal?: string): Promise<void> {
this.process.kill(signal);
}
public async disconnect(): Promise<void> {
this.process.disconnect();
}
public async ref(): Promise<void> {
this.process.ref();
}
public async unref(): Promise<void> {
this.process.unref();
}
// tslint:disable-next-line no-any
public async send(message: any): Promise<void> {
return new Promise((resolve, reject) => {
this.process.send(message, (error) => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
}
public async getPid(): Promise<number> {
return this.process.pid;
}
public async onDone(cb: () => void): Promise<void> {
this.process.on("close", cb);
}
public async dispose(): Promise<void> {
this.kill();
setTimeout(() => this.kill("SIGKILL"), 5000); // Double tap.
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
this.process.on("close", (code, signal) => cb("close", code, signal));
this.process.on("disconnect", () => cb("disconnect"));
this.process.on("error", (error) => cb("error", error));
this.process.on("exit", (exitCode, signal) => cb("exit", exitCode, signal));
this.process.on("message", (message) => cb("message", message));
}
}
export interface ChildProcessProxies {
childProcess: ChildProcessProxy;
stdin?: WritableProxy;
stdout?: ReadableProxy;
stderr?: ReadableProxy;
}
export class ChildProcessModuleProxy {
public constructor(private readonly forkProvider?: ForkProvider) {}
public async exec(
command: string,
options?: { encoding?: string | null } & cp.ExecOptions | null,
callback?: ((error: cp.ExecException | null, stdin: string | Buffer, stdout: string | Buffer) => void),
): Promise<ChildProcessProxies> {
preserveEnv(options);
return this.returnProxies(cp.exec(command, options, callback));
}
public async fork(modulePath: string, args?: string[], options?: cp.ForkOptions): Promise<ChildProcessProxies> {
preserveEnv(options);
return this.returnProxies((this.forkProvider || cp.fork)(modulePath, args, options));
}
public async spawn(command: string, args?: string[], options?: cp.SpawnOptions): Promise<ChildProcessProxies> {
preserveEnv(options);
return this.returnProxies(cp.spawn(command, args, options));
}
private returnProxies(process: cp.ChildProcess): ChildProcessProxies {
return {
childProcess: new ChildProcessProxy(process),
stdin: process.stdin && new WritableProxy(process.stdin),
stdout: process.stdout && new ReadableProxy(process.stdout),
stderr: process.stderr && new ReadableProxy(process.stderr),
};
}
}

250
packages/protocol/src/node/modules/fs.ts Normal file
View File

@@ -0,0 +1,250 @@
import * as fs from "fs";
import { promisify } from "util";
import { ServerProxy } from "../../common/proxy";
import { IEncodingOptions } from "../../common/util";
import { WritableProxy } from "./stream";
/**
* A serializable version of fs.Stats.
*/
export interface Stats {
dev: number;
ino: number;
mode: number;
nlink: number;
uid: number;
gid: number;
rdev: number;
size: number;
blksize: number;
blocks: number;
atimeMs: number;
mtimeMs: number;
ctimeMs: number;
birthtimeMs: number;
atime: Date | string;
mtime: Date | string;
ctime: Date | string;
birthtime: Date | string;
_isFile: boolean;
_isDirectory: boolean;
_isBlockDevice: boolean;
_isCharacterDevice: boolean;
_isSymbolicLink: boolean;
_isFIFO: boolean;
_isSocket: boolean;
}
export class WriteStreamProxy extends WritableProxy<fs.WriteStream> {
public async close(): Promise<void> {
this.stream.close();
}
public async dispose(): Promise<void> {
super.dispose();
this.stream.close();
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
super.onEvent(cb);
this.stream.on("open", (fd) => cb("open", fd));
}
}
export class WatcherProxy implements ServerProxy {
public constructor(private readonly watcher: fs.FSWatcher) {}
public async close(): Promise<void> {
this.watcher.close();
}
public async dispose(): Promise<void> {
this.watcher.close();
this.watcher.removeAllListeners();
}
public async onDone(cb: () => void): Promise<void> {
this.watcher.on("close", cb);
this.watcher.on("error", cb);
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
this.watcher.on("change", (event, filename) => cb("change", event, filename));
this.watcher.on("close", () => cb("close"));
this.watcher.on("error", (error) => cb("error", error));
}
}
export class FsModuleProxy {
public access(path: fs.PathLike, mode?: number): Promise<void> {
return promisify(fs.access)(path, mode);
}
// tslint:disable-next-line no-any
public appendFile(file: fs.PathLike | number, data: any, options?: fs.WriteFileOptions): Promise<void> {
return promisify(fs.appendFile)(file, data, options);
}
public chmod(path: fs.PathLike, mode: string | number): Promise<void> {
return promisify(fs.chmod)(path, mode);
}
public chown(path: fs.PathLike, uid: number, gid: number): Promise<void> {
return promisify(fs.chown)(path, uid, gid);
}
public close(fd: number): Promise<void> {
return promisify(fs.close)(fd);
}
public copyFile(src: fs.PathLike, dest: fs.PathLike, flags?: number): Promise<void> {
return promisify(fs.copyFile)(src, dest, flags);
}
// tslint:disable-next-line no-any
public async createWriteStream(path: fs.PathLike, options?: any): Promise<WriteStreamProxy> {
return new WriteStreamProxy(fs.createWriteStream(path, options));
}
public exists(path: fs.PathLike): Promise<boolean> {
return promisify(fs.exists)(path);
}
public fchmod(fd: number, mode: string | number): Promise<void> {
return promisify(fs.fchmod)(fd, mode);
}
public fchown(fd: number, uid: number, gid: number): Promise<void> {
return promisify(fs.fchown)(fd, uid, gid);
}
public fdatasync(fd: number): Promise<void> {
return promisify(fs.fdatasync)(fd);
}
public async fstat(fd: number): Promise<Stats> {
return this.makeStatsSerializable(await promisify(fs.fstat)(fd));
}
public fsync(fd: number): Promise<void> {
return promisify(fs.fsync)(fd);
}
public ftruncate(fd: number, len?: number | null): Promise<void> {
return promisify(fs.ftruncate)(fd, len);
}
public futimes(fd: number, atime: string | number | Date, mtime: string | number | Date): Promise<void> {
return promisify(fs.futimes)(fd, atime, mtime);
}
public lchmod(path: fs.PathLike, mode: string | number): Promise<void> {
return promisify(fs.lchmod)(path, mode);
}
public lchown(path: fs.PathLike, uid: number, gid: number): Promise<void> {
return promisify(fs.lchown)(path, uid, gid);
}
public link(existingPath: fs.PathLike, newPath: fs.PathLike): Promise<void> {
return promisify(fs.link)(existingPath, newPath);
}
public async lstat(path: fs.PathLike): Promise<Stats> {
return this.makeStatsSerializable(await promisify(fs.lstat)(path));
}
public mkdir(path: fs.PathLike, mode: number | string | fs.MakeDirectoryOptions | undefined | null): Promise<void> {
return promisify(fs.mkdir)(path, mode);
}
public mkdtemp(prefix: string, options: IEncodingOptions): Promise<string | Buffer> {
return promisify(fs.mkdtemp)(prefix, options);
}
public open(path: fs.PathLike, flags: string | number, mode: string | number | undefined | null): Promise<number> {
return promisify(fs.open)(path, flags, mode);
}
public read(fd: number, length: number, position: number | null): Promise<{ bytesRead: number, buffer: Buffer }> {
const buffer = new Buffer(length);
return promisify(fs.read)(fd, buffer, 0, length, position);
}
public readFile(path: fs.PathLike | number, options: IEncodingOptions): Promise<string | Buffer> {
return promisify(fs.readFile)(path, options);
}
public readdir(path: fs.PathLike, options: IEncodingOptions): Promise<Buffer[] | fs.Dirent[] | string[]> {
return promisify(fs.readdir)(path, options);
}
public readlink(path: fs.PathLike, options: IEncodingOptions): Promise<string | Buffer> {
return promisify(fs.readlink)(path, options);
}
public realpath(path: fs.PathLike, options: IEncodingOptions): Promise<string | Buffer> {
return promisify(fs.realpath)(path, options);
}
public rename(oldPath: fs.PathLike, newPath: fs.PathLike): Promise<void> {
return promisify(fs.rename)(oldPath, newPath);
}
public rmdir(path: fs.PathLike): Promise<void> {
return promisify(fs.rmdir)(path);
}
public async stat(path: fs.PathLike): Promise<Stats> {
return this.makeStatsSerializable(await promisify(fs.stat)(path));
}
public symlink(target: fs.PathLike, path: fs.PathLike, type?: fs.symlink.Type | null): Promise<void> {
return promisify(fs.symlink)(target, path, type);
}
public truncate(path: fs.PathLike, len?: number | null): Promise<void> {
return promisify(fs.truncate)(path, len);
}
public unlink(path: fs.PathLike): Promise<void> {
return promisify(fs.unlink)(path);
}
public utimes(path: fs.PathLike, atime: string | number | Date, mtime: string | number | Date): Promise<void> {
return promisify(fs.utimes)(path, atime, mtime);
}
public async write(fd: number, buffer: Buffer, offset?: number, length?: number, position?: number): Promise<{ bytesWritten: number, buffer: Buffer }> {
return promisify(fs.write)(fd, buffer, offset, length, position);
}
// tslint:disable-next-line no-any
public writeFile (path: fs.PathLike | number, data: any, options: IEncodingOptions): Promise<void> {
return promisify(fs.writeFile)(path, data, options);
}
public async watch(filename: fs.PathLike, options?: IEncodingOptions): Promise<WatcherProxy> {
return new WatcherProxy(fs.watch(filename, options));
}
private makeStatsSerializable(stats: fs.Stats): Stats {
return {
...stats,
/**
* We need to check if functions exist because nexe's implemented FS
* lib doesnt implement fs.stats properly.
*/
_isBlockDevice: stats.isBlockDevice ? stats.isBlockDevice() : false,
_isCharacterDevice: stats.isCharacterDevice ? stats.isCharacterDevice() : false,
_isDirectory: stats.isDirectory(),
_isFIFO: stats.isFIFO ? stats.isFIFO() : false,
_isFile: stats.isFile(),
_isSocket: stats.isSocket ? stats.isSocket() : false,
_isSymbolicLink: stats.isSymbolicLink ? stats.isSymbolicLink() : false,
};
}
}

6
packages/protocol/src/node/modules/index.ts Normal file
View File

@@ -0,0 +1,6 @@
export * from "./child_process";
export * from "./fs";
export * from "./net";
export * from "./node-pty";
export * from "./spdlog";
export * from "./trash";

90
packages/protocol/src/node/modules/net.ts Normal file
View File

@@ -0,0 +1,90 @@
import * as net from "net";
import { ServerProxy } from "../../common/proxy";
import { DuplexProxy } from "./stream";
export class NetSocketProxy extends DuplexProxy<net.Socket> {
public async connect(options: number | string | net.SocketConnectOpts, host?: string): Promise<void> {
this.stream.connect(options as any, host as any); // tslint:disable-line no-any this works fine
}
public async unref(): Promise<void> {
this.stream.unref();
}
public async ref(): Promise<void> {
this.stream.ref();
}
public async dispose(): Promise<void> {
this.stream.removeAllListeners();
this.stream.end();
this.stream.destroy();
this.stream.unref();
}
public async onDone(cb: () => void): Promise<void> {
this.stream.on("close", cb);
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
super.onEvent(cb);
this.stream.on("connect", () => cb("connect"));
this.stream.on("lookup", (error, address, family, host) => cb("lookup", error, address, family, host));
this.stream.on("timeout", () => cb("timeout"));
}
}
export class NetServerProxy implements ServerProxy {
public constructor(private readonly server: net.Server) {}
public async listen(handle?: net.ListenOptions | number | string, hostname?: string | number, backlog?: number): Promise<void> {
this.server.listen(handle, hostname as any, backlog as any); // tslint:disable-line no-any this is fine
}
public async ref(): Promise<void> {
this.server.ref();
}
public async unref(): Promise<void> {
this.server.unref();
}
public async close(): Promise<void> {
this.server.close();
}
public async onConnection(cb: (proxy: NetSocketProxy) => void): Promise<void> {
this.server.on("connection", (socket) => cb(new NetSocketProxy(socket)));
}
public async dispose(): Promise<void> {
this.server.close();
this.server.removeAllListeners();
}
public async onDone(cb: () => void): Promise<void> {
this.server.on("close", cb);
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
this.server.on("close", () => cb("close"));
this.server.on("error", (error) => cb("error", error));
this.server.on("listening", () => cb("listening"));
}
}
export class NetModuleProxy {
public async createSocket(options?: net.SocketConstructorOpts): Promise<NetSocketProxy> {
return new NetSocketProxy(new net.Socket(options));
}
public async createConnection(target: string | number | net.NetConnectOpts, host?: string): Promise<NetSocketProxy> {
return new NetSocketProxy(net.createConnection(target as any, host)); // tslint:disable-line no-any defeat stubborness
}
public async createServer(options?: { allowHalfOpen?: boolean, pauseOnConnect?: boolean }): Promise<NetServerProxy> {
return new NetServerProxy(net.createServer(options));
}
}

75
packages/protocol/src/node/modules/node-pty.ts Normal file
View File

@@ -0,0 +1,75 @@
/// <reference path="../../../../../lib/vscode/src/typings/node-pty.d.ts" />
import { EventEmitter } from "events";
import * as pty from "node-pty";
import { ServerProxy } from "../../common/proxy";
import { preserveEnv } from "../../common/util";
/**
* Server-side IPty proxy.
*/
export class NodePtyProcessProxy implements ServerProxy {
private readonly emitter = new EventEmitter();
public constructor(private readonly process: pty.IPty) {
let name = process.process;
setTimeout(() => { // Need to wait for the caller to listen to the event.
this.emitter.emit("process", name);
}, 1);
const timer = setInterval(() => {
if (process.process !== name) {
name = process.process;
this.emitter.emit("process", name);
}
}, 200);
this.onDone(() => clearInterval(timer));
}
public async getPid(): Promise<number> {
return this.process.pid;
}
public async getProcess(): Promise<string> {
return this.process.process;
}
public async kill(signal?: string): Promise<void> {
this.process.kill(signal);
}
public async resize(columns: number, rows: number): Promise<void> {
this.process.resize(columns, rows);
}
public async write(data: string): Promise<void> {
this.process.write(data);
}
public async onDone(cb: () => void): Promise<void> {
this.process.on("exit", cb);
}
public async dispose(): Promise<void> {
this.kill();
setTimeout(() => this.kill("SIGKILL"), 5000); // Double tap.
this.emitter.removeAllListeners();
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
this.emitter.on("process", (process) => cb("process", process));
this.process.on("data", (data) => cb("data", data));
this.process.on("exit", (exitCode, signal) => cb("exit", exitCode, signal));
}
}
/**
* Server-side node-pty proxy.
*/
export class NodePtyModuleProxy {
public async spawn(file: string, args: string[] | string, options: pty.IPtyForkOptions): Promise<NodePtyProcessProxy> {
preserveEnv(options);
return new NodePtyProcessProxy(require("node-pty").spawn(file, args, options));
}
}

46
packages/protocol/src/node/modules/spdlog.ts Normal file
View File

@@ -0,0 +1,46 @@
/// <reference path="../../../../../lib/vscode/src/typings/spdlog.d.ts" />
import { EventEmitter } from "events";
import * as spdlog from "spdlog";
import { ServerProxy } from "../../common/proxy";
export class RotatingLoggerProxy implements ServerProxy {
private readonly emitter = new EventEmitter();
public constructor(private readonly logger: spdlog.RotatingLogger) {}
public async trace (message: string): Promise<void> { this.logger.trace(message); }
public async debug (message: string): Promise<void> { this.logger.debug(message); }
public async info (message: string): Promise<void> { this.logger.info(message); }
public async warn (message: string): Promise<void> { this.logger.warn(message); }
public async error (message: string): Promise<void> { this.logger.error(message); }
public async critical (message: string): Promise<void> { this.logger.critical(message); }
public async setLevel (level: number): Promise<void> { this.logger.setLevel(level); }
public async clearFormatters (): Promise<void> { this.logger.clearFormatters(); }
public async flush (): Promise<void> { this.logger.flush(); }
public async drop (): Promise<void> { this.logger.drop(); }
public async onDone(cb: () => void): Promise<void> {
this.emitter.on("dispose", cb);
}
public async dispose(): Promise<void> {
this.flush();
this.emitter.emit("dispose");
this.emitter.removeAllListeners();
}
// tslint:disable-next-line no-any
public async onEvent(_cb: (event: string, ...args: any[]) => void): Promise<void> {
// No events.
}
}
export class SpdlogModuleProxy {
public async createLogger(name: string, filePath: string, fileSize: number, fileCount: number): Promise<RotatingLoggerProxy> {
return new RotatingLoggerProxy(new (require("spdlog") as typeof import("spdlog")).RotatingLogger(name, filePath, fileSize, fileCount));
}
public async setAsyncMode(bufferSize: number, flushInterval: number): Promise<void> {
require("spdlog").setAsyncMode(bufferSize, flushInterval);
}
}

107
packages/protocol/src/node/modules/stream.ts Normal file
View File

@@ -0,0 +1,107 @@
import * as stream from "stream";
import { ServerProxy } from "../../common/proxy";
export class WritableProxy<T extends stream.Writable = stream.Writable> implements ServerProxy {
public constructor(protected readonly stream: T) {}
public async destroy(): Promise<void> {
this.stream.destroy();
}
// tslint:disable-next-line no-any
public async end(data?: any, encoding?: string): Promise<void> {
return new Promise((resolve): void => {
this.stream.end(data, encoding, () => {
resolve();
});
});
}
public async setDefaultEncoding(encoding: string): Promise<void> {
this.stream.setDefaultEncoding(encoding);
}
// tslint:disable-next-line no-any
public async write(data: any, encoding?: string): Promise<void> {
return new Promise((resolve, reject): void => {
this.stream.write(data, encoding, (error) => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
}
public async dispose(): Promise<void> {
this.stream.end();
this.stream.removeAllListeners();
}
public async onDone(cb: () => void): Promise<void> {
this.stream.on("close", cb);
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
// Sockets have an extra argument on "close".
// tslint:disable-next-line no-any
this.stream.on("close", (...args: any[]) => cb("close", ...args));
this.stream.on("drain", () => cb("drain"));
this.stream.on("error", (error) => cb("error", error));
this.stream.on("finish", () => cb("finish"));
}
}
/**
* This noise is because we can't do multiple extends and we also can't seem to
* do `extends WritableProxy<T> implement ReadableProxy<T>` (for `DuplexProxy`).
*/
export interface IReadableProxy extends ServerProxy {
destroy(): Promise<void>;
setEncoding(encoding: string): Promise<void>;
dispose(): Promise<void>;
onDone(cb: () => void): Promise<void>;
}
export class ReadableProxy<T extends stream.Readable = stream.Readable> implements IReadableProxy {
public constructor(protected readonly stream: T) {}
public async destroy(): Promise<void> {
this.stream.destroy();
}
public async setEncoding(encoding: string): Promise<void> {
this.stream.setEncoding(encoding);
}
public async dispose(): Promise<void> {
this.stream.destroy();
}
public async onDone(cb: () => void): Promise<void> {
this.stream.on("close", cb);
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
this.stream.on("close", () => cb("close"));
this.stream.on("data", (chunk) => cb("data", chunk));
this.stream.on("end", () => cb("end"));
this.stream.on("error", (error) => cb("error", error));
}
}
export class DuplexProxy<T extends stream.Duplex = stream.Duplex> extends WritableProxy<T> implements IReadableProxy {
public async setEncoding(encoding: string): Promise<void> {
this.stream.setEncoding(encoding);
}
// tslint:disable-next-line no-any
public async onEvent(cb: (event: string, ...args: any[]) => void): Promise<void> {
super.onEvent(cb);
this.stream.on("data", (chunk) => cb("data", chunk));
this.stream.on("end", () => cb("end"));
}
}

7
packages/protocol/src/node/modules/trash.ts Normal file
View File

@@ -0,0 +1,7 @@
import * as trash from "trash";
export class TrashModuleProxy {
public async trash(path: string, options?: trash.Options): Promise<void> {
return trash(path, options);
}
}

333
packages/protocol/src/node/server.ts
View File

@@ -1,10 +1,13 @@
import { mkdirp } from "fs-extra";
import * as os from "os";
import { logger, field } from "@coder/logger";
import { Pong, ClientMessage, WorkingInitMessage, ServerMessage } from "../proto";
import { evaluate, ActiveEvaluation } from "./evaluate";
import { ForkProvider } from "../common/helpers";
import { field, logger} from "@coder/logger";
import { ReadWriteConnection } from "../common/connection";
import { Module, ServerProxy } from "../common/proxy";
import { isPromise, isProxy, moduleToProto, parse, platformToProto, protoToModule, stringify } from "../common/util";
import { CallbackMessage, ClientMessage, EventMessage, FailMessage, MethodMessage, NamedCallbackMessage, NamedEventMessage, NumberedCallbackMessage, NumberedEventMessage, Pong, ServerMessage, SuccessMessage, WorkingInitMessage } from "../proto";
import { ChildProcessModuleProxy, ForkProvider, FsModuleProxy, NetModuleProxy, NodePtyModuleProxy, SpdlogModuleProxy, TrashModuleProxy } from "./modules";
// tslint:disable no-any
export interface ServerOptions {
readonly workingDirectory: string;
@@ -14,27 +17,59 @@ export interface ServerOptions {
readonly fork?: ForkProvider;
}
interface ProxyData {
disposeTimeout?: number | NodeJS.Timer;
instance: any;
}
export class Server {
private readonly evals = new Map<number, ActiveEvaluation>();
private proxyId = 0;
private readonly proxies = new Map<number | Module, ProxyData>();
private disconnected: boolean = false;
private responseTimeout = 10000;
public constructor(
private readonly connection: ReadWriteConnection,
private readonly options?: ServerOptions,
) {
connection.onMessage((data) => {
connection.onMessage(async (data) => {
try {
this.handleMessage(ClientMessage.deserializeBinary(data));
await this.handleMessage(ClientMessage.deserializeBinary(data));
} catch (ex) {
logger.error("Failed to handle client message", field("length", data.byteLength), field("exception", {
message: ex.message,
stack: ex.stack,
}));
logger.error(
"Failed to handle client message",
field("length", data.byteLength),
field("exception", {
message: ex.message,
stack: ex.stack,
}),
);
}
});
connection.onClose(() => {
this.evals.forEach((e) => e.dispose());
this.disconnected = true;
logger.trace(() => [
"disconnected from client",
field("proxies", this.proxies.size),
]);
this.proxies.forEach((proxy, proxyId) => {
if (isProxy(proxy.instance)) {
proxy.instance.dispose();
}
this.removeProxy(proxyId);
});
});
this.storeProxy(new ChildProcessModuleProxy(this.options ? this.options.fork : undefined), Module.ChildProcess);
this.storeProxy(new FsModuleProxy(), Module.Fs);
this.storeProxy(new NetModuleProxy(), Module.Net);
this.storeProxy(new NodePtyModuleProxy(), Module.NodePty);
this.storeProxy(new SpdlogModuleProxy(), Module.Spdlog);
this.storeProxy(new TrashModuleProxy(), Module.Trash);
if (!this.options) {
logger.warn("No server options provided. InitMessage will not be sent.");
@@ -55,53 +90,19 @@ export class Server {
initMsg.setBuiltinExtensionsDir(this.options.builtInExtensionsDirectory);
initMsg.setHomeDirectory(os.homedir());
initMsg.setTmpDirectory(os.tmpdir());
const platform = os.platform();
let operatingSystem: WorkingInitMessage.OperatingSystem;
switch (platform) {
case "win32":
operatingSystem = WorkingInitMessage.OperatingSystem.WINDOWS;
break;
case "linux":
operatingSystem = WorkingInitMessage.OperatingSystem.LINUX;
break;
case "darwin":
operatingSystem = WorkingInitMessage.OperatingSystem.MAC;
break;
default:
throw new Error(`unrecognized platform "${platform}"`);
}
initMsg.setOperatingSystem(operatingSystem);
initMsg.setOperatingSystem(platformToProto(os.platform()));
initMsg.setShell(os.userInfo().shell || global.process.env.SHELL);
const srvMsg = new ServerMessage();
srvMsg.setInit(initMsg);
connection.send(srvMsg.serializeBinary());
}
private handleMessage(message: ClientMessage): void {
if (message.hasNewEval()) {
const evalMessage = message.getNewEval()!;
logger.trace(() => [
"EvalMessage",
field("id", evalMessage.getId()),
field("args", evalMessage.getArgsList()),
field("function", evalMessage.getFunction()),
]);
const resp = evaluate(this.connection, evalMessage, () => {
this.evals.delete(evalMessage.getId());
logger.trace(() => [
`dispose ${evalMessage.getId()}, ${this.evals.size} left`,
]);
}, this.options ? this.options.fork : undefined);
if (resp) {
this.evals.set(evalMessage.getId(), resp);
}
} else if (message.hasEvalEvent()) {
const evalEventMessage = message.getEvalEvent()!;
const e = this.evals.get(evalEventMessage.getId());
if (!e) {
return;
}
e.onEvent(evalEventMessage);
/**
* Handle all messages from the client.
*/
private async handleMessage(message: ClientMessage): Promise<void> {
if (message.hasMethod()) {
await this.runMethod(message.getMethod()!);
} else if (message.hasPing()) {
logger.trace("ping");
const srvMsg = new ServerMessage();
@@ -111,4 +112,230 @@ export class Server {
throw new Error("unknown message type");
}
}
/**
* Run a method on a proxy.
*/
private async runMethod(message: MethodMessage): Promise<void> {
const proxyMessage = message.getNamedProxy()! || message.getNumberedProxy()!;
const id = proxyMessage.getId();
const proxyId = message.hasNamedProxy()
? protoToModule(message.getNamedProxy()!.getModule())
: message.getNumberedProxy()!.getProxyId();
const method = proxyMessage.getMethod();
const args = proxyMessage.getArgsList().map((a) => parse(
a,
(id, args) => this.sendCallback(proxyId, id, args),
));
logger.trace(() => [
"received",
field("id", id),
field("proxyId", proxyId),
field("method", method),
field("args", proxyMessage.getArgsList()),
]);
let response: any;
try {
const proxy = this.getProxy(proxyId);
if (typeof proxy.instance[method] !== "function") {
throw new Error(`"${method}" is not a function`);
}
response = proxy.instance[method](...args);
// We wait for the client to call "dispose" instead of doing it onDone to
// ensure all the messages it sent get processed before we get rid of it.
if (method === "dispose") {
this.removeProxy(proxyId);
}
// Proxies must always return promises.
if (!isPromise(response)) {
throw new Error('"${method}" must return a promise');
}
} catch (error) {
logger.error(
error.message,
field("type", typeof response),
field("proxyId", proxyId),
);
this.sendException(id, error);
}
try {
this.sendResponse(id, await response);
} catch (error) {
this.sendException(id, error);
}
}
/**
* Send a callback to the client.
*/
private sendCallback(proxyId: number | Module, callbackId: number, args: any[]): void {
const stringifiedArgs = args.map((a) => this.stringify(a));
logger.trace(() => [
"sending callback",
field("proxyId", proxyId),
field("callbackId", callbackId),
field("args", stringifiedArgs),
]);
const message = new CallbackMessage();
let callbackMessage: NamedCallbackMessage | NumberedCallbackMessage;
if (typeof proxyId === "string") {
callbackMessage = new NamedCallbackMessage();
callbackMessage.setModule(moduleToProto(proxyId));
message.setNamedCallback(callbackMessage);
} else {
callbackMessage = new NumberedCallbackMessage();
callbackMessage.setProxyId(proxyId);
message.setNumberedCallback(callbackMessage);
}
callbackMessage.setCallbackId(callbackId);
callbackMessage.setArgsList(stringifiedArgs);
const serverMessage = new ServerMessage();
serverMessage.setCallback(message);
this.connection.send(serverMessage.serializeBinary());
}
/**
* Store a proxy and bind events to send them back to the client.
*/
private storeProxy(instance: ServerProxy): number;
private storeProxy(instance: any, moduleProxyId: Module): Module;
private storeProxy(instance: ServerProxy | any, moduleProxyId?: Module): number | Module {
// In case we disposed while waiting for a function to return.
if (this.disconnected) {
if (isProxy(instance)) {
instance.dispose();
}
throw new Error("disposed");
}
const proxyId = moduleProxyId || this.proxyId++;
logger.trace(() => [
"storing proxy",
field("proxyId", proxyId),
]);
this.proxies.set(proxyId, { instance });
if (isProxy(instance)) {
instance.onEvent((event, ...args) => this.sendEvent(proxyId, event, ...args));
instance.onDone(() => {
// It might have finished because we disposed it due to a disconnect.
if (!this.disconnected) {
this.sendEvent(proxyId, "done");
this.getProxy(proxyId).disposeTimeout = setTimeout(() => {
instance.dispose();
this.removeProxy(proxyId);
}, this.responseTimeout);
}
});
}
return proxyId;
}
/**
* Send an event to the client.
*/
private sendEvent(proxyId: number | Module, event: string, ...args: any[]): void {
const stringifiedArgs = args.map((a) => this.stringify(a));
logger.trace(() => [
"sending event",
field("proxyId", proxyId),
field("event", event),
field("args", stringifiedArgs),
]);
const message = new EventMessage();
let eventMessage: NamedEventMessage | NumberedEventMessage;
if (typeof proxyId === "string") {
eventMessage = new NamedEventMessage();
eventMessage.setModule(moduleToProto(proxyId));
message.setNamedEvent(eventMessage);
} else {
eventMessage = new NumberedEventMessage();
eventMessage.setProxyId(proxyId);
message.setNumberedEvent(eventMessage);
}
eventMessage.setEvent(event);
eventMessage.setArgsList(stringifiedArgs);
const serverMessage = new ServerMessage();
serverMessage.setEvent(message);
this.connection.send(serverMessage.serializeBinary());
}
/**
* Send a response back to the client.
*/
private sendResponse(id: number, response: any): void {
const stringifiedResponse = this.stringify(response);
logger.trace(() => [
"sending resolve",
field("id", id),
field("response", stringifiedResponse),
]);
const successMessage = new SuccessMessage();
successMessage.setId(id);
successMessage.setResponse(stringifiedResponse);
const serverMessage = new ServerMessage();
serverMessage.setSuccess(successMessage);
this.connection.send(serverMessage.serializeBinary());
}
/**
* Send an exception back to the client.
*/
private sendException(id: number, error: Error): void {
const stringifiedError = stringify(error);
logger.trace(() => [
"sending reject",
field("id", id) ,
field("response", stringifiedError),
]);
const failedMessage = new FailMessage();
failedMessage.setId(id);
failedMessage.setResponse(stringifiedError);
const serverMessage = new ServerMessage();
serverMessage.setFail(failedMessage);
this.connection.send(serverMessage.serializeBinary());
}
/**
* Call after disposing a proxy.
*/
private removeProxy(proxyId: number | Module): void {
clearTimeout(this.getProxy(proxyId).disposeTimeout as any);
this.proxies.delete(proxyId);
logger.trace(() => [
"disposed and removed proxy",
field("proxyId", proxyId),
field("proxies", this.proxies.size),
]);
}
private stringify(value: any): string {
return stringify(value, undefined, (p) => this.storeProxy(p));
}
private getProxy(proxyId: number | Module): ProxyData {
if (!this.proxies.has(proxyId)) {
throw new Error(`proxy ${proxyId} disposed too early`);
}
return this.proxies.get(proxyId)!;
}
}