Improve retry

Registering returns an instance that lets you retry and recover without
needing to keep passing the name everywhere.

Also refactored the shared process a little to make better use of the
retry and downgraded stderr messages to warnings because they aren't
critical.

packages/ide/src/fill/client.ts

@@ -11,7 +11,7 @@ class WebsocketConnection implements ReadWriteConnection {
 	private activeSocket: WebSocket | undefined;
 	private readonly messageBuffer = <Uint8Array[]>[];
 	private readonly socketTimeoutDelay = 60 * 1000;
-	private readonly retryName = "Socket";
+	private readonly retry = retry.register("Socket", () => this.connect());
 	private isUp: boolean = false;
 	private closed: boolean = false;
 
@@ -26,11 +26,14 @@ class WebsocketConnection implements ReadWriteConnection {
 	public readonly onMessage = this.messageEmitter.event;
 
 	public constructor() {
-		retry.register(this.retryName, () => this.connect());
+		this.retry.block();
-		retry.block(this.retryName);
+		this.retry.run();
-		retry.run(this.retryName);
 	}
 
+	/**
+	 * Send data across the socket. If closed, will error. If connecting, will
+	 * queue.
+	 */
 	public send(data: Buffer | Uint8Array): void {
 		if (this.closed) {
 			throw new Error("web socket is closed");
@@ -42,6 +45,9 @@ class WebsocketConnection implements ReadWriteConnection {
 		}
 	}
 
+	/**
+	 * Close socket connection.
+	 */
 	public close(): void {
 		this.closed = true;
 		this.dispose();
@@ -75,8 +81,8 @@ class WebsocketConnection implements ReadWriteConnection {
 				field("wasClean", event.wasClean),
 			);
 			if (!this.closed) {
-				retry.block(this.retryName);
+				this.retry.block();
-				retry.run(this.retryName);
+				this.retry.run();
 			}
 		});
 
@@ -108,15 +114,19 @@ class WebsocketConnection implements ReadWriteConnection {
 		}, this.socketTimeoutDelay);
 
 		await new Promise((resolve, reject): void => {
-			const onClose = (): void => {
+			const doReject = (): void => {
 				clearTimeout(socketWaitTimeout);
-				socket.removeEventListener("close", onClose);
+				socket.removeEventListener("error", doReject);
+				socket.removeEventListener("close", doReject);
 				reject();
 			};
-			socket.addEventListener("close", onClose);
+			socket.addEventListener("error", doReject);
+			socket.addEventListener("close", doReject);
 
-			socket.addEventListener("open", async () => {
+			socket.addEventListener("open", () => {
 				clearTimeout(socketWaitTimeout);
+				socket.removeEventListener("error", doReject);
+				socket.removeEventListener("close", doReject);
 				resolve();
 			});
 		});

packages/ide/src/retry.ts

@@ -1,14 +1,64 @@
 import { logger, field } from "@coder/logger";
 import { NotificationService, INotificationHandle, INotificationService, Severity } from "./fill/notification";
 
+// tslint:disable no-any can have different return values
+
 interface IRetryItem {
+	/**
+	 * How many times this item has been retried.
+	 */
 	count?: number;
-	delay?: number; // In seconds.
+
-	end?: number;   // In ms.
+	/**
-	fn(): any | Promise<any>; // tslint:disable-line no-any can have different return values
+	 * In seconds.
+	 */
+	delay?: number;
+
+	/**
+	 * In milliseconds.
+	 */
+	end?: number;
+
+	/**
+	 * Function to run when retrying.
+	 */
+	fn(): any;
+
+	/**
+	 * Timer for running this item.
+	 */
 	timeout?: number | NodeJS.Timer;
+
+	/**
+	 * Whether the item is retrying or waiting to retry.
+	 */
 	running?: boolean;
-	showInNotification: boolean;
+}
+
+/**
+ * An retry-able instance.
+ */
+export interface RetryInstance {
+	/**
+	 * Run this retry.
+	 */
+	run(error?: Error): void;
+
+	/**
+	 * Block on this instance.
+	 */
+	block(): void;
+}
+
+/**
+ * A retry-able instance that doesn't use a promise so it must be manually
+ * ran again on failure and recovered on success.
+ */
+export interface ManualRetryInstance extends RetryInstance {
+	/**
+	 * Mark this item as recovered.
+	 */
+	recover(): void;
 }
 
 /**
@@ -21,7 +71,7 @@ interface IRetryItem {
  * to the user explaining what is happening with an option to immediately retry.
  */
 export class Retry {
-	private items = new Map<string, IRetryItem>();
+	private readonly items = new Map<string, IRetryItem>();
 
 	// Times are in seconds.
 	private readonly retryMinDelay = 1;
@@ -50,13 +100,54 @@ export class Retry {
 	}
 
 	/**
-	 * Block retries when we know they will fail (for example when starting Wush
+	 * Register a function to retry that starts/connects to a service.
-	 * back up). If a name is passed, that service will still be allowed to retry
+	 *
+	 * The service is automatically retried or recovered when the promise resolves
+	 * or rejects. If the service dies after starting, it must be manually
+	 * retried.
+	 */
+	public register(name: string, fn: () => Promise<any>): RetryInstance;
+	/**
+	 * Register a function to retry that starts/connects to a service.
+	 *
+	 * Must manually retry if it fails to start again or dies after restarting and
+	 * manually recover if it succeeds in starting again.
+	 */
+	public register(name: string, fn: () => any): ManualRetryInstance;
+	/**
+	 * Register a function to retry that starts/connects to a service.
+	 */
+	public register(name: string, fn: () => any): RetryInstance | ManualRetryInstance {
+		if (this.items.has(name)) {
+			throw new Error(`"${name}" is already registered`);
+		}
+		this.items.set(name, { fn });
+
+		return {
+			block: (): void => this.block(name),
+			run: (error?: Error): void => this.run(name, error),
+			recover: (): void => this.recover(name),
+		};
+	}
+
+	/**
+	 * Un-register a function to retry.
+	 */
+	public unregister(name: string): void {
+		if (!this.items.has(name)) {
+			throw new Error(`"${name}" is not registered`);
+		}
+		this.items.delete(name);
+	}
+
+	/**
+	 * Block retries when we know they will fail (for example when the socket is
+	 * down ). If a name is passed, that service will still be allowed to retry
 	 * (unless we have already blocked).
 	 *
 	 * Blocking without a name will override a block with a name.
 	 */
-	public block(name?: string): void {
+	private block(name?: string): void {
 		if (!this.blocked || !name) {
 			this.blocked = name || true;
 			this.items.forEach((item) => {
@@ -68,7 +159,7 @@ export class Retry {
 	/**
 	 * Unblock retries and run any that are pending.
 	 */
-	public unblock(): void {
+	private unblock(): void {
 		this.blocked = false;
 		this.items.forEach((item, name) => {
 			if (item.running) {
@@ -77,35 +168,10 @@ export class Retry {
 		});
 	}
 
-	/**
-	 * Register a function to retry that starts/connects to a service.
-	 *
-	 * If the function returns a promise, it will automatically be retried,
-	 * recover, & unblock after calling `run` once (otherwise they need to be
-	 * called manually).
-	 */
-	// tslint:disable-next-line no-any can have different return values
-	public register(name: string, fn: () => any | Promise<any>, showInNotification: boolean = true): void {
-		if (this.items.has(name)) {
-			throw new Error(`"${name}" is already registered`);
-		}
-		this.items.set(name, { fn, showInNotification });
-	}
-
-	/**
-	 * Unregister a function to retry.
-	 */
-	public unregister(name: string): void {
-		if (!this.items.has(name)) {
-			throw new Error(`"${name}" is not registered`);
-		}
-		this.items.delete(name);
-	}
-
 	/**
 	 * Retry a service.
 	 */
-	public run(name: string, error?: Error): void {
+	private run(name: string, error?: Error): void {
 		if (!this.items.has(name)) {
 			throw new Error(`"${name}" is not registered`);
 		}
@@ -149,7 +215,7 @@ export class Retry {
 	/**
 	 * Reset a service after a successfully recovering.
 	 */
-	public recover(name: string): void {
+	private recover(name: string): void {
 		if (!this.items.has(name)) {
 			throw new Error(`"${name}" is not registered`);
 		}
@@ -191,9 +257,9 @@ export class Retry {
 					if (this.blocked === name) {
 						this.unblock();
 					}
-				}).catch(() => {
+				}).catch((error) => {
 					endItem();
-					this.run(name);
+					this.run(name, error);
 				});
 			} else {
 				endItem();
@@ -214,8 +280,7 @@ export class Retry {
 
 		const now = Date.now();
 		const items = Array.from(this.items.entries()).filter(([_, item]) => {
-			return item.showInNotification
+			return typeof item.end !== "undefined"
-				&& typeof item.end !== "undefined"
 				&& item.end > now
 				&& item.delay && item.delay >= this.notificationThreshold;
 		}).sort((a, b) => {

packages/server/src/vscode/sharedProcess.ts

@@ -1,5 +1,6 @@
 import { ChildProcess } from "child_process";
 import * as fs from "fs";
+import * as fse from "fs-extra";
 import * as os from "os";
 import * as path from "path";
 import { forkModule } from "./bootstrapFork";
@@ -7,7 +8,7 @@ import { StdioIpcHandler } from "../ipc";
 import { ParsedArgs } from "vs/platform/environment/common/environment";
 import { Emitter } from "@coder/events/src";
 import { retry } from "@coder/ide/src/retry";
-import { logger, field, Level } from "@coder/logger";
+import { logger, Level } from "@coder/logger";
 
 export enum SharedProcessState {
 	Stopped,
@@ -23,129 +24,143 @@ export type SharedProcessEvent = {
 };
 
 export class SharedProcess {
-	public readonly socketPath: string = os.platform() === "win32" ? path.join("\\\\?\\pipe", os.tmpdir(), `.code-server${Math.random().toString()}`) : path.join(os.tmpdir(), `.code-server${Math.random().toString()}`);
+	public readonly socketPath: string = os.platform() === "win32"
+		? path.join("\\\\?\\pipe", os.tmpdir(), `.code-server${Math.random().toString()}`)
+		: path.join(os.tmpdir(), `.code-server${Math.random().toString()}`);
 	private _state: SharedProcessState = SharedProcessState.Stopped;
 	private activeProcess: ChildProcess | undefined;
 	private ipcHandler: StdioIpcHandler | undefined;
 	private readonly onStateEmitter = new Emitter<SharedProcessEvent>();
 	public readonly onState = this.onStateEmitter.event;
-	private readonly retryName = "Shared process";
 	private readonly logger = logger.named("shared");
+	private readonly retry = retry.register("Shared process", () => this.connect());
+	private disposed: boolean = false;
 
 	public constructor(
 		private readonly userDataDir: string,
 		private readonly builtInExtensionsDir: string,
 	) {
-		retry.register(this.retryName, () => this.restart());
+		this.retry.run();
-		retry.run(this.retryName);
 	}
 
 	public get state(): SharedProcessState {
 		return this._state;
 	}
 
-	public restart(): void {
+	/**
-		if (this.activeProcess && !this.activeProcess.killed) {
+	 * Signal the shared process to terminate.
-			this.activeProcess.kill();
+	 */
-		}
-
-		const extensionsDir = path.join(this.userDataDir, "extensions");
-		const mkdir = (dir: string): void => {
-			try {
-				fs.mkdirSync(dir);
-			} catch (ex) {
-				if (ex.code !== "EEXIST" && ex.code !== "EISDIR") {
-					throw ex;
-				}
-			}
-		};
-		mkdir(this.userDataDir);
-		mkdir(extensionsDir);
-		const backupsDir = path.join(this.userDataDir, "Backups");
-		mkdir(backupsDir);
-		const workspacesFile = path.join(backupsDir, "workspaces.json");
-		if (!fs.existsSync(workspacesFile)) {
-			fs.closeSync(fs.openSync(workspacesFile, "w"));
-		}
-
-		this.setState({
-			state: SharedProcessState.Starting,
-		});
-		let resolved: boolean = false;
-		const maybeStop = (error: string): void => {
-			if (resolved) {
-				return;
-			}
-			this.setState({
-				error,
-				state: SharedProcessState.Stopped,
-			});
-			if (!this.activeProcess) {
-				return;
-			}
-			this.activeProcess.kill();
-		};
-		this.activeProcess = forkModule("vs/code/electron-browser/sharedProcess/sharedProcessMain", [], {
-			env: {
-				VSCODE_ALLOW_IO: "true",
-				VSCODE_LOGS: process.env.VSCODE_LOGS,
-			},
-		}, this.userDataDir);
-		if (this.logger.level <= Level.Trace) {
-			this.activeProcess.stdout.on("data", (data) => {
-				this.logger.trace(() => ["stdout", field("data", data.toString())]);
-			});
-		}
-		this.activeProcess.on("error", (error) => {
-			this.logger.error("error", field("error", error));
-			maybeStop(error.message);
-		});
-		this.activeProcess.on("exit", (err) => {
-			if (this._state !== SharedProcessState.Stopped) {
-				this.setState({
-					error: `Exited with ${err}`,
-					state: SharedProcessState.Stopped,
-				});
-			}
-			retry.run(this.retryName, new Error(`Exited with ${err}`));
-		});
-		this.ipcHandler = new StdioIpcHandler(this.activeProcess);
-		this.ipcHandler.once("handshake:hello", () => {
-			const data: {
-				sharedIPCHandle: string;
-				args: Partial<ParsedArgs>;
-				logLevel: Level;
-			} = {
-				args: {
-					"builtin-extensions-dir": this.builtInExtensionsDir,
-					"user-data-dir": this.userDataDir,
-					"extensions-dir": extensionsDir,
-				},
-				logLevel: this.logger.level,
-				sharedIPCHandle: this.socketPath,
-			};
-			this.ipcHandler!.send("handshake:hey there", "", data);
-		});
-		this.ipcHandler.once("handshake:im ready", () => {
-			resolved = true;
-			retry.recover(this.retryName);
-			this.setState({
-				state: SharedProcessState.Ready,
-			});
-		});
-		this.activeProcess.stderr.on("data", (data) => {
-			this.logger.error("stderr", field("data", data.toString()));
-			maybeStop(data.toString());
-		});
-	}
-
 	public dispose(): void {
+		this.disposed = true;
 		if (this.ipcHandler) {
 			this.ipcHandler.send("handshake:goodbye");
 		}
 		this.ipcHandler = undefined;
 	}
 
+	/**
+	 * Start and connect to the shared process.
+	 */
+	private async connect(): Promise<void> {
+		this.setState({ state: SharedProcessState.Starting });
+		const activeProcess = await this.restart();
+
+		activeProcess.stderr.on("data", (data) => {
+			// Warn instead of error to prevent panic. It's unlikely stderr here is
+			// about anything critical to the functioning of the editor.
+			logger.warn(data.toString());
+		});
+
+		activeProcess.on("exit", (exitCode) => {
+			const error = new Error(`Exited with ${exitCode}`);
+			this.setState({
+				error: error.message,
+				state: SharedProcessState.Stopped,
+			});
+			if (!this.disposed) {
+				this.retry.run(error);
+			}
+		});
+
+		this.setState({ state: SharedProcessState.Ready });
+	}
+
+	/**
+	 * Restart the shared process. Kill existing process if running. Resolve when
+	 * the shared process is ready and reject when it errors or dies before being
+	 * ready.
+	 */
+	private async restart(): Promise<ChildProcess> {
+		if (this.activeProcess && !this.activeProcess.killed) {
+			this.activeProcess.kill();
+		}
+
+		const extensionsDir = path.join(this.userDataDir, "extensions");
+		const backupsDir = path.join(this.userDataDir, "Backups");
+		await Promise.all([
+			fse.mkdirp(extensionsDir),
+			fse.mkdirp(backupsDir),
+		]);
+
+		const workspacesFile = path.join(backupsDir, "workspaces.json");
+		if (!fs.existsSync(workspacesFile)) {
+			fs.appendFileSync(workspacesFile, "");
+		}
+
+		const activeProcess = forkModule("vs/code/electron-browser/sharedProcess/sharedProcessMain", [], {
+			env: {
+				VSCODE_ALLOW_IO: "true",
+				VSCODE_LOGS: process.env.VSCODE_LOGS,
+			},
+		}, this.userDataDir);
+		this.activeProcess = activeProcess;
+
+		await new Promise((resolve, reject): void => {
+			const doReject = (error: Error | number): void => {
+				if (typeof error === "number") {
+					error = new Error(`Exited with ${error}`);
+				}
+				activeProcess.removeAllListeners();
+				this.setState({
+					error: error.message,
+					state: SharedProcessState.Stopped,
+				});
+				reject(error);
+			};
+
+			activeProcess.on("error", doReject);
+			activeProcess.on("exit", doReject);
+
+			this.ipcHandler = new StdioIpcHandler(activeProcess);
+			this.ipcHandler.once("handshake:hello", () => {
+				const data: {
+					sharedIPCHandle: string;
+					args: Partial<ParsedArgs>;
+					logLevel: Level;
+				} = {
+					args: {
+						"builtin-extensions-dir": this.builtInExtensionsDir,
+						"user-data-dir": this.userDataDir,
+						"extensions-dir": extensionsDir,
+					},
+					logLevel: this.logger.level,
+					sharedIPCHandle: this.socketPath,
+				};
+				this.ipcHandler!.send("handshake:hey there", "", data);
+			});
+			this.ipcHandler.once("handshake:im ready", () => {
+				activeProcess.removeListener("error", doReject);
+				activeProcess.removeListener("exit", doReject);
+				resolve();
+			});
+		});
+
+		return activeProcess;
+	}
+
+	/**
+	 * Set the internal shared process state and emit the state event.
+	 */
 	private setState(event: SharedProcessEvent): void {
 		this._state = event.state;
 		this.onStateEmitter.emit(event);

scripts/vscode.patch

@@ -917,17 +917,15 @@ index 0592910..0ce7e35 100644
 @@ -33,0 +34 @@ function getSystemExtensionsRoot(): string {
 +	return (require('vs/../../../../packages/vscode/src/fill/paths') as typeof import ('vs/../../../../packages/vscode/src/fill/paths')).getBuiltInExtensionsDirectory();
 diff --git a/src/vs/workbench/services/extensions/electron-browser/extensionService.ts b/src/vs/workbench/services/extensions/electron-browser/extensionService.ts
-index 2c2f9c7..69fa321 100644
+index 2c2f9c7..e2ab620 100644
 --- a/src/vs/workbench/services/extensions/electron-browser/extensionService.ts
 +++ b/src/vs/workbench/services/extensions/electron-browser/extensionService.ts
-@@ -34,0 +35 @@ import { Schemas } from 'vs/base/common/network';
+@@ -92,0 +93 @@ export class ExtensionService extends Disposable implements IExtensionService {
-+const retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry;
++	private readonly retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry.register('Extension Host', () => this.startExtensionHost());
-@@ -117,0 +119 @@ export class ExtensionService extends Disposable implements IExtensionService {
+@@ -435,0 +437 @@ export class ExtensionService extends Disposable implements IExtensionService {
-+		retry.register('Extension Host', () => this.startExtensionHost());
++		extHostProcessWorker.start().then(() => this.retry.recover());
-@@ -435,0 +438 @@ export class ExtensionService extends Disposable implements IExtensionService {
+@@ -458,0 +461 @@ export class ExtensionService extends Disposable implements IExtensionService {
-+		extHostProcessWorker.start().then(() => retry.recover('Extension Host'));
++		return this.retry.run();
-@@ -458,0 +462 @@ export class ExtensionService extends Disposable implements IExtensionService {
-+		return retry.run('Extension Host');
 diff --git a/src/vs/workbench/services/extensions/node/extensionHostProcess.ts b/src/vs/workbench/services/extensions/node/extensionHostProcess.ts
 index 484cef9..f728fc8 100644
 --- a/src/vs/workbench/services/extensions/node/extensionHostProcess.ts
@@ -936,45 +934,39 @@ index 484cef9..f728fc8 100644
 -					process.kill(initData.parentPid, 0); // throws an exception if the main process doesn't exist anymore.
 +					// process.kill(initData.parentPid, 0); // throws an exception if the main process doesn't exist anymore.
 diff --git a/src/vs/workbench/services/files/node/watcher/nsfw/watcherService.ts b/src/vs/workbench/services/files/node/watcher/nsfw/watcherService.ts
-index ca03fc9..e3dcd08 100644
+index ca03fc9..e8b6326 100644
 --- a/src/vs/workbench/services/files/node/watcher/nsfw/watcherService.ts
 +++ b/src/vs/workbench/services/files/node/watcher/nsfw/watcherService.ts
-@@ -18,0 +19 @@ import { getPathFromAmdModule } from 'vs/base/common/amd';
+@@ -26,0 +27 @@ export class FileWatcher {
-+const retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry;
++	private readonly retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry.register('Watcher', () => this.startWatching());
-@@ -35,0 +37 @@ export class FileWatcher {
+@@ -56,0 +58,2 @@ export class FileWatcher {
-+		retry.register('Watcher', () => this.startWatching());
-@@ -56,0 +59,2 @@ export class FileWatcher {
 +				this.toDispose = dispose(this.toDispose);
-+				return retry.run('Watcher');
++				return this.retry.run();
-@@ -113 +117 @@ export class FileWatcher {
+@@ -113 +116 @@ export class FileWatcher {
 -		}));
-+		})).then(() => retry.recover('Watcher'));
++		})).then(() => this.retry.recover());
 diff --git a/src/vs/workbench/services/files/node/watcher/unix/watcherService.ts b/src/vs/workbench/services/files/node/watcher/unix/watcherService.ts
-index 7e3a324..b9ccd63 100644
+index 7e3a324..a880182 100644
 --- a/src/vs/workbench/services/files/node/watcher/unix/watcherService.ts
 +++ b/src/vs/workbench/services/files/node/watcher/unix/watcherService.ts
-@@ -18,0 +19 @@ import { getPathFromAmdModule } from 'vs/base/common/amd';
+@@ -26,0 +27 @@ export class FileWatcher {
-+const retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry;
++	private readonly retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry.register('Watcher', () => this.startWatching());
-@@ -36,0 +38 @@ export class FileWatcher {
+@@ -59,0 +61,2 @@ export class FileWatcher {
-+		retry.register('Watcher', () => this.startWatching());
-@@ -59,0 +62,2 @@ export class FileWatcher {
 +				this.toDispose = dispose(this.toDispose);
-+				return retry.run('Watcher');
++				return this.retry.run();
-@@ -116 +120 @@ export class FileWatcher {
+@@ -116 +119 @@ export class FileWatcher {
 -		}));
-+		})).then(() => retry.recover('Watcher'));
++		})).then(() => this.retry.recover());
 diff --git a/src/vs/workbench/services/files/node/watcher/win32/csharpWatcherService.ts b/src/vs/workbench/services/files/node/watcher/win32/csharpWatcherService.ts
-index 74dad64..34cd83b 100644
+index 74dad64..7bc591a 100644
 --- a/src/vs/workbench/services/files/node/watcher/win32/csharpWatcherService.ts
 +++ b/src/vs/workbench/services/files/node/watcher/win32/csharpWatcherService.ts
-@@ -14,0 +15 @@ import { getPathFromAmdModule } from 'vs/base/common/amd';
+@@ -25,0 +26 @@ export class OutOfProcessWin32FolderWatcher {
-+const retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry;
++	private readonly retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry.register('Watcher', () => this.startWatcher());
-@@ -40,0 +42 @@ export class OutOfProcessWin32FolderWatcher {
+@@ -52,0 +54 @@ export class OutOfProcessWin32FolderWatcher {
-+		retry.register('Watcher', () => this.startWatcher());
++		this.handle.stdout.once('data', () => this.retry.recover());
-@@ -52,0 +55 @@ export class OutOfProcessWin32FolderWatcher {
+@@ -110,0 +113 @@ export class OutOfProcessWin32FolderWatcher {
-+		this.handle.stdout.once('data', () => retry.recover('Watcher'));
++			return this.retry.run();
-@@ -110,0 +114 @@ export class OutOfProcessWin32FolderWatcher {
-+			return retry.run('Watcher');
 diff --git a/src/vs/workbench/services/keybinding/electron-browser/keybindingService.ts b/src/vs/workbench/services/keybinding/electron-browser/keybindingService.ts
 index 3c78990..545d91a 100644
 --- a/src/vs/workbench/services/keybinding/electron-browser/keybindingService.ts
@@ -986,39 +978,36 @@ index 3c78990..545d91a 100644
 -		if (OS === OperatingSystem.Windows) {
 +		if (isNative && OS === OperatingSystem.Windows) {
 diff --git a/src/vs/workbench/services/search/node/searchService.ts b/src/vs/workbench/services/search/node/searchService.ts
-index 3eaafa4..0345bad 100644
+index 3eaafa4..3b4cb5f 100644
 --- a/src/vs/workbench/services/search/node/searchService.ts
 +++ b/src/vs/workbench/services/search/node/searchService.ts
 @@ -11 +11 @@ import { Event } from 'vs/base/common/event';
 -import { Disposable, IDisposable, toDisposable } from 'vs/base/common/lifecycle';
 +import { Disposable, IDisposable, toDisposable, dispose } from 'vs/base/common/lifecycle';
-@@ -32,0 +33 @@ import { IConfigurationService } from 'vs/platform/configuration/common/configur
+@@ -433,0 +434 @@ export class DiskSearch implements ISearchResultProvider {
-+const retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry;
-@@ -433,0 +435 @@ export class DiskSearch implements ISearchResultProvider {
 +	private toDispose: IDisposable[] = [];
-@@ -470,6 +472,16 @@ export class DiskSearch implements ISearchResultProvider {
+@@ -470,6 +471,15 @@ export class DiskSearch implements ISearchResultProvider {
 -		const client = new Client(
 -			getPathFromAmdModule(require, 'bootstrap-fork'),
 -			opts);
 -
 -		const channel = getNextTickChannel(client.getChannel('search'));
 -		this.raw = new SearchChannelClient(channel);
-+		const connect = (): void => {
++		const connect = (): Promise<void> => {
 +			const client = new Client(
 +				getPathFromAmdModule(require, 'bootstrap-fork'),
 +				opts);
 +			client.onDidProcessExit(() => {
 +				this.toDispose = dispose(this.toDispose);
-+				retry.run('Searcher');
++				retry.run();
 +			}, null, this.toDispose);
 +			this.toDispose.push(client);
-+
 +			const channel = getNextTickChannel(client.getChannel('search'));
 +			this.raw = new SearchChannelClient(channel);
-+			this.raw.clearCache('test-connectivity').then(() => retry.recover('Searcher'));
++			return this.raw.clearCache('test-connectivity');
 +		};
-+		retry.register('Searcher', connect);
++		const retry = (require('vs/../../../../packages/vscode/src/workbench') as typeof import ('vs/../../../../packages/vscode/src/workbench')).workbench.retry.register('Searcher', connect);
-+		connect();
++		retry.run();
 diff --git a/src/vs/workbench/services/timer/electron-browser/timerService.ts b/src/vs/workbench/services/timer/electron-browser/timerService.ts
 index 6e6fbcc..645bd72 100644
 --- a/src/vs/workbench/services/timer/electron-browser/timerService.ts