Run each e2e test in a new workspace

The workspaces also have settings to prevent the welcome page from
appearing.

test/e2e/models/CodeServer.ts

@@ -1,21 +1,24 @@
+import { promises as fs } from "fs"
+import * as path from "path"
 import { Page } from "playwright"
-import { CODE_SERVER_ADDRESS } from "../../utils/constants"
+import { CODE_SERVER_ADDRESS, workspaceDir } from "../../utils/constants"
+import { tmpdir } from "../../utils/helpers"
+
 // This is a Page Object Model
 // We use these to simplify e2e test authoring
 // See Playwright docs: https://playwright.dev/docs/pom/
 export class CodeServer {
-  page: Page
-  editorSelector = "div.monaco-workbench"
+  private readonly editorSelector = "div.monaco-workbench"
 
-  constructor(page: Page) {
-    this.page = page
-  }
+  constructor(public readonly page: Page) {}
 
   /**
-   * Navigates to CODE_SERVER_ADDRESS
+   * Navigates to CODE_SERVER_ADDRESS. It will open a newly created random
+   * directory.
    */
   async navigate() {
-    await this.page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+    const dir = await this.createWorkspace()
+    await this.page.goto(`${CODE_SERVER_ADDRESS}?folder=${dir}`, { waitUntil: "networkidle" })
   }
 
   /**
@@ -113,4 +116,20 @@ export class CodeServer {
     await this.navigate()
     await this.reloadUntilEditorIsReady()
   }
+
+  /**
+   * Create a random workspace and seed it with settings.
+   */
+  private async createWorkspace(): Promise<string> {
+    const dir = await tmpdir(workspaceDir)
+    await fs.mkdir(path.join(dir, ".vscode"))
+    await fs.writeFile(
+      path.join(dir, ".vscode/settings.json"),
+      JSON.stringify({
+        "workbench.startupEditor": "none",
+      }),
+      "utf8",
+    )
+    return dir
+  }
 }

test/playwright.config.ts

@@ -6,7 +6,7 @@ import path from "path"
 const config: PlaywrightTestConfig = {
   testDir: path.join(__dirname, "e2e"), // Search for tests in this directory.
   timeout: 60000, // Each test is given 60 seconds.
-  retries: 3, // Retry failing tests 2 times
+  retries: process.env.CI ? 2 : 1, // Retry twice in CI due to flakiness.
   workers: 1,
   globalSetup: require.resolve("./utils/globalSetup.ts"),
   reporter: "list",
@@ -34,10 +34,4 @@ const config: PlaywrightTestConfig = {
   ],
 }
 
-if (process.env.CI) {
-  // In CI, retry failing tests 2 times
-  // in the event of flakiness
-  config.retries = 2
-}
-
 export default config

test/utils/constants.ts

@@ -1,3 +1,4 @@
 export const CODE_SERVER_ADDRESS = process.env.CODE_SERVER_ADDRESS || "http://localhost:8080"
 export const PASSWORD = process.env.PASSWORD || "e45432jklfdsab"
 export const storageState = JSON.parse(process.env.STORAGE || "{}")
+export const workspaceDir = "workspaces"

test/utils/globalSetup.ts

@@ -1,15 +1,20 @@
-// This setup runs before our e2e tests
-// so that it authenticates us into code-server
-// ensuring that we're logged in before we run any tests
 import { chromium } from "playwright"
 import { hash } from "../../src/node/util"
-import { PASSWORD } from "./constants"
+import { PASSWORD, workspaceDir } from "./constants"
+import { clean } from "./helpers"
 import * as wtfnode from "./wtfnode"
 
+/**
+ * Perform workspace cleanup and authenticate. This should be set up to run
+ * before our tests execute.
+ */
 export default async function () {
   console.log("\n🚨 Running Global Setup for Playwright End-to-End Tests")
   console.log("   Please hang tight...")
 
+  // Cleanup workspaces from previous tests.
+  await clean(workspaceDir)
+
   const cookieToStore = {
     sameSite: "Lax" as const,
     name: "key",

test/utils/helpers.ts

@@ -1,4 +1,4 @@
-import * as fs from "fs"
+import { promises as fs } from "fs"
 import * as os from "os"
 import * as path from "path"
 
@@ -20,13 +20,20 @@ export function createLoggerMock() {
 }
 
 /**
- * Create a uniquely named temporary directory.
- *
- * These directories are placed under a single temporary code-server directory.
+ * Clean up directories left by a test. It is recommended to do this when a test
+ * starts to avoid potentially accumulating infinite test directories.
+ */
+export async function clean(testName: string): Promise<void> {
+  const dir = path.join(os.tmpdir(), `code-server/tests/${testName}`)
+  await fs.rm(dir, { recursive: true })
+}
+
+/**
+ * Create a uniquely named temporary directory for a test.
  */
 export async function tmpdir(testName: string): Promise<string> {
-  const dir = path.join(os.tmpdir(), "code-server/tests")
-  await fs.promises.mkdir(dir, { recursive: true })
+  const dir = path.join(os.tmpdir(), `code-server/tests/${testName}`)
+  await fs.mkdir(dir, { recursive: true })
 
-  return await fs.promises.mkdtemp(path.join(dir, `${testName}-`), { encoding: "utf8" })
+  return await fs.mkdtemp(path.join(dir, `${testName}-`), { encoding: "utf8" })
 }