Merge pull request #2086 from nhooyr/master

Integrate Coder Cloud Agent

.github/workflows/ci.yaml

@@ -8,7 +8,7 @@ jobs:
     steps:
       - uses: actions/checkout@v1
       - name: Run ./ci/steps/fmt.sh
-        uses: ./ci/images/debian8
+        uses: ./ci/images/debian10
         with:
           args: ./ci/steps/fmt.sh
 
@@ -17,7 +17,7 @@ jobs:
     steps:
       - uses: actions/checkout@v1
       - name: Run ./ci/steps/lint.sh
-        uses: ./ci/images/debian8
+        uses: ./ci/images/debian10
         with:
           args: ./ci/steps/lint.sh
 
@@ -26,7 +26,7 @@ jobs:
     steps:
       - uses: actions/checkout@v1
       - name: Run ./ci/steps/test.sh
-        uses: ./ci/images/debian8
+        uses: ./ci/images/debian10
         with:
           args: ./ci/steps/test.sh
 
@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@v1
       - name: Run ./ci/steps/release.sh
-        uses: ./ci/images/debian8
+        uses: ./ci/images/debian10
         with:
           args: ./ci/steps/release.sh
       - name: Upload npm package artifact
@@ -116,7 +116,7 @@ jobs:
           name: release-packages
           path: ./release-packages
       - name: Run ./ci/steps/build-docker-image.sh
-        uses: ./ci/images/debian8
+        uses: ./ci/images/debian10
         with:
           args: ./ci/steps/build-docker-image.sh
       - name: Upload release image

.github/workflows/publish.yaml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@v1
       - name: Run ./ci/steps/publish-npm.sh
-        uses: ./ci/images/debian8
+        uses: ./ci/images/debian10
         with:
           args: ./ci/steps/publish-npm.sh
         env:
@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v1
       - name: Run ./ci/steps/push-docker-manifest.sh
-        uses: ./ci/images/debian8
+        uses: ./ci/images/debian10
         with:
           args: ./ci/steps/push-docker-manifest.sh
         env:

.gitignore

@@ -11,3 +11,5 @@ release-images/
 node_modules
 node-*
 /plugins
+/lib/coder-cloud-agent
+.home

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "lib/vscode"]
 	path = lib/vscode
 	url = https://github.com/microsoft/vscode
+	ignore = dirty

ci/build/build-code-server.sh

@@ -18,6 +18,12 @@ main() {
     chmod +x out/node/entry.js
   fi
 
+  if ! [ -f ./lib/coder-cloud-agent ]; then
+    OS="$(uname | tr '[:upper:]' '[:lower:]')"
+    curl -fsSL "https://storage.googleapis.com/coder-cloud-releases/agent/latest/$OS/cloud-agent" -o ./lib/coder-cloud-agent
+    chmod +x ./lib/coder-cloud-agent
+  fi
+
   parcel build \
     --public-url "." \
     --out-dir dist \

ci/build/build-packages.sh

@@ -11,15 +11,6 @@ main() {
   mkdir -p release-packages
 
   release_archive
-  # Will stop the auto update issues and allow people to upgrade their scripts
-  # for the new release structure.
-  if [[ $ARCH == "amd64" ]]; then
-    if [[ $OS == "linux" ]]; then
-      ARCH=x86_64 release_archive
-    elif [[ $OS == "macos" ]]; then
-      OS=darwin ARCH=x86_64 release_archive
-    fi
-  fi
 
   if [[ $OS == "linux" ]]; then
     release_nfpm
@@ -30,12 +21,6 @@ release_archive() {
   local release_name="code-server-$VERSION-$OS-$ARCH"
   if [[ $OS == "linux" ]]; then
     tar -czf "release-packages/$release_name.tar.gz" --transform "s/^\.\/release-standalone/$release_name/" ./release-standalone
-  elif [[ $OS == "darwin" && $ARCH == "x86_64" ]]; then
-    # Just exists to make autoupdating from 3.2.0 work again.
-    mv ./release-standalone "./$release_name"
-    zip -r "release-packages/$release_name.zip" "./$release_name"
-    mv "./$release_name" ./release-standalone
-    return
   else
     tar -czf "release-packages/$release_name.tar.gz" -s "/^release-standalone/$release_name/" release-standalone
   fi

ci/build/build-release.sh

@@ -25,6 +25,7 @@ main() {
   rsync README.md "$RELEASE_PATH"
   rsync LICENSE.txt "$RELEASE_PATH"
   rsync ./lib/vscode/ThirdPartyNotices.txt "$RELEASE_PATH"
+  rsync ./lib/coder-cloud-agent "$RELEASE_PATH/lib"
 
   # code-server exports types which can be imported and used by plugins. Those
   # types import ipc.d.ts but it isn't included in the final vscode build so
@@ -57,7 +58,6 @@ EOF
   rsync yarn.lock "$RELEASE_PATH"
   rsync ci/build/npm-postinstall.sh "$RELEASE_PATH/postinstall.sh"
 
-
   if [ "$KEEP_MODULES" = 1 ]; then
     rsync node_modules/ "$RELEASE_PATH/node_modules"
   fi

ci/build/clean.sh

@@ -5,16 +5,7 @@ main() {
   cd "$(dirname "${0}")/../.."
   source ./ci/lib.sh
 
-  rm -rf \
+  git clean -Xffd
-    out \
-    release \
-    release-standalone \
-    release-packages \
-    release-gcp \
-    release-images \
-    dist \
-    .cache \
-    node-*
 
   pushd lib/vscode
   git clean -xffd

ci/dev/image/run.sh

@@ -4,16 +4,22 @@ set -euo pipefail
 main() {
   cd "$(dirname "$0")/../../.."
   source ./ci/lib.sh
+  mkdir -p .home
 
   docker run \
     -it \
     --rm \
     -v "$PWD:/src" \
+    -e HOME="/src/.home" \
+    -e USER="coder" \
+    -e GITHUB_TOKEN \
+    -e KEEP_MODULES \
+    -e MINIFY \
     -w /src \
     -p 127.0.0.1:8080:8080 \
     -u "$(id -u):$(id -g)" \
     -e CI \
-    "$(docker_build ./ci/images/debian8)" \
+    "$(docker_build ./ci/images/"${IMAGE-debian10}")" \
     "$@"
 }
 

ci/dev/lint.sh

@@ -7,10 +7,7 @@ main() {
   eslint --max-warnings=0 --fix $(git ls-files "*.ts" "*.tsx" "*.js")
   stylelint $(git ls-files "*.css")
   tsc --noEmit
-  # See comment in ./ci/image/debian8
-  if [[ ! ${CI-} ]]; then
   shellcheck -e SC2046,SC2164,SC2154,SC1091,SC1090,SC2002 $(git ls-files "*.sh")
-  fi
 }
 
 main "$@"

ci/images/centos7/Dockerfile

@@ -15,11 +15,16 @@ RUN npm config set python python2
 RUN yum install -y epel-release && yum install -y jq
 RUN yum install -y rsync
 
-# Copied from ../debian8/Dockerfile
+# Copied from ../debian10/Dockerfile
-# Install Go dependencies
+# Install Go.
 RUN ARCH="$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/')" && \
     curl -fsSL "https://dl.google.com/go/go1.14.3.linux-$ARCH.tar.gz" | tar -C /usr/local -xz
-ENV PATH=/usr/local/go/bin:/root/go/bin:$PATH
+ENV GOPATH=/gopath
+# Ensures running this image as another user works.
+RUN mkdir -p $GOPATH && chmod -R 777 $GOPATH
+ENV PATH=/usr/local/go/bin:$GOPATH/bin:$PATH
+
+# Install Go dependencies
 ENV GO111MODULE=on
 RUN go get mvdan.cc/sh/v3/cmd/shfmt
 RUN go get github.com/goreleaser/nfpm/cmd/nfpm

ci/images/debian10/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:8
+FROM debian:10
 
 RUN apt-get update
 
@@ -24,28 +24,23 @@ RUN apt-get install -y build-essential \
 RUN apt-get install -y gettext-base
 
 # Misc build dependencies.
-RUN apt-get install -y git rsync unzip
+RUN apt-get install -y git rsync unzip jq
-
-# We need latest jq from debian buster for date support.
-RUN ARCH="$(dpkg --print-architecture)" && \
-    curl -fsSOL http://http.us.debian.org/debian/pool/main/libo/libonig/libonig5_6.9.1-1_$ARCH.deb && \
-    dpkg -i libonig*.deb && \
-    curl -fsSOL http://http.us.debian.org/debian/pool/main/j/jq/libjq1_1.5+dfsg-2+b1_$ARCH.deb && \
-    dpkg -i libjq*.deb && \
-    curl -fsSOL http://http.us.debian.org/debian/pool/main/j/jq/jq_1.5+dfsg-2+b1_$ARCH.deb && \
-    dpkg -i jq*.deb && rm *.deb
 
 # Installs shellcheck.
-# Unfortunately coredumps on debian:8 so disabled for now.
+RUN curl -fsSL https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.$(uname -m).tar.xz | \
-#RUN curl -fsSL https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.$(uname -m).tar.xz | \
+    tar -xJ && \
-#    tar -xJ && \
+    mv shellcheck*/shellcheck /usr/local/bin && \
-#    mv shellcheck*/shellcheck /usr/local/bin && \
+    rm -R shellcheck*
-#    rm -R shellcheck*
 
-# Install Go dependencies
+# Install Go.
 RUN ARCH="$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/')" && \
     curl -fsSL "https://dl.google.com/go/go1.14.3.linux-$ARCH.tar.gz" | tar -C /usr/local -xz
-ENV PATH=/usr/local/go/bin:/root/go/bin:$PATH
+ENV GOPATH=/gopath
+# Ensures running this image as another user works.
+RUN mkdir -p $GOPATH && chmod -R 777 $GOPATH
+ENV PATH=/usr/local/go/bin:$GOPATH/bin:$PATH
+
+# Install Go dependencies
 ENV GO111MODULE=on
 RUN go get mvdan.cc/sh/v3/cmd/shfmt
 RUN go get github.com/goreleaser/nfpm/cmd/nfpm

ci/release-image/entrypoint.sh

@@ -2,7 +2,8 @@
 set -eu
 
 # This isn't set by default.
-export USER="$(whoami)"
+USER="$(whoami)"
+export USER
 
 if [ "${DOCKER_USER-}" != "$USER" ]; then
   echo "$DOCKER_USER ALL=(ALL) NOPASSWD:ALL" | sudo tee -a /etc/sudoers.d/nopasswd > /dev/null
@@ -11,7 +12,7 @@ if [ "${DOCKER_USER-}" != "$USER" ]; then
   sudo usermod --login "$DOCKER_USER" coder
   sudo groupmod -n "$DOCKER_USER" coder
 
-  export USER="$(whoami)"
+  USER="$DOCKER_USER"
 
   sudo sed -i "/coder/d" /etc/sudoers.d/nopasswd
   sudo sed -i "s/coder/$DOCKER_USER/g" /etc/fixuid/config.yml

doc/CONTRIBUTING.md

@@ -32,7 +32,7 @@ Differences:
 - We require a minimum of node v12 but later versions should work.
 - We use [nfpm](https://github.com/goreleaser/nfpm) to build `.deb` and `.rpm` packages.
 - We use [jq](https://stedolan.github.io/jq/) to build code-server releases.
-- The [CI container](../ci/images/debian8/Dockerfile) is a useful reference for all our dependencies.
+- The [CI container](../ci/images/debian10/Dockerfile) is a useful reference for all our dependencies.
 
 ## Development Workflow
 
@@ -76,7 +76,7 @@ node .
 Build release packages (make sure you run `./ci/steps/release.sh` first):
 
 ```
-./ci/dev/image/run.sh ./ci/steps/release-packages.sh
+IMAGE=centos7 ./ci/dev/image/run.sh ./ci/steps/release-packages.sh
 # The standalone release is in ./release-standalone
 # .deb, .rpm and the standalone archive are in ./release-packages
 ```
@@ -99,6 +99,13 @@ yarn test:standalone-release
 yarn package
 ```
 
+For a faster release build you can also run:
+
+```
+KEEP_MODULES=1 ./ci/steps/release.sh
+node ./release
+```
+
 ## Structure
 
 The `code-server` script serves an HTTP API to login and start a remote VS Code process.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "code-server",
   "license": "MIT",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "description": "Run VS Code on a remote server.",
   "homepage": "https://github.com/cdr/code-server",
   "bugs": {
@@ -39,6 +39,7 @@
     "@types/pem": "^1.9.5",
     "@types/safe-compare": "^1.1.0",
     "@types/semver": "^7.1.0",
+    "@types/split2": "^2.1.6",
     "@types/tar-fs": "^2.0.0",
     "@types/tar-stream": "^2.1.0",
     "@types/ws": "^7.2.6",
@@ -76,6 +77,7 @@
     "safe-buffer": "^5.1.1",
     "safe-compare": "^1.1.4",
     "semver": "^7.1.3",
+    "split2": "^3.2.2",
     "tar": "^6.0.1",
     "tar-fs": "^2.0.0",
     "ws": "^7.2.0",

src/node/cli.ts

@@ -47,6 +47,8 @@ export interface Args extends VsArgs {
   readonly _: string[]
   readonly "reuse-window"?: boolean
   readonly "new-window"?: boolean
+
+  readonly "coder-bind"?: OptionalString
 }
 
 interface Option<T> {
@@ -63,6 +65,11 @@ interface Option<T> {
    * Description of the option. Leave blank to hide the option.
    */
   description?: string
+
+  /**
+   * If marked as beta, the option is not printed unless $CS_BETA is set.
+   */
+  beta?: boolean
 }
 
 type OptionType<T> = T extends boolean
@@ -130,7 +137,8 @@ const options: Options<Required<Args>> = {
   "install-extension": {
     type: "string[]",
     description:
-      "Install or update a VS Code extension by id or vsix. The identifier of an extension is `${publisher}.${name}`. To install a specific version provide `@${version}`. For example: 'vscode.csharp@1.2.3'.",
+      "Install or update a VS Code extension by id or vsix. The identifier of an extension is `${publisher}.${name}`.\n" +
+      "To install a specific version provide `@${version}`. For example: 'vscode.csharp@1.2.3'.",
   },
   "enable-proposed-api": {
     type: "string[]",
@@ -155,6 +163,18 @@ const options: Options<Required<Args>> = {
   locale: { type: "string" },
   log: { type: LogLevel },
   verbose: { type: "boolean", short: "vvv", description: "Enable verbose logging." },
+
+  "coder-bind": {
+    type: OptionalString,
+    description: `
+      Securely bind code-server via Coder Cloud with the passed name. You'll get a URL like
+      https://myname.coder-cloud.com at which you can easily access your code-server instance.
+      Authorization is done via GitHub.
+      This is presently beta and requires being accepted for testing.
+      See https://github.com/cdr/code-server/discussions/2137
+    `,
+    beta: true,
+  },
 }
 
 export const optionDescriptions = (): string[] => {
@@ -166,12 +186,32 @@ export const optionDescriptions = (): string[] => {
     }),
     { short: 0, long: 0 },
   )
-  return entries.map(
+  return entries
-    ([k, v]) =>
+    .filter(([, v]) => {
-      `${" ".repeat(widths.short - (v.short ? v.short.length : 0))}${v.short ? `-${v.short}` : " "} --${k}${" ".repeat(
+      // If CS_BETA is set, we show beta options but if not, then we do not want
-        widths.long - k.length,
+      // to show beta options.
-      )} ${v.description}${typeof v.type === "object" ? ` [${Object.values(v.type).join(", ")}]` : ""}`,
+      return process.env.CS_BETA || !v.beta
+    })
+    .map(([k, v]) => {
+      const help = `${" ".repeat(widths.short - (v.short ? v.short.length : 0))}${
+        v.short ? `-${v.short}` : " "
+      } --${k} `
+      return (
+        help +
+        v.description
+          ?.trim()
+          .split(/\n/)
+          .map((line, i) => {
+            line = line.trim()
+            if (i === 0) {
+              return " ".repeat(widths.long - k.length) + line
+            }
+            return " ".repeat(widths.long + widths.short + 6) + line
+          })
+          .join("\n") +
+        (typeof v.type === "object" ? ` [${Object.values(v.type).join(", ")}]` : "")
       )
+    })
 }
 
 export const parse = (

src/node/coder-cloud.ts

@@ -0,0 +1,42 @@
+import { logger } from "@coder/logger"
+import { spawn } from "child_process"
+import path from "path"
+import split2 from "split2"
+
+const coderCloudAgent = path.resolve(__dirname, "../../lib/coder-cloud-agent")
+
+function runAgent(...args: string[]): Promise<void> {
+  logger.debug(`running agent with ${args}`)
+
+  const agent = spawn(coderCloudAgent, args, {
+    stdio: ["inherit", "inherit", "pipe"],
+  })
+
+  agent.stderr.pipe(split2()).on("data", (line) => {
+    line = line.replace(/^[0-9-]+ [0-9:]+ [^ ]+\t/, "")
+    logger.info(line)
+  })
+
+  return new Promise((res, rej) => {
+    agent.on("error", rej)
+
+    agent.on("close", (code) => {
+      if (code !== 0) {
+        rej({
+          message: `coder cloud agent exited with ${code}`,
+        })
+        return
+      }
+      res()
+    })
+  })
+}
+
+export function coderCloudBind(csAddr: string, serverName = ""): Promise<void> {
+  logger.info("Remember --coder-bind is a beta feature and requires being accepted for testing")
+  logger.info("See https://github.com/cdr/code-server/discussions/2137")
+  // addr needs to be in host:port format.
+  // So we trim the protocol.
+  csAddr = csAddr.replace(/^https?:\/\//, "")
+  return runAgent("bind", `--code-server-addr=${csAddr}`, serverName)
+}

src/node/entry.ts

@@ -12,6 +12,7 @@ import { StaticHttpProvider } from "./app/static"
 import { UpdateHttpProvider } from "./app/update"
 import { VscodeHttpProvider } from "./app/vscode"
 import { Args, bindAddrFromAllSources, optionDescriptions, parse, readConfigFile, setDefaults } from "./cli"
+import { coderCloudBind } from "./coder-cloud"
 import { AuthType, HttpServer, HttpServerOptions } from "./http"
 import { loadPlugins } from "./plugin"
 import { generateCertificate, hash, humanPath, open } from "./util"
@@ -34,7 +35,20 @@ try {
 const version = pkg.version || "development"
 const commit = pkg.commit || "development"
 
-const main = async (args: Args, cliArgs: Args, configArgs: Args): Promise<void> => {
+const main = async (args: Args, configArgs: Args): Promise<void> => {
+  if (args["coder-bind"]) {
+    // If we're being exposed to the cloud, we listen on a random address and disable auth.
+    args = {
+      ...args,
+      host: "localhost",
+      port: 0,
+      auth: AuthType.None,
+      socket: undefined,
+      cert: undefined,
+    }
+    logger.info("coder-bind: disabling auth and listening on random localhost port")
+  }
+
   if (!args.auth) {
     args = {
       ...args,
@@ -51,7 +65,7 @@ const main = async (args: Args, cliArgs: Args, configArgs: Args): Promise<void>
   if (args.auth === AuthType.Password && !password) {
     throw new Error("Please pass in a password via the config file or $PASSWORD")
   }
-  const [host, port] = bindAddrFromAllSources(cliArgs, configArgs)
+  const [host, port] = bindAddrFromAllSources(args, configArgs)
 
   // Spawn the main HTTP server.
   const options: HttpServerOptions = {
@@ -128,24 +142,33 @@ const main = async (args: Args, cliArgs: Args, configArgs: Args): Promise<void>
     await open(openAddress).catch(console.error)
     logger.info(`Opened ${openAddress}`)
   }
+
+  if (args["coder-bind"]) {
+    try {
+      await coderCloudBind(serverAddress!, args["coder-bind"].value)
+    } catch (err) {
+      logger.error(err.message)
+      ipcMain().exit(1)
+    }
+  }
 }
 
 async function entry(): Promise<void> {
-  const tryParse = async (): Promise<[Args, Args, Args]> => {
+  const tryParse = async (): Promise<[Args, Args]> => {
     try {
       const cliArgs = parse(process.argv.slice(2))
       const configArgs = await readConfigFile(cliArgs.config)
       // This prioritizes the flags set in args over the ones in the config file.
       let args = Object.assign(configArgs, cliArgs)
       args = await setDefaults(args)
-      return [args, cliArgs, configArgs]
+      return [args, configArgs]
     } catch (error) {
       console.error(error.message)
       process.exit(1)
     }
   }
 
-  const [args, cliArgs, configArgs] = await tryParse()
+  const [args, configArgs] = await tryParse()
   if (args.help) {
     console.log("code-server", version, commit)
     console.log("")
@@ -240,7 +263,7 @@ async function entry(): Promise<void> {
     vscode.write(JSON.stringify(pipeArgs))
     vscode.end()
   } else {
-    wrap(() => main(args, cliArgs, configArgs))
+    wrap(() => main(args, configArgs))
   }
 }
 

yarn.lock

@@ -1107,6 +1107,13 @@
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.3.tgz#3ad6ed949e7487e7bda6f886b4a2434a2c3d7b1a"
   integrity sha512-jQxClWFzv9IXdLdhSaTf16XI3NYe6zrEbckSpb5xhKfPbWgIyAY0AFyWWWfaiDcBuj3UHmMkCIwSRqpKMTZL2Q==
 
+"@types/split2@^2.1.6":
+  version "2.1.6"
+  resolved "https://registry.yarnpkg.com/@types/split2/-/split2-2.1.6.tgz#b095c9e064853824b22c67993d99b066777402b1"
+  integrity sha512-ddaFSOMuy2Rp97l6q/LEteQygvTQJuEZ+SRhxFKR0uXGsdbFDqX/QF2xoGcOqLQ8XV91v01SnAv2vpgihNgW/Q==
+  dependencies:
+    "@types/node" "*"
+
 "@types/tar-fs@^2.0.0":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/@types/tar-fs/-/tar-fs-2.0.0.tgz#db94cb4ea1cccecafe3d1a53812807efb4bbdbc1"
@@ -5996,7 +6003,7 @@ readable-stream@^2.0.2, readable-stream@^2.2.2, readable-stream@^2.3.3, readable
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
+readable-stream@^3.0.0, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
   integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
@@ -6621,6 +6628,13 @@ split-string@^3.0.1, split-string@^3.0.2:
   dependencies:
     extend-shallow "^3.0.0"
 
+split2@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f"
+  integrity sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==
+  dependencies:
+    readable-stream "^3.0.0"
+
 sprintf-js@~1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"