From 3e7582880efe85c71d93ed90760e5bb154e28dc5 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Wed, 6 May 2020 23:15:30 -0400 Subject: [PATCH 1/6] Make CI faster --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 652ef953..7e6b2e2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jobs: include: - name: Test if: tag IS blank - script: ./ci/container/exec.sh ./ci/steps/test.sh + script: ./ci/steps/test.sh deploy: null install: null - name: Linux Release From 193a45113c520094bb82871f88d803220c070d45 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Thu, 7 May 2020 00:06:55 -0400 Subject: [PATCH 2/6] Add back ARM with GH Actions --- .github/workflows/ci.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..c8c3b39f --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,15 @@ +name: ci + +on: [push, pull_request] + +jobs: + arm64-release: + name: linux-arm64-release + runs-on: [Linux, ARM64] + steps: + - uses: actions/checkout@v1 + - run: ./ci/steps/linux-release.sh + - uses: actions/upload-artifact@v2 + with: + name: release-github + path: ./release-github/* From 0ec1c69c0635c0ee5c209b5f1198fde1acfaf321 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Thu, 7 May 2020 20:44:32 -0400 Subject: [PATCH 3/6] Switch fully to GH Actions --- .github/workflows/ci.yaml | 57 +++++++++++++-- .travis.yml | 80 ---------------------- ci/README.md | 12 ++-- ci/container/Dockerfile | 63 ++++++++++------- ci/container/exec.sh | 24 ------- ci/dev/vscode.patch | 16 +++++ ci/release-container/Dockerfile | 7 +- ci/steps/linux-release.sh | 13 ---- ci/steps/publish-npm.sh | 11 --- ci/steps/{static-release.sh => release.sh} | 4 ++ 10 files changed, 115 insertions(+), 172 deletions(-) delete mode 100644 .travis.yml delete mode 100755 ci/container/exec.sh delete mode 100755 ci/steps/linux-release.sh delete mode 100755 ci/steps/publish-npm.sh rename ci/steps/{static-release.sh => release.sh} (81%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c8c3b39f..c0a107ac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,13 +3,60 @@ name: ci on: [push, pull_request] jobs: - arm64-release: - name: linux-arm64-release - runs-on: [Linux, ARM64] + test: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - run: ./ci/steps/linux-release.sh - - uses: actions/upload-artifact@v2 + - name: Run ./ci/steps/test.sh + uses: ./ci/container + with: + args: ./ci/steps/test.sh + + linux-amd64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Run ./ci/steps/release.sh + uses: ./ci/container + with: + args: ./ci/steps/release.sh + - name: Upload release artifacts + uses: actions/upload-artifact@v2 + with: + name: release-github + path: ./release-github/* + - name: Upload npm package artifact + uses: actions/upload-artifact@v2 + with: + name: npm-package + path: ./release + + linux-arm64: + runs-on: ubuntu-arm64-latest + steps: + - uses: actions/checkout@v1 + - name: Run ./ci/steps/release.sh + uses: ./ci/container + with: + args: ./ci/steps/release.sh + - name: Upload release artifacts + uses: actions/upload-artifact@v2 + with: + name: release-github + path: ./release-github/* + + macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v1 + - run: brew unlink node@12 + - run: brew install node + - run: ./ci/steps/release.sh + env: + # Otherwise we get a rate limited when fetching the ripgrep binary. + GITHUB_TOKEN: ${{ secrets.github_token }} + - name: Upload release artifacts + uses: actions/upload-artifact@v2 with: name: release-github path: ./release-github/* diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7e6b2e2b..00000000 --- a/.travis.yml +++ /dev/null @@ -1,80 +0,0 @@ -language: node_js -node_js: node - -jobs: - include: - - name: Test - if: tag IS blank - script: ./ci/steps/test.sh - deploy: null - install: null - - name: Linux Release - if: tag IS present - script: ./ci/steps/linux-release.sh - install: null - # Unfortunately ARM on travis is very unreliable. - # We see random build failures, logging output being truncated, build being killed - # due to no output even though we use travis_wait etc. - # So we've disabled it for now. - # - name: Linux Release - # if: tag IS present - # arch: arm64 - # script: | - # sudo apt-get update && sudo apt-get install -y jq || exit 1 - # travis_wait 60 ./ci/steps/linux-release.sh - # install: null - - name: MacOS Release - if: tag IS present - os: osx - # node 13/14 crashes in the build process for some reason. - node_js: 12 - script: | - HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install jq || exit 1 - travis_wait 60 ./ci/steps/static-release.sh || exit 1 - install: null - -before_deploy: - - echo "$JSON_KEY" | base64 --decode > ./ci/key.json - -deploy: - - provider: releases - edge: true - draft: true - overwrite: true - tag_name: $TRAVIS_TAG - target_commitish: $TRAVIS_COMMIT - name: $TRAVIS_TAG - file: - - release-github/*.tar.gz - - release-github/*.zip - - release-github/*.deb - - release-github/*.rpm - on: - tags: true - - provider: gcs - edge: true - bucket: "codesrv-ci.cdr.sh" - upload_dir: "releases" - key_file: ./ci/key.json - local_dir: ./release-gcp - on: - tags: true - # TODO: The gcs provider fails to install on arm64. - condition: $TRAVIS_CPU_ARCH == amd64 - - provider: script - edge: true - # We do not use the travis npm deploy integration as it does not allow us to - # deploy a subpath and and v2 which should, just errors out that the src does not exist - script: ./ci/steps/publish-npm.sh - on: - tags: true - condition: $TRAVIS_CPU_ARCH == amd64 && $TRAVIS_OS_NAME == linux - -cache: - timeout: 600 - yarn: true - directories: - - .cache - - out - - dist - - lib/vscode/.build/extensions diff --git a/ci/README.md b/ci/README.md index 5c1c1c2b..0c561742 100644 --- a/ci/README.md +++ b/ci/README.md @@ -68,15 +68,11 @@ This directory contains the container for CI. ## steps -This directory contains a few scripts used in CI. Just helps avoid clobbering .travis.yml. +This directory contains a few scripts used in CI. +Just helps avoid clobbering .travis.yml. - [./steps/test.sh](./steps/test.sh) - Runs `yarn ci` after ensuring VS Code is patched -- [./steps/static-release.sh](./steps/static-release.sh) +- [./steps/release.sh](./steps/release.sh) - Runs the full static build process for CI -- [./steps/linux-release.sh](./steps/linux-release.sh) - - Runs the full static build process for CI - - Packages the release into a .deb and .rpm - - Builds and pushes a docker release -- [./steps/publish-npm.sh](./steps/publish-npm.sh) - - Authenticates yarn and publishes the built package from `./release` + - Packages the release into a .deb and .rpm for linux diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile index dcbdd6aa..e40fe9aa 100644 --- a/ci/container/Dockerfile +++ b/ci/container/Dockerfile @@ -1,30 +1,41 @@ -FROM centos:7 +FROM debian -RUN yum update -y && yum install -y \ - devtoolset-6 \ - gcc-c++ \ - xz \ - ccache \ - git \ - wget \ - openssl \ - libxkbfile-devel \ - libsecret-devel \ - libx11-devel \ - gettext +RUN apt-get update -RUN yum install -y epel-release && \ - yum install -y ShellCheck jq golang +# Needed for debian repositories added below. +RUN apt-get install -y curl gnupg +# Installs node. +RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \ + apt-get install -y nodejs + +# Installs yarn. +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ + echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ + apt-get update && apt-get install -y yarn + +# Installs VS Code build deps. +RUN apt-get install -y build-essential \ + libsecret-1-dev \ + libx11-dev \ + libxkbfile-dev + +# Installs envsubst. +RUN apt-get install -y gettext-base + +# Misc build dependencies. +RUN apt-get install -y jq git rsync + +# Installs shellcheck. +RUN curl -L https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.$(uname -m).tar.xz | \ + tar -xJ && \ + mv shellcheck*/shellcheck /usr/local/bin && \ + rm -R shellcheck* + +# Install Go dependencies +RUN ARCH="$(dpkg --print-architecture)" && \ + curl "https://dl.google.com/go/go1.14.2.linux-$ARCH.tar.gz" | tar -C /usr/local -xz +ENV PATH=/usr/local/go/bin:/root/go/bin:$PATH +ENV GO111MODULE=on +RUN go get mvdan.cc/sh/v3/cmd/shfmt RUN go get github.com/goreleaser/nfpm/cmd/nfpm -ENV PATH=$PATH:/root/go/bin - -RUN mkdir /usr/share/node && cd /usr/share/node \ - && curl "https://nodejs.org/dist/v12.16.3/node-v12.16.3-linux-$(uname -m | sed 's/86_//; s/aarch/arm/').tar.xz" | tar xJ --strip-components=1 -- -ENV PATH "$PATH:/usr/share/node/bin" -RUN npm install -g yarn@1.22.4 - -RUN curl -L "https://github.com/mvdan/sh/releases/download/v3.0.1/shfmt_v3.0.1_linux_$(uname -m | sed 's/x86_/amd/; s/aarch64/arm/')" > /usr/local/bin/shfmt \ - && chmod +x /usr/local/bin/shfmt - -ENTRYPOINT ["/bin/bash", "-c"] diff --git a/ci/container/exec.sh b/ci/container/exec.sh deleted file mode 100755 index 57627b06..00000000 --- a/ci/container/exec.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - docker build ci/container - imageTag="$(docker build -q ci/container)" - docker run \ - --rm \ - -e CI \ - -e GITHUB_TOKEN \ - -e TRAVIS_TAG \ - -e NPM_TOKEN \ - -v "$(yarn cache dir):/usr/local/share/.cache/yarn/v6" \ - $(if [[ -f ~/.npmrc ]]; then echo -v "$HOME/.npmrc:/root/.npmrc"; fi) \ - -v "$PWD:/repo" \ - -w /repo \ - $(if [[ -t 0 ]]; then echo -it; fi) \ - "$imageTag" \ - "$*" -} - -main "$@" diff --git a/ci/dev/vscode.patch b/ci/dev/vscode.patch index 0f1302ff..e967b3b2 100644 --- a/ci/dev/vscode.patch +++ b/ci/dev/vscode.patch @@ -48,6 +48,22 @@ index 7a2320d828..5768890636 100644 +// yarnInstall('test/smoke'); // node modules required for smoketest +// yarnInstall('test/integration/browser'); // node modules required for integration yarnInstallBuildDependencies(); // node modules for watching, specific to host node version, not electron +diff --git a/build/npm/preinstall.js b/build/npm/preinstall.js +index cb88d37ade..6b3253af0a 100644 +--- a/build/npm/preinstall.js ++++ b/build/npm/preinstall.js +@@ -8,8 +8,9 @@ let err = false; + const majorNodeVersion = parseInt(/^(\d+)\./.exec(process.versions.node)[1]); + + if (majorNodeVersion < 10 || majorNodeVersion >= 13) { +- console.error('\033[1;31m*** Please use node >=10 and <=12.\033[0;0m'); +- err = true; ++ // We are ok building above Node 12. ++ // console.error('\033[1;31m*** Please use node >=10 and <=12.\033[0;0m'); ++ // err = true; + } + + const cp = require('child_process'); diff --git a/coder.js b/coder.js new file mode 100644 index 0000000000..0170b47241 diff --git a/ci/release-container/Dockerfile b/ci/release-container/Dockerfile index 0a99242f..cafcea28 100644 --- a/ci/release-container/Dockerfile +++ b/ci/release-container/Dockerfile @@ -27,11 +27,8 @@ ENV SHELL=/bin/bash RUN adduser --gecos '' --disabled-password coder && \ echo "coder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/nopasswd -SHELL ["/bin/bash", "-c"] - -COPY ci/lib.sh /tmp/lib.sh -RUN source /tmp/lib.sh && rm /tmp/lib.sh && \ - curl -L "https://github.com/boxboat/fixuid/releases/download/v0.4.1/fixuid-0.4.1-linux-$(arch).tar.gz" | tar -C /usr/local/bin -xzf - && \ +RUN ARCH="$(dpkg --print-architecture)" && \ + curl -L "https://github.com/boxboat/fixuid/releases/download/v0.4.1/fixuid-0.4.1-linux-$ARCH.tar.gz" | tar -C /usr/local/bin -xzf - && \ chown root:root /usr/local/bin/fixuid && \ chmod 4755 /usr/local/bin/fixuid && \ mkdir -p /etc/fixuid && \ diff --git a/ci/steps/linux-release.sh b/ci/steps/linux-release.sh deleted file mode 100755 index 2d5b34d0..00000000 --- a/ci/steps/linux-release.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - source ./ci/lib.sh - - ./ci/container/exec.sh ./ci/steps/static-release.sh - ./ci/container/exec.sh yarn pkg - ./ci/release-container/push.sh -} - -main "$@" diff --git a/ci/steps/publish-npm.sh b/ci/steps/publish-npm.sh deleted file mode 100755 index c4bcb967..00000000 --- a/ci/steps/publish-npm.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -main() { - cd "$(dirname "$0")/../.." - - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc - ./ci/container/exec.sh yarn publish --non-interactive release -} - -main "$@" diff --git a/ci/steps/static-release.sh b/ci/steps/release.sh similarity index 81% rename from ci/steps/static-release.sh rename to ci/steps/release.sh index c0a70431..b8ac7202 100755 --- a/ci/steps/static-release.sh +++ b/ci/steps/release.sh @@ -11,6 +11,10 @@ main() { STATIC=1 yarn release ./ci/build/test-static-release.sh ./ci/build/archive-static-release.sh + + if [[ $OSTYPE == linux* ]]; then + yarn pkg + fi } main "$@" From bc453b5f0d62135d221264379eaba7933c3eebff Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Thu, 7 May 2020 23:48:49 -0400 Subject: [PATCH 4/6] Switch to a single job to build the npm package The architecture specific jobs pull it in and then build releases. Much faster! --- .github/workflows/ci.yaml | 44 +++++++++++++++++++------ ci/README.md | 16 ++++++--- ci/build/build-release.sh | 56 ++++---------------------------- ci/build/build-static-release.sh | 25 ++++++++++++++ ci/build/lib.sh | 10 ++++++ ci/steps/release.sh | 8 +---- ci/steps/static-release.sh | 16 +++++++++ package.json | 1 + 8 files changed, 104 insertions(+), 72 deletions(-) create mode 100755 ci/build/build-static-release.sh create mode 100644 ci/build/lib.sh create mode 100644 ci/steps/static-release.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c0a107ac..c6413922 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: with: args: ./ci/steps/test.sh - linux-amd64: + release: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 @@ -20,25 +20,44 @@ jobs: uses: ./ci/container with: args: ./ci/steps/release.sh - - name: Upload release artifacts - uses: actions/upload-artifact@v2 - with: - name: release-github - path: ./release-github/* - name: Upload npm package artifact uses: actions/upload-artifact@v2 with: name: npm-package path: ./release + linux-amd64: + needs: release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Download npm package + uses: actions/download-artifact@v1 + with: + name: npm-package + - name: Run ./ci/steps/static-release.sh + uses: ./ci/container + with: + args: ./ci/steps/static-release.sh + - name: Upload release artifacts + uses: actions/upload-artifact@v2 + with: + name: release-github + path: ./release-github/* + linux-arm64: + needs: release runs-on: ubuntu-arm64-latest steps: - uses: actions/checkout@v1 - - name: Run ./ci/steps/release.sh + - name: Download npm package + uses: actions/download-artifact@v1 + with: + name: npm-package + - name: Run ./ci/steps/static-release.sh uses: ./ci/container with: - args: ./ci/steps/release.sh + args: ./ci/steps/static-release.sh - name: Upload release artifacts uses: actions/upload-artifact@v2 with: @@ -46,14 +65,19 @@ jobs: path: ./release-github/* macos: + needs: release runs-on: macos-latest steps: - uses: actions/checkout@v1 + - name: Download npm package + uses: actions/download-artifact@v1 + with: + name: npm-package - run: brew unlink node@12 - run: brew install node - - run: ./ci/steps/release.sh + - run: ./ci/steps/static-release.sh env: - # Otherwise we get a rate limited when fetching the ripgrep binary. + # Otherwise we get rate limited when fetching the ripgrep binary. GITHUB_TOKEN: ${{ secrets.github_token }} - name: Upload release artifacts uses: actions/upload-artifact@v2 diff --git a/ci/README.md b/ci/README.md index 0c561742..a5397f59 100644 --- a/ci/README.md +++ b/ci/README.md @@ -33,14 +33,17 @@ This directory contains scripts used for the development of code-server. This directory contains the scripts used to build code-server. +- [./lib.sh](./lib.sh) + - Contains code duplicated across these scripts. - [./build/build-code-server.sh](./build/build-code-server.sh) (`yarn build`) - Builds code-server into ./out and bundles the frontend into ./dist. - [./build/build-vscode.sh](./build/build-vscode.sh) (`yarn build:vscode`) - Builds vscode into ./lib/vscode/out-vscode. - [./build/build-release.sh](./build/build-release.sh) (`yarn release`) - - Bundles the output of the above two scripts into a single node module at ./release. - - Will build a static release with node/node_modules into `./release-static` - if `STATIC=1` is set. + - Bundles the output of the above two scripts into a single node module at `./release`. +- [./build/build-static-release.sh](./build/build-static-release.sh) (`yarn release:static`) + - Requires a release already built in `./release`. + - Will build a static release with node and node_modules into `./release-static` - [./build/clean.sh](./build/clean.sh) (`yarn clean`) - Removes all git ignored files like build artifacts. - Will also `git reset --hard lib/vscode` @@ -74,5 +77,8 @@ Just helps avoid clobbering .travis.yml. - [./steps/test.sh](./steps/test.sh) - Runs `yarn ci` after ensuring VS Code is patched - [./steps/release.sh](./steps/release.sh) - - Runs the full static build process for CI - - Packages the release into a .deb and .rpm for linux + - Runs the full release process + - Generates the npm package at `./release` +- [./steps/static-release.sh](./steps/static-release.sh) + - Takes the output of the previous script and bundles it into a self-contained archive into `./github-release` + - Also outputs .deb/.rpm if on linux. diff --git a/ci/build/build-release.sh b/ci/build/build-release.sh index 4a86874c..0205488d 100755 --- a/ci/build/build-release.sh +++ b/ci/build/build-release.sh @@ -1,28 +1,18 @@ #!/usr/bin/env bash set -euo pipefail -# This script requires code-server and vscode to be built with -# matching MINIFY. - -# RELEASE_PATH is the destination directory for the release from the root. -# Defaults to release -RELEASE_PATH="${RELEASE_PATH-release}" - -# STATIC controls whether node and node_modules are packaged into the release. -# Disabled by default. -STATIC="${STATIC-}" +# This script requires code-server and vscode to be built with matching MINIFY. # MINIFY controls whether minified vscode is bundled and whether # any included node_modules are pruned for production. MINIFY="${MINIFY-true}" -VSCODE_SRC_PATH="lib/vscode" - -VSCODE_OUT_PATH="$RELEASE_PATH/lib/vscode" - main() { cd "$(dirname "${0}")/../.." - source ./ci/lib.sh + source ./ci/build/lib.sh + + VSCODE_SRC_PATH="lib/vscode" + VSCODE_OUT_PATH="$RELEASE_PATH/lib/vscode" mkdir -p "$RELEASE_PATH" @@ -32,20 +22,6 @@ main() { rsync README.md "$RELEASE_PATH" rsync LICENSE.txt "$RELEASE_PATH" rsync ./lib/vscode/ThirdPartyNotices.txt "$RELEASE_PATH" - - if [[ $STATIC ]]; then - rsync "$RELEASE_PATH/" "$RELEASE_PATH-static" - RELEASE_PATH+=-static - VSCODE_OUT_PATH="$RELEASE_PATH/lib/vscode" - - bundle_node - else - rm -Rf "$VSCODE_OUT_PATH/extensions/node_modules" - fi -} - -rsync() { - command rsync -a --del "$@" } bundle_code_server() { @@ -76,6 +52,7 @@ bundle_vscode() { mkdir -p "$VSCODE_OUT_PATH" rsync "$VSCODE_SRC_PATH/out-vscode${MINIFY+-min}/" "$VSCODE_OUT_PATH/out" rsync "$VSCODE_SRC_PATH/.build/extensions/" "$VSCODE_OUT_PATH/extensions" + rm -Rf "$VSCODE_OUT_PATH/extensions/node_modules" rsync "$VSCODE_SRC_PATH/extensions/package.json" "$VSCODE_OUT_PATH/extensions" rsync "$VSCODE_SRC_PATH/extensions/yarn.lock" "$VSCODE_OUT_PATH/extensions" rsync "$VSCODE_SRC_PATH/extensions/postinstall.js" "$VSCODE_OUT_PATH/extensions" @@ -103,25 +80,4 @@ EOF jq 'del(.scripts)' < "$VSCODE_SRC_PATH/package.json" > "$VSCODE_OUT_PATH/package.json" } -bundle_node() { - # We cannot find the path to node from $PATH because yarn shims a script to ensure - # we use the same version it's using so we instead run a script with yarn that - # will print the path to node. - local node_path - node_path="$(yarn -s node <<< 'console.info(process.execPath)')" - - mkdir -p "$RELEASE_PATH/bin" - rsync ./ci/build/code-server.sh "$RELEASE_PATH/bin/code-server" - rsync "$node_path" "$RELEASE_PATH/lib/node" - - rsync node_modules "$RELEASE_PATH" - rsync "$VSCODE_SRC_PATH/node_modules" "$VSCODE_OUT_PATH" - - if [[ $MINIFY ]]; then - pushd "$RELEASE_PATH" - yarn --production - popd - fi -} - main "$@" diff --git a/ci/build/build-static-release.sh b/ci/build/build-static-release.sh new file mode 100755 index 00000000..f2e28fa3 --- /dev/null +++ b/ci/build/build-static-release.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "${0}")/../.." + source ./ci/build/lib.sh + + rsync "$RELEASE_PATH/" "$RELEASE_PATH-static" + RELEASE_PATH+=-static + + # We cannot find the path to node from $PATH because yarn shims a script to ensure + # we use the same version it's using so we instead run a script with yarn that + # will print the path to node. + local node_path + node_path="$(yarn -s node <<< 'console.info(process.execPath)')" + + mkdir -p "$RELEASE_PATH/bin" + rsync ./ci/build/code-server.sh "$RELEASE_PATH/bin/code-server" + rsync "$node_path" "$RELEASE_PATH/lib/node" + + cd "$RELEASE_PATH" + yarn --production +} + +main "$@" diff --git a/ci/build/lib.sh b/ci/build/lib.sh new file mode 100644 index 00000000..f263adb8 --- /dev/null +++ b/ci/build/lib.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +source ./ci/lib.sh + +# RELEASE_PATH is the destination directory for the release from the root. +# Defaults to release +RELEASE_PATH="${RELEASE_PATH-release}" + +rsync() { + command rsync -a --del "$@" +} diff --git a/ci/steps/release.sh b/ci/steps/release.sh index b8ac7202..5c935522 100755 --- a/ci/steps/release.sh +++ b/ci/steps/release.sh @@ -8,13 +8,7 @@ main() { yarn vscode yarn build yarn build:vscode - STATIC=1 yarn release - ./ci/build/test-static-release.sh - ./ci/build/archive-static-release.sh - - if [[ $OSTYPE == linux* ]]; then - yarn pkg - fi + yarn release } main "$@" diff --git a/ci/steps/static-release.sh b/ci/steps/static-release.sh new file mode 100644 index 00000000..62c5e8be --- /dev/null +++ b/ci/steps/static-release.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + + yarn release:static + ./ci/build/test-static-release.sh + ./ci/build/archive-static-release.sh + + if [[ $OSTYPE == linux* ]]; then + yarn pkg + fi +} + +main "$@" diff --git a/package.json b/package.json index 975f3be2..461e42a0 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "build": "./ci/build/build-code-server.sh", "build:vscode": "./ci/build/build-vscode.sh", "release": "./ci/build/build-release.sh", + "release:static": "./ci/build/build-static-release.sh", "pkg": "./ci/build/build-static-pkgs.sh", "_____": "", "fmt": "./ci/dev/fmt.sh", From 6282cd7e7b84c4de32c7a38f68ee9db208e35a71 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Fri, 8 May 2020 00:34:20 -0400 Subject: [PATCH 5/6] Simplify packaging and improve scripts Much better test now as well. --- .dockerignore | 2 +- .github/workflows/ci.yaml | 31 +++++----- .gitignore | 2 +- ci/README.md | 11 ++-- ci/build/archive-static-release.sh | 41 ------------- ci/build/build-packages.sh | 59 +++++++++++++++++++ ci/build/build-release.sh | 5 +- ci/build/build-static-pkgs.sh | 24 -------- ci/build/lib.sh | 0 ci/build/test-static-release.sh | 17 ++++-- ci/release-container/Dockerfile | 2 +- .../{static-release.sh => release-static.sh} | 6 +- package.json | 2 +- 13 files changed, 100 insertions(+), 102 deletions(-) delete mode 100755 ci/build/archive-static-release.sh create mode 100755 ci/build/build-packages.sh delete mode 100755 ci/build/build-static-pkgs.sh mode change 100644 => 100755 ci/build/lib.sh rename ci/steps/{static-release.sh => release-static.sh} (61%) mode change 100644 => 100755 diff --git a/.dockerignore b/.dockerignore index b8940df7..9bcce7a8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,3 @@ ** -!release-github +!release-packages !ci diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c6413922..411f5e9d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,18 +32,19 @@ jobs: steps: - uses: actions/checkout@v1 - name: Download npm package - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: npm-package - - name: Run ./ci/steps/static-release.sh + path: ./release + - name: Run ./ci/steps/release-static.sh uses: ./ci/container with: - args: ./ci/steps/static-release.sh + args: ./ci/steps/release-static.sh - name: Upload release artifacts uses: actions/upload-artifact@v2 with: - name: release-github - path: ./release-github/* + name: release-packages + path: ./release-packages linux-arm64: needs: release @@ -51,18 +52,19 @@ jobs: steps: - uses: actions/checkout@v1 - name: Download npm package - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: npm-package - - name: Run ./ci/steps/static-release.sh + path: ./release + - name: Run ./ci/steps/release-static.sh uses: ./ci/container with: - args: ./ci/steps/static-release.sh + args: ./ci/steps/release-static.sh - name: Upload release artifacts uses: actions/upload-artifact@v2 with: - name: release-github - path: ./release-github/* + name: release-packages + path: ./release-packages macos: needs: release @@ -70,17 +72,18 @@ jobs: steps: - uses: actions/checkout@v1 - name: Download npm package - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: npm-package + path: ./release - run: brew unlink node@12 - run: brew install node - - run: ./ci/steps/static-release.sh + - run: ./ci/steps/release-static.sh env: # Otherwise we get rate limited when fetching the ripgrep binary. GITHUB_TOKEN: ${{ secrets.github_token }} - name: Upload release artifacts uses: actions/upload-artifact@v2 with: - name: release-github - path: ./release-github/* + name: release-packages + path: ./release-packages diff --git a/.gitignore b/.gitignore index 86557de4..35daca8b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,6 @@ dist* out* release/ release-static/ -release-github/ +release-packages/ release-gcp/ node_modules diff --git a/ci/README.md b/ci/README.md index a5397f59..bf9e1875 100644 --- a/ci/README.md +++ b/ci/README.md @@ -32,6 +32,7 @@ This directory contains scripts used for the development of code-server. ## build This directory contains the scripts used to build code-server. +You can disable minification by setting `MINIFY=`. - [./lib.sh](./lib.sh) - Contains code duplicated across these scripts. @@ -50,12 +51,11 @@ This directory contains the scripts used to build code-server. - Useful to do a clean build. - [./build/code-server.sh](./build/code-server.sh) - Copied into static releases to run code-server with the bundled node binary. -- [./build/archive-static-release.sh](./build/archive-static-release.sh) - - Archives `./release-static` into a tar/zip for CI with the proper directory name scheme - [./build/test-release.sh](./build/test-static-release.sh) - Ensures code-server in the `./release-static` directory runs -- [./build/build-static-pkgs.sh](./build/build-static-pkgs.sh) (`yarn pkg`) - - Uses [nfpm](https://github.com/goreleaser/nfpm) to generate .deb and .rpm from a static release +- [./build/build-packages.sh](./build/build-static-pkgs.sh) (`yarn package`) + - Packages `./release-static` into an archive in `./release-packages` + - If on linux, [nfpm](https://github.com/goreleaser/nfpm) is used to generate .deb and .rpm - [./build/nfpm.yaml](./build/nfpm.yaml) - Used to configure [nfpm](https://github.com/goreleaser/nfpm) to generate .deb and .rpm - [./build/code-server-nfpm.sh](./build/code-server-nfpm.sh) @@ -80,5 +80,4 @@ Just helps avoid clobbering .travis.yml. - Runs the full release process - Generates the npm package at `./release` - [./steps/static-release.sh](./steps/static-release.sh) - - Takes the output of the previous script and bundles it into a self-contained archive into `./github-release` - - Also outputs .deb/.rpm if on linux. + - Takes the output of the previous script and generates a static release and packages diff --git a/ci/build/archive-static-release.sh b/ci/build/archive-static-release.sh deleted file mode 100755 index 37a85617..00000000 --- a/ci/build/archive-static-release.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Generates static code-server releases for CI. -# This script assumes that a static release is built already. - -main() { - cd "$(dirname "${0}")/../.." - source ./ci/lib.sh - - VERSION="$(pkg_json_version)" - - local OS - OS="$(os)" - - local ARCH - ARCH="$(arch)" - - local archive_name="code-server-$VERSION-$OS-$ARCH" - mkdir -p release-github - - local ext - if [[ $OS == "linux" ]]; then - ext=".tar.gz" - tar -czf "release-github/$archive_name$ext" --transform "s/^\.\/release-static/$archive_name/" ./release-static - else - mv ./release-static "./$archive_name" - ext=".zip" - zip -r "release-github/$archive_name$ext" "./$archive_name" - mv "./$archive_name" ./release-static - fi - - echo "done (release-github/$archive_name)" - - mkdir -p "release-gcp/$VERSION" - cp "release-github/$archive_name$ext" "./release-gcp/$VERSION/$OS-$ARCH$ext" - mkdir -p "release-gcp/latest" - cp "./release-github/$archive_name$ext" "./release-gcp/latest/$OS-$ARCH$ext" -} - -main "$@" diff --git a/ci/build/build-packages.sh b/ci/build/build-packages.sh new file mode 100755 index 00000000..7dc65df0 --- /dev/null +++ b/ci/build/build-packages.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Packages code-server for the current OS and architecture into ./release-packages. +# This script assumes that a static release is built already into ./release-static. + +main() { + cd "$(dirname "${0}")/../.." + source ./ci/lib.sh + + export VERSION + VERSION="$(pkg_json_version)" + + local OS + OS="$(os)" + + export ARCH + ARCH="$(arch)" + + local archive_name="code-server-$VERSION-$OS-$ARCH" + mkdir -p release-packages + + local ext + if [[ $OS == "linux" ]]; then + ext=".tar.gz" + tar -czf "release-packages/$archive_name$ext" --transform "s/^\.\/release-static/$archive_name/" ./release-static + else + mv ./release-static "./$archive_name" + ext=".zip" + zip -r "release-packages/$archive_name$ext" "./$archive_name" + mv "./$archive_name" ./release-static + fi + + echo "done (release-packages/$archive_name)" + + release_gcp + + if [[ $OSTYPE == linux* ]]; then + release_nfpm + fi +} + +release_gcp() { + mkdir -p "release-gcp/$VERSION" + cp "release-packages/$archive_name$ext" "./release-gcp/$VERSION/$OS-$ARCH$ext" + mkdir -p "release-gcp/latest" + cp "./release-packages/$archive_name$ext" "./release-gcp/latest/$OS-$ARCH$ext" +} + +# Generates deb and rpm packages. +release_nfpm() { + local nfpm_config + nfpm_config=$(envsubst < ./ci/build/nfpm.yaml) + + nfpm pkg -f <(echo "$nfpm_config") --target release-packages/code-server-"$VERSION-$ARCH.deb" + nfpm pkg -f <(echo "$nfpm_config") --target release-packages/code-server-"$VERSION-$ARCH.rpm" +} + +main "$@" diff --git a/ci/build/build-release.sh b/ci/build/build-release.sh index 0205488d..d96a466d 100755 --- a/ci/build/build-release.sh +++ b/ci/build/build-release.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -# This script requires code-server and vscode to be built with matching MINIFY. +# This script requires vscode to be built with matching MINIFY. -# MINIFY controls whether minified vscode is bundled and whether -# any included node_modules are pruned for production. +# MINIFY controls whether minified vscode is bundled. MINIFY="${MINIFY-true}" main() { diff --git a/ci/build/build-static-pkgs.sh b/ci/build/build-static-pkgs.sh deleted file mode 100755 index 659039ae..00000000 --- a/ci/build/build-static-pkgs.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Generates deb and rpm packages for CI. -# Assumes a static release has already been built. - -main() { - cd "$(dirname "${0}")/../.." - source ./ci/lib.sh - - VERSION="$(pkg_json_version)" - export VERSION - - ARCH="$(arch)" - export ARCH - - local nfpm_config - nfpm_config=$(envsubst < ./ci/build/nfpm.yaml) - - nfpm pkg -f <(echo "$nfpm_config") --target release-github/code-server-"$VERSION-$ARCH.deb" - nfpm pkg -f <(echo "$nfpm_config") --target release-github/code-server-"$VERSION-$ARCH.rpm" -} - -main "$@" diff --git a/ci/build/lib.sh b/ci/build/lib.sh old mode 100644 new mode 100755 diff --git a/ci/build/test-static-release.sh b/ci/build/test-static-release.sh index 02177fe3..93d9e991 100755 --- a/ci/build/test-static-release.sh +++ b/ci/build/test-static-release.sh @@ -7,14 +7,21 @@ set -euo pipefail main() { cd "$(dirname "${0}")/../.." - local output - output=$(./release-static/bin/code-server --list-extensions 2>&1) - if echo "$output" | grep 'was compiled against a different Node.js version'; then - echo "$output" + local EXTENSIONS_DIR + EXTENSIONS_DIR="$(mktemp -d)" + + echo "Testing static release" + + ./release-static/bin/code-server --extensions-dir "$EXTENSIONS_DIR" --install-extension ms-python.python + local installed_extensions + installed_extensions="$(./release-static/bin/code-server --extensions-dir "$EXTENSIONS_DIR" --list-extensions 2>&1)" + if [[ $installed_extensions != "ms-python.python" ]]; then + echo "Unexpected output from listing extensions:" + echo "$installed_extensions" exit 1 fi - echo "Build ran successfully" + echo "Static release works correctly" } main "$@" diff --git a/ci/release-container/Dockerfile b/ci/release-container/Dockerfile index cafcea28..a0f20936 100644 --- a/ci/release-container/Dockerfile +++ b/ci/release-container/Dockerfile @@ -34,7 +34,7 @@ RUN ARCH="$(dpkg --print-architecture)" && \ mkdir -p /etc/fixuid && \ printf "user: coder\ngroup: coder\n" > /etc/fixuid/config.yml -COPY release-github/code-server*.deb /tmp/ +COPY release-packages/code-server*.deb /tmp/ RUN dpkg -i /tmp/code-server*.deb && rm /tmp/code-server*.deb EXPOSE 8080 diff --git a/ci/steps/static-release.sh b/ci/steps/release-static.sh old mode 100644 new mode 100755 similarity index 61% rename from ci/steps/static-release.sh rename to ci/steps/release-static.sh index 62c5e8be..6d565421 --- a/ci/steps/static-release.sh +++ b/ci/steps/release-static.sh @@ -6,11 +6,7 @@ main() { yarn release:static ./ci/build/test-static-release.sh - ./ci/build/archive-static-release.sh - - if [[ $OSTYPE == linux* ]]; then - yarn pkg - fi + yarn package } main "$@" diff --git a/package.json b/package.json index 461e42a0..b6a311c6 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build:vscode": "./ci/build/build-vscode.sh", "release": "./ci/build/build-release.sh", "release:static": "./ci/build/build-static-release.sh", - "pkg": "./ci/build/build-static-pkgs.sh", + "package": "./ci/build/build-packages.sh", "_____": "", "fmt": "./ci/dev/fmt.sh", "lint": "./ci/dev/lint.sh", From e9fe4c04668301bad313287299f0a895dac7e48b Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Fri, 8 May 2020 01:42:46 -0400 Subject: [PATCH 6/6] Document release process --- .github/workflows/ci.yaml | 2 +- ci/README.md | 14 ++++++++++++++ package.json | 2 +- yarn.lock | 8 ++++---- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 411f5e9d..ffb864fd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -66,7 +66,7 @@ jobs: name: release-packages path: ./release-packages - macos: + macos-amd64: needs: release runs-on: macos-latest steps: diff --git a/ci/README.md b/ci/README.md index bf9e1875..038f3aab 100644 --- a/ci/README.md +++ b/ci/README.md @@ -6,6 +6,20 @@ Many of these scripts contain more detailed documentation and options in comment Any file and directory added into this tree should be documented here. +## Publishing a release + +1. Change the version of code-server in `package.json` and push this commit. +1. CI will run and generate an NPM package and release packages that you can download + as artifacts on Github Actions. +1. Create a new draft release with the built release packages. +1. Run some basic sanity tests on one of the released packages. +1. Publish. +1. Download the built npm package and publish it. +1. Place the debian releases into `./release-packages` and then push the docker + image with `./ci/release-container/push.sh`. + 1. This will need to be ran on an ARM64 instance as well. + 1. At some point we need to automate this. + ## dev This directory contains scripts used for the development of code-server. diff --git a/package.json b/package.json index b6a311c6..05287881 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "vfile-message": "^2.0.2" }, "dependencies": { - "@coder/logger": "1.1.14", + "@coder/logger": "1.1.11", "adm-zip": "^0.4.14", "fs-extra": "^8.1.0", "http-proxy": "^1.18.0", diff --git a/yarn.lock b/yarn.lock index 8a5195a6..473107bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -792,10 +792,10 @@ lodash "^4.17.13" to-fast-properties "^2.0.0" -"@coder/logger@1.1.14": - version "1.1.14" - resolved "https://registry.yarnpkg.com/@coder/logger/-/logger-1.1.14.tgz#0242da33e0245834361dd078e31280fc1c976b7e" - integrity sha512-NuTvsOH3dqrXn/8Pbs5zy7l0gLqOSC/TPRl3nexdP/897lgG/vtHNQHrUwTBTzTzihH1ON4lklDxJjY0hD4UPg== +"@coder/logger@1.1.11": + version "1.1.11" + resolved "https://registry.yarnpkg.com/@coder/logger/-/logger-1.1.11.tgz#e6f36dba9436ae61e66e3f66787d75c768617605" + integrity sha512-EEh1dqSU0AaqjjjMsVqumgZGbrZimKFKIb4t5E6o3FLfVUxJCReSME78Yj2N1xWUVAHMnqafDCxLostpuIotzw== "@iarna/toml@^2.2.0": version "2.2.5"