From 231e31656a5f5db42d1306631f79dadd75f2e680 Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Fri, 8 May 2020 03:08:30 -0400 Subject: [PATCH 1/2] Automate release process --- .github/workflows/ci.yaml | 2 +- .github/workflows/publish.yaml | 41 +++++++++++++++++++++++++++++++++ ci/README.md | 27 +++++++++++++--------- ci/container/Dockerfile | 8 +++---- ci/release-container/Dockerfile | 4 ++-- ci/release-container/push.sh | 4 ---- ci/steps/lib.sh | 29 +++++++++++++++++++++++ ci/steps/publish-docker.sh | 16 +++++++++++++ ci/steps/publish-npm.sh | 16 +++++++++++++ 9 files changed, 125 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/publish.yaml create mode 100755 ci/steps/lib.sh create mode 100755 ci/steps/publish-docker.sh create mode 100755 ci/steps/publish-npm.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ffb864fd..4a10a92e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,7 +81,7 @@ jobs: - run: ./ci/steps/release-static.sh env: # Otherwise we get rate limited when fetching the ripgrep binary. - GITHUB_TOKEN: ${{ secrets.github_token }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload release artifacts uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..4230a995 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,41 @@ +name: publish + +on: + release: + types: [published] + +jobs: + npm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Run ./ci/steps/publish-npm.sh + uses: ./ci/container + with: + args: ./ci/steps/publish-npm.sh + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + + docker-amd64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Run ./ci/steps/publish-docker.sh + uses: ./ci/container + with: + args: ./ci/steps/publish-docker.sh + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + docker-arm64: + runs-on: ubuntu-arm64-latest + steps: + - uses: actions/checkout@v1 + - name: Run ./ci/steps/publish-docker.sh + uses: ./ci/container + with: + args: ./ci/steps/publish-docker.sh + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/ci/README.md b/ci/README.md index 038f3aab..add6b081 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,17 +8,15 @@ Any file and directory added into this tree should be documented here. ## Publishing a release -1. Change the version of code-server in `package.json` and push this commit. -1. CI will run and generate an NPM package and release packages that you can download - as artifacts on Github Actions. -1. Create a new draft release with the built release packages. -1. Run some basic sanity tests on one of the released packages. -1. Publish. -1. Download the built npm package and publish it. -1. Place the debian releases into `./release-packages` and then push the docker - image with `./ci/release-container/push.sh`. - 1. This will need to be ran on an ARM64 instance as well. - 1. At some point we need to automate this. +1. Update the version of code-server in `package.json` and push a commit +1. CI will run and generate the `npm-package` and `release-packages` artifacts on the GH actions workflow +1. Create a new draft release and attach all the files in `release-packages` +1. Run some basic sanity tests on one of the released packages +1. Publish the release +1. CI will automatically grab the artifacts and then + 1. Publish the NPM package. + 1. Publish the AMD64 docker image. + 1. Publish the ARM64 docker image. ## dev @@ -95,3 +93,10 @@ Just helps avoid clobbering .travis.yml. - Generates the npm package at `./release` - [./steps/static-release.sh](./steps/static-release.sh) - Takes the output of the previous script and generates a static release and packages +- [./steps/lib.sh](./steps/lib.sh) + - Contains helpers to download artifacts from github actions workflow runs +- [./steps/publish-npm.sh](./steps/publish-npm.sh) + - Grabs the `npm-package` release artifact for the current commit and publishes it on NPM +- [./steps/publish-docker.sh](./steps/publish-docker.sh) + - Grabs the `release-packages` release artifact for the current commit and builds a docker + image with it and publishes that onto docker hub diff --git a/ci/container/Dockerfile b/ci/container/Dockerfile index e40fe9aa..87578576 100644 --- a/ci/container/Dockerfile +++ b/ci/container/Dockerfile @@ -6,11 +6,11 @@ RUN apt-get update RUN apt-get install -y curl gnupg # Installs node. -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \ +RUN curl -sSL https://deb.nodesource.com/setup_14.x | bash - && \ apt-get install -y nodejs # Installs yarn. -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ +RUN curl -sSL https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \ echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \ apt-get update && apt-get install -y yarn @@ -27,14 +27,14 @@ RUN apt-get install -y gettext-base RUN apt-get install -y jq git rsync # Installs shellcheck. -RUN curl -L https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.$(uname -m).tar.xz | \ +RUN curl -sSL https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.$(uname -m).tar.xz | \ tar -xJ && \ mv shellcheck*/shellcheck /usr/local/bin && \ rm -R shellcheck* # Install Go dependencies RUN ARCH="$(dpkg --print-architecture)" && \ - curl "https://dl.google.com/go/go1.14.2.linux-$ARCH.tar.gz" | tar -C /usr/local -xz + curl -sSL "https://dl.google.com/go/go1.14.2.linux-$ARCH.tar.gz" | tar -C /usr/local -xz ENV PATH=/usr/local/go/bin:/root/go/bin:$PATH ENV GO111MODULE=on RUN go get mvdan.cc/sh/v3/cmd/shfmt diff --git a/ci/release-container/Dockerfile b/ci/release-container/Dockerfile index a0f20936..4dbc4784 100644 --- a/ci/release-container/Dockerfile +++ b/ci/release-container/Dockerfile @@ -28,14 +28,14 @@ RUN adduser --gecos '' --disabled-password coder && \ echo "coder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/nopasswd RUN ARCH="$(dpkg --print-architecture)" && \ - curl -L "https://github.com/boxboat/fixuid/releases/download/v0.4.1/fixuid-0.4.1-linux-$ARCH.tar.gz" | tar -C /usr/local/bin -xzf - && \ + curl -sSL "https://github.com/boxboat/fixuid/releases/download/v0.4.1/fixuid-0.4.1-linux-$ARCH.tar.gz" | tar -C /usr/local/bin -xzf - && \ chown root:root /usr/local/bin/fixuid && \ chmod 4755 /usr/local/bin/fixuid && \ mkdir -p /etc/fixuid && \ printf "user: coder\ngroup: coder\n" > /etc/fixuid/config.yml COPY release-packages/code-server*.deb /tmp/ -RUN dpkg -i /tmp/code-server*.deb && rm /tmp/code-server*.deb +RUN dpkg -i /tmp/code-server*-$(dpkg --print-architecture).deb && rm /tmp/code-server*.deb EXPOSE 8080 USER coder diff --git a/ci/release-container/push.sh b/ci/release-container/push.sh index 99faa67c..635f11b2 100755 --- a/ci/release-container/push.sh +++ b/ci/release-container/push.sh @@ -7,10 +7,6 @@ main() { source ./ci/lib.sh VERSION="$(pkg_json_version)" - if [[ ${CI-} ]]; then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - fi - imageTag="codercom/code-server:$VERSION" if [[ $(arch) == "arm64" ]]; then imageTag+="-arm64" diff --git a/ci/steps/lib.sh b/ci/steps/lib.sh new file mode 100755 index 00000000..781e9e61 --- /dev/null +++ b/ci/steps/lib.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +source ./ci/lib.sh + +# Grabs the most recent ci.yaml github workflow run that was successful and triggered from the same commit being pushd. +# This will contain the artifacts we want. +# https://developer.github.com/v3/actions/workflow-runs/#list-workflow-runs +get_artifacts_url() { + curl -sSL 'https://api.github.com/repos/cdr/code-server/actions/workflows/ci.yaml/runs?status=success&event=push' | jq -r ".workflow_runs[] | select(.head_sha == \"$(git rev-parse HEAD)\") | .artifacts_url" | head -n 1 +} + +# Grabs the artifact's download url. +# https://developer.github.com/v3/actions/artifacts/#list-workflow-run-artifacts +get_artifact_url() { + local artifact_name="$1" + curl -sSL "$(get_artifacts_url)" | jq -r ".artifacts[] | select(.name == \"$artifact_name\") | .archive_download_url" | head -n 1 +} + +# Uses the above two functions to download a artifact into a directory. +download_artifact() { + local artifact_name="$1" + local dst="$2" + + local tmp_file + tmp_file="$(mktemp)" + + curl -sSL "$(get_artifact_url "$artifact_name")" > "$tmp_file" + unzip -o "$tmp_file" -d "$dst" + rm "$tmp_file" +} diff --git a/ci/steps/publish-docker.sh b/ci/steps/publish-docker.sh new file mode 100755 index 00000000..d5e0835d --- /dev/null +++ b/ci/steps/publish-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + source ./ci/steps/lib.sh + + if [[ ${CI-} ]]; then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + fi + + download_artifact release-packages ./release-packages + ./ci/release-container/push.sh +} + +main "$@" diff --git a/ci/steps/publish-npm.sh b/ci/steps/publish-npm.sh new file mode 100755 index 00000000..3ca56f88 --- /dev/null +++ b/ci/steps/publish-npm.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + source ./ci/steps/lib.sh + + if [[ ${CI-} ]]; then + echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc + fi + + download_artifact npm-package ./release + yarn publish --non-interactive release +} + +main "$@" From 7c7f62d3f3377058169945f60109f0592502031d Mon Sep 17 00:00:00 2001 From: Anmol Sethi Date: Fri, 8 May 2020 16:31:42 -0400 Subject: [PATCH 2/2] Fixes for CI from @code-asher's review --- .github/workflows/ci.yaml | 1 + ci/README.md | 31 ++++++++++++++++++++----------- ci/release-container/build.sh | 14 ++++++++++++++ ci/release-container/push.sh | 9 ++------- ci/steps/publish-docker.sh | 1 + ci/steps/release-static.sh | 2 +- doc/CONTRIBUTING.md | 4 +++- package.json | 1 + 8 files changed, 43 insertions(+), 20 deletions(-) create mode 100755 ci/release-container/build.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a10a92e..f20b60f3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,6 +81,7 @@ jobs: - run: ./ci/steps/release-static.sh env: # Otherwise we get rate limited when fetching the ripgrep binary. + # For whatever reason only MacOS needs it. GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload release artifacts uses: actions/upload-artifact@v2 diff --git a/ci/README.md b/ci/README.md index add6b081..ce54ac34 100644 --- a/ci/README.md +++ b/ci/README.md @@ -10,13 +10,15 @@ Any file and directory added into this tree should be documented here. 1. Update the version of code-server in `package.json` and push a commit 1. CI will run and generate the `npm-package` and `release-packages` artifacts on the GH actions workflow -1. Create a new draft release and attach all the files in `release-packages` -1. Run some basic sanity tests on one of the released packages +1. Create a new draft release and attach all files in `release-packages` + 1. Run some basic sanity tests on one of the released packages +1. Summarize the major changes in the release notes and link to the relevant issues. + 1. Make sure to mention the VS Code version in the release notes 1. Publish the release -1. CI will automatically grab the artifacts and then - 1. Publish the NPM package. - 1. Publish the AMD64 docker image. - 1. Publish the ARM64 docker image. + 1. CI will automatically grab the artifacts and then + 1. Publish the NPM package + 1. Publish the AMD64 docker image + 1. Publish the ARM64 docker image ## dev @@ -63,9 +65,9 @@ You can disable minification by setting `MINIFY=`. - Useful to do a clean build. - [./build/code-server.sh](./build/code-server.sh) - Copied into static releases to run code-server with the bundled node binary. -- [./build/test-release.sh](./build/test-static-release.sh) +- [./build/test-static-release.sh](./build/test-static-release.sh) (`yarn test:static-release`) - Ensures code-server in the `./release-static` directory runs -- [./build/build-packages.sh](./build/build-static-pkgs.sh) (`yarn package`) +- [./build/build-packages.sh](./build/build-packages.sh) (`yarn package`) - Packages `./release-static` into an archive in `./release-packages` - If on linux, [nfpm](https://github.com/goreleaser/nfpm) is used to generate .deb and .rpm - [./build/nfpm.yaml](./build/nfpm.yaml) @@ -77,6 +79,12 @@ You can disable minification by setting `MINIFY=`. This directory contains the release docker container. +- [./release-container/build.sh](./release-container/build.sh) + - Builds the release container + - Assumes debian releases are ready in `./release-packages` +- [./release-container/push.sh](./release-container/push.sh) + - Pushes the built release container to docker hub and updates the latest tag + ## container This directory contains the container for CI. @@ -84,7 +92,7 @@ This directory contains the container for CI. ## steps This directory contains a few scripts used in CI. -Just helps avoid clobbering .travis.yml. +Just helps avoid clobbering the CI configuration. - [./steps/test.sh](./steps/test.sh) - Runs `yarn ci` after ensuring VS Code is patched @@ -98,5 +106,6 @@ Just helps avoid clobbering .travis.yml. - [./steps/publish-npm.sh](./steps/publish-npm.sh) - Grabs the `npm-package` release artifact for the current commit and publishes it on NPM - [./steps/publish-docker.sh](./steps/publish-docker.sh) - - Grabs the `release-packages` release artifact for the current commit and builds a docker - image with it and publishes that onto docker hub + - Grabs the `release-packages` release artifact for the current commit and + builds a docker image with it and publishes that onto docker hub with the + correct tag and updates latest diff --git a/ci/release-container/build.sh b/ci/release-container/build.sh new file mode 100755 index 00000000..27941a9b --- /dev/null +++ b/ci/release-container/build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + cd "$(dirname "$0")/../.." + source ./ci/lib.sh + VERSION="$(pkg_json_version)" + + imageTag="codercom/code-server:$VERSION" + + docker build -t "$imageTag" -f ./ci/release-container/Dockerfile . +} + +main "$@" diff --git a/ci/release-container/push.sh b/ci/release-container/push.sh index 635f11b2..0e44a4e4 100755 --- a/ci/release-container/push.sh +++ b/ci/release-container/push.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - set -euo pipefail main() { @@ -8,14 +7,10 @@ main() { VERSION="$(pkg_json_version)" imageTag="codercom/code-server:$VERSION" - if [[ $(arch) == "arm64" ]]; then - imageTag+="-arm64" - fi - docker build \ - -t "$imageTag" \ - -f ./ci/release-container/Dockerfile . docker push "$imageTag" + docker tag "$imageTag" codercom/code-server:latest + docker push codercom/code-server:latest } main "$@" diff --git a/ci/steps/publish-docker.sh b/ci/steps/publish-docker.sh index d5e0835d..48bf8c1a 100755 --- a/ci/steps/publish-docker.sh +++ b/ci/steps/publish-docker.sh @@ -10,6 +10,7 @@ main() { fi download_artifact release-packages ./release-packages + ./ci/release-container/build.sh ./ci/release-container/push.sh } diff --git a/ci/steps/release-static.sh b/ci/steps/release-static.sh index 6d565421..63c1b778 100755 --- a/ci/steps/release-static.sh +++ b/ci/steps/release-static.sh @@ -5,7 +5,7 @@ main() { cd "$(dirname "$0")/../.." yarn release:static - ./ci/build/test-static-release.sh + yarn test:static-release yarn package } diff --git a/doc/CONTRIBUTING.md b/doc/CONTRIBUTING.md index 824eaad2..37be0f7d 100644 --- a/doc/CONTRIBUTING.md +++ b/doc/CONTRIBUTING.md @@ -41,5 +41,7 @@ yarn vscode yarn build yarn build:vscode yarn release -node ./release # Run the built JavaScript with Node. +cd release +yarn --production +node . # Run the built JavaScript with Node. ``` diff --git a/package.json b/package.json index 05287881..c8b2bab9 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "build:vscode": "./ci/build/build-vscode.sh", "release": "./ci/build/build-release.sh", "release:static": "./ci/build/build-static-release.sh", + "test:static-release": "./ci/build/test-static-release.sh", "package": "./ci/build/build-packages.sh", "_____": "", "fmt": "./ci/dev/fmt.sh",