Improve CI caching (#416)

* Adjust linux distro to ubuntu 14.04

* Cache lib directory for speedy builds

* Fix path linking for default extensions

* Update reset

* Reset to head

* Improve caching

* Still run yarn in CI

* Update yarn before install

* Increase cache timeout

* Install vscode from vstar

* Undo data-dir changes to CLI, add back clean, remove unused CI func

* Remove additional flags added

* Remove unused dependency

* Reset vscode install dir so patching always works
This commit is contained in:
Kyle Carberry 2019-04-03 14:24:00 -05:00 committed by Asher
parent 2683b7c734
commit dcf409aecb
7 changed files with 145 additions and 84 deletions

1
.gitignore vendored
View File

@ -4,3 +4,4 @@ dist
out out
.DS_Store .DS_Store
release release
.cache

View File

@ -6,11 +6,12 @@ env:
matrix: matrix:
include: include:
- os: linux - os: linux
dist: ubuntu dist: trusty
- os: osx - os: osx
before_install: before_install:
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev
libsecret-1-dev; fi libsecret-1-dev; fi
- npm install -g yarn@1.12.3
script: script:
- scripts/build.sh - scripts/build.sh
before_deploy: before_deploy:
@ -35,4 +36,8 @@ deploy:
on: on:
repo: codercom/code-server repo: codercom/code-server
branch: master branch: master
cache: yarn cache:
yarn: true
timeout: 1000
directories:
- .cache

View File

@ -4,21 +4,22 @@ import * as fse from "fs-extra";
import * as os from "os"; import * as os from "os";
import * as path from "path"; import * as path from "path";
import * as zlib from "zlib"; import * as zlib from "zlib";
import * as https from "https";
import * as tar from "tar";
const isWin = os.platform() === "win32"; const isWin = os.platform() === "win32";
const libPath = path.join(__dirname, "../lib"); const libPath = path.join(__dirname, "../lib");
const vscodePath = path.join(libPath, "vscode"); const vscodePath = path.join(libPath, "vscode");
const defaultExtensionsPath = path.join(libPath, "extensions");
const pkgsPath = path.join(__dirname, "../packages"); const pkgsPath = path.join(__dirname, "../packages");
const defaultExtensionsPath = path.join(libPath, "VSCode-linux-x64/resources/app/extensions");
const vscodeVersion = process.env.VSCODE_VERSION || "1.32.0"; const vscodeVersion = process.env.VSCODE_VERSION || "1.32.0";
const vsSourceUrl = `https://codesrv-ci.cdr.sh/vstar-${vscodeVersion}.tar.gz`;
const buildServerBinary = register("build:server:binary", async (runner) => { const buildServerBinary = register("build:server:binary", async (runner) => {
await ensureInstalled(); await ensureInstalled();
await copyForDefaultExtensions();
await Promise.all([ await Promise.all([
buildBootstrapFork(), buildBootstrapFork(),
buildWeb(), buildWeb(),
buildDefaultExtensions(),
buildServerBundle(), buildServerBundle(),
buildAppBrowser(), buildAppBrowser(),
]); ]);
@ -129,97 +130,50 @@ const buildWeb = register("build:web", async (runner) => {
await runner.execute(isWin ? "npm.cmd" : "npm", ["run", "build"]); await runner.execute(isWin ? "npm.cmd" : "npm", ["run", "build"]);
}); });
const extDirPath = path.join("lib", "vscode-default-extensions");
const copyForDefaultExtensions = register("build:copy-vscode", async (runner) => {
if (!fs.existsSync(defaultExtensionsPath)) {
await ensureClean();
await ensureInstalled();
await new Promise((resolve, reject): void => {
fse.remove(extDirPath, (err) => {
if (err) {
return reject(err);
}
resolve();
});
});
await new Promise((resolve, reject): void => {
fse.copy(vscodePath, extDirPath, (err) => {
if (err) {
return reject(err);
}
resolve();
});
});
}
});
const buildDefaultExtensions = register("build:default-extensions", async (runner) => {
if (!fs.existsSync(defaultExtensionsPath)) {
await copyForDefaultExtensions();
runner.cwd = extDirPath;
const resp = await runner.execute(isWin ? "npx.cmd" : "npx", [isWin ? "gulp.cmd" : "gulp", "vscode-linux-x64", "--max-old-space-size=32384"]);
if (resp.exitCode !== 0) {
throw new Error(`Failed to build default extensions: ${resp.stderr}`);
}
}
});
const ensureInstalled = register("vscode:install", async (runner) => { const ensureInstalled = register("vscode:install", async (runner) => {
await ensureCloned(); runner.cwd = libPath;
runner.cwd = vscodePath; if (fs.existsSync(vscodePath) && fs.existsSync(defaultExtensionsPath)) {
const install = await runner.execute(isWin ? "yarn.cmd" : "yarn", []); const pkgVersion = JSON.parse(fs.readFileSync(path.join(vscodePath, "package.json")).toString("utf8")).version;
if (install.exitCode !== 0) { if (pkgVersion === vscodeVersion) {
throw new Error(`Failed to install vscode dependencies: ${install.stderr}`); runner.cwd = vscodePath;
}
});
const ensureCloned = register("vscode:clone", async (runner) => { const reset = await runner.execute("git", ["reset", "--hard"]);
if (fs.existsSync(vscodePath)) { if (reset.exitCode !== 0) {
await ensureClean(); throw new Error(`Failed to clean git repository: ${reset.stderr}`);
} else { }
fse.mkdirpSync(libPath);
runner.cwd = libPath; return;
const clone = await runner.execute("git", ["clone", "https://github.com/microsoft/vscode", "--branch", vscodeVersion, "--single-branch", "--depth=1"]);
if (clone.exitCode !== 0) {
throw new Error(`Failed to clone: ${clone.exitCode}`);
} }
} }
runner.cwd = vscodePath; fse.removeSync(libPath);
const checkout = await runner.execute("git", ["checkout", vscodeVersion]); fse.mkdirpSync(libPath);
if (checkout.exitCode !== 0) {
throw new Error(`Failed to checkout: ${checkout.stderr}`);
}
});
const ensureClean = register("vscode:clean", async (runner) => { await new Promise<void>((resolve, reject): void => {
runner.cwd = vscodePath; https.get(vsSourceUrl, (res) => {
if (res.statusCode !== 200) {
return reject(res.statusMessage);
}
const status = await runner.execute("git", ["status", "--porcelain"]); res.pipe(tar.x({
if (status.stdout.trim() !== "") { C: libPath,
const clean = await runner.execute("git", ["clean", "-f", "-d", "-X"]); }).on("finish", () => {
if (clean.exitCode !== 0) { resolve();
throw new Error(`Failed to clean git repository: ${clean.stderr}`); }).on("error", (err: Error) => {
} reject(err);
const removeUnstaged = await runner.execute("git", ["checkout", "--", "."]); }));
if (removeUnstaged.exitCode !== 0) { }).on("error", (err) => {
throw new Error(`Failed to remove unstaged files: ${removeUnstaged.stderr}`); reject(err);
} });
} });
const fetch = await runner.execute("git", ["fetch", "--prune"]);
if (fetch.exitCode !== 0) {
throw new Error(`Failed to fetch latest changes: ${fetch.stderr}`);
}
}); });
const ensurePatched = register("vscode:patch", async (runner) => { const ensurePatched = register("vscode:patch", async (runner) => {
if (!fs.existsSync(vscodePath)) { if (!fs.existsSync(vscodePath)) {
throw new Error("vscode must be cloned to patch"); throw new Error("vscode must be cloned to patch");
} }
await ensureClean(); await ensureInstalled();
runner.cwd = vscodePath; runner.cwd = vscodePath;
const patchPath = path.join(__dirname, "../scripts/vscode.patch"); const patchPath = path.join(__dirname, "../scripts/vscode.patch");

View File

@ -15,7 +15,9 @@
"devDependencies": { "devDependencies": {
"@types/fs-extra": "^5.0.4", "@types/fs-extra": "^5.0.4",
"@types/node": "^10.12.18", "@types/node": "^10.12.18",
"@types/tar": "^4.0.0",
"@types/trash": "^4.3.1", "@types/trash": "^4.3.1",
"cache-loader": "^2.0.1",
"cross-env": "^5.2.0", "cross-env": "^5.2.0",
"crypto-browserify": "^3.12.0", "crypto-browserify": "^3.12.0",
"css-loader": "^2.1.0", "css-loader": "^2.1.0",
@ -34,6 +36,8 @@
"sass-loader": "^7.1.0", "sass-loader": "^7.1.0",
"string-replace-loader": "^2.1.1", "string-replace-loader": "^2.1.1",
"style-loader": "^0.23.1", "style-loader": "^0.23.1",
"tar": "^4.4.8",
"terser-webpack-plugin": "^1.2.3",
"ts-loader": "^5.3.3", "ts-loader": "^5.3.3",
"ts-node": "^7.0.1", "ts-node": "^7.0.1",
"tsconfig-paths": "^3.8.0", "tsconfig-paths": "^3.8.0",

24
scripts/vstar.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
set -euxo pipefail
# Builds a tarfile containing vscode sourcefiles neccessary for CI.
# Done outside the CI and uploaded to object storage to reduce CI time.
branch=1.32.0
dir=/tmp/vstar
outfile=/tmp/vstar-$branch.tar.gz
rm -rf $dir
mkdir -p $dir
cd $dir
git clone https://github.com/microsoft/vscode --branch $branch --single-branch --depth=1
cd vscode
yarn
npx gulp vscode-linux-x64 --max-old-space-size=32384
rm -rf extensions build out* test
cd ..
mv *-x64/resources/app/extensions ./extensions
rm -rf *-x64
tar -czvf $outfile .

View File

@ -3,6 +3,7 @@ const os = require("os");
const environment = process.env.NODE_ENV || "development"; const environment = process.env.NODE_ENV || "development";
const HappyPack = require("happypack"); const HappyPack = require("happypack");
const webpack = require("webpack"); const webpack = require("webpack");
const TerserPlugin = require("terser-webpack-plugin");
const root = path.join(__dirname, ".."); const root = path.join(__dirname, "..");
@ -107,6 +108,11 @@ module.exports = (options = {}) => ({
id: "ts", id: "ts",
threads: Math.max(os.cpus().length - 1, 1), threads: Math.max(os.cpus().length - 1, 1),
loaders: [{ loaders: [{
path: "cache-loader",
query: {
cacheDirectory: path.join(__dirname, "..", ".cache"),
},
}, {
path: "ts-loader", path: "ts-loader",
query: { query: {
happyPackMode: true, happyPackMode: true,
@ -121,6 +127,14 @@ module.exports = (options = {}) => ({
"process.env.VERSION": `"${process.env.VERSION || ""}"`, "process.env.VERSION": `"${process.env.VERSION || ""}"`,
}), }),
], ],
optimization: {
minimizer: [
new TerserPlugin({
cache: path.join(__dirname, "..", ".cache", "terser"),
parallel: true,
}),
],
},
stats: { stats: {
all: false, // Fallback for options not defined. all: false, // Fallback for options not defined.
errors: true, errors: true,

View File

@ -24,6 +24,13 @@
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.18.tgz#1d3ca764718915584fcd9f6344621b7672665c67" resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.18.tgz#1d3ca764718915584fcd9f6344621b7672665c67"
integrity sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ== integrity sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==
"@types/tar@^4.0.0":
version "4.0.0"
resolved "https://registry.yarnpkg.com/@types/tar/-/tar-4.0.0.tgz#e3239d969eeb693a012200613860d0eb871c94f0"
integrity sha512-YybbEHNngcHlIWVCYsoj7Oo1JU9JqONuAlt1LlTH/lmL8BMhbzdFUgReY87a05rY1j8mfK47Del+TCkaLAXwLw==
dependencies:
"@types/node" "*"
"@types/trash@^4.3.1": "@types/trash@^4.3.1":
version "4.3.1" version "4.3.1"
resolved "https://registry.yarnpkg.com/@types/trash/-/trash-4.3.1.tgz#4880ff17c4eb467f1a26774ea6328428403b5c57" resolved "https://registry.yarnpkg.com/@types/trash/-/trash-4.3.1.tgz#4880ff17c4eb467f1a26774ea6328428403b5c57"
@ -723,6 +730,17 @@ cache-base@^1.0.1:
union-value "^1.0.0" union-value "^1.0.0"
unset-value "^1.0.0" unset-value "^1.0.0"
cache-loader@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/cache-loader/-/cache-loader-2.0.1.tgz#5758f41a62d7c23941e3c3c7016e6faeb03acb07"
integrity sha512-V99T3FOynmGx26Zom+JrVBytLBsmUCzVG2/4NnUKgvXN4bEV42R1ERl1IyiH/cvFIDA1Ytq2lPZ9tXDSahcQpQ==
dependencies:
loader-utils "^1.1.0"
mkdirp "^0.5.1"
neo-async "^2.6.0"
normalize-path "^3.0.0"
schema-utils "^1.0.0"
camel-case@3.0.x: camel-case@3.0.x:
version "3.0.0" version "3.0.0"
resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-3.0.0.tgz#ca3c3688a4e9cf3a4cda777dc4dcbc713249cf73" resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-3.0.0.tgz#ca3c3688a4e9cf3a4cda777dc4dcbc713249cf73"
@ -916,6 +934,11 @@ commander@^2.12.1, commander@^2.18.0:
resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a" resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"
integrity sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg== integrity sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==
commander@^2.19.0:
version "2.20.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==
commondir@^1.0.1: commondir@^1.0.1:
version "1.0.1" version "1.0.1"
resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
@ -3305,7 +3328,7 @@ negotiator@0.6.1:
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9"
integrity sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk= integrity sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=
neo-async@^2.5.0: neo-async@^2.5.0, neo-async@^2.6.0:
version "2.6.0" version "2.6.0"
resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.0.tgz#b9d15e4d71c6762908654b5183ed38b753340835" resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.0.tgz#b9d15e4d71c6762908654b5183ed38b753340835"
integrity sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA== integrity sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA==
@ -3452,6 +3475,11 @@ normalize-path@^2.1.1:
dependencies: dependencies:
remove-trailing-separator "^1.0.1" remove-trailing-separator "^1.0.1"
normalize-path@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
npm-bundled@^1.0.1: npm-bundled@^1.0.1:
version "1.0.5" version "1.0.5"
resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.5.tgz#3c1732b7ba936b3a10325aef616467c0ccbcc979" resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.5.tgz#3c1732b7ba936b3a10325aef616467c0ccbcc979"
@ -4625,6 +4653,14 @@ source-map-support@^0.5.6, source-map-support@~0.5.6:
buffer-from "^1.0.0" buffer-from "^1.0.0"
source-map "^0.6.0" source-map "^0.6.0"
source-map-support@~0.5.10:
version "0.5.11"
resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.11.tgz#efac2ce0800355d026326a0ca23e162aeac9a4e2"
integrity sha512-//sajEx/fGL3iw6fltKMdPvy8kL3kJ2O3iuYlRoT3k9Kb4BjOoZ+BZzaNHeuaruSt+Kf3Zk9tnfAQg9/AJqUVQ==
dependencies:
buffer-from "^1.0.0"
source-map "^0.6.0"
source-map-url@^0.4.0: source-map-url@^0.4.0:
version "0.4.0" version "0.4.0"
resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3" resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3"
@ -4924,7 +4960,7 @@ tar@^2.0.0:
fstream "^1.0.2" fstream "^1.0.2"
inherits "2" inherits "2"
tar@^4: tar@^4, tar@^4.4.8:
version "4.4.8" version "4.4.8"
resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.8.tgz#b19eec3fde2a96e64666df9fdb40c5ca1bc3747d" resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.8.tgz#b19eec3fde2a96e64666df9fdb40c5ca1bc3747d"
integrity sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ== integrity sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==
@ -4951,6 +4987,29 @@ terser-webpack-plugin@^1.1.0:
webpack-sources "^1.1.0" webpack-sources "^1.1.0"
worker-farm "^1.5.2" worker-farm "^1.5.2"
terser-webpack-plugin@^1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.2.3.tgz#3f98bc902fac3e5d0de730869f50668561262ec8"
integrity sha512-GOK7q85oAb/5kE12fMuLdn2btOS9OBZn4VsecpHDywoUC/jLhSAKOiYo0ezx7ss2EXPMzyEWFoE0s1WLE+4+oA==
dependencies:
cacache "^11.0.2"
find-cache-dir "^2.0.0"
schema-utils "^1.0.0"
serialize-javascript "^1.4.0"
source-map "^0.6.1"
terser "^3.16.1"
webpack-sources "^1.1.0"
worker-farm "^1.5.2"
terser@^3.16.1:
version "3.17.0"
resolved "https://registry.yarnpkg.com/terser/-/terser-3.17.0.tgz#f88ffbeda0deb5637f9d24b0da66f4e15ab10cb2"
integrity sha512-/FQzzPJmCpjAH9Xvk2paiWrFq+5M6aVOf+2KRbwhByISDX/EujxsK+BAvrhb6H+2rtrLCHK9N01wO014vrIwVQ==
dependencies:
commander "^2.19.0"
source-map "~0.6.1"
source-map-support "~0.5.10"
terser@^3.8.1: terser@^3.8.1:
version "3.14.1" version "3.14.1"
resolved "https://registry.yarnpkg.com/terser/-/terser-3.14.1.tgz#cc4764014af570bc79c79742358bd46926018a32" resolved "https://registry.yarnpkg.com/terser/-/terser-3.14.1.tgz#cc4764014af570bc79c79742358bd46926018a32"