Update docs and remove separate wasm CI step

Integrated into the appropriate existing steps.

.github/CONTRIBUTING.md

@@ -27,25 +27,29 @@ CI must pass on your changes for them to be merged.
 
 ### CI
 
-CI will ensure your code is formatted correctly, passes linting and tests.
+CI will ensure your code is formatted, lints and passes tests.
 It will collect coverage and report it to [codecov](https://codecov.io/gh/nhooyr/websocket)
-and also upload a `coverage` artifact that you can download to inspect browse coverage.
+and also upload a html `coverage` artifact that you can download to browse coverage.
 
-You can run CI locally. The various steps are located in [ci/\*.sh](../ci).
+You can run CI locally. You only need [Go](https://golang.org), [nodejs](https://nodejs.org/en/) and [yarn](https://yarnpkg.com).
 
-1. [fmt.sh](../ci/fmt.sh) which requires node (specifically prettier).
-1. [lint.sh](../ci/lint.sh) which requires [shellcheck](https://github.com/koalaman/shellcheck#installing).
-1. [test.sh](../ci/test.sh)
-1. [run.sh](../ci/run.sh) which runs the above scripts in order.
+See the scripts in [package.json](../package.json).
 
-For coverage details locally, see `ci/out/coverage.html` after running [test.sh](../ci/test.sh).
+1. `yarn fmt` performs code generation and formatting.
+1. `yarn lint` performs linting.
+1. `yarn test` runs tests.
+1. `yarn all` runs the above scripts in parallel.
+
+For coverage details locally, see `ci/out/coverage.html` after running `yarn test`.
+
+CI is written with nodejs to enable running as much as possible concurrently.
 
 See [ci/image/Dockerfile](../ci/image/Dockerfile) for the installation of the CI dependencies on Ubuntu.
 
-You can also run tests normally with `go test`. [test.sh](../ci/test.sh) just passes a default set of flags to
-`go test` to collect coverage and also prettify the output.
+You can also run tests normally with `go test`. `yarn test` just passes a default set of flags to
+`go test` to collect coverage and runs the WASM tests.
 
-You can pass flags to [test.sh](../ci/test.sh) if you want to run a specific test or otherwise
+You can pass flags to `yarn test` if you want to run a specific test or otherwise
 control the behaviour of `go test` but also get coverage.
 
 Coverage percentage from codecov and the CI scripts will be different because they are calculated differently.

.github/workflows/ci.yml

@@ -4,19 +4,19 @@ on: [push]
 jobs:
   fmt:
     runs-on: ubuntu-latest
-    container: nhooyr/websocket-ci@sha256:7f5513545dcbaa3ed06a2919acfd1cfbff1e6e0decc1602c98672a4aad2f68ab
+    container: nhooyr/websocket-ci@sha256:f8b6e53a9fd256bcf6c90029276385b9ec730b76a0d7ccf3ff19084bce210c50
     steps:
       - uses: actions/checkout@v1
       - run: yarn --frozen-lockfile && yarn fmt
   lint:
     runs-on: ubuntu-latest
-    container: nhooyr/websocket-ci@sha256:7f5513545dcbaa3ed06a2919acfd1cfbff1e6e0decc1602c98672a4aad2f68ab
+    container: nhooyr/websocket-ci@sha256:f8b6e53a9fd256bcf6c90029276385b9ec730b76a0d7ccf3ff19084bce210c50
     steps:
       - uses: actions/checkout@v1
       - run: yarn --frozen-lockfile && yarn lint
   test:
     runs-on: ubuntu-latest
-    container: nhooyr/websocket-ci@sha256:7f5513545dcbaa3ed06a2919acfd1cfbff1e6e0decc1602c98672a4aad2f68ab
+    container: nhooyr/websocket-ci@sha256:f8b6e53a9fd256bcf6c90029276385b9ec730b76a0d7ccf3ff19084bce210c50
     steps:
       - uses: actions/checkout@v1
       - run: yarn --frozen-lockfile && yarn test
@@ -27,9 +27,3 @@ jobs:
         with:
           name: coverage
           path: ci/out/coverage.html
-  wasm:
-    runs-on: ubuntu-latest
-    container: nhooyr/websocket-ci@sha256:7f5513545dcbaa3ed06a2919acfd1cfbff1e6e0decc1602c98672a4aad2f68ab
-    steps:
-      - uses: actions/checkout@v1
-      - run: yarn --frozen-lockfile && yarn wasm

ci/all.ts

@@ -4,7 +4,6 @@ import { fmt, gen } from "./fmt"
 import { main } from "./lib"
 import { lint } from "./lint"
 import { test } from "./test"
-import { wasm } from "./wasm"
 
 main(run)
 
@@ -15,6 +14,5 @@ async function run(ctx: Promise<unknown>) {
     fmt(ctx),
     lint(ctx),
     test(ctx),
-    wasm(ctx),
   ])
 }

ci/fmt.ts

@@ -2,7 +2,7 @@
 
 import { exec, main } from "./lib"
 
-if (process.argv[1] === __filename) {
+if (require.main === module) {
   main(async (ctx: Promise<unknown>) => {
     await gen(ctx)
     await fmt(ctx)

ci/image/Dockerfile

@@ -6,7 +6,10 @@ RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
 RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
 RUN apt-get install -y nodejs chromium yarn
 
-RUN git config --global color.ui always
+COPY ./ci/image/gitignore /etc/git/ignore
+RUN git config --system color.ui always
+# Need to set set this explicitly for the system since github uses HOME=/home/github.
+RUN git config --system core.excludesfile /etc/git/ignore
 
 ENV GOPATH=/root/gopath
 ENV PATH=$GOPATH/bin:$PATH
@@ -14,9 +17,6 @@ ENV GOFLAGS="-mod=readonly"
 ENV PAGER=cat
 ENV CI=true
 
-RUN mkdir -p ~/.config/git
-COPY ./ci/image/gitignore ~/.config/git/ignore
-
 # Cache go modules, build cache and yarn cache.
 COPY . /tmp/websocket
 RUN cd /tmp/websocket && \

ci/image/push.ts

@@ -13,11 +13,9 @@ async function run(ctx: Promise<unknown>) {
 
   try {
     await spawn(ctx, "docker build -f ./ci/image/Dockerfile -t nhooyr/websocket-ci .", [], {
-      timeout: 180_000,
       stdio: "inherit",
     })
     await spawn(ctx, "docker push nhooyr/websocket-ci", [], {
-      timeout: 30_000,
       stdio: "inherit",
     })
   } finally {

ci/lib.ts

@@ -5,7 +5,7 @@ import { ExecOptions, SpawnOptions } from "child_process"
 export async function main(fn: (ctx: Promise<unknown>) => void, opts: {
   timeout: number
 } = {
-  timeout: 180_000,
+  timeout: 3 * 60_000,
 }) {
 
   const timer = new Timeout();
@@ -45,7 +45,6 @@ export async function main(fn: (ctx: Promise<unknown>) => void, opts: {
 // TODO promisify native versions
 export async function exec(ctx: Promise<unknown>, cmd: string, opts?: ExecOptions) {
   opts = {
-    timeout: 60_000,
     ...opts,
   }
   const p = cp.exec(cmd, opts)
@@ -62,7 +61,6 @@ export async function spawn(ctx: Promise<unknown>, cmd: string, args: string[],
     args = []
   }
   opts = {
-    timeout: 60_000,
     shell: true,
     ...opts,
   }
@@ -115,3 +113,10 @@ export function withCancel<T>(p: Promise<T>) {
     p: p,
   }
 }
+
+
+export const wasmEnv = {
+  ...process.env,
+  GOOS: "js",
+  GOARCH: "wasm",
+}

ci/lint.ts

 #!/usr/bin/env -S npx ts-node -P ci/tsconfig.json
 
-import { exec, main } from "./lib"
+import { exec, main, wasmEnv } from "./lib"
 
-if (process.argv[1] === __filename) {
+if (require.main === module) {
   main(lint)
 }
 
@@ -13,6 +13,15 @@ export async function lint(ctx: Promise<unknown>) {
       exec(ctx, "git ls-files '*.ts' | xargs npx eslint --max-warnings 0 --fix", {
         cwd: "ci",
       }),
+      wasmLint(ctx),
     ],
   )
 }
+
+
+async function wasmLint(ctx: Promise<unknown>) {
+  await exec(ctx, "go install golang.org/x/lint/golint")
+  await exec(ctx, "golint -set_exit_status ./...", {
+    env: wasmEnv,
+  })
+}

ci/test.ts

 #!/usr/bin/env -S npx ts-node -P ci/tsconfig.json
 
-import * as https from "https"
+import cp from "child_process"
+import * as events from "events"
+import * as readline from "readline"
 import replaceInFile from "replace-in-file"
-import { exec, main, selectCtx, spawn } from "./lib"
+import { exec, main, selectCtx, spawn, wasmEnv } from "./lib"
 
-if (process.argv[1] === __filename) {
+if (require.main === module) {
   main(test)
 }
 
@@ -26,19 +28,20 @@ export async function test(ctx: Promise<unknown>) {
     args.push("./...")
   }
 
-  await spawn(ctx, "go", ["test", ...args], {
-    timeout: 60_000,
+  const p1 = spawn(ctx, "go", ["test", ...args], {
     stdio: "inherit",
   })
+  const p2 = wasmTest(ctx)
+  await Promise.all([p1, p2])
 
   // Depending on the code tested, we may not have replaced anything so we do not
   // check whether anything was replaced.
   await selectCtx(ctx, replaceInFile({
     files: "./ci/out/coverage.prof",
     from: [
-      /.+frame_stringer.go:.+\n/g,
-      /.+wsjstest:.+\n/g,
-      /.+wsecho:.+\n/g,
+      /.+frame_stringer.go.+\n/g,
+      /.+wsjstest.+\n/g,
+      /.+wsecho.+\n/g,
     ],
     to: "",
   }))
@@ -46,12 +49,46 @@ export async function test(ctx: Promise<unknown>) {
   let p: Promise<unknown> = exec(ctx, "go tool cover -html=ci/out/coverage.prof -o=ci/out/coverage.html")
 
   if (process.env.CI) {
-    const script = https.get("https://codecov.io/bash")
-    const p2 = spawn(ctx, "bash -Z -R . -f ci/out/coverage.prof", [], {
-      stdio: [script],
-    })
-    p = Promise.all([p, p2])
+    p = Promise.all([p, codecov(ctx)])
   }
 
   await p
 }
+
+
+async function wasmTest(ctx: Promise<unknown>) {
+  await Promise.all([
+    exec(ctx, "go install ./internal/wsjstest"),
+    exec(ctx, "go install github.com/agnivade/wasmbrowsertest"),
+  ])
+
+  const url = await startWasmTestServer(ctx)
+
+  await exec(ctx, "go test -exec=wasmbrowsertest ./...", {
+    env: {
+      ...wasmEnv,
+      WS_ECHO_SERVER_URL: url,
+    },
+  })
+}
+
+async function startWasmTestServer(ctx: Promise<unknown>): Promise<string> {
+  const wsjstest = cp.spawn("wsjstest")
+  ctx.finally(wsjstest.kill.bind(wsjstest))
+
+  const rl = readline.createInterface({
+    input: wsjstest.stdout!,
+  })
+
+  try {
+    const p = events.once(rl, "line")
+    const a = await selectCtx(ctx, p)
+    return a[0]
+  } finally {
+    rl.close()
+  }
+}
+
+function codecov(ctx: Promise<unknown>) {
+  return exec(ctx, "curl -s https://codecov.io/bash | bash -s -- -Z -f ci/out/coverage.prof")
+}

ci/wasm.ts

-#!/usr/bin/env -S npx ts-node -P ci/tsconfig.json
-
-import cp from "child_process"
-import * as events from "events"
-import * as readline from "readline"
-import { exec, main, selectCtx } from "./lib"
-
-if (process.argv[1] === __filename) {
-  main(wasm)
-}
-
-const wasmEnv = {
-  ...process.env,
-  GOOS: "js",
-  GOARCH: "wasm",
-}
-
-export async function wasm(ctx: Promise<unknown>) {
-  await Promise.all([
-      exec(ctx, "go vet ./...", {
-        env: wasmEnv,
-      }),
-      goLint(ctx),
-      wasmTest(ctx),
-    ],
-  )
-}
-
-async function goLint(ctx: Promise<unknown>) {
-  await exec(ctx, "go install golang.org/x/lint/golint")
-  await exec(ctx, "golint -set_exit_status ./...", {
-    env: wasmEnv,
-  })
-}
-
-async function wasmTest(ctx: Promise<unknown>) {
-  await Promise.all([
-    exec(ctx, "go install ./internal/wsjstest"),
-    exec(ctx, "go install github.com/agnivade/wasmbrowsertest"),
-  ])
-
-  const url = await startServer(ctx)
-
-  await exec(ctx, "go test -exec=wasmbrowsertest ./...", {
-    env: {
-      ...wasmEnv,
-      WS_ECHO_SERVER_URL: url,
-    },
-  })
-}
-
-async function startServer(ctx: Promise<unknown>): Promise<string> {
-  const wsjstest = cp.spawn("wsjstest")
-  ctx.finally(wsjstest.kill.bind(wsjstest))
-
-  const rl = readline.createInterface({
-    input: wsjstest.stdout!,
-  })
-
-  try {
-    const p = events.once(rl, "line")
-    const a = await selectCtx(ctx, p)
-    return a[0]
-  } finally {
-    rl.close()
-  }
-}

package.json

@@ -7,6 +7,7 @@
 		"@typescript-eslint/eslint-plugin": "^2.3.2",
 		"@typescript-eslint/parser": "^2.3.2",
 		"await-timeout": "^0.6.0",
+		"axios": "^0.19.0",
 		"child-process-promise": "^2.2.1",
 		"eslint": "^6.5.1",
 		"eslint-config-prettier": "^6.3.0",
@@ -20,7 +21,6 @@
 		"fmt": "./ci/fmt.ts",
 		"lint": "./ci/lint.ts",
 		"test": "./ci/test.ts",
-		"wasm": "./ci/wasm.ts",
 		"ci": "./ci/all.ts",
 		"push-ci-image": "ci/image/push.ts"
 	}

yarn.lock

@@ -157,6 +157,14 @@ await-timeout@^0.6.0:
   resolved "https://registry.yarnpkg.com/await-timeout/-/await-timeout-0.6.0.tgz#efb52f5dba4d5fea6cff043705b09c97e7c403dc"
   integrity sha512-eMGnZxbqnmYTxCPqSYk5paIbPgcX8auC0UyVMScYMPY8pYmdk6o2wqQGw+SyN2hrhaDVZIQHVZX8DaHuBoRZcg==
 
+axios@^0.19.0:
+  version "0.19.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.0.tgz#8e09bff3d9122e133f7b8101c8fbdd00ed3d2ab8"
+  integrity sha512-1uvKqKQta3KBxIz14F2v06AEHZ/dIoeKfbTRkK1E5oqjDnuEerLmYTgJB5AiQZHJcljpg1TuRzdjDR06qNk0DQ==
+  dependencies:
+    follow-redirects "1.5.10"
+    is-buffer "^2.0.2"
+
 balanced-match@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
@@ -270,6 +278,13 @@ cross-spawn@^6.0.5:
     shebang-command "^1.2.0"
     which "^1.2.9"
 
+debug@=3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
+  integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==
+  dependencies:
+    ms "2.0.0"
+
 debug@^2.6.8, debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -570,6 +585,13 @@ flatted@^2.0.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.1.tgz#69e57caa8f0eacbc281d2e2cb458d46fdb449e08"
   integrity sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==
 
+follow-redirects@1.5.10:
+  version "1.5.10"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a"
+  integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==
+  dependencies:
+    debug "=3.1.0"
+
 fs.realpath@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
@@ -708,6 +730,11 @@ is-arrayish@^0.2.1:
   resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
   integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=
 
+is-buffer@^2.0.2:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.4.tgz#3e572f23c8411a5cfd9557c849e3665e0b290623"
+  integrity sha512-Kq1rokWXOPXWuaMAqZiJW4XxsmD9zGx9q4aePabbn3qCRGedtH7Cm+zV8WETitMfu1wdh+Rvd6w5egwSngUX2A==
+
 is-callable@^1.1.4:
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.4.tgz#1e1adf219e1eeb684d691f9d6a05ff0d30a24d75"