Switch CI to Typescript

Significantly faster locally since **everything** runs in parallel.

Also much easier to maintain than bash.

.github/workflows/ci.yml

@@ -4,22 +4,22 @@ on: [push]
 jobs:
   fmt:
     runs-on: ubuntu-latest
-    container: docker://nhooyr/websocket-ci@sha256:b6331f8f64803c8b1bbd2a0ee9e2547317e0de2348bccd9c8dbcc1d88ff5747f
+    container: nhooyr/websocket-ci@sha256:7f5513545dcbaa3ed06a2919acfd1cfbff1e6e0decc1602c98672a4aad2f68ab
     steps:
       - uses: actions/checkout@v1
-      - run: ./ci/fmt.sh
+      - run: yarn --frozen-lockfile && yarn fmt
   lint:
     runs-on: ubuntu-latest
-    container: docker://nhooyr/websocket-ci@sha256:b6331f8f64803c8b1bbd2a0ee9e2547317e0de2348bccd9c8dbcc1d88ff5747f
+    container: nhooyr/websocket-ci@sha256:7f5513545dcbaa3ed06a2919acfd1cfbff1e6e0decc1602c98672a4aad2f68ab
     steps:
       - uses: actions/checkout@v1
-      - run: ./ci/lint.sh
+      - run: yarn --frozen-lockfile && yarn lint
   test:
     runs-on: ubuntu-latest
-    container: docker://nhooyr/websocket-ci@sha256:b6331f8f64803c8b1bbd2a0ee9e2547317e0de2348bccd9c8dbcc1d88ff5747f
+    container: nhooyr/websocket-ci@sha256:7f5513545dcbaa3ed06a2919acfd1cfbff1e6e0decc1602c98672a4aad2f68ab
     steps:
       - uses: actions/checkout@v1
-      - run: ./ci/test.sh
+      - run: yarn --frozen-lockfile && yarn test
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       - name: Upload coverage.html
@@ -29,7 +29,7 @@ jobs:
           path: ci/out/coverage.html
   wasm:
     runs-on: ubuntu-latest
-    container: docker://nhooyr/websocket-ci@sha256:b6331f8f64803c8b1bbd2a0ee9e2547317e0de2348bccd9c8dbcc1d88ff5747f
+    container: nhooyr/websocket-ci@sha256:7f5513545dcbaa3ed06a2919acfd1cfbff1e6e0decc1602c98672a4aad2f68ab
     steps:
       - uses: actions/checkout@v1
-      - run: ./ci/wasm.sh
+      - run: yarn --frozen-lockfile && yarn wasm

ci/.eslintrc.yaml

+parser: "@typescript-eslint/parser"
+env:
+  node: true
+
+parserOptions:
+  ecmaVersion: 2018
+  sourceType: module
+
+extends:
+  # https://github.com/typescript-eslint/typescript-eslint/tree/master/packages/eslint-plugin#usage
+  - eslint:recommended
+  - plugin:@typescript-eslint/eslint-recommended
+  - plugin:@typescript-eslint/recommended
+  # https://www.npmjs.com/package/eslint-plugin-import#typescript
+  - plugin:import/recommended
+  - plugin:import/typescript
+  #  https://dev.to/robertcoopercode/using-eslint-and-prettier-in-a-typescript-project-53jb
+  - prettier/@typescript-eslint
+
+rules:
+  "@typescript-eslint/no-use-before-define": off
+  "@typescript-eslint/explicit-function-return-type": off
+  "@typescript-eslint/no-non-null-assertion": off

ci/.gitignore

-out

ci/all.ts

+#!/usr/bin/env -S npx ts-node -P ci/tsconfig.json
+
+import { fmt, gen } from "./fmt"
+import { main } from "./lib"
+import { lint } from "./lint"
+import { test } from "./test"
+import { wasm } from "./wasm"
+
+main(run)
+
+async function run(ctx: Promise<unknown>) {
+  await gen(ctx)
+
+  await Promise.all([
+    fmt(ctx),
+    lint(ctx),
+    test(ctx),
+    wasm(ctx),
+  ])
+}

ci/fmt.sh

-#!/usr/bin/env bash
-
-set -euo pipefail
-cd "$(dirname "${0}")"
-cd "$(git rev-parse --show-toplevel)"
-
-gen() {
-  # Unfortunately, this is the only way to ensure go.mod and go.sum are correct.
-  # See https://github.com/golang/go/issues/27005
-  go list ./... > /dev/null
-  go mod tidy
-
-  go generate ./...
-}
-
-fmt() {
-  gofmt -w -s .
-  go run go.coder.com/go-tools/cmd/goimports -w "-local=$(go list -m)" .
-  go run mvdan.cc/sh/cmd/shfmt -i 2 -w -s -sr .
-  # shellcheck disable=SC2046
-  npx -q prettier \
-    --write \
-    --print-width 120 \
-    --no-semi \
-    --trailing-comma all \
-    --loglevel silent \
-    $(git ls-files "*.yaml" "*.yml" "*.md")
-}
-
-unstaged_files() {
-  git ls-files --other --modified --exclude-standard
-}
-
-check() {
-  if [[ ${CI-} && $(unstaged_files) != "" ]]; then
-    echo
-    echo "Files need generation or are formatted incorrectly."
-    echo "Run:"
-    echo "./ci/fmt.sh"
-    echo
-    git status
-    git diff
-    exit 1
-  fi
-}
-
-gen
-fmt
-check

ci/fmt.ts

+#!/usr/bin/env -S npx ts-node -P ci/tsconfig.json
+
+import { exec, main } from "./lib"
+
+if (process.argv[1] === __filename) {
+  main(async (ctx: Promise<unknown>) => {
+    await gen(ctx)
+    await fmt(ctx)
+  })
+}
+
+export async function fmt(ctx: Promise<unknown>) {
+  await Promise.all([
+      exec(ctx, "go mod tidy"),
+      exec(ctx, "gofmt -w -s ."),
+      exec(ctx, `go run go.coder.com/go-tools/cmd/goimports -w "-local=$(go list -m)" .`),
+      exec(ctx, `npx prettier --write --print-width=120 --no-semi --trailing-comma=all --loglevel=silent $(git ls-files "*.yaml" "*.yml" "*.md")`),
+    ],
+  )
+
+  if (process.env.CI) {
+    const r = await exec(ctx, "git ls-files --other --modified --exclude-standard")
+    const files = r.stdout.toString().trim()
+    if (files.length) {
+      console.log(`files need generation or are formatted incorrectly:
+${files}
+
+please run:
+  ./ci/fmt.js`)
+      process.exit(1)
+    }
+  }
+}
+
+export async function gen(ctx: Promise<unknown>) {
+  await exec(ctx, "go generate ./...")
+}

ci/image/Dockerfile

 FROM golang:1
 
-ENV DEBIAN_FRONTEND=noninteractive
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
+  echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get install -y nodejs chromium yarn
+
+RUN git config --global color.ui always
+
 ENV GOPATH=/root/gopath
 ENV PATH=$GOPATH/bin:$PATH
 ENV GOFLAGS="-mod=readonly"
 ENV PAGER=cat
 ENV CI=true
 
-RUN apt-get update && \
-  apt-get install -y shellcheck npm chromium && \
-  npm install -g prettier
-
-# https://github.com/golang/go/wiki/WebAssembly#running-tests-in-the-browser
-RUN go get github.com/agnivade/wasmbrowsertest && \
-  mv $GOPATH/bin/wasmbrowsertest $GOPATH/bin/go_js_wasm_exec
-
-RUN git config --global color.ui always
+RUN mkdir -p ~/.config/git
+COPY ./ci/image/gitignore ~/.config/git/ignore
 
-# Cache go modules and build cache.
+# Cache go modules, build cache and yarn cache.
 COPY . /tmp/websocket
 RUN cd /tmp/websocket && \
-  CI= ./ci/run.sh && \
+  yarn && CI= yarn ci && \
   rm -rf /tmp/websocket

ci/image/dockerignore

+node_modules

ci/image/gitignore

+node_modules
+.DS_Store
+.idea
+.gitignore
+.dockerignore

ci/image/push.sh

-#!/usr/bin/env bash
-
-set -euo pipefail
-cd "$(dirname "${0}")"
-cd "$(git rev-parse --show-toplevel)"
-
-docker build -f ./ci/image/Dockerfile -t nhooyr/websocket-ci .
-docker push nhooyr/websocket-ci

ci/image/push.ts

+#!/usr/bin/env -S npx ts-node -P ci/tsconfig.json
+
+import fs from "fs"
+import { promisify } from "util"
+import { main, spawn } from "../lib"
+
+main(run, {
+  timeout: 10 * 60_000,
+})
+
+async function run(ctx: Promise<unknown>) {
+  await promisify(fs.copyFile)("./ci/image/dockerignore", ".dockerignore")
+
+  try {
+    await spawn(ctx, "docker build -f ./ci/image/Dockerfile -t nhooyr/websocket-ci .", [], {
+      timeout: 180_000,
+      stdio: "inherit",
+    })
+    await spawn(ctx, "docker push nhooyr/websocket-ci", [], {
+      timeout: 30_000,
+      stdio: "inherit",
+    })
+  } finally {
+    await promisify(fs.unlink)(".dockerignore")
+  }
+}

ci/lib.ts

+import Timeout from "await-timeout"
+import cp from "child-process-promise"
+import { ExecOptions, SpawnOptions } from "child_process"
+
+export async function main(fn: (ctx: Promise<unknown>) => void, opts: {
+  timeout: number
+} = {
+  timeout: 180_000,
+}) {
+
+  const timer = new Timeout();
+  let ctx: Promise<unknown> = timer.set(opts.timeout, "context timed out")
+
+  const interrupted = new Promise((res, rej) => {
+    let int = 0
+    process.on("SIGINT", () => {
+      int++
+      if (int === 2) {
+        console.log("force exited")
+        process.exit(1)
+      }
+      rej("")
+    })
+  })
+
+  ctx = Promise.race([ctx, interrupted])
+  const {res, rej, p} = withCancel(ctx)
+  ctx = p
+
+  try {
+    await init(ctx)
+    await fn(ctx)
+    res!()
+  } catch (e) {
+    console.log(e)
+    rej!()
+    process.on("beforeExit", () => {
+      process.exit(1)
+    })
+  } finally {
+    timer.clear()
+  }
+}
+
+// TODO promisify native versions
+export async function exec(ctx: Promise<unknown>, cmd: string, opts?: ExecOptions) {
+  opts = {
+    timeout: 60_000,
+    ...opts,
+  }
+  const p = cp.exec(cmd, opts)
+
+  try {
+    return await selectCtx(ctx, p)
+  } finally {
+    p.childProcess.kill()
+  }
+}
+
+export async function spawn(ctx: Promise<unknown>, cmd: string, args: string[], opts?: SpawnOptions) {
+  if (args === undefined) {
+    args = []
+  }
+  opts = {
+    timeout: 60_000,
+    shell: true,
+    ...opts,
+  }
+  const p = cp.spawn(cmd, args, opts)
+
+  try {
+    return await selectCtx(ctx, p)
+  } finally {
+    p.childProcess.kill()
+  }
+}
+
+async function init(ctx: Promise<unknown>) {
+  const r = await exec(ctx, "git rev-parse --show-toplevel", {
+    cwd: __dirname,
+  })
+
+  process.chdir(r.stdout.toString().trim())
+}
+
+export async function selectCtx<T>(ctx: Promise<unknown>, p: Promise<T>): Promise<T> {
+  return await Promise.race([ctx, p]) as Promise<T>
+}
+
+const cancelSymbol = Symbol()
+
+export function withCancel<T>(p: Promise<T>) {
+  let rej: () => void;
+  let res: () => void;
+  const p2 = new Promise<T>((res2, rej2) => {
+    res = res2
+    rej = () => {
+      rej2(cancelSymbol)
+    }
+  })
+
+  p = Promise.race<T>([p, p2])
+  p = p.catch(e => {
+    // We need this catch to prevent node from complaining about it being unhandled.
+    // Look into why more later.
+    if (e === cancelSymbol) {
+      return
+    }
+    throw e
+  }) as Promise<T>
+
+  return {
+    res: res!,
+    rej: rej!,
+    p: p,
+  }
+}

ci/lint.sh

-#!/usr/bin/env bash
-
-set -euo pipefail
-cd "$(dirname "${0}")"
-cd "$(git rev-parse --show-toplevel)"
-
-# shellcheck disable=SC2046
-shellcheck -x $(git ls-files "*.sh")
-go vet ./...
-go run golang.org/x/lint/golint -set_exit_status ./...

ci/lint.ts

+#!/usr/bin/env -S npx ts-node -P ci/tsconfig.json
+
+import { exec, main } from "./lib"
+
+if (process.argv[1] === __filename) {
+  main(lint)
+}
+
+export async function lint(ctx: Promise<unknown>) {
+  await Promise.all([
+      exec(ctx, "go vet ./..."),
+      exec(ctx, "go run golang.org/x/lint/golint -set_exit_status ./..."),
+      exec(ctx, "git ls-files '*.ts' | xargs npx eslint --max-warnings 0 --fix", {
+        cwd: "ci",
+      }),
+    ],
+  )
+}

ci/out/.gitignore

+*

ci/run.sh

-#!/usr/bin/env bash
-
-# This script is for local testing. See .github/workflows/ci.yml for CI.
-
-set -euo pipefail
-cd "$(dirname "${0}")"
-cd "$(git rev-parse --show-toplevel)"
-
-echo "--- fmt"
-./ci/fmt.sh
-
-echo "--- lint"
-./ci/lint.sh
-
-echo "--- test"
-./ci/test.sh
-
-echo "--- wasm"
-./ci/wasm.sh

ci/test.sh

-#!/usr/bin/env bash
-
-set -euo pipefail
-cd "$(dirname "${0}")"
-cd "$(git rev-parse --show-toplevel)"
-
-argv+=(go test
-  "-parallel=1024"
-  "-coverprofile=ci/out/coverage.prof"
-  "-coverpkg=./..."
-)
-
-if [[ ${CI-} ]]; then
-  argv+=(
-    "-race"
-  )
-fi
-
-if [[ $# -gt 0 ]]; then
-  argv+=(
-    "$@"
-  )
-else
-  argv+=(./...)
-fi
-
-mkdir -p ci/out/websocket
-"${argv[@]}"
-
-# Removes coverage of generated/test related files.
-sed -i.bak '/_stringer.go/d' ci/out/coverage.prof
-sed -i.bak '/wsjstest/d' ci/out/coverage.prof
-sed -i.bak '/wsecho/d' ci/out/coverage.prof
-rm ci/out/coverage.prof.bak
-
-go tool cover -html=ci/out/coverage.prof -o=ci/out/coverage.html
-if [[ ${CI-} ]]; then
-  bash <(curl -s https://codecov.io/bash) -Z -R . -f ci/out/coverage.prof
-fi

ci/test.ts

+#!/usr/bin/env -S npx ts-node -P ci/tsconfig.json
+
+import * as https from "https"
+import replaceInFile from "replace-in-file"
+import { exec, main, selectCtx, spawn } from "./lib"
+
+if (process.argv[1] === __filename) {
+  main(test)
+}
+
+export async function test(ctx: Promise<unknown>) {
+  const args = [
+    "-parallel=1024",
+    "-coverprofile=ci/out/coverage.prof",
+    "-coverpkg=./...",
+  ]
+
+  if (process.env.CI) {
+    args.push("-race")
+  }
+
+  const cliArgs = process.argv.splice(2)
+  if (cliArgs.length > 0) {
+    args.push(...cliArgs)
+  } else {
+    args.push("./...")
+  }
+
+  await spawn(ctx, "go", ["test", ...args], {
+    timeout: 60_000,
+    stdio: "inherit",
+  })
+
+  // Depending on the code tested, we may not have replaced anything so we do not
+  // check whether anything was replaced.
+  await selectCtx(ctx, replaceInFile({
+    files: "./ci/out/coverage.prof",
+    from: [
+      /.+frame_stringer.go:.+\n/g,
+      /.+wsjstest:.+\n/g,
+      /.+wsecho:.+\n/g,
+    ],
+    to: "",
+  }))
+
+  let p: Promise<unknown> = exec(ctx, "go tool cover -html=ci/out/coverage.prof -o=ci/out/coverage.html")
+
+  if (process.env.CI) {
+    const script = https.get("https://codecov.io/bash")
+    const p2 = spawn(ctx, "bash -Z -R . -f ci/out/coverage.prof", [], {
+      stdio: [script],
+    })
+    p = Promise.all([p, p2])
+  }
+
+  await p
+}

ci/tsconfig.json

+{
+  "compilerOptions": {
+    /* Basic Options */
+    "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    // "lib": [],                             /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                       /* Allow javascript files to be compiled. */
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./",                       /* Concatenate and emit output to single file. */
+    // "outDir": "./",                        /* Redirect output structure to the directory. */
+     "rootDir": "./ci",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                     /* Enable project compilation */
+    // "incremental": true,                   /* Enable incremental compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+     "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                /* Report errors on unused locals. */
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+    /* Module Resolution Options */
+    // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                       /* List of folders to include type definitions from. */
+    // "types": [],                           /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true                   /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+  }
+}

ci/wasm.sh

-#!/usr/bin/env bash
-
-set -euo pipefail
-cd "$(dirname "${0}")"
-cd "$(git rev-parse --show-toplevel)"
-
-GOOS=js GOARCH=wasm go vet ./...
-
-go install golang.org/x/lint/golint
-GOOS=js GOARCH=wasm golint -set_exit_status ./...
-
-wsjstestOut="$(mktemp)"
-go install ./internal/wsjstest
-timeout 30s wsjstest >> "$wsjstestOut" 2>&1 &
-wsjstestPID=$!
-
-# See https://superuser.com/a/900134
-WS_ECHO_SERVER_URL="$( (tail -f -n0 "$wsjstestOut" &) | timeout 10s head -n 1)"
-if [[ -z $WS_ECHO_SERVER_URL ]]; then
-  echo "./internal/wsjstest failed to start in 10s"
-  exit 1
-fi
-
-go install github.com/agnivade/wasmbrowsertest
-export WS_ECHO_SERVER_URL
-GOOS=js GOARCH=wasm go test -exec=wasmbrowsertest ./...
-
-kill "$wsjstestPID" || true
-if ! wait "$wsjstestPID"; then
-  echo "--- wsjstest exited unsuccessfully"
-  echo "output:"
-  cat "$wsjstestOut"
-  exit 1
-fi