Merge upstream, include GitHub Actions to deploy PRs to staging environment

.firebaserc

@@ -7,9 +7,12 @@
       "hosting": {
         "app": [
           "neuroglancer"
+        ],
+        "docs": [
+          "neuroglancer-docs"
         ]
       }
     }
   },
   "etags": {}
-}
+}

.github/workflows/build.yml

@@ -40,6 +40,7 @@ jobs:
         run: |
           build_info="{'tag':'$(git describe --always --tags)', 'url':'https://github.com/google/neuroglancer/commit/$(git rev-parse HEAD)', 'timestamp':'$(date)'}"
           npm run build -- --no-typecheck --no-lint --define NEUROGLANCER_BUILD_INFO="${build_info}"
+          echo $build_info > ./dist/client/version.json
         shell: bash
       - name: Build Python client bundles
         run: npm run build-python -- --no-typecheck --no-lint
@@ -49,6 +50,10 @@ jobs:
       - uses: ./.github/actions/setup-firefox
       - name: Run JavaScript tests (including WebGL)
         run: npm test
+        if: ${{ runner.os != 'macOS' }}
+      - name: Run JavaScript tests (excluding WebGL)
+        run: npm test -- --project node
+        if: ${{ runner.os == 'macOS' }}
       - name: Run JavaScript benchmarks
         run: npm run benchmark
       - name: Upload NPM package as artifact
@@ -96,7 +101,7 @@ jobs:
       # - name: Setup tmate session
       #   uses: mxschmitt/action-tmate@v3
       - name: Install Python packaging/test tools
-        run: python -m pip install --upgrade pip tox nox wheel numpy -r python/requirements-test.txt
+        run: pip install --upgrade pip tox nox wheel numpy -r python/requirements-test.txt
       - uses: ./.github/actions/setup-firefox
       - run: nox -s lint format mypy
       - name: Check for dirty working directory
@@ -173,13 +178,37 @@ jobs:
             dist/*.whl
             dist/*.tar.gz
 
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Setup Graphviz
+        uses: ts-graphviz/setup-graphviz@b1de5da23ed0a6d14e0aeee8ed52fdd87af2363c # v2.0.2
+        with:
+          macos-skip-brew-update: "true"
+      - name: Install nox
+        run: pip install nox
+      - name: Build docs
+        run: nox -s docs
+      - name: Upload docs as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs
+          path: |
+            dist/docs
+
   publish-package:
     # Only publish package on push to tag or default branch.
     if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master') }}
     runs-on: ubuntu-latest
     needs:
       - "client"
       - "python-build-package"
+      - "docs"
     steps:
       - uses: actions/checkout@v4
       - name: Use Node.js
@@ -219,12 +248,26 @@ jobs:
         with:
           name: client
           path: dist/client
-      - name: Publish to Firebase hosting
+      - name: Publish client to Firebase hosting
+        uses: FirebaseExtended/action-hosting-deploy@v0
+        with:
+          firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}"
+          projectId: neuroglancer-demo
+          channelId: live
+          target: app
+      # Download dist/docs after publishing to PyPI, because PyPI publish
+      # action fails if dist/docs directory is present.
+      - uses: actions/download-artifact@v4
+        with:
+          name: docs
+          path: dist/docs
+      - name: Publish docs to Firebase hosting
         uses: FirebaseExtended/action-hosting-deploy@v0
         with:
           firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}"
           projectId: neuroglancer-demo
           channelId: live
+          target: docs
 
   ngauth:
     strategy:

.github/workflows/build_docs_preview.yml

@@ -0,0 +1,29 @@
+name: Build docs preview
+
+on:
+  pull_request:
+
+jobs:
+  upload:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Setup Graphviz
+        uses: ts-graphviz/setup-graphviz@b1de5da23ed0a6d14e0aeee8ed52fdd87af2363c # v2.0.2
+        with:
+          macos-skip-brew-update: "true"
+      - name: Install nox
+        run: pip install nox
+      - name: Build documentation
+        run: nox -s docs
+      - name: Upload client as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs
+          path: |
+            dist/docs/

.github/workflows/deploy_docs_preview.yml

@@ -0,0 +1,67 @@
+name: Deploy docs preview
+
+on:
+  workflow_run:
+    workflows: ["Build docs preview"]
+    types: [completed]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: "Create commit status"
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const commitId = "${{ github.event.workflow_run.head_commit.id }}";
+            await github.rest.repos.createCommitStatus({
+                context: "docs-preview",
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                sha: commitId,
+                state: "pending",
+                description: `Creating preview`,
+                target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+            });
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: docs
+          path: dist/docs
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+          run-id: "${{ github.event.workflow_run.id }}"
+      - name: Get PR ID
+        # https://github.com/orgs/community/discussions/25220#discussioncomment-7532132
+        id: pr-id
+        run: |
+          PR_ID=$(gh run view -R ${{ github.repository }} ${{ github.event.workflow_run.id }} | grep -oP '#[0-9]+ . ${{ github.event.workflow_run.id }}' | grep -oP '#[0-9]+' | cut -c 2-)
+          echo "pr-id=${PR_ID}" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+      - uses: FirebaseExtended/action-hosting-deploy@v0
+        id: deploy
+        with:
+          repoToken: "${{ secrets.GITHUB_TOKEN }}"
+          firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}"
+          expires: 30d
+          channelId: "pr${{ steps.pr-id.outputs.pr-id }}"
+          projectId: neuroglancer-demo
+          target: docs
+      - name: "Update commit status"
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const expires = new Date("${{ steps.deploy.outputs.expire_time }}");
+            const commitId = "${{ github.event.workflow_run.head_commit.id }}";
+            await github.rest.repos.createCommitStatus({
+                context: "docs-preview",
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                sha: commitId,
+                state: "success",
+                target_url: "${{ steps.deploy.outputs.details_url }}",
+                description: `Preview created, expires at: ${expires.toISOString()}`,
+            });

.github/workflows/deploy_preview.yml

@@ -1,66 +1,47 @@
-name: Deploy preview
+name: Build and Deploy Staging
 
 on:
-  workflow_run:
-    workflows: ["Build preview"]
-    types: [completed]
+  push:
+    branches:
+      - staging
+      - "**"
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
-      - name: "Create commit status"
-        uses: actions/github-script@v7
+      - name: Checkout Repository
+        uses: actions/checkout@v4
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const commitId = "${{ github.event.workflow_run.head_commit.id }}";
-            await github.rest.repos.createCommitStatus({
-                context: "client-preview",
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                sha: commitId,
-                state: "pending",
-                description: `Creating preview`,
-                target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
-            });
-      - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
-        with:
-          name: client
-          path: dist/client
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-          run-id: "${{ github.event.workflow_run.id }}"
-      - name: Get PR ID
-        # https://github.com/orgs/community/discussions/25220#discussioncomment-7532132
-        id: pr-id
-        run: |
-          PR_ID=$(gh run view -R ${{ github.repository }} ${{ github.event.workflow_run.id }} | grep -oP '#[0-9]+ . ${{ github.event.workflow_run.id }}' | grep -oP '#[0-9]+' | cut -c 2-)
-          echo "pr-id=${PR_ID}" >> $GITHUB_OUTPUT
-        env:
-          GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-      - uses: FirebaseExtended/action-hosting-deploy@v0
-        id: deploy
+          ref: ${{ github.event.inputs.commit_sha || github.sha }}
+
+      - name: Set short sha name for sub-directory
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Check outputs
+        run: echo ${{ steps.vars.outputs.sha_short }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
         with:
-          repoToken: "${{ secrets.GITHUB_TOKEN }}"
-          firebaseServiceAccount: "${{ secrets.FIREBASE_HOSTING_SERVICE_ACCOUNT_KEY }}"
-          expires: 30d
-          channelId: "pr${{ steps.pr-id.outputs.pr-id }}"
-          projectId: neuroglancer-demo
-      - name: "Update commit status"
-        uses: actions/github-script@v7
+          node-version: 20.x
+
+      - name: Install Dependencies
+        run: npm install
+
+      - name: Build Project
+        run: npm run build
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const expires = new Date("${{ steps.deploy.outputs.expire_time }}");
-            const commitId = "${{ github.event.workflow_run.head_commit.id }}";
-            await github.rest.repos.createCommitStatus({
-                context: "client-preview",
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                sha: commitId,
-                state: "success",
-                target_url: "${{ steps.deploy.outputs.details_url }}",
-                description: `Preview created, expires at: ${expires.toISOString()}`,
-            });
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+
+      - name: Upload to S3
+        run: |
+          aws s3 sync dist/client/ ${{ secrets.CLOUDFRONT_DEPLOYMENT_LOCATION }}/staging/${{ steps.vars.outputs.sha_short }}/ --delete
+
+      - name: Display URL for Neuroglancer
+        run: echo "https://neuroglancer.lincbrain.org/cloudfront/frontend/staging/${{ steps.vars.outputs.sha_short }}/index.html"

.gitignore

@@ -21,3 +21,6 @@ tsconfig.tsbuildinfo
 .eslintcache
 /lib
 /.firebase
+.idea/
+yarn.lock
+.DS_Store

.prettierignore

@@ -5,3 +5,5 @@
 .parcel-cache
 dist
 /lib
+/docs/_build/
+/.ruff_cache

README.md

@@ -2,7 +2,7 @@
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![PyPI](https://img.shields.io/pypi/v/neuroglancer)](https://pypi.org/project/neuroglancer)
-![Build](https://github.com/google/neuroglancer/workflows/Build/badge.svg)
+![Build](https://github.com/google/neuroglancer/actions/workflows/build.yml/badge.svg)
 [![DOI](https://zenodo.org/badge/59798355.svg)](https://zenodo.org/badge/latestdoi/59798355)
 
 Neuroglancer is a WebGL-based viewer for volumetric data. It is capable of displaying arbitrary (non axis-aligned) cross-sectional views of volumetric data, as well as 3-D meshes and line-segment based models (skeletons).

build_tools/build-package.ts

@@ -14,10 +14,24 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import esbuild from "esbuild";
 import { glob } from "glob";
+import ts from "typescript";
 import yargs from "yargs";
 
 const rootDir = path.resolve(import.meta.dirname, "..");
 
+function buildDeclarationFiles(
+  fileNames: string[],
+  options: ts.CompilerOptions,
+): void {
+  options = {
+    ...options,
+    declaration: true,
+    emitDeclarationOnly: true,
+  };
+  const program = ts.createProgram(fileNames, options);
+  program.emit();
+}
+
 async function buildPackage(options: { inplace?: boolean }) {
   const { inplace = false } = options;
 
@@ -45,13 +59,32 @@ async function buildPackage(options: { inplace?: boolean }) {
     nodir: true,
   });
 
+  const entryPoints = typescriptSources.map((name) =>
+    path.resolve(srcDir, name),
+  );
+
   await esbuild.build({
-    entryPoints: typescriptSources.map((name) => path.resolve(srcDir, name)),
+    entryPoints,
     outbase: srcDir,
     bundle: false,
     outdir: libDir,
   });
 
+  let compilerOptionsFromConfigFile: ts.CompilerOptions = {};
+  const configFileName = ts.findConfigFile("../", ts.sys.fileExists);
+  if (configFileName) {
+    const configFile = ts.readConfigFile(configFileName, ts.sys.readFile);
+    compilerOptionsFromConfigFile = ts.parseJsonConfigFileContent(
+      configFile.config,
+      ts.sys,
+      "./",
+    ).options;
+  }
+  buildDeclarationFiles(entryPoints, {
+    ...compilerOptionsFromConfigFile,
+    outDir: libDir,
+  });
+
   const otherSources = await glob(["**/*.{css,js,html,wasm}"], {
     cwd: srcDir,
     nodir: true,