Better CI (#98)

* Amend a cargo-clippy issues present on nightly that do not show up on stable. * Split GHA and add modified GHA for the benchmarking tracking (pushing against gh-pages). * Add release-plz * Rename htsget-http-actix -> htsget-actix, htsget-http-lambda -> htsget-lambda, htsget-http-core -> htsget-http, and htsget-test-utils -> htsget-test * Version bump all deps * Better names for the lib side of this crate: get() and post() exposed to third party devs * Temporarily disable htsget-test as suggested by cargo maintainers Co-authored-by: Marko Malenic <[email protected]>
umccr · Jan 15, 2023 · 4437cce · 4437cce
1 parent 4ed6d8e
commit 4437cce
Show file tree

Hide file tree

Showing 73 changed files with 508 additions and 368 deletions.
diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml
@@ -0,0 +1,61 @@
+name: benchmarks
+
+on:
+  push:
+    branches:
+      - main
+      - better_ci
+      - iai
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        rust: [stable]
+        os: [ubuntu-latest]
+    steps:
+      - name: Check out
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ matrix.rust }}
+            override: true
+            components: rustfmt, clippy
+      - name: Cargo cache
+        uses: actions/cache@v3
+        continue-on-error: false
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-cache3-${{ hashFiles('**/Cargo.toml') }}
+          restore-keys: ${{ runner.os }}-cargo-cache3-
+      - name: Install cargo-criterion
+        uses: baptiste0928/cargo-install@v1
+        with:
+          crate: cargo-criterion
+      - name: Run search benchmarks
+        run: cargo criterion --bench search-benchmarks --message-format=json -- LIGHT 1> search-benchmarks-output.json
+      - name: Store search benchmark result
+        uses: brainstorm/github-action-benchmark@cargo-criterion-v3
+        with:
+          name: Search benchmark
+          tool: 'cargo-criterion'
+          output-file-path: search-benchmarks-output.json
+          native-benchmark-data-dir-path: target/criterion
+          fail-on-alert: false
+          github-token: ${{ secrets.HTSGET_RS_BENCHMARKS_TOKEN }}
+          auto-push: true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,23 @@
+name: release
+
+on:
+  push:
+    branches:
+      - main
+      - better_ci
+      - iai
+
+jobs:
+  release:
+    name: Release-plz
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Run release-plz
+        uses: MarcoIeni/release-plz-action@main
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ secrets.HTSGET_RS_CRATES_IO_TOKEN }}
diff --git a/.github/workflows/action.yml → .github/workflows/tests.yml b/.github/workflows/action.yml → .github/workflows/tests.yml
@@ -1,6 +1,11 @@
 name: tests
 
-on: [push]
+on:
+  push:
+    branches:
+      - main
+      - better_ci
+      - iai
 
 jobs:
   test:
@@ -10,62 +15,54 @@ jobs:
         rust: [stable]
         os: [ubuntu-latest]
     steps:
-      # Run tests
-      - uses: actions/checkout@v2
+      - name: Check out
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
             toolchain: ${{ matrix.rust }}
             override: true
             components: rustfmt, clippy
-
-      - name: Run cargo fmt
+      - name: Cargo cache
+        uses: actions/cache@v3
+        continue-on-error: false
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-cache3-${{ hashFiles('**/Cargo.toml') }}
+          restore-keys: ${{ runner.os }}-cargo-cache3-
+      - name: Cargo fmt
         uses: actions-rs/cargo@v1
         with:
           command: fmt
-
+      - name: Clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: -- -D warnings
       - name: Build all features
         uses: actions-rs/cargo@v1
         with:
           command: build
           args: --all-targets --all-features
-
       - name: Build no default features
         uses: actions-rs/cargo@v1
         with:
           command: build
           args: --all-targets --no-default-features
-
       - name: Build s3-storage feature
         uses: actions-rs/cargo@v1
         with:
           command: build
           args: --all-targets --features s3-storage
-
-      - name: Run cargo clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets --all-features
-
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
           command: test
           args: --all-features
-
-      # Run benchmarks
-      - name: Run benchmarks
-        run: cargo bench --bench search-benchmarks --bench request-benchmarks -- LIGHT --output-format bencher | tee output.txt
-      - name: Download previous benchmark data
-        uses: actions/cache@v3
-        with:
-          path: ./cache
-          key: ${{ runner.os }}-benchmark
-      - name: Store benchmark result
-        uses: benchmark-action/github-action-benchmark@v1
-        with:
-          tool: 'cargo'
-          output-file-path: output.txt
-          external-data-json-path: ./cache/benchmark-data.json
-          fail-on-alert: false
diff --git a/.gitignore b/.gitignore
@@ -7,3 +7,6 @@ Cargo.lock
 package-lock.json
 deploy/.build
 output.txt
+cert.pem
+key.pem
+request-benchmarks.json
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -31,7 +31,7 @@ We also use [clippy] and [rustfmt] for code style, linting and formatting.
 Please run the following commands to check tests, lints, code style and formatting before submitting a pull request:
 
 ```sh
-cargo clippy --all-targets --all-features
+cargo clippy --all-targets --all-features --fix
 cargo fmt
 cargo test --tests --all-features
 ```

diff --git a/Cargo.toml b/Cargo.toml
@@ -1,9 +1,9 @@
 [workspace]
 members = [
     "htsget-config",
-    "htsget-http-actix",
-    "htsget-http-core",
-    "htsget-http-lambda",
+    "htsget-actix",
+    "htsget-http",
+    "htsget-lambda",
     "htsget-search",
-    "htsget-test-utils",
+    "htsget-test",
 ]
diff --git a/README.md b/README.md
@@ -11,9 +11,9 @@
 
 A **server** implementation of the [htsget protocol][htsget-protocol] for bioinformatics in Rust. It is:
 * **Fully-featured**: supports BAM and CRAM for reads, and VCF and BCF for variants, as well as other aspects of the protocol such as TLS, and CORS.
-* **Serverless**: supports local server instances using [Actix Web][actix-web], and serverless instances using AWS Lambda.
+* **Serverless**: supports local server instances using [Actix Web][actix-web], and serverless instances using [AWS Lambda Rust Runtime][aws-lambda-rust-runtime].
 * **Storage interchangeable**: supports local filesystem storage, and storage on AWS S3.
-* **Thoroughly tested and benchmarked**: tested using a purpose-built [test suite][htsget-test-utils], and benchmarked using [criterion-rs].
+* **Thoroughly tested and benchmarked**: tested using a purpose-built [test suite][htsget-test], and benchmarked using [criterion-rs].
 
 To get started, see [Usage].
 
@@ -56,15 +56,15 @@ htsget-rs implements the following components of the protocol:
 Htsget-rs is configured using environment variables, for details on how to set them, see [htsget-config].
 
 ### Local
-To run a local instance htsget-rs, run [htsget-http-actix] by executing the following:
+To run a local instance htsget-rs, run [htsget-actix] by executing the following:
 ```sh
-cargo run -p htsget-http-actix
+cargo run -p htsget-actix
 ```
 Using the default configuration, this will start a ticket server on `127.0.0.1:8080` and a data block server on `127.0.0.1:8081`
-with data accessible from the [data] directory. See [htsget-http-actix] for more information.
+with data accessible from the [data] directory. See [htsget-actix] for more information.
 
 ### Cloud
-Cloud based htsget-rs uses [htsget-http-lambda]. For more information and an example deployment of this crate see 
+Cloud based htsget-rs uses [htsget-lambda]. For more information and an example deployment of this crate see 
 [deploy].
 
 ### Tests
@@ -75,42 +75,42 @@ Tests can be run tests by executing:
 cargo test --all-features
 ```
 
-To run benchmarks, see the benchmark sections of [htsget-http-actix][htsget-http-actix-benches] and [htsget-search][htsget-search-benches].
+To run benchmarks, see the benchmark sections of [htsget-actix][htsget-actix-benches] and [htsget-search][htsget-search-benches].
 
-[htsget-http-actix-benches]: htsget-http-actix/README.md#Benchmarks
+[htsget-actix-benches]: htsget-actix/README.md#Benchmarks
 [htsget-search-benches]: htsget-search/README.md#Benchmarks
 
 ## Project Layout
 
 This repository consists of a workspace composed of the following crates:
 
 - [htsget-config]: Configuration of the server.
-- [htsget-http-actix]: Local instance of the htsget server. Contains framework dependent code using [Actix Web][actix-web].
-- [htsget-http-core]: Handling of htsget HTTP requests. Framework independent code.
-- [htsget-http-lambda]: Cloud based instance of the htsget server. Contains framework dependent
+- [htsget-actix]: Local instance of the htsget server. Contains framework dependent code using [Actix Web][actix-web].
+- [htsget-http]: Handling of htsget HTTP requests. Framework independent code.
+- [htsget-lambda]: Cloud based instance of the htsget server. Contains framework dependent
 code using the [Rust Runtime for AWS Lambda][aws-lambda-rust-runtime].
 - [htsget-search]: Core logic needed to search bioinformatics files based on htsget queries.
-- [htsget-test-utils]: Test suite used by other crates in the project.
+- [htsget-test]: Test suite used by other crates in the project.
 
 Other directories contain further applications or data:
 - [data]: Contains example data files which can be used by htsget-rs, in folders denoting the file type.
 This directory also contains example events used by a cloud instance of htsget-rs in the [`events`][data-events] subdirectory.
-- [deploy]: An example deployment of [htsget-http-lambda].
+- [deploy]: An example deployment of [htsget-lambda].
 
-In htsget-rs the ticket server handled by [htsget-http-actix] or [htsget-http-lambda], and the data
+In htsget-rs the ticket server handled by [htsget-actix] or [htsget-lambda], and the data
 block server is handled by the [storage backend][storage-backend], either [locally][local-storage], or using [AWS S3][s3-storage].
 This project layout is structured to allow for extensibility and modularity. For example, a new ticket server and data server could 
 be implemented using Cloudflare Workers in a `htsget-http-workers` crate and Cloudflare R2 in [htsget-search].
 
 See the [htsget-search overview][htsget-search-overview] for more information on the storage backend.
 
 [htsget-config]: htsget-config
-[htsget-http-actix]: htsget-http-actix
-[htsget-http-core]: htsget-http-core
-[htsget-http-lambda]: htsget-http-lambda
+[htsget-actix]: htsget-actix
+[htsget-http]: htsget-http
+[htsget-lambda]: htsget-lambda
 [htsget-search]: htsget-search
 [htsget-search-overview]: htsget-search/README.md#Overview
-[htsget-test-utils]: htsget-test-utils
+[htsget-test]: htsget-test
 
 [storage-backend]: htsget-search/src/storage
 [local-storage]: htsget-search/src/storage/local.rs
@@ -134,6 +134,7 @@ See the [contributing guide][contributing] for more information.
 
 This project is licensed under the [MIT license][license].
 
-[htsget-http-actix]: htsget-http-actix
-[htsget-http-lambda]: htsget-http-lambda
+[htsget-actix]: htsget-actix
+[htsget-lambda]: htsget-lambda
 [license]: LICENSE
+[aws-lambda-rust-runtime]: https://github.com/awslabs/aws-lambda-rust-runtime
diff --git a/deploy/.dockerignore b/deploy/.dockerignore
@@ -0,0 +1,9 @@
+target
+Cargo.lock
+*.code-workspace
+*.vcf.gz
+.vscode
+.idea
+package-lock.json
+deploy/.build
+output.txt
diff --git a/deploy/Dockerfile b/deploy/Dockerfile
@@ -0,0 +1,25 @@
+# syntax=docker/dockerfile:1.4
+FROM rust:1.61.0 AS builder
+
+ARG TARGETPLATFORM
+
+WORKDIR /root
+
+RUN --mount=type=cache,target=/usr/local/cargo/registry,id=${TARGETPLATFORM} \
+    cargo install cargo-strip
+
+COPY . .
+
+RUN --mount=type=cache,target=/usr/local/cargo/registry,id=${TARGETPLATFORM} --mount=type=cache,target=/root/target,id=${TARGETPLATFORM} \
+    cargo build --release && \
+    cargo strip && \
+    mv /root/target/release/htsget-actix /root
+
+
+FROM gcr.io/distroless/cc-debian11
+
+COPY --from=builder /root/htsget-actix /
+
+ENTRYPOINT ["./htsget-actix"]
+
+EXPOSE 3000