Switch from Python quilt3 to quiltcore-java (#135)

Co-authored-by: Ernest Prabhakar <[email protected]>

.github/workflows/mega-linter.yml

@@ -27,17 +27,17 @@ jobs:
     steps:
       # Git Checkout
       - name: Checkout Code
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
         with:
-          token: ${{ secrets.GITHUB_TOKEN }} # secrets.PAT || 
+          token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
           fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances
 
       # MegaLinter
       - name: MegaLinter
         id: ml
         # You can override MegaLinter flavor used to have faster performances
         # More info at https://megalinter.github.io/flavors/
-        uses: oxsecurity/megalinter/flavors/dotnet@v7.4.0
+        uses: oxsecurity/megalinter/flavors/dotnet@v7.3.0
         env:
           # All available variables are described in documentation
           # https://megalinter.github.io/configuration/
@@ -48,6 +48,7 @@ jobs:
 
       # Upload MegaLinter artifacts
       - name: Archive production artifacts
+        if: ${{ success() }} || ${{ failure() }}
         uses: actions/upload-artifact@v3
         with:
           name: MegaLinter reports

.github/workflows/pkg-test.yml

@@ -0,0 +1,61 @@
+name: Test
+
+on:
+  # Trigger at every push. Action will also be visible from Pull Requests to master
+  push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions)
+  pull_request:
+    branches: [master]
+
+permissions: read-all
+
+jobs:
+  build:
+    name: Package Test
+    permissions:
+      contents: read
+      id-token: write
+      issues: write
+      pull-requests: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        java_version: [19]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    # Git Checkout
+    - name: Checkout Code
+      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        fetch-depth: 0
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v3
+      with:
+        role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt
+        aws-region: us-east-1
+
+    - name: Setup Java ${{matrix.java_version}}
+      uses: actions/setup-java@v3
+      with:
+        java-version: ${{matrix.java_version}}
+        distribution: 'temurin'
+        architecture: x64
+        cache: gradle
+
+    - name: Setup Gradle
+      uses: gradle/gradle-build-action@v2
+
+    - name: Run Package Tests
+      run: make pkg-test
+
+    - name: Archive production artifacts
+      if: ${{ success() }} || ${{ failure() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: nf-quilt-pkg-test
+        path: |
+          /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/

.github/workflows/test.yml

@@ -12,8 +12,8 @@ jobs:
   build:
     name: Test
     permissions:
-      # contents: read
-      # id-token: write
+      contents: read
+      id-token: write
       issues: write
       pull-requests: write
 
@@ -27,17 +27,16 @@ jobs:
     steps:
     # Git Checkout
     - name: Checkout Code
-      uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         fetch-depth: 0
 
-    - name: Install Quilt3
-      uses: actions/setup-python@v4
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v3
       with:
-        python-version: '3.10' 
-    - run: python -m pip install quilt3
-    - run: which quilt3
+        role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt
+        aws-region: us-east-1
 
     - name: Setup Java ${{matrix.java_version}}
       uses: actions/setup-java@v3
@@ -54,6 +53,7 @@ jobs:
       run: make test
 
     - name: Archive production artifacts
+      if: ${{ success() }} || ${{ failure() }}
       uses: actions/upload-artifact@v3
       with:
         name: nf-quilt-test-reports

CHANGELOG.md

@@ -1,13 +1,24 @@
 # Changelog
 
+## [0.7.0] 2023-10-05
+
+- Officially QuiltCore 0.1.0 instead of Python
+
 ## [0.6.0] 2023-10-03
 
+(interim release still using Python)
+
 - Stop overwriting README.md (use README_NF_QUILT.md instead)
   - improve formatting
 - Smarter quilt_summarize.json
   - Top level (only): md, html, pdf, csv, tsv
   - multiqc sub-folder HTML
 
+## [0.5.0] 2023-09-04
+
+- Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows]
+- Do not pre-install packages (only install before write)
+
 ## [0.4.5] 2023-08-23
 
 - fix metadata in README.md

Makefile

@@ -4,13 +4,10 @@ WRITE_BUCKET ?= quilt-example
 FRAGMENT ?= &path=.
 NF_DIR ?= ../nextflow
 PID ?= $$$$
-PIP ?= python -m pip
 PIPELINE ?= sarek
 QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com
 TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT)
 QUILT_URI ?=  quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE)
-PIP ?= pip3
-QUILT3 ?= /usr/local/bin/quilt3
 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html
 
 verify: #compile
@@ -35,13 +32,9 @@ compile:
 
 nextflow:
 	if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git  "$(NF_DIR)"; fi
-	pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd
+	cd "$(NF_DIR)"; git checkout && make compile && git restore .; cd ..
 
-install-python:
-	if [ ! -x "$(QUILT3)" ]; then $(PIP) install quilt3 ; fi
-	which quilt3
-
-compile-all: install-python nextflow compile
+compile-all: nextflow compile
 
 check:
 	./gradlew check --warning-mode all

README.md

@@ -6,33 +6,15 @@ NextFlow plugin for reading and writing Quilt packages as a FileSystem
 developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly
 to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs.
 
-Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs
+Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs.
 
 ## I. Using the nf-quilt plugin in Production
 
 This plugin allows your existing pipelines, without modification,
 to read and write versioned Quilt packages stored on Amazon S3.
 
-Use the following four steps to configure NextFlow Tower or your command-line environment.
-
-1. Install the `quilt3` command-line tool
-
-This is distributed as an open source Python package you can install using `pip3`,
-and must be available in the PATH used by `nextflow`.
-
-```bash
-yum install python3-pip -y
-yum install git -y        
-pip3 install quilt3
-which quilt3
-```
-
-The above instructions use the 'yum' package manager,
-which NextFlow Tower uses in the "Pre-run script"
-when you edit the Pipeline settings from the Launchpad.
-
-If you are running from the command-line, you may need to use your own package manager
-(or just skip those lines if you already have Python and Git).
+Use the following three steps to configure NextFlow Tower or your command-line environment.
+[Note: versions 0.7.0 and later no longer require the `quilt3` Python client.]
 
 1. Enable the `nf-quilt` plugin
 
@@ -93,8 +75,8 @@ From the command-line, do, e.g.:
 
 ```bash
 # export NXF_VER=23.04.3
-export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.6.0/nf-quilt-0.6.0-meta.json
-nextflow run main.nf -plugins nf-quilt@0.6.0
+export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.0/nf-quilt-0.7.0-meta.json
+nextflow run main.nf -plugins nf-quilt@0.7.0
 ```
 
 For Tower, you can use the "Pre-run script" to set the environment variables.
@@ -183,13 +165,6 @@ git clone https://github.com/quiltdata/nf-quilt.git
 cd ./nf-quilt
 ```
 
-You also need to use Python to install the `quilt3` command-line tool used by `nf-quilt`:
-
-```bash
-pip install quilt3
-which quilt3
-```
-
 ### Unit Testing
 
 You can compile run unit tests with:

plugins/build.gradle

@@ -108,6 +108,7 @@ subprojects {
     task copyPluginLibs(type: Sync) {
         from configurations.runtimeClasspath
         into 'build/target/libs'
+        duplicatesStrategy 'exclude'
     }
 
     /*

plugins/nf-quilt/build.gradle

@@ -59,6 +59,9 @@ ext{
 }
 
 dependencies {
+    // quiltcore
+    implementation 'com.quiltdata:quiltcore:0.1'
+
     // This dependency is exported to consumers, that is to say found on their compile classpath.
     compileOnly "io.nextflow:nextflow:$nextflowVersion"
     compileOnly 'org.slf4j:slf4j-api:2.0.9'