54 avoid metadata failure (#66)

.github/workflows/mega-linter.yml

@@ -13,7 +13,7 @@ env: # Comment env block if you do not want to apply fixes
   #APPLY_FIXES_MODE: pull_request # If APPLY_FIXES is used, defines if the fixes are directly committed (commit) or posted in a PR (pull_request)
   DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD
   FILTER_REGEX_EXCLUDE: .*/.*gradle
-  
+
 concurrency:
   group: ${{ github.ref }}-${{ github.workflow }}
   cancel-in-progress: true

.github/workflows/test.yml

@@ -9,7 +9,13 @@ on:
 jobs:
   build:
     name: Test
-    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+
     steps:
     # Git Checkout
     - name: Checkout Code
@@ -18,14 +24,29 @@ jobs:
         token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
         fetch-depth: 0
 
+    - name: Install Quilt3
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10' 
+    - run: python -m pip install quilt3
+    - run: which quilt3
+
     - name: Setup Java
       uses: actions/setup-java@v3
       with:
         distribution: temurin
         java-version: 17
-            
+
     - name: Setup Gradle
       uses: gradle/gradle-build-action@v2
 
     - name: Run Gradle Tests
       run: make test
+
+    - name: Archive production artifacts
+      if: ${{ success() }} || ${{ failure() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: nf-quilt-test-reports
+        path: |
+          /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/

.gitignore

@@ -12,6 +12,7 @@ build
 work
 out
 results
+bin
 
 # Ignore Gradle GUI config
 gradle-app.setting

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "java.compile.nullAnalysis.mode": "disabled"
+}

Makefile

@@ -1,17 +1,25 @@
 sinclude .env # create from example.env
-PROJECT := nf-quilt
-BUCKET := quilt-ernest-staging
-NF_DIR := ../nextflow
-PID := $$$$
-PIP := python -m pip
-PIPELINE := sarek
-QUILT_URI :=  quilt+s3://$(BUCKET)\#package=$(PROJECT)/$(PIPELINE)
-QUILT3 := /usr/local/bin/pip3
-REPORT := ./plugins/$(PROJECT)/build/reports/tests/test/index.html
+PROJECT ?= nf-quilt
+WRITE_BUCKET ?= quilt-example
+NF_DIR ?= ../nextflow
+PID ?= $$$$
+PIP ?= python -m pip
+PIPELINE ?= sarek
+TEST_URI ?= "quilt+s3://$(WRITE_BUCKET)\#package=test/hurdat"
+QUILT_URI ?=  quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE)
+PIP ?= pip3
+QUILT3 ?= /usr/local/bin/quilt3
+REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html
 
 verify: #compile
+	echo $(WRITE_BUCKET)
 	./gradlew check || open $(REPORT)
 
+check-env:
+	echo $(WRITE_BUCKET)
+	echo "Use 'make WRITE_BUCKET=<value>' to override" 
+	printenv MAKEFLAGS
+
 clean:
 	./gradlew clean
 	rm -rf work
@@ -40,7 +48,7 @@ check:
 coverage: compile
 	./gradlew cloverInstrumentCodeForTest
 
-.PHONY: clean test test-all all
+.PHONY: clean test test-all all pkg-test tower-test
 
 test: clean compile check #coverage
 
@@ -52,15 +60,14 @@ test-all: clean compile-all check #coverage
 # Create packages
 #
 
-# use 'make pkg-test BUCKET=my-s3-bucket' to publish test output to a Quilt package
-
-pkg-test: compile-all
-	./launch.sh run ./main.nf -profile standard -plugins $(PROJECT) --pub "quilt+s3://$(BUCKET)#package=test/hurdat"
+pkg-test: #compile-all
+	echo $(TEST_URI)
+	./launch.sh run ./main.nf -profile standard -plugins $(PROJECT) --pub "$(TEST_URI)"
 
 tower-test: compile-all
-	./launch.sh run ./main.nf -with-tower -profile standard -plugins $(PROJECT) --pub "quilt+s3://$(BUCKET)#package=test/hurdat"
+	./launch.sh run ./main.nf -with-tower -profile standard -plugins $(PROJECT) --pub "$(TEST_URI)"
 
-# use `make $(PIPELINE) BUCKET=my-s3-bucket` to publish `--outdir` to a Quilt package
+# use `make $(PIPELINE) WRITE_BUCKET=my-s3-bucket` to publish `--outdir` to a Quilt package
 
 $(PIPELINE): compile-all
 	./launch.sh pull nf-core/$(PIPELINE)

README.md

@@ -8,17 +8,24 @@ to Quilt packages using `quilt+s3` URIs wherever you currently use `s3`, `az` or
 
 Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs
 
-# Prerequisite:  `quilt3` command-line tool
+## Prerequisite:  `quilt3` command-line tool
 
 Depending on your configuration, you may need to first install the `quilt3` command-line tool.
 This is distributed as an open source Python package you can install using `pip`,
-and must be available in the PATH used by `nextflow`
+and must be available in the PATH used by `nextflow`.
+
+For NextFlow Tower, do this in the "Pre-run script" when you edit the Pipeline settings from the Launchpad:
 
 ```bash
-pip install quilt3
+yum install python3-pip -y # optional in CLI
+yum install git -y         # optional in CLI
+pip3 install quilt3
 which quilt3
 ```
 
+If you are running from the command-line, you likely already have Python and GIT,
+so the first two lines are optional.
+
 ## I. QuickStart
 
 To quickly run `nf-quilt` from this GitHub repository:
@@ -27,24 +34,25 @@ To quickly run `nf-quilt` from this GitHub repository:
 git clone https://github.com/quiltdata/nf-quilt.git
 cd nf-quilt
 make test-all # runs unit tests and installs depdencies
-make pkg-test BUCKET=destination-bucket # create "test/hurdat" package
+make pkg-test WRITE_BUCKET=destination-bucket # create "test/hurdat" package
 ./launch.sh run nf-core/sarek -profile test,docker -plugins nf-quilt \
             --outdir "quilt+s3://destination-bucket#package=nf-quilt/sarek"
 
 ```
+
 ## II. Usage
 
 _NOTE: If using a published version of `nf-quilt` (e.g., v0.3.2 when it is avialable),
-you can start using this immediately with [NextFlow Tower](https://cloud.tower.nf) or 
-the `nextflow` [command-line tool](https://github.com/nextflow-io/nextflow).  
-Otherwise, see "Using Development Versions" below._
+you can start using this immediately with [NextFlow Tower](https://cloud.tower.nf) or
+the `nextflow` [command-line tool](https://github.com/nextflow-io/nextflow).
+Otherwise, see "Using Development Versions" below.
 
 There are three simple steps to redirecting your existing NextFlow pipelines to read and write from
 Quilt packages. Please note that it has only been tested on NextFlow version 22.10.6.
 
 ### 1. Construct a Quilt+ URI for each package
 
-Each Quilt+ package URI has the form: 
+Each Quilt+ package URI has the form:
 
 ```string
 quilt+s3://bucket#package=prefix/suffix
@@ -58,7 +66,7 @@ If you are running the Quilt web catalog, you can find the Quilt+ URI
 for an existing package in the `<> CODE | URI` section at the top.
 You can also manually create URIs for new packages that don't exist.
 
-#### Quilt+ URIs for Metadata Workflows 
+#### Quilt+ URIs for Metadata Workflows
 
 Sometimes you may want to ensure the created package contains specific metadata.
 This is done using [Quilt workflows](https://docs.quiltdata.com/advanced/workflows).
@@ -73,7 +81,6 @@ Note that specifying a workflow means that package creation will fail (and nothi
 if the query string does not contain all the required metadata,
 so you should carefully test it beforehand.
 
-
 ### 2. Set the appropriate parameter(s) in your pipeline
 
 There is no formal convention, but most pipelines specify a parameter such as 'outdir' or 'pub'
@@ -84,7 +91,7 @@ Note that keys specified on the command-line override the corresponding 'params.
 
 ### 3. Tell NextFlow to use the nf-quilt plugin
 
-There are two ways to tell NextFlow to use this plugin.  
+There are two ways to tell NextFlow to use this plugin.
 
 You can edit `nextflow.config` to add it permanently:
 
@@ -119,7 +126,7 @@ pip install quilt3
 which quilt3
 ```
 
-### Unit Testing 
+### Unit Testing
 
 You can compile run unit tests with:
 
@@ -129,7 +136,7 @@ make check
 
 ### Verifying NextFlow
 
-If this is your first time using NextFlow, you may also need to install a recent 
+If this is your first time using NextFlow, you may also need to install a recent
 [version of Java](https://www.java.com/en/download/help/download_options.html) for your platform.
 NextFlow itself will take care of all the other dependencies.
 
@@ -143,6 +150,7 @@ make nextflow-22-10
 
 To verify that the plugin, nextflow, and your AWS credentials have been properly installed,
 type:
+
 ```bash
    ./launch.sh run ./main.nf -profile standard -plugins $(PROJECT) --pub "quilt+s3://bucket#package=test/hurdat"
 ```
@@ -164,7 +172,7 @@ and replace `outdir` with the appropriate parameter for `publishDir`.
 
 ## IV. Package, upload and publish
 
-The project should be hosted in a GitHub repository whose name should match the name of the plugin, 
+The project should be hosted in a GitHub repository whose name should match the name of the plugin,
 that is the name of the directory in the `plugins` folder (e.g. `nf-quilt`).
 
 Follow these steps to package, upload and publish the plugin:
@@ -177,15 +185,17 @@ Follow these steps to package, upload and publish the plugin:
    * `github_commit_email`: The email address associated with your GitHub account.
 
 2. Use the following command to package and create a release for your plugin on GitHub:
-```bash
-./gradlew :plugins:nf-quilt:upload
-```
+
+   ```bash
+   ./gradlew :plugins:nf-quilt:upload
+   ```
 
 3. Fork the [nextflow-io/plugins](https://github.com/nextflow-io/plugins) repository to one you can write to
 
 4. Use the following command to publish your plugin to your fork:
- ```bash
- ./gradlew :plugins:publishIndex
- ```
+
+   ```bash
+   ./gradlew :plugins:publishIndex
+   ```
 
 5. Create a pull request to push your changes back to [nextflow-io/plugins](https://github.com/nextflow-io/plugins/blob/main/plugins.json)

gradlew

@@ -144,15 +144,15 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac