75 push uri if benchlingexperiment id in query string (#79)

.github/workflows/mega-linter.yml

@@ -13,7 +13,7 @@ env: # Comment env block if you do not want to apply fixes
   APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool)
   #APPLY_FIXES_EVENT: pull_request # Decide which event triggers application of fixes in a commit or a PR (pull_request, push, all)
   #APPLY_FIXES_MODE: pull_request # If APPLY_FIXES is used, defines if the fixes are directly committed (commit) or posted in a PR (pull_request)
-  DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD
+  DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD,REPOSITORY_GITLEAKS
   FILTER_REGEX_EXCLUDE: .*/.*gradle
 
 concurrency:

CHANGELOG.md

@@ -2,9 +2,11 @@
 
 ## UNRELEASED
 
-## [0.3.4] 2023-03-28
+## [0.3.4] 2023-04-04
 
-- Call Benchling API
+- Alpha: Call Benchling API after publishing package
+- accept "?commit_message=text+string&readme=GString" in Quilt+ URI
+- Support NextFlow 23+
 
 ## [0.3.3] 2023-03-08
 

Makefile

@@ -1,12 +1,13 @@
 sinclude .env # create from example.env
 PROJECT ?= nf-quilt
 WRITE_BUCKET ?= quilt-example
+FRAGMENT ?= &path=.
 NF_DIR ?= ../nextflow
 PID ?= $$$$
 PIP ?= python -m pip
 PIPELINE ?= sarek
 QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com
-TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat
+TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT)
 QUILT_URI ?=  quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE)
 PIP ?= pip3
 QUILT3 ?= /usr/local/bin/quilt3
@@ -32,16 +33,15 @@ compile:
 	./gradlew compileGroovy exportClasspath
 	@echo "DONE `date`"
 
-nextflow-22-10:
+nextflow:
 	if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git  "$(NF_DIR)"; fi
-	pushd "$(NF_DIR)"; git checkout 4f776ef -b v22_10_6_$(PID) && make compile && git restore .; popd
-	# https://github.com/nextflow-io/nextflow/releases/tag/v22.10.6
+	pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd
 
 install-python:
 	if [ ! -x "$(QUILT3)" ]; then $(PIP) install quilt3 ; fi
 	which quilt3
 
-compile-all: install-python nextflow-22-10 compile
+compile-all: install-python nextflow compile
 
 check:
 	./gradlew check --warning-mode all
@@ -53,7 +53,7 @@ coverage: compile
 
 test: clean compile check #coverage
 
-test-nextflow: clean nextflow-22-10 compile check #coverage
+test-nextflow: clean nextflow compile check #coverage
 
 test-all: clean compile-all check #coverage
 

README.md

@@ -31,7 +31,7 @@ The above instructions use the 'yum' package manager, which NextFlow Tower uses
 If you are running from the command-line, you may need to use your own package manager
 (or just skip those lines if you already have Python and Git).
 
-1. Enable the `nf-quilt` plugin
+. Enable the `nf-quilt` plugin
 
 The usual way to enable a plugin is to add the following to your `nextflow.config` file,
 or (in Tower) the "Advanced Options ->  Nextflow config file":
@@ -130,7 +130,7 @@ NextFlow itself will take care of all the other dependencies.
 You can verify and compile NextFlow with:
 
 ```bash
-make nextflow-22-10
+make nextflow
 ```
 
 ### Testing Installation
@@ -186,3 +186,65 @@ Follow these steps to package, upload and publish the plugin:
    ```
 
 5. Create a pull request to push your changes back to [nextflow-io/plugins](https://github.com/nextflow-io/plugins/blob/main/plugins.json)
+
+## IV. Advanced Options
+
+You can use the Makefile to test the following options,
+as long as you set a WRITE_BUCKET:
+
+```bash
+export WRITE_BUCKET=bucket-with-write-access
+make pkg-test  # create "test/hurdat" package on s3://$WRITE_BUCKET
+```
+
+### A. Quilt+ URIs for Metadata Workflows
+
+Sometimes you may want to ensure the created package contains specific metadata.
+This is done using [Quilt workflows](https://docs.quiltdata.com/advanced/workflows).
+Specify the workflow name as an additional `workflow=` fragment parameter,
+and any metadata properties as part of the query string.
+
+```bash
+make pkg-test QUERY='?mkey1=val1&mkey2=val2' FRAGMENT='&workflow=my_workflow'
+```
+
+Note that specifying a workflow means that package creation will fail (and nothing will be saved)
+if the query string does not contain all the required metadata,
+so you should carefully test it before running long jobs.
+
+### B. Quilt+ URIs for Custom Data Products
+
+Version 0.3.4 and later allow you to customize both the `commit_message`
+and `readme` via metadata query keys:
+
+```bash
+make pkg-test QUERY='?commit_message=text+str&readme=GStr+%24msg+%24now+%24%7Bmeta[%22quilt%22]%7D'
+```
+
+The `readme` parameter is a Groovy GString template which expands the variables:
+
+* `msg`: the current commit_message
+* `now`: the ISO 8601 date and time
+* `meta`: the complete metadata (very large! use only  subsets)
+
+### C. Benchling Integration (Preview)
+
+Version 0.3.4 includes alpha support for a `benchling.experiment_id`
+metadata key in the query parameter:
+
+```bash
+make pkg-test QUERY='?benchling.experiment_id=123'
+```
+
+After package push, this will add the URI of the output quilt package to a
+metadata field in the Benchling notebook.
+
+In order to use this, you must also export two environment variables:
+
+```string
+# The URL of your custom Benchling domain (only for paid plans)
+export BENCHLING_TENANT=https://mock-benchling.proxy.beeceptor.com/
+
+# The base64 API Key from your Benchling Developer settings
+export BENCHLING_API_KEY=R4nd0mB4se64N0mb3r
+```

VERSION

@@ -1 +1 @@
-22.10.7
+23.04.0

example.env

@@ -1,3 +1,3 @@
-export NXF_VER=22.10.6
+export NXF_VER=23.04.0
 export TOWER_ACCESS_TOKEN=eyxxxxxxxxxxxxxxxQ1ZTE=
 export TOWER_WORKSPACE_ID=000000000000000

plugins/nf-quilt/build.gradle

@@ -55,7 +55,7 @@ sourceSets {
 }
 
 ext{
-    nextflowVersion = '22.10.6'
+    nextflowVersion = '23.04.0'
 }
 
 dependencies {

plugins/nf-quilt/src/main/nextflow/quilt/QuiltBenchling.groovy

@@ -31,6 +31,8 @@ import benchling.api.EntriesApi
 @CompileStatic
 class QuiltBenchling {
 
+    public static final String EXPERIMENT_ID = 'benchling.experiment_id'
+
     private final EntriesApi api
 
     QuiltBenchling() {

plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy

@@ -16,14 +16,10 @@
 package nextflow.quilt
 
 import nextflow.quilt.jep.QuiltParser
-import nextflow.quilt.jep.QuiltPackage
 import nextflow.quilt.nio.QuiltPath
 import nextflow.quilt.nio.QuiltPathFactory
 
-import java.nio.file.Files
 import java.nio.file.Path
-import java.nio.file.Paths
-import java.time.LocalDateTime
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
@@ -39,38 +35,9 @@ import nextflow.trace.TraceObserver
 @CompileStatic
 class QuiltObserver implements TraceObserver {
 
-    private final static String[] BIG_KEYS = [
-        'nextflow', 'commandLine', 'scriptFile', 'projectDir',
-        'homeDir', 'workDir', 'launchDir', 'manifest', 'configFiles'
-    ]
-    private final Set<QuiltPackage> pkgs = [] as Set
-    private Map config
-    private Map quiltConfig
+    private final Set<QuiltPath> paths = [] as Set
     private Session session
 
-    static void printMap(Map map, String title) {
-        log.info("\n\n\n# $title")
-        map.each {
-            key, value -> log.info("\n## ${key}: ${value}")
-        }
-    }
-
-    static void writeString(String text, QuiltPackage pkg, String filename) {
-        String dir = pkg.packageDest()
-        Path path  = Paths.get(dir, filename)
-        try {
-            Files.write(path, text.bytes)
-        }
-        catch (Exception e) {
-            log.error("writeString: cannot write `$text` to `$path` for `${pkg}`")
-        }
-    }
-
-    static String now() {
-        LocalDateTime time = LocalDateTime.now()
-        return time.toString()
-    }
-
     static QuiltPath asQuiltPath(Path path) {
         if (path in QuiltPath) {
             return (QuiltPath) path
@@ -101,9 +68,7 @@ class QuiltObserver implements TraceObserver {
     void onFlowCreate(Session session) {
         log.debug("`onFlowCreate` $this")
         this.session = session
-        this.config = session.config
-        this.quiltConfig = session.config.navigate('quilt') as Map
-        this.pkgs
+        this.paths
     }
 
     @Override
@@ -113,85 +78,18 @@ class QuiltObserver implements TraceObserver {
 
         if (qPath) {
             QuiltPath npath = normalizePath(qPath, session.getParams())
-            this.pkgs.add(npath.pkg())
-            log.debug("onFilePublish.QuiltPath[$qPath]: pkgs=${pkgs}")
+            this.paths.add(npath)
+            log.debug("onFilePublish.QuiltPath[$qPath]: paths=${paths}")
         } else {
             log.warn("onFilePublish.QuiltPath missing: $path")
         }
     }
 
     @Override
     void onFlowComplete() {
-        log.debug("`onFlowComplete` ${pkgs}")
+        log.debug("`onFlowComplete` ${paths}")
         // publish pkgs to repository
-        this.pkgs.each { pkg -> publish(pkg) }
-    }
-
-    String readme(Map meta, String msg) {
-        return """
-            # ${now()}
-            ## $msg
-
-            ## workflow
-            ### scriptFile: ${meta['workflow']['scriptFile']}
-            ### sessionId: ${meta['workflow']['sessionId']}
-            - start: ${meta['time_start']}
-            - complete: ${meta['time_complete']}
-
-            ## processes
-            ${meta['workflow']['stats']['processes']}
-
-            """.stripIndent()
-    }
-
-    int publish(QuiltPackage pkg) {
-        String msg = pkg
-        Map meta = [pkg: msg]
-        String text = 'Stub README'
-        try {
-            meta = getMetadata()
-            msg = "${meta['config']['runName']}: ${meta['cmd']}"
-            text = readme(meta, msg)
-            meta.remove('config')
-        }
-        catch (Exception e) {
-            log.error('publish: cannot generate metadata (QuiltObserver uninitialized?)', e)
-        }
-        writeString(text, pkg, 'README.md')
-        writeString("$meta", pkg, 'quilt_metadata.txt')
-        def rc = pkg.push(msg, meta)
-        log.info("$rc: pushed package[$pkg] $msg")
-        if (rc > 0) {
-            print("ERROR[package push failed]: $pkg")
-        }
-
-        return rc
-    }
-
-    Map getMetadata() {
-        // TODO: Write out config files
-        Map cf = config
-        cf.remove('params')
-        cf.remove('session')
-        cf.remove('executor')
-        printMap(cf, 'config')
-        Map params = session.getParams()
-        params.remove('genomes')
-        params.remove('test_data')
-        printMap(params, 'params')
-        Map wf = session.getWorkflowMetadata().toMap()
-        String start = wf['start']
-        String complete = wf['complete']
-        String cmd = wf['commandLine']
-        BIG_KEYS.each { k -> wf[k] = "${wf[k]}" }
-        wf.remove('container')
-        wf.remove('start')
-        wf.remove('complete')
-        wf.remove('workflowStats')
-        wf.remove('commandLine')
-        printMap(wf, 'workflow')
-        log.info("\npublishing: ${wf['runName']}")
-        return [params: params, config: cf, workflow: wf, cmd: cmd, time_start: start, time_complete: complete]
+        this.paths.each { path -> new QuiltProduct(path, session) }
     }
 
 }