71 path input (#174)

- Make input URIs with `&path=` actually work
- also support query arrays

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.7.7] 2024-01-24
+
+- Properly implement and test getFilename()
+- Install package just before download
+- Add and pass `path-input` integration test
+- Add unit test for "&path=" Quilt+ URIs
+
 ## [0.7.6] 2024-01-10 UNOFFICIAL
 
 - Re-enable crash on failure

Makefile

@@ -3,7 +3,7 @@ PROJECT ?= nf-quilt
 WRITE_BUCKET ?= quilt-example
 FRAGMENT ?= &path=.
 NF_DIR ?= ../nextflow
-NF_BIN ?= ./nextflow
+NF_BIN ?= ./launch.sh
 PID ?= $$$$
 PIPELINE ?= sarek
 QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com
@@ -36,10 +36,6 @@ compile:
 	./gradlew compileGroovy exportClasspath
 	@echo "DONE `date`"
 
-$(NF_BIN):
-	curl -s https://get.nextflow.io | bash
-	chmod +x $(NF_BIN)
-
 nextflow-git:
 	if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git  "$(NF_DIR)"; fi
 	cd "$(NF_DIR)"; git checkout && make compile && git restore .; cd ..
@@ -61,14 +57,18 @@ test-all: clean compile-all check #coverage
 # Create packages
 #
 
-pkg-test: compile-all
+pkg-test: compile #-all
 	echo "$(TEST_URI)"
 	$(NF_BIN) run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)"
 
 pkg-fail: compile
 	echo "$(TEST_URI)"
 	$(NF_BIN) run ./fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)"
 
+path-input: clean compile #-all
+	echo "$(TEST_URI)"
+	$(NF_BIN) run ./main.path.nf -profile standard -plugins $(PROJECT) --outdir "./results"
+
 tower-test: $(NF_BIN)
 	$(NF_BIN) run "https://github.com/quiltdata/nf-quilt" -name local_einstein  -with-tower -r main -latest --pub "$(TEST_URI)"
 

README.md

@@ -75,8 +75,8 @@ From the command-line, do, e.g.:
 
 ```bash
 # export NXF_VER=23.04.3
-export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.6/nf-quilt-0.7.6-meta.json
-nextflow run main.nf -plugins [email protected].6
+export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.7/nf-quilt-0.7.7-meta.json
+nextflow run main.nf -plugins [email protected].7
 ```
 
 For Tower, you can use the "Pre-run script" to set the environment variables.

main.path.nf

@@ -0,0 +1,29 @@
+#!/usr/bin/env nextflow
+/* groovylint-disable CompileStatic */
+
+nextflow.enable.dsl=2
+
+test_file_local = 'README.md'
+test_file_s3 = 's3://quilt-example/examples/hurdat2/README.md'
+test_file_quilt = 'quilt+s3://quilt-example#package=examples/hurdat2&path=README.md'
+
+myFileChannel = Channel.fromList([file(test_file_local), file(test_file_s3), file(test_file_quilt)])
+
+process CHECK_INPUT {
+    input:
+        path input
+
+    output:
+        path 'README.md', emit: output
+
+    script:
+    """
+    ls -l
+    echo $input
+    cp -f $input ../../tmp
+    """
+}
+
+workflow {
+    CHECK_INPUT(myFileChannel)
+}

plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy

@@ -145,6 +145,7 @@ ${nextflow}
             log.error("Exception: ${e}")
             print("FAILED: $pkg\n")
             e.printStackTrace()
+            /* groovylint-disable-next-line ThrowRuntimeException */
             throw new RuntimeException(e)
         }
         print("SUCCESS: $pkg\n")

plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy

@@ -121,7 +121,7 @@ class QuiltPackage {
         this.hash = parsed.getHash()
         this.meta = parsed.getMetadata()
         this.folder = Paths.get(INSTALL_ROOT.toString(), this.toString())
-        //log.debug("QuiltParser.folder[${this.folder}]")
+        log.debug("QuiltPackage.folder[${this.folder}]")
         this.setup()
     }
 
@@ -187,11 +187,12 @@ class QuiltPackage {
             String resolvedHash = (hash == 'latest' || hash == null || hash == 'null')
               ? namespace.getHash('latest')
               : hash
-            log.info("hash: $hash -> $resolvedHash")
+            log.debug("hash: $hash -> $resolvedHash")
             Manifest manifest = namespace.getManifest(resolvedHash)
 
             manifest.install(dest)
-            log.info('done')
+            log.debug("done: installed into $dest)")
+            println("Children: ${relativeChildren('')}")
         } catch (IOException e) {
             log.error("failed to install $packageName")
             // this is non-fatal error, so we don't want to stop the pipeline

plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy

@@ -1,3 +1,4 @@
+/* groovylint-disable Instanceof */
 /*
  * Copyright 2022, Quilt Data Inc
  *
@@ -75,18 +76,50 @@ class QuiltParser {
         return new QuiltParser(uri.authority, pkg, path, options, metadata)
     }
 
-    static Map<String,Object> parseQuery(String query) {
+    static String decode(String str) {
+        return URLDecoder.decode(str, StandardCharsets.UTF_8)
+    }
+
+    static String encode(String str) {
+        return URLEncoder.encode(str, StandardCharsets.UTF_8)
+    }
+
+    static Map<String, Object> parseQuery(String query) {
         if (!query) { return [:] } // skip for urls without query params
-        final queryParams = query.split('&')
-        return queryParams.collectEntries { params -> params.split('=').collect { param -> URLDecoder.decode(param) } }
+        def params = query.split('&')
+        def result = [:]
+        params.each { param ->
+            def keyValue = param.split('=')
+            if (keyValue.size() == 2) {
+                String key = decode(keyValue[0])
+                String value = decode(keyValue[1])
+                if (result.containsKey(key)) {
+                    if (result[key] instanceof List) {
+                        result[key].add(value)
+                    } else {
+                        result[key] = [result[key], value]
+                    }
+                } else {
+                    result[key] = value
+                }
+            }
+
+        }
+        return result
+    }
+
+    static String encodePair(String key, String value) {
+        return "${QuiltParser.encode(key)}=${QuiltParser.encode(value)}"
     }
 
     static String unparseQuery(Map<String,Object> query) {
         if (!query) { return '' } // skip for urls without query params
-        List<String> params = query.collect {  key, val ->
-            String k = URLEncoder.encode(key, StandardCharsets.UTF_8)
-            String v = URLEncoder.encode(val.toString(), StandardCharsets.UTF_8)
-            "${k}=${v}".toString()
+        List<String> params = query.collect {  key, value ->
+            if (value instanceof List) {
+                value.collect { QuiltParser.encodePair(key, it.toString()) }.join('&')
+            } else {
+                QuiltParser.encodePair(key, value.toString())
+            }
         }
         return params.join('&')
     }

plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy

@@ -106,7 +106,7 @@ final class QuiltFileSystem extends FileSystem implements Closeable {
     }
 
     QuiltFileAttributes readAttributes(QuiltPath path)  {
-        //log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)")
+        log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)")
         Path installedPath = path.localPath()
         try {
             BasicFileAttributes attrs = Files.readAttributes(installedPath, BasicFileAttributes)
@@ -139,7 +139,7 @@ final class QuiltFileSystem extends FileSystem implements Closeable {
 
     @Override
     QuiltPath getPath(String root, String... more) {
-        //log.debug("QuiltFileSystem.getPath`[${root}]: $more")
+        log.debug("QuiltFileSystem.getPath`[${root}]: $more")
 
         QuiltParser p = QuiltParser.forBarePath(root)
         return new QuiltPath(this, p)