From 0f743ad5ec6ed58c3077fb770cda2070f5c8743f Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Wed, 19 Jul 2023 19:26:27 -0700 Subject: [PATCH 01/27] fix Makefile --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4dff5385..6e6be666 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,6 @@ QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+UR TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT) QUILT_URI ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE) PIP ?= pip3 -QUILT3 ?= /usr/local/bin/quilt3 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile @@ -38,7 +37,7 @@ nextflow: pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd install-python: - if [ ! -x "$(QUILT3)" ]; then $(PIP) install quilt3 ; fi + if ! which quilt3; then $(PIP) install quilt3 ; fi which quilt3 compile-all: install-python nextflow compile From 69691375d03e3d1028b5b451e305e6005a1859f9 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Thu, 20 Jul 2023 02:07:51 -0700 Subject: [PATCH 02/27] WIP: Use quiltcore to install packages --- plugins/build.gradle | 1 + plugins/nf-quilt/build.gradle | 3 ++ .../nextflow/quilt/jep/QuiltPackage.groovy | 32 +++++++++++++++---- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/plugins/build.gradle b/plugins/build.gradle index 1e067cdd..ed48d2c5 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -108,6 +108,7 @@ subprojects { task copyPluginLibs(type: Sync) { from configurations.runtimeClasspath into 'build/target/libs' + duplicatesStrategy 'exclude' } /* diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index fa81bef7..11580db5 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -59,6 +59,9 @@ ext{ } dependencies { + // quiltcore + implementation zipTree('../../../quiltcore-java/lib/build/distributions/quiltcore.zip') + // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" compileOnly 'org.slf4j:slf4j-api:2.0.7' diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index b1d84fd8..bfe55480 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -28,6 +28,11 @@ import java.nio.file.attribute.BasicFileAttributes import java.util.stream.Collectors import java.time.LocalDate +import com.quiltdata.quiltcore.Registry +import com.quiltdata.quiltcore.Namespace +import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.S3PhysicalKey + @Slf4j @CompileStatic class QuiltPackage { @@ -86,7 +91,7 @@ class QuiltPackage { log.debug("Installing `${pkg}` for.pkgKey $pkgKey") pkg.install() } - catch (Exception e) { + catch (IOException e) { log.warn("Package `${parsed.toUriString()}` does not yet exist") } return pkg @@ -249,17 +254,30 @@ class QuiltPackage { } } - // usage: quilt3 install [-h] [--registry REGISTRY] [--top-hash TOP_HASH] - // [--dest DEST] [--dest-registry DEST_REGISTRY] [--path PATH] name Path install() { - int exitCode = call('install', packageName, key_registry(), key_hash(), key_dest()) - if (exitCode != 0) { - log.error("`install.fail.exitCode` ${exitCode}: ${packageName}") + Path dest = packageDest() + + try { + log.info("installing $packageName from $bucket...") + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + log.info("registryPath: $registryPath") + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash + log.info("hash: $hash -> $resolvedHash") + Manifest manifest = namespace.getManifest(resolvedHash) + + manifest.install(dest) + log.info("done") + } catch (IOException e) { + log.error("failed to install $packageName") return null } + installed = true recursiveDeleteOnExit() - return packageDest() + + return dest } // https://stackoverflow.com/questions/15022219 From 725802c34e6dc6bec3c09de9aa87de77cddae5e2 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Tue, 22 Aug 2023 17:24:11 -0700 Subject: [PATCH 03/27] Replace push with quiltcore --- Makefile | 8 +- .../nextflow/quilt/jep/QuiltPackage.groovy | 112 ++++++------------ 2 files changed, 35 insertions(+), 85 deletions(-) diff --git a/Makefile b/Makefile index 6e6be666..693e3407 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,10 @@ WRITE_BUCKET ?= quilt-example FRAGMENT ?= &path=. NF_DIR ?= ../nextflow PID ?= $$$$ -PIP ?= python -m pip PIPELINE ?= sarek QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT) QUILT_URI ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE) -PIP ?= pip3 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile @@ -36,11 +34,7 @@ nextflow: if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd -install-python: - if ! which quilt3; then $(PIP) install quilt3 ; fi - which quilt3 - -compile-all: install-python nextflow compile +compile-all: nextflow compile check: ./gradlew check --warning-mode all diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index bfe55480..9ec5b2f7 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -28,11 +28,16 @@ import java.nio.file.attribute.BasicFileAttributes import java.util.stream.Collectors import java.time.LocalDate +import com.quiltdata.quiltcore.Entry import com.quiltdata.quiltcore.Registry import com.quiltdata.quiltcore.Namespace import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.LocalPhysicalKey import com.quiltdata.quiltcore.key.S3PhysicalKey +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + @Slf4j @CompileStatic class QuiltPackage { @@ -48,7 +53,6 @@ class QuiltPackage { private final Path folder private final Map meta private boolean installed - private String cmd = 'N/A' static String today() { LocalDate date = LocalDate.now() @@ -170,51 +174,10 @@ class QuiltPackage { this.installed = false } - String key_dest() { - return "--dest ${packageDest()}" - } - - String key_dir() { - return "--dir ${packageDest()}" - } - boolean is_force() { return parsed.options[QuiltParser.P_FORCE] } - String key_force() { - log.debug("key_force.options[${parsed.options[QuiltParser.P_FORCE]}] ${parsed.options}") - return is_force() ? '--force' : '' - } - - String key_hash() { - return (hash == 'latest' || hash == null || hash == 'null') ? '' : "--top-hash $hash" - } - - String key_meta(Map srcMeta = [:]) { - log.debug("key_meta.srcMeta $srcMeta") - log.debug("key_meta.uriMeta ${meta}") - Map metas = srcMeta + meta - if (metas.isEmpty()) { return '' } - - String jsonMeta = toJson(metas) - log.debug("key_meta.jsonMeta $jsonMeta") - return "--meta '$jsonMeta'" - } - - String key_msg(String message='') { - String msg = meta_overrides('msg', "nf-quilt:${today()}-${message}") - return "--message '${sanitize(msg)}'" - } - - String key_registry() { - return "--registry s3://${bucket}" - } - - String key_workflow() { - return (parsed.workflowName) ? "--workflow ${parsed.workflowName}" : '' - } - boolean isInstalled() { return installed } @@ -223,44 +186,12 @@ class QuiltPackage { return folder } - String lastCommand() { - return cmd - } - - int call(String... args) { - def command = ['quilt3'] - command.addAll(args) - cmd = command.join(' ') - log.debug("call `${cmd}`") - - try { - ProcessBuilder pb = new ProcessBuilder('bash', '-c', cmd) - pb.redirectErrorStream(true) - - Process p = pb.start() - log.debug("call.start ${p}") - String result = new String(p.getInputStream().readAllBytes()) - log.debug("call.result ${result}") - int exitCode = p.waitFor() - log.debug("call.exitCode ${exitCode}") - if (exitCode != 0) { - log.debug("`call.fail` rc=${exitCode}[${cmd}]: ${result}\n") - } - return exitCode - } - catch (Exception e) { - log.error("Failed `${cmd}` ${this}", e) - return -1 - } - } - Path install() { Path dest = packageDest() try { log.info("installing $packageName from $bucket...") S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) - log.info("registryPath: $registryPath") Registry registry = new Registry(registryPath) Namespace namespace = registry.getNamespace(packageName) String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash @@ -301,10 +232,35 @@ class QuiltPackage { } // https://docs.quiltdata.com/v/version-5.0.x/examples/gitlike#install-a-package int push(String msg = 'update', Map meta = [:]) { - int exitCode = 0 - String args = [key_dir(), key_registry(), key_meta(meta), key_msg(msg), key_force()].join(' ') - exitCode = call('push', packageName, args, key_workflow()) - return exitCode + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + + Manifest.Builder builder = Manifest.builder() + + Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { + System.out.println(f) + String logicalKey = packageDest().relativize(f) + LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) + long size = Files.size(f) + builder.addEntry(logicalKey, new Entry(physicalKey, size, null, null)) + }); + + Map fullMeta = [ + "version": Manifest.VERSION, + "user_meta": meta + this.meta, + ] + ObjectMapper mapper = new ObjectMapper() + builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta)) + + Manifest m = builder.build() + + try { + m.push(namespace, "nf-quilt:${today()}-${msg}") + } catch (IOException e) { + return 1 + } + return 0 } @Override From f68b84ac584249d629864df88585cd785c4ed185 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Tue, 22 Aug 2023 22:38:38 -0700 Subject: [PATCH 04/27] Use a maven repo at s3://quilt-dima --- plugins/nf-quilt/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 11580db5..498f4fba 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -38,6 +38,7 @@ repositories { maven { url = 'https://jitpack.io' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' } + maven { url = 'https://quilt-dima.s3.amazonaws.com/maven/releases' } } configurations { @@ -60,7 +61,7 @@ ext{ dependencies { // quiltcore - implementation zipTree('../../../quiltcore-java/lib/build/distributions/quiltcore.zip') + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.1' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" From 00c1e08e9a6aad72a407132018cc8bee546c8865 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Wed, 19 Jul 2023 19:26:27 -0700 Subject: [PATCH 05/27] fix Makefile --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 134ac862..e599e704 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,6 @@ QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+UR TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT) QUILT_URI ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE) PIP ?= pip3 -QUILT3 ?= /usr/local/bin/quilt3 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile @@ -38,7 +37,7 @@ nextflow: pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd install-python: - if [ ! -x "$(QUILT3)" ]; then $(PIP) install quilt3 ; fi + if ! which quilt3; then $(PIP) install quilt3 ; fi which quilt3 compile-all: install-python nextflow compile From 46716ae2fd58692f762e17fafdb3524f90ac0975 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Thu, 20 Jul 2023 02:07:51 -0700 Subject: [PATCH 06/27] WIP: Use quiltcore to install packages --- plugins/build.gradle | 1 + plugins/nf-quilt/build.gradle | 3 ++ .../nextflow/quilt/jep/QuiltPackage.groovy | 32 +++++++++++++++---- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/plugins/build.gradle b/plugins/build.gradle index 1e067cdd..ed48d2c5 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -108,6 +108,7 @@ subprojects { task copyPluginLibs(type: Sync) { from configurations.runtimeClasspath into 'build/target/libs' + duplicatesStrategy 'exclude' } /* diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 1627df82..66584cae 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -59,6 +59,9 @@ ext{ } dependencies { + // quiltcore + implementation zipTree('../../../quiltcore-java/lib/build/distributions/quiltcore.zip') + // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" compileOnly 'org.slf4j:slf4j-api:2.0.7' diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 797d567a..45c92b89 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -28,6 +28,11 @@ import java.nio.file.attribute.BasicFileAttributes import java.util.stream.Collectors import java.time.LocalDate +import com.quiltdata.quiltcore.Registry +import com.quiltdata.quiltcore.Namespace +import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.S3PhysicalKey + @Slf4j @CompileStatic class QuiltPackage { @@ -86,7 +91,7 @@ class QuiltPackage { log.debug("${pkg}: attempting install for.pkgKey $pkgKey (okay if fails)") pkg.install() } - catch (Exception e) { + catch (IOException e) { log.warn("Package `${parsed.toUriString()}` does not yet exist") } return pkg @@ -250,17 +255,30 @@ class QuiltPackage { } } - // usage: quilt3 install [-h] [--registry REGISTRY] [--top-hash TOP_HASH] - // [--dest DEST] [--dest-registry DEST_REGISTRY] [--path PATH] name Path install() { - int exitCode = call('install', packageName, key_registry(), key_hash(), key_dest()) - if (exitCode != 0) { - log.warn("${exitCode}: ${packageName} failed to install (may not exist)") + Path dest = packageDest() + + try { + log.info("installing $packageName from $bucket...") + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + log.info("registryPath: $registryPath") + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash + log.info("hash: $hash -> $resolvedHash") + Manifest manifest = namespace.getManifest(resolvedHash) + + manifest.install(dest) + log.info("done") + } catch (IOException e) { + log.error("failed to install $packageName") return null } + installed = true recursiveDeleteOnExit() - return packageDest() + + return dest } // https://stackoverflow.com/questions/15022219 From 26c82e8a263df6e5c0a55b6284680ee13eb88efd Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Tue, 22 Aug 2023 17:24:11 -0700 Subject: [PATCH 07/27] Replace push with quiltcore --- Makefile | 8 +- .../nextflow/quilt/jep/QuiltPackage.groovy | 112 ++++++------------ 2 files changed, 35 insertions(+), 85 deletions(-) diff --git a/Makefile b/Makefile index e599e704..35439606 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,10 @@ WRITE_BUCKET ?= quilt-example FRAGMENT ?= &path=. NF_DIR ?= ../nextflow PID ?= $$$$ -PIP ?= python -m pip PIPELINE ?= sarek QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com TEST_URI ?= quilt+s3://$(WRITE_BUCKET)$(QUERY)\#package=test/hurdat$(FRAGMENT) QUILT_URI ?= quilt+s3://$(WRITE_BUCKET)\#package=$(PROJECT)/$(PIPELINE) -PIP ?= pip3 REPORT ?= ./plugins/$(PROJECT)/build/reports/tests/test/index.html verify: #compile @@ -36,11 +34,7 @@ nextflow: if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi pushd "$(NF_DIR)"; git checkout && make compile && git restore .; popd -install-python: - if ! which quilt3; then $(PIP) install quilt3 ; fi - which quilt3 - -compile-all: install-python nextflow compile +compile-all: nextflow compile check: ./gradlew check --warning-mode all diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 45c92b89..7ed0cc77 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -28,11 +28,16 @@ import java.nio.file.attribute.BasicFileAttributes import java.util.stream.Collectors import java.time.LocalDate +import com.quiltdata.quiltcore.Entry import com.quiltdata.quiltcore.Registry import com.quiltdata.quiltcore.Namespace import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.LocalPhysicalKey import com.quiltdata.quiltcore.key.S3PhysicalKey +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + @Slf4j @CompileStatic class QuiltPackage { @@ -48,7 +53,6 @@ class QuiltPackage { private final Path folder private final Map meta private boolean installed - private String cmd = 'N/A' static String today() { LocalDate date = LocalDate.now() @@ -171,51 +175,10 @@ class QuiltPackage { this.installed = false } - String key_dest() { - return "--dest ${packageDest()}" - } - - String key_dir() { - return "--dir ${packageDest()}" - } - boolean is_force() { return parsed.options[QuiltParser.P_FORCE] } - String key_force() { - //log.debug("key_force.options[${parsed.options[QuiltParser.P_FORCE]}] ${parsed.options}") - return is_force() ? '--force' : '' - } - - String key_hash() { - return (hash == 'latest' || hash == null || hash == 'null') ? '' : "--top-hash $hash" - } - - String key_meta(Map srcMeta = [:]) { - //log.debug("key_meta.srcMeta $srcMeta") - //log.debug("key_meta.uriMeta ${meta}") - Map metas = srcMeta + meta - if (metas.isEmpty()) { return '' } - - String jsonMeta = toJson(metas) - log.debug("key_meta.jsonMeta $jsonMeta") - return "--meta '$jsonMeta'" - } - - String key_msg(String message='') { - String msg = meta_overrides('msg', "nf-quilt:${today()}-${message}") - return "--message '${sanitize(msg)}'" - } - - String key_registry() { - return "--registry s3://${bucket}" - } - - String key_workflow() { - return (parsed.workflowName) ? "--workflow ${parsed.workflowName}" : '' - } - boolean isInstalled() { return installed } @@ -224,44 +187,12 @@ class QuiltPackage { return folder } - String lastCommand() { - return cmd - } - - int call(String... args) { - def command = ['quilt3'] - command.addAll(args) - cmd = command.join(' ') - log.debug("QuiltPackage.call `${cmd}`") - - try { - ProcessBuilder pb = new ProcessBuilder('bash', '-c', cmd) - pb.redirectErrorStream(true) - - Process p = pb.start() - //log.debug("call.start ${p}") - String result = new String(p.getInputStream().readAllBytes()) - //log.debug("call.result ${result}") - int exitCode = p.waitFor() - //log.debug("call.exitCode ${exitCode}") - if (exitCode != 0) { - log.debug("`call.fail` rc=${exitCode}[${cmd}]: ${result}\n") - } - return exitCode - } - catch (Exception e) { - log.warn("${e.getMessage()}: `${cmd}` ${this}") - return -1 - } - } - Path install() { Path dest = packageDest() try { log.info("installing $packageName from $bucket...") S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) - log.info("registryPath: $registryPath") Registry registry = new Registry(registryPath) Namespace namespace = registry.getNamespace(packageName) String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash @@ -302,10 +233,35 @@ class QuiltPackage { } // https://docs.quiltdata.com/v/version-5.0.x/examples/gitlike#install-a-package int push(String msg = 'update', Map meta = [:]) { - int exitCode = 0 - String args = [key_dir(), key_registry(), key_meta(meta), key_msg(msg), key_force()].join(' ') - exitCode = call('push', packageName, args, key_workflow()) - return exitCode + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + + Manifest.Builder builder = Manifest.builder() + + Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { + System.out.println(f) + String logicalKey = packageDest().relativize(f) + LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) + long size = Files.size(f) + builder.addEntry(logicalKey, new Entry(physicalKey, size, null, null)) + }); + + Map fullMeta = [ + "version": Manifest.VERSION, + "user_meta": meta + this.meta, + ] + ObjectMapper mapper = new ObjectMapper() + builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta)) + + Manifest m = builder.build() + + try { + m.push(namespace, "nf-quilt:${today()}-${msg}") + } catch (IOException e) { + return 1 + } + return 0 } @Override From 97ea35c735eabc5bb5126d1dfb376d9f289aa78b Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Tue, 22 Aug 2023 22:38:38 -0700 Subject: [PATCH 08/27] Use a maven repo at s3://quilt-dima --- plugins/nf-quilt/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 66584cae..0edbe4e9 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -38,6 +38,7 @@ repositories { maven { url = 'https://jitpack.io' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' } maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' } + maven { url = 'https://quilt-dima.s3.amazonaws.com/maven/releases' } } configurations { @@ -60,7 +61,7 @@ ext{ dependencies { // quiltcore - implementation zipTree('../../../quiltcore-java/lib/build/distributions/quiltcore.zip') + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.1' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" From c0f2961953bb69a39d546dc0c8786eaa44aede44 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 11:26:05 -0700 Subject: [PATCH 09/27] v0.5 w/o Python --- CHANGELOG.md | 4 +++ README.md | 35 +++---------------- .../src/resources/META-INF/MANIFEST.MF | 2 +- 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929a2a7a..2064fae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [0.5.0] 2023-09-04+ + +- Switch to quiltcore-java + ## [0.4.5] 2023-08-23 - fix metadata in README.md diff --git a/README.md b/README.md index 04cc61cf..defce8ed 100644 --- a/README.md +++ b/README.md @@ -6,33 +6,15 @@ NextFlow plugin for reading and writing Quilt packages as a FileSystem developed by [Quilt Data](https://quiltdata.com/) that enables you read and write directly to Quilt packages using `quilt+s3` URIs wherever your NextFlow pipeline currently use `s3` URIs. -Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs +Inspired by the original [`nf-quilt`](https://github.com/nextflow-io/nf-quilt) plugin (v0.2.0) developed by Seqera labs. ## I. Using the nf-quilt plugin in Production This plugin allows your existing pipelines, without modification, to read and write versioned Quilt packages stored on Amazon S3. -Use the following four steps to configure NextFlow Tower or your command-line environment. - -1. Install the `quilt3` command-line tool - -This is distributed as an open source Python package you can install using `pip3`, -and must be available in the PATH used by `nextflow`. - -```bash -yum install python3-pip -y -yum install git -y -pip3 install quilt3 -which quilt3 -``` - -The above instructions use the 'yum' package manager, -which NextFlow Tower uses in the "Pre-run script" -when you edit the Pipeline settings from the Launchpad. - -If you are running from the command-line, you may need to use your own package manager -(or just skip those lines if you already have Python and Git). +Use the following three steps to configure NextFlow Tower or your command-line environment. +[Note: versions 0.5.0 and later no longer require the `quilt3` Python client.] 1. Enable the `nf-quilt` plugin @@ -93,8 +75,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.4.5/nf-quilt-0.4.5-meta.json -nextflow run main.nf -plugins nf-quilt@0.4.5 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.5.0/nf-quilt-0.5.0-meta.json +nextflow run main.nf -plugins nf-quilt@0.5.0 ``` For Tower, you can use the "Pre-run script" to set the environment variables. @@ -183,13 +165,6 @@ git clone https://github.com/quiltdata/nf-quilt.git cd ./nf-quilt ``` -You also need to use Python to install the `quilt3` command-line tool used by `nf-quilt`: - -```bash -pip install quilt3 -which quilt3 -``` - ### Unit Testing You can compile run unit tests with: diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index ec3f6d05..2d73a071 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.4.5 +Plugin-Version: 0.5.0 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 From 04f7635e2b078848e24e4a80c36884fc0bcd68f7 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 11:44:27 -0700 Subject: [PATCH 10/27] stop pre-installing packages --- CHANGELOG.md | 1 + .../nextflow/quilt/jep/QuiltPackage.groovy | 34 ++++++++----------- .../nextflow/quilt/QuiltProductTest.groovy | 2 ++ .../quilt/jep/QuiltPackageTest.groovy | 2 ++ 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2064fae8..896fc1e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [0.5.0] 2023-09-04+ - Switch to quiltcore-java +- Stop trying to pre-install every package ## [0.4.5] 2023-08-23 diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 7ed0cc77..dfe1c32a 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -35,8 +35,8 @@ import com.quiltdata.quiltcore.Manifest import com.quiltdata.quiltcore.key.LocalPhysicalKey import com.quiltdata.quiltcore.key.S3PhysicalKey -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode @Slf4j @CompileStatic @@ -91,13 +91,7 @@ class QuiltPackage { return pkg } - try { - log.debug("${pkg}: attempting install for.pkgKey $pkgKey (okay if fails)") - pkg.install() - } - catch (IOException e) { - log.warn("Package `${parsed.toUriString()}` does not yet exist") - } + log.debug("${pkg}: Do NOT pre-install every package (use list API instead)") return pkg } @@ -195,21 +189,21 @@ class QuiltPackage { S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) Registry registry = new Registry(registryPath) Namespace namespace = registry.getNamespace(packageName) - String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash + boolean defaultHash = (hash == 'latest' || hash == null || hash == 'null') + String resolvedHash = defaultHash ? namespace.getHash('latest') : hash log.info("hash: $hash -> $resolvedHash") Manifest manifest = namespace.getManifest(resolvedHash) manifest.install(dest) - log.info("done") + log.info('done') + installed = true + recursiveDeleteOnExit() + + return dest } catch (IOException e) { log.error("failed to install $packageName") - return null } - - installed = true - recursiveDeleteOnExit() - - return dest + return null } // https://stackoverflow.com/questions/15022219 @@ -240,7 +234,7 @@ class QuiltPackage { Manifest.Builder builder = Manifest.builder() Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { - System.out.println(f) + println(f) String logicalKey = packageDest().relativize(f) LocalPhysicalKey physicalKey = new LocalPhysicalKey(f) long size = Files.size(f) @@ -248,8 +242,8 @@ class QuiltPackage { }); Map fullMeta = [ - "version": Manifest.VERSION, - "user_meta": meta + this.meta, + 'version': Manifest.VERSION, + 'user_meta': meta + this.meta, ] ObjectMapper mapper = new ObjectMapper() builder.setMetadata((ObjectNode)mapper.valueToTree(fullMeta)) diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index 398b6ca1..81851a25 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -22,6 +22,7 @@ import nextflow.quilt.QuiltProduct import nextflow.quilt.jep.QuiltParser import groovy.transform.CompileDynamic +import spock.lang.Ignore import spock.lang.IgnoreIf /** @@ -123,6 +124,7 @@ class QuiltProductTest extends QuiltSpecification { } @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) + @Ignore('Does NOT support re-using existing metadata to satisfy workflow') void 'pushes previous metadata if metadata=SKIP'() { given: Map meta = [ diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 4e04570f..b74d1d92 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -192,6 +192,7 @@ class QuiltPackageTest extends QuiltSpecification { opkg.push('msg', meta) == 0 } + @Ignore('QuiltCore-java does not support workflows yet') @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'should fail if invalid workflow'() { given: @@ -201,6 +202,7 @@ class QuiltPackageTest extends QuiltSpecification { bad_wf.push('missing-workflow first time', [:]) == 1 } + @Ignore('QuiltCore-java does not support workflows yet') @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'should fail push if unsatisfied workflow'() { given: From 062e2225853e6f5dad814d2bd838f57ace972921 Mon Sep 17 00:00:00 2001 From: Dima Ryazanov Date: Mon, 4 Sep 2023 13:29:40 -0700 Subject: [PATCH 11/27] Bump quiltcore to 0.0.2 --- plugins/nf-quilt/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 0edbe4e9..1c6c832f 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -61,7 +61,7 @@ ext{ dependencies { // quiltcore - implementation 'com.quiltdata.quiltcore:quiltcore:0.0.1' + implementation 'com.quiltdata.quiltcore:quiltcore:0.0.2' // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" From ab4b16a905e952ebe0d698b583f2bd0889163641 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 15:13:27 -0700 Subject: [PATCH 12/27] packageManifest() wrap hash, namespace, manifest --- .../nextflow/quilt/jep/QuiltPackage.groovy | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index dfe1c32a..c4776590 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -130,32 +130,39 @@ class QuiltPackage { this.setup() } + Namespace packageNamespace() { + S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) + Registry registry = new Registry(registryPath) + Namespace namespace = registry.getNamespace(packageName) + return namespace + } + + String resolveHash() { + if (hash == 'latest' || hash == null || hash == 'null') { + Namespace namespace = packageNamespace() + return namespace.getHash('latest') + } + return hash + } + + Manifest packageManifest() { + Namespace namespace = packageNamespace() + Manifest manifest = namespace.getManifest(resolveHash()) + return manifest + } + /** * Returns {@code List} of object keys below a subpath. * - *

Because the `quilt3` CLI does not provide a direct way to list - * the logical keys ("files") inside a package, we have to infer it - * from the actual files inside the install folder. - * - *

To do this, we list the full path of the files directly inside - * the subfolder, then remove the top-level folder ("base") to get - * the relative keys. - * * @param subpath * folder inside the package (use '' for top-level) * * @return List of the child object keys, as Strings */ + List relativeChildren(String subpath) { - Path subfolder = folder.resolve(subpath) - String base = subfolder.toString() + '/' - List result = [] - final String[] children = subfolder.list().sort() - //log.debug("relativeChildren[${base}] $children") - for (String pathString : children) { - def relative = pathString.replace(base, '') - result.add(relative) - } + Manifest manifest = packageManifest() + List result = manifest.listDirectory(subpath) return result } @@ -186,13 +193,7 @@ class QuiltPackage { try { log.info("installing $packageName from $bucket...") - S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) - Registry registry = new Registry(registryPath) - Namespace namespace = registry.getNamespace(packageName) - boolean defaultHash = (hash == 'latest' || hash == null || hash == 'null') - String resolvedHash = defaultHash ? namespace.getHash('latest') : hash - log.info("hash: $hash -> $resolvedHash") - Manifest manifest = namespace.getManifest(resolvedHash) + Manifest manifest = packageManifest() manifest.install(dest) log.info('done') @@ -227,12 +228,8 @@ class QuiltPackage { } // https://docs.quiltdata.com/v/version-5.0.x/examples/gitlike#install-a-package int push(String msg = 'update', Map meta = [:]) { - S3PhysicalKey registryPath = new S3PhysicalKey(bucket, '', null) - Registry registry = new Registry(registryPath) - Namespace namespace = registry.getNamespace(packageName) - + Namespace namespace = packageNamespace() Manifest.Builder builder = Manifest.builder() - Files.walk(packageDest()).filter(f -> Files.isRegularFile(f)).forEach(f -> { println(f) String logicalKey = packageDest().relativize(f) From 6847c152b5054ca1cb71129b57b49c24679b14e5 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 15:32:37 -0700 Subject: [PATCH 13/27] manifest.getEntries --- .../src/main/nextflow/quilt/jep/QuiltPackage.groovy | 10 ++++++++-- .../src/test/nextflow/quilt/nio/QuiltNioTest.groovy | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index c4776590..43ee951b 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -162,8 +162,14 @@ class QuiltPackage { List relativeChildren(String subpath) { Manifest manifest = packageManifest() - List result = manifest.listDirectory(subpath) - return result + // get names of entries + List children = manifest.getEntries().keySet().collect { String key -> + if (key.startsWith(subpath)) { + return key + } + return null + } + return children } void reset() { diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 4bffd441..047eaaf6 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -245,7 +245,7 @@ class QuiltNioTest extends QuiltSpecification { itr.hasNext() itr.next().toString().contains('path=data') - itr.next().toString().contains('path=folder') //whuh? + //itr.next().toString().contains('path=folder') //whuh? itr.next().toString().contains('path=notebooks') itr.next().toString().contains('path=quilt_summarize.json') From 0abe9d33fff9d7de79c7d8e5488438cc412090c8 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 15:52:29 -0700 Subject: [PATCH 14/27] install just before read --- .../nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy | 1 - .../main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy | 5 +++++ .../nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy index b2fad63f..9bf56c13 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy @@ -78,7 +78,6 @@ class QuiltObserver implements TraceObserver { params.each { k, value -> String uri = "$value" if (uri.startsWith(QuiltParser.SCHEME)) { - log.debug("checkParams.uri[$k]: $uri") QuiltPath path = QuiltPathFactory.parse(uri) checkPath(path) } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index 44e86f29..8469307e 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -44,6 +44,7 @@ import groovy.util.logging.Slf4j import nextflow.Global import nextflow.Session import nextflow.quilt.jep.QuiltParser +import nextflow.quilt.jep.QuiltPackage /** * Implements NIO File system provider for Quilt Blob Storage @@ -270,6 +271,10 @@ class QuiltFileSystemProvider extends FileSystemProvider { //options = [WRITE,CREATE] as Set attributesCache = [:] // reset cache notifyFilePublish(qPath) + } else { + // ensure QuiltPackage installed + QuiltPackage pkg = qPath.pkg() + pkg.install() } //log.debug("\tOpening channel to: $installedPath") FileChannel channel = FileChannel.open(installedPath, options) diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 047eaaf6..9720176d 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -67,8 +67,11 @@ class QuiltNioTest extends QuiltSpecification { @IgnoreIf({ System.getProperty('os.name').contains('indows') }) void 'should read from a path'() { + // quilt+s3://quilt-example#package=examples/hurdat@f8d1478d93&path=data%2Fatlantic-storms.csv + // https://open.quiltdata.com/b/quilt-example/packages/examples/hurdat/tree/latest/data/atlantic-storms.csv given: Path path = Paths.get(new URI(READ_URL)) + log.info("QuiltNioTest:Reading from $path") when: String text = readObject(path) From 20a053d421bd3d10d9501120152f5a4ce77267d7 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 15:53:29 -0700 Subject: [PATCH 15/27] only install if needed --- .../nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 43ee951b..d76ccb95 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -196,6 +196,10 @@ class QuiltPackage { Path install() { Path dest = packageDest() + if (isInstalled()) { + log.debug("QuiltPackage.install: already installed: $dest") + return dest + } try { log.info("installing $packageName from $bucket...") From 91f53db50a15fa16416526f8822d250372c8902b Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 16:09:19 -0700 Subject: [PATCH 16/27] GitHub-Testing-QuiltCore --- .github/workflows/test.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 836d056c..54aef491 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,12 +32,11 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 - - name: Install Quilt3 - uses: actions/setup-python@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 with: - python-version: '3.10' - - run: python -m pip install quilt3 - - run: which quilt3 + role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Testing-QuiltCore + aws-region: us-east-1 - name: Setup Java ${{matrix.java_version}} uses: actions/setup-java@v3 From b5e72f1747bb99c2759bbcc32477bdfc316bea06 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 16:42:57 -0700 Subject: [PATCH 17/27] id-token --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54aef491..c14acc39 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,8 +12,8 @@ jobs: build: name: Test permissions: - # contents: read - # id-token: write + contents: read + id-token: write issues: write pull-requests: write From d26b4b23a804612186edafedcc7fccd0413ac5b0 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 16:49:25 -0700 Subject: [PATCH 18/27] GitHub-Testing-NF-Quilt --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c14acc39..de231cff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v3 with: - role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Testing-QuiltCore + role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Testing-NF-Quilt aws-region: us-east-1 - name: Setup Java ${{matrix.java_version}} From 67b9cc276ed9efa4520fe2a41e0c3e4f732f65a6 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 17:03:41 -0700 Subject: [PATCH 19/27] role/github --- .github/workflows/test.yml | 2 +- CHANGELOG.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de231cff..4f3a22e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v3 with: - role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Testing-NF-Quilt + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt aws-region: us-east-1 - name: Setup Java ${{matrix.java_version}} diff --git a/CHANGELOG.md b/CHANGELOG.md index 896fc1e5..e439c532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ ## [0.5.0] 2023-09-04+ -- Switch to quiltcore-java -- Stop trying to pre-install every package +- Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows] +- Do not pre-install packages (only install before write) + ## [0.4.5] 2023-08-23 From 85ff1bc6d2125d242dd84926c02699fb0c7cc97b Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 17:23:02 -0700 Subject: [PATCH 20/27] manual lint --- CHANGELOG.md | 1 - Makefile | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e439c532..656c1cd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ - Switch to quiltcore-java [NOTE: this pre-release does NOT check workflows] - Do not pre-install packages (only install before write) - ## [0.4.5] 2023-08-23 - fix metadata in README.md diff --git a/Makefile b/Makefile index 35439606..0fa72f4a 100644 --- a/Makefile +++ b/Makefile @@ -83,19 +83,17 @@ deps: deps-all: ./gradlew -q dependencyInsight -# ./gradlew -q ${mm}dependencies --configuration ${CONFIG} --dependency ${module} # # Refresh SNAPSHOTs dependencies # + refresh: ./gradlew --refresh-dependencies dependencies # # Run all tests or selected ones # -#test: -# ./gradlew ${mm}test test-class: ./gradlew ${mm}test --tests ${class} @@ -105,6 +103,7 @@ fast: # # Upload JAR artifacts to Maven Central +# publish: echo "Ensure you have set 'github_organization=' in gradle.properties" From f1668441f08359e2612c5b200bc0b0d8c7c56f9e Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 17:34:52 -0700 Subject: [PATCH 21/27] install before readAttributes --- plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy | 3 +-- .../src/main/nextflow/quilt/nio/QuiltFileSystem.groovy | 2 +- plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index add707f2..5b129d75 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -231,8 +231,7 @@ ${meta['workflow']['stats']['processes']} } List match(String glob) throws IOException { - String dir = pkg.packageDest() - Path folder = Paths.get(dir) + Path folder = pkg.install() FileSystem fs = FileSystems.getDefault() PathMatcher pathMatcher = fs.getPathMatcher("glob:${glob}") List matches = [] diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy index 62d71e00..d349f333 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy @@ -107,7 +107,7 @@ final class QuiltFileSystem extends FileSystem implements Closeable { QuiltFileAttributes readAttributes(QuiltPath path) { //log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") - Path installedPath = path.localPath() + Path installedPath = path.localPath(true) try { BasicFileAttributes attrs = Files.readAttributes(installedPath, BasicFileAttributes) return new QuiltFileAttributes(path, path.toString(), attrs) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy index 2d7aefe3..fe549bee 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy @@ -71,8 +71,8 @@ final class QuiltPath implements Path, Comparable { return sub_paths() } - Path localPath() { - Path pkgPath = pkg().packageDest() + Path localPath(boolean shouldInstall = false) { + Path pkgPath = shouldInstall ? pkg().install() : pkg().packageDest() assert pkgPath return Paths.get(pkgPath.toUriString(), sub_paths()) } From c07fafbfc4cacd6e20f4b7645808064e1501eac0 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 4 Sep 2023 22:29:24 -0600 Subject: [PATCH 22/27] pkg-test fail --- .github/workflows/pkg-test.yml | 61 +++++++++++++++++++ .../main/nextflow/quilt/QuiltProduct.groovy | 5 +- .../nextflow/quilt/jep/QuiltPackage.groovy | 7 +-- .../quilt/nio/QuiltFileSystemProvider.groovy | 7 ++- 4 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/pkg-test.yml diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml new file mode 100644 index 00000000..48acbbd6 --- /dev/null +++ b/.github/workflows/pkg-test.yml @@ -0,0 +1,61 @@ +name: Test + +on: + # Trigger at every push. Action will also be visible from Pull Requests to master + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) + pull_request: + branches: [master] + +permissions: read-all + +jobs: + build: + name: Test + permissions: + contents: read + id-token: write + issues: write + pull-requests: write + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + java_version: [19] + runs-on: ${{ matrix.os }} + + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 + + - name: Setup Java ${{matrix.java_version}} + uses: actions/setup-java@v3 + with: + java-version: ${{matrix.java_version}} + distribution: 'temurin' + architecture: x64 + cache: gradle + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run Package Tests + run: make pkg-test + + - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: nf-quilt-pkg-test + path: | + /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/ diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 5b129d75..6a14192b 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -230,8 +230,9 @@ ${meta['workflow']['stats']['processes']} return template.toString() } - List match(String glob) throws IOException { - Path folder = pkg.install() + List match(String glob, boolean shouldInstall = false) throws IOException { + log.debug("QuiltProduct.match[$shouldInstall]: $glob") + Path folder = shouldInstall ? pkg.install() : pkg.packageDest() FileSystem fs = FileSystems.getDefault() PathMatcher pathMatcher = fs.getPathMatcher("glob:${glob}") List matches = [] diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index d76ccb95..34e10406 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -137,9 +137,8 @@ class QuiltPackage { return namespace } - String resolveHash() { + String resolveHash(Namespace namespace) { if (hash == 'latest' || hash == null || hash == 'null') { - Namespace namespace = packageNamespace() return namespace.getHash('latest') } return hash @@ -147,7 +146,8 @@ class QuiltPackage { Manifest packageManifest() { Namespace namespace = packageNamespace() - Manifest manifest = namespace.getManifest(resolveHash()) + String newHash = resolveHash(namespace) + Manifest manifest = namespace.getManifest(newHash) return manifest } @@ -204,7 +204,6 @@ class QuiltPackage { try { log.info("installing $packageName from $bucket...") Manifest manifest = packageManifest() - manifest.install(dest) log.info('done') installed = true diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index 8469307e..b7ae4c53 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -272,12 +272,13 @@ class QuiltFileSystemProvider extends FileSystemProvider { attributesCache = [:] // reset cache notifyFilePublish(qPath) } else { + // log.debug("\tEnsure installed: $installedPath") // ensure QuiltPackage installed - QuiltPackage pkg = qPath.pkg() - pkg.install() + qPath.pkg().install() + log.debug("\tReading from: $installedPath") } - //log.debug("\tOpening channel to: $installedPath") FileChannel channel = FileChannel.open(installedPath, options) + log.debug("FileChannel.open: ${channel}") return channel } catch (java.nio.file.NoSuchFileException e) { From c4b3018604d6dd2d175f30240a68be4b206f30de Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Thu, 7 Sep 2023 15:32:07 -0700 Subject: [PATCH 23/27] Update QuiltPackageTest.groovy --- .../src/test/nextflow/quilt/jep/QuiltPackageTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index b74d1d92..263f715d 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -119,11 +119,11 @@ class QuiltPackageTest extends QuiltSpecification { @IgnoreIf({ System.getProperty('os.name').contains('indows') }) void 'should deinstall files'() { expect: - Files.exists(qpath.localPath()) + Files.exists(qpath.localPath(true)) when: qpath.deinstall() then: - !Files.exists(qpath.localPath()) + !Files.exists(qpath.localPath(false)) /* when: Files.readAttributes(qpath, BasicFileAttributes) then: From db6b12a80fd2d42704048f2b33929e20d9368460 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Fri, 8 Sep 2023 13:18:11 -0700 Subject: [PATCH 24/27] error.print stack trace --- .../nextflow/quilt/nio/QuiltFileSystem.groovy | 3 ++- .../quilt/nio/QuiltFileSystemProvider.groovy | 2 +- .../nextflow/quilt/jep/QuiltPackageTest.groovy | 7 ++++--- .../test/nextflow/quilt/nio/QuiltNioTest.groovy | 15 +++++++++------ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy index d349f333..fd0c04ca 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy @@ -106,8 +106,9 @@ final class QuiltFileSystem extends FileSystem implements Closeable { } QuiltFileAttributes readAttributes(QuiltPath path) { - //log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") + log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") Path installedPath = path.localPath(true) + log.debug(".readAttributes.installedPath $installedPath exists(${Files.exists(installedPath)}))") try { BasicFileAttributes attrs = Files.readAttributes(installedPath, BasicFileAttributes) return new QuiltFileAttributes(path, path.toString(), attrs) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index b7ae4c53..a96283bf 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -400,7 +400,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override def A readAttributes(Path path, Class type, LinkOption... options) throws IOException { - //log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' + log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' def attr = attributesCache.get(path) if (attr) { return attr diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 263f715d..443ae4b7 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -120,17 +120,18 @@ class QuiltPackageTest extends QuiltSpecification { void 'should deinstall files'() { expect: Files.exists(qpath.localPath(true)) + Files.readAttributes(qpath, BasicFileAttributes) when: qpath.deinstall() then: !Files.exists(qpath.localPath(false)) - /* when: + + /*when: Files.readAttributes(qpath, BasicFileAttributes) then: - thrown(java.nio.file.NoSuchFileException) */ + thrown(java.nio.file.NoSuchFileException)*/ } - @Ignore() void 'should iterate over installed files '() { given: def root = qpath.getRoot() diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 9720176d..52b7c17e 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -79,7 +79,7 @@ class QuiltNioTest extends QuiltSpecification { text.startsWith('id') } - @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + // @IgnoreIf({ System.getProperty('os.name').contains('ux') }) @IgnoreIf({ System.getProperty('os.name').contains('indows') }) void 'should read file attributes'() { given: @@ -197,7 +197,7 @@ class QuiltNioTest extends QuiltSpecification { if (source) { Files.delete(source) } } - @Ignore + @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'copy a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) @@ -212,15 +212,18 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT } - @Ignore + @Ignore('QuiltFileSystem.copy not implemented') void 'move a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) - final source_url = WRITE_URL.replace('test_folder', 'source') + println("QuiltNioTest:move:path $path") + final source_url = WRITE_URL.replace('folder', 'source') final source = Paths.get(new URI(source_url)) + println("QuiltNioTest:move:source $source") Files.write(source, TEXT.bytes) and: Files.move(source, path) + println("QuiltNioTest:move: $source -> $path") expect: !existsPath(source) existsPath(path) @@ -471,7 +474,7 @@ class QuiltNioTest extends QuiltSpecification { thrown(FileSystemException) } - @Ignore + @Ignore('Can not write to null_path') void 'should stream directory content'() { given: makeObject(null_path('foo/file1.txt'), 'A') @@ -515,7 +518,7 @@ class QuiltNioTest extends QuiltSpecification { list == [ 'file4.txt' ] } - @Ignore + @Ignore('Can not write to null_path') void 'should check walkTree'() { given: makeObject(null_path('foo/file1.txt'), 'A') From 51eb5575c8c7a3a6a19a0b2448412518321c6bf9 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Fri, 8 Sep 2023 16:57:22 -0700 Subject: [PATCH 25/27] FileSystemTransferAware --- .../quilt/nio/QuiltFileSystemProvider.groovy | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index a96283bf..637f059b 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -43,6 +43,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Global import nextflow.Session +import nextflow.file.FileSystemTransferAware import nextflow.quilt.jep.QuiltParser import nextflow.quilt.jep.QuiltPackage @@ -54,7 +55,7 @@ import nextflow.quilt.jep.QuiltPackage @Slf4j @CompileStatic -class QuiltFileSystemProvider extends FileSystemProvider { +class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { private final Map myEnv = new HashMap<>(System.getenv()) private final Map fileSystems = [:] @@ -84,6 +85,24 @@ class QuiltFileSystemProvider extends FileSystemProvider { return path.getFileSystem().provider() } + boolean canUpload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canUpload[${source}] -> ${target}") + return false + } + + boolean canDownload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canDownload[${source}] -> ${target}") + return false + } + + void download(Path source, Path target, CopyOption... options) throws IOException { + throw new UnsupportedOperationException("Operation 'download' is not supported by QuiltFileSystem") + } + + void upload(Path source, Path target, CopyOption... options) throws IOException { + throw new UnsupportedOperationException("Operation 'upload' is not supported by QuiltFileSystem") + } + /** * @inheritDoc */ From 3770b5dc773dd0521a10c47138dfc0ede20538a1 Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Sat, 9 Sep 2023 20:25:31 -0700 Subject: [PATCH 26/27] Files.copy(tempFile, installedFile) --- .../quilt/nio/QuiltFileSystemProvider.groovy | 27 +++++++++++----- .../quilt/jep/QuiltPackageTest.groovy | 31 ++++++++++++++++--- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index 637f059b..d5aa194d 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -46,6 +46,8 @@ import nextflow.Session import nextflow.file.FileSystemTransferAware import nextflow.quilt.jep.QuiltParser import nextflow.quilt.jep.QuiltPackage +import sun.nio.fs.UnixFileSystemProvider +import sun.nio.fs.MacOSXFileSystemProvider /** * Implements NIO File system provider for Quilt Blob Storage @@ -85,22 +87,33 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr return path.getFileSystem().provider() } - boolean canUpload(Path source, Path target) { - log.debug("QuiltFileSystemProvider.canUpload[${source}] -> ${target}") - return false + static boolean localProvider(Path path) { + FileSystemProvider provider = provider(path) + log.debug("QuiltFileSystemProvider.localProvider[${path}] -> ${provider}") + return provider instanceof UnixFileSystemProvider || + provider instanceof MacOSXFileSystemProvider } boolean canDownload(Path source, Path target) { log.debug("QuiltFileSystemProvider.canDownload[${source}] -> ${target}") - return false + return localProvider(target) && source instanceof QuiltPath + } + + boolean canUpload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canUpload[${source}] -> ${target}") + return localProvider(source) && target instanceof QuiltPath } void download(Path source, Path target, CopyOption... options) throws IOException { - throw new UnsupportedOperationException("Operation 'download' is not supported by QuiltFileSystem") + QuiltPath qSource = asQuiltPath(source) + Path local_source = qSource.localPath() + Files.copy(local_source, target, options) } void upload(Path source, Path target, CopyOption... options) throws IOException { - throw new UnsupportedOperationException("Operation 'upload' is not supported by QuiltFileSystem") + QuiltPath qTarget = asQuiltPath(target) + Path local_target = qTarget.localPath() + Files.copy(source, local_target, options) } /** @@ -359,7 +372,7 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr @Override void copy(Path from, Path to, CopyOption... options) throws IOException { - //log.debug("Attempting `copy`: ${from} -> ${to}") + log.debug("Attempting `copy`: ${from} -> ${to}") assert provider(from) == provider(to) if (from == to) { return // nothing to do -- just return diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 443ae4b7..87a273f8 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -86,6 +86,31 @@ class QuiltPackageTest extends QuiltSpecification { Files.exists(installPath) } + void 'should copy temp files into install folder'() { + given: + String filename = 'test.txt' + Path installPath = pkg.packageDest() + Path tempFile = File.createTempFile('test', '.txt').toPath() + println("tempFile: "+tempFile); + Path installedFile = Paths.get(installPath.toString(), filename) + expect: + Files.exists(tempFile) + Files.exists(installPath) + !Files.exists(installedFile) + Files.copy(tempFile, installedFile) + Files.exists(installedFile) + } + + void 'should copy package files to temp Path'() { + given: + Path installPath = pkg.packageDest() + expect: + Files.exists(installPath) + Files.isDirectory(installPath) + Files.readAttributes(installPath, BasicFileAttributes) + } + + void 'should get attributes for package folder'() { given: def root = qpath.getRoot() @@ -106,6 +131,7 @@ class QuiltPackageTest extends QuiltSpecification { Files.readAttributes(qpath, BasicFileAttributes) } + void 'should return null on failed install'() { given: def url2 = TEST_URL.replace('quilt-', 'quilted-') @@ -125,11 +151,6 @@ class QuiltPackageTest extends QuiltSpecification { qpath.deinstall() then: !Files.exists(qpath.localPath(false)) - - /*when: - Files.readAttributes(qpath, BasicFileAttributes) - then: - thrown(java.nio.file.NoSuchFileException)*/ } void 'should iterate over installed files '() { From 9259c77f7c31fce8aa256d5235e34dcd3838dd9f Mon Sep 17 00:00:00 2001 From: Ernest Prabhakar Date: Mon, 11 Sep 2023 14:17:18 -0700 Subject: [PATCH 27/27] pass pkg-test --- .../nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy | 8 +++++++- .../src/test/nextflow/quilt/jep/QuiltPackageTest.groovy | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 6a14192b..25318104 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -80,11 +80,17 @@ ${meta['workflow']['stats']['processes']} static void writeString(String text, QuiltPackage pkg, String filename) { String dir = pkg.packageDest() Path path = Paths.get(dir, filename.split('/') as String[]) + File parent = path.getParent().toFile() + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IllegalStateException("Couldn't create dir: " + parent); + } + try { Files.write(path, text.bytes) } catch (Exception e) { - log.error("writeString: cannot write `$text` to `$path` for `${pkg}`") + log.error("writeString[${e.getMessage()}]: fail write `$path` for `${pkg}`") + e.printStackTrace() } } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 87a273f8..fdf77d74 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -91,7 +91,6 @@ class QuiltPackageTest extends QuiltSpecification { String filename = 'test.txt' Path installPath = pkg.packageDest() Path tempFile = File.createTempFile('test', '.txt').toPath() - println("tempFile: "+tempFile); Path installedFile = Paths.get(installPath.toString(), filename) expect: Files.exists(tempFile)