From 7137ce59bec778def651282058075079ce6687af Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Wed, 22 May 2024 06:47:57 -0700 Subject: [PATCH 1/4] Create QuiltRenderer Created a placeholder QuiltRenderer. Builds successfully. --- nextflow.config | 107 +++++++++++--- plugins/nf-prov/build.gradle | 4 + .../main/nextflow/prov/ProvObserver.groovy | 5 +- .../main/nextflow/prov/QuiltRenderer.groovy | 135 ++++++++++++++++++ 4 files changed, 229 insertions(+), 22 deletions(-) create mode 100644 plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy diff --git a/nextflow.config b/nextflow.config index 6219b1b..8e557e7 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,24 +1,89 @@ +/* + * Copyright 2021-2022, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + plugins { - id 'nf-prov' -} - -params { - outdir = 'results' -} - -prov { - formats { - bco { - file = "${params.outdir}/bco.json" - overwrite = true - } - dag { - file = "${params.outdir}/dag.html" - overwrite = true - } - legacy { - file = "${params.outdir}/manifest.json" - overwrite = true - } + // Apply the groovy plugin to add support for Groovy + id 'groovy' + id 'idea' +} + +group = 'io.nextflow' +// DO NOT SET THE VERSION HERE +// THE VERSION FOR PLUGINS IS DEFINED IN THE `/resources/META-INF/MANIFEST.NF` file +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } +} + +idea { + module.inheritOutputDirs = true +} + +repositories { + mavenCentral() + maven { url = 'https://jitpack.io' } + maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' } + maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' } +} + +configurations { + // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies + runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api' +} + +sourceSets { + main.java.srcDirs = [] + main.groovy.srcDirs = ['src/main'] + main.resources.srcDirs = ['src/resources'] + test.groovy.srcDirs = ['src/test'] + test.java.srcDirs = [] + test.resources.srcDirs = ['src/testResources'] +} + +dependencies { + // This dependency is exported to consumers, that is to say found on their compile classpath. + compileOnly 'io.nextflow:nextflow:23.04.0' + compileOnly 'org.slf4j:slf4j-api:1.7.10' + compileOnly 'org.pf4j:pf4j:3.4.1' + + // quiltcore + implementation 'com.quiltdata:quiltcore:0.1.2' + + // add here plugins depepencies + + // test configuration + testImplementation "org.codehaus.groovy:groovy:3.0.8" + testImplementation "org.codehaus.groovy:groovy-nio:3.0.8" + testImplementation 'io.nextflow:nextflow:23.04.0' + testImplementation ("org.codehaus.groovy:groovy-test:3.0.8") { exclude group: 'org.codehaus.groovy' } + testImplementation ("cglib:cglib-nodep:3.3.0") + testImplementation ("org.objenesis:objenesis:3.1") + testImplementation ("org.spockframework:spock-core:2.0-M3-groovy-3.0") { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } + testImplementation ('org.spockframework:spock-junit4:2.0-M3-groovy-3.0') { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } + testImplementation ('com.google.jimfs:jimfs:1.1') + + // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sec:module_replacement + modules { + module("commons-logging:commons-logging") { replacedBy("org.slf4j:jcl-over-slf4j") } } } + +// use JUnit 5 platform +test { + useJUnitPlatform() +} + diff --git a/plugins/nf-prov/build.gradle b/plugins/nf-prov/build.gradle index 3eee961..8e557e7 100644 --- a/plugins/nf-prov/build.gradle +++ b/plugins/nf-prov/build.gradle @@ -59,6 +59,10 @@ dependencies { compileOnly 'io.nextflow:nextflow:23.04.0' compileOnly 'org.slf4j:slf4j-api:1.7.10' compileOnly 'org.pf4j:pf4j:3.4.1' + + // quiltcore + implementation 'com.quiltdata:quiltcore:0.1.2' + // add here plugins depepencies // test configuration diff --git a/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy b/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy index 658de6d..ad092f6 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy @@ -38,7 +38,7 @@ import nextflow.trace.TraceRecord @CompileStatic class ProvObserver implements TraceObserver { - public static final List VALID_FORMATS = ['bco', 'dag', 'legacy'] + public static final List VALID_FORMATS = ['bco', 'dag', 'legacy', 'quilt'] private Session session @@ -66,6 +66,9 @@ class ProvObserver implements TraceObserver { if( name == 'legacy' ) return new LegacyRenderer(opts) + + if( name == 'quilt' ) + return new QuiltRenderer(opts) throw new IllegalArgumentException("Invalid provenance format -- valid formats are ${VALID_FORMATS.join(', ')}") } diff --git a/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy b/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy new file mode 100644 index 0000000..d31e611 --- /dev/null +++ b/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy @@ -0,0 +1,135 @@ +/* + * Copyright 2023, Quilt Data + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.prov + +import java.nio.file.Path + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic +import nextflow.Session +import nextflow.file.FileHolder +import nextflow.processor.TaskRun + +import com.quiltdata.quiltcore.Entry +import com.quiltdata.quiltcore.Registry +import com.quiltdata.quiltcore.Namespace +import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.LocalPhysicalKey +import com.quiltdata.quiltcore.key.S3PhysicalKey + +/** + * Renderer for the Quilt manifest format. + * + * @author Kevin Moore + */ +@CompileStatic +class QuiltRenderer implements Renderer { + + private Path path + + private boolean overwrite + + QuiltRenderer(Map opts) { + path = opts.file as Path + overwrite = opts.overwrite as Boolean + + ProvHelper.checkFileOverwrite(path, overwrite) + } + + private static def jsonify(root) { + if ( root instanceof Map ) + root.collectEntries( (k, v) -> [k, jsonify(v)] ) + + else if ( root instanceof Collection ) + root.collect( v -> jsonify(v) ) + + else if ( root instanceof FileHolder ) + jsonify(root.storePath) + + else if ( root instanceof Path ) + root.toUriString() + + else if ( root instanceof Boolean || root instanceof Number ) + root + + else + root.toString() + } + + Map renderTask(TaskRun task) { + // TODO: Figure out what the '$' input/output means + // Omitting them from manifest for now + return [ + 'id': task.id as String, + 'name': task.name, + 'cached': task.cached, + 'process': task.processor.name, + 'script': task.script, + 'inputs': task.inputs.findResults { inParam, object -> + def inputMap = [ + 'name': inParam.getName(), + 'value': jsonify(object) + ] + inputMap['name'] != '$' ? inputMap : null + }, + 'outputs': task.outputs.findResults { outParam, object -> + def outputMap = [ + 'name': outParam.getName(), + 'emit': outParam.getChannelEmitName(), + 'value': jsonify(object) + ] + outputMap['name'] != '$' ? outputMap : null + } + ] + } + + @Override + void render(Session session, Set tasks, Map outputs) { + // generate task manifest + def tasksMap = tasks.inject([:]) { accum, task -> + accum[task.id] = renderTask(task) + accum + } + + // generate temporary output-task map + def taskLookup = tasksMap.inject([:]) { accum, id, task -> + task['outputs'].each { output -> + // Make sure to handle tuples of outputs + def values = output['value'] + if ( values instanceof Collection ) + values.each { accum.put(it, task['id']) } + else + accum.put(values, task['id']) + } + accum + } + + // render JSON output + def manifest = [ + 'pipeline': session.config.manifest, + 'published': outputs.collect { source, target -> [ + 'source': source.toUriString(), + 'target': target.toUriString(), + 'publishingTaskId': taskLookup[source.toUriString()], + ] }, + 'tasks': tasksMap + ] + + path.text = JsonOutput.prettyPrint(JsonOutput.toJson(manifest)) + } + +} From b526dbcc2b70c4bd2360899dae1440cc5ec49764 Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Wed, 22 May 2024 09:01:14 -0700 Subject: [PATCH 2/4] Broken config --- plugins/nf-prov/build.gradle | 2 ++ .../src/main/nextflow/prov/QuiltRenderer.groovy | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/plugins/nf-prov/build.gradle b/plugins/nf-prov/build.gradle index 8e557e7..f1836f2 100644 --- a/plugins/nf-prov/build.gradle +++ b/plugins/nf-prov/build.gradle @@ -61,6 +61,8 @@ dependencies { compileOnly 'org.pf4j:pf4j:3.4.1' // quiltcore + implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.1' + implementation 'com.upplication:s3fs:2.2.2' implementation 'com.quiltdata:quiltcore:0.1.2' // add here plugins depepencies diff --git a/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy b/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy index d31e611..82f176f 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy @@ -16,6 +16,7 @@ package nextflow.prov +import java.net.URI; import java.nio.file.Path import groovy.json.JsonOutput @@ -24,10 +25,13 @@ import nextflow.Session import nextflow.file.FileHolder import nextflow.processor.TaskRun +import com.fasterxml.jackson.databind.node.ObjectNode; + import com.quiltdata.quiltcore.Entry import com.quiltdata.quiltcore.Registry import com.quiltdata.quiltcore.Namespace import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.PhysicalKey import com.quiltdata.quiltcore.key.LocalPhysicalKey import com.quiltdata.quiltcore.key.S3PhysicalKey @@ -118,6 +122,17 @@ class QuiltRenderer implements Renderer { accum } + for(Path source: outputs.keySet()){ + println "Source" + source.toUriString() + println "Target" + outputs.get(source).toUriString() + def Entry e = new Entry( + PhysicalKey.fromUri(new URI(outputs.get(source).toUriString())), + 0l, + new Entry.Hash(Entry.HashType.SHA256, ""), + (ObjectNode) null + ) + } + // render JSON output def manifest = [ 'pipeline': session.config.manifest, From 46392535825fe34614e37acdd169a7044b9be295 Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Wed, 22 May 2024 10:30:00 -0700 Subject: [PATCH 3/4] Fix broken Nextflow config Fixing the config and bumping the version --- nextflow.config | 111 ++++-------------- plugins/nf-prov/build.gradle | 3 +- .../src/resources/META-INF/MANIFEST.MF | 2 +- 3 files changed, 27 insertions(+), 89 deletions(-) diff --git a/nextflow.config b/nextflow.config index 8e557e7..ce1c180 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,89 +1,28 @@ -/* - * Copyright 2021-2022, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - plugins { - // Apply the groovy plugin to add support for Groovy - id 'groovy' - id 'idea' -} - -group = 'io.nextflow' -// DO NOT SET THE VERSION HERE -// THE VERSION FOR PLUGINS IS DEFINED IN THE `/resources/META-INF/MANIFEST.NF` file -java { - toolchain { - languageVersion = JavaLanguageVersion.of(11) - } -} - -idea { - module.inheritOutputDirs = true -} - -repositories { - mavenCentral() - maven { url = 'https://jitpack.io' } - maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/releases' } - maven { url = 'https://s3-eu-west-1.amazonaws.com/maven.seqera.io/snapshots' } -} - -configurations { - // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies - runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api' -} - -sourceSets { - main.java.srcDirs = [] - main.groovy.srcDirs = ['src/main'] - main.resources.srcDirs = ['src/resources'] - test.groovy.srcDirs = ['src/test'] - test.java.srcDirs = [] - test.resources.srcDirs = ['src/testResources'] -} - -dependencies { - // This dependency is exported to consumers, that is to say found on their compile classpath. - compileOnly 'io.nextflow:nextflow:23.04.0' - compileOnly 'org.slf4j:slf4j-api:1.7.10' - compileOnly 'org.pf4j:pf4j:3.4.1' - - // quiltcore - implementation 'com.quiltdata:quiltcore:0.1.2' - - // add here plugins depepencies - - // test configuration - testImplementation "org.codehaus.groovy:groovy:3.0.8" - testImplementation "org.codehaus.groovy:groovy-nio:3.0.8" - testImplementation 'io.nextflow:nextflow:23.04.0' - testImplementation ("org.codehaus.groovy:groovy-test:3.0.8") { exclude group: 'org.codehaus.groovy' } - testImplementation ("cglib:cglib-nodep:3.3.0") - testImplementation ("org.objenesis:objenesis:3.1") - testImplementation ("org.spockframework:spock-core:2.0-M3-groovy-3.0") { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } - testImplementation ('org.spockframework:spock-junit4:2.0-M3-groovy-3.0') { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } - testImplementation ('com.google.jimfs:jimfs:1.1') - - // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sec:module_replacement - modules { - module("commons-logging:commons-logging") { replacedBy("org.slf4j:jcl-over-slf4j") } + id 'nf-prov@1.2.3' +} + +params { + outdir = 'results' +} + +prov { + formats { + bco { + file = "${params.outdir}/bco.json" + overwrite = true + } + dag { + file = "${params.outdir}/dag.html" + overwrite = true + } + legacy { + file = "${params.outdir}/manifest.json" + overwrite = true + } + quilt { + file = "${params.outdir}/manifest.json" + overwrite = true + } } } - -// use JUnit 5 platform -test { - useJUnitPlatform() -} - diff --git a/plugins/nf-prov/build.gradle b/plugins/nf-prov/build.gradle index f1836f2..7ff2eec 100644 --- a/plugins/nf-prov/build.gradle +++ b/plugins/nf-prov/build.gradle @@ -87,5 +87,4 @@ dependencies { // use JUnit 5 platform test { useJUnitPlatform() -} - +} \ No newline at end of file diff --git a/plugins/nf-prov/src/resources/META-INF/MANIFEST.MF b/plugins/nf-prov/src/resources/META-INF/MANIFEST.MF index 2db3eb5..0f2e693 100644 --- a/plugins/nf-prov/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-prov/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Id: nf-prov -Plugin-Version: 1.2.2 +Plugin-Version: 1.2.3 Plugin-Class: nextflow.prov.ProvPlugin Plugin-Provider: nextflow Plugin-Requires: >=23.04.0 From b559d9f1bc9caa38b280a3bb04801bd39eb2c824 Mon Sep 17 00:00:00 2001 From: Kevin Moore Date: Thu, 20 Jun 2024 13:36:17 -0700 Subject: [PATCH 4/4] WIP Commiting in-progress changes from the end of the hackathon. --- nextflow.config | 12 ---- .../main/nextflow/prov/QuiltRenderer.groovy | 61 +++++++++++++------ 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/nextflow.config b/nextflow.config index ce1c180..62e59fb 100644 --- a/nextflow.config +++ b/nextflow.config @@ -8,18 +8,6 @@ params { prov { formats { - bco { - file = "${params.outdir}/bco.json" - overwrite = true - } - dag { - file = "${params.outdir}/dag.html" - overwrite = true - } - legacy { - file = "${params.outdir}/manifest.json" - overwrite = true - } quilt { file = "${params.outdir}/manifest.json" overwrite = true diff --git a/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy b/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy index 82f176f..274513c 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy @@ -16,7 +16,10 @@ package nextflow.prov +import java.io.FileOutputStream +import java.io.OutputStream import java.net.URI; +import java.nio.file.Files import java.nio.file.Path import groovy.json.JsonOutput @@ -122,29 +125,51 @@ class QuiltRenderer implements Renderer { accum } + + + // render JSON output + //def manifest = [ + // 'pipeline': session.config.manifest, + // 'published': outputs.collect { source, target -> [ + // 'source': source.toUriString(), + // 'target': target.toUriString(), + // 'publishingTaskId': taskLookup[source.toUriString()], + // ] }, + // 'tasks': tasksMap + //] + + println "Building manifest" + def builder = new Manifest.Builder() + int i=0 for(Path source: outputs.keySet()){ - println "Source" + source.toUriString() - println "Target" + outputs.get(source).toUriString() + println "Source URI " + source.toUriString() + println "Target URI " + outputs.get(source).toUriString() + def target = outputs.get(source) + if(target == null){ + println "NULL TARGET" + break + } + + def logicalKey = target.getFileName() + + // Get the file size + def fileSize = Files.size(target) def Entry e = new Entry( - PhysicalKey.fromUri(new URI(outputs.get(source).toUriString())), - 0l, - new Entry.Hash(Entry.HashType.SHA256, ""), - (ObjectNode) null + PhysicalKey.fromUri(target.toUri()), + Files.size(target), + null, + null ) + builder.addEntry(logicalKey.toString(), e.withHash()) + println("Added ${logicalKey}") } + def pkg = builder.build() + OutputStream outputStream = new FileOutputStream(path.toString()) + pkg.serializeToOutputStream(outputStream) + outputStream.close() + println "Writing manifest to " + path.toString() - // render JSON output - def manifest = [ - 'pipeline': session.config.manifest, - 'published': outputs.collect { source, target -> [ - 'source': source.toUriString(), - 'target': target.toUriString(), - 'publishingTaskId': taskLookup[source.toUriString()], - ] }, - 'tasks': tasksMap - ] - - path.text = JsonOutput.prettyPrint(JsonOutput.toJson(manifest)) + //path.text = JsonOutput.prettyPrint(JsonOutput.toJson(manifest)) } }