diff --git a/nextflow.config b/nextflow.config index 6219b1b..62e59fb 100644 --- a/nextflow.config +++ b/nextflow.config @@ -1,5 +1,5 @@ plugins { - id 'nf-prov' + id 'nf-prov@1.2.3' } params { @@ -8,15 +8,7 @@ params { prov { formats { - bco { - file = "${params.outdir}/bco.json" - overwrite = true - } - dag { - file = "${params.outdir}/dag.html" - overwrite = true - } - legacy { + quilt { file = "${params.outdir}/manifest.json" overwrite = true } diff --git a/plugins/nf-prov/build.gradle b/plugins/nf-prov/build.gradle index 3eee961..7ff2eec 100644 --- a/plugins/nf-prov/build.gradle +++ b/plugins/nf-prov/build.gradle @@ -59,6 +59,12 @@ dependencies { compileOnly 'io.nextflow:nextflow:23.04.0' compileOnly 'org.slf4j:slf4j-api:1.7.10' compileOnly 'org.pf4j:pf4j:3.4.1' + + // quiltcore + implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.1' + implementation 'com.upplication:s3fs:2.2.2' + implementation 'com.quiltdata:quiltcore:0.1.2' + // add here plugins depepencies // test configuration @@ -81,5 +87,4 @@ dependencies { // use JUnit 5 platform test { useJUnitPlatform() -} - +} \ No newline at end of file diff --git a/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy b/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy index 658de6d..ad092f6 100644 --- a/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy +++ b/plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy @@ -38,7 +38,7 @@ import nextflow.trace.TraceRecord @CompileStatic class ProvObserver implements TraceObserver { - public static final List VALID_FORMATS = ['bco', 'dag', 'legacy'] + public static final List VALID_FORMATS = ['bco', 'dag', 'legacy', 'quilt'] private Session session @@ -66,6 +66,9 @@ class ProvObserver implements TraceObserver { if( name == 'legacy' ) return new LegacyRenderer(opts) + + if( name == 'quilt' ) + return new QuiltRenderer(opts) throw new IllegalArgumentException("Invalid provenance format -- valid formats are ${VALID_FORMATS.join(', ')}") } diff --git a/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy b/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy new file mode 100644 index 0000000..274513c --- /dev/null +++ b/plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy @@ -0,0 +1,175 @@ +/* + * Copyright 2023, Quilt Data + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.prov + +import java.io.FileOutputStream +import java.io.OutputStream +import java.net.URI; +import java.nio.file.Files +import java.nio.file.Path + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic +import nextflow.Session +import nextflow.file.FileHolder +import nextflow.processor.TaskRun + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import com.quiltdata.quiltcore.Entry +import com.quiltdata.quiltcore.Registry +import com.quiltdata.quiltcore.Namespace +import com.quiltdata.quiltcore.Manifest +import com.quiltdata.quiltcore.key.PhysicalKey +import com.quiltdata.quiltcore.key.LocalPhysicalKey +import com.quiltdata.quiltcore.key.S3PhysicalKey + +/** + * Renderer for the Quilt manifest format. + * + * @author Kevin Moore + */ +@CompileStatic +class QuiltRenderer implements Renderer { + + private Path path + + private boolean overwrite + + QuiltRenderer(Map opts) { + path = opts.file as Path + overwrite = opts.overwrite as Boolean + + ProvHelper.checkFileOverwrite(path, overwrite) + } + + private static def jsonify(root) { + if ( root instanceof Map ) + root.collectEntries( (k, v) -> [k, jsonify(v)] ) + + else if ( root instanceof Collection ) + root.collect( v -> jsonify(v) ) + + else if ( root instanceof FileHolder ) + jsonify(root.storePath) + + else if ( root instanceof Path ) + root.toUriString() + + else if ( root instanceof Boolean || root instanceof Number ) + root + + else + root.toString() + } + + Map renderTask(TaskRun task) { + // TODO: Figure out what the '$' input/output means + // Omitting them from manifest for now + return [ + 'id': task.id as String, + 'name': task.name, + 'cached': task.cached, + 'process': task.processor.name, + 'script': task.script, + 'inputs': task.inputs.findResults { inParam, object -> + def inputMap = [ + 'name': inParam.getName(), + 'value': jsonify(object) + ] + inputMap['name'] != '$' ? inputMap : null + }, + 'outputs': task.outputs.findResults { outParam, object -> + def outputMap = [ + 'name': outParam.getName(), + 'emit': outParam.getChannelEmitName(), + 'value': jsonify(object) + ] + outputMap['name'] != '$' ? outputMap : null + } + ] + } + + @Override + void render(Session session, Set tasks, Map outputs) { + // generate task manifest + def tasksMap = tasks.inject([:]) { accum, task -> + accum[task.id] = renderTask(task) + accum + } + + // generate temporary output-task map + def taskLookup = tasksMap.inject([:]) { accum, id, task -> + task['outputs'].each { output -> + // Make sure to handle tuples of outputs + def values = output['value'] + if ( values instanceof Collection ) + values.each { accum.put(it, task['id']) } + else + accum.put(values, task['id']) + } + accum + } + + + + // render JSON output + //def manifest = [ + // 'pipeline': session.config.manifest, + // 'published': outputs.collect { source, target -> [ + // 'source': source.toUriString(), + // 'target': target.toUriString(), + // 'publishingTaskId': taskLookup[source.toUriString()], + // ] }, + // 'tasks': tasksMap + //] + + println "Building manifest" + def builder = new Manifest.Builder() + int i=0 + for(Path source: outputs.keySet()){ + println "Source URI " + source.toUriString() + println "Target URI " + outputs.get(source).toUriString() + def target = outputs.get(source) + if(target == null){ + println "NULL TARGET" + break + } + + def logicalKey = target.getFileName() + + // Get the file size + def fileSize = Files.size(target) + def Entry e = new Entry( + PhysicalKey.fromUri(target.toUri()), + Files.size(target), + null, + null + ) + builder.addEntry(logicalKey.toString(), e.withHash()) + println("Added ${logicalKey}") + } + def pkg = builder.build() + OutputStream outputStream = new FileOutputStream(path.toString()) + pkg.serializeToOutputStream(outputStream) + outputStream.close() + println "Writing manifest to " + path.toString() + + //path.text = JsonOutput.prettyPrint(JsonOutput.toJson(manifest)) + } + +} diff --git a/plugins/nf-prov/src/resources/META-INF/MANIFEST.MF b/plugins/nf-prov/src/resources/META-INF/MANIFEST.MF index 2db3eb5..0f2e693 100644 --- a/plugins/nf-prov/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-prov/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Id: nf-prov -Plugin-Version: 1.2.2 +Plugin-Version: 1.2.3 Plugin-Class: nextflow.prov.ProvPlugin Plugin-Provider: nextflow Plugin-Requires: >=23.04.0