Create QuiltRenderer

nextflow.config

@@ -1,5 +1,5 @@
 plugins {
-    id 'nf-prov'
+    id 'nf-prov@1.2.3'
 }
 
 params {
@@ -8,15 +8,7 @@ params {
 
 prov {
     formats {
-        bco {
-            file = "${params.outdir}/bco.json"
-            overwrite = true
-        }
-        dag {
-            file = "${params.outdir}/dag.html"
-            overwrite = true
-        }
-        legacy {
+        quilt {
             file = "${params.outdir}/manifest.json"
             overwrite = true
         }

plugins/nf-prov/build.gradle

@@ -59,6 +59,12 @@ dependencies {
     compileOnly 'io.nextflow:nextflow:23.04.0'
     compileOnly 'org.slf4j:slf4j-api:1.7.10'
     compileOnly 'org.pf4j:pf4j:3.4.1'
+
+    // quiltcore
+    implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.1'
+    implementation 'com.upplication:s3fs:2.2.2'
+    implementation 'com.quiltdata:quiltcore:0.1.2'
+
     // add here plugins depepencies
 
     // test configuration
@@ -81,5 +87,4 @@ dependencies {
 // use JUnit 5 platform
 test {
     useJUnitPlatform()
-}
-
+}

plugins/nf-prov/src/main/nextflow/prov/ProvObserver.groovy

@@ -38,7 +38,7 @@ import nextflow.trace.TraceRecord
 @CompileStatic
 class ProvObserver implements TraceObserver {
 
-    public static final List<String> VALID_FORMATS = ['bco', 'dag', 'legacy']
+    public static final List<String> VALID_FORMATS = ['bco', 'dag', 'legacy', 'quilt']
 
     private Session session
 
@@ -66,6 +66,9 @@ class ProvObserver implements TraceObserver {
 
         if( name == 'legacy' )
             return new LegacyRenderer(opts)
+
+        if( name == 'quilt' )
+            return new QuiltRenderer(opts)
 
         throw new IllegalArgumentException("Invalid provenance format -- valid formats are ${VALID_FORMATS.join(', ')}")
     }

plugins/nf-prov/src/main/nextflow/prov/QuiltRenderer.groovy

@@ -0,0 +1,175 @@
+/*
+ * Copyright 2023, Quilt Data
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nextflow.prov
+
+import java.io.FileOutputStream
+import java.io.OutputStream
+import java.net.URI;
+import java.nio.file.Files
+import java.nio.file.Path
+
+import groovy.json.JsonOutput
+import groovy.transform.CompileStatic
+import nextflow.Session
+import nextflow.file.FileHolder
+import nextflow.processor.TaskRun
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import com.quiltdata.quiltcore.Entry
+import com.quiltdata.quiltcore.Registry
+import com.quiltdata.quiltcore.Namespace
+import com.quiltdata.quiltcore.Manifest
+import com.quiltdata.quiltcore.key.PhysicalKey
+import com.quiltdata.quiltcore.key.LocalPhysicalKey
+import com.quiltdata.quiltcore.key.S3PhysicalKey
+
+/**
+ * Renderer for the Quilt manifest format.
+ *
+ * @author Kevin Moore <[email protected]>
+ */
+@CompileStatic
+class QuiltRenderer implements Renderer {
+
+    private Path path
+
+    private boolean overwrite
+
+    QuiltRenderer(Map opts) {
+        path = opts.file as Path
+        overwrite = opts.overwrite as Boolean
+
+        ProvHelper.checkFileOverwrite(path, overwrite)
+    }
+
+    private static def jsonify(root) {
+        if ( root instanceof Map )
+            root.collectEntries( (k, v) -> [k, jsonify(v)] )
+
+        else if ( root instanceof Collection )
+            root.collect( v -> jsonify(v) )
+
+        else if ( root instanceof FileHolder )
+            jsonify(root.storePath)
+
+        else if ( root instanceof Path )
+            root.toUriString()
+
+        else if ( root instanceof Boolean || root instanceof Number )
+            root
+
+        else
+            root.toString()
+    }
+
+    Map renderTask(TaskRun task) {
+        // TODO: Figure out what the '$' input/output means
+        //       Omitting them from manifest for now
+        return [
+            'id': task.id as String,
+            'name': task.name,
+            'cached': task.cached,
+            'process': task.processor.name,
+            'script': task.script,
+            'inputs': task.inputs.findResults { inParam, object -> 
+                def inputMap = [ 
+                    'name': inParam.getName(),
+                    'value': jsonify(object) 
+                ] 
+                inputMap['name'] != '$' ? inputMap : null
+            },
+            'outputs': task.outputs.findResults { outParam, object -> 
+                def outputMap = [
+                    'name': outParam.getName(),
+                    'emit': outParam.getChannelEmitName(),
+                    'value': jsonify(object) 
+                ] 
+                outputMap['name'] != '$' ? outputMap : null
+            }
+        ]
+    }
+
+    @Override
+    void render(Session session, Set<TaskRun> tasks, Map<Path,Path> outputs) {
+        // generate task manifest
+        def tasksMap = tasks.inject([:]) { accum, task ->
+            accum[task.id] = renderTask(task)
+            accum
+        }
+
+        // generate temporary output-task map
+        def taskLookup = tasksMap.inject([:]) { accum, id, task ->
+            task['outputs'].each { output ->
+                // Make sure to handle tuples of outputs
+                def values = output['value']
+                if ( values instanceof Collection )
+                    values.each { accum.put(it, task['id']) }
+                else
+                    accum.put(values, task['id'])
+            }
+            accum
+        }
+
+
+
+        // render JSON output
+        //def manifest = [
+        //    'pipeline': session.config.manifest,
+        //    'published': outputs.collect { source, target -> [
+        //        'source': source.toUriString(),
+        //        'target': target.toUriString(),
+        //        'publishingTaskId': taskLookup[source.toUriString()],
+        //    ] },
+        //    'tasks': tasksMap
+        //]
+
+        println "Building manifest"
+        def builder = new Manifest.Builder()
+        int i=0
+        for(Path source: outputs.keySet()){
+            println "Source URI " + source.toUriString()
+            println "Target URI " + outputs.get(source).toUriString()
+            def target = outputs.get(source)
+            if(target == null){
+                println "NULL TARGET"
+                break
+            }
+
+            def logicalKey = target.getFileName()
+
+            // Get the file size
+            def fileSize = Files.size(target)
+            def Entry e = new Entry(
+                PhysicalKey.fromUri(target.toUri()),
+                Files.size(target),
+                null,
+                null
+            )
+            builder.addEntry(logicalKey.toString(), e.withHash())
+            println("Added ${logicalKey}")
+        }
+        def pkg = builder.build()
+        OutputStream outputStream = new FileOutputStream(path.toString())
+        pkg.serializeToOutputStream(outputStream)
+        outputStream.close()
+        println "Writing manifest to " + path.toString()
+
+        //path.text = JsonOutput.prettyPrint(JsonOutput.toJson(manifest))
+    }
+
+}

plugins/nf-prov/src/resources/META-INF/MANIFEST.MF

@@ -1,6 +1,6 @@
 Manifest-Version: 1.0
 Plugin-Id: nf-prov
-Plugin-Version: 1.2.2
+Plugin-Version: 1.2.3
 Plugin-Class: nextflow.prov.ProvPlugin
 Plugin-Provider: nextflow
 Plugin-Requires: >=23.04.0