Ignore internal pushes (#257)

Don't try to create a quilt package from an internal pubDir
(test whether the bucket exists)

---------

Co-authored-by: Dr. Ernie Prabhakar <[email protected]>

CHANGELOG.md

@@ -8,6 +8,10 @@
 - Improve handling of dynamically-specified URIs
 - Rewrite README.md, splitting out developer documentation to README_DEV.md
 
+## [0.8.9] 2024-10-31 UNPUBLISHED
+
+- Handle multiple/internal publishDir calls
+
 ## [0.8.8] 2024-10-31 UNPUBLISHED
 
 - Debug build

plugins/nf-quilt/build.gradle

@@ -125,7 +125,7 @@ jacocoTestCoverageVerification {
     violationRules {
         rule {
             limit {
-                minimum = 0.65
+                minimum = 0.7
             }
         }
 

plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy

@@ -43,10 +43,11 @@ class QuiltObserver implements TraceObserver {
 
     boolean checkExtractedPath(QuiltPathify pathify) {
         String key = pathify.pkgKey()
+        println("checkExtractedPath[$key]: $pathify [$publishedPaths]")
         if (key in publishedPaths) {
             return true
         }
-        log.debug("checkExtractedPath: $key not in publishedPaths")
+        println("checkExtractedPath: $key not in publishedPaths")
         addPublishedPath(key, pathify)
         return false
     }
@@ -58,24 +59,37 @@ class QuiltObserver implements TraceObserver {
         } finally {
             lock.unlock()
         }
+        println("addPublishedPath[$key]: $pathify [$publishedPaths]")
+    }
+
+    int countPublishedPaths() {
+        return publishedPaths.size()
     }
 
     @Override
     void onFlowCreate(Session session) {
-        log.debug("`onFlowCreate` $this")
+        log.info("`onFlowCreate` $session")
         this.session = session
         this.workDir = session.config.workDir
     }
 
     @Override
     void onFilePublish(Path destination, Path source) {
         // Path source may be null, won't work with older versions of Nextflow
-        log.info("\nonFilePublish.dest:$destination <- src:$source")
-        if (!session) {
-            log.debug('onFilePublish: no session intialized')
+        log.info("onFilePublish.dest:$destination <- src:$source")
+        println("\tonFilePublish.session: $session")
+        if (session == null) {
+            log.info('onFilePublish: no session intialized')
             return
         }
+        println('\tonFilePublish.QuiltPathify')
         QuiltPathify pathify = new QuiltPathify(destination)
+        println("\tonFilePublish.pathify: $pathify")
+        if (!pathify.isBucketAccessible()) {
+            log.debug("onFilePublish.isBucketAccessible[false]: $pathify")
+            return
+        }
+        println("\tonFilePublish.isOverlay: ${pathify.isOverlay}")
         if (pathify.isOverlay && source == null) {
             log.error("onFilePublish.isOverlay: no source for $pathify")
             return

plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy

@@ -155,6 +155,10 @@ class QuiltPathify  {
         return true
     }
 
+    boolean isBucketAccessible() {
+        return pkg.isBucketAccessible()
+    }
+
     String pkgKey() {
         return pkg.toKey()
     }

plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy

@@ -216,10 +216,25 @@ class QuiltPackage {
         return installed
     }
 
+    boolean isBucketAccessible() {
+        S3PhysicalKey key = new S3PhysicalKey(bucket, '', null)
+        try {
+            key.listRecursively()
+        } catch (Exception e) {
+            log.error("isBucketAccessible: failed to check $bucket", e)
+            return false
+        }
+        return true
+    }
+
     Path packageDest() {
         return folder
     }
 
+    /*
+     * Package methods
+     */
+
     Path install(boolean implicit=false) {
         if (isNull()) {
             log.debug('null bucket: no need to install')

plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF

@@ -1,7 +1,7 @@
 Manifest-Version: 1.0
 Plugin-Class: nextflow.quilt.QuiltPlugin
 Plugin-Id: nf-quilt
-Plugin-Version: 0.8.8
+Plugin-Version: 0.8.9
 Plugin-Provider: Quilt Data
 Plugin-Requires: >=22.10.6
 

plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy

@@ -22,6 +22,7 @@ import nextflow.Session
 //import spock.lang.Ignore
 
 import java.nio.file.Paths
+import java.nio.file.Path
 import groovy.transform.CompileDynamic
 
 /**
@@ -49,14 +50,62 @@ class QuiltObserverTest extends QuiltSpecification {
     void 'should not error on onFlowComplete success'() {
         given:
         String quilt_uri = 'quilt+s3://bucket#package=prefix%2fsuffix'
+        QuiltObserver observer = makeObserver()
+        QuiltPath badPath = QuiltPathFactory.parse(quilt_uri)
+        when:
+        observer.onFlowCreate(mockSession(false))
+        observer.onFilePublish(badPath)
+        observer.onFlowComplete()
+        then:
+        true
+    }
+
+    void 'should not add publishedPaths if uninitialized'() {
+        given:
         QuiltObserver observer = new QuiltObserver()
-        QuiltPath qPath = QuiltPathFactory.parse(quilt_uri)
+        QuiltPath validPath = QuiltPathFactory.parse(SpecURI())
+
+        when:
+        // uninitialized
+        observer.onFilePublish(validPath, validPath.localPath())
+        then:
+        validPath.pkg().isBucketAccessible() == true
+        observer.publishedPaths.size() == 0
+    }
+
+    void 'should only add publishedPaths if valid path'() {
+        given:
+        QuiltObserver observer = makeObserver()
+        QuiltPath badPath = QuiltPathFactory.parse('quilt+s3://bucket#package=prefix%2fsuffix')
+        QuiltPath validPath = QuiltPathFactory.parse(SpecURI())
+        Path localPath = Paths.get('/work/bkt/prefix/suffix')
+
+        when:
+        // bad path
         observer.onFlowCreate(mockSession(false))
-        observer.onFilePublish(qPath.localPath())
+        observer.onFilePublish(badPath, badPath.localPath())
+        then:
+        observer.publishedPaths.size() == 0
+
+        when:
+        // no source
+        observer.onFilePublish(badPath)
+        then:
+        observer.publishedPaths.size() == 0
+
+        when:
+        // local path (treated as overlay)
+        observer.onFilePublish(localPath)
+        then:
+        observer.publishedPaths.size() == 0
+
         when:
+        // valid bucket
+        observer.onFilePublish(validPath, validPath.localPath())
         observer.onFlowComplete()
         then:
-        true
+        validPath.pkg().isBucketAccessible() == true
+        observer.publishedPaths.size() == 1
     }
 
 }

plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy

@@ -52,4 +52,54 @@ class QuiltFileSystemTest extends QuiltSpecification {
         fs.supportedFileAttributeViews() == ['basic'] as Set
     }
 
+    void 'should test QuiltPath-only operations'() {
+        when:
+        String BUCKET_NAME = 'bucket'
+        QuiltFileSystemProvider provider = Stub(QuiltFileSystemProvider)
+        QuiltFileSystem fs = new QuiltFileSystem(BUCKET_NAME, provider)
+        QuiltPath quiltPath = QuiltPathFactory.parse('quilt+s3://bkt#package=a/b&path=f.txt')
+        Path localPath = Paths.get('f.txt')
+
+        then:
+        !fs.exists(quiltPath)
+        fs.toUriString(quiltPath)
+        !fs.toUriString(localPath)
+        fs.getBashLib(quiltPath)
+        !fs.getBashLib(localPath)
+        fs.getUploadCmd(BUCKET_NAME, quiltPath)
+        !fs.getUploadCmd(BUCKET_NAME, localPath)
+    }
+
+    void 'should test unimplemented operations'() {
+        given:
+        String BUCKET_NAME = 'bucket'
+        QuiltFileSystemProvider provider = Stub(QuiltFileSystemProvider)
+        QuiltFileSystem fs = new QuiltFileSystem(BUCKET_NAME, provider)
+
+        when:
+        fs.getRootDirectories()
+        then:
+        thrown(UnsupportedOperationException)
+
+        when:
+        fs.getFileStores()
+        then:
+        thrown(UnsupportedOperationException)
+
+        when:
+        fs.getPathMatcher('*')
+        then:
+        thrown(UnsupportedOperationException)
+
+        when:
+        fs.getUserPrincipalLookupService()
+        then:
+        thrown(UnsupportedOperationException)
+
+        when:
+        fs.newWatchService()
+        then:
+        thrown(UnsupportedOperationException)
+    }
+
 }