diff --git a/CHANGELOG.md b/CHANGELOG.md index d678ce0d..b11cd220 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ - Improve handling of dynamically-specified URIs - Rewrite README.md, splitting out developer documentation to README_DEV.md +## [0.8.9] 2024-10-31 UNPUBLISHED + +- Handle multiple/internal publishDir calls + ## [0.8.8] 2024-10-31 UNPUBLISHED - Debug build diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index e84db762..5c842dca 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -125,7 +125,7 @@ jacocoTestCoverageVerification { violationRules { rule { limit { - minimum = 0.65 + minimum = 0.7 } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy index c9a58525..d40ad544 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy @@ -43,10 +43,11 @@ class QuiltObserver implements TraceObserver { boolean checkExtractedPath(QuiltPathify pathify) { String key = pathify.pkgKey() + println("checkExtractedPath[$key]: $pathify [$publishedPaths]") if (key in publishedPaths) { return true } - log.debug("checkExtractedPath: $key not in publishedPaths") + println("checkExtractedPath: $key not in publishedPaths") addPublishedPath(key, pathify) return false } @@ -58,11 +59,16 @@ class QuiltObserver implements TraceObserver { } finally { lock.unlock() } + println("addPublishedPath[$key]: $pathify [$publishedPaths]") + } + + int countPublishedPaths() { + return publishedPaths.size() } @Override void onFlowCreate(Session session) { - log.debug("`onFlowCreate` $this") + log.info("`onFlowCreate` $session") this.session = session this.workDir = session.config.workDir } @@ -70,12 +76,20 @@ class QuiltObserver implements TraceObserver { @Override void onFilePublish(Path destination, Path source) { // Path source may be null, won't work with older versions of Nextflow - log.info("\nonFilePublish.dest:$destination <- src:$source") - if (!session) { - log.debug('onFilePublish: no session intialized') + log.info("onFilePublish.dest:$destination <- src:$source") + println("\tonFilePublish.session: $session") + if (session == null) { + log.info('onFilePublish: no session intialized') return } + println('\tonFilePublish.QuiltPathify') QuiltPathify pathify = new QuiltPathify(destination) + println("\tonFilePublish.pathify: $pathify") + if (!pathify.isBucketAccessible()) { + log.debug("onFilePublish.isBucketAccessible[false]: $pathify") + return + } + println("\tonFilePublish.isOverlay: ${pathify.isOverlay}") if (pathify.isOverlay && source == null) { log.error("onFilePublish.isOverlay: no source for $pathify") return diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy index fc8f163c..1a771351 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy @@ -155,6 +155,10 @@ class QuiltPathify { return true } + boolean isBucketAccessible() { + return pkg.isBucketAccessible() + } + String pkgKey() { return pkg.toKey() } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index c01e6a03..537305ac 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -216,10 +216,25 @@ class QuiltPackage { return installed } + boolean isBucketAccessible() { + S3PhysicalKey key = new S3PhysicalKey(bucket, '', null) + try { + key.listRecursively() + } catch (Exception e) { + log.error("isBucketAccessible: failed to check $bucket", e) + return false + } + return true + } + Path packageDest() { return folder } + /* + * Package methods + */ + Path install(boolean implicit=false) { if (isNull()) { log.debug('null bucket: no need to install') diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index 34ad02d9..57de8d5f 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.8.8 +Plugin-Version: 0.8.9 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy index 2270d87d..b7e05243 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy @@ -22,6 +22,7 @@ import nextflow.Session //import spock.lang.Ignore import java.nio.file.Paths +import java.nio.file.Path import groovy.transform.CompileDynamic /** @@ -49,14 +50,62 @@ class QuiltObserverTest extends QuiltSpecification { void 'should not error on onFlowComplete success'() { given: String quilt_uri = 'quilt+s3://bucket#package=prefix%2fsuffix' + QuiltObserver observer = makeObserver() + QuiltPath badPath = QuiltPathFactory.parse(quilt_uri) + when: + observer.onFlowCreate(mockSession(false)) + observer.onFilePublish(badPath) + observer.onFlowComplete() + then: + true + } + + void 'should not add publishedPaths if uninitialized'() { + given: QuiltObserver observer = new QuiltObserver() - QuiltPath qPath = QuiltPathFactory.parse(quilt_uri) + QuiltPath validPath = QuiltPathFactory.parse(SpecURI()) + + when: + // uninitialized + observer.onFilePublish(validPath, validPath.localPath()) + then: + validPath.pkg().isBucketAccessible() == true + observer.publishedPaths.size() == 0 + } + + void 'should only add publishedPaths if valid path'() { + given: + QuiltObserver observer = makeObserver() + QuiltPath badPath = QuiltPathFactory.parse('quilt+s3://bucket#package=prefix%2fsuffix') + QuiltPath validPath = QuiltPathFactory.parse(SpecURI()) + Path localPath = Paths.get('/work/bkt/prefix/suffix') + + when: + // bad path observer.onFlowCreate(mockSession(false)) - observer.onFilePublish(qPath.localPath()) + observer.onFilePublish(badPath, badPath.localPath()) + then: + observer.publishedPaths.size() == 0 + + when: + // no source + observer.onFilePublish(badPath) + then: + observer.publishedPaths.size() == 0 + + when: + // local path (treated as overlay) + observer.onFilePublish(localPath) + then: + observer.publishedPaths.size() == 0 + when: + // valid bucket + observer.onFilePublish(validPath, validPath.localPath()) observer.onFlowComplete() then: - true + validPath.pkg().isBucketAccessible() == true + observer.publishedPaths.size() == 1 } } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy index 5f23e2c4..7e32cefb 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy @@ -52,4 +52,54 @@ class QuiltFileSystemTest extends QuiltSpecification { fs.supportedFileAttributeViews() == ['basic'] as Set } + void 'should test QuiltPath-only operations'() { + when: + String BUCKET_NAME = 'bucket' + QuiltFileSystemProvider provider = Stub(QuiltFileSystemProvider) + QuiltFileSystem fs = new QuiltFileSystem(BUCKET_NAME, provider) + QuiltPath quiltPath = QuiltPathFactory.parse('quilt+s3://bkt#package=a/b&path=f.txt') + Path localPath = Paths.get('f.txt') + + then: + !fs.exists(quiltPath) + fs.toUriString(quiltPath) + !fs.toUriString(localPath) + fs.getBashLib(quiltPath) + !fs.getBashLib(localPath) + fs.getUploadCmd(BUCKET_NAME, quiltPath) + !fs.getUploadCmd(BUCKET_NAME, localPath) + } + + void 'should test unimplemented operations'() { + given: + String BUCKET_NAME = 'bucket' + QuiltFileSystemProvider provider = Stub(QuiltFileSystemProvider) + QuiltFileSystem fs = new QuiltFileSystem(BUCKET_NAME, provider) + + when: + fs.getRootDirectories() + then: + thrown(UnsupportedOperationException) + + when: + fs.getFileStores() + then: + thrown(UnsupportedOperationException) + + when: + fs.getPathMatcher('*') + then: + thrown(UnsupportedOperationException) + + when: + fs.getUserPrincipalLookupService() + then: + thrown(UnsupportedOperationException) + + when: + fs.newWatchService() + then: + thrown(UnsupportedOperationException) + } + }