diff --git a/CHANGELOG.md b/CHANGELOG.md index 31750da5..13028869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [0.8.1] UNRELEASED + +- Fix bug in path extraction from S3 URIs + +## [0.8.0] 2024-08-31 + +- Add and improve code coverage using jacoco +- Support S3 URIs as an overlay plugin +- Fix bug with pathless input URIs + ## [0.7.17] UNRELEASED - support ChunkedChecksums diff --git a/Makefile b/Makefile index 7bf8744f..964c9d16 100644 --- a/Makefile +++ b/Makefile @@ -49,11 +49,21 @@ check: .PHONY: clean test test-all all pkg-test tower-test -test: clean compile check #coverage +test: clean compile check verifyCoverage -test-nextflow: clean nextflow-git compile check #coverage +test-nextflow: clean nextflow-git compile check -test-all: clean compile-all check #coverage +test-all: clean compile-all check coverage + +coverage: + ./gradlew jacocoTestReport + open plugins/nf-quilt/build/reports/jacoco/test/html/index.html + +verifyCoverage: + ./gradlew jacocoTestCoverageVerification + +groovysh: + ./gradlew -q --no-daemon --console=plain --init-script groovysh-task.gradle groovysh # # Create packages diff --git a/gradle-groovysh-init.gradle b/gradle-groovysh-init.gradle new file mode 100644 index 00000000..f10c6575 --- /dev/null +++ b/gradle-groovysh-init.gradle @@ -0,0 +1,49 @@ +gradle.projectsLoaded { + rootProject { + afterEvaluate { project -> + if (!project.repositories.any{it.name == 'MavenRepo'}) { + project.repositories { + // To be able to load org.codehaus.groovy:groovy-groovysh + mavenCentral() + } + } + + project.configurations { + groovyshdependencies + } + + project.dependencies { + groovyshdependencies("org.codehaus.groovy:groovy-groovysh:${GroovySystem.version}") { + exclude group: 'org.codehaus.groovy' + } + } + + project.tasks.register('groovysh') { + group 'debug' + description 'Runs an interactive shell in the context of the project.' + doLast { + URLClassLoader groovyObjectClassLoader = GroovyObject.class.classLoader + def groovyshClass + def groovyShell + + // Add dependency jars to classloader + configurations.groovyshdependencies.each {File file -> + groovyObjectClassLoader.addURL(file.toURL()) + } + Class.forName('jline.console.history.FileHistory', true, groovyObjectClassLoader) + groovyshClass = Class.forName('org.codehaus.groovy.tools.shell.Groovysh', true, groovyObjectClassLoader) + + if (groovyshClass) { + groovyShell = groovyshClass.newInstance() + } + if (groovyShell) { + groovyShell.interp.context.variables.put("gradle", gradle) + groovyShell.interp.context.variables.put("settings", gradle.settings) + groovyShell.interp.context.variables.put("project", project) + groovyShell.run('') + } + } + } + } + } +} \ No newline at end of file diff --git a/groovysh-task.gradle b/groovysh-task.gradle new file mode 100644 index 00000000..5bde2aab --- /dev/null +++ b/groovysh-task.gradle @@ -0,0 +1,54 @@ +gradle.projectsLoaded { + rootProject { + afterEvaluate { project -> + if (!project.repositories.any{it.name == 'MavenRepo'}) { + project.repositories { + // To be able to load org.apache.groovy:groovy-groovysh and dependencies + mavenCentral { + content { + includeGroup 'org.apache.groovy' + includeGroup 'jline' + includeGroup 'com.github.javaparser' + includeGroup 'org.ow2.asm' + includeGroup 'org.abego.treelayout' + includeGroup 'org.apache.ivy' + } + } + } + } + project.configurations { + groovyshdependencies + } + + project.dependencies { + groovyshdependencies "org.apache.groovy:groovy-groovysh:4.0.13" + } + + project.tasks.register('groovysh') { + group 'debug' + description 'Runs an interactive shell in the context of the project. Use :inspect command to inspect project, gradle, settings or other objects.' + doLast { + URLClassLoader groovyshClassLoader = new URLClassLoader(); + configurations.groovyshdependencies.each {File file -> + groovyshClassLoader.addURL(file.toURI().toURL()) + } + + def fileHistoryClass + def groovyshClass + def groovyShell + fileHistoryClass = Class.forName('jline.console.history.FileHistory', true, groovyshClassLoader) + groovyshClass = Class.forName('org.apache.groovy.groovysh.Groovysh', true, groovyshClassLoader) + if (groovyshClass) { + groovyShell = groovyshClass.newInstance() + if (groovyShell) { + groovyShell.interp.context.variables.put("gradle", gradle) + groovyShell.interp.context.variables.put("settings", gradle.settings) + groovyShell.interp.context.variables.put("project", project) + groovyShell.run('# Available objects: gradle, settings, project\n# Try :inspect project') + } + } + } + } + } + } +} diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 840650a2..8b77bf35 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -20,6 +20,7 @@ plugins { id 'idea' id 'se.patrikerdes.use-latest-versions' version '0.2.18' id 'com.github.ben-manes.versions' version '0.51.0' + id 'jacoco' } useLatestVersions { @@ -115,3 +116,29 @@ test { useJUnitPlatform() } +jacocoTestReport { + dependsOn test // tests are required to run before generating the report +} + +jacocoTestCoverageVerification { + dependsOn jacocoTestReport // tests are required to run before generating the report + violationRules { + rule { + limit { + minimum = 0.65 + } + } + + rule { + enabled = false + element = 'CLASS' + includes = ['org.gradle.*'] + + limit { + counter = 'LINE' + value = 'TOTALCOUNT' + maximum = 0.3 + } + } + } +} diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index a0b5b45b..47fa2a05 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -114,6 +114,9 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr log.debug "QuiltFileSystemProvider.download: ${remoteFile} -> ${localDestination}" QuiltPath qPath = asQuiltPath(remoteFile) Path cachedFile = qPath.localPath() + /* + * UNUSED: QuiltPackage is always installed + * QuiltPackage pkg = qPath.pkg() if (!pkg.installed) { log.info "download.install Quilt package: ${pkg}" @@ -124,6 +127,7 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr } log.info "download.installed Quilt package to: $dest" } + */ if (!Files.exists(cachedFile)) { log.error "download: File ${cachedFile} not found" @@ -164,6 +168,9 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr if (Files.exists(cachedFile)) { throw new FileAlreadyExistsException(remoteDestination.toString()) } + if (!Files.exists(localFile)) { + throw new NoSuchFileException(localFile.toString()) + } Files.copy(localFile, cachedFile, options) } @@ -418,9 +425,9 @@ class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTr @Override void copy(Path from, Path to, CopyOption... options) throws IOException { - // log.debug("Attempting `copy`: ${from} -> ${to}") + log.debug("Attempting `copy`: ${from} -> ${to}") assert provider(from) == provider(to) - if (from == to) { + if (from.toString() == to.toString()) { return // nothing to do -- just return } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy index 6aecb4a3..439c2ceb 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy @@ -23,6 +23,7 @@ import nextflow.Session import java.nio.file.Path import java.nio.file.Paths import groovy.transform.CompileDynamic +import spock.lang.Ignore /** * @@ -95,4 +96,55 @@ class QuiltObserverTest extends QuiltSpecification { '/bucket/file.ext' | 'quilt+s3://bucket#package=default_prefix%2fdefault_suffix&path=file.ext' } + void 'should recover URI from onFilePublish QuiltPath'() { + given: + QuiltObserver observer = new QuiltObserver() + Path path = Paths.get(key) + Path quiltPath = QuiltPathFactory.parse(quilt_uri) + observer.onFilePublish(quiltPath, path) + String pkgKey = observer.pkgKey(quiltPath) + expect: + pkgKey == key + observer.uniqueURIs[key] == quilt_uri + observer.publishedURIs[key] == quilt_uri + where: + key | quilt_uri + 'bucket/prefix/suffix' | 'quilt+s3://bucket#package=prefix%2fsuffix' + } + + @Ignore('FIXME: handle onFilePublish with local Path') + void 'should extract URI from onFilePublish local Path'() { + given: + QuiltObserver observer = new QuiltObserver() + Path quiltPath = QuiltPathFactory.parse(quilt_uri) + Path path = Paths.get('/'+key) + observer.onFilePublish(path, quiltPath) + String pkgKey = observer.pkgKey(quiltPath) + expect: + pkgKey == key + observer.uniqueURIs[key] == quilt_uri + observer.publishedURIs[key] == quilt_uri + where: + key | quilt_uri + 'bucket/prefix/suffix' | 'quilt+s3://bucket#package=prefix%2fsuffix' + } + + void 'should not error on onFlowComplete'() { + given: + String quilt_uri = 'quilt+s3://bucket#package=prefix%2fsuffix' + QuiltObserver observer = new QuiltObserver() + QuiltPath qPath = QuiltPathFactory.parse(quilt_uri) + Session session = GroovyMock(Session) { + // getWorkflowMetadata() >> metadata + getParams() >> [outdir: quilt_uri] + isSuccess() >> false + } + observer.onFlowCreate(session) + observer.onFilePublish(qPath, qPath) + when: + observer.onFlowComplete() + then: + true + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index 111ffa41..a1840fcd 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -17,6 +17,8 @@ package nextflow.quilt.nio import nextflow.Session +import nextflow.script.WorkflowMetadata + import nextflow.quilt.QuiltSpecification import nextflow.quilt.QuiltProduct import nextflow.quilt.jep.QuiltParser @@ -37,9 +39,16 @@ import spock.lang.Unroll @CompileDynamic class QuiltProductTest extends QuiltSpecification { - QuiltProduct makeProduct(String query=null) { + QuiltProduct makeProduct(String query=null, boolean success = false) { String subURL = query ? fullURL.replace('key=val&key2=val2', query) : fullURL - Session session = Mock(Session) + WorkflowMetadata metadata = GroovyMock(WorkflowMetadata) { + toMap() >> [start:'2022-01-01', complete:'2022-01-02'] + } + Session session = GroovyMock(Session) { + getWorkflowMetadata() >> metadata + getParams() >> [outdir: subURL] + isSuccess() >> success + } QuiltPath path = QuiltPathFactory.parse(subURL) return new QuiltProduct(path, session) } @@ -50,12 +59,23 @@ class QuiltProductTest extends QuiltSpecification { String query = QuiltParser.unparseQuery(meta) subURL = subURL.replace('#', "?${query}#") } - Session session = Mock(Session) + Session session = GroovyMock(Session) QuiltPath path = QuiltPathFactory.parse(subURL) return new QuiltProduct(path, session) } - void 'now should generate solid string for timestamp'() { + void 'should generate mocks from makeProduct'() { + given: + QuiltProduct product = makeProduct() + + expect: + product + product.pkg + product.session != null + product.session.getWorkflowMetadata() != null + } + + void 'should generate solid string for timestamp from now'() { when: def now = QuiltProduct.now() then: @@ -216,4 +236,47 @@ class QuiltProductTest extends QuiltSpecification { Files.exists(Paths.get(sumPkg.packageDest().toString(), QuiltProduct.SUMMARY_FILE)) } + void 'should getMetadata from Map'() { + given: + Map meta = [ + 'Name': 'QuiltPackageTest', + 'Owner': 'Ernest', + 'Date': '1967-10-08', + 'Type': 'NGS' + ] + QuiltProduct product = makeProduct() + Map quilt_meta = product.getMetadata(meta) + + expect: + quilt_meta != null + } + + void 'should setupMeta from session'() { + given: + QuiltProduct product = makeProduct() + Map quilt_meta = product.setupMeta() + + expect: + quilt_meta != null + } + + void 'should throw error on publish'() { + given: + QuiltProduct product = makeProduct() + + when: + product.publish() + + then: + thrown(RuntimeException) + } + + void 'should throw error if session.isSuccess'() { + when: + makeProduct(query: null, success: true) + + then: + thrown(RuntimeException) + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy index b89f1992..562a9465 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy @@ -4,6 +4,7 @@ package nextflow.quilt.nio import nextflow.quilt.QuiltSpecification import groovy.transform.CompileDynamic import java.nio.file.Path +import java.nio.file.Paths import java.nio.file.Files import java.nio.file.CopyOption import java.nio.file.StandardCopyOption @@ -17,6 +18,14 @@ import groovy.util.logging.Slf4j @Slf4j class QuiltFileSystemProviderTest extends QuiltSpecification { + static Path parsedURIWithPath(boolean withPath = false) { + String packageURI = 'quilt+s3://udp-spec#package=nf-quilt/source' + if (withPath) { + packageURI += '&path=COPY_THIS.md' + } + return QuiltPathFactory.parse(packageURI) + } + void 'should return Quilt storage scheme'() { given: QuiltFileSystemProvider provider = new QuiltFileSystemProvider() @@ -24,6 +33,29 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { provider.getScheme() == 'quilt+s3' } + void 'should error asQuiltPath with non-Quilt path'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + Path path = Paths.get('README.md') + when: + provider.asQuiltPath(path) + then: + thrown IllegalArgumentException + } + + void 'should canUpload and canDownload based on local and remote paths'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + Path localPath = Paths.get('README.md') + Path remotePath = parsedURIWithPath(true) + + expect: + provider.canUpload(localPath, remotePath) + provider.canDownload(remotePath, localPath) + !provider.canUpload(remotePath, localPath) + !provider.canDownload(localPath, remotePath) + } + // newDirectoryStream returns local path for read // newDirectoryStream returns package path for write // do we need a new schema for quilt+local? @@ -31,8 +63,8 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { void 'should download file from remote to local destination'() { given: QuiltFileSystemProvider provider = new QuiltFileSystemProvider() - String filename = 'README.md' - Path remoteFile = QuiltPathFactory.parse('quilt+s3://quilt-example#package=examples%2fhurdat2&path=' + filename) + Path remoteFile = parsedURIWithPath(true) + String filename = remoteFile.getFileName() Path tempFolder = Files.createTempDirectory('quilt') Path tempFile = tempFolder.resolve(filename) @@ -47,7 +79,7 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { void 'should download folders from remote to local destination'() { given: QuiltFileSystemProvider provider = new QuiltFileSystemProvider() - Path remoteFolder = QuiltPathFactory.parse('quilt+s3://quilt-example#package=examples%2fhurdat2') + Path remoteFolder = parsedURIWithPath(false) Path tempFolder = Files.createTempDirectory('quilt') CopyOption opt = StandardCopyOption.REPLACE_EXISTING when: @@ -58,4 +90,43 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { Files.list(tempFolder).count() > 0 } + void 'should fail to upload a file to itself'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + Path remoteFile = parsedURIWithPath(true) + + when: + provider.upload(remoteFile, remoteFile) + + then: + thrown java.nio.file.FileAlreadyExistsException + } + + void 'should error when copying from remote to local path'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + Path remoteFile = parsedURIWithPath(true) + String filename = remoteFile.getFileName() + Path tempFolder = Files.createTempDirectory('quilt') + Path tempFile = tempFolder.resolve(filename) + + when: + provider.copy(remoteFile, tempFile) + + then: + thrown org.codehaus.groovy.runtime.powerassert.PowerAssertionError + } + + void 'should do nothing when copying a path to itself'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + Path remoteFile = parsedURIWithPath(true) + + when: + provider.copy(remoteFile, remoteFile) + + then: + Files.exists(remoteFile.localPath()) + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 6c166018..79a04465 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -79,8 +79,8 @@ class QuiltNioTest extends QuiltSpecification { text.startsWith('id') } - @IgnoreIf({ System.getProperty('os.name').toLowerCase().contains('windows') || - System.getProperty('os.name').toLowerCase().contains('linux') }) + @IgnoreIf({ System.getProperty('os.name').toLowerCase().contains('windows') }) + @IgnoreIf({ System.getProperty('os.name').toLowerCase().contains('linux') }) void 'should read file attributes'() { given: final start = System.currentTimeMillis() diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy index 5aae1bf7..65bb9050 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy @@ -6,8 +6,10 @@ import nextflow.quilt.jep.QuiltParser import nextflow.quilt.jep.QuiltPackage import java.nio.file.Path +import java.nio.file.Paths import groovy.util.logging.Slf4j import groovy.transform.CompileDynamic +import java.nio.file.ProviderMismatchException import spock.lang.Unroll import spock.lang.Ignore @@ -161,7 +163,6 @@ class QuiltPathTest extends QuiltSpecification { void 'should validate resolve: base:=#base; path=#path'() { expect: pathify(base).resolve(path) == pathify(expected) - //pathify(base).resolve( pathify(path) ) == pathify(expected) where: base | path | expected @@ -171,6 +172,26 @@ class QuiltPathTest extends QuiltSpecification { 'bucket' | 'some%2ffile-name.txt' | 'bucket#path=some%2ffile-name.txt' } + void 'should resolve another QuiltPath'() { + given: + QuiltPath basePath = pathify('bucket#package=so%2fme') + QuiltPath otherPath = pathify('bucket#package=da/ta') + + expect: + basePath.resolve(otherPath) == otherPath + } + + void 'should resolve error for non-QuiltPath'() { + given: + QuiltPath basePath = pathify('bucket#package=so%2fme') + Path nonQuiltPath = Paths.get('file-name.txt') + when: + basePath.resolve(nonQuiltPath) + + then: + thrown ProviderMismatchException + } + @Ignore('FIXME: subpath not yet implemented') void 'should validate subpath: #expected'() { expect: