From 98e14100eabc590114b28ecbd2a4982ec57c0a09 Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" Date: Fri, 26 Jan 2024 20:53:12 -0800 Subject: [PATCH] 71 path input (#174) - Make input URIs with `&path=` actually work - also support query arrays --- .github/workflows/pkg-test.yml | 60 ---------- CHANGELOG.md | 7 ++ Makefile | 12 +- README.md | 4 +- main.path.nf | 29 +++++ .../main/nextflow/quilt/QuiltProduct.groovy | 1 + .../nextflow/quilt/jep/QuiltPackage.groovy | 7 +- .../nextflow/quilt/jep/QuiltParser.groovy | 47 ++++++-- .../nextflow/quilt/nio/QuiltFileSystem.groovy | 4 +- .../quilt/nio/QuiltFileSystemProvider.groovy | 113 ++++++++++++++---- .../main/nextflow/quilt/nio/QuiltPath.groovy | 15 ++- .../quilt/nio/QuiltPathFactory.groovy | 6 +- .../src/resources/META-INF/MANIFEST.MF | 2 +- .../quilt/jep/QuiltPackageTest.groovy | 24 ++++ .../nextflow/quilt/jep/QuiltParserTest.groovy | 13 ++ .../nio/QuiltFileSystemProviderTest.groovy | 23 ++++ .../quilt/nio/QuiltPathFactoryTest.groovy | 5 +- .../nextflow/quilt/nio/QuiltPathTest.groovy | 4 +- 18 files changed, 264 insertions(+), 112 deletions(-) delete mode 100644 .github/workflows/pkg-test.yml create mode 100644 main.path.nf diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml deleted file mode 100644 index 030162b0..00000000 --- a/.github/workflows/pkg-test.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Test - -on: - # Trigger at every push. Action will also be visible from Pull Requests to master - push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) - pull_request: - branches: [master] - -permissions: read-all - -jobs: - build: - name: Package Test - permissions: - contents: read - id-token: write - issues: write - pull-requests: write - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - java_version: [19] - runs-on: ${{ matrix.os }} - - steps: - # Git Checkout - - name: Checkout Code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - fetch-depth: 0 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::712023778557:role/github/GitHub-Testing-NF-Quilt - aws-region: us-east-1 - - - name: Setup Java ${{matrix.java_version}} - uses: actions/setup-java@v4 - with: - java-version: ${{matrix.java_version}} - distribution: 'temurin' - architecture: x64 - cache: gradle - - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - - - name: Run Package Tests - run: make pkg-test - - - name: Archive production artifacts - uses: actions/upload-artifact@v3 - with: - name: nf-quilt-pkg-test - path: | - /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c48f397e..c368343e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.7.7] 2024-01-24 + +- Properly implement and test getFilename() +- Install package just before download +- Add and pass `path-input` integration test +- Add unit test for "&path=" Quilt+ URIs + ## [0.7.6] 2024-01-10 UNOFFICIAL - Re-enable crash on failure diff --git a/Makefile b/Makefile index 4b6ad401..4e9d660f 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PROJECT ?= nf-quilt WRITE_BUCKET ?= quilt-example FRAGMENT ?= &path=. NF_DIR ?= ../nextflow -NF_BIN ?= ./nextflow +NF_BIN ?= ./launch.sh PID ?= $$$$ PIPELINE ?= sarek QUERY ?= ?Name=$(USER)&Owner=Kevin+Moore&Date=2023-03-07&Type=CRISPR&Notebook+URL=http%3A%2F%2Fexample.com @@ -36,10 +36,6 @@ compile: ./gradlew compileGroovy exportClasspath @echo "DONE `date`" -$(NF_BIN): - curl -s https://get.nextflow.io | bash - chmod +x $(NF_BIN) - nextflow-git: if [ ! -d "$(NF_DIR)" ]; then git clone https://github.com/nextflow-io/nextflow.git "$(NF_DIR)"; fi cd "$(NF_DIR)"; git checkout && make compile && git restore .; cd .. @@ -61,7 +57,7 @@ test-all: clean compile-all check #coverage # Create packages # -pkg-test: compile-all +pkg-test: compile #-all echo "$(TEST_URI)" $(NF_BIN) run ./main.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" @@ -69,6 +65,10 @@ pkg-fail: compile echo "$(TEST_URI)" $(NF_BIN) run ./fail.nf -profile standard -plugins $(PROJECT) --outdir "$(TEST_URI)" +path-input: clean compile #-all + echo "$(TEST_URI)" + $(NF_BIN) run ./main.path.nf -profile standard -plugins $(PROJECT) --outdir "./results" + tower-test: $(NF_BIN) $(NF_BIN) run "https://github.com/quiltdata/nf-quilt" -name local_einstein -with-tower -r main -latest --pub "$(TEST_URI)" diff --git a/README.md b/README.md index e1fec9a2..32a6b19e 100644 --- a/README.md +++ b/README.md @@ -75,8 +75,8 @@ From the command-line, do, e.g.: ```bash # export NXF_VER=23.04.3 -export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.6/nf-quilt-0.7.6-meta.json -nextflow run main.nf -plugins nf-quilt@0.7.6 +export NXF_PLUGINS_TEST_REPOSITORY=https://github.com/quiltdata/nf-quilt/releases/download/0.7.7/nf-quilt-0.7.7-meta.json +nextflow run main.nf -plugins nf-quilt@0.7.7 ``` For Tower, you can use the "Pre-run script" to set the environment variables. diff --git a/main.path.nf b/main.path.nf new file mode 100644 index 00000000..7d91f301 --- /dev/null +++ b/main.path.nf @@ -0,0 +1,29 @@ +#!/usr/bin/env nextflow +/* groovylint-disable CompileStatic */ + +nextflow.enable.dsl=2 + +test_file_local = 'README.md' +test_file_s3 = 's3://quilt-example/examples/hurdat2/README.md' +test_file_quilt = 'quilt+s3://quilt-example#package=examples/hurdat2&path=README.md' + +myFileChannel = Channel.fromList([file(test_file_local), file(test_file_s3), file(test_file_quilt)]) + +process CHECK_INPUT { + input: + path input + + output: + path 'README.md', emit: output + + script: + """ + ls -l + echo $input + cp -f $input ../../tmp + """ +} + +workflow { + CHECK_INPUT(myFileChannel) +} diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 43209e2a..23b39013 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -145,6 +145,7 @@ ${nextflow} log.error("Exception: ${e}") print("FAILED: $pkg\n") e.printStackTrace() + /* groovylint-disable-next-line ThrowRuntimeException */ throw new RuntimeException(e) } print("SUCCESS: $pkg\n") diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index d54c0b80..9123c622 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -121,7 +121,7 @@ class QuiltPackage { this.hash = parsed.getHash() this.meta = parsed.getMetadata() this.folder = Paths.get(INSTALL_ROOT.toString(), this.toString()) - //log.debug("QuiltParser.folder[${this.folder}]") + log.debug("QuiltPackage.folder[${this.folder}]") this.setup() } @@ -187,11 +187,12 @@ class QuiltPackage { String resolvedHash = (hash == 'latest' || hash == null || hash == 'null') ? namespace.getHash('latest') : hash - log.info("hash: $hash -> $resolvedHash") + log.debug("hash: $hash -> $resolvedHash") Manifest manifest = namespace.getManifest(resolvedHash) manifest.install(dest) - log.info('done') + log.debug("done: installed into $dest)") + println("Children: ${relativeChildren('')}") } catch (IOException e) { log.error("failed to install $packageName") // this is non-fatal error, so we don't want to stop the pipeline diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy index fd159ec4..b4c71f6e 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable Instanceof */ /* * Copyright 2022, Quilt Data Inc * @@ -75,18 +76,50 @@ class QuiltParser { return new QuiltParser(uri.authority, pkg, path, options, metadata) } - static Map parseQuery(String query) { + static String decode(String str) { + return URLDecoder.decode(str, StandardCharsets.UTF_8) + } + + static String encode(String str) { + return URLEncoder.encode(str, StandardCharsets.UTF_8) + } + + static Map parseQuery(String query) { if (!query) { return [:] } // skip for urls without query params - final queryParams = query.split('&') - return queryParams.collectEntries { params -> params.split('=').collect { param -> URLDecoder.decode(param) } } + def params = query.split('&') + def result = [:] + params.each { param -> + def keyValue = param.split('=') + if (keyValue.size() == 2) { + String key = decode(keyValue[0]) + String value = decode(keyValue[1]) + if (result.containsKey(key)) { + if (result[key] instanceof List) { + result[key].add(value) + } else { + result[key] = [result[key], value] + } + } else { + result[key] = value + } + } + + } + return result + } + + static String encodePair(String key, String value) { + return "${QuiltParser.encode(key)}=${QuiltParser.encode(value)}" } static String unparseQuery(Map query) { if (!query) { return '' } // skip for urls without query params - List params = query.collect { key, val -> - String k = URLEncoder.encode(key, StandardCharsets.UTF_8) - String v = URLEncoder.encode(val.toString(), StandardCharsets.UTF_8) - "${k}=${v}".toString() + List params = query.collect { key, value -> + if (value instanceof List) { + value.collect { QuiltParser.encodePair(key, it.toString()) }.join('&') + } else { + QuiltParser.encodePair(key, value.toString()) + } } return params.join('&') } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy index 62d71e00..cf6ffc11 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy @@ -106,7 +106,7 @@ final class QuiltFileSystem extends FileSystem implements Closeable { } QuiltFileAttributes readAttributes(QuiltPath path) { - //log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") + log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") Path installedPath = path.localPath() try { BasicFileAttributes attrs = Files.readAttributes(installedPath, BasicFileAttributes) @@ -139,7 +139,7 @@ final class QuiltFileSystem extends FileSystem implements Closeable { @Override QuiltPath getPath(String root, String... more) { - //log.debug("QuiltFileSystem.getPath`[${root}]: $more") + log.debug("QuiltFileSystem.getPath`[${root}]: $more") QuiltParser p = QuiltParser.forBarePath(root) return new QuiltPath(this, p) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index 44e86f29..b7bb108d 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable ExplicitCallToEqualsMethod, Instanceof */ /* * Copyright 2022, Quilt Data Inc * @@ -38,12 +39,18 @@ import java.nio.file.attribute.BasicFileAttributes import java.nio.file.attribute.FileAttribute import java.nio.file.attribute.FileAttributeView import java.nio.file.spi.FileSystemProvider +import java.nio.file.FileSystems +import java.nio.file.FileAlreadyExistsException import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Global import nextflow.Session import nextflow.quilt.jep.QuiltParser +import nextflow.quilt.jep.QuiltPackage +import nextflow.file.FileSystemTransferAware +import nextflow.file.CopyOptions +import nextflow.file.FileHelper /** * Implements NIO File system provider for Quilt Blob Storage @@ -53,20 +60,25 @@ import nextflow.quilt.jep.QuiltParser @Slf4j @CompileStatic -class QuiltFileSystemProvider extends FileSystemProvider { +class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { private final Map myEnv = new HashMap<>(System.getenv()) private final Map fileSystems = [:] private Map attributesCache = [:] static QuiltPath asQuiltPath(Path path) { - if (path !in QuiltPath) { - String pathClassName = path?.class?.name ?: '-' - throw new IllegalArgumentException( - "Not a valid Quilt blob storage path object: `${path}` [${pathClassName}]" - ) + if (path in QuiltPath) { + return (QuiltPath)path + } + String pathString = path?.toString() ?: '-' + if (pathString.startsWith(QuiltParser.SCHEME + ':/')) { + QuiltPath qPath = QuiltPathFactory.parse(pathString) + return qPath } - return (QuiltPath)path + String pathClassName = path?.class?.name ?: '-' + throw new IllegalArgumentException( + "Not a valid Quilt blob storage path object: `${path}` [${pathClassName}]" + ) } static QuiltFileSystem getQuiltFilesystem(Path path) { @@ -83,6 +95,65 @@ class QuiltFileSystemProvider extends FileSystemProvider { return path.getFileSystem().provider() } + /** + * QuiltFileSystemProvider + */ + + boolean canUpload(Path source, Path target) { + // log.debug "QuiltFileSystemProvider.canUpload: ${source} -> ${target}" + return FileSystems.getDefault().equals(source.getFileSystem()) && target instanceof QuiltPath + } + + boolean canDownload(Path source, Path target) { + // log.debug "QuiltFileSystemProvider.canDownload: ${source} -> ${target}" + return source instanceof QuiltPath && FileSystems.getDefault().equals(target.getFileSystem()) + } + + void download(Path remoteFile, Path localDestination, CopyOption... options) throws IOException { + // log.debug "QuiltFileSystemProvider.download: ${remoteFile} -> ${localDestination}" + QuiltPath qPath = asQuiltPath(remoteFile) + Path cachedFile = qPath.localPath() + QuiltPackage pkg = qPath.pkg() + if (!pkg.installed) { + pkg.install() + } + + if (!Files.exists(cachedFile)) { + throw new NoSuchFileException(remoteFile.toString()) + } + + final CopyOptions opts = CopyOptions.parse(options) + // delete target if it exists and REPLACE_EXISTING is specified + if (opts.replaceExisting()) { + FileHelper.deletePath(localDestination) + } + else if (Files.exists(localDestination)) { + throw new FileAlreadyExistsException(localDestination.toString()) + } + + Files.copy(cachedFile, localDestination, options) + } + + void upload(Path localFile, Path remoteDestination, CopyOption... options) throws IOException { + // log.debug "QuiltFileSystemProvider.upload: ${localFile} -> ${remoteDestination}" + final CopyOptions opts = CopyOptions.parse(options) + // delete target if it exists and REPLACE_EXISTING is specified + if (opts.replaceExisting()) { + QuiltPath qPath = asQuiltPath(remoteDestination) + qPath.deinstall() + } + else if (Files.exists(remoteDestination)) { + throw new FileAlreadyExistsException(remoteDestination.toString()) + } + + QuiltPath qPath = asQuiltPath(remoteDestination) + Path cachedFile = qPath.localPath() + if (Files.exists(cachedFile)) { + throw new FileAlreadyExistsException(remoteDestination.toString()) + } + Files.copy(localFile, cachedFile, options) + } + /** * @inheritDoc */ @@ -226,7 +297,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override QuiltPath getPath(URI uri) { QuiltParser parsed = QuiltParser.forURI(uri) - log.debug("QuiltFileSystemProvider.getPath`[${uri}]") + // log.debug("QuiltFileSystemProvider.getPath`[${uri}]") final fs = getFileSystem(parsed.quiltIDS(), true) return new QuiltPath(fs, parsed) } @@ -257,7 +328,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override SeekableByteChannel newByteChannel( Path path, Set options, FileAttribute... attrs) throws IOException { - //log.debug("Creating `newByteChannel`: ${path} <- ${options}") + // log.debug("Creating `newByteChannel`: ${path} <- ${options}") final modeWrite = options.contains(WRITE) || options.contains(APPEND) final QuiltPath qPath = asQuiltPath(path) @@ -271,7 +342,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { attributesCache = [:] // reset cache notifyFilePublish(qPath) } - //log.debug("\tOpening channel to: $installedPath") + // log.debug("\tOpening channel to: $installedPath") FileChannel channel = FileChannel.open(installedPath, options) return channel } @@ -297,10 +368,10 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override DirectoryStream newDirectoryStream(Path obj, DirectoryStream.Filter filter) throws IOException { final qPath = asQuiltPath(obj) - //final dirPath = qPath.localPath() - //Files.newDirectoryStream(dirPath, filter) + // final dirPath = qPath.localPath() + // Files.newDirectoryStream(dirPath, filter) - //log.debug("QuiltFileSystemProvider.newDirectoryStream[${qPath.file_key()}]: ${qPath}") + // log.debug("QuiltFileSystemProvider.newDirectoryStream[${qPath.file_key()}]: ${qPath}") return new DirectoryStream() { @@ -319,7 +390,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override void createDirectory(Path dir, FileAttribute... attrs) throws IOException { final path = asQuiltPath(dir).localPath() - //log.debug("Calling createDirectory[${path}]: ${dir} ") + // log.debug("Calling createDirectory[${path}]: ${dir} ") Files.createDirectories(path) } /* groovylint-enable BuilderMethodWithSideEffects */ @@ -334,7 +405,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override void copy(Path from, Path to, CopyOption... options) throws IOException { - //log.debug("Attempting `copy`: ${from} -> ${to}") + // log.debug("Attempting `copy`: ${from} -> ${to}") assert provider(from) == provider(to) if (from == to) { return // nothing to do -- just return @@ -370,7 +441,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override void checkAccess(Path path, AccessMode... modes) throws IOException { - //log.debug("Calling `checkAccess`: ${path}") + // log.debug("Calling `checkAccess`: ${path}") checkRoot(path) QuiltPath qPath = asQuiltPath(path) readAttributes(qPath, QuiltFileAttributes) @@ -381,7 +452,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override def V getFileAttributeView(Path path, Class type, LinkOption... options) { - //log.debug("Calling `getFileAttributeView`: ${path}") + // log.debug("Calling `getFileAttributeView`: ${path}") checkRoot(path) if (type == BasicFileAttributeView || type == QuiltFileAttributesView) { QuiltPath qPath = asQuiltPath(path) @@ -394,7 +465,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override def A readAttributes(Path path, Class type, LinkOption... options) throws IOException { - //log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' + // log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' def attr = attributesCache.get(path) if (attr) { return attr @@ -407,15 +478,15 @@ class QuiltFileSystemProvider extends FileSystemProvider { attributesCache[path] = result return result } - //log.debug("readAttributes: File ${qPath.localPath()} not found") + // log.debug("readAttributes: File ${qPath.localPath()} not found") throw new NoSuchFileException(qPath.toUriString()) } throw new UnsupportedOperationException("Not a valid Quilt Storage file attribute type: $type") - } + } @Override Map readAttributes(Path path, String attributes, LinkOption... options) throws IOException { - //log.debug 'Map QuiltFileSystemProvider.readAttributes()' + // log.debug 'Map QuiltFileSystemProvider.readAttributes()' throw new UnsupportedOperationException("Operation Map 'readAttributes' is not supported by QuiltFileSystem") } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy index 2d7aefe3..d6b8ac90 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy @@ -43,12 +43,14 @@ final class QuiltPath implements Path, Comparable { private final QuiltFileSystem filesystem private final QuiltParser parsed private final String[] paths + private final boolean isFileName - QuiltPath(QuiltFileSystem filesystem, QuiltParser parsed) { + QuiltPath(QuiltFileSystem filesystem, QuiltParser parsed, boolean isFileName = false) { this.filesystem = filesystem this.parsed = parsed this.paths = parsed.getPaths() - //log.debug("Creating QuiltPath: $parsed") + this.isFileName = isFileName + log.debug("Creating QuiltPath: $parsed") } String getBucket() { @@ -110,8 +112,10 @@ final class QuiltPath implements Path, Comparable { @Override Path getFileName() { - //log.debug("getFileName`[${this}]: paths=$paths") - return isJustPackage() ? this : new QuiltPath(filesystem, parsed.lastPath()) // IF DIRECTORY + log.debug("getFileName[${this}]: paths=$paths") + // if (isJustPackage()) { return this } // IF DIRECTORY + QuiltPath filePath = new QuiltPath(filesystem, parsed.lastPath(), true) + return filePath } @Override @@ -215,6 +219,9 @@ final class QuiltPath implements Path, Comparable { @Override String toString() { + if (isFileName) { + return paths.size() > 0 ? paths[-1] : '' + } return parsed.toString() } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathFactory.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathFactory.groovy index 52a4716d..b0a21eb5 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathFactory.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathFactory.groovy @@ -40,10 +40,12 @@ class QuiltPathFactory extends FileSystemPathFactory { @Override protected Path parseUri(String uriString) { - if (!uriString.startsWith(QuiltParser.PREFIX)) { + if (!uriString.startsWith(QuiltParser.SCHEME)) { return null } - final uri = new URI(uriString) + String[] split = uriString.split('/') + String newString = (split[1] == '') ? uriString : uriString.replaceFirst(/\/+/, '//') + final uri = new URI(newString) return FileHelper.getOrCreateFileSystemFor(uri).provider().getPath(uri) } diff --git a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF index 0a53cefd..f85e13db 100644 --- a/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-quilt/src/resources/META-INF/MANIFEST.MF @@ -1,7 +1,7 @@ Manifest-Version: 1.0 Plugin-Class: nextflow.quilt.QuiltPlugin Plugin-Id: nf-quilt -Plugin-Version: 0.7.6 +Plugin-Version: 0.7.7 Plugin-Provider: Quilt Data Plugin-Requires: >=22.10.6 diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index 685b2724..203d9b89 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -107,6 +107,30 @@ class QuiltPackageTest extends QuiltSpecification { Files.readAttributes(qpath, BasicFileAttributes) } + @IgnoreIf({ System.getProperty('os.name').contains('indows') }) + void 'should download the specified path'() { + given: + def qpath = factory.parseUri(TEST_URL) + def qpkg = qpath.pkg() + Path outputFolder = pkg.packageDest() + Path readmeFile = outputFolder.resolve('README.md') + println("qpkg: ${qpkg} -> ${qpath.localPath()} == ${readmeFile}") + + expect: + !qpath.isJustPackage() + Files.isDirectory(outputFolder) + //!Files.exists(readmeFile) + + qpkg.install() + Files.exists(qpath.localPath()) + Files.isRegularFile(qpath.localPath()) + + Files.exists(readmeFile) + Files.isRegularFile(readmeFile) + Files.isReadable(readmeFile) + readObject(readmeFile).startsWith('# Quilt Smart Reports') + } + void 'should return null on failed install'() { given: def url2 = TEST_URL.replace('quilt-', 'quilted-') diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy index e0dcc4d8..0d570e14 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy @@ -123,4 +123,17 @@ class QuiltParserTest extends QuiltSpecification { unparsed.replace('%2f', '/') == fullURL } + void 'should collect array parameters from query string'() { + when: + String query = 'key=val1,val2&quay=vale1&quay=vale2' + Map result = QuiltParser.parseQuery(query) + println "QuiltParserTest[$query] -> ${result}" + String unparsed = QuiltParser.unparseQuery(result) + + then: + result['key'] == 'val1,val2' + result['quay'] == ['vale1', 'vale2'] + unparsed == query.replace(',', '%2C') + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy index fc18641e..b9e02856 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy @@ -3,12 +3,18 @@ package nextflow.quilt.nio import nextflow.quilt.QuiltSpecification import groovy.transform.CompileDynamic +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.Files +import groovy.util.logging.Slf4j +import spock.lang.IgnoreIf /** * * @author Ernest Prabhakar */ @CompileDynamic +@Slf4j class QuiltFileSystemProviderTest extends QuiltSpecification { void 'should return Quilt storage scheme'() { @@ -22,4 +28,21 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { // newDirectoryStream returns package path for write // do we need a new schema for quilt+local? + @IgnoreIf({ System.getProperty('os.name').contains('indows') }) + void 'should download file from remote to local destination'() { + given: + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() + String filename = 'README.md' + Path remoteFile = Paths.get('quilt+s3://quilt-example#package=examples%2fhurdat2&path=' + filename) + Path tempFolder = Files.createTempDirectory('quilt') + Path tempFile = tempFolder.resolve(filename) + + when: + provider.download(remoteFile, tempFile) + + then: + Files.exists(tempFile) + Files.size(tempFile) > 0 + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathFactoryTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathFactoryTest.groovy index 5343a39a..d0811b97 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathFactoryTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathFactoryTest.groovy @@ -53,8 +53,9 @@ class QuiltPathFactoryTest extends QuiltSpecification { QuiltPathFactory.parse(PATH).toString() == STR where: - _ | PATH | STR - _ | 'quilt+s3://reg#package=user/pkg' | 'reg#package=user%2fpkg' + _ | PATH | STR + _ | 'quilt+s3://reg#package=user/pkg' | 'reg#package=user%2fpkg' + _ | 'quilt+s3:/reg#package=user/pkg&path=test.md' | 'reg#package=user%2fpkg&path=test.md' } void 'should create Channel from URL'() { diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy index 727053b0..0f06fc4e 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy @@ -115,13 +115,13 @@ class QuiltPathTest extends QuiltSpecification { '#path=file-name.txt' | false | 'bucket.so.me' } - @Ignore void 'should validate getFileName'() { expect: - pathify(path).getFileName() == pathify(fileName) + pathify(path).getFileName().toString() == fileName where: path | fileName + 'bucket' | '' 'bucket#path=file.txt' | 'file.txt' 'bucket#path=some%2fdata%2ffile.txt' | 'file.txt' '#path=file-name.txt' | 'file-name.txt'