diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml new file mode 100644 index 00000000..0e2bf335 --- /dev/null +++ b/.github/workflows/mega-linter.yml @@ -0,0 +1,55 @@ +# MegaLinter GitHub Action configuration file +# More info at https://megalinter.github.io +name: MegaLinter + +on: + # Trigger mega-linter at every push. Action will also be visible from Pull Requests to main + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) + +env: # Comment env block if you do not want to apply fixes + # Apply linter fixes configuration + APPLY_FIXES: all # When active, APPLY_FIXES must also be defined as environment variable (in github/workflows/mega-linter.yml or other CI tool) + #APPLY_FIXES_EVENT: pull_request # Decide which event triggers application of fixes in a commit or a PR (pull_request, push, all) + #APPLY_FIXES_MODE: pull_request # If APPLY_FIXES is used, defines if the fixes are directly committed (commit) or posted in a PR (pull_request) + DISABLE_LINTERS: SPELL_CSPELL,COPYPASTE_JSCPD + FILTER_REGEX_EXCLUDE: .*/.*gradle + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + name: MegaLinter + runs-on: ubuntu-latest + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@v3 + with: + token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} + fetch-depth: 0 # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to improve performances + + # MegaLinter + - name: MegaLinter + id: ml + # You can override MegaLinter flavor used to have faster performances + # More info at https://megalinter.github.io/flavors/ + uses: oxsecurity/megalinter/flavors/dotnet@v6.18.0 + env: + # All available variables are described in documentation + # https://megalinter.github.io/configuration/ + VALIDATE_ALL_CODEBASE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} # Validates all source when push on main, else just the git diff with main. Override with true if you always want to lint all sources + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # ADD YOUR CUSTOM ENV VARIABLES HERE OR DEFINE THEM IN A FILE .mega-linter.yml AT THE ROOT OF YOUR REPOSITORY + DISABLE: COPYPASTE,SPELL # Uncomment to disable copy-paste and spell checks + + # Upload MegaLinter artifacts + - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: MegaLinter reports + path: | + megalinter-reports + mega-linter.log diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..e0641bda --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,31 @@ +name: Test + +on: + # Trigger at every push. Action will also be visible from Pull Requests to master + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) + pull_request: + branches: [master] + +jobs: + build: + name: Test + runs-on: ubuntu-latest + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@v3 + with: + token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 17 + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run Gradle Tests + run: make test \ No newline at end of file diff --git a/.groovylintrc.json b/.groovylintrc.json new file mode 100644 index 00000000..7e971c84 --- /dev/null +++ b/.groovylintrc.json @@ -0,0 +1,83 @@ +{ + "extends": "recommended", + "rules": { + "MethodCount": { + "enabled": false + }, + "JavadocMissingParamDescription": { + "enabled": false + }, + "JavadocMissingThrowsDescription": { + "enabled": false + }, + "JavadocEmptyReturnTag": { + "enabled": false + }, + "JavadocEmptyFirstLine": { + "enabled": false + }, + "JUnitPublicNonTestMethod": { + "enabled": false + }, + "CatchException": { + "enabled": false + }, + "CatchThrowable": { + "enabled": false + }, + "ClassJavadoc": { + "enabled": false + }, + "ClosureAsLastMethodParameter": { + "enabled": false + }, + "DuplicateNumberLiteral": { + "enabled": false + }, + "DuplicateStringLiteral": { + "enabled": false + }, + "FieldTypeRequired": { + "enabled": false + }, + "JavaIoPackageAccess": { + "enabled": false + }, + "MethodParameterTypeRequired": { + "enabled": false + }, + "MethodSize": { + "enabled": false + }, + "NoDef": { + "enabled": false + }, + "PrintStackTrace": { + "enabled": false + }, + "PropertyName": { + "enabled": false + }, + "SpaceAroundMapEntryColon": { + "enabled": false + }, + "SystemExit": { + "enabled": false + }, + "UnnecessaryGetter": { + "enabled": false + }, + "UnnecessaryObjectReferences": { + "enabled": false + }, + "UnnecessarySetter": { + "enabled": false + }, + "VariableName": { + "enabled": false + }, + "VariableTypeRequired": { + "enabled": false + } + } +} diff --git a/Makefile b/Makefile index bfc6529b..cc3c4836 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,13 @@ compile: @echo "DONE `date`" check: - ./gradlew check + ./gradlew check --warning-mode all + +coverage: compile + ./gradlew cloverInstrumentCodeForTest + +.PHONY: clean test all +test: clean compile check #coverage # diff --git a/README.md b/README.md index e67baaa4..269b4ce1 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ which quilt3 # e.g., /usr/local/bin/quilt3 ### Loading the nf-quilt plugin -Once this plugin is officially published on `nextflow-io/plugins`, +Once this plugin is officially published on `nextflow-io/plugins`, you can enable it by modifying `nextflow.config`. Add the following snippet, or just add that one 'id' if you already have other plugins): ```groovy @@ -43,28 +43,28 @@ Next, create a Quilt URL for the S3 bucket where you want to store (and eventual You must specify a package name containing exactly one '/', such as `instrument/experiment` e.g. "quilt+s3://raw-bucket#package=nf-quilt/sarek" -Note your command-line environment must have -[AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) +Note your command-line environment must have +[AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) that allow you to read/write that bucket. Finally, run your Nextflow pipeline as usual, setting that URL as your output directory, .e.g.: -``` +```bash ./launch.sh run nf-core/sarek -profile test,docker --outdir quilt+s3://raw-bucket#package=nf-quilt/sarek&path=. ``` You can also use Quilt packages as input to nextflow jobs, e.g.: -``` +```bash nextflow run my/analysis --indir quilt+s3://raw-bucket#package=experiment/instrument --outdir quilt+s3://prod-bucket#package=experiment/analysis ``` -# Development +## Development _Based on [nf-hello](https://github.com/nextflow-io/nf-hello)_ -## Unit testing +### Unit testing Run the following command in the project root directory (ie. where the file `settings.gradle` is located): @@ -72,7 +72,7 @@ Run the following command in the project root directory (ie. where the file `set make check ``` -## Testing and debugging +### Testing and debugging 1. Clone the Nextflow repository into a sibling directory, .e.g: @@ -99,7 +99,7 @@ make pkg-test BUCKET=my-s3-bucket # default, simply copies a package make sarek BUCKET=my-s3-bucket # runs nf-core/sarek, or any other pipeline that uses `--outdir` ``` -## Package, upload and publish +### Package, upload and publish The project should be hosted in a GitHub repository whose name should match the name of the plugin, that is the name of the directory in the `plugins` folder (e.g. `nf-quilt`). @@ -113,14 +113,15 @@ Follow these steps to package, upload and publish the plugin: * `github_commit_email`: The email address associated with your GitHub account. 3. Use the following command to package and create a release for your plugin on GitHub: - ```bash - ./gradlew :plugins:nf-quilt:upload - ``` +```bash +./gradlew :plugins:nf-quilt:upload +``` 4. Fork the [nextflow-io/plugins](https://github.com/nextflow-io/plugins) repository to one you can write to 5. Use the following command to publish your plugin to your fork: - ```bash - ./gradlew :plugins:publishIndex - ``` + ```bash + ./gradlew :plugins:publishIndex + ``` + 6. Create a pull request to push your changes back to [nextflow-io/plugins](https://github.com/nextflow-io/plugins/blob/main/plugins.json) diff --git a/build.gradle b/build.gradle index 90fa720f..4fa81ecf 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ */ plugins { - id "io.codearte.nexus-staging" version "0.30.0" + id 'io.codearte.nexus-staging' version '0.30.0' id 'java' id 'idea' } @@ -34,10 +34,12 @@ if (groovyVer) { configurations.all { resolutionStrategy.eachDependency { DependencyResolveDetails details -> if (details.requested.group == 'org.codehaus.groovy') { - if( groovyVer.contains(':') ) + if (groovyVer.contains(':')) { details.useTarget(groovyVer) - else + } + else { details.useVersion(groovyVer) + } println ">> Overriding $details.requested with version: $groovyVer" } } @@ -46,7 +48,7 @@ if (groovyVer) { } def projects(String...args) { - args.collect {project(it)} + args.collect { project(it) } } group = 'io.nextflow' @@ -94,15 +96,17 @@ allprojects { dependencies { // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sec:module_replacement + implementation 'org.codehaus.groovy:groovy-all:3.0.14' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.14.1' + modules { - module("commons-logging:commons-logging") { replacedBy("org.slf4j:jcl-over-slf4j") } + module('commons-logging:commons-logging') { replacedBy('org.slf4j:jcl-over-slf4j') } } // Documentation required libraries groovyDoc 'org.fusesource.jansi:jansi:2.4.0' - groovyDoc "org.codehaus.groovy:groovy-groovydoc:3.0.14" - groovyDoc "org.codehaus.groovy:groovy-ant:3.0.14" + groovyDoc 'org.codehaus.groovy:groovy-groovydoc:3.0.14' + groovyDoc 'org.codehaus.groovy:groovy-ant:3.0.14' } test { @@ -133,13 +137,13 @@ allprojects { // http://forums.gradle.org/gradle/topics/gradle_task_groovydoc_failing_with_noclassdeffounderror tasks.withType(Groovydoc) { groovyClasspath = project.configurations.groovyDoc - includes = ["nextflow/**"] + includes = ['nextflow/**'] } // Required to run tests on Java 9 and higher in compatibility mode if (JavaVersion.current() >= JavaVersion.VERSION_1_9) { tasks.withType(Test) { - jvmArgs ([ + jvmArgs([ '--add-opens=java.base/java.lang=ALL-UNNAMED', '--add-opens=java.base/java.io=ALL-UNNAMED', '--add-opens=java.base/java.nio=ALL-UNNAMED', @@ -162,19 +166,21 @@ allprojects { def getRuntimeConfigs() { def names = subprojects - .findAll { prj -> prj.name in ['nextflow','nf-commons','nf-httpfs'] } - .collect { it.name } + .findAll { prj -> prj.name in ['nextflow', 'nf-commons', 'nf-httpfs'] } + .collect { p -> p.name } FileCollection result = null - for( def it : names ) { + for ( def it : names ) { def cfg = project(it).configurations.getByName('runtimeClasspath') - if( result==null ) + if ( result == null ) { result = cfg - else + } + else { result += cfg - // this include the module actual jar file - // note: migrating to gradle 7 does not work any more - //result = result + cfg.getOutgoing().getArtifacts().getFiles() + } + // this include the module actual jar file + // note: migrating to gradle 7 does not work any more + //result = result + cfg.getOutgoing().getArtifacts().getFiles() } return result } @@ -188,9 +194,7 @@ task exportClasspath { def home = System.getProperty('user.home') def all = getRuntimeConfigs() def libs = all.collect { File file -> /*println file.canonicalPath.replace(home, '$HOME');*/ file.canonicalPath; } - ['nextflow','nf-commons','nf-httfs'].each {libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath } + ['nextflow', 'nf-commons', 'nf-httfs'].each { libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath } file('.launch.classpath').text = libs.unique().join(':') } } - - diff --git a/gradlew b/gradlew index 65dcd68d..80c7fc5c 100755 --- a/gradlew +++ b/gradlew @@ -91,11 +91,11 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum -warn () { +warn() { echo "$*" } >&2 -die () { +die() { echo echo "$*" echo diff --git a/launch.sh b/launch.sh index 20ae0fd1..ebee5335 100755 --- a/launch.sh +++ b/launch.sh @@ -1,2 +1,3 @@ +#!/usr/bin/env bash export NXF_PLUGINS_DEV=$PWD/plugins - ../nextflow/launch.sh "$@" +../nextflow/launch.sh "$@" diff --git a/main.nf b/main.nf index 74c2326e..c42c2e7a 100644 --- a/main.nf +++ b/main.nf @@ -1,20 +1,22 @@ #!/usr/bin/env nextflow -nextflow.enable.dsl=2 +/* groovylint-disable CompileStatic */ + +nextflow.enable.dsl = 2 params.src = 'quilt+s3://quilt-example#package=examples/hurdat' params.pub = '/var/tmp' params.out = 'output' -pkg_files = Channel.fromPath(params.src) +packageFiles = Channel.fromPath(params.src) process transfer { publishDir params.pub, mode: 'copy', overwrite: true input: - path x + path x output: - path params.out + '/*' + path params.out + '/*' """ mkdir -p $params.out @@ -24,5 +26,5 @@ process transfer { } workflow { - pkg_files | transfer | view { it } + packageFiles | transfer | view { file -> file } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy index e5eff27f..4ebb75fa 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserver.groovy @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt import nextflow.quilt.jep.QuiltParser @@ -21,21 +20,17 @@ import nextflow.quilt.jep.QuiltPackage import nextflow.quilt.nio.QuiltPath import nextflow.quilt.nio.QuiltPathFactory -import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import java.nio.file.PathMatcher -import java.text.SimpleDateFormat +import java.time.LocalDateTime import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.Global import nextflow.Session import nextflow.trace.TraceObserver - /** * Plugin observer of workflow events * @@ -44,31 +39,42 @@ import nextflow.trace.TraceObserver @Slf4j @CompileStatic class QuiltObserver implements TraceObserver { - private Session session + + private final static String[] BIG_KEYS = [ + 'nextflow', 'commandLine', 'scriptFile', 'projectDir', + 'homeDir', 'workDir', 'launchDir', 'manifest', 'configFiles' + ] + private final Set pkgs = [] as Set private Map config - private Map quilt_config - private Set pkgs = new HashSet<>() + private Map quiltConfig + private Session session + + static void printMap(Map map, String title) { + log.debug "\n\n\n# $title" + map.each { + key, value -> log.debug "\n## ${key}: ${value}" + } + } - public static void writeString(String text, QuiltPackage pkg, String filename) { - String dir = pkg.packageDest().toString() - def path = Paths.get(dir, filename) + static void writeString(String text, QuiltPackage pkg, String filename) { + String dir = pkg.packageDest() + Path path = Paths.get(dir, filename) Files.write(path, text.bytes) } - public static String now(){ - def date = new Date() - def sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss") - return sdf.format(date) + static String now() { + LocalDateTime time = LocalDateTime.now() + return time.toString() } - public static QuiltPath asQuiltPath(Path path){ - if( path instanceof QuiltPath ) { + static QuiltPath asQuiltPath(Path path) { + if (path in QuiltPath) { return (QuiltPath) path } - String strPath = path.getFileName().toString() + String strPath = path.getFileName() if (strPath.contains('#package')) { String url = "${QuiltParser.SCHEME}://${strPath}" - return QuiltPathFactory.Parse(url) + return QuiltPathFactory.parse(url) } return null } @@ -78,7 +84,7 @@ class QuiltObserver implements TraceObserver { log.debug "`onFlowCreate` $this" this.session = session this.config = session.config - this.quilt_config = session.config.navigate('quilt') as Map + this.quiltConfig = session.config.navigate('quilt') as Map this.pkgs } @@ -87,7 +93,7 @@ class QuiltObserver implements TraceObserver { log.debug "onFilePublish.Path[$path].Source[$source]" QuiltPath qPath = asQuiltPath(path) - if( qPath ) { + if (qPath) { QuiltPackage pkg = qPath.pkg() this.pkgs.add(pkg) log.debug "onFilePublish.QuiltPath[$qPath]: pkgs=${pkgs}" @@ -104,7 +110,7 @@ class QuiltObserver implements TraceObserver { } String readme(Map meta, String msg) { -""" + return """ # ${now()} ## $msg @@ -121,14 +127,14 @@ ${meta['workflow']['stats']['processes']} } void publish(QuiltPackage pkg) { - String msg = pkg.toString() + String msg = pkg Map meta = [pkg: msg] - String text = "Stub README" + String text = 'Stub README' String jsonMeta = JsonOutput.toJson(meta) try { meta = getMetadata() msg = "${meta['config']['runName']}: ${meta['workflow']['commandLine']}" - text = readme(meta,msg) + text = readme(meta, msg) //meta.remove('config') jsonMeta = JsonOutput.toJson(meta) } @@ -137,40 +143,30 @@ ${meta['workflow']['stats']['processes']} } writeString(text, pkg, 'README.md') writeString("$meta", pkg, 'quilt_metadata.txt') - def rc = pkg.push(msg,jsonMeta) + def rc = pkg.push(msg, jsonMeta) log.info "$rc: pushed package[$pkg] $msg" } - private static String[] bigKeys = [ - 'nextflow','commandLine','scriptFile','projectDir','homeDir','workDir','launchDir','manifest','configFiles' - ] - - static void printMap(Map map, String title) { - log.debug "\n\n\n# $title" - map.each{ - key, value -> log.debug "\n## ${key}: ${value}"; - } - } - Map getMetadata() { // TODO: Write out config files Map cf = config cf.remove('params') cf.remove('session') cf.remove('executor') - printMap(cf, "config") + printMap(cf, 'config') Map params = session.getParams() params.remove('genomes') - printMap(params, "params") + printMap(params, 'params') Map wf = session.getWorkflowMetadata().toMap() String start = wf['start'] String complete = wf['complete'] - bigKeys.each { k -> wf[k] = "${wf[k]}" } + BIG_KEYS.each { k -> wf[k] = "${wf[k]}" } wf.remove('container') wf.remove('start') wf.remove('complete') - printMap(wf, "workflow") + printMap(wf, 'workflow') log.info "\npublishing: ${wf['runName']}" - [params: params, config: cf, workflow: wf, time_start: start, time_complete: complete] + return [params: params, config: cf, workflow: wf, time_start: start, time_complete: complete] } + } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserverFactory.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserverFactory.groovy index 53227af6..92a672d7 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserverFactory.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltObserverFactory.groovy @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt import groovy.transform.CompileStatic @@ -34,6 +33,7 @@ class QuiltObserverFactory implements TraceObserverFactory { @Override Collection create(Session session) { log.debug "`create` ${this}" - (Collection) [new QuiltObserver()] + return (Collection) [new QuiltObserver()] } + } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltOpts.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltOpts.groovy deleted file mode 100644 index 752da2ac..00000000 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltOpts.groovy +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2022, Quilt Data Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.quilt - -import groovy.json.JsonSlurper -import groovy.transform.CompileStatic -import groovy.transform.Memoized -import groovy.transform.ToString -import groovy.util.logging.Slf4j -import nextflow.Session -import nextflow.exception.AbortOperationException -/** - * Model Quilt config options - * - * @author Ernest Prabhakar - */ -@Slf4j -@ToString(includeNames = true, includePackage = false) -@CompileStatic -class QuiltOpts { - - static Map env = System.getenv() - - private String registry - private String profile - - String getProjectId() { projectId } - - @Memoized - static QuiltOpts fromSession(Session session) { - try { - return fromSession0(session.config) - } - catch (Exception e) { - if(session) session.abort() - throw e - } - } - - protected static QuiltOpts fromSession0(Map config) { - final result = new QuiltOpts() - result.registry = config.navigate("Quilt3.registry") as String - result.profile = config.navigate("Quilt3.profile") as String - return result - } - - static QuiltOpts create(Session session) { - // Typically the credentials picked up are the "AWS Credentials" - // as described at: - // https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html - // - // If not using the default, the profile needs to be set in the nextflow config file. - - return fromSession(session) - } - -} diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltID.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltID.groovy index d1888a72..867b42c0 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltID.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltID.groovy @@ -13,42 +13,45 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.jep +/* groovylint-disable-next-line ImportFromSamePackage */ +import nextflow.quilt.jep.QuiltParser + import groovy.transform.CompileStatic -import groovy.transform.Memoized import groovy.util.logging.Slf4j @Slf4j @CompileStatic class QuiltID { - public static String[] DEFAULT_PACKAGE=["null","default"] - private static final Map ids = [:] + + private static final String[] DEFAULT_PACKAGE = ['null', 'default'] + private static final Map QIDS = [:] private final String bucket private final String pkgPrefix private final String pkgSuffix - static public QuiltID Fetch(String bucket, String pkg_name) { + static QuiltID fetch(String bucket, String packageName) { if (!bucket) { - log.error "null == QuiltID.Fetch($bucket, $pkg_name)" + log.error "null == QuiltID.fetch($bucket, $packageName)" return null } - if (!pkg_name || pkg_name.size() packages = [:] - private static final String INSTALL_PREFIX = "QuiltPackage" - public static final Path INSTALL_ROOT = Files.createTempDirectory(INSTALL_PREFIX) + + private static final Map PKGS = [:] + private static final String INSTALL_PREFIX = 'QuiltPackage' + static final Path INSTALL_ROOT = Files.createTempDirectory(INSTALL_PREFIX) private final String bucket - private final String pkg_name + private final String packageName private final String hash private final Path folder private boolean installed - static public QuiltPackage ForParsed(QuiltParser parsed) { + static String today() { + LocalDate date = LocalDate.now() + return date.toString() + } + + static QuiltPackage forParsed(QuiltParser parsed) { def pkgKey = parsed.toPackageString() - def pkg = packages.get(pkgKey) - if( pkg ) return pkg + def pkg = PKGS.get(pkgKey) + if (pkg) { return pkg } + pkg = new QuiltPackage(parsed) - packages[pkgKey] = pkg + PKGS[pkgKey] = pkg try { - log.debug "Installing `${pkg}` ForParsed(${parsed}) " + log.debug "Installing `${pkg}` forParsed(${parsed}) " pkg.install() } catch (Exception e) { @@ -61,33 +60,31 @@ class QuiltPackage { return pkg } - static protected List listDirectory(Path rootPath) { - Files.walk(rootPath).sorted(Comparator.reverseOrder()).collect(Collectors.toList()) + static List listDirectory(Path rootPath) { + return Files.walk(rootPath).sorted(Comparator.reverseOrder()).collect(Collectors.toList()) } - static protected boolean deleteDirectory(Path rootPath) { - if (!Files.exists(rootPath)) return false + static boolean deleteDirectory(Path rootPath) { + if (!Files.exists(rootPath)) { return false } try { final List pathsToDelete = listDirectory(rootPath) - for(Path path : pathsToDelete) { - Files.deleteIfExists(path); + for (Path path : pathsToDelete) { + Files.deleteIfExists(path) } } - catch (java.nio.file.NoSuchFileException e) { } + catch (java.nio.file.NoSuchFileException e) { + log.debug 'deleteDirectory: ignore non-existent files' + } return true } - static public String today() { - Date dateObj = new Date() - new SimpleDateFormat('yyyy-MM-dd').format(dateObj) - } - QuiltPackage(QuiltParser parsed) { this.installed = false - this.bucket = parsed.bucket() - this.pkg_name = parsed.pkg_name() - this.hash = parsed.hash() + this.bucket = parsed.getBucket() + this.packageName = parsed.getPackageName() + this.hash = parsed.getHash() this.folder = Paths.get(INSTALL_ROOT.toString(), this.toString()) + log.debug "QuiltParser.folder[${this.folder}] ${parsed}" assert this.folder this.setup() } @@ -95,14 +92,14 @@ class QuiltPackage { List relativeChildren(String subpath) { Path subfolder = folder.resolve(subpath) String base = subfolder.toString() + '/' - List result = new ArrayList() + List result = [] final String[] children = subfolder.list().sort() log.debug "relativeChildren[${base}] $children" - for(String pathString : children) { - def relative = pathString.replace(base,'') + for (String pathString : children) { + def relative = pathString.replace(base, '') result.add(relative) } - result + return result } void reset() { @@ -115,80 +112,80 @@ class QuiltPackage { } String key_dest() { - "--dest ${packageDest()}" + return "--dest ${packageDest()}" } String key_dir() { - "--dir ${packageDest()}" + return "--dir ${packageDest()}" } - String key_force() { - "--force true" + return '--force true' } String key_hash() { - "--top-hash $hash" + return "--top-hash $hash" } - String key_meta(String meta="[]") { - "--meta '$meta'" + String key_meta(String meta='[]') { + return "--meta '$meta'" } - String key_msg(prefix="") { - "--message 'nf-quilt:${prefix}@${today()}'" + String key_msg(prefix='') { + return "--message 'nf-quilt:${prefix}@${today()}'" } String key_path() { - "--path=${packageDest()}" + return "--path=${packageDest()}" } String key_registry() { - "--registry s3://${bucket}" + return "--registry s3://${bucket}" } int call(String... args) { def command = ['quilt3'] command.addAll(args) - def cmd = command.join(" ") + def cmd = command.join(' ') log.debug "call `${cmd}`" - ProcessBuilder pb = new ProcessBuilder('bash','-c', cmd) - pb.redirectErrorStream(true); + ProcessBuilder pb = new ProcessBuilder('bash', '-c', cmd) + pb.redirectErrorStream(true) - Process p = pb.start(); - String result = new String(p.getInputStream().readAllBytes()); - int exitCode = p.waitFor(); + Process p = pb.start() + String result = new String(p.getInputStream().readAllBytes()) + int exitCode = p.waitFor() if (exitCode > 0) { log.warn "`call.exitCode` ${exitCode}: ${result}" } - exitCode + return exitCode } - // usage: quilt3 install [-h] [--registry REGISTRY] [--top-hash TOP_HASH] [--dest DEST] [--dest-registry DEST_REGISTRY] [--path PATH] name + // usage: quilt3 install [-h] [--registry REGISTRY] [--top-hash TOP_HASH] + // [--dest DEST] [--dest-registry DEST_REGISTRY] [--path PATH] name Path install() { - if ('latest' == hash || hash == null || hash == "null") { - call('install',pkg_name,key_registry(),key_dest()) + if (hash == 'latest' || hash == null || hash == 'null') { + call('install', packageName, key_registry(), key_dest()) } else { - call('install',pkg_name,key_registry(),key_hash(),key_dest()) + call('install', packageName, key_registry(), key_hash(), key_dest()) } installed = true - packageDest() + return packageDest() } boolean isInstalled() { - installed + return installed } Path packageDest() { - folder + return folder } // https://docs.quiltdata.com/v/version-5.0.x/examples/gitlike#install-a-package - boolean push(String msg = "update", String meta = "[]") { + boolean push(String msg = 'update', String meta = '[]') { log.debug "`push` $this" try { - call('push',pkg_name,key_dir(),key_registry(),key_meta(meta),key_msg(msg)) + call('push', packageName, key_dir(), key_registry(), key_meta(meta), key_msg(msg)) } catch (Exception e) { log.error "Failed `push` ${this}: ${e}" @@ -199,7 +196,7 @@ class QuiltPackage { @Override String toString() { - "${bucket}_${pkg_name}".replaceAll(/[-\/]/,'_') + return "${bucket}_${packageName}".replaceAll(/[-\/]/, '_') } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy index 4af18364..d754b49d 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltParser.groovy @@ -13,131 +13,123 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.jep -import nextflow.quilt.nio.QuiltPath import groovy.transform.CompileStatic -import groovy.transform.Memoized import groovy.util.logging.Slf4j -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - -import java.text.SimpleDateFormat -import java.util.Date -import java.lang.ProcessBuilder @Slf4j @CompileStatic class QuiltParser { - public static final String SCHEME = 'quilt+s3' - public static final String SEP = '/' - public static final String PREFIX = SCHEME+'://' - public static final int MIN_SIZE = 2 - public static final String P_PKG = 'package' - public static final String P_PATH = 'path' + static final String SCHEME = 'quilt+s3' + static final String SEP = '/' + static final String PREFIX = SCHEME + '://' + static final int MIN_SIZE = 2 + + static final String P_PKG = 'package' + static final String P_PATH = 'path' private final String bucket - private final String pkg_name + private final String packageName private String[] paths private String hash private String tag - //private final String catalog private final Map options - static public QuiltParser ForBarePath(String path) { - QuiltParser.ForUriString(PREFIX+path) + static QuiltParser forBarePath(String path) { + return QuiltParser.forUriString(PREFIX + path) } - static public QuiltParser ForUriString(String uri_string) { - URI uri = new URI(uri_string) - QuiltParser.ForURI(uri) + static QuiltParser forUriString(String uriString) { + URI uri = new URI(uriString) + return QuiltParser.forURI(uri) } - static public QuiltParser ForURI(URI uri) { - log.debug("ForURI[${uri.scheme}] for $uri") - if (uri.scheme != SCHEME) - throw new IllegalArgumentException("Scheme[$uri] URI:${uri.scheme}] != SCHEME:${SCHEME}") + static QuiltParser forURI(URI uri) { + log.debug("forURI[${uri.scheme}] for ${uri}") + if (uri.scheme != SCHEME) { + String msg = "Scheme[${uri}] URI:${uri.scheme}] != SCHEME:${SCHEME}" + throw new IllegalArgumentException(msg) + } def options = parseQuery(uri.fragment) String pkg = options.get(P_PKG) String path = options.get(P_PATH) - new QuiltParser(uri.authority, pkg, path, options) + return new QuiltParser(uri.authority, pkg, path, options) } - static private Map parseQuery(String query) { - if (!query) return [:] // skip for urls without query params - final queryParams = query.split('&') - queryParams.collectEntries { param -> param.split('=').collect { URLDecoder.decode(it) }} + static Map parseQuery(String query) { + if (!query) { return [:] } // skip for urls without query params + final queryParams = query.split('&') + return queryParams.collectEntries { params -> params.split('=').collect { param -> URLDecoder.decode(param) } } } QuiltParser(String bucket, String pkg, String path, Map options = [:]) { this.bucket = bucket this.paths = path ? path.split(SEP) : [] as String[] - this.pkg_name = parsePkg(pkg) + this.packageName = parsePkg(pkg) this.options = options } String parsePkg(String pkg) { - if (! pkg) return null - if (! pkg.contains('/')) { + if (!pkg) { return null } + if (!pkg.contains('/')) { log.error("Invalid package[$pkg]") } if (pkg.contains('@')) { def split = pkg.split('@') - this.hash = split[1] + hash = split[1] return split[0] } if (pkg.contains(':')) { def split = pkg.split(':') - this.tag = split[1] + tag = split[1] return split[0] } + String new_pkg = pkg String[] split = pkg.split(SEP) if (split.size() > MIN_SIZE) { String[] head = split[0..1] String[] tail = split[2..-1] - pkg = head.join(SEP) - this.paths += tail + new_pkg = head.join(SEP) + paths += tail } - pkg + return new_pkg } QuiltParser appendPath(String tail) { - String path2 = [path(),tail].join(SEP) + String path2 = [getPath(), tail].join(SEP) while (path2.startsWith(SEP)) { path2 = path2.substring(1) } - new QuiltParser(bucket(), pkg_name(), path2, options) + return new QuiltParser(getBucket(), getPackageName(), path2, options) } QuiltParser dropPath() { - String[] subpath = ((paths.size() > 1) ? paths[0..-2] : []) as String[] + String[] subpath = ((paths.size() > 1) ? paths[0..-2] : []) as String[] String path2 = subpath.join(SEP) - new QuiltParser(bucket(), pkg_name(), path2, options) + return new QuiltParser(getBucket(), getPackageName(), path2, options) } QuiltParser lastPath() { String path2 = paths.size() > 0 ? paths[-1] : '' - new QuiltParser(bucket(), pkg_name(), path2, options) + return new QuiltParser(getBucket(), getPackageName(), path2, options) } - QuiltParser subPath(int beginIndex, int endIndex) { String path2 = path(beginIndex, endIndex) - new QuiltParser(bucket(), pkg_name(), path2, options) + return new QuiltParser(getBucket(), getPackageName(), path2, options) } QuiltParser normalized() { boolean skip = false String[] rnorms = paths.reverse().findAll { String x -> - if (x == "..") { + if (x == '..') { skip = true false - } else if(skip) { + } else if (skip) { skip = false - false + false } else { true } @@ -146,75 +138,75 @@ class QuiltParser { log.debug("normalized: ${paths} -> ${rnorms}") String path2 = rnorms.reverse().join(SEP) log.debug("normalized: -> ${path2}") - new QuiltParser(bucket(), pkg_name(), path2, options) + return new QuiltParser(getBucket(), getPackageName(), path2, options) } QuiltID quiltID() { - QuiltID.Fetch(bucket(), pkg_name()) + return QuiltID.fetch(getBucket(), getPackageName()) } String quiltIDS() { - quiltID().toString() + return quiltID().toString() } - String bucket() { - bucket?.toLowerCase() + String getBucket() { + return bucket?.toLowerCase() } - String pkg_name() { - pkg_name + String getPackageName() { + return packageName } - String hash() { - hash + String getHash() { + return hash } - String tag() { - tag + String getTag() { + return tag } - String path() { - paths.join(SEP) + String getPath() { + return paths.join(SEP) } String path(int beginIndex, int endIndex) { String[] sub = paths[beginIndex.. 0 + return paths.size() > 0 } - String options(String key) { - options?.get(key) + String getOptions(String key) { + return options?.get(key) } String toPackageString() { - String str = "${bucket()}" - if ( pkg_name ) { - String pkg = pkg_name - if ( hash ) { pkg += "@$hash" } - if ( tag ) { pkg += ":$tag" } - str += "#package=${pkg.replace('/','%2f')}" + String str = "${getBucket()}" + if (packageName) { + String pkg = packageName + if (hash) { pkg += "@$hash" } + if (tag) { pkg += ":$tag" } + str += "#package=${pkg.replace('/', '%2f')}" } - str + return str } String toString() { String str = toPackageString() - if (! hasPath() ) return str - str += ( pkg_name ) ? "&" : "#" - str += "path=${path().replace('/','%2f')}" + if (!hasPath()) { return str } + str += (packageName) ? '&' : '#' + str += "path=${getPath().replace('/', '%2f')}" + return str } String toUriString() { - PREFIX + toString() + return PREFIX + toString() } - } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltBashLib.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltBashLib.groovy index 72027c69..47e5fd3b 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltBashLib.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltBashLib.groovy @@ -13,21 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio -import nextflow.Global -import nextflow.Session import nextflow.executor.BashFunLib +import groovy.transform.CompileStatic /** * Quilt helper class */ +@CompileStatic class QuiltBashLib extends BashFunLib { - private String storageClass = 'STANDARD' - static String script() { - "quilt3 install" + return 'quilt3 install' } + } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileAttributes.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileAttributes.groovy index bcdcb605..c6e4ee93 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileAttributes.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileAttributes.groovy @@ -13,12 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio -import static java.lang.String.format; -import java.nio.file.Path -import java.nio.file.attribute.BasicFileAttributeView import java.nio.file.attribute.BasicFileAttributes import java.nio.file.attribute.FileTime import groovy.transform.CompileStatic @@ -33,65 +29,69 @@ import groovy.util.logging.Slf4j @Slf4j @CompileStatic class QuiltFileAttributes implements BasicFileAttributes { - private final QuiltPath path; - private final String key; - private final BasicFileAttributes attrs; - public QuiltFileAttributes(QuiltPath path, String key, BasicFileAttributes attrs) { + private final QuiltPath path + private final String key + private final String origKey + private final BasicFileAttributes attrs + + QuiltFileAttributes(QuiltPath path, String key, BasicFileAttributes attrs) { this.path = path - this.key = path.isJustPackage() ? "/" : path.file_key() - this.attrs = attrs + this.key = path.isJustPackage() ? '/' : path.file_key() + this.origKey = key + this.attrs = attrs log.debug "QuiltFileAttributes($path): this=$this" - } - - @Override - public FileTime lastModifiedTime() { - return attrs.lastModifiedTime(); - } - - @Override - public FileTime lastAccessTime() { - return attrs.lastAccessTime(); - } - - @Override - public FileTime creationTime() { - return attrs.creationTime(); - } - - @Override - public boolean isRegularFile() { - return attrs.isRegularFile(); - } - - @Override - public boolean isDirectory() { - return attrs.isDirectory(); - } - - @Override - public boolean isSymbolicLink() { - return attrs.isSymbolicLink(); - } - - @Override - public boolean isOther() { - return attrs.isOther(); - } - - @Override - public long size() { - return attrs.size(); - } - - @Override - public Object fileKey() { + } + + @Override + FileTime lastModifiedTime() { + return attrs.lastModifiedTime() + } + + @Override + FileTime lastAccessTime() { + return attrs.lastAccessTime() + } + + @Override + FileTime creationTime() { + return attrs.creationTime() + } + + @Override + boolean isRegularFile() { + return attrs.isRegularFile() + } + + @Override + boolean isDirectory() { + return attrs.isDirectory() + } + + @Override + boolean isSymbolicLink() { + return attrs.isSymbolicLink() + } + + @Override + boolean isOther() { + return attrs.isOther() + } + + @Override + long size() { + return attrs.size() + } + + @Override + Object fileKey() { log.debug "QuiltFileAttributes.fileKey: $key" - return key; - } + return key + } - @Override - public String toString() { - return "[${key}: lastModified=${lastModifiedTime()}, size=${size()}, isDirectory=${isDirectory()}]" + @Override + String toString() { + return "[${key}: lastModified=${lastModifiedTime()}, size=${size()}, isDirectory=${isDirectory()}]" } - } + +} diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileCopyStrategy.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileCopyStrategy.groovy index 33009c7e..fdb08bef 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileCopyStrategy.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileCopyStrategy.groovy @@ -13,9 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - // https://github.com/LukeGoodsell/nextflow/blob/master/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy - package nextflow.quilt.nio import java.nio.file.Path @@ -43,6 +41,7 @@ class QuiltFileCopyStrategy extends SimpleFileCopyStrategy { } static String uploadCmd(String source, Path target) { - "quilt3 push ${Escape.path(source)} ${Escape.uriPath(target)}" + return "quilt3 push ${Escape.path(source)} ${Escape.uriPath(target)}" } + } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy index 1313a01c..62dfb9ea 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy @@ -13,11 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio -import java.nio.channels.Channels -import java.nio.channels.SeekableByteChannel import java.nio.file.Files import java.nio.file.FileSystem import java.nio.file.FileStore @@ -28,15 +25,11 @@ import java.nio.file.attribute.UserPrincipalLookupService import java.nio.file.spi.FileSystemProvider import java.nio.file.NoSuchFileException import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.attribute.FileTime -import java.util.regex.Matcher import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.Global -import nextflow.Session -import nextflow.quilt.QuiltOpts import nextflow.quilt.jep.QuiltParser + /** * Implements FileSystem interface for Quilt registries * Each bucket/package pair (QuiltID) is a FileSystem @@ -45,44 +38,44 @@ import nextflow.quilt.jep.QuiltParser * @author Ernest Prabhakar */ - -// cf. https://cloud.google.com/java/docs/reference/google-cloud-nio/latest/com.google.cloud.storage.contrib.nio.CloudStorageFileSystem +// cf. https://cloud.google.com/java/docs/reference/google-cloud-nio/latest/ +// com.google.cloud.storage.contrib.nio.CloudStorageFileSystem // https://github.com/nextflow-io/nextflow-s3fs/tree/master/src/main/java/com/upplication/s3fs @Slf4j @CompileStatic -public final class QuiltFileSystem extends FileSystem { +final class QuiltFileSystem extends FileSystem implements Closeable { - protected final String quiltIDS; - protected final QuiltFileSystemProvider provider + private final String quiltIDS + private final QuiltFileSystemProvider myProvider - public QuiltFileSystem(String quiltIDS, QuiltFileSystemProvider provider) { - this.quiltIDS = quiltIDS; - this.provider = provider; + QuiltFileSystem(String quiltIDS, QuiltFileSystemProvider provider) { + this.quiltIDS = quiltIDS + this.myProvider = provider } @Override String toString() { - quiltIDS + return quiltIDS } void copy(QuiltPath source, QuiltPath target) { - throw new UnsupportedOperationException("NOT Implemented 'QuiltFileSystem.copy' `$source` -> `$target`") + throw new UnsupportedOperationException("NOT Implemented 'QuiltFileSystem.copy' `$source` -> `$target`") } void delete(QuiltPath path) { log.debug "QuiltFileSystem.delete: $path" path.deinstall() - //throw new UnsupportedOperationException("Operation 'delete' is not supported by QuiltFileSystem") + //throw new UnsupportedOperationException("Operation 'delete' is not supported by QuiltFileSystem") } @Override FileSystemProvider provider() { - return provider + return myProvider } @Override void close() throws IOException { - // nothing to do + // nothing to do } @Override @@ -102,7 +95,7 @@ public final class QuiltFileSystem extends FileSystem { QuiltFileAttributesView getFileAttributeView(QuiltPath path) { log.debug "QuiltFileAttributesView QuiltFileSystem.getFileAttributeView($path)" - def pathString = path.toUriString() + String pathString = path.toUriString() try { QuiltFileAttributes attrs = readAttributes(path) return new QuiltFileAttributesView(attrs) @@ -117,7 +110,7 @@ public final class QuiltFileSystem extends FileSystem { Path installedPath = path.localPath() try { BasicFileAttributes attrs = Files.readAttributes(installedPath, BasicFileAttributes) - return new QuiltFileAttributes(path,path.toString(),attrs) + return new QuiltFileAttributes(path, path.toString(), attrs) } catch (NoSuchFileException e) { log.debug "No attributes yet for: ${installedPath}" @@ -141,29 +134,27 @@ public final class QuiltFileSystem extends FileSystem { @Override Set supportedFileAttributeViews() { log.debug "Calling `supportedFileAttributeViews`: ${this}" - return Collections.unmodifiableSet( ['basic'] as Set ) + return Collections.unmodifiableSet(['basic'] as Set) } @Override QuiltPath getPath(String root, String... more) { log.debug "QuiltFileSystem.getPath`[${root}]: $more" - QuiltParser p = QuiltParser.ForBarePath(root) - new QuiltPath(this, p) + QuiltParser p = QuiltParser.forBarePath(root) + return new QuiltPath(this, p) } - protected String toUriString(Path path) { - return path instanceof QuiltPath ? ((QuiltPath)path).toUriString() : null + String toUriString(Path path) { + return path in QuiltPath ? ((QuiltPath)path).toUriString() : null } - protected String getBashLib(Path path) { - throw new UnsupportedOperationException("Operation 'getBashLib' is not supported by QuiltFileSystem") - return path instanceof QuiltPath ? QuiltBashLib.script() : null + String getBashLib(Path path) { + return path in QuiltPath ? QuiltBashLib.script() : null } - protected String getUploadCmd(String source, Path target) { - throw new UnsupportedOperationException("Operation 'getUploadCmd' is not supported by QuiltFileSystem") - return target instanceof QuiltPath ? QuiltFileCopyStrategy.uploadCmd(source, target) : null + String getUploadCmd(String source, Path target) { + return target in QuiltPath ? QuiltFileCopyStrategy.uploadCmd(source, target) : null } @Override @@ -173,7 +164,9 @@ public final class QuiltFileSystem extends FileSystem { @Override UserPrincipalLookupService getUserPrincipalLookupService() { - throw new UnsupportedOperationException("Operation 'getUserPrincipalLookupService' is not supported by QuiltFileSystem") + throw new UnsupportedOperationException( + "Operation 'getUserPrincipalLookupService' is not supported by QuiltFileSystem" + ) } @Override diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index 98bad25a..9b08abda 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -13,40 +13,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio -import static java.nio.file.StandardCopyOption.* -import static java.nio.file.StandardOpenOption.* +import static java.nio.file.StandardOpenOption.APPEND +import static java.nio.file.StandardOpenOption.WRITE -import java.nio.channels.FileChannel; -import java.nio.channels.SeekableByteChannel; +import java.nio.channels.FileChannel +import java.nio.channels.SeekableByteChannel import java.nio.file.AccessDeniedException import java.nio.file.AccessMode import java.nio.file.CopyOption import java.nio.file.DirectoryStream -import java.nio.file.FileAlreadyExistsException import java.nio.file.Files import java.nio.file.FileStore import java.nio.file.FileSystem -import java.nio.file.FileSystemAlreadyExistsException import java.nio.file.FileSystemNotFoundException import java.nio.file.LinkOption import java.nio.file.NoSuchFileException -import java.nio.file.NotDirectoryException; -import java.nio.file.OpenOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; +import java.nio.file.OpenOption +import java.nio.file.Path +import java.nio.file.StandardOpenOption import java.nio.file.attribute.BasicFileAttributeView import java.nio.file.attribute.BasicFileAttributes import java.nio.file.attribute.FileAttribute import java.nio.file.attribute.FileAttributeView -import java.nio.file.attribute.FileTime import java.nio.file.spi.FileSystemProvider import groovy.transform.CompileStatic -import groovy.transform.Memoized import groovy.util.logging.Slf4j import nextflow.Global import nextflow.Session @@ -58,40 +51,49 @@ import nextflow.quilt.jep.QuiltParser * @author Ernest Prabhakar */ - @Slf4j @CompileStatic class QuiltFileSystemProvider extends FileSystemProvider { - private final Map env = new HashMap<>(System.getenv()) + private final Map myEnv = new HashMap<>(System.getenv()) private final Map fileSystems = [:] private Map attributesCache = [:] - /** - * @inheritDoc - */ - @Override - String getScheme() { - return QuiltParser.SCHEME - } - - static private QuiltPath asQuiltPath(Path path ) { - if( path !instanceof QuiltPath ) - throw new IllegalArgumentException("Not a valid Quilt blob storage path object: `$path` [${path?.class?.name?:'-'}]" ) + static QuiltPath asQuiltPath(Path path) { + if (path !in QuiltPath) { + String pathClassName = path?.class?.name ?: '-' + throw new IllegalArgumentException( + "Not a valid Quilt blob storage path object: `${path}` [${pathClassName}]" + ) + } return (QuiltPath)path } - static private QuiltFileSystem getQuiltFilesystem(Path path ) { + static QuiltFileSystem getQuiltFilesystem(Path path) { final qPath = asQuiltPath(path) final fs = qPath.getFileSystem() - if( fs !instanceof QuiltFileSystem ) - throw new IllegalArgumentException("Not a valid Quilt file system: `$fs` [${path?.class?.name?:'-'}]" ) + if (fs !in QuiltFileSystem) { + String pathClassName = path?.class?.name ?: '-' + throw new IllegalArgumentException("Not a valid Quilt file system: `$fs` [${pathClassName}]") + } return (QuiltFileSystem)fs } - protected String getQuiltIDS(URI uri) { + static FileSystemProvider provider(Path path) { + return path.getFileSystem().provider() + } + + /** + * @inheritDoc + */ + @Override + String getScheme() { + return QuiltParser.SCHEME + } + + String getQuiltIDS(URI uri) { assert uri - QuiltParser parsed = QuiltParser.ForURI(uri) + QuiltParser parsed = QuiltParser.forURI(uri) return parsed.quiltID().toString() } @@ -136,13 +138,14 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override QuiltFileSystem newFileSystem(URI uri, Map config) throws IOException { final quiltIDS = getQuiltIDS(uri) - newFileSystem(quiltIDS,config) + return newFileSystem(quiltIDS, config) } + /* groovylint-disable-next-line UnusedMethodParameter */ QuiltFileSystem newFileSystem(String quiltIDS, Map env) throws IOException { final fs = new QuiltFileSystem(quiltIDS, this) fileSystems[quiltIDS] = fs - fs + return fs } /** @@ -182,17 +185,16 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override FileSystem getFileSystem(URI uri) { final quiltIDS = getQuiltIDS(uri) - getFileSystem0(quiltIDS,false) + return getFileSystem0(quiltIDS, false) } - protected QuiltFileSystem getFileSystem0(String quiltIDS, boolean canCreate) { - - def fs = fileSystems.get(quiltIDS) - if( fs ) return fs - if( canCreate ) - return newFileSystem(quiltIDS, env) - else - throw new FileSystemNotFoundException("Missing Quilt file system for quiltIDS: `$quiltIDS`") + QuiltFileSystem getFileSystem0(String quiltIDS, boolean canCreate) { + QuiltFileSystem fs = fileSystems.get(quiltIDS) + if (fs) { return fs } + if (canCreate) { + return newFileSystem(quiltIDS, myEnv) + } + throw new FileSystemNotFoundException("Missing Quilt file system for quiltIDS: `$quiltIDS`") } /** @@ -223,23 +225,18 @@ class QuiltFileSystemProvider extends FileSystemProvider { */ @Override QuiltPath getPath(URI uri) { - QuiltParser parsed = QuiltParser.ForURI(uri) + QuiltParser parsed = QuiltParser.forURI(uri) log.debug "QuiltFileSystemProvider.getPath`[${uri}]: parsed=$parsed" - final fs = getFileSystem0(parsed.quiltIDS(),true) - new QuiltPath(fs, parsed) - } - - - static private FileSystemProvider provider( Path path ) { - path.getFileSystem().provider() + final fs = getFileSystem0(parsed.quiltIDS(), true) + return new QuiltPath(fs, parsed) } - private void checkRoot(Path path) { - if( path.toString() == '/' ) + void checkRoot(Path path) { + if (path.toString() == '/') { throw new UnsupportedOperationException("Operation 'checkRoot' not supported on root path") + } } - /** * Open a file for reading or writing. @@ -249,20 +246,21 @@ class QuiltFileSystemProvider extends FileSystemProvider { * @return * @throws IOException */ - protected void notifyFilePublish(QuiltPath destination, Path source=null) { + void notifyFilePublish(QuiltPath destination, Path source=null) { final sess = Global.session + /* groovylint-disable-next-line Instanceof */ if (sess instanceof Session) { sess.notifyFilePublish((Path)destination, source) } } @Override - public SeekableByteChannel newByteChannel( + SeekableByteChannel newByteChannel( Path path, Set options, FileAttribute... attrs) throws IOException { log.debug "Creating `newByteChannel`: ${path} <- ${options}" final modeWrite = options.contains(WRITE) || options.contains(APPEND) - final qPath = asQuiltPath(path) + final QuiltPath qPath = asQuiltPath(path) Path installedPath = qPath.localPath() Path parent = installedPath.getParent() Files.createDirectories(parent) @@ -274,22 +272,25 @@ class QuiltFileSystemProvider extends FileSystemProvider { notifyFilePublish(qPath) } log.debug "\tOpening channel to: $installedPath" - def channel = FileChannel.open(installedPath, options) + FileChannel channel = FileChannel.open(installedPath, options) return channel } catch (java.nio.file.NoSuchFileException e) { log.error "Failed `FileChannel.open`: ${installedPath} <- ${options}" } - } + } DirectoryStream emptyStream() throws IOException { return new DirectoryStream() { + @Override Iterator iterator() { return Collections.emptyIterator() } + /* groovylint-disable-next-line CloseWithoutCloseable */ @Override void close() throws IOException { } + } } @@ -302,25 +303,26 @@ class QuiltFileSystemProvider extends FileSystemProvider { log.debug "QuiltFileSystemProvider.newDirectoryStream[${qPath.file_key()}]: ${qPath}" return new DirectoryStream() { - @Override - public void close() throws IOException { - // nothing to do here - } @Override - public Iterator iterator() { + Iterator iterator() { return new QuiltPathIterator(qPath, filter) } - }; + /* groovylint-disable-next-line CloseWithoutCloseable */ + @Override void close() throws IOException { } + + } } + /* groovylint-disable BuilderMethodWithSideEffects */ @Override void createDirectory(Path dir, FileAttribute... attrs) throws IOException { final path = asQuiltPath(dir).localPath() log.debug "Calling createDirectory[${path}]: ${dir} " Files.createDirectories(path) } + /* groovylint-enable BuilderMethodWithSideEffects */ @Override void delete(Path obj) throws IOException { @@ -334,8 +336,9 @@ class QuiltFileSystemProvider extends FileSystemProvider { void copy(Path from, Path to, CopyOption... options) throws IOException { log.debug "Attempting `copy`: ${from} -> ${to}" assert provider(from) == provider(to) - if( from == to ) + if (from == to) { return // nothing to do -- just return + } checkRoot(from); checkRoot(to) final source = asQuiltPath(from) @@ -346,7 +349,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override void move(Path source, Path target, CopyOption... options) throws IOException { - copy(source,target,options) + copy(source, target, options) delete(source) } @@ -369,18 +372,19 @@ class QuiltFileSystemProvider extends FileSystemProvider { void checkAccess(Path path, AccessMode... modes) throws IOException { log.debug "Calling `checkAccess`: ${path}" checkRoot(path) - final qPath = asQuiltPath(path) - readAttributes(qPath, QuiltFileAttributes.class) - if( AccessMode.EXECUTE in modes) + QuiltPath qPath = asQuiltPath(path) + readAttributes(qPath, QuiltFileAttributes) + if (AccessMode.EXECUTE in modes) { throw new AccessDeniedException(qPath.toUriString(), null, 'Execute permission not allowed') + } } @Override def V getFileAttributeView(Path path, Class type, LinkOption... options) { log.debug "Calling `getFileAttributeView`: ${path}" checkRoot(path) - if( type == BasicFileAttributeView || type == QuiltFileAttributesView ) { - def qPath = asQuiltPath(path) + if (type == BasicFileAttributeView || type == QuiltFileAttributesView) { + QuiltPath qPath = asQuiltPath(path) QuiltFileSystem fs = qPath.filesystem return (V)fs.getFileAttributeView(qPath) } @@ -388,17 +392,18 @@ class QuiltFileSystemProvider extends FileSystemProvider { } @Override - def A readAttributes(Path path, Class type, LinkOption... options) throws IOException { + def A readAttributes(Path path, Class type, LinkOption... options) + throws IOException { log.debug "BasicFileAttributes QuiltFileSystemProvider.readAttributes($path)" def attr = attributesCache.get(path) - if ( attr ) { + if (attr) { return attr } - if( type == BasicFileAttributes || type == QuiltFileAttributes ) { - def qPath = asQuiltPath(path) + if (type == BasicFileAttributes || type == QuiltFileAttributes) { + QuiltPath qPath = asQuiltPath(path) QuiltFileSystem fs = qPath.filesystem def result = (A)fs.readAttributes(qPath) - if( result ) { + if (result) { attributesCache[path] = result return result } @@ -406,7 +411,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { throw new NoSuchFileException(qPath.toUriString()) } throw new UnsupportedOperationException("Not a valid Quilt Storage file attribute type: $type") - } + } @Override Map readAttributes(Path path, String attributes, LinkOption... options) throws IOException { diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy index 045adbd0..7aa83555 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio import nextflow.quilt.jep.QuiltPackage @@ -30,10 +29,6 @@ import java.nio.file.WatchService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.Global -import nextflow.Session -import nextflow.quilt.QuiltOpts -import nextflow.quilt.jep.QuiltParser /** * Implements Path interface for Quilt storage @@ -43,45 +38,46 @@ import nextflow.quilt.jep.QuiltParser @Slf4j @CompileStatic -public final class QuiltPath implements Path { +final class QuiltPath implements Path, Comparable { + private final QuiltFileSystem filesystem private final QuiltParser parsed private final String[] paths - public QuiltPath(QuiltFileSystem filesystem, QuiltParser parsed) { + QuiltPath(QuiltFileSystem filesystem, QuiltParser parsed) { this.filesystem = filesystem this.parsed = parsed - this.paths = parsed.paths() + this.paths = parsed.getPaths() log.debug "Creating QuiltPath[$parsed]@$filesystem" } - public String bucket() { - parsed.bucket() + String getBucket() { + return parsed.getBucket() } - public String pkg_name() { - parsed.pkg_name() + String getPackageName() { + return parsed.getPackageName() } - public String sub_paths() { - parsed.path() + String sub_paths() { + return parsed.getPath() } - public QuiltPackage pkg() { - isAbsolute() ? QuiltPackage.ForParsed(parsed) : null + QuiltPackage pkg() { + return isAbsolute() ? QuiltPackage.forParsed(parsed) : null } - public String file_key() { - sub_paths() + String file_key() { + return sub_paths() } Path localPath() { Path pkgPath = pkg().packageDest() assert pkgPath - Paths.get(pkgPath.toUriString(), sub_paths()) + return Paths.get(pkgPath.toUriString(), sub_paths()) } - public boolean deinstall() { + boolean deinstall() { Path path = localPath() log.debug "QuiltPath.deinstall: $path" return Files.delete(path) @@ -94,72 +90,72 @@ public final class QuiltPath implements Path { @Override boolean isAbsolute() { - log.debug "isAbsolute[${pkg_name()}] : ${parsed}" - filesystem && pkg_name() + log.debug "isAbsolute[${getPackageName()}] : ${parsed}" + return filesystem && getPackageName() } boolean isJustPackage() { - !parsed.hasPath() + return !parsed.hasPath() } QuiltPath getJustPackage() { - if ( isJustPackage() ) return this - QuiltParser pkg_parsed = QuiltParser.ForBarePath(parsed.toPackageString()) - new QuiltPath(filesystem, pkg_parsed) + if (isJustPackage()) { return this } + QuiltParser packageParsed = QuiltParser.forBarePath(parsed.toPackageString()) + return new QuiltPath(filesystem, packageParsed) } @Override Path getRoot() { - isAbsolute() ? getJustPackage() : null + return isAbsolute() ? getJustPackage() : null } @Override Path getFileName() { log.debug "getFileName`[${this}]: paths=$paths" - isJustPackage() ? this : new QuiltPath(filesystem, parsed.lastPath()) // IF DIRECTORY + return isJustPackage() ? this : new QuiltPath(filesystem, parsed.lastPath()) // IF DIRECTORY } @Override Path getParent() { log.debug "${this}.getParent: ${paths}`" - new QuiltPath(filesystem, parsed.dropPath()) + return new QuiltPath(filesystem, parsed.dropPath()) } @Override int getNameCount() { - paths.size() + return paths.size() } @Override Path getName(int index) { throw new UnsupportedOperationException("Operation 'getName' is not supported by QuiltPath") - //new QuiltPath(filesystem, pkg_name, sub_paths[0,index], options) + //new QuiltPath(filesystem, packageName, sub_paths[0,index], options) } @Override Path subpath(int beginIndex, int endIndex) { - QuiltParser p2 = parsed.subPath(beginIndex,endIndex) - new QuiltPath(filesystem, p2) + QuiltParser p2 = parsed.subPath(beginIndex, endIndex) + return new QuiltPath(filesystem, p2) } @Override boolean startsWith(Path other) { - startsWith(other.toString()) + return startsWith(other.toString()) } @Override boolean startsWith(String other) { - toString().startsWith(other) + return toString().startsWith(other) } @Override boolean endsWith(Path other) { - endsWith(other.toString()) + return endsWith(other.toString()) } @Override boolean endsWith(String other) { - toString().endsWith(other) + return toString().endsWith(other) } @Override @@ -169,24 +165,26 @@ public final class QuiltPath implements Path { @Override Path normalize() { - new QuiltPath(filesystem, parsed.normalized()) + return new QuiltPath(filesystem, parsed.normalized()) } @Override QuiltPath resolve(Path other) { - if( other.class != QuiltPath ) + if (other.class != QuiltPath) { throw new ProviderMismatchException() + } final that = (QuiltPath)other - if( other.isAbsolute() ) + if (other.isAbsolute()) { return that + } throw new UnsupportedOperationException("Operation 'resolve'[$that] is not supported by QuiltPath[$this]") - //new QuiltPath(filesystem, pkg_name, other.toString(), options) + //new QuiltPath(filesystem, packageName, other.toString(), options) } @Override QuiltPath resolve(String other) { - new QuiltPath(filesystem, parsed.appendPath(other)) + return new QuiltPath(filesystem, parsed.appendPath(other)) } @Override @@ -196,48 +194,52 @@ public final class QuiltPath implements Path { @Override Path resolveSibling(String other) { - new QuiltPath(filesystem, parsed.dropPath().appendPath(other)) + return new QuiltPath(filesystem, parsed.dropPath().appendPath(other)) } @Override Path relativize(Path other) { - if (this == other) return null - String file = (other instanceof QuiltPath) ? ((QuiltPath)other).localPath() : other.toString() - String base = [pkg().toString(),parsed.path()].join(QuiltParser.SEP) + if (this == other) { return null } + String file = (other in QuiltPath) ? ((QuiltPath)other).localPath() : other.toString() + String base = [pkg().toString(), parsed.getPath()].join(QuiltParser.SEP) log.debug "relativize[$base] in [$file]" int i = file.indexOf(base) - if (i<1) { + if (i < 1) { throw new UnsupportedOperationException("other[$file] does not contain package[$base]") } String tail = file.substring(i + base.size()) - if (tail.size() > 0 && tail[0] == '/') tail = tail.substring(1) // drop leading "/" + if (tail.size() > 0 && tail[0] == '/') { tail = tail.substring(1) } // drop leading "/" log.debug "tail[$i] -> $tail" - Paths.get(tail) + return Paths.get(tail) } @Override String toString() { - parsed.toString() + return parsed.toString() } String toUriString() { - parsed.toUriString() + return parsed.toUriString() } @Override URI toUri() { - new URI(toUriString()) + return new URI(toUriString()) } @Override - int hashCode() { - toString().hashCode() - } + int hashCode() { + return toString().hashCode() + } + + boolean equals(Object other) { + return hashCode() == other.hashCode() + } @Override Path toAbsolutePath() { - if(isAbsolute()) return this + if (isAbsolute()) { return this } throw new UnsupportedOperationException("Operation 'toAbsolutePath' is not supported by QuiltPath") } @@ -252,9 +254,10 @@ public final class QuiltPath implements Path { } @Override - WatchKey register(WatchService watcher, WatchEvent.Kind[] events, WatchEvent.Modifier... modifiers) throws IOException { + WatchKey register(WatchService watcher, WatchEvent.Kind[] events, + WatchEvent.Modifier... modifiers) throws IOException { throw new UnsupportedOperationException("Operation 'register' is not supported by QuiltPath") - } + } @Override WatchKey register(WatchService watcher, WatchEvent.Kind... events) throws IOException { @@ -263,7 +266,7 @@ public final class QuiltPath implements Path { @Override Iterator iterator() { - throw new UnsupportedOperationException("Operation 'iterator' is not supported by QuiltPath") + throw new UnsupportedOperationException("Operation 'iterator' is not supported by QuiltPath") } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathFactory.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathFactory.groovy index 2ac746ec..fce62e85 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathFactory.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathFactory.groovy @@ -13,18 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio import java.nio.file.Path import groovy.transform.CompileStatic -import nextflow.Global -import nextflow.Session -import nextflow.quilt.QuiltOpts import nextflow.quilt.jep.QuiltParser import nextflow.file.FileSystemPathFactory import nextflow.file.FileHelper + /** * Implements FileSystemPathFactory interface for Google storage * @@ -33,33 +30,34 @@ import nextflow.file.FileHelper @CompileStatic class QuiltPathFactory extends FileSystemPathFactory { - static public QuiltPath Parse(String path) { + static QuiltPath parse(String path) { QuiltPathFactory factory = new QuiltPathFactory() - (QuiltPath) factory.parseUri(path) + return (QuiltPath) factory.parseUri(path) } @Override - protected Path parseUri(String uri_string) { - if( !uri_string.startsWith(QuiltParser.PREFIX) ) + protected Path parseUri(String uriString) { + if (!uriString.startsWith(QuiltParser.PREFIX)) { return null - final uri = new URI(uri_string) + } + final uri = new URI(uriString) return FileHelper.getOrCreateFileSystemFor(uri).provider().getPath(uri) } @Override protected String toUriString(Path p) { - if( p instanceof QuiltPath ) { - return p.toUriString() - } - return null + if (p in QuiltPath) { + return p.toUriString() + } + return null } protected String getBashLib(Path path) { - return path instanceof QuiltPath ? QuiltBashLib.script() : null + return path in QuiltPath ? QuiltBashLib.script() : null } protected String getUploadCmd(String source, Path target) { - return target instanceof QuiltPath ? QuiltFileCopyStrategy.uploadCmd(source, target) : null + return target in QuiltPath ? QuiltFileCopyStrategy.uploadCmd(source, target) : null } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathIterator.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathIterator.groovy index a91397d7..3f04eda8 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathIterator.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPathIterator.groovy @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio import java.nio.file.DirectoryStream @@ -30,17 +29,13 @@ import groovy.util.logging.Slf4j @Slf4j @CompileStatic -public class QuiltPathIterator implements Iterator { - - private Iterator itr - - private DirectoryStream.Filter filter - - private String[] children +class QuiltPathIterator implements Iterator { - private QuiltPath next - - private QuiltPath dir + private final Iterator itr + private final DirectoryStream.Filter filter + private final String[] children + private final QuiltPath dir + private QuiltPath nextPath QuiltPathIterator(QuiltPath dir, DirectoryStream.Filter filter) { this.dir = dir @@ -50,33 +45,17 @@ public class QuiltPathIterator implements Iterator { advance() } - private void advance() { - - QuiltPath result = null - while( result == null && itr.hasNext() ) { - def item = itr.next() - def path = dir.resolve(item) - if( path == dir) // make sure to skip the original path - result = null - else if( filter ) - result = filter.accept(path) ? path : null - else - result = path - } - - next = result - } - @Override boolean hasNext() { - return next != null + return nextPath != null } @Override Path next() { - def result = next - if( result == null ) + def result = nextPath + if (result == null) { throw new NoSuchElementException() + } advance() return result } @@ -85,4 +64,24 @@ public class QuiltPathIterator implements Iterator { void remove() { throw new UnsupportedOperationException("Operation 'remove' is not supported by QuiltPathIterator") } + + private void advance() { + QuiltPath result = null + while (result == null && itr.hasNext()) { + def item = itr.next() + Path path = dir.resolve(item) + if (path == dir) { // make sure to skip the original path + result = null + } + else if (filter) { + result = filter.accept(path) ? path : null + } + else { + result = path + } + } + + nextPath = result + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy index 04838424..82957a0d 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltObserverTest.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable MethodName */ /* * Copyright 2022, Quilt Data Inc * @@ -13,22 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio + import nextflow.quilt.QuiltSpecification import nextflow.quilt.QuiltObserver import java.nio.file.Path import java.nio.file.Paths -import spock.lang.Unroll +import groovy.transform.CompileDynamic /** * * @author Ernest Prabhakar */ +@CompileDynamic class QuiltObserverTest extends QuiltSpecification { - def 'should generate solid string for timestamp'() { + void 'should generate solid string for timestamp'() { when: def now = QuiltObserver.now() then: @@ -36,13 +38,13 @@ class QuiltObserverTest extends QuiltSpecification { now.contains('T') } - def 'should extract Quilt path from appropriate UNIX Path'() { + void 'should extract Quilt path from appropriate UNIX Path'() { given: - def pkg = Paths.get('/var/tmp/output/quilt-example#package=examples%2fhurdat') - def unpkg = Paths.get('/var/tmp/output/quilt-example/examples/hurdat') + Path pkg = Paths.get('/var/tmp/output/quilt-example#package=examples%2fhurdat') + Path unpkg = Paths.get('/var/tmp/output/quilt-example/examples/hurdat') expect: - null == QuiltObserver.asQuiltPath(unpkg) - 'quilt-example#package=examples%2fhurdat' == QuiltObserver.asQuiltPath(pkg).toString() + QuiltObserver.asQuiltPath(unpkg) == null + QuiltObserver.asQuiltPath(pkg).toString() == 'quilt-example#package=examples%2fhurdat' } } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy index d4ca2f1a..fa404593 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltSpecification.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable MethodName */ /* * Copyright 2022, Quilt Data Inc * @@ -13,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt + import nextflow.quilt.nio.QuiltPath import java.nio.ByteBuffer @@ -32,116 +33,116 @@ import nextflow.plugin.TestPluginManager import nextflow.plugin.extension.PluginExtensionProvider import org.pf4j.PluginDescriptorFinder import groovy.util.logging.Slf4j +import groovy.transform.CompileDynamic import spock.lang.Shared -import spock.lang.Timeout import spock.lang.Specification -import sun.nio.fs.UnixPath /** * * @author Ernest Prabhakar */ @Slf4j -abstract class QuiltSpecification extends Specification { +@CompileDynamic +class QuiltSpecification extends Specification { @Shared String pluginsMode - def setupSpec() { + void setupSpec() { // reset previous instances PluginExtensionProvider.reset() // this need to be set *before* the plugin manager class is created pluginsMode = System.getProperty('pf4j.mode') System.setProperty('pf4j.mode', 'dev') // the plugin root should - def root = Path.of('.').toAbsolutePath().normalize() + Path root = Path.of('.').toAbsolutePath().normalize() def manager = new TestPluginManager(root){ + @Override protected PluginDescriptorFinder createPluginDescriptorFinder() { return new TestPluginDescriptorFinder(){ + @Override protected Path getManifestPath(Path pluginPath) { return pluginPath.resolve('src/resources/META-INF/MANIFEST.MF') } + } } + } Plugins.init(root, 'dev', manager) Plugins.startIfMissing('nf-quilt') } - def cleanupSpec() { + void cleanupSpec() { Plugins.stop() PluginExtensionProvider.reset() - pluginsMode ? System.setProperty('pf4j.mode',pluginsMode) : System.clearProperty('pf4j.mode') + pluginsMode ? System.setProperty('pf4j.mode', pluginsMode) : System.clearProperty('pf4j.mode') } - Path createObject(String url, String text) { + Path makeObject(String url, String text) { assert url - def path = Paths.get(new URI(url)) - createObject(path, text) + Path path = Paths.get(new URI(url)) + return makeObject(path, text) } - Path createObject(Path path, String text) { + Path makeObject(Path path, String text) { assert path log.debug "Write String[$text] to '$path'" Files.write(path, text.bytes) - path.localPath() + return path.localPath() } boolean existsPath(String path) { assert path log.debug "Check path string exists '$path'" - Files.exists(Paths.get(path)) + return Files.exists(Paths.get(path)) } boolean existsPath(QuiltPath path) { log.debug "Check path object exists '$path'" final local = path.localPath() - existsPath(local.toString()) + return existsPath(local.toString()) } String readObject(Path path) { log.debug "Read String from '$path'" - new String(Files.readAllBytes(path)) + return new String(Files.readAllBytes(path)) } - String readChannel(SeekableByteChannel sbc, int buffLen ) { - def buffer = new ByteArrayOutputStream() + String readChannel(SeekableByteChannel sbc, int buffLen) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream() ByteBuffer bf = ByteBuffer.allocate(buffLen) - while((sbc.read(bf))>0) { - bf.flip(); + while ((sbc.read(bf)) > 0) { + bf.flip() buffer.write(bf.array(), 0, bf.limit()) - bf.clear(); + bf.clear() } - buffer.toString() + return buffer.toString() } - void writeChannel( SeekableByteChannel channel, String content, int buffLen ) { - + void writeChannel(SeekableByteChannel channel, String content, int buffLen) { def bytes = content.getBytes() - ByteBuffer buf = ByteBuffer.allocate(buffLen); - int i=0 - while( i < bytes.size()) { - - def len = Math.min(buffLen, bytes.size()-i); - buf.clear(); - buf.put(bytes, i, len); - buf.flip(); - channel.write(buf); + ByteBuffer buf = ByteBuffer.allocate(buffLen) + int i = 0 + while (i < bytes.size()) { + int len = Math.min(buffLen, bytes.size() - i) + buf.clear() + buf.put(bytes, i, len) + buf.flip() + channel.write(buf) i += len } - } - protected Path mockQuiltPath(String path, boolean isDir=false) { assert path.startsWith('quilt+s3://') - def tokens = path.tokenize('/') - def bucket = tokens[1] - def file = '/' + tokens[2..-1].join('/') + List tokens = path.tokenize('/') + String bucket = tokens[1] + String file = '/' + tokens[2..-1].join('/') def attr = Mock(BasicFileAttributes) attr.isDirectory() >> isDir @@ -158,17 +159,19 @@ abstract class QuiltSpecification extends Specification { def uri = GroovyMock(URI) uri.toString() >> path - def result = GroovyMock(Path) - result.bucket() >> bucket + result.getBucket() >> bucket result.toUriString() >> path result.toString() >> file result.getFileSystem() >> fs result.toUri() >> uri - result.resolve(_) >> { mockQuiltPath("$path/${it[0]}") } + result.resolve(_) >> { p -> mockQuiltPath("${path}/${p[0]}") } result.toAbsolutePath() >> result result.asBoolean() >> true - result.getParent() >> { def p=path.lastIndexOf('/'); p!=-1 ? mockQuiltPath("${path.substring(0,p)}", true) : null } + result.getParent() >> { + int p = path.lastIndexOf('/') + return (p == -1) ? null : mockQuiltPath("${path.substring(0, p)}", true) + } result.getFileName() >> { Paths.get(tokens[-1]) } result.getName() >> tokens[1] return result diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltIDTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltIDTest.groovy index 5c2a7488..aac91d55 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltIDTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltIDTest.groovy @@ -1,41 +1,44 @@ +/* groovylint-disable MethodName */ package nextflow.quilt.jep -import nextflow.quilt.QuiltSpecification -import spock.lang.Unroll -import spock.lang.Ignore +import nextflow.quilt.QuiltSpecification import groovy.util.logging.Slf4j +import groovy.transform.CompileDynamic /** * * @author Ernest Prabhakar */ @Slf4j +@CompileDynamic class QuiltIDTest extends QuiltSpecification { - def 'should null on missing bucket'() { + + void 'should null on missing bucket'() { when: - def id = QuiltID.Fetch(null, "pkg/name") + QuiltID id = QuiltID.fetch(null, 'pkg/name') then: - null == id + id == null } - def 'should default on missing pgk_suffix'() { + void 'should default on missing pgk_suffix'() { when: - def id = QuiltID.Fetch("bucket", "pkg") + QuiltID id = QuiltID.fetch('bucket', 'pkg') then: - id.toString() == "bucket.pkg.default" + id.toString() == 'bucket.pkg.default' } - def 'should default on missing pgk_name'() { + void 'should default on missing pgk_name'() { when: - def id = QuiltID.Fetch("bucket", null) + QuiltID id = QuiltID.fetch('bucket', null) then: - id.toString() == "bucket.null.default" + id.toString() == 'bucket.null.default' } - def 'should decompose pkg names'() { + void 'should decompose pkg names'() { when: - def id = QuiltID.Fetch("bucket", "pkg/name") + QuiltID id = QuiltID.fetch('bucket', 'pkg/name') then: - id.toString() == "bucket.pkg.name" + id.toString() == 'bucket.pkg.name' } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index acc88a3a..f328afb7 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable MethodName */ /* * Copyright 2022, Quilt Data Inc * @@ -13,23 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.jep import nextflow.quilt.QuiltSpecification import nextflow.quilt.nio.QuiltPathFactory import nextflow.quilt.nio.QuiltPath -import nextflow.quilt.nio.QuiltFileAttributesView -import nextflow.Global -import nextflow.Session -import spock.lang.Unroll -import spock.lang.Shared +import spock.lang.IgnoreIf import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.nio.file.attribute.BasicFileAttributes import groovy.util.logging.Slf4j +import groovy.transform.CompileDynamic /** * @@ -37,60 +34,61 @@ import groovy.util.logging.Slf4j */ @Slf4j +@CompileDynamic class QuiltPackageTest extends QuiltSpecification { - QuiltPathFactory factory - QuiltPath qpath - QuiltPackage pkg - static String pkg_url = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9' - static String url = pkg_url + '&path=README.md' - static String out_url = 'quilt+s3://quilt_dev_null#package=nf-quilt%2ftest' + private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples%2fsmart-report@d68a7e9' + private final static String TEST_URL = PACKAGE_URL + '&path=README.md' + + private QuiltPathFactory factory + private QuiltPath qpath + private QuiltPackage pkg - def setup() { + void setup() { factory = new QuiltPathFactory() - qpath = factory.parseUri(url) + qpath = factory.parseUri(TEST_URL) pkg = qpath.pkg() } - def 'should create unique Package for associated Paths' () { + void 'should create unique Package for associated Paths'() { given: def pkgPath = qpath.getJustPackage() def pkg2 = pkgPath.pkg() expect: pkg != null - pkg.toString() == "quilt_example_examples_smart_report" - pkgPath.toUriString() == pkg_url + pkg.toString() == 'quilt_example_examples_smart_report' + pkgPath.toUriString() == PACKAGE_URL pkg == pkg2 } - def 'should distinguish Packages with same name in different Buckets ' () { + void 'should distinguish Packages with same name in different Buckets '() { given: - def url2 = url.replace('quilt-','quilted-') + def url2 = TEST_URL.replace('quilt-', 'quilted-') def qpath2 = factory.parseUri(url2) def pkg2 = qpath2.pkg() expect: - url != url2 + TEST_URL != url2 pkg != pkg2 pkg.toString() != pkg2.toString() !Files.exists(qpath2.localPath()) } - def 'should create an install folder ' () { + void 'should create an install folder '() { given: Path installPath = pkg.packageDest() - String tmpDirsLocation = System.getProperty("java.io.tmpdir") + String tmpDirsLocation = System.getProperty('java.io.tmpdir') expect: installPath.toString().startsWith(tmpDirsLocation) Files.exists(installPath) } - def 'should get attributes for package folder' () { + void 'should get attributes for package folder'() { given: def root = qpath.getRoot() - def qroot = factory.parseUri(pkg_url) + def qroot = factory.parseUri(PACKAGE_URL) expect: root == qroot qroot.isJustPackage() @@ -98,7 +96,8 @@ class QuiltPackageTest extends QuiltSpecification { Files.readAttributes(qroot, BasicFileAttributes) } - def 'should pre-install files and get attributes' () { + @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + void 'should pre-install files and get attributes'() { expect: pkg.install() pkg.isInstalled() @@ -106,24 +105,24 @@ class QuiltPackageTest extends QuiltSpecification { Files.readAttributes(qpath, BasicFileAttributes) } - def 'should deinstall files' () { + @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + void 'should deinstall files'() { expect: Files.exists(qpath.localPath()) when: qpath.deinstall() then: !Files.exists(qpath.localPath()) - when: + /* when: Files.readAttributes(qpath, BasicFileAttributes) then: - thrown(java.nio.file.NoSuchFileException) + thrown(java.nio.file.NoSuchFileException) */ } - - def 'should iterate over installed files ' () { + void 'should iterate over installed files '() { given: def root = qpath.getRoot() - def qroot = factory.parseUri(pkg_url) + def qroot = factory.parseUri(PACKAGE_URL) expect: root @@ -131,28 +130,27 @@ class QuiltPackageTest extends QuiltSpecification { root == qroot Files.isDirectory(qroot) pkg.install() - //vs!Files.isDirectory(qpath) + //vs!Files.isDirectory(qpath) } - def 'should write new files back to bucket ' () { + void 'should write new files back to bucket '() { given: def cleanDate = QuiltPackage.today() - def qout = factory.parseUri(out_url) + //def qout = factory.parseUri(OUT_URL) def opkg = qpath.pkg() def outPath = Paths.get(opkg.packageDest().toString(), "${cleanDate}.txt") // remove path // re-install package // verify file exists - Files.writeString(outPath, cleanDate); + Files.writeString(outPath, cleanDate) expect: Files.exists(outPath) - //opkg.push() - //opkg.uninstall() - //!Files.exists(outPath) - //pkg.isInstalled() + //opkg.push() + //opkg.uninstall() + //!Files.exists(outPath) + //pkg.isInstalled() } + // void 'Package should return Attributes IFF the file exists'() { } - def 'Package should return Attributes IFF the file exists' () { - } } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy index ad9cb9f3..2226c629 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltParserTest.groovy @@ -1,80 +1,79 @@ +/* groovylint-disable MethodName */ package nextflow.quilt.jep -import nextflow.quilt.QuiltSpecification -import spock.lang.Unroll -import spock.lang.Ignore +import nextflow.quilt.QuiltSpecification import groovy.util.logging.Slf4j +import groovy.transform.CompileDynamic /** * * @author Ernest Prabhakar */ @Slf4j +@CompileDynamic class QuiltParserTest extends QuiltSpecification { - static String path_url = 'quilt+s3://bucket-name#package=quilt/test@abc1&path=sub%2Fpath' - static String rel_url = 'quilt+s3://bucket-name#package=quilt/test@abc1&path=sub%2F..%2Fpath' - static String tag_url = 'quilt+s3://bucket-name#package=quilt/test:later&path=sub%2Fpath' - static String hash_url = 'quilt+s3://quilt-ernest-staging#package=test/hurdat@e4bed47503f9dde90a00b915ef75bd1ad294378870ba2e388084266e4f7ed909' - static String test_url = 'quilt+s3://quilt-ernest-staging#package=nf-quilt/sarek/pipeline_info/execution_trace_2022-10-13_01-01-31.txt' + private static final String REL_URL = 'quilt+s3://bucket-name#package=quilt/test@abc1&path=sub%2F..%2Fpath' + private static final String TEST_URL = + 'quilt+s3://quilt-ernest-staging#package=nf-quilt/sarek/pipeline_info/execution_trace_2022-10-13_01-01-31.txt' - def 'should host Quilt URL scheme'() { + void 'should host Quilt URL scheme'() { expect: QuiltParser.SCHEME == 'quilt+s3' QuiltParser.PREFIX == 'quilt+s3://' } - def 'should error on invalid schema'() { + void 'should error on invalid schema'() { when: - def parser = QuiltParser.ForUriString("quilt3://bucket/") + QuiltParser.forUriString('quilt3://bucket/') then: thrown(IllegalArgumentException) } - def 'should parse over-long packages into path'() { + void 'should parse over-long packages into path'() { when: - def parser = QuiltParser.ForUriString(test_url) + QuiltParser parser = QuiltParser.forUriString(TEST_URL) then: - parser.bucket() == "quilt-ernest-staging" - parser.pkg_name() == "nf-quilt/sarek" - parser.path() == "pipeline_info/execution_trace_2022-10-13_01-01-31.txt" - } + parser.getBucket() == 'quilt-ernest-staging' + parser.getPackageName() == 'nf-quilt/sarek' + parser.getPath() == 'pipeline_info/execution_trace_2022-10-13_01-01-31.txt' + } - def 'should modify path segments appropriately'() { + void 'should modify path segments appropriately'() { when: - def parser = QuiltParser.ForUriString(rel_url) + QuiltParser parser = QuiltParser.forUriString(REL_URL) then: - parser.path() == "sub/../path" - parser.appendPath("child").path() == "sub/../path/child" - parser.normalized().path() == "path" - def p1 = parser.dropPath() - p1.path() == "sub/.." - def p2 = p1.dropPath() - p2.path() == "sub" - def p3 = p2.dropPath() - p3.path() == "" - def p4 = p3.dropPath() - p4.path() == "" + parser.getPath() == 'sub/../path' + parser.appendPath('child').getPath() == 'sub/../path/child' + parser.normalized().getPath() == 'path' + QuiltParser p1 = parser.dropPath() + p1.getPath() == 'sub/..' + QuiltParser p2 = p1.dropPath() + p2.getPath() == 'sub' + QuiltParser p3 = p2.dropPath() + p3.getPath() == '' + QuiltParser p4 = p3.dropPath() + p4.getPath() == '' } - def 'should decompose URIs'() { + void 'should decompose URIs'() { when: - def parser = QuiltParser.ForBarePath(bare) + QuiltParser parser = QuiltParser.forBarePath(bare) then: - parser.bucket() == bucket - parser.pkg_name() == pkg - parser.path() == path - parser.hash() == hash - parser.tag() == tag + parser.getBucket() == bucket + parser.getPackageName() == pkg + parser.getPath() == path + parser.getHash() == hash + parser.getTag() == tag where: bare | bucket | query | pkg | path | hash | tag - 'bucket' | 'bucket' | null | null | "" | null | null - 'BuCKet' | 'bucket' | null | null | "" | null | null - 'b#frag' | 'b' | 'frag' | null | "" | null | null - 'B#package=q%2Fp' | 'b' | 'package=q/p' | 'q/p' | "" | null | null - 'B#package=q%2Fp@hash' | 'b' | 'package=q/p@hash' | 'q/p' | "" | 'hash' | null - 'B#package=q%2Fp:tag&path=a%2Fb' | 'b' | 'package=q/p:tag&path=a/b'| 'q/p' | 'a/b'| null | 'tag' + 'bucket' | 'bucket' | null | null | '' | null | null + 'BuCKet' | 'bucket' | null | null | '' | null | null + 'b#frag' | 'b' | 'frag' | null | '' | null | null + 'B#package=q%2Fp' | 'b' | 'package=q/p' | 'q/p' | '' | null | null + 'B#package=q%2Fp@hash' | 'b' | 'package=q/p@hash' | 'q/p' | '' | 'hash' | null + 'B#package=q%2Fp:tag&path=a%2Fb' | 'b' | 'package=q/p:tag&path=a/b' | 'q/p' | 'a/b' | null | 'tag' } } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy index b61edf94..fc18641e 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy @@ -1,21 +1,19 @@ +/* groovylint-disable MethodName */ package nextflow.quilt.nio -import nextflow.quilt.QuiltSpecification - -import java.nio.file.FileSystemAlreadyExistsException -import spock.lang.Specification -import spock.lang.Unroll -import spock.lang.Ignore +import nextflow.quilt.QuiltSpecification +import groovy.transform.CompileDynamic /** * * @author Ernest Prabhakar */ +@CompileDynamic class QuiltFileSystemProviderTest extends QuiltSpecification { - def 'should return Quilt storage scheme'() { + void 'should return Quilt storage scheme'() { given: - def provider = new QuiltFileSystemProvider() + QuiltFileSystemProvider provider = new QuiltFileSystemProvider() expect: provider.getScheme() == 'quilt+s3' } @@ -25,5 +23,3 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { // do we need a new schema for quilt+local? } - - diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy index 56dc4b52..5f23e2c4 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemTest.groovy @@ -1,28 +1,30 @@ +/* groovylint-disable MethodName */ package nextflow.quilt.nio + import nextflow.quilt.QuiltSpecification import nextflow.quilt.jep.QuiltParser - +import java.nio.file.Path import java.nio.file.Paths - -import spock.lang.Ignore +import groovy.transform.CompileDynamic /** * * @author Paolo Di Tommaso */ +@CompileDynamic class QuiltFileSystemTest extends QuiltSpecification { - def 'should test getPath' () { + void 'should test getPath'() { given: - def BUCKET_NAME = 'bucket' - def provider = Stub(QuiltFileSystemProvider) + String BUCKET_NAME = 'bucket' + QuiltFileSystemProvider provider = Stub(QuiltFileSystemProvider) and: - def fs = new QuiltFileSystem(BUCKET_NAME, provider) + QuiltFileSystem fs = new QuiltFileSystem(BUCKET_NAME, provider) when: - def fs_path = fs.getPath(path) - def url = QuiltParser.PREFIX + path - def url_path = Paths.get(new URI(url)) + Path fs_path = fs.getPath(path) + String url = QuiltParser.PREFIX + path + Path url_path = Paths.get(new URI(url)) then: fs_path @@ -30,19 +32,18 @@ class QuiltFileSystemTest extends QuiltSpecification { fs_path.toString() == path where: - call| path + call | path 1 | 'file-name.txt' 1 | 'bucket#package=alpha%2fbravo' } - def 'should test basic properties' () { - + void 'should test basic properties'() { given: - def BUCKET_NAME = 'bucket' - def provider = Stub(QuiltFileSystemProvider) + String BUCKET_NAME = 'bucket' + QuiltFileSystemProvider provider = Stub(QuiltFileSystemProvider) when: - def fs = new QuiltFileSystem(BUCKET_NAME, provider) + QuiltFileSystem fs = new QuiltFileSystem(BUCKET_NAME, provider) then: fs.getSeparator() == '/' fs.isOpen() diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 2a29e3b6..efe2ac05 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -1,9 +1,10 @@ +/* groovylint-disable MethodName */ package nextflow.quilt.nio + import nextflow.quilt.QuiltSpecification -//import nextflow.quilt.jep.QuiltPackage +import nextflow.quilt.jep.QuiltPackage import java.nio.charset.Charset -import java.nio.file.DirectoryNotEmptyException import java.nio.file.FileAlreadyExistsException import java.nio.file.FileVisitResult import java.nio.file.Files @@ -18,43 +19,45 @@ import java.nio.file.attribute.BasicFileAttributeView import java.nio.file.attribute.BasicFileAttributes import spock.lang.IgnoreIf -import spock.lang.Requires -import spock.lang.Shared import spock.lang.Ignore import groovy.util.logging.Slf4j +import groovy.transform.CompileDynamic /** * * @author Ernest Prabhakar */ @Slf4j +@CompileDynamic class QuiltNioTest extends QuiltSpecification { - static String null_url = 'quilt+s3://quilt-dev-null#package=test/null' - public static def null_path(f) { null_url+"&path=$f" } + + private static final String NULL_URL = 'quilt+s3://quilt-dev-null#package=test/null' //https://open.quiltdata.com/b/quilt-example/tree/examples/hurdat/ - static String pkg_url = 'quilt+s3://quilt-example#package=examples/hurdat' - public static def pkg_path(f) { pkg_url+"&path=$f" } - static String write_url = pkg_path('folder/file-name.txt') - static String read_url = pkg_path('data/atlantic-storms.csv') - static String TEXT = "Hello world!" + private static final String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples/hurdat' + private static final String WRITE_URL = packagePath('folder/file-name.txt') + private static final String READ_URL = packagePath('data/atlantic-storms.csv') + private static final String TEXT = 'Hello world!' - def 'should create valid URIs' () { + static String null_path(String f) { return NULL_URL + "&path=$f" } + static String packagePath(String f) { return PACKAGE_URL + "&path=$f" } + + void 'should create valid URIs'() { given: - def uri = new URI(write_url) + URI uri = new URI(WRITE_URL) expect: uri when: - def path = Paths.get(uri) + Path path = Paths.get(uri) then: path } - def 'should write to a path' () { + void 'should write to a path'() { given: - def path = Paths.get(new URI(write_url)) + Path path = Paths.get(new URI(WRITE_URL)) when: - Path dest = createObject(path, TEXT) + Path dest = makeObject(path, TEXT) then: existsPath(dest.toString()) new String(Files.readAllBytes(path)).trim() == TEXT @@ -62,9 +65,10 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT } - def 'should read from a path' () { + @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + void 'should read from a path'() { given: - def path = Paths.get(new URI(read_url)) + Path path = Paths.get(new URI(READ_URL)) when: String text = readObject(path) @@ -72,22 +76,23 @@ class QuiltNioTest extends QuiltSpecification { text.startsWith('id') } - def 'should read file attributes' () { + @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + void 'should read file attributes'() { given: final start = System.currentTimeMillis() final root = 'folder' final start_path = "${root}/${start}.txt" - final start_url = pkg_path(start_path) - def path = Paths.get(new URI(start_url)) + final start_url = packagePath(start_path) + Path path = Paths.get(new URI(start_url)) when: - createObject(path, TEXT) + makeObject(path, TEXT) and: // // -- readAttributes // - def attrs = Files.readAttributes(path, BasicFileAttributes) + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes) then: attrs.isRegularFile() !attrs.isDirectory() @@ -96,8 +101,8 @@ class QuiltNioTest extends QuiltSpecification { !attrs.isOther() attrs.fileKey() == start_path //attrs.lastAccessTime() == null - attrs.lastModifiedTime().toMillis()-start < 5_000 - attrs.creationTime().toMillis()-start < 5_000 + attrs.lastModifiedTime().toMillis() - start < 5_000 + attrs.creationTime().toMillis() - start < 5_000 // // -- getLastModifiedTime @@ -111,7 +116,7 @@ class QuiltNioTest extends QuiltSpecification { // -- getFileAttributeView // when: - def view = Files.getFileAttributeView(path, BasicFileAttributeView) + BasicFileAttributeView view = Files.getFileAttributeView(path, BasicFileAttributeView) then: view.readAttributes().toString() == attrs.toString() @@ -135,7 +140,7 @@ class QuiltNioTest extends QuiltSpecification { // -- readAttributes for a package // when: - attrs = Files.readAttributes(Paths.get(new URI(pkg_url)), BasicFileAttributes) + attrs = Files.readAttributes(Paths.get(new URI(PACKAGE_URL)), BasicFileAttributes) then: !attrs.isRegularFile() attrs.isDirectory() @@ -145,14 +150,14 @@ class QuiltNioTest extends QuiltSpecification { attrs.fileKey() == '/' attrs.creationTime() != null //attrs.lastAccessTime() == null - attrs.lastModifiedTime().toMillis()-start < 5_000 + attrs.lastModifiedTime().toMillis() - start < 5_000 } - def 'should copy a stream to path' () { + void 'should copy a stream to path'() { given: - def surl = pkg_path("stream.txt") - def path = Paths.get(new URI(surl)) - def stream = new ByteArrayInputStream(new String(TEXT).bytes) + String surl = packagePath('stream.txt') + Path path = Paths.get(new URI(surl)) + ByteArrayInputStream stream = new ByteArrayInputStream(new String(TEXT).bytes) Files.copy(stream, path) and: existsPath(path) @@ -172,10 +177,10 @@ class QuiltNioTest extends QuiltSpecification { thrown(FileAlreadyExistsException) } - def 'copy local file to a bucket' () { + void 'copy local file to a bucket'() { given: - def path = Paths.get(new URI(write_url)) - def source = Files.createTempFile('test','nf') + Path path = Paths.get(new URI(WRITE_URL)) + Path source = Files.createTempFile('test', 'nf') source.text = TEXT Files.deleteIfExists(path.localPath()) @@ -185,14 +190,14 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT cleanup: - if( source ) Files.delete(source) + if (source) { Files.delete(source) } } @Ignore - def 'copy a remote file to a bucket' () { + void 'copy a remote file to a bucket'() { given: - def path = Paths.get(new URI(write_url)) - final source_url = write_url.replace("test_folder","source") + Path path = Paths.get(new URI(WRITE_URL)) + final source_url = WRITE_URL.replace('test_folder', 'source') final source = Paths.get(new URI(source_url)) Files.write(source, TEXT.bytes) and: @@ -204,10 +209,10 @@ class QuiltNioTest extends QuiltSpecification { } @Ignore - def 'move a remote file to a bucket' () { + void 'move a remote file to a bucket'() { given: - def path = Paths.get(new URI(write_url)) - final source_url = write_url.replace("test_folder","source") + Path path = Paths.get(new URI(WRITE_URL)) + final source_url = WRITE_URL.replace('test_folder', 'source') final source = Paths.get(new URI(source_url)) Files.write(source, TEXT.bytes) and: @@ -218,21 +223,22 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT } - def 'should create a package' () { + void 'should create a package'() { given: - def path = Paths.get(new URI(pkg_url)) + Path path = Paths.get(new URI(PACKAGE_URL)) when: - def pkg = path.pkg() + QuiltPackage pkg = path.pkg() then: pkg - pkg.relativeChildren("") + pkg.relativeChildren('') } - def 'should iterate over package folders/files' () { + @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + void 'should iterate over package folders/files'() { given: - def path = Paths.get(new URI(pkg_url)) + Path path = Paths.get(new URI(PACKAGE_URL)) when: - def itr = new QuiltPathIterator(path, null) + QuiltPathIterator itr = new QuiltPathIterator(path, null) then: itr != null @@ -243,17 +249,17 @@ class QuiltNioTest extends QuiltSpecification { itr.next().toString().contains('path=quilt_summarize.json') when: - def spath = itr.next() - def sitr = new QuiltPathIterator(spath, null) + Path spath = itr.next() + QuiltPathIterator sitr = new QuiltPathIterator(spath, null) then: spath.toString().contains('path=scripts') sitr.hasNext() sitr.next().toString().contains('path=scripts%2fbuild.py') } - def 'should create a directory' () { + void 'should create a directory'() { given: - def dir = Paths.get(new URI(pkg_url)) + Path dir = Paths.get(new URI(PACKAGE_URL)) when: Files.createDirectory(dir) @@ -261,51 +267,51 @@ class QuiltNioTest extends QuiltSpecification { existsPath(dir) } - def 'should create a directory tree' () { + void 'should create a directory tree'() { given: - def dir = Paths.get(new URI(pkg_path("alpha/bravo/omega/"))) + Path dir = Paths.get(new URI(packagePath('alpha/bravo/omega/'))) when: Files.createDirectories(dir) then: - existsPath(Paths.get(new URI(pkg_path("alpha")))) - existsPath(Paths.get(new URI(pkg_path("alpha/bravo")))) - existsPath(Paths.get(new URI(pkg_path("alpha/bravo/omega")))) + existsPath(Paths.get(new URI(packagePath('alpha')))) + existsPath(Paths.get(new URI(packagePath('alpha/bravo')))) + existsPath(Paths.get(new URI(packagePath('alpha/bravo/omega')))) when: - Files.createDirectories(dir) + Files.createDirectory(dir) then: noExceptionThrown() } - def 'should create a file' () { + void 'should create a file'() { given: - def curl = pkg_path("create.txt") - def path = Paths.get(new URI(curl)) + String curl = packagePath('create.txt') + Path path = Paths.get(new URI(curl)) Files.createFile(path) expect: existsPath(path) } @Ignore - def 'should create temp file and directory' () { + void 'should create temp file and directory'() { given: - def base = Paths.get(new URI(pkg_url)) + Path base = Paths.get(new URI(PACKAGE_URL)) when: - def t1 = Files.createTempDirectory(base, 'test') + Path t1 = Files.createTempDirectory(base, 'test') then: existsPaths(t1) when: - def t2 = Files.createTempFile(base, 'prefix', 'suffix') + Path t2 = Files.createTempFile(base, 'prefix', 'suffix') then: existsPaths(t2) } - def 'should delete a file' () { + void 'should delete a file'() { given: - def path = Paths.get(new URI(write_url)) - Path dest = createObject(path,TEXT) + Path path = Paths.get(new URI(WRITE_URL)) + Path dest = makeObject(path, TEXT) when: Files.delete(dest) @@ -314,40 +320,40 @@ class QuiltNioTest extends QuiltSpecification { !existsPath(dest.toString()) } - def 'should throw a NoSuchFileException when deleting an object not existing' () { + void 'should throw a NoSuchFileException when deleting an object not existing'() { when: - def path = Paths.get(new URI(write_url)) + Path path = Paths.get(new URI(WRITE_URL)) Files.delete(path) then: thrown(NoSuchFileException) } - def 'should validate exists method' () { + void 'should validate exists method'() { when: - def vurl = pkg_path("bolder/file.txt") - def path = Paths.get(new URI(vurl)) - createObject(path,TEXT) + String vurl = packagePath('bolder/file.txt') + Path path = Paths.get(new URI(vurl)) + makeObject(path, TEXT) QuiltPath p = Paths.get(new URI(file)) as QuiltPath then: existsPath(p) == flag where: flag | file - true | pkg_path("bolder/file.txt") - true | pkg_path("bolder/file.txt") - false| pkg_path("bolder/fooooo.txt") + true | packagePath('bolder/file.txt') + true | packagePath('bolder/file.txt') + false | packagePath('bolder/fooooo.txt') } - def 'should check if it is a directory' () { + void 'should check if it is a directory'() { given: - def dir = Paths.get(new URI(pkg_url)) + Path dir = Paths.get(new URI(PACKAGE_URL)) expect: Files.isDirectory(dir) !Files.isRegularFile(dir) when: - def file = dir.resolve('this/and/that') - Path dest = createObject(file, 'Hello world') + Path file = dir.resolve('this/and/that') + makeObject(file, 'Hello world') then: !Files.isDirectory(file) Files.isRegularFile(file) @@ -365,23 +371,21 @@ class QuiltNioTest extends QuiltSpecification { !Files.isSymbolicLink(file) } - def 'should check that is the same file' () { - + void 'should check that is the same file'() { given: - def file1 = Paths.get(new URI(pkg_path("some/data/file.txt"))) - def file2 = Paths.get(new URI(pkg_path("some/data/file.txt"))) - def file3 = Paths.get(new URI(pkg_path("some/data/fooo.txt"))) + Path file1 = Paths.get(new URI(packagePath('some/data/file.txt'))) + Path file2 = Paths.get(new URI(packagePath('some/data/file.txt'))) + Path file3 = Paths.get(new URI(packagePath('some/data/fooo.txt'))) expect: Files.isSameFile(file1, file2) !Files.isSameFile(file1, file3) - } - def 'should create a newBufferedReader' () { + void 'should create a newBufferedReader'() { given: - def path = Paths.get(new URI(write_url)) - createObject(path, TEXT) + Path path = Paths.get(new URI(WRITE_URL)) + makeObject(path, TEXT) when: def reader = Files.newBufferedReader(path, Charset.forName('UTF-8')) @@ -389,26 +393,26 @@ class QuiltNioTest extends QuiltSpecification { reader.text == TEXT when: - def unknown = Paths.get(new URI(pkg_path("unknown.txt"))) + Path unknown = Paths.get(new URI(packagePath('unknown.txt'))) Files.newBufferedReader(unknown, Charset.forName('UTF-8')) then: thrown(NullPointerException) } - def 'should create a newBufferedWriter' () { + void 'should create a newBufferedWriter'() { given: - def path = Paths.get(new URI(write_url)) + Path path = Paths.get(new URI(WRITE_URL)) def writer = Files.newBufferedWriter(path, Charset.forName('UTF-8')) - TEXT.readLines().each { it -> writer.println(it) } + TEXT.readLines().each { line -> writer.println(line) } writer.close() expect: readObject(path).trim() == TEXT } - def 'should create a newInputStream' () { + void 'should create a newInputStream'() { given: - def path = Paths.get(new URI(write_url)) - createObject(path, TEXT) + Path path = Paths.get(new URI(WRITE_URL)) + makeObject(path, TEXT) when: def reader = Files.newInputStream(path) @@ -416,12 +420,12 @@ class QuiltNioTest extends QuiltSpecification { reader.text == TEXT } - def 'should create a newOutputStream' () { + void 'should create a newOutputStream'() { given: - def path = Paths.get(new URI(write_url)) + Path path = Paths.get(new URI(WRITE_URL)) def writer = Files.newOutputStream(path) - TEXT.readLines().each { it -> - writer.write(it.bytes); + TEXT.readLines().each { line -> + writer.write(line.bytes) writer.write((int)('\n' as char)) } writer.close() @@ -429,10 +433,10 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT } - def 'should read a newByteChannel' () { + void 'should read a newByteChannel'() { given: - def path = Paths.get(new URI(write_url)) - createObject(path, TEXT) + Path path = Paths.get(new URI(WRITE_URL)) + makeObject(path, TEXT) when: def channel = Files.newByteChannel(path) @@ -440,9 +444,9 @@ class QuiltNioTest extends QuiltSpecification { readChannel(channel, 10).trim() == TEXT } - def 'should write a byte channel' () { + void 'should write a byte channel'() { given: - def path = Paths.get(new URI(write_url)) + Path path = Paths.get(new URI(WRITE_URL)) def channel = Files.newByteChannel(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE) writeChannel(channel, TEXT, 200) channel.close() @@ -450,10 +454,10 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT } - def 'should check file size' () { + void 'should check file size'() { given: - def path = Paths.get(new URI(write_url)) - createObject(path, TEXT) + Path path = Paths.get(new URI(WRITE_URL)) + makeObject(path, TEXT) expect: Files.size(path) == TEXT.size() @@ -464,72 +468,77 @@ class QuiltNioTest extends QuiltSpecification { } @Ignore - def 'should stream directory content' () { + void 'should stream directory content'() { given: - createObject(null_path("foo/file1.txt"),'A') - createObject(null_path("foo/file2.txt"),'BB') - createObject(null_path("foo/bar/file3.txt"),'CCC') - createObject(null_path("foo/bar/baz/file4.txt"),'DDDD') - createObject(null_path("foo/bar/file5.txt"),'EEEEE') - createObject(null_path("foo/file6.txt"),'FFFFFF') + makeObject(null_path('foo/file1.txt'), 'A') + makeObject(null_path('foo/file2.txt'), 'BB') + makeObject(null_path('foo/bar/file3.txt'), 'CCC') + makeObject(null_path('foo/bar/baz/file4.txt'), 'DDDD') + makeObject(null_path('foo/bar/file5.txt'), 'EEEEE') + makeObject(null_path('foo/file6.txt'), 'FFFFFF') when: - def p = Paths.get(new URI(null_url)) - def list = Files.newDirectoryStream(p).collect { - it.getFileName().toString() + Path p = Paths.get(new URI(NULL_URL)) + List list = Files.newDirectoryStream(p).collect { + path -> path.getFileName().toString() } then: list.size() == 1 list == [ 'foo' ] when: - list = Files.newDirectoryStream(Paths.get(new URI(null_path("foo")))).collect { it.getFileName().toString() } + list = Files.newDirectoryStream(Paths.get(new URI(null_path('foo')))).collect { + path -> path.getFileName().toString() + } then: list.size() == 4 list as Set == [ 'file1.txt', 'file2.txt', 'bar', 'file6.txt' ] as Set when: - list = Files.newDirectoryStream(Paths.get(new URI(null_path("foo/bar")))).collect { it.getFileName().toString() } + list = Files.newDirectoryStream(Paths.get(new URI(null_path('foo/bar')))).collect { + path -> path.getFileName().toString() + } then: list.size() == 3 list as Set == [ 'file3.txt', 'baz', 'file5.txt' ] as Set when: - list = Files.newDirectoryStream(Paths.get(new URI(null_path("foo/bar/baz")))).collect { it.getFileName().toString() } + list = Files.newDirectoryStream(Paths.get(new URI(null_path('foo/bar/baz')))).collect { + path -> path.getFileName().toString() + } then: list.size() == 1 list == [ 'file4.txt' ] } @Ignore - def 'should check walkTree' () { + void 'should check walkTree'() { given: - createObject(null_path("foo/file1.txt"),'A') - createObject(null_path("foo/file2.txt"),'BB') - createObject(null_path("foo/bar/file3.txt"),'CCC') - createObject(null_path("foo/bar/baz/file4.txt"),'DDDD') - createObject(null_path("foo/bar/file5.txt"),'EEEEE') - createObject(null_path("foo/file6.txt"),'FFFFFF') + makeObject(null_path('foo/file1.txt'), 'A') + makeObject(null_path('foo/file2.txt'), 'BB') + makeObject(null_path('foo/bar/file3.txt'), 'CCC') + makeObject(null_path('foo/bar/baz/file4.txt'), 'DDDD') + makeObject(null_path('foo/bar/file5.txt'), 'EEEEE') + makeObject(null_path('foo/file6.txt'), 'FFFFFF') when: List dirs = [] Map files = [:] - def base = Paths.get(new URI(null_url)) + Path base = Paths.get(new URI(NULL_URL)) Files.walkFileTree(base, new SimpleFileVisitor() { @Override - FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException - { + FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { dirs << base.relativize(dir).toString() - return FileVisitResult.CONTINUE; + return FileVisitResult.CONTINUE } @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException - { + FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { files[file.getFileName().toString()] = attrs - return FileVisitResult.CONTINUE; + return FileVisitResult.CONTINUE } + }) then: @@ -546,30 +555,28 @@ class QuiltNioTest extends QuiltSpecification { dirs.contains('foo/bar') dirs.contains('foo/bar/baz') - when: dirs = [] files = [:] - base = Paths.get(new URI(null_path("foo/bar/"))) + base = Paths.get(new URI(null_path('foo/bar/'))) Files.walkFileTree(base, new SimpleFileVisitor() { @Override - FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException - { + FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { dirs << base.relativize(dir).toString() - return FileVisitResult.CONTINUE; + return FileVisitResult.CONTINUE } @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException - { + FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { files[file.getFileName().toString()] = attrs - return FileVisitResult.CONTINUE; + return FileVisitResult.CONTINUE } + }) then: - files.size()==3 + files.size() == 3 files.containsKey('file3.txt') files.containsKey('file4.txt') files.containsKey('file5.txt') @@ -579,36 +586,36 @@ class QuiltNioTest extends QuiltSpecification { } @Ignore - def 'should handle dir and files having the same name' () { + void 'should handle dir and files having the same name'() { given: - createObject(pkg_path("foo"),'file-1') - createObject(pkg_path("foo/bar"),'file-2') - createObject(pkg_path("foo/baz"),'file-3') + makeObject(packagePath('foo'), 'file-1') + makeObject(packagePath('foo/bar'), 'file-2') + makeObject(packagePath('foo/baz'), 'file-3') and: - def root = Paths.get(new URI(pkg_url)) + Path root = Paths.get(new URI(PACKAGE_URL)) when: - def file1 = root.resolve('foo') + Path file1 = root.resolve('foo') then: Files.isRegularFile(file1) !Files.isDirectory(file1) file1.text == 'file-1' when: - def dir1 = root.resolve('foo/') + Path dir1 = root.resolve('foo/') then: !Files.isRegularFile(dir1) Files.isDirectory(dir1) when: - def file2 = root.resolve('foo/bar') + Path file2 = root.resolve('foo/bar') then: Files.isRegularFile(file2) !Files.isDirectory(file2) file2.text == 'file-2' when: - def parent = file2.parent + Path parent = file2.parent then: !Files.isRegularFile(parent) Files.isDirectory(parent) @@ -619,18 +626,17 @@ class QuiltNioTest extends QuiltSpecification { Files.walkFileTree(root, new SimpleFileVisitor() { @Override - FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException - { + FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { dirs << root.relativize(dir).toString() - return FileVisitResult.CONTINUE; + return FileVisitResult.CONTINUE } @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException - { + FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { files[root.relativize(file).toString()] = attrs - return FileVisitResult.CONTINUE; + return FileVisitResult.CONTINUE } + }) then: dirs.size() == 2 @@ -640,29 +646,28 @@ class QuiltNioTest extends QuiltSpecification { files.containsKey('foo') files.containsKey('foo/bar') files.containsKey('foo/baz') - } - def 'should handle file names with same prefix' () { + void 'should handle file names with same prefix'() { given: - createObject(pkg_path("transcript_index.junctions.fa"), 'foo') - createObject(pkg_path("alpha-beta/file1"), 'bar') - createObject(pkg_path("alpha/file2"), 'baz') + makeObject(packagePath('transcript_index.junctions.fa'), 'foo') + makeObject(packagePath('alpha-beta/file1'), 'bar') + makeObject(packagePath('alpha/file2'), 'baz') when: - def uri = new URI(pkg_path(file)) + URI uri = new URI(packagePath(file)) QuiltPath p = Paths.get(uri) as QuiltPath then: existsPath(p) == flag where: flag | file - true | "transcript_index.junctions.fa" - false | "transcript_index.junctions" - true | "alpha-beta/file1" - true | "alpha/file2" - true | "alpha-beta" - true | "alpha" + true | 'transcript_index.junctions.fa' + false | 'transcript_index.junctions' + true | 'alpha-beta/file1' + true | 'alpha/file2' + true | 'alpha-beta' + true | 'alpha' } } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathFactoryTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathFactoryTest.groovy index af2dc1f1..6a86f2b2 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathFactoryTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathFactoryTest.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable MethodName */ /* * Copyright 2022, Quilt Data Inc * @@ -13,52 +14,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio -import nextflow.quilt.QuiltSpecification +import nextflow.quilt.QuiltSpecification import nextflow.Channel import nextflow.Global import nextflow.Session -import spock.lang.Unroll +import java.nio.file.Path +import groovy.transform.CompileDynamic /** * * @author Ernest Prabhakar */ - +@CompileDynamic class QuiltPathFactoryTest extends QuiltSpecification { - static String pkg_url = 'quilt+s3://quilt-example#package=examples/hurdat@f8d1478d93' - static String url = pkg_url + '&path=scripts/build.py' + private final static String PACKAGE_URL = 'quilt+s3://quilt-example#package=examples/hurdat@f8d1478d93' + private final static String URL = PACKAGE_URL + '&path=scripts/build.py' - def 'should decompose Quilt URLs' () { + void 'should decompose Quilt URLs'() { given: - def qpath = QuiltPathFactory.Parse(url) + Path qpath = QuiltPathFactory.parse(URL) expect: qpath != null - qpath.bucket() == 'quilt-example' - qpath.pkg_name() == 'examples/hurdat' + qpath.getBucket() == 'quilt-example' + qpath.getPackageName() == 'examples/hurdat' qpath.file_key() == 'scripts/build.py' } - def 'should create quilt path #PATH' () { + void 'should create quilt path #PATH'() { given: Global.session = Mock(Session) { getConfig() >> [quilt:[project:'foo', region:'x']] } expect: - QuiltPathFactory.Parse(PATH).toString() == STR + QuiltPathFactory.parse(PATH).toString() == STR where: _ | PATH | STR _ | 'quilt+s3://reg#package=user/pkg' | 'reg#package=user%2fpkg' } - def 'should create Channel from URL' () { + void 'should create Channel from URL'() { expect: - def channel = Channel.fromPath(url) // +'/*' + def channel = Channel.fromPath(URL) // +'/*' channel } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathSerializerTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathSerializerTest.groovy index f7af1fb2..269b4ea6 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathSerializerTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathSerializerTest.groovy @@ -1,3 +1,4 @@ +/* groovylint-disable MethodName */ /* * Copyright 2022, Quilt Data Inc * @@ -13,35 +14,38 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package nextflow.quilt.nio + import nextflow.quilt.QuiltSpecification import java.nio.file.Path import java.nio.file.Paths import nextflow.Global import nextflow.Session +import groovy.transform.CompileDynamic /** * * @author Ernest Prabhakar */ +@CompileDynamic class QuiltPathSerializerTest extends QuiltSpecification { - static String url = 'quilt+s3://bucket#package=pkg%2fname&path=sample.csv' + private static final String URL = 'quilt+s3://bucket#package=pkg%2fname&path=sample.csv' - def 'should serialize a Quilt path'() { + void 'should serialize a Quilt path'() { given: Global.session = Mock(Session) { getConfig() >> [quilt:[project:'foo', region:'x']] } when: - def uri = URI.create(url) - def path = Paths.get(uri) + URI uri = URI.create(URL) + Path path = Paths.get(uri) then: - path instanceof QuiltPath + path in QuiltPath path.toUri() == uri - path.toUriString() == url + path.toUriString() == URL } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy index e25a3846..8ee36e9e 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltPathTest.groovy @@ -1,41 +1,44 @@ +/* groovylint-disable MethodName */ package nextflow.quilt.nio + import nextflow.quilt.QuiltSpecification import nextflow.quilt.jep.QuiltParser +import java.nio.file.Path import java.nio.file.Paths import groovy.util.logging.Slf4j +import groovy.transform.CompileDynamic -import spock.lang.Shared import spock.lang.Unroll import spock.lang.Ignore + /** * * @author Paolo Di Tommaso */ @Slf4j +@CompileDynamic class QuiltPathTest extends QuiltSpecification { - static final String BKT = "bucket" - static final String PKG_URL = "${QuiltParser.PREFIX}${BKT}#package=so%2fme" - static QuiltPath pkgPath - static QuiltFileSystem QFS - static String sub_path = "${BKT}#package=s/d&path=f%2ffile.txt" + private static final String BKT = 'bucket' + private static final String PKG_URL = "${QuiltParser.PREFIX}${BKT}#package=so%2fme" + private static final String SUB_PATH = "${BKT}#package=s/d&path=f%2ffile.txt" - private QuiltPath pathify(String path) { - if (!pkgPath) { pkgPath = Paths.get(new URI(PKG_URL)) } - if (!QFS) { QFS = pkgPath.getFileSystem() } + QuiltPath pathify(String path) { if (!path.contains(BKT)) { - return QFS.getPath(path) + URI uri = new URI(PKG_URL) + QuiltPath pkgPath = Paths.get(uri) + QuiltFileSystem qfs = pkgPath.getFileSystem() + return qfs.getPath(path) } String url = QuiltParser.PREFIX + path - Paths.get(new URI(url)) + return Paths.get(new URI(url)) } @Unroll - def 'should create a path: #objectName'() { - + void 'should create a path: #objectName'() { when: - def path = pathify(objectName) + Path path = pathify(objectName) then: path.toString() == expected //path.directory == dir @@ -50,13 +53,12 @@ class QuiltPathTest extends QuiltSpecification { '#package=o%2fs&path=b.c' | 'null#package=o%2fs&path=b.c' | false } - def 'should validate equals and hashCode'() { - + void 'should validate equals and hashCode'() { when: - def path1 = pathify('bucket#package=so%2fme&path=file-name.txt') - def path2 = pathify('bucket#package=so%2fme&path=file-name.txt') - def path3 = pathify('bucket#package=ot%2fher&path=file-name.txt') - def path4 = pathify('bucket2#package=so%2fme&path=file-name.txt') + Path path1 = pathify('bucket#package=so%2fme&path=file-name.txt') + Path path2 = pathify('bucket#package=so%2fme&path=file-name.txt') + Path path3 = pathify('bucket#package=ot%2fher&path=file-name.txt') + Path path4 = pathify('bucket2#package=so%2fme&path=file-name.txt') then: path1 == path2 @@ -67,28 +69,27 @@ class QuiltPathTest extends QuiltSpecification { path1.hashCode() != path3.hashCode() when: - def rel1 = pathify('file.txt') - def rel2 = pathify('file.txt') + Path rel1 = pathify('file.txt') + Path rel2 = pathify('file.txt') then: rel1 == rel2 rel1.hashCode() == rel2.hashCode() - } - def 'should validate isAbsolute'() { + void 'should validate isAbsolute'() { when: - def path1 = pathify('bucket#package=so%2fme&path=file-name.txt') - def path2 = pathify('file-name.txt') + Path path1 = pathify('bucket#package=so%2fme&path=file-name.txt') + Path path2 = pathify('file-name.txt') then: path1.isAbsolute() !path2.isAbsolute() } - def 'should validate getRoot'() { + void 'should validate getRoot'() { when: - def path1 = pathify('bucket#package=so%2fme&path=file-name.txt') - def path2 = pathify('#path=file-name.txt') + Path path1 = pathify('bucket#package=so%2fme&path=file-name.txt') + Path path2 = pathify('#path=file-name.txt') then: path1.getRoot() == pathify('bucket#package=so%2fme') @@ -97,9 +98,9 @@ class QuiltPathTest extends QuiltSpecification { } @Unroll - def 'should return fsName and isJustPackage' () { + void 'should return fsName and isJustPackage'() { when: - def p = pathify(path) + Path p = pathify(path) then: p.isJustPackage() == expected p.getFileSystem().toString() == fsName @@ -114,7 +115,7 @@ class QuiltPathTest extends QuiltSpecification { } @Ignore - def 'should validate getFileName'() { + void 'should validate getFileName'() { expect: pathify(path).getFileName() == pathify(fileName) @@ -126,22 +127,22 @@ class QuiltPathTest extends QuiltSpecification { } @Unroll - def 'should validate getParent: #path'() { + void 'should validate getParent: #path'() { given: - def parent_path = (parent ? pathify(parent) : null) + Path parent_path = (parent ? pathify(parent) : null) expect: pathify(path).getParent() == parent_path where: path | parent - 'bucket#path=some%2fdata%2ffile.txt'| 'bucket#path=some%2fdata' + 'bucket#path=some%2fdata%2ffile.txt' | 'bucket#path=some%2fdata' 'bucket#path=data%2ffile.txt' | 'bucket#path=data' 'bucket#path=file-name.txt' | 'bucket#path=/' 'bucket' | 'bucket' } @Unroll - def 'should validate toUri: #uri'() { + void 'should validate toUri: #uri'() { expect: pathify(path).toUri() == new URI(uri) pathify(path).toUri().scheme == new URI(uri).scheme @@ -156,10 +157,8 @@ class QuiltPathTest extends QuiltSpecification { '#path=some-file.txt' | 'quilt+s3://null#path=some-file.txt' } - @Unroll - def 'should validate resolve: base:=#base; path=#path'() { - + void 'should validate resolve: base:=#base; path=#path'() { expect: pathify(base).resolve(path) == pathify(expected) //pathify(base).resolve( pathify(path) ) == pathify(expected) @@ -173,7 +172,7 @@ class QuiltPathTest extends QuiltSpecification { } @Ignore - def 'should validate subpath: #expected'() { + void 'should validate subpath: #expected'() { expect: pathify(path).subpath(from, to) == pathify(expected) where: @@ -184,50 +183,50 @@ class QuiltPathTest extends QuiltSpecification { } @Unroll - def 'should validate startsWith: #prefix'() { + void 'should validate startsWith: #prefix'() { expect: pathify(path).startsWith(prefix) == expected pathify(path).startsWith(pathify(prefix)) == expected where: path | prefix | expected - 'bucket#package=s/d/file.txt'| 'bucket#package=s%2fd'| true - 'bucket#package=s/d/file.txt'| 'bucket#package=s' | true - 'bucket#package=s/d/file.txt'| 'bucket' | true - 'bucket#package=s/d/file.txt'| 'file.txt' | false + 'bucket#package=s/d/file.txt' | 'bucket#package=s%2fd' | true + 'bucket#package=s/d/file.txt' | 'bucket#package=s' | true + 'bucket#package=s/d/file.txt' | 'bucket' | true + 'bucket#package=s/d/file.txt' | 'file.txt' | false 'data%2ffile.txt' | 'data' | true 'data%2ffile.txt' | 'file.txt' | false } @Unroll - def 'should validate endsWith'() { + void 'should validate endsWith'() { expect: pathify(path).endsWith(suffix) == expected //pathify(path).endsWith(pathify(suffix)) == expected where: path | suffix | expected - sub_path | 'file.txt' | true - sub_path | 'f%2ffile.txt' | true - sub_path | '/f%2ffile.txt' | false - sub_path | 'bucket' | false - 'data%2ffile.txt'| 'data' | false - 'data%2ffile.txt'| 'file.txt' | true + SUB_PATH | 'file.txt' | true + SUB_PATH | 'f%2ffile.txt' | true + SUB_PATH | '/f%2ffile.txt' | false + SUB_PATH | 'bucket' | false + 'data%2ffile.txt' | 'data' | false + 'data%2ffile.txt' | 'file.txt' | true } @Unroll - def 'should validate normalise'() { + void 'should validate normalise'() { expect: pathify(path).normalize() == pathify(expected) where: path | expected 'bucket#path=s/d/file.txt' | 'bucket#path=s/d/file.txt' - 'bucket#path=some%2f..%2ffile.txt'| 'bucket#path=file.txt' + 'bucket#path=some%2f..%2ffile.txt' | 'bucket#path=file.txt' 'file.txt' | 'file.txt' } @Unroll - def 'should validate resolveSibling: #path' () { + void 'should validate resolveSibling: #path'() { expect: pathify(base).resolveSibling(path) == pathify(expected) @@ -239,7 +238,7 @@ class QuiltPathTest extends QuiltSpecification { } @Unroll - def 'should validate relativize' () { + void 'should validate relativize'() { expect: pathify(path).relativize(pathify(other)).toString() == pathify(expected).toString() where: @@ -248,4 +247,5 @@ class QuiltPathTest extends QuiltSpecification { 'bucket#package=so%2fme%2fdata' | 'bucket#package=so%2fme%2fdata%2ffile.txt' | 'file.txt' 'bucket#package=so%2fme&path=foo' | 'bucket#package=so%2fme&path=foo%2fbar' | 'bar' } + }