From f6fca9c8523beeb0b86422e456b587ef306a48ab Mon Sep 17 00:00:00 2001 From: "Dr. Ernie Prabhakar" <19791+drernie@users.noreply.github.com> Date: Thu, 31 Oct 2024 22:27:45 -0700 Subject: [PATCH] getRootPackage From strings with more than one --- .../main/nextflow/quilt/QuiltPathify.groovy | 37 +++++++++---- .../main/nextflow/quilt/nio/QuiltPath.groovy | 29 +++++++++- .../nextflow/quilt/QuiltPathifyTest.groovy | 54 ++++++++++++++++--- .../nextflow/quilt/QuiltProductTest.groovy | 2 +- 4 files changed, 105 insertions(+), 17 deletions(-) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy index 26dde791..fc8f163c 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltPathify.groovy @@ -41,6 +41,9 @@ class QuiltPathify { QuiltPackage pkg String uri + /* + * Copy a file from source to destRoot/relpath + */ static void copyFile(Path source, String destRoot, String relpath) { Path dest = Paths.get(destRoot, relpath.split('/') as String[]) try { @@ -95,43 +98,59 @@ class QuiltPathify { // Constructor takes a Path and finds QuiltPath and QuiltPackage QuiltPathify(Path path) { + println("QuiltPathify: $path") if (path in QuiltPath) { this.path = (QuiltPath) path + println("\tQuiltPathify.QuiltPath: $this.path") this.uri = this.path.toUriString() + println("\t\tQuiltPathify.QuiltPath.uri: $this.uri") this.pkg = this.path.pkg() + println("\t\tQuiltPathify.QuiltPath.pkg: $this.pkg") } else if (!findQuiltPath(path.getFileName().toString())) { - makeQuiltPath(path) + makeQuiltPath(path.toString()) this.isOverlay = true } } boolean findQuiltPath(String filename) { - // check for '#package' in filename - if (!filename.toString().contains('#package')) { + println("findQuiltPath: $filename") + String base = QuiltPath.getRootPackage(filename) + if (base == null) { return false } - uri = "${QuiltParser.SCHEME}://${filename}" + uri = "${QuiltParser.SCHEME}://${base}" + println("\tfindQuiltPath.uri: $uri") path = QuiltPathFactory.parse(this.uri) + println("\tfindQuiltPath.path: $path") pkg = path.pkg() + println("\tfindQuiltPath.pkg: $pkg") return true } - boolean makeQuiltPath(Path path) { - String quiltURI = uriFromS3File(path.toString()) + boolean makeQuiltPath(String s3File) { + println("makeQuiltPath: $s3File") + String quiltURI = uriFromS3File(s3File) + println("\tmakeQuiltPath.quiltURI: $quiltURI") this.path = QuiltPathFactory.parse(quiltURI) + println("\tmakeQuiltPath.path: $path") this.uri = this.path.toUriString() + println("\tmakeQuiltPath.uri: $uri") this.pkg = this.path.pkg() + println("\tmakeQuiltPath.pkg: $pkg") return true } - boolean copyToPackage(Path source) { + boolean copyToCache(Path source) { + println("copyToCache: $source -> $path") if (!this.isOverlay) { return false } - String localPath = path.sub_paths() + String localPath = source.getFileName() // FIXME: should be relative to workdir + println("\tcopyToCache.localPath: $localPath") Path destDir = pkg.packageDest() - log.debug("copyToPackage: $source -> $destDir / $localPath") + println("\tcopyToCache.destDir: $destDir") + println("copyToCache: $source -> $destDir / $localPath") copyFile(source, destDir.toString(), localPath) return true } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy index b33d8c79..256e554b 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltPath.groovy @@ -17,6 +17,7 @@ package nextflow.quilt.nio import nextflow.quilt.jep.QuiltPackage import nextflow.quilt.jep.QuiltParser + import java.nio.file.Files import java.nio.file.FileSystem import java.nio.file.LinkOption @@ -26,6 +27,7 @@ import java.nio.file.ProviderMismatchException import java.nio.file.WatchEvent import java.nio.file.WatchKey import java.nio.file.WatchService +import java.util.regex.Matcher import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -45,6 +47,30 @@ final class QuiltPath implements Path, Comparable { private final String[] paths private final boolean isFileName + static String getRootPackage(String filename) { + int n_package = filename.count('#package') + if (n_package < 1) { + return null + } + String base = filename + if (n_package > 1) { + log.info("\tfindQuiltPath: multiple '#package' in $base") + println('\t\tTO MATCH') + Matcher matches = (filename =~ /^([^#]+#package=.*)?(?:(?!%2f).)*#package=/) + if (!matches) { + log.error("findQuiltPath: no match found for $filename") + return null + } + println('\t\tDID MATCH') + println("\tfindQuiltPath.matches: $matches") + List parts = matches[0] as List + println("\tfindQuiltPath.parts: $parts") + base = parts[1] + log.info("\tfindQuiltPath: trimmed to $base") + } + return base + } + QuiltPath(QuiltFileSystem filesystem, QuiltParser parsed, boolean isFileName = false) { this.filesystem = filesystem this.parsed = parsed @@ -239,7 +265,8 @@ final class QuiltPath implements Path, Comparable { } String toUriString() { - return parsed.toUriString() + String rawString = parsed.toUriString() + return getRootPackage(rawString) ?: rawString } @Override diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPathifyTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPathifyTest.groovy index bf29d2e4..0e8f7414 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPathifyTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltPathifyTest.groovy @@ -20,8 +20,9 @@ import nextflow.quilt.jep.QuiltParser import nextflow.quilt.nio.QuiltPath import nextflow.quilt.nio.QuiltPathFactory -// import java.nio.file.Path -// import java.nio.file.Paths +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.Path //import spock.lang.Ignore import groovy.transform.CompileDynamic @@ -82,9 +83,9 @@ class QuiltPathifyTest extends QuiltSpecification { rc | path false | 'FILE.md' true | 'bucket#package=prefix%2fsuffix&path=FILE.md' + true | 'bkt#package=pre%2fsuffix&path=inputs%2fbkt#package=pre%2fsuffix@af541d2%2fdata.tsv' } - // Test findQuiltPath updates uri/path/pkg void 'test findQuiltPath overrides attributes'() { when: QuiltPathify pathify = getPathify() @@ -119,8 +120,49 @@ class QuiltPathifyTest extends QuiltSpecification { pathify2.pkg.toUriString() == uriWith } - // Test makeQuiltPath creates new uri/path/pkg - // Test makeQuiltPath sets isOverlay - // Test copyToPackage copies overly file to package folder + // TODO: Test findQuiltPath works for dynamic URIs + + void 'test makeQuiltPath'() { + when: + QuiltPathify pathify = getPathify() + pathify.makeQuiltPath(s3File) + + then: + pathify.isOverlay == false + pathify.uri == "quilt+s3://${uri}" + pathify.path.toString() == uri + pathify.pkg.toUriString() == "quilt+s3://${uri}" + + where: + s3File | uri + '/bkt/pre/suf/fold/FILE.md' | 'bkt#package=pre%2fsuf&path=fold%2fFILE.md' + } + + void 'test copyToCache copies overlay file to package folder'() { + when: + Path tempFolder = Files.createTempDirectory('bkt') + Path source = Paths.get(tempFolder.toString(), sub_path) + Files.createDirectories(source.getParent()) + Files.createFile(source) + + then: + Files.exists(source) + + when: + QuiltPathify pathify = new QuiltPathify(source) + Path destFolder = pathify.pkg.packageDest() + String file = source.getFileName() + Path dest = Paths.get(destFolder.toString(), file) + pathify.copyToCache(source) + + then: + pathify.isOverlay == true + Files.exists(dest) + Files.delete(dest) + Files.delete(source) + + where: + sub_path << ['FILE.md'] + } } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy index 5f4f0bd1..c45313c0 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/QuiltProductTest.groovy @@ -49,7 +49,7 @@ class QuiltProductTest extends QuiltSpecification { getWorkflowMetadata() >> wf_meta getParams() >> [outdir: url] isSuccess() >> success - config >> [quilt: [meta: [cfkey: 'cfval']], runName: 'my-run', publishing: false] + config >> [quilt: [meta: [cfkey: 'cfval']]] } return new QuiltProduct(pathify, session) }