diff --git a/.github/workflows/pkg-test.yml b/.github/workflows/pkg-test.yml new file mode 100644 index 00000000..51317a7c --- /dev/null +++ b/.github/workflows/pkg-test.yml @@ -0,0 +1,61 @@ +name: Test + +on: + # Trigger at every push. Action will also be visible from Pull Requests to master + push: # Comment this line to trigger action only on pull-requests (not recommended if you don't pay for GH Actions) + pull_request: + branches: [master] + +permissions: read-all + +jobs: + build: + name: Package Test + permissions: + contents: read + id-token: write + issues: write + pull-requests: write + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + java_version: [19] + runs-on: ${{ matrix.os }} + + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::712023778557:role/GitHub-Testing-NF-Quilt + aws-region: us-east-1 + + - name: Setup Java ${{matrix.java_version}} + uses: actions/setup-java@v3 + with: + java-version: ${{matrix.java_version}} + distribution: 'temurin' + architecture: x64 + cache: gradle + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run Package Tests + run: make pkg-test + + - name: Archive production artifacts + if: ${{ success() }} || ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: nf-quilt-pkg-test + path: | + /home/runner/work/nf-quilt/nf-quilt/plugins/nf-quilt/build/reports/tests/test/ diff --git a/plugins/nf-quilt/build.gradle b/plugins/nf-quilt/build.gradle index 87a98eb1..c9d3ae5e 100644 --- a/plugins/nf-quilt/build.gradle +++ b/plugins/nf-quilt/build.gradle @@ -66,7 +66,7 @@ dependencies { // This dependency is exported to consumers, that is to say found on their compile classpath. compileOnly "io.nextflow:nextflow:$nextflowVersion" compileOnly 'org.slf4j:slf4j-api:2.0.9' - compileOnly 'org.pf4j:pf4j:3.9.0' + compileOnly 'org.pf4j:pf4j:3.10.0' // add here plugins depepencies compileOnly 'org.slf4j:slf4j-simple:2.0.9' compileOnly 'black.ninia:jep:4.1.1' diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy index 6a14192b..25318104 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/QuiltProduct.groovy @@ -80,11 +80,17 @@ ${meta['workflow']['stats']['processes']} static void writeString(String text, QuiltPackage pkg, String filename) { String dir = pkg.packageDest() Path path = Paths.get(dir, filename.split('/') as String[]) + File parent = path.getParent().toFile() + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IllegalStateException("Couldn't create dir: " + parent); + } + try { Files.write(path, text.bytes) } catch (Exception e) { - log.error("writeString: cannot write `$text` to `$path` for `${pkg}`") + log.error("writeString[${e.getMessage()}]: fail write `$path` for `${pkg}`") + e.printStackTrace() } } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy index 54bd7874..420b056c 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/jep/QuiltPackage.groovy @@ -162,11 +162,10 @@ class QuiltPackage { List relativeChildren(String subpath) { Set keys = packageManifest().getEntries().keySet() - println("relativeChildren[${subpath}]: ${keys}") + log.debug("relativeChildren[${subpath}]: ${keys}") Collection children = keys.findResults { String key -> key.startsWith(subpath) ? key : null } - println("relativeChildren.children: ${children}") return children as List } diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy index d349f333..fd0c04ca 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystem.groovy @@ -106,8 +106,9 @@ final class QuiltFileSystem extends FileSystem implements Closeable { } QuiltFileAttributes readAttributes(QuiltPath path) { - //log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") + log.debug("QuiltFileAttributes QuiltFileSystem.readAttributes($path)") Path installedPath = path.localPath(true) + log.debug(".readAttributes.installedPath $installedPath exists(${Files.exists(installedPath)}))") try { BasicFileAttributes attrs = Files.readAttributes(installedPath, BasicFileAttributes) return new QuiltFileAttributes(path, path.toString(), attrs) diff --git a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy index b7ae4c53..972b072d 100644 --- a/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy +++ b/plugins/nf-quilt/src/main/nextflow/quilt/nio/QuiltFileSystemProvider.groovy @@ -43,6 +43,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Global import nextflow.Session +import nextflow.file.FileSystemTransferAware import nextflow.quilt.jep.QuiltParser import nextflow.quilt.jep.QuiltPackage @@ -54,7 +55,7 @@ import nextflow.quilt.jep.QuiltPackage @Slf4j @CompileStatic -class QuiltFileSystemProvider extends FileSystemProvider { +class QuiltFileSystemProvider extends FileSystemProvider implements FileSystemTransferAware { private final Map myEnv = new HashMap<>(System.getenv()) private final Map fileSystems = [:] @@ -84,6 +85,36 @@ class QuiltFileSystemProvider extends FileSystemProvider { return path.getFileSystem().provider() } + static boolean isLocalProvider(Path path) { + FileSystemProvider provider = provider(path) + String providerName = provider?.class?.name?.toLowerCase() ?: 'N/A' + println("QuiltFileSystemProvider.isLocalProvider[${path}] -> ${providerName}") + return providerName.contains("xfile") || providerName.contains("win") ||\ + providerName.contains("fat") || providerName == 'N/A' + } + + boolean canDownload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canDownload[${source}] -> ${target}") + return isLocalProvider(target) && source instanceof QuiltPath + } + + boolean canUpload(Path source, Path target) { + log.debug("QuiltFileSystemProvider.canUpload[${source}] -> ${target}") + return isLocalProvider(source) && target instanceof QuiltPath + } + + void download(Path source, Path target, CopyOption... options) throws IOException { + QuiltPath qSource = asQuiltPath(source) + Path local_source = qSource.localPath() + Files.copy(local_source, target, options) + } + + void upload(Path source, Path target, CopyOption... options) throws IOException { + QuiltPath qTarget = asQuiltPath(target) + Path local_target = qTarget.localPath() + Files.copy(source, local_target, options) + } + /** * @inheritDoc */ @@ -340,7 +371,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override void copy(Path from, Path to, CopyOption... options) throws IOException { - //log.debug("Attempting `copy`: ${from} -> ${to}") + log.debug("Attempting `copy`: ${from} -> ${to}") assert provider(from) == provider(to) if (from == to) { return // nothing to do -- just return @@ -366,7 +397,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override boolean isHidden(Path path) throws IOException { - return path.getFileName()?.toString()?.startsWith('.') + return path.getFileName()?.toString()?.contains('.') } @Override @@ -400,7 +431,7 @@ class QuiltFileSystemProvider extends FileSystemProvider { @Override def A readAttributes(Path path, Class type, LinkOption... options) throws IOException { - //log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' + log.debug 'BasicFileAttributes QuiltFileSystemProvider.readAttributes()' def attr = attributesCache.get(path) if (attr) { return attr diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy index f56472e9..45cfbba0 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/jep/QuiltPackageTest.groovy @@ -77,9 +77,9 @@ class QuiltPackageTest extends QuiltSpecification { subpath | expected_size '' | 8 '.ipynb_checkpoints' | 1 - '/.ipynb_checkpoints' | 1 + '.ipynb_checkpoints/' | 1 '.ipynb' | 1 - '/.ipynb' | 0 + '.ipynb/' | 0 } @@ -106,6 +106,30 @@ class QuiltPackageTest extends QuiltSpecification { Files.exists(installPath) } + void 'should copy temp files into install folder'() { + given: + String filename = 'test.txt' + Path installPath = pkg.packageDest() + Path tempFile = File.createTempFile('test', '.txt').toPath() + Path installedFile = Paths.get(installPath.toString(), filename) + expect: + Files.exists(tempFile) + Files.exists(installPath) + !Files.exists(installedFile) + Files.copy(tempFile, installedFile) + Files.exists(installedFile) + } + + void 'should copy package files to temp Path'() { + given: + Path installPath = pkg.packageDest() + expect: + Files.exists(installPath) + Files.isDirectory(installPath) + Files.readAttributes(installPath, BasicFileAttributes) + } + + void 'should get attributes for package folder'() { given: def root = qpath.getRoot() @@ -126,6 +150,7 @@ class QuiltPackageTest extends QuiltSpecification { Files.readAttributes(qpath, BasicFileAttributes) } + void 'should return null on failed install'() { given: def url2 = TEST_URL.replace('quilt-', 'quilted-') @@ -140,17 +165,13 @@ class QuiltPackageTest extends QuiltSpecification { void 'should deinstall files'() { expect: Files.exists(qpath.localPath(true)) + Files.readAttributes(qpath, BasicFileAttributes) when: qpath.deinstall() then: !Files.exists(qpath.localPath(false)) - /* when: - Files.readAttributes(qpath, BasicFileAttributes) - then: - thrown(java.nio.file.NoSuchFileException) */ } - @Ignore() void 'should iterate over installed files '() { given: def root = qpath.getRoot() diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy index fc18641e..0f841007 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltFileSystemProviderTest.groovy @@ -3,6 +3,8 @@ package nextflow.quilt.nio import nextflow.quilt.QuiltSpecification import groovy.transform.CompileDynamic +import java.nio.file.Path +import java.nio.file.Paths /** * @@ -22,4 +24,14 @@ class QuiltFileSystemProviderTest extends QuiltSpecification { // newDirectoryStream returns package path for write // do we need a new schema for quilt+local? + void 'should recognize isLocalProvider'() { + given: + Path local = File.createTempFile('test', '.txt').toPath() + Path remote = Paths.get(new URI(fullURL)) + + expect: + QuiltFileSystemProvider.isLocalProvider(local) == true + QuiltFileSystemProvider.isLocalProvider(remote) == false + } + } diff --git a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy index 9720176d..02450348 100644 --- a/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy +++ b/plugins/nf-quilt/src/test/nextflow/quilt/nio/QuiltNioTest.groovy @@ -79,7 +79,7 @@ class QuiltNioTest extends QuiltSpecification { text.startsWith('id') } - @IgnoreIf({ System.getProperty('os.name').contains('ux') }) + // @IgnoreIf({ System.getProperty('os.name').contains('ux') }) @IgnoreIf({ System.getProperty('os.name').contains('indows') }) void 'should read file attributes'() { given: @@ -132,7 +132,7 @@ class QuiltNioTest extends QuiltSpecification { then: !attrs.isRegularFile() attrs.isDirectory() - attrs.size() == 128 + attrs.size() > 100 // differs by platform !attrs.isSymbolicLink() !attrs.isOther() attrs.fileKey() == root @@ -148,7 +148,7 @@ class QuiltNioTest extends QuiltSpecification { then: !attrs.isRegularFile() attrs.isDirectory() - attrs.size() == 224 + attrs.size() > 100 // differs by platform !attrs.isSymbolicLink() !attrs.isOther() attrs.fileKey() == '/' @@ -197,7 +197,7 @@ class QuiltNioTest extends QuiltSpecification { if (source) { Files.delete(source) } } - @Ignore + @IgnoreIf({ env.WRITE_BUCKET == 'quilt-example' || env.WRITE_BUCKET == null }) void 'copy a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) @@ -212,11 +212,11 @@ class QuiltNioTest extends QuiltSpecification { readObject(path).trim() == TEXT } - @Ignore + @Ignore('QuiltFileSystem.copy not implemented') void 'move a remote file to a bucket'() { given: Path path = Paths.get(new URI(WRITE_URL)) - final source_url = WRITE_URL.replace('test_folder', 'source') + final source_url = WRITE_URL.replace('folder', 'source') final source = Paths.get(new URI(source_url)) Files.write(source, TEXT.bytes) and: @@ -471,7 +471,7 @@ class QuiltNioTest extends QuiltSpecification { thrown(FileSystemException) } - @Ignore + @Ignore('Can not write to null_path') void 'should stream directory content'() { given: makeObject(null_path('foo/file1.txt'), 'A') @@ -515,7 +515,7 @@ class QuiltNioTest extends QuiltSpecification { list == [ 'file4.txt' ] } - @Ignore + @Ignore('Can not write to null_path') void 'should check walkTree'() { given: makeObject(null_path('foo/file1.txt'), 'A')