From 0c37d99c4f19b2458918f28e755cdd802ff49fad Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Fri, 10 Nov 2023 14:20:26 -0500 Subject: [PATCH 1/7] oci: support --app w/"run" command --- .../runtime/launcher/oci/launcher_linux.go | 4 -- .../pkg/runtime/launcher/oci/process_linux.go | 43 ++++++++++++++++++- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 9201664ca1..d2437a1c58 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -192,10 +192,6 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "ContainAll") } - if lo.AppName != "" { - badOpt = append(badOpt, "AppName") - } - if lo.KeyInfo != nil { badOpt = append(badOpt, "KeyInfo") } diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 5d31cf36df..5a66b545ac 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "syscall" @@ -25,7 +26,10 @@ import ( "golang.org/x/term" ) -const singularityLibs = "/.singularity.d/libs" +const ( + singularityLibs = "/.singularity.d/libs" + scifExecutableName = "scif" +) // Script that can be run by /bin/sh to emulate native mode shell behavior. // Set Singularity> prompt, try bash --norc, fall back to sh. @@ -87,6 +91,20 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, bund return nil, nil, fmt.Errorf("while getting ProcessArgs: %w", err) } sylog.Debugf("Native SIF container process/args: %v", args) + case l.cfg.AppName != "": + sylog.Debugf("SCIF app %q requested", l.cfg.AppName) + specArgs := getSpecArgs(imgSpec) + if len(specArgs) < 1 { + return nil, nil, fmt.Errorf("could not determine executable for container") + } + if filepath.Base(specArgs[0]) != scifExecutableName { + sylog.Warningf("OCI mode: SCIF app requested (%q) but container entrypoint does not seem to be a %s executable (container command-line: %q)", l.cfg.AppName, scifExecutableName, strings.Join(specArgs, " ")) + } + args, err = l.prepareArgsForSCIF(specArgs, ep) + if err != nil { + return nil, nil, err + } + sylog.Debugf("args after prepareArgsForSCIF(): %v", args) case ep.Action == "shell": // OCI-SIF shell handling to emulate native runtime shell args = []string{"/bin/sh", "-c", ociShellScript} @@ -108,6 +126,23 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, bund return &p, rtEnv, nil } +func (l *Launcher) prepareArgsForSCIF(specArgs []string, ep launcher.ExecParams) ([]string, error) { + switch ep.Action { + case "run": + args := []string{specArgs[0], "run", l.cfg.AppName} + args = append(args, specArgs[1:]...) + if ep.Process != "" { + args = append(args, ep.Process) + } + if len(ep.Args) > 0 { + args = append(args, ep.Args...) + } + return args, nil + } + + return []string{}, fmt.Errorf("unrecognized action %q", ep.Action) +} + // getProcessTerminal determines whether the container process should run with a terminal. func getProcessTerminal() bool { // Sets the default Process.Terminal to false if our stdin is not a terminal. @@ -135,6 +170,12 @@ func getProcessArgs(imageSpec imgspecv1.Image, ep launcher.ExecParams) []string return processArgs } +// getSpecArgs attempts to get the command-line args that the OCI container was +// built to run. +func getSpecArgs(imageSpec imgspecv1.Image) []string { + return append(imageSpec.Config.Entrypoint, imageSpec.Config.Cmd...) +} + // getProcessCwd computes the Cwd that the container process should start in. // Default in OCI mode, like native --compat, is $HOME. // In native emulation (--no-compat), we use the CWD. From 63ef171f71d26681c116bc628546820a6fceb373 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Fri, 10 Nov 2023 15:37:55 -0500 Subject: [PATCH 2/7] oci: extend SCIF --app support to all "action" cmds --- internal/pkg/runtime/launcher/oci/process_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 5a66b545ac..0084bb5cd3 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -128,8 +128,8 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, bund func (l *Launcher) prepareArgsForSCIF(specArgs []string, ep launcher.ExecParams) ([]string, error) { switch ep.Action { - case "run": - args := []string{specArgs[0], "run", l.cfg.AppName} + case "run", "exec", "shell": + args := []string{specArgs[0], ep.Action, l.cfg.AppName} args = append(args, specArgs[1:]...) if ep.Process != "" { args = append(args, ep.Process) From ede5acb0573ff8b33868796f3cb1a19d10f1b209 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Mon, 13 Nov 2023 20:10:27 -0500 Subject: [PATCH 3/7] move Go template execution for tests into gen-purpose pkg --- e2e/imgbuild/imgbuild.go | 46 +++++------------------------ internal/pkg/test/tool/tmpl/tmpl.go | 46 +++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 38 deletions(-) create mode 100644 internal/pkg/test/tool/tmpl/tmpl.go diff --git a/e2e/imgbuild/imgbuild.go b/e2e/imgbuild/imgbuild.go index c063328c5d..9d60f93d24 100644 --- a/e2e/imgbuild/imgbuild.go +++ b/e2e/imgbuild/imgbuild.go @@ -20,13 +20,13 @@ import ( "path/filepath" "strings" "testing" - "text/template" "time" "github.com/sylabs/singularity/v4/e2e/ecl" "github.com/sylabs/singularity/v4/e2e/internal/e2e" "github.com/sylabs/singularity/v4/e2e/internal/testhelper" "github.com/sylabs/singularity/v4/internal/pkg/test/tool/require" + "github.com/sylabs/singularity/v4/internal/pkg/test/tool/tmpl" "github.com/sylabs/singularity/v4/internal/pkg/util/fs" ) @@ -1962,7 +1962,7 @@ func (c imgBuildTests) buildUseExistingBuildkitd(t *testing.T) { imageNoPrefix := strings.TrimPrefix(c.env.TestRegistryImage, "docker://") tmplValues := struct{ Source string }{Source: imageNoPrefix} - dockerfileSimple := c.createDockerfileFromTmpl(t, tmpdir, filepath.Join("..", "test", "defs", "Dockerfile.simple.tmpl"), tmplValues) + dockerfileSimple := tmpl.Execute(t, tmpdir, "Dockerfile-", filepath.Join("..", "test", "defs", "Dockerfile.simple.tmpl"), tmplValues) outputImgPath := filepath.Join(tmpdir, "image.oci.sif") buildkitd, err := exec.LookPath("buildkitd") @@ -2091,12 +2091,12 @@ func (c imgBuildTests) buildDockerfile(t *testing.T) { Source: imageNoPrefix, AddFile: "/this_should_not_exist/this_should_not_exist_either", } - dockerfileSimple := c.createDockerfileFromTmpl(t, tmpdir, filepath.Join("..", "test", "defs", "Dockerfile.simple.tmpl"), tmplValues) - dockerfileBroken := c.createDockerfileFromTmpl(t, tmpdir, filepath.Join("..", "test", "defs", "Dockerfile.broken.tmpl"), tmplValues) - dockerfileBuildArgs := c.createDockerfileFromTmpl(t, tmpdir, filepath.Join("..", "test", "defs", "Dockerfile.buildargs.tmpl"), tmplValues) - dockerfileBuildArgsNoDef := c.createDockerfileFromTmpl(t, tmpdir, filepath.Join("..", "test", "defs", "Dockerfile.buildargs-nodefault.tmpl"), tmplValues) - dockerfileAdd := c.createDockerfileFromTmpl(t, tmpdir, filepath.Join("..", "test", "defs", "Dockerfile.add.tmpl"), tmplValues) - dockerfileAddBad := c.createDockerfileFromTmpl(t, tmpdir, filepath.Join("..", "test", "defs", "Dockerfile.add.tmpl"), badAddValues) + dockerfileSimple := tmpl.Execute(t, tmpdir, "Dockerfile-", filepath.Join("..", "test", "defs", "Dockerfile.simple.tmpl"), tmplValues) + dockerfileBroken := tmpl.Execute(t, tmpdir, "Dockerfile-", filepath.Join("..", "test", "defs", "Dockerfile.broken.tmpl"), tmplValues) + dockerfileBuildArgs := tmpl.Execute(t, tmpdir, "Dockerfile-", filepath.Join("..", "test", "defs", "Dockerfile.buildargs.tmpl"), tmplValues) + dockerfileBuildArgsNoDef := tmpl.Execute(t, tmpdir, "Dockerfile-", filepath.Join("..", "test", "defs", "Dockerfile.buildargs-nodefault.tmpl"), tmplValues) + dockerfileAdd := tmpl.Execute(t, tmpdir, "Dockerfile-", filepath.Join("..", "test", "defs", "Dockerfile.add.tmpl"), tmplValues) + dockerfileAddBad := tmpl.Execute(t, tmpdir, "Dockerfile-", filepath.Join("..", "test", "defs", "Dockerfile.add.tmpl"), badAddValues) outputImgPath := filepath.Join(tmpdir, "image.oci.sif") @@ -2286,36 +2286,6 @@ func (c imgBuildTests) buildDockerfile(t *testing.T) { } } -func (c imgBuildTests) createDockerfileFromTmpl(t *testing.T, tmpdir, tmplPath string, values any) string { - dockerfile, err := os.CreateTemp(tmpdir, "Dockerfile-") - if err != nil { - t.Fatalf("failed to open temp file: %v", err) - } - dockerfileName := dockerfile.Name() - t.Cleanup(func() { - if !t.Failed() { - os.Remove(dockerfileName) - } - }) - defer dockerfile.Close() - - tmplBytes, err := os.ReadFile(tmplPath) - if err != nil { - t.Fatalf("While trying to read template file %q: %v", tmplPath, err) - } - tmpl, err := template.New(filepath.Base(dockerfileName)).Parse(string(tmplBytes)) - if err != nil { - t.Fatalf("While trying to parse template file %q: %v", tmplPath, err) - } - - err = tmpl.Execute(dockerfile, values) - if err != nil { - t.Fatalf("While trying to execute template %q: %v", tmplPath, err) - } - - return dockerfileName -} - func (c imgBuildTests) buildWithAuth(t *testing.T) { e2e.EnsureImage(t, c.env) diff --git a/internal/pkg/test/tool/tmpl/tmpl.go b/internal/pkg/test/tool/tmpl/tmpl.go new file mode 100644 index 0000000000..4cff69fda1 --- /dev/null +++ b/internal/pkg/test/tool/tmpl/tmpl.go @@ -0,0 +1,46 @@ +// Copyright (c) 2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package tmpl + +import ( + "html/template" + "os" + "path/filepath" + "testing" +) + +// Execute creates a file in tmpdir based on namePattern whose contents are the +// result of executing the Go template found in tmplPath over the struct passed +// in the values argument. Returns the full path of the created file. +func Execute(t *testing.T, tmpdir, namePattern, tmplPath string, values any) string { + outfile, err := os.CreateTemp(tmpdir, namePattern) + if err != nil { + t.Fatalf("failed to open temp file: %v", err) + } + outfilePath := outfile.Name() + t.Cleanup(func() { + if !t.Failed() { + os.Remove(outfilePath) + } + }) + defer outfile.Close() + + tmplBytes, err := os.ReadFile(tmplPath) + if err != nil { + t.Fatalf("While trying to read template file %q: %v", tmplPath, err) + } + tmpl, err := template.New(filepath.Base(outfilePath)).Parse(string(tmplBytes)) + if err != nil { + t.Fatalf("While trying to parse template file %q: %v", tmplPath, err) + } + + err = tmpl.Execute(outfile, values) + if err != nil { + t.Fatalf("While trying to execute template %q: %v", tmplPath, err) + } + + return outfilePath +} From 08d86a9d3732710538808a079b33955819b43114 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Mon, 13 Nov 2023 20:36:39 -0500 Subject: [PATCH 4/7] e2e: initial infrastructure for testing OCI-mode SCIF --- e2e/docker/docker.go | 68 +++++++++++++++++++++++++++++ internal/pkg/test/tool/tmpl/tmpl.go | 5 ++- test/defs/Dockerfile.scif.tmpl | 9 ++++ test/defs/scif_recipe | 14 ++++++ 4 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 test/defs/Dockerfile.scif.tmpl create mode 100644 test/defs/scif_recipe diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index b2d0a211ae..74be549de4 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -29,6 +29,7 @@ import ( "github.com/sylabs/singularity/v4/e2e/internal/e2e" "github.com/sylabs/singularity/v4/e2e/internal/testhelper" "github.com/sylabs/singularity/v4/internal/pkg/test/tool/require" + "github.com/sylabs/singularity/v4/internal/pkg/test/tool/tmpl" "github.com/sylabs/singularity/v4/internal/pkg/util/fs" "golang.org/x/sys/unix" "gotest.tools/assert" @@ -1571,6 +1572,72 @@ func verifyImgArch(t *testing.T, imgPath, arch string) { assert.Equal(t, arch, cg.Architecture) } +// Test support for SCIF containers in OCI mode +func (c ctx) testDockerSCIF(t *testing.T) { + tmpdir, tmpdirCleanup := e2e.MakeTempDir(t, "", "docker-scif-", "dir") + t.Cleanup(func() { + if !t.Failed() { + tmpdirCleanup(t) + } + }) + + scifRecipeFilename := "local_scif_recipe" + scifRecipeFullpath := filepath.Join(tmpdir, scifRecipeFilename) + scifRecipeSource := filepath.Join("..", "test", "defs", "scif_recipe") + if err := fs.CopyFile(scifRecipeSource, scifRecipeFullpath, 0o755); err != nil { + t.Fatalf("While trying to copy %q to %q: %v", scifRecipeSource, scifRecipeFullpath, err) + } + + tmplValues := struct{ SCIFRecipeFilename string }{SCIFRecipeFilename: scifRecipeFilename} + scifDockerfile := tmpl.Execute(t, tmpdir, "Dockerfile-", filepath.Join("..", "test", "defs", "Dockerfile.scif.tmpl"), tmplValues) + scifImageFilename := "scif-image.oci.sif" + scifImageFullpath := filepath.Join(tmpdir, scifImageFilename) + + c.env.RunSingularity( + t, + e2e.AsSubtest("build"), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("build"), + e2e.WithDir(tmpdir), + e2e.WithArgs("--oci", scifImageFilename, scifDockerfile), + e2e.ExpectExit(0), + ) + + tests := []struct { + name string + cmd string + app string + expects []e2e.SingularityCmdResultOp + expectExit int + }{ + { + name: "echo", + cmd: "run", + app: "hello-world-echo", + expects: []e2e.SingularityCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "The best app is hello-world-echo"), + }, + expectExit: 0, + }, + } + + for _, tt := range tests { + args := []string{} + if tt.app != "" { + args = append(args, "--app", tt.app) + } + args = append(args, scifImageFullpath) + c.env.RunSingularity( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand(tt.cmd), + e2e.WithArgs(args...), + e2e.ExpectExit(0, tt.expects...), + ) + } +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := ctx{ @@ -1597,6 +1664,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { t.Run("user", c.testDockerUSER) t.Run("platform", c.testDockerPlatform) t.Run("crossarch buildkit", c.testDockerCrossArchBk) + t.Run("scif", c.testDockerSCIF) // Regressions t.Run("issue 4524", c.issue4524) t.Run("issue 1286", c.issue1286) diff --git a/internal/pkg/test/tool/tmpl/tmpl.go b/internal/pkg/test/tool/tmpl/tmpl.go index 4cff69fda1..e06dec6805 100644 --- a/internal/pkg/test/tool/tmpl/tmpl.go +++ b/internal/pkg/test/tool/tmpl/tmpl.go @@ -13,8 +13,9 @@ import ( ) // Execute creates a file in tmpdir based on namePattern whose contents are the -// result of executing the Go template found in tmplPath over the struct passed -// in the values argument. Returns the full path of the created file. +// result of executing the Go template in tmplPath, over the struct passed in +// the values argument. Returns the full path of the created file. The created +// file will be automatically removed at the end of the test t unless t fails. func Execute(t *testing.T, tmpdir, namePattern, tmplPath string, values any) string { outfile, err := os.CreateTemp(tmpdir, namePattern) if err != nil { diff --git a/test/defs/Dockerfile.scif.tmpl b/test/defs/Dockerfile.scif.tmpl new file mode 100644 index 0000000000..4157a8c8b5 --- /dev/null +++ b/test/defs/Dockerfile.scif.tmpl @@ -0,0 +1,9 @@ +FROM continuumio/miniconda3 + +RUN pip install scif + +ADD ./{{ .SCIFRecipeFilename }} / + +RUN scif install /{{ .SCIFRecipeFilename }} + +CMD ["scif"] diff --git a/test/defs/scif_recipe b/test/defs/scif_recipe new file mode 100644 index 0000000000..170e46609f --- /dev/null +++ b/test/defs/scif_recipe @@ -0,0 +1,14 @@ +%appenv hello-world-echo + THEBESTAPP=$SCIF_APPNAME + export THEBESTAPP +%apprun hello-world-echo + echo "The best app is $THEBESTAPP" + +%appinstall hello-world-script + echo "echo 'Hello World!'" >> bin/hello-world.sh + chmod u+x bin/hello-world.sh +%appenv hello-world-script + THEBESTAPP=$SCIF_APPNAME + export THEBESTAPP +%apprun hello-world-script + /bin/bash hello-world.sh From 1340fa6b60732bacadd7776c154952e56119289a Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Tue, 14 Nov 2023 10:41:40 -0500 Subject: [PATCH 5/7] e2e: fill out e2e tests for docker scif --- e2e/docker/docker.go | 86 +++++++++++++++++++++++- test/defs/scif_recipe.inspect_output.all | 32 +++++++++ test/defs/scif_recipe.inspect_output.one | 19 ++++++ 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 test/defs/scif_recipe.inspect_output.all create mode 100644 test/defs/scif_recipe.inspect_output.one diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index 74be549de4..6dbf661147 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -1593,6 +1593,24 @@ func (c ctx) testDockerSCIF(t *testing.T) { scifImageFilename := "scif-image.oci.sif" scifImageFullpath := filepath.Join(tmpdir, scifImageFilename) + scifInspectOutAllPath := filepath.Join("..", "test", "defs", "scif_recipe.inspect_output.all") + scifInspectOutAllBytes, err := os.ReadFile(scifInspectOutAllPath) + if err != nil { + t.Fatalf("While trying to read contents of %s: %v", scifInspectOutAllPath, err) + } + scifInspectOutOnePath := filepath.Join("..", "test", "defs", "scif_recipe.inspect_output.one") + scifInspectOutOneBytes, err := os.ReadFile(scifInspectOutOnePath) + if err != nil { + t.Fatalf("While trying to read contents of %s: %v", scifInspectOutOnePath, err) + } + + testInspectOutput := func(bytes []byte) func(t *testing.T, r *e2e.SingularityCmdResult) { + return func(t *testing.T, r *e2e.SingularityCmdResult) { + got := string(r.Stdout) + assert.Equal(t, got, string(bytes)) + } + } + c.env.RunSingularity( t, e2e.AsSubtest("build"), @@ -1607,11 +1625,13 @@ func (c ctx) testDockerSCIF(t *testing.T) { name string cmd string app string + preArgs []string + args []string expects []e2e.SingularityCmdResultOp expectExit int }{ { - name: "echo", + name: "run echo", cmd: "run", app: "hello-world-echo", expects: []e2e.SingularityCmdResultOp{ @@ -1619,14 +1639,76 @@ func (c ctx) testDockerSCIF(t *testing.T) { }, expectExit: 0, }, + { + name: "exec echo", + cmd: "exec", + app: "hello-world-echo", + args: []string{"echo", "This is different text that should still include [e]SCIF_APPNAME"}, + expects: []e2e.SingularityCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "This is different text that should still include hello-world-echo"), + }, + expectExit: 0, + }, + { + name: "run script", + cmd: "run", + app: "hello-world-script", + expects: []e2e.SingularityCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "Hello World!"), + }, + expectExit: 0, + }, + { + name: "exec script", + cmd: "exec", + app: "hello-world-script", + args: []string{"echo", "This is different text that should still include [e]SCIF_APPNAME"}, + expects: []e2e.SingularityCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "This is different text that should still include hello-world-script"), + }, + expectExit: 0, + }, + { + name: "exec script2", + cmd: "exec", + app: "hello-world-script", + args: []string{"/bin/bash hello-world.sh"}, + expects: []e2e.SingularityCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "Hello World!"), + }, + expectExit: 0, + }, + { + name: "insp all", + cmd: "inspect", + preArgs: []string{"--oci"}, + expects: []e2e.SingularityCmdResultOp{ + testInspectOutput(scifInspectOutAllBytes), + }, + expectExit: 0, + }, + { + name: "insp one", + cmd: "inspect", + app: "hello-world-script", + preArgs: []string{"--oci"}, + expects: []e2e.SingularityCmdResultOp{ + testInspectOutput(scifInspectOutOneBytes), + }, + expectExit: 0, + }, } for _, tt := range tests { - args := []string{} + args := tt.preArgs[:] if tt.app != "" { args = append(args, "--app", tt.app) } args = append(args, scifImageFullpath) + if len(tt.args) > 0 { + args = append(args, tt.args...) + } + c.env.RunSingularity( t, e2e.AsSubtest(tt.name), diff --git a/test/defs/scif_recipe.inspect_output.all b/test/defs/scif_recipe.inspect_output.all new file mode 100644 index 0000000000..208bda31e4 --- /dev/null +++ b/test/defs/scif_recipe.inspect_output.all @@ -0,0 +1,32 @@ +{ + "hello-world-echo": { + "appenv": [ + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP" + ], + "apprun": [ + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP", + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP", + " echo \"The best app is $THEBESTAPP\"" + ] + }, + "hello-world-script": { + "appinstall": [ + " echo \"echo 'Hello World!'\" >> bin/hello-world.sh", + " chmod u+x bin/hello-world.sh" + ], + "appenv": [ + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP" + ], + "apprun": [ + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP", + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP", + " /bin/bash hello-world.sh" + ] + } +} diff --git a/test/defs/scif_recipe.inspect_output.one b/test/defs/scif_recipe.inspect_output.one new file mode 100644 index 0000000000..65dc6ee7b7 --- /dev/null +++ b/test/defs/scif_recipe.inspect_output.one @@ -0,0 +1,19 @@ +{ + "hello-world-script": { + "appinstall": [ + " echo \"echo 'Hello World!'\" >> bin/hello-world.sh", + " chmod u+x bin/hello-world.sh" + ], + "appenv": [ + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP" + ], + "apprun": [ + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP", + "THEBESTAPP=$SCIF_APPNAME", + "export THEBESTAPP", + " /bin/bash hello-world.sh" + ] + } +} From cd9d0987c157a556234d65528fa0b1d6dffddbdb Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Tue, 14 Nov 2023 10:53:01 -0500 Subject: [PATCH 6/7] CHANGELOG entry --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15186a0fc3..0fb484a992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,16 @@ builds, `--authfile` and other authentication options, and more. See the [user guide](https://docs.sylabs.io/guides/latest/user-guide/build_a_container.html#dockerfile) for more information. +- Docker-style SCIF containers + ([https://sci-f.github.io/tutorial-preview-install](https://sci-f.github.io/tutorial-preview-install)) + are now supported. If the entrypoint of an OCI container is the `scif` + executable, then the `run` / `exec` / `shell` commands in `--oci` mode can be + given the `--app ` flag, and will automatically invoke the relevant + SCIF command. For example: + `singularity run --oci --app myapp mycontainer.oci.sif arg1 arg2` will + automatically run `scif run myapp arg1 arg2` inside the container. The + `inspect --oci` command will also run `scif inspect`, or + `scif inspect ` if used with the `--app appname` flag. ### Bug Fixes From 8ddbff6ddc9ebbf3178e804e4697124d9b6ed887 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Tue, 14 Nov 2023 11:51:10 -0500 Subject: [PATCH 7/7] address review comments --- CHANGELOG.md | 6 +- e2e/docker/docker.go | 77 ++++++++++--------- .../pkg/runtime/launcher/oci/process_linux.go | 4 +- 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fb484a992..dd1e7cb21b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,11 +32,7 @@ are now supported. If the entrypoint of an OCI container is the `scif` executable, then the `run` / `exec` / `shell` commands in `--oci` mode can be given the `--app ` flag, and will automatically invoke the relevant - SCIF command. For example: - `singularity run --oci --app myapp mycontainer.oci.sif arg1 arg2` will - automatically run `scif run myapp arg1 arg2` inside the container. The - `inspect --oci` command will also run `scif inspect`, or - `scif inspect ` if used with the `--app appname` flag. + SCIF command. ### Bug Fixes diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index 6dbf661147..9e199bd32d 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -1593,23 +1593,26 @@ func (c ctx) testDockerSCIF(t *testing.T) { scifImageFilename := "scif-image.oci.sif" scifImageFullpath := filepath.Join(tmpdir, scifImageFilename) - scifInspectOutAllPath := filepath.Join("..", "test", "defs", "scif_recipe.inspect_output.all") - scifInspectOutAllBytes, err := os.ReadFile(scifInspectOutAllPath) - if err != nil { - t.Fatalf("While trying to read contents of %s: %v", scifInspectOutAllPath, err) - } - scifInspectOutOnePath := filepath.Join("..", "test", "defs", "scif_recipe.inspect_output.one") - scifInspectOutOneBytes, err := os.ReadFile(scifInspectOutOnePath) - if err != nil { - t.Fatalf("While trying to read contents of %s: %v", scifInspectOutOnePath, err) - } - - testInspectOutput := func(bytes []byte) func(t *testing.T, r *e2e.SingularityCmdResult) { - return func(t *testing.T, r *e2e.SingularityCmdResult) { - got := string(r.Stdout) - assert.Equal(t, got, string(bytes)) - } - } + // Uncomment when `singularity inspect --oci` for Docker-style SCIF + // containers is enabled. + // See: https://github.com/sylabs/singularity/pull/2360 + // scifInspectOutAllPath := filepath.Join("..", "test", "defs", "scif_recipe.inspect_output.all") + // scifInspectOutAllBytes, err := os.ReadFile(scifInspectOutAllPath) + // if err != nil { + // t.Fatalf("While trying to read contents of %s: %v", scifInspectOutAllPath, err) + // } + // scifInspectOutOnePath := filepath.Join("..", "test", "defs", "scif_recipe.inspect_output.one") + // scifInspectOutOneBytes, err := os.ReadFile(scifInspectOutOnePath) + // if err != nil { + // t.Fatalf("While trying to read contents of %s: %v", scifInspectOutOnePath, err) + // } + + // testInspectOutput := func(bytes []byte) func(t *testing.T, r *e2e.SingularityCmdResult) { + // return func(t *testing.T, r *e2e.SingularityCmdResult) { + // got := string(r.Stdout) + // assert.Equal(t, got, string(bytes)) + // } + // } c.env.RunSingularity( t, @@ -1678,25 +1681,27 @@ func (c ctx) testDockerSCIF(t *testing.T) { }, expectExit: 0, }, - { - name: "insp all", - cmd: "inspect", - preArgs: []string{"--oci"}, - expects: []e2e.SingularityCmdResultOp{ - testInspectOutput(scifInspectOutAllBytes), - }, - expectExit: 0, - }, - { - name: "insp one", - cmd: "inspect", - app: "hello-world-script", - preArgs: []string{"--oci"}, - expects: []e2e.SingularityCmdResultOp{ - testInspectOutput(scifInspectOutOneBytes), - }, - expectExit: 0, - }, + // Uncomment when `singularity inspect --oci` for Docker-style SCIF + // containers is enabled. + // See: https://github.com/sylabs/singularity/pull/2360 + // { + // name: "insp all", + // cmd: "inspect", + // preArgs: []string{"--oci", "--list-apps"}, + // expects: []e2e.SingularityCmdResultOp{ + // testInspectOutput(scifInspectOutAllBytes), + // }, + // expectExit: 0, + // }, { + // name: "insp one", + // cmd: "inspect", + // app: "hello-world-script", + // preArgs: []string{"--oci"}, + // expects: []e2e.SingularityCmdResultOp{ + // testInspectOutput(scifInspectOutOneBytes), + // }, + // expectExit: 0, + // }, } for _, tt := range tests { diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 0084bb5cd3..5cabe998f7 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -100,7 +100,7 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, bund if filepath.Base(specArgs[0]) != scifExecutableName { sylog.Warningf("OCI mode: SCIF app requested (%q) but container entrypoint does not seem to be a %s executable (container command-line: %q)", l.cfg.AppName, scifExecutableName, strings.Join(specArgs, " ")) } - args, err = l.prepareArgsForSCIF(specArgs, ep) + args, err = l.argsForSCIF(specArgs, ep) if err != nil { return nil, nil, err } @@ -126,7 +126,7 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, bund return &p, rtEnv, nil } -func (l *Launcher) prepareArgsForSCIF(specArgs []string, ep launcher.ExecParams) ([]string, error) { +func (l *Launcher) argsForSCIF(specArgs []string, ep launcher.ExecParams) ([]string, error) { switch ep.Action { case "run", "exec", "shell": args := []string{specArgs[0], ep.Action, l.cfg.AppName}