start of work to add singularity build from Dockerfile support

This is a prototype or proof of concept that we can build from Dockerfile The work here currently includes support for FROM/ADD/COPY/ENV/LABEL/CMD/ENTRYPOINT and is setup to also include support for multistage builds (although this needs to be finished and I need to look into how singularity handles this with respect to copying files between stages. There is no expectation of merging this, it was a lot of fun to work on / figure out and I learned a lot! Signed-off-by: vsoch <[email protected]>
sylabs · Jun 22, 2022 · 8ae1631 · 8ae1631
1 parent 34dedda
commit 8ae1631
Show file tree

Hide file tree

Showing 101 changed files with 1,029 additions and 93 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,7 @@
 - `DOCKER_USERNAME` and `DOCKER_PASSWORD` supported without `SINGULARITY_` prefix.
 - The `--no-mount` flag now accepts the value `bind-paths` to disable mounting of
   all `bind path` entries in `singularity.conf.
+- ability to build directly from `Dockerfile`
 
 ### Bug Fixes
 

diff --git a/LICENSE_DEPENDENCIES.md b/LICENSE_DEPENDENCIES.md
@@ -23,6 +23,12 @@ The dependencies and their licenses are as follows:
 
 **License URL:** <https://github.com/containerd/containerd/blob/master/LICENSE>
 
+## github.com/containerd/typeurl
+
+**License:** Apache-2.0
+
+**License URL:** <https://github.com/containerd/typeurl/blob/master/LICENSE>
+
 ## github.com/containernetworking/cni
 
 **License:** Apache-2.0
@@ -137,6 +143,12 @@ The dependencies and their licenses are as follows:
 
 **License URL:** <https://github.com/matttproud/golang_protobuf_extensions/blob/master/pbutil/LICENSE>
 
+## github.com/moby/buildkit
+
+**License:** Apache-2.0
+
+**License URL:** <https://github.com/moby/buildkit/blob/master/LICENSE>
+
 ## github.com/moby/locker
 
 **License:** Apache-2.0
@@ -365,11 +377,11 @@ The dependencies and their licenses are as follows:
 
 **License URL:** <https://github.com/cyphar/filepath-securejoin/blob/master/LICENSE>
 
-## github.com/gogo/protobuf/proto
+## github.com/gogo/protobuf
 
 **License:** BSD-3-Clause
 
-**License URL:** <https://github.com/gogo/protobuf/blob/master/proto/LICENSE>
+**License URL:** <https://github.com/gogo/protobuf/blob/master/LICENSE>
 
 ## github.com/golang/protobuf
 

diff --git a/docs/content.go b/docs/content.go
@@ -47,14 +47,15 @@ Enterprise Performance Computing (EPC)`
 
   BUILD SPEC:
 
-  The build spec target is a definition (def) file, local image, or URI that can 
-  be used to create a Singularity container. Several different local target 
-  formats exist:
-
-      def file  : This is a recipe for building a container (examples below)
-      directory:  A directory structure containing a (ch)root file system
-      image:      A local image on your machine (will convert to sif if
-                  it is legacy format)
+  The build spec target is a definition (def) file, local image, a Dockerfile
+  (with more limited support) or URI that can be used to create a Singularity
+  container. Several different local target formats exist:
+
+      def file   : This is a recipe for building a container (examples below)
+      Dockerfile : with support for ADD/COPY/FROM/RUN/ENTRYPOINT/CMD/LABEL/ENV
+      directory: : A directory structure containing a (ch)root file system
+      image:     : A local image on your machine (will convert to sif if
+                   it is legacy format)
 
   Targets can also be remote and defined by a URI of the following formats:
 

diff --git a/go.mod b/go.mod
@@ -21,6 +21,7 @@ require (
 	github.com/go-log/log v0.2.0
 	github.com/google/uuid v1.3.0
 	github.com/gosimple/slug v1.12.0
+	github.com/moby/buildkit v0.10.3
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84
@@ -65,6 +66,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cilium/ebpf v0.7.0 // indirect
 	github.com/containerd/cgroups v1.0.3 // indirect
+	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
 	github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f // indirect
 	github.com/containers/storage v1.40.0 // indirect
@@ -106,7 +108,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
 	github.com/moby/locker v1.0.1 // indirect
-	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mount v0.3.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

diff --git a/go.sum b/go.sum
diff --git a/internal/pkg/build/build.go b/internal/pkg/build/build.go
@@ -534,7 +534,8 @@ func MakeAllDefs(spec string) ([]types.Definition, error) {
 	}
 	defer defFile.Close()
 
-	d, err := parser.All(defFile)
+	// The Dockerfile parser needs to read the raw file
+	d, err := parser.All(defFile, spec)
 	if err != nil {
 		return nil, fmt.Errorf("while parsing definition: %s: %v", spec, err)
 	}

diff --git a/internal/pkg/build/sources/conveyorPacker_scratch_test.go b/internal/pkg/build/sources/conveyorPacker_scratch_test.go
@@ -17,7 +17,7 @@ import (
 	"github.com/sylabs/singularity/pkg/build/types/parser"
 )
 
-const scratchDef = "../../../../pkg/build/types/parser/testdata_good/scratch/scratch"
+const scratchDef = "../../../../pkg/build/types/parser/deffile_test/testdata_good/scratch/scratch"
 
 func TestScratchConveyor(t *testing.T) {
 	if testing.Short() {

diff --git a/pkg/build/types/definition_test.go b/pkg/build/types/definition_test.go
@@ -42,7 +42,7 @@ func TestNewDefinitionFromJSON(t *testing.T) {
 		{JSON: `{"Key1": "Value1", "Key2": "Value2."}`, shouldPass: true},
 	}
 
-	const singularityJSON = "parser/testdata_good/docker/docker.json"
+	const singularityJSON = "parser/deffile_test/testdata_good/docker/docker.json"
 	// We do not have a valid example file that we can use to reach the corner cases, so we define a fake JSON
 	const validSingularityJSON = `
 {

diff --git a/pkg/build/types/parser/deffile.go b/pkg/build/types/parser/deffile.go
@@ -132,6 +132,23 @@ func getSectionName(line string) string {
 	return lineSplit[0]
 }
 
+// parseFileLin is a shared function to get a src, dest from a single line
+func parseFileLine(line string) (string, string) {
+	var src, dst string
+	// Split at space, but not within double quotes
+	lineSubs := fileSplitter.FindAllString(line, -1)
+	if len(lineSubs) < 2 {
+		src = strings.TrimSpace(lineSubs[0])
+		dst = ""
+	} else {
+		src = strings.TrimSpace(lineSubs[0])
+		dst = strings.TrimSpace(lineSubs[1])
+	}
+	src = strings.Trim(src, "\"")
+	dst = strings.Trim(dst, "\"")
+	return src, dst
+}
+
 // parseTokenSection into appropriate components to be placed into a types.Script struct
 func parseTokenSection(tok string, sections map[string]*types.Script, files *[]types.Files, appOrder *[]string) error {
 	split := strings.SplitN(tok, "\n", 2)
@@ -156,18 +173,7 @@ func parseTokenSection(tok string, sections map[string]*types.Script, files *[]t
 			if line = strings.TrimSpace(line); line == "" || strings.Index(line, "#") == 0 {
 				continue
 			}
-			var src, dst string
-			// Split at space, but not within double quotes
-			lineSubs := fileSplitter.FindAllString(line, -1)
-			if len(lineSubs) < 2 {
-				src = strings.TrimSpace(lineSubs[0])
-				dst = ""
-			} else {
-				src = strings.TrimSpace(lineSubs[0])
-				dst = strings.TrimSpace(lineSubs[1])
-			}
-			src = strings.Trim(src, "\"")
-			dst = strings.Trim(dst, "\"")
+			src, dst := parseFileLine(line)
 			f.Files = append(f.Files, types.FileTransport{Src: src, Dst: dst})
 		}
 
@@ -448,16 +454,33 @@ func ParseDefinitionFile(r io.Reader) (d types.Definition, err error) {
 // All receives a reader from a definition file
 // and parses it into a slice of Definition structs or returns error if
 // an error is encounter while parsing
-func All(r io.Reader) ([]types.Definition, error) {
-	var stages []types.Definition
-
+func All(r io.Reader, spec string) ([]types.Definition, error) {
 	raw, err := ioutil.ReadAll(r)
 	if err != nil {
 		return nil, fmt.Errorf("while attempting to read in definition: %v", err)
 	}
 
 	// copy raw data for parsing
 	buf := raw
+
+	// Determine if Singularity recipe (or Docker)
+	dockerRgx := regexp.MustCompile(`(?mi)^FROM `)
+	dfrom := dockerRgx.FindAllIndex(buf, -1)
+
+	// If we found a FROM with space, assume Dockerfile
+	if len(dfrom) > 0 {
+		return ParseDockerfile(spec, raw)
+	}
+	return ParseSingularityDefinition(raw)
+}
+
+// ParseSingularityDefinition breaks a buffer into sections (stages) and returns them parsed
+func ParseSingularityDefinition(raw []byte) ([]types.Definition, error) {
+	var stages []types.Definition
+
+	// copy raw data for parsing
+	buf := raw
+
 	rgx := regexp.MustCompile(`(?mi)^bootstrap:`)
 	i := rgx.FindAllIndex(buf, -1)
 
@@ -474,6 +497,7 @@ func All(r io.Reader) ([]types.Definition, error) {
 	// handles case of no header
 	splitBuf = append([][]byte{buf[:]}, splitBuf...)
 
+	// Second attempt to quit if everything empty!
 	if len(splitBuf) == 0 {
 		return nil, errEmptyDefinition
 	}
@@ -483,6 +507,8 @@ func All(r io.Reader) ([]types.Definition, error) {
 			continue
 		}
 
+		// Both regular definition parser and Docker return
+		// equivalent sections to be further processed
 		d, err := ParseDefinitionFile(bytes.NewReader(stage))
 		if err != nil {
 			if err == errEmptyDefinition {
@@ -493,10 +519,8 @@ func All(r io.Reader) ([]types.Definition, error) {
 
 		stages = append(stages, d)
 	}
-
 	// set raw of last stage to be entire specification
 	stages[len(stages)-1].Raw = raw
-
 	return stages, nil
 }