From d0212a9f8271518a760e298a36e9ef2c31c7890c Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Mon, 9 Dec 2024 16:08:32 -0600 Subject: [PATCH] bake: various fixes for composable attributes This changes how the composable attributes are implemented and provides various fixes to the first iteration. Cache-from and cache-to now no longer print sensitive values that are automatically added. These automatically added attributes are added when the protobuf is created rather than at the time of parsing so they will no longer be printed. If they are part of the original configuration file, they will still be printed. Empty strings will now be skipped. This was the original behavior and composable attributes removed this functionality accidentally. This functionality is now restored. This also expands the available syntax that works with each of the composable attributes. It is now possible to interleave the csv syntax with the object syntax without any problems. The canonical form is still the object syntax and variables are resolved according to that syntax. Signed-off-by: Jonathan A. Sternberg --- bake/bake.go | 103 ++++++++-------- bake/bake_test.go | 20 ++++ bake/compose.go | 10 +- bake/hcl_test.go | 12 +- bake/hclparser/type_implied_ext.go | 97 +++++++++------ tests/bake.go | 125 +++++++++++++++++++- util/buildflags/attests.go | 2 +- util/buildflags/cache.go | 77 ++++++++---- util/buildflags/cache_cty.go | 86 ++++++++++++++ util/buildflags/cache_test.go | 39 ++++++ util/buildflags/context.go | 5 + util/buildflags/cty.go | 183 ----------------------------- util/buildflags/entitlements.go | 4 + util/buildflags/export.go | 48 +++++++- util/buildflags/export_cty.go | 90 ++++++++++++++ util/buildflags/secrets.go | 37 ++++++ util/buildflags/secrets_cty.go | 96 +++++++++++++++ util/buildflags/ssh.go | 37 ++++++ util/buildflags/ssh_cty.go | 105 +++++++++++++++++ util/buildflags/utils.go | 54 +++++++++ 20 files changed, 919 insertions(+), 311 deletions(-) create mode 100644 util/buildflags/cache_cty.go create mode 100644 util/buildflags/cache_test.go delete mode 100644 util/buildflags/cty.go create mode 100644 util/buildflags/export_cty.go create mode 100644 util/buildflags/secrets_cty.go create mode 100644 util/buildflags/ssh_cty.go create mode 100644 util/buildflags/utils.go diff --git a/bake/bake.go b/bake/bake.go index 6f72fb88f07..cd4c1e01b9c 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -698,30 +698,30 @@ type Target struct { // Inherits is the only field that cannot be overridden with --set Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"` - Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"` - Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"` - Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"` - Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"` - Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"` - DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"` - Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"` - Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"` - Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"` - CacheFrom []*buildflags.CacheOptionsEntry `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"` - CacheTo []*buildflags.CacheOptionsEntry `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"` - Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"` - Secrets []*buildflags.Secret `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"` - SSH []*buildflags.SSH `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"` - Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"` - Outputs []*buildflags.ExportEntry `json:"output,omitempty" hcl:"output,optional" cty:"output"` - Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"` - NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"` - NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"` - NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"` - ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"` - Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"` - Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"` - Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"` + Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"` + Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"` + Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"` + Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"` + Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"` + DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"` + Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"` + Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"` + Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"` + CacheFrom buildflags.CacheOptions `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"` + CacheTo buildflags.CacheOptions `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"` + Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"` + Secrets buildflags.Secrets `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"` + SSH buildflags.SSHKeys `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"` + Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"` + Outputs buildflags.Exports `json:"output,omitempty" hcl:"output,optional" cty:"output"` + Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"` + NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"` + NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"` + NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"` + ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"` + Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"` + Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"` + Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"` // IMPORTANT: if you add more fields here, do not forget to update newOverrides/AddOverrides and docs/bake-reference.md. // linked is a private field to mark a target used as a linked one @@ -739,12 +739,12 @@ func (t *Target) normalize() { t.Annotations = removeDupesStr(t.Annotations) t.Attest = removeAttestDupes(t.Attest) t.Tags = removeDupesStr(t.Tags) - t.Secrets = removeDupes(t.Secrets) + t.Secrets = t.Secrets.Normalize() t.SSH = removeDupes(t.SSH) t.Platforms = removeDupesStr(t.Platforms) - t.CacheFrom = removeDupes(t.CacheFrom) - t.CacheTo = removeDupes(t.CacheTo) - t.Outputs = removeDupes(t.Outputs) + t.CacheFrom = t.CacheFrom.Normalize() + t.CacheTo = t.CacheTo.Normalize() + t.Outputs = t.Outputs.Normalize() t.NoCacheFilter = removeDupesStr(t.NoCacheFilter) t.Ulimits = removeDupesStr(t.Ulimits) @@ -815,16 +815,16 @@ func (t *Target) Merge(t2 *Target) { t.Attest = removeAttestDupes(t.Attest) } if t2.Secrets != nil { // merge - t.Secrets = append(t.Secrets, t2.Secrets...) + t.Secrets = t.Secrets.Merge(t2.Secrets) } if t2.SSH != nil { // merge - t.SSH = append(t.SSH, t2.SSH...) + t.SSH = t.SSH.Merge(t2.SSH) } if t2.Platforms != nil { // no merge t.Platforms = t2.Platforms } if t2.CacheFrom != nil { // merge - t.CacheFrom = append(t.CacheFrom, t2.CacheFrom...) + t.CacheFrom = t.CacheFrom.Merge(t2.CacheFrom) } if t2.CacheTo != nil { // no merge t.CacheTo = t2.CacheTo @@ -1333,13 +1333,8 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { } bo.Platforms = platforms - secrets := make([]*controllerapi.Secret, len(t.Secrets)) - for i, s := range t.Secrets { - secrets[i] = s.ToPB() - } - bo.SecretSpecs = secrets - - secretAttachment, err := controllerapi.CreateSecrets(secrets) + bo.SecretSpecs = t.Secrets.ToPB() + secretAttachment, err := controllerapi.CreateSecrets(bo.SecretSpecs) if err != nil { return nil, err } @@ -1372,24 +1367,14 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { } } - cacheImports := make([]*controllerapi.CacheOptionsEntry, len(t.CacheFrom)) - for i, ci := range t.CacheFrom { - cacheImports[i] = ci.ToPB() - } - bo.CacheFrom = controllerapi.CreateCaches(cacheImports) - - cacheExports := make([]*controllerapi.CacheOptionsEntry, len(t.CacheTo)) - for i, ce := range t.CacheTo { - cacheExports[i] = ce.ToPB() + if t.CacheFrom != nil { + bo.CacheFrom = controllerapi.CreateCaches(t.CacheFrom.ToPB()) } - bo.CacheTo = controllerapi.CreateCaches(cacheExports) - - outputs := make([]*controllerapi.ExportEntry, len(t.Outputs)) - for i, output := range t.Outputs { - outputs[i] = output.ToPB() + if t.CacheTo != nil { + bo.CacheTo = controllerapi.CreateCaches(t.CacheTo.ToPB()) } - bo.Exports, bo.ExportsLocalPathsTemporary, err = controllerapi.CreateExports(outputs) + bo.Exports, bo.ExportsLocalPathsTemporary, err = controllerapi.CreateExports(t.Outputs.ToPB()) if err != nil { return nil, err } @@ -1616,6 +1601,10 @@ type arrValue[B any] interface { func parseArrValue[T any, PT arrValue[T]](s []string) ([]*T, error) { outputs := make([]*T, 0, len(s)) for _, text := range s { + if text == "" { + continue + } + output := new(T) if err := PT(output).UnmarshalText([]byte(text)); err != nil { return nil, err @@ -1625,9 +1614,13 @@ func parseArrValue[T any, PT arrValue[T]](s []string) ([]*T, error) { return outputs, nil } -func parseCacheArrValues(s []string) ([]*buildflags.CacheOptionsEntry, error) { - outs := make([]*buildflags.CacheOptionsEntry, 0, len(s)) +func parseCacheArrValues(s []string) (buildflags.CacheOptions, error) { + var outs buildflags.CacheOptions for _, in := range s { + if in == "" { + continue + } + if !strings.Contains(in, "=") { // This is ref only format. Each field in the CSV is its own entry. fields, err := csvvalue.Fields(in, nil) diff --git a/bake/bake_test.go b/bake/bake_test.go index 0abc35ed1ad..efa75502943 100644 --- a/bake/bake_test.go +++ b/bake/bake_test.go @@ -2019,6 +2019,26 @@ target "app" { }) } +// https://github.com/docker/buildx/pull/428 +// https://github.com/docker/buildx/issues/2822 +func TestEmptyAttribute(t *testing.T) { + fp := File{ + Name: "docker-bake.hcl", + Data: []byte(` +target "app" { + output = [""] +} +`), + } + + ctx := context.TODO() + + m, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, nil, nil, &EntitlementConf{}) + require.Equal(t, 1, len(m)) + require.Len(t, m["app"].Outputs, 0) + require.NoError(t, err) +} + func stringify[V fmt.Stringer](values []V) []string { s := make([]string, len(values)) for i, v := range values { diff --git a/bake/compose.go b/bake/compose.go index 4a4facb159f..58cfc80c953 100644 --- a/bake/compose.go +++ b/bake/compose.go @@ -353,28 +353,28 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error { if err != nil { return err } - t.CacheFrom = removeDupes(append(t.CacheFrom, cacheFrom...)) + t.CacheFrom = t.CacheFrom.Merge(cacheFrom) } if len(xb.CacheTo) > 0 { cacheTo, err := parseCacheArrValues(xb.CacheTo) if err != nil { return err } - t.CacheTo = removeDupes(append(t.CacheTo, cacheTo...)) + t.CacheTo = t.CacheTo.Merge(cacheTo) } if len(xb.Secrets) > 0 { secrets, err := parseArrValue[buildflags.Secret](xb.Secrets) if err != nil { return err } - t.Secrets = removeDupes(append(t.Secrets, secrets...)) + t.Secrets = t.Secrets.Merge(secrets) } if len(xb.SSH) > 0 { ssh, err := parseArrValue[buildflags.SSH](xb.SSH) if err != nil { return err } - t.SSH = removeDupes(append(t.SSH, ssh...)) + t.SSH = t.SSH.Merge(ssh) slices.SortFunc(t.SSH, func(a, b *buildflags.SSH) int { return a.Less(b) }) @@ -387,7 +387,7 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error { if err != nil { return err } - t.Outputs = removeDupes(append(t.Outputs, outputs...)) + t.Outputs = t.Outputs.Merge(outputs) } if xb.Pull != nil { t.Pull = xb.Pull diff --git a/bake/hcl_test.go b/bake/hcl_test.go index facf7b790e6..a0d37db15e4 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -606,7 +606,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) { target "app" { cache-from = [ { type = "registry", ref = "user/app:cache" }, - { type = "local", src = "path/to/cache" }, + "type=local,src=path/to/cache", ] cache-to = [ @@ -615,6 +615,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) { output = [ { type = "oci", dest = "../out.tar" }, + "type=local,dest=../out", ] secret = [ @@ -633,7 +634,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(c.Targets)) - require.Equal(t, []string{"type=oci,dest=../out.tar"}, stringify(c.Targets[0].Outputs)) + require.Equal(t, []string{"type=local,dest=../out", "type=oci,dest=../out.tar"}, stringify(c.Targets[0].Outputs)) require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheFrom)) require.Equal(t, []string{"type=local,dest=path/to/cache"}, stringify(c.Targets[0].CacheTo)) require.Equal(t, []string{"id=mysecret,src=/local/secret", "id=mysecret2,env=TOKEN"}, stringify(c.Targets[0].Secrets)) @@ -649,13 +650,14 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) { target "app" { cache-from = [ { type = "registry", ref = "user/app:cache" }, - { type = "local", src = "path/to/cache" }, + "type=local,src=path/to/cache", ] cache-to = [ target.app.cache-from[0] ] output = [ { type = "oci", dest = "../out.tar" }, + "type=local,dest=../out", ] secret = [ @@ -674,7 +676,7 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) { output = [ "type=oci,dest=../${foo}.tar" ] secret = [ - { id = target.app.output[0].type, src = "/local/secret" }, + { id = target.app.output[0].type, src = "/${target.app.cache-from[1].type}/secret" }, ] } `) @@ -696,7 +698,7 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) { } app := findTarget(t, "app") - require.Equal(t, []string{"type=oci,dest=../out.tar"}, stringify(app.Outputs)) + require.Equal(t, []string{"type=local,dest=../out", "type=oci,dest=../out.tar"}, stringify(app.Outputs)) require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(app.CacheFrom)) require.Equal(t, []string{"user/app:cache"}, stringify(app.CacheTo)) require.Equal(t, []string{"id=mysecret,src=/local/secret"}, stringify(app.Secrets)) diff --git a/bake/hclparser/type_implied_ext.go b/bake/hclparser/type_implied_ext.go index fad53f7074c..e1fbd340b38 100644 --- a/bake/hclparser/type_implied_ext.go +++ b/bake/hclparser/type_implied_ext.go @@ -10,44 +10,60 @@ import ( "github.com/zclconf/go-cty/cty/gocty" ) -type CapsuleValue interface { - // FromCtyValue will initialize this value using a cty.Value. - FromCtyValue(in cty.Value, path cty.Path) error - +type ToCtyValueConverter interface { // ToCtyValue will convert this capsule value into a native // cty.Value. This should not return a capsule type. ToCtyValue() cty.Value } +type FromCtyValueConverter interface { + // FromCtyValue will initialize this value using a cty.Value. + FromCtyValue(in cty.Value, path cty.Path) error +} + type extensionType int const ( - nativeTypeExtension extensionType = iota + unwrapCapsuleValueExtension extensionType = iota ) func impliedTypeExt(rt reflect.Type, _ cty.Path) (cty.Type, error) { - if rt.AssignableTo(capsuleValueType) { + if rt.Kind() != reflect.Pointer { + rt = reflect.PointerTo(rt) + } + + if isCapsuleType(rt) { return capsuleValueCapsuleType(rt), nil } return cty.NilType, errdefs.ErrNotImplemented } -var ( - capsuleValueType = reflect.TypeFor[CapsuleValue]() - capsuleValueTypes sync.Map -) +func isCapsuleType(rt reflect.Type) bool { + fromCtyValueType := reflect.TypeFor[FromCtyValueConverter]() + toCtyValueType := reflect.TypeFor[ToCtyValueConverter]() + return rt.Implements(fromCtyValueType) && rt.Implements(toCtyValueType) +} + +var capsuleValueTypes sync.Map func capsuleValueCapsuleType(rt reflect.Type) cty.Type { - if val, loaded := capsuleValueTypes.Load(rt); loaded { + if rt.Kind() != reflect.Pointer { + panic("capsule value must be a pointer") + } + + elem := rt.Elem() + if val, loaded := capsuleValueTypes.Load(elem); loaded { return val.(cty.Type) } - // First time used. - ety := cty.CapsuleWithOps(rt.Name(), rt.Elem(), &cty.CapsuleOps{ + toCtyValueType := reflect.TypeFor[ToCtyValueConverter]() + + // First time used. Initialize new capsule ops. + ops := &cty.CapsuleOps{ ConversionTo: func(_ cty.Type) func(cty.Value, cty.Path) (any, error) { return func(in cty.Value, p cty.Path) (any, error) { - rv := reflect.New(rt.Elem()).Interface() - if err := rv.(CapsuleValue).FromCtyValue(in, p); err != nil { + rv := reflect.New(elem).Interface() + if err := rv.(FromCtyValueConverter).FromCtyValue(in, p); err != nil { return nil, err } return rv, nil @@ -55,30 +71,39 @@ func capsuleValueCapsuleType(rt reflect.Type) cty.Type { }, ConversionFrom: func(want cty.Type) func(any, cty.Path) (cty.Value, error) { return func(in any, _ cty.Path) (cty.Value, error) { - v := in.(CapsuleValue).ToCtyValue() + rv := reflect.ValueOf(in).Convert(toCtyValueType) + v := rv.Interface().(ToCtyValueConverter).ToCtyValue() return convert.Convert(v, want) } }, ExtensionData: func(key any) any { switch key { - case nativeTypeExtension: - zero := reflect.Zero(rt).Interface() - return zero.(CapsuleValue).ToCtyValue().Type() - default: - return nil + case unwrapCapsuleValueExtension: + zero := reflect.Zero(elem).Interface() + if conv, ok := zero.(ToCtyValueConverter); ok { + return conv.ToCtyValue().Type() + } + + zero = reflect.Zero(rt).Interface() + if conv, ok := zero.(ToCtyValueConverter); ok { + return conv.ToCtyValue().Type() + } } + return nil }, - }) + } - // Attempt to store the new type. Use whichever was loaded first in the case of a race condition. - val, _ := capsuleValueTypes.LoadOrStore(rt, ety) + // Attempt to store the new type. Use whichever was loaded first in the case + // of a race condition. + ety := cty.CapsuleWithOps(elem.Name(), elem, ops) + val, _ := capsuleValueTypes.LoadOrStore(elem, ety) return val.(cty.Type) } -// ToNativeValue will convert a value to only native cty types which will -// remove capsule types if possible. -func ToNativeValue(in cty.Value) cty.Value { - want := toNativeType(in.Type()) +// UnwrapCtyValue will unwrap capsule type values into their native cty value +// equivalents if possible. +func UnwrapCtyValue(in cty.Value) cty.Value { + want := toCtyValueType(in.Type()) if in.Type().Equals(want) { return in } else if out, err := convert.Convert(in, want); err == nil { @@ -87,17 +112,17 @@ func ToNativeValue(in cty.Value) cty.Value { return cty.NullVal(want) } -func toNativeType(in cty.Type) cty.Type { +func toCtyValueType(in cty.Type) cty.Type { if et := in.MapElementType(); et != nil { - return cty.Map(toNativeType(*et)) + return cty.Map(toCtyValueType(*et)) } if et := in.SetElementType(); et != nil { - return cty.Set(toNativeType(*et)) + return cty.Set(toCtyValueType(*et)) } if et := in.ListElementType(); et != nil { - return cty.List(toNativeType(*et)) + return cty.List(toCtyValueType(*et)) } if in.IsObjectType() { @@ -105,7 +130,7 @@ func toNativeType(in cty.Type) cty.Type { inAttrTypes := in.AttributeTypes() outAttrTypes := make(map[string]cty.Type, len(inAttrTypes)) for name, typ := range inAttrTypes { - outAttrTypes[name] = toNativeType(typ) + outAttrTypes[name] = toCtyValueType(typ) if in.AttributeOptional(name) { optional = append(optional, name) } @@ -117,13 +142,13 @@ func toNativeType(in cty.Type) cty.Type { inTypes := in.TupleElementTypes() outTypes := make([]cty.Type, len(inTypes)) for i, typ := range inTypes { - outTypes[i] = toNativeType(typ) + outTypes[i] = toCtyValueType(typ) } return cty.Tuple(outTypes) } if in.IsCapsuleType() { - if out := in.CapsuleExtensionData(nativeTypeExtension); out != nil { + if out := in.CapsuleExtensionData(unwrapCapsuleValueExtension); out != nil { return out.(cty.Type) } return cty.DynamicPseudoType @@ -137,5 +162,5 @@ func ToCtyValue(val any, ty cty.Type) (cty.Value, error) { if err != nil { return out, err } - return ToNativeValue(out), nil + return UnwrapCtyValue(out), nil } diff --git a/tests/bake.go b/tests/bake.go index 8f92cb8e668..66e4aa04e79 100644 --- a/tests/bake.go +++ b/tests/bake.go @@ -32,6 +32,7 @@ func bakeCmd(sb integration.Sandbox, opts ...cmdOpt) (string, error) { var bakeTests = []func(t *testing.T, sb integration.Sandbox){ testBakePrint, + testBakePrintSensitive, testBakeLocal, testBakeLocalMulti, testBakeRemote, @@ -77,7 +78,8 @@ target "build" { HELLO = "foo" } } -`)}, +`), + }, { "Compose", "compose.yml", @@ -88,7 +90,8 @@ services: context: . args: HELLO: foo -`)}, +`), + }, } for _, tc := range testCases { @@ -149,6 +152,124 @@ RUN echo "Hello ${HELLO}" } } +func testBakePrintSensitive(t *testing.T, sb integration.Sandbox) { + testCases := []struct { + name string + f string + dt []byte + }{ + { + "HCL", + "docker-bake.hcl", + []byte(` +target "build" { + args = { + HELLO = "foo" + } + + cache-from = [ + "type=gha", + "type=s3,region=us-west-2,bucket=my_bucket,name=my_image", + ] +} +`), + }, + { + "Compose", + "compose.yml", + []byte(` +services: + build: + build: + context: . + args: + HELLO: foo + cache_from: + - type=gha + - type=s3,region=us-west-2,bucket=my_bucket,name=my_image +`), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dir := tmpdir( + t, + fstest.CreateFile(tc.f, tc.dt, 0600), + fstest.CreateFile("Dockerfile", []byte(` +FROM busybox +ARG HELLO +RUN echo "Hello ${HELLO}" + `), 0600), + ) + + cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--print", "build"), + withEnv( + "ACTIONS_RUNTIME_TOKEN=sensitive_token", + "ACTIONS_CACHE_URL=https://cache.github.com", + "AWS_ACCESS_KEY_ID=definitely_dont_look_here", + "AWS_SECRET_ACCESS_KEY=hackers_please_dont_steal", + "AWS_SESSION_TOKEN=not_a_mitm_attack", + ), + ) + stdout := bytes.Buffer{} + stderr := bytes.Buffer{} + cmd.Stdout = &stdout + cmd.Stderr = &stderr + require.NoError(t, cmd.Run(), stdout.String(), stderr.String()) + + var def struct { + Group map[string]*bake.Group `json:"group,omitempty"` + Target map[string]*bake.Target `json:"target"` + } + require.NoError(t, json.Unmarshal(stdout.Bytes(), &def)) + + require.Len(t, def.Group, 1) + require.Contains(t, def.Group, "default") + + require.Equal(t, []string{"build"}, def.Group["default"].Targets) + require.Len(t, def.Target, 1) + require.Contains(t, def.Target, "build") + require.Equal(t, ".", *def.Target["build"].Context) + require.Equal(t, "Dockerfile", *def.Target["build"].Dockerfile) + require.Equal(t, map[string]*string{"HELLO": ptrstr("foo")}, def.Target["build"].Args) + require.NotNil(t, def.Target["build"].CacheFrom) + require.Len(t, def.Target["build"].CacheFrom, 2) + + require.JSONEq(t, `{ + "group": { + "default": { + "targets": [ + "build" + ] + } + }, + "target": { + "build": { + "context": ".", + "dockerfile": "Dockerfile", + "args": { + "HELLO": "foo" + }, + "cache-from": [ + { + "type": "gha" + }, + { + "type": "s3", + "region": "us-west-2", + "bucket": "my_bucket", + "name": "my_image" + } + ] + } + } +} +`, stdout.String()) + }) + } +} + func testBakeLocal(t *testing.T, sb integration.Sandbox) { dockerfile := []byte(` FROM scratch diff --git a/util/buildflags/attests.go b/util/buildflags/attests.go index 600b755e0d9..daef2b96538 100644 --- a/util/buildflags/attests.go +++ b/util/buildflags/attests.go @@ -21,7 +21,7 @@ func CanonicalizeAttest(attestType string, in string) string { } func ParseAttests(in []string) ([]*controllerapi.Attest, error) { - out := []*controllerapi.Attest{} + var out []*controllerapi.Attest found := map[string]struct{}{} for _, in := range in { in := in diff --git a/util/buildflags/cache.go b/util/buildflags/cache.go index fcd36ff5e04..db982e786c5 100644 --- a/util/buildflags/cache.go +++ b/util/buildflags/cache.go @@ -15,6 +15,41 @@ import ( jsoncty "github.com/zclconf/go-cty/cty/json" ) +type CacheOptions []*CacheOptionsEntry + +func (o CacheOptions) Merge(other CacheOptions) CacheOptions { + if other == nil { + return o.Normalize() + } else if o == nil { + return other.Normalize() + } + + return append(o, other...).Normalize() +} + +func (o CacheOptions) Normalize() CacheOptions { + if len(o) == 0 { + return nil + } + return removeDupes(o) +} + +func (o CacheOptions) ToPB() []*controllerapi.CacheOptionsEntry { + if len(o) == 0 { + return nil + } + + var outs []*controllerapi.CacheOptionsEntry + for _, entry := range o { + pb := entry.ToPB() + if !isActive(pb) { + continue + } + outs = append(outs, pb) + } + return outs +} + type CacheOptionsEntry struct { Type string `json:"type"` Attrs map[string]string `json:"attrs,omitempty"` @@ -46,10 +81,13 @@ func (e *CacheOptionsEntry) String() string { } func (e *CacheOptionsEntry) ToPB() *controllerapi.CacheOptionsEntry { - return &controllerapi.CacheOptionsEntry{ + ci := &controllerapi.CacheOptionsEntry{ Type: e.Type, Attrs: maps.Clone(e.Attrs), } + addGithubToken(ci) + addAwsCredentials(ci) + return ci } func (e *CacheOptionsEntry) MarshalJSON() ([]byte, error) { @@ -74,14 +112,6 @@ func (e *CacheOptionsEntry) UnmarshalJSON(data []byte) error { return e.validate(data) } -func (e *CacheOptionsEntry) IsActive() bool { - // Always active if not gha. - if e.Type != "gha" { - return true - } - return e.Attrs["token"] != "" && e.Attrs["url"] != "" -} - func (e *CacheOptionsEntry) UnmarshalText(text []byte) error { in := string(text) fields, err := csvvalue.Fields(in, nil) @@ -116,8 +146,6 @@ func (e *CacheOptionsEntry) UnmarshalText(text []byte) error { if e.Type == "" { return errors.Errorf("type required form> %q", in) } - addGithubToken(e) - addAwsCredentials(e) return e.validate(text) } @@ -140,23 +168,22 @@ func (e *CacheOptionsEntry) validate(gv interface{}) error { } func ParseCacheEntry(in []string) ([]*controllerapi.CacheOptionsEntry, error) { - outs := make([]*controllerapi.CacheOptionsEntry, 0, len(in)) + if len(in) == 0 { + return nil, nil + } + + opts := make(CacheOptions, 0, len(in)) for _, in := range in { var out CacheOptionsEntry if err := out.UnmarshalText([]byte(in)); err != nil { return nil, err } - - if !out.IsActive() { - // Skip inactive cache entries. - continue - } - outs = append(outs, out.ToPB()) + opts = append(opts, &out) } - return outs, nil + return opts.ToPB(), nil } -func addGithubToken(ci *CacheOptionsEntry) { +func addGithubToken(ci *controllerapi.CacheOptionsEntry) { if ci.Type != "gha" { return } @@ -172,7 +199,7 @@ func addGithubToken(ci *CacheOptionsEntry) { } } -func addAwsCredentials(ci *CacheOptionsEntry) { +func addAwsCredentials(ci *controllerapi.CacheOptionsEntry) { if ci.Type != "s3" { return } @@ -201,3 +228,11 @@ func addAwsCredentials(ci *CacheOptionsEntry) { ci.Attrs["session_token"] = credentials.SessionToken } } + +func isActive(pb *controllerapi.CacheOptionsEntry) bool { + // Always active if not gha. + if pb.Type != "gha" { + return true + } + return pb.Attrs["token"] != "" && pb.Attrs["url"] != "" +} diff --git a/util/buildflags/cache_cty.go b/util/buildflags/cache_cty.go new file mode 100644 index 00000000000..abfcd008a6a --- /dev/null +++ b/util/buildflags/cache_cty.go @@ -0,0 +1,86 @@ +package buildflags + +import ( + "sync" + + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" +) + +var cacheOptionsEntryType = sync.OnceValue(func() cty.Type { + return cty.Map(cty.String) +}) + +func (o *CacheOptions) FromCtyValue(in cty.Value, p cty.Path) error { + got := in.Type() + if got.IsTupleType() || got.IsListType() { + return o.fromCtyValue(in, p) + } + + want := cty.List(cacheOptionsEntryType()) + return p.NewErrorf("%s", convert.MismatchMessage(got, want)) +} + +func (o *CacheOptions) fromCtyValue(in cty.Value, p cty.Path) error { + *o = make([]*CacheOptionsEntry, 0, in.LengthInt()) + for elem := in.ElementIterator(); elem.Next(); { + _, value := elem.Element() + + if isEmpty(value) { + continue + } + + entry := &CacheOptionsEntry{} + if err := entry.FromCtyValue(value, p); err != nil { + return err + } + *o = append(*o, entry) + } + return nil +} + +func (o CacheOptions) ToCtyValue() cty.Value { + if len(o) == 0 { + return cty.ListValEmpty(cacheOptionsEntryType()) + } + + vals := make([]cty.Value, len(o)) + for i, entry := range o { + vals[i] = entry.ToCtyValue() + } + return cty.ListVal(vals) +} + +func (o *CacheOptionsEntry) FromCtyValue(in cty.Value, p cty.Path) error { + if in.Type() == cty.String { + if err := o.UnmarshalText([]byte(in.AsString())); err != nil { + return p.NewError(err) + } + return nil + } + + conv, err := convert.Convert(in, cty.Map(cty.String)) + if err != nil { + return err + } + + m := conv.AsValueMap() + if err := getAndDelete(m, "type", &o.Type); err != nil { + return err + } + o.Attrs = asMap(m) + return o.validate(in) +} + +func (o *CacheOptionsEntry) ToCtyValue() cty.Value { + if o == nil { + return cty.NullVal(cty.Map(cty.String)) + } + + vals := make(map[string]cty.Value, len(o.Attrs)+1) + for k, v := range o.Attrs { + vals[k] = cty.StringVal(v) + } + vals["type"] = cty.StringVal(o.Type) + return cty.MapVal(vals) +} diff --git a/util/buildflags/cache_test.go b/util/buildflags/cache_test.go new file mode 100644 index 00000000000..a25544011d7 --- /dev/null +++ b/util/buildflags/cache_test.go @@ -0,0 +1,39 @@ +package buildflags + +import ( + "testing" + + "github.com/docker/buildx/controller/pb" + "github.com/stretchr/testify/require" +) + +func TestCacheOptions_DerivedVars(t *testing.T) { + t.Setenv("ACTIONS_RUNTIME_TOKEN", "sensitive_token") + t.Setenv("ACTIONS_CACHE_URL", "https://cache.github.com") + t.Setenv("AWS_ACCESS_KEY_ID", "definitely_dont_look_here") + t.Setenv("AWS_SECRET_ACCESS_KEY", "hackers_please_dont_steal") + t.Setenv("AWS_SESSION_TOKEN", "not_a_mitm_attack") + + cacheFrom, err := ParseCacheEntry([]string{"type=gha", "type=s3,region=us-west-2,bucket=my_bucket,name=my_image"}) + require.NoError(t, err) + require.Equal(t, []*pb.CacheOptionsEntry{ + { + Type: "gha", + Attrs: map[string]string{ + "token": "sensitive_token", + "url": "https://cache.github.com", + }, + }, + { + Type: "s3", + Attrs: map[string]string{ + "region": "us-west-2", + "bucket": "my_bucket", + "name": "my_image", + "access_key_id": "definitely_dont_look_here", + "secret_access_key": "hackers_please_dont_steal", + "session_token": "not_a_mitm_attack", + }, + }, + }, cacheFrom) +} diff --git a/util/buildflags/context.go b/util/buildflags/context.go index ac2e80742c1..e4974bc6b92 100644 --- a/util/buildflags/context.go +++ b/util/buildflags/context.go @@ -11,8 +11,13 @@ func ParseContextNames(values []string) (map[string]string, error) { if len(values) == 0 { return nil, nil } + result := make(map[string]string, len(values)) for _, value := range values { + if value == "" { + continue + } + kv := strings.SplitN(value, "=", 2) if len(kv) != 2 { return nil, errors.Errorf("invalid context value: %s, expected key=value", value) diff --git a/util/buildflags/cty.go b/util/buildflags/cty.go deleted file mode 100644 index d8b6a19a03e..00000000000 --- a/util/buildflags/cty.go +++ /dev/null @@ -1,183 +0,0 @@ -package buildflags - -import ( - "encoding" - "sync" - - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/convert" - "github.com/zclconf/go-cty/cty/gocty" -) - -func (e *CacheOptionsEntry) FromCtyValue(in cty.Value, p cty.Path) error { - conv, err := convert.Convert(in, cty.Map(cty.String)) - if err == nil { - m := conv.AsValueMap() - if err := getAndDelete(m, "type", &e.Type); err != nil { - return err - } - e.Attrs = asMap(m) - return e.validate(in) - } - return unmarshalTextFallback(in, e, err) -} - -func (e *CacheOptionsEntry) ToCtyValue() cty.Value { - if e == nil { - return cty.NullVal(cty.Map(cty.String)) - } - - vals := make(map[string]cty.Value, len(e.Attrs)+1) - for k, v := range e.Attrs { - vals[k] = cty.StringVal(v) - } - vals["type"] = cty.StringVal(e.Type) - return cty.MapVal(vals) -} - -func (e *ExportEntry) FromCtyValue(in cty.Value, p cty.Path) error { - conv, err := convert.Convert(in, cty.Map(cty.String)) - if err == nil { - m := conv.AsValueMap() - if err := getAndDelete(m, "type", &e.Type); err != nil { - return err - } - if err := getAndDelete(m, "dest", &e.Destination); err != nil { - return err - } - e.Attrs = asMap(m) - return e.validate() - } - return unmarshalTextFallback(in, e, err) -} - -func (e *ExportEntry) ToCtyValue() cty.Value { - if e == nil { - return cty.NullVal(cty.Map(cty.String)) - } - - vals := make(map[string]cty.Value, len(e.Attrs)+2) - for k, v := range e.Attrs { - vals[k] = cty.StringVal(v) - } - vals["type"] = cty.StringVal(e.Type) - vals["dest"] = cty.StringVal(e.Destination) - return cty.MapVal(vals) -} - -var secretType = sync.OnceValue(func() cty.Type { - return cty.ObjectWithOptionalAttrs( - map[string]cty.Type{ - "id": cty.String, - "src": cty.String, - "env": cty.String, - }, - []string{"id", "src", "env"}, - ) -}) - -func (e *Secret) FromCtyValue(in cty.Value, p cty.Path) (err error) { - conv, err := convert.Convert(in, secretType()) - if err == nil { - if id := conv.GetAttr("id"); !id.IsNull() { - e.ID = id.AsString() - } - if src := conv.GetAttr("src"); !src.IsNull() { - e.FilePath = src.AsString() - } - if env := conv.GetAttr("env"); !env.IsNull() { - e.Env = env.AsString() - } - return nil - } - return unmarshalTextFallback(in, e, err) -} - -func (e *Secret) ToCtyValue() cty.Value { - if e == nil { - return cty.NullVal(secretType()) - } - - return cty.ObjectVal(map[string]cty.Value{ - "id": cty.StringVal(e.ID), - "src": cty.StringVal(e.FilePath), - "env": cty.StringVal(e.Env), - }) -} - -var sshType = sync.OnceValue(func() cty.Type { - return cty.ObjectWithOptionalAttrs( - map[string]cty.Type{ - "id": cty.String, - "paths": cty.List(cty.String), - }, - []string{"id", "paths"}, - ) -}) - -func (e *SSH) FromCtyValue(in cty.Value, p cty.Path) (err error) { - conv, err := convert.Convert(in, sshType()) - if err == nil { - if id := conv.GetAttr("id"); !id.IsNull() { - e.ID = id.AsString() - } - if paths := conv.GetAttr("paths"); !paths.IsNull() { - if err := gocty.FromCtyValue(paths, &e.Paths); err != nil { - return err - } - } - return nil - } - return unmarshalTextFallback(in, e, err) -} - -func (e *SSH) ToCtyValue() cty.Value { - if e == nil { - return cty.NullVal(sshType()) - } - - var ctyPaths cty.Value - if len(e.Paths) > 0 { - paths := make([]cty.Value, len(e.Paths)) - for i, path := range e.Paths { - paths[i] = cty.StringVal(path) - } - ctyPaths = cty.ListVal(paths) - } else { - ctyPaths = cty.ListValEmpty(cty.String) - } - - return cty.ObjectVal(map[string]cty.Value{ - "id": cty.StringVal(e.ID), - "paths": ctyPaths, - }) -} - -func getAndDelete(m map[string]cty.Value, attr string, gv interface{}) error { - if v, ok := m[attr]; ok { - delete(m, attr) - return gocty.FromCtyValue(v, gv) - } - return nil -} - -func asMap(m map[string]cty.Value) map[string]string { - out := make(map[string]string, len(m)) - for k, v := range m { - out[k] = v.AsString() - } - return out -} - -func unmarshalTextFallback[V encoding.TextUnmarshaler](in cty.Value, v V, inErr error) (outErr error) { - // Attempt to convert this type to a string. - conv, err := convert.Convert(in, cty.String) - if err != nil { - // Cannot convert. Do not attempt to convert and return the original error. - return inErr - } - - // Conversion was successful. Use UnmarshalText on the string and return any - // errors associated with that. - return v.UnmarshalText([]byte(conv.AsString())) -} diff --git a/util/buildflags/entitlements.go b/util/buildflags/entitlements.go index 31f04304f29..970739bc45d 100644 --- a/util/buildflags/entitlements.go +++ b/util/buildflags/entitlements.go @@ -5,6 +5,10 @@ import "github.com/moby/buildkit/util/entitlements" func ParseEntitlements(in []string) ([]entitlements.Entitlement, error) { out := make([]entitlements.Entitlement, 0, len(in)) for _, v := range in { + if v == "" { + continue + } + e, err := entitlements.Parse(v) if err != nil { return nil, err diff --git a/util/buildflags/export.go b/util/buildflags/export.go index 90eaa743d7b..ba364c9fef9 100644 --- a/util/buildflags/export.go +++ b/util/buildflags/export.go @@ -16,6 +16,39 @@ import ( "github.com/tonistiigi/go-csvvalue" ) +type Exports []*ExportEntry + +func (e Exports) Merge(other Exports) Exports { + if other == nil { + e.Normalize() + return e + } else if e == nil { + other.Normalize() + return other + } + + return append(e, other...).Normalize() +} + +func (e Exports) Normalize() Exports { + if len(e) == 0 { + return nil + } + return removeDupes(e) +} + +func (e Exports) ToPB() []*controllerapi.ExportEntry { + if len(e) == 0 { + return nil + } + + entries := make([]*controllerapi.ExportEntry, len(e)) + for i, entry := range e { + entries[i] = entry.ToPB() + } + return entries +} + type ExportEntry struct { Type string `json:"type"` Attrs map[string]string `json:"attrs,omitempty"` @@ -131,18 +164,23 @@ func (e *ExportEntry) validate() error { } func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) { - var outs []*controllerapi.ExportEntry if len(inp) == 0 { return nil, nil } + + export := make(Exports, 0, len(inp)) for _, s := range inp { + if s == "" { + continue + } + var out ExportEntry if err := out.UnmarshalText([]byte(s)); err != nil { return nil, err } - outs = append(outs, out.ToPB()) + export = append(export, &out) } - return outs, nil + return export.ToPB(), nil } func ParseAnnotations(inp []string) (map[exptypes.AnnotationKey]string, error) { @@ -153,6 +191,10 @@ func ParseAnnotations(inp []string) (map[exptypes.AnnotationKey]string, error) { annotations := make(map[exptypes.AnnotationKey]string) for _, inp := range inp { + if inp == "" { + continue + } + k, v, ok := strings.Cut(inp, "=") if !ok { return nil, errors.Errorf("invalid annotation %q, expected key=value", inp) diff --git a/util/buildflags/export_cty.go b/util/buildflags/export_cty.go new file mode 100644 index 00000000000..4e77956b8b9 --- /dev/null +++ b/util/buildflags/export_cty.go @@ -0,0 +1,90 @@ +package buildflags + +import ( + "sync" + + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" +) + +var exportEntryType = sync.OnceValue(func() cty.Type { + return cty.Map(cty.String) +}) + +func (e *Exports) FromCtyValue(in cty.Value, p cty.Path) error { + got := in.Type() + if got.IsTupleType() || got.IsListType() { + return e.fromCtyValue(in, p) + } + + want := cty.List(exportEntryType()) + return p.NewErrorf("%s", convert.MismatchMessage(got, want)) +} + +func (e *Exports) fromCtyValue(in cty.Value, p cty.Path) error { + *e = make([]*ExportEntry, 0, in.LengthInt()) + for elem := in.ElementIterator(); elem.Next(); { + _, value := elem.Element() + + if isEmpty(value) { + continue + } + + entry := &ExportEntry{} + if err := entry.FromCtyValue(value, p); err != nil { + return err + } + *e = append(*e, entry) + } + return nil +} + +func (e Exports) ToCtyValue() cty.Value { + if len(e) == 0 { + return cty.ListValEmpty(exportEntryType()) + } + + vals := make([]cty.Value, len(e)) + for i, entry := range e { + vals[i] = entry.ToCtyValue() + } + return cty.ListVal(vals) +} + +func (e *ExportEntry) FromCtyValue(in cty.Value, p cty.Path) error { + if in.Type() == cty.String { + if err := e.UnmarshalText([]byte(in.AsString())); err != nil { + return p.NewError(err) + } + return nil + } + + conv, err := convert.Convert(in, cty.Map(cty.String)) + if err != nil { + return err + } + + m := conv.AsValueMap() + if err := getAndDelete(m, "type", &e.Type); err != nil { + return err + } + if err := getAndDelete(m, "dest", &e.Destination); err != nil { + return err + } + e.Attrs = asMap(m) + return e.validate() +} + +func (e *ExportEntry) ToCtyValue() cty.Value { + if e == nil { + return cty.NullVal(cty.Map(cty.String)) + } + + vals := make(map[string]cty.Value, len(e.Attrs)+2) + for k, v := range e.Attrs { + vals[k] = cty.StringVal(v) + } + vals["type"] = cty.StringVal(e.Type) + vals["dest"] = cty.StringVal(e.Destination) + return cty.MapVal(vals) +} diff --git a/util/buildflags/secrets.go b/util/buildflags/secrets.go index d480e6860a9..75a7f1613c2 100644 --- a/util/buildflags/secrets.go +++ b/util/buildflags/secrets.go @@ -8,6 +8,39 @@ import ( "github.com/tonistiigi/go-csvvalue" ) +type Secrets []*Secret + +func (s Secrets) Merge(other Secrets) Secrets { + if other == nil { + s.Normalize() + return s + } else if s == nil { + other.Normalize() + return other + } + + return append(s, other...).Normalize() +} + +func (s Secrets) Normalize() Secrets { + if len(s) == 0 { + return nil + } + return removeDupes(s) +} + +func (s Secrets) ToPB() []*controllerapi.Secret { + if len(s) == 0 { + return nil + } + + entries := make([]*controllerapi.Secret, len(s)) + for i, entry := range s { + entries[i] = entry.ToPB() + } + return entries +} + type Secret struct { ID string `json:"id,omitempty"` FilePath string `json:"src,omitempty"` @@ -85,6 +118,10 @@ func (s *Secret) UnmarshalText(text []byte) error { func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) { fs := make([]*controllerapi.Secret, 0, len(sl)) for _, v := range sl { + if v == "" { + continue + } + s, err := parseSecret(v) if err != nil { return nil, err diff --git a/util/buildflags/secrets_cty.go b/util/buildflags/secrets_cty.go new file mode 100644 index 00000000000..2ea02ccb588 --- /dev/null +++ b/util/buildflags/secrets_cty.go @@ -0,0 +1,96 @@ +package buildflags + +import ( + "sync" + + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" +) + +var secretType = sync.OnceValue(func() cty.Type { + return cty.ObjectWithOptionalAttrs( + map[string]cty.Type{ + "id": cty.String, + "src": cty.String, + "env": cty.String, + }, + []string{"id", "src", "env"}, + ) +}) + +func (s *Secrets) FromCtyValue(in cty.Value, p cty.Path) error { + got := in.Type() + if got.IsTupleType() || got.IsListType() { + return s.fromCtyValue(in, p) + } + + want := cty.List(secretType()) + return p.NewErrorf("%s", convert.MismatchMessage(got, want)) +} + +func (s *Secrets) fromCtyValue(in cty.Value, p cty.Path) error { + *s = make([]*Secret, 0, in.LengthInt()) + for elem := in.ElementIterator(); elem.Next(); { + _, value := elem.Element() + + if isEmpty(value) { + continue + } + + entry := &Secret{} + if err := entry.FromCtyValue(value, p); err != nil { + return err + } + *s = append(*s, entry) + } + return nil +} + +func (s Secrets) ToCtyValue() cty.Value { + if len(s) == 0 { + return cty.ListValEmpty(secretType()) + } + + vals := make([]cty.Value, len(s)) + for i, entry := range s { + vals[i] = entry.ToCtyValue() + } + return cty.ListVal(vals) +} + +func (e *Secret) FromCtyValue(in cty.Value, p cty.Path) error { + if in.Type() == cty.String { + if err := e.UnmarshalText([]byte(in.AsString())); err != nil { + return p.NewError(err) + } + return nil + } + + conv, err := convert.Convert(in, secretType()) + if err != nil { + return err + } + + if id := conv.GetAttr("id"); !id.IsNull() { + e.ID = id.AsString() + } + if src := conv.GetAttr("src"); !src.IsNull() { + e.FilePath = src.AsString() + } + if env := conv.GetAttr("env"); !env.IsNull() { + e.Env = env.AsString() + } + return nil +} + +func (e *Secret) ToCtyValue() cty.Value { + if e == nil { + return cty.NullVal(secretType()) + } + + return cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal(e.ID), + "src": cty.StringVal(e.FilePath), + "env": cty.StringVal(e.Env), + }) +} diff --git a/util/buildflags/ssh.go b/util/buildflags/ssh.go index e7944db8eb3..200b701024d 100644 --- a/util/buildflags/ssh.go +++ b/util/buildflags/ssh.go @@ -9,6 +9,39 @@ import ( "github.com/moby/buildkit/util/gitutil" ) +type SSHKeys []*SSH + +func (s SSHKeys) Merge(other SSHKeys) SSHKeys { + if other == nil { + s.Normalize() + return s + } else if s == nil { + other.Normalize() + return other + } + + return append(s, other...).Normalize() +} + +func (s SSHKeys) Normalize() SSHKeys { + if len(s) == 0 { + return nil + } + return removeDupes(s) +} + +func (s SSHKeys) ToPB() []*controllerapi.SSH { + if len(s) == 0 { + return nil + } + + entries := make([]*controllerapi.SSH, len(s)) + for i, entry := range s { + entries[i] = entry.ToPB() + } + return entries +} + type SSH struct { ID string `json:"id,omitempty" cty:"id"` Paths []string `json:"paths,omitempty" cty:"paths"` @@ -62,6 +95,10 @@ func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) { } for _, s := range sl { + if s == "" { + continue + } + var out SSH if err := out.UnmarshalText([]byte(s)); err != nil { return nil, err diff --git a/util/buildflags/ssh_cty.go b/util/buildflags/ssh_cty.go new file mode 100644 index 00000000000..bd211c77c17 --- /dev/null +++ b/util/buildflags/ssh_cty.go @@ -0,0 +1,105 @@ +package buildflags + +import ( + "sync" + + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" + "github.com/zclconf/go-cty/cty/gocty" +) + +var sshType = sync.OnceValue(func() cty.Type { + return cty.ObjectWithOptionalAttrs( + map[string]cty.Type{ + "id": cty.String, + "paths": cty.List(cty.String), + }, + []string{"id", "paths"}, + ) +}) + +func (s *SSHKeys) FromCtyValue(in cty.Value, p cty.Path) error { + got := in.Type() + if got.IsTupleType() || got.IsListType() { + return s.fromCtyValue(in, p) + } + + want := cty.List(sshType()) + return p.NewErrorf("%s", convert.MismatchMessage(got, want)) +} + +func (s *SSHKeys) fromCtyValue(in cty.Value, p cty.Path) error { + *s = make([]*SSH, 0, in.LengthInt()) + for elem := in.ElementIterator(); elem.Next(); { + _, value := elem.Element() + + if isEmpty(value) { + continue + } + + entry := &SSH{} + if err := entry.FromCtyValue(value, p); err != nil { + return err + } + *s = append(*s, entry) + } + return nil +} + +func (s SSHKeys) ToCtyValue() cty.Value { + if len(s) == 0 { + return cty.ListValEmpty(sshType()) + } + + vals := make([]cty.Value, len(s)) + for i, entry := range s { + vals[i] = entry.ToCtyValue() + } + return cty.ListVal(vals) +} + +func (e *SSH) FromCtyValue(in cty.Value, p cty.Path) error { + if in.Type() == cty.String { + if err := e.UnmarshalText([]byte(in.AsString())); err != nil { + return p.NewError(err) + } + return nil + } + + conv, err := convert.Convert(in, sshType()) + if err != nil { + return err + } + + if id := conv.GetAttr("id"); !id.IsNull() { + e.ID = id.AsString() + } + if paths := conv.GetAttr("paths"); !paths.IsNull() { + if err := gocty.FromCtyValue(paths, &e.Paths); err != nil { + return err + } + } + return nil +} + +func (e *SSH) ToCtyValue() cty.Value { + if e == nil { + return cty.NullVal(sshType()) + } + + var ctyPaths cty.Value + if len(e.Paths) > 0 { + paths := make([]cty.Value, len(e.Paths)) + for i, path := range e.Paths { + paths[i] = cty.StringVal(path) + } + ctyPaths = cty.ListVal(paths) + } else { + ctyPaths = cty.ListValEmpty(cty.String) + } + + return cty.ObjectVal(map[string]cty.Value{ + "id": cty.StringVal(e.ID), + "paths": ctyPaths, + }) +} diff --git a/util/buildflags/utils.go b/util/buildflags/utils.go new file mode 100644 index 00000000000..44d4a1f9f4d --- /dev/null +++ b/util/buildflags/utils.go @@ -0,0 +1,54 @@ +package buildflags + +import ( + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/gocty" +) + +type comparable[E any] interface { + Equal(other E) bool +} + +func removeDupes[E comparable[E]](s []E) []E { + // Move backwards through the slice. + // For each element, any elements after the current element are unique. + // If we find our current element conflicts with an existing element, + // then we swap the offender with the end of the slice and chop it off. + + // Start at the second to last element. + // The last element is always unique. + for i := len(s) - 2; i >= 0; i-- { + elem := s[i] + // Check for duplicates after our current element. + for j := i + 1; j < len(s); j++ { + if elem.Equal(s[j]) { + // Found a duplicate, exchange the + // duplicate with the last element. + s[j], s[len(s)-1] = s[len(s)-1], s[j] + s = s[:len(s)-1] + break + } + } + } + return s +} + +func getAndDelete(m map[string]cty.Value, attr string, gv interface{}) error { + if v, ok := m[attr]; ok { + delete(m, attr) + return gocty.FromCtyValue(v, gv) + } + return nil +} + +func asMap(m map[string]cty.Value) map[string]string { + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v.AsString() + } + return out +} + +func isEmpty(v cty.Value) bool { + return v.Type() == cty.String && v.AsString() == "" +}