Skip to content

Commit

Permalink
bake: cache-from/cache-to options no longer print sensitive values
Browse files Browse the repository at this point in the history 
This refactors how the cache-from/cache-to composable attributes work so
they no longer print sensitive values that are automatically added.

This also expands the available syntax that works with the cache
options. It is now possible to interleave the csv syntax with the object
syntax without any problems. The canonical form is still the object
syntax and variables are resolved according to that syntax.

`cache-from` and `cache-to` now correctly ignore empty string inputs so
these can be used with variables.

Signed-off-by: Jonathan A. Sternberg <[email protected]>
  • Loading branch information
@jsternberg
jsternberg committed Dec 2, 2024
1 parent d6d713a commit c15110e
Show file tree
Hide file tree
Showing 10 changed files with 409 additions and 103 deletions.
83 changes: 45 additions & 38 deletions bake/bake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -698,30 +698,30 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"`

Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom []*buildflags.CacheOptionsEntry `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo []*buildflags.CacheOptionsEntry `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets []*buildflags.Secret `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH []*buildflags.SSH `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs []*buildflags.ExportEntry `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest []string `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom *buildflags.CacheOptions `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo *buildflags.CacheOptions `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets []*buildflags.Secret `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH []*buildflags.SSH `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs []*buildflags.ExportEntry `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides/AddOverrides and docs/bake-reference.md.

// linked is a private field to mark a target used as a linked one
Expand All @@ -742,8 +742,12 @@ func (t *Target) normalize() {
t.Secrets = removeDupes(t.Secrets)
t.SSH = removeDupes(t.SSH)
t.Platforms = removeDupesStr(t.Platforms)
t.CacheFrom = removeDupes(t.CacheFrom)
t.CacheTo = removeDupes(t.CacheTo)
if t.CacheFrom != nil {
t.CacheFrom.Normalize()
}
if t.CacheTo != nil {
t.CacheTo.Normalize()
}
t.Outputs = removeDupes(t.Outputs)
t.NoCacheFilter = removeDupesStr(t.NoCacheFilter)
t.Ulimits = removeDupesStr(t.Ulimits)
Expand Down Expand Up @@ -824,7 +828,7 @@ func (t *Target) Merge(t2 *Target) {
t.Platforms = t2.Platforms
}
if t2.CacheFrom != nil { // merge
t.CacheFrom = append(t.CacheFrom, t2.CacheFrom...)
t.CacheFrom = t.CacheFrom.Merge(t2.CacheFrom)
}
if t2.CacheTo != nil { // no merge
t.CacheTo = t2.CacheTo
Expand Down Expand Up @@ -1338,17 +1342,12 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
}

cacheImports := make([]*controllerapi.CacheOptionsEntry, len(t.CacheFrom))
for i, ci := range t.CacheFrom {
cacheImports[i] = ci.ToPB()
if t.CacheFrom != nil {
bo.CacheFrom = controllerapi.CreateCaches(t.CacheFrom.ToPB())
}
bo.CacheFrom = controllerapi.CreateCaches(cacheImports)

cacheExports := make([]*controllerapi.CacheOptionsEntry, len(t.CacheTo))
for i, ce := range t.CacheTo {
cacheExports[i] = ce.ToPB()
if t.CacheTo != nil {
bo.CacheTo = controllerapi.CreateCaches(t.CacheTo.ToPB())
}
bo.CacheTo = controllerapi.CreateCaches(cacheExports)

outputs := make([]*controllerapi.ExportEntry, len(t.Outputs))
for i, output := range t.Outputs {
Expand Down Expand Up @@ -1591,9 +1590,13 @@ func parseArrValue[T any, PT arrValue[T]](s []string) ([]*T, error) {
return outputs, nil
}

func parseCacheArrValues(s []string) ([]*buildflags.CacheOptionsEntry, error) {
func parseCacheArrValues(s []string) (*buildflags.CacheOptions, error) {
outs := make([]*buildflags.CacheOptionsEntry, 0, len(s))
for _, in := range s {
if in == "" {
continue
}

if !strings.Contains(in, "=") {
// This is ref only format. Each field in the CSV is its own entry.
fields, err := csvvalue.Fields(in, nil)
Expand All @@ -1618,5 +1621,9 @@ func parseCacheArrValues(s []string) ([]*buildflags.CacheOptionsEntry, error) {
}
outs = append(outs, &out)
}
return outs, nil

if len(outs) == 0 {
return nil, nil
}
return &buildflags.CacheOptions{Entries: outs}, nil
}
4 changes: 2 additions & 2 deletions bake/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,14 +353,14 @@ func (t *Target) composeExtTarget(exts map[string]interface{}) error {
if err != nil {
return err
}
t.CacheFrom = removeDupes(append(t.CacheFrom, cacheFrom...))
t.CacheFrom = t.CacheFrom.Merge(cacheFrom)
}
if len(xb.CacheTo) > 0 {
cacheTo, err := parseCacheArrValues(xb.CacheTo)
if err != nil {
return err
}
t.CacheTo = removeDupes(append(t.CacheTo, cacheTo...))
t.CacheTo = t.CacheTo.Merge(cacheTo)
}
if len(xb.Secrets) > 0 {
secrets, err := parseArrValue[buildflags.Secret](xb.Secrets)
Expand Down
12 changes: 6 additions & 6 deletions bake/compose_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ secrets:
require.Equal(t, "Dockerfile-alternate", *c.Targets[1].Dockerfile)
require.Equal(t, 1, len(c.Targets[1].Args))
require.Equal(t, ptrstr("123"), c.Targets[1].Args["buildno"])
require.Equal(t, []string{"type=local,src=path/to/cache"}, stringify(c.Targets[1].CacheFrom))
require.Equal(t, []string{"type=local,dest=path/to/cache"}, stringify(c.Targets[1].CacheTo))
require.Equal(t, []string{"type=local,src=path/to/cache"}, stringify(c.Targets[1].CacheFrom.Entries))
require.Equal(t, []string{"type=local,dest=path/to/cache"}, stringify(c.Targets[1].CacheTo.Entries))
require.Equal(t, "none", *c.Targets[1].NetworkMode)
require.Equal(t, []string{"default", "key=path/to/key"}, stringify(c.Targets[1].SSH))
require.Equal(t, []string{
Expand Down Expand Up @@ -336,8 +336,8 @@ services:
require.Equal(t, map[string]*string{"CT_ECR": ptrstr("foo"), "CT_TAG": ptrstr("bar")}, c.Targets[0].Args)
require.Equal(t, []string{"ct-addon:baz", "ct-addon:foo", "ct-addon:alp"}, c.Targets[0].Tags)
require.Equal(t, []string{"linux/amd64", "linux/arm64"}, c.Targets[0].Platforms)
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheFrom))
require.Equal(t, []string{"type=local,dest=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheTo))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheFrom.Entries))
require.Equal(t, []string{"type=local,dest=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheTo.Entries))
require.Equal(t, []string{"default", "key=path/to/key", "other=path/to/otherkey"}, stringify(c.Targets[0].SSH))
require.Equal(t, newBool(true), c.Targets[0].Pull)
require.Equal(t, map[string]string{"alpine": "docker-image://alpine:3.13"}, c.Targets[0].Contexts)
Expand Down Expand Up @@ -383,8 +383,8 @@ services:
require.NoError(t, err)
require.Equal(t, 1, len(c.Targets))
require.Equal(t, []string{"ct-addon:foo", "ct-addon:baz"}, c.Targets[0].Tags)
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheFrom))
require.Equal(t, []string{"type=local,dest=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheTo))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheFrom.Entries))
require.Equal(t, []string{"type=local,dest=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheTo.Entries))
require.Equal(t, []string{"default", "key=path/to/key"}, stringify(c.Targets[0].SSH))
}

Expand Down
16 changes: 8 additions & 8 deletions bake/hcl_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ func TestHCLAttrsCapsuleType(t *testing.T) {
target "app" {
cache-from = [
{ type = "registry", ref = "user/app:cache" },
{ type = "local", src = "path/to/cache" },
"type=local,src=path/to/cache",
]
cache-to = [
Expand Down Expand Up @@ -634,8 +634,8 @@ func TestHCLAttrsCapsuleType(t *testing.T) {

require.Equal(t, 1, len(c.Targets))
require.Equal(t, []string{"type=oci,dest=../out.tar"}, stringify(c.Targets[0].Outputs))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheFrom))
require.Equal(t, []string{"type=local,dest=path/to/cache"}, stringify(c.Targets[0].CacheTo))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(c.Targets[0].CacheFrom.Entries))
require.Equal(t, []string{"type=local,dest=path/to/cache"}, stringify(c.Targets[0].CacheTo.Entries))
require.Equal(t, []string{"id=mysecret,src=/local/secret", "id=mysecret2,env=TOKEN"}, stringify(c.Targets[0].Secrets))
require.Equal(t, []string{"default", "key=path/to/key"}, stringify(c.Targets[0].SSH))
}
Expand All @@ -649,7 +649,7 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
target "app" {
cache-from = [
{ type = "registry", ref = "user/app:cache" },
{ type = "local", src = "path/to/cache" },
"type=local,src=path/to/cache",
]
cache-to = [ target.app.cache-from[0] ]
Expand All @@ -674,7 +674,7 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {
output = [ "type=oci,dest=../${foo}.tar" ]
secret = [
{ id = target.app.output[0].type, src = "/local/secret" },
{ id = target.app.output[0].type, src = "/${target.app.cache-from[1].type}/secret" },
]
}
`)
Expand All @@ -697,14 +697,14 @@ func TestHCLAttrsCapsuleTypeVars(t *testing.T) {

app := findTarget(t, "app")
require.Equal(t, []string{"type=oci,dest=../out.tar"}, stringify(app.Outputs))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(app.CacheFrom))
require.Equal(t, []string{"user/app:cache"}, stringify(app.CacheTo))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(app.CacheFrom.Entries))
require.Equal(t, []string{"user/app:cache"}, stringify(app.CacheTo.Entries))
require.Equal(t, []string{"id=mysecret,src=/local/secret"}, stringify(app.Secrets))
require.Equal(t, []string{"default", "key=path/to/oci"}, stringify(app.SSH))

web := findTarget(t, "web")
require.Equal(t, []string{"type=oci,dest=../bar.tar"}, stringify(web.Outputs))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(web.CacheFrom))
require.Equal(t, []string{"type=local,src=path/to/cache", "user/app:cache"}, stringify(web.CacheFrom.Entries))
require.Equal(t, []string{"id=oci,src=/local/secret"}, stringify(web.Secrets))
}

Expand Down
125 changes: 123 additions & 2 deletions tests/bake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ func bakeCmd(sb integration.Sandbox, opts ...cmdOpt) (string, error) {

var bakeTests = []func(t *testing.T, sb integration.Sandbox){
testBakePrint,
testBakePrintSensitive,
testBakeLocal,
testBakeLocalMulti,
testBakeRemote,
Expand Down Expand Up @@ -77,7 +78,8 @@ target "build" {
HELLO = "foo"
}
}
`)},
`),
},
{
"Compose",
"compose.yml",
Expand All @@ -88,7 +90,8 @@ services:
context: .
args:
HELLO: foo
`)},
`),
},
}

for _, tc := range testCases {
Expand Down Expand Up @@ -149,6 +152,124 @@ RUN echo "Hello ${HELLO}"
}
}

func testBakePrintSensitive(t *testing.T, sb integration.Sandbox) {
testCases := []struct {
name string
f string
dt []byte
}{
{
"HCL",
"docker-bake.hcl",
[]byte(`
target "build" {
args = {
HELLO = "foo"
}
cache-from = [
"type=gha",
"type=s3,region=us-west-2,bucket=my_bucket,name=my_image",
]
}
`),
},
{
"Compose",
"compose.yml",
[]byte(`
services:
build:
build:
context: .
args:
HELLO: foo
cache_from:
- type=gha
- type=s3,region=us-west-2,bucket=my_bucket,name=my_image
`),
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
dir := tmpdir(
t,
fstest.CreateFile(tc.f, tc.dt, 0600),
fstest.CreateFile("Dockerfile", []byte(`
FROM busybox
ARG HELLO
RUN echo "Hello ${HELLO}"
`), 0600),
)

cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--print", "build"),
withEnv(
"ACTIONS_RUNTIME_TOKEN=sensitive_token",
"ACTIONS_CACHE_URL=https://cache.github.com",
"AWS_ACCESS_KEY_ID=definitely_dont_look_here",
"AWS_SECRET_ACCESS_KEY=hackers_please_dont_steal",
"AWS_SESSION_TOKEN=not_a_mitm_attack",
),
)
stdout := bytes.Buffer{}
stderr := bytes.Buffer{}
cmd.Stdout = &stdout
cmd.Stderr = &stderr
require.NoError(t, cmd.Run(), stdout.String(), stderr.String())

var def struct {
Group map[string]*bake.Group `json:"group,omitempty"`
Target map[string]*bake.Target `json:"target"`
}
require.NoError(t, json.Unmarshal(stdout.Bytes(), &def))

require.Len(t, def.Group, 1)
require.Contains(t, def.Group, "default")

require.Equal(t, []string{"build"}, def.Group["default"].Targets)
require.Len(t, def.Target, 1)
require.Contains(t, def.Target, "build")
require.Equal(t, ".", *def.Target["build"].Context)
require.Equal(t, "Dockerfile", *def.Target["build"].Dockerfile)
require.Equal(t, map[string]*string{"HELLO": ptrstr("foo")}, def.Target["build"].Args)
require.NotNil(t, def.Target["build"].CacheFrom)
require.Len(t, def.Target["build"].CacheFrom.Entries, 2)

require.JSONEq(t, `{
"group": {
"default": {
"targets": [
"build"
]
}
},
"target": {
"build": {
"context": ".",
"dockerfile": "Dockerfile",
"args": {
"HELLO": "foo"
},
"cache-from": [
{
"type": "gha"
},
{
"type": "s3",
"region": "us-west-2",
"bucket": "my_bucket",
"name": "my_image"
}
]
}
}
}
`, stdout.String())
})
}
}

func testBakeLocal(t *testing.T, sb integration.Sandbox) {
dockerfile := []byte(`
FROM scratch
Expand Down
Loading

0 comments on commit c15110e

Please sign in to comment.