diff --git a/go.mod b/go.mod index 55727b261f..c0bbeb9c22 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,8 @@ require ( github.com/klauspost/compress v1.13.6 github.com/ldelossa/responserecorder v1.0.2-0.20210711162258-40bec93a9325 github.com/mattn/go-sqlite3 v1.11.0 // indirect + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.0.2 github.com/prometheus/client_golang v1.9.0 github.com/quay/clair/config v1.0.0 github.com/quay/claircore v1.2.0 diff --git a/go.sum b/go.sum index f48020dfd0..54ddb87e28 100644 --- a/go.sum +++ b/go.sum @@ -720,8 +720,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go. github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -922,7 +923,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= @@ -1106,7 +1106,6 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1317,7 +1316,6 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/httptransport/indexer_v1.go b/httptransport/indexer_v1.go index 944a90cad6..212bd49332 100644 --- a/httptransport/indexer_v1.go +++ b/httptransport/indexer_v1.go @@ -6,9 +6,11 @@ import ( "fmt" "net/http" "path" + "strings" "time" "github.com/ldelossa/responserecorder" + oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/prometheus/client_golang/prometheus" "github.com/quay/claircore" "github.com/quay/zlog" @@ -90,8 +92,8 @@ func (h *IndexerV1) indexReport(w http.ResponseWriter, r *http.Request) { apiError(w, http.StatusInternalServerError, "could not retrieve indexer state: %v", err) return } - var m claircore.Manifest - if err := dec.Decode(&m); err != nil { + m, err := decodeManifest(ctx, r, dec) + if err != nil { apiError(w, http.StatusBadRequest, "failed to deserialize manifest: %v", err) return } @@ -109,9 +111,9 @@ func (h *IndexerV1) indexReport(w http.ResponseWriter, r *http.Request) { return } - // TODO Do we need some sort of background context embedded in the HTTP - // struct? - report, err := h.srv.Index(ctx, &m) + // TODO(hank) We should switch on the content-type header and not send + // back the report if we've received an OCI manifest. + report, err := h.srv.Index(ctx, m) if err != nil { apiError(w, http.StatusInternalServerError, "failed to start scan: %v", err) return @@ -342,3 +344,84 @@ var indexerv1wrapper = &wrapper{ []string{"handler"}, ), } + +const ( + // Known manifest types we ingest. + typeOCIManifest = oci.MediaTypeImageManifest + typeNativeManifest = `application/vnd.projectquay.clair.mainfest.v1+json` +) + +// DecodeManifest switches on the Request's Content-Type to consume the body. +// +// Defaults to expecting a native Claircore Manifest. +func decodeManifest(ctx context.Context, r *http.Request, dec *codec.Decoder) (*claircore.Manifest, error) { + var m claircore.Manifest + + t := r.Header.Get("content-type") + if i := strings.IndexByte(t, ';'); i != -1 { + t = strings.TrimSpace(t[:i]) + } + switch t { + case typeOCIManifest: + var om oci.Manifest + if err := dec.Decode(&om); err != nil { + return nil, err + } + if err := nativeFromOCI(&m, &om); err != nil { + return nil, err + } + case typeNativeManifest, "application/json", "": + if err := dec.Decode(&m); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown content-type %q", t) + } + return &m, nil +} + +// These are the layer types we accept inside an OCI Manifest. +var ociLayerTypes = map[string]struct{}{ + oci.MediaTypeImageLayer: {}, + oci.MediaTypeImageLayerGzip: {}, + oci.MediaTypeImageLayer + "+zstd": {}, // The specs package doesn't have zstd, oddly. +} + +// NativeFromOCI populates the Manifest from the OCI Manifest, reporting an +// error if something is invalid. +func nativeFromOCI(m *claircore.Manifest, o *oci.Manifest) error { + const header = `header:` + var err error + + m.Hash, err = claircore.ParseDigest(o.Config.Digest.String()) + if err != nil { + return fmt.Errorf("unable to parse manifest digest %q: %w", o.Config.Digest, err) + } + + for _, u := range o.Layers { + if len(u.URLs) == 0 { + // Manifest is missing URLs. + // They're optional in the spec, but we need them for obvious reasons. + return fmt.Errorf("missing URLs for layer %q", u.Digest) + } + if _, ok := ociLayerTypes[u.MediaType]; !ok { + return fmt.Errorf("invalid media type for layer %q", u.Digest) + } + l := claircore.Layer{ + URI: u.URLs[0], + } + l.Hash, err = claircore.ParseDigest(u.Digest.String()) + if err != nil { + return fmt.Errorf("unable to parse layer digest %q: %w", u.Digest, err) + } + for k, v := range u.Annotations { + if !strings.HasPrefix(k, header) { + continue + } + l.Headers[strings.TrimPrefix(k, header)] = []string{v} + } + m.Layers = append(m.Layers, &l) + } + + return nil +} diff --git a/httptransport/indexer_v1_test.go b/httptransport/indexer_v1_test.go index 6e32501a7e..61bc892331 100644 --- a/httptransport/indexer_v1_test.go +++ b/httptransport/indexer_v1_test.go @@ -11,12 +11,17 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/quay/claircore" "github.com/quay/zlog" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel/trace" "github.com/quay/clair/v4/indexer" + "github.com/quay/clair/v4/internal/codec" ) func TestIndexerV1(t *testing.T) { @@ -194,3 +199,229 @@ func TestIndexerV1(t *testing.T) { }) }) } + +func TestNativeFromOCI(t *testing.T) { + t.Parallel() + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b claircore.Digest) bool { return a.String() == b.String() }), + cmpopts.IgnoreUnexported(claircore.Layer{}), + } + type testcase struct { + Name string + Want claircore.Manifest + In oci.Manifest + Err bool + } + Run := func(tc *testcase) func(*testing.T) { + return func(t *testing.T) { + var got claircore.Manifest + + err := nativeFromOCI(&got, &tc.In) + if (err != nil) != tc.Err { + t.Errorf("unexpected error: %v", err) + } + + if got, want := &got, &tc.Want; !cmp.Equal(got, want, cmpOpts) { + t.Error(cmp.Diff(got, want, cmpOpts)) + } + } + } + + tt := []testcase{ + { + Name: "EmptyDigest", + In: oci.Manifest{}, + Err: true, + }, + { + Name: "BadDigest", + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.Digest("xxx:yyy"), + }, + }, + Err: true, + }, + { + Name: "BadURLs", + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.FromString("good manifest"), + }, + Layers: []oci.Descriptor{ + {URLs: nil}, + }, + }, + Want: claircore.Manifest{ + Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"), + }, + Err: true, + }, + { + Name: "BadMediaType", + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.FromString("good manifest"), + }, + Layers: []oci.Descriptor{ + { + MediaType: `fake/media-type`, + URLs: []string{"http://localhost/real/layer"}, + }, + }, + }, + Want: claircore.Manifest{ + Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"), + }, + Err: true, + }, + { + Name: "BadLayerDigest", + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.FromString("good manifest"), + }, + Layers: []oci.Descriptor{ + { + Digest: digest.Digest("xxx:yyy"), + MediaType: oci.MediaTypeImageLayer, + URLs: []string{"http://localhost/real/layer"}, + }, + }, + }, + Want: claircore.Manifest{ + Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"), + }, + Err: true, + }, + { + Name: "OK", + Want: claircore.Manifest{ + Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"), + Layers: []*claircore.Layer{ + { + Hash: claircore.MustParseDigest("sha256:ba54d2c66022c637137ad0896ba5fb790847755be51b08bc472ffab5fdd76b1b"), + URI: "http://localhost/real/layer", + }, + }, + }, + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.FromString("good manifest"), + }, + Layers: []oci.Descriptor{ + { + Digest: digest.FromString("cool layer"), + MediaType: oci.MediaTypeImageLayer, + URLs: []string{"http://localhost/real/layer"}, + }, + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, Run(&tc)) + } +} + +func TestDecodeManifest(t *testing.T) { + t.Parallel() + ctx := context.Background() + + type testcase struct { + Name string + In *http.Request + Want claircore.Manifest + Err bool + } + Run := func(tc *testcase) func(*testing.T) { + return func(t *testing.T) { + dec := codec.GetDecoder(tc.In.Body) + defer codec.PutDecoder(dec) + got, err := decodeManifest(ctx, tc.In, dec) + if err != nil { + t.Log(err) + } + if (err != nil) != tc.Err { + t.Errorf("unexpected error: %v", err) + } + _ = got + } + } + + const ( + goodOCI = `{ + "mediaType":"` + oci.MediaTypeImageManifest + `", + "config":{"digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"}, + "layers":[{ + "mediaType":"` + oci.MediaTypeImageLayer + `", + "digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19", + "urls":["http://example.com/layer"] + }]}` + errorOCI = `{ + "mediaType":"` + oci.MediaTypeImageManifest + `", + "config":{"digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"}, + "layers":[{ + "mediaType":"` + oci.MediaTypeImageLayer + `", + "digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19" + }]}` + ) + tt := []testcase{ + { + Name: "NoHeaders", + In: httptest.NewRequest("", "/", strings.NewReader(`{}`)), + }, + { + Name: "BadContentType", + In: httptest.NewRequest("", "/", strings.NewReader(`{}`)), + Err: true, + }, + { + Name: "Default", + In: httptest.NewRequest("", "/", strings.NewReader(`{}`)), + }, + { + Name: "Default+Error", + In: httptest.NewRequest("", "/", strings.NewReader(`""`)), + Err: true, + }, + { + Name: "Claircore", + In: httptest.NewRequest("", "/", strings.NewReader(`{}`)), + }, + { + Name: "OCIManifest", + In: httptest.NewRequest("", "/", strings.NewReader(goodOCI)), + }, + { + Name: "OCIManifest+DecodeError", + In: httptest.NewRequest("", "/", strings.NewReader(`""`)), + Err: true, + }, + { + Name: "OCIManifest+TranslateError", + In: httptest.NewRequest("", "/", strings.NewReader(errorOCI)), + Err: true, + }, + } + // Adjust headers + for _, tc := range tt { + switch tc.Name { + case "NoHeaders": + case "BadContentType": + tc.In.Header.Set(`content-type`, `text/plain; charset=UTF-8`) + case "OCIManifest", "OCIManifest+DecodeError", "OCIManifest+TranslateError": + tc.In.Header.Set(`content-type`, oci.MediaTypeImageManifest) + case "Claircore": + tc.In.Header.Set(`content-type`, `application/json; charset=UTF-8`) + default: + tc.In.Header.Set(`content-type`, `application/json; charset=UTF-8`) + } + } + + for _, tc := range tt { + t.Run(tc.Name, Run(&tc)) + } +}