From a6cad056815072ca85b23f84ae2e6bd675180b8d Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 17 Dec 2024 20:49:03 +0330 Subject: [PATCH] feat: added github_container_package and github_artifact_dockerfile describers --- go.mod | 10 + go.sum | 28 ++ pkg/sdk/es/resources_clients.go | 402 +++++++++++++++++- provider/describer/artifact_dockerfile.go | 229 ++++++++++ provider/describer/container_package.go | 197 +++++++-- provider/describer/utils.go | 1 + provider/describer_wrapper.go | 1 + provider/model/model.go | 120 ++++++ provider/resource_types.go | 14 + provider/resource_types/resource-types.json | 14 +- steampipe-plugin-github/github/plugin.go | 1 + .../table_github_artifact_dockerfile.go | 32 ++ .../github/table_github_container_package.go | 17 +- steampipe/table_index_map.go | 5 +- 14 files changed, 1024 insertions(+), 47 deletions(-) create mode 100644 provider/describer/artifact_dockerfile.go create mode 100644 steampipe-plugin-github/github/table_github_artifact_dockerfile.go diff --git a/go.mod b/go.mod index adee00e0..0aaea9b2 100755 --- a/go.mod +++ b/go.mod @@ -7,12 +7,14 @@ require ( github.com/buildkite/go-pipeline v0.3.1 github.com/go-errors/errors v1.5.1 github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/google/go-containerregistry v0.20.2 github.com/google/go-github/v55 v55.0.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-hclog v1.6.3 github.com/nats-io/nats.go v1.36.0 github.com/opengovern/og-util v1.1.9 github.com/opengovern/opencomply v0.475.2 + github.com/opengovern/resilient-bridge v0.0.0-20241217072313-40e2d7a377f6 github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 github.com/spf13/cobra v1.8.1 github.com/turbot/steampipe-plugin-github v1.0.0 @@ -74,10 +76,14 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.3.8 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/docker/cli v27.1.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eko/gocache/lib/v4 v4.1.5 // indirect github.com/eko/gocache/store/bigcache/v4 v4.2.1 // indirect @@ -158,6 +164,8 @@ require ( github.com/oklog/run v1.1.0 // indirect github.com/oleiade/reflections v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opensearch-project/opensearch-go/v2 v2.3.0 // indirect github.com/pganalyze/pg_query_go/v4 v4.2.3 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect @@ -170,12 +178,14 @@ require ( github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sethvargo/go-retry v0.2.4 // indirect github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stevenle/topsort v0.2.0 // indirect github.com/tkrajina/go-reflector v0.5.6 // indirect github.com/ulikunitz/xz v0.5.11 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect + github.com/vbatts/tar-split v0.11.3 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zclconf/go-cty v1.14.4 // indirect go.opencensus.io v0.24.0 // indirect diff --git a/go.sum b/go.sum index a68d6eb8..154b719b 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,7 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mo github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3 h1:6LyjnnaLpcOKK0fbYisI+mb8CE7iNe7i89nMNQxFxs8= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.3/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -309,8 +310,11 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= @@ -327,6 +331,12 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= +github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -465,6 +475,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= +github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts= github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg= @@ -725,10 +737,16 @@ github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0 github.com/oleiade/reflections v1.0.1/go.mod h1:rdFxbxq4QXVZWj0F+e9jqjDkc7dbp97vkRixKo2JR60= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opengovern/og-util v1.1.9 h1:Pswy4yOEcUS9FZW/3ThTBNNtRdyhhGdW9oj/GJ5WpwY= github.com/opengovern/og-util v1.1.9/go.mod h1:dyn8rhmxq59o1jnbgGfmcUvW7iB/eN6OxoTUUx6jEHA= github.com/opengovern/opencomply v0.475.2 h1:JHs12+mhJb9HzFZYMLLgw079PRa+qLKntugvwWiCVL0= github.com/opengovern/opencomply v0.475.2/go.mod h1:FsJTfdLz+DDRAzW2/KyCFRxVxM89jMysP7ZfQjaS8+A= +github.com/opengovern/resilient-bridge v0.0.0-20241217072313-40e2d7a377f6 h1:9IEO50lNreg+kMPgWq5RU3/4cMDLbT3KMjJ1GlZUJds= +github.com/opengovern/resilient-bridge v0.0.0-20241217072313-40e2d7a377f6/go.mod h1:zEulaGgccj5tPXGiudGz17NRhXAk2HHh+GPj2VkjNq0= github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= github.com/pganalyze/pg_query_go/v4 v4.2.3 h1:cNLqyiVMasV7YGWyYV+fkXyHp32gDfXVNCqoHztEGNk= @@ -790,6 +808,9 @@ github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29/go.mod h1:AuYgA5K github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -828,11 +849,14 @@ github.com/turbot/steampipe-plugin-sdk/v5 v5.10.4/go.mod h1:FzW+0aq4x1PoCkklCRx4 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1129,9 +1153,11 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1482,6 +1508,8 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/sdk/es/resources_clients.go b/pkg/sdk/es/resources_clients.go index b0c9d3f4..2d5ea733 100644 --- a/pkg/sdk/es/resources_clients.go +++ b/pkg/sdk/es/resources_clients.go @@ -8424,7 +8424,7 @@ var listWorkflowFilters = map[string]string{ "pipeline": "Description.Pipeline", "repository_full_name": "Description.RepositoryFullName", "state": "Description.State", - "url": "Description.State", + "url": "Description.URL", "workflow_file_content": "Description.WorkFlowFileContent", "workflow_file_content_json": "Description.WorkFlowFileContentJson", } @@ -8499,7 +8499,7 @@ var getWorkflowFilters = map[string]string{ "pipeline": "Description.Pipeline", "repository_full_name": "Description.RepositoryFullName", "state": "Description.State", - "url": "Description.State", + "url": "Description.URL", "workflow_file_content": "Description.WorkFlowFileContent", "workflow_file_content_json": "Description.WorkFlowFileContentJson", } @@ -8774,6 +8774,205 @@ func GetCodeOwner(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateDat // ========================== END: CodeOwner ============================= +// ========================== START: PackageOutputVersion ============================= + +type PackageOutputVersion struct { + ResourceID string `json:"resource_id"` + PlatformID string `json:"platform_id"` + Description github.PackageOutputVersionDescription `json:"description"` + Metadata github.Metadata `json:"metadata"` + DescribedBy string `json:"described_by"` + ResourceType string `json:"resource_type"` + IntegrationType string `json:"integration_type"` + IntegrationID string `json:"integration_id"` +} + +type PackageOutputVersionHit struct { + ID string `json:"_id"` + Score float64 `json:"_score"` + Index string `json:"_index"` + Type string `json:"_type"` + Version int64 `json:"_version,omitempty"` + Source PackageOutputVersion `json:"_source"` + Sort []interface{} `json:"sort"` +} + +type PackageOutputVersionHits struct { + Total essdk.SearchTotal `json:"total"` + Hits []PackageOutputVersionHit `json:"hits"` +} + +type PackageOutputVersionSearchResponse struct { + PitID string `json:"pit_id"` + Hits PackageOutputVersionHits `json:"hits"` +} + +type PackageOutputVersionPaginator struct { + paginator *essdk.BaseESPaginator +} + +func (k Client) NewPackageOutputVersionPaginator(filters []essdk.BoolFilter, limit *int64) (PackageOutputVersionPaginator, error) { + paginator, err := essdk.NewPaginator(k.ES(), "github_package_container", filters, limit) + if err != nil { + return PackageOutputVersionPaginator{}, err + } + + p := PackageOutputVersionPaginator{ + paginator: paginator, + } + + return p, nil +} + +func (p PackageOutputVersionPaginator) HasNext() bool { + return !p.paginator.Done() +} + +func (p PackageOutputVersionPaginator) Close(ctx context.Context) error { + return p.paginator.Deallocate(ctx) +} + +func (p PackageOutputVersionPaginator) NextPage(ctx context.Context) ([]PackageOutputVersion, error) { + var response PackageOutputVersionSearchResponse + err := p.paginator.Search(ctx, &response) + if err != nil { + return nil, err + } + + var values []PackageOutputVersion + for _, hit := range response.Hits.Hits { + values = append(values, hit.Source) + } + + hits := int64(len(response.Hits.Hits)) + if hits > 0 { + p.paginator.UpdateState(hits, response.Hits.Hits[hits-1].Sort, response.PitID) + } else { + p.paginator.UpdateState(hits, nil, "") + } + + return values, nil +} + +var listPackageOutputVersionFilters = map[string]string{} + +func ListPackageOutputVersion(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("ListPackageOutputVersion") + runtime.GC() + + // create service + cfg := essdk.GetConfig(d.Connection) + ke, err := essdk.NewClientCached(cfg, d.ConnectionCache, ctx) + if err != nil { + plugin.Logger(ctx).Error("ListPackageOutputVersion NewClientCached", "error", err) + return nil, err + } + k := Client{Client: ke} + + sc, err := steampipesdk.NewSelfClientCached(ctx, d.ConnectionCache) + if err != nil { + plugin.Logger(ctx).Error("ListPackageOutputVersion NewSelfClientCached", "error", err) + return nil, err + } + integrationID, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyIntegrationID) + if err != nil { + plugin.Logger(ctx).Error("ListPackageOutputVersion GetConfigTableValueOrNil for OpenGovernanceConfigKeyIntegrationID", "error", err) + return nil, err + } + encodedResourceCollectionFilters, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyResourceCollectionFilters) + if err != nil { + plugin.Logger(ctx).Error("ListPackageOutputVersion GetConfigTableValueOrNil for OpenGovernanceConfigKeyResourceCollectionFilters", "error", err) + return nil, err + } + clientType, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyClientType) + if err != nil { + plugin.Logger(ctx).Error("ListPackageOutputVersion GetConfigTableValueOrNil for OpenGovernanceConfigKeyClientType", "error", err) + return nil, err + } + + paginator, err := k.NewPackageOutputVersionPaginator(essdk.BuildFilter(ctx, d.QueryContext, listPackageOutputVersionFilters, integrationID, encodedResourceCollectionFilters, clientType), d.QueryContext.Limit) + if err != nil { + plugin.Logger(ctx).Error("ListPackageOutputVersion NewPackageOutputVersionPaginator", "error", err) + return nil, err + } + + for paginator.HasNext() { + page, err := paginator.NextPage(ctx) + if err != nil { + plugin.Logger(ctx).Error("ListPackageOutputVersion paginator.NextPage", "error", err) + return nil, err + } + + for _, v := range page { + d.StreamListItem(ctx, v) + } + } + + err = paginator.Close(ctx) + if err != nil { + return nil, err + } + + return nil, nil +} + +var getPackageOutputVersionFilters = map[string]string{} + +func GetPackageOutputVersion(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("GetPackageOutputVersion") + runtime.GC() + // create service + cfg := essdk.GetConfig(d.Connection) + ke, err := essdk.NewClientCached(cfg, d.ConnectionCache, ctx) + if err != nil { + return nil, err + } + k := Client{Client: ke} + + sc, err := steampipesdk.NewSelfClientCached(ctx, d.ConnectionCache) + if err != nil { + return nil, err + } + integrationID, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyIntegrationID) + if err != nil { + return nil, err + } + encodedResourceCollectionFilters, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyResourceCollectionFilters) + if err != nil { + return nil, err + } + clientType, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyClientType) + if err != nil { + return nil, err + } + + limit := int64(1) + paginator, err := k.NewPackageOutputVersionPaginator(essdk.BuildFilter(ctx, d.QueryContext, getPackageOutputVersionFilters, integrationID, encodedResourceCollectionFilters, clientType), &limit) + if err != nil { + return nil, err + } + + for paginator.HasNext() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, err + } + + for _, v := range page { + return v, nil + } + } + + err = paginator.Close(ctx) + if err != nil { + return nil, err + } + + return nil, nil +} + +// ========================== END: PackageOutputVersion ============================= + // ========================== START: Package ============================= type Package struct { @@ -9171,3 +9370,202 @@ func GetPackageVersion(ctx context.Context, d *plugin.QueryData, _ *plugin.Hydra } // ========================== END: PackageVersion ============================= + +// ========================== START: ArtifactDockerFile ============================= + +type ArtifactDockerFile struct { + ResourceID string `json:"resource_id"` + PlatformID string `json:"platform_id"` + Description github.ArtifactDockerFileDescription `json:"description"` + Metadata github.Metadata `json:"metadata"` + DescribedBy string `json:"described_by"` + ResourceType string `json:"resource_type"` + IntegrationType string `json:"integration_type"` + IntegrationID string `json:"integration_id"` +} + +type ArtifactDockerFileHit struct { + ID string `json:"_id"` + Score float64 `json:"_score"` + Index string `json:"_index"` + Type string `json:"_type"` + Version int64 `json:"_version,omitempty"` + Source ArtifactDockerFile `json:"_source"` + Sort []interface{} `json:"sort"` +} + +type ArtifactDockerFileHits struct { + Total essdk.SearchTotal `json:"total"` + Hits []ArtifactDockerFileHit `json:"hits"` +} + +type ArtifactDockerFileSearchResponse struct { + PitID string `json:"pit_id"` + Hits ArtifactDockerFileHits `json:"hits"` +} + +type ArtifactDockerFilePaginator struct { + paginator *essdk.BaseESPaginator +} + +func (k Client) NewArtifactDockerFilePaginator(filters []essdk.BoolFilter, limit *int64) (ArtifactDockerFilePaginator, error) { + paginator, err := essdk.NewPaginator(k.ES(), "github_artifactdockerfile", filters, limit) + if err != nil { + return ArtifactDockerFilePaginator{}, err + } + + p := ArtifactDockerFilePaginator{ + paginator: paginator, + } + + return p, nil +} + +func (p ArtifactDockerFilePaginator) HasNext() bool { + return !p.paginator.Done() +} + +func (p ArtifactDockerFilePaginator) Close(ctx context.Context) error { + return p.paginator.Deallocate(ctx) +} + +func (p ArtifactDockerFilePaginator) NextPage(ctx context.Context) ([]ArtifactDockerFile, error) { + var response ArtifactDockerFileSearchResponse + err := p.paginator.Search(ctx, &response) + if err != nil { + return nil, err + } + + var values []ArtifactDockerFile + for _, hit := range response.Hits.Hits { + values = append(values, hit.Source) + } + + hits := int64(len(response.Hits.Hits)) + if hits > 0 { + p.paginator.UpdateState(hits, response.Hits.Hits[hits-1].Sort, response.PitID) + } else { + p.paginator.UpdateState(hits, nil, "") + } + + return values, nil +} + +var listArtifactDockerFileFilters = map[string]string{} + +func ListArtifactDockerFile(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("ListArtifactDockerFile") + runtime.GC() + + // create service + cfg := essdk.GetConfig(d.Connection) + ke, err := essdk.NewClientCached(cfg, d.ConnectionCache, ctx) + if err != nil { + plugin.Logger(ctx).Error("ListArtifactDockerFile NewClientCached", "error", err) + return nil, err + } + k := Client{Client: ke} + + sc, err := steampipesdk.NewSelfClientCached(ctx, d.ConnectionCache) + if err != nil { + plugin.Logger(ctx).Error("ListArtifactDockerFile NewSelfClientCached", "error", err) + return nil, err + } + integrationID, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyIntegrationID) + if err != nil { + plugin.Logger(ctx).Error("ListArtifactDockerFile GetConfigTableValueOrNil for OpenGovernanceConfigKeyIntegrationID", "error", err) + return nil, err + } + encodedResourceCollectionFilters, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyResourceCollectionFilters) + if err != nil { + plugin.Logger(ctx).Error("ListArtifactDockerFile GetConfigTableValueOrNil for OpenGovernanceConfigKeyResourceCollectionFilters", "error", err) + return nil, err + } + clientType, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyClientType) + if err != nil { + plugin.Logger(ctx).Error("ListArtifactDockerFile GetConfigTableValueOrNil for OpenGovernanceConfigKeyClientType", "error", err) + return nil, err + } + + paginator, err := k.NewArtifactDockerFilePaginator(essdk.BuildFilter(ctx, d.QueryContext, listArtifactDockerFileFilters, integrationID, encodedResourceCollectionFilters, clientType), d.QueryContext.Limit) + if err != nil { + plugin.Logger(ctx).Error("ListArtifactDockerFile NewArtifactDockerFilePaginator", "error", err) + return nil, err + } + + for paginator.HasNext() { + page, err := paginator.NextPage(ctx) + if err != nil { + plugin.Logger(ctx).Error("ListArtifactDockerFile paginator.NextPage", "error", err) + return nil, err + } + + for _, v := range page { + d.StreamListItem(ctx, v) + } + } + + err = paginator.Close(ctx) + if err != nil { + return nil, err + } + + return nil, nil +} + +var getArtifactDockerFileFilters = map[string]string{} + +func GetArtifactDockerFile(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("GetArtifactDockerFile") + runtime.GC() + // create service + cfg := essdk.GetConfig(d.Connection) + ke, err := essdk.NewClientCached(cfg, d.ConnectionCache, ctx) + if err != nil { + return nil, err + } + k := Client{Client: ke} + + sc, err := steampipesdk.NewSelfClientCached(ctx, d.ConnectionCache) + if err != nil { + return nil, err + } + integrationID, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyIntegrationID) + if err != nil { + return nil, err + } + encodedResourceCollectionFilters, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyResourceCollectionFilters) + if err != nil { + return nil, err + } + clientType, err := sc.GetConfigTableValueOrNil(ctx, steampipesdk.OpenGovernanceConfigKeyClientType) + if err != nil { + return nil, err + } + + limit := int64(1) + paginator, err := k.NewArtifactDockerFilePaginator(essdk.BuildFilter(ctx, d.QueryContext, getArtifactDockerFileFilters, integrationID, encodedResourceCollectionFilters, clientType), &limit) + if err != nil { + return nil, err + } + + for paginator.HasNext() { + page, err := paginator.NextPage(ctx) + if err != nil { + return nil, err + } + + for _, v := range page { + return v, nil + } + } + + err = paginator.Close(ctx) + if err != nil { + return nil, err + } + + return nil, nil +} + +// ========================== END: ArtifactDockerFile ============================= diff --git a/provider/describer/artifact_dockerfile.go b/provider/describer/artifact_dockerfile.go new file mode 100644 index 00000000..8f5312ff --- /dev/null +++ b/provider/describer/artifact_dockerfile.go @@ -0,0 +1,229 @@ +package describer + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "github.com/opengovern/og-describer-github/pkg/sdk/models" + "github.com/opengovern/og-describer-github/provider/model" + resilientbridge "github.com/opengovern/resilient-bridge" + "github.com/opengovern/resilient-bridge/adapters" + "log" + "net/url" + "strings" + "time" +) + +func ListArtifactDockerFiles(ctx context.Context, githubClient GitHubClient, organizationName string, stream *models.StreamSender) ([]models.Resource, error) { + repositories, err := getRepositories(ctx, githubClient.RestClient, organizationName) + if err != nil { + return nil, err + } + var artifacts []model.ArtifactDockerFileDescription + for _, repo := range repositories { + outputs := fetchArtifactDockerFiles(ctx, githubClient, repo.GetFullName()) + artifacts = append(artifacts, outputs...) + } + var values []models.Resource + for _, artifact := range artifacts { + value := models.Resource{ + ID: artifact.URI, + Name: artifact.Name, + Description: JSONAllFieldsMarshaller{ + Value: artifact, + }, + } + if stream != nil { + if err := (*stream)(value); err != nil { + return nil, err + } + } else { + values = append(values, value) + } + } + + return values, nil +} + +func fetchArtifactDockerFiles(ctx context.Context, githubClient GitHubClient, repoName string) []model.ArtifactDockerFileDescription { + sdk := resilientbridge.NewResilientBridge() + sdk.SetDebug(false) // Disable debug + + sdk.RegisterProvider("github", adapters.NewGitHubAdapter(githubClient.Token), &resilientbridge.ProviderConfig{ + UseProviderLimits: true, + MaxRetries: 3, + BaseBackoff: time.Second, + }) + + // Construct the query + queryParts := []string{ + fmt.Sprintf("repo:%s", repoName), + "filename:Dockerfile", + } + + finalQuery := strings.Join(queryParts, " ") + + perPage := 100 + page := 1 + totalCollected := 0 + + var allItems []model.CodeSearchHit + + for { + if totalCollected >= 500 { + break + } + + q := url.QueryEscape(finalQuery) + searchEndpoint := fmt.Sprintf("/search/code?q=%s&per_page=%d&page=%d", q, perPage, page) + + searchReq := &resilientbridge.NormalizedRequest{ + Method: "GET", + Endpoint: searchEndpoint, + Headers: map[string]string{"Accept": "application/vnd.github+json"}, + } + + searchResp, err := sdk.Request("github", searchReq) + if err != nil { + log.Printf("Error searching code: %v\n", err) + break + } + + if searchResp.StatusCode >= 400 { + log.Printf("HTTP error %d searching code: %s\n", searchResp.StatusCode, string(searchResp.Data)) + break + } + + var result model.CodeSearchResult + if err := json.Unmarshal(searchResp.Data, &result); err != nil { + log.Printf("Error parsing code search response: %v\n", err) + break + } + + if len(result.Items) == 0 { + // No more results + break + } + + for _, item := range result.Items { + allItems = append(allItems, item) + totalCollected++ + if totalCollected >= 500 { + break + } + } + + if totalCollected >= 500 || len(result.Items) < perPage { + break + } + page++ + } + + var outputs []model.ArtifactDockerFileDescription + + for _, item := range allItems { + contentReq := &resilientbridge.NormalizedRequest{ + Method: "GET", + Endpoint: strings.TrimPrefix(item.URL, "https://api.github.com"), + Headers: map[string]string{"Accept": "application/vnd.github+json"}, + } + + contentResp, err := sdk.Request("github", contentReq) + if err != nil { + log.Printf("Error fetching content for %s/%s: %v\n", item.Repository.FullName, item.Path, err) + continue + } + + if contentResp.StatusCode >= 400 { + log.Printf("HTTP error %d fetching content for %s/%s: %s\n", contentResp.StatusCode, item.Repository.FullName, item.Path, string(contentResp.Data)) + continue + } + + var contentData model.ContentResponse + if err := json.Unmarshal(contentResp.Data, &contentData); err != nil { + log.Printf("Error parsing content response for %s/%s: %v\n", item.Repository.FullName, item.Path, err) + continue + } + + var fileContent string + if contentData.Encoding == "base64" { + decoded, err := base64.StdEncoding.DecodeString(contentData.Content) + if err != nil { + log.Printf("Error decoding base64 content for %s/%s: %v\n", item.Repository.FullName, item.Path, err) + continue + } + fileContent = string(decoded) + } else { + fileContent = contentData.Content + } + + lines := strings.Split(fileContent, "\n") + if len(lines) > 200 { + // Skip if more than 200 lines + log.Printf("Skipping %s/%s: more than 200 lines (%d lines)\n", item.Repository.FullName, item.Path, len(lines)) + continue + } + + // Fetch last_updated_at via commits API + lastUpdatedAt := "" + if item.Repository.FullName != "" && item.Path != "" { + commitsEndpoint := fmt.Sprintf("/repos/%s/commits?path=%s&per_page=1", item.Repository.FullName, url.QueryEscape(item.Path)) + commitReq := &resilientbridge.NormalizedRequest{ + Method: "GET", + Endpoint: commitsEndpoint, + Headers: map[string]string{"Accept": "application/vnd.github+json"}, + } + + commitResp, err := sdk.Request("github", commitReq) + if err != nil { + log.Printf("Error fetching commits for %s/%s: %v\n", item.Repository.FullName, item.Path, err) + } else if commitResp.StatusCode < 400 { + var commits []model.CommitResponse + if err := json.Unmarshal(commitResp.Data, &commits); err == nil && len(commits) > 0 { + if commits[0].Commit.Author.Date != "" { + lastUpdatedAt = commits[0].Commit.Author.Date + } else if commits[0].Commit.Committer.Date != "" { + lastUpdatedAt = commits[0].Commit.Committer.Date + } + } + } + } + + repoObj := map[string]interface{}{ + "id": item.Repository.ID, + "node_id": item.Repository.NodeID, + "name": item.Repository.Name, + "full_name": item.Repository.FullName, + "private": item.Repository.Private, + "public": !item.Repository.Private, + "owner": map[string]interface{}{ + "login": item.Repository.Owner.Login, + "id": item.Repository.Owner.ID, + "node_id": item.Repository.Owner.NodeID, + "html_url": item.Repository.Owner.HTMLURL, + "type": item.Repository.Owner.Type, + }, + "html_url": item.Repository.HTMLURL, + "description": item.Repository.Description, + "fork": item.Repository.Fork, + } + + output := model.ArtifactDockerFileDescription{ + Sha: item.Sha, + Name: item.Name, + Path: item.Path, + LastUpdatedAt: lastUpdatedAt, + GitURL: item.GitURL, + HTMLURL: item.HTMLURL, + URI: item.HTMLURL, // Unique identifier + DockerfileContent: fileContent, + DockerfileContentBase64: contentData.Content, + Repository: repoObj, + } + + outputs = append(outputs, output) + } + + return outputs +} diff --git a/provider/describer/container_package.go b/provider/describer/container_package.go index 651db0da..f6ef271d 100644 --- a/provider/describer/container_package.go +++ b/provider/describer/container_package.go @@ -2,56 +2,60 @@ package describer import ( "context" - "github.com/google/go-github/v55/github" + "encoding/json" + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/opengovern/og-describer-github/pkg/sdk/models" "github.com/opengovern/og-describer-github/provider/model" + resilientbridge "github.com/opengovern/resilient-bridge" + "github.com/opengovern/resilient-bridge/adapters" "strconv" ) func GetContainerPackageList(ctx context.Context, githubClient GitHubClient, organizationName string, stream *models.StreamSender) ([]models.Resource, error) { - client := githubClient.RestClient - page := 1 - var values []models.Resource - for { - packageType := "container" - var opts = &github.PackageListOptions{ - PackageType: &packageType, - ListOptions: github.ListOptions{ - Page: page, - PerPage: packagePageSize, - }, - } - respPackages, resp, err := client.Organizations.ListPackages(ctx, organizationName, opts) + sdk := resilientbridge.NewResilientBridge() + sdk.RegisterProvider("github", adapters.NewGitHubAdapter(githubClient.Token), &resilientbridge.ProviderConfig{ + UseProviderLimits: true, + MaxRetries: 3, + BaseBackoff: 0, + }) + packages, err := fetchPackages(sdk, organizationName, "container") + if err != nil { + return nil, err + } + var allResults []model.PackageOutputVersionDescription + for _, p := range packages { + packageName := p.Name + versions, err := fetchVersions(sdk, organizationName, "container", packageName) if err != nil { return nil, err } - for _, packageData := range respPackages { - value := models.Resource{ - ID: strconv.Itoa(int(packageData.GetID())), - Name: packageData.GetName(), - Description: JSONAllFieldsMarshaller{ - Value: model.PackageDescription{ - ID: strconv.Itoa(int(packageData.GetID())), - RegistryID: packageData.Registry.GetURL(), - Name: packageData.GetName(), - URL: packageData.GetURL(), - CreatedAt: packageData.GetCreatedAt(), - UpdatedAt: packageData.GetUpdatedAt(), - }, - }, - } - if stream != nil { - if err := (*stream)(value); err != nil { - return nil, err - } - } else { - values = append(values, value) + for _, v := range versions { + results, err := getVersionOutput(githubClient.Token, organizationName, packageName, v) + if err != nil { + return nil, err } + allResults = append(allResults, results...) } - if resp.After == "" { - break + } + var values []models.Resource + for _, result := range allResults { + value := models.Resource{ + ID: strconv.Itoa(result.ID), + Name: result.Name, + Description: JSONAllFieldsMarshaller{ + Value: result, + }, + } + if stream != nil { + if err := (*stream)(value); err != nil { + return nil, err + } + } else { + values = append(values, value) } - opts.ListOptions.Page += 1 } return values, nil @@ -86,3 +90,120 @@ func GetContainerPackage(ctx context.Context, githubClient GitHubClient, organiz return &value, nil } + +func getVersionOutput(apiToken, org, packageName string, version model.PackageVersion) ([]model.PackageOutputVersionDescription, error) { + // Each version can have multiple tags. We'll produce one output object per tag. + var results []model.PackageOutputVersionDescription + for _, tag := range version.Metadata.Container.Tags { + imageRef := fmt.Sprintf("ghcr.io/%s/%s:%s", org, packageName, tag) + ov, err := fetchAndAssembleOutput(apiToken, version, imageRef) + if err != nil { + return nil, err + } + results = append(results, *ov) + } + return results, nil +} + +func fetchAndAssembleOutput(apiToken string, version model.PackageVersion, imageRef string) (*model.PackageOutputVersionDescription, error) { + authOption := remote.WithAuth(&authn.Basic{ + Username: "github", + Password: apiToken, + }) + + ref, err := name.ParseReference(imageRef) + if err != nil { + return nil, err + } + + desc, err := remote.Get(ref, authOption) + if err != nil { + return nil, err + } + + var manifestStruct struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config struct { + Size int64 `json:"size"` + Digest string `json:"digest"` + MediaType string `json:"mediaType"` + } `json:"config"` + Layers []struct { + Size int64 `json:"size"` + Digest string `json:"digest"` + MediaType string `json:"mediaType"` + } `json:"layers"` + } + if err := json.Unmarshal(desc.Manifest, &manifestStruct); err != nil { + return nil, err + } + + totalSize := manifestStruct.Config.Size + for _, layer := range manifestStruct.Layers { + totalSize += layer.Size + } + + var manifestInterface interface{} + if err := json.Unmarshal(desc.Manifest, &manifestInterface); err != nil { + return nil, err + } + + return &model.PackageOutputVersionDescription{ + ID: version.ID, + Digest: version.Name, // version digest from "name" + URL: version.URL, + PackageURI: imageRef, + PackageHTMLURL: version.PackageHTMLURL, + CreatedAt: version.CreatedAt, + UpdatedAt: version.UpdatedAt, + HTMLURL: version.HTMLURL, + Name: imageRef, + MediaType: string(desc.Descriptor.MediaType), + TotalSize: totalSize, + Metadata: version.Metadata, + Manifest: manifestInterface, + }, nil +} + +func fetchPackages(sdk *resilientbridge.ResilientBridge, org, packageType string) ([]model.Package, error) { + listReq := &resilientbridge.NormalizedRequest{ + Method: "GET", + Endpoint: fmt.Sprintf("/orgs/%s/packages?package_type=%s", org, packageType), + Headers: map[string]string{"Accept": "application/vnd.github+json"}, + } + listResp, err := sdk.Request("github", listReq) + if err != nil { + return nil, err + } + if listResp.StatusCode >= 400 { + return nil, fmt.Errorf("HTTP error %d: %s", listResp.StatusCode, string(listResp.Data)) + } + var packages []model.Package + if err := json.Unmarshal(listResp.Data, &packages); err != nil { + return nil, fmt.Errorf("error parsing packages list response: %v", err) + } + return packages, nil +} + +func fetchVersions(sdk *resilientbridge.ResilientBridge, org, packageType, packageName string) ([]model.PackageVersion, error) { + versionsReq := &resilientbridge.NormalizedRequest{ + Method: "GET", + Endpoint: fmt.Sprintf("/orgs/%s/packages/%s/%s/versions", org, packageType, packageName), + Headers: map[string]string{"Accept": "application/vnd.github+json"}, + } + + versionsResp, err := sdk.Request("github", versionsReq) + if err != nil { + return nil, err + } + if versionsResp.StatusCode >= 400 { + return nil, fmt.Errorf("HTTP error %d: %s", versionsResp.StatusCode, string(versionsResp.Data)) + } + + var versions []model.PackageVersion + if err := json.Unmarshal(versionsResp.Data, &versions); err != nil { + return nil, fmt.Errorf("error parsing package versions response: %v", err) + } + return versions, nil +} diff --git a/provider/describer/utils.go b/provider/describer/utils.go index 908ad38c..ef7fddfd 100644 --- a/provider/describer/utils.go +++ b/provider/describer/utils.go @@ -13,6 +13,7 @@ import ( type GitHubClient struct { RestClient *github.Client GraphQLClient *githubv4.Client + Token string } const ( diff --git a/provider/describer_wrapper.go b/provider/describer_wrapper.go index c92f81ba..9b48a565 100755 --- a/provider/describer_wrapper.go +++ b/provider/describer_wrapper.go @@ -35,6 +35,7 @@ func DescribeByGithub(describe func(context.Context, describer.GitHubClient, str client := describer.GitHubClient{ RestClient: restClient, GraphQLClient: graphQLClient, + Token: cfg.PatToken, } organizationName := additionalParameters["OrganizationName"] diff --git a/provider/model/model.go b/provider/model/model.go index 3df7efaf..ab8e539a 100755 --- a/provider/model/model.go +++ b/provider/model/model.go @@ -709,6 +709,44 @@ type CodeOwnerDescription struct { LineComment string } +type Owner struct { + Login string `json:"login"` +} + +type Package struct { + ID int `json:"id"` + Name string `json:"name"` + PackageType string `json:"package_type"` + Visibility string `json:"visibility"` + HTMLURL string `json:"html_url"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + Owner Owner `json:"owner"` + URL string `json:"url"` +} + +type ContainerMetadata struct { + Container struct { + Tags []string `json:"tags"` + } `json:"container"` +} + +type PackageOutputVersionDescription struct { + ID int `json:"id"` + Digest string `json:"digest"` + URL string `json:"url"` + PackageURI string `json:"package_uri"` + PackageHTMLURL string `json:"package_html_url"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + HTMLURL string `json:"html_url"` + Name string `json:"name"` + MediaType string `json:"media_type"` + TotalSize int64 `json:"total_size"` + Metadata ContainerMetadata `json:"metadata"` + Manifest interface{} `json:"manifest"` +} + type PackageDescription struct { ID string RegistryID string @@ -718,6 +756,17 @@ type PackageDescription struct { UpdatedAt github.Timestamp } +type PackageVersion struct { + ID int `json:"id"` + Name string `json:"name"` + URL string `json:"url"` + PackageHTMLURL string `json:"package_html_url"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + HTMLURL string `json:"html_url"` + Metadata ContainerMetadata `json:"metadata"` +} + type PackageVersionDescription struct { ID int Name string @@ -727,3 +776,74 @@ type PackageVersionDescription struct { CreatedAt github.Timestamp UpdatedAt github.Timestamp } + +type CodeSearchResult struct { + TotalCount int `json:"total_count"` + IncompleteResults bool `json:"incomplete_results"` + Items []CodeSearchHit `json:"items"` +} + +type CodeSearchHit struct { + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + URL string `json:"url"` + GitURL string `json:"git_url"` + HTMLURL string `json:"html_url"` + Repository struct { + ID int `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Private bool `json:"private"` + Owner struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + Type string `json:"type"` + } `json:"owner"` + HTMLURL string `json:"html_url"` + Description string `json:"description"` + Fork bool `json:"fork"` + } `json:"repository"` + Score float64 `json:"score"` +} + +type ContentResponse struct { + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + Size int `json:"size"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + GitURL string `json:"git_url"` + Type string `json:"type"` + Content string `json:"content"` // base64 + Encoding string `json:"encoding"` +} + +type CommitResponse struct { + Commit struct { + Author struct { + Date string `json:"date"` + } `json:"author"` + Committer struct { + Date string `json:"date"` + } `json:"committer"` + } `json:"commit"` +} + +type ArtifactDockerFileDescription struct { + Sha string `json:"sha"` + Name string `json:"name"` + Path string `json:"path"` + LastUpdatedAt string `json:"last_updated_at"` + GitURL string `json:"git_url"` + HTMLURL string `json:"html_url"` + URI string `json:"uri"` // Unique identifier + DockerfileContent string `json:"dockerfile_content"` + DockerfileContentBase64 string `json:"dockerfile_content_base64"` + Repository map[string]interface{} `json:"repository"` +} diff --git a/provider/resource_types.go b/provider/resource_types.go index 0b7db4d6..c444c783 100644 --- a/provider/resource_types.go +++ b/provider/resource_types.go @@ -621,4 +621,18 @@ var ResourceTypes = map[string]model.ResourceType{ ListDescriber: DescribeByGithub(describer.GetAllPackageVersionList), GetDescriber: nil, }, + + "Github/ArtifactDockerFile": { + IntegrationType: configs.IntegrationName, + ResourceName: "Github/ArtifactDockerFile", + Tags: map[string][]string{ + "category": {"artifact_dockerfile"}, + }, + Labels: map[string]string{ + }, + Annotations: map[string]string{ + }, + ListDescriber: DescribeByGithub(describer.ListArtifactDockerFiles), + GetDescriber: nil, + }, } diff --git a/provider/resource_types/resource-types.json b/provider/resource_types/resource-types.json index 31687aea..9852f44e 100644 --- a/provider/resource_types/resource-types.json +++ b/provider/resource_types/resource-types.json @@ -465,7 +465,7 @@ "ListDescriber": "DescribeByGithub(describer.GetContainerPackageList)", "GetDescriber": "DescribeSingleByRepo(describer.GetContainerPackage)", "SteampipeTable": "github_container_package", - "Model": "Package" + "Model": "PackageOutputVersion" }, { "ResourceName": "Github/Package/Maven", @@ -526,5 +526,17 @@ "GetDescriber": "", "SteampipeTable": "github_package_version", "Model": "PackageVersion" + }, + { + "ResourceName": "Github/ArtifactDockerFile", + "Tags": { + "category": [ + "artifact_dockerfile" + ] + }, + "ListDescriber": "DescribeByGithub(describer.ListArtifactDockerFiles)", + "GetDescriber": "", + "SteampipeTable": "github_artifact_dockerfile", + "Model": "ArtifactDockerFile" } ] \ No newline at end of file diff --git a/steampipe-plugin-github/github/plugin.go b/steampipe-plugin-github/github/plugin.go index 8ede841f..88a0acfb 100644 --- a/steampipe-plugin-github/github/plugin.go +++ b/steampipe-plugin-github/github/plugin.go @@ -66,6 +66,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "github_nuget_package": tableGitHubNugetPackage(), "github_ruby_gems_package": tableGitHubRubyGemsPackage(), "github_package_version": tableGitHubPackageVersion(), + "github_artifact_dockerfile": tableGitHubArtifactDockerFile(), }, } for key, table := range p.TableMap { diff --git a/steampipe-plugin-github/github/table_github_artifact_dockerfile.go b/steampipe-plugin-github/github/table_github_artifact_dockerfile.go new file mode 100644 index 00000000..2c86afa3 --- /dev/null +++ b/steampipe-plugin-github/github/table_github_artifact_dockerfile.go @@ -0,0 +1,32 @@ +package github + +import ( + "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" +) + +func tableGitHubArtifactDockerFile() *plugin.Table { + return &plugin.Table{ + Name: "github_artifact_dockerfile", + List: &plugin.ListConfig{ + Hydrate: nil, + }, + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"sha"}), + Hydrate: nil, + }, + Columns: []*plugin.Column{ + // Basic details columns + {Name: "sha", Type: proto.ColumnType_STRING, Description: "SHA hash of the Dockerfile."}, + {Name: "name", Type: proto.ColumnType_STRING, Description: "Name of the Dockerfile."}, + {Name: "path", Type: proto.ColumnType_STRING, Description: "Path to the Dockerfile in the repository."}, + {Name: "last_updated_at", Type: proto.ColumnType_TIMESTAMP, Description: "Timestamp when the Dockerfile was last updated."}, + {Name: "git_url", Type: proto.ColumnType_STRING, Description: "Git URL where the Dockerfile can be accessed."}, + {Name: "html_url", Type: proto.ColumnType_STRING, Description: "HTML URL where the Dockerfile can be accessed."}, + {Name: "uri", Type: proto.ColumnType_STRING, Description: "Unique URI for the Dockerfile."}, + {Name: "dockerfile_content", Type: proto.ColumnType_STRING, Description: "Content of the Dockerfile."}, + {Name: "dockerfile_content_base64", Type: proto.ColumnType_STRING, Description: "Base64-encoded content of the Dockerfile."}, + {Name: "repository", Type: proto.ColumnType_JSON, Description: "Repository metadata associated with the Dockerfile."}, + }, + } +} diff --git a/steampipe-plugin-github/github/table_github_container_package.go b/steampipe-plugin-github/github/table_github_container_package.go index 62d52054..e5a90f91 100644 --- a/steampipe-plugin-github/github/table_github_container_package.go +++ b/steampipe-plugin-github/github/table_github_container_package.go @@ -18,12 +18,19 @@ func tableGitHubContainerPackage() *plugin.Table { }, Columns: []*plugin.Column{ // Basic details columns - {Name: "id", Type: proto.ColumnType_STRING, Description: "Unique identifier for the package."}, - {Name: "registryId", Type: proto.ColumnType_STRING, Description: "Registry ID associated with the package."}, - {Name: "name", Type: proto.ColumnType_STRING, Description: "Name of the package."}, + {Name: "id", Type: proto.ColumnType_INT, Description: "Unique identifier for the package."}, + {Name: "digest", Type: proto.ColumnType_STRING, Description: "Digest of the package."}, {Name: "url", Type: proto.ColumnType_STRING, Description: "URL where the package can be accessed."}, - {Name: "createdAt", Type: proto.ColumnType_TIMESTAMP, Description: "Timestamp when the package was created."}, - {Name: "updatedAt", Type: proto.ColumnType_TIMESTAMP, Description: "Timestamp when the package was last updated."}, + {Name: "package_uri", Type: proto.ColumnType_STRING, Description: "URI of the package."}, + {Name: "package_html_url", Type: proto.ColumnType_STRING, Description: "HTML URL of the package."}, + {Name: "created_at", Type: proto.ColumnType_TIMESTAMP, Description: "Timestamp when the package was created."}, + {Name: "updated_at", Type: proto.ColumnType_TIMESTAMP, Description: "Timestamp when the package was last updated."}, + {Name: "html_url", Type: proto.ColumnType_STRING, Description: "HTML URL for the package."}, + {Name: "name", Type: proto.ColumnType_STRING, Description: "Name of the package."}, + {Name: "media_type", Type: proto.ColumnType_STRING, Description: "Media type of the package."}, + {Name: "total_size", Type: proto.ColumnType_INT, Description: "Total size of the package."}, + {Name: "metadata", Type: proto.ColumnType_JSON, Description: "Metadata of the package."}, + {Name: "manifest", Type: proto.ColumnType_JSON, Description: "Manifest of the package."}, }, } } diff --git a/steampipe/table_index_map.go b/steampipe/table_index_map.go index 92af57a1..bc06f4ca 100644 --- a/steampipe/table_index_map.go +++ b/steampipe/table_index_map.go @@ -49,6 +49,7 @@ var Map = map[string]string{ "Github/Package/RubyGems": "github_rubygems_package", "Github/Package/Nuget": "github_nuget_package", "Github/Package/Version": "github_package_version", + "Github/ArtifactDockerFile": "github_artifact_dockerfile", } var DescriptionMap = map[string]interface{}{ @@ -90,12 +91,13 @@ var DescriptionMap = map[string]interface{}{ "Github/User": opengovernance.User{}, "Github/Workflow": opengovernance.Workflow{}, "Github/CodeOwner": opengovernance.CodeOwner{}, - "Github/Package/Container": opengovernance.Package{}, + "Github/Package/Container": opengovernance.PackageOutputVersion{}, "Github/Package/Maven": opengovernance.Package{}, "Github/Package/NPM": opengovernance.Package{}, "Github/Package/RubyGems": opengovernance.Package{}, "Github/Package/Nuget": opengovernance.Package{}, "Github/Package/Version": opengovernance.PackageVersion{}, + "Github/ArtifactDockerFile": opengovernance.ArtifactDockerFile{}, } var ReverseMap = map[string]string{ @@ -143,4 +145,5 @@ var ReverseMap = map[string]string{ "github_rubygems_package": "Github/Package/RubyGems", "github_nuget_package": "Github/Package/Nuget", "github_package_version": "Github/Package/Version", + "github_artifact_dockerfile": "Github/ArtifactDockerFile", }