feat: update build process

opengovern · Dec 22, 2024 · 63871b9 · 63871b9
1 parent 370801a
commit 63871b9
Show file tree

Hide file tree

Showing 8 changed files with 98 additions and 305 deletions.
diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
@@ -151,6 +151,44 @@ jobs:
           path: build.tar.gz
           retention-days: 1
   deploy-steampipe:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+      - tag
+      - deploy-steampipe-base
+    permissions:
+      id-token: write
+      contents: read
+    environment: docker
+    if: (needs.build.outputs.steampipe-base == 'true' || needs.build.outputs.steampipe == 'true') && github.event_name != 'pull_request'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: build
+          path: .
+      - name: Unpack artifact
+        run: |
+          tar -xvf build.tar.gz
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GHCR_PAT }}
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/steampipe-service:${{ needs.tag.outputs.latest_tag }}
+          file: docker/SteampipeServiceDockerfile
+          build-args: |
+            PLUGIN_REGISTRY=ghcr.io/opengovern
+          context: .
+  deploy-steampipe-base:
     runs-on: ubuntu-latest
     needs:
       - build
@@ -184,7 +222,7 @@ jobs:
           push: true
           tags: |
             ghcr.io/${{ github.repository_owner }}/steampipe-service:${{ needs.tag.outputs.latest_tag }}
-          file: docker/SteampipeServiceDockerfile
+          file: docker/SteampipeBaseImageDockerfile
           build-args: |
             PLUGIN_REGISTRY=ghcr.io/opengovern
           context: .

diff --git a/docker/AuditJobDockerfile b/docker/AuditJobDockerfile
@@ -1,57 +1,6 @@
-ARG PLUGIN_REGISTRY
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-aws:0.0.1 as aws
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-azure:0.0.1 as azure
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-entraid:0.0.1 as entraid
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-github:0.0.1 as github
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-digitalocean:0.0.1 as digitalocean
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-cloudflare:0.0.1 as cloudflare
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-openai:0.0.1 as openai
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-linode:0.0.1 as linode
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-cohereai:0.0.1 as cohereai
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-oci:0.0.1 as oci
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-render:0.0.1 as render
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-doppler:0.0.1 as doppler
-
-
-
-FROM ${PLUGIN_REGISTRY}/cloudql:0.0.1 as opengovernance
-
-FROM ubuntu:20.04 AS base
-RUN apt-get update && apt-get install -y \
-    curl \
-    sudo \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)"
-
-COPY --from=aws /steampipe-plugin-aws.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/aws@latest/steampipe-plugin-aws.plugin
-COPY --from=azure /steampipe-plugin-azure.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azure@latest/steampipe-plugin-azure.plugin
-COPY --from=entraid /steampipe-plugin-entraid.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/entraid@latest/steampipe-plugin-entraid.plugin
-COPY --from=github /steampipe-plugin-github.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/github@latest/steampipe-plugin-github.plugin
-COPY --from=digitalocean /steampipe-plugin-digitalocean.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/digitalocean@latest/steampipe-plugin-digitalocean.plugin
-COPY --from=cloudflare /steampipe-plugin-cloudflare.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/cloudflare@latest/steampipe-plugin-cloudflare.plugin
-COPY --from=openai /steampipe-plugin-openai.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/openai@latest/steampipe-plugin-openai.plugin
-COPY --from=linode /steampipe-plugin-linode.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/linode@latest/steampipe-plugin-linode.plugin
-COPY --from=cohereai /steampipe-plugin-cohereai.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/cohereai@latest/steampipe-plugin-cohereai.plugin
-COPY --from=oci /steampipe-plugin-oci.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/oci@latest/steampipe-plugin-oci.plugin
-COPY --from=render /steampipe-plugin-render.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/render@latest/steampipe-plugin-render.plugin
-COPY --from=doppler /steampipe-plugin-doppler.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/doppler@latest/steampipe-plugin-doppler.plugin
-
-COPY --from=opengovernance /cloudql.plugin /home/steampipe/.steampipe/plugins/local/opengovernance/opengovernance.plugin
-
-USER root
-RUN useradd -ms /bin/bash steampipe
-RUN mkdir -p /home/steampipe/.steampipe/config
-RUN mkdir -p /home/steampipe/.steampipe/db
-RUN mkdir -p /home/steampipe/.steampipe/db/14.2.0
-RUN chown -R steampipe:steampipe /home/steampipe
-RUN chmod -R 755 /home/steampipe
-RUN apt update
-RUN apt install -y procps htop
-USER steampipe
-
-RUN steampipe plugin list
 
+ARG PLUGIN_REGISTRY
+FROM ${PLUGIN_REGISTRY}/steampipe-plugin-base:0.0.1 as base
 COPY ./build/audit-job /
 
 ENTRYPOINT [ "/audit-job" ]

diff --git a/docker/ComplianceReportJobDockerfile b/docker/ComplianceReportJobDockerfile
@@ -1,58 +1,5 @@
 ARG PLUGIN_REGISTRY
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-aws:0.0.1 as aws
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-azure:0.0.1 as azure
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-entraid:0.0.1 as entraid
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-github:0.0.1 as github
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-digitalocean:0.0.1 as digitalocean
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-cloudflare:0.0.1 as cloudflare
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-openai:0.0.1 as openai
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-linode:0.0.1 as linode
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-cohereai:0.0.1 as cohereai
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-googleworkspace:0.0.1 as googleworkspace
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-oci:0.0.1 as oci
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-render:0.0.1 as render
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-doppler:0.0.1 as doppler
-
-
-FROM ${PLUGIN_REGISTRY}/cloudql:0.0.1 as opengovernance
-
-FROM ubuntu:20.04 AS base
-RUN apt-get update && apt-get install -y \
-    curl \
-    sudo \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)"
-
-COPY --from=aws /steampipe-plugin-aws.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/aws@latest/steampipe-plugin-aws.plugin
-COPY --from=azure /steampipe-plugin-azure.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azure@latest/steampipe-plugin-azure.plugin
-COPY --from=entraid /steampipe-plugin-entraid.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/entraid@latest/steampipe-plugin-entraid.plugin
-COPY --from=github /steampipe-plugin-github.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/github@latest/steampipe-plugin-github.plugin
-COPY --from=digitalocean /steampipe-plugin-digitalocean.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/digitalocean@latest/steampipe-plugin-digitalocean.plugin
-COPY --from=cloudflare /steampipe-plugin-cloudflare.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/cloudflare@latest/steampipe-plugin-cloudflare.plugin
-COPY --from=openai /steampipe-plugin-openai.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/openai@latest/steampipe-plugin-openai.plugin
-COPY --from=linode /steampipe-plugin-linode.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/linode@latest/steampipe-plugin-linode.plugin
-COPY --from=cohereai /steampipe-plugin-cohereai.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/cohereai@latest/steampipe-plugin-cohereai.plugin
-COPY --from=googleworkspace /steampipe-plugin-googleworkspace.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/googleworkspace@latest/steampipe-plugin-googleworkspace.plugin
-COPY --from=oci /steampipe-plugin-oci.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/oci@latest/steampipe-plugin-oci.plugin
-COPY  --from=render /steampipe-plugin-render.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/render@latest/steampipe-plugin-render.plugin
-COPY --from=doppler /steampipe-plugin-doppler.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/doppler@latest/steampipe-plugin-doppler.plugin
-
-COPY --from=opengovernance /cloudql.plugin /home/steampipe/.steampipe/plugins/local/opengovernance/opengovernance.plugin
-
-USER root
-RUN useradd -ms /bin/bash steampipe
-RUN mkdir -p /home/steampipe/.steampipe/config
-RUN mkdir -p /home/steampipe/.steampipe/db
-RUN mkdir -p /home/steampipe/.steampipe/db/14.2.0
-RUN chown -R steampipe:steampipe /home/steampipe
-RUN chmod -R 755 /home/steampipe
-RUN apt update
-RUN apt install -y procps htop
-USER steampipe
-
-RUN steampipe plugin list
-
+FROM ${PLUGIN_REGISTRY}/steampipe-plugin-base:0.0.1 as base
 COPY ./build/compliance-report-job /
 
 ENTRYPOINT [ "/compliance-report-job" ]

diff --git a/docker/QueryRunnerJobDockerfile b/docker/QueryRunnerJobDockerfile
@@ -1,54 +1,5 @@
 ARG PLUGIN_REGISTRY
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-aws:0.0.1 as aws
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-azure:0.0.1 as azure
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-entraid:0.0.1 as entraid
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-github:0.0.1 as github
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-digitalocean:0.0.1 as digitalocean
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-cloudflare:0.0.1 as cloudflare
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-openai:0.0.1 as openai
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-linode:0.0.1 as linode
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-cohereai:0.0.1 as cohereai
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-googleworkspace:0.0.1 as googleworkspace
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-oci:0.0.1 as oci
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-render:0.0.1 as render
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-doppler:0.0.1 as doppler
-
-FROM ${PLUGIN_REGISTRY}/cloudql:0.0.1 as opengovernance
-
-FROM ubuntu:20.04 AS base
-RUN apt-get update && apt-get install -y \
-    curl \
-    sudo \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)"
-
-COPY --from=aws /steampipe-plugin-aws.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/aws@latest/steampipe-plugin-aws.plugin
-COPY --from=azure /steampipe-plugin-azure.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azure@latest/steampipe-plugin-azure.plugin
-COPY --from=entraid /steampipe-plugin-entraid.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/entraid@latest/steampipe-plugin-entraid.plugin
-COPY --from=github /steampipe-plugin-github.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/github@latest/steampipe-plugin-github.plugin
-COPY --from=digitalocean /steampipe-plugin-digitalocean.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/digitalocean@latest/steampipe-plugin-digitalocean.plugin
-COPY --from=cloudflare /steampipe-plugin-cloudflare.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/cloudflare@latest/steampipe-plugin-cloudflare.plugin
-COPY --from=openai /steampipe-plugin-openai.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/openai@latest/steampipe-plugin-openai.plugin
-COPY --from=linode /steampipe-plugin-linode.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/linode@latest/steampipe-plugin-linode.plugin
-COPY --from=cohereai /steampipe-plugin-cohereai.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/cohereai@latest/steampipe-plugin-cohereai.plugin
-COPY --from=googleworkspace /steampipe-plugin-googleworkspace.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/googleworkspace@latest/steampipe-plugin-googleworkspace.plugin
-COPY --from=oci /steampipe-plugin-oci.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/oci@latest/steampipe-plugin-oci.plugin
-COPY --from=render /steampipe-plugin-render.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/render@latest/steampipe-plugin-render.plugin
-COPY --from=opengovernance /cloudql.plugin /home/steampipe/.steampipe/plugins/local/opengovernance/opengovernance.plugin
-COPY --from=doppler /steampipe-plugin-doppler.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/doppler@latest/steampipe-plugin-doppler.plugin
-USER root
-RUN useradd -ms /bin/bash steampipe
-RUN mkdir -p /home/steampipe/.steampipe/config
-RUN mkdir -p /home/steampipe/.steampipe/db
-RUN mkdir -p /home/steampipe/.steampipe/db/14.2.0
-RUN chown -R steampipe:steampipe /home/steampipe
-RUN chmod -R 755 /home/steampipe
-RUN apt update
-RUN apt install -y procps htop
-USER steampipe
-
-RUN steampipe plugin list
+FROM ${PLUGIN_REGISTRY}/steampipe-plugin-base:0.0.1 as base
 
 COPY ./build/query-runner-job /
 

diff --git a/docker/QueryValidatorJobDockerfile b/docker/QueryValidatorJobDockerfile
@@ -1,54 +1,5 @@
 ARG PLUGIN_REGISTRY
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-aws:0.0.1 as aws
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-azure:0.0.1 as azure
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-entraid:0.0.1 as entraid
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-github:0.0.1 as github
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-digitalocean:0.0.1 as digitalocean
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-cloudflare:0.0.1 as cloudflare
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-openai:0.0.1 as openai
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-linode:0.0.1 as linode
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-cohereai:0.0.1 as cohereai
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-googleworkspace:0.0.1 as googleworkspace
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-oci:0.0.1 as oci
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-render:0.0.1 as render
-FROM ${PLUGIN_REGISTRY}/steampipe-plugin-doppler:0.0.1 as doppler
-FROM ${PLUGIN_REGISTRY}/cloudql:0.0.1 as opengovernance
-
-FROM ubuntu:20.04 AS base
-RUN apt-get update && apt-get install -y \
-    curl \
-    sudo \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)"
-
-COPY --from=aws /steampipe-plugin-aws.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/aws@latest/steampipe-plugin-aws.plugin
-COPY --from=azure /steampipe-plugin-azure.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/azure@latest/steampipe-plugin-azure.plugin
-COPY --from=entraid /steampipe-plugin-entraid.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/entraid@latest/steampipe-plugin-entraid.plugin
-COPY --from=github /steampipe-plugin-github.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/github@latest/steampipe-plugin-github.plugin
-COPY --from=digitalocean /steampipe-plugin-digitalocean.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/digitalocean@latest/steampipe-plugin-digitalocean.plugin
-COPY --from=cloudflare /steampipe-plugin-cloudflare.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/cloudflare@latest/steampipe-plugin-cloudflare.plugin
-COPY --from=openai /steampipe-plugin-openai.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/openai@latest/steampipe-plugin-openai.plugin
-COPY --from=linode /steampipe-plugin-linode.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/linode@latest/steampipe-plugin-linode.plugin
-COPY --from=cohereai /steampipe-plugin-cohereai.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/cohereai@latest/steampipe-plugin-cohereai.plugin
-COPY --from=googleworkspace /steampipe-plugin-googleworkspace.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/googleworkspace@latest/steampipe-plugin-googleworkspace.plugin
-COPY --from=oci /steampipe-plugin-oci.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/oci@latest/steampipe-plugin-oci.plugin
-COPY --from=render /steampipe-plugin-render.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/render@latest/steampipe-plugin-render.plugin
-COPY --from=opengovernance /cloudql.plugin /home/steampipe/.steampipe/plugins/local/opengovernance/opengovernance.plugin
-COPY --from=doppler /steampipe-plugin-doppler.plugin /home/steampipe/.steampipe/plugins/hub.steampipe.io/plugins/turbot/doppler@latest/steampipe-plugin-doppler.plugin
-USER root
-RUN useradd -ms /bin/bash steampipe
-RUN mkdir -p /home/steampipe/.steampipe/config
-RUN mkdir -p /home/steampipe/.steampipe/db
-RUN mkdir -p /home/steampipe/.steampipe/db/14.2.0
-RUN chown -R steampipe:steampipe /home/steampipe
-RUN chmod -R 755 /home/steampipe
-RUN apt update
-RUN apt install -y procps htop
-USER steampipe
-
-RUN steampipe plugin list
-
+FROM ${PLUGIN_REGISTRY}/steampipe-plugin-base:0.0.1 as base
 COPY ./build/query-validator-job /
 
 ENTRYPOINT [ "/query-validator-job" ]