chore(ci): s3 pkg uploads, new release workflow

Packages & install scripts are now uploaded to AWS S3. Added new releases workflow & documentation on how to trigger a release. Signed-off-by: Justin Kolberg <[email protected]>
SumoLogic · Nov 6, 2024 · f052486 · f052486
1 parent c80cbe4
commit f052486
Show file tree

Hide file tree

Showing 48 changed files with 1,953 additions and 1,722 deletions.
diff --git a/.github/workflows/_reusable_build_package.yml b/.github/workflows/_reusable_build_package.yml
@@ -38,6 +38,10 @@ on:
         required: false
         type: boolean
         default: false
+      packagecloud_go_version:
+        required: false
+        type: string
+        default: "0.2.2"
     secrets:
       gh_artifacts_token:
         required: true
@@ -61,6 +65,14 @@ on:
         required: true
       packagecloud_token:
         required: true
+      aws_access_key_id:
+        required: true
+      aws_secret_access_key:
+        required: true
+    outputs:
+      otc_build_number:
+        description: "The build number of the package"
+        value: ${{ jobs.build_package.outputs.otc_build_number }}
 
 defaults:
   run:
@@ -72,23 +84,12 @@ jobs:
     name: Build (CMake)
     if: inputs.build_tool == 'cmake'
     outputs:
+      otc_build_number: ${{ steps.get-build-number.outputs.otc_build_number }}
       package_path: ${{ steps.package.outputs.path }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Download packagecloud-go tool
-        run: |
-          baseURL="https://github.com/amdprophet/packagecloud-go/releases/download"
-          version="0.1.5"
-          file="packagecloud-go_${version}_linux_amd64.tar.gz"
-          curl -Lo /tmp/packagecloud-go.tar.gz $baseURL/$version/$file
-
-      - name: Install packagecloud-go tool
-        run: |
-          tar -C /tmp -zxf /tmp/packagecloud-go.tar.gz
-          sudo mv /tmp/packagecloud /usr/local/bin
-
       - name: Workflow URL for sumologic-otel-collector
         if: ${{ !inputs.use_release_artifacts && inputs.workflow_id != '' }}
         run: |
@@ -97,6 +98,15 @@ jobs:
           workflow_id="${{ inputs.workflow_id }}"
           echo "https://github.com/${org}/${repo}/actions/runs/${workflow_id}"
 
+      # Only output build number on one target so that it can be read by other
+      # jobs
+      - name: Output Build Number
+        if: inputs.cmake_target == 'otc_linux_amd64_deb'
+        id: get-build-number
+        run: |
+          build_number="${{ inputs.otc_build_number }}"
+          echo "otc_build_number=${build_number}" >> $GITHUB_OUTPUT
+
       - name: Determine if MacOS package should be signed
         if: runner.os == 'macOS'
         env:
@@ -228,12 +238,37 @@ jobs:
           path: ./build/${{ steps.package.outputs.path }}
           if-no-files-found: error
 
-      - name: Publish package to Packagecloud
+      - name: Publish packages
         if: runner.os == 'Linux'
         uses: ./ci/github-actions/make
         with:
           target: publish-package
           packagecloud-token: ${{ secrets.PACKAGECLOUD_TOKEN }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Publish packages
+        if: runner.os != 'Linux'
+        working-directory: build
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+        run: make publish-package
+
+      - name: Wait for Packagecloud packages to be indexed
+        if: runner.os == 'Linux'
+        uses: ./ci/github-actions/make
+        with:
+          target: wait-for-packagecloud-indexing
+          packagecloud-token: ${{ secrets.PACKAGECLOUD_TOKEN }}
+
+      - name: Wait for Packagecloud packages to be indexed
+        if: runner.os == 'Linux'
+        uses: ./ci/github-actions/make
+        with:
+          target: wait-for-packagecloud-indexing
+          packagecloud-token: ${{ secrets.PACKAGECLOUD_TOKEN }}
 
   test_package:
     runs-on: ${{ inputs.runs_on }}
@@ -259,6 +294,7 @@ jobs:
     name: Build (WiX) ${{ inputs.fips && 'FIPS' || '' }}
     if: inputs.build_tool == 'wix'
     env:
+      OTC_VERSION: ${{ inputs.otc_version }}-${{ inputs.otc_build_number }}
       PRODUCT_VERSION: ${{ inputs.otc_version }}.${{ inputs.otc_build_number }}
     steps:
       - name: Checkout
@@ -363,6 +399,17 @@ jobs:
           path: ${{ steps.build.outputs.package_path }}
           if-no-files-found: error
 
+      - name: Publish packages
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-west-2
+          PKG_PATH: ${{ steps.build.outputs.package_path }}
+          PKG_NAME: ${{ steps.build.outputs.package_filename }}
+          S3_BUCKET: sumologic-osc-ci-builds
+        run: |
+          aws.exe s3 cp --debug $PKG_PATH s3://${S3_BUCKET}/${OTC_VERSION}/${PKG_NAME}
+
   test_wixext:
     name: Test (SumoLogic.wixext)
     if: inputs.build_tool == 'wix'

diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml
@@ -75,6 +75,10 @@ on:
         type: boolean
         required: false
         default: false
+      is_latest:
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   determine_workflow:
@@ -199,6 +203,8 @@ jobs:
       microsoft_description: ${{ secrets.MICROSOFT_DESCRIPTION }}
       gh_ci_token: ${{ secrets.GH_CI_TOKEN }}
       packagecloud_token: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
     strategy:
       matrix:
@@ -249,6 +255,16 @@ jobs:
   install-script:
     name: Store install script
     runs-on: ubuntu-latest
+    needs:
+      - determine_version
+      - build_packages
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_S3_BUCKET: sumologic-osc-ci-builds
+      OTC_VERSION: ${{ needs.determine_version.outputs.otc_version }}
+      BUILD_NUMBER: ${{ needs.build_packages.outputs.otc_build_number }}
     steps:
       - uses: actions/checkout@v4
 
@@ -266,6 +282,22 @@ jobs:
           path: ./install-script/install.ps1
           if-no-files-found: error
 
+      - name: Store install scripts on S3
+        run: |
+          version="${OTC_VERSION}-${BUILD_NUMBER}"
+          s3_path="${version}/"
+          aws s3 cp install-script/install.ps1 s3://${AWS_S3_BUCKET}/${s3_path}
+          aws s3 cp install-script/install.sh s3://${AWS_S3_BUCKET}/${s3_path}
+
+      - name: Create latest_version file
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          version="${OTC_VERSION}-${BUILD_NUMBER}"
+          echo "${version}" >> latest_version
+          aws s3 cp --content-type "text/plain" latest_version s3://${AWS_S3_BUCKET}/
+
   publish_release:
     name: Publish Release
     runs-on: ubuntu-latest
@@ -331,3 +363,74 @@ jobs:
             This release packages [${{ env.OTC_APP_VERSION }}](https://github.com/SumoLogic/sumologic-otel-collector/releases/tag/${{ env.OTC_APP_VERSION }}).
 
             The changelog below is for the package itself, rather than the Sumo Logic Distribution for OpenTelemetry Collector.
+
+  test-install-script:
+    name: Test Install Script
+    runs-on: ${{ matrix.runs_on }}
+    timeout-minutes: 60
+    needs:
+      - determine_version
+      - build_packages
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch_os: linux_amd64
+            runs_on: ubuntu-20.04
+          - arch_os: darwin_amd64
+            runs_on: macos-latest
+          - arch_os: windows_amd64
+            runs_on: windows-2022
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GH_CI_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      OTC_VERSION: ${{ needs.determine_version.outputs.otc_version }}
+      OTC_BUILD_NUMBER: ${{ github.run_number }}
+      PACKAGECLOUD_MASTER_TOKEN: ${{ secrets.PACKAGECLOUD_MASTER_TOKEN }}
+      PACKAGECLOUD_REPO: ci-builds
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check if test related files changed
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+          files: |
+            install-script/**/*
+            .github/**
+
+      - name: Setup go
+        if: steps.changed-files.outputs.any_changed == 'true'
+        uses: WillAbides/setup-go-faster@v1
+        with:
+          go-version: stable
+
+      - name: Download macOS package and use it for install.sh
+        if: ${{ steps.changed-files.outputs.any_changed == 'true' && runner.os == 'macOS' }}
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts/
+          pattern: otelcol-sumo_*-intel.pkg
+
+      - name: Show packages
+        if: ${{ steps.changed-files.outputs.any_changed == 'true' && runner.os == 'macOS' }}
+        run: |
+          ls -l artifacts/
+          ls -l artifacts/**/*
+
+      - name: Set DARWIN_PKG_URL
+        if: ${{ steps.changed-files.outputs.any_changed == 'true' && runner.os == 'macOS' }}
+        run: |
+          fp="$(readlink -f artifacts/otelcol-sumo_*-intel.pkg/otelcol-sumo_*-intel.pkg)"
+          echo DARWIN_PKG_URL="file://${fp}" >> $GITHUB_ENV
+
+      - name: Run install script tests (*nix)
+        if: steps.changed-files.outputs.any_changed == 'true' && runner.os != 'Windows'
+        working-directory: install-script/test
+        run: make test
+
+      - name: Run install script tests (Windows)
+        shell: powershell
+        if: steps.changed-files.outputs.any_changed == 'true' && runner.os == 'Windows'
+        working-directory: install-script/test
+        run: make test