upload packages & install scripts to s3

Signed-off-by: Justin Kolberg <[email protected]>

.github/workflows/_reusable_build_package.yml

@@ -38,6 +38,10 @@ on:
         required: false
         type: boolean
         default: false
+      packagecloud_go_version:
+        required: false
+        type: string
+        default: "0.2.2"
     secrets:
       gh_artifacts_token:
         required: true
@@ -61,6 +65,14 @@ on:
         required: true
       packagecloud_token:
         required: true
+      aws_access_key_id:
+        required: true
+      aws_secret_access_key:
+        required: true
+    outputs:
+      otc_build_number:
+        description: "The build number of the package"
+        value: ${{ jobs.build_package.outputs.otc_build_number }}
 
 defaults:
   run:
@@ -72,6 +84,7 @@ jobs:
     name: Build (CMake)
     if: inputs.build_tool == 'cmake'
     outputs:
+      otc_build_number: ${{ steps.get-build-number.outputs.otc_build_number }}
       package_path: ${{ steps.package.outputs.path }}
     steps:
       - name: Checkout
@@ -85,6 +98,15 @@ jobs:
           workflow_id="${{ inputs.workflow_id }}"
           echo "https://github.com/${org}/${repo}/actions/runs/${workflow_id}"
 
+      # Only output build number on one target so that it can be read by other
+      # jobs
+      - name: Output Build Number
+        if: inputs.cmake_target == 'otc_linux_amd64_deb'
+        id: get-build-number
+        run: |
+          build_number="${{ inputs.otc_build_number }}"
+          echo "otc_build_number=${build_number}" >> $GITHUB_OUTPUT
+
       - name: Determine if MacOS package should be signed
         if: runner.os == 'macOS'
         env:
@@ -216,12 +238,23 @@ jobs:
           path: ./build/${{ steps.package.outputs.path }}
           if-no-files-found: error
 
-      - name: Publish package to Packagecloud
+      - name: Publish packages
         if: runner.os == 'Linux'
         uses: ./ci/github-actions/make
         with:
           target: publish-package
           packagecloud-token: ${{ secrets.PACKAGECLOUD_TOKEN }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Publish packages
+        if: runner.os != 'Linux'
+        working-directory: build
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+        run: make publish-package
 
       - name: Wait for Packagecloud packages to be indexed
         if: runner.os == 'Linux'

.github/workflows/build_packages.yml

@@ -75,6 +75,10 @@ on:
         type: boolean
         required: false
         default: false
+      is_latest:
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   determine_workflow:
@@ -199,6 +203,8 @@ jobs:
       microsoft_description: ${{ secrets.MICROSOFT_DESCRIPTION }}
       gh_ci_token: ${{ secrets.GH_CI_TOKEN }}
       packagecloud_token: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
     strategy:
       matrix:
@@ -249,6 +255,16 @@ jobs:
   install-script:
     name: Store install script
     runs-on: ubuntu-latest
+    needs:
+      - determine_version
+      - build_packages
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_S3_BUCKET: sumologic-osc-ci-builds
+      OTC_VERSION: ${{ needs.determine_version.outputs.otc_version }}
+      BUILD_NUMBER: ${{ needs.build_packages.outputs.otc_build_number }}
     steps:
       - uses: actions/checkout@v4
 
@@ -266,6 +282,23 @@ jobs:
           path: ./install-script/install.ps1
           if-no-files-found: error
 
+      - name: Store install scripts on S3
+        run: |
+          version="${OTC_VERSION}-${BUILD_NUMBER}"
+          s3_path="${version}/"
+          aws s3 cp install-script/install.ps1 s3://${AWS_S3_BUCKET}/${s3_path}
+          aws s3 cp install-script/install.sh s3://${AWS_S3_BUCKET}/${s3_path}
+
+      - name: Create latest_version file
+        if: inputs.is_latest
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          version="${OTC_VERSION}-${BUILD_NUMBER}"
+          echo "${version}" >> latest_version
+          aws s3 cp --content-type "text/plain" latest_version s3://${AWS_S3_BUCKET}/
+
   publish_release:
     name: Publish Release
     runs-on: ubuntu-latest

Dockerfile

@@ -16,7 +16,8 @@ RUN apk add --no-cache \
     curl \
     bash \
     tar \
-    gzip
+    gzip \
+    aws-cli
 
 COPY docker/install-deps.sh /install-deps.sh
 

Makefile

@@ -42,6 +42,8 @@ publish-package:
 		-v "$(mkfile_dir):/src" \
 		-v "$(build_dir):/build" \
 		-e PACKAGECLOUD_TOKEN="$(PACKAGECLOUD_TOKEN)" \
+		-e AWS_ACCESS_KEY_ID="$(AWS_ACCESS_KEY_ID)" \
+		-e AWS_SECRET_ACCESS_KEY="$(AWS_SECRET_ACCESS_KEY)"
 		otelcol-sumo/cmake \
 		make publish-package
 

ci/github-actions/make/action.yml

@@ -7,12 +7,20 @@ inputs:
   packagecloud-token:
     required: false
     type: string
+  aws-access-key-id:
+    required: false
+    type: string
+  aws-secret-access-key:
+    required: false
+    type: string
 
 runs:
   using: 'docker'
   image: '../../../Dockerfile'
   env:
     PACKAGECLOUD_TOKEN: ${{ inputs.packagecloud-token }}
+    AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+    AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
   args:
     - make
     - ${{ inputs.target }}

distributions.cmake

@@ -1,3 +1,6 @@
+# Checks if the Packagecloud distro supports the package architecture and if it
+# does it will be added to the list of Packagecloud distributions to upload the
+# package to.
 macro(check_architecture_support)
   if(NOT ${package_arch} IN_LIST _supported_architectures)
     message(FATAL_ERROR "${_distro_name} does not support architecture: ${package_arch}")

packages.cmake

@@ -135,6 +135,13 @@ macro(build_cpack_config)
     create_packagecloud_publish_target(${_pc_user} ${_pc_repo} ${_pc_distro} ${_package_output})
   endforeach()
 
+  # Add a target for uploading the package to Amazon S3
+  set(_s3_channel "ci-builds")
+  set(_version "${OTC_VERSION}-${BUILD_NUMBER}")
+  set(_s3_bucket "sumologic-osc-${_s3_channel}")
+  set(_s3_path "${_version}/")
+  create_s3_cp_target(${_s3_bucket} ${_s3_path} ${_package_output})
+
   # Add a publish-package target to publish the package built above
   get_property(_all_publish_targets GLOBAL PROPERTY _all_publish_targets)
   add_custom_target(publish-package
@@ -182,6 +189,18 @@ function(create_wait_for_packagecloud_indexing_target _pc_user _pc_repo _pkg_pat
     DEPENDS ${_pc_output})
 endfunction()
 
+# Create an Amazon S3 publish target for uploading a package to an S3 bucket.
+function(create_s3_cp_target _s3_bucket _s3_path _pkg_path)
+    set(_s3_output "${_pkg_path}-s3-${_s3_bucket}")
+    separate_arguments(_s3_cp_cmd UNIX_COMMAND "aws s3 cp ${_pkg_path} s3://${_s3_bucket}/${_s3_path}")
+    add_custom_command(OUTPUT ${_s3_output}
+        COMMAND ${_s3_cp_cmd}
+        DEPENDS ${_pkg_path}
+        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+        VERBATIM)
+    append_to_publish_targets(${_s3_output})
+endfunction()
+
 # Sets a GitHub output parameter by appending a statement to the file defined by
 # the GITHUB_OUTPUT environment variable. It enables the passing of data from
 # this CMake project to GitHub Actions.