diff --git a/.github/actions/generate-package-build-matrix/action.yml b/.github/actions/generate-package-build-matrix/action.yml new file mode 100644 index 0000000000..7e90f27be5 --- /dev/null +++ b/.github/actions/generate-package-build-matrix/action.yml @@ -0,0 +1,44 @@ +name: Generate target matrix. +description: Matrix creation for building Valkey for different architectures and platforms. + +inputs: + ref: + description: The commit, tag or branch of Valkey to checkout to determine what version to use. + required: true +outputs: + x86_64-build-matrix: + description: The x86_64 build matrix. + value: ${{ steps.set-matrix.outputs.x86matrix }} + arm64-build-matrix: + description: The arm64 build matrix. + value: ${{ steps.set-matrix.outputs.armmatrix }} + +runs: + using: "composite" + steps: + - name: Checkout code for version check + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + path: version-check + + - name: Get targets + run: | + x86_arch=$(jq -c '[.linux_targets[] | select(.arch=="x86_64")]' utils/releasetools/build-config.json) + x86_matrix=$(echo "{ \"distro\" : $x86_arch }" | jq -c .) + echo "X86_MATRIX=$x86_matrix" >> $GITHUB_ENV + + arm_arch=$(jq -c '[.linux_targets[] | select(.arch=="arm64")]' utils/releasetools/build-config.json) + arm_matrix=$(echo "{ \"distro\" : $arm_arch }" | jq -c .) + echo "ARM_MATRIX=$arm_matrix" >> $GITHUB_ENV + shell: bash + + - id: set-matrix + run: | + echo $X86_MATRIX + echo $X86_MATRIX| jq . + echo "x86matrix=$X86_MATRIX" >> $GITHUB_OUTPUT + echo $ARM_MATRIX + echo $ARM_MATRIX| jq . + echo "armmatrix=$ARM_MATRIX" >> $GITHUB_OUTPUT + shell: bash diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml new file mode 100644 index 0000000000..8c32548f9e --- /dev/null +++ b/.github/workflows/build-release-packages.yml @@ -0,0 +1,89 @@ +name: Build Release Packages + +on: + release: + types: [published] + + workflow_dispatch: + inputs: + version: + description: Version of Valkey to build + required: true + +permissions: + contents: read + +jobs: + # This job provides the version metadata from the tag for the other jobs to use. + release-build-get-meta: + name: Get metadata to build + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get_version.outputs.VERSION }} + steps: + + - run: | + echo "Version: ${{ inputs.version || github.ref_name }}" + shell: bash + + # This step is to consolidate the three different triggers into a single "version" + # 1. If manual dispatch - use the version provided. + # 3. If tag trigger, use that tag. + - name: Get the version + id: get_version + run: | + VERSION="${INPUT_VERSION}" + if [ -z "${VERSION}" ]; then + exit 1 + fi + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + shell: bash + env: + # Use the dispatch variable in preference, if empty use the context ref_name which should + # only ever be a tag + INPUT_VERSION: ${{ inputs.version || github.ref_name }} + + generate-build-matrix: + name: Generating build matrix + runs-on: ubuntu-latest + outputs: + x86_64-build-matrix: ${{ steps.set-matrix.outputs.x86_64-build-matrix }} + arm64-build-matrix: ${{ steps.set-matrix.outputs.arm64-build-matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Set up the list of target to build so we can pass the JSON to the reusable job + - uses: ./.github/actions/generate-package-build-matrix + id: set-matrix + with: + ref: ${{ inputs.version || github.ref_name }} + + release-build-linux-x86-packages: + needs: + - release-build-get-meta + - generate-build-matrix + uses: ./.github/workflows/call-build-linux-x86-packages.yml + with: + version: ${{ needs.release-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + build_matrix: ${{ needs.generate-build-matrix.outputs.x86_64-build-matrix }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + bucket: ${{ secrets.AWS_S3_BUCKET }} + access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }} + + release-build-linux-arm-packages: + needs: + - release-build-get-meta + - generate-build-matrix + uses: ./.github/workflows/call-build-linux-arm-packages.yml + with: + version: ${{ needs.release-build-get-meta.outputs.version }} + ref: ${{ inputs.version || github.ref_name }} + build_matrix: ${{ needs.generate-build-matrix.outputs.arm64-build-matrix }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + bucket: ${{ secrets.AWS_S3_BUCKET }} + access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} + secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }} diff --git a/.github/workflows/call-build-linux-arm-packages.yml b/.github/workflows/call-build-linux-arm-packages.yml new file mode 100644 index 0000000000..db987b3816 --- /dev/null +++ b/.github/workflows/call-build-linux-arm-packages.yml @@ -0,0 +1,79 @@ +name: Builds Linux arm binary packages into S3 bucket. + +on: + workflow_call: + inputs: + version: + description: The version of Valkey to create. + type: string + required: true + ref: + description: The commit, tag or branch of Valkey to checkout for building that creates the version above. + type: string + required: true + build_matrix: + description: The build targets to produce as a JSON matrix. + type: string + required: true + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + +permissions: + contents: read + +jobs: + build-valkey: + # Capture source tarball and generate checksum for it + name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }} + runs-on: 'ubuntu-latest' + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.build_matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.version }} + + - name: Make Valkey + uses: uraimo/run-on-arch-action@v2 + with: + arch: aarch64 + distro: ${{matrix.distro.target}} + install: apt-get update && apt-get install -y build-essential libssl-dev + run: make -C src all BUILD_TLS=yes + + - name: Create Tarball and SHA256sums + run: | + TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}} + mkdir -p $TAR_FILE_NAME/bin $TAR_FILE_NAME/share + cp -rfv src/valkey-* $TAR_FILE_NAME/bin + cp -v /home/runner/work/valkey/valkey/COPYING $TAR_FILE_NAME/share/LICENSE + tar -czvf $TAR_FILE_NAME.tar.gz $TAR_FILE_NAME + sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256 + mkdir -p packages-files + cp -rfv $TAR_FILE_NAME.tar* packages-files/ + + - name: Install AWS cli. + run: | + sudo apt-get install -y awscli + + - name: Configure AWS credentials + run: | + aws configure set region us-west-2 + aws configure set aws_access_key_id ${{ secrets.access_key_id }} + aws configure set aws_secret_access_key ${{ secrets.secret_access_key }} + + - name: Sync to S3 + run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/ diff --git a/.github/workflows/call-build-linux-x86-packages.yml b/.github/workflows/call-build-linux-x86-packages.yml new file mode 100644 index 0000000000..59c9fac52f --- /dev/null +++ b/.github/workflows/call-build-linux-x86-packages.yml @@ -0,0 +1,73 @@ +name: Builds Linux X86 binary packages into S3 bucket. + +on: + workflow_call: + inputs: + version: + description: The version of Valkey to create. + type: string + required: true + ref: + description: The commit, tag or branch of Valkey to checkout for building that creates the version above. + type: string + required: true + build_matrix: + description: The build targets to produce as a JSON matrix. + type: string + required: true + secrets: + token: + description: The Github token or similar to authenticate with. + required: true + bucket: + description: The name of the S3 bucket to push packages into. + required: false + access_key_id: + description: The S3 access key id for the bucket. + required: false + secret_access_key: + description: The S3 secret access key for the bucket. + required: false + +permissions: + contents: read + +jobs: + build-valkey: + # Capture source tarball and generate checksum for it + name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }} + runs-on: 'ubuntu-latest' + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.build_matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.version }} + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential libssl-dev jq wget awscli + + - name: Make Valkey + run: make -C src all BUILD_TLS=yes + + - name: Create Tarball and SHA256sums + run: | + TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}} + mkdir -p $TAR_FILE_NAME/bin $TAR_FILE_NAME/share + cp -rfv src/valkey-* $TAR_FILE_NAME/bin + cp -v /home/runner/work/valkey/valkey/COPYING $TAR_FILE_NAME/share/LICENSE + tar -czvf $TAR_FILE_NAME.tar.gz $TAR_FILE_NAME + sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256 + mkdir -p packages-files + cp -rfv $TAR_FILE_NAME.tar* packages-files/ + + - name: Configure AWS credentials + run: | + aws configure set region us-west-2 + aws configure set aws_access_key_id ${{ secrets.access_key_id }} + aws configure set aws_secret_access_key ${{ secrets.secret_access_key }} + + - name: Sync to S3 + run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/ diff --git a/utils/releasetools/build-config.json b/utils/releasetools/build-config.json new file mode 100644 index 0000000000..5ee9308b3b --- /dev/null +++ b/utils/releasetools/build-config.json @@ -0,0 +1,28 @@ +{ + "linux_targets": [ + { + "arch": "x86_64", + "target": "ubuntu18.04", + "type": "deb", + "platform": "bionic" + }, + { + "arch": "x86_64", + "target": "ubuntu20.04", + "type": "deb", + "platform": "focal" + }, + { + "arch": "arm64", + "target": "ubuntu18.04", + "type": "deb", + "platform": "bionic" + }, + { + "arch": "arm64", + "target": "ubuntu20.04", + "type": "deb", + "platform": "focal" + } + ] +} \ No newline at end of file