Skip to content

Introduce reproducible branch-based coverage #4264

Introduce reproducible branch-based coverage

Introduce reproducible branch-based coverage #4264

Workflow file for this run

name: CI
# Controls when the workflow will run.
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
paths-ignore:
- 'docs/**'
permissions: read-all
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
nativelink-dot-com-cloud-rbe-main-legacy-dockerfile-test:
runs-on: ubuntu-22.04
environment: production
name: NativeLink.com Cloud / RBE on Main (Legacy Dockerfile Test)
if: github.ref == 'refs/heads/main'
steps:
- name: Checkout
uses: >- # v4.1.1
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Set up AWS CLI
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502
with:
aws-access-key-id: ${{ secrets.RBE_ECR_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.RBE_ECR_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.RBE_ECR_AWS_ACCOUNT_REGION }}
- name: Calculate Dockerfile hash and Retrieve Image URI for RBE
run: |
DOCKERFILE_HASH=$(sha256sum "$GITHUB_WORKSPACE/tools/toolchain-nativelink/Dockerfile" | awk '{print $1}')
IMAGE_DETAILS=$(aws ecr describe-images --repository-name ${{ secrets.RBE_ECR_REPOSITORY_NAME }} --image-ids imageTag=$DOCKERFILE_HASH)
if [ $? -ne 0 ]; then
exit 1;
fi
echo "RBE_IMAGE=${{ secrets.RBE_ECR_AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.RBE_ECR_AWS_ACCOUNT_REGION }}.amazonaws.com/${{ secrets.RBE_ECR_REPOSITORY_NAME }}:$DOCKERFILE_HASH" >> $GITHUB_ENV
- name: Setup Bazelisk
uses: >- # v0.8.1
bazel-contrib/setup-bazel@b388b84bb637e50cdae241d0f255670d4bd79f29
with:
bazelisk-cache: true
- name: Run Bazel tests
shell: bash
# remove digest_function when #1325 is resolved
run: |
bazel --digest_function=sha256 test \
--remote_cache=grpcs://cas-tracemachina-shared.build-faster.nativelink.net \
--remote_header=x-nativelink-api-key=${{ secrets.NATIVELINK_COM_API_HEADER }} \
--remote_instance_name=main \
--bes_backend=grpcs://bes-tracemachina-shared.build-faster.nativelink.net \
--bes_header=x-nativelink-api-key=${{ secrets.NATIVELINK_COM_API_HEADER }} \
--bes_results_url=https://app.nativelink.com/a/e3b1e0e0-4b73-45d6-85bc-5cb7b02edea5/build \
--remote_header=x-nativelink-project=nativelink-ci \
--remote_executor=grpcs://scheduler-tracemachina-shared.build-faster.nativelink.net:443 \
--remote_default_exec_properties="container-image=docker://$RBE_IMAGE" \
--jobs=200 \
//...
nativelink-dot-com-cloud-cache-test:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, macos-14]
runs-on: ${{ matrix.os }}
environment: production
name: NativeLink.com Cloud / Remote Cache / ${{ matrix.os }}
env:
NL_COM_API_KEY: ${{ secrets.NATIVELINK_COM_API_HEADER || '065f02f53f26a12331d5cfd00a778fb243bfb4e857b8fcd4c99273edfb15deae' }}
steps:
- name: Checkout
uses: >- # v4.1.1
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Free disk space
uses: >- # v2.0.0
endersonmenezes/free-disk-space@3f9ec39ebae520864ac93467ee395f5237585c21
with:
remove_android: true
remove_dotnet: true
remove_haskell: true
remove_tool_cache: false
- name: Install Nix
uses: >- # v10
DeterminateSystems/nix-installer-action@de22e16c4711fca50c816cc9081563429d1cf563
- name: Cache Nix derivations
uses: >- # v4
DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41
- name: Run Bazel tests
run: >
nix develop --impure --command
bash -c "bazel test \
--remote_header=x-nativelink-api-key=$NL_COM_API_KEY \
--bes_backend=grpcs://bes-tracemachina-shared.build-faster.nativelink.net \
--bes_header=x-nativelink-api-key=$NL_COM_API_KEY \
--bes_results_url=https://app.nativelink.com/a/e3b1e0e0-4b73-45d6-85bc-5cb7b02edea5/build \
${{ github.ref == 'refs/heads/main' && '--remote_upload_local_results=true' || '--nogenerate_json_trace_profile --remote_upload_local_results=false' }} \
//..."
docker-compose-compiles-nativelink:
# The type of runner that the job will run on.
runs-on: ubuntu-22.04
strategy:
matrix:
# Which OS versions we will test on.
os_version: [ 22.04 ]
steps:
- name: Checkout
uses: >- # v4.1.1
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Set up Docker Buildx
uses: >- # v3.2.0
docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
- name: Build Nativelink image
uses: >- # v5.3.0
docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
with:
context: .
file: ./deployment-examples/docker-compose/Dockerfile
build-args: |
OPT_LEVEL=opt
OS_VERSION=${{ matrix.os_version }}
ADDITIONAL_SETUP_WORKER_CMD=apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc g++ lld pkg-config python3
load: true # This brings the build into `docker images` from buildx.
tags: trace_machina/nativelink:latest
- name: Build builder image
uses: >- # v5.3.0
docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
with:
context: .
file: ./deployment-examples/docker-compose/Dockerfile
build-args: |
OPT_LEVEL=opt
OS_VERSION=${{ matrix.os_version }}
load: true # This brings the build into `docker images` from buildx.
tags: trace_machina/nativelink:builder
target: builder
- name: Compile NativeLink with NativeLink
run: |
mkdir -p ~/.cache && \
cd deployment-examples/docker-compose && \
(docker-compose up -d || docker compose up -d) && \
cd ../../ && \
docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
bazel clean && \
bazel test //... \
--remote_instance_name=main \
--remote_cache=grpc://127.0.0.1:50051 \
--remote_executor=grpc://127.0.0.1:50052 \
--remote_default_exec_properties=cpu_count=1 \
' && \
docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
bazel clean && \
bazel test //... \
--remote_instance_name=main \
--remote_cache=grpc://127.0.0.1:50051 \
--remote_executor=grpc://127.0.0.1:50052 \
--remote_default_exec_properties=cpu_count=1 \
' 2>&1 | ( ! grep ' PASSED in ' ) # If we get PASSED without (cache) it means there's a cache issue.
integration-tests:
runs-on: ubuntu-22.04
strategy:
matrix:
# Which OS versions we will test on.
os_version: [ 22.04 ]
steps:
- name: Checkout
uses: >- # v4.1.1
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Set up Docker Buildx
uses: >- # v3.2.0
docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
- name: Build image
uses: >- # v5.3.0
docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
with:
context: .
file: ./deployment-examples/docker-compose/Dockerfile
build-args: |
OPT_LEVEL=fastbuild
OS_VERSION=${{ matrix.os_version }}
load: true # This brings the build into `docker images` from buildx.
tags: trace_machina/nativelink:latest
- name: Run tests
run: ./run_integration_tests.sh