diff --git a/.container b/.container new file mode 100644 index 0000000000..6c5dba6f71 --- /dev/null +++ b/.container @@ -0,0 +1 @@ +299166832260.dkr.ecr.us-east-2.amazonaws.com/nativelink-rbe:v0.3 HASH diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 770af682e0..f3e0690b92 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,7 @@ name: CI # Controls when the workflow will run. on: + workflow_dispatch: push: branches: [ main ] pull_request: @@ -19,8 +20,10 @@ jobs: nativelink-dot-com-build-on-main-test: runs-on: ubuntu-22.04 environment: production - # Only run on main branch. - if: github.ref == 'refs/heads/main' + env: + NL_COM_API_KEY: ${{ secrets.NATIVELINK_COM_API_HEADER || '065f02f53f26a12331d5cfd00a778fb243bfb4e857b8fcd4c99273edfb15deae' }} + NL_COM_RO_FLAGS: ${{ github.ref == 'refs/heads/main' && ' ' || '--nogenerate_json_trace_profile --remote_upload_local_results=false' }} + NL_COM_RE_FLAGS: ${{ github.ref == 'refs/heads/main' && '--remote_executor=grpcs://scheduler-tracemachina-shared.build-faster.nativelink.net:443 --remote_default_exec_properties=container-image=docker://default' || ' ' }} steps: - name: Checkout uses: >- # v4.1.1 @@ -34,12 +37,44 @@ jobs: shell: bash run: | bazel test \ - --remote_cache=${{ vars.NATIVELINK_COM_REMOTE_CACHE_URL }} \ - --remote_header=${{ secrets.NATIVELINK_COM_API_HEADER }} \ - --bes_backend=${{ vars.NATIVELINK_COM_BES_URL }} \ - --bes_header=${{ secrets.NATIVELINK_COM_API_HEADER }} \ - --bes_results_url=${{ vars.NATIVELINK_COM_BES_RESULTS_URL }} \ + --remote_cache=grpcs://cas-tracemachina-shared.build-faster.nativelink.net \ + --remote_header=x-nativelink-api-key=${NL_COM_API_KEY} \ + --remote_instance_name=main \ + --bes_backend=grpcs://bes-tracemachina-shared.build-faster.nativelink.net \ + --bes_header=x-nativelink-api-key=${NL_COM_API_KEY} \ + --bes_results_url=https://app.nativelink.com/a/e3b1e0e0-4b73-45d6-85bc-5cb7b02edea5/build \ + --remote_header=x-nativelink-project=nativelink-ci \ + ${NL_COM_RE_FLAGS} \ + ${NL_COM_RO_FLAGS} \ + //... + + nativelink-dot-com-rbe-test: + runs-on: ubuntu-22.04 + environment: production + steps: + - name: Checkout + uses: >- # v4.1.1 + actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - name: Setup Bazelisk + uses: >- # v0.8.1 + bazel-contrib/setup-bazel@b388b84bb637e50cdae241d0f255670d4bd79f29 + with: + bazelisk-cache: true + - name: Run Bazel tests + shell: bash + run: | + bazel test \ + --remote_cache=grpcs://cas-bclark8923-d21fad.build-faster.nativelink.net \ + --remote_header=x-nativelink-api-key=05929e1fb5253a96df099dbdd41a6304b4c9feef9fecfdbdcbbb02f105600053 \ + --remote_instance_name=main \ + --bes_backend=grpcs://bes-bclark8923-d21fad.build-faster.nativelink.net \ + --bes_header=x-nativelink-api-key=05929e1fb5253a96df099dbdd41a6304b4c9feef9fecfdbdcbbb02f105600053 \ + --bes_results_url=https://app.nativelink.com/a/bclark8923/build \ --remote_header=x-nativelink-project=nativelink-ci \ + --remote_executor=grpcs://scheduler-bclark8923-d21fad.build-faster.nativelink.net:443 \ + --remote_default_exec_properties=container-image=docker://default \ + --verbose_failures \ + --platform_suffix=${{ github.run_id }}-${{ github.run_attempt }} \ //... docker-compose-compiles-nativelink: diff --git a/tools/toolchain-nativelink/Dockerfile b/tools/toolchain-nativelink/Dockerfile new file mode 100644 index 0000000000..0889e38059 --- /dev/null +++ b/tools/toolchain-nativelink/Dockerfile @@ -0,0 +1,35 @@ +# Copyright 2022-2024 The NativeLink Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:22.04@sha256:f9d633ff6640178c2d0525017174a688e2c1aef28f0a0130b26bd5554491f0da + +# Set shell to bash and enable pipefail +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Get Ubuntu packages +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + gcc=4:11.2.0-1ubuntu1 \ + g++=4:11.2.0-1ubuntu1 \ + python3=3.10.6-1~22.04 \ + python3-minimal=3.10.6-1~22.04 \ + libpython3-stdlib=3.10.6-1~22.04 \ + curl=7.81.0-1ubuntu1.17 \ + ca-certificates=20230311ubuntu0.22.04.1 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Get Rust +RUN curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.79.0 + +RUN echo "source \"$HOME/.cargo/env\"" >> "$HOME/.bashrc" diff --git a/tools/toolchain-nativelink/toolchain-nativelink.sh b/tools/toolchain-nativelink/toolchain-nativelink.sh new file mode 100755 index 0000000000..1aa84c7ba0 --- /dev/null +++ b/tools/toolchain-nativelink/toolchain-nativelink.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -xeuo pipefail + +ECR=${ECR:?Error: ECR is not set} +ECR_PROFILE=${ECR_PROFILE:?Error: ECR_PROFILE is not set} +ECR_USER=${ECR_USER:?Error: ECR_USER is not set} +ECR_REGION=${ECR_REGION:?Error: ECR_REGION is not set} +BUILDX_NO_CACHE=${BUILDX_NO_CACHE:-true} + +function ecr_login() { + aws ecr get-login-password --profile ${ECR_PROFILE} --region ${ECR_REGION} | docker login --username ${ECR_USER} --password-stdin ${ECR}.dkr.ecr.${ECR_REGION}.amazonaws.com +} + +# Check OS and calculate the SHA256 hash of the Dockerfile +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + IMAGE_TAG=$(sha256sum 'Dockerfile' | awk '{print $1}') +elif [[ "$OSTYPE" == "darwin"* ]]; then + IMAGE_TAG=$(shasum -a 256 'Dockerfile' | awk '{print $1}') +else + echo "Unsupported OS" + exit 1 +fi + +# Build the Docker image and tag it with the hash +docker buildx build --no-cache=${BUILDX_NO_CACHE} --platform linux/amd64 -t "${ECR}.dkr.ecr.${ECR_REGION}.amazonaws.com/nativelink-rbe:$IMAGE_TAG" -f 'Dockerfile' . + +ecr_login +docker push ${ECR}.dkr.ecr.${ECR_REGION}.amazonaws.com/nativelink-rbe:$IMAGE_TAG + +# Output the tag of the built image +echo "Docker image tagged as ${ECR}.dkr.ecr.${ECR_REGION}.amazonaws.com/nativelink-rbe:$IMAGE_TAG"