-
Notifications
You must be signed in to change notification settings - Fork 125
202 lines (180 loc) · 7.7 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
name: CI
# Controls when the workflow will run.
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
paths-ignore:
- 'docs/**'
permissions: read-all
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
nativelink-dot-com-cloud-rbe-main-legacy-dockerfile-test:
runs-on: ubuntu-22.04
environment: production
name: NativeLink.com Cloud / RBE on Main (Legacy Dockerfile Test)
if: github.ref == 'refs/heads/main'
steps:
- name: Checkout
uses: >- # v4.1.1
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Set up AWS CLI
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502
with:
aws-access-key-id: ${{ secrets.RBE_ECR_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.RBE_ECR_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.RBE_ECR_AWS_ACCOUNT_REGION }}
- name: Calculate Dockerfile hash and Retrieve Image URI for RBE
run: |
DOCKERFILE_HASH=$(sha256sum "$GITHUB_WORKSPACE/tools/toolchain-nativelink/Dockerfile" | awk '{print $1}')
IMAGE_DETAILS=$(aws ecr describe-images --repository-name ${{ secrets.RBE_ECR_REPOSITORY_NAME }} --image-ids imageTag=$DOCKERFILE_HASH)
if [ $? -ne 0 ]; then
exit 1;
fi
echo "RBE_IMAGE=${{ secrets.RBE_ECR_AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.RBE_ECR_AWS_ACCOUNT_REGION }}.amazonaws.com/${{ secrets.RBE_ECR_REPOSITORY_NAME }}:$DOCKERFILE_HASH" >> $GITHUB_ENV
- name: Setup Bazelisk
uses: >- # v0.8.1
bazel-contrib/setup-bazel@b388b84bb637e50cdae241d0f255670d4bd79f29
with:
bazelisk-cache: true
- name: Run Bazel tests
shell: bash
# remove digest_function when #1325 is resolved
run: |
bazel --digest_function=sha256 test \
--remote_cache=grpcs://cas-tracemachina-shared.build-faster.nativelink.net \
--remote_header=x-nativelink-api-key=${{ secrets.NATIVELINK_COM_API_HEADER }} \
--remote_instance_name=main \
--bes_backend=grpcs://bes-tracemachina-shared.build-faster.nativelink.net \
--bes_header=x-nativelink-api-key=${{ secrets.NATIVELINK_COM_API_HEADER }} \
--bes_results_url=https://app.nativelink.com/a/e3b1e0e0-4b73-45d6-85bc-5cb7b02edea5/build \
--remote_header=x-nativelink-project=nativelink-ci \
--remote_executor=grpcs://scheduler-tracemachina-shared.build-faster.nativelink.net:443 \
--remote_default_exec_properties="container-image=docker://$RBE_IMAGE" \
--jobs=200 \
//...
nativelink-dot-com-cloud-cache-test:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, macos-14]
runs-on: ${{ matrix.os }}
environment: production
name: NativeLink.com Cloud / Remote Cache / ${{ matrix.os }}
env:
NL_COM_API_KEY: ${{ secrets.NATIVELINK_COM_API_HEADER || '065f02f53f26a12331d5cfd00a778fb243bfb4e857b8fcd4c99273edfb15deae' }}
steps:
- name: Checkout
uses: >- # v4.1.1
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Free disk space
uses: >- # v2.0.0
endersonmenezes/free-disk-space@3f9ec39ebae520864ac93467ee395f5237585c21
with:
remove_android: true
remove_dotnet: true
remove_haskell: true
remove_tool_cache: false
- name: Install Nix
uses: >- # v10
DeterminateSystems/nix-installer-action@de22e16c4711fca50c816cc9081563429d1cf563
- name: Cache Nix derivations
uses: >- # v4
DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41
- name: Run Bazel tests
run: >
nix develop --impure --command
bash -c "bazel test \
--remote_header=x-nativelink-api-key=$NL_COM_API_KEY \
--bes_backend=grpcs://bes-tracemachina-shared.build-faster.nativelink.net \
--bes_header=x-nativelink-api-key=$NL_COM_API_KEY \
--bes_results_url=https://app.nativelink.com/a/e3b1e0e0-4b73-45d6-85bc-5cb7b02edea5/build \
${{ github.ref == 'refs/heads/main' && '--remote_upload_local_results=true' || '--nogenerate_json_trace_profile --remote_upload_local_results=false' }} \
//..."
docker-compose-compiles-nativelink:
# The type of runner that the job will run on.
runs-on: ubuntu-22.04
strategy:
matrix:
# Which OS versions we will test on.
os_version: [ 22.04 ]
steps:
- name: Checkout
uses: >- # v4.1.1
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Set up Docker Buildx
uses: >- # v3.2.0
docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
- name: Build Nativelink image
uses: >- # v5.3.0
docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
with:
context: .
file: ./deployment-examples/docker-compose/Dockerfile
build-args: |
OPT_LEVEL=opt
OS_VERSION=${{ matrix.os_version }}
ADDITIONAL_SETUP_WORKER_CMD=apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc g++ lld pkg-config python3
load: true # This brings the build into `docker images` from buildx.
tags: trace_machina/nativelink:latest
- name: Build builder image
uses: >- # v5.3.0
docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
with:
context: .
file: ./deployment-examples/docker-compose/Dockerfile
build-args: |
OPT_LEVEL=opt
OS_VERSION=${{ matrix.os_version }}
load: true # This brings the build into `docker images` from buildx.
tags: trace_machina/nativelink:builder
target: builder
- name: Compile NativeLink with NativeLink
run: |
mkdir -p ~/.cache && \
cd deployment-examples/docker-compose && \
(docker-compose up -d || docker compose up -d) && \
cd ../../ && \
docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
bazel clean && \
bazel test //... \
--remote_cache=grpc://127.0.0.1:50051 \
--remote_executor=grpc://127.0.0.1:50052 \
--remote_default_exec_properties=cpu_count=1 \
' && \
docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
bazel clean && \
bazel test //... \
--remote_cache=grpc://127.0.0.1:50051 \
--remote_executor=grpc://127.0.0.1:50052 \
--remote_default_exec_properties=cpu_count=1 \
' 2>&1 | ( ! grep ' PASSED in ' ) # If we get PASSED without (cache) it means there's a cache issue.
integration-tests:
runs-on: ubuntu-22.04
strategy:
matrix:
# Which OS versions we will test on.
os_version: [ 22.04 ]
steps:
- name: Checkout
uses: >- # v4.1.1
actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
- name: Set up Docker Buildx
uses: >- # v3.2.0
docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
- name: Build image
uses: >- # v5.3.0
docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
with:
context: .
file: ./deployment-examples/docker-compose/Dockerfile
build-args: |
OPT_LEVEL=fastbuild
OS_VERSION=${{ matrix.os_version }}
load: true # This brings the build into `docker images` from buildx.
tags: trace_machina/nativelink:latest
- name: Run tests
run: ./run_integration_tests.sh