-
Notifications
You must be signed in to change notification settings - Fork 99
214 lines (205 loc) · 8.22 KB
/
build.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
name: Build and Test
on:
push:
tags: ["v**"]
branches: ["master"]
pull_request:
branches: ["**"]
workflow_dispatch:
inputs:
persistent_storage:
description: Run the persistent storage tests
type: boolean
pypi_publish:
type: boolean
publish_env:
description: Environment to publish to
type: environment
cmake_preset_type:
description: Override CMAKE preset type
type: choice
options: ["-", debug, release]
dev_image_tag:
description: Tag of the ArcticCB development image to use for the Linux C++ tests build
type: string
default: latest
run-name: Building ${{github.ref_name}} on ${{github.event_name}} by ${{github.actor}}
concurrency:
group: ${{github.ref}}
cancel-in-progress: true
jobs:
cibw_docker_image:
uses: ./.github/workflows/cibw_docker_image.yml
permissions: {packages: write}
with:
cibuildwheel_ver: "2.12.1"
force_update: false
common_config:
needs: [cibw_docker_image]
runs-on: ubuntu-22.04
strategy:
matrix:
include:
# Use a matrix to build the common hierarchical structure used by multiple matrix jobs in build_steps.yaml
# Please declare any key you added below in build_steps.yaml's dummy matrix as well to aid the linting tools
- linux_matrix:
- os: linux
distro: ubuntu-22.04
cmake_preset_prefix: linux
cibw_build_suffix: manylinux_x86_64
build_dir: /tmp/cpp_build
vcpkg_packages_dir: ""
symbols: "*.gz" # Please update publish.yml when changing this!!
do_not_archive: ["*.so", "*.[ao]", "vcpkg_installed"]
test_services:
mongodb:
image: "mongo:4.4"
container:
image: ghcr.io/man-group/arcticdb-dev:${{inputs.dev_image_tag || 'latest'}}
volumes:
- /:/mnt
steps:
- run: |
if ${{startsWith(github.ref, 'refs/tags/v')}}; then
# Can only upload to Pypi once per version, so only auto upload on tag builds:
echo -e "PUBLISH_ENV=ProdPypi\nCMAKE_PRESET_TYPE=release\nPYPI_PUBLISH=1" | tee -a $GITHUB_ENV
elif $GITHUB_REF_PROTECTED || ${{github.ref == 'refs/heads/master'}} ; then
echo -e "PUBLISH_ENV=TestPypi\nCMAKE_PRESET_TYPE=release" | tee -a $GITHUB_ENV
else
echo -e "PUBLISH_ENV=${{vars.DEFAULT_PUBLISH_ENV}}\nCMAKE_PRESET_TYPE=$DEFAULT_PRESET" | tee -a $GITHUB_ENV
fi
env:
DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' || vars.DEFAULT_CMAKE_PRESET_TYPE || 'debug'}}
outputs:
cibuildwheel_ver: ${{needs.cibw_docker_image.outputs.cibuildwheel_ver}}
publish_env: ${{inputs.publish_env || env.PUBLISH_ENV || ''}}
pypi_publish: ${{inputs.pypi_publish || env.PYPI_PUBLISH}}
cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type || env.CMAKE_PRESET_TYPE}}
linux_matrix: ${{toJson(matrix.linux_matrix)}}
pre_seed_cleanup:
if: inputs.persistent_storage == true
name: Cleanup persistent storages
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
with:
job_type: cleanup
persistent_storage_seed_linux:
needs: [common_config, pre_seed_cleanup]
if: inputs.persistent_storage == true
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
arcticdb_version: ["oldest", "latest"]
include:
- python_deps_ids: [""]
matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
name: Seed Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: seed
python3: ${{matrix.python3}}
arcticdb_version: ${{matrix.arcticdb_version}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
cpp-test-linux:
needs: [cibw_docker_image, common_config]
name: Linux C++ Tests
uses: ./.github/workflows/build_steps.yml
secrets: inherit
with:
job_type: cpp-tests
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
matrix: ${{needs.common_config.outputs.linux_matrix}}
build-python-wheels-linux:
# Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels
needs: [cibw_docker_image, common_config]
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
include:
- python_deps_ids: [""]
matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
pytest_xdist_mode: "--dist worksteal"
- python3: 6
python_deps_ids: ["", -compat36]
matrix_override:
- ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
- python_deps_id: -compat36
python_deps: requirements-compatibility-py36.txt
pytest_xdist_mode: "" # worksteal Not supported on Python 3.6
- python3: 8
python_deps_ids: ["", -compat38]
matrix_override:
- ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}}
- python_deps_id: -compat38
python_deps: requirements-compatibility-py38.txt
name: 3.${{matrix.python3}} Linux
uses: ./.github/workflows/build_steps.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: build-python-wheels
python3: ${{matrix.python3}}
cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}}
cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}}
cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
persistent_storage: ${{inputs.persistent_storage}}
pytest_xdist_mode: ${{matrix.pytest_xdist_mode}}
persistent_storage_verify_linux:
needs: [common_config, build-python-wheels-linux]
if: inputs.persistent_storage == true
strategy:
fail-fast: false
matrix:
python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}}
arcticdb_version: ["oldest", "latest"]
include:
- python_deps_ids: [""]
matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}}
name: Verify Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
permissions: {packages: write}
with:
job_type: verify
python3: ${{matrix.python3}}
arcticdb_version: ${{matrix.arcticdb_version}}
matrix: ${{toJson(matrix.matrix_override)}}
python_deps_ids: ${{toJson(matrix.python_deps_ids)}}
post_verify_cleanup:
needs: persistent_storage_verify_linux
if: inputs.persistent_storage == true
name: Cleanup persistent storages
uses: ./.github/workflows/persistent_storage.yml
secrets: inherit
with:
job_type: cleanup
can_merge:
needs: [cpp-test-linux, build-python-wheels-linux, persistent_storage_verify_linux]
if: |
always() &&
!failure() &&
!cancelled()
runs-on: ubuntu-22.04
steps:
- run: echo Dummy job to simplify PR merge checks configuration
# FUTURE: add some test stats/reporting
publish:
needs: [common_config, can_merge]
if: |
always() &&
!failure() &&
!cancelled() &&
(needs.common_config.outputs.pypi_publish && needs.common_config.outputs.publish_env)
uses: ./.github/workflows/publish.yml
secrets: inherit
permissions: {contents: write}
with:
environment: ${{needs.common_config.outputs.publish_env}}