-
Notifications
You must be signed in to change notification settings - Fork 97
179 lines (160 loc) · 6.15 KB
/
flow.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
name: Flow build and test
on:
pull_request:
branches: [main]
push:
branches: [main]
jobs:
flow_test:
strategy:
matrix:
runner: [ubicloud-standard-16-ubuntu-2204-arm]
runs-on: ${{ matrix.runner }}
timeout-minutes: 30
services:
catalog:
image: imresamu/postgis:15-3.4-alpine
ports:
- 5432:5432
env:
PGUSER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
POSTGRES_INITDB_ARGS: --locale=C.UTF-8
elasticsearch:
image: elasticsearch:8.13.0
ports:
- 9200:9200
env:
discovery.type: single-node
xpack.security.enabled: false
xpack.security.enrollment.enabled: false
minio:
image: bitnami/minio:2024.11.7
ports:
- 9999:9999
env:
MINIO_ROOT_USER: minio
MINIO_ROOT_PASSWORD: miniosecret
MINIO_API_PORT_NUMBER: 9999
AWS_EC2_METADATA_DISABLED: true
MINIO_DEFAULT_BUCKETS: peerdb
steps:
- uses: actions/checkout@v4
- name: generate or hydrate protos
uses: ./.github/actions/genprotos
- uses: ubicloud/setup-go@v5
with:
go-version: '1.23.0'
cache-dependency-path: flow/go.sum
- name: install lib-geos
run: |
sudo apt-get update
sudo apt-get install libgeos-dev
- run: go mod download
working-directory: ./flow
- name: setup gcp service account
id: gcp-service-account
uses: jsdaniell/[email protected]
with:
name: "bq_service_account.json"
json: ${{ secrets.GCP_GH_CI_PKEY }}
- name: setup snowflake credentials
id: sf-credentials
uses: jsdaniell/[email protected]
with:
name: "snowflake_creds.json"
json: ${{ secrets.SNOWFLAKE_GH_CI_PKEY }}
- name: setup S3 credentials
id: s3-credentials
uses: jsdaniell/[email protected]
with:
name: "s3_creds.json"
json: ${{ secrets.S3_CREDS }}
- name: setup GCS credentials
id: gcs-credentials
uses: jsdaniell/[email protected]
with:
name: "gcs_creds.json"
json: ${{ secrets.GCS_CREDS }}
- name: setup Eventhubs credentials
id: eventhubs-credentials
uses: jsdaniell/[email protected]
with:
name: "eh_creds.json"
json: ${{ secrets.EH_CREDS }}
- name: create hstore extension, increase logical replication limits, and setup catalog database
run: >
docker exec "${{ job.services.catalog.id }}" psql -U postgres -c "CREATE EXTENSION hstore;"
-c "ALTER SYSTEM SET wal_level=logical;"
-c "ALTER SYSTEM SET max_replication_slots=192;"
-c "ALTER SYSTEM SET max_wal_senders=256;"
-c "ALTER SYSTEM SET max_connections=2048;" &&
(cat ./nexus/catalog/migrations/V{?,??}__* | docker exec -i "${{ job.services.catalog.id }}" psql -U postgres) &&
docker restart "${{ job.services.catalog.id }}"
env:
PGPASSWORD: postgres
- name: start redpanda
uses: redpanda-data/[email protected]
with:
version: "latest"
- uses: ubicloud/cache@v4
id: cache-clickhouse
with:
path: ./clickhouse
key: ${{ runner.os }}-clickhouse
- name: Install ClickHouse
if: steps.cache-clickhouse.outputs.cache-hit != 'true'
run: |
curl https://clickhouse.com | sh
- name: Run ClickHouse
run: |
./clickhouse server &
- name: Install Temporal CLI
uses: temporalio/setup-temporal@v0
- name: run tests
run: |
temporal server start-dev --namespace default --headless &
go build -ldflags="-s -w" -o peer-flow
temporal operator search-attribute create --name MirrorName --type Text --namespace default
./peer-flow worker &
./peer-flow snapshot-worker &
go test -p 32 ./... -timeout 900s
working-directory: ./flow
env:
AWS_ENDPOINT_URL_S3: http://localhost:9999
AWS_ACCESS_KEY_ID: minio
AWS_SECRET_ACCESS_KEY: miniosecret
AWS_REGION: us-east-1
PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio
PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret
PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_REGION: us-east-1
PEERDB_CLICKHOUSE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999
PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME: peerdb
PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ACCESS_KEY_ID: minio
PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_SECRET_ACCESS_KEY: miniosecret
PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_REGION: us-east-1
PEERDB_SNOWFLAKE_AWS_CREDENTIALS_AWS_ENDPOINT_URL_S3: http://localhost:9999
PEERDB_SNOWFLAKE_AWS_S3_BUCKET_NAME: peerdb
TEST_BQ_CREDS: ${{ github.workspace }}/bq_service_account.json
TEST_SF_CREDS: ${{ github.workspace }}/snowflake_creds.json
TEST_S3_CREDS: ${{ github.workspace }}/s3_creds.json
TEST_GCS_CREDS: ${{ github.workspace }}/gcs_creds.json
TEST_EH_CREDS: ${{ github.workspace }}/eh_creds.json
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
ENABLE_SQLSERVER_TESTS: "true"
SQLSERVER_HOST: ${{ secrets.SQLSERVER_HOST }}
SQLSERVER_PORT: ${{ secrets.SQLSERVER_PORT }}
SQLSERVER_USER: ${{ secrets.SQLSERVER_USER }}
SQLSERVER_PASSWORD: ${{ secrets.SQLSERVER_PASSWORD }}
SQLSERVER_DB: ${{ secrets.SQLSERVER_DB }}
PEERDB_CATALOG_HOST: localhost
PEERDB_CATALOG_PORT: 5432
PEERDB_CATALOG_USER: postgres
PEERDB_CATALOG_PASSWORD: postgres
PEERDB_CATALOG_DATABASE: postgres
PEERDB_QUEUE_FORCE_TOPIC_CREATION: "true"
ELASTICSEARCH_TEST_ADDRESS: http://localhost:9200