Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: split packages #1496

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

chore: split packages #1496

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b39a86d
chore: split packages
kristof-mattei Oct 24, 2023
f50c395
chore: remove CI trigger lines
kristof-mattei Jan 5, 2024
d92f066
Merge branch 'master' into feature/split-docker-packages
kristof-mattei Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 93 additions & 64 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
# These environment variables must be set in CircleCI UI
#
# DOCKERHUB_REPO - docker hub repo, format: <username>/<repo>
# DOCKERHUB_REPO_SPANNER - docker hub repo, format: <username>/<repo>
# DOCKERHUB_REPO_MYSQL - docker hub repo, format: <username>/<repo>
# DOCKER_USER - login info for docker hub
# DOCKER_PASS
# DOCKERHUB_REPO_UTILS=mozilla/sync-spanner-py-utils
#
version: 2.1
commands:
display-rust:
steps:
- run:
name: Display Rust Version
command:
rustc --version
command: rustc --version
setup-rust-check:
steps:
- run:
Expand Down Expand Up @@ -95,45 +96,45 @@ commands:
name: quota test
command: cargo test --workspace --verbose
environment:
SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1
SYNC_SYNCSTORAGE__ENFORCE_QUOTA: 1

run-e2e-mysql-tests:
steps:
- run:
name: e2e tests (syncstorage mysql)
command: >
/usr/local/bin/docker-compose
-f docker-compose.mysql.yaml
-f docker-compose.e2e.mysql.yaml
up
--exit-code-from mysql-e2e-tests
--abort-on-container-exit
/usr/local/bin/docker-compose
-f docker-compose.mysql.yaml
-f docker-compose.e2e.mysql.yaml
up
--exit-code-from mysql-e2e-tests
--abort-on-container-exit
environment:
SYNCSTORAGE_RS_IMAGE: app:build
SYNCSTORAGE_RS_IMAGE: app:build

run-tokenserver-scripts-tests:
steps:
- run:
name: Tokenserver scripts tests
command: >
pip3 install -r tools/tokenserver/requirements.txt &&
python3 tools/tokenserver/run_tests.py
pip3 install -r tools/tokenserver/requirements.txt &&
python3 tools/tokenserver/run_tests.py
environment:
SYNCSTORAGE_RS_IMAGE: app:build
SYNCSTORAGE_RS_IMAGE: app:build

run-e2e-spanner-tests:
steps:
- run:
name: e2e tests (syncstorage spanner)
command: >
/usr/local/bin/docker-compose
-f docker-compose.spanner.yaml
-f docker-compose.e2e.spanner.yaml
up
--exit-code-from spanner-e2e-tests
--abort-on-container-exit
/usr/local/bin/docker-compose
-f docker-compose.spanner.yaml
-f docker-compose.e2e.spanner.yaml
up
--exit-code-from spanner-e2e-tests
--abort-on-container-exit
environment:
SYNCSTORAGE_RS_IMAGE: app:build
SYNCSTORAGE_RS_IMAGE: app:build

setup-sccache:
steps:
Expand All @@ -158,6 +159,39 @@ commands:
key: sccache-cache-stable-{{ .Branch }}-{{ .Environment.CIRCLE_JOB }}-{{ epoch }}
paths:
- "~/.cache/sccache"
deploy:
parameters:
repo-env-name:
type: env_var_name
steps:
- run:
name: Restore Docker image cache
command: docker load -i /home/circleci/cache/docker.tar
- run:
name: Deploy to Dockerhub
command: |
export DOCKERHUB_REPO=${<< parameters.repo-env-name >>}
if [ "${CIRCLE_BRANCH}" == "master" ]; then
DOCKER_TAG="${CIRCLE_SHA1}"
fi

if echo "${CIRCLE_BRANCH}" | grep '^feature\..*' > /dev/null; then
DOCKER_TAG="${CIRCLE_BRANCH}"
fi

if [ -n "${CIRCLE_TAG}" ]; then
DOCKER_TAG="$CIRCLE_TAG"
fi

if [ -n "${DOCKER_TAG}" ]; then
echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
echo ${DOCKERHUB_REPO}:${DOCKER_TAG}
docker tag app:build ${DOCKERHUB_REPO}:${DOCKER_TAG}
docker images
docker push "${DOCKERHUB_REPO}:${DOCKER_TAG}"
else
echo "Not pushing to dockerhub for tag=${CIRCLE_TAG} branch=${CIRCLE_BRANCH}"
fi
jobs:
checks:
docker:
Expand All @@ -183,20 +217,20 @@ jobs:
username: $DOCKER_USER
password: $DOCKER_PASS
environment:
SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:[email protected]/syncstorage
SYNC_TOKENSERVER__DATABASE_URL: mysql://test:[email protected]/tokenserver
RUST_BACKTRACE: 1
# XXX: begin_test_transaction doesn't play nice over threaded tests
RUST_TEST_THREADS: 1
SYNC_SYNCSTORAGE__DATABASE_URL: mysql://test:[email protected]/syncstorage
SYNC_TOKENSERVER__DATABASE_URL: mysql://test:[email protected]/tokenserver
RUST_BACKTRACE: 1
# XXX: begin_test_transaction doesn't play nice over threaded tests
RUST_TEST_THREADS: 1
- image: circleci/mysql:5.7-ram
auth:
username: $DOCKER_USER
password: $DOCKER_PASS
environment:
MYSQL_ROOT_PASSWORD: password
MYSQL_USER: test
MYSQL_PASSWORD: test
MYSQL_DATABASE: syncstorage
MYSQL_ROOT_PASSWORD: password
MYSQL_USER: test
MYSQL_PASSWORD: test
MYSQL_DATABASE: syncstorage
resource_class: large
steps:
- checkout
Expand Down Expand Up @@ -310,7 +344,7 @@ jobs:
command: cp /home/circleci/cache/docker-compose*.yaml .
- run-e2e-spanner-tests

deploy:
deploy-mysql:
docker:
- image: docker:18.02.0-ce
auth:
Expand All @@ -319,34 +353,22 @@ jobs:
steps:
- setup_remote_docker
- restore_cache:
key: spanner-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}
- run:
name: Restore Docker image cache
command: docker load -i /home/circleci/cache/docker.tar
- run:
name: Deploy to Dockerhub
command: |
if [ "${CIRCLE_BRANCH}" == "master" ]; then
DOCKER_TAG="${CIRCLE_SHA1}"
fi

if echo "${CIRCLE_BRANCH}" | grep '^feature\..*' > /dev/null; then
DOCKER_TAG="${CIRCLE_BRANCH}"
fi

if [ -n "${CIRCLE_TAG}" ]; then
DOCKER_TAG="$CIRCLE_TAG"
fi
key: mysql-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}
- deploy:
repo-env-name: DOCKERHUB_REPO_MYSQL

if [ -n "${DOCKER_TAG}" ]; then
echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
echo ${DOCKERHUB_REPO}:${DOCKER_TAG}
docker tag app:build ${DOCKERHUB_REPO}:${DOCKER_TAG}
docker images
docker push "${DOCKERHUB_REPO}:${DOCKER_TAG}"
else
echo "Not pushing to dockerhub for tag=${CIRCLE_TAG} branch=${CIRCLE_BRANCH}"
fi
deploy-spanner:
docker:
- image: docker:18.02.0-ce
auth:
username: $DOCKER_USER
password: $DOCKER_PASS
steps:
- setup_remote_docker
- restore_cache:
key: spanner-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}
- deploy:
repo-env-name: DOCKERHUB_REPO_SPANNER

deploy-python-utils:
docker:
Expand All @@ -361,7 +383,6 @@ jobs:
- run:
name: Build and deploy to Dockerhub
command: |
export UTILS_DOCKERHUB_REPO=mozilla/sync-spanner-py-utils
if [ "${CIRCLE_BRANCH}" == "master" ]; then
DOCKER_TAG="latest"
fi
Expand All @@ -376,11 +397,11 @@ jobs:

if [ -n "${DOCKER_TAG}" ]; then
echo "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
echo ${UTILS_DOCKERHUB_REPO}:${DOCKER_TAG}
echo ${DOCKERHUB_REPO_UTILS}:${DOCKER_TAG}
cd tools/spanner
docker build -t ${UTILS_DOCKERHUB_REPO}:${DOCKER_TAG} .
docker build -t ${DOCKERHUB_REPO_UTILS}:${DOCKER_TAG} .
docker images
docker push "${UTILS_DOCKERHUB_REPO}:${DOCKER_TAG}"
docker push "${DOCKERHUB_REPO_UTILS}:${DOCKER_TAG}"
else
echo "Not building or pushing to dockerhub for tag=${CIRCLE_TAG} branch=${CIRCLE_BRANCH}"
fi
Expand Down Expand Up @@ -421,7 +442,16 @@ workflows:
filters:
tags:
only: /.*/
- deploy:
- deploy-mysql:
requires:
- mysql-e2e-tests
- spanner-e2e-tests
filters:
tags:
only: /.*/
branches:
only: master
- deploy-spanner:
requires:
- mysql-e2e-tests
- spanner-e2e-tests
Expand All @@ -430,7 +460,6 @@ workflows:
only: /.*/
branches:
only: master
# touch: 1676417203
- deploy-python-utils:
requires:
- mysql-e2e-tests
Expand Down