Skip to content

Integration

Integration #1269

Workflow file for this run

name: Integration
on:
# Integration tests always run on pull requests
pull_request:
# Re-run integration tests in main, to make sure there are no issues from the merge
push:
branches:
- main
# Allow manual triggering of integration tests
workflow_dispatch:
# Use baseline language versions for integration builds
env:
JAVA_VERSION: 17
JAVA_DISTRIBUTION: zulu
PYTHON_VERSION: "3.11"
NODE_VERSION: 18
jobs:
web-api:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup - Java
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
# Turn on Gradle dependency caching
cache: gradle
- name: Setup -Node.js
uses: actions/setup-node@v3
with:
node-version: ${{ env.NODE_VERSION }}
- name: Platform - Build distribution files
run: ./gradlew installDist
- name: Platform - Install the sandbox
run: |
VERSION=`dev/version.sh`
mkdir -p build/dist/tracdap-sandbox-${VERSION}
for MODULE in build/modules/*/install/*; do
cp -R $MODULE/* build/dist/tracdap-sandbox-${VERSION}
done
cd build/dist
ln -s tracdap-sandbox-${VERSION}/ current
# Currently not starting svc-orch, we'd need to build the runtime and deploy an execution venv
# OUr JavaScript tests only do data and metadata so far, no spawning / checking jobs
# We should aim to get there though!
- name: Platform - Run setup tools
run: |
cd build/dist/current
deploy_dir=`pwd`
mkdir metadata
mkdir -p data/storage1
echo SECRET_KEY=xHjFeFlo$k= >> etc/env.sh
sed -i "s#/path/to#${deploy_dir}#g" etc/trac-platform.yaml
bin/secret-tool run --task init_secrets
bin/secret-tool run --task create_root_auth_key EC 256
bin/deploy-metadb run --task deploy_schema
bin/deploy-metadb run --task add_tenant ACME_CORP "ACME Rockets always get the Coyote!"
- name: Platform - Start
run: |
cd build/dist/current
bin/tracdap-svc-meta start
bin/tracdap-svc-data start
bin/tracdap-gateway start
- name: Web API - Install dependencies
run: |
cd tracdap-api/packages/web
npm install
- name: Web API - Set TRAC version
run: |
cd tracdap-api/packages/web
npm run tracVersion:posix
- name: Web API - Build package
run: |
cd tracdap-api/packages/web
npm run buildApi
- name: Examples - Install dependencies
run: |
cd examples/apps/javascript
npm install
- name: Examples - Run all
run: |
cd examples/apps/javascript
npm run examples
- name: Platform - Shut down
if: always()
run: |
cd build/dist/current
bin/tracdap-gateway stop
bin/tracdap-svc-data stop
bin/tracdap-svc-meta stop
slow-tests:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
# fetch-depth = 0 is needed to get tags for version info
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Java
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
cache: gradle
- name: Build
run: ./gradlew testClasses
- name: Slow tests
run: ./gradlew integration -DintegrationTags="slow"
# If the tests fail, make the output available for download
- name: Store failed test results
uses: actions/upload-artifact@v3
if: failure()
with:
name: junit-test-results
path: build/modules/*/reports/**
retention-days: 7
end-to-end:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
# fetch-depth = 0 is needed to get tags for version info
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Java - Setup
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
# Turn on Gradle dependency caching
cache: gradle
- name: Java - Build platform
run: ./gradlew testClasses
- name: Python - setup
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Python - PIP
run: python -m pip install --upgrade pip
- name: Python - Install build dependencies
run: |
pip install -r tracdap-runtime/python/requirements.txt
- name: Python - Build runtime package
run: python tracdap-runtime/python/build_runtime.py --target codegen dist
- name: Run end to end tests
run: ./gradlew integration -DintegrationTags="int-e2e"
# If the tests fail, make the output available for download
- name: Store failed test results
uses: actions/upload-artifact@v3
if: failure()
with:
name: junit-test-results
path: build/modules/*/reports/**
retention-days: 7
int-metadb:
runs-on: ubuntu-latest
container:
image: ubuntu:latest
strategy:
# Try to finish all jobs - it can be helpful to see if some succeed and others fail
fail-fast: false
matrix:
database:
- { DB_NAME: MySQL,
DB_IMAGE: 'mysql:8.0',
DB_PORT: 3306,
DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3',
BUILD_sql_mysql: true,
TRAC_CONFIG_FILE: '.github/config/int-metadb-mysql.yaml',
TRAC_SECRET_KEY: wDeq3x-NjaLL7,
MYSQL_DATABASE: trac,
MYSQL_USER: trac_admin,
MYSQL_PASSWORD: trac_admin,
METADB_SECRET: trac_admin,
MYSQL_ALLOW_EMPTY_PASSWORD: yes }
- { DB_NAME: MariaDB,
DB_IMAGE: 'mariadb:10.3',
DB_PORT: 3306,
DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3',
BUILD_sql_mariadb: true,
TRAC_CONFIG_FILE: '.github/config/int-metadb-mariadb.yaml',
TRAC_SECRET_KEY: uYhnKwq8+esS,
MYSQL_DATABASE: trac,
MYSQL_USER: trac_admin,
MYSQL_PASSWORD: trac_admin,
METADB_SECRET: trac_admin,
MYSQL_ALLOW_EMPTY_PASSWORD: yes }
- { DB_NAME: PostgreSQL,
DB_IMAGE: 'postgres:10-alpine',
DB_PORT: 5432,
DB_OPTIONS: '--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5',
BUILD_sql_postgresql: true,
TRAC_CONFIG_FILE: '.github/config/int-metadb-postgresql.yaml',
TRAC_SECRET_KEY: hjXks83bX=wxMr,
POSTGRES_DB: trac,
POSTGRES_USER: trac_admin,
METADB_SECRET: trac_admin,
POSTGRES_PASSWORD: trac_admin }
- { DB_NAME: SQLServer,
DB_IMAGE: 'mcr.microsoft.com/mssql/server:2019-latest',
DB_PORT: 1433,
DB_OPTIONS: '-e "NO_DB_OPTIONS=not_used"', # docker run -e flag sets an env variable, passing '' causes errors
BUILD_sql_sqlserver: true,
TRAC_CONFIG_FILE: '.github/config/int-metadb-sqlserver.yaml',
TRAC_SECRET_KEY: unHkj>weN2jSl,
MSSQL_PID: Developer,
ACCEPT_EULA: Y,
SA_PASSWORD: "tR4c_aDm!n",
METADB_SECRET: "tR4c_aDm!n" }
env: ${{ matrix.database }}
services:
metadb:
image: ${{ matrix.database.DB_IMAGE }}
env: ${{ matrix.database }}
ports:
- ${{ matrix.database.DB_PORT }}:${{ matrix.database.DB_PORT }}
options: ${{ matrix.database.DB_OPTIONS }}
steps:
# fetch-depth = 0 is needed to get tags for version info
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Java
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
cache: gradle
- name: Build
run: ./gradlew trac-svc-meta:testClasses
# Auth tool will also create the secrets file if it doesn't exist
- name: Prepare secrets
run: |
./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task init_secrets"
./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task create_root_auth_key EC 256"
echo "${METADB_SECRET}" | ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task add_secret metadb_secret"
# The name and description of the test tenant are verified in one of the test cases so they need to match
# MetadataReapApiTest listTenants()
- name: Prepare database
run: |
./gradlew deploy-metadb:run --args="\
--config ${{ env.TRAC_CONFIG_FILE }} \
--secret-key ${{ env.TRAC_SECRET_KEY }} \
--task deploy_schema \
--task add_tenant ACME_CORP 'Test tenant [ACME_CORP]'"
- name: Integration tests
run: ./gradlew trac-svc-meta:integration -DintegrationTags="int-metadb"
# If the tests fail, make the output available for download
- name: Store failed test results
uses: actions/upload-artifact@v3
if: failure()
with:
name: junit-test-results
path: build/modules/*/reports/**
retention-days: 7
int-storage:
# Storage integration tests not available in CI yet
# (test cases must be run manually until CI resources are available)
if: ${{ false }}
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
# Try to finish all jobs - it can be helpful to see if some succeed and others fail
fail-fast: false
matrix:
storage:
- { PROTOCOL: S3,
BUILD_aws_storage: true,
TRAC_CONFIG_FILE: '.github/config/int-storage-s3.yaml',
TRAC_SECRET_KEY: storage_s3_secrets,
S3_BUCKET: not-configured,
STORGE_SECRET: "not_configured" }
env: ${{ matrix.storage }}
steps:
# fetch-depth = 0 is needed to get tags for version info
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Java
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
cache: gradle
- name: Build
run: ./gradlew testClasses
# Auth tool will also create the secrets file if it doesn't exist
- name: Prepare secrets
run: |
./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task init_secrets"
./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task create_root_auth_key EC 256"
echo "${STORAGE_SECRET}" | ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task add_secret storage_secret"
# No need to prepare DB, it is done by the platform test setup
- name: Integration tests
run: ./gradlew integration -DintegrationTags="int-storage"
# If the tests fail, make the output available for download
- name: Store failed test results
uses: actions/upload-artifact@v3
if: failure()
with:
name: junit-test-results
path: build/modules/*/reports/**
retention-days: 7