Skip to content

Compliance update (MySQL CVE-2023-22102) #1272

Compliance update (MySQL CVE-2023-22102)

Compliance update (MySQL CVE-2023-22102) #1272

Workflow file for this run

name: Integration
on:
# Integration tests always run on pull requests
pull_request:
# Re-run integration tests in main, to make sure there are no issues from the merge
push:
branches:
- main
# Allow manual triggering of integration tests
workflow_dispatch:
# Use baseline language versions for integration builds
env:
JAVA_VERSION: 17
JAVA_DISTRIBUTION: zulu
PYTHON_VERSION: "3.11"
NODE_VERSION: 18
jobs:
web-api:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup - Java
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
# Turn on Gradle dependency caching
cache: gradle
- name: Setup -Node.js
uses: actions/setup-node@v3
with:
node-version: ${{ env.NODE_VERSION }}
- name: Platform - Build distribution files
run: ./gradlew installDist
- name: Platform - Install the sandbox
run: |
VERSION=`dev/version.sh`
mkdir -p build/dist/tracdap-sandbox-${VERSION}
for MODULE in build/modules/*/install/*; do
cp -R $MODULE/* build/dist/tracdap-sandbox-${VERSION}
done
cd build/dist
ln -s tracdap-sandbox-${VERSION}/ current
# Currently not starting svc-orch, we'd need to build the runtime and deploy an execution venv
# OUr JavaScript tests only do data and metadata so far, no spawning / checking jobs
# We should aim to get there though!
- name: Platform - Run setup tools
run: |
cd build/dist/current
deploy_dir=`pwd`
mkdir metadata
mkdir -p data/storage1
echo SECRET_KEY=xHjFeFlo$k= >> etc/env.sh
sed -i "s#/path/to#${deploy_dir}#g" etc/trac-platform.yaml
bin/secret-tool run --task init_secrets
bin/secret-tool run --task create_root_auth_key EC 256
bin/deploy-metadb run --task deploy_schema
bin/deploy-metadb run --task add_tenant ACME_CORP "ACME Rockets always get the Coyote!"
- name: Platform - Start
run: |
cd build/dist/current
bin/tracdap-svc-meta start
bin/tracdap-svc-data start
bin/tracdap-gateway start
- name: Web API - Install dependencies
run: |
cd tracdap-api/packages/web
npm install
- name: Web API - Set TRAC version
run: |
cd tracdap-api/packages/web
npm run tracVersion:posix
- name: Web API - Build package
run: |
cd tracdap-api/packages/web
npm run buildApi
- name: Examples - Install dependencies
run: |
cd examples/apps/javascript
npm install
- name: Examples - Run all
run: |
cd examples/apps/javascript
npm run examples
- name: Platform - Shut down
if: always()
run: |
cd build/dist/current
bin/tracdap-gateway stop
bin/tracdap-svc-data stop
bin/tracdap-svc-meta stop
slow-tests:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
# fetch-depth = 0 is needed to get tags for version info
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Java
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
cache: gradle
- name: Build
run: ./gradlew testClasses
- name: Slow tests
run: ./gradlew integration -DintegrationTags="slow"
# If the tests fail, make the output available for download
- name: Store failed test results
uses: actions/upload-artifact@v3
if: failure()
with:
name: junit-test-results
path: build/modules/*/reports/**
retention-days: 7
end-to-end:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
# fetch-depth = 0 is needed to get tags for version info
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Java - Setup
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
# Turn on Gradle dependency caching
cache: gradle
- name: Java - Build platform
run: ./gradlew testClasses
- name: Python - setup
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Python - PIP
run: python -m pip install --upgrade pip
- name: Python - Install build dependencies
run: |
pip install -r tracdap-runtime/python/requirements.txt
- name: Python - Build runtime package
run: python tracdap-runtime/python/build_runtime.py --target codegen dist
- name: Run end to end tests
run: ./gradlew integration -DintegrationTags="int-e2e"
# If the tests fail, make the output available for download
- name: Store failed test results
uses: actions/upload-artifact@v3
if: failure()
with:
name: junit-test-results
path: build/modules/*/reports/**
retention-days: 7
int-metadb:
runs-on: ubuntu-latest
timeout-minutes: 20
container:
image: ubuntu:latest
strategy:
# Try to finish all jobs - it can be helpful to see if some succeed and others fail
fail-fast: false
matrix:
database:
- { DB_NAME: MySQL,
DB_IMAGE: 'mysql:8.0',
DB_PORT: 3306,
DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3',
BUILD_sql_mysql: true,
TRAC_CONFIG_FILE: '.github/config/int-metadb-mysql.yaml',
TRAC_SECRET_KEY: wDeq3x-NjaLL7,
MYSQL_DATABASE: trac,
MYSQL_USER: trac_admin,
MYSQL_PASSWORD: trac_admin,
METADB_SECRET: trac_admin,
MYSQL_ALLOW_EMPTY_PASSWORD: yes }
- { DB_NAME: MariaDB,
DB_IMAGE: 'mariadb:10.3',
DB_PORT: 3306,
DB_OPTIONS: '--health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3',
BUILD_sql_mariadb: true,
TRAC_CONFIG_FILE: '.github/config/int-metadb-mariadb.yaml',
TRAC_SECRET_KEY: uYhnKwq8+esS,
MYSQL_DATABASE: trac,
MYSQL_USER: trac_admin,
MYSQL_PASSWORD: trac_admin,
METADB_SECRET: trac_admin,
MYSQL_ALLOW_EMPTY_PASSWORD: yes }
- { DB_NAME: PostgreSQL,
DB_IMAGE: 'postgres:10-alpine',
DB_PORT: 5432,
DB_OPTIONS: '--health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5',
BUILD_sql_postgresql: true,
TRAC_CONFIG_FILE: '.github/config/int-metadb-postgresql.yaml',
TRAC_SECRET_KEY: hjXks83bX=wxMr,
POSTGRES_DB: trac,
POSTGRES_USER: trac_admin,
METADB_SECRET: trac_admin,
POSTGRES_PASSWORD: trac_admin }
- { DB_NAME: SQLServer,
DB_IMAGE: 'mcr.microsoft.com/mssql/server:2019-latest',
DB_PORT: 1433,
DB_OPTIONS: '-e "NO_DB_OPTIONS=not_used"', # docker run -e flag sets an env variable, passing '' causes errors
BUILD_sql_sqlserver: true,
TRAC_CONFIG_FILE: '.github/config/int-metadb-sqlserver.yaml',
TRAC_SECRET_KEY: unHkj>weN2jSl,
MSSQL_PID: Developer,
ACCEPT_EULA: Y,
SA_PASSWORD: "tR4c_aDm!n",
METADB_SECRET: "tR4c_aDm!n" }
env: ${{ matrix.database }}
services:
metadb:
image: ${{ matrix.database.DB_IMAGE }}
env: ${{ matrix.database }}
ports:
- ${{ matrix.database.DB_PORT }}:${{ matrix.database.DB_PORT }}
options: ${{ matrix.database.DB_OPTIONS }}
steps:
# fetch-depth = 0 is needed to get tags for version info
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Java
uses: actions/setup-java@v3
with:
distribution: ${{ env.JAVA_DISTRIBUTION }}
java-version: ${{ env.JAVA_VERSION }}
cache: gradle
- name: Build
run: ./gradlew trac-svc-meta:testClasses
# Auth tool will also create the secrets file if it doesn't exist
- name: Prepare secrets
run: |
./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task init_secrets"
./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task create_root_auth_key EC 256"
echo "${METADB_SECRET}" | ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task add_secret metadb_secret"
# The name and description of the test tenant are verified in one of the test cases so they need to match
# MetadataReapApiTest listTenants()
- name: Prepare database
run: |
./gradlew deploy-metadb:run --args="\
--config ${{ env.TRAC_CONFIG_FILE }} \
--secret-key ${{ env.TRAC_SECRET_KEY }} \
--task deploy_schema \
--task add_tenant ACME_CORP 'Test tenant [ACME_CORP]'"
- name: Integration tests
run: ./gradlew trac-svc-meta:integration -DintegrationTags="int-metadb"
# If the tests fail, make the output available for download
- name: Store failed test results
uses: actions/upload-artifact@v3
if: failure()
with:
name: junit-test-results
path: build/modules/*/reports/**
retention-days: 7
int-storage:
strategy:
# Try to finish all jobs - it can be helpful to see if some succeed and others fail
fail-fast: false
matrix:
storage:
- { SERVICE: storage,
TARGET: aws,
PROTOCOL: S3,
ENABLE_IF: TRAC_AWS_BUCKET,
BUILD_aws_storage: true,
INSTALL_SCRIPT: ".github/scripts/int-storage-s3-install.sh",
SETUP_SCRIPT: ".github/scripts/int-storage-s3-setup.sh",
TRAC_CONFIG_FILE: '.github/config/int-storage-s3.yaml',
TRAC_SECRET_KEY: short-lived-secret,
PYTHON_TESTS: int_storage_aws*.py,
JAVA_TESTS: int-storage }
- { SERVICE: storage,
TARGET: gcp,
PROTOCOL: GCS,
ENABLE_IF: TRAC_GCP_BUCKET,
BUILD_gcp_storage: true,
INSTALL_SCRIPT: ".github/scripts/int-storage-gcs-install.sh",
SETUP_SCRIPT: ".github/scripts/int-storage-gcs-setup.sh",
TRAC_CONFIG_FILE: '.github/config/int-storage-gcs.yaml',
TRAC_SECRET_KEY: short-lived-secret,
PYTHON_TESTS: int_storage_gcp*.py,
JAVA_TESTS: int-storage }
- { SERVICE: storage,
TARGET: azure,
PROTOCOL: BLOB,
ENABLE_IF: TRAC_AZURE_CONTAINER,
BUILD_azure_storage: true,
INSTALL_SCRIPT: ".github/scripts/int-storage-azure-install.sh",
SETUP_SCRIPT: ".github/scripts/int-storage-azure-setup.sh",
TRAC_CONFIG_FILE: '.github/config/int-storage-azure.yaml',
TRAC_SECRET_KEY: short-lived-secret,
PYTHON_TESTS: int_storage_azure*.py,
JAVA_TESTS: int-storage }
uses: ./.github/workflows/integration-cloud.yaml
with:
service: ${{ matrix.storage.SERVICE }}
target: ${{ matrix.storage.TARGET }}
protocol: ${{ matrix.storage.PROTOCOL }}
matrix: ${{ toJson( matrix.storage ) }}
enabled: ${{ vars[ matrix.storage.ENABLE_IF ] != '' }}
has_java: ${{ matrix.storage.JAVA_TESTS != '' }}
has_python: ${{ matrix.storage.PYTHON_TESTS != '' }}
secrets: inherit
# Permissions required to use IdP to connect to cloud platforms
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout