comment #840
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: wipac ci/cd | |
| on: [push] | |
| jobs: | |
| ############################################################################# | |
| # PACKAGING & LINTERS | |
| ############################################################################# | |
| py-versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.versions.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: versions | |
| uses: WIPACrepo/[email protected] | |
| flake8: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| - uses: WIPACrepo/[email protected] | |
| mypy: | |
| needs: [py-versions] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - uses: WIPACrepo/[email protected] | |
| py-setup: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # dependabot can't access normal secrets | |
| # & don't run non-branch triggers (like tags) | |
| # & we don't want to trigger an update on PR's merge to main/master/default (which is a branch) | |
| # IOW: only for non-dependabot branches | |
| - if: | | |
| github.actor != 'dependabot[bot]' && | |
| github.ref_type == 'branch' && | |
| format('refs/heads/{0}', github.event.repository.default_branch) != github.ref | |
| name: checkout (only for non-dependabot non-default branches) | |
| uses: actions/checkout@v3 | |
| with: | |
| token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| - if: | | |
| github.actor != 'dependabot[bot]' && | |
| github.ref_type == 'branch' && | |
| format('refs/heads/{0}', github.event.repository.default_branch) != github.ref | |
| name: wipac-dev-py-setup-action (only for non-dependabot non-default branches) | |
| uses: WIPACrepo/[email protected] | |
| with: | |
| base-keywords: "WIPAC IceCube" | |
| ############################################################################# | |
| # TESTS | |
| ############################################################################# | |
| example: | |
| needs: [py-versions] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| broker_client: [nats, pulsar, rabbitmq] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - name: example | |
| run: | | |
| # NOTE: these examples are intended to be the most basic in terms of config: | |
| # no auth, no unnecessary env vars, etc. | |
| # Pulsar | |
| if [ "${{ matrix.broker_client }}" = "pulsar" ]; then | |
| ./resources/docker-pulsar.sh "${{ matrix.broker_client }}" | |
| # RabbitMQ | |
| elif [ "${{ matrix.broker_client }}" = "rabbitmq" ]; then | |
| ./resources/docker-rabbitmq.sh "${{ matrix.broker_client }}" | |
| # NATS | |
| elif [ "${{ matrix.broker_client }}" = "nats" ]; then | |
| ./resources/local-nats-js.sh | |
| fi | |
| echo "--------------------------------------------------------------" | |
| echo "running examples..." | |
| pip install --upgrade pip wheel setuptools | |
| pip install .[dev,${{ matrix.broker_client }}] | |
| id=$(date +%s) | |
| python examples/worker.py \ | |
| --broker-client ${{ matrix.broker_client }} \ | |
| --in-queue "queue1-$id" \ | |
| --out-queue "queue2-$id" \ | |
| >> worker.out 2>&1 & | |
| python examples/server.py \ | |
| --broker-client ${{ matrix.broker_client }} \ | |
| --work-queue "queue1-$id" \ | |
| --result-queue "queue2-$id" | |
| - name: worker output | |
| if: always() | |
| run: | | |
| cat worker.out | |
| - name: broker output | |
| if: always() | |
| run: | | |
| set -x | |
| cat ./broker.out | |
| if [ -n "$(ls -A ./broker_logs/ 2>/dev/null)" ]; then | |
| ls ./broker_logs/ | |
| more ./broker_logs/* | cat # cats with filenames (delimited by :::::::) | |
| fi | |
| - name: broker docker logs | |
| if: always() | |
| run: | | |
| docker logs "${{ matrix.broker_client }}" || true | |
| unit-test: | |
| needs: [py-versions] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| type: [brokerless, pulsar, rabbitmq] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - name: tests | |
| run: | | |
| pip install --upgrade pip wheel setuptools | |
| if [ "${{ matrix.type }}" = "brokerless" ]; then | |
| pip install .[dev] | |
| else | |
| pip install .[dev,${{ matrix.type }}] | |
| fi | |
| pytest -vvv tests/unit/"${{ matrix.type }}" | |
| integration-test: | |
| needs: [py-versions] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| py3: ${{ fromJSON(needs.py-versions.outputs.matrix) }} | |
| broker_client: [nats, pulsar, rabbitmq] | |
| services: | |
| keycloak: | |
| image: ghcr.io/wipacrepo/keycloak-rest-services:test-keycloak-master | |
| env: | |
| KEYCLOAK_ADMIN: admin | |
| KEYCLOAK_ADMIN_PASSWORD: admin | |
| CMD: "start --hostname-url=https://localhost:8443/auth" | |
| ports: | |
| #- 8081:8080 # host 8080 is used by pulsar | |
| - 8443:8443 | |
| env: | |
| KEYCLOAK_URL: https://localhost:8443 | |
| KEYCLOAK_REALM: testrealm | |
| KEYCLOAK_CLIENT_ID: testclient | |
| USERNAME: admin | |
| PASSWORD: admin | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.py3 }} | |
| - name: install deps | |
| run: | | |
| pip install --upgrade pip wheel setuptools | |
| pip install .[dev,${{ matrix.broker_client }},integration] | |
| - name: get keycloak cert | |
| run: | | |
| docker run --rm -v $PWD:/mnt -u root --entrypoint=/usr/bin/keytool ghcr.io/wipacrepo/keycloak-rest-services:test-keycloak-master -export -alias server -keystore /opt/keycloak/conf/server.keystore -rfc -file /mnt/public.cert -storepass password | |
| echo "REQUESTS_CA_BUNDLE=$PWD/public.cert" >> $GITHUB_ENV | |
| - name: bootstrap keycloak | |
| run: | | |
| python ./resources/keycloak_bootstrap.py | |
| cat KEYCLOAK_OIDC_URL.txt | |
| cat KEYCLOAK_CLIENT_ID.txt | |
| cat KEYCLOAK_CLIENT_SECRET.txt | |
| - name: tests | |
| run: | | |
| export KEYCLOAK_OIDC_URL=$(cat KEYCLOAK_OIDC_URL.txt) | |
| echo $KEYCLOAK_OIDC_URL | |
| export KEYCLOAK_CLIENT_ID=$(cat KEYCLOAK_CLIENT_ID.txt) | |
| echo $KEYCLOAK_CLIENT_ID | |
| export KEYCLOAK_CLIENT_SECRET=$(cat KEYCLOAK_CLIENT_SECRET.txt) | |
| echo $KEYCLOAK_CLIENT_SECRET | |
| # Pulsar | |
| if [ "${{ matrix.broker_client }}" = "pulsar" ]; then | |
| ./resources/docker-pulsar.sh "${{ matrix.broker_client }}" | |
| export PYTEST_DO_AUTH_FOR_MQCLIENT=no | |
| # RabbitMQ | |
| elif [ "${{ matrix.broker_client }}" = "rabbitmq" ]; then | |
| ./resources/docker-rabbitmq.sh "${{ matrix.broker_client }}" ./resources/rabbitmq-custom.conf | |
| export PYTEST_DO_AUTH_FOR_MQCLIENT=yes | |
| # NATS | |
| elif [ "${{ matrix.broker_client }}" = "nats" ]; then | |
| ./resources/local-nats-js.sh | |
| export PYTEST_DO_AUTH_FOR_MQCLIENT=no | |
| fi | |
| echo "--------------------------------------------------------------" | |
| echo "running tests..." | |
| pip install --upgrade pip wheel setuptools | |
| pip install .[dev,${{ matrix.broker_client }},integration] | |
| pytest -vvv tests/integrate/test_${{ matrix.broker_client }}.py -n 10 | |
| - name: broker output | |
| if: always() | |
| run: | | |
| set -x | |
| cat ./broker.out | |
| if [ -n "$(ls -A ./broker_logs/ 2>/dev/null)" ]; then | |
| ls ./broker_logs/ | |
| more ./broker_logs/* | cat # cats with filenames (delimited by :::::::) | |
| fi | |
| - name: broker docker logs | |
| if: always() | |
| run: | | |
| docker logs "${{ matrix.broker_client }}" || true | |
| ############################################################################# | |
| # GITHUB RELEASE | |
| ############################################################################# | |
| release: | |
| # only run on main/master/default | |
| if: format('refs/heads/{0}', github.event.repository.default_branch) == github.ref | |
| needs: [flake8, mypy, py-setup, example, unit-test, integration-test] | |
| runs-on: ubuntu-latest | |
| concurrency: release | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - uses: python-semantic-release/[email protected] | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| repository_username: __token__ | |
| repository_password: ${{ secrets.PYPI_TOKEN }} |