change ID #98
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-and-test | |
| on: [push] | |
| jobs: | |
| build_container_base: | |
| if: false # Temporarily disable workflow | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set image tag | |
| id: image_tag | |
| run: | | |
| if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then | |
| echo "image_tag=latest" >> $GITHUB_OUTPUT; | |
| else | |
| echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT; | |
| fi | |
| - name: Docker Login | |
| run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }} | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| version: v0.17.1 | |
| - name: Run Buildx | |
| env: | |
| image_name: galaxy-container-base | |
| run: | | |
| for i in {1..4}; do | |
| set +e | |
| docker buildx build \ | |
| --output "type=image,name=${{ secrets.docker_registry }}/${{ secrets.docker_registry_username }}/$image_name:${{ steps.image_tag.outputs.image_tag }},push=true" \ | |
| --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \ | |
| --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \ | |
| --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \ | |
| $image_name && break || echo "Fail.. Retrying" | |
| done; | |
| shell: bash | |
| working-directory: ./compose/base-images | |
| build_cluster_base: | |
| needs: build_container_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set image tag | |
| id: image_tag | |
| run: | | |
| if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then | |
| echo "image_tag=latest" >> $GITHUB_OUTPUT; | |
| else | |
| echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT; | |
| fi | |
| - name: Docker Login | |
| run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }} | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| version: v0.17.1 | |
| - name: Run Buildx | |
| env: | |
| image_name: galaxy-cluster-base | |
| run: | | |
| for i in {1..4}; do | |
| set +e | |
| docker buildx build \ | |
| --output "type=image,name=${{ secrets.docker_registry }}/${{ secrets.docker_registry_username }}/$image_name:${{ steps.image_tag.outputs.image_tag }},push=true" \ | |
| --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \ | |
| --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \ | |
| --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \ | |
| $image_name && break || echo "Fail.. Retrying" | |
| done; | |
| shell: bash | |
| working-directory: ./compose/base-images | |
| build: | |
| needs: build_cluster_base | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| image: | |
| - name: galaxy-server | |
| - name: galaxy-nginx | |
| - name: galaxy-htcondor | |
| - name: galaxy-slurm | |
| - name: galaxy-slurm-node-discovery | |
| - name: galaxy-kind | |
| - name: pulsar | |
| - name: galaxy-configurator | |
| - name: galaxy-bioblend-test | |
| subdir: tests/ | |
| - name: galaxy-workflow-test | |
| subdir: tests/ | |
| - name: galaxy-selenium-test | |
| subdir: tests/ | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set image tag | |
| id: image_tag | |
| run: | | |
| if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then | |
| echo "image_tag=latest" >> $GITHUB_OUTPUT; | |
| else | |
| echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT; | |
| fi | |
| - name: Docker Login | |
| run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }} | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| version: v0.17.1 | |
| - name: Run Buildx | |
| run: | | |
| for i in {1..4}; do | |
| set +e | |
| docker buildx build \ | |
| --output "type=image,name=${{ secrets.docker_registry }}/${{ secrets.docker_registry_username }}/${{ matrix.image.name }}:${{ steps.image_tag.outputs.image_tag }},push=true" \ | |
| --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \ | |
| --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \ | |
| --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \ | |
| --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy \ | |
| ${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying" | |
| done; | |
| shell: bash | |
| working-directory: ./compose | |
| test: | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| infrastructure: | |
| - name: galaxy-base | |
| files: -f docker-compose.yml | |
| - name: galaxy-proxy-prefix | |
| files: -f docker-compose.yml | |
| env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix EXTRA_SKIP_TESTS_BIOBLEND="not test_import_export_workflow_dict and not test_import_export_workflow_from_local_path" | |
| exclude_test: | |
| - selenium | |
| - name: galaxy-htcondor | |
| files: -f docker-compose.yml -f docker-compose.htcondor.yml | |
| - name: galaxy-slurm | |
| files: -f docker-compose.yml -f docker-compose.slurm.yml | |
| env: SLURM_NODE_COUNT=3 | |
| options: --scale slurm_node=3 | |
| - name: galaxy-pulsar | |
| files: -f docker-compose.yml -f docker-compose.pulsar.yml | |
| exclude_test: | |
| - workflow_quality_control | |
| env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job" | |
| - name: galaxy-pulsar-mq | |
| files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml | |
| exclude_test: | |
| - workflow_quality_control | |
| env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job" | |
| - name: galaxy-k8s | |
| files: -f docker-compose.yml -f docker-compose.k8s.yml | |
| - name: galaxy-singularity | |
| files: -f docker-compose.yml -f docker-compose.singularity.yml | |
| env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" | |
| - name: galaxy-pulsar-mq-singularity | |
| files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml -f docker-compose.singularity.yml | |
| env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job and not test_get_container_resolvers and not test_show_container_resolver" | |
| exclude_test: | |
| - workflow_quality_control | |
| - name: galaxy-slurm-singularity | |
| files: -f docker-compose.yml -f docker-compose.slurm.yml -f docker-compose.singularity.yml | |
| env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" | |
| - name: galaxy-htcondor-singularity | |
| files: -f docker-compose.yml -f docker-compose.htcondor.yml -f docker-compose.singularity.yml | |
| env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver" | |
| test: | |
| - name: bioblend | |
| files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.bioblend.yml | |
| exit-from: galaxy-bioblend-test | |
| timeout: 60 | |
| second_run: "true" | |
| - name: workflow_ard | |
| files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml | |
| exit-from: galaxy-workflow-test | |
| workflow: sklearn/ard/ard.ga | |
| timeout: 60 | |
| second_run: "true" | |
| - name: workflow_quality_control | |
| files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml | |
| exit-from: galaxy-workflow-test | |
| workflow: training/sequence-analysis/quality-control/quality_control.ga | |
| timeout: 60 | |
| - name: workflow_example1 | |
| files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml | |
| exit-from: galaxy-workflow-test | |
| workflow: example1/wf3-shed-tools.ga | |
| timeout: 60 | |
| - name: selenium | |
| files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.selenium.yml | |
| exit-from: galaxy-selenium-test | |
| timeout: 60 | |
| fail-fast: false | |
| steps: | |
| # Self-made `exclude` as Github Actions currently does not support | |
| # exclude/including of dicts in matrices | |
| - name: Check if test should be run | |
| id: run_check | |
| if: contains(matrix.infrastructure.exclude_test, matrix.test.name) != true | |
| run: echo "run=true" >> $GITHUB_OUTPUT | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set image tag in env | |
| run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV | |
| - name: Master branch - Set image to to 'latest' | |
| if: github.ref == 'refs/heads/master' | |
| run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV | |
| - name: Set WORKFLOWS env for worfklows-test | |
| if: matrix.test.workflow | |
| run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV | |
| - name: Install Docker Compose | |
| run: | | |
| sudo apt-get update -qq && sudo apt-get install docker-compose -y | |
| - name: Run tests for the first time | |
| if: steps.run_check.outputs.run | |
| run: | | |
| export DOCKER_REGISTRY=${{ secrets.docker_registry }} | |
| export DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} | |
| export ${{ matrix.infrastructure.env }} | |
| export TIMEOUT=${{ matrix.test.timeout }} | |
| docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} config | |
| env | |
| for i in {1..4}; do | |
| echo "Running test - try \#$i" | |
| echo "Removing export directory if existent"; | |
| sudo rm -rf export | |
| docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} pull | |
| set +e | |
| docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }} | |
| test_exit_code=$? | |
| error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1) | |
| docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} down | |
| if [ $error_exit_codes_count != 0 ] || [ $test_exit_code != 0 ] ; then | |
| echo "Test failed.."; | |
| continue; | |
| else | |
| exit $test_exit_code; | |
| fi | |
| done; | |
| exit 1 | |
| shell: bash | |
| working-directory: ./compose | |
| continue-on-error: false | |
| - name: Fix file names before saving artifacts | |
| if: failure() | |
| run: | | |
| sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \; | |
| - name: Allow upload-artifact read access | |
| if: failure() | |
| run: sudo chmod -R +r ./compose/export/galaxy/database | |
| - name: Save artifacts for debugging a failed test | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_first-run | |
| path: ./compose/export/galaxy/database | |
| - name: Clean up after first run | |
| if: matrix.test.second_run == 'true' | |
| run: | | |
| sudo rm -rf export/postgres | |
| sudo rm -rf export/galaxy/database | |
| working-directory: ./compose | |
| - name: Run tests a second time | |
| if: matrix.test.second_run == 'true' && steps.run_check.outputs.run | |
| run: | | |
| export DOCKER_REGISTRY=${{ secrets.docker_registry }} | |
| export DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} | |
| export ${{ matrix.infrastructure.env }} | |
| export TIMEOUT=${{ matrix.test.timeout }} | |
| for i in {1..4}; do | |
| echo "Running test - try \#$i" | |
| echo "Removing export directory if existent"; | |
| sudo rm -rf export | |
| set +e | |
| docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }} | |
| test_exit_code=$? | |
| error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1) | |
| docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} down | |
| if [ $error_exit_codes_count != 0 ] || [ $test_exit_code != 0 ] ; then | |
| echo "Test failed.."; | |
| continue; | |
| else | |
| exit $test_exit_code; | |
| fi | |
| done; | |
| exit 1 | |
| shell: bash | |
| working-directory: ./compose | |
| continue-on-error: false | |
| - name: Fix file names before saving artifacts | |
| if: failure() && matrix.test.second_run == 'true' | |
| run: | | |
| sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \; | |
| - name: Allow upload-artifact read access | |
| if: failure() && matrix.test.second_run == 'true' | |
| run: sudo chmod -R +r ./compose/export/galaxy/database | |
| - name: Save artifacts for debugging a failed test | |
| uses: actions/upload-artifact@v4 | |
| if: failure() && matrix.test.second_run == 'true' | |
| with: | |
| name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_second-run | |
| path: ./compose/export/galaxy/database |