compose.yml

name: build-and-test
on: [push]
jobs:
  build_container_base:
    if: false  # Temporarily disable workflow 
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set image tag
        id: image_tag
        run: |
          if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then
            echo "image_tag=latest" >> $GITHUB_OUTPUT;
          else
            echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT;
          fi
      - name: Docker Login
        run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }}
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: v0.17.1
      - name: Run Buildx
        env:
          image_name: galaxy-container-base
        run: |
          for i in {1..4}; do
            set +e
            docker buildx build  \
            --output "type=image,name=${{ secrets.docker_registry }}/${{ secrets.docker_registry_username }}/$image_name:${{ steps.image_tag.outputs.image_tag }},push=true" \
            --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \
            --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \
            --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \
            $image_name && break || echo "Fail.. Retrying"
          done;
        shell: bash
        working-directory: ./compose/base-images
  build_cluster_base:
    needs: build_container_base
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set image tag
        id: image_tag
        run: |
          if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then
            echo "image_tag=latest" >> $GITHUB_OUTPUT;
          else
            echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT;
          fi
      - name: Docker Login
        run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }}
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: v0.17.1
      - name: Run Buildx
        env:
          image_name: galaxy-cluster-base
        run: |
          for i in {1..4}; do
            set +e
            docker buildx build  \
            --output "type=image,name=${{ secrets.docker_registry }}/${{ secrets.docker_registry_username }}/$image_name:${{ steps.image_tag.outputs.image_tag }},push=true" \
            --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \
            --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \
            --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \
            $image_name && break || echo "Fail.. Retrying"
          done;
        shell: bash
        working-directory: ./compose/base-images
  build:
    needs: build_cluster_base
    runs-on: ubuntu-latest
    strategy:
      matrix:
        image:
          - name: galaxy-server
          - name: galaxy-nginx
          - name: galaxy-htcondor
          - name: galaxy-slurm
          - name: galaxy-slurm-node-discovery
          - name: galaxy-kind
          - name: pulsar
          - name: galaxy-configurator
          - name: galaxy-bioblend-test
            subdir: tests/
          - name: galaxy-workflow-test
            subdir: tests/
          - name: galaxy-selenium-test
            subdir: tests/
      fail-fast: false
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set image tag
        id: image_tag
        run: |
          if [ "${GITHUB_REF#refs/heads/}" = "master" ]; then
            echo "image_tag=latest" >> $GITHUB_OUTPUT;
          else
            echo "image_tag=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT;
          fi
      - name: Docker Login
        run: echo "${{ secrets.docker_registry_password }}" | docker login -u ${{ secrets.docker_registry_username }} --password-stdin ${{ secrets.docker_registry }}
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: v0.17.1
      - name: Run Buildx
        run: |
          for i in {1..4}; do
            set +e
            docker buildx build \
            --output "type=image,name=${{ secrets.docker_registry }}/${{ secrets.docker_registry_username }}/${{ matrix.image.name }}:${{ steps.image_tag.outputs.image_tag }},push=true" \
            --build-arg IMAGE_TAG=${{ steps.image_tag.outputs.image_tag }} \
            --build-arg DOCKER_REGISTRY=${{ secrets.docker_registry }} \
            --build-arg DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }} \
            --build-arg GALAXY_REPO=https://github.com/galaxyproject/galaxy \
            ${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying"
          done;
        shell: bash
        working-directory: ./compose
  test:
    needs: [build]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        infrastructure:
          - name: galaxy-base
            files: -f docker-compose.yml
          - name: galaxy-proxy-prefix
            files: -f docker-compose.yml
            env: GALAXY_PROXY_PREFIX=/arbitrary_Galaxy-prefix GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=http://localhost/arbitrary_Galaxy-prefix EXTRA_SKIP_TESTS_BIOBLEND="not test_import_export_workflow_dict and not test_import_export_workflow_from_local_path"
            exclude_test:
              - selenium
          - name: galaxy-htcondor
            files: -f docker-compose.yml -f docker-compose.htcondor.yml
          - name: galaxy-slurm
            files: -f docker-compose.yml -f docker-compose.slurm.yml
            env: SLURM_NODE_COUNT=3
            options: --scale slurm_node=3
          - name: galaxy-pulsar
            files: -f docker-compose.yml -f docker-compose.pulsar.yml
            exclude_test:
              - workflow_quality_control
            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job"
          - name: galaxy-pulsar-mq
            files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml
            exclude_test:
              - workflow_quality_control
            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job"
          - name: galaxy-k8s
            files: -f docker-compose.yml -f docker-compose.k8s.yml
          - name: galaxy-singularity
            files: -f docker-compose.yml -f docker-compose.singularity.yml
            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
          - name: galaxy-pulsar-mq-singularity
            files: -f docker-compose.yml -f docker-compose.pulsar.yml -f docker-compose.pulsar.mq.yml -f docker-compose.singularity.yml
            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_wait_for_job and not test_get_container_resolvers and not test_show_container_resolver"
            exclude_test:
              - workflow_quality_control
          - name: galaxy-slurm-singularity
            files: -f docker-compose.yml -f docker-compose.slurm.yml -f docker-compose.singularity.yml
            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
          - name: galaxy-htcondor-singularity
            files: -f docker-compose.yml -f docker-compose.htcondor.yml -f docker-compose.singularity.yml
            env: EXTRA_SKIP_TESTS_BIOBLEND="not test_get_container_resolvers and not test_show_container_resolver"
        test:
          - name: bioblend
            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.bioblend.yml
            exit-from: galaxy-bioblend-test
            timeout: 60
            second_run: "true"
          - name: workflow_ard
            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
            exit-from: galaxy-workflow-test
            workflow: sklearn/ard/ard.ga
            timeout: 60
            second_run: "true"
          - name: workflow_quality_control
            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
            exit-from: galaxy-workflow-test
            workflow: training/sequence-analysis/quality-control/quality_control.ga
            timeout: 60
          - name: workflow_example1
            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
            exit-from: galaxy-workflow-test
            workflow: example1/wf3-shed-tools.ga
            timeout: 60
          - name: selenium
            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.selenium.yml
            exit-from: galaxy-selenium-test
            timeout: 60
      fail-fast: false
    steps:
      # Self-made `exclude` as Github Actions currently does not support
      # exclude/including of dicts in matrices
      - name: Check if test should be run
        id: run_check
        if: contains(matrix.infrastructure.exclude_test, matrix.test.name) != true
        run: echo "run=true" >> $GITHUB_OUTPUT
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set image tag in env
        run: echo "IMAGE_TAG=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
      - name: Master branch - Set image to to 'latest'
        if: github.ref == 'refs/heads/master'
        run: echo "IMAGE_TAG=latest" >> $GITHUB_ENV
      - name: Set WORKFLOWS env for worfklows-test
        if: matrix.test.workflow
        run: echo "WORKFLOWS=${{ matrix.test.workflow }}" >> $GITHUB_ENV
      - name: Install Docker Compose
        run: |
          sudo apt-get update -qq && sudo apt-get install docker-compose -y
      - name: Run tests for the first time
        if: steps.run_check.outputs.run
        run: |
          export DOCKER_REGISTRY=${{ secrets.docker_registry }}
          export DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }}
          export ${{ matrix.infrastructure.env }}
          export TIMEOUT=${{ matrix.test.timeout }}
          docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} config
          env
          for i in {1..4}; do
            echo "Running test - try \#$i"
            echo "Removing export directory if existent";
            sudo rm -rf export
            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} pull
            set +e
            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }}
            test_exit_code=$?
            error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1)
            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} down
            if [ $error_exit_codes_count != 0 ] || [ $test_exit_code != 0 ] ; then
              echo "Test failed..";
              continue;
            else
              exit $test_exit_code;
            fi
          done;
          exit 1
        shell: bash
        working-directory: ./compose
        continue-on-error: false
      - name: Fix file names before saving artifacts
        if: failure()
        run: |
          sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \;
      - name: Allow upload-artifact read access
        if: failure()
        run: sudo chmod -R +r ./compose/export/galaxy/database
      - name: Save artifacts for debugging a failed test
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_first-run
          path: ./compose/export/galaxy/database
      - name: Clean up after first run
        if: matrix.test.second_run == 'true'
        run: |
          sudo rm -rf export/postgres
          sudo rm -rf export/galaxy/database
        working-directory: ./compose
      - name: Run tests a second time
        if: matrix.test.second_run == 'true' && steps.run_check.outputs.run
        run: |
          export DOCKER_REGISTRY=${{ secrets.docker_registry }}
          export DOCKER_REGISTRY_USERNAME=${{ secrets.docker_registry_username }}
          export ${{ matrix.infrastructure.env }}
          export TIMEOUT=${{ matrix.test.timeout }}
          for i in {1..4}; do
            echo "Running test - try \#$i"
            echo "Removing export directory if existent";
            sudo rm -rf export
            set +e
            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} up ${{ matrix.infrastructure.options }} --exit-code-from ${{ matrix.test.exit-from }}
            test_exit_code=$?
            error_exit_codes_count=$(expr $(docker ps -a --filter exited=1 | wc -l) - 1)
            docker-compose ${{ matrix.infrastructure.files }} ${{ matrix.test.files }} down
            if [ $error_exit_codes_count != 0 ] || [ $test_exit_code != 0 ] ; then
              echo "Test failed..";
              continue;
            else
              exit $test_exit_code;
            fi
          done;
          exit 1
        shell: bash
        working-directory: ./compose
        continue-on-error: false
      - name: Fix file names before saving artifacts
        if: failure() && matrix.test.second_run == 'true'
        run: |
          sudo find ./compose/export/galaxy/database -depth -name '*:*' -execdir bash -c 'mv "$1" "${1//:/-}"' bash {} \;
      - name: Allow upload-artifact read access
        if: failure() && matrix.test.second_run == 'true'
        run: sudo chmod -R +r ./compose/export/galaxy/database
      - name: Save artifacts for debugging a failed test
        uses: actions/upload-artifact@v4
        if: failure() && matrix.test.second_run == 'true'
        with:
          name: ${{ matrix.infrastructure.name }}_${{ matrix.test.name }}_second-run
          path: ./compose/export/galaxy/database