CI: add new workflows (#84)

.github/workflows/ci.yaml

@@ -0,0 +1,118 @@
+name: Test
+on:
+  pull_request:
+    branches: ["main"]
+  push:
+    branches: ["main"]
+env:
+    DISTRO: shotgun
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    outputs:
+      latest-dev-tag: 2024.5.0.dev0
+      latest-stable-tag: 2024.2.0
+      commit-msg: ${{ steps.get-commit-msg.outputs.commit-msg }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            fetch-depth: 0
+
+      - name: Checkout utilities
+        uses: actions/checkout@v4
+        with:
+          repository: bokulich-lab/utilities
+          path: utilities
+
+      - name: Get last commit
+        id: get-commit-msg
+        run: |
+          echo "$(git log --pretty=%B -n 1 --skip 1)"
+          echo "commit-msg=$(git log --pretty=%B -n 1 --skip 1)" >> $GITHUB_OUTPUT
+
+      - name: Install dependencies
+        run: python -m pip install requests yq
+
+      - name: Create conda yaml
+        id: create-conda-yaml
+        run: |
+          commit_msg="${{ steps.get-commit-msg.outputs.commit-msg }}"
+          if [[ "$commit_msg" == *"[stable]"* ]] || [[ "$commit_msg" == *"[prod]"* ]]; then
+              tag="2024.2.0"
+          else
+              tag="2024.5.0.dev0"
+          fi
+          bash utilities/ci/get-dependencies.sh "$DISTRO" $tag utilities/ci/repositories.yaml
+          cat environment.yml >> $GITHUB_STEP_SUMMARY
+          echo "qiime-deps=$(tr '\n' ' ' < repo-urls.txt | xargs)" >> $GITHUB_OUTPUT
+
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: 3.8
+          mamba-version: "*"
+          channels: conda-forge,defaults
+          channel-priority: true
+          activate-environment: conda-env
+          condarc-file: utilities/ci/condarc
+          # use-only-tar-bz2: true
+
+      - name: Get date
+        id: get-date
+        run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Cache conda env
+        uses: actions/cache@v3
+        with:
+          path: /usr/share/miniconda/envs
+          key:
+            conda-${{ runner.os }}--${{ runner.arch }}--${{
+            steps.get-date.outputs.today }}-${{
+            hashFiles('environment.yml') }}-${{ env.CACHE_NUMBER
+            }}
+        env:
+          # Increase this value to reset cache if environment.yml has not changed
+          CACHE_NUMBER: 0
+        id: cache
+
+      - name: Update environment
+        run: mamba env update -n conda-env -f environment.yml
+        if: steps.cache.outputs.cache-hit != 'true'
+
+#      - name: Install dev versions of QIIME repos
+#        run: mamba run -n conda-env pip install ${{ steps.create-conda-yaml.outputs.qiime-deps }}
+
+      - name: Update specific dependency, if requested
+        if: contains(${{ steps.get-commit-msg.outputs.commit-msg }}, '[add:')
+        run: |
+            commit_msg="${{ steps.get-commit-msg.outputs.commit-msg }}"
+            IFS=',' read -ra ADDR <<< "$commit_msg"
+            for i in "${ADDR[@]}"; do
+                pkg_name=$(echo "$i" | awk -F':' '/\[add/{print $2}')
+                commit_hash=$(echo "$i" | awk -F':' '/\[add/{print $3}' | awk -F']' '{print $1}')
+                if [[ $(yq '.repositories[].name' utilities/ci/repositories.yaml | grep -c $pkg_name) -eq 1 ]]; then
+                    pkg_url=$(yq ".repositories[] | select(.name == \"$pkg_name\") | .url" utilities/ci/repositories.yaml | tr -d '"')
+                    mamba run -n conda-env pip install "git+$pkg_url@$commit_hash"
+                fi
+            done
+
+      - name: Install plugin
+        run: |
+          mamba run -n conda-env pip install .
+          mamba run -n conda-env qiime dev refresh-cache
+
+      - name: Install dev dependencies
+        run: mamba run -n conda-env pip install pytest coverage parameterized
+
+      - name: Run tests
+        id: test
+        run: mamba run -n conda-env make test-cov
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        if: steps.test.outcome == 'success'
+        with:
+          name: coverage
+          path: coverage.xml

.github/workflows/ci-dev.yaml → .github/workflows/q2-ci.yaml

@@ -1,15 +1,23 @@
-name: CI
-on:
-  pull_request:
-    branches: ["main"]
-  push:
-    branches: ["main"]
+name: "QIIME2: build and test"
+on: issue_comment
 
 jobs:
+  react:
+    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/q2ci') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          body: |
+            <br /> **Edit:** Running QIIME 2 CI...
+          reactions: "+1"
+
   ci:
+    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/q2ci') }}
     uses: qiime2/distributions/.github/workflows/lib-ci-dev.yaml@dev
     with:
-      distro: metagenome
+      distro: shotgun
       additional-reports-path: ./coverage.xml
       additional-reports-name: coverage
 

.github/workflows/upload-coverage.yaml

@@ -0,0 +1,64 @@
+name: Upload coverage report
+
+on:
+  workflow_run:
+    workflows: ["Test"]
+    types:
+      - completed
+
+jobs:
+  upload-coverage:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+    steps:
+      - name: 'Download artifact'
+        uses: actions/[email protected]
+        with:
+          script: |
+            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{github.event.workflow_run.id }},
+            });
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "coverage"
+            })[0];
+            var download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/coverage.zip', Buffer.from(download.data));
+
+      - run: unzip coverage.zip
+
+      - name: Find associated PR
+        id: pr
+        uses: actions/[email protected]
+        with:
+          script: |
+            const response = await github.rest.search.issuesAndPullRequests({
+              q: 'repo:${{ github.repository }} is:pr sha:${{ github.event.workflow_run.head_sha }}',
+              per_page: 1,
+            })
+            const items = response.data.items
+            if (items.length < 1) {
+              console.error('No PRs found')
+              return
+            }
+            const pullRequestNumber = items[0].number
+            console.info("Pull request number is", pullRequestNumber)
+            return pullRequestNumber
+
+      - uses: codecov/codecov-action@v4
+        name: Upload coverage report
+        with:
+          files: ./coverage.xml
+          fail_ci_if_error: true
+          override_pr: ${{ steps.pr.outputs.result }}
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

ci/recipe/meta.yaml

@@ -13,11 +13,11 @@ build:
 
 requirements:
   host:
-    - python {{ python }}
+    - python=3.8.*
     - setuptools
 
   run:
-    - python {{ python }}
+    - python=3.8.*
     - qiime2 {{ qiime2_epoch }}.*
     - q2-types {{ qiime2_epoch }}.*
     - q2templates {{ qiime2_epoch }}.*