Merge branch 'master' into bgruening-patch-3

.circleci/check-for-additional-platforms.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Check to see if any changed recipes have specified the key
+# extra:additional-platforms, and if so, if they match the platform of the
+# currently-running machine.
+
+# arguments
+git_range=$1
+job_name=$2
+
+# Download ARM version of yq
+yq_platform=$(uname)
+yq_arch=$(uname -m)
+[[ $yq_arch = "aarch64" ]] && yq_arch="arm64"
+mkdir -p ${HOME}/bin
+wget https://github.com/mikefarah/yq/releases/latest/download/yq_${yq_platform}_${yq_arch} -O ${HOME}/bin/yq
+chmod +x ${HOME}/bin/yq
+
+# Find recipes changed from this merge
+files=`git diff --name-only --diff-filter AMR ${git_range} | grep -E 'meta.yaml$' `
+build=0
+
+for file in $files; do
+    echo $file
+    # To create a properly-formatted yaml that yq can parse, comment out jinja2
+    # variable setting with {% %} and remove variable use with {{ }}.
+    additional_platforms=$(cat $file \
+    | sed -E 's/(.*)\{%(.*)%\}(.*)/# \1\2\3/g' \
+    | tr -d '{{' | tr -d '}}' \
+    | ${HOME}/bin/yq '.extra.additional-platforms[]')
+    # Check if any additional platforms match this job
+    for additional_platform in $additional_platforms; do
+    if [ "${CIRCLE_JOB}" = "${job_name}-${additional_platform}" ]
+    then
+        build=1
+        break
+    fi
+    done
+done
+
+# If no changed recipes apply to this platform, skip remaining steps
+if [[ build -lt 1 ]]
+then
+    echo "No recipes using this platform, skipping rest of job."
+    circleci-agent step halt
+fi

.circleci/config.yml

@@ -0,0 +1,260 @@
+# .circleci/config.yml
+version: 2.1
+
+executors:
+  # osx-arm64:
+  #   macos:
+  #     xcode: 14.2.0 # indicate your selected version of Xcode
+  #   resource_class: macos.m1.large.gen1
+  linux-aarch64:
+    machine:
+      image: ubuntu-2204:current
+    resource_class: arm.medium
+
+jobs: # a basic unit of work in a run
+  build_and_test:
+    parameters:
+      os:
+        type: executor
+    executor: << parameters.os >>
+    steps:
+      - checkout
+
+      - run:
+          name: Check for Additional Platforms
+          command: ./.circleci/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test"
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt/
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      # We reconfigure conda to use the right channel setup.
+      # This has to be done after the cache is restored, because
+      # the channel setup is not cached as it resides in the home directory.
+      # We could use a system-wide (and therefore cached) channel setup,
+      # but mamba does not support that at the time of implementation
+      # (it ignores settings made by --system).
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      - run:
+          name: Build and test
+          command: |
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            bioconda-utils build recipes config.yml \
+                --lint --docker --mulled-test \
+                --docker-base-image quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch) \
+                --git-range origin/master HEAD
+
+      - run:
+          name: Prepare artifacts
+          command: |
+            (
+            mkdir -p /tmp/artifacts/packages
+            cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0
+            find -name .cache | xargs rm -rf || true
+            for n in index.html channeldata.json linux-aarch64 linux-64 osx-64 noarch; do
+              cp -rv $n /tmp/artifacts/packages || true
+            done
+            if command -V docker >/dev/null; then
+              mkdir -p /tmp/artifacts/images
+              cd /tmp/artifacts/images
+              docker image ls --format='{{.Repository}}:{{.Tag}}' | \
+                { grep biocontainers || true ; } | \
+                xargs -n1 -P4 bash -c '
+                  test -n "${1+x}" || exit 0
+                  echo "Start compressing docker image ${1} ..."
+                  docker save "${1}" | gzip -c > "${1##*/}.tar.gz"
+                  echo "Finished compressing docker image ${1} ."
+                  ' --
+            fi
+            ) || true
+
+      - store_artifacts:
+          path: /tmp/artifacts
+
+  build_and_upload:
+    parameters:
+      os:
+        type: executor
+    executor: << parameters.os >>
+    steps:
+      - checkout
+
+      - run:
+          name: Check for Additional Platforms
+          command: ./.circleci/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload"
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      # We reconfigure conda to use the right channel setup.
+      # This has to be done after the cache is restored, because
+      # the channel setup is not cached as it resides in the home directory.
+      # We could use a system-wide (and therefore cached) channel setup,
+      # but mamba does not support that at the time of implementation
+      # (it ignores settings made by --system).
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      - run:
+          name: Build and push
+          command: |
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            bioconda-utils handle-merged-pr recipes config.yml \
+              --repo bioconda/bioconda-recipes \
+              --git-range ${CIRCLE_SHA1}~1 ${CIRCLE_SHA1} \
+              --fallback build \
+              --artifact-source circleci
+
+  bulk_build:
+    parameters:
+      os:
+        type: executor
+      runner:
+        type: integer
+    executor: << parameters.os >>
+    steps:
+      - add_ssh_keys:
+          fingerprints:
+            - 1e:85:74:42:35:5f:c5:a2:35:c2:ec:b7:80:c0:7c:40
+
+      - checkout
+
+      - run:
+          name: Check for [ci run]
+          command: |
+            commit_message="$(git log --format=oneline -n 1 $CIRCLE_SHA1)"
+            if [[ $commit_message  =~ "[ci run]" ]]; then
+              echo "[ci run] found, continuing."
+            else
+              echo "[ci run] not found, exiting."
+              circleci-agent step halt
+            fi
+
+      - run:
+          name: set git user
+          command: |
+            git config user.name BiocondaBot
+            git config user.email [email protected]
+            git branch --set-upstream-to=origin/$CIRCLE_BRANCH $CIRCLE_BRANCH
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/bulk/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt/
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      # We reconfigure conda to use the right channel setup.
+      # This has to be done after the cache is restored, because
+      # the channel setup is not cached as it resides in the home directory.
+      # We could use a system-wide (and therefore cached) channel setup,
+      # but mamba does not support that at the time of implementation
+      # (it ignores settings made by --system).
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      # For now, do not upload ARM images to biocontainers: --mulled-upload-target biocontainers
+      - run:
+          name: Build and upload
+          command: |
+            set -e
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            echo '============'
+            conda info --all
+            conda config --show-sources
+            python -c 'import bioconda_utils.utils as u ; import pathlib as p ; print(*(f"{f}:\n{p.Path(f).read_text()}" for f in u.load_conda_build_config().exclusive_config_files), sep="\n")'
+            echo '============'
+            bioconda-utils build recipes config.yml \
+              --worker-offset << parameters.runner >> --n-workers 6 \
+              --docker --mulled-test --docker-base-image quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch) \
+              --anaconda-upload \
+              --mulled-upload-target biocontainers \
+              --record-build-failures \
+              --skiplist-leafs
+            conda clean -y --all
+
+
+workflows:
+  build and test (ARM):
+    jobs:
+      - build_and_test:
+          filters:
+            branches:
+              ignore:
+                - master
+                - bulk
+          matrix:
+            parameters:
+              os:
+                #- osx-arm64
+                - linux-aarch64
+
+  build and upload (ARM):
+    jobs:
+      - build_and_upload:
+          filters:
+            branches:
+              only: master
+          matrix:
+            parameters:
+              os:
+                #- osx-arm64
+                - linux-aarch64
+
+  Bulk branch (ARM):
+    jobs:
+      - bulk_build:
+          filters:
+            branches:
+              only: bulk
+          matrix:
+            parameters:
+              os: 
+                #- osx-arm64
+                - linux-aarch64
+              runner: [0, 1, 2, 3, 4, 5]

.github/workflows/Bulk.yml

@@ -77,7 +77,7 @@ jobs:
   build-osx:
     name: Bulk OSX Builds
     if: "contains(github.event.head_commit.message, '[ci run]')"
-    runs-on: macOS-latest
+    runs-on: macos-13
     strategy:
       fail-fast: false
       max-parallel: 4

.github/workflows/PR.yml

@@ -165,7 +165,7 @@ jobs:
 
   build-osx:
     name: OSX Tests
-    runs-on: macOS-latest
+    runs-on: macos-13
     strategy:
       fail-fast: true
       max-parallel: 4

.github/workflows/master.yml

@@ -71,7 +71,7 @@ jobs:
   build-osx:
     name: OSX Upload
     if: github.repository == 'bioconda/bioconda-recipes'
-    runs-on: macOS-latest
+    runs-on: macos-13
     strategy:
       fail-fast: false
       max-parallel: 4

.github/workflows/nightly.yml

@@ -65,7 +65,7 @@ jobs:
   build-osx:
     name: OSX Tests
     if: github.repository == 'bioconda/bioconda-recipes'
-    runs-on: macOS-latest
+    runs-on: macos-13
     strategy:
       fail-fast: false
       max-parallel: 4

azure-pipeline-master.yml

@@ -118,7 +118,6 @@ jobs:
           conda activate bioconda
           export OSTYPE="darwin"
           export CI="true"
-          export HOME=`pwd`
 
           bioconda-utils handle-merged-pr recipes config.yml \
             --repo bioconda/bioconda-recipes \

azure-pipeline-nightly.yml

@@ -76,7 +76,7 @@ jobs:
           # build and push all leftover packages
           bioconda-utils build recipes config.yml \
               --docker --mulled-test --anaconda-upload --mulled-upload-target biocontainers \
-              --prelint
+              --prelint --exclude bioconda-repodata-patches
           docker rmi quay.io/dpryan79/mulled_container:latest
         env:
           QUAY_OAUTH_TOKEN: $(QUAY_OAUTH_TOKEN)
@@ -133,7 +133,6 @@ jobs:
           conda activate bioconda
           export OSTYPE="darwin"
           export CI="true"
-          export HOME=`pwd`
           bioconda-utils build recipes config.yml \
               --anaconda-upload \
               --prelint