From 2b7dd936a81904430f357732e54d978f93eea862 Mon Sep 17 00:00:00 2001
From: Dima Ryazanov <dima@gmail.com>
Date: Tue, 6 Feb 2024 15:29:18 -0800
Subject: [PATCH] Split the workflows into catalog.yml, client.yml, and
 lambdas.yml

It makes more sense than JS vs Python, and lets us filter by paths and skip unnecessary testing.
---
 .github/workflows/{js-ci.yml => catalog.yml} |  7 +-
 .github/workflows/{py-ci.yml => client.yml}  | 78 +++--------------
 .github/workflows/lambdas.yml                | 90 ++++++++++++++++++++
 3 files changed, 106 insertions(+), 69 deletions(-)
 rename .github/workflows/{js-ci.yml => catalog.yml} (90%)
 rename .github/workflows/{py-ci.yml => client.yml} (62%)
 create mode 100644 .github/workflows/lambdas.yml

diff --git a/.github/workflows/js-ci.yml b/.github/workflows/catalog.yml
similarity index 90%
rename from .github/workflows/js-ci.yml
rename to .github/workflows/catalog.yml
index a852a2daf18..391ab2241eb 100644
--- a/.github/workflows/js-ci.yml
+++ b/.github/workflows/catalog.yml
@@ -1,4 +1,9 @@
-on: [push]
+on:
+  push:
+    paths:
+      - '.github/workflows/catalog.yml'
+      - 'catalog/**'
+      - 'shared/**'
 jobs:
   test-catalog:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/py-ci.yml b/.github/workflows/client.yml
similarity index 62%
rename from .github/workflows/py-ci.yml
rename to .github/workflows/client.yml
index 62243b2b907..82fb919e5d1 100644
--- a/.github/workflows/py-ci.yml
+++ b/.github/workflows/client.yml
@@ -1,4 +1,10 @@
-on: [push]
+on:
+  push:
+    paths:
+      - '.github/workflows/client.yml'
+      - 'api/python/**'
+      - 'gendocs/**'
+      - 'testdocs/**'
 jobs:
   linter:
     runs-on: ubuntu-latest
@@ -13,13 +19,13 @@ jobs:
           python -m pip install 'pylint==2.10.2' 'pycodestyle>=2.6.1' isort
       - name: Run pylint
         run: |
-          pylint $(find -name '*.py')
+          pylint $(find ./api/python/ -name '*.py')
       - name: Run pycodestyle
         run: |
-          pycodestyle $(find -name '*.py')
+          pycodestyle $(find ./api/python/ -name '*.py')
       - name: Run isort
         run: |
-          isort --check --diff .
+          isort --check --diff ./api/python/
 
   test-gendocs:
     runs-on: ubuntu-latest
@@ -129,67 +135,3 @@ jobs:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: twine upload dist/*
-
-  test-lambda:
-    strategy:
-      fail-fast: false
-      matrix:
-        path:
-          - access_counts
-          - es/indexer
-          - molecule
-          - pkgevents
-          - pkgpush
-          - pkgselect
-          - preview
-          - s3hash
-          - s3select
-          - shared
-          - status_reports
-          - tabular_preview
-          - transcode
-        python-version: ['3.8']
-        include:
-          - path: thumbnail
-            python-version: '3.9'
-    runs-on: ubuntu-latest
-    env:
-      QUILT_DISABLE_USAGE_METRICS: true
-      PYTHONPATH: deps
-      JUPYTER_PATH: deps/share/jupyter  # Jupyter is not smart enough to handle custom Python paths
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          # Due to behavior change in pip>=23.1 installing tifffile==0.15.1
-          # from thumbnail lambda fails whithout installed wheel.
-          # See https://github.com/pypa/pip/issues/8559.
-          python -m pip install wheel
-          if [ ${{ matrix.path }} == "shared" ]
-            python -m pip install -e lambdas/shared[tests]
-          then
-            python -m pip install -e lambdas/shared
-            python -m pip install -e lambdas/${{ matrix.path }}
-          fi
-          python -m pip install -r lambdas/${{ matrix.path }}/test-requirements.txt
-
-          # Try to simulate the lambda .zip file:
-          # - Use --no-deps to ensure that second-order dependencies are included in the requirements file
-          # - Remove "tests" directories
-          # - Run "strip" on shared libraries
-          python -m pip install -t deps --no-deps -r lambdas/${{ matrix.path }}/requirements.txt
-          find deps -name tests -type d -exec rm -r \{} \+
-          find deps \( -name '*.so.*' -o -name '*.so' \) -type f -exec strip \{} \+
-      - name: Pytest
-        run: |
-          pytest --cov=lambdas lambdas/${{ matrix.path }}
-      - uses: codecov/codecov-action@v3
-        env:
-          LAMBDA: ${{ matrix.path }}
-        with:
-          flags: lambda
-          name: ${{ github.job }}
-          env_vars: LAMBDA
diff --git a/.github/workflows/lambdas.yml b/.github/workflows/lambdas.yml
new file mode 100644
index 00000000000..432ef5cb355
--- /dev/null
+++ b/.github/workflows/lambdas.yml
@@ -0,0 +1,90 @@
+on:
+    push:
+      paths:
+        - '.github/workflows/lambdas.yml'
+        - 'lambdas/**'
+jobs:
+  linter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.7'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools
+          python -m pip install 'pylint==2.10.2' 'pycodestyle>=2.6.1' isort
+      - name: Run pylint
+        run: |
+          pylint $(find ./lambdas/ -name '*.py')
+      - name: Run pycodestyle
+        run: |
+          pycodestyle $(find ./lambdas/ -name '*.py')
+      - name: Run isort
+        run: |
+          isort --check --diff ./lambdas/
+
+  test-lambda:
+    strategy:
+      fail-fast: false
+      matrix:
+        path:
+          - access_counts
+          - es/indexer
+          - molecule
+          - pkgevents
+          - pkgpush
+          - pkgselect
+          - preview
+          - s3hash
+          - s3select
+          - shared
+          - status_reports
+          - tabular_preview
+          - transcode
+        python-version: ['3.8']
+        include:
+          - path: thumbnail
+            python-version: '3.9'
+    runs-on: ubuntu-latest
+    env:
+      QUILT_DISABLE_USAGE_METRICS: true
+      PYTHONPATH: deps
+      JUPYTER_PATH: deps/share/jupyter  # Jupyter is not smart enough to handle custom Python paths
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          # Due to behavior change in pip>=23.1 installing tifffile==0.15.1
+          # from thumbnail lambda fails whithout installed wheel.
+          # See https://github.com/pypa/pip/issues/8559.
+          python -m pip install wheel
+          if [ ${{ matrix.path }} == "shared" ]
+            python -m pip install -e lambdas/shared[tests]
+          then
+            python -m pip install -e lambdas/shared
+            python -m pip install -e lambdas/${{ matrix.path }}
+          fi
+          python -m pip install -r lambdas/${{ matrix.path }}/test-requirements.txt
+
+          # Try to simulate the lambda .zip file:
+          # - Use --no-deps to ensure that second-order dependencies are included in the requirements file
+          # - Remove "tests" directories
+          # - Run "strip" on shared libraries
+          python -m pip install -t deps --no-deps -r lambdas/${{ matrix.path }}/requirements.txt
+          find deps -name tests -type d -exec rm -r \{} \+
+          find deps \( -name '*.so.*' -o -name '*.so' \) -type f -exec strip \{} \+
+      - name: Pytest
+        run: |
+          pytest --cov=lambdas lambdas/${{ matrix.path }}
+      - uses: codecov/codecov-action@v3
+        env:
+          LAMBDA: ${{ matrix.path }}
+        with:
+          flags: lambda
+          name: ${{ github.job }}
+          env_vars: LAMBDA