Automate GH Pages build

.github/static/get_dic2owl_deps.py

@@ -4,8 +4,8 @@
 Retrieve dependencies from a `requirements.txt`-style file.
 """
 import argparse
-from pathlib import Path
 import re
+from pathlib import Path
 from typing import Set
 
 
@@ -24,22 +24,17 @@ def main(argv_input: list = None) -> Set[str]:
     )
     args = parser.parse_args(argv_input)
 
-    requirements_regex = re.compile(
-        r"^(?P<name>[A-Za-z0-9_-]+)(~=|>=|==).*\n$"
-    )
+    requirements_regex = re.compile(r"^(?P<name>[A-Za-z0-9_-]+)(~=|>=|==).*\n$")
     dependencies = set()
     for file in args.requirements_files:
         if not file.exists():
             raise FileNotFoundError(f"Could not find {file} !")
-        with open(file.resolve(), "r") as handle:
+        with open(file.resolve(), "r", encoding="utf8") as handle:
             for line in handle.readlines():
                 match = requirements_regex.fullmatch(line)
                 if match is None:
                     raise ValueError(
-                        (
-                            "Couldn't determine package name from line:\n\n  "
-                            f"{line}"
-                        )
+                        ("Couldn't determine package name from line:\n\n  " f"{line}")
                     )
                 dependencies.add(match.group("name"))
     return dependencies

.github/workflows/ci.yml

@@ -4,22 +4,32 @@ on: [push]
 
 jobs:
   pre-commit-basic:
-    runs-on: ubuntu-latest
+    name: External
+    uses: SINTEF/ci-cd/.github/workflows/[email protected]
+    with:
+      # general setup
+      python_version: "3.7"
+      install_extras: "/dic2owl[dev] && pip install -U --upgrade-strategy=eager -r requirements_docs.txt"
+      run_pylint: false
+      run_safety: false
+      run_build_package: false
 
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      # pre-commit
+      run_pre-commit: true
+      skip_pre-commit_hooks: "pylint,mypy,black,bandit,requirements-txt-fixer"
 
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.7
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -U setuptools wheel
-        pip install pre-commit
-
-    - name: Run pre-commit
-      run: SKIP=pylint,mypy,black,bandit,requirements-txt-fixer pre-commit run --all-files
+      # build docs
+      run_build_docs: true
+      package_dir: "dic2owl/dic2owl"
+      exclude_dirs: |
+        __pycache__
+        _mkdocs
+      exclude_files: |
+        __init__.py
+        _utils.py
+      landing_page_replacements: |
+        (LICENSE),(LICENSE.md)
+        (dic2owl/README.md),(dic2owl/index.md)
+        (ontology),(https://github.com/emmo-repo/CIF-ontology/tree/main/ontology)
+        (ontology/catalog-v001.xml),(https://github.com/emmo-repo/CIF-ontology/blob/main/ontology/catalog-v001.xml)
+      relative: true

.github/workflows/ci_automerge_dependabot.yml

@@ -5,26 +5,9 @@ on:
     branches: [ci/dependabot-updates]
 
 jobs:
-  update-dependabot-branch:
-    name: Update permanent dependabot branch
+  automerge-dependency-prs:
+    name: External
+    uses: SINTEF/ci-cd/.github/workflows/[email protected]
     if: github.repository_owner == 'emmo-repo' && startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]'
-    runs-on: ubuntu-latest
-
-    env:
-      DEPENDABOT_BRANCH: ci/dependabot-updates
-      GIT_USER_NAME: CIF-ontology Team
-      GIT_USER_EMAIL: "[email protected]"
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        ref: ${{ github.event.pull_request.head.ref }}
-
-    - name: Activate auto-merge
-      run: |
-        PR_ID="$(gh api graphql -F owner='{owner}' -F name='{repo}' -f query='query($owner: String!, $name: String!) {repository(owner: $owner, name: $name) {pullRequest(number: ${{ github.event.pull_request.number }}) {id}}}' --jq '.data.repository.pullRequest.id')"
-        gh api graphql -f pr_id="$PR_ID" -f query='mutation($pr_id: ID!) {enablePullRequestAutoMerge(input:{mergeMethod:SQUASH,pullRequestId:$pr_id }) {pullRequest {number}}}'
-      env:
-        GITHUB_TOKEN: ${{ secrets.RELEASE_PAT_CWA }}
+    secrets:
+      PAT: ${{ secrets.RELEASE_PAT }}

.github/workflows/ci_cd_updated_main.yml

@@ -4,55 +4,16 @@ on:
   push:
     branches: [main]
 
-env:
-  DEPENDABOT_BRANCH: ci/dependabot-updates
-  GIT_USER_NAME: CIF-ontology Team
-  GIT_USER_EMAIL: "[email protected]"
-  DEFAULT_REPO_BRANCH: main
-
 jobs:
   update-dependabot-branch:
-    name: Update permanent dependabot branch
+    name: External
+    uses: SINTEF/ci-cd/.github/workflows/[email protected]
     if: github.repository_owner == 'emmo-repo'
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-      with:
-        ref: ${{ env.DEPENDABOT_BRANCH }}
-        fetch-depth: 0
-
-    - name: Set up git config
-      run: |
-        git config --global user.name "${{ env.GIT_USER_NAME }}"
-        git config --global user.email "${{ env.GIT_USER_EMAIL }}"
-
-    - name: Update '${{ env.DEPENDABOT_BRANCH }}'
-      run: |
-        git fetch origin
-
-        LATEST_PR_BODY="$(gh api /repos/${{ github.repository}}/pulls -X GET -f state=closed -f per_page=1 -f sort=updated -f direction=desc --jq '.[].body')"
-        cat .github/static/single_dependency_pr_body.txt | head -8 > .tmp_file.txt
-        if [ -z "$(printf '%s\n' "${LATEST_PR_BODY}" | head -8 | diff - .tmp_file.txt --strip-trailing-cr)" ]; then
-          # The dependency branch has just been merged into `${{ env.DEFAULT_REPO_BRANCH }}`
-          # The dependency branch should be reset to `${{ env.DEFAULT_REPO_BRANCH }}`
-          echo "The dependencies have just been updated! Reset to ${{ env.DEFAULT_REPO_BRANCH }}."
-          git reset --hard origin/${{ env.DEFAULT_REPO_BRANCH }}
-          echo "FORCE_PUSH=yes" >> $GITHUB_ENV
-        else
-          # Normal procedure: Merge `${{ env.DEFAULT_REPO_BRANCH }}` into `${{ env.DEPENDABOT_BRANCH }}`
-          echo "Merge new updates to ${{ env.DEFAULT_REPO_BRANCH }} into ${{ env.DEPENDABOT_BRANCH }}"
-          git merge -m "Keep '${{ env.DEPENDABOT_BRANCH }}' up-to-date with '${{ env.DEFAULT_REPO_BRANCH }}'" origin/${{ env.DEFAULT_REPO_BRANCH }}
-          echo "FORCE_PUSH=no" >> $GITHUB_ENV
-        fi
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Push to '${{ env.DEPENDABOT_BRANCH }}'
-      uses: CasperWA/push-protected@v2
-      with:
-        token: ${{ secrets.RELEASE_PAT_CWA }}
-        branch: ${{ env.DEPENDABOT_BRANCH }}
-        sleep: 15
-        force: ${{ env.FORCE_PUSH }}
+    with:
+      git_username: "TEAM 4.0[bot]"
+      git_email: "[email protected]"
+      permanent_dependencies_branch: ci/dependabot-updates
+      default_repo_branch: main
+      relative: true
+    secrets:
+      PAT: ${{ secrets.RELEASE_PAT }}

.github/workflows/ci_dependabot.yml

@@ -9,90 +9,19 @@ on:
 
 jobs:
   create-collected-pr:
-    name: Single dependabot PR
+    name: External
+    uses: SINTEF/ci-cd/.github/workflows/[email protected]
     if: github.repository_owner == 'emmo-repo'
-    runs-on: ubuntu-latest
-    env:
-      DEPENDABOT_BRANCH: ci/dependabot-updates
-      GIT_USER_NAME: CIF-ontology Team
-      GIT_USER_EMAIL: "[email protected]"
-      DEFAULT_REPO_BRANCH: main
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-      with:
-        ref: ${{ env.DEFAULT_REPO_BRANCH }}
-
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.7
-
-    - name: Install `pre-commit` and dependencies
-      run: |
-        python -m pip install -U pip
-        pip install -U setuptools wheel
-
-        while IFS="" read -r line || [ -n "${line}" ]; do
-          if [[ "${line}" =~ ^pre-commit.*$ ]]; then
-            pre_commit="${line}"
-          fi
-        done < dic2owl/requirements_dev.txt
-
-        pip install ${pre_commit}
-
-    - name: Set up git user info
-      run: |
-        git config --global user.name "${{ env.GIT_USER_NAME }}"
-        git config --global user.email "${{ env.GIT_USER_EMAIL }}"
-
-    - name: Reset to '${{ env.DEPENDABOT_BRANCH }}'
-      run: |
-        git fetch origin ${{ env.DEPENDABOT_BRANCH }}:${{ env.DEPENDABOT_BRANCH }}
-        git reset --hard ${{ env.DEPENDABOT_BRANCH }}
-
-    - name: Auto-update `pre-commit` hooks
-      run: |
-        pre-commit autoupdate
-
-        if [ -n "$(git status --porcelain .pre-commit-config.yaml)" ]; then
-          # Set environment variable notifying next steps that the hooks changed
-          echo "Pre-commit hooks have been updated !"
-          echo "UPDATED_PRE_COMMIT_HOOKS=true" >> $GITHUB_ENV
-        else
-          echo "No pre-commit hooks have been updated."
-          echo "UPDATED_PRE_COMMIT_HOOKS=false" >> $GITHUB_ENV
-        fi
-
-    - name: Possibly run `pre-commit` with updated hooks
-      if: env.UPDATED_PRE_COMMIT_HOOKS == 'true'
-      continue-on-error: true  # Still create the PR if this step fails
-      run: SKIP=pylint pre-commit run --all-files
-
-    - name: Possibly commit changes and updates
-      if: env.UPDATED_PRE_COMMIT_HOOKS == 'true'
-      run: git commit -am "Update \`pre-commit\` hooks"
-
-    - name: Fetch PR body
-      id: pr_body
-      uses: chuhlomin/[email protected]
-      with:
-        template: .github/static/single_dependency_pr_body.txt
-
-    - name: Create PR
-      id: cpr
-      uses: peter-evans/create-pull-request@v4
-      with:
-        token: ${{ secrets.RELEASE_PAT_CWA }}
-        commit-message: New @dependabot-fueled updates
-        committer: "${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>"
-        author: "${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>"
-        branch: ci/update-dependencies
-        delete-branch: true
-        title: "[Auto-generated] Update dependencies"
-        body: ${{ steps.pr_body.outputs.result }}
-        labels: dependencies,github_actions
-
-    - name: Information
-      run: 'echo "${{ steps.cpr.outputs.pull-request-operation }} PR #${{ steps.cpr.outputs.pull-request-number }}: ${{ steps.cpr.outputs.pull-request-url }}"'
+    with:
+      git_username: "TEAM 4.0[bot]"
+      git_email: "[email protected]"
+      permanent_dependencies_branch: ci/dependabot-updates
+      default_repo_branch: main
+      pr_labels: dependencies,github_actions
+      extra_to_dos: "- [ ] Make sure that the PR is **squash** merged, with a sensible commit message."
+      update_pre-commit: true
+      skip_pre-commit_hooks: pylint
+      python_version: "3.7"
+      install_extras: "/dic2owl[dev]"
+    secrets:
+      PAT: ${{ secrets.RELEASE_PAT }}

.github/workflows/dic2owl_ci.yml

@@ -8,63 +8,46 @@ on:
       - '.github/**'
 
 jobs:
-  pre-commit-dic2owl:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.7
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -U setuptools wheel
-        pip install pre-commit
-
-    - name: Run pre-commit
-      run: SKIP=pylint pre-commit run --all-files
-
-  pylint-safety:
-    name: PyLint and safety
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.7
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -U setuptools wheel
-
-        pip install -U -r dic2owl/requirements.txt -r dic2owl/requirements_dev.txt
-        pip install safety
-
-    - name: Run PyLint
-      run: pylint --rcfile=dic2owl/pyproject.toml dic2owl/**/*.py
-
+  dic2owl-basic:
+    name: External
+    uses: SINTEF/ci-cd/.github/workflows/[email protected]
+    with:
+      # general setup
+      python_version: "3.7"
+      install_extras: "/dic2owl[dev]"
+      run_build_docs: false
+
+      # pre-commit
+      run_pre-commit: true
+      skip_pre-commit_hooks: pylint
+
+      # pylint
+      run_pylint: true
+      pylint_targets: "dic2owl/**/*.py"
+      pylint_options: "--rcfile=dic2owl/pyproject.toml"
+
+      # safety
+      run_safety: true
       # Ignore ID 44715 for now.
       # See this NumPy issue for more information: https://github.com/numpy/numpy/issues/19038
       # Remove ignoring 48547 as soon as RDFLib/rdflib#1844 has been fixed and the fix
       # has been released.
-    - name: Run safety
-      run: pip freeze | safety check --stdin --ignore 44715 --ignore 44716 --ignore 44717 --ignore 48547
+      safety_options: |
+        --ignore=44715
+        --ignore=44716
+        --ignore=44717
+        --ignore=48547
+
+      # build dist
+      run_build_package: true
+      build_libs: build
+      build_cmd: python -m build -o dist dic2owl
 
   dic2owl-test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: ["3.7", "3.8", "3.9"]
 
     steps:
       - uses: actions/checkout@v3
@@ -78,7 +61,7 @@ jobs:
         run: |
           python -m pip install -U pip
           pip install -U setuptools wheel
-          pip install -e dic2owl[dev]
+          pip install -e ./dic2owl[dev]
 
       - name: Run unit tests with PyTest
         run: pytest -c dic2owl/pyproject.toml