Merge to main to release v5.3.0

.dockerignore

@@ -1,4 +1,4 @@
-Dockerfile
+Dockerfile*
 .husky
 .github
 .vscode
@@ -8,3 +8,9 @@ node_modules/
 **/__pycache__
 karma-result.json
 install.log
+**/.pytest_cache
+.ruff*
+.eslint*
+.prettier*
+.stylelintrc*
+.coverage*

.github/codeql.yml

@@ -3,8 +3,6 @@ name: "Code scanning - action"
 on:
   push:
   pull_request:
-  schedule:
-    - cron: '0 19 * * 0'
 
 jobs:
   CodeQL-Build:
@@ -14,7 +12,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
@@ -24,18 +22,18 @@ jobs:
     # the head of the pull request instead of the merge commit.
     - run: git checkout HEAD^2
       if: ${{ github.event_name == 'pull_request' }}
-      
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v3
       # Override language selection by uncommenting this and choosing your languages
       # with:
       #   languages: go, javascript, csharp, python, cpp, java
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -49,4 +47,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v3

.github/dependabot.yml

@@ -5,16 +5,33 @@ updates:
   schedule:
     interval: monthly
     time: '11:00'
+  groups:
+    python-dependencies:
+      patterns:
+        - "*"
   open-pull-requests-limit: 20
+  ignore:
+    - dependency-name: "notebook"
+      versions: '>= 7.0.0'
 - package-ecosystem: npm
   directory: "/"
   schedule:
     interval: monthly
     time: '11:00'
+  groups:
+    npm-dependencies:
+      patterns:
+        "*"
   open-pull-requests-limit: 20
 - package-ecosystem: docker
   directory: "/"
   schedule:
     interval: monthly
     time: '11:00'
   open-pull-requests-limit: 20
+- package-ecosystem: github-actions
+  directory: "/.github"
+  schedule:
+    interval: monthly
+    time: '11:00'
+  open-pull-requests-limit: 20

.github/pull_request_template.md

@@ -24,7 +24,7 @@ Related Jira ticket: https://kbase-jira.atlassian.net/browse/DATAUP-X
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] New and existing unit tests pass locally with my changes
 - [ ] (JavaScript) I have run Prettier and ESLint on changed code manually or with a git precommit hook
-- [ ] (Python) I have run Black and Flake8 on changed Python code manually or with a git precommit hook
+- [ ] (Python) I have run Ruff `format` and `check` on changed Python code manually or with a git precommit hook
 - [ ] Any dependent changes have been merged and published in downstream modules
 
 # Updating Version and Release Notes (if applicable)

.github/workflows/build_and_push_image.yml

@@ -18,7 +18,7 @@ jobs:
       name: Check out GitHub Repo
       with:
         ref: "${{ github.event.pull_request.head.sha }}"
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     -
       name: Set up environment
       run: |
@@ -42,17 +42,17 @@ jobs:
       run: echo "date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
     -
       name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
     -
       name: Log in to the Container registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ secrets.GHCR_USERNAME }}
         password: ${{ secrets.GHCR_TOKEN }}
     -
       name: Build narrative image
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile
@@ -67,7 +67,7 @@ jobs:
           TAG='${{ github.ref }}'
     -
       name: Build version image
-      uses: docker/build-push-action@v3
+      uses: docker/build-push-action@v5
       if: ${{ github.base_ref }} == 'main' || ${{ github.base_ref }} == 'develop'
       with:
         context: .

.github/workflows/build_prodrc_pr.yaml

@@ -22,6 +22,7 @@ jobs:
     secrets:
       GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
       GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   run_integration_tests:
     uses: ./.github/workflows/integration_test.yml

.github/workflows/build_test_pr.yaml

@@ -17,7 +17,7 @@ jobs:
       GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
 
   trivy-scans:
-    if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false
+    if: (github.base_ref == 'develop' || github.base_ref == 'main' ) && github.event.pull_request.merged == false
     uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main
     secrets: inherit
 
@@ -27,6 +27,7 @@ jobs:
     secrets:
       GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
       GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   run_integration_tests:
     uses: ./.github/workflows/integration_test.yml

.github/workflows/integration_test.yml

@@ -16,14 +16,14 @@ jobs:
     steps:
     -
       name: Log in to the Container registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ secrets.GHCR_USERNAME }}
         password: ${{ secrets.GHCR_TOKEN }}
     -
       name: Repo checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     -
       name: Set up environment
       run: |
@@ -47,10 +47,10 @@ jobs:
       shell: bash -l {0}
       run: docker pull ${{ env.APP_IMAGE_TAG }}
     -
-      name: Use Node JS 16
-      uses: actions/setup-node@v3
+      name: Use Node JS 20
+      uses: actions/setup-node@v4
       with:
-        node-version: 16
+        node-version: 20
     -
       name: Install JS dependencies
       run: |

.github/workflows/pr_build.yml

@@ -25,20 +25,20 @@ jobs:
       tags: pr-${{ github.event.number }},latest
     secrets: inherit
   build-main-open:
-    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false
+    if: github.base_ref == 'main' && github.event.pull_request.merged == false
     uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
     with:
       name: '${{ github.event.repository.name }}'
       tags: pr-${{ github.event.number }}
     secrets: inherit
   build-main-merge:
-    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true
+    if: github.base_ref == 'main' && github.event.pull_request.merged == true
     uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
     with:
       name: '${{ github.event.repository.name }}'
       tags: pr-${{ github.event.number }},latest-rc
     secrets: inherit
   trivy-scans:
-    if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false
+    if: (github.base_ref == 'develop' || github.base_ref == 'main') && github.event.pull_request.merged == false
     uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main
     secrets: inherit

.github/workflows/release-main.yml

@@ -2,9 +2,6 @@
 name: Release - Build & Push Image
 on:
   release:
-    branches:
-      - main
-      - master
     types: [ published ]
 jobs:
   check-source-branch:

.github/workflows/tag_latest_image.yaml

@@ -16,10 +16,10 @@ jobs:
     - name: Check out GitHub Repo
       with:
         ref: "${{ github.event.pull_request.head.sha }}"
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     -
       name: Log in to the Container registry
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ secrets.GHCR_USERNAME }}

.github/workflows/unit_test.yml

@@ -7,21 +7,23 @@ on:
         required: true
       GHCR_TOKEN:
         required: true
+      CODECOV_TOKEN:
+        required: true
 
 jobs:
   run_unit_tests:
     runs-on: ubuntu-latest
     steps:
     -
       name: Log in to the Container registry
-      uses: docker/login-action@v1
+      uses: docker/login-action@v3
       with:
         registry: ghcr.io
         username: ${{ secrets.GHCR_USERNAME }}
         password: ${{ secrets.GHCR_TOKEN }}
     -
       name: Repo checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     -
       name: Set up environment
       run: |
@@ -52,13 +54,13 @@ jobs:
         mkdir -p ${{ github.workspace }}/output
         sudo chown nobody ${{ github.workspace }}/output
         sudo chmod -R 0777 ${{ github.workspace }}/output
-        docker run -v "${{ github.workspace }}/output:/tmp/output" --user nobody ${{ env.APP_IMAGE_TAG }} /bin/bash scripts/narrative_backend_tests.sh
+        docker run -v "${{ github.workspace }}/output:/tmp/output" --user nobody ${{ env.APP_IMAGE_TAG }} /bin/bash scripts/run_backend_tests.sh
       continue-on-error: true
     -
-      name: Use Node JS 16
-      uses: actions/setup-node@v2
+      name: Use Node JS 20
+      uses: actions/setup-node@v4
       with:
-        node-version: 16
+        node-version: 20
     -
       name: Install JS dependencies
       run: |
@@ -74,9 +76,10 @@ jobs:
     -
       name: Send to Codecov
       id: send_to_codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
+        token: ${{ secrets.CODECOV_TOKEN }}
         files: ./output/coverage.xml,./js-coverage/lcov/lcov.info
         fail_ci_if_error: true
     -