Develop->Main

.github/codeql.yml

@@ -0,0 +1,52 @@
+name: "Code scanning - action"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 19 * * 0'
+
+jobs:
+  CodeQL-Build:
+
+    # CodeQL runs on ubuntu-latest and windows-latest
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      # Override language selection by uncommenting this and choosing your languages
+      # with:
+      #   languages: go, javascript, csharp, python, cpp, java
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+- package-ecosystem: docker
+  directory: "/"
+  schedule:
+    interval: weekly
+    time: '11:00'
+  open-pull-requests-limit: 10
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: weekly
+    time: '11:00'
+  open-pull-requests-limit: 10

.github/workflows/pr_build.yml

@@ -37,7 +37,7 @@ jobs:
       name: '${{ github.event.repository.name }}'
       tags: pr-${{ github.event.number }},latest-rc
     secrets: inherit
-  trivy-scans:
-    if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false
-    uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main
-    secrets: inherit
+#  trivy-scans:
+#    if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false
+#    uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main
+#    secrets: inherit

.github/workflows/test.yml

@@ -0,0 +1,88 @@
+name: KBase Catalog test
+
+on:
+  pull_request:
+    types:
+    - opened
+    - reopened
+    - synchronize
+    - ready_for_review
+  push:
+    # run workflow when merging to main or develop
+    branches:
+      - main
+      - master
+      - develop
+
+jobs:
+  catalog_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - python-version: '3.7'
+            mongo-version: '3.6'
+          # - python-version: '3.7'
+          #   mongo-version: '7.0.4'
+    services:
+      mongo:
+        image: mongo:${{matrix.mongo-version}}
+        ports:
+            - 27017:27017
+        options: --name mongo${{matrix.mongo-version}}
+
+    steps:
+      - name: Repo checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.python-version}}
+
+      - name: Install dependencies and set up test config
+        shell: bash
+        env:
+          KBASE_CI_TOKEN: ${{ secrets.KBASE_CI_TOKEN }}
+          ADMIN_USER: ${{ secrets.KBASE_BOT_USER_CI }}
+
+        run: |
+
+          # test mongo connection
+          curl http://localhost:27017
+          returncode=$?
+          if [ $returncode != 0 ]; then exit $returncode; fi
+
+          # set HOMEDIR
+          export HOMEDIR=`pwd`
+
+          # move to parent dir to install binaries etc
+          cd ..
+
+          # setup kb-sdk
+          mkdir -p $(pwd)/bin
+          docker run ghcr.io/kbase/kb_sdk_patch-develop:br-0.0.4 genscript > $(pwd)/bin/kb-sdk
+          chmod 755 $(pwd)/bin/kb-sdk
+          export PATH=$(pwd)/bin:$PATH
+
+          # install catalog dependencies
+          cd $HOMEDIR
+          pip install -r requirements.txt
+
+          # setup test config
+          cp -n test/test.cfg.example test/test.cfg
+          sed -i "s#^nms-admin-token.*#nms-admin-token=$KBASE_CI_TOKEN#" test/test.cfg
+          sed -i "s#^method-spec-admin-users.*#method-spec-admin-users=$ADMIN_USER#" test/test.cfg
+
+      - name: Run tests
+        shell: bash
+        run: make test
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true

lib/biokbase/catalog/registrar.py

@@ -252,7 +252,7 @@ def sanity_checks_and_parse(self, basedir, git_commit_hash):
         with codecs.open(os.path.join(basedir, yaml_filename), 'r', "utf-8",
                          errors='ignore') as kb_yaml_file:
             kb_yaml_string = kb_yaml_file.read()
-        self.kb_yaml = yaml.load(kb_yaml_string)
+        self.kb_yaml = yaml.safe_load(kb_yaml_string)
         self.log('=====kbase.yaml parse:')
         self.log(pprint.pformat(self.kb_yaml))
         self.log('=====end kbase.yaml')

requirements.txt

@@ -1,4 +1,4 @@
-pymongo==3.7
+pymongo==3.10
 docker>=3.5
 gitpython
 pyyaml

test/core_registration_test.py

@@ -37,6 +37,7 @@ def test_full_module_lifecycle(self):
         self.assertEqual(state['registration'], 'complete')
 
         # (3) check the log
+        sleep(3)  # sleep to make sure the catalog db gets the final log messages
         parsed_log = self.catalog.get_parsed_build_log(self.cUtil.anonymous_ctx(),
                                                        {'registration_id': registration_id})[0]
         self.assertEqual(parsed_log['registration'], 'complete')

test/local_function_module_test.py

@@ -16,7 +16,7 @@ def test_local_function_module(self):
         # assume test user is already approved as a developer
         # (1) register the test repo
         giturl = self.cUtil.get_test_repo_1()
-        githash = 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970'  # branch local_method_module
+        githash = 'a8915afe6811de9199897d710348befad8f6f7ab'  # branch local_method_module
         registration_id = self.catalog.register_repo(self.cUtil.user_ctx(),
                                                      {'git_url': giturl,
                                                       'git_commit_hash': githash})[0]
@@ -59,7 +59,7 @@ def test_local_function_module(self):
         self.assertEqual(len(specs), 1)
         info = specs[0]['info']
         self.assertEqual(info['function_id'], 'powerpoint_to_genome')
-        self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970')
+        self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab')
         self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter')
         self.assertEqual(info['name'], 'Powerpoint to Genome')
         self.assertEqual(info['release_tag'], ['dev'])
@@ -122,7 +122,7 @@ def test_local_function_module(self):
         self.assertEqual(len(specs), 1)
         info = specs[0]['info']
         self.assertEqual(info['function_id'], 'powerpoint_to_genome')
-        self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970')
+        self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab')
         self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter')
         self.assertEqual(info['name'], 'Powerpoint to Genome')
         self.assertEqual(info['release_tag'], ['beta', 'dev'])
@@ -179,12 +179,12 @@ def test_local_function_module(self):
         # make sure we can fetch it by commit hash
         specs = self.catalog.get_local_function_details(self.cUtil.user_ctx(), {'functions': [
             {'module_name': 'GenomeTopowerpointConverter', 'function_id': 'powerpoint_to_genome',
-             'git_commit_hash': 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970'}]})[0]
+             'git_commit_hash': 'a8915afe6811de9199897d710348befad8f6f7ab'}]})[0]
 
         self.assertEqual(len(specs), 1)
         info = specs[0]['info']
         self.assertEqual(info['function_id'], 'powerpoint_to_genome')
-        self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970')
+        self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab')
         self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter')
         self.assertEqual(info['name'], 'Powerpoint to Genome')
         self.assertEqual(info['release_tag'], ['release', 'beta', 'dev'])