diff --git a/.github/codeql.yml b/.github/codeql.yml new file mode 100644 index 0000000..0f2f982 --- /dev/null +++ b/.github/codeql.yml @@ -0,0 +1,52 @@ +name: "Code scanning - action" + +on: + push: + pull_request: + schedule: + - cron: '0 19 * * 0' + +jobs: + CodeQL-Build: + + # CodeQL runs on ubuntu-latest and windows-latest + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + # Override language selection by uncommenting this and choosing your languages + # with: + # languages: go, javascript, csharp, python, cpp, java + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..fe38181 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: +- package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + time: '11:00' + open-pull-requests-limit: 10 +- package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + time: '11:00' + open-pull-requests-limit: 10 diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml index 0fa1c46..53b0d18 100644 --- a/.github/workflows/pr_build.yml +++ b/.github/workflows/pr_build.yml @@ -37,7 +37,7 @@ jobs: name: '${{ github.event.repository.name }}' tags: pr-${{ github.event.number }},latest-rc secrets: inherit - trivy-scans: - if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false - uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main - secrets: inherit +# trivy-scans: +# if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false +# uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main +# secrets: inherit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6a4109b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,88 @@ +name: KBase Catalog test + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + # run workflow when merging to main or develop + branches: + - main + - master + - develop + +jobs: + catalog_tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - python-version: '3.7' + mongo-version: '3.6' + # - python-version: '3.7' + # mongo-version: '7.0.4' + services: + mongo: + image: mongo:${{matrix.mongo-version}} + ports: + - 27017:27017 + options: --name mongo${{matrix.mongo-version}} + + steps: + - name: Repo checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python-version}} + + - name: Install dependencies and set up test config + shell: bash + env: + KBASE_CI_TOKEN: ${{ secrets.KBASE_CI_TOKEN }} + ADMIN_USER: ${{ secrets.KBASE_BOT_USER_CI }} + + run: | + + # test mongo connection + curl http://localhost:27017 + returncode=$? + if [ $returncode != 0 ]; then exit $returncode; fi + + # set HOMEDIR + export HOMEDIR=`pwd` + + # move to parent dir to install binaries etc + cd .. + + # setup kb-sdk + mkdir -p $(pwd)/bin + docker run ghcr.io/kbase/kb_sdk_patch-develop:br-0.0.4 genscript > $(pwd)/bin/kb-sdk + chmod 755 $(pwd)/bin/kb-sdk + export PATH=$(pwd)/bin:$PATH + + # install catalog dependencies + cd $HOMEDIR + pip install -r requirements.txt + + # setup test config + cp -n test/test.cfg.example test/test.cfg + sed -i "s#^nms-admin-token.*#nms-admin-token=$KBASE_CI_TOKEN#" test/test.cfg + sed -i "s#^method-spec-admin-users.*#method-spec-admin-users=$ADMIN_USER#" test/test.cfg + + - name: Run tests + shell: bash + run: make test + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index beae4e4..0000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -sudo: required -language: python -python: - - '3.6' -services: - - docker - -before_install: - #- docker run -d -p 5000:5000 --name registry registry:2 - - docker run -p 27017:27017 --name mongo3.6 -d mongo:3.6 - - sleep 3 - - mongo --eval "printjson(db.serverStatus())" - - sudo apt install ant - - mkdir $(pwd)/bin - - docker run kbase/kb-sdk genscript > $(pwd)/bin/kb-sdk - - chmod 755 $(pwd)/bin/kb-sdk - -install: - - pip install -r requirements.txt - - pip install python-coveralls - - pip install coverage - - cd .. - - git clone https://github.com/kbase/jars - - docker images - - export PATH=$(pwd)/bin:$PATH - - cd catalog - -script: - - cp -n test/test.cfg.example test/test.cfg - - sed -i 's/^nms-admin-toke.*/nms-admin-token = '$NMS_ADMIN_TOKEN'/' test/test.cfg - - make test - -after_success: - - mv test/.coverage . - - coveralls - -env: - global: - - secure: "o0xBDxPBVpqmezRtuDf4gdFYUIlFReH3kV1BOx75taCqzMvsvNnIDHzLITFTvGhNBzOT/hG/eEVDiKx/bnFoq0HJWsKYeKahOsiMlFavIR5jTIdMLq2zbZDkxV/+e0HoRRArWj53NV9wx0pWTv3DYxqbzPR1uu5letxdWnY3DQXggspcnJuAVYGn6LtgNM7MV+h3jdHe/E61LkwMff8KwpoBF+64AKbsZv1ddcFo0gbo9Z9CjLeBq4le2TwziLeuUYV885mPsRCvhToRMzpeVlbiNkzf7w5oh5g3FFsPOOYFeu22a+RrmKoM0hv9/pTTYG0G8/ylzOMIKWJvVrOJYptRvyQKmpmdcN7/CPYIQkf6ZBp0rP6G30Q2xt49+ST40ko1L0jhL42FfwXSmuNjklRMZAjpMNRZM2fIazEhDw3ND9l4T6HZnIwr/SQvKCs4rYBjqmSLo/dMISiFDLjWpMor5ew9fpZSLwJcPglGT0Anh4OIZUY2JGlO4TuOkq3m5kvXjjKLBc3Domd10c+DLfkldwwt6uqShsDECqFwSMf/liPEtLfi+03djrWvP26QeiEKL93ujAMS4oXAYxdquOy2Ib9ihd3XWTJzYc4s1H7furYqAbQDAhU0Dx3nvcyEsu5djdRAFe0eKUdjdAqsMAxe84h/EHZHrlE+KOOZo8s=" - - MONGODB_VERSION=2.7 diff --git a/lib/biokbase/catalog/registrar.py b/lib/biokbase/catalog/registrar.py index b10d158..d8226cb 100644 --- a/lib/biokbase/catalog/registrar.py +++ b/lib/biokbase/catalog/registrar.py @@ -252,7 +252,7 @@ def sanity_checks_and_parse(self, basedir, git_commit_hash): with codecs.open(os.path.join(basedir, yaml_filename), 'r', "utf-8", errors='ignore') as kb_yaml_file: kb_yaml_string = kb_yaml_file.read() - self.kb_yaml = yaml.load(kb_yaml_string) + self.kb_yaml = yaml.safe_load(kb_yaml_string) self.log('=====kbase.yaml parse:') self.log(pprint.pformat(self.kb_yaml)) self.log('=====end kbase.yaml') diff --git a/requirements.txt b/requirements.txt index e319061..b94e0b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pymongo==3.7 +pymongo==3.10 docker>=3.5 gitpython pyyaml diff --git a/test/core_registration_test.py b/test/core_registration_test.py index e222bb8..c1e63c7 100644 --- a/test/core_registration_test.py +++ b/test/core_registration_test.py @@ -37,6 +37,7 @@ def test_full_module_lifecycle(self): self.assertEqual(state['registration'], 'complete') # (3) check the log + sleep(3) # sleep to make sure the catalog db gets the final log messages parsed_log = self.catalog.get_parsed_build_log(self.cUtil.anonymous_ctx(), {'registration_id': registration_id})[0] self.assertEqual(parsed_log['registration'], 'complete') diff --git a/test/local_function_module_test.py b/test/local_function_module_test.py index bf510e5..0b17303 100644 --- a/test/local_function_module_test.py +++ b/test/local_function_module_test.py @@ -16,7 +16,7 @@ def test_local_function_module(self): # assume test user is already approved as a developer # (1) register the test repo giturl = self.cUtil.get_test_repo_1() - githash = 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970' # branch local_method_module + githash = 'a8915afe6811de9199897d710348befad8f6f7ab' # branch local_method_module registration_id = self.catalog.register_repo(self.cUtil.user_ctx(), {'git_url': giturl, 'git_commit_hash': githash})[0] @@ -59,7 +59,7 @@ def test_local_function_module(self): self.assertEqual(len(specs), 1) info = specs[0]['info'] self.assertEqual(info['function_id'], 'powerpoint_to_genome') - self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970') + self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab') self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter') self.assertEqual(info['name'], 'Powerpoint to Genome') self.assertEqual(info['release_tag'], ['dev']) @@ -122,7 +122,7 @@ def test_local_function_module(self): self.assertEqual(len(specs), 1) info = specs[0]['info'] self.assertEqual(info['function_id'], 'powerpoint_to_genome') - self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970') + self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab') self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter') self.assertEqual(info['name'], 'Powerpoint to Genome') self.assertEqual(info['release_tag'], ['beta', 'dev']) @@ -179,12 +179,12 @@ def test_local_function_module(self): # make sure we can fetch it by commit hash specs = self.catalog.get_local_function_details(self.cUtil.user_ctx(), {'functions': [ {'module_name': 'GenomeTopowerpointConverter', 'function_id': 'powerpoint_to_genome', - 'git_commit_hash': 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970'}]})[0] + 'git_commit_hash': 'a8915afe6811de9199897d710348befad8f6f7ab'}]})[0] self.assertEqual(len(specs), 1) info = specs[0]['info'] self.assertEqual(info['function_id'], 'powerpoint_to_genome') - self.assertEqual(info['git_commit_hash'], 'a01e1a20b9c504a0136c75323b00b1cd4c7f7970') + self.assertEqual(info['git_commit_hash'], 'a8915afe6811de9199897d710348befad8f6f7ab') self.assertEqual(info['module_name'], 'GenomeToPowerpointConverter') self.assertEqual(info['name'], 'Powerpoint to Genome') self.assertEqual(info['release_tag'], ['release', 'beta', 'dev'])