From 8fed151e18da5ac1140d7ba5ca53162ac6c5e1b8 Mon Sep 17 00:00:00 2001 From: Gavin Date: Thu, 8 Aug 2024 12:55:23 -0700 Subject: [PATCH 1/3] Add GHA and testing infrastructure Also license, update readme with basics, etc. --- .github/dependabot.yml | 23 +++ .github/workflows/codeql.yml | 38 ++++ .github/workflows/manual-build.yml | 12 ++ .github/workflows/pr_build.yml | 43 +++++ .github/workflows/release-main.yml | 25 +++ .github/workflows/test.yml | 49 +++++ LICENSE.md | 22 +++ Pipfile | 16 ++ Pipfile.lock | 279 +++++++++++++++++++++++++++++ README.md | 45 +++++ RELEASE_NOTES.md | 5 + cdmtaskservice/dockerimages.py | 30 ++++ test/dockerimages_test.py | 9 + 13 files changed, 596 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/manual-build.yml create mode 100644 .github/workflows/pr_build.yml create mode 100644 .github/workflows/release-main.yml create mode 100644 .github/workflows/test.yml create mode 100644 LICENSE.md create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 RELEASE_NOTES.md create mode 100644 cdmtaskservice/dockerimages.py create mode 100644 test/dockerimages_test.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1cadfe8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,23 @@ +version: 2 +updates: + + # Docker + - package-ecosystem: docker + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 25 + + # Python + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "monthly" + open-pull-requests-limit: 25 + + # GitHub Actions + - package-ecosystem: "github-actions" + directory: ".github/workflows" + schedule: + interval: "monthly" + open-pull-requests-limit: 25 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..54e37ee --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,38 @@ +name: "CodeQL" + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + # run workflow when merging to main or develop + branches: + - main + - master + - develop + +jobs: + CodeQL-Build: + # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest + runs-on: ubuntu-latest + + permissions: + # required for all workflows + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + # Override language selection by uncommenting this and choosing your languages + with: + languages: python + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml new file mode 100644 index 0000000..11607c8 --- /dev/null +++ b/.github/workflows/manual-build.yml @@ -0,0 +1,12 @@ +--- +name: Manual Build & Push +on: + workflow_dispatch: +jobs: + build-push: + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: br-${{ github.ref_name }} + secrets: inherit + diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml new file mode 100644 index 0000000..0fa1c46 --- /dev/null +++ b/.github/workflows/pr_build.yml @@ -0,0 +1,43 @@ +--- +name: Pull Request Build, Tag, & Push +on: + pull_request: + branches: + - develop + - main + - master + types: + - opened + - reopened + - synchronize + - closed +jobs: + build-develop-open: + if: github.base_ref == 'develop' && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build.yml@main + secrets: inherit + build-develop-merge: + if: github.base_ref == 'develop' && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}-develop' + tags: pr-${{ github.event.number }},latest + secrets: inherit + build-main-open: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }} + secrets: inherit + build-main-merge: + if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: pr-${{ github.event.number }},latest-rc + secrets: inherit + trivy-scans: + if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false + uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main + secrets: inherit diff --git a/.github/workflows/release-main.yml b/.github/workflows/release-main.yml new file mode 100644 index 0000000..a254678 --- /dev/null +++ b/.github/workflows/release-main.yml @@ -0,0 +1,25 @@ +--- +name: Release - Build & Push Image +on: + release: + branches: + - main + - master + types: [ published ] +jobs: + check-source-branch: + uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main + with: + build_branch: '${{ github.event.release.target_commitish }}' + validate-release-tag: + needs: check-source-branch + uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main + with: + release_tag: '${{ github.event.release.tag_name }}' + build-push: + needs: validate-release-tag + uses: kbase/.github/.github/workflows/reusable_build-push.yml@main + with: + name: '${{ github.event.repository.name }}' + tags: '${{ github.event.release.tag_name }},latest' + secrets: inherit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6066286 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,49 @@ +name: KBase CDM Task Service tests + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + # run workflow when merging to main or develop + branches: + - main + - master + - develop + +jobs: + + task_service_tests: + runs-on: ubuntu-24.04 + strategy: + matrix: + python-version: ["3.11"] + + steps: + + - name: Repo checkout + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependecies + shell: bash + run: | + pip install pipenv + pipenv sync --system --dev + + - name: Run tests + shell: bash + run: PYTHONPATH=. pytest --cov=src --cov-report=xml test + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..f04bf0f --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2024-present KBase Software + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..a666d7c --- /dev/null +++ b/Pipfile @@ -0,0 +1,16 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] +pytest = "==8.3.2" +coverage = "==7.6.1" +ipython = "==8.26.0" +pytest-asyncio = "==0.23.8" +pytest-cov = "==5.0.0" + +[requires] +python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..53e2a8d --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,279 @@ +{ + "_meta": { + "hash": { + "sha256": "6db16383fe5de9079ef53b33b9ab07ec4f5074d92c5e07d3b767aa460b00d7c5" + }, + "pipfile-spec": 6, + "requires": { + "python_full_version": "3.11.3", + "python_version": "3.11" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": { + "asttokens": { + "hashes": [ + "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24", + "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0" + ], + "version": "==2.4.1" + }, + "coverage": { + "hashes": [ + "sha256:06a737c882bd26d0d6ee7269b20b12f14a8704807a01056c80bb881a4b2ce6ca", + "sha256:07e2ca0ad381b91350c0ed49d52699b625aab2b44b65e1b4e02fa9df0e92ad2d", + "sha256:0c0420b573964c760df9e9e86d1a9a622d0d27f417e1a949a8a66dd7bcee7bc6", + "sha256:0dbde0f4aa9a16fa4d754356a8f2e36296ff4d83994b2c9d8398aa32f222f989", + "sha256:1125ca0e5fd475cbbba3bb67ae20bd2c23a98fac4e32412883f9bcbaa81c314c", + "sha256:13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b", + "sha256:166811d20dfea725e2e4baa71fffd6c968a958577848d2131f39b60043400223", + "sha256:170d444ab405852903b7d04ea9ae9b98f98ab6d7e63e1115e82620807519797f", + "sha256:1f4aa8219db826ce6be7099d559f8ec311549bfc4046f7f9fe9b5cea5c581c56", + "sha256:225667980479a17db1048cb2bf8bfb39b8e5be8f164b8f6628b64f78a72cf9d3", + "sha256:260933720fdcd75340e7dbe9060655aff3af1f0c5d20f46b57f262ab6c86a5e8", + "sha256:2bdb062ea438f22d99cba0d7829c2ef0af1d768d1e4a4f528087224c90b132cb", + "sha256:2c09f4ce52cb99dd7505cd0fc8e0e37c77b87f46bc9c1eb03fe3bc9991085388", + "sha256:3115a95daa9bdba70aea750db7b96b37259a81a709223c8448fa97727d546fe0", + "sha256:3e0cadcf6733c09154b461f1ca72d5416635e5e4ec4e536192180d34ec160f8a", + "sha256:3f1156e3e8f2872197af3840d8ad307a9dd18e615dc64d9ee41696f287c57ad8", + "sha256:4421712dbfc5562150f7554f13dde997a2e932a6b5f352edcce948a815efee6f", + "sha256:44df346d5215a8c0e360307d46ffaabe0f5d3502c8a1cefd700b34baf31d411a", + "sha256:502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962", + "sha256:547f45fa1a93154bd82050a7f3cddbc1a7a4dd2a9bf5cb7d06f4ae29fe94eaf8", + "sha256:5621a9175cf9d0b0c84c2ef2b12e9f5f5071357c4d2ea6ca1cf01814f45d2391", + "sha256:609b06f178fe8e9f89ef676532760ec0b4deea15e9969bf754b37f7c40326dbc", + "sha256:645786266c8f18a931b65bfcefdbf6952dd0dea98feee39bd188607a9d307ed2", + "sha256:6878ef48d4227aace338d88c48738a4258213cd7b74fd9a3d4d7582bb1d8a155", + "sha256:6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb", + "sha256:6db04803b6c7291985a761004e9060b2bca08da6d04f26a7f2294b8623a0c1a0", + "sha256:6e2cd258d7d927d09493c8df1ce9174ad01b381d4729a9d8d4e38670ca24774c", + "sha256:6e81d7a3e58882450ec4186ca59a3f20a5d4440f25b1cff6f0902ad890e6748a", + "sha256:702855feff378050ae4f741045e19a32d57d19f3e0676d589df0575008ea5004", + "sha256:78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060", + "sha256:7bb65125fcbef8d989fa1dd0e8a060999497629ca5b0efbca209588a73356232", + "sha256:7dea0889685db8550f839fa202744652e87c60015029ce3f60e006f8c4462c93", + "sha256:8284cf8c0dd272a247bc154eb6c95548722dce90d098c17a883ed36e67cdb129", + "sha256:877abb17e6339d96bf08e7a622d05095e72b71f8afd8a9fefc82cf30ed944163", + "sha256:8929543a7192c13d177b770008bc4e8119f2e1f881d563fc6b6305d2d0ebe9de", + "sha256:8ae539519c4c040c5ffd0632784e21b2f03fc1340752af711f33e5be83a9d6c6", + "sha256:8f59d57baca39b32db42b83b2a7ba6f47ad9c394ec2076b084c3f029b7afca23", + "sha256:9054a0754de38d9dbd01a46621636689124d666bad1936d76c0341f7d71bf569", + "sha256:953510dfb7b12ab69d20135a0662397f077c59b1e6379a768e97c59d852ee51d", + "sha256:95cae0efeb032af8458fc27d191f85d1717b1d4e49f7cb226cf526ff28179778", + "sha256:9bc572be474cafb617672c43fe989d6e48d3c83af02ce8de73fff1c6bb3c198d", + "sha256:9c56863d44bd1c4fe2abb8a4d6f5371d197f1ac0ebdee542f07f35895fc07f36", + "sha256:9e0b2df163b8ed01d515807af24f63de04bebcecbd6c3bfeff88385789fdf75a", + "sha256:a09ece4a69cf399510c8ab25e0950d9cf2b42f7b3cb0374f95d2e2ff594478a6", + "sha256:a1ac0ae2b8bd743b88ed0502544847c3053d7171a3cff9228af618a068ed9c34", + "sha256:a318d68e92e80af8b00fa99609796fdbcdfef3629c77c6283566c6f02c6d6704", + "sha256:a4acd025ecc06185ba2b801f2de85546e0b8ac787cf9d3b06e7e2a69f925b106", + "sha256:a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9", + "sha256:a78d169acd38300060b28d600344a803628c3fd585c912cacc9ea8790fe96862", + "sha256:a95324a9de9650a729239daea117df21f4b9868ce32e63f8b650ebe6cef5595b", + "sha256:abd5fd0db5f4dc9289408aaf34908072f805ff7792632250dcb36dc591d24255", + "sha256:b06079abebbc0e89e6163b8e8f0e16270124c154dc6e4a47b413dd538859af16", + "sha256:b43c03669dc4618ec25270b06ecd3ee4fa94c7f9b3c14bae6571ca00ef98b0d3", + "sha256:b48f312cca9621272ae49008c7f613337c53fadca647d6384cc129d2996d1133", + "sha256:b5d7b556859dd85f3a541db6a4e0167b86e7273e1cdc973e5b175166bb634fdb", + "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657", + "sha256:c3c02d12f837d9683e5ab2f3d9844dc57655b92c74e286c262e0fc54213c216d", + "sha256:c44fee9975f04b33331cb8eb272827111efc8930cfd582e0320613263ca849ca", + "sha256:cf4b19715bccd7ee27b6b120e7e9dd56037b9c0681dcc1adc9ba9db3d417fa36", + "sha256:d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c", + "sha256:d3296782ca4eab572a1a4eca686d8bfb00226300dcefdf43faa25b5242ab8a3e", + "sha256:d85f5e9a5f8b73e2350097c3756ef7e785f55bd71205defa0bfdaf96c31616ff", + "sha256:da511e6ad4f7323ee5702e6633085fb76c2f893aaf8ce4c51a0ba4fc07580ea7", + "sha256:e05882b70b87a18d937ca6768ff33cc3f72847cbc4de4491c8e73880766718e5", + "sha256:e61c0abb4c85b095a784ef23fdd4aede7a2628478e7baba7c5e3deba61070a02", + "sha256:e6a08c0be454c3b3beb105c0596ebdc2371fab6bb90c0c0297f4e58fd7e1012c", + "sha256:e9a6e0eb86070e8ccaedfbd9d38fec54864f3125ab95419970575b42af7541df", + "sha256:ed37bd3c3b063412f7620464a9ac1314d33100329f39799255fb8d3027da50d3", + "sha256:f1adfc8ac319e1a348af294106bc6a8458a0f1633cc62a1446aebc30c5fa186a", + "sha256:f5796e664fe802da4f57a168c85359a8fbf3eab5e55cd4e4569fbacecc903959", + "sha256:fc5a77d0c516700ebad189b587de289a20a78324bc54baee03dd486f0855d234", + "sha256:fd21f6ae3f08b41004dfb433fa895d858f3f5979e7762d052b12aef444e29afc" + ], + "index": "pypi", + "version": "==7.6.1" + }, + "decorator": { + "hashes": [ + "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", + "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186" + ], + "markers": "python_version >= '3.5'", + "version": "==5.1.1" + }, + "executing": { + "hashes": [ + "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147", + "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc" + ], + "markers": "python_version >= '3.5'", + "version": "==2.0.1" + }, + "iniconfig": { + "hashes": [ + "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", + "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.0" + }, + "ipython": { + "hashes": [ + "sha256:1cec0fbba8404af13facebe83d04436a7434c7400e59f47acf467c64abd0956c", + "sha256:e6b347c27bdf9c32ee9d31ae85defc525755a1869f14057e900675b9e8d6e6ff" + ], + "index": "pypi", + "version": "==8.26.0" + }, + "jedi": { + "hashes": [ + "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd", + "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0" + ], + "markers": "python_version >= '3.6'", + "version": "==0.19.1" + }, + "matplotlib-inline": { + "hashes": [ + "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90", + "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca" + ], + "markers": "python_version >= '3.8'", + "version": "==0.1.7" + }, + "packaging": { + "hashes": [ + "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", + "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" + ], + "markers": "python_version >= '3.8'", + "version": "==24.1" + }, + "parso": { + "hashes": [ + "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", + "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d" + ], + "markers": "python_version >= '3.6'", + "version": "==0.8.4" + }, + "pexpect": { + "hashes": [ + "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", + "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f" + ], + "markers": "sys_platform != 'win32' and sys_platform != 'emscripten'", + "version": "==4.9.0" + }, + "pluggy": { + "hashes": [ + "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", + "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669" + ], + "markers": "python_version >= '3.8'", + "version": "==1.5.0" + }, + "prompt-toolkit": { + "hashes": [ + "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10", + "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.0.47" + }, + "ptyprocess": { + "hashes": [ + "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", + "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220" + ], + "version": "==0.7.0" + }, + "pure-eval": { + "hashes": [ + "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", + "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42" + ], + "version": "==0.2.3" + }, + "pygments": { + "hashes": [ + "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199", + "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a" + ], + "markers": "python_version >= '3.8'", + "version": "==2.18.0" + }, + "pytest": { + "hashes": [ + "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5", + "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce" + ], + "index": "pypi", + "version": "==8.3.2" + }, + "pytest-asyncio": { + "hashes": [ + "sha256:50265d892689a5faefb84df80819d1ecef566eb3549cf915dfb33569359d1ce2", + "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3" + ], + "index": "pypi", + "version": "==0.23.8" + }, + "pytest-cov": { + "hashes": [ + "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652", + "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857" + ], + "index": "pypi", + "version": "==5.0.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "stack-data": { + "hashes": [ + "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", + "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695" + ], + "version": "==0.6.3" + }, + "traitlets": { + "hashes": [ + "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7", + "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f" + ], + "markers": "python_version >= '3.8'", + "version": "==5.14.3" + }, + "typing-extensions": { + "hashes": [ + "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", + "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8" + ], + "markers": "python_version < '3.12'", + "version": "==4.12.2" + }, + "wcwidth": { + "hashes": [ + "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", + "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5" + ], + "version": "==0.2.13" + } + } +} diff --git a/README.md b/README.md index f75135c..045deeb 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ # CDM Task Service + +**This is currently a prototype** + +Enables running jobs on remote compute from the KBase CDM cluster. + +## Development + +### Adding code + +* In this alpha / prototype stage, we will be PRing (do not push directly) to `main`. In the + future, once we want to deploy beyond CI, we will add a `develop` branch. +* The PR creator merges the PR and deletes branches (after builds / tests / linters complete). + +### Code requirements for prototype code + +* Any code committed must at least have a test file that imports it and runs a noop test so that + the code is shown with no coverage in the coverage statistics. This will make it clear what + code needs tests when we move beyond the prototype stage. +* Each module should have its own test file. Eventually these will be expanded into unit tests + (or integration tests in the case of app.py) +* Any code committed must have regular code and user documentation so that future devs + converting the code to production can understand it. +* Release notes are not strictly necessary while deploying to CI, but a concrete version (e.g. + no `-dev*` or `-prototype*` suffix) will be required outside of that environment. On a case by + case basis, add release notes and bump the prototype version (e.g. 0.1.0-prototype3 -> + 0.1.0-prototype4) for changes that should be documented. + +### Running tests + +Python 3.11 must be installed on the system. + +``` +pipenv sync --dev # only the first time or when Pipfile.lock changes +pipenv shell +PYTHONPATH=. pytest test +``` + +### Exit from prototype status + +* Coverage badge in Readme +* Run through all code, refactor to production quality +* Add tests where missing (which is a lot) and inspect current tests for completeness and quality + * E.g. don't assume existing tests are any good + * Async testing help + https://tonybaloney.github.io/posts/async-test-patterns-for-pytest-and-unittest.html \ No newline at end of file diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..792dbfc --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,5 @@ +# CDM Task Service Release Notes + +## 0.1.0-prototype1 + +* Initial prototype release \ No newline at end of file diff --git a/cdmtaskservice/dockerimages.py b/cdmtaskservice/dockerimages.py new file mode 100644 index 0000000..82bb222 --- /dev/null +++ b/cdmtaskservice/dockerimages.py @@ -0,0 +1,30 @@ +""" +Contains code for querying container information without access to a docker server +""" + +from pathlib import Path + +class DockerImageInfo: + """ + Provides information about a remote docker image + """ + + def __init__(self, crane_absolute_path: Path): + """ + Initialize the image info provider. + + crane_absolute_path - the absolute path to the crane executable. + """ + # Require an absolute path to avoid various malicious attacks + # TODO check is absolute path + # TODO run crane sync and check version, store path + + def get_digest_from_name(self, image_name: str): + """ + Get an image digest given the image name. + """ + # TODO validate container_name + # TODO run crane asynchronously - watch out for shell injection + # TODO throw an exception if something goes wrong + return "sha256:fake" + diff --git a/test/dockerimages_test.py b/test/dockerimages_test.py new file mode 100644 index 0000000..99dd099 --- /dev/null +++ b/test/dockerimages_test.py @@ -0,0 +1,9 @@ +import pytest +from cdmtaskservice.dockerimages import DockerImageInfo + + +@pytest.mark.asyncio +async def test_get_digest_from_name(): + dig = DockerImageInfo("fake_path").get_digest_from_name("fake_image") + + assert dig == "sha256:fake" From 866c561fbc7e734bd5718f7c0e0aa2bb3a72f8f6 Mon Sep 17 00:00:00 2001 From: Gavin Date: Thu, 8 Aug 2024 16:27:46 -0700 Subject: [PATCH 2/3] Add docker infrastructure & fix coverage ... to allow GHA tasks to complete --- .github/workflows/test.yml | 2 +- Dockerfile | 21 +++++++++++++++++++++ cdmtaskservice/git_commit.py | 7 +++++++ scripts/entrypoint.sh | 4 ++++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 cdmtaskservice/git_commit.py create mode 100755 scripts/entrypoint.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6066286..059abbe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,7 @@ jobs: - name: Run tests shell: bash - run: PYTHONPATH=. pytest --cov=src --cov-report=xml test + run: PYTHONPATH=. pytest --cov=cdmtaskservice --cov-report=xml test - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..962bd7a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.11 + +RUN mkdir -p /cdmtaskservice +WORKDIR /cdmtaskservice + +# install pipenv +RUN pip install --upgrade pip && \ + pip install pipenv + +# install deps +COPY Pipfile* ./ +RUN pipenv sync --system + +COPY ./ /cdmtaskservice + +# Write the git commit for the service +ARG VCS_REF=no_git_commit_passed_to_build +RUN echo "GIT_COMMIT=\"$VCS_REF\"" > cdmtaskservice/git_commit.py + +ENTRYPOINT ["scripts/entrypoint.sh"] + diff --git a/cdmtaskservice/git_commit.py b/cdmtaskservice/git_commit.py new file mode 100644 index 0000000..6ce8e55 --- /dev/null +++ b/cdmtaskservice/git_commit.py @@ -0,0 +1,7 @@ +''' +The git commit for the current version of the CDM Task Service code. +''' + +# This file will be overwritten during the Docker build to contain the current git commit. + +GIT_COMMIT = "fake_commit" \ No newline at end of file diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100755 index 0000000..dd17be6 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# TODO fill this out - see collections service for an example +# TODO docker compose that starts up the service From 2f0966a7c7daefeabdb62d7c98ffdfcde49a8398 Mon Sep 17 00:00:00 2001 From: Gavin Date: Thu, 8 Aug 2024 18:49:56 -0700 Subject: [PATCH 3/3] Bump codecov to v4 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 059abbe..af6e47e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,7 @@ jobs: run: PYTHONPATH=. pytest --cov=cdmtaskservice --cov-report=xml test - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true