D->M 0.0.14 (#482)

* Update RELEASE_NOTES.md * add dependabot.yml && fix .pre-commit-config.yaml file (#478) * add dependabot * fix pre-comit config * black format fix * flake8 fix * SECURITY-36 3.10.10 (#477) * Testing upgrades * Update mongo version * Update python to 3.10.10 * Update ee2-tests.yml * SECURITY-36 Mongo3.6/Mongo7 Matrix (#484) * Linting * Bump version and release notes --------- Co-authored-by: Boris <[email protected]> * Bump actions/setup-python from 4 to 5 in /.github/workflows (#481) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](actions/setup-python@v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Sijie Xiang <[email protected]> Co-authored-by: Boris <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
kbase · Aug 7, 2024 · d5786fc · d5786fc
1 parent 60557e6
commit d5786fc
Show file tree

Hide file tree

Showing 21 changed files with 352 additions and 228 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,23 @@
+version: 2
+updates:
+
+  # Docker
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
+
+  # Python
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
+
+  # GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: ".github/workflows"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
diff --git a/.github/workflows/check_build.yml b/.github/workflows/check_build.yml
@@ -0,0 +1,21 @@
+name: Build EE2 Docker Image
+
+on:
+  pull_request
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      -
+        name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: false
+          tags: ee2/test:test
diff --git a/.github/workflows/ee2-tests.yml b/.github/workflows/ee2-tests.yml
@@ -1,12 +1,6 @@
 # This workflow will install Python dependencies, run tests and lint
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-# To ssh into this build add the following:
-#- name: Start SSH session
-#      uses: luchihoratiu/debug-via-ssh@main
-#      with:
-#        NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }}
-#        SSH_PASS: ${{ secrets.SSH_PASS }}
 
 name: Execution Engine 2 Test Suite
 
@@ -18,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Lint With Black
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           options: "--check --verbose"
@@ -35,11 +29,11 @@ jobs:
     name: Lint With Flake8
     steps:
       - name: Check out source repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Python environment
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.10.14"
       - name: flake8 Lint Lib
         uses: py-actions/flake8@v2
         with:
@@ -50,22 +44,38 @@ jobs:
           path: "./test"
 
 
+
   Build_and_Run_Tests_and_CodeCov:
     name: Build and Run Tests and CodeCov
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        mongo-config:
+          - version: "7.0"
+            init-path: "./test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/"
+          - version: "3.6"
+            init-path: "./test/dockerfiles/mongo/docker-entrypoint-initdb.d-3.6/"
     steps:
       - name: Check out source repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      - name: Set up Python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10.14"
       - name: Install dependencies
         run: |
           pip install -r requirements.txt
           git clone https://github.com/kbase/jars /opt/jars
-      - name: Build Docker Image
+      - name: Set MongoDB Version and Init Path
         run: |
-          docker build . -t execution_engine2:test
-      - name: Run Tests
+          echo "MONGO_VERSION=${{ matrix.mongo-config.version }}" >> $GITHUB_ENV
+          echo "INIT_PATH=${{ matrix.mongo-config.init-path }}" >> $GITHUB_ENV
+      - name: Start Services and Run Tests
         run: |
-          docker-compose up -d
+          docker compose up -d
           cp test/env/test.travis.env test.env
           make test-coverage
-          codecov
+          codecov --token="${{ secrets.CODECOV_TOKEN }}"
+      - name: Cleanup
+        run: |
+          docker compose down
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,7 +4,7 @@ repos:
     hooks:
     - id: black
       exclude: ".*execution_engine2Impl.py"
--   repo: https://gitlab.com/pycqa/flake8
-    rev: '3.9.2'
+-   repo: https://github.com/PyCQA/flake8
+    rev: 3.9.2
     hooks:
     - id: flake8
diff --git a/Dockerfile b/Dockerfile
@@ -26,23 +26,8 @@ RUN curl -o /tmp/dockerize.tgz https://raw.githubusercontent.com/kbase/dockerize
       rm /tmp/dockerize.tgz
 
 
-# install mongodb
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 \
-    && echo "deb http://repo.mongodb.org/apt/debian stretch/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list  \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends mongodb-org=3.6.11 mongodb-org-server=3.6.11 mongodb-org-shell=3.6.11 mongodb-org-mongos=3.6.11 mongodb-org-tools=3.6.11 \
-    && apt-get install -y --no-install-recommends mongodb \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN echo "mongodb-org hold" | dpkg --set-selections \
-    && echo "mongodb-org-server hold" | dpkg --set-selections \
-    && echo "mongodb-org-shell hold" | dpkg --set-selections \
-    && echo "mongodb-org-mongos hold" | dpkg --set-selections \
-    && echo "mongodb-org-tools hold" | dpkg --set-selections
-
 #Install Python3 and Libraries (source /root/miniconda/bin/activate)
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-py310_24.5.0-0-Linux-x86_64.sh -O ~/miniconda.sh \
 && bash ~/miniconda.sh -b -p /miniconda-latest
 
 # Setup Cron

diff --git a/Makefile b/Makefile
@@ -38,7 +38,7 @@ setup-database:
 test-coverage:
 	# Set up travis user in mongo
 	@echo "Run tests for $(TESTS)"
-	PYTHONPATH=.:lib:test pytest --cov-report=xml --cov lib/execution_engine2/ --verbose $(TESTS)
+	PYTHONPATH=.:lib:test pytest --cov-report=xml --cov-report=term --cov lib/execution_engine2/  --verbose $(TESTS)
 
 build-condor-test-image:
 	cd test/dockerfiles/condor && echo `pwd` && docker build -f Dockerfile . -t $(CONDOR_DOCKER_IMAGE_TAG_NAME)

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -1,8 +1,16 @@
 # execution_engine2 (ee2) release notes
 =========================================
 
+## 0.0.14
+- Update clients to work with mongo7 by updating pymongo and mongoengine
+-  Fix linting issues from flake8 and black
+- Update outdated libraries that can no longer be pip installed
+- The latest Dockerfile build contains conda which installs python 3.12.4 so these changes will pin python to 3.10 and save us headache in the future
+- Speed up tests by moving docker build step out
+- Create matrix of tests to run against mongo3.6 and mongo7
+
 ## 0.0.13
-* Bump version in impl file
+* Bump version in impl file and sync branches
 
 ## 0.0.12 
 * Forcing black to 22.1.0 to make sure that GHA doesn't suddenly fail

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,57 +2,58 @@ version: "3.1"
 
 services:
 
+  # For Local Dev
   # Requires a ` docker build . -t execution_engine2:test `
   # Requires a ` make database `
-  ee2:
-    image: execution_engine2:test
-    command:
-      - "-poll"
-      - "-template"
-      - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config"
-      - "-template"
-      - "/kb/module/test/deploy.cfg:/kb/module/deploy.cfg"
-      - "-timeout"
-      - "120s"
-      - "-wait"
-      - "tcp://mongodb:27017"
-      - "-stdout"
-      - "/kb/deployment/jettybase/logs/request.log"
-      - "./scripts/entrypoint.sh"
-    entrypoint: [ "dockerize" ]
-    depends_on: ["mongodb","condor"]
-    environment:
-      - POOL_PASSWORD=weakpassword
-    env_file: test/deploy.cfg
-    volumes:
-      - ./:/ee2
-
-  ee2_with_ssh:
-    image: execution_engine2:test
-    command:
-      - "-poll"
-      - "-template"
-      - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config"
-      - "-template"
-      - "/kb/module/build/templates/deploy.docker.cfg.templ:/kb/module/deploy.cfg"
-      - "-timeout"
-      - "120s"
-      - "-wait"
-      - "tcp://mongodb:27017"
-      - "-stdout"
-      - "/kb/deployment/jettybase/logs/request.log"
-      - "./scripts/entrypoint.sh"
-    entrypoint: [ "dockerize" ]
-    depends_on: ["mongodb","condor"]
-    environment:
-      - POOL_PASSWORD=weakpassword
-      - LOCAL_DEV=1
-      - KB_DEPLOYMENT_CONFIG=/kb/module/deploy.cfg
-    env_file: test/deploy.cfg
-    ports: ["23:22","5678:5678"]
-    volumes:
-      - ./:/ee2
-      - /Users:/Users
+#  ee2:
+#    image: execution_engine2:test
+#    command:
+#      - "-poll"
+#      - "-template"
+#      - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config"
+#      - "-template"
+#      - "/kb/module/test/deploy.cfg:/kb/module/deploy.cfg"
+#      - "-timeout"
+#      - "120s"
+#      - "-wait"
+#      - "tcp://mongodb:27017"
+#      - "-stdout"
+#      - "/kb/deployment/jettybase/logs/request.log"
+#      - "./scripts/entrypoint.sh"
+#    entrypoint: [ "dockerize" ]
+#    depends_on: ["mongodb","condor"]
+#    environment:
+#      - POOL_PASSWORD=weakpassword
+#    env_file: test/deploy.cfg
+#    volumes:
+#      - ./:/ee2
+#
+#  ee2_with_ssh:
+#    image: execution_engine2:test
+#    command:
+#      - "-poll"
+#      - "-template"
+#      - "/kb/module/build/templates/condor_config.templ:/etc/condor/condor_config"
+#      - "-template"
+#      - "/kb/module/build/templates/deploy.docker.cfg.templ:/kb/module/deploy.cfg"
+#      - "-timeout"
+#      - "120s"
+#      - "-wait"
+#      - "tcp://mongodb:27017"
+#      - "-stdout"
+#      - "/kb/deployment/jettybase/logs/request.log"
+#      - "./scripts/entrypoint.sh"
+#    entrypoint: [ "dockerize" ]
+#    depends_on: ["mongodb","condor"]
+#    environment:
+#      - POOL_PASSWORD=weakpassword
+#      - LOCAL_DEV=1
+#      - KB_DEPLOYMENT_CONFIG=/kb/module/deploy.cfg
+#    env_file: test/deploy.cfg
+#    ports: ["23:22","5678:5678"]
+#    volumes:
+#      - ./:/ee2
+#      - /Users:/Users
 
 
 
@@ -146,13 +147,15 @@ services:
       ZOOKEEPER_SYNC_LIMIT: 2docker-compose rm
       ZOOKEEPER_SERVERS: zookeeper:12888:13888
 
+
   mongodb:
-    image: mongo:3.2
+    image: mongo:${MONGO_VERSION:-7.0} # Use 7.0 as the default if MONGO_VERSION is not set
     environment:
-      # provide your credentials here
       - MONGO_INITDB_ROOT_USERNAME=travis
       - MONGO_INITDB_ROOT_PASSWORD=travis
       - MONGO_INITDB_DATABASE=ee2
-    ports: ["27018:27017","27017:27017"]
+    ports:
+      - "27018:27017"
+      - "27017:27017"
     volumes:
-      - "./test/dockerfiles/mongo/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d/"
+      - ${INIT_PATH:-./test/dockerfiles/mongo/docker-entrypoint-initdb.d-7.0/}:/docker-entrypoint-initdb.d/
diff --git a/kbase.yml b/kbase.yml
@@ -8,7 +8,7 @@ service-language:
     python
 
 module-version:
-    0.0.13
+    0.0.14
 
 owners:
     [bsadkhin, tgu2, wjriehl, gaprice]
-Original file line number
+Diff line change
@@ Expand Up / @@ -8,7 +8,7 @@ service-language: @@
         python
     module-version:
-.0.13
+.0.14
     owners:
         [bsadkhin, tgu2, wjriehl, gaprice]
@@ Expand Down @@