chore: lint & make integration tests in autopush-rs (#518)

* chore: lint & make integration tests in autopush-rs - move the integration test code to the tests\integration folder - convert the integration test dependency manager to poetry - lint integration test code - update make to lint and execute integration tests - update CI to lint integration test code SYNC-3833 --------- Co-authored-by: JR Conlin <[email protected]>
mozilla-services · Nov 16, 2023 · e3d38c6 · e3d38c6
1 parent 7fe9d48
commit e3d38c6
Show file tree

Hide file tree

Showing 21 changed files with 2,362 additions and 2,089 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -41,13 +41,25 @@ jobs:
       - run:
           command: cargo audit
 
-  load-test-checks:
+  python-checks:
     docker:
-      - image: cimg/python:3.11
-    working_directory: "~/autopush-rs"
+      - image: python:3.10-slim-bullseye
+        auth:
+          username: $DOCKER_USER
+          password: $DOCKER_PASS
     steps:
-      - checkout:
-          path: ~/autopush-rs/
+      - checkout
+      - run:
+          name: Set up system
+          command: |
+            apt update
+            apt install git -y
+            apt install cmake -y
+      - run:
+          name: Set up Python
+          command: |
+            pip install --upgrade pip
+            pip install poetry
       - run:
           name: isort, black, flake8 and mypy
           command: make lint
@@ -68,11 +80,11 @@ jobs:
       AWS_LOCAL_DYNAMODB: http://localhost:8000
     steps:
       - checkout
-      # Need to download the requirements.txt files so we can use their
+      # Need to download the poetry.lock files so we can use their
       # checksums in restore_cache.
       - restore_cache:
           name: Restoring Python cache
-          key: python-v1-{{ checksum "tests/requirements.txt" }}-{{ checksum "tests/test_integration_all_rust.py"}}
+          key: python-v1-{{ checksum "tests/poetry.lock" }}-{{ checksum "tests/integration/test_integration_all_rust.py"}}
       - restore_cache:
           name: Restoring Rust cache
           key: rust-v1-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
@@ -85,8 +97,7 @@ jobs:
           name: Set up Python
           command: |
             pip install --upgrade pip
-            pip install bottle
-            pip install -r tests/requirements.txt
+            pip install poetry
       - run:
           name: Set up Rust
           command: |
@@ -116,20 +127,13 @@ jobs:
           command: cargo test --features=bigtable --jobs=8
       - run:
           name: Integration tests (Autopush Legacy)
-          command: |
-            cd tests
-            py.test -v
+          command: make integration-test-legacy
       - run:
           name: Integration tests (Autoconnect)
-          environment:
-            CONNECTION_BINARY: autoconnect
-            CONNECTION_SETTINGS_PREFIX: autoconnect__
-          command: |
-            cd tests
-            py.test -v
+          command: make integration-test
       - save_cache:
           name: Save Python cache
-          key: python-v1-{{ checksum "tests/requirements.txt" }}-{{ checksum "tests/test_integration_all_rust.py"}}
+          key: python-v1-{{ checksum "tests/poetry.lock" }}-{{ checksum "tests/integration/test_integration_all_rust.py"}}
           paths:
             - /home/circleci/.local/bin/
             - /home/circleci/.local/lib/
@@ -235,7 +239,7 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - load-test-checks:
+      - python-checks:
           filters:
             tags:
               only: /.*/

diff --git a/.gitignore b/.gitignore
@@ -32,6 +32,7 @@ target
 requirements.txt
 test-requirements.txt
 venv
+.python-version
 
 # docs
 docs/output

diff --git a/Makefile b/Makefile
@@ -1,10 +1,12 @@
 SHELL := /bin/sh
 CARGO = cargo
-LOAD_TEST_DIR := tests/load
-POETRY := poetry --directory $(LOAD_TEST_DIR)
+TESTS_DIR := tests
+INTEGRATION_TEST_FILE := $(TESTS_DIR)/integration/test_integration_all_rust.py
+LOAD_TEST_DIR := $(TESTS_DIR)/load
+POETRY := poetry --directory $(TESTS_DIR)
 DOCKER_COMPOSE := docker compose
-PYPROJECT_TOML := $(LOAD_TEST_DIR)/pyproject.toml
-FLAKE8_CONFIG := $(LOAD_TEST_DIR)/.flake8
+PYPROJECT_TOML := $(TESTS_DIR)/pyproject.toml
+FLAKE8_CONFIG := $(TESTS_DIR)/.flake8
 STAGE_SERVER_URL := "wss://autopush.stage.mozaws.net"
 STAGE_ENDPOINT_URL := "https://updates-autopush.stage.mozaws.net"
 
@@ -20,13 +22,25 @@ upgrade:
 	$(CARGO) upgrade
 	$(CARGO) update
 
+integration-test-legacy:
+	$(POETRY) -V
+	$(POETRY) install --without dev,load
+	$(POETRY) run pytest $(INTEGRATION_TEST_FILE) -v
+
+integration-test:
+	$(POETRY) -V
+	$(POETRY) install --without dev,load
+	CONNECTION_BINARY=autoconnect \
+		CONNECTION_SETTINGS_PREFIX=autoconnect__ \
+		$(POETRY) run pytest $(INTEGRATION_TEST_FILE) -v
+
 lint:
 	$(POETRY) -V
 	$(POETRY) install
-	$(POETRY) run isort --sp $(PYPROJECT_TOML) -c $(LOAD_TEST_DIR)
-	$(POETRY) run black --quiet --diff --config $(PYPROJECT_TOML) --check $(LOAD_TEST_DIR)
-	$(POETRY) run flake8 --config $(FLAKE8_CONFIG) $(LOAD_TEST_DIR)
-	$(POETRY) run mypy $(LOAD_TEST_DIR) --config-file=$(PYPROJECT_TOML)
+	$(POETRY) run isort --sp $(PYPROJECT_TOML) -c $(TESTS_DIR)
+	$(POETRY) run black --quiet --diff --config $(PYPROJECT_TOML) --check $(TESTS_DIR)
+	$(POETRY) run flake8 --config $(FLAKE8_CONFIG) $(TESTS_DIR)
+	$(POETRY) run mypy $(TESTS_DIR) --config-file=$(PYPROJECT_TOML)
 
 load:
 	SERVER_URL=$(STAGE_SERVER_URL) ENDPOINT_URL=$(STAGE_ENDPOINT_URL) \

diff --git a/tests/.flake8 b/tests/.flake8
@@ -0,0 +1,7 @@
+[flake8]
+exclude =
+    .git,
+    __pycache__,
+# E203: Whitespace before ':'. Addresses a discrepancy with Black slice formatting.
+ignore = E203
+max-line-length = 99
diff --git a/tests/__init__.py → tests/integration/__init__.py b/tests/__init__.py → tests/integration/__init__.py
diff --git a/tests/bt_test.json → tests/integration/bt_test.json b/tests/bt_test.json → tests/integration/bt_test.json