Add Monkeytype and Poe the Poet plugin for Poetry (#1731)

* Add monkeytype and poe the poet plugin for poetry

* Fix some wording

* Fix typo in trace-tests task

* Improve poe tasks

- Added monkeytype config
- Specified code filter so we only trace packages and modules we actually want traced

* Only allow monkeytype to trace our modules

- Fixed typos in Poe tasks

* Update lock file

* Refresh lock file

* Formatting

* Update build backend

- Install wheel before awslambdaric

* Update pip before running installs

* Quick typo fix

* Messing with build deps

* Test updated alpine image

* Migrate make targets to poe

* Update poetry lock file

* Fix Dockerfile.lambda

- Added rust and cargo to fix failing wheel build

* Add rust and cargo to Dockerfile

* Bump docker image to 3.18

- these images are going to be the end of me...

* Update lock file

* update lock file

---------

Co-authored-by: Jumana B <[email protected]>

.devcontainer/scripts/installations.sh

@@ -11,6 +11,7 @@ echo -e "alias ls='exa'" >> ~/.zshrc
 echo -e "alias l='exa -alh'" >> ~/.zshrc
 echo -e "alias ll='exa -alh@ --git'" >> ~/.zshrc
 echo -e "alias lt='exa -al -T -L 2'" >> ~/.zshrc
+echo -e "alias poe='poetry run poe'" >> ~/.zshrc
 
 # Poetry autocomplete
 echo -e "fpath+=/.zfunc" >> ~/.zshrc
@@ -33,6 +34,10 @@ poetry completions zsh > ~/.zfunc/_poetry
 cd /workspaces/notification-admin
 poetry install
 
+# Poe the Poet plugin tab completions
+touch ~/.zfunc/_poe
+poetry run poe _zsh_completion > ~/.zfunc/_poe
+
 npm rebuild node-sass
 make generate-version-file
 make babel

.gitignore

@@ -50,6 +50,9 @@ test_results.xml
 *.pot
 app/translations/csv/en.csv
 
+# MonkeyType
+monkeytype.sqlite3
+
 # Django stuff:
 *.log
 

app/__init__.py

@@ -56,6 +56,7 @@
 from app.models.organisation import Organisation
 from app.models.service import Service
 from app.models.user import AnonymousUser, User
+from app.monkeytype_config import MonkeytypeConfig
 from app.navigation import (
     AdminNavigation,
     HeaderNavigation,
@@ -92,7 +93,6 @@
 login_manager = LoginManager()
 csrf = CSRFProtect()
 
-
 # The current service attached to the request stack.
 current_service: Service = LocalProxy(lambda: g.current_service)  # type: ignore
 
@@ -231,6 +231,13 @@ def get_locale():
             application.config["CRM_ORG_LIST_URL"], application.config["CRM_GITHUB_PERSONAL_ACCESS_TOKEN"], application.logger
         )
 
+    # Specify packages to be traced by MonkeyType. This can be overriden
+    # via the MONKEYTYPE_TRACE_MODULES environment variable. e.g:
+    # MONKEYTYPE_TRACE_MODULES="app.,notifications_utils."
+    if application.config["NOTIFY_ENVIRONMENT"] == "development":
+        packages_prefix = ["app.", "notifications_utils."]
+        application.monkeytype_config = MonkeytypeConfig(packages_prefix)
+
 
 def init_app(application):
     application.after_request(useful_headers_after_request)

app/monkeytype_config.py

@@ -0,0 +1,19 @@
+import os
+
+from monkeytype.config import DefaultConfig
+
+
+class MonkeytypeConfig(DefaultConfig):
+    def __init__(self, package_prefixes):
+        self.package_prefixes = package_prefixes
+
+    def code_filter(self, code):
+        # Get the module name from the code object
+        # and convert the file path to a module name
+        module_name = code.co_filname.replace(os.path.sep, ".")[:-3]
+
+        for prefix in self.package_prefixes:
+            if module_name.startswith(prefix):
+                return True
+
+        return False

ci/Dockerfile

@@ -4,7 +4,7 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV POETRY_VERSION="1.7.1"
 ENV POETRY_VIRTUALENVS_CREATE false
 
-RUN apk add --no-cache bash build-base libxml2-dev libxslt-dev git nodejs npm g++ make libffi-dev && rm -rf /var/cache/apk/*
+RUN apk add --no-cache bash build-base libxml2-dev libxslt-dev git nodejs npm g++ make libffi-dev rust cargo && rm -rf /var/cache/apk/*
 
 # update pip
 RUN python -m pip install wheel poetry==${POETRY_VERSION}

ci/Dockerfile.lambda

@@ -5,7 +5,7 @@ ARG POETRY_VERSION="1.7.1"
 ARG POETRY_VIRTUALENVS_CREATE="false"
 
 # Build image
-FROM python:3.10-alpine3.16@sha256:afe68972cc00883d70b3760ee0ffbb7375cf09706c122dda7063ffe64c5be21b as base
+FROM python:3.10-alpine3.18@sha256:d5ee9613c89c9bd4c4112465d2136512ea8629bce6ff15fa27144f3cc16b5c6b as base
 
 ARG APP_DIR
 ARG APP_VENV
@@ -26,28 +26,35 @@ RUN apk add --no-cache \
     cmake \
     g++ \
     git \
-    libexecinfo-dev \
     libffi-dev \
     libtool \
     libxml2-dev \
     libxslt-dev \
     nodejs \
     npm \
-    make
+    make \
+    rust \
+    cargo
+
+# wheel requires rustc 1.70 or later so we'll pull it in from the 3.18 package repo
+# We could update the base image to 3.18 but libexecinfo-dev was dropped in 3.17, so
+# either way we need to pull a package in from a different repo to build the base image
+RUN apk add --no-cache --update --repository=https://dl-cdn.alpinelinux.org/alpine/v3.16/main/ libexecinfo-dev
 
 RUN mkdir -p ${APP_DIR}
 WORKDIR ${APP_DIR}
 
 # Install Poetry and isolate it from the project
-RUN python -m venv ${POETRY_HOME} \
-    && ${POETRY_HOME}/bin/pip3 install poetry==${POETRY_VERSION}
+RUN python -m venv ${POETRY_HOME}
+RUN ${POETRY_HOME}/bin/pip3 install --upgrade pip
+RUN ${POETRY_HOME}/bin/pip3 install poetry==${POETRY_VERSION}
 
 COPY pyproject.toml poetry.lock ${APP_DIR}/
 
 RUN python -m venv ${APP_VENV} \
     && . ${APP_VENV}/bin/activate \
     && poetry install \
-    && poetry add awslambdaric wheel
+    && poetry add wheel awslambdaric
 
 COPY package.json package-lock.json ${APP_DIR}/
 RUN npm ci
@@ -61,7 +68,7 @@ RUN . ${APP_VENV}/bin/activate \
     && make generate-version-file
 
 # Final image
-FROM python:3.10-alpine3.16@sha256:afe68972cc00883d70b3760ee0ffbb7375cf09706c122dda7063ffe64c5be21b as lambda
+FROM python:3.10-alpine3.18@sha256:d5ee9613c89c9bd4c4112465d2136512ea8629bce6ff15fa27144f3cc16b5c6b as lambda
 
 ARG APP_DIR
 ARG APP_VENV