Accommodate slower NewRelic initialization (+goodies) (#2351)

* Bringing in changes from newrelic slower initialization work sessions

* Fix the unavail POETRY_VENV_PATH env var to the vscode launchers

* Upgrading to NewRelic 9.1.1 and locking dependencies

* Simplifying launcher; it won't work for now but we'll look at it later

* Remove .gitattributes from being git managed

.devcontainer/Dockerfile

@@ -11,13 +11,17 @@ RUN apt-get update \
     2>&1 \
     && apt-get -y install \
     curl \
+    dbus-x11 \
     dnsutils \
     emacs \
     exa \
     fd-find \
+    fzf \
     git \
+    graphviz \
     iproute2 \
     iputils-ping \
+    kcachegrind \
     less \
     libsodium-dev \
     lsb-release \
@@ -28,6 +32,8 @@ RUN apt-get update \
     npm \
     openssh-client \
     procps \
+    pyprof2calltree \
+    ripgrep \
     sudo \
     tldr \
     unzip \

.devcontainer/devcontainer.json

@@ -25,7 +25,8 @@
 				"eamodio.gitlens",
 				"fill-labs.dependi",
 				"GitHub.copilot",
-				"GitHub.copilot-labs",
+				"github.copilot-chat",
+				"github.vscode-pull-request-github",
 				"googlecloudtools.cloudcode",
 				"kaiwood.center-editor-window",
 				"matangover.mypy",

.devcontainer/docker-compose.yml

@@ -12,10 +12,6 @@ services:
     volumes:
       - ..:/workspace:cached
     command: sleep infinity
-    ports:
-      - 8000:8000
-      - 8001:8001
-      - 6011:6011
     links:
       - db
     expose:

.devcontainer/scripts/notify-dev-entrypoint.sh

@@ -30,6 +30,10 @@ echo -e "alias smoke-staging='cd /workspace && cp .env_smoke_staging tests_smoke
 echo -e "alias smoke-prod='cd /workspace && cp .env_smoke_prod tests_smoke/.env && poetry run make smoke-test'" >> ~/.zshrc
 echo -e "alias smoke-dev='cd /workspace && cp .env_smoke_dev tests_smoke/.env && poetry run make smoke-test'" >> ~/.zshrc
 
+echo -e "# fzf key bindings and completion" >> ~/.zshrc
+echo -e "source /usr/share/doc/fzf/examples/key-bindings.zsh" >> ~/.zshrc
+echo -e "source /usr/share/doc/fzf/examples/completion.zsh" >> ~/.zshrc
+
 cd /workspace
 
 # Poetry autocomplete

.dockerignore

@@ -1 +1,2 @@
-*.env
+*.env
+tests*

.vscode/launch.json

@@ -3,14 +3,28 @@
     "configurations": [
         {
             "name": "Python: current file",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal"
         },
+        {
+            "name": "Python: Gunicorn",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "gunicorn",
+            "gevent": true,
+            "args": ["--config", "gunicorn_config.py", "application"],
+            "env": {
+                "FLASK_APP": "application.py",
+                "FLASK_ENV": "development"
+            },
+            "justMyCode": false,
+            "console": "integratedTerminal"
+        },
         {
             "name": "Python: Flask",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "flask",
             "env": {
@@ -28,7 +42,7 @@
         },
         {
             "name": "Python: Celery",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "celery",
             "console": "integratedTerminal",
@@ -48,7 +62,7 @@
         },
         {
             "name": "Locust",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "locust",
             "args": [
@@ -64,6 +78,28 @@
             ],
             "gevent": true,
             "console": "integratedTerminal"
+        },
+        {
+            "name": "Python: Current File with profiler",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "cProfile",
+            "env": {
+                "FLASK_APP": "application.py",
+                "FLASK_ENV": "development"
+            },
+            "args": [
+                "-o",
+                "/tmp/tmp.prof",
+                "${file}",
+                "flask",
+                "run",
+                "--no-debugger",
+                "-p 6011",
+                "--host=0.0.0.0"
+            ],
+            "jinja": true,
+            "justMyCode": false
         }
     ]
 }

application.py

@@ -42,6 +42,6 @@
 
 
 def handler(event, context):
-    newrelic.agent.initialize()  # noqa: E402
+    newrelic.agent.initialize(environment=app.config["NOTIFY_ENVIRONMENT"])  # noqa: E402
     newrelic.agent.register_application(timeout=20.0)
     return apig_wsgi_handler(event, context)

gunicorn_config.py

@@ -5,7 +5,7 @@
 import gunicorn  # type: ignore
 import newrelic.agent  # See https://bit.ly/2xBVKBH
 
-newrelic.agent.initialize()  # noqa: E402
+newrelic.agent.initialize(environment=os.getenv("NOTIFY_ENVIRONMENT"))  # noqa: E402
 
 workers = 4
 worker_class = "gevent"
@@ -25,15 +25,26 @@
     keepalive = 75
 
     # The default graceful timeout period for Kubernetes is 30 seconds, so
-    # want a lower graceful timeout value for gunicorn so that proper instance
-    # shutdowns.
+    # make sure that the timeouts defined here are less than the configured
+    # Kubernetes timeout. This ensures that the gunicorn worker will exit
+    # before the Kubernetes pod is terminated. This is important because
+    # Kubernetes will send a SIGKILL to the pod if it does not terminate
+    # within the grace period. If the worker is still processing requests
+    # when it receives the SIGKILL, it will be terminated abruptly and
+    # will not be able to finish processing the request. This can lead to
+    # 502 errors being returned to the client.
+    #
+    # Also, some libraries such as NewRelic might need some time to finish
+    # initialization before the worker can start processing requests. The
+    # timeout values should consider these factors.
     #
     # Gunicorn config:
     # https://docs.gunicorn.org/en/stable/settings.html#graceful-timeout
     #
     # Kubernetes config:
     # https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
-    graceful_timeout = 20
+    graceful_timeout = 25
+    timeout = 30
 
 
 def on_starting(server):

newrelic.ini

@@ -192,10 +192,12 @@ error_collector.ignore_errors = app.v2.errors:BadRequestError jsonschema.excepti
 
 [newrelic:development]
 # monitor_mode = false
+log_level = debug
 
 [newrelic:staging]
 # app_name = Python Application (Staging)
 # monitor_mode = true
+log_level = debug
 
 [newrelic:production]
 # monitor_mode = true
@@ -205,5 +207,6 @@ error_collector.ignore_errors = app.v2.errors:BadRequestError jsonschema.excepti
 
 [newrelic:dev]
 # monitor_mode = false
+log_level = debug
 
 # ---------------------------------------------------------------------------

pyproject.toml

@@ -49,7 +49,7 @@ PyYAML = "6.0.1"
 
 cachelib = "0.12.0"
 SQLAlchemy = "1.4.52"
-newrelic = "8.10.0"
+newrelic = "9.1.1"
 notifications-python-client = "6.4.1"
 python-dotenv = "1.0.1"
 pwnedpasswords = "2.0.0"
@@ -80,6 +80,7 @@ aws-xray-sdk = "2.14.0"
 
 [tool.poetry.group.test.dependencies]
 flake8 = "6.1.0"
+gprof2dot = "2024.6.6"
 isort = "5.13.2"
 moto = "4.2.14"
 idna = "2.10"
@@ -91,6 +92,7 @@ coveralls = "3.3.1"
 pytest-xdist = "2.5.0"
 freezegun = "1.5.1"
 requests-mock = "1.12.1"
+snakeviz = "2.2.0"
 # optional requirements for jsonschema
 strict-rfc3339 = "0.7"
 rfc3987 = "1.3.8"