From 9f745f9110f1491c3ceab46034c193326638ccd2 Mon Sep 17 00:00:00 2001 From: Jimmy Royer Date: Thu, 14 Nov 2024 15:15:15 -0500 Subject: [PATCH] Accommodate slower NewRelic initialization (+goodies) (#2351) * Bringing in changes from newrelic slower initialization work sessions * Fix the unavail POETRY_VENV_PATH env var to the vscode launchers * Upgrading to NewRelic 9.1.1 and locking dependencies * Simplifying launcher; it won't work for now but we'll look at it later * Remove .gitattributes from being git managed --- .devcontainer/Dockerfile | 6 ++ .devcontainer/devcontainer.json | 3 +- .devcontainer/docker-compose.yml | 4 - .../scripts/notify-dev-entrypoint.sh | 4 + .dockerignore | 3 +- .vscode/launch.json | 44 ++++++++++- application.py | 2 +- gunicorn_config.py | 19 ++++- newrelic.ini | 3 + poetry.lock | 79 +++++++++++++++---- pyproject.toml | 4 +- 11 files changed, 138 insertions(+), 33 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 84a565d3de..0532009828 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -11,13 +11,17 @@ RUN apt-get update \ 2>&1 \ && apt-get -y install \ curl \ + dbus-x11 \ dnsutils \ emacs \ exa \ fd-find \ + fzf \ git \ + graphviz \ iproute2 \ iputils-ping \ + kcachegrind \ less \ libsodium-dev \ lsb-release \ @@ -28,6 +32,8 @@ RUN apt-get update \ npm \ openssh-client \ procps \ + pyprof2calltree \ + ripgrep \ sudo \ tldr \ unzip \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 243b733518..1e9bc814ab 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -25,7 +25,8 @@ "eamodio.gitlens", "fill-labs.dependi", "GitHub.copilot", - "GitHub.copilot-labs", + "github.copilot-chat", + "github.vscode-pull-request-github", "googlecloudtools.cloudcode", "kaiwood.center-editor-window", "matangover.mypy", diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index fe6ff395e3..50d2f7fd71 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -12,10 +12,6 @@ services: volumes: - ..:/workspace:cached command: sleep infinity - ports: - - 8000:8000 - - 8001:8001 - - 6011:6011 links: - db expose: diff --git a/.devcontainer/scripts/notify-dev-entrypoint.sh b/.devcontainer/scripts/notify-dev-entrypoint.sh index 6db49cc637..e9bc5df109 100755 --- a/.devcontainer/scripts/notify-dev-entrypoint.sh +++ b/.devcontainer/scripts/notify-dev-entrypoint.sh @@ -30,6 +30,10 @@ echo -e "alias smoke-staging='cd /workspace && cp .env_smoke_staging tests_smoke echo -e "alias smoke-prod='cd /workspace && cp .env_smoke_prod tests_smoke/.env && poetry run make smoke-test'" >> ~/.zshrc echo -e "alias smoke-dev='cd /workspace && cp .env_smoke_dev tests_smoke/.env && poetry run make smoke-test'" >> ~/.zshrc +echo -e "# fzf key bindings and completion" >> ~/.zshrc +echo -e "source /usr/share/doc/fzf/examples/key-bindings.zsh" >> ~/.zshrc +echo -e "source /usr/share/doc/fzf/examples/completion.zsh" >> ~/.zshrc + cd /workspace # Poetry autocomplete diff --git a/.dockerignore b/.dockerignore index 4f509e525f..d8d4252864 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ -*.env \ No newline at end of file +*.env +tests* \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 711f43a9c6..a4b6a47d81 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -3,14 +3,28 @@ "configurations": [ { "name": "Python: current file", - "type": "python", + "type": "debugpy", "request": "launch", "program": "${file}", "console": "integratedTerminal" }, + { + "name": "Python: Gunicorn", + "type": "debugpy", + "request": "launch", + "program": "gunicorn", + "gevent": true, + "args": ["--config", "gunicorn_config.py", "application"], + "env": { + "FLASK_APP": "application.py", + "FLASK_ENV": "development" + }, + "justMyCode": false, + "console": "integratedTerminal" + }, { "name": "Python: Flask", - "type": "python", + "type": "debugpy", "request": "launch", "module": "flask", "env": { @@ -28,7 +42,7 @@ }, { "name": "Python: Celery", - "type": "python", + "type": "debugpy", "request": "launch", "module": "celery", "console": "integratedTerminal", @@ -48,7 +62,7 @@ }, { "name": "Locust", - "type": "python", + "type": "debugpy", "request": "launch", "module": "locust", "args": [ @@ -64,6 +78,28 @@ ], "gevent": true, "console": "integratedTerminal" + }, + { + "name": "Python: Current File with profiler", + "type": "debugpy", + "request": "launch", + "module": "cProfile", + "env": { + "FLASK_APP": "application.py", + "FLASK_ENV": "development" + }, + "args": [ + "-o", + "/tmp/tmp.prof", + "${file}", + "flask", + "run", + "--no-debugger", + "-p 6011", + "--host=0.0.0.0" + ], + "jinja": true, + "justMyCode": false } ] } \ No newline at end of file diff --git a/application.py b/application.py index 7f554ecd3f..ce8d1d9276 100644 --- a/application.py +++ b/application.py @@ -42,6 +42,6 @@ def handler(event, context): - newrelic.agent.initialize() # noqa: E402 + newrelic.agent.initialize(environment=app.config["NOTIFY_ENVIRONMENT"]) # noqa: E402 newrelic.agent.register_application(timeout=20.0) return apig_wsgi_handler(event, context) diff --git a/gunicorn_config.py b/gunicorn_config.py index e43a7cb288..814e477ea5 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -5,7 +5,7 @@ import gunicorn # type: ignore import newrelic.agent # See https://bit.ly/2xBVKBH -newrelic.agent.initialize() # noqa: E402 +newrelic.agent.initialize(environment=os.getenv("NOTIFY_ENVIRONMENT")) # noqa: E402 workers = 4 worker_class = "gevent" @@ -25,15 +25,26 @@ keepalive = 75 # The default graceful timeout period for Kubernetes is 30 seconds, so - # want a lower graceful timeout value for gunicorn so that proper instance - # shutdowns. + # make sure that the timeouts defined here are less than the configured + # Kubernetes timeout. This ensures that the gunicorn worker will exit + # before the Kubernetes pod is terminated. This is important because + # Kubernetes will send a SIGKILL to the pod if it does not terminate + # within the grace period. If the worker is still processing requests + # when it receives the SIGKILL, it will be terminated abruptly and + # will not be able to finish processing the request. This can lead to + # 502 errors being returned to the client. + # + # Also, some libraries such as NewRelic might need some time to finish + # initialization before the worker can start processing requests. The + # timeout values should consider these factors. # # Gunicorn config: # https://docs.gunicorn.org/en/stable/settings.html#graceful-timeout # # Kubernetes config: # https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ - graceful_timeout = 20 + graceful_timeout = 25 + timeout = 30 def on_starting(server): diff --git a/newrelic.ini b/newrelic.ini index eddd873bfd..f8eb87bd36 100644 --- a/newrelic.ini +++ b/newrelic.ini @@ -192,10 +192,12 @@ error_collector.ignore_errors = app.v2.errors:BadRequestError jsonschema.excepti [newrelic:development] # monitor_mode = false +log_level = debug [newrelic:staging] # app_name = Python Application (Staging) # monitor_mode = true +log_level = debug [newrelic:production] # monitor_mode = true @@ -205,5 +207,6 @@ error_collector.ignore_errors = app.v2.errors:BadRequestError jsonschema.excepti [newrelic:dev] # monitor_mode = false +log_level = debug # --------------------------------------------------------------------------- diff --git a/poetry.lock b/poetry.lock index b2ee531e85..afe7624cec 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1647,6 +1647,17 @@ benchmarks = ["httplib2", "httpx", "requests", "urllib3"] dev = ["dpkt", "pytest", "requests"] examples = ["oauth2"] +[[package]] +name = "gprof2dot" +version = "2024.6.6" +description = "Generate a dot graph from the output of several profilers." +optional = false +python-versions = ">=3.8" +files = [ + {file = "gprof2dot-2024.6.6-py2.py3-none-any.whl", hash = "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696"}, + {file = "gprof2dot-2024.6.6.tar.gz", hash = "sha256:fa1420c60025a9eb7734f65225b4da02a10fc6dd741b37fa129bc6b41951e5ab"}, +] + [[package]] name = "greenlet" version = "2.0.2" @@ -2604,26 +2615,26 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "newrelic" -version = "8.10.0" +version = "9.1.1" description = "New Relic Python Agent" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ - {file = "newrelic-8.10.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:cf3b67327e64d2b50aec855821199b2bc46bc0c2d142df269d420748dd49b31b"}, - {file = "newrelic-8.10.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9601d886669fe1e0c23bbf91fb68ab23086011816ba96c6dd714c60dc0a74088"}, - {file = "newrelic-8.10.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:55a64d2abadf69bbc7bb01178332c4f25247689a97b01a62125d162ea7ec8974"}, - {file = "newrelic-8.10.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:b6cddd869ac8f7f32f6de8212ae878a21c9e63f2183601d239a76d38c5d5a366"}, - {file = "newrelic-8.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9af0130e1f1ca032c606d15a6d5558d27273a063b7c53702218b3beccd50b23"}, - {file = "newrelic-8.10.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2fd24b32dbf510e4e3fe40b71ad395dd73a4bb9f5eaf59eb5ff22ed76ba2d41"}, - {file = "newrelic-8.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2567ba9e29fd7b9f4c23cf16a5a149097eb0e5da587734c5a40732d75aaec189"}, - {file = "newrelic-8.10.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9c9f7842234a51e4a2fdafe42c42ebe0b6b1966279f2f91ec8a9c16480c2236"}, - {file = "newrelic-8.10.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:365d3b1a10d1021217beeb28a93c1356a9feb94bd24f02972691dc71227e40dc"}, - {file = "newrelic-8.10.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecd0666557419dbe11b04e3b38480b3113b3c4670d42619420d60352a1956dd8"}, - {file = "newrelic-8.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722072d57e2d416de68b650235878583a2a8809ea39c7dd5c8c11a19089b7665"}, - {file = "newrelic-8.10.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbda843100c99ac3291701c0a70fedb705c0b0707800c60b93657d3985aae357"}, - {file = "newrelic-8.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ed36fb91f152128825459eae9a52da364352ea95bcd78b405b0a5b8057b2ed7"}, - {file = "newrelic-8.10.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc975c29548e25805ead794d9de7ab3cb8ba4a6a106098646e1ab03112d1432e"}, - {file = "newrelic-8.10.0.tar.gz", hash = "sha256:8a2271b76ea684a63936302579d6085d46a2b54042cb91dc9b0d71a0cd4dd38b"}, + {file = "newrelic-9.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:639abcaa1baee5a1a137036e328617e3a6bbb33a63814ef6227a8059d9062f0d"}, + {file = "newrelic-9.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:1bd46363c71c3fc5dbcc0e96e1698757f4e7ff82c73a0ccb28f18c2b9f9c5de5"}, + {file = "newrelic-9.1.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:fbebaf8d9801eef85827ca2906a7bdbc2a458a226455da49c71857e2ce2b264a"}, + {file = "newrelic-9.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:c1a91322fdb301a7a17aab8df2e70925c1e2a01a61136d86a0e433f6ff167c5c"}, + {file = "newrelic-9.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d677f8f11bd69d92a4b359c53a04e02094e5198ed1aa49c9b1ca235522d5efd5"}, + {file = "newrelic-9.1.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:289cebbb86eb92b6c133e18cd5640115a918db41ae791d11c1ded5b592dd7c23"}, + {file = "newrelic-9.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1b93effe99be14a1747a70e9368274091abbed2ef3c593b08a29732d72d83e4"}, + {file = "newrelic-9.1.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45966c5f083d1f1abb76828c4864d20296f6929053fbe4d122e164d84c78e388"}, + {file = "newrelic-9.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18e8e6af37cc084007913b991bced2ea2b8b31ed5318a397f210a9625c214459"}, + {file = "newrelic-9.1.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:437dfae553012dbe6e0f288b1f600ef27b0ebc2c367641723529b1a71fa541ae"}, + {file = "newrelic-9.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5489becb7cff1ba992405c9d891fd1743e8042910c97f89ac072fd562501cdb1"}, + {file = "newrelic-9.1.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8403b48807eae524ee1b2da254f4222834f3aef8326dd4475f0b22f924774bde"}, + {file = "newrelic-9.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:603158cb23ed6604845e4fc8ee2a052e60003ccbb7da0f4b00667cd0c7c77c11"}, + {file = "newrelic-9.1.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96dedf6c1a385bf7b4b1336b3e4b5a0d6e983b5305022176c7c43d327e768bf3"}, + {file = "newrelic-9.1.1.tar.gz", hash = "sha256:968a3662ccfb881498789105a52eec866b57c22b8cbdef43889ad76881c62a95"}, ] [package.extras] @@ -3904,6 +3915,20 @@ files = [ {file = "smartypants-2.0.1-py2.py3-none-any.whl", hash = "sha256:8db97f7cbdf08d15b158a86037cd9e116b4cf37703d24e0419a0d64ca5808f0d"}, ] +[[package]] +name = "snakeviz" +version = "2.2.0" +description = "A web-based viewer for Python profiler output" +optional = false +python-versions = ">=3.7" +files = [ + {file = "snakeviz-2.2.0-py2.py3-none-any.whl", hash = "sha256:569e2d71c47f80a886aa6e70d6405cb6d30aa3520969ad956b06f824c5f02b8e"}, + {file = "snakeviz-2.2.0.tar.gz", hash = "sha256:7bfd00be7ae147eb4a170a471578e1cd3f41f803238958b6b8efcf2c698a6aa9"}, +] + +[package.dependencies] +tornado = ">=2.0" + [[package]] name = "sqlalchemy" version = "1.4.52" @@ -4061,6 +4086,26 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tornado" +version = "6.4.1" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8"}, + {file = "tornado-6.4.1-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842"}, + {file = "tornado-6.4.1-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4"}, + {file = "tornado-6.4.1-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698"}, + {file = "tornado-6.4.1-cp38-abi3-win32.whl", hash = "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d"}, + {file = "tornado-6.4.1-cp38-abi3-win_amd64.whl", hash = "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7"}, + {file = "tornado-6.4.1.tar.gz", hash = "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9"}, +] + [[package]] name = "tqdm" version = "4.66.5" @@ -4623,4 +4668,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "~3.10.9" -content-hash = "d6297cc2dcb00a6b90e6a5914a1f5357cce172e853e12f9d221375f276c516d3" +content-hash = "75a9f5035a4c3598836f34a7e9440aff6b4400ced2494a8e5e7625b50e906fef" diff --git a/pyproject.toml b/pyproject.toml index fc9abadfaa..35987c796e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ PyYAML = "6.0.1" cachelib = "0.12.0" SQLAlchemy = "1.4.52" -newrelic = "8.10.0" +newrelic = "9.1.1" notifications-python-client = "6.4.1" python-dotenv = "1.0.1" pwnedpasswords = "2.0.0" @@ -80,6 +80,7 @@ aws-xray-sdk = "2.14.0" [tool.poetry.group.test.dependencies] flake8 = "6.1.0" +gprof2dot = "2024.6.6" isort = "5.13.2" moto = "4.2.14" idna = "2.10" @@ -91,6 +92,7 @@ coveralls = "3.3.1" pytest-xdist = "2.5.0" freezegun = "1.5.1" requests-mock = "1.12.1" +snakeviz = "2.2.0" # optional requirements for jsonschema strict-rfc3339 = "0.7" rfc3987 = "1.3.8"