From 1e5f2b90d9a60dcc16f287d300085554e32c496e Mon Sep 17 00:00:00 2001 From: Darrell Malone Jr Date: Mon, 21 Aug 2023 20:14:35 -0500 Subject: [PATCH 01/10] Add Mixpanel - Add mixpanel and ua-parser to requirements - Track API events --- README.md | 1 + backend/mixpanel/__init__.py | 0 backend/mixpanel/mix.py | 29 +++++++++++++++++++++++++++++ backend/routes/auth.py | 6 ++++++ backend/routes/incidents.py | 11 ++++++++++- backend/routes/partners.py | 12 +++++++++++- requirements/_core.in | 4 +++- requirements/dev_unix.txt | 17 +++++++++++++---- requirements/dev_windows.txt | 17 +++++++++++++---- requirements/docs.txt | 4 ++-- requirements/prod.txt | 17 +++++++++++++---- 11 files changed, 101 insertions(+), 17 deletions(-) create mode 100644 backend/mixpanel/__init__.py create mode 100644 backend/mixpanel/mix.py diff --git a/README.md b/README.md index a3300334..ebbded24 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ POSTGRES_DB=police_data POSTGRES_HOST=db POSTGRES_PORT=5432 PDT_API_PORT=5000 +MIXPANEL_TOKEN=your_mixpanel_token ``` > **Note** diff --git a/backend/mixpanel/__init__.py b/backend/mixpanel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/mixpanel/mix.py b/backend/mixpanel/mix.py new file mode 100644 index 00000000..c85c4693 --- /dev/null +++ b/backend/mixpanel/mix.py @@ -0,0 +1,29 @@ +from mixpanel import Mixpanel +from ua_parser import user_agent_parser +from flask_jwt_extended import get_jwt + +from backend import config +# Path: backend/mixpanel/mix.py + +# Mixpanel connection +mp = Mixpanel.init(config.MIXPANEL_TOKEN) + + +def track_to_mp(request, event_name, properties): + parsed = user_agent_parser.Parse(request.headers["User-Agent"]) + + # Set parsed values as properties. + # You can also parse out the browser/device/os versions. + properties.update({ + "$browser": parsed["user_agent"]["family"], + "$device": parsed["device"]["family"], + "$os": parsed["os"]["family"], + }) + + if properties["user_id"] is None: + user_id = get_jwt()["sub"] + else: + user_id = properties["user_id"] + + properties["ip"] = request.remote_addr + mp.track(user_id, event_name, properties) diff --git a/backend/routes/auth.py b/backend/routes/auth.py index 6659ac6e..f3e505b5 100644 --- a/backend/routes/auth.py +++ b/backend/routes/auth.py @@ -9,6 +9,7 @@ ) from pydantic.main import BaseModel from ..auth import min_role_required, user_manager +from ..mixpanel.mix import track_to_mp from ..database import User, UserRole, db from ..dto import LoginUserDTO, RegisterUserDTO from ..schemas import UserSchema, validate @@ -94,6 +95,11 @@ def register(): } ) set_access_cookies(resp, token) + + track_to_mp(request, "register", { + 'user_id': user.id, + 'success': True, + }) return resp, 200 # In case of missing fields, return error message indicating # required fields. diff --git a/backend/routes/incidents.py b/backend/routes/incidents.py index 6f376b2b..7170be51 100644 --- a/backend/routes/incidents.py +++ b/backend/routes/incidents.py @@ -2,6 +2,7 @@ from typing import Optional from backend.auth.jwt import min_role_required, contributor_has_partner +from backend.mixpanel.mix import track_to_mp from backend.database.models.user import UserRole from flask import Blueprint, abort, current_app, request from flask_jwt_extended.view_decorators import jwt_required @@ -47,6 +48,8 @@ def create_incident(): abort(400) created = incident.create() + track_to_mp(request, "create_incident", { + "source_id": incident.source_id}) return incident_orm_to_json(created) @@ -107,6 +110,12 @@ def search_incidents(): ) try: + track_to_mp(request, "search_incidents", { + "description": body.description, + "location": body.location, + "dateStart": body.dateStart, + "dateEnd": body.dateEnd, + }) return { "results": [ incident_orm_to_json(result) for result in results.items @@ -116,4 +125,4 @@ def search_incidents(): "totalResults": results.total, } except Exception as e: - abort(500, description=str(e)) + abort(500, description=str(e)) \ No newline at end of file diff --git a/backend/routes/partners.py b/backend/routes/partners.py index b2df88ab..72058e98 100644 --- a/backend/routes/partners.py +++ b/backend/routes/partners.py @@ -1,4 +1,5 @@ from backend.auth.jwt import min_role_required +from backend.mixpanel.mix import track_to_mp from backend.database.models.user import User, UserRole from flask import Blueprint, abort, current_app, request from flask_jwt_extended import get_jwt @@ -48,11 +49,15 @@ def create_partner(): created = partner.create() make_admin = PartnerMember( partner_id=created.id, - user_id=User.get(get_jwt()["sub"]).id, + user_id=get_jwt()["sub"], role=MemberRole.ADMIN, ) make_admin.create() + track_to_mp(request, "create_partner", { + "partner_name": partner.name, + "partner_contact": partner.contact_email + }) return partner_orm_to_json(created) @@ -182,4 +187,9 @@ def add_member_to_partner(partner_id: int): created = partner_member.create() + track_to_mp(request, "add_partner_member", { + "partner_id": partner_id, + "user_id": partner_member.user_id, + "role": partner_member.role, + }) return partner_member_orm_to_json(created) diff --git a/requirements/_core.in b/requirements/_core.in index 2be53650..36578321 100644 --- a/requirements/_core.in +++ b/requirements/_core.in @@ -32,4 +32,6 @@ openpyxl xlsxwriter numpy spectree -jupyter \ No newline at end of file +jupyter +mixpanel +ua-parser \ No newline at end of file diff --git a/requirements/dev_unix.txt b/requirements/dev_unix.txt index 1a59f603..6474ce9a 100644 --- a/requirements/dev_unix.txt +++ b/requirements/dev_unix.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements/dev_unix.in # @@ -205,6 +205,8 @@ mirakuru==2.4.1 # via pytest-postgresql mistune==0.8.4 # via nbconvert +mixpanel==4.10.0 + # via -r requirements/_core.in mypy-extensions==0.4.3 # via black nbclient==0.5.13 @@ -362,7 +364,9 @@ qtpy==2.0.1 regex==2021.4.4 # via black requests==2.25.1 - # via -r requirements/_core.in + # via + # -r requirements/_core.in + # mixpanel send2trash==1.8.0 # via notebook six==1.16.0 @@ -372,6 +376,7 @@ six==1.16.0 # bleach # click-repl # flask-cors + # mixpanel # python-dateutil # sqlalchemy-utils soupsieve==2.3.1 @@ -424,8 +429,12 @@ traitlets==5.1.1 # qtconsole typing-extensions==3.10.0.0 # via pydantic +ua-parser==0.18.0 + # via -r requirements/_core.in urllib3==1.26.5 - # via requests + # via + # mixpanel + # requests vine==5.0.0 # via # amqp diff --git a/requirements/dev_windows.txt b/requirements/dev_windows.txt index 8b9e8a33..3573529a 100644 --- a/requirements/dev_windows.txt +++ b/requirements/dev_windows.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements/dev_windows.in # @@ -205,6 +205,8 @@ mirakuru==2.4.1 # via pytest-postgresql mistune==0.8.4 # via nbconvert +mixpanel==4.10.0 + # via -r requirements/_core.in mypy-extensions==0.4.3 # via black nbclient==0.5.13 @@ -362,7 +364,9 @@ qtpy==2.0.1 regex==2021.4.4 # via black requests==2.25.1 - # via -r requirements/_core.in + # via + # -r requirements/_core.in + # mixpanel send2trash==1.8.0 # via notebook six==1.16.0 @@ -372,6 +376,7 @@ six==1.16.0 # bleach # click-repl # flask-cors + # mixpanel # python-dateutil # sqlalchemy-utils soupsieve==2.3.1 @@ -424,8 +429,12 @@ traitlets==5.1.1 # qtconsole typing-extensions==3.10.0.0 # via pydantic +ua-parser==0.18.0 + # via -r requirements/_core.in urllib3==1.26.5 - # via requests + # via + # mixpanel + # requests vine==5.0.0 # via # amqp diff --git a/requirements/docs.txt b/requirements/docs.txt index beb862b4..b4969fee 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements/docs.in # diff --git a/requirements/prod.txt b/requirements/prod.txt index ea36c678..d0209f5e 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile requirements/prod.in # @@ -205,6 +205,8 @@ mirakuru==2.4.1 # via pytest-postgresql mistune==0.8.4 # via nbconvert +mixpanel==4.10.0 + # via -r requirements/_core.in mypy-extensions==0.4.3 # via black nbclient==0.5.13 @@ -362,7 +364,9 @@ qtpy==2.0.1 regex==2021.4.4 # via black requests==2.25.1 - # via -r requirements/_core.in + # via + # -r requirements/_core.in + # mixpanel send2trash==1.8.0 # via notebook six==1.16.0 @@ -372,6 +376,7 @@ six==1.16.0 # bleach # click-repl # flask-cors + # mixpanel # python-dateutil # sqlalchemy-utils soupsieve==2.3.1 @@ -424,8 +429,12 @@ traitlets==5.1.1 # qtconsole typing-extensions==3.10.0.0 # via pydantic +ua-parser==0.18.0 + # via -r requirements/_core.in urllib3==1.26.5 - # via requests + # via + # mixpanel + # requests vine==5.0.0 # via # amqp From db421778626f4469a345b35140df89256b2593d5 Mon Sep 17 00:00:00 2001 From: Darrell Malone Jr Date: Mon, 21 Aug 2023 22:06:48 -0500 Subject: [PATCH 02/10] Mixpanel Token Not leading --- backend/config.py | 4 ++++ backend/mixpanel/mix.py | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/config.py b/backend/config.py index 308f3d2e..9f8407b4 100644 --- a/backend/config.py +++ b/backend/config.py @@ -59,6 +59,10 @@ def SQLALCHEMY_DATABASE_URI(self): self.POSTGRES_DB, ) + @property + def MIXPANEL_TOKEN(self): + return os.environ.get("MIXPANEL_TOKEN", None) + SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_ECHO = False diff --git a/backend/mixpanel/mix.py b/backend/mixpanel/mix.py index c85c4693..05234c62 100644 --- a/backend/mixpanel/mix.py +++ b/backend/mixpanel/mix.py @@ -1,12 +1,14 @@ from mixpanel import Mixpanel +from backend.config import Config from ua_parser import user_agent_parser from flask_jwt_extended import get_jwt -from backend import config # Path: backend/mixpanel/mix.py +config = Config() + # Mixpanel connection -mp = Mixpanel.init(config.MIXPANEL_TOKEN) +mp = Mixpanel(config.MIXPANEL_TOKEN) def track_to_mp(request, event_name, properties): @@ -20,7 +22,7 @@ def track_to_mp(request, event_name, properties): "$os": parsed["os"]["family"], }) - if properties["user_id"] is None: + if "user_id" not in properties: user_id = get_jwt()["sub"] else: user_id = properties["user_id"] From 32ce37322e679d70ad0bfdc3f7cd07d2dd813ac0 Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 29 Aug 2023 20:24:14 -0400 Subject: [PATCH 03/10] pass MIXPANEL_TOKEN env var --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index cf722a0a..a95312ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,7 @@ services: POSTGRES_HOST: db FLASK_ENV: ${FLASK_ENV:-development} WAIT_HOSTS: db:${POSTGRES_PORT:-5432} + MIXPANEL_TOKEN: ${MIXPANEL_TOKEN:-notset} ports: - ${PDT_API_PORT:-5000}:${PDT_API_PORT:-5000} From caa26d16d36510ca520641a143949c04b4668ca2 Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 14 Nov 2023 19:55:34 -0500 Subject: [PATCH 04/10] fix CORS with next config; move mp tracking up in incident search --- backend/routes/incidents.py | 13 +++++++------ frontend/next.config.js | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/backend/routes/incidents.py b/backend/routes/incidents.py index 7170be51..57f47ca6 100644 --- a/backend/routes/incidents.py +++ b/backend/routes/incidents.py @@ -109,13 +109,14 @@ def search_incidents(): page=body.page, per_page=body.perPage, max_per_page=100 ) + track_to_mp(request, "search_incidents", { + "description": body.description, + "location": body.location, + "dateStart": body.dateStart, + "dateEnd": body.dateEnd, + }) + try: - track_to_mp(request, "search_incidents", { - "description": body.description, - "location": body.location, - "dateStart": body.dateStart, - "dateEnd": body.dateEnd, - }) return { "results": [ incident_orm_to_json(result) for result in results.items diff --git a/frontend/next.config.js b/frontend/next.config.js index f450b757..e5e181e6 100644 --- a/frontend/next.config.js +++ b/frontend/next.config.js @@ -1,4 +1,22 @@ module.exports = { + headers() { + return [ + { + // matching all API routes + source: "/api/:path*", + headers: [ + { key: "Access-Control-Allow-Credentials", value: "true" }, + { key: "Access-Control-Allow-Origin", value: "*" }, // replace this your actual origin + { key: "Access-Control-Allow-Methods", value: "GET,DELETE,PATCH,POST,PUT" }, + { + key: "Access-Control-Allow-Headers", + value: + "X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version" + } + ] + } + ] + }, eslint: { dirs: ["tests", "shared-components", "pages", "models", "helpers", "compositions"] } From 36e9ec82f6a0362764de28b6cf9f6333eac14cfd Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 14 Nov 2023 20:01:14 -0500 Subject: [PATCH 05/10] | fix no newline --- backend/routes/incidents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/routes/incidents.py b/backend/routes/incidents.py index 57f47ca6..816774e9 100644 --- a/backend/routes/incidents.py +++ b/backend/routes/incidents.py @@ -126,4 +126,4 @@ def search_incidents(): "totalResults": results.total, } except Exception as e: - abort(500, description=str(e)) \ No newline at end of file + abort(500, description=str(e)) From 983d0ebaaf56a6bf1df437c535cf9ce57f2338ac Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 14 Nov 2023 20:19:07 -0500 Subject: [PATCH 06/10] | catch mixpanel error --- backend/routes/incidents.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/backend/routes/incidents.py b/backend/routes/incidents.py index 816774e9..10be67b1 100644 --- a/backend/routes/incidents.py +++ b/backend/routes/incidents.py @@ -81,6 +81,7 @@ def search_incidents(): """Search Incidents.""" body: SearchIncidentsSchema = request.context.json query = db.session.query(Incident) + logger = logging.getLogger('incidents') try: if body.location: @@ -109,12 +110,15 @@ def search_incidents(): page=body.page, per_page=body.perPage, max_per_page=100 ) - track_to_mp(request, "search_incidents", { - "description": body.description, - "location": body.location, - "dateStart": body.dateStart, - "dateEnd": body.dateEnd, - }) + try: + track_to_mp(request, "search_incidents", { + "description": body.description, + "location": body.location, + "dateStart": body.dateStart, + "dateEnd": body.dateEnd, + }) + except MixpanelException as e: + logger.error(e); try: return { From 519146379b66b054cea7157f53712929e311f02a Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 14 Nov 2023 20:34:46 -0500 Subject: [PATCH 07/10] | fix imports --- backend/routes/incidents.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/routes/incidents.py b/backend/routes/incidents.py index 10be67b1..d832c6d6 100644 --- a/backend/routes/incidents.py +++ b/backend/routes/incidents.py @@ -1,8 +1,10 @@ +import logging from datetime import datetime from typing import Optional from backend.auth.jwt import min_role_required, contributor_has_partner from backend.mixpanel.mix import track_to_mp +from mixpanel import MixpanelException from backend.database.models.user import UserRole from flask import Blueprint, abort, current_app, request from flask_jwt_extended.view_decorators import jwt_required From 7343c755acf60593acc7b1bbd3eca221d1c7927b Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 14 Nov 2023 20:37:24 -0500 Subject: [PATCH 08/10] fix javascript bad habit --- backend/routes/incidents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/routes/incidents.py b/backend/routes/incidents.py index d832c6d6..a33b528f 100644 --- a/backend/routes/incidents.py +++ b/backend/routes/incidents.py @@ -120,7 +120,7 @@ def search_incidents(): "dateEnd": body.dateEnd, }) except MixpanelException as e: - logger.error(e); + logger.error(e) try: return { From fae9766bcf014a1bacc3805560ff3319a81a55e5 Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 21 Nov 2023 20:27:32 -0500 Subject: [PATCH 09/10] add MIXPANEL_TOKEN to test config --- backend/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/config.py b/backend/config.py index 9f8407b4..cffc281b 100644 --- a/backend/config.py +++ b/backend/config.py @@ -94,6 +94,7 @@ class TestingConfig(Config): POSTGRES_DB = "police_data_test" SECRET_KEY = "my-secret-key" JWT_SECRET_KEY = "my-jwt-secret-key" + MIXPANEL_TOKEN = "mixpanel-token" def get_config_from_env(env: str) -> Config: From 466d990126d4182a423a20f66fecbf3310bc2563 Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 21 Nov 2023 20:33:05 -0500 Subject: [PATCH 10/10] | add MIXPANEL_TOKEN to test action yml --- .github/workflows/python-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 4b482326..f8142af5 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -41,3 +41,4 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: police_data + MIXPANEL_TOKEN: mixpanel_token