From 33ded2cfd3427c10491f252d11fabae9fd873089 Mon Sep 17 00:00:00 2001 From: jburchard Date: Sat, 7 Oct 2023 16:48:08 -0500 Subject: [PATCH 01/10] Push code. --- docs/empower.rst | 47 ++++ docs/index.rst | 1 + parsons/__init__.py | 1 + parsons/empower/__init__.py | 3 + parsons/empower/empower.py | 256 ++++++++++++++++++ test/test_empower/dummy_empower_data.py | 337 ++++++++++++++++++++++++ test/test_empower/test_empower.py | 164 ++++++++++++ 7 files changed, 809 insertions(+) create mode 100644 docs/empower.rst create mode 100644 parsons/empower/__init__.py create mode 100644 parsons/empower/empower.py create mode 100644 test/test_empower/dummy_empower_data.py create mode 100644 test/test_empower/test_empower.py diff --git a/docs/empower.rst b/docs/empower.rst new file mode 100644 index 0000000000..1d2a43e2b7 --- /dev/null +++ b/docs/empower.rst @@ -0,0 +1,47 @@ +Empower +======= + +******** +Overview +******** + +The Empower class allows you to interact with the Empower API. Documentation for the Empower API can be found +in their `GitHub `_ repo. + +.. note:: + The Empower API only has a single endpoint to access all account data. As such, it has a very high overhead. This + connector employs caching in order to allow the user to specify the tables to extract without additional API calls. + You can disable caching as an argument when instantiating the class. + +========== +Quickstart +========== + +To instantiate the Empower class, you can either store your ``EMPOWER_API_KEY`` an environment +variables or pass them in as arguments: + +.. code-block:: python + + from parsons import Empower + + # First approach: Use API key environment variables + + # In bash, set your environment variables like so: + # export EMPOWER_API_KEY='MY_API_KEY' + empower = Empower() + + # Second approach: Pass API keys as arguments + empower = Empower(api_key='MY_API_KEY') + +You can then request tables in the following manner: + +.. code-block:: python + + tbl = empower.get_profiles() + +*** +API +*** + +.. autoclass :: parsons.Empower + :inherited-members: \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 093849d15a..0604998c04 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -199,6 +199,7 @@ Indices and tables crowdtangle databases donorbox + empower facebook_ads freshdesk github diff --git a/parsons/__init__.py b/parsons/__init__.py index 132d90db75..c292a9c3ea 100644 --- a/parsons/__init__.py +++ b/parsons/__init__.py @@ -87,6 +87,7 @@ ("parsons.turbovote.turbovote", "TurboVote"), ("parsons.twilio.twilio", "Twilio"), ("parsons.zoom.zoom", "Zoom"), + ("parsons.empower.empower", "Empower"), ): try: globals()[connector_name] = getattr( diff --git a/parsons/empower/__init__.py b/parsons/empower/__init__.py new file mode 100644 index 0000000000..9e660e7165 --- /dev/null +++ b/parsons/empower/__init__.py @@ -0,0 +1,3 @@ +from parsons.empower.empower import Empower + +__all__ = ["Empower"] diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py new file mode 100644 index 0000000000..b060b3a6a5 --- /dev/null +++ b/parsons/empower/empower.py @@ -0,0 +1,256 @@ +from parsons.utilities.api_connector import APIConnector +from parsons.utilities import check_env +from parsons.etl import Table +import logging +from datetime import datetime + +logger = logging.getLogger(__name__) + +EMPOWER_API_ENDPOINT = "https://api.getempower.com/v1/export" + + +class Empower(object): + """ + Instantiate class. + + `Args:` + api_key: str + The Empower provided API key.The Empower provided Client UUID. Not + required if ``EMPOWER_API_KEY`` env variable set. + empower_uri: str + The URI to access the Empower API. The default is currently set to + https://api.getempower.com/v1/export. You can set an ``EMPOWER_URI`` env + variable or use this URI parameter if a different endpoint is necessary. + cache: boolean + The Empower API returns all account data after each call. Setting cache + to ``True`` stores the blob and then extracts Parsons tables for each method. + Setting cache to ``False`` will download all account data for each method call. + """ + + def __init__(self, api_key=None, empower_uri=None, cache=True): + + self.api_key = check_env.check("EMPOWER_API_KEY", api_key) + self.empower_uri = ( + check_env.check("EMPOWER_URI", empower_uri, optional=True) + or EMPOWER_API_ENDPOINT + ) + self.headers = {"accept": "application/json", "secret-token": self.api_key} + self.client = APIConnector( + self.empower_uri, + headers=self.headers, + ) + self.data = None + self.data = self._get_data(cache) + + def _get_data(self, cache): + """ + Gets fresh data from Empower API based on cache setting. + """ + + if not cache or self.data is None: + r = self.client.get_request(self.empower_uri) + logger.info("Empower data downloaded.") + return r + + else: + return self.data + + def _unix_convert(self, ts): + """ + Converts UNIX timestamps to readable timestamps. + """ + + ts = datetime.utcfromtimestamp(int(ts) / 1000) + ts = ts.strftime("%Y-%m-%d %H:%M:%S UTC") + return ts + + def _empty_obj(self, obj_name): + """ + Determine if a dict object is empty. + """ + + if len(self.data[obj_name]) == 0: + return True + else: + return False + + def get_profiles(self): + """ + Get Empower profiles. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["profiles"]) + for col in ["createdMts", "lastUsedEmpowerMts", "updatedMts"]: + tbl.convert_column(col, lambda x: self._unix_convert(x)) + tbl.remove_column( + "activeCtaIds" + ) # Get as a method via get_profiles_active_ctas + return tbl + + def get_profiles_active_ctas(self): + """ + Get active ctas assigned to Empower profiles. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["profiles"]).long_table("eid", "activeCtaIds") + return tbl + + def get_regions(self): + """ + Get Empower regions. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["regions"]) + tbl.convert_column("inviteCodeCreatedMts", lambda x: self._unix_convert(x)) + return tbl + + def get_cta_results(self): + """ + Get Empower call to action results. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["ctaResults"]) + tbl.convert_column("contactedMts", lambda x: self._unix_convert(x)) + tbl = tbl.unpack_nested_columns_as_rows( + "answerIdsByPromptId", key="profileEid", expand_original=True + ) + tbl.unpack_list("answerIdsByPromptId_value", replace=True) + col_list = [v for v in tbl.columns if v.find("value") != -1] + tbl.coalesce_columns("answer_id", col_list, remove_source_columns=True) + tbl.remove_column("uid") + tbl.remove_column("answers") # Per docs, this is deprecated. + return tbl + + def _split_ctas(self): + """ + Internal method to split CTA objects into tables. + """ + + ctas = Table(self.data["ctas"]) + for col in [ + "createdMts", + "scheduledLaunchTimeMts", + "updatedMts", + "activeUntilMts", + ]: + ctas.convert_column(col, lambda x: self._unix_convert(x)) + ctas.remove_column("regionIds") # Get as a table via get_cta_regions() + ctas.remove_column("shareables") # Get as a table via get_cta_shareables() + ctas.remove_column( + "prioritizations" + ) # Get as a table via get_cta_prioritizations() + ctas.remove_column("questions") # This column has been deprecated. + cta_prompts = ctas.long_table( + "id", "prompts", prepend=False, retain_original=False + ) + cta_prompts.remove_column("ctaId") + cta_prompt_answers = cta_prompts.long_table("id", "answers", prepend=False) + + return [ctas, cta_prompts, cta_prompt_answers] + + def get_ctas(self): + """ + Get Empower calls to action. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + return self._split_ctas()[0] + + def get_cta_prompts(self): + """ + Get Empower calls to action prompts. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + return self._split_ctas()[1] + + def get_cta_prompt_answers(self): + """ + Get Empower calls to action prompt answers. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + return self._split_ctas()[2] + + def get_cta_regions(self): + """ + Get a list of regions that each call to active is active in. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["ctas"]).long_table("id", "regionIds") + return tbl + + def get_cta_shareables(self): + """ + Get a list of shareables associated with calls to action. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["ctas"]).long_table("id", "shareables") + return tbl + + def get_cta_prioritizations(self): + """ + Get a list prioritizations associated with calls to action. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["ctas"]).long_table("id", "prioritizations") + return tbl + + def get_outreach_entries(self): + """ + Get outreach entries. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + if self._empty_obj("outreachEntries"): + logger.info("No Outreach Entries found.") + return Table([]) + + tbl = Table(self.data["outreachEntries"]) + for col in [ + "outreachCreatedMts", + "outreachSnoozeUntilMts", + "outreachScheduledFollowUpMts", + ]: + tbl.convert_column(col, lambda x: self._unix_convert(x)) + logger.info(f"Unable to find column {col}") + return tbl diff --git a/test/test_empower/dummy_empower_data.py b/test/test_empower/dummy_empower_data.py new file mode 100644 index 0000000000..ae7a2c2338 --- /dev/null +++ b/test/test_empower/dummy_empower_data.py @@ -0,0 +1,337 @@ +dummy_data = { + "success": True, + "profiles": [ + { + "eid": "1zpyd08a6w9ulq", + "parentEid": "63nvsmpvx46870", + "role": "campaignDirector", + "firstName": "Bob", + "lastName": "Loblaw", + "email": "bob@gmail.com", + "phone": None, + "city": None, + "state": None, + "zip": None, + "address": None, + "address2": None, + "vanId": None, + "myCampaignVanId": None, + "lastUsedEmpowerMts": 1696522334107, + "notes": "", + "regionId": 5125, + "createdMts": 1695755343766, + "updatedMts": None, + "currentCtaId": 8311, + "activeCtaIds": [8311], + }, + { + "eid": "frzst1fnprr6n2", + "parentEid": "63nvsmpvx46870", + "role": "campaignDirector", + "firstName": "John", + "lastName": "Smith", + "email": "Jon@gmail.com", + "phone": None, + "city": None, + "state": None, + "zip": None, + "address": None, + "address2": None, + "vanId": None, + "myCampaignVanId": None, + "lastUsedEmpowerMts": 1696626131104, + "notes": "", + "regionId": 5125, + "createdMts": 1695755297395, + "updatedMts": 1696011335276, + "currentCtaId": 8311, + "activeCtaIds": [8311], + }, + { + "eid": "63nvsmpvx46870", + "parentEid": None, + "role": "campaignDirector", + "firstName": "Sally", + "lastName": "Jones", + "email": "sally@gmail.com", + "phone": None, + "city": None, + "state": None, + "zip": None, + "address": None, + "address2": None, + "vanId": None, + "myCampaignVanId": None, + "lastUsedEmpowerMts": 1695755125095, + "notes": "", + "regionId": 5125, + "createdMts": 1695752839622, + "updatedMts": 1695753353687, + "currentCtaId": 8311, + "activeCtaIds": [8311], + }, + ], + "ctas": [ + { + "id": 8156, + "name": "Let's get started!", + "description": "This is where you will get actions you can take to make change. Can you let us know how you want to get involved?", + "instructionsHtml": '

To get started, please answer the question below to complete your first action!

If you have any questions about using Empower, please use the "Have a question?" button below to contact your organizer.

', + "prompts": [ + { + "id": 16719, + "ctaId": 8156, + "promptText": "How do you want to volunteer?", + "vanId": None, + "isDeleted": False, + "answerInputType": "CHECKBOX", + "ordering": 1, + "dependsOnInitialDispositionResponse": None, + "answers": [ + { + "id": 55950, + "promptId": 16719, + "answerText": "Attend Events", + "vanId": None, + "isDeleted": False, + "ordering": 1, + }, + { + "id": 55951, + "promptId": 16719, + "answerText": "Talk to friends and family", + "vanId": None, + "isDeleted": False, + "ordering": 2, + }, + { + "id": 55952, + "promptId": 16719, + "answerText": "Canvassing", + "vanId": None, + "isDeleted": False, + "ordering": 3, + }, + { + "id": 55953, + "promptId": 16719, + "answerText": "Phone banking", + "vanId": None, + "isDeleted": False, + "ordering": 4, + }, + { + "id": 55954, + "promptId": 16719, + "answerText": "Text banking", + "vanId": None, + "isDeleted": False, + "ordering": 5, + }, + { + "id": 55955, + "promptId": 16719, + "answerText": "Other (specify in notes)", + "vanId": None, + "isDeleted": False, + "ordering": 6, + }, + ], + } + ], + "shareables": [], + "createdMts": 1695752752987, + "updatedMts": None, + "organizationId": 1095, + "regionIds": [5125], + "recruitmentQuestionType": "invite", + "recruitmentTrainingUrl": None, + "prioritizations": [], + "isIntroCta": True, + "scheduledLaunchTimeMts": 1695752752987, + "activeUntilMts": None, + "shouldUseAdvancedTargeting": False, + "advancedTargetingFilter": None, + "defaultPriorityLabelKey": None, + "actionType": "personal", + "spokeCampaignId": None, + "textCanvassingType": None, + "turfCuttingType": None, + "conversationStarter": None, + "isPersonal": True, + "isGeocodingDone": True, + "customRecruitmentPromptText": None, + "isBatchImportDone": True, + "hasAssignableTurfs": False, + "associatedElectionId": None, + "shouldDisplayElectionDayPollingLocation": False, + "shouldDisplayEarlyVotingPollingLocation": False, + "shouldShowMatchButton": False, + "questions": [ + { + "key": 1, + "type": "Normal", + "text": "How do you want to volunteer?", + "options": [ + "Attend Events", + "Talk to friends and family", + "Canvassing", + "Phone banking", + "Text banking", + "Other (specify in notes)", + ], + } + ], + }, + { + "id": 8311, + "name": "TEST", + "description": "Please test this for us!", + "instructionsHtml": "

Please load at least 25 contacts and mark random responses for them all.

", + "prompts": [], + "shareables": [ + { + "type": "link", + "url": "http://bioinfo.uib.es/~joemiro/RecEscr/PoliticsandEngLang.pdf", + "displayLabel": "Leisure reading", + } + ], + "createdMts": 1696624915388, + "updatedMts": None, + "organizationId": 1095, + "regionIds": [5125], + "recruitmentQuestionType": "training", + "recruitmentTrainingUrl": "https://forms.gle/Q1YX66WWQq8VGHeV8", + "prioritizations": [], + "isIntroCta": False, + "scheduledLaunchTimeMts": 1696624915388, + "activeUntilMts": None, + "shouldUseAdvancedTargeting": False, + "advancedTargetingFilter": None, + "defaultPriorityLabelKey": None, + "actionType": "relational", + "spokeCampaignId": None, + "textCanvassingType": "manual", + "turfCuttingType": None, + "conversationStarter": "Hey {recipientname} -- it's {sendername}. Have you considered voting yet this year?", + "isPersonal": False, + "isGeocodingDone": True, + "customRecruitmentPromptText": None, + "isBatchImportDone": True, + "hasAssignableTurfs": False, + "associatedElectionId": None, + "shouldDisplayElectionDayPollingLocation": False, + "shouldDisplayEarlyVotingPollingLocation": False, + "shouldShowMatchButton": False, + "questions": [], + }, + ], + "ctaResults": [ + { + "profileEid": "m48oukqkfxf6in", + "ctaId": 8311, + "contactedMts": 1696630492605, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "mqms2mh3hhen5y", + "ctaId": 8311, + "contactedMts": 1696630502526, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "eskhmei145adzx", + "ctaId": 8311, + "contactedMts": 1696630508282, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "xdm0xtu62jsf29", + "ctaId": 8311, + "contactedMts": 1696630513369, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "rciszn1qtuqxfj", + "ctaId": 8311, + "contactedMts": 1696630517176, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "480opsrmfjzns4", + "ctaId": 8311, + "contactedMts": 1696630520827, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "k6xgo161z4g6r2", + "ctaId": 8311, + "contactedMts": 1696628059352, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "ndpgftmbzc3gn8", + "ctaId": 8311, + "contactedMts": 1696628061778, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "yjg44ywsbfxfmq", + "ctaId": 8311, + "contactedMts": 1696628062789, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "fj7qxof19e0e6c", + "ctaId": 8311, + "contactedMts": 1696628063833, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "kncehygy5zvfru", + "ctaId": 8311, + "contactedMts": 1696628065465, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + ], + "regions": [ + { + "id": 5125, + "name": "Default", + "inviteCode": None, + "inviteCodeCreatedMts": None, + "ctaId": 8311, + "organizationId": 1095, + "description": "Volunteers who sign up using the organization's inviteCode will be sent here", + } + ], + "outreachEntries": [], + "profileOrganizationTags": [ + {"profileEid": "ah9xl5wlv3i2t3", "tagId": 456196}, + {"profileEid": "akwimque10cov3", "tagId": 456196}, + {"profileEid": "aq0dx4hm8esd7u", "tagId": 456196}, + + ], +} diff --git a/test/test_empower/test_empower.py b/test/test_empower/test_empower.py new file mode 100644 index 0000000000..eeb956873d --- /dev/null +++ b/test/test_empower/test_empower.py @@ -0,0 +1,164 @@ +import unittest +import requests_mock +from parsons import Empower, Table +from test.test_empower.dummy_empower_data import dummy_data + +TEST_EMPOWER_API_KEY = "MYKEY" +EMPOWER_API_ENDPOINT = "https://api.getempower.com/v1/export" + + +class TestEmpower(unittest.TestCase): + @requests_mock.Mocker() + def setUp(self, m): + m.get(EMPOWER_API_ENDPOINT, json=dummy_data) + self.empower = Empower(api_key=TEST_EMPOWER_API_KEY) + + @requests_mock.Mocker() + def test_get_profiles(self, m): + + exp_columns = [ + "eid", + "parentEid", + "role", + "firstName", + "lastName", + "email", + "phone", + "city", + "state", + "zip", + "address", + "address2", + "vanId", + "myCampaignVanId", + "lastUsedEmpowerMts", + "notes", + "regionId", + "createdMts", + "updatedMts", + "currentCtaId", + ] + + assert self.empower.get_profiles().columns == exp_columns + + @requests_mock.Mocker() + def test_get_profiles_active_ctas(self, m): + + exp_columns = ["eid", "activeCtaIds"] + + assert self.empower.get_profiles_active_ctas().columns == exp_columns + + @requests_mock.Mocker() + def test_get_regions(self, m): + + exp_columns = Table(dummy_data["regions"]).columns + + assert self.empower.get_regions().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_results(self, m): + + exp_columns = [ + "profileEid", + "ctaId", + "contactedMts", + "notes", + "answerIdsByPromptId", + "answer_id", + ] + + assert self.empower.get_cta_results().columns == exp_columns + + @requests_mock.Mocker() + def test_get_ctas(self, m): + + exp_columns = [ + "id", + "name", + "description", + "instructionsHtml", + "createdMts", + "updatedMts", + "organizationId", + "recruitmentQuestionType", + "recruitmentTrainingUrl", + "isIntroCta", + "scheduledLaunchTimeMts", + "activeUntilMts", + "shouldUseAdvancedTargeting", + "advancedTargetingFilter", + "defaultPriorityLabelKey", + "actionType", + "spokeCampaignId", + "textCanvassingType", + "turfCuttingType", + "conversationStarter", + "isPersonal", + "isGeocodingDone", + "customRecruitmentPromptText", + "isBatchImportDone", + "hasAssignableTurfs", + "associatedElectionId", + "shouldDisplayElectionDayPollingLocation", + "shouldDisplayEarlyVotingPollingLocation", + "shouldShowMatchButton", + ] + + assert self.empower.get_ctas().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_prompts(self, m): + + exp_columns = [ + "id", + "answerInputType", + "dependsOnInitialDispositionResponse", + "id", + "isDeleted", + "ordering", + "promptText", + "vanId", + ] + + assert self.empower.get_cta_prompts().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_prompt_answers(self, m): + + exp_columns = [ + "id", + "answerText", + "id", + "isDeleted", + "ordering", + "promptId", + "vanId", + ] + + assert self.empower.get_cta_prompt_answers().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_regions(self, m): + + exp_columns = ["id", "regionIds"] + + assert self.empower.get_cta_regions().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_shareables(self, m): + + exp_columns = [ + "id", + "shareables_displayLabel", + "shareables_type", + "shareables_url", + ] + + assert self.empower.get_cta_shareables().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_prioritizations(self, m): + + exp_columns = ["id", "prioritizations"] + + assert self.empower.get_cta_prioritizations().columns == exp_columns From ebf5c9d2d5439ac93aca7e6e3537875f3670bfa2 Mon Sep 17 00:00:00 2001 From: jburchard Date: Sat, 7 Oct 2023 16:59:41 -0500 Subject: [PATCH 02/10] Some linting that snuck through... --- test/test_empower/dummy_empower_data.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/test_empower/dummy_empower_data.py b/test/test_empower/dummy_empower_data.py index ae7a2c2338..f537dbd220 100644 --- a/test/test_empower/dummy_empower_data.py +++ b/test/test_empower/dummy_empower_data.py @@ -75,8 +75,8 @@ { "id": 8156, "name": "Let's get started!", - "description": "This is where you will get actions you can take to make change. Can you let us know how you want to get involved?", - "instructionsHtml": '

To get started, please answer the question below to complete your first action!

If you have any questions about using Empower, please use the "Have a question?" button below to contact your organizer.

', + "description": "This is where...", + "instructionsHtml": '

To get started...', "prompts": [ { "id": 16719, @@ -187,7 +187,7 @@ "id": 8311, "name": "TEST", "description": "Please test this for us!", - "instructionsHtml": "

Please load at least 25 contacts and mark random responses for them all.

", + "instructionsHtml": "

Please load at least 25", "prompts": [], "shareables": [ { @@ -213,7 +213,7 @@ "spokeCampaignId": None, "textCanvassingType": "manual", "turfCuttingType": None, - "conversationStarter": "Hey {recipientname} -- it's {sendername}. Have you considered voting yet this year?", + "conversationStarter": "Hey {recipientname} -- it's {sendername}.", "isPersonal": False, "isGeocodingDone": True, "customRecruitmentPromptText": None, @@ -324,7 +324,7 @@ "inviteCodeCreatedMts": None, "ctaId": 8311, "organizationId": 1095, - "description": "Volunteers who sign up using the organization's inviteCode will be sent here", + "description": "Volunteers who...", } ], "outreachEntries": [], @@ -332,6 +332,5 @@ {"profileEid": "ah9xl5wlv3i2t3", "tagId": 456196}, {"profileEid": "akwimque10cov3", "tagId": 456196}, {"profileEid": "aq0dx4hm8esd7u", "tagId": 456196}, - ], } From 082fa1032e91275298fc538efe62183c496cf56e Mon Sep 17 00:00:00 2001 From: jburchard Date: Sat, 7 Oct 2023 17:02:39 -0500 Subject: [PATCH 03/10] Black v2. --- test/test_empower/dummy_empower_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_empower/dummy_empower_data.py b/test/test_empower/dummy_empower_data.py index f537dbd220..c6b9e6fc7b 100644 --- a/test/test_empower/dummy_empower_data.py +++ b/test/test_empower/dummy_empower_data.py @@ -76,7 +76,7 @@ "id": 8156, "name": "Let's get started!", "description": "This is where...", - "instructionsHtml": '

To get started...', + "instructionsHtml": "

To get started...", "prompts": [ { "id": 16719, From 5314e44c15b6da86f4e4f8ca8d428ebf6500dce8 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:57:12 -0600 Subject: [PATCH 04/10] Update empower.py --- parsons/empower/empower.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py index b060b3a6a5..257f8e2cab 100644 --- a/parsons/empower/empower.py +++ b/parsons/empower/empower.py @@ -254,3 +254,16 @@ def get_outreach_entries(self): tbl.convert_column(col, lambda x: self._unix_convert(x)) logger.info(f"Unable to find column {col}") return tbl + + def get_raw_data(self): + """ + Get a table of the complete, raw data as returned by the API. + Meant to facilitate pure ELT pipelines + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data) + return tbl From f2650daaee8c6128b86f28f38819e616f34d32de Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:37:51 -0600 Subject: [PATCH 05/10] edit method name --- parsons/empower/empower.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py index 257f8e2cab..2a23163897 100644 --- a/parsons/empower/empower.py +++ b/parsons/empower/empower.py @@ -255,7 +255,7 @@ def get_outreach_entries(self): logger.info(f"Unable to find column {col}") return tbl - def get_raw_data(self): + def get_full_export(self): """ Get a table of the complete, raw data as returned by the API. Meant to facilitate pure ELT pipelines @@ -265,5 +265,5 @@ def get_raw_data(self): See :ref:`parsons-table` for output options. """ - tbl = Table(self.data) + tbl = Table([self.data]) return tbl From 64ec2845b1055709deed9cdd0941e1a19f3e36f1 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:50:07 -0600 Subject: [PATCH 06/10] ruff format --- parsons/empower/empower.py | 16 ++++------------ test/test_empower/test_empower.py | 10 ---------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py index 2a23163897..f2f553472d 100644 --- a/parsons/empower/empower.py +++ b/parsons/empower/empower.py @@ -28,11 +28,9 @@ class Empower(object): """ def __init__(self, api_key=None, empower_uri=None, cache=True): - self.api_key = check_env.check("EMPOWER_API_KEY", api_key) self.empower_uri = ( - check_env.check("EMPOWER_URI", empower_uri, optional=True) - or EMPOWER_API_ENDPOINT + check_env.check("EMPOWER_URI", empower_uri, optional=True) or EMPOWER_API_ENDPOINT ) self.headers = {"accept": "application/json", "secret-token": self.api_key} self.client = APIConnector( @@ -86,9 +84,7 @@ def get_profiles(self): tbl = Table(self.data["profiles"]) for col in ["createdMts", "lastUsedEmpowerMts", "updatedMts"]: tbl.convert_column(col, lambda x: self._unix_convert(x)) - tbl.remove_column( - "activeCtaIds" - ) # Get as a method via get_profiles_active_ctas + tbl.remove_column("activeCtaIds") # Get as a method via get_profiles_active_ctas return tbl def get_profiles_active_ctas(self): @@ -152,13 +148,9 @@ def _split_ctas(self): ctas.convert_column(col, lambda x: self._unix_convert(x)) ctas.remove_column("regionIds") # Get as a table via get_cta_regions() ctas.remove_column("shareables") # Get as a table via get_cta_shareables() - ctas.remove_column( - "prioritizations" - ) # Get as a table via get_cta_prioritizations() + ctas.remove_column("prioritizations") # Get as a table via get_cta_prioritizations() ctas.remove_column("questions") # This column has been deprecated. - cta_prompts = ctas.long_table( - "id", "prompts", prepend=False, retain_original=False - ) + cta_prompts = ctas.long_table("id", "prompts", prepend=False, retain_original=False) cta_prompts.remove_column("ctaId") cta_prompt_answers = cta_prompts.long_table("id", "answers", prepend=False) diff --git a/test/test_empower/test_empower.py b/test/test_empower/test_empower.py index eeb956873d..c501fab669 100644 --- a/test/test_empower/test_empower.py +++ b/test/test_empower/test_empower.py @@ -15,7 +15,6 @@ def setUp(self, m): @requests_mock.Mocker() def test_get_profiles(self, m): - exp_columns = [ "eid", "parentEid", @@ -43,21 +42,18 @@ def test_get_profiles(self, m): @requests_mock.Mocker() def test_get_profiles_active_ctas(self, m): - exp_columns = ["eid", "activeCtaIds"] assert self.empower.get_profiles_active_ctas().columns == exp_columns @requests_mock.Mocker() def test_get_regions(self, m): - exp_columns = Table(dummy_data["regions"]).columns assert self.empower.get_regions().columns == exp_columns @requests_mock.Mocker() def test_get_cta_results(self, m): - exp_columns = [ "profileEid", "ctaId", @@ -71,7 +67,6 @@ def test_get_cta_results(self, m): @requests_mock.Mocker() def test_get_ctas(self, m): - exp_columns = [ "id", "name", @@ -108,7 +103,6 @@ def test_get_ctas(self, m): @requests_mock.Mocker() def test_get_cta_prompts(self, m): - exp_columns = [ "id", "answerInputType", @@ -124,7 +118,6 @@ def test_get_cta_prompts(self, m): @requests_mock.Mocker() def test_get_cta_prompt_answers(self, m): - exp_columns = [ "id", "answerText", @@ -139,14 +132,12 @@ def test_get_cta_prompt_answers(self, m): @requests_mock.Mocker() def test_get_cta_regions(self, m): - exp_columns = ["id", "regionIds"] assert self.empower.get_cta_regions().columns == exp_columns @requests_mock.Mocker() def test_get_cta_shareables(self, m): - exp_columns = [ "id", "shareables_displayLabel", @@ -158,7 +149,6 @@ def test_get_cta_shareables(self, m): @requests_mock.Mocker() def test_get_cta_prioritizations(self, m): - exp_columns = ["id", "prioritizations"] assert self.empower.get_cta_prioritizations().columns == exp_columns From a950c17da01c8b947d23dcbc92fa6b16b38f9ff4 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Tue, 19 Nov 2024 15:33:10 -0600 Subject: [PATCH 07/10] respond to feedback --- docs/empower.rst | 12 +++++--- parsons/empower/empower.py | 58 ++++++++++++++++++----------------- parsons/utilities/datetime.py | 11 +++++++ 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/docs/empower.rst b/docs/empower.rst index 1d2a43e2b7..54b3064fac 100644 --- a/docs/empower.rst +++ b/docs/empower.rst @@ -8,17 +8,19 @@ Overview The Empower class allows you to interact with the Empower API. Documentation for the Empower API can be found in their `GitHub `_ repo. +The Empower API only has a single endpoint to access all account data. As such, it has a very high overhead. This +connector employs caching in order to allow the user to specify the tables to extract without additional API calls. +You can disable caching as an argument when instantiating the class. + .. note:: - The Empower API only has a single endpoint to access all account data. As such, it has a very high overhead. This - connector employs caching in order to allow the user to specify the tables to extract without additional API calls. - You can disable caching as an argument when instantiating the class. + To authenticate, request a secret token from Empower. ========== Quickstart ========== -To instantiate the Empower class, you can either store your ``EMPOWER_API_KEY`` an environment -variables or pass them in as arguments: +To instantiate the Empower class, you can either store your ``EMPOWER_API_KEY`` as an environment +variables or pass it in as an argument: .. code-block:: python diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py index f2f553472d..64f7ebbfac 100644 --- a/parsons/empower/empower.py +++ b/parsons/empower/empower.py @@ -1,8 +1,8 @@ from parsons.utilities.api_connector import APIConnector from parsons.utilities import check_env +from parsons.utilities.datetime import unix_convert from parsons.etl import Table import logging -from datetime import datetime logger = logging.getLogger(__name__) @@ -30,21 +30,21 @@ class Empower(object): def __init__(self, api_key=None, empower_uri=None, cache=True): self.api_key = check_env.check("EMPOWER_API_KEY", api_key) self.empower_uri = ( - check_env.check("EMPOWER_URI", empower_uri, optional=True) or EMPOWER_API_ENDPOINT + check_env.check("EMPOWER_URI", empower_uri, optional=True) + or EMPOWER_API_ENDPOINT ) self.headers = {"accept": "application/json", "secret-token": self.api_key} self.client = APIConnector( self.empower_uri, headers=self.headers, ) - self.data = None self.data = self._get_data(cache) def _get_data(self, cache): """ Gets fresh data from Empower API based on cache setting. """ - + self.get("data", None) if not cache or self.data is None: r = self.client.get_request(self.empower_uri) logger.info("Empower data downloaded.") @@ -53,15 +53,6 @@ def _get_data(self, cache): else: return self.data - def _unix_convert(self, ts): - """ - Converts UNIX timestamps to readable timestamps. - """ - - ts = datetime.utcfromtimestamp(int(ts) / 1000) - ts = ts.strftime("%Y-%m-%d %H:%M:%S UTC") - return ts - def _empty_obj(self, obj_name): """ Determine if a dict object is empty. @@ -83,8 +74,10 @@ def get_profiles(self): tbl = Table(self.data["profiles"]) for col in ["createdMts", "lastUsedEmpowerMts", "updatedMts"]: - tbl.convert_column(col, lambda x: self._unix_convert(x)) - tbl.remove_column("activeCtaIds") # Get as a method via get_profiles_active_ctas + tbl.convert_column(col, lambda x: unix_convert(x)) + tbl.remove_column( + "activeCtaIds" + ) # Get as a method via get_profiles_active_ctas return tbl def get_profiles_active_ctas(self): @@ -109,7 +102,7 @@ def get_regions(self): """ tbl = Table(self.data["regions"]) - tbl.convert_column("inviteCodeCreatedMts", lambda x: self._unix_convert(x)) + tbl.convert_column("inviteCodeCreatedMts", lambda x: unix_convert(x)) return tbl def get_cta_results(self): @@ -121,8 +114,9 @@ def get_cta_results(self): See :ref:`parsons-table` for output options. """ + # unpacks answerIdsByPromptId into standalone rows tbl = Table(self.data["ctaResults"]) - tbl.convert_column("contactedMts", lambda x: self._unix_convert(x)) + tbl.convert_column("contactedMts", lambda x: unix_convert(x)) tbl = tbl.unpack_nested_columns_as_rows( "answerIdsByPromptId", key="profileEid", expand_original=True ) @@ -145,16 +139,25 @@ def _split_ctas(self): "updatedMts", "activeUntilMts", ]: - ctas.convert_column(col, lambda x: self._unix_convert(x)) - ctas.remove_column("regionIds") # Get as a table via get_cta_regions() - ctas.remove_column("shareables") # Get as a table via get_cta_shareables() - ctas.remove_column("prioritizations") # Get as a table via get_cta_prioritizations() + ctas.convert_column(col, lambda x: unix_convert(x)) + # Get following data as their own tables via their own methods + ctas.remove_column("regionIds") # get_cta_regions() + ctas.remove_column("shareables") # get_cta_shareables() + ctas.remove_column("prioritizations") # get_cta_prioritizations() ctas.remove_column("questions") # This column has been deprecated. - cta_prompts = ctas.long_table("id", "prompts", prepend=False, retain_original=False) + + cta_prompts = ctas.long_table( + "id", "prompts", prepend=False, retain_original=False + ) cta_prompts.remove_column("ctaId") + cta_prompt_answers = cta_prompts.long_table("id", "answers", prepend=False) - return [ctas, cta_prompts, cta_prompt_answers] + return { + "ctas": ctas, + "cta_prompts": cta_prompts, + "cta_prompt_answers": cta_prompt_answers, + } def get_ctas(self): """ @@ -165,7 +168,7 @@ def get_ctas(self): See :ref:`parsons-table` for output options. """ - return self._split_ctas()[0] + return self._split_ctas()["ctas"] def get_cta_prompts(self): """ @@ -176,7 +179,7 @@ def get_cta_prompts(self): See :ref:`parsons-table` for output options. """ - return self._split_ctas()[1] + return self._split_ctas()["cta_prompts"] def get_cta_prompt_answers(self): """ @@ -187,7 +190,7 @@ def get_cta_prompt_answers(self): See :ref:`parsons-table` for output options. """ - return self._split_ctas()[2] + return self._split_ctas()["cta_prompt_answers"] def get_cta_regions(self): """ @@ -243,8 +246,7 @@ def get_outreach_entries(self): "outreachSnoozeUntilMts", "outreachScheduledFollowUpMts", ]: - tbl.convert_column(col, lambda x: self._unix_convert(x)) - logger.info(f"Unable to find column {col}") + tbl.convert_column(col, lambda x: unix_convert(x)) return tbl def get_full_export(self): diff --git a/parsons/utilities/datetime.py b/parsons/utilities/datetime.py index a3ace88700..d4fd6d69a1 100644 --- a/parsons/utilities/datetime.py +++ b/parsons/utilities/datetime.py @@ -25,6 +25,17 @@ def date_to_timestamp(value, tzinfo=datetime.timezone.utc): return int(parsed_date.timestamp()) +def unix_convert(ts): + """ + Converts UNIX timestamps to readable timestamps. + """ + + ts = datetime.utcfromtimestamp(int(ts) / 1000) + ts = ts.strftime("%Y-%m-%d %H:%M:%S UTC") + + return ts + + def parse_date(value, tzinfo=datetime.timezone.utc): """Parse an arbitrary date value into a Python datetime. From c811091b1877501b45567bc62f408ab5404c7bd6 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:49:36 -0600 Subject: [PATCH 08/10] rename function --- parsons/empower/empower.py | 14 ++++++++------ parsons/utilities/datetime.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py index 64f7ebbfac..77e618669b 100644 --- a/parsons/empower/empower.py +++ b/parsons/empower/empower.py @@ -1,6 +1,6 @@ from parsons.utilities.api_connector import APIConnector from parsons.utilities import check_env -from parsons.utilities.datetime import unix_convert +from parsons.utilities.datetime import convert_unix_to_readable from parsons.etl import Table import logging @@ -74,7 +74,7 @@ def get_profiles(self): tbl = Table(self.data["profiles"]) for col in ["createdMts", "lastUsedEmpowerMts", "updatedMts"]: - tbl.convert_column(col, lambda x: unix_convert(x)) + tbl.convert_column(col, lambda x: convert_unix_to_readable(x)) tbl.remove_column( "activeCtaIds" ) # Get as a method via get_profiles_active_ctas @@ -102,7 +102,9 @@ def get_regions(self): """ tbl = Table(self.data["regions"]) - tbl.convert_column("inviteCodeCreatedMts", lambda x: unix_convert(x)) + tbl.convert_column( + "inviteCodeCreatedMts", lambda x: convert_unix_to_readable(x) + ) return tbl def get_cta_results(self): @@ -116,7 +118,7 @@ def get_cta_results(self): # unpacks answerIdsByPromptId into standalone rows tbl = Table(self.data["ctaResults"]) - tbl.convert_column("contactedMts", lambda x: unix_convert(x)) + tbl.convert_column("contactedMts", lambda x: convert_unix_to_readable(x)) tbl = tbl.unpack_nested_columns_as_rows( "answerIdsByPromptId", key="profileEid", expand_original=True ) @@ -139,7 +141,7 @@ def _split_ctas(self): "updatedMts", "activeUntilMts", ]: - ctas.convert_column(col, lambda x: unix_convert(x)) + ctas.convert_column(col, lambda x: convert_unix_to_readable(x)) # Get following data as their own tables via their own methods ctas.remove_column("regionIds") # get_cta_regions() ctas.remove_column("shareables") # get_cta_shareables() @@ -246,7 +248,7 @@ def get_outreach_entries(self): "outreachSnoozeUntilMts", "outreachScheduledFollowUpMts", ]: - tbl.convert_column(col, lambda x: unix_convert(x)) + tbl.convert_column(col, lambda x: convert_unix_to_readable(x)) return tbl def get_full_export(self): diff --git a/parsons/utilities/datetime.py b/parsons/utilities/datetime.py index d4fd6d69a1..175dedb6de 100644 --- a/parsons/utilities/datetime.py +++ b/parsons/utilities/datetime.py @@ -25,7 +25,7 @@ def date_to_timestamp(value, tzinfo=datetime.timezone.utc): return int(parsed_date.timestamp()) -def unix_convert(ts): +def convert_unix_to_readable(ts): """ Converts UNIX timestamps to readable timestamps. """ From a25d9d184e5c34181d92eeda63640e0697cc0d98 Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:55:26 -0600 Subject: [PATCH 09/10] moving that logic didn't work --- parsons/empower/empower.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py index 77e618669b..8bad67c5c7 100644 --- a/parsons/empower/empower.py +++ b/parsons/empower/empower.py @@ -38,13 +38,13 @@ def __init__(self, api_key=None, empower_uri=None, cache=True): self.empower_uri, headers=self.headers, ) + self.data = None self.data = self._get_data(cache) def _get_data(self, cache): """ Gets fresh data from Empower API based on cache setting. """ - self.get("data", None) if not cache or self.data is None: r = self.client.get_request(self.empower_uri) logger.info("Empower data downloaded.") From 1d62e64f609858b407d4bf038e4b40c31041748e Mon Sep 17 00:00:00 2001 From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:37:34 -0600 Subject: [PATCH 10/10] ruff --- parsons/empower/empower.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py index 8bad67c5c7..b085787bf2 100644 --- a/parsons/empower/empower.py +++ b/parsons/empower/empower.py @@ -30,8 +30,7 @@ class Empower(object): def __init__(self, api_key=None, empower_uri=None, cache=True): self.api_key = check_env.check("EMPOWER_API_KEY", api_key) self.empower_uri = ( - check_env.check("EMPOWER_URI", empower_uri, optional=True) - or EMPOWER_API_ENDPOINT + check_env.check("EMPOWER_URI", empower_uri, optional=True) or EMPOWER_API_ENDPOINT ) self.headers = {"accept": "application/json", "secret-token": self.api_key} self.client = APIConnector( @@ -75,9 +74,7 @@ def get_profiles(self): tbl = Table(self.data["profiles"]) for col in ["createdMts", "lastUsedEmpowerMts", "updatedMts"]: tbl.convert_column(col, lambda x: convert_unix_to_readable(x)) - tbl.remove_column( - "activeCtaIds" - ) # Get as a method via get_profiles_active_ctas + tbl.remove_column("activeCtaIds") # Get as a method via get_profiles_active_ctas return tbl def get_profiles_active_ctas(self): @@ -102,9 +99,7 @@ def get_regions(self): """ tbl = Table(self.data["regions"]) - tbl.convert_column( - "inviteCodeCreatedMts", lambda x: convert_unix_to_readable(x) - ) + tbl.convert_column("inviteCodeCreatedMts", lambda x: convert_unix_to_readable(x)) return tbl def get_cta_results(self): @@ -148,9 +143,7 @@ def _split_ctas(self): ctas.remove_column("prioritizations") # get_cta_prioritizations() ctas.remove_column("questions") # This column has been deprecated. - cta_prompts = ctas.long_table( - "id", "prompts", prepend=False, retain_original=False - ) + cta_prompts = ctas.long_table("id", "prompts", prepend=False, retain_original=False) cta_prompts.remove_column("ctaId") cta_prompt_answers = cta_prompts.long_table("id", "answers", prepend=False)