diff --git a/docs/empower.rst b/docs/empower.rst new file mode 100644 index 0000000000..54b3064fac --- /dev/null +++ b/docs/empower.rst @@ -0,0 +1,49 @@ +Empower +======= + +******** +Overview +******** + +The Empower class allows you to interact with the Empower API. Documentation for the Empower API can be found +in their `GitHub `_ repo. + +The Empower API only has a single endpoint to access all account data. As such, it has a very high overhead. This +connector employs caching in order to allow the user to specify the tables to extract without additional API calls. +You can disable caching as an argument when instantiating the class. + +.. note:: + To authenticate, request a secret token from Empower. + +========== +Quickstart +========== + +To instantiate the Empower class, you can either store your ``EMPOWER_API_KEY`` as an environment +variables or pass it in as an argument: + +.. code-block:: python + + from parsons import Empower + + # First approach: Use API key environment variables + + # In bash, set your environment variables like so: + # export EMPOWER_API_KEY='MY_API_KEY' + empower = Empower() + + # Second approach: Pass API keys as arguments + empower = Empower(api_key='MY_API_KEY') + +You can then request tables in the following manner: + +.. code-block:: python + + tbl = empower.get_profiles() + +*** +API +*** + +.. autoclass :: parsons.Empower + :inherited-members: \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index bb057fbca3..1f2bf8817d 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -199,6 +199,7 @@ Indices and tables crowdtangle databases donorbox + empower facebook_ads formstack freshdesk diff --git a/parsons/__init__.py b/parsons/__init__.py index 1219713152..5de93f62a8 100644 --- a/parsons/__init__.py +++ b/parsons/__init__.py @@ -91,6 +91,7 @@ ("parsons.turbovote.turbovote", "TurboVote"), ("parsons.twilio.twilio", "Twilio"), ("parsons.zoom.zoom", "Zoom"), + ("parsons.empower.empower", "Empower"), ): try: globals()[connector_name] = getattr(importlib.import_module(module_path), connector_name) diff --git a/parsons/empower/__init__.py b/parsons/empower/__init__.py new file mode 100644 index 0000000000..9e660e7165 --- /dev/null +++ b/parsons/empower/__init__.py @@ -0,0 +1,3 @@ +from parsons.empower.empower import Empower + +__all__ = ["Empower"] diff --git a/parsons/empower/empower.py b/parsons/empower/empower.py new file mode 100644 index 0000000000..b085787bf2 --- /dev/null +++ b/parsons/empower/empower.py @@ -0,0 +1,258 @@ +from parsons.utilities.api_connector import APIConnector +from parsons.utilities import check_env +from parsons.utilities.datetime import convert_unix_to_readable +from parsons.etl import Table +import logging + +logger = logging.getLogger(__name__) + +EMPOWER_API_ENDPOINT = "https://api.getempower.com/v1/export" + + +class Empower(object): + """ + Instantiate class. + + `Args:` + api_key: str + The Empower provided API key.The Empower provided Client UUID. Not + required if ``EMPOWER_API_KEY`` env variable set. + empower_uri: str + The URI to access the Empower API. The default is currently set to + https://api.getempower.com/v1/export. You can set an ``EMPOWER_URI`` env + variable or use this URI parameter if a different endpoint is necessary. + cache: boolean + The Empower API returns all account data after each call. Setting cache + to ``True`` stores the blob and then extracts Parsons tables for each method. + Setting cache to ``False`` will download all account data for each method call. + """ + + def __init__(self, api_key=None, empower_uri=None, cache=True): + self.api_key = check_env.check("EMPOWER_API_KEY", api_key) + self.empower_uri = ( + check_env.check("EMPOWER_URI", empower_uri, optional=True) or EMPOWER_API_ENDPOINT + ) + self.headers = {"accept": "application/json", "secret-token": self.api_key} + self.client = APIConnector( + self.empower_uri, + headers=self.headers, + ) + self.data = None + self.data = self._get_data(cache) + + def _get_data(self, cache): + """ + Gets fresh data from Empower API based on cache setting. + """ + if not cache or self.data is None: + r = self.client.get_request(self.empower_uri) + logger.info("Empower data downloaded.") + return r + + else: + return self.data + + def _empty_obj(self, obj_name): + """ + Determine if a dict object is empty. + """ + + if len(self.data[obj_name]) == 0: + return True + else: + return False + + def get_profiles(self): + """ + Get Empower profiles. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["profiles"]) + for col in ["createdMts", "lastUsedEmpowerMts", "updatedMts"]: + tbl.convert_column(col, lambda x: convert_unix_to_readable(x)) + tbl.remove_column("activeCtaIds") # Get as a method via get_profiles_active_ctas + return tbl + + def get_profiles_active_ctas(self): + """ + Get active ctas assigned to Empower profiles. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["profiles"]).long_table("eid", "activeCtaIds") + return tbl + + def get_regions(self): + """ + Get Empower regions. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["regions"]) + tbl.convert_column("inviteCodeCreatedMts", lambda x: convert_unix_to_readable(x)) + return tbl + + def get_cta_results(self): + """ + Get Empower call to action results. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + # unpacks answerIdsByPromptId into standalone rows + tbl = Table(self.data["ctaResults"]) + tbl.convert_column("contactedMts", lambda x: convert_unix_to_readable(x)) + tbl = tbl.unpack_nested_columns_as_rows( + "answerIdsByPromptId", key="profileEid", expand_original=True + ) + tbl.unpack_list("answerIdsByPromptId_value", replace=True) + col_list = [v for v in tbl.columns if v.find("value") != -1] + tbl.coalesce_columns("answer_id", col_list, remove_source_columns=True) + tbl.remove_column("uid") + tbl.remove_column("answers") # Per docs, this is deprecated. + return tbl + + def _split_ctas(self): + """ + Internal method to split CTA objects into tables. + """ + + ctas = Table(self.data["ctas"]) + for col in [ + "createdMts", + "scheduledLaunchTimeMts", + "updatedMts", + "activeUntilMts", + ]: + ctas.convert_column(col, lambda x: convert_unix_to_readable(x)) + # Get following data as their own tables via their own methods + ctas.remove_column("regionIds") # get_cta_regions() + ctas.remove_column("shareables") # get_cta_shareables() + ctas.remove_column("prioritizations") # get_cta_prioritizations() + ctas.remove_column("questions") # This column has been deprecated. + + cta_prompts = ctas.long_table("id", "prompts", prepend=False, retain_original=False) + cta_prompts.remove_column("ctaId") + + cta_prompt_answers = cta_prompts.long_table("id", "answers", prepend=False) + + return { + "ctas": ctas, + "cta_prompts": cta_prompts, + "cta_prompt_answers": cta_prompt_answers, + } + + def get_ctas(self): + """ + Get Empower calls to action. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + return self._split_ctas()["ctas"] + + def get_cta_prompts(self): + """ + Get Empower calls to action prompts. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + return self._split_ctas()["cta_prompts"] + + def get_cta_prompt_answers(self): + """ + Get Empower calls to action prompt answers. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + return self._split_ctas()["cta_prompt_answers"] + + def get_cta_regions(self): + """ + Get a list of regions that each call to active is active in. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["ctas"]).long_table("id", "regionIds") + return tbl + + def get_cta_shareables(self): + """ + Get a list of shareables associated with calls to action. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["ctas"]).long_table("id", "shareables") + return tbl + + def get_cta_prioritizations(self): + """ + Get a list prioritizations associated with calls to action. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table(self.data["ctas"]).long_table("id", "prioritizations") + return tbl + + def get_outreach_entries(self): + """ + Get outreach entries. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + if self._empty_obj("outreachEntries"): + logger.info("No Outreach Entries found.") + return Table([]) + + tbl = Table(self.data["outreachEntries"]) + for col in [ + "outreachCreatedMts", + "outreachSnoozeUntilMts", + "outreachScheduledFollowUpMts", + ]: + tbl.convert_column(col, lambda x: convert_unix_to_readable(x)) + return tbl + + def get_full_export(self): + """ + Get a table of the complete, raw data as returned by the API. + Meant to facilitate pure ELT pipelines + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + tbl = Table([self.data]) + return tbl diff --git a/parsons/utilities/datetime.py b/parsons/utilities/datetime.py index a3ace88700..175dedb6de 100644 --- a/parsons/utilities/datetime.py +++ b/parsons/utilities/datetime.py @@ -25,6 +25,17 @@ def date_to_timestamp(value, tzinfo=datetime.timezone.utc): return int(parsed_date.timestamp()) +def convert_unix_to_readable(ts): + """ + Converts UNIX timestamps to readable timestamps. + """ + + ts = datetime.utcfromtimestamp(int(ts) / 1000) + ts = ts.strftime("%Y-%m-%d %H:%M:%S UTC") + + return ts + + def parse_date(value, tzinfo=datetime.timezone.utc): """Parse an arbitrary date value into a Python datetime. diff --git a/test/test_empower/dummy_empower_data.py b/test/test_empower/dummy_empower_data.py new file mode 100644 index 0000000000..c6b9e6fc7b --- /dev/null +++ b/test/test_empower/dummy_empower_data.py @@ -0,0 +1,336 @@ +dummy_data = { + "success": True, + "profiles": [ + { + "eid": "1zpyd08a6w9ulq", + "parentEid": "63nvsmpvx46870", + "role": "campaignDirector", + "firstName": "Bob", + "lastName": "Loblaw", + "email": "bob@gmail.com", + "phone": None, + "city": None, + "state": None, + "zip": None, + "address": None, + "address2": None, + "vanId": None, + "myCampaignVanId": None, + "lastUsedEmpowerMts": 1696522334107, + "notes": "", + "regionId": 5125, + "createdMts": 1695755343766, + "updatedMts": None, + "currentCtaId": 8311, + "activeCtaIds": [8311], + }, + { + "eid": "frzst1fnprr6n2", + "parentEid": "63nvsmpvx46870", + "role": "campaignDirector", + "firstName": "John", + "lastName": "Smith", + "email": "Jon@gmail.com", + "phone": None, + "city": None, + "state": None, + "zip": None, + "address": None, + "address2": None, + "vanId": None, + "myCampaignVanId": None, + "lastUsedEmpowerMts": 1696626131104, + "notes": "", + "regionId": 5125, + "createdMts": 1695755297395, + "updatedMts": 1696011335276, + "currentCtaId": 8311, + "activeCtaIds": [8311], + }, + { + "eid": "63nvsmpvx46870", + "parentEid": None, + "role": "campaignDirector", + "firstName": "Sally", + "lastName": "Jones", + "email": "sally@gmail.com", + "phone": None, + "city": None, + "state": None, + "zip": None, + "address": None, + "address2": None, + "vanId": None, + "myCampaignVanId": None, + "lastUsedEmpowerMts": 1695755125095, + "notes": "", + "regionId": 5125, + "createdMts": 1695752839622, + "updatedMts": 1695753353687, + "currentCtaId": 8311, + "activeCtaIds": [8311], + }, + ], + "ctas": [ + { + "id": 8156, + "name": "Let's get started!", + "description": "This is where...", + "instructionsHtml": "

To get started...", + "prompts": [ + { + "id": 16719, + "ctaId": 8156, + "promptText": "How do you want to volunteer?", + "vanId": None, + "isDeleted": False, + "answerInputType": "CHECKBOX", + "ordering": 1, + "dependsOnInitialDispositionResponse": None, + "answers": [ + { + "id": 55950, + "promptId": 16719, + "answerText": "Attend Events", + "vanId": None, + "isDeleted": False, + "ordering": 1, + }, + { + "id": 55951, + "promptId": 16719, + "answerText": "Talk to friends and family", + "vanId": None, + "isDeleted": False, + "ordering": 2, + }, + { + "id": 55952, + "promptId": 16719, + "answerText": "Canvassing", + "vanId": None, + "isDeleted": False, + "ordering": 3, + }, + { + "id": 55953, + "promptId": 16719, + "answerText": "Phone banking", + "vanId": None, + "isDeleted": False, + "ordering": 4, + }, + { + "id": 55954, + "promptId": 16719, + "answerText": "Text banking", + "vanId": None, + "isDeleted": False, + "ordering": 5, + }, + { + "id": 55955, + "promptId": 16719, + "answerText": "Other (specify in notes)", + "vanId": None, + "isDeleted": False, + "ordering": 6, + }, + ], + } + ], + "shareables": [], + "createdMts": 1695752752987, + "updatedMts": None, + "organizationId": 1095, + "regionIds": [5125], + "recruitmentQuestionType": "invite", + "recruitmentTrainingUrl": None, + "prioritizations": [], + "isIntroCta": True, + "scheduledLaunchTimeMts": 1695752752987, + "activeUntilMts": None, + "shouldUseAdvancedTargeting": False, + "advancedTargetingFilter": None, + "defaultPriorityLabelKey": None, + "actionType": "personal", + "spokeCampaignId": None, + "textCanvassingType": None, + "turfCuttingType": None, + "conversationStarter": None, + "isPersonal": True, + "isGeocodingDone": True, + "customRecruitmentPromptText": None, + "isBatchImportDone": True, + "hasAssignableTurfs": False, + "associatedElectionId": None, + "shouldDisplayElectionDayPollingLocation": False, + "shouldDisplayEarlyVotingPollingLocation": False, + "shouldShowMatchButton": False, + "questions": [ + { + "key": 1, + "type": "Normal", + "text": "How do you want to volunteer?", + "options": [ + "Attend Events", + "Talk to friends and family", + "Canvassing", + "Phone banking", + "Text banking", + "Other (specify in notes)", + ], + } + ], + }, + { + "id": 8311, + "name": "TEST", + "description": "Please test this for us!", + "instructionsHtml": "

Please load at least 25", + "prompts": [], + "shareables": [ + { + "type": "link", + "url": "http://bioinfo.uib.es/~joemiro/RecEscr/PoliticsandEngLang.pdf", + "displayLabel": "Leisure reading", + } + ], + "createdMts": 1696624915388, + "updatedMts": None, + "organizationId": 1095, + "regionIds": [5125], + "recruitmentQuestionType": "training", + "recruitmentTrainingUrl": "https://forms.gle/Q1YX66WWQq8VGHeV8", + "prioritizations": [], + "isIntroCta": False, + "scheduledLaunchTimeMts": 1696624915388, + "activeUntilMts": None, + "shouldUseAdvancedTargeting": False, + "advancedTargetingFilter": None, + "defaultPriorityLabelKey": None, + "actionType": "relational", + "spokeCampaignId": None, + "textCanvassingType": "manual", + "turfCuttingType": None, + "conversationStarter": "Hey {recipientname} -- it's {sendername}.", + "isPersonal": False, + "isGeocodingDone": True, + "customRecruitmentPromptText": None, + "isBatchImportDone": True, + "hasAssignableTurfs": False, + "associatedElectionId": None, + "shouldDisplayElectionDayPollingLocation": False, + "shouldDisplayEarlyVotingPollingLocation": False, + "shouldShowMatchButton": False, + "questions": [], + }, + ], + "ctaResults": [ + { + "profileEid": "m48oukqkfxf6in", + "ctaId": 8311, + "contactedMts": 1696630492605, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "mqms2mh3hhen5y", + "ctaId": 8311, + "contactedMts": 1696630502526, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "eskhmei145adzx", + "ctaId": 8311, + "contactedMts": 1696630508282, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "xdm0xtu62jsf29", + "ctaId": 8311, + "contactedMts": 1696630513369, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "rciszn1qtuqxfj", + "ctaId": 8311, + "contactedMts": 1696630517176, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "480opsrmfjzns4", + "ctaId": 8311, + "contactedMts": 1696630520827, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "k6xgo161z4g6r2", + "ctaId": 8311, + "contactedMts": 1696628059352, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "ndpgftmbzc3gn8", + "ctaId": 8311, + "contactedMts": 1696628061778, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "yjg44ywsbfxfmq", + "ctaId": 8311, + "contactedMts": 1696628062789, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "fj7qxof19e0e6c", + "ctaId": 8311, + "contactedMts": 1696628063833, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + { + "profileEid": "kncehygy5zvfru", + "ctaId": 8311, + "contactedMts": 1696628065465, + "answerIdsByPromptId": {}, + "notes": None, + "answers": {}, + }, + ], + "regions": [ + { + "id": 5125, + "name": "Default", + "inviteCode": None, + "inviteCodeCreatedMts": None, + "ctaId": 8311, + "organizationId": 1095, + "description": "Volunteers who...", + } + ], + "outreachEntries": [], + "profileOrganizationTags": [ + {"profileEid": "ah9xl5wlv3i2t3", "tagId": 456196}, + {"profileEid": "akwimque10cov3", "tagId": 456196}, + {"profileEid": "aq0dx4hm8esd7u", "tagId": 456196}, + ], +} diff --git a/test/test_empower/test_empower.py b/test/test_empower/test_empower.py new file mode 100644 index 0000000000..c501fab669 --- /dev/null +++ b/test/test_empower/test_empower.py @@ -0,0 +1,154 @@ +import unittest +import requests_mock +from parsons import Empower, Table +from test.test_empower.dummy_empower_data import dummy_data + +TEST_EMPOWER_API_KEY = "MYKEY" +EMPOWER_API_ENDPOINT = "https://api.getempower.com/v1/export" + + +class TestEmpower(unittest.TestCase): + @requests_mock.Mocker() + def setUp(self, m): + m.get(EMPOWER_API_ENDPOINT, json=dummy_data) + self.empower = Empower(api_key=TEST_EMPOWER_API_KEY) + + @requests_mock.Mocker() + def test_get_profiles(self, m): + exp_columns = [ + "eid", + "parentEid", + "role", + "firstName", + "lastName", + "email", + "phone", + "city", + "state", + "zip", + "address", + "address2", + "vanId", + "myCampaignVanId", + "lastUsedEmpowerMts", + "notes", + "regionId", + "createdMts", + "updatedMts", + "currentCtaId", + ] + + assert self.empower.get_profiles().columns == exp_columns + + @requests_mock.Mocker() + def test_get_profiles_active_ctas(self, m): + exp_columns = ["eid", "activeCtaIds"] + + assert self.empower.get_profiles_active_ctas().columns == exp_columns + + @requests_mock.Mocker() + def test_get_regions(self, m): + exp_columns = Table(dummy_data["regions"]).columns + + assert self.empower.get_regions().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_results(self, m): + exp_columns = [ + "profileEid", + "ctaId", + "contactedMts", + "notes", + "answerIdsByPromptId", + "answer_id", + ] + + assert self.empower.get_cta_results().columns == exp_columns + + @requests_mock.Mocker() + def test_get_ctas(self, m): + exp_columns = [ + "id", + "name", + "description", + "instructionsHtml", + "createdMts", + "updatedMts", + "organizationId", + "recruitmentQuestionType", + "recruitmentTrainingUrl", + "isIntroCta", + "scheduledLaunchTimeMts", + "activeUntilMts", + "shouldUseAdvancedTargeting", + "advancedTargetingFilter", + "defaultPriorityLabelKey", + "actionType", + "spokeCampaignId", + "textCanvassingType", + "turfCuttingType", + "conversationStarter", + "isPersonal", + "isGeocodingDone", + "customRecruitmentPromptText", + "isBatchImportDone", + "hasAssignableTurfs", + "associatedElectionId", + "shouldDisplayElectionDayPollingLocation", + "shouldDisplayEarlyVotingPollingLocation", + "shouldShowMatchButton", + ] + + assert self.empower.get_ctas().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_prompts(self, m): + exp_columns = [ + "id", + "answerInputType", + "dependsOnInitialDispositionResponse", + "id", + "isDeleted", + "ordering", + "promptText", + "vanId", + ] + + assert self.empower.get_cta_prompts().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_prompt_answers(self, m): + exp_columns = [ + "id", + "answerText", + "id", + "isDeleted", + "ordering", + "promptId", + "vanId", + ] + + assert self.empower.get_cta_prompt_answers().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_regions(self, m): + exp_columns = ["id", "regionIds"] + + assert self.empower.get_cta_regions().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_shareables(self, m): + exp_columns = [ + "id", + "shareables_displayLabel", + "shareables_type", + "shareables_url", + ] + + assert self.empower.get_cta_shareables().columns == exp_columns + + @requests_mock.Mocker() + def test_get_cta_prioritizations(self, m): + exp_columns = ["id", "prioritizations"] + + assert self.empower.get_cta_prioritizations().columns == exp_columns