From c657332f30621e4c8a5f2fd4ef418bd45a741198 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Wed, 4 Mar 2020 21:27:01 +0000 Subject: [PATCH 1/5] Remove whitelist of receipt keys, update tests --- tap_shopify/streams/transactions.py | 12 ++++++++---- tests/test_transaction_canonicalize.py | 10 +++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index a9652f1b..f6b1bff4 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -20,7 +20,13 @@ # prefer `token` if both are present and equal, convert `Token` -> `token` # if only `Token` is present, and throw an error if both are present and # their values are not equal -def canonicalize(transaction_dict, field_name): +def canonicalize(transaction_dict): + dict_keys = list(transaction_dict.get('receipt', {}).keys()) + for field_name in dict_keys: + if field_name.capitalize() == field_name: + canonicalize_field(transaction_dict, field_name.lower()) + +def canonicalize_field(transaction_dict, field_name): field_name_upper = field_name.capitalize() value_lower = transaction_dict.get('receipt', {}).get(field_name) value_upper = transaction_dict.get('receipt', {}).get(field_name_upper) @@ -46,7 +52,6 @@ def canonicalize(transaction_dict, field_name): elif value_upper: transaction_dict["receipt"][field_name] = transaction_dict['receipt'].pop(field_name_upper) - class Transactions(Stream): name = 'transactions' replication_key = 'created_at' @@ -89,8 +94,7 @@ def get_objects(self): def sync(self): for transaction in self.get_objects(): transaction_dict = transaction.to_dict() - for field_name in ['token', 'version', 'ack']: - canonicalize(transaction_dict, field_name) + canonicalize(transaction_dict) yield transaction_dict Context.stream_objects['transactions'] = Transactions diff --git a/tests/test_transaction_canonicalize.py b/tests/test_transaction_canonicalize.py index fe1168a2..068f20aa 100644 --- a/tests/test_transaction_canonicalize.py +++ b/tests/test_transaction_canonicalize.py @@ -7,28 +7,28 @@ def test_unmodified_if_not_present(self): # record to compare record = {"receipt": {"foo": "bar"}, "id": 2} expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "token") + canonicalize(record) self.assertEqual(record, expected_record) def test_unmodified_if_only_lower_exists(self): record = {"receipt": {"foo": "bar"}, "id": 2} expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") + canonicalize(record) self.assertEqual(record, expected_record) def test_lowercases_if_capital_only_exists(self): record = {"receipt": {"Foo": "bar"}, "id": 2} expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") + canonicalize(record) self.assertEqual(record, expected_record) def test_removes_uppercase_if_both_exist_and_are_equal(self): record = {"receipt": {"Foo": "bar", "foo": "bar"}, "id": 2} expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record, "foo") + canonicalize(record) self.assertEqual(record, expected_record) def test_throws_if_both_exist_and_are_not_equal(self): record = {"receipt": {"Foo": "bark", "foo": "bar"}, "id": 2} with self.assertRaises(ValueError): - canonicalize(record, "foo") + canonicalize(record) From 5ae88a1f70a0e73a8b7661182bf9b6ef14a29229 Mon Sep 17 00:00:00 2001 From: Andy Lu Date: Thu, 5 Mar 2020 15:06:46 +0000 Subject: [PATCH 2/5] Update function name --- tap_shopify/streams/transactions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index f6b1bff4..0f663d76 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -20,7 +20,7 @@ # prefer `token` if both are present and equal, convert `Token` -> `token` # if only `Token` is present, and throw an error if both are present and # their values are not equal -def canonicalize(transaction_dict): +def canonicalize_receipts(transaction_dict): dict_keys = list(transaction_dict.get('receipt', {}).keys()) for field_name in dict_keys: if field_name.capitalize() == field_name: @@ -94,7 +94,7 @@ def get_objects(self): def sync(self): for transaction in self.get_objects(): transaction_dict = transaction.to_dict() - canonicalize(transaction_dict) + canonicalize_receipts(transaction_dict) yield transaction_dict Context.stream_objects['transactions'] = Transactions From 965afd95a27aa17c036c1d5dc1db0722d8667fc2 Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Fri, 23 Apr 2021 17:32:35 +0000 Subject: [PATCH 3/5] Update test to call correct function --- tests/unittests/test_transaction_canonicalize.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unittests/test_transaction_canonicalize.py b/tests/unittests/test_transaction_canonicalize.py index 068f20aa..74ba6e98 100644 --- a/tests/unittests/test_transaction_canonicalize.py +++ b/tests/unittests/test_transaction_canonicalize.py @@ -1,34 +1,34 @@ import unittest -from tap_shopify.streams.transactions import canonicalize +from tap_shopify.streams.transactions import canonicalize_receipts -class TestTransactionCanonicalize(unittest.TestCase): +class TestTransactionCanonicalize_Receipts(unittest.TestCase): def test_unmodified_if_not_present(self): - # Note: Canonicalize has a side effect with pop(), must copy test + # Note: Canonicalize_Receipts has a side effect with pop(), must copy test # record to compare record = {"receipt": {"foo": "bar"}, "id": 2} expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record) + canonicalize_receipts(record) self.assertEqual(record, expected_record) def test_unmodified_if_only_lower_exists(self): record = {"receipt": {"foo": "bar"}, "id": 2} expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record) + canonicalize_receipts(record) self.assertEqual(record, expected_record) def test_lowercases_if_capital_only_exists(self): record = {"receipt": {"Foo": "bar"}, "id": 2} expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record) + canonicalize_receipts(record) self.assertEqual(record, expected_record) def test_removes_uppercase_if_both_exist_and_are_equal(self): record = {"receipt": {"Foo": "bar", "foo": "bar"}, "id": 2} expected_record = {"receipt": {"foo": "bar"}, "id": 2} - canonicalize(record) + canonicalize_receipts(record) self.assertEqual(record, expected_record) def test_throws_if_both_exist_and_are_not_equal(self): record = {"receipt": {"Foo": "bark", "foo": "bar"}, "id": 2} with self.assertRaises(ValueError): - canonicalize(record) + canonicalize_receipts(record) From 25321f3e3bda39a8add122491b22269d9b8ad0e5 Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Fri, 23 Apr 2021 17:46:51 +0000 Subject: [PATCH 4/5] Fix test class name --- tests/unittests/test_transaction_canonicalize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_transaction_canonicalize.py b/tests/unittests/test_transaction_canonicalize.py index 74ba6e98..69df3721 100644 --- a/tests/unittests/test_transaction_canonicalize.py +++ b/tests/unittests/test_transaction_canonicalize.py @@ -1,7 +1,7 @@ import unittest from tap_shopify.streams.transactions import canonicalize_receipts -class TestTransactionCanonicalize_Receipts(unittest.TestCase): +class TestTransactionCanonicalizeReceipts(unittest.TestCase): def test_unmodified_if_not_present(self): # Note: Canonicalize_Receipts has a side effect with pop(), must copy test # record to compare From 41d5e7d9908d93c8b4088577232b4b84ae6fbb1b Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Fri, 23 Apr 2021 18:00:58 +0000 Subject: [PATCH 5/5] Update function name to more accurately reflect work --- tap_shopify/streams/transactions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_shopify/streams/transactions.py b/tap_shopify/streams/transactions.py index 0f663d76..fe1569fa 100644 --- a/tap_shopify/streams/transactions.py +++ b/tap_shopify/streams/transactions.py @@ -24,9 +24,9 @@ def canonicalize_receipts(transaction_dict): dict_keys = list(transaction_dict.get('receipt', {}).keys()) for field_name in dict_keys: if field_name.capitalize() == field_name: - canonicalize_field(transaction_dict, field_name.lower()) + canonicalize_receipt_field(transaction_dict, field_name.lower()) -def canonicalize_field(transaction_dict, field_name): +def canonicalize_receipt_field(transaction_dict, field_name): field_name_upper = field_name.capitalize() value_lower = transaction_dict.get('receipt', {}).get(field_name) value_upper = transaction_dict.get('receipt', {}).get(field_name_upper)