From 95e9bec5759b1f930c95c9524b17a56271dc511b Mon Sep 17 00:00:00 2001 From: danielpdwalker Date: Tue, 11 Apr 2023 11:32:07 +0100 Subject: [PATCH 1/6] Fixed install link in README from shopify to google-sheets --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b5f3964..16cee35 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ Currently if have duplicate column names, a database will either: Use pip to install a release from GitHub. ```bash -pip install git+https://github.com/Matatika/tap-shopify@vx.x.x +pip install git+https://github.com/Matatika/tap-google-sheets@vx.x.x ``` ## Usage From 7f35b15c6c9bde46caffaef1caa98bb3e9726ec3 Mon Sep 17 00:00:00 2001 From: danielpdwalker Date: Tue, 11 Apr 2023 13:39:49 +0100 Subject: [PATCH 2/6] Added new optional setting key_properties to let users choose primary key columns from their google sheets --- tap_google_sheets/tap.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tap_google_sheets/tap.py b/tap_google_sheets/tap.py index 3f88b14..f87c8ee 100644 --- a/tap_google_sheets/tap.py +++ b/tap_google_sheets/tap.py @@ -45,6 +45,12 @@ class TapGoogleSheets(Tap): + " your Google Sheet", required=False, ), + th.Property( + "key_properties", + th.ArrayType(th.StringType), + description="Optionally choose one or more primary key columns", + required=False, + ), ).to_dict() def discover_streams(self) -> List[Stream]: @@ -53,6 +59,7 @@ def discover_streams(self) -> List[Stream]: stream_name = self.config.get("stream_name") or self.get_sheet_name() stream_name = stream_name.replace(" ", "_") + key_properties = self.config.get("key_properties", []) google_sheet_data = self.get_sheet_data() @@ -68,6 +75,7 @@ def discover_streams(self) -> List[Stream]: ) stream.child_sheet_name = child_sheet_name stream.selected + stream.primary_keys = key_properties streams.append(stream) return streams From 2697f6fd738632eef0ffdefe40a163ef3d26c2de Mon Sep 17 00:00:00 2001 From: danielpdwalker Date: Tue, 11 Apr 2023 13:44:30 +0100 Subject: [PATCH 3/6] Updated README with new key_properties setting --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16cee35..4360851 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Setting | Required | Type | Description | `sheet_id` | Required | String | Your target google sheet id `stream_name` | Optional | String | Optionailly rename the stream and output file or table from the tap `child_sheet_name` | Optional | String | Optionally choose a different sheet from your Google Sheet file +`key_properties` | Optional | Array of Strings | Optionally choose primary key column(s) from your Google Sheet file. Example: `["column_one", "column_two"]` ### Environment Variable @@ -66,6 +67,7 @@ These settings expand into environment variables of: - `TAP_GOOGLE_SHEETS_SHEET_ID` - `TAP_GOOGLE_SHEETS_STREAM_NAME` - `TAP_GOOGLE_SHEETS_CHILD_SHEET_NAME` +- `TAP_GOOGLE_SHEETS_KEY_PROPERTIES` --- @@ -85,6 +87,8 @@ These settings expand into environment variables of: * The tap will again replace any spaces in column names with underscores. +* When using the `key_properties` setting, you must choose columns with no null values. + ### Loaders Tested - [target-jsonl](https://hub.meltano.com/targets/jsonl) @@ -98,8 +102,6 @@ These settings expand into environment variables of: ## Roadmap - [ ] Add setting to optionally allow the selection of a range of data from a sheet. (Add an optional range setting). -- [ ] Add setting to enable primary key, and select primary key(s) column(s). - - [ ] Improve default behavior of a sheet with multiple columns of the same name and `target-postgres`. From 2c9bd566a946f17b81e1042f6540aa9cda0863af Mon Sep 17 00:00:00 2001 From: danielpdwalker Date: Tue, 11 Apr 2023 14:02:25 +0100 Subject: [PATCH 4/6] Added unit test for key_properties setting, asserting that once initialized, the tap's streams have the key_properties values set to the settings value --- .../tests/test_key_properties_setting.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tap_google_sheets/tests/test_key_properties_setting.py diff --git a/tap_google_sheets/tests/test_key_properties_setting.py b/tap_google_sheets/tests/test_key_properties_setting.py new file mode 100644 index 0000000..f62a6b4 --- /dev/null +++ b/tap_google_sheets/tests/test_key_properties_setting.py @@ -0,0 +1,56 @@ +"""Tests tap setting key_properties.""" + +import unittest + +import responses + +from tap_google_sheets.tap import TapGoogleSheets + + +class TestKeyPropertiesSetting(unittest.TestCase): + """Test class for tap setting key_properties""" + + def setUp(self): + self.mock_config = { + "oauth_credentials": { + "client_id": "123", + "client_secret": "123", + "refresh_token": "123", + }, + "sheet_id": "12345", + } + self.mock_config["key_properties"] = ["column_one", "column_two"] + + @responses.activate() + def test_key_properties_being_set_in_stream(self): + """""" + self.column_response = {"values": [["Column One", "Column Two"], ["1", "1"]]} + + responses.add( + responses.POST, + "https://oauth2.googleapis.com/token", + json={"access_token": "new_token"}, + status=200, + ), + responses.add( + responses.GET, + "https://www.googleapis.com/drive/v2/files/12345", + json={"title": "File Name One"}, + status=200, + ), + responses.add( + responses.GET, + "https://sheets.googleapis.com/v4/spreadsheets/12345/values/!1:1", + json={ + "range": "!1:1", + "values": [["Column One", "Column Two"]], + }, + status=200, + ) + + tap = TapGoogleSheets(config=self.mock_config) + + # Assert that the key_properties in a stream is equal to the setting key_properties + for stream in tap.catalog_dict.get("streams"): + self.assertEquals(stream.get("key_properties"), tap.config.get("key_properties")) + \ No newline at end of file From 5d7409362c486b607bf477f7abf812c06b9a8b4a Mon Sep 17 00:00:00 2001 From: danielpdwalker Date: Tue, 11 Apr 2023 14:07:22 +0100 Subject: [PATCH 5/6] Linting fixes --- tap_google_sheets/tests/test_ignoring_unnamed_columns.py | 1 - tap_google_sheets/tests/test_key_properties_setting.py | 5 +++-- tap_google_sheets/tests/test_underscoring_column_names.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tap_google_sheets/tests/test_ignoring_unnamed_columns.py b/tap_google_sheets/tests/test_ignoring_unnamed_columns.py index a1b40f1..0ffda7c 100644 --- a/tap_google_sheets/tests/test_ignoring_unnamed_columns.py +++ b/tap_google_sheets/tests/test_ignoring_unnamed_columns.py @@ -22,7 +22,6 @@ def setUp(self): @responses.activate() def test_ignoring_unnamed_columns(self): - self.missing_column_response = { "values": [ ["Column_One", "", "Column_Two"], diff --git a/tap_google_sheets/tests/test_key_properties_setting.py b/tap_google_sheets/tests/test_key_properties_setting.py index f62a6b4..4ec367b 100644 --- a/tap_google_sheets/tests/test_key_properties_setting.py +++ b/tap_google_sheets/tests/test_key_properties_setting.py @@ -52,5 +52,6 @@ def test_key_properties_being_set_in_stream(self): # Assert that the key_properties in a stream is equal to the setting key_properties for stream in tap.catalog_dict.get("streams"): - self.assertEquals(stream.get("key_properties"), tap.config.get("key_properties")) - \ No newline at end of file + self.assertEquals( + stream.get("key_properties"), tap.config.get("key_properties") + ) diff --git a/tap_google_sheets/tests/test_underscoring_column_names.py b/tap_google_sheets/tests/test_underscoring_column_names.py index 9bd0010..5673bbf 100644 --- a/tap_google_sheets/tests/test_underscoring_column_names.py +++ b/tap_google_sheets/tests/test_underscoring_column_names.py @@ -22,7 +22,6 @@ def setUp(self): @responses.activate() def test_underscoring_column_names(self): - self.column_response = {"values": [["Column One", "Column Two"], ["1", "1"]]} responses.add( From a48276c5dddf252e5a70393ee080b64b91821a82 Mon Sep 17 00:00:00 2001 From: danielpdwalker Date: Tue, 11 Apr 2023 14:11:07 +0100 Subject: [PATCH 6/6] Lint fixes --- tap_google_sheets/tests/test_key_properties_setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_google_sheets/tests/test_key_properties_setting.py b/tap_google_sheets/tests/test_key_properties_setting.py index 4ec367b..531ebbc 100644 --- a/tap_google_sheets/tests/test_key_properties_setting.py +++ b/tap_google_sheets/tests/test_key_properties_setting.py @@ -50,7 +50,7 @@ def test_key_properties_being_set_in_stream(self): tap = TapGoogleSheets(config=self.mock_config) - # Assert that the key_properties in a stream is equal to the setting key_properties + # Assert that key_properties in tap streams equal to the setting key_properties for stream in tap.catalog_dict.get("streams"): self.assertEquals( stream.get("key_properties"), tap.config.get("key_properties")