From b14f8d7bd9f79aa47af93a9593a36deb006dd3cf Mon Sep 17 00:00:00 2001 From: Adil Ahmed Date: Sat, 20 Apr 2024 02:26:29 +0500 Subject: [PATCH 1/7] Added attachable stream. --- tap_quickbooks/__init__.py | 2 +- tap_quickbooks/quickbooks/rest.py | 2 +- .../quickbooks/schemas/object_definition.json | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tap_quickbooks/__init__.py b/tap_quickbooks/__init__.py index 204fbae..ade826a 100644 --- a/tap_quickbooks/__init__.py +++ b/tap_quickbooks/__init__.py @@ -184,7 +184,7 @@ def do_sync(qb, catalog, state, state_passed): else: LOGGER.info("Starting sync") - for catalog_entry in catalog["streams"]: + for catalog_entry in catalog["streams"]: stream_version = get_stream_version(catalog_entry, state) stream = catalog_entry['stream'] stream_alias = catalog_entry.get('stream_alias') diff --git a/tap_quickbooks/quickbooks/rest.py b/tap_quickbooks/quickbooks/rest.py index 4d50545..bb832db 100644 --- a/tap_quickbooks/quickbooks/rest.py +++ b/tap_quickbooks/quickbooks/rest.py @@ -107,7 +107,7 @@ def _sync_records(self, url, headers, params, stream): headers.update(self.qb._get_standard_headers()) records_deleted = [] excluded_entities = ["Bill", "Payment", "Transfer", "CompanyInfo", "CreditMemo", "Invoice", - "JournalEntry", "Preferences", "Purchase", "SalesReceipt", "TimeActivity", "BillPayment","Estimate"] + "JournalEntry", "Preferences", "Purchase", "SalesReceipt", "TimeActivity", "BillPayment","Estimate","Attachable"] if self.qb.include_deleted and stream not in excluded_entities: # Get the deleted records first if "WHERE" in query: diff --git a/tap_quickbooks/quickbooks/schemas/object_definition.json b/tap_quickbooks/quickbooks/schemas/object_definition.json index e960141..9c902fa 100644 --- a/tap_quickbooks/quickbooks/schemas/object_definition.json +++ b/tap_quickbooks/quickbooks/schemas/object_definition.json @@ -666,5 +666,19 @@ {"name": "TaxExemptionRef", "type": "string"}, {"name": "HomeTotalAmt", "type": "string"}, {"name": "FreeFormAddress", "type": "boolean"} + ], + + "Attachable": [ + {"name": "FileName", "type": "string"}, + {"name": "FileAccessUri", "type": "string"}, + {"name": "TempDownloadUri", "type": "string"}, + {"name": "Size", "type": "number"}, + {"name": "ContentType", "type": "string"}, + {"name": "domain", "type": "string"}, + {"name": "sparse", "type": "boolean"}, + {"name": "Id", "type": "string"}, + {"name": "SyncToken", "type": "string"}, + {"name": "MetaData", "type": "string"}, + {"name": "AttachableRef", "type": "array","child_type": "string"} ] } From 3c9173f135167e97cf2492fe46973d84139030a2 Mon Sep 17 00:00:00 2001 From: xacadil <92389481+xacadil@users.noreply.github.com> Date: Sat, 20 Apr 2024 02:30:02 +0500 Subject: [PATCH 2/7] Added attachable stream. Cleanup. --- tap_quickbooks/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_quickbooks/__init__.py b/tap_quickbooks/__init__.py index ade826a..b1bf7bb 100644 --- a/tap_quickbooks/__init__.py +++ b/tap_quickbooks/__init__.py @@ -184,7 +184,7 @@ def do_sync(qb, catalog, state, state_passed): else: LOGGER.info("Starting sync") - for catalog_entry in catalog["streams"]: + for catalog_entry in catalog["streams"]: stream_version = get_stream_version(catalog_entry, state) stream = catalog_entry['stream'] stream_alias = catalog_entry.get('stream_alias') @@ -319,4 +319,4 @@ def main(): raise e if __name__ == "__main__": - main() \ No newline at end of file + main() From 6a0a64159329a8aa56a54f78b8f7cb2154af3db4 Mon Sep 17 00:00:00 2001 From: Adil Ahmed Date: Sat, 20 Apr 2024 02:32:54 +0500 Subject: [PATCH 3/7] Added attachable stream.Cleanup. --- tap_quickbooks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_quickbooks/__init__.py b/tap_quickbooks/__init__.py index b1bf7bb..204fbae 100644 --- a/tap_quickbooks/__init__.py +++ b/tap_quickbooks/__init__.py @@ -319,4 +319,4 @@ def main(): raise e if __name__ == "__main__": - main() + main() \ No newline at end of file From 01b43cfad2169700df841d122e0fb2def6d30384 Mon Sep 17 00:00:00 2001 From: Adil Ahmed Date: Mon, 22 Apr 2024 20:46:16 +0500 Subject: [PATCH 4/7] Download attachable files. --- .DS_Store | Bin 8196 -> 8196 bytes tap_quickbooks/sync.py | 21 ++++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.DS_Store b/.DS_Store index ded2b90b457505e86ff8c8bb1813e5505025affe..4bc919bcd6fa889871a50d131e4d43c2041b8e02 100644 GIT binary patch delta 121 zcmZp1XmQw}Eg-x-sURn_xWvHV8Y2@k3o9EtJNslifoLyI&UgXw>S_ZW1tW9wS{;RI zLvtew9R&*mquN?d4pC)&>!A4ToZP(pP6jYwWQ5QRyigiOb!~nmpu)bHUE&+d=076r F%mAtp8Danc delta 120 zcmZp1XmQw}Eg-xtsURn_xWvHV8Y2@k3o9EtJLhCOfoM+-&UgW_>S_~X9R-709ffL3 z14A7JLjzOO+FDKyQDuGWp!n>Z+`Rk_1~6b`gwPDUP#Q*cZhkDF!oHbJ;ycUce Date: Wed, 24 Apr 2024 19:22:10 -0400 Subject: [PATCH 5/7] write to correct output dir --- tap_quickbooks/__init__.py | 4 +++- tap_quickbooks/quickbooks/__init__.py | 2 ++ tap_quickbooks/sync.py | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tap_quickbooks/__init__.py b/tap_quickbooks/__init__.py index 204fbae..e4756ec 100644 --- a/tap_quickbooks/__init__.py +++ b/tap_quickbooks/__init__.py @@ -285,7 +285,9 @@ def main_impl(): gl_full_sync = CONFIG.get('gl_full_sync'), gl_weekly = CONFIG.get('gl_weekly', False), gl_daily = CONFIG.get('gl_daily', False), - gl_basic_fields = CONFIG.get('gl_basic_fields', False)) + gl_basic_fields = CONFIG.get('gl_basic_fields', False), + hg_sync_output = CONFIG.get('hg_sync_output') + ) qb.login() if args.discover: diff --git a/tap_quickbooks/quickbooks/__init__.py b/tap_quickbooks/quickbooks/__init__.py index 45cb495..b3c6e2f 100644 --- a/tap_quickbooks/quickbooks/__init__.py +++ b/tap_quickbooks/quickbooks/__init__.py @@ -246,6 +246,7 @@ def __init__(self, gl_weekly = None, gl_daily = None, gl_basic_fields = None, + hg_sync_output = None, realm_id=None): self.api_type = api_type.upper() if api_type else None self.report_period_days = report_period_days @@ -262,6 +263,7 @@ def __init__(self, self.qb_client_secret = qb_client_secret self.session = requests.Session() self.access_token = None + self.hg_sync_output = hg_sync_output self.base_url = "https://sandbox-quickbooks.api.intuit.com/v3/company/" if is_sandbox is True else 'https://quickbooks.api.intuit.com/v3/company/' diff --git a/tap_quickbooks/sync.py b/tap_quickbooks/sync.py index 94b230a..5ec13d7 100644 --- a/tap_quickbooks/sync.py +++ b/tap_quickbooks/sync.py @@ -1,3 +1,4 @@ +import os import time import singer import singer.utils as singer_utils @@ -96,7 +97,7 @@ def sync_records(qb, catalog_entry, state, counter, state_passed): for rec in query_func(catalog_entry, state, state_passed): #Check if it is Attachable stream with a downloadable file if stream == 'Attachable' and "TempDownloadUri" in rec: - download_file(rec['TempDownloadUri'],rec['FileName']) + download_file(rec['TempDownloadUri'], os.path.join(qb.hg_sync_output, rec['FileName'])) counter.increment() with Transformer(pre_hook=transform_data_hook) as transformer: rec = transformer.transform(rec, schema) From a49762af22aa6c283422471a096268611eba6611 Mon Sep 17 00:00:00 2001 From: Hassan Syyid Date: Wed, 24 Apr 2024 19:31:24 -0400 Subject: [PATCH 6/7] fix --- tap_quickbooks/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_quickbooks/sync.py b/tap_quickbooks/sync.py index 5ec13d7..98fdcc9 100644 --- a/tap_quickbooks/sync.py +++ b/tap_quickbooks/sync.py @@ -97,7 +97,7 @@ def sync_records(qb, catalog_entry, state, counter, state_passed): for rec in query_func(catalog_entry, state, state_passed): #Check if it is Attachable stream with a downloadable file if stream == 'Attachable' and "TempDownloadUri" in rec: - download_file(rec['TempDownloadUri'], os.path.join(qb.hg_sync_output, rec['FileName'])) + download_file(rec['TempDownloadUri'], os.path.join(qb.hg_sync_output or "", rec['FileName'])) counter.increment() with Transformer(pre_hook=transform_data_hook) as transformer: rec = transformer.transform(rec, schema) From 3caed105d401b7cf0fdcbe4fe98920608821e2ac Mon Sep 17 00:00:00 2001 From: xacadil <92389481+xacadil@users.noreply.github.com> Date: Fri, 24 May 2024 23:24:59 +0500 Subject: [PATCH 7/7] Change attachable filename format to contain type and object id (#104) --- .DS_Store | Bin 8196 -> 6148 bytes tap_quickbooks/sync.py | 12 +++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.DS_Store b/.DS_Store index 4bc919bcd6fa889871a50d131e4d43c2041b8e02..80f31a26c8a74f90dd3c76a1c897bcfc5d8f3d43 100644 GIT binary patch delta 201 zcmZp1XfcprU|?W$DortDU=RQ@Ie-{MGjdEU6q~50$jCG?VE1GL8J5ZX0y2!-ChG_k zx{6g-8|Ww))aocy8=4zg=qMN(m|ND?a&m|&>strKXXoVR<#zzh02#vwp&59gG>q!p zJVPv>abv?amc{HG9D>XcaRF{1?FzDeW8rt^$^0^w3LpufAxu!31w;cmAa{bC&agS2 HXAUy}5Ir9% literal 8196 zcmeHMU1%It6uxJYHZzV%ZfgFf(R5>4T6LR0C5fgA_s)jq&xbxprE_8K znRCxQckcP-J9Be3gb?V-7|n!K6GCLXn4~Ip8#HET*SaPJBZVjd{Rv4Ella6-=DeA8 zv%`T9fe?WZfe?WZfe?Wk0RcR-S(7(8_k}g=Lj*zuZb}5$_aTB8ld(W93duhm*zpno zB`X1V!+Wd)xKAXIu|O^g$z5?xQ9VH53PFhh!X51qHXUUwkc&bJcLKtlK=5P;Diq*P zN52T4P9P>^*oO#&2&_kdrH@P+h=-MX>e~7}X}d|ZT}sZ_mghG$eF;)Q;g+pM*wmsE z@hLYK_jNzx_2}sdrVYEUoy_0s$zzt8OVILO*YS1BG1J_^GEAH@IchnEo9l5ij^T0N zxG*T#60~Z5z9|}MX=_`EM&_H_TH2$L_NJDF1x2o?Z|oRKoVKSO_bCwx1YZMEye6`9 z8--h|;GeJu|jO-w)N7H5&BN|P)c4pe~ z60|I(+o_DL`)0Rov&lF{GoCqQ;+T~+`_STz`R1(8jN_i0_ItVSxIQ}^j3($dw&>uD zZ@G?l%uIVI>tjtK7Ug1bWldz~T@ClPclJCuFt}V&O3PHWykazKc~;UkkLzB_bPUTm zHIlOoe`*|2bbRZiWu_IS3WPs?wOJdn z^`vPl)DX|PPi@skvm!c$4v4|~)edbe56+kl+tm`< zFpAz!z3gj1ty#X z7jiHQXJ8)A!qe~!JPXgk%kU<=1#iPexCBe^5nP5(;0k;SpTSq~J^TPa!tWADl2jm7 zN>x&|R3|lvAf$YLj0rv8ON?Xd(jN0;M&yT-*)Tt9)B^_(wB9KBakZFb#J1bYD=K%e zG-=z{y}~F#!944(BglMnR%Z02MZxSl>t$p< z-4dv~_nv?5qY_uun&uV-bBPuNYP1zq5Q_Rhhk{u}3zxT7YgL`Ki59R47BrI)2%4|IC*sgO5ZFw@Ub_=!ZMW8@9R0 and "EntityRef" in attachable_ref[0]: + attachable_ref = attachable_ref[0]['EntityRef'] + if attachable_ref: + file_name = f"{attachable_ref['type']}-{attachable_ref['value']}-{file_name}" + #Save the newly formatted file name + rec['FileName'] = file_name + download_file( + rec["TempDownloadUri"], os.path.join(qb.hg_sync_output or "", file_name) + ) counter.increment() with Transformer(pre_hook=transform_data_hook) as transformer: rec = transformer.transform(rec, schema)