From 19ca9dad115d5d1f9cf2b8b9eb93b997008b0c5c Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 11 Dec 2019 14:03:12 -0500 Subject: [PATCH 1/3] Make validation optional --- target_csv.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/target_csv.py b/target_csv.py index c372545..1d060e2 100755 --- a/target_csv.py +++ b/target_csv.py @@ -35,7 +35,7 @@ def flatten(d, parent_key='', sep='__'): items.append((new_key, str(v) if type(v) is list else v)) return dict(items) -def persist_messages(delimiter, quotechar, messages, destination_path): +def persist_messages(delimiter, quotechar, messages, destination_path, validate_records): state = None schemas = {} key_properties = {} @@ -55,8 +55,8 @@ def persist_messages(delimiter, quotechar, messages, destination_path): if o['stream'] not in schemas: raise Exception("A record for stream {}" "was encountered before a corresponding schema".format(o['stream'])) - - validators[o['stream']].validate(o['record']) + if validate_records: + validators[o['stream']].validate(o['record']) filename = o['stream'] + '-' + now + '.csv' filename = os.path.expanduser(os.path.join(destination_path, filename)) @@ -141,7 +141,8 @@ def main(): state = persist_messages(config.get('delimiter', ','), config.get('quotechar', '"'), input_messages, - config.get('destination_path', '')) + config.get('destination_path', ''), + config.get('validate_records', '')) emit_state(state) logger.debug("Exiting normally") From b70762ab8a6df616deb7adc214ddb24a5dbd31f6 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 11 Dec 2019 14:03:58 -0500 Subject: [PATCH 2/3] Set default to true --- .idea/.gitignore | 3 +++ target_csv.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .idea/.gitignore diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..0e40fe8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ + +# Default ignored files +/workspace.xml \ No newline at end of file diff --git a/target_csv.py b/target_csv.py index 1d060e2..2e36527 100755 --- a/target_csv.py +++ b/target_csv.py @@ -142,7 +142,7 @@ def main(): config.get('quotechar', '"'), input_messages, config.get('destination_path', ''), - config.get('validate_records', '')) + config.get('validate_records', True)) emit_state(state) logger.debug("Exiting normally") From b0269653d1a07b26896b9a2993ac281d92ceb8e5 Mon Sep 17 00:00:00 2001 From: Peter Dannemann Date: Wed, 11 Dec 2019 14:10:24 -0500 Subject: [PATCH 3/3] Remove .idea file --- .idea/.gitignore | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .idea/.gitignore diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 0e40fe8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ - -# Default ignored files -/workspace.xml \ No newline at end of file