From b792244cf63e05f805b73aa957a3571a4b006435 Mon Sep 17 00:00:00 2001 From: Stephen Jefferson Date: Sun, 16 Jul 2023 11:28:41 +0100 Subject: [PATCH 1/6] Add secondary upload destination --- enviro/__init__.py | 13 +++++++++++++ enviro/config_defaults.py | 8 +++++++- enviro/config_template.py | 6 ++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/enviro/__init__.py b/enviro/__init__.py index 236f280..bdd6892 100644 --- a/enviro/__init__.py +++ b/enviro/__init__.py @@ -415,15 +415,28 @@ def upload_readings(): return False destination = config.destination + secondary_destination = config.secondary_destination + try: exec(f"import enviro.destinations.{destination}") destination_module = sys.modules[f"enviro.destinations.{destination}"] destination_module.log_destination() + + exec(f"import enviro.destinations.{secondary_destination}") + secondary_destination_module = sys.modules[f"enviro.destinations.{secondary_destination}"] + secondary_destination_module.log_destination() for cache_file in os.ilistdir("uploads"): try: with open(f"uploads/{cache_file[0]}", "r") as upload_file: status = destination_module.upload_reading(ujson.load(upload_file)) + if status == UPLOAD_SUCCESS: + logging.info(f" - Primary destination upload success for {cache_file[0]}") + secondary_status = secondary_destination_module.upload_reading(ujson.load(upload_file)) + if secondary_status == UPLOAD_SUCCESS: + logging.info(f" - Secondary destination upload success for {cache_file[0]}") + # Delete if primary upload succeeds regardless of secondary - prioritise stability over data coverage + # This will mean multiple uploads to secondary if primary fails - may need to improve destination success management dpeending on destination duplicate handling if status == UPLOAD_SUCCESS: os.remove(f"uploads/{cache_file[0]}") logging.info(f" - uploaded {cache_file[0]}") diff --git a/enviro/config_defaults.py b/enviro/config_defaults.py index cfdf5dd..4bb6f17 100644 --- a/enviro/config_defaults.py +++ b/enviro/config_defaults.py @@ -2,7 +2,7 @@ from phew import logging DEFAULT_USB_POWER_TEMPERATURE_OFFSET = 4.5 - +DEFAULT_SECONDARY_DESTINATION = None def add_missing_config_settings(): try: @@ -17,6 +17,12 @@ def add_missing_config_settings(): except AttributeError: warn_missing_config_setting("usb_power_temperature_offset") config.usb_power_temperature_offset = DEFAULT_USB_POWER_TEMPERATURE_OFFSET + + try: + config.secondary_destination + except AttributeError: + warn_missing_config_setting("secondary_destination") + config.secondary_destination = DEFAULT_SECONDARY_DESTINATION def warn_missing_config_setting(setting): diff --git a/enviro/config_template.py b/enviro/config_template.py index a345cc0..5846ce5 100644 --- a/enviro/config_template.py +++ b/enviro/config_template.py @@ -20,6 +20,12 @@ # where to upload to ("http", "mqtt", "adafruit_io", "influxdb") destination = None +# Optional secondary destination - this will consume more battery +# Cached uploads cleanup occurs only on primary destination success, this means +# secondary data will not retry if primary is successful, also secondary data +# will be reuploaded if the primary fails, ensure the secondary destination can +# handle duplicate uploads +secondary_destination = None # how often to upload data (number of cached readings) upload_frequency = 5 From e7fccb994ebdac91187b61417f7b3ea6bbb6246f Mon Sep 17 00:00:00 2001 From: Stephen Jefferson Date: Sun, 16 Jul 2023 11:46:44 +0100 Subject: [PATCH 2/6] Adjusting secondary upload operation order --- enviro/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/enviro/__init__.py b/enviro/__init__.py index bdd6892..452952e 100644 --- a/enviro/__init__.py +++ b/enviro/__init__.py @@ -420,26 +420,22 @@ def upload_readings(): try: exec(f"import enviro.destinations.{destination}") destination_module = sys.modules[f"enviro.destinations.{destination}"] - destination_module.log_destination() exec(f"import enviro.destinations.{secondary_destination}") secondary_destination_module = sys.modules[f"enviro.destinations.{secondary_destination}"] - secondary_destination_module.log_destination() for cache_file in os.ilistdir("uploads"): try: with open(f"uploads/{cache_file[0]}", "r") as upload_file: + filename = cache_file[0] + destination_module.log_destination() status = destination_module.upload_reading(ujson.load(upload_file)) - if status == UPLOAD_SUCCESS: - logging.info(f" - Primary destination upload success for {cache_file[0]}") - secondary_status = secondary_destination_module.upload_reading(ujson.load(upload_file)) - if secondary_status == UPLOAD_SUCCESS: - logging.info(f" - Secondary destination upload success for {cache_file[0]}") # Delete if primary upload succeeds regardless of secondary - prioritise stability over data coverage # This will mean multiple uploads to secondary if primary fails - may need to improve destination success management dpeending on destination duplicate handling if status == UPLOAD_SUCCESS: + logging.info(f" - Primary destination upload success for {filename}") os.remove(f"uploads/{cache_file[0]}") - logging.info(f" - uploaded {cache_file[0]}") + logging.info(f" - removing file {cache_file[0]}") elif status == UPLOAD_RATE_LIMITED: # write out that we want to attempt a reupload with open("reattempt_upload.txt", "w") as attemptfile: @@ -465,6 +461,11 @@ def upload_readings(): else: logging.error(f" ! failed to upload '{cache_file[0]}' to {destination}") return False + + secondary_destination_module.log_destination() + secondary_status = secondary_destination_module.upload_reading(ujson.load(upload_file)) + if secondary_status == UPLOAD_SUCCESS: + logging.info(f" - Secondary destination upload success for {filename}") except OSError: logging.error(f" ! failed to open '{cache_file[0]}'") From 984eb5f013e18476278a2f8a0518ec82fb3ee609 Mon Sep 17 00:00:00 2001 From: Stephen Jefferson Date: Sun, 16 Jul 2023 12:13:41 +0100 Subject: [PATCH 3/6] Fix data not available for secondary upload --- enviro/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/enviro/__init__.py b/enviro/__init__.py index 452952e..b1e9048 100644 --- a/enviro/__init__.py +++ b/enviro/__init__.py @@ -428,8 +428,9 @@ def upload_readings(): try: with open(f"uploads/{cache_file[0]}", "r") as upload_file: filename = cache_file[0] + json = ujson.load(upload_file) destination_module.log_destination() - status = destination_module.upload_reading(ujson.load(upload_file)) + status = destination_module.upload_reading(json) # Delete if primary upload succeeds regardless of secondary - prioritise stability over data coverage # This will mean multiple uploads to secondary if primary fails - may need to improve destination success management dpeending on destination duplicate handling if status == UPLOAD_SUCCESS: @@ -463,7 +464,7 @@ def upload_readings(): return False secondary_destination_module.log_destination() - secondary_status = secondary_destination_module.upload_reading(ujson.load(upload_file)) + secondary_status = secondary_destination_module.upload_reading(json) if secondary_status == UPLOAD_SUCCESS: logging.info(f" - Secondary destination upload success for {filename}") From 82746c9befc26fef086999f06f1dbf35aa47a378 Mon Sep 17 00:00:00 2001 From: Stephen Jefferson Date: Sun, 16 Jul 2023 12:27:39 +0100 Subject: [PATCH 4/6] Comment improvement --- enviro/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/enviro/__init__.py b/enviro/__init__.py index b1e9048..dfae4fc 100644 --- a/enviro/__init__.py +++ b/enviro/__init__.py @@ -431,8 +431,12 @@ def upload_readings(): json = ujson.load(upload_file) destination_module.log_destination() status = destination_module.upload_reading(json) - # Delete if primary upload succeeds regardless of secondary - prioritise stability over data coverage - # This will mean multiple uploads to secondary if primary fails - may need to improve destination success management dpeending on destination duplicate handling + + # Delete if primary upload succeeds regardless of secondary + # Prioritise stability over secondary destination data coverage + # This will mean multiple uploads to secondary if primary fails + # May need to improve destination success management depending on + # destination duplicate handling if status == UPLOAD_SUCCESS: logging.info(f" - Primary destination upload success for {filename}") os.remove(f"uploads/{cache_file[0]}") From e20cc1536c60e78b9c532eed64523062ec4120cc Mon Sep 17 00:00:00 2001 From: Stephen Jefferson Date: Fri, 1 Sep 2023 21:29:43 +0100 Subject: [PATCH 5/6] Quick fix to set Secondary destination to None --- enviro/__init__.py | 14 ++++++++------ enviro/config_template.py | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/enviro/__init__.py b/enviro/__init__.py index dfae4fc..1328b20 100644 --- a/enviro/__init__.py +++ b/enviro/__init__.py @@ -421,8 +421,9 @@ def upload_readings(): exec(f"import enviro.destinations.{destination}") destination_module = sys.modules[f"enviro.destinations.{destination}"] - exec(f"import enviro.destinations.{secondary_destination}") - secondary_destination_module = sys.modules[f"enviro.destinations.{secondary_destination}"] + if secondary_destination: + exec(f"import enviro.destinations.{secondary_destination}") + secondary_destination_module = sys.modules[f"enviro.destinations.{secondary_destination}"] for cache_file in os.ilistdir("uploads"): try: @@ -467,10 +468,11 @@ def upload_readings(): logging.error(f" ! failed to upload '{cache_file[0]}' to {destination}") return False - secondary_destination_module.log_destination() - secondary_status = secondary_destination_module.upload_reading(json) - if secondary_status == UPLOAD_SUCCESS: - logging.info(f" - Secondary destination upload success for {filename}") + if secondary_destination: + secondary_destination_module.log_destination() + secondary_status = secondary_destination_module.upload_reading(json) + if secondary_status == UPLOAD_SUCCESS: + logging.info(f" - Secondary destination upload success for {filename}") except OSError: logging.error(f" ! failed to open '{cache_file[0]}'") diff --git a/enviro/config_template.py b/enviro/config_template.py index 5846ce5..e1a22af 100644 --- a/enviro/config_template.py +++ b/enviro/config_template.py @@ -25,6 +25,7 @@ # secondary data will not retry if primary is successful, also secondary data # will be reuploaded if the primary fails, ensure the secondary destination can # handle duplicate uploads +# set to None if not in use secondary_destination = None # how often to upload data (number of cached readings) From 0d2d62a6a87a3bee0414ad78ced79cf5f8da0d3b Mon Sep 17 00:00:00 2001 From: Stephen Jefferson Date: Mon, 13 Nov 2023 18:15:44 +0000 Subject: [PATCH 6/6] Better handling of secondary destination config --- enviro/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/enviro/__init__.py b/enviro/__init__.py index 1328b20..06c7cda 100644 --- a/enviro/__init__.py +++ b/enviro/__init__.py @@ -416,12 +416,13 @@ def upload_readings(): destination = config.destination secondary_destination = config.secondary_destination + valid_secondary_destinations = ["http", "mqtt", "adafruit_io", "influxdb", "wunderground"] try: exec(f"import enviro.destinations.{destination}") destination_module = sys.modules[f"enviro.destinations.{destination}"] - if secondary_destination: + if secondary_destination in valid_secondary_destinations and secondary_destination != destination: exec(f"import enviro.destinations.{secondary_destination}") secondary_destination_module = sys.modules[f"enviro.destinations.{secondary_destination}"] @@ -468,7 +469,7 @@ def upload_readings(): logging.error(f" ! failed to upload '{cache_file[0]}' to {destination}") return False - if secondary_destination: + if secondary_destination in valid_secondary_destinations and secondary_destination != destination: secondary_destination_module.log_destination() secondary_status = secondary_destination_module.upload_reading(json) if secondary_status == UPLOAD_SUCCESS: