Merge pull request #69 from ghrcdaac/mlh0079-optional-verification

 - Verifying the output is now configurable.

README.md

@@ -213,10 +213,18 @@ variable.
 When making the subprocess call, stdout and stderr will default to `None` to prevent an issue from occurring where the 
 timeout is not respected. This can be configured in two ways.
 1. Setting the `ENABLE_SUBPROCESS_LOGGING` environment variable in terraform
-2. Adding `enable_subprocess_logging` to the collection definition: `collection.meta.dmrpp`. Can be `true` or `false`.
+2. Adding `enable_subprocess_logging` to the collection definition: `collection.meta.dmrpp`.  
 
 If the value is provided in the collection definition this will take precedence over the environment
-variable.
+variable. Can be `true` or `false`.
+
+## Verify Output Configuration
+The processing code will verify that dmrpp outputs are produced and if not an exception will be raised. This behavior can be disabled if needed. This can be configured in two ways.
+1. Setting the `VERIFY_OUTPUT` environment variable in terraform
+2. Adding `verify_output` to the collection definition: `collection.meta.dmrpp`.  
+
+If the value is provided in the collection definition this will take precedence over the environment
+variable. Can be `true` or `false`.
 
 # DMR++ Python CLI
 # How to install

dmrpp_generator/main.py

@@ -40,20 +40,23 @@ def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.path = self.path.rstrip('/') + "/"
         # Enable logging the default is True
-        enable_logging = os.getenv('ENABLE_CW_LOGGING', 'True') in [True, "true", "t", 1]
+        enable_logging = (os.getenv('ENABLE_CW_LOGGING', 'true').lower() == 'true')
         self.dmrpp_version = f"DMRPP {__version__}"
         self.logger_to_cw = LOGGER_TO_CW if enable_logging else logging
         self.logger_to_cw.info(f'config: {self.config}')
         self.timeout = int(self.dmrpp_meta.get(
             'get_dmrpp_timeout', os.getenv('GET_DMRPP_TIMEOUT', '600'))
         )
         self.enable_subprocess_logging = self.dmrpp_meta.get(
-            'enable_subprocess_logging', os.getenv('ENABLE_SUBPROCESS_LOGGING', False)
+            'enable_subprocess_logging', os.getenv('ENABLE_SUBPROCESS_LOGGING', 'false').lower() == 'true'
+        )
+        self.verify_output = self.dmrpp_meta.get(
+            'verify_output', os.getenv('VERIFY_OUTPUT', 'true').lower() == 'true'
         )
-
         self.logger_to_cw.info(f'get_dmrpp_timeout: {self.timeout}')
         self.logger_to_cw.info(f'enagled_cw_logging: {enable_logging}')
         self.logger_to_cw.info(f'enable_subprocess_logging: {self.enable_subprocess_logging}')
+        self.logger_to_cw.info(f'verify_output: {self.verify_output}')
 
     @property
     def input_keys(self):
@@ -129,14 +132,16 @@ def process(self):
                         dmrpp_files.append(dmrpp_file)
                         self.upload_file_to_s3(output_file_path, f's3://{dmrpp_file["bucket"]}/{dmrpp_file["key"]}')
 
-            if dmrpp_files == 0:
-                raise Exception(f'No dmrpp files were produced for {granule}')
-
+            if len(dmrpp_files) == 0:
+                message = f'No dmrpp files were produced for {granule}'
+                if self.verify_output:
+                    raise Exception(message)
+                else:
+                    self.logger_to_cw.warning(message)
+
             self.strip_old_dmrpp_files(granule)
             granule['files'] += dmrpp_files
 
-        self.verify_outputs_produced(granules)
-
         return self.input
 
     @staticmethod
@@ -150,20 +155,6 @@ def strip_old_dmrpp_files(granule):
             else:
                 i += 1
 
-    @staticmethod
-    def verify_outputs_produced(granules):
-        has_output = False
-        for granule in granules:
-            for file in granule['files']:
-                if str(file.get('fileName')).endswith('dmrpp'):
-                    has_output = True
-                    break
-            if has_output:
-                break
-
-        if not has_output:
-            raise Exception('No dmrpp outputs produced.')
-
     def get_dmrpp_command(self, dmrpp_meta, input_path, output_filename, local=False):
         """
         Getting the command line to create DMRPP files

dmrpp_generator/version.py

@@ -1 +1 @@
-__version__ = "v5.1.0"
+__version__ = "v5.1.1"