verbose option added, many of the print changed to logger.debug and info

catalogbuilder/intakebuilder/CSVwriter.py

@@ -47,7 +47,7 @@ def listdict_to_csv(dict_info,headerlist, csvfile, overwrite, append,slow):
         if overwrite:
             with open(csvfile, 'w') as csvfile:
                 writer = csv.DictWriter(csvfile, fieldnames=headerlist)
-                print("writing..")
+                logger.debug("writing..")
                 writer.writeheader()
                 for data in dict_info:
                     if len(data.keys()) > 2:

catalogbuilder/intakebuilder/configparser.py

@@ -1,41 +1,40 @@
 import yaml
 import os
 class Config:
-    def __init__(self, config):
+    def __init__(self, config,logger):
         self.config = config
         with open(self.config, 'r') as file:
             configfile = yaml.safe_load(file)
         try:
             self.input_path = configfile['input_path']
         except:
             self.input_path = None
-            print("input_path does not exist in config")
+            logger.debug("input_path does not exist in config")
             pass
         try:
             self.output_path = configfile['output_path']
-            #print("output_path :",self.output_path)
         except:
             self.output_path = None
-            print("output_path does not exist in config")
+            logger.debug("output_path does not exist in config")
             pass 
         try:
             self.headerlist = configfile['headerlist']
-            print("headerlist :",self.headerlist)
+            logger.debug("headerlist :"+(str)(self.headerlist))
         except:
             raise KeyError("headerlist does not exist in config")
         try:
             self.output_path_template = configfile['output_path_template']
-            print("output_path_template :",self.output_path_template)
+            logger.debug("output_path_template :"+(str)(self.output_path_template))
         except:
             raise KeyError("output_path_template does not exist in config")
         try:
             self.output_file_template = configfile['output_file_template']
-            print("output_file_template :", self.output_file_template)
+            logger.debug("output_file_template :"+ (str)(self.output_file_template))
         except:
             raise KeyError("output_file_template does not exist in config")
         try:
             self.schema = configfile['schema']
-            print("schema:", self.schema)
+            logger.info("schema:"+ self.schema)
         except:
             self.schema = None
             pass

catalogbuilder/intakebuilder/gfdlcrawler.py

@@ -57,7 +57,7 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow):
     if ( len(set_ftemplate) > 0 ):
       missingcols = [col for col in diffcols if col not in set_ftemplate]
       missingcols.remove("path") #because we get this anyway
-      print("Missing cols from metadata sources:", missingcols)
+      logger.debug("Missing cols from metadata sources:", missingcols)
 
 
     #TODO INCLUDE filter in traversing through directories at the top
@@ -75,7 +75,7 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow):
                #    logger.debug("Skipping hidden file", filepath)
                #    continue
                if not filename.endswith(".nc"):
-                   logger.debug("FILE does not end with .nc. Skipping", filepath)
+                   logger.debug("FILE does not end with .nc. Skipping "+ filepath)
                    continue
                #if our filename expectations are not met compared to the output_file_path_template in config, skip the loop. TODO revisit for statics
                if ("static" not in filename):

catalogbuilder/scripts/gen_intake_gfdl.py

@@ -9,6 +9,7 @@
 
 logger = logging.getLogger('local')
 logger.setLevel(logging.INFO)
+logging.basicConfig(stream=sys.stdout)
 
 try:
    from catalogbuilder.intakebuilder import gfdlcrawler, CSVwriter, builderconfig, configparser, getinfo
@@ -30,19 +31,25 @@
 #template_path = os.path.join(package_dir, '../cats/gfdl_template.json')
 
 def create_catalog(input_path=None, output_path=None, config=None, filter_realm=None, filter_freq=None, filter_chunk=None,
-         overwrite=False, append=False, slow = False):
+         overwrite=False, append=False, slow = False, verbose=False):
+    if verbose:
+       logger.setLevel(logging.DEBUG)
+       logger.info("Verbose log activated.")
+    else:
+       logger.setLevel(logging.INFO)
+       logger.info("[Mostly] silent log activated")
     configyaml = None
     # TODO error catching
     if (config is not None):
-        configyaml = configparser.Config(config)
+        configyaml = configparser.Config(config,logger)
         if(input_path is None):     
             input_path = configyaml.input_path
         if(output_path is None):
             output_path = configyaml.output_path
     if((input_path is None) or (output_path is None)):
        sys.exit("Missing: input_path or output_path. Pass it in the config yaml or as command-line option")     
     if config is None or not configyaml.schema:
-            print("We will use catalog builder catalogbuilder/cats/gfdl_template.json as your json schema")
+            logger.info("Default schema: catalogbuilder/cats/gfdl_template.json")
             template_path = os.path.join(package_dir, '../cats/gfdl_template.json')
     else:
             template_path = configyaml.schema
@@ -51,7 +58,8 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm=
         sys.exit("Input path does not exist. Adjust configuration.")
     if not os.path.exists(Path(output_path).parent.absolute()):
         sys.exit("Output path parent directory does not exist. Adjust configuration.")
-    print("input path: ",input_path, " output path: ", output_path)
+    logger.info("input path: "+ input_path)
+    logger.info( " output path: "+ output_path)
     project_dir = input_path
     csv_path = "{0}.csv".format(output_path)
     json_path = "{0}.json".format(output_path) 
@@ -78,7 +86,7 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm=
     '''
     dictInfo = {}
     project_dir = project_dir.rstrip("/")
-    logger.info("Calling gfdlcrawler.crawlLocal")
+    logger.debug("Calling gfdlcrawler.crawlLocal")
     list_files = gfdlcrawler.crawlLocal(project_dir, dictFilter, dictFilterIgnore, logger, configyaml,slow)
     #Grabbing data from template JSON, changing CSV path to match output path, and dumping data in new JSON
     with open(template_path, "r") as jsonTemplate:
@@ -139,6 +147,7 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm=
 @click.option('--overwrite', is_flag=True, default=False)
 @click.option('--append', is_flag=True, default=False)
 @click.option('--slow','-s', is_flag=True, default=False)
+@click.option('--verbose/--silent', default=False, is_flag=True) #default has silent option. Use --verbose for detailed logging
 
 def create_catalog_cli(**kwargs):
     return create_catalog(**kwargs)