diff --git a/python/configure-datastream.py b/python/configure-datastream.py index 7369955e..54aba4bb 100644 --- a/python/configure-datastream.py +++ b/python/configure-datastream.py @@ -1,19 +1,17 @@ -import argparse, json, os, datetime +import argparse, json, os +from datetime import datetime, timedelta from pathlib import Path -def write_conf(conf, out_dir, name): +def write_json(conf, out_dir, name): conf_path = Path(out_dir,name) with open(conf_path,'w') as fp: - json.dump(conf, fp) + json.dump(conf, fp, indent=2) return conf_path -def create_confs_daily(conf): - date = datetime.datetime.now() - date = date.strftime('%Y%m%d') - hourminute = '0000' +def create_ds_confs_daily(conf, today, tomorrow): - conf['globals']['start_date'] = date + hourminute - conf['globals']['end_date'] = date + hourminute + conf['globals']['start_date'] = today + conf['globals']['end_date'] = tomorrow fp_conf = { "forcing" : { @@ -37,8 +35,8 @@ def create_confs_daily(conf): nwm_conf = { "forcing_type" : "operational_archive", - "start_date" : conf['globals']['start_date'], - "end_date" : conf['globals']['end_date'], + "start_date" : today, + "end_date" : today, # If we specify this as tomorrow, we will get 2 days worth of data, just how nwmurl works "runinput" : 1, "varinput" : 5, "geoinput" : 1, @@ -47,7 +45,7 @@ def create_confs_daily(conf): "fcst_cycle" : [0], "lead_time" : [1], } - # "lead_time" : [x+1 for x in range(24)] + # "lead_time" : [x+1 for x in range(24)] HACK REMOVE AFTER DEVELOPMENT conf['forcingprcoessor'] = fp_conf conf['nwmurl'] = nwm_conf @@ -55,20 +53,56 @@ def create_confs_daily(conf): return conf, fp_conf, nwm_conf def create_confs(conf): - if conf['globals']['start_date'] == "DAILY": - ds_conf, fp_conf, nwm_conf = create_confs_daily(conf) + + today = datetime.now() + today = today.replace(hour=0, minute=0, second=0, microsecond=0) + tomorrow = today + timedelta(days=1) + + today_ds_confs = today.strftime('%Y%m%d%H%M') + tomorrow_ds_confs = tomorrow.strftime('%Y%m%d%H%M') + + today_realization = today.strftime('%Y-%m-%d %H:%M:%S') + tomorrow_realization = tomorrow.strftime('%Y-%m-%d %H:%M:%S') if "relative_to" in conf['globals'].keys(): - out_dir = Path(conf['globals']['relative_to'],conf['globals']['data_dir']) + data_dir = Path(conf['globals']['relative_to'],conf['globals']['data_dir']) else: - out_dir = conf['globals']['data_dir'] - out_dir = Path(out_dir,'datastream-configs') + data_dir = conf['globals']['data_dir'] + ngen_config_dir = Path(data_dir,'ngen-run','config') + datastream_config_dir = Path(data_dir,'datastream-configs') + resources_dir = Path(data_dir,'datastream-resources') + resources_config_dir = Path(resources_dir,'ngen-configs') + + if conf['globals']['start_date'] == "DAILY": + + ds_conf, fp_conf, nwm_conf = create_ds_confs_daily(conf, today_ds_confs, tomorrow_ds_confs) + + for path, _, files in os.walk(ngen_config_dir): + for jfile in files: + jfile_path = os.path.join(path,jfile) + if jfile_path.find('realization') >= 0: + realization_file = jfile_path + break + + with open(realization_file,'r') as fp: + data = json.load(fp) + + template_realization = Path(resources_config_dir,"realization.json") + template_realization_rename = Path(resources_config_dir,"realization_template.json") + os.system(f'mv {template_realization} {template_realization_rename}') + + data['time']['start_time'] = today_realization + data['time']['end_time'] = tomorrow_realization + write_json(data,ngen_config_dir,'realization.json') + + else: # Trusting config has been constructed properly + pass - write_conf(nwm_conf,out_dir,'conf_nwmurl.json') - write_conf(fp_conf,out_dir,'conf_fp.json') - write_conf(ds_conf,out_dir,'conf_datastream.json') + write_json(nwm_conf,datastream_config_dir,'conf_nwmurl.json') + write_json(fp_conf,datastream_config_dir,'conf_fp.json') + write_json(ds_conf,datastream_config_dir,'conf_datastream.json') - print(f'\ndatastream configs have been generated and placed here\n{out_dir}\n') + print(f'\ndatastream configs have been generated and placed here\n{datastream_config_dir}\n') if __name__ == "__main__": parser = argparse.ArgumentParser()