From eb611f9c7d445b60457efd7b19a1a9b0af9b9152 Mon Sep 17 00:00:00 2001
From: Shu <Qiang.Shu@water.ca.gov>
Date: Fri, 22 Nov 2024 17:27:43 -0800
Subject: [PATCH] add alternate name for short-long wave radiation in hrrr3,
 switch to a new data repos for hycom

---
 dms_datastore/download_hycom.py | 57 ++++++++++++++++++++++++++++++---
 dms_datastore/hrrr3.py          | 12 ++++---
 2 files changed, 61 insertions(+), 8 deletions(-)

diff --git a/dms_datastore/download_hycom.py b/dms_datastore/download_hycom.py
index 572933d..1288b5a 100644
--- a/dms_datastore/download_hycom.py
+++ b/dms_datastore/download_hycom.py
@@ -2,7 +2,6 @@
 import urllib.request
 import xarray as xr
 import pandas as pd
-import pyproj 
 import os
 import numpy as np
 import time
@@ -81,6 +80,56 @@ def hycom_schism_opendap(start=None,end=None,dest=None):
                            encoding = {'salinity': {'_FillValue': -9999.0}, 'water_temp': {'_FillValue': -9999.0}} )
         s = s + days(1)
         e = e + hours(24)
+        
+def hycom_schism_opendap_alt2(start=None,end=None,dest=None):
+    """ Download hycom  opendap data for all time based on a bounding set of lat/lon   
+        from a seperate repos. 
+    
+    This particular variant is available from 8/10/2024
+    """
+    url="https://tds.hycom.org/thredds/dodsC/ESPC-D-V02/s3z?lat,lon,time,salinity"
+    url2="https://tds.hycom.org/thredds/dodsC/ESPC-D-V02/t3z?lat,lon,time,water_temp"
+    data = xr.open_dataset(url)
+    data2 = xr.open_dataset(url2)
+
+    if start is None:
+        start = pd.Timestamp(2024,9,1)
+    if end is None:
+        end = pd.Timestamp.now()
+    if dest is None:
+        dest = './raw'
+        
+    if os.path.exists(dest) is False:
+        os.mkdir(dest)
+        print("Destination path created: %s"%dest)
+        
+    s = copy.copy(start)
+    nnday = (end - start).days+1
+    print(nnday)
+    print("Start=",start," End=",end," dest=",dest," nday=",nnday)        
+
+    
+    for nday in range(nnday):
+        print("Downloading: ",s)
+        e = s + days(1)
+        #subset = data.sel(time=slice(s.to_datetime64(),e.to_datetime64()),lat=slice(37.2,38.801),lon=slice(236.48,238.021))
+        subset = data.sel(time=slice(s.to_datetime64(),e.to_datetime64()),lat=slice(37,39),lon=slice(236,239))
+        subset2 = data2.sel(time=slice(s.to_datetime64(),e.to_datetime64()),lat=slice(37,39),lon=slice(236,239))
+        ##sub['salinity' = sub['salinity']*sub['salinity']get
+        #newtime = pd.date_range(s,freq='1H',periods=24)
+        #resampled= subset.interp(time=newtime)
+        datestr = s.strftime("%Y%m%d")        
+        #filename = os.path.join(dest,"hycom_processed_"+datestr+".nc")
+        filename = os.path.join(dest,"hycom_raw_"+datestr+".nc")
+        subset.to_netcdf(filename,mode='w',\
+                           format='NETCDF4_CLASSIC',unlimited_dims=['time'],\
+                           encoding = {'salinity': {'_FillValue': -9999.0}} )
+        subset2.to_netcdf(filename,mode='a',\
+                           format='NETCDF4_CLASSIC',unlimited_dims=['time'],\
+                           encoding = {'water_temp': {'_FillValue': -9999.0}} )
+        s = s + days(1)
+        e = e + hours(24)
+
 
 def hycom_schism_opendap_alt():
     """ Alternate interface that seems slower, so this is currently not used"""
@@ -129,7 +178,7 @@ def process_hycom(start=None, end=None,dest=None):
         os.mkdir(dest)
         print("Destination path created: %s"%dest)
 
-    nnday = (end - start).days  
+    nnday = (end - start).days + 1
     s = copy.copy(start)
 
     for nday in range(nnday):
@@ -186,10 +235,10 @@ def main():
     
     start_date = pd.to_datetime(args.sdate, format='%Y-%m-%d')
     if end_date is None:
-        end_date = pd.Timestamp.today().date()
+        end_date = pd.Timestamp.today()
     else:
         end_date = pd.to_datetime(args.edate, format='%Y-%m-%d')
-    hycom_schism_opendap(start_date,end_date,raw_dest)
+    hycom_schism_opendap_alt2(start_date,end_date,raw_dest)
     process_hycom(start_date,end_date,processed_dest)
 
 if __name__ == '__main__':
diff --git a/dms_datastore/hrrr3.py b/dms_datastore/hrrr3.py
index 19f76ce..532e77f 100644
--- a/dms_datastore/hrrr3.py
+++ b/dms_datastore/hrrr3.py
@@ -180,7 +180,7 @@ def gen_sflux(self, date, record, pscr):
         Vars = {
             'group1': {'sh2': ['174096', spfh], 't2m': ['167', stmp],'u10': ['165', uwind], 'v10': ['166', vwind]},
             'group2': {'mslma': ['meanSea', prmsl]},
-            'group3': {'prate': ['surface', prate], 'dlwrf': ['surface', dlwrf], 'dswrf': ['surface', dswrf]}
+            'group3': {'prate': ['surface', prate,'prate'], 'dlwrf': ['surface', dlwrf,'sdlwrf'], 'dswrf': ['surface', dswrf,'sdswrf']}
         }
 
         
@@ -219,13 +219,15 @@ def gen_sflux(self, date, record, pscr):
                     
                         logger.info('meanSea'+' not found in %s'%file)
                         print('meanSea'+' not found in %s'%file)
-                   
-
-                else:
+                else:                    
                     ds=xr.open_dataset(file,
                     engine='cfgrib',
                     backend_kwargs=dict(filter_by_keys={'stepType': 'instant','typeOfLevel': 'surface'}))
+                    available_key = [key for key in ds.data_vars]
                     for key2, value2 in value.items():
+                      if key2 not in available_key:
+                          # try alternate name
+                          key2 = value2[2]
                       try:
                           value2[1].append(ds[key2][idx_ymin:idx_ymax+1, idx_xmin:idx_xmax+1].astype('float32'))
                       except KeyError:
@@ -233,6 +235,8 @@ def gen_sflux(self, date, record, pscr):
                           logger.info(f'instant {key2} not found in %s'%file)
                           print(f'instant {key2} not found in %s'%file)     
                     ds.close() 
+
+                  
                    
         #write netcdf
         fout = xr.Dataset({