Merge pull request #78 from NCAR/devel

More function definitions and minor mods for compliance

scripts/iconform

@@ -166,10 +166,12 @@ def fill_missing_glob_attributes(attr, table, v, grids):
                 attr["variable_id"] = v["variable_id"]
     if "branch_method" in attr.keys():
         if "none" not in attr["branch_method"]:
-            #if "branch_time_in_child" in attr.keys():
-            #    attr["branch_time_in_child"] = "Get correct date format"
-            #if "branch_time_in_parent" in attr.keys():
-            #    attr["branch_time_in_parent"] = "Get correct date format"
+            if "branch_time_in_child" in attr.keys():
+              if len(attr["branch_time_in_child"])>0:
+                attr["branch_time_in_child"] = float(attr["branch_time_in_child"].split('D')[0]) 
+            if "branch_time_in_parent" in attr.keys():
+              if len(attr["branch_time_in_parent"])>0:
+                attr["branch_time_in_parent"] = float(attr["branch_time_in_parent"].split('D')[0])
             if "parent_mip_era" in attr.keys():
                 attr["parent_mip_era"] = attr["mip_era"]
             if "parent_source_id" in attr.keys():
@@ -190,13 +192,13 @@ def fill_missing_glob_attributes(attr, table, v, grids):
 
     if "variant_label" in attr.keys():
         pre = attr["variant_label"].split('r')[1]
-        attr["realization_index"] = (pre.split('i')[0])
+        attr["realization_index"] = int(pre.split('i')[0])
         pre = pre.split('i')[1]
-        attr["initialization_index"] = (pre.split('p')[0])
+        attr["initialization_index"] = int(pre.split('p')[0])
         pre = pre.split('p')[1]
-        attr["physics_index"] = (pre.split('f')[0])
+        attr["physics_index"] = int(pre.split('f')[0])
         pre = int(pre.split('f')[1])
-        attr["forcing_index"] = str(pre)
+        attr["forcing_index"] = int(pre)
 
     if "further_info_url" in attr.keys():
         if "__FILL__" in attr["further_info_url"]:       
@@ -225,7 +227,7 @@ def fill_missing_glob_attributes(attr, table, v, grids):
             else:
                 ripf = ''
             info_url = "{0}.{1}.{2}.{3}.{4}.{5}".format(mip_era, institution_id, source_id, experiment_id, sub_experiment_id, ripf)
-            attr['further_info_url'] = "http://furtherinfo.es-doc.org/" + info_url
+            attr['further_info_url'] = "https://furtherinfo.es-doc.org/" + info_url
     if "grid" in attr.keys():
         if len(attr["realm"])>0:
             attr["grid"] = grids[attr["realm"].split()[0]]
@@ -260,10 +262,10 @@ def defineVar(v, varName, attr, table_info, definition, ig, experiment, out_dir)
     # Get variables needed to piece together the filename
     ripf_list = ['realization_index','initialization_index','physics_index','forcing_index']
     if all (ripf in attributes for ripf in ripf_list):
-        ripf = ("r{0}i{1}p{2}f{3}".format(attributes['realization_index'],
-                                                     attributes['initialization_index'],
-                                                     attributes['physics_index'],
-                                                     attributes['forcing_index']))
+        ripf = ("r{0}i{1}p{2}f{3}".format(str(attributes['realization_index']),
+                                                     str(attributes['initialization_index']),
+                                                     str(attributes['physics_index']),
+                                                     str(attributes['forcing_index'])))
     else:
         ripf = ''
 
@@ -467,6 +469,7 @@ def create_output(exp_dict, definitions, input_glob, attributes, output_path, ar
             ts_key = None
             mip = d['mipTable']
             if mip in definitions.keys():
+                ig = ""
                 if v in definitions[mip].keys():
                     if "N/A" in definitions[mip][v].upper():
                         v_def = ""
@@ -514,15 +517,17 @@ def create_output(exp_dict, definitions, input_glob, attributes, output_path, ar
             with open(f, 'w') as outfile:
                 json.dump(t, outfile, sort_keys=True, indent=4)
     else:
+        ignore = ['latitude','longitude','olevel','plev19','time','time1','alevhalf','ygre','xgre','vegtype','spectband','areacellg','alevel','xant','yant','rho','tau','plev3','gridlatitude','plev39','plev4','plev27','plev3','plev7h','plev7c','plev8','plev7h','sdepth','siline','basin','olevel','site','soilpools','snowdepth','snowband','vegtype']
         for n,t in TableSpec.iteritems():
             for vn,var in t.iteritems():
-                varD={}
-                varD[vn]=var
-                for d in var["dimensions"]:
-                    varD[d]=t[d]
-                f = output_path + "/" + experiment + '_' + n + '_' + vn + '_spec.json'
-                with open(f, 'w') as outfile:
-                    json.dump(varD, outfile, sort_keys=True, indent=4)
+                if vn not in ignore:
+                    varD={}
+                    varD[vn]=var
+                    for d in var["dimensions"]:
+                        varD[d]=t[d]
+                    f = output_path + "/" + experiment + '_' + n + '_' + vn + '_spec.json'
+                    with open(f, 'w') as outfile:
+                        json.dump(varD, outfile, sort_keys=True, indent=4)
 
     #f1 = output_path + '/MISSING_DEFS.json'
     #with open(f1, 'w') as outfile:
@@ -612,6 +617,9 @@ def main(argv=None):
                 if "json" in gaFile:
                     with open(gaFile) as gaF: 
                         ga = json.load(gaF)
+                    for k in ga.keys():
+                        if ga[k] is None:
+                            ga[k] = ""
                     attributes.update(ga)             
                 else:
                     with open(gaFile) as y_attributes:

source/pyconform/miptableparser.py

@@ -421,6 +421,7 @@ def parse_table(self,exp,mips,tables,v_list,table_var_fields,table_axes_fields,t
                         var['mipTable']=c_var.mipTable
                         if c_var.mipTable in tables or '--ALL--' in tables: 
                             var["_FillValue"] = "1e+20"
+                            #var["missing_value"] = "1e+20"
                             if hasattr(c_var,'deflate'):
                                 var['deflate']= c_var.deflate
                             if hasattr(c_var,'deflate_level'):

source/pyconform/modules/CLM_pft_to_CMIP6_vegtype.py

@@ -251,6 +251,3 @@ def main(argv=None):
 if __name__ == '__main__':
     main()
 
-
-
-

source/pyconform/modules/commonfunctions.py

@@ -259,6 +259,75 @@ def __getitem__(self, index):
         return PhysArray(a, name = new_name, units=p_data.units)
 
 
+
+#===================================================================================================
+# sftofFunction
+#===================================================================================================
+class sftofFunction(Function):
+    key = 'sftof'
+
+    def __init__(self, KMT):
+        super(sftofFunction, self).__init__(KMT)
+
+    def __getitem__(self, index):
+        p_KMT = self.arguments[0][index]
+
+        if index is None:
+            return PhysArray(np.zeros((0,0)), dimensions=[p_KMT.dimensions[0],p_KMT.dimensions[1]])
+
+        KMT = p_KMT.data
+
+        a = np.zeros((KMT.shape[0],KMT.shape[1]))
+
+        for j in range(KMT.shape[0]):
+            for i in range(KMT.shape[1]):
+                if KMT[j,i] > 0:
+                    a[j,i] = 1
+
+        new_name = 'sftof({})'.format( p_KMT.name)
+
+        return PhysArray(a, name = new_name, dimensions=[p_KMT.dimensions[0],p_KMT.dimensions[1]], units=p_KMT.units)
+
+
+
+#===================================================================================================
+# POP_bottom_layer_multFunction
+#===================================================================================================
+class POP_bottom_layer_multaddFunction(Function):
+    key = 'POP_bottom_layer_multadd'
+
+    def __init__(self, KMT, data1, data2):
+        super(POP_bottom_layer_multaddFunction, self).__init__(KMT, data1, data2)
+
+    def __getitem__(self, index):
+        p_KMT = self.arguments[0][index]
+        p_data1 = self.arguments[1][index]
+        p_data2 = self.arguments[2][index]
+
+        data1 = p_data1.data
+        data2 = p_data2.data
+        KMT = p_KMT.data
+
+        a1 = np.zeros((p_data2.shape[0],p_data2.shape[2],p_data2.shape[3]))
+        a2 = np.zeros((p_data2.shape[0]))
+
+        for t in range(p_data2.shape[0]):
+            for j in range(KMT.shape[0]):
+                for i in range(KMT.shape[1]):
+                    k = KMT[j,i]-1
+                    if data2[t,k,j,i] < 1e+16:
+                        a1[t,j,i] = data1[k]*data2[t,k,j,i]
+                        #print a1[t,j,i],data1[k],data2[t,k,j,i]
+        for t in range(p_data2.shape[0]):
+            a2[t] = np.ma.sum(a1[t,:,:])
+            #print a2[t]
+
+        new_name = 'POP_bottom_layer_multadd({}{}{})'.format(KMT.name, data1.name, data2.name)
+        return PhysArray(a2, name=new_name, dimensions=[p_data2.dimensions[0]], units=p_data2.units)
+
+
+
+
 #===================================================================================================
 # masked_invalidFunction
 #===================================================================================================
@@ -354,7 +423,200 @@ def __getitem__(self, index):
                 a[t,:,:] = np.ma.where(a1[t,:,:] < a2, var, value)
             elif '>' in condition:
                 a[t,:,:] = np.ma.where(a1[t,:,:] > a2, var, value)
-        return PhysArray(a, dimensions=[a1.dimensions[0],a1.dimensions[1],a1.dimensions[2]], units=var.units) 
 
+        new_name = 'cice_where()'.format()
+        return PhysArray(a, name=new_name, dimensions=[a1.dimensions[0],a1.dimensions[1],a1.dimensions[2]], units=var.units) 
+
+
+#===================================================================================================
+# cice_regions
+#===================================================================================================
+class cice_regionsFunction(Function):
+    key = 'cice_regions'
+
+    def __init__(self, p_aice, p_uvel, p_vvel, p_HTE, p_HTN, p_siline, multiple):
+        super(cice_regionsFunction, self).__init__(p_aice, p_uvel, p_vvel, p_HTE, p_HTN, p_siline, multiple)
+
+    def __getitem__(self, index):
+        p_aice = self.arguments[0][index]
+        p_uvel = self.arguments[1][index]
+        p_vvel = self.arguments[2][index]
+        p_HTE = self.arguments[3][index]
+        p_HTN = self.arguments[4][index]
+        p_siline = self.arguments[5][index]
+        multiple = self.arguments[6]
+
+        aice=p_aice
+        uvel=p_uvel
+        vvel=p_vvel
+        HTE=p_HTE
+        HTN=p_HTN
+        siline=p_siline
+        a = np.ma.zeros((aice.data.shape[0],siline.data.shape[0]))
+
+        for t in range(aice.data.shape[0]):
+            #1
+            i = 92
+            for j in range(370, 381):
+                if aice[t,j,i] < 1e+16 and aice[t,j,i+1] < 1e+16 and HTE[j,i] < 1e+16 and uvel[t,j,i] < 1e+16 and uvel[t,j-1,i] < 1e+16: 
+                    a[t,0] += 0.5*(aice[t,j,i]+aice[t,j,i+1])*0.5*(HTE[j,i]*uvel[t,j,i]+HTE[j,i]*uvel[t,j-1,i])
+            #2
+            i = 214
+            for j in range(375,377):
+                if aice[t,j,i] < 1e+16 and aice[t,j,i+1] < 1e+16 and HTE[j,i] < 1e+16 and uvel[t,j,i] < 1e+16 and uvel[t,j-1,i] < 1e+16 and aice[t,j+1,i] < 1e+16 and HTN[j,i] < 1e+16 and vvel[t,j,i] < 1e+16 and vvel[t,j,i-1] < 1e+16: 
+                    a[t,1] += 0.5*(aice[t,j,i]+aice[t,j,i+1])*0.5*(HTE[j,i]*uvel[t,j,i]+HTE[j,i]*uvel[t,j-1,i]) + 0.5*(aice[t,j,i]+aice[t,j+1,i])*0.5*(HTN[j,i]*vvel[t,j,i]+HTN[j,i]*vvel[t,j,i-1])
+            j = 366
+            for i in range(240,244):
+                if aice[t,j,i] < 1e+16 and aice[t,j,i+1] < 1e+16 and HTE[j,i] < 1e+16 and uvel[t,j,i] < 1e+16 and uvel[t,j-1,i] < 1e+16 and aice[t,j+1,i] < 1e+16 and HTN[j,i] < 1e+16 and vvel[t,j,i] < 1e+16 and vvel[t,j,i-1] < 1e+16: 
+                    a[t,1] += 0.5*(aice[t,j,i]+aice[t,j,i+1])*0.5*(HTE[j,i]*uvel[t,j,i]+HTE[j,i]*uvel[t,j-1,i]) + 0.5*(aice[t,j,i]+aice[t,j+1,i])*0.5*(HTN[j,i]*vvel[t,j,i]+HTN[j,i]*vvel[t,j,i-1])
+            #3
+            i = 85
+            for j in range(344,366):
+                if aice[t,j,i] < 1e+16 and aice[t,j,i+1] < 1e+16 and HTE[j,i] < 1e+16 and uvel[t,j,i] < 1e+16 and uvel[t,j-1,i] < 1e+16: 
+                    a[t,2] += 0.5*(aice[t,j,i]+aice[t,j,i+1])*0.5*(HTE[j,i]*uvel[t,j,i]+HTE[j,i]*uvel[t,j-1,i])
+            #4
+            j = 333
+            for i in range(198,201):
+                if aice[t,j,i] < 1e+16 and aice[t,j+1,i] < 1e+16 and HTN[j,i] < 1e+16 and vvel[t,j,i] < 1e+16 and vvel[t,j,i-1] < 1e+16: 
+                    a[t,3] +=  0.5*(aice[t,j,i]+aice[t,j+1,i])*0.5*(HTN[j,i]*vvel[t,j,i]+HTN[j,i]*vvel[t,j,i-1])
+
+        a = a*multiple
+
+        new_name = 'cice_regions()'.format()
+        return PhysArray(a, name=new_name, dimensions=[p_aice.dimensions[0],p_siline.dimensions[0]], units=uvel.units)
+
+
+#===================================================================================================
+# reduce_luFunction
+#===================================================================================================
+class reduce_luFunction(Function):
+    key = 'reduce_lu'
+
+    # np.where(x < 5, x, -1) 
+
+    def __init__(self, p_data,p_lu):
+        super(reduce_luFunction, self).__init__(p_data,p_lu)
+
+    def __getitem__(self, index):
+        p_data = self.arguments[0][index]
+        p_lu = self.arguments[1][index]
+
+        #if index is None:
+        #    return PhysArray(p_data.data, dimensions=[p_data.dimensions[0],p_lu.dimensions[0],p_data.dimensions[2],p_data.dimensions[3]])
+
+        data = p_data.data
+        lu = p_lu.data 
+
+        data2 = np.ma.zeros((data.shape[0],4,data.shape[2],data.shape[3]))
+
+        for t in range(data.shape[0]):
+            for x in range(data.shape[2]):
+                for y in range(data.shape[3]):
+                    data2[t,0,x,y] = data[t,0,x,y]
+                    data2[t,1,x,y] = 0
+                    data2[t,2,x,y] = data[t,1,x,y]
+                    data2[t,3,x,y] = data[t,6,x,y]+data[t,7,x,y]+data[t,8,x,y]
+        data2[data2>=1e+16] = 1e+20
+
+        new_name = 'reduce_lu({}{})'.format(p_data.name,p_lu.name)
+        return PhysArray(data2, name=new_name, dimensions=[p_data.dimensions[0],p_lu.dimensions[0],p_data.dimensions[2],p_data.dimensions[3]], units=p_data.units)
+
+
+#===================================================================================================
+# soilpoolsFunction
+#===================================================================================================
+class get_soilpoolsFunction(Function):
+    key = 'get_soilpools'
+
+    def __init__(self, p_data1,p_data2,p_data3,p_soilpool):
+        super(get_soilpoolsFunction, self).__init__(p_data1,p_data2,p_data3,p_soilpool)
+
+    def __getitem__(self, index):
+        p_data1 = self.arguments[0][index]
+        p_data2 = self.arguments[1][index]
+        p_data3 = self.arguments[2][index]
+        p_soilpool = self.arguments[3][index]
+
+        data1 = p_data1.data
+        data2 = p_data2.data
+        data3 = p_data3.data
+        soilpool = p_soilpool.data
+
+        data = np.ma.zeros((data1.shape[0],3,data1.shape[1],data1.shape[2]))
+
+        data[:,0,:,:] = data1
+        data[:,1,:,:] = data2
+        data[:,2,:,:] = data3
+
+        data[data>=1e+16] = 1e+20
+        data = np.ma.masked_values(data, 1e+20)
+
+        new_name = 'soilpools({}{}{}{})'.format(p_data1.name,p_data2.name,p_data3.name,p_soilpool.name)
+        return PhysArray(data, name=new_name, dimensions=[p_data1.dimensions[0],p_soilpool.dimensions[0],p_data1.dimensions[1],p_data1.dimensions[2]], units=p_data1.units)
+
+
+#===================================================================================================
+# expand_latlonFunction
+#===================================================================================================
+class expand_latlonFunction(Function):
+    key = 'expand_latlon'
+
+    def __init__(self, p_data1,p_lat,p_lon):
+        super(expand_latlonFunction, self).__init__(p_data1,p_lat,p_lon)
+
+    def __getitem__(self, index):
+        p_data1 = self.arguments[0][index]
+        p_lat = self.arguments[1][index]
+        p_lon = self.arguments[2][index]
+
+        data1 = p_data1.data
+        lat = p_lat.data
+        lon = p_lon.data
+
+        data = np.ma.zeros((data1.shape[0],lat.shape[0],lon.shape[0]))
+
+        for x in range(lat.shape[0]):
+            for y in range(lon.shape[0]):
+                data[:,x,y] = data1 
+
+        data[data>=1e+16] = 1e+20
+        data = np.ma.masked_values(data, 1e+20)
+
+        new_name = 'expand_latlon({}{}{})'.format( p_data1.name, p_lat.name, p_lon.name)
+        return PhysArray(data, name=new_name, dimensions=[p_data1.dimensions[0],p_lat.dimensions[0],p_lon.dimensions[0]], units=p_data1.units)
+
+
+#===================================================================================================
+# ocean_basinFunction
+#===================================================================================================
+class ocean_basinFunction(Function):
+    key = 'ocean_basin'
+
+    def __init__(self, p_data1, p_comp, p_basin):
+        super(ocean_basinFunction, self).__init__(p_data1, p_comp, p_basin)
+
+    def __getitem__(self, index):
+        p_data1 = self.arguments[0][index]
+        p_comp = self.arguments[1]
+        p_basin = self.arguments[2][index]
+
+        data1 = p_data1.data
+        comp = int(p_comp)
+        basin = p_basin.data
+
+        data = np.ma.zeros((data1.shape[0],data1.shape[4],data1.shape[3],basin.shape[0]))
+
+        for t in range(data1.shape[0]):
+            for x in range(data1.shape[4]):
+                for y in range(data1.shape[3]):
+                    data[t,x,y,0] = data1[t,1,comp,y,x]
+                    data[t,x,y,1] = data1[t,0,comp,y,x]-data1[t,1,comp,y,x]
+                    data[t,x,y,2] = data1[t,0,comp,y,x]
+
+        data[data>=1e+16] = 1e+20
+        data = np.ma.masked_values(data, 1e+20)
+
+        new_name = 'ocean_basin({}{})'.format(p_data1.name, p_basin.name)
+        return PhysArray(data, name=new_name, dimensions=[p_data1.dimensions[0],p_data1.dimensions[4],p_data1.dimensions[3],p_basin.dimensions[0]], units=p_data1.units)
 
 

source/pyconform/version.py

@@ -1,2 +1,2 @@
 # Single place for version information
-__version__ = '0.2.5'
+__version__ = '0.2.6'