From f6fe37b7d5a390487c306a2a942c611b5af8299f Mon Sep 17 00:00:00 2001 From: Weber <Weber.Marc@epa.gov> Date: Mon, 22 Jul 2024 16:06:14 -0700 Subject: [PATCH] Local updates --- ControlTable_StreamCat.csv | 30 +++++++++++---- PartitionDownscaledResults.py | 3 +- StreamCat.py | 72 +++++++++++++++++------------------ 3 files changed, 60 insertions(+), 45 deletions(-) diff --git a/ControlTable_StreamCat.csv b/ControlTable_StreamCat.csv index a52c826..9ab94da 100644 --- a/ControlTable_StreamCat.csv +++ b/ControlTable_StreamCat.csv @@ -1,7 +1,21 @@ FullTableName,accum_type,MetricName,AppendMetric,LandscapeLayer,summaryfield,Final_Table_Name,MetricType,Conversion,by_RPU,use_mask,run,notes,Published,Metadata Published,InAPIDatabase,Date Added AgDrain,Categorical,lookup/AgDrain_lookup.csv,none,AgDrain_stlvl_FINAL.tif,,AgDrain,Percent,1,0,0,0,,Yes,Yes,,7/10/2020 -Ag2006HiSlp,Categorical,lookup/Ag2006HiSlp_lookup.csv,none,Ag2006HiSlp.tif,,AgMidHiSlopes,Percent,1,0,0,0,,Yes,no,,1/21/2021 -Ag2006MidSlp,Categorical,lookup/Ag2006MidSlp_lookup.csv,none,Ag2006MidSlp.tif,,AgMidHiSlopes,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2001HiSlp,Categorical,lookup/Ag2001HiSlp_lookup.csv,none,Ag2001HiSlp.tif,,AgMidHiSlopes2001,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2001MidSlp,Categorical,lookup/Ag2001MidSlp_lookup.csv,none,Ag2001MidSlp.tif,,AgMidHiSlopes2001,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2004HiSlp,Categorical,lookup/Ag2006HiSlp_lookup.csv,none,Ag2004HiSlp.tif,,AgMidHiSlopes2004,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2004MidSlp,Categorical,lookup/Ag2006MidSlp_lookup.csv,none,Ag2004MidSlp.tif,,AgMidHiSlopes2004,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2006HiSlp,Categorical,lookup/Ag2006HiSlp_lookup.csv,none,Ag2006HiSlp.tif,,AgMidHiSlopes2006,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2006MidSlp,Categorical,lookup/Ag2006MidSlp_lookup.csv,none,Ag2006MidSlp.tif,,AgMidHiSlopes2006,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2008HiSlp,Categorical,lookup/Ag2008HiSlp_lookup.csv,none,Ag2008HiSlp.tif,,AgMidHiSlopes2008,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2008MidSlp,Categorical,lookup/Ag2008MidSlp_lookup.csv,none,Ag2008MidSlp.tif,,AgMidHiSlopes2008,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2011HiSlp,Categorical,lookup/Ag2011HiSlp_lookup.csv,none,Ag2011HiSlp.tif,,AgMidHiSlopes2011,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2011MidSlp,Categorical,lookup/Ag2011MidSlp_lookup.csv,none,Ag2011MidSlp.tif,,AgMidHiSlopes2011,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2013HiSlp,Categorical,lookup/Ag2013HiSlp_lookup.csv,none,Ag2013HiSlp.tif,,AgMidHiSlopes2013,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2013MidSlp,Categorical,lookup/Ag2013MidSlp_lookup.csv,none,Ag2013MidSlp.tif,,AgMidHiSlopes2013,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2016HiSlp,Categorical,lookup/Ag2016HiSlp_lookup.csv,none,Ag2016HiSlp.tif,,AgMidHiSlopes2016,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2016MidSlp,Categorical,lookup/Ag2016MidSlp_lookup.csv,none,Ag2016MidSlp.tif,,AgMidHiSlopes2016,Percent,1,0,0,0,,Yes,no,,1/21/2021 +Ag2019HiSlp,Categorical,lookup/Ag2019HiSlp_lookup.csv,none,Ag2019HiSlp.tif,,AgMidHiSlopes2019,Percent,1,0,0,1,,Yes,no,,1/21/2021 +Ag2019MidSlp,Categorical,lookup/Ag2019MidSlp_lookup.csv,none,Ag2019MidSlp.tif,,AgMidHiSlopes2019,Percent,1,0,0,0,,Yes,no,,1/21/2021 CBNF,Continuous,CBNF,none,cbnf.tif,,AgriculturalNitrogen,Mean,1,0,0,0,,Yes,yes,Yes,1/21/2021 Fert,Continuous,Fert,none,fert.tif,,AgriculturalNitrogen,Mean,1,0,0,0,,Yes,yes,Yes,1/21/2021 Manure,Continuous,Manure,none,manure.tif,,AgriculturalNitrogen,Mean,1,0,0,0,,Yes,yes,Yes,1/22/2021 @@ -403,9 +417,9 @@ WWTPAll,Point,WWTPAllDens,none,WWTP_All_CWA_Active_2013_CONUS.shp,,WWTP,Density, WWTPMajor,Point,WWTPMajorDens,none,WWTP_Major_CWA_Active_2013_CONUS.shp,,WWTP,Density,1,0,0,0,,Yes,yes,,12/30/2021 WWTPMinor,Point,WWTPMinorDens,none,WWTP_Minor_CWA_Active_2013_CONUS.shp,,WWTP,Density,1,0,0,0,,Yes,yes,,12/30/2021 WetnessIndex,Continuous,WetIndex,none,wetness_index.tif,,WetIndex,Mean,1,0,0,0,,Yes,yes,Yes,7/4/2020 -NPP_YrMean,Continuous,NPP_YrMean,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,1,,No,No,No,5/6/2024 -NPP,Continuous,NPP,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,1,,No,No,No,5/6/2024 -Precip_YrMean,Continuous,Precip_YrMean,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,1,,No,No,No,5/6/2024 -LST_YrMean,Continuous,LST_YrMean,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,1,,No,No,No,5/6/2024 -LST,Continuous,LST,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,1,,No,No,No,5/6/2024 -SNOW_YrMean,Continuous,SNOW_YrMean,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,1,,No,No,No,5/6/2024 +NPP_YrMean,Continuous,NPP_YrMean,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,0,,No,No,No,5/6/2024 +NPP,Continuous,NPP,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,0,,No,No,No,5/6/2024 +Precip_YrMean,Continuous,Precip_YrMean,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,0,,No,No,No,5/6/2024 +LST_YrMean,Continuous,LST_YrMean,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,0,,No,No,No,5/6/2024 +LST,Continuous,LST,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,0,,No,No,No,5/6/2024 +SNOW_YrMean,Continuous,SNOW_YrMean,none,E:\WorkingData\To_Be_Flow_Accumulated,,ClimTerms_2012_10,Mean,1,0,0,0,,No,No,No,5/6/2024 diff --git a/PartitionDownscaledResults.py b/PartitionDownscaledResults.py index a90471e..fd55860 100644 --- a/PartitionDownscaledResults.py +++ b/PartitionDownscaledResults.py @@ -37,7 +37,8 @@ # select columns - this part we can modify to iterate through columns final = nut[['COMID', 'SNOW_YrMean', 'CatAreaSqKm', 'VPU']] final = final.rename(columns={'SNOW_YrMean': 'CatSum'}) -final['CatCount'] = final['CatAreaSqKm'] +final['CatCount'] = final['CatAreaSqKm'] +final['CatSum'] = final['CatSum'] * final['CatCount'] final['CatPctFull'] = 100 final = final.set_axis(['COMID', 'CatSum', 'CatAreaSqKm','VPU', 'CatCount', 'CatPctFull'], axis=1) diff --git a/StreamCat.py b/StreamCat.py index 466f675..6fd0cc6 100644 --- a/StreamCat.py +++ b/StreamCat.py @@ -113,34 +113,34 @@ end="", flush=True, ) - # for zone, hydroregion in INPUTS.items(): - # if not os.path.exists(f"{OUT_DIR}/{row.FullTableName}_{zone}.csv"): - # print(zone, end=", ", flush=True) - # pre = f"{NHD_DIR}/NHDPlus{hydroregion}/NHDPlus{zone}" - # if not row.accum_type == "Point": - # izd = ( - # f"{mask_dir}/{zone}.tif" - # if mask_dir - # else f"{pre}/NHDPlusCatchment/cat" - # ) - # cat = createCatStats( - # row.accum_type, - # layer, - # izd, - # OUT_DIR, - # zone, - # row.by_RPU, - # mask_dir, - # NHD_DIR, - # hydroregion, - # apm, - # ) - # if row.accum_type == "Point": - # izd = f"{pre}/NHDPlusCatchment/Catchment.shp" - # cat = PointInPoly( - # points, zone, izd, pct_full, mask_dir, apm, summary - # ) - # cat.to_csv(f"{OUT_DIR}/{row.FullTableName}_{zone}.csv", index=False) + for zone, hydroregion in INPUTS.items(): + if not os.path.exists(f"{OUT_DIR}/{row.FullTableName}_{zone}.csv"): + print(zone, end=", ", flush=True) + pre = f"{NHD_DIR}/NHDPlus{hydroregion}/NHDPlus{zone}" + if not row.accum_type == "Point": + izd = ( + f"{mask_dir}/{zone}.tif" + if mask_dir + else f"{pre}/NHDPlusCatchment/cat" + ) + cat = createCatStats( + row.accum_type, + layer, + izd, + OUT_DIR, + zone, + row.by_RPU, + mask_dir, + NHD_DIR, + hydroregion, + apm, + ) + if row.accum_type == "Point": + izd = f"{pre}/NHDPlusCatchment/Catchment.shp" + cat = PointInPoly( + points, zone, izd, pct_full, mask_dir, apm, summary + ) + cat.to_csv(f"{OUT_DIR}/{row.FullTableName}_{zone}.csv", index=False) print("done!") print("Accumulating...", end="", flush=True) for zone in INPUTS: @@ -184,11 +184,11 @@ final = pd.merge(cat, upFinal, on="COMID") final.to_csv(f"{OUT_DIR}/{row.FullTableName}_{zone}.csv", index=False) print(end="") if processed else print("done!") -if already_processed: - print( - "\n!!!Processing Problem!!!\n\n" - f"{', '.join(already_processed)} already run!\n" - "Be sure to delete the associated files in your `OUTDIR` to rerun:" - f"\n\t> {OUT_DIR}\n\n!!! `$OUT_DIR/DBF_stash/*` " - f"output used in 'Continuous' and 'Categorical' metrics!!!" - ) \ No newline at end of file + if already_processed: + print( + "\n!!!Processing Problem!!!\n\n" + f"{', '.join(already_processed)} already run!\n" + "Be sure to delete the associated files in your `OUTDIR` to rerun:" + f"\n\t> {OUT_DIR}\n\n!!! `$OUT_DIR/DBF_stash/*` " + f"output used in 'Continuous' and 'Categorical' metrics!!!" + ) \ No newline at end of file