From 263a28a2e8fa702aa79bdbb5f9b1b1d83f0e5f27 Mon Sep 17 00:00:00 2001 From: James McManus Date: Mon, 26 Jun 2023 14:35:56 -0400 Subject: [PATCH] Changes from main, and removal of output kalpana directory --- kalpana/downscaling.py | 56 ++++++++++++++++++++++++++++++++++++++---- kalpana/kalpana.py | 43 ++++++++++++++++++-------------- 2 files changed, 76 insertions(+), 23 deletions(-) diff --git a/kalpana/downscaling.py b/kalpana/downscaling.py index 01685c1..2f3cdc5 100644 --- a/kalpana/downscaling.py +++ b/kalpana/downscaling.py @@ -656,7 +656,7 @@ def runStatic(ncFile, levels, epsgOut, pathOut, grassVer, pathRasFiles, rasterF same path of the extracted shape file (pathout). createGrassLocation: boolean. DEFAULT True True for creating a new location and loading DEMs, false to use an existing location with DEMs already imported - createLocMethod: str. DEFAULT 'from_raster' + createLocMethod: str. DEFAULT 'from_raster' Two options "from_epsg" (default) or "from_raster" otherwise an error will be thrown. attrCol: str. DEFAULT 'avgVal' name of the attribute column @@ -762,7 +762,7 @@ def runStatic(ncFile, levels, epsgOut, pathOut, grassVer, pathRasFiles, rasterF logger.info(f'Output files saved on: {pathaux}') # Changed def meshRepLen2raster(fort14, epsgIn, epsgOut, pathOut, grassVer, pathRasFiles, rasterFiles, subDomain=None, - nameGrassLocation=None, createGrassLocation=True, createLocMethod='from_raster'): + nameGrassLocation=None, createGrassLocation=True, createLocMethod='from_raster', exportDEM=False): ''' Function to rasterize mesh shapefile created from the fort.14 file Parameters fort14: str @@ -786,10 +786,10 @@ def meshRepLen2raster(fort14, epsgIn, epsgOut, pathOut, grassVer, pathRasFiles, same path of the extracted shape file (pathout). createGrassLocation: boolean. DEFAULT True True for creating a new location and loading DEMs, false to use an existing location with DEMs already imported - createLocMethod: str. DEFAULT 'from_raster' - Two options "from_epsg" (default) or "from_raster" otherwise an error will be thrown. + createLocMethod: str. DEFAULT 'from_raster' + Two options "from_epsg" (default) or "from_raster" otherwise an error will be thrown. Returns - NOne + None ''' ## create gdf from fort14 file with elements as geometries t0 = time.time() @@ -829,6 +829,9 @@ def meshRepLen2raster(fort14, epsgIn, epsgOut, pathOut, grassVer, pathRasFiles, ## setup grass env setGrassEnv(grassVer, pathGrassLocation, createGrassLocation, gs, gsetup, pathRasFiles, rasterFiles, createLocMethod, epsgOut) + if exportDEM == True: + gs.run_command('r.out.gdal', input = 'dem', flags = 'cm', format = 'GTiff', nodata = -9999, + output = os.path.join(os.path.dirname(pathOut), 'downscaling_dem.tif')) ## get minimum area minArea = gdfMesh.elemArea.min() t0 = time.time() @@ -907,3 +910,46 @@ def reprojectRas(filein, pathout, epsgOut=None, res='same'): rasOut.rio.to_raster(os.path.join(pathout, bname + f'_epsg{epsgOut}_res{res}.tif'), driver="COG") # Changed logger.info('Wrote '+bname+' to COG') # Changed +def mergeDEMs(grassVer, pathRasFiles, rasterFiles, pathOut, epsgOut, + nameGrassLocation=None, createGrassLocation=True, createLocMethod='from_raster'): + ''' Function to rasterize mesh shapefile created from the fort.14 file + Parameters + grassVer: float + Version of the grass software (The code was writen for v8.0). + pathRasters: str + path of the raster files + rasterFiles: list or str + name(s) of the raster file(s). If 'all' is input, all files in pathRasters are used + pathOut: str + full path of the output DEM + nameGrassLocation: str. DEFAULT None + path and name of the grass location. If None the grass location will be called 'grassLoc' and save in the + same path of the extracted shape file (pathout). + createGrassLocation: boolean. DEFAULT True + True for creating a new location and loading DEMs, false to use an existing location with DEMs already imported + createLocMethod: str. DEFAULT 'from_raster' + Two options "from_epsg" (default) or "from_raster" otherwipathOut, epsgOut,se an error will be thrown. + Returns + None + ''' + pathaux = os.path.dirname(pathOut) + ## add grass to the environment variables + grassEnvVar(grassVer) + ## import grass + import grass.script as gs + import grass.script.setup as gsetup + ## grass location path + if nameGrassLocation == None: + pathGrassLocation = os.path.join(pathaux, 'grassLoc') + else: + pathGrassLocation = os.path.join(pathaux, nameGrassLocation) + + if rasterFiles == 'all': + rasterFiles = os.listdir(pathRasFiles) + + ## setup grass env + setGrassEnv(grassVer, pathGrassLocation, createGrassLocation, gs, gsetup, + pathRasFiles, rasterFiles, createLocMethod, epsgOut) + + gs.run_command('r.out.gdal', input = 'dem', flags = 'cm', format = 'GTiff', nodata = -9999, + output = pathOut) diff --git a/kalpana/kalpana.py b/kalpana/kalpana.py index 6e23faf..4603fbf 100644 --- a/kalpana/kalpana.py +++ b/kalpana/kalpana.py @@ -235,10 +235,16 @@ def main(args): if not os.path.exists(outputDir): mode = 0o777 os.makedirs(outputDir, mode, exist_ok=True) + logger.info('Made directory '+outputDir) + else: + logger.info('Directory '+outputDir+' already made.') + + if not os.path.exists(finalDir): + mode = 0o777 os.makedirs(finalDir, mode, exist_ok=True) - logger.info('Made directories '+outputDir+ ' and '+finalDir+'.') + logger.info('Made directory '+finalDir) else: - logger.info('Directories '+outputDir+' and '+finalDir+' already made.') + logger.info('Directory '+finalDir+' already made.') # log start of runStatic run logger.info('Start runScript with the following inputs: '+runScript+', '+str(epsgIn)+', '+str(epsgOut)+', '+pathOut+', '+grassVer+', '+ncFile+', '+meshFile+', '+conLevelsLog+', '+vUnitIn+', '+vUnitOut+', '+adcircVar+', '+conType+', '+str(subDomain)+', '+str(epsgSubDom)+', '+str(exportMesh)+', '+dzFile+', '+str(zeroDif)+', '+nameGrassLocation+', '+str(createGrassLocation)+', '+createLocMethod+', '+attrCol+', '+str(repLenGrowing)+', '+str(compAdcirc2dem)+', '+str(floodDepth)+', '+clumpThreshold+', '+str(perMinElemArea)+', '+str(ras2vec)+', '+str(exportOrg)) @@ -250,15 +256,16 @@ def main(args): compAdcirc2dem, floodDepth, clumpThreshold, perMinElemArea, ras2vec, exportOrg) # move cog tiff to final directory - for finalPathFile in glob.glob(outputDir+'*_epsg4326.tif'): - try: - shutil.move(finalPathFile, finalDir) - logger.info('Moved cog file '+finalPathFile.split("/")[-1]+' to '+finalDir+' directory.') - shutil.rmtree('/data/'+modelRunID+'/kalpana') - logger.info('Removed TIFF file /data/'+modelRunID+'/kalpana.') - except OSError as err: - logger.error('Failed to move cog file '+finalPathFile.split("/")[-1]+' to '+finalDir+' directory.') - sys.exit(1) + finalPathFile = glob.glob(outputDir+'*_epsg4326.tif')[0] + logger.info('The length of the finalPathFile is: '+str(len(finalPathFile))) + try: + shutil.move(finalPathFile, finalDir) + logger.info('Moved cog file '+finalPathFile+' to '+finalDir+' directory.') + shutil.rmtree('/data/'+modelRunID+'/kalpana', ignore_errors=True) + logger.info('Removed TIFF file /data/'+modelRunID+'/kalpana.') + except OSError as err: + logger.error(err) + sys.exit(1) else: logger.info('ncFile '+ncFile+' does not use the NCSC_SAB_v1.22 grid or the hsofs grid, it uses the '+grid+' grid, so do not process') @@ -374,13 +381,13 @@ def main(args): # create cog # move cog tiff to final directory - for finalPathFile in glob.glob(outputDir+'*_epsg4326.tif'): - try: - shutil.move(finalPathFiles, finalDir) - logger.info('Created cog file '+finalPathFiles.split("/")[-1]+' and move to '+finalDir+' directory.') - except OSError as err: - logger.error('Failed to create cog file '+finalPathFiles.split("/")[-1]+' and move to '+finalDir+' directory.') - sys.exit(1) + finalPathFile = glob.glob(outputDir+'*_epsg4326.tif')[0] + try: + shutil.move(finalPathFile, finalDir) + logger.info('Created cog file '+finalPathFile+' and move to '+finalDir+' directory.') + except OSError as err: + logger.error(err) + sys.exit(1) if __name__ == "__main__": ''' Takes argparse inputs and passes theme to the main function