diff --git a/Dockerfile b/Dockerfile index b2ee223d..9ad8cd6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,10 +57,10 @@ RUN ln -s /usr/bin/python3 /usr/bin/python RUN git config --global user.email dagibbs22@gmail.com ## Check out the branch that I'm currently using for model development -#RUN git checkout model_v_1.2.0 +#RUN git checkout model_v_1.2.1 # ## Makes sure the latest version of the current branch is downloaded -#RUN git pull origin model_v_1.2.0 +#RUN git pull origin model_v_1.2.1 ## Compile C++ scripts #RUN g++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.exe -lgdal && \ diff --git a/analyses/aggregate_results_to_4_km.py b/analyses/aggregate_results_to_4_km.py index 4390fcf9..b9889679 100644 --- a/analyses/aggregate_results_to_4_km.py +++ b/analyses/aggregate_results_to_4_km.py @@ -24,7 +24,7 @@ # Converts the 10x10 degree Hansen tiles that are in windows of 40000x1 pixels to windows of 400x400 pixels, # which is the resolution of the output tiles. This will allow the 30x30 m pixels in each window to be summed. -def rewindow(tile): +def rewindow(tile, no_upload): # start time start = datetime.datetime.now() @@ -127,7 +127,7 @@ def rewindow(tile): uu.print_log("No mangrove tile found for {}".format(tile_id)) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, '{}_rewindow'.format(tile_type)) + uu.end_of_fx_summary(start, tile_id, '{}_rewindow'.format(tile_type), no_upload) # Converts the existing (per ha) values to per pixel values (e.g., emissions/ha to emissions/pixel) @@ -136,7 +136,7 @@ def rewindow(tile): # 0.1x0.1 degree resolution (approximately 10m in the tropics). # Each pixel in that raster is the sum of the 30m pixels converted to value/pixel (instead of value/ha). # The 0.1x0.1 degree tile is output. -def aggregate(tile, thresh, sensit_type): +def aggregate(tile, thresh, sensit_type, no_upload): # start time start = datetime.datetime.now() @@ -284,12 +284,12 @@ def aggregate(tile, thresh, sensit_type): # aggregated.close() # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, '{}_0_4deg'.format(tile_type)) + uu.end_of_fx_summary(start, tile_id, '{}_0_4deg'.format(tile_type), no_upload) # Calculates the percent difference between the standard model's net flux output # and the sensitivity model's net flux output -def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type): +def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload): # start time start = datetime.datetime.now() @@ -315,11 +315,11 @@ def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type): uu.log_subprocess_output_full(cmd) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux) + uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux, no_upload) # Maps where the sources stay sources, sinks stay sinks, sources become sinks, and sinks become sources -def sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type): +def sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload): # start time start = datetime.datetime.now() @@ -372,4 +372,4 @@ def sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type): # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux) \ No newline at end of file + uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux, no_upload) \ No newline at end of file diff --git a/analyses/create_supplementary_outputs.py b/analyses/create_supplementary_outputs.py index 94da476b..5e21c769 100644 --- a/analyses/create_supplementary_outputs.py +++ b/analyses/create_supplementary_outputs.py @@ -24,7 +24,7 @@ import constants_and_names as cn import universal_util as uu -def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit_type): +def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit_type, no_upload): # start time start = datetime.datetime.now() @@ -145,4 +145,4 @@ def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit uu.print_log(" Output tiles created for {}...".format(tile_id)) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, output_patterns[0]) \ No newline at end of file + uu.end_of_fx_summary(start, tile_id, output_patterns[0], no_upload) \ No newline at end of file diff --git a/analyses/loss_in_raster.py b/analyses/loss_in_raster.py deleted file mode 100644 index 1815f564..00000000 --- a/analyses/loss_in_raster.py +++ /dev/null @@ -1,63 +0,0 @@ -from subprocess import Popen, PIPE, STDOUT, check_call -import datetime -import os -import sys -sys.path.append('../') -import constants_and_names as cn -import universal_util as uu - -# Calculates a range of tile statistics -def loss_in_raster(tile_id, raster_type, output_name, lat, mask): - - uu.print_log("Calculating loss area for tile id {0}...".format(tile_id)) - - xmin, ymin, xmax, ymax = uu.coords(tile_id) - - # start time - start = datetime.datetime.now() - - # Name of the loss time - loss_tile = '{0}.tif'.format(tile_id) - - # The raster that loss is being analyzed inside - raster_of_interest = '{0}_{1}.tif'.format(tile_id, raster_type) - - # Output file name - outname = '{0}_{1}.tif'.format(tile_id, output_name) - - # Only processes the tile if it is inside the latitude band (north of the specified latitude) - if ymax > lat and os.path.exists(raster_of_interest): - - uu.print_log("{} inside latitude band and peat tile exists. Processing tile.".format(tile_id)) - - # If the user has asked to create just a mask of loss as opposed to the actual output values - if mask == "True": - - calc = '--calc=(A>=1)*(A+1)/(A+1)*B' - - # If the user has asked to output the actual loss values - if mask == "False": - - # Equation argument for converting emissions from per hectare to per pixel. - # First, multiplies the per hectare emissions by the area of the pixel in m2, then divides by the number of m2 in a hectare. - calc = '--calc=A*B' - - # Argument for outputting file - out = '--outfile={}'.format(outname) - - uu.print_log("Masking loss in {} by raster of interest...".format(tile_id)) - cmd = ['gdal_calc.py', '-A', loss_tile, '-B', raster_of_interest, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=LZW', - '--overwrite', '--quiet'] - # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging - process = Popen(cmd, stdout=PIPE, stderr=STDOUT) - with process.stdout: - uu.log_subprocess_output(process.stdout) - - uu.print_log("{} masked".format(tile_id)) - - else: - - uu.print_log("{} outside of latitude band. Skipped tile.".format(tile_id)) - - # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, output_name) \ No newline at end of file diff --git a/analyses/mp_aggregate_results_to_4_km.py b/analyses/mp_aggregate_results_to_4_km.py index e108ece7..d72d5961 100644 --- a/analyses/mp_aggregate_results_to_4_km.py +++ b/analyses/mp_aggregate_results_to_4_km.py @@ -29,7 +29,7 @@ sys.path.append(os.path.join(cn.docker_app,'analyses')) import aggregate_results_to_4_km -def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux = None, run_date = None): +def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux = None, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) @@ -44,15 +44,15 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux # Files to download for this script download_dict = { - # cn.annual_gain_AGC_all_types_dir: [cn.pattern_annual_gain_AGC_all_types], - # cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types], + cn.annual_gain_AGC_all_types_dir: [cn.pattern_annual_gain_AGC_all_types], + cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types], cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil], cn.net_flux_dir: [cn.pattern_net_flux] } # Checks whether the canopy cover argument is valid if thresh < 0 or thresh > 99: - uu.exception_log('Invalid tcd. Please provide an integer between 0 and 99.') + uu.exception_log(no_upload, 'Invalid tcd. Please provide an integer between 0 and 99.') # Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles @@ -131,7 +131,7 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux processes = 8 uu.print_log('Rewindow max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(aggregate_results_to_4_km.rewindow, tile_list) + pool.map(partial(aggregate_results_to_4_km.rewindow, no_upload=no_upload), tile_list) # Added these in response to error12: Cannot allocate memory error. # This fix was mentioned here: of https://stackoverflow.com/questions/26717120/python-cannot-allocate-memory-using-multiprocessing-pool # Could also try this: https://stackoverflow.com/questions/42584525/python-multiprocessing-debugging-oserror-errno-12-cannot-allocate-memory @@ -141,7 +141,7 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux # # For single processor use # for tile in tile_list: # - # aggregate_results_to_4_km.rewindow(tile) + # aggregate_results_to_4_km.rewindow(til, no_upload) # Converts the existing (per ha) values to per pixel values (e.g., emissions/ha to emissions/pixel) # and sums those values in each 400x400 pixel window. @@ -159,14 +159,15 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux processes = 8 uu.print_log('Conversion to per pixel and aggregate max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(partial(aggregate_results_to_4_km.aggregate, thresh=thresh, sensit_type=sensit_type), tile_list) + pool.map(partial(aggregate_results_to_4_km.aggregate, thresh=thresh, sensit_type=sensit_type, + no_upload=no_upload), tile_list) pool.close() pool.join() # # For single processor use # for tile in tile_list: # - # aggregate_results_to_4_km.aggregate(tile, thresh, sensit_type) + # aggregate_results_to_4_km.aggregate(tile, thresh, sensit_type, no_upload) # Makes a vrt of all the output 10x10 tiles (10 km resolution) out_vrt = "{}_0_4deg.vrt".format(pattern) @@ -207,10 +208,11 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux uu.log_subprocess_output_full(cmd) - uu.print_log("Tiles processed. Uploading to s3 now...") + # If no_upload flag is not activated, output is uploaded + if not no_upload: - # Uploads all output tiles to s3 - uu.upload_final_set(output_dir_list[0], out_pattern) + uu.print_log("Tiles processed. Uploading to s3 now...") + uu.upload_final_set(output_dir_list[0], out_pattern) # Cleans up the folder before starting on the next raster type vrtList = glob.glob('*vrt') @@ -254,12 +256,16 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux uu.print_log('Cannot do comparison. One of the input flux tiles is not valid. Verify that both net flux rasters are on the spot machine.') uu.print_log("Creating map of percent difference between standard and {} net flux".format(sensit_type)) - aggregate_results_to_4_km.percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type) - uu.upload_final_set(output_dir_list[0], cn.pattern_aggreg_sensit_perc_diff) + aggregate_results_to_4_km.percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload) uu.print_log("Creating map of which pixels change sign and which stay the same between standard and {}".format(sensit_type)) - aggregate_results_to_4_km.sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type) - uu.upload_final_set(output_dir_list[0], cn.pattern_aggreg_sensit_sign_change) + aggregate_results_to_4_km.sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload) + + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + uu.upload_final_set(output_dir_list[0], cn.pattern_aggreg_sensit_perc_diff) + uu.upload_final_set(output_dir_list[0], cn.pattern_aggreg_sensit_sign_change) else: @@ -279,18 +285,23 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux help='Tree cover density threshold above which pixels will be included in the aggregation.') parser.add_argument('--std-net-flux-aggreg', '-sagg', required=False, help='The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list std_net_flux = args.std_net_flux_aggreg thresh = args.tcd_threshold thresh = int(thresh) + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, thresh=thresh, std_net_flux=std_net_flux) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, thresh=thresh, std_net_flux=std_net_flux, + no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_aggregate_results_to_4_km(sensit_type=sensit_type, tile_id_list=tile_id_list, thresh=thresh, std_net_flux=std_net_flux) \ No newline at end of file + mp_aggregate_results_to_4_km(sensit_type=sensit_type, tile_id_list=tile_id_list, thresh=thresh, + std_net_flux=std_net_flux, no_upload=no_upload) \ No newline at end of file diff --git a/analyses/mp_create_supplementary_outputs.py b/analyses/mp_create_supplementary_outputs.py index 489e0c68..a108d028 100644 --- a/analyses/mp_create_supplementary_outputs.py +++ b/analyses/mp_create_supplementary_outputs.py @@ -28,7 +28,7 @@ sys.path.append(os.path.join(cn.docker_app,'analyses')) import create_supplementary_outputs -def mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = None): +def mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) @@ -45,7 +45,7 @@ def mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = None): # Files to download for this script download_dict = { - # cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types], + cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types], cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil], cn.net_flux_dir: [cn.pattern_net_flux] } @@ -130,7 +130,7 @@ def mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = None): elif "net_flux" in input_pattern: output_patterns = output_pattern_list[6:9] else: - uu.exception_log("No output patterns found for input pattern. Please check.") + uu.exception_log(no_upload, "No output patterns found for input pattern. Please check.") uu.print_log("Input pattern:", input_pattern) uu.print_log("Output patterns:", output_patterns) @@ -144,13 +144,13 @@ def mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = None): uu.print_log("Creating derivative outputs for {0} with {1} processors...".format(input_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(create_supplementary_outputs.create_supplementary_outputs, input_pattern=input_pattern, - output_patterns=output_patterns, sensit_type=sensit_type), tile_id_list_input) + output_patterns=output_patterns, sensit_type=sensit_type, no_upload=no_upload), tile_id_list_input) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list_input: - # create_supplementary_outputs.create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit_type) + # create_supplementary_outputs.create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit_type, no_upload) # Checks the two forest extent output tiles created from each input tile for whether there is data in them. # Because the extent is restricted in the forest extent pixels, some tiles with pixels in the full extent @@ -171,9 +171,12 @@ def mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = None): pool.close() pool.join() - # Uploads output tiles to s3 - for i in range(0, len(output_dir_list)): - uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) + + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + for i in range(0, len(output_dir_list)): + uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) if __name__ == '__main__': @@ -187,16 +190,20 @@ def mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = None): help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_create_supplementary_outputs(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date) \ No newline at end of file + mp_create_supplementary_outputs(sensit_type=sensit_type, tile_id_list=tile_id_list, + run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/analyses/mp_loss_in_raster.py b/analyses/mp_loss_in_raster.py deleted file mode 100644 index 0df00983..00000000 --- a/analyses/mp_loss_in_raster.py +++ /dev/null @@ -1,83 +0,0 @@ -### Creates rasters of loss in some other raster of interest. This can then be put through the tile statistics script to -### get the pixel count and total loss area in the raster of interest in each tile. -### This script has three arguments: the directory for the raster of interest (required), whether the output loss is -### masked to 1s or kept as actual loss years (required), and the latitude above which tiles will be processed (optional). -### e.g., python mp_loss_in_raster.py -r s3://gfw2-data/climate/carbon_model/other_emissions_inputs/peatlands/processed/20190429/00N_000E_peat_mask_processed.tif -m True -l 30 - -from multiprocessing.pool import Pool -from functools import partial -import argparse -import os -import loss_in_raster -import sys -sys.path.append('../') -import constants_and_names as cn -import universal_util as uu - -os.chdir(cn.docker_base_dir) - -tile_list = uu.tile_list_s3(cn.loss_dir, sensit_type) -# tile_list = ['00N_110E'] # test tiles -# tile_list = ['00N_110E', '70N_100W'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest -uu.print_log(tile_list) -uu.print_log("There are {} tiles to process".format(str(len(tile_list)))) - -parser = argparse.ArgumentParser(description='Create rasters of loss masked by some other raster') -parser.add_argument('--raster-of-interest', '-r', required=True, - help='one raster in the s3 directory of the raster that loss will be masked by.') -parser.add_argument('--mask-output', '-m', required=True, - help='If True, all output loss is set to 0. If False, loss year data is maintained.') -parser.add_argument('--latitude-constraint', '-l', required=False, - help='Enter a latitude in the format of 20, 0, -30, etc. Only tiles north of that will be evaluated. For example, entering 20 means that the tiles with a southern edge of 20 will be processed.') -args = parser.parse_args() - -# Gets the path, full name, and type of the raster that loss is being considered in. -args_index = os.path.split(args.raster_of_interest) -raster_path = args_index[0] -raster_name = args_index[1] -raster_type = raster_name[9:-4] - -# The name of the output rasters-- a combination of loss and the type of the raster of interest -output_name = 'loss_in_{}'.format(raster_type) - -# The latitude above which loss will be analyzed -lat = args.latitude_constraint -lat = int(lat) - -# The list of valid mask arguments -mask = args.mask_output -valid_masks = ['True', 'False'] - -# If the mask argument isn't valid, the script terminates -if mask not in valid_masks: - uu.print_log("Mask argument is not valid. Use either True or False.") - sys.exit() - -# For downloading all tiles in the input folders -download_list = [cn.loss_dir, '{}/'.format(raster_path)] - -for input in download_list: - uu.s3_folder_download('{}'.format(input), cn.docker_base_dir) - -# # For copying individual tiles to spot machine for testing -# for tile in tile_list: -# -# uu.s3_file_download('{0}{1}.tif'.format(cn.loss_dir, tile), cn.docker_base_dir) # loss tiles -# uu.s3_file_download('{0}/{1}_{2}.tif'.format(raster_path, tile, raster_type), cn.docker_base_dir) # raster of interest - -# 14 processors maxed out at about 70 GB on an m4.16xlarge for peat mask processing. -num_of_processes = 45 -pool = Pool(num_of_processes) -pool.map(partial(loss_in_raster.loss_in_raster, raster_type=raster_type, output_name=output_name, lat=lat, mask=mask), tile_list) -pool.close() -pool.join() - -# # For single processor use -# for tile in tile_list: -# -# loss_in_raster.loss_in_raster(tile, raster_type, output_name, lat, mask) - -uu.print_log("Tiles processed. Uploading to s3 now...") - -# Uploads all output tiles to s3 -uu.upload_final_set('s3://gfw2-data/climate/carbon_model/loss_in_peat/20190917/', output_name) \ No newline at end of file diff --git a/analyses/mp_net_flux.py b/analyses/mp_net_flux.py index e4d10167..b943fc2a 100644 --- a/analyses/mp_net_flux.py +++ b/analyses/mp_net_flux.py @@ -13,7 +13,7 @@ sys.path.append(os.path.join(cn.docker_app,'analyses')) import net_flux -def mp_net_flux(sensit_type, tile_id_list, run_date = None): +def mp_net_flux(sensit_type, tile_id_list, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) @@ -70,17 +70,18 @@ def mp_net_flux(sensit_type, tile_id_list, run_date = None): processes = 9 uu.print_log('Net flux max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(partial(net_flux.net_calc, pattern=pattern, sensit_type=sensit_type), tile_id_list) + pool.map(partial(net_flux.net_calc, pattern=pattern, sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # net_flux.net_calc(tile_id, output_pattern_list[0], sensit_type) + # net_flux.net_calc(tile_id, output_pattern_list[0], sensit_type, no_upload) - # Uploads output tiles to s3 - uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) + # If no_upload flag is not activated, output is uploaded + if not no_upload: + uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) if __name__ == '__main__': @@ -94,16 +95,19 @@ def mp_net_flux(sensit_type, tile_id_list, run_date = None): help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_net_flux(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date) \ No newline at end of file + mp_net_flux(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/analyses/net_flux.py b/analyses/net_flux.py index 50b116d4..7e16c2be 100644 --- a/analyses/net_flux.py +++ b/analyses/net_flux.py @@ -9,7 +9,7 @@ import constants_and_names as cn import universal_util as uu -def net_calc(tile_id, pattern, sensit_type): +def net_calc(tile_id, pattern, sensit_type, no_upload): uu.print_log("Calculating net flux for", tile_id) @@ -92,4 +92,4 @@ def net_calc(tile_id, pattern, sensit_type): net_flux_dst.write_band(1, dst_data, window=window) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, pattern) \ No newline at end of file + uu.end_of_fx_summary(start, tile_id, pattern, no_upload) \ No newline at end of file diff --git a/burn_date/clip_year_tiles.py b/burn_date/clip_year_tiles.py index 3e5939d2..17ecae64 100644 --- a/burn_date/clip_year_tiles.py +++ b/burn_date/clip_year_tiles.py @@ -11,7 +11,7 @@ parentdir = os.path.dirname(currentdir) sys.path.insert(0, parentdir) -def clip_year_tiles(tile_year_list): +def clip_year_tiles(tile_year_list, no_upload): # Start time start = datetime.datetime.now() @@ -57,6 +57,6 @@ def clip_year_tiles(tile_year_list): uu.print_log(" Tile copied to", cn.burn_year_warped_to_Hansen_dir) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, "ba_{}".format(year)) + uu.end_of_fx_summary(start, tile_id, "ba_{}".format(year), no_upload) diff --git a/burn_date/hansen_burnyear_final.py b/burn_date/hansen_burnyear_final.py index 845d4914..345ac450 100644 --- a/burn_date/hansen_burnyear_final.py +++ b/burn_date/hansen_burnyear_final.py @@ -12,7 +12,7 @@ import universal_util as uu -def hansen_burnyear(tile_id): +def hansen_burnyear(tile_id, no_upload): # Start time start = datetime.datetime.now() @@ -160,5 +160,5 @@ def hansen_burnyear(tile_id): os.remove(out_tile_no_tag) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, cn.pattern_burn_year) + uu.end_of_fx_summary(start, tile_id, cn.pattern_burn_year, no_upload) diff --git a/burn_date/mp_burn_year.py b/burn_date/mp_burn_year.py index b568b428..66af73ca 100644 --- a/burn_date/mp_burn_year.py +++ b/burn_date/mp_burn_year.py @@ -40,7 +40,7 @@ import hansen_burnyear_final -def mp_burn_year(tile_id_list, run_date = None): +def mp_burn_year(tile_id_list, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) @@ -191,13 +191,13 @@ def mp_burn_year(tile_id_list, run_date = None): # year burned and NoData. count = multiprocessing.cpu_count() pool = multiprocessing.Pool(processes=count-5) - pool.map(clip_year_tiles.clip_year_tiles, tile_year_list) + pool.map(partial(clip_year_tiles.clip_year_tiles, no_upload=no_upload), tile_year_list) pool.close() pool.join() # # For single processor use # for tile_year in tile_year_list: - # clip_year_tiles.clip_year_tiles(tile_year) + # clip_year_tiles.clip_year_tiles(tile_year, no_upload) uu.print_log("Processing for {} done. Moving to next year.".format(year)) @@ -219,19 +219,19 @@ def mp_burn_year(tile_id_list, run_date = None): else: processes = 1 pool = multiprocessing.Pool(processes) - pool.map(hansen_burnyear_final.hansen_burnyear, tile_id_list) + pool.map(partial(hansen_burnyear_final.hansen_burnyear, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # hansen_burnyear_final.hansen_burnyear(tile_id) + # hansen_burnyear_final.hansen_burnyear(tile_id, no_upload) - # Uploads output tiles to s3 - uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) - + # If no_upload flag is not activated, output is uploaded + if not no_upload: + uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) if __name__ == '__main__': @@ -244,14 +244,17 @@ def mp_burn_year(tile_id_list, run_date = None): help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type='std', run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type='std', run_date=run_date, no_upload=no_upload) # Checks whether the tile_id_list argument is valid tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_burn_year(tile_id_list=tile_id_list, run_date=run_date) \ No newline at end of file + mp_burn_year(tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/carbon_pools/create_carbon_pools.py b/carbon_pools/create_carbon_pools.py index df536c62..ee03b213 100644 --- a/carbon_pools/create_carbon_pools.py +++ b/carbon_pools/create_carbon_pools.py @@ -39,7 +39,7 @@ def mangrove_pool_ratio_dict(gain_table_simplified, tropical_dry, tropical_wet, # Creates aboveground carbon emitted_pools in 2000 and/or the year of loss (loss pixels only) -def create_AGC(tile_id, sensit_type, carbon_pool_extent): +def create_AGC(tile_id, sensit_type, carbon_pool_extent, no_upload): # Start time start = datetime.datetime.now() @@ -252,13 +252,13 @@ def create_AGC(tile_id, sensit_type, carbon_pool_extent): # Prints information about the tile that was just processed if 'loss' in carbon_pool_extent: - uu.end_of_fx_summary(start, tile_id, cn.pattern_AGC_emis_year) + uu.end_of_fx_summary(start, tile_id, cn.pattern_AGC_emis_year, no_upload) else: - uu.end_of_fx_summary(start, tile_id, cn.pattern_AGC_2000) + uu.end_of_fx_summary(start, tile_id, cn.pattern_AGC_2000, no_upload) # Creates belowground carbon tiles (both in 2000 and loss year) -def create_BGC(tile_id, mang_BGB_AGB_ratio, carbon_pool_extent, sensit_type): +def create_BGC(tile_id, mang_BGB_AGB_ratio, carbon_pool_extent, sensit_type, no_upload): start = datetime.datetime.now() @@ -370,13 +370,13 @@ def create_BGC(tile_id, mang_BGB_AGB_ratio, carbon_pool_extent, sensit_type): # Prints information about the tile that was just processed if 'loss' in carbon_pool_extent: - uu.end_of_fx_summary(start, tile_id, cn.pattern_BGC_emis_year) + uu.end_of_fx_summary(start, tile_id, cn.pattern_BGC_emis_year, no_upload) else: - uu.end_of_fx_summary(start, tile_id, cn.pattern_BGC_2000) + uu.end_of_fx_summary(start, tile_id, cn.pattern_BGC_2000, no_upload) # Creates deadwood and litter carbon tiles (in 2000 and/or in loss year) -def create_deadwood_litter(tile_id, mang_deadwood_AGB_ratio, mang_litter_AGB_ratio, carbon_pool_extent, sensit_type): +def create_deadwood_litter(tile_id, mang_deadwood_AGB_ratio, mang_litter_AGB_ratio, carbon_pool_extent, sensit_type, no_upload): start = datetime.datetime.now() @@ -675,13 +675,13 @@ def create_deadwood_litter(tile_id, mang_deadwood_AGB_ratio, mang_litter_AGB_rat # Prints information about the tile that was just processed if 'loss' in carbon_pool_extent: - uu.end_of_fx_summary(start, tile_id, cn.pattern_deadwood_emis_year_2000) + uu.end_of_fx_summary(start, tile_id, cn.pattern_deadwood_emis_year_2000, no_upload) else: - uu.end_of_fx_summary(start, tile_id, cn.pattern_deadwood_2000) + uu.end_of_fx_summary(start, tile_id, cn.pattern_deadwood_2000, no_upload) # Creates soil carbon tiles in loss pixels only -def create_soil_emis_extent(tile_id, pattern, sensit_type): +def create_soil_emis_extent(tile_id, pattern, sensit_type, no_upload): start = datetime.datetime.now() @@ -750,11 +750,11 @@ def create_soil_emis_extent(tile_id, pattern, sensit_type): dst_soil_emis_year.write_band(1, soil_output, window=window) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, pattern) + uu.end_of_fx_summary(start, tile_id, pattern, no_upload) # Creates total carbon tiles (both in 2000 and loss year) -def create_total_C(tile_id, carbon_pool_extent, sensit_type): +def create_total_C(tile_id, carbon_pool_extent, sensit_type, no_upload): start = datetime.datetime.now() @@ -880,6 +880,6 @@ def create_total_C(tile_id, carbon_pool_extent, sensit_type): # Prints information about the tile that was just processed if 'loss' in carbon_pool_extent: - uu.end_of_fx_summary(start, tile_id, cn.pattern_total_C_emis_year) + uu.end_of_fx_summary(start, tile_id, cn.pattern_total_C_emis_year, no_upload) else: - uu.end_of_fx_summary(start, tile_id, cn.pattern_total_C_2000) + uu.end_of_fx_summary(start, tile_id, cn.pattern_total_C_2000, no_upload) diff --git a/carbon_pools/create_soil_C.py b/carbon_pools/create_soil_C.py index be3aa27b..cb9cfb3a 100644 --- a/carbon_pools/create_soil_C.py +++ b/carbon_pools/create_soil_C.py @@ -24,7 +24,7 @@ import constants_and_names as cn # Creates 10x10 mangrove soil C tiles -def create_mangrove_soil_C(tile_id): +def create_mangrove_soil_C(tile_id, no_upload): # Start time start = datetime.datetime.now() @@ -50,21 +50,18 @@ def create_mangrove_soil_C(tile_id): uu.print_log("Masking mangrove soil to mangrove biomass for", tile_id) cmd = ['gdal_calc.py', '-A', mangrove_soil, '-B', mangrove_biomass, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE', '--overwrite', datatype, '--quiet'] - # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging - process = Popen(cmd, stdout=PIPE, stderr=STDOUT) - with process.stdout: - uu.log_subprocess_output(process.stdout) + uu.log_subprocess_output_full(cmd) else: uu.print_log("No mangrove aboveground biomass tile for", tile_id) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, 'mangrove_masked_to_mangrove') + uu.end_of_fx_summary(start, tile_id, 'mangrove_masked_to_mangrove', no_upload) # Overlays the mangrove soil C tiles with the mineral soil C tiles, giving precedence to the mangrove soil C -def create_combined_soil_C(tile_id): +def create_combined_soil_C(tile_id, no_upload): # Start time start = datetime.datetime.now() @@ -122,16 +119,4 @@ def create_combined_soil_C(tile_id): os.rename('{0}_{1}.tif'.format(tile_id, 'mineral_soil'), combined_soil) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, cn.pattern_soil_C_full_extent_2000) - - -def create_soil_C_stdev(tile_id, vrt_CI05, vrt_CI95, out_pattern): - - # Start time - start = datetime.datetime.now() - - - - - # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, out_pattern) + uu.end_of_fx_summary(start, tile_id, cn.pattern_soil_C_full_extent_2000, no_upload) diff --git a/carbon_pools/mp_create_carbon_pools.py b/carbon_pools/mp_create_carbon_pools.py index c714b335..eb5fd76a 100644 --- a/carbon_pools/mp_create_carbon_pools.py +++ b/carbon_pools/mp_create_carbon_pools.py @@ -35,16 +35,16 @@ sys.path.append(os.path.join(cn.docker_app,'carbon_pools')) import create_carbon_pools -def mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_date = None): +def mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) if (sensit_type != 'std') & (carbon_pool_extent != 'loss'): - uu.exception_log("Sensitivity analysis run must use 'loss' extent") + uu.exception_log(no_upload, "Sensitivity analysis run must use 'loss' extent") # Checks the validity of the carbon_pool_extent argument if (carbon_pool_extent not in ['loss', '2000', 'loss,2000', '2000,loss']): - uu.exception_log("Invalid carbon_pool_extent input. Please choose loss, 2000, loss,2000 or 2000,loss.") + uu.exception_log(no_upload, "Invalid carbon_pool_extent input. Please choose loss, 2000, loss,2000 or 2000,loss.") # If a full model run is specified, the correct set of tiles for the particular script is listed. @@ -206,19 +206,23 @@ def mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_da uu.print_log('AGC loss year max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(partial(create_carbon_pools.create_AGC, - sensit_type=sensit_type, carbon_pool_extent=carbon_pool_extent), tile_id_list) + sensit_type=sensit_type, carbon_pool_extent=carbon_pool_extent, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # create_carbon_pools.create_AGC(tile_id, sensit_type, carbon_pool_extent) + # create_carbon_pools.create_AGC(tile_id, sensit_type, carbon_pool_extent, no_upload) + + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + if carbon_pool_extent in ['loss', '2000']: + uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) + else: + uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) + uu.upload_final_set(output_dir_list[6], output_pattern_list[6]) - if carbon_pool_extent in ['loss', '2000']: - uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) - else: - uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) - uu.upload_final_set(output_dir_list[6], output_pattern_list[6]) uu.check_storage() uu.print_log(":::::Freeing up memory for belowground carbon creation; deleting unneeded tiles") @@ -249,19 +253,23 @@ def mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_da pool = multiprocessing.Pool(processes) pool.map(partial(create_carbon_pools.create_BGC, mang_BGB_AGB_ratio=mang_BGB_AGB_ratio, carbon_pool_extent=carbon_pool_extent, - sensit_type=sensit_type), tile_id_list) + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # create_carbon_pools.create_BGC(tile_id, mang_BGB_AGB_ratio, carbon_pool_extent, sensit_type) + # create_carbon_pools.create_BGC(tile_id, mang_BGB_AGB_ratio, carbon_pool_extent, sensit_type, no_upload) + + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + if carbon_pool_extent in ['loss', '2000']: + uu.upload_final_set(output_dir_list[1], output_pattern_list[1]) + else: + uu.upload_final_set(output_dir_list[1], output_pattern_list[1]) + uu.upload_final_set(output_dir_list[7], output_pattern_list[7]) - if carbon_pool_extent in ['loss', '2000']: - uu.upload_final_set(output_dir_list[1], output_pattern_list[1]) - else: - uu.upload_final_set(output_dir_list[1], output_pattern_list[1]) - uu.upload_final_set(output_dir_list[7], output_pattern_list[7]) uu.check_storage() @@ -306,22 +314,26 @@ def mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_da partial(create_carbon_pools.create_deadwood_litter, mang_deadwood_AGB_ratio=mang_deadwood_AGB_ratio, mang_litter_AGB_ratio=mang_litter_AGB_ratio, carbon_pool_extent=carbon_pool_extent, - sensit_type=sensit_type), tile_id_list) + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # create_carbon_pools.create_deadwood_litter(tile_id, mang_deadwood_AGB_ratio, mang_litter_AGB_ratio, carbon_pool_extent, sensit_type) + # create_carbon_pools.create_deadwood_litter(tile_id, mang_deadwood_AGB_ratio, mang_litter_AGB_ratio, carbon_pool_extent, sensit_type, no_upload) + + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + if carbon_pool_extent in ['loss', '2000']: + uu.upload_final_set(output_dir_list[2], output_pattern_list[2]) # deadwood + uu.upload_final_set(output_dir_list[3], output_pattern_list[3]) # litter + else: + uu.upload_final_set(output_dir_list[2], output_pattern_list[2]) # deadwood + uu.upload_final_set(output_dir_list[3], output_pattern_list[3]) # litter + uu.upload_final_set(output_dir_list[8], output_pattern_list[8]) # deadwood + uu.upload_final_set(output_dir_list[9], output_pattern_list[9]) # litter - if carbon_pool_extent in ['loss', '2000']: - uu.upload_final_set(output_dir_list[2], output_pattern_list[2]) # deadwood - uu.upload_final_set(output_dir_list[3], output_pattern_list[3]) # litter - else: - uu.upload_final_set(output_dir_list[2], output_pattern_list[2]) # deadwood - uu.upload_final_set(output_dir_list[3], output_pattern_list[3]) # litter - uu.upload_final_set(output_dir_list[8], output_pattern_list[8]) # deadwood - uu.upload_final_set(output_dir_list[9], output_pattern_list[9]) # litter uu.check_storage() uu.print_log(":::::Freeing up memory for soil and total carbon creation; deleting unneeded tiles") @@ -364,20 +376,23 @@ def mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_da uu.print_log('Soil carbon loss year max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(partial(create_carbon_pools.create_soil_emis_extent, pattern=pattern, - sensit_type=sensit_type), tile_id_list) + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # create_carbon_pools.create_soil_emis_extent(tile_id, pattern, sensit_type) + # create_carbon_pools.create_soil_emis_extent(tile_id, pattern, sensit_type, no_upload) - # If pools in 2000 weren't generated, soil carbon in emissions extent is 4. - # If pools in 2000 were generated, soil carbon in emissions extent is 10. - if '2000' not in carbon_pool_extent: - uu.upload_final_set(output_dir_list[4], output_pattern_list[4]) - else: - uu.upload_final_set(output_dir_list[10], output_pattern_list[10]) + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + # If pools in 2000 weren't generated, soil carbon in emissions extent is 4. + # If pools in 2000 were generated, soil carbon in emissions extent is 10. + if '2000' not in carbon_pool_extent: + uu.upload_final_set(output_dir_list[4], output_pattern_list[4]) + else: + uu.upload_final_set(output_dir_list[10], output_pattern_list[10]) uu.check_storage() @@ -418,19 +433,23 @@ def mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_da uu.print_log('Total carbon loss year max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(partial(create_carbon_pools.create_total_C, carbon_pool_extent=carbon_pool_extent, - sensit_type=sensit_type), tile_id_list) + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # create_carbon_pools.create_total_C(tile_id, carbon_pool_extent, sensit_type) + # create_carbon_pools.create_total_C(tile_id, carbon_pool_extent, sensit_type, no_upload) + + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + if carbon_pool_extent in ['loss', '2000']: + uu.upload_final_set(output_dir_list[5], output_pattern_list[5]) + else: + uu.upload_final_set(output_dir_list[5], output_pattern_list[5]) + uu.upload_final_set(output_dir_list[11], output_pattern_list[11]) - if carbon_pool_extent in ['loss', '2000']: - uu.upload_final_set(output_dir_list[5], output_pattern_list[5]) - else: - uu.upload_final_set(output_dir_list[5], output_pattern_list[5]) - uu.upload_final_set(output_dir_list[11], output_pattern_list[11]) uu.check_storage() @@ -447,18 +466,22 @@ def mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_da help='Extent over which carbon emitted_pools should be calculated: loss, 2000, loss,2000, or 2000,loss') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list carbon_pool_extent = args.carbon_pool_extent # Tells the pool creation functions to calculate carbon emitted_pools as they were at the year of loss in loss pixels only run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, carbon_pool_extent=carbon_pool_extent) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, + carbon_pool_extent=carbon_pool_extent, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) mp_create_carbon_pools(sensit_type=sensit_type, tile_id_list=tile_id_list, - carbon_pool_extent=carbon_pool_extent, run_date=run_date) \ No newline at end of file + carbon_pool_extent=carbon_pool_extent, run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/carbon_pools/mp_create_soil_C.py b/carbon_pools/mp_create_soil_C.py index c2d2e8b0..6aacb5c1 100644 --- a/carbon_pools/mp_create_soil_C.py +++ b/carbon_pools/mp_create_soil_C.py @@ -27,7 +27,7 @@ import constants_and_names as cn import universal_util as uu -def mp_create_soil_C(tile_id_list): +def mp_create_soil_C(tile_id_list, no_upload=None): os.chdir(cn.docker_base_dir) sensit_type = 'std' @@ -85,14 +85,14 @@ def mp_create_soil_C(tile_id_list): processes = int(cn.count/3) uu.print_log('Mangrove soil C max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(create_soil_C.create_mangrove_soil_C, tile_id_list) + pool.map(partial(create_soil_C.create_mangrove_soil_C, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: # - # create_soil_C.create_mangrove_soil_C(tile_id) + # create_soil_C.create_mangrove_soil_C(tile_id, no_Upload) uu.print_log('Done making mangrove soil C tiles', '\n') @@ -110,7 +110,8 @@ def mp_create_soil_C(tile_id_list): processes = int(cn.count/2) uu.print_log("Creating mineral soil C density tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, + no_upload=no_upload), tile_id_list) pool.close() pool.join() @@ -130,19 +131,23 @@ def mp_create_soil_C(tile_id_list): processes = int(cn.count/2) uu.print_log('Combined soil C max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(create_soil_C.create_combined_soil_C, tile_id_list) + pool.map(partial(create_soil_C.create_combined_soil_C, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile in tile_list: # - # create_soil_C.create_combined_soil_C(tile_id) + # create_soil_C.create_combined_soil_C(tile_id, no_upload) uu.print_log("Done making combined soil C tiles") - uu.print_log("Uploading soil C density tiles") - uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + uu.print_log("Uploading soil C density tiles") + uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) + # Need to delete soil c density rasters because they have the same pattern as the standard deviation rasters uu.print_log("Deleting raw soil C density rasters") @@ -215,7 +220,8 @@ def mp_create_soil_C(tile_id_list): processes = 2 uu.print_log("Creating mineral soil C stock stdev tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, + no_upload=no_upload), tile_id_list) pool.close() pool.join() @@ -248,8 +254,12 @@ def mp_create_soil_C(tile_id_list): pool.close() pool.join() - uu.print_log("Uploading soil C density standard deviation tiles") - uu.upload_final_set(output_dir_list[1], output_pattern_list[1]) + + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + uu.print_log("Uploading soil C density standard deviation tiles") + uu.upload_final_set(output_dir_list[1], output_pattern_list[1]) if __name__ == '__main__': @@ -260,12 +270,15 @@ def mp_create_soil_C(tile_id_list): help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log uu.initiate_log(tile_id_list, run_date=run_date) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_create_soil_C(tile_id_list=tile_id_list) \ No newline at end of file + mp_create_soil_C(tile_id_list=tile_id_list, no_upload=no_upload) \ No newline at end of file diff --git a/data_prep/model_extent.py b/data_prep/model_extent.py index e91fe0c2..bc2d615d 100644 --- a/data_prep/model_extent.py +++ b/data_prep/model_extent.py @@ -8,7 +8,7 @@ import constants_and_names as cn import universal_util as uu -def model_extent(tile_id, pattern, sensit_type): +def model_extent(tile_id, pattern, sensit_type, no_upload): # I don't know why, but this needs to be here and not just in mp_model_extent os.chdir(cn.docker_base_dir) @@ -156,4 +156,4 @@ def model_extent(tile_id, pattern, sensit_type): # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, pattern) \ No newline at end of file + uu.end_of_fx_summary(start, tile_id, pattern, no_upload) \ No newline at end of file diff --git a/data_prep/mp_mangrove_processing.py b/data_prep/mp_mangrove_processing.py index f2d2ca0f..89287cb8 100644 --- a/data_prep/mp_mangrove_processing.py +++ b/data_prep/mp_mangrove_processing.py @@ -13,10 +13,9 @@ import constants_and_names as cn import universal_util as uu -def mp_mangrove_processing(tile_id_list, run_date = None): +def mp_mangrove_processing(tile_id_list, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) - sensit_type = 'std' # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': @@ -33,10 +32,8 @@ def mp_mangrove_processing(tile_id_list, run_date = None): # Unzips mangrove images into a flat structure (all tifs into main folder using -j argument) # NOTE: Unzipping some tifs (e.g., Australia, Indonesia) takes a very long time, so don't worry if the script appears to stop on that. cmd = ['unzip', '-o', '-j', cn.mangrove_biomass_raw_file] - # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging - process = Popen(cmd, stdout=PIPE, stderr=STDOUT) - with process.stdout: - uu.log_subprocess_output(process.stdout) + uu.log_subprocess_output_full(cmd) + # Creates vrt for the Saatchi biomass rasters mangrove_vrt = 'mangrove_biomass.vrt' @@ -49,12 +46,13 @@ def mp_mangrove_processing(tile_id_list, run_date = None): processes=int(cn.count/4) uu.print_log('Mangrove preprocessing max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, + no_upload=no_upload), tile_id_list) # # For single processor use, for testing purposes # for tile_id in tile_id_list: # - # mangrove_processing.create_mangrove_tiles(tile_id, source_raster, out_pattern) + # mangrove_processing.create_mangrove_tiles(tile_id, source_raster, out_pattern, no_upload) # Checks if each tile has data in it. Only tiles with data are uploaded. upload_dir = cn.mangrove_biomass_2000_dir @@ -73,11 +71,14 @@ def mp_mangrove_processing(tile_id_list, run_date = None): help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) - mp_mangrove_processing(tile_id_list=tile_id_list, run_date=run_date) \ No newline at end of file + mp_mangrove_processing(tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/data_prep/mp_model_extent.py b/data_prep/mp_model_extent.py index 4ddce956..4486c184 100644 --- a/data_prep/mp_model_extent.py +++ b/data_prep/mp_model_extent.py @@ -22,7 +22,7 @@ sys.path.append(os.path.join(cn.docker_app,'data_prep')) import model_extent -def mp_model_extent(sensit_type, tile_id_list, run_date = None): +def mp_model_extent(sensit_type, tile_id_list, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) @@ -98,13 +98,13 @@ def mp_model_extent(sensit_type, tile_id_list, run_date = None): processes = 3 uu.print_log('Model extent processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(partial(model_extent.model_extent, pattern=pattern, sensit_type=sensit_type), tile_id_list) + pool.map(partial(model_extent.model_extent, pattern=pattern, sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # model_extent.model_extent(tile_id, pattern, sensit_type) + # model_extent.model_extent(tile_id, pattern, sensit_type, no_upload) output_pattern = output_pattern_list[0] if cn.count <= 2: # For local tests @@ -124,8 +124,10 @@ def mp_model_extent(sensit_type, tile_id_list, run_date = None): pool.join() - # Uploads output tiles to s3 - uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) if __name__ == '__main__': @@ -140,17 +142,20 @@ def mp_model_extent(sensit_type, tile_id_list, run_date = None): help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_model_extent(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date) + mp_model_extent(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) diff --git a/data_prep/mp_prep_other_inputs.py b/data_prep/mp_prep_other_inputs.py index da0dbeef..89252ffa 100644 --- a/data_prep/mp_prep_other_inputs.py +++ b/data_prep/mp_prep_other_inputs.py @@ -19,7 +19,7 @@ import constants_and_names as cn import universal_util as uu -def mp_prep_other_inputs(tile_id_list, run_date): +def mp_prep_other_inputs(tile_id_list, run_date, no_upload = None): os.chdir(cn.docker_base_dir) sensit_type='std' @@ -129,7 +129,8 @@ def mp_prep_other_inputs(tile_id_list, run_date): processes = int(cn.count/2) uu.print_log("Creating tree cover loss driver tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, + no_upload=no_upload), tile_id_list) pool.close() pool.join() @@ -144,7 +145,7 @@ def mp_prep_other_inputs(tile_id_list, run_date): # processes = int(cn.count/2) # uu.print_log("Creating young natural forest gain rate tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) - # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # @@ -158,7 +159,7 @@ def mp_prep_other_inputs(tile_id_list, run_date): # processes = int(cn.count/2) # uu.print_log("Creating standard deviation for young natural forest removal rate tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) - # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # @@ -196,7 +197,7 @@ def mp_prep_other_inputs(tile_id_list, run_date): # processes = int(cn.count/2) # uu.print_log("Creating European natural forest gain rate tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) - # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # @@ -210,7 +211,7 @@ def mp_prep_other_inputs(tile_id_list, run_date): # processes = int(cn.count/2) # uu.print_log("Creating standard deviation for European natural forest gain rate tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) - # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # @@ -231,7 +232,7 @@ def mp_prep_other_inputs(tile_id_list, run_date): # processes = int(cn.count/2) # uu.print_log("Creating primary forest tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) - # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # @@ -259,7 +260,7 @@ def mp_prep_other_inputs(tile_id_list, run_date): # processes = int(cn.count/2) # uu.print_log("Creating US forest age category tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) - # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # @@ -273,7 +274,7 @@ def mp_prep_other_inputs(tile_id_list, run_date): # processes = int(cn.count/2) # uu.print_log("Creating US forest group tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) - # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # @@ -287,7 +288,7 @@ def mp_prep_other_inputs(tile_id_list, run_date): # processes = int(cn.count/2) # uu.print_log("Creating US forest region tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) - # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # @@ -345,14 +346,17 @@ def mp_prep_other_inputs(tile_id_list, run_date): help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) # Checks whether the tile_id_list argument is valid tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_prep_other_inputs(tile_id_list=tile_id_list, run_date=run_date) \ No newline at end of file + mp_prep_other_inputs(tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/emissions/calculate_gross_emissions.py b/emissions/calculate_gross_emissions.py index f82955c9..91992ce6 100644 --- a/emissions/calculate_gross_emissions.py +++ b/emissions/calculate_gross_emissions.py @@ -9,7 +9,7 @@ import universal_util as uu # Calls the c++ script to calculate gross emissions -def calc_emissions(tile_id, emitted_pools, sensit_type, folder): +def calc_emissions(tile_id, emitted_pools, sensit_type, folder, no_upload): uu.print_log("Calculating gross emissions for", tile_id, "using", sensit_type, "model type...") @@ -29,7 +29,7 @@ def calc_emissions(tile_id, emitted_pools, sensit_type, folder): cmd = ['{0}/calc_gross_emissions_generic.exe'.format(cn.c_emis_compile_dst), tile_id, sensit_type, folder] else: - uu.exception_log('Pool and/or sensitivity analysis option not valid') + uu.exception_log(no_upload, 'Pool and/or sensitivity analysis option not valid') uu.log_subprocess_output_full(cmd) @@ -46,10 +46,10 @@ def calc_emissions(tile_id, emitted_pools, sensit_type, folder): pattern = pattern.replace('biomass_soil', 'soil_only') else: - uu.exception_log('Pool option not valid') + uu.exception_log(no_upload, 'Pool option not valid') # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, pattern) + uu.end_of_fx_summary(start, tile_id, pattern, no_upload) # Adds metadata tags to the output rasters diff --git a/emissions/mp_calculate_gross_emissions.py b/emissions/mp_calculate_gross_emissions.py index 18191cfc..2735f47c 100644 --- a/emissions/mp_calculate_gross_emissions.py +++ b/emissions/mp_calculate_gross_emissions.py @@ -34,7 +34,7 @@ sys.path.append(os.path.join(cn.docker_app,'emissions')) import calculate_gross_emissions -def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_date = None): +def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) @@ -77,7 +77,7 @@ def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_d # Checks the validity of the emitted_pools argument if (emitted_pools not in ['soil_only', 'biomass_soil']): - uu.exception_log('Invalid pool input. Please choose soil_only or biomass_soil.') + uu.exception_log(no_upload, 'Invalid pool input. Please choose soil_only or biomass_soil.') # Checks if the correct c++ script has been compiled for the pool option selected @@ -113,12 +113,12 @@ def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_d if os.path.exists('{0}/calc_gross_emissions_{1}.exe'.format(cn.c_emis_compile_dst, sensit_type)): uu.print_log("C++ for {} already compiled.".format(sensit_type)) else: - uu.exception_log('Must compile {} model C++...'.format(sensit_type)) + uu.exception_log(no_upload, 'Must compile {} model C++...'.format(sensit_type)) else: if os.path.exists('{0}/calc_gross_emissions_generic.exe'.format(cn.c_emis_compile_dst)): uu.print_log("C++ for generic emissions already compiled.") else: - uu.exception_log('Must compile generic emissions C++...') + uu.exception_log(no_upload, 'Must compile generic emissions C++...') elif (emitted_pools == 'soil_only') & (sensit_type == 'std'): if os.path.exists('{0}/calc_gross_emissions_soil_only.exe'.format(cn.c_emis_compile_dst)): @@ -148,10 +148,10 @@ def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_d cn.pattern_gross_emis_nodes_soil_only] else: - uu.exception_log('Must compile soil_only C++...') + uu.exception_log(no_upload, 'Must compile soil_only C++...') else: - uu.exception_log('Pool and/or sensitivity analysis option not valid') + uu.exception_log(no_upload, 'Pool and/or sensitivity analysis option not valid') # Downloads input files or entire directories, depending on how many tiles are in the tile_id_list @@ -217,13 +217,14 @@ def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_d processes = 9 uu.print_log('Gross emissions max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(partial(calculate_gross_emissions.calc_emissions, emitted_pools=emitted_pools, sensit_type=sensit_type, folder=folder), tile_id_list) + pool.map(partial(calculate_gross_emissions.calc_emissions, emitted_pools=emitted_pools, sensit_type=sensit_type, + folder=folder, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile in tile_id_list: - # calculate_gross_emissions.calc_emissions(tile, emitted_pools, sensit_type, folder) + # calculate_gross_emissions.calc_emissions(tile, emitted_pools, sensit_type, folder, no_upload) # Print the list of blank created tiles, delete the tiles, and delete their text file @@ -250,9 +251,11 @@ def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_d # calculate_gross_emissions.add_metadata_tags(tile_id, pattern, sensit_type) - # Uploads emissions to appropriate directory for the carbon emitted_pools chosen - for i in range(0, len(output_dir_list)): - uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + for i in range(0, len(output_dir_list)): + uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) if __name__ == '__main__': @@ -268,14 +271,18 @@ def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_d help='{}'.format(cn.model_type_arg_help)) parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list emitted_pools = args.emitted_pools_to_use run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, emitted_pools=emitted_pools) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, + emitted_pools=emitted_pools, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) @@ -285,4 +292,5 @@ def mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_d else: tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_calculate_gross_emissions(sensit_type=sensit_type, tile_id_list=tile_id_list, emitted_pools=emitted_pools, run_date=run_date) \ No newline at end of file + mp_calculate_gross_emissions(sensit_type=sensit_type, tile_id_list=tile_id_list, emitted_pools=emitted_pools, + run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/gain/annual_gain_rate_AGC_BGC_all_forest_types.py b/gain/annual_gain_rate_AGC_BGC_all_forest_types.py index 366b348b..a285bded 100644 --- a/gain/annual_gain_rate_AGC_BGC_all_forest_types.py +++ b/gain/annual_gain_rate_AGC_BGC_all_forest_types.py @@ -8,7 +8,7 @@ import constants_and_names as cn import universal_util as uu -def annual_gain_rate_AGC_BGC_all_forest_types(tile_id, output_pattern_list, sensit_type): +def annual_gain_rate_AGC_BGC_all_forest_types(tile_id, output_pattern_list, sensit_type, no_upload): uu.print_log("Mapping removal rate source and AGB and BGB removal rates:", tile_id) @@ -323,4 +323,4 @@ def annual_gain_rate_AGC_BGC_all_forest_types(tile_id, output_pattern_list, sens stdev_annual_gain_AGC_all_forest_types_dst.write_band(1, stdev_annual_gain_AGC_all_forest_types_window, window=window) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, cn.pattern_removal_forest_type) \ No newline at end of file + uu.end_of_fx_summary(start, tile_id, cn.pattern_removal_forest_type, no_upload) \ No newline at end of file diff --git a/gain/annual_gain_rate_IPCC_defaults.py b/gain/annual_gain_rate_IPCC_defaults.py index b2686442..674ff906 100644 --- a/gain/annual_gain_rate_IPCC_defaults.py +++ b/gain/annual_gain_rate_IPCC_defaults.py @@ -10,7 +10,7 @@ # Necessary to suppress a pandas error later on np.set_printoptions(threshold=np.nan) -def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict, output_pattern_list): +def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict, output_pattern_list, no_upload): # Converts the forest age category decision tree output values to the three age categories-- # 10000: primary forest; 20000: secondary forest > 20 years; 30000: secondary forest <= 20 years @@ -147,4 +147,4 @@ def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict, ou dst_stdev_above.write_band(1, gain_stdev_AGB, window=window) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, output_pattern_list[0]) + uu.end_of_fx_summary(start, tile_id, output_pattern_list[0], no_upload) diff --git a/gain/forest_age_category_IPCC.py b/gain/forest_age_category_IPCC.py index 49d48660..68802219 100644 --- a/gain/forest_age_category_IPCC.py +++ b/gain/forest_age_category_IPCC.py @@ -8,7 +8,7 @@ import constants_and_names as cn import universal_util as uu -def forest_age_category(tile_id, gain_table_dict, pattern, sensit_type): +def forest_age_category(tile_id, gain_table_dict, pattern, sensit_type, no_upload): uu.print_log("Assigning forest age categories:", tile_id) @@ -208,4 +208,4 @@ def forest_age_category(tile_id, gain_table_dict, pattern, sensit_type): dst.write_band(1, dst_data, window=window) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, pattern) \ No newline at end of file + uu.end_of_fx_summary(start, tile_id, pattern, no_upload) \ No newline at end of file diff --git a/gain/gain_year_count_all_forest_types.py b/gain/gain_year_count_all_forest_types.py index ee7f2246..deb69503 100644 --- a/gain/gain_year_count_all_forest_types.py +++ b/gain/gain_year_count_all_forest_types.py @@ -23,7 +23,7 @@ def tile_names(tile_id, sensit_type): # Creates gain year count tiles for pixels that only had loss -def create_gain_year_count_loss_only(tile_id, sensit_type): +def create_gain_year_count_loss_only(tile_id, sensit_type, no_upload): uu.print_log("Gain year count for loss only pixels:", tile_id) @@ -45,11 +45,11 @@ def create_gain_year_count_loss_only(tile_id, sensit_type): uu.print_log("No loss tile found for {}. Skipping loss only pixel gain year count.".format(tile_id)) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_only') + uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_only', no_upload) # Creates gain year count tiles for pixels that only had gain -def create_gain_year_count_gain_only_standard(tile_id, sensit_type): +def create_gain_year_count_gain_only_standard(tile_id, sensit_type, no_upload): uu.print_log("Gain year count for gain only pixels using standard function:", tile_id) @@ -77,11 +77,11 @@ def create_gain_year_count_gain_only_standard(tile_id, sensit_type): uu.log_subprocess_output_full(cmd) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, 'growth_years_gain_only') + uu.end_of_fx_summary(start, tile_id, 'growth_years_gain_only', no_upload) # Creates gain year count tiles for pixels that only had gain -def create_gain_year_count_gain_only_maxgain(tile_id, sensit_type): +def create_gain_year_count_gain_only_maxgain(tile_id, sensit_type, no_upload): uu.print_log("Gain year count for gain only pixels using maxgain function:", tile_id) @@ -109,12 +109,12 @@ def create_gain_year_count_gain_only_maxgain(tile_id, sensit_type): uu.log_subprocess_output_full(cmd) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, 'growth_years_gain_only') + uu.end_of_fx_summary(start, tile_id, 'growth_years_gain_only', no_upload) # Creates gain year count tiles for pixels that had neither loss not gain. # For all models except legal_Amazon_loss. -def create_gain_year_count_no_change_standard(tile_id, sensit_type): +def create_gain_year_count_no_change_standard(tile_id, sensit_type, no_upload): uu.print_log("Gain year count for pixels with neither loss nor gain:", tile_id) @@ -142,12 +142,12 @@ def create_gain_year_count_no_change_standard(tile_id, sensit_type): uu.log_subprocess_output_full(cmd) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change') + uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change', no_upload) # Creates gain year count tiles for pixels that did not have loss (doesn't matter if they had gain or not). # For legal_Amazon_loss sensitivity analysis. -def create_gain_year_count_no_change_legal_Amazon_loss(tile_id, sensit_type): +def create_gain_year_count_no_change_legal_Amazon_loss(tile_id, sensit_type, no_upload): uu.print_log("Gain year count for pixels without loss for legal_Amazon_loss:", tile_id) @@ -173,11 +173,11 @@ def create_gain_year_count_no_change_legal_Amazon_loss(tile_id, sensit_type): os.remove(loss_vrt) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change') + uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change', no_upload) # Creates gain year count tiles for pixels that had both loss and gain -def create_gain_year_count_loss_and_gain_standard(tile_id, sensit_type): +def create_gain_year_count_loss_and_gain_standard(tile_id, sensit_type, no_upload): uu.print_log("Loss and gain pixel processing using standard function:", tile_id) @@ -200,11 +200,11 @@ def create_gain_year_count_loss_and_gain_standard(tile_id, sensit_type): # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain') + uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain', no_upload) # Creates gain year count tiles for pixels that had both loss and gain -def create_gain_year_count_loss_and_gain_maxgain(tile_id, sensit_type): +def create_gain_year_count_loss_and_gain_maxgain(tile_id, sensit_type, no_upload): uu.print_log("Loss and gain pixel processing using maxgain function:", tile_id) @@ -226,11 +226,11 @@ def create_gain_year_count_loss_and_gain_maxgain(tile_id, sensit_type): uu.print_log("No loss tile found for {}. Skipping loss and gain pixel gain year count.".format(tile_id)) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain') + uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain', no_upload) # Merges the four gain year count tiles above to create a single gain year count tile -def create_gain_year_count_merge(tile_id, pattern, sensit_type): +def create_gain_year_count_merge(tile_id, pattern, sensit_type, no_upload): uu.print_log("Merging loss, gain, no change, and loss/gain pixels into single raster for {}".format(tile_id)) @@ -327,4 +327,4 @@ def create_gain_year_count_merge(tile_id, pattern, sensit_type): gain_year_count_merged_dst.write_band(1, gain_year_count_merged_window, window=window) # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, pattern) \ No newline at end of file + uu.end_of_fx_summary(start, tile_id, pattern, no_upload) \ No newline at end of file diff --git a/gain/gross_removals_all_forest_types.py b/gain/gross_removals_all_forest_types.py index 6329cb6a..d66240d9 100644 --- a/gain/gross_removals_all_forest_types.py +++ b/gain/gross_removals_all_forest_types.py @@ -7,7 +7,7 @@ import universal_util as uu # Calculates cumulative aboveground carbon dioxide gain in mangroves -def gross_removals_all_forest_types(tile_id, output_pattern_list, sensit_type): +def gross_removals_all_forest_types(tile_id, output_pattern_list, sensit_type, no_upload): uu.print_log("Calculating cumulative CO2 removals:", tile_id) @@ -106,4 +106,4 @@ def gross_removals_all_forest_types(tile_id, output_pattern_list, sensit_type): # Prints information about the tile that was just processed - uu.end_of_fx_summary(start, tile_id, output_pattern_list[0]) + uu.end_of_fx_summary(start, tile_id, output_pattern_list[0], no_upload) diff --git a/gain/mp_annual_gain_rate_AGC_BGC_all_forest_types.py b/gain/mp_annual_gain_rate_AGC_BGC_all_forest_types.py index a37c13d4..9fef8c0d 100644 --- a/gain/mp_annual_gain_rate_AGC_BGC_all_forest_types.py +++ b/gain/mp_annual_gain_rate_AGC_BGC_all_forest_types.py @@ -24,7 +24,7 @@ sys.path.append(os.path.join(cn.docker_app,'gain')) import annual_gain_rate_AGC_BGC_all_forest_types -def mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type, tile_id_list, run_date = None): +def mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type, tile_id_list, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) @@ -97,13 +97,13 @@ def mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type, tile_id_list, run_ uu.print_log('Removal factor processors=', processes) pool = multiprocessing.Pool(processes) pool.map(partial(annual_gain_rate_AGC_BGC_all_forest_types.annual_gain_rate_AGC_BGC_all_forest_types, - output_pattern_list=output_pattern_list, sensit_type=sensit_type), tile_id_list) + output_pattern_list=output_pattern_list, sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # annual_gain_rate_AGC_BGC_all_forest_types.annual_gain_rate_AGC_BGC_all_forest_types(tile_id, sensit_type) + # annual_gain_rate_AGC_BGC_all_forest_types.annual_gain_rate_AGC_BGC_all_forest_types(tile_id, sensit_type, no_upload) # Checks the gross removals outputs for tiles with no data for output_pattern in output_pattern_list: @@ -125,9 +125,11 @@ def mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type, tile_id_list, run_ pool.join() - # Uploads output tiles to s3 - for i in range(0, len(output_dir_list)): - uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + for i in range(0, len(output_dir_list)): + uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) if __name__ == '__main__': @@ -142,17 +144,21 @@ def mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type, tile_id_list, run_ help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date) + mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type=sensit_type, tile_id_list=tile_id_list, + run_date=run_date, no_upload=no_upload) diff --git a/gain/mp_annual_gain_rate_IPCC_defaults.py b/gain/mp_annual_gain_rate_IPCC_defaults.py index 1e01e0e8..6390fdd3 100644 --- a/gain/mp_annual_gain_rate_IPCC_defaults.py +++ b/gain/mp_annual_gain_rate_IPCC_defaults.py @@ -31,7 +31,7 @@ os.chdir(cn.docker_base_dir) -def mp_annual_gain_rate_IPCC_defaults(sensit_type, tile_id_list, run_date = None): +def mp_annual_gain_rate_IPCC_defaults(sensit_type, tile_id_list, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) pd.options.mode.chained_assignment = None @@ -206,7 +206,7 @@ def mp_annual_gain_rate_IPCC_defaults(sensit_type, tile_id_list, run_date = None pool = multiprocessing.Pool(processes) pool.map(partial(annual_gain_rate_IPCC_defaults.annual_gain_rate, sensit_type=sensit_type, gain_table_dict=gain_table_dict, stdev_table_dict=stdev_table_dict, - output_pattern_list=output_pattern_list), tile_id_list) + output_pattern_list=output_pattern_list, no_upload=no_upload), tile_id_list) pool.close() pool.join() @@ -214,12 +214,14 @@ def mp_annual_gain_rate_IPCC_defaults(sensit_type, tile_id_list, run_date = None # for tile_id in tile_id_list: # # annual_gain_rate_IPCC_defaults.annual_gain_rate(tile_id, sensit_type, - # gain_table_dict, stdev_table_dict, output_pattern_list) + # gain_table_dict, stdev_table_dict, output_pattern_list, no_upload) - # Uploads output tiles to s3 - for i in range(0, len(output_dir_list)): - uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + for i in range(0, len(output_dir_list)): + uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) if __name__ == '__main__': @@ -234,16 +236,19 @@ def mp_annual_gain_rate_IPCC_defaults(sensit_type, tile_id_list, run_date = None help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_annual_gain_rate_IPCC_defaults(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date) \ No newline at end of file + mp_annual_gain_rate_IPCC_defaults(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/gain/mp_forest_age_category_IPCC.py b/gain/mp_forest_age_category_IPCC.py index b9908a00..24c50a85 100644 --- a/gain/mp_forest_age_category_IPCC.py +++ b/gain/mp_forest_age_category_IPCC.py @@ -26,7 +26,7 @@ sys.path.append(os.path.join(cn.docker_app,'gain')) import forest_age_category_IPCC -def mp_forest_age_category_IPCC(sensit_type, tile_id_list, run_date = None): +def mp_forest_age_category_IPCC(sensit_type, tile_id_list, run_date = None, no_upload = None): os.chdir(cn.docker_base_dir) @@ -121,17 +121,19 @@ def mp_forest_age_category_IPCC(sensit_type, tile_id_list, run_date = None): uu.print_log('Natural forest age category max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(partial(forest_age_category_IPCC.forest_age_category, gain_table_dict=gain_table_dict, - pattern=pattern, sensit_type=sensit_type), tile_id_list) + pattern=pattern, sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: # - # forest_age_category_IPCC.forest_age_category(tile_id, gain_table_dict, pattern, sensit_type) + # forest_age_category_IPCC.forest_age_category(tile_id, gain_table_dict, pattern, sensit_type, no_upload) - # Uploads output tiles to s3 - uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) if __name__ == '__main__': @@ -146,17 +148,20 @@ def mp_forest_age_category_IPCC(sensit_type, tile_id_list, run_date = None): help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_forest_age_category_IPCC(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date) + mp_forest_age_category_IPCC(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) diff --git a/gain/mp_gain_year_count_all_forest_types.py b/gain/mp_gain_year_count_all_forest_types.py index 06fbd6e8..0ac5a1d6 100644 --- a/gain/mp_gain_year_count_all_forest_types.py +++ b/gain/mp_gain_year_count_all_forest_types.py @@ -21,7 +21,7 @@ import constants_and_names as cn import universal_util as uu -def mp_gain_year_count_all_forest_types(sensit_type, tile_id_list, run_date = None): +def mp_gain_year_count_all_forest_types(sensit_type, tile_id_list, run_date = None, no_upload = True): os.chdir(cn.docker_base_dir) @@ -83,7 +83,8 @@ def mp_gain_year_count_all_forest_types(sensit_type, tile_id_list, run_date = No processes = int(cn.count/2) uu.print_log('Gain year count loss only pixels max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_loss_only, sensit_type=sensit_type), tile_id_list) + pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_loss_only, + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) if cn.count == 96: processes = 90 # 66 = 330 GB peak; 75 = 380 GB peak; 90 = 530 GB peak @@ -93,12 +94,14 @@ def mp_gain_year_count_all_forest_types(sensit_type, tile_id_list, run_date = No pool = multiprocessing.Pool(processes) if sensit_type == 'maxgain': # Creates gain year count tiles using only pixels that had only gain - pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_gain_only_maxgain, sensit_type=sensit_type), tile_id_list) + pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_gain_only_maxgain, + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) if sensit_type == 'legal_Amazon_loss': uu.print_log("Gain-only pixels do not apply to legal_Amazon_loss sensitivity analysis. Skipping this step.") else: # Creates gain year count tiles using only pixels that had only gain - pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_gain_only_standard, sensit_type=sensit_type), tile_id_list) + pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_gain_only_standard, + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) # Creates gain year count tiles using only pixels that had neither loss nor gain pixels if cn.count == 96: @@ -108,9 +111,11 @@ def mp_gain_year_count_all_forest_types(sensit_type, tile_id_list, run_date = No uu.print_log('Gain year count no change pixels max processors=', processes) pool = multiprocessing.Pool(processes) if sensit_type == 'legal_Amazon_loss': - pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_no_change_legal_Amazon_loss, sensit_type=sensit_type), tile_id_list) + pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_no_change_legal_Amazon_loss, + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) else: - pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_no_change_standard, sensit_type=sensit_type), tile_id_list) + pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_no_change_standard, + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) if cn.count == 96: processes = 90 # 66 = 370 GB peak; 88 = 430 GB peak; 90 = 550 GB peak @@ -120,10 +125,12 @@ def mp_gain_year_count_all_forest_types(sensit_type, tile_id_list, run_date = No pool = multiprocessing.Pool(processes) if sensit_type == 'maxgain': # Creates gain year count tiles using only pixels that had only gain - pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_loss_and_gain_maxgain, sensit_type=sensit_type), tile_id_list) + pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_loss_and_gain_maxgain, + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) else: # Creates gain year count tiles using only pixels that had only gain - pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_loss_and_gain_standard, sensit_type=sensit_type), tile_id_list) + pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_loss_and_gain_standard, + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) # Combines the four above gain year count tiles for each Hansen tile into a single output tile if cn.count == 96: @@ -134,42 +141,46 @@ def mp_gain_year_count_all_forest_types(sensit_type, tile_id_list, run_date = No processes = int(cn.count/4) uu.print_log('Gain year count gain merge all combos max processors=', processes) pool = multiprocessing.Pool(processes) - pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_merge, pattern=pattern, sensit_type=sensit_type), tile_id_list) + pool.map(partial(gain_year_count_all_forest_types.create_gain_year_count_merge, + pattern=pattern, sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # gain_year_count_all_forest_types.create_gain_year_count_loss_only(tile_id) + # gain_year_count_all_forest_types.create_gain_year_count_loss_only(tile_id, no_upload) # # for tile_id in tile_id_list: # if sensit_type == 'maxgain': - # gain_year_count_all_forest_types.create_gain_year_count_gain_only_maxgain(tile_id) + # gain_year_count_all_forest_types.create_gain_year_count_gain_only_maxgain(tile_id, no_upload) # else: - # gain_year_count_all_forest_types.create_gain_year_count_gain_only_standard(tile_id) + # gain_year_count_all_forest_types.create_gain_year_count_gain_only_standard(tile_id, no_upload) # # for tile_id in tile_id_list: - # gain_year_count_all_forest_types.create_gain_year_count_no_change_standard(tile_id) + # gain_year_count_all_forest_types.create_gain_year_count_no_change_standard(tile_id, no_upload) # # for tile_id in tile_id_list: # if sensit_type == 'maxgain': - # gain_year_count_all_forest_types.create_gain_year_count_loss_and_gain_maxgain(tile_id) + # gain_year_count_all_forest_types.create_gain_year_count_loss_and_gain_maxgain(tile_id, no_upload) # else: - # gain_year_count_all_forest_types.create_gain_year_count_loss_and_gain_standard(tile_id) + # gain_year_count_all_forest_types.create_gain_year_count_loss_and_gain_standard(tile_id, no_upload) # # for tile_id in tile_id_list: - # gain_year_count_all_forest_types.create_gain_year_count_merge(tile_id, pattern, sensit_type) + # gain_year_count_all_forest_types.create_gain_year_count_merge(tile_id, pattern, sensit_type, no_upload) - # Intermediate output tiles for checking outputs - uu.upload_final_set(output_dir_list[0], "growth_years_loss_only") - uu.upload_final_set(output_dir_list[0], "growth_years_gain_only") - uu.upload_final_set(output_dir_list[0], "growth_years_no_change") - uu.upload_final_set(output_dir_list[0], "growth_years_loss_and_gain") + # If no_upload flag is not activated, output is uploaded + if not no_upload: - # This is the final output used later in the model - uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) + # Intermediate output tiles for checking outputs + uu.upload_final_set(output_dir_list[0], "growth_years_loss_only") + uu.upload_final_set(output_dir_list[0], "growth_years_gain_only") + uu.upload_final_set(output_dir_list[0], "growth_years_no_change") + uu.upload_final_set(output_dir_list[0], "growth_years_loss_and_gain") + + # This is the final output used later in the model + uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) if __name__ == '__main__': @@ -184,16 +195,19 @@ def mp_gain_year_count_all_forest_types(sensit_type, tile_id_list, run_date = No help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_gain_year_count_all_forest_types(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date) \ No newline at end of file + mp_gain_year_count_all_forest_types(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/gain/mp_gross_removals_all_forest_types.py b/gain/mp_gross_removals_all_forest_types.py index 3b1c11fa..f810e10d 100644 --- a/gain/mp_gross_removals_all_forest_types.py +++ b/gain/mp_gross_removals_all_forest_types.py @@ -19,7 +19,7 @@ sys.path.append(os.path.join(cn.docker_app,'gain')) import gross_removals_all_forest_types -def mp_gross_removals_all_forest_types(sensit_type, tile_id_list, run_date = None): +def mp_gross_removals_all_forest_types(sensit_type, tile_id_list, run_date = None, no_upload = True): os.chdir(cn.docker_base_dir) @@ -79,13 +79,13 @@ def mp_gross_removals_all_forest_types(sensit_type, tile_id_list, run_date = Non uu.print_log('Gross removals max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(partial(gross_removals_all_forest_types.gross_removals_all_forest_types, output_pattern_list=output_pattern_list, - sensit_type=sensit_type), tile_id_list) + sensit_type=sensit_type, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: - # gross_removals_all_forest_types.gross_removals_all_forest_types(tile_id, output_pattern_list, sensit_type) + # gross_removals_all_forest_types.gross_removals_all_forest_types(tile_id, output_pattern_list, sensit_type, no_upload) # Checks the gross removals outputs for tiles with no data for output_pattern in output_pattern_list: @@ -106,10 +106,11 @@ def mp_gross_removals_all_forest_types(sensit_type, tile_id_list, run_date = Non pool.close() pool.join() + # If no_upload flag is not activated, output is uploaded + if not no_upload: - # Uploads output tiles to s3 - for i in range(0, len(output_dir_list)): - uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) + for i in range(0, len(output_dir_list)): + uu.upload_final_set(output_dir_list[i], output_pattern_list[i]) if __name__ == '__main__': @@ -124,16 +125,19 @@ def mp_gross_removals_all_forest_types(sensit_type, tile_id_list, run_date = Non help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') args = parser.parse_args() sensit_type = args.model_type tile_id_list = args.tile_id_list run_date = args.run_date + no_upload = args.no_upload # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date) + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload) # Checks whether the sensitivity analysis and tile_id_list arguments are valid uu.check_sensit_type(sensit_type) tile_id_list = uu.tile_id_list_check(tile_id_list) - mp_gross_removals_all_forest_types(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date) \ No newline at end of file + mp_gross_removals_all_forest_types(sensit_type=sensit_type, tile_id_list=tile_id_list, run_date=run_date, no_upload=no_upload) \ No newline at end of file diff --git a/readme.md b/readme.md index 8b8d3ff6..70105c04 100644 --- a/readme.md +++ b/readme.md @@ -16,7 +16,7 @@ use transitions). The model is described and published in Harris et al. (2021) N ### Inputs Well over twenty inputs are needed to run this model. Most are spatial, but some are tabular. All spatial data are converted to 10 x 10 degree raster tiles at 0.00025 x 0.00025 degree resolution -(approximately 30 x 30 m at the equator). The tabular data are generally annual biomass removal (i.e. +(approximately 30 x 30 m at the equator) before inclusion in the model. The tabular data are generally annual biomass removal (i.e. sequestration) factors (e.g., mangroves, planted forests, natural forests), which are then applied to spatial data. Spatial data include annual tree cover loss, biomass densities in 2000, drivers of tree cover loss, ecozones, tree cover extent in 2000, elevation, etc. Different inputs are needed for different @@ -31,17 +31,22 @@ These are produced at two resolutions: 0.00025 x 0.00025 degrees (approximately 30 x 30 m at the equator) in 10 x 10 degree rasters (to make outputs a manageable size), and 0.04 x 0.04 degrees (approximately 4 x 4 km at the equator) as global rasters. -Model runs also automatically generate a txt log that is saved to s3. This log includes nearly everything that is output in the console. +Model runs also automatically generate a txt log. This log includes nearly everything that is output in the console. This log is useful for documenting model runs and checking for mistakes/errors in retrospect, although it does not capture errors that terminate the model. For example, users can examine it to see if the correct input tiles were downloaded or if the intended tiles were used during the model run. +Output rasters and model logs are uploaded to s3 unless the `--no-upload` flag (`-nu`) is activated as a command line argument. +When this is activated for any model stage, neither raster outputs nor logs are uploaded to s3. This is good for local test runs or versions +of the model that are independent of s3 (that is, inputs are stored locally and no on s3, and the user does not have +a connection to s3 storage or s3 credentials). + #### 30-m outputs The 30-m outputs are used for zonal statistics analyses (i.e. emissions, removals, or net in polygons of interest) and mapping on the Global Forest Watch web platform or at small scales (where 30-m pixels can be distinguished). Individual emissions can be assigned years based on Hansen loss during further analyses but removals and net flux are cumulative over the entire model run and cannot be assigned specific years. -This 30-m output is in megagrams CO2e/ha 2001-2020 (i.e. densities) and includes all tree cover densities ("full extent"): +This 30-m output is in megagrams (Mg) CO2e/ha 2001-2020 (i.e. densities) and includes all tree cover densities ("full extent"): `(((TCD2000>0 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations)`. However, the model is designed to be used specifically for forests, so the model creates three derivative 30-m outputs for each key output (gross emissions, gross removals, net flux) as well @@ -55,7 +60,7 @@ outputs for each key output (gross emissions, gross removals, net flux) as well `(((TCD2000>30 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations)` The per hectare outputs are used for making pixel-level maps (essentially showing emission and removal factors), -while the per pixel outputs are used for analyses because the values +while the per pixel outputs are used for getting total values within areas because the values of those pixels can be summed within areas of interest. (The pixels of the per hectare outputs should not be summed but they can be averaged in areas of interest.) Statistics from this model are always based on the "forest extent" rasters, not the "full extent" rasters. @@ -70,7 +75,7 @@ carbon pool densities in the year of tree cover loss, and the number of years in Almost all model output have metadata associated with them, viewable using the `gdalinfo` command line utility (https://gdal.org/programs/gdalinfo.html). Metadata includes units, date created, model version, geographic extent, and more. Unfortunately, the metadata are not viewable in ArcMap -or in the versions of these files downloadable from the Global Forest Watch Open Data Portal. +or in the versions of these files downloadable from the Global Forest Watch Open Data Portal (https://data.globalforestwatch.org/). #### 4-km outputs @@ -88,11 +93,13 @@ There are two ways to run the model: as a series of individual scripts, or from Which one to use depends on what you are trying to do. Generally, the individual scripts (which correspond to model stages) are more appropriate for development and testing, while the master script is better for running the main part of the model from start to finish in one go. In either case, the code must be cloned from this repository. -Run at scale, both options iterate through a list of ~275 10x10 degree tiles. (Different model stages have different numbers of tiles.) +Run globally, both options iterate through a list of ~275 10x10 degree tiles. (Different model stages have different numbers of tiles.) Run all tiles in the model extent fully through one model stage before starting on the next stage. (The master script does this automatically.) If a user wants to run the model on just one or a few tiles, that can be done through a command line argument (`--tile-id-list` or `-l`). -If individual tiles are listed, only those will be downloaded and run. This is a natural system for testing. +If individual tiles are listed, only those will be downloaded and run. This is a natural system for testing or for +running the model for individual countries. +For example, to run the model for Madagascar, only tiles 10S_040E, 10S_050E, and 20S_040E need to be run. The model runs inside a Docker container. Once you have Docker configured on your system, have cloned this repository, and have configured access to Amazon Web Services, you can do the following on the command line in the same folder as the repository on your system. @@ -100,7 +107,9 @@ This will enter the command line in the Docker container. For runs on my local computer, I use `docker-compose` so that the Docker is mapped to my computer's drives. I do this for development and testing. If running on another computer, you will need to change the local -folder being mapped in `docker-compose.yaml` to fit your computer's directory structure. +folder being mapped in `docker-compose.yaml` to fit your computer's directory structure. +You will also need to either provide your own AWS secret key and access key or change the docker-compose code +to not use them. `docker-compose build` `docker-compose run --rm -e AWS_SECRET_ACCESS_KEY=... -e AWS_ACCESS_KEY_ID=... carbon-budget` @@ -113,8 +122,9 @@ Depending on what exactly the user is running, the user may have to change lots Unfortunately, I can't really give better guidance than that; it really depends on what part of the model is being run and how. (I want to make the situations under which users change folder dates more consistent eventually.) -The model can be run either using multiple processors or one processor. The former is for full model runs, -while the latter is for model development. The user can switch between these two versions by commenting out +The model can be run either using multiple processors or one processor. The former is for large scale model runs, +while the latter is for model development or running on small-ish countries that use only a few tiles. +The user can switch between these two versions by commenting out the appropriate code chunks in each script. The single-processor option is commented out by default. One important thing to note is that if a user tries to use too many processors, the system will run out of memory and can crash (particularly on AWS EC2 instances). Thus, it is important not to use too many processors at once. @@ -139,7 +149,7 @@ then initiate the actual work done on each tile in the script without the `mp_` The order in which the individual stages must be run is very specific; many scripts depend on the outputs of other scripts. Looking at the files that must be downloaded for the script to run will show what files must already be created and therefore what scripts must have already been -run. Alternatively, you can look at `run_full_model.py` to see the order in which model stages are run. +run. Alternatively, you can look at the top of `run_full_model.py` to see the order in which model stages are run. The date component of the output directory on s3 generally must be changed in `constants_and_names.py` for each output file unless a date argument is provided on the command line. @@ -150,27 +160,30 @@ the appropriate code chunks. #### Master script The master script runs through all of the non-preparatory scripts in the model: some removal factor creation, gross removals, carbon pool generation, gross emissions, net flux, and aggregation. It includes all the arguments needed to run -every script. The user can control what model components are run to some extent and set the date part of +every script. Thus, the table below also explains the potential arguments for the individual model stages. +The user can control what model components are run to some extent and set the date part of the output directories. The emissions C++ code has to be be compiled before running the master script (see below). Preparatory scripts like creating soil carbon tiles or mangrove tiles are not included in the master script because they are run very infrequently. `python run_full_model.py -t std -s all -r true -d 20200822 -l all -ce loss -p biomass_soil -tcd 30 -ma true -us true -ln "This will run the entire standard model, including creating mangrove and US removal factor tiles, on all tiles and output everything in s3 folders with the date 20200822."` -| Argument | Required/Optional | Description | -| -------- | ----------- | ------ | -| `model-type` | Required | Standard model (`std`) or a sensitivity analysis. Refer to `constants_and_names.py` for valid list of sensitivity analyses. | -| `stages` | Required | The model stage at which the model should start. `all` will run the following stages in this order: model_extent, forest_age_category_IPCC, annual_removals_IPCC, annual_removals_all_forest_types, gain_year_count, gross_removals_all_forest_types, carbon_pools, gross_emissions, net_flux, aggregate, create_supplementary_outputs | -| `run-through` | Required | Options: true or false. true: run stage provided in `stages` argument and all following stages. false: run only stage in `stages` argument. | -| `run-date` | Required | Date of run. Must be format YYYYMMDD. This sets the output folder in s3. | -| `tile-id-list` | Required | List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all | -| `carbon-pool-extent` | Optional | Extent over which carbon pools should be calculated: loss or 2000 or loss,2000 or 2000,loss | -| `pools-to-use` | Optional | Options are soil_only or biomass_soil. Former only considers emissions from soil. Latter considers emissions from biomass and soil. | -| `tcd-threshold` | Optional | Tree cover density threshold above which pixels will be included in the aggregation. | -| `std-net-flux-aggreg` | Optional | The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map. | -| `mangroves` | Optional | Create mangrove removal factor tiles as the first stage. true or false | -| `us-rates` | Optional | Create US-specific removal factor tiles as the first stage (or second stage, if mangroves are enabled). true or false | -| `log-note` | Optional | Adds text to the beginning of the log | +| Argument | Short argument | Required/Optional | Relevant stage | Description | +| -------- | ----- | ----------- | ------- | ------ | +| `model-type` | `-t` | Required | All | Standard model (`std`) or a sensitivity analysis. Refer to `constants_and_names.py` for valid list of sensitivity analyses. | +| `stages` | `-s` | Required | All | The model stage at which the model should start. `all` will run the following stages in this order: model_extent, forest_age_category_IPCC, annual_removals_IPCC, annual_removals_all_forest_types, gain_year_count, gross_removals_all_forest_types, carbon_pools, gross_emissions, net_flux, aggregate, create_supplementary_outputs | +| `run-through` | `-r` | Optional | All | If activated, run stage provided in `stages` argument and all following stages. Otherwise, run only stage in `stages` argument. Activated with flag. | +| `run-date` | `-d` | Required | All | Date of run. Must be format YYYYMMDD. This sets the output folder in s3. | +| `tile-id-list` | `-l` | Required | All | List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all | +| `no-upload` | `-nu` | Optional | All | No files are uploaded to s3 during or after model run (including logs and model outputs). Use for testing or completely local runs. | +| `carbon-pool-extent` | `-ce` | Optional | Carbon pool creation | Extent over which carbon pools should be calculated: loss or 2000 or loss,2000 or 2000,loss | +| `pools-to-use` | `-p` | Optional | Emissions| Options are soil_only or biomass_soil. Former only considers emissions from soil. Latter considers emissions from biomass and soil. | +| `tcd-threshold` | `-tcd`| Optional | Aggregation | Tree cover density threshold above which pixels will be included in the aggregation. | +| `std-net-flux-aggreg` | `-std` | Optional | Aggregation | The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map. | +| `mangroves` | `-ma` | Optional | `run_full_model.py` | Create mangrove removal factor tiles as the first stage. Activate with flag. | +| `us-rates` | `-us` | Optional | `run_full_model.py` | Create US-specific removal factor tiles as the first stage (or second stage, if mangroves are enabled). Activate with flag. | +| `save-intermdiates` | `-si`| Optional | `run_full_model.py` | Intermediate outputs are not deleted within `run_full_model.py`. Use for local model runs. | +| `log-note` | `-ln`| Optional | All | Adds text to the beginning of the log | ##### Running the emissions model The gross emissions script is the only part of the model that uses C++. Thus, it must be manually compiled before running. @@ -180,9 +193,13 @@ The command for compiling the C++ script is (subbing in the actual file name): `c++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_[VERSION].cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_[VERSION].exe -lgdal` +For the standard model and the sensitivity analyses that don't specifically affect emissions, it is: + +`c++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.exe -lgdal` + ### Sensitivity analysis Several variations of the model are included; these are the sensitivity variants, as they use different inputs or parameters. -They can be run by changing the `model-type` argument from `std` to an option found in `constants_and_names.py`. +They can be run by changing the `--model-type` (`-t`) argument from `std` to an option found in `constants_and_names.py`. Each sensitivity analysis variant starts at a different stage in the model and runs to the final stage, except that sensitivity analyses do not include the creation of the supplementary outputs (per pixel tiles, forest extent tiles). Some use all tiles and some use a smaller extent. @@ -262,8 +279,8 @@ At the least, it needs a certain type of memory configuration on the EC2 instanc Otherwise, I do not know the limitations and constraints on running this model. ### Contact information -David Gibbs +David Gibbs: david.gibbs@wri.org -Global Forest Watch, World Resources Institute, Washington, D.C. +Nancy Harris: nancy.harris@wri.org -david.gibbs@wri.org +Global Forest Watch, World Resources Institute, Washington, D.C. diff --git a/run_full_model.py b/run_full_model.py index e9a5ae3f..1f992e26 100644 --- a/run_full_model.py +++ b/run_full_model.py @@ -48,8 +48,8 @@ def main (): parser.add_argument('--model-type', '-t', required=True, help='{}'.format(cn.model_type_arg_help)) parser.add_argument('--stages', '-s', required=True, help='Stages for running the flux model. Options are {}'.format(model_stages)) - parser.add_argument('--run-through', '-r', required=True, - help='Options: true or false. true: run named stage and following stages. false: run only named stage.') + parser.add_argument('--run-through', '-r', action='store_true', + help='If activated, run named stage and all following stages. If not activated, run the selected stage only.') parser.add_argument('--run-date', '-d', required=False, help='Date of run. Must be format YYYYMMDD.') parser.add_argument('--tile-id-list', '-l', required=True, @@ -62,10 +62,14 @@ def main (): help='Tree cover density threshold above which pixels will be included in the aggregation.') parser.add_argument('--std-net-flux-aggreg', '-sagg', required=False, help='The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map') - parser.add_argument('--mangroves', '-ma', required=False, - help='Include mangrove removal rate and standard deviation tile creation step (before model extent). true or false.') - parser.add_argument('--us-rates', '-us', required=False, - help='Include US removal rate and standard deviation tile creation step (before model extent). true or false.') + parser.add_argument('--mangroves', '-ma', action='store_true', + help='Include mangrove removal rate and standard deviation tile creation step (before model extent).') + parser.add_argument('--us-rates', '-us', action='store_true', + help='Include US removal rate and standard deviation tile creation step (before model extent).') + parser.add_argument('--no-upload', '-nu', action='store_true', + help='Disables uploading of outputs to s3') + parser.add_argument('--save-intermediates', '-si', action='store_true', + help='Saves intermediate model outputs rather than deleting them to save storage') parser.add_argument('--log-note', '-ln', required=False, help='Note to include in log header about model run.') args = parser.parse_args() @@ -83,24 +87,24 @@ def main (): std_net_flux = args.std_net_flux_aggreg include_mangroves = args.mangroves include_us = args.us_rates + no_upload = args.no_upload + save_intermediates = args.save_intermediates log_note = args.log_note # Start time for script script_start = datetime.datetime.now() # Create the output log - uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, stage_input=stage_input, run_through=run_through, - carbon_pool_extent=carbon_pool_extent, emitted_pools=emitted_pools, thresh=thresh, std_net_flux=std_net_flux, + uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date, no_upload=no_upload, + save_intermediates=save_intermediates, + stage_input=stage_input, run_through=run_through, carbon_pool_extent=carbon_pool_extent, + emitted_pools=emitted_pools, thresh=thresh, std_net_flux=std_net_flux, include_mangroves=include_mangroves, include_us=include_us, log_note=log_note) # Checks the validity of the model stage arguments. If either one is invalid, the script ends. if (stage_input not in model_stages): - uu.exception_log('Invalid stage selection. Please provide a stage from', model_stages) - else: - pass - if (run_through not in ['true', 'false']): - uu.exception_log('Invalid run through option. Please enter true or false.') + uu.exception_log(no_upload, 'Invalid stage selection. Please provide a stage from', model_stages) else: pass @@ -118,7 +122,7 @@ def main (): # Checks if the carbon pool type is specified if the stages to run includes carbon pool generation. # Does this up front so the user knows before the run begins that information is missing. if ('carbon_pools' in actual_stages) & (carbon_pool_extent not in ['loss', '2000', 'loss,2000', '2000,loss']): - uu.exception_log("Invalid carbon_pool_extent input. Please choose loss, 2000, loss,2000 or 2000,loss.") + uu.exception_log(no_upload, "Invalid carbon_pool_extent input. Please choose loss, 2000, loss,2000 or 2000,loss.") # Checks if the correct c++ script has been compiled for the pool option selected. # Does this up front so that the user is prompted to compile the C++ before the script starts running, if necessary. @@ -131,26 +135,26 @@ def main (): if os.path.exists('{0}/calc_gross_emissions_{1}.exe'.format(cn.c_emis_compile_dst, sensit_type)): uu.print_log("C++ for {} already compiled.".format(sensit_type)) else: - uu.exception_log('Must compile standard {} model C++...'.format(sensit_type)) + uu.exception_log(no_upload, 'Must compile standard {} model C++...'.format(sensit_type)) else: if os.path.exists('{0}/calc_gross_emissions_generic.exe'.format(cn.c_emis_compile_dst)): uu.print_log("C++ for generic emissions already compiled.") else: - uu.exception_log('Must compile generic emissions C++...') + uu.exception_log(no_upload, 'Must compile generic emissions C++...') elif (emitted_pools == 'soil_only') & (sensit_type == 'std'): if os.path.exists('{0}/calc_gross_emissions_soil_only.exe'.format(cn.c_emis_compile_dst)): uu.print_log("C++ for generic emissions already compiled.") else: - uu.exception_log('Must compile soil_only C++...') + uu.exception_log(no_upload, 'Must compile soil_only C++...') else: - uu.exception_log('Pool and/or sensitivity analysis option not valid for gross emissions') + uu.exception_log(no_upload, 'Pool and/or sensitivity analysis option not valid for gross emissions') # Checks whether the canopy cover argument is valid up front. if 'aggregate' in actual_stages: if thresh < 0 or thresh > 99: - uu.exception_log('Invalid tcd. Please provide an integer between 0 and 99.') + uu.exception_log(no_upload, 'Invalid tcd. Please provide an integer between 0 and 99.') else: pass @@ -278,7 +282,7 @@ def main (): uu.print_log(":::::Creating tiles of model extent") start = datetime.datetime.now() - mp_model_extent(sensit_type, tile_id_list, run_date = run_date) + mp_model_extent(sensit_type, tile_id_list, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -292,7 +296,7 @@ def main (): uu.print_log(":::::Creating tiles of forest age categories for IPCC removal rates") start = datetime.datetime.now() - mp_forest_age_category_IPCC(sensit_type, tile_id_list, run_date = run_date) + mp_forest_age_category_IPCC(sensit_type, tile_id_list, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -306,7 +310,7 @@ def main (): uu.print_log(":::::Creating tiles of annual aboveground and belowground removal rates using IPCC defaults") start = datetime.datetime.now() - mp_annual_gain_rate_IPCC_defaults(sensit_type, tile_id_list, run_date = run_date) + mp_annual_gain_rate_IPCC_defaults(sensit_type, tile_id_list, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -319,7 +323,7 @@ def main (): uu.print_log(":::::Creating tiles of annual aboveground and belowground removal rates for all forest types") start = datetime.datetime.now() - mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type, tile_id_list, run_date=run_date) + mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type, tile_id_list, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -330,35 +334,38 @@ def main (): # Creates tiles of the number of years of removals for all model pixels (across all forest types) if 'gain_year_count' in actual_stages: - uu.print_log(":::::Freeing up memory for gain year count creation by deleting unneeded tiles") - tiles_to_delete = [] - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_mangrove_biomass_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_WHRC_biomass_2000_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_mangrove))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_mangrove))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_natrl_forest_US))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_natrl_forest_young))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_age_cat_IPCC))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_IPCC_defaults))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_IPCC_defaults))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_all_types))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_ifl_primary))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_planted_forest_type_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGB_mangrove))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_planted_forest_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_natrl_forest_young))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGB_IPCC_defaults))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_all_types))) - uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + if not save_intermediates: + + uu.print_log(":::::Freeing up memory for gain year count creation by deleting unneeded tiles") + tiles_to_delete = [] + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_mangrove_biomass_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_WHRC_biomass_2000_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_mangrove))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_mangrove))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_natrl_forest_US))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_natrl_forest_young))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_age_cat_IPCC))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_IPCC_defaults))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_IPCC_defaults))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_all_types))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_ifl_primary))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_planted_forest_type_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGB_mangrove))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_planted_forest_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_natrl_forest_young))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGB_IPCC_defaults))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_all_types))) + uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + + for tile_to_delete in tiles_to_delete: + os.remove(tile_to_delete) + uu.print_log(":::::Deleted unneeded tiles") - for tile_to_delete in tiles_to_delete: - os.remove(tile_to_delete) - uu.print_log(":::::Deleted unneeded tiles") uu.check_storage() uu.print_log(":::::Creating tiles of gain year count for all removal pixels") @@ -378,7 +385,7 @@ def main (): uu.print_log(":::::Creating gross removals for all forest types combined (above + belowground) tiles'") start = datetime.datetime.now() - mp_gross_removals_all_forest_types(sensit_type, tile_id_list, run_date = run_date) + mp_gross_removals_all_forest_types(sensit_type, tile_id_list, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -389,45 +396,48 @@ def main (): # Creates carbon emitted_pools in loss year if 'carbon_pools' in actual_stages: - uu.print_log(":::::Freeing up memory for carbon pool creation by deleting unneeded tiles") - tiles_to_delete = [] - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_model_extent))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_mangrove))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_mangrove))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_natrl_forest_US))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_natrl_forest_young))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_age_cat_IPCC))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_IPCC_defaults))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_IPCC_defaults))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGC_all_types))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_all_types))) - tiles_to_delete.extend(glob.glob('*growth_years*tif')) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gain_year_count))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_cumul_gain_BGCO2_all_types))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_cumul_gain_AGCO2_BGCO2_all_types))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_ifl_primary))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_planted_forest_type_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGB_mangrove))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_planted_forest_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_natrl_forest_young))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGB_IPCC_defaults))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_all_types))) - uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + if not save_intermediates: + + uu.print_log(":::::Freeing up memory for carbon pool creation by deleting unneeded tiles") + tiles_to_delete = [] + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_model_extent))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_mangrove))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_mangrove))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_natrl_forest_US))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_natrl_forest_young))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_age_cat_IPCC))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_IPCC_defaults))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_IPCC_defaults))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGC_all_types))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_BGC_all_types))) + tiles_to_delete.extend(glob.glob('*growth_years*tif')) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gain_year_count))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_cumul_gain_BGCO2_all_types))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_cumul_gain_AGCO2_BGCO2_all_types))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_ifl_primary))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_planted_forest_type_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGB_mangrove))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_planted_forest_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_natrl_forest_young))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGB_IPCC_defaults))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_stdev_annual_gain_AGC_all_types))) + uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + + for tile_to_delete in tiles_to_delete: + os.remove(tile_to_delete) + uu.print_log(":::::Deleted unneeded tiles") - for tile_to_delete in tiles_to_delete: - os.remove(tile_to_delete) - uu.print_log(":::::Deleted unneeded tiles") uu.check_storage() uu.print_log(":::::Creating carbon pool tiles") start = datetime.datetime.now() - mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_date = run_date) + mp_create_carbon_pools(sensit_type, tile_id_list, carbon_pool_extent, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -438,35 +448,38 @@ def main (): # Creates gross emissions tiles by driver, gas, and all emissions combined if 'gross_emissions' in actual_stages: - uu.print_log(":::::Freeing up memory for gross emissions creation by deleting unneeded tiles") - tiles_to_delete = [] - # tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_removal_forest_type))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_AGC_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_BGC_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_deadwood_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_litter_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_total_C_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_elevation))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_precip))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_all_types))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_cumul_gain_AGCO2_all_types))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_cont_eco_processed))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_WHRC_biomass_2000_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_mangrove_biomass_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_removal_forest_type))) - uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") - - uu.print_log(tiles_to_delete) + if not save_intermediates: + + uu.print_log(":::::Freeing up memory for gross emissions creation by deleting unneeded tiles") + tiles_to_delete = [] + # tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_removal_forest_type))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_AGC_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_BGC_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_deadwood_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_litter_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_total_C_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_elevation))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_precip))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_all_types))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_cumul_gain_AGCO2_all_types))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_cont_eco_processed))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_WHRC_biomass_2000_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_mangrove_biomass_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_removal_forest_type))) + uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + + uu.print_log(tiles_to_delete) + + for tile_to_delete in tiles_to_delete: + os.remove(tile_to_delete) + uu.print_log(":::::Deleted unneeded tiles") - for tile_to_delete in tiles_to_delete: - os.remove(tile_to_delete) - uu.print_log(":::::Deleted unneeded tiles") uu.check_storage() uu.print_log(":::::Creating gross emissions tiles") start = datetime.datetime.now() - mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_date = run_date) + mp_calculate_gross_emissions(sensit_type, tile_id_list, emitted_pools, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -477,42 +490,45 @@ def main (): # Creates net flux tiles (gross emissions - gross removals) if 'net_flux' in actual_stages: - uu.print_log(":::::Freeing up memory for net flux creation by deleting unneeded tiles") - tiles_to_delete = [] - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_non_co2_all_drivers_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_co2_only_all_drivers_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_commod_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_shifting_ag_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_forestry_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_wildfire_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_urban_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_no_driver_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_nodes_biomass_soil))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_AGC_emis_year))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_BGC_emis_year))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_deadwood_emis_year_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_litter_emis_year_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_soil_C_emis_year_2000))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_total_C_emis_year))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_peat_mask))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_ifl_primary))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_planted_forest_type_unmasked))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_drivers))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_climate_zone))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_bor_tem_trop_processed))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_burn_year))) - tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000))) - uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + if not save_intermediates: + + uu.print_log(":::::Freeing up memory for net flux creation by deleting unneeded tiles") + tiles_to_delete = [] + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_non_co2_all_drivers_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_co2_only_all_drivers_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_commod_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_shifting_ag_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_forestry_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_wildfire_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_urban_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_no_driver_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_gross_emis_nodes_biomass_soil))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_AGC_emis_year))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_BGC_emis_year))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_deadwood_emis_year_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_litter_emis_year_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_soil_C_emis_year_2000))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_total_C_emis_year))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_peat_mask))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_ifl_primary))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_planted_forest_type_unmasked))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_drivers))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_climate_zone))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_bor_tem_trop_processed))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_burn_year))) + tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000))) + uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + + for tile_to_delete in tiles_to_delete: + os.remove(tile_to_delete) + uu.print_log(":::::Deleted unneeded tiles") - for tile_to_delete in tiles_to_delete: - os.remove(tile_to_delete) - uu.print_log(":::::Deleted unneeded tiles") uu.check_storage() uu.print_log(":::::Creating net flux tiles") start = datetime.datetime.now() - mp_net_flux(sensit_type, tile_id_list, run_date = run_date) + mp_net_flux(sensit_type, tile_id_list, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -524,10 +540,24 @@ def main (): # For sensitivity analyses, creates percent difference and sign change maps compared to standard model net flux. if 'aggregate' in actual_stages: + # aux.xml files need to be deleted because otherwise they'll be included in the aggregation iteration. + # They are created by using check_and_delete_if_empty_light() + uu.print_log(":::::Deleting any aux.xml files") + tiles_to_delete = [] + tiles_to_delete.extend(glob.glob('*aux.xml')) + + uu.print_log(" Deleting", len(tiles_to_delete), "aux.xml files:", tiles_to_delete) + + for tile_to_delete in tiles_to_delete: + os.remove(tile_to_delete) + uu.print_log(":::::Deleted unneeded tiles") + + uu.print_log(":::::Creating 4x4 km aggregate maps") start = datetime.datetime.now() - mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux = std_net_flux, run_date = run_date) + mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux=std_net_flux, + run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -538,20 +568,23 @@ def main (): # Converts gross emissions, gross removals and net flux from per hectare rasters to per pixel rasters if 'create_supplementary_outputs' in actual_stages: - uu.print_log(":::::Deleting rewindowed tiles") - tiles_to_delete = [] - tiles_to_delete.extend(glob.glob('*rewindow*tif')) - uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + if not save_intermediates: + + uu.print_log(":::::Deleting rewindowed tiles") + tiles_to_delete = [] + tiles_to_delete.extend(glob.glob('*rewindow*tif')) + uu.print_log(" Deleting", len(tiles_to_delete), "tiles...") + + for tile_to_delete in tiles_to_delete: + os.remove(tile_to_delete) + uu.print_log(":::::Deleted unneeded tiles") - for tile_to_delete in tiles_to_delete: - os.remove(tile_to_delete) - uu.print_log(":::::Deleted unneeded tiles") uu.check_storage() uu.print_log(":::::Creating supplementary versions of main model outputs (forest extent, per pixel)") start = datetime.datetime.now() - mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = run_date) + mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date=run_date, no_upload=no_upload) end = datetime.datetime.now() elapsed_time = end - start @@ -559,30 +592,37 @@ def main (): uu.print_log(":::::Processing time for supplementary output raster creation:", elapsed_time, "\n", "\n") - uu.print_log(":::::Counting tiles output to each folder") + # If no_upload flag is activated, tiles on s3 aren't counted + if not no_upload: + + uu.print_log(":::::Counting tiles output to each folder") - # Modifies output directory names to make them match those used during the model run. - # The tiles in each of these directories and counted and logged. - # If the model run isn't the standard one, the output directory and file names are changed - if sensit_type != 'std': - uu.print_log("Modifying output directory and file name pattern based on sensitivity analysis") - output_dir_list = uu.alter_dirs(sensit_type, output_dir_list) + # Modifies output directory names to make them match those used during the model run. + # The tiles in each of these directories and counted and logged. + # If the model run isn't the standard one, the output directory and file names are changed + if sensit_type != 'std': + uu.print_log("Modifying output directory and file name pattern based on sensitivity analysis") + output_dir_list = uu.alter_dirs(sensit_type, output_dir_list) - # Changes the date in the output directories. This date was used during the model run. - # This replaces the date in constants_and_names. - if run_date: - output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) + # Changes the date in the output directories. This date was used during the model run. + # This replaces the date in constants_and_names. + if run_date: + output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) - for output in output_dir_list: + for output in output_dir_list: - tile_count = uu.count_tiles_s3(output) - uu.print_log("Total tiles in", output, ": ", tile_count) + tile_count = uu.count_tiles_s3(output) + uu.print_log("Total tiles in", output, ": ", tile_count) script_end = datetime.datetime.now() script_elapsed_time = script_end - script_start uu.print_log(":::::Processing time for entire run:", script_elapsed_time, "\n") - uu.upload_log() + + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + uu.upload_log() if __name__ == '__main__': main() \ No newline at end of file diff --git a/sensitivity_analysis/mp_Mekong_loss.py b/sensitivity_analysis/mp_Mekong_loss.py index 84441529..04a3bf19 100644 --- a/sensitivity_analysis/mp_Mekong_loss.py +++ b/sensitivity_analysis/mp_Mekong_loss.py @@ -15,6 +15,8 @@ def main (): + no_upload = False + # Create the output log uu.initiate_log() @@ -58,7 +60,7 @@ def main (): source_raster = loss_composite out_pattern = cn.pattern_Mekong_loss_processed dt = 'Byte' - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # This is necessary for changing NoData values to 0s (so they are recognized as 0s) pool.map(Mekong_loss.recode_tiles, tile_id_list) diff --git a/sensitivity_analysis/mp_Saatchi_biomass_prep.py b/sensitivity_analysis/mp_Saatchi_biomass_prep.py index 0fcfa3fb..fe7b49ae 100644 --- a/sensitivity_analysis/mp_Saatchi_biomass_prep.py +++ b/sensitivity_analysis/mp_Saatchi_biomass_prep.py @@ -15,6 +15,8 @@ def main (): + no_upload = False + # Create the output log uu.initiate_log() @@ -38,7 +40,7 @@ def main (): out_pattern = cn.pattern_JPL_unmasked_processed dt = 'Float32' pool = multiprocessing.Pool(cn.count-5) # count-5 peaks at 320GB of memory - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # Checks if each tile has data in it. Only tiles with data are uploaded. upload_dir = cn.JPL_processed_dir diff --git a/sensitivity_analysis/mp_US_removal_rates.py b/sensitivity_analysis/mp_US_removal_rates.py index 965f7491..613e18cd 100644 --- a/sensitivity_analysis/mp_US_removal_rates.py +++ b/sensitivity_analysis/mp_US_removal_rates.py @@ -40,6 +40,8 @@ def main (): + no_upload=False + # Create the output log uu.initiate_log() @@ -113,7 +115,8 @@ def main (): out_pattern = cn.pattern_US_forest_age_cat_processed dt = 'Int16' pool = multiprocessing.Pool(int(cn.count/2)) - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), US_tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, + no_upload=no_upload), US_tile_id_list) uu.upload_final_set(cn.US_forest_age_cat_processed_dir, cn.pattern_US_forest_age_cat_processed) @@ -135,7 +138,8 @@ def main (): out_pattern = cn.pattern_FIA_forest_group_processed dt = 'Byte' pool = multiprocessing.Pool(int(cn.count/2)) - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), US_tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, + no_upload=no_upload), US_tile_id_list) uu.upload_final_set(cn.FIA_forest_group_processed_dir, cn.pattern_FIA_forest_group_processed) diff --git a/sensitivity_analysis/mp_legal_AMZ_loss.py b/sensitivity_analysis/mp_legal_AMZ_loss.py index 6ae3ca34..86037ee5 100644 --- a/sensitivity_analysis/mp_legal_AMZ_loss.py +++ b/sensitivity_analysis/mp_legal_AMZ_loss.py @@ -21,6 +21,10 @@ def main (): + no_upload = False + + sensit_type = "legal_Amazon_loss" + # Create the output log uu.initiate_log() @@ -42,15 +46,15 @@ def main (): # Checks the validity of the two arguments. If either one is invalid, the script ends. if (stage_input not in Brazil_stages): - uu.exception_log('Invalid stage selection. Please provide a stage from', Brazil_stages) + uu.exception_log(no_upload, 'Invalid stage selection. Please provide a stage from', Brazil_stages) else: pass if (run_through not in ['true', 'false']): - uu.exception_log('Invalid run through option. Please enter true or false.') + uu.exception_log(no_upload, 'Invalid run through option. Please enter true or false.') else: pass - actual_stages = uu.analysis_stages(Brazil_stages, stage_input, run_through) + actual_stages = uu.analysis_stages(Brazil_stages, stage_input, run_through, sensit_type) uu.print_log(actual_stages) @@ -105,7 +109,8 @@ def main (): out_pattern = cn.pattern_Brazil_forest_extent_2000_processed dt = 'Byte' pool = multiprocessing.Pool(int(cn.count/2)) - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, + no_upload=no_upload), tile_id_list) # Checks if each tile has data in it. Only tiles with data are uploaded. upload_dir = master_output_dir_list[0] @@ -158,7 +163,8 @@ def main (): out_pattern = cn.pattern_Brazil_annual_loss_processed dt = 'Byte' pool = multiprocessing.Pool(int(cn.count/2)) - pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) + pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, + no_upload=no_upload), tile_id_list) uu.print_log(" PRODES composite loss raster warped to Hansen tiles") # Checks if each tile has data in it. Only tiles with data are uploaded. @@ -669,7 +675,7 @@ def main (): uu.upload_final_set(stage_output_dir_list[0], stage_output_pattern_list[0]) else: - uu.exception_log("Extent argument not valid") + uu.exception_log(no_upload, "Extent argument not valid") uu.print_log("Creating tiles of belowground carbon") # 18 processors used between 300 and 400 GB memory, so it was okay on a r4.16xlarge spot machine @@ -743,7 +749,7 @@ def main (): uu.print_log("Skipping soil for 2000 carbon pool calculation") else: - uu.exception_log("Extent argument not valid") + uu.exception_log(no_upload, "Extent argument not valid") uu.print_log("Creating tiles of total carbon") # I tried several different processor numbers for this. Ended up using 14 processors, which used about 380 GB memory diff --git a/universal_util.py b/universal_util.py index 544c5f62..27343615 100644 --- a/universal_util.py +++ b/universal_util.py @@ -39,7 +39,8 @@ def upload_log(): # Creates the log with a starting line -def initiate_log(tile_id_list=None, sensit_type=None, run_date=None, stage_input=None, run_through=None, carbon_pool_extent=None, +def initiate_log(tile_id_list=None, sensit_type=None, run_date=None, no_upload=None, save_intermediates=None, + stage_input=None, run_through=None, carbon_pool_extent=None, emitted_pools=None, thresh=None, std_net_flux=None, include_mangroves=None, include_us=None, log_note=None): @@ -50,7 +51,7 @@ def initiate_log(tile_id_list=None, sensit_type=None, run_date=None, stage_input logging.info("This is the start of the log for this model run. Below are the command line arguments for this run.") logging.info("Sensitivity analysis type: {}".format(sensit_type)) logging.info("Model stage argument: {}".format(stage_input)) - logging.info("Run through model: {}".format(run_through)) + logging.info("Run model stages after the initial selected stage: {}".format(run_through)) logging.info("Run date: {}".format(run_date)) logging.info("Tile ID list: {}".format(tile_id_list)) logging.info("Carbon emitted_pools to generate (optional): {}".format(carbon_pool_extent)) @@ -59,6 +60,8 @@ def initiate_log(tile_id_list=None, sensit_type=None, run_date=None, stage_input logging.info("Standard net flux for comparison with sensitivity analysis net flux (optional): {}".format(std_net_flux)) logging.info("Include mangrove removal scripts in model run (optional): {}".format(include_mangroves)) logging.info("Include US removal scripts in model run (optional): {}".format(include_us)) + logging.info("Do not upload anything to s3: {}".format(no_upload)) + logging.info("Save intermediate outputs: {}".format(save_intermediates)) logging.info("AWS ec2 instance type and AMI ID:") # https://stackoverflow.com/questions/13735051/how-to-capture-curl-output-to-a-file @@ -92,7 +95,9 @@ def initiate_log(tile_id_list=None, sensit_type=None, run_date=None, stage_input logging.getLogger("rasterio").setLevel(logging.ERROR) # https://www.tutorialspoint.com/How-to-disable-logging-from-imported-modules-in-Python logging.getLogger("botocore").setLevel(logging.ERROR) # "Found credentials in environment variables." is logged by botocore: https://github.com/boto/botocore/issues/1841 - upload_log() + # If no_upload flag is not activated, log is uploaded + if not no_upload: + upload_log() # Prints the output statement in the console and adds it to the log. It can handle an indefinite number of string to print @@ -119,7 +124,7 @@ def print_log(*args): # Logs fatal errors to the log txt, uploads to s3, and then terminates the program with an exception in the console -def exception_log(*args): +def exception_log(no_upload, *args): # Empty string full_statement = str(object='') @@ -129,10 +134,13 @@ def exception_log(*args): full_statement = full_statement + str(arg) + " " # Adds the exception to the log txt - logging.debug(full_statement, stack_info=True) + logging.info(full_statement, stack_info=True) - # Need to upload log before the exception stops the script - upload_log() + # If no_upload flag is not activated, output is uploaded + if not no_upload: + + # Need to upload log before the exception stops the script + upload_log() # Prints to console, ending the program raise Exception(full_statement) @@ -885,7 +893,7 @@ def upload_final_set(upload_dir, pattern): except: print_log("Error uploading output tile(s)") - # Uploads the log as each model stage is finished + # Uploads the log as each model output tile set is finished upload_log() @@ -904,7 +912,8 @@ def upload_final(upload_dir, tile_id, pattern): # This version of checking for data is bad because it can miss tiles that have very little data in them. -# But it takes less memory than using rasterio, so it's good for local tests +# But it takes less memory than using rasterio, so it's good for local tests. +# This method creates a tif.aux.xml file that I tried to add a line to delete but couldn't get to work. def check_and_delete_if_empty_light(tile_id, output_pattern): tile_name = '{0}_{1}.tif'.format(tile_id, output_pattern) @@ -922,6 +931,10 @@ def check_and_delete_if_empty_light(tile_id, output_pattern): print_log(" No data found. Deleting {}...".format(tile_name)) os.remove(tile_name) + # Using this gdal data check method creates a tif.aux.xml file that is unnecessary. + # This does not work, however; it returns an error that there is no such file or directory. + # os.remove("{0}{1}.aux.xml".format(cn.docker_base_dir, tile_name)) + # This version of checking for data in a tile is more robust def check_for_data(tile): @@ -996,19 +1009,21 @@ def get_raster_nodata_value(tile): # Prints information about the tile that was just processed: how long it took and how many tiles have been completed -def end_of_fx_summary(start, tile_id, pattern): +def end_of_fx_summary(start, tile_id, pattern, no_upload): end = datetime.datetime.now() elapsed_time = end-start print_log("Processing time for tile", tile_id, ":", elapsed_time) count_completed_tiles(pattern) - # Uploads the log as each tile is finished - upload_log() + # If no_upload flag is not activated, log is uploaded + if not no_upload: + # Uploads the log as each tile is finished + upload_log() # Warps raster to Hansen tiles using multiple processors -def mp_warp_to_Hansen(tile_id, source_raster, out_pattern, dt): +def mp_warp_to_Hansen(tile_id, source_raster, out_pattern, dt, no_upload): # Start time start = datetime.datetime.now() @@ -1024,7 +1039,7 @@ def mp_warp_to_Hansen(tile_id, source_raster, out_pattern, dt): with process.stdout: log_subprocess_output(process.stdout) - end_of_fx_summary(start, tile_id, out_pattern) + end_of_fx_summary(start, tile_id, out_pattern, no_upload) def warp_to_Hansen(in_file, out_file, xmin, ymin, xmax, ymax, dt): @@ -1275,10 +1290,11 @@ def analysis_stages(stage_list, stage_input, run_through, sensit_type, stage_output = stage_list[1:] + # If the user selected a specific stage, the run_through argument is evaluated else: # If the user wants to run through all stages after the selected one, a new list is created - if run_through == 'true': + if run_through: stage_output = stage_list[stage_list.index(stage_input):] @@ -1288,10 +1304,10 @@ def analysis_stages(stage_list, stage_input, run_through, sensit_type, stage_output = stage_input.split() # Flags to include mangrove forest removal rates and US-specific removal rates in the stages to run - if include_us == 'true': + if include_us: stage_output.insert(0, 'annual_removals_us') - if include_mangroves == 'true': + if include_mangroves: stage_output.insert(0, 'annual_removals_mangrove') # Step create_supplementary_outputs only run for standard model