Creating soil C emissions for 2021 update (v1.2.2).

analyses/aggregate_results_to_4_km.py

@@ -13,7 +13,7 @@
 For sensitivity analysis runs, it only processes outputs which actually have a sensitivity analysis version.
 The user has to supply a tcd threshold for which forest pixels to include in the results. Defaults to cn.canopy_threshold.
 For sensitivity analysis, the s3 folder with the aggregations for the standard model must be specified.
-sample command: python mp_aggregate_results_to_4_km.py -tcd 30 -t no_shifting_ag -sagg s3://gfw2-data/climate/carbon_model/0_4deg_output_aggregation/biomass_soil/standard/20200901/net_flux_Mt_CO2e_biomass_soil_per_year_tcd30_0_4deg_modelv1_2_0_std_20200901.tif
+sample command: python mp_aggregate_results_to_4_km.py -tcd 30 -t no_shifting_ag -sagg s3://gfw2-data/climate/carbon_model/0_04deg_output_aggregation/biomass_soil/standard/20200901/net_flux_Mt_CO2e_biomass_soil_per_year_tcd30_0_4deg_modelv1_2_0_std_20200901.tif
 '''
 
 
@@ -71,7 +71,7 @@ def aggregate(tile, thresh, sensit_type, no_upload):
     #2D array in which the 0.04x0.04 deg aggregated sums will be stored
     sum_array = np.zeros([250,250], 'float32')
 
-    out_raster = "{0}_{1}_0_4deg.tif".format(tile_id, tile_type)
+    out_raster = "{0}_{1}_0_04deg.tif".format(tile_id, tile_type)
 
     uu.check_memory()
 
@@ -185,7 +185,7 @@ def aggregate(tile, thresh, sensit_type, no_upload):
         # aggregated.close()
 
     # Prints information about the tile that was just processed
-    uu.end_of_fx_summary(start, tile_id, '{}_0_4deg'.format(tile_type), no_upload)
+    uu.end_of_fx_summary(start, tile_id, '{}_0_04deg'.format(tile_type), no_upload)
 
 
 # Calculates the percent difference between the standard model's net flux output

analyses/download_tile_set.py

@@ -27,26 +27,26 @@ def download_tile_set(sensit_type, tile_id_list):
     os.chdir(wd)
 
     download_dict = {
-        # cn.model_extent_dir: [cn.pattern_model_extent],
-        # cn.age_cat_IPCC_dir: [cn.pattern_age_cat_IPCC],
-        # cn.annual_gain_AGB_IPCC_defaults_dir: [cn.pattern_annual_gain_AGB_IPCC_defaults],
-        # cn.annual_gain_BGB_IPCC_defaults_dir: [cn.pattern_annual_gain_BGB_IPCC_defaults],
-        # cn.stdev_annual_gain_AGB_IPCC_defaults_dir: [cn.pattern_stdev_annual_gain_AGB_IPCC_defaults],
-        # cn.removal_forest_type_dir: [cn.pattern_removal_forest_type],
-        # cn.annual_gain_AGC_all_types_dir: [cn.pattern_annual_gain_AGC_all_types],
-        # cn.annual_gain_BGC_all_types_dir: [cn.pattern_annual_gain_BGC_all_types],
-        # cn.annual_gain_AGC_BGC_all_types_dir: [cn.pattern_annual_gain_AGC_BGC_all_types],
-        # cn.stdev_annual_gain_AGC_all_types_dir: [cn.pattern_stdev_annual_gain_AGC_all_types],
-        # cn.gain_year_count_dir: [cn.pattern_gain_year_count],
-        # cn.cumul_gain_AGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_all_types],
-        # cn.cumul_gain_BGCO2_all_types_dir: [cn.pattern_cumul_gain_BGCO2_all_types],
-        # cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
-        # cn.AGC_emis_year_dir: [cn.pattern_AGC_emis_year],
-        # cn.BGC_emis_year_dir: [cn.pattern_BGC_emis_year],
-        # cn.deadwood_emis_year_2000_dir: [cn.pattern_deadwood_emis_year_2000],
-        # cn.litter_emis_year_2000_dir: [cn.pattern_litter_emis_year_2000],
-        # cn.soil_C_emis_year_2000_dir: [cn.pattern_soil_C_emis_year_2000],
-        # cn.total_C_emis_year_dir: [cn.pattern_total_C_emis_year],
+        cn.model_extent_dir: [cn.pattern_model_extent],
+        cn.age_cat_IPCC_dir: [cn.pattern_age_cat_IPCC],
+        cn.annual_gain_AGB_IPCC_defaults_dir: [cn.pattern_annual_gain_AGB_IPCC_defaults],
+        cn.annual_gain_BGB_IPCC_defaults_dir: [cn.pattern_annual_gain_BGB_IPCC_defaults],
+        cn.stdev_annual_gain_AGB_IPCC_defaults_dir: [cn.pattern_stdev_annual_gain_AGB_IPCC_defaults],
+        cn.removal_forest_type_dir: [cn.pattern_removal_forest_type],
+        cn.annual_gain_AGC_all_types_dir: [cn.pattern_annual_gain_AGC_all_types],
+        cn.annual_gain_BGC_all_types_dir: [cn.pattern_annual_gain_BGC_all_types],
+        cn.annual_gain_AGC_BGC_all_types_dir: [cn.pattern_annual_gain_AGC_BGC_all_types],
+        cn.stdev_annual_gain_AGC_all_types_dir: [cn.pattern_stdev_annual_gain_AGC_all_types],
+        cn.gain_year_count_dir: [cn.pattern_gain_year_count],
+        cn.cumul_gain_AGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_all_types],
+        cn.cumul_gain_BGCO2_all_types_dir: [cn.pattern_cumul_gain_BGCO2_all_types],
+        cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
+        cn.AGC_emis_year_dir: [cn.pattern_AGC_emis_year],
+        cn.BGC_emis_year_dir: [cn.pattern_BGC_emis_year],
+        cn.deadwood_emis_year_2000_dir: [cn.pattern_deadwood_emis_year_2000],
+        cn.litter_emis_year_2000_dir: [cn.pattern_litter_emis_year_2000],
+        cn.soil_C_emis_year_2000_dir: [cn.pattern_soil_C_emis_year_2000],
+        cn.total_C_emis_year_dir: [cn.pattern_total_C_emis_year],
 
         # cn.gross_emis_commod_biomass_soil_dir: [cn.pattern_gross_emis_commod_biomass_soil],
         # cn.gross_emis_shifting_ag_biomass_soil_dir: [cn.pattern_gross_emis_shifting_ag_biomass_soil],

analyses/mp_aggregate_results_to_4_km.py

@@ -13,7 +13,7 @@
 For sensitivity analysis runs, it only processes outputs which actually have a sensitivity analysis version.
 The user has to supply a tcd threshold for which forest pixels to include in the results. Defaults to cn.canopy_threshold.
 For sensitivity analysis, the s3 folder with the aggregations for the standard model must be specified.
-sample command: python mp_aggregate_results_to_4_km.py -tcd 30 -t no_shifting_ag -sagg s3://gfw2-data/climate/carbon_model/0_4deg_output_aggregation/biomass_soil/standard/20200901/net_flux_Mt_CO2e_biomass_soil_per_year_tcd30_0_4deg_modelv1_2_0_std_20200901.tif
+sample command: python mp_aggregate_results_to_4_km.py -tcd 30 -t no_shifting_ag -sagg s3://gfw2-data/climate/carbon_model/0_04deg_output_aggregation/biomass_soil/standard/20200901/net_flux_Mt_CO2e_biomass_soil_per_year_tcd30_0_4deg_modelv1_2_0_std_20200901.tif
 '''
 
 
@@ -38,9 +38,9 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux
 
     # Files to download for this script
     download_dict = {
-             # cn.annual_gain_AGC_all_types_dir: [cn.pattern_annual_gain_AGC_all_types],
-             # cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
-             # cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil],
+             cn.annual_gain_AGC_all_types_dir: [cn.pattern_annual_gain_AGC_all_types],
+             cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
+             cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil],
              cn.net_flux_dir: [cn.pattern_net_flux]
              }
 
@@ -110,7 +110,7 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux
         # from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
         tile_list = [i for i in tile_list if not ('hanson_2013' in i)]
         tile_list = [i for i in tile_list if not ('rewindow' in i)]
-        tile_list = [i for i in tile_list if not ('0_4deg' in i)]
+        tile_list = [i for i in tile_list if not ('0_04deg' in i)]
         tile_list = [i for i in tile_list if not ('.ovr' in i)]
 
         # tile_list = ['00N_070W_cumul_gain_AGCO2_BGCO2_t_ha_all_forest_types_2001_15_biomass_swap.tif']  # test tiles
@@ -169,8 +169,8 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux
         #     aggregate_results_to_4_km.aggregate(tile, thresh, sensit_type, no_upload)
 
         # Makes a vrt of all the output 10x10 tiles (10 km resolution)
-        out_vrt = "{}_0_4deg.vrt".format(pattern)
-        os.system('gdalbuildvrt -tr 0.04 0.04 {0} *{1}_0_4deg*.tif'.format(out_vrt, pattern))
+        out_vrt = "{}_0_04deg.vrt".format(pattern)
+        os.system('gdalbuildvrt -tr 0.04 0.04 {0} *{1}_0_04deg*.tif'.format(out_vrt, pattern))
 
         # Creates the output name for the 10km map
         out_pattern = uu.name_aggregated_output(download_pattern_name, thresh, sensit_type)
@@ -221,7 +221,7 @@ def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux
         for tile_name in tile_list:
             tile_id = uu.get_tile_id(tile_name)
             os.remove('{0}_{1}_rewindow.tif'.format(tile_id, pattern))
-            os.remove('{0}_{1}_0_4deg.tif'.format(tile_id, pattern))
+            os.remove('{0}_{1}_0_04deg.tif'.format(tile_id, pattern))
 
     # Need to delete rewindowed tiles so they aren't confused with the normal tiles for creation of supplementary outputs
     rewindow_list = glob.glob('*rewindow*tif')

analyses/mp_create_supplementary_outputs.py

@@ -45,7 +45,7 @@ def mp_create_supplementary_outputs(sensit_type, tile_id_list, run_date = None,
 
     # Files to download for this script
     download_dict = {
-        # cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
+        cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
         cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil],
         cn.net_flux_dir: [cn.pattern_net_flux]
     }

constants_and_names.py

@@ -542,7 +542,7 @@
 ### Emissions from soil only
 
 # Date to include in the output directory
-emis_run_date_soil_only = '20220316'
+emis_run_date_soil_only = '20220318'
 
 pattern_gross_emis_commod_soil_only = 'gross_emis_commodity_Mg_CO2e_ha_soil_only_2001_{}'.format(loss_years)
 gross_emis_commod_soil_only_dir = '{0}gross_emissions/commodities/soil_only/standard/{1}/'.format(s3_base_dir, emis_run_date_soil_only)
@@ -618,11 +618,11 @@
 ### 4x4 km aggregation tiles for mapping
 ######
 
-pattern_aggreg = '0_4deg_modelv{}'.format(version_filename)
-pattern_aggreg_sensit_perc_diff = 'net_flux_0_4deg_modelv{}_perc_diff_std'.format(version_filename)
-pattern_aggreg_sensit_sign_change = 'net_flux_0_4deg_modelv{}_sign_change_std'.format(version_filename)
+pattern_aggreg = '0_04deg_modelv{}'.format(version_filename)
+pattern_aggreg_sensit_perc_diff = 'net_flux_0_04deg_modelv{}_perc_diff_std'.format(version_filename)
+pattern_aggreg_sensit_sign_change = 'net_flux_0_04deg_modelv{}_sign_change_std'.format(version_filename)
 
-output_aggreg_dir = os.path.join(s3_base_dir, '0_4deg_output_aggregation/biomass_soil/standard/20220316/')
+output_aggreg_dir = os.path.join(s3_base_dir, '0_04deg_output_aggregation/biomass_soil/standard/20220316/')
 
 
 

readme.md

@@ -239,7 +239,7 @@ Run three tiles in biomass_swap sensitivity analysis; don't upload intermediates
 run model_extent stage; don't continue after that stage (no run-through); get carbon pools at time of loss; emissions from biomass and soil;
 compare aggregated outputs to specified file (although not used in this specific launch because only the first step runs):
 
-`python run_full_model.py -nu -t biomass_swap -s model_extent -r false -d 20229999 -l 00N_000E,00N_110E,40N_90W -ce loss -p biomass_soil -tcd 30 -sagg s3://gfw2-data/climate/carbon_model/0_4deg_output_aggregation/biomass_soil/standard/20200914/net_flux_Mt_CO2e_biomass_soil_per_year_tcd30_0_4deg_modelv1_2_0_std_20200914.tif -ln "Multi-tile test"`
+`python run_full_model.py -nu -t biomass_swap -s model_extent -r false -d 20229999 -l 00N_000E,00N_110E,40N_90W -ce loss -p biomass_soil -tcd 30 -sagg s3://gfw2-data/climate/carbon_model/0_04deg_output_aggregation/biomass_soil/standard/20200914/net_flux_Mt_CO2e_biomass_soil_per_year_tcd30_0_4deg_modelv1_2_0_std_20200914.tif -ln "Multi-tile test"`
 
 
 ##### Running the emissions model

removals/annual_gain_rate_IPCC_defaults.py

@@ -19,7 +19,7 @@ def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict, ou
     # The key in the dictionary is the forest age category decision tree endpoints.
     age_dict = {0: 0, 1: 10000, 2: 20000, 3: 30000}
 
-    uu.print_log("Processing:", tile_id)
+    uu.print_log("Creating IPCC default biomass removals rates and standard deviation for {}".format(tile_id))
 
     # Start time
     start = datetime.datetime.now()
@@ -33,20 +33,18 @@ def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict, ou
     BGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[1])
     AGB_IPCC_default_gain_stdev = '{0}_{1}.tif'.format(tile_id, output_pattern_list[2])
 
-    uu.print_log("  Creating IPCC default biomass removals rates and standard deviation for {}".format(tile_id))
-
     # Opens the input tiles if they exist. kips tile if either input doesn't exist.
     try:
         age_cat_src = rasterio.open(age_cat)
-        uu.print_log("   Age category tile found for {}".format(tile_id))
+        uu.print_log("  Age category tile found for {}".format(tile_id))
     except:
-        return uu.print_log("   No age category tile found for {}. Skipping tile.".format(tile_id))
+        return uu.print_log("  No age category tile found for {}. Skipping tile.".format(tile_id))
 
     try:
         cont_eco_src = rasterio.open(cont_eco)
-        uu.print_log("   Continent-ecozone tile found for {}".format(tile_id))
+        uu.print_log("  Continent-ecozone tile found for {}".format(tile_id))
     except:
-        return uu.print_log("   No continent-ecozone tile found for {}. Skipping tile.".format(tile_id))
+        return uu.print_log("  No continent-ecozone tile found for {}. Skipping tile.".format(tile_id))
 
     # Grabs metadata about the continent ecozone tile, like its location/projection/cellsize
     kwargs = cont_eco_src.meta

removals/forest_age_category_IPCC.py

@@ -51,8 +51,6 @@ def forest_age_category(tile_id, gain_table_dict, pattern, sensit_type, no_uploa
         loss = '{0}_{1}.tif'.format(cn.pattern_loss, tile_id)
         uu.print_log("Using Hansen loss tile {0} for {1} model run".format(tile_id, sensit_type))
 
-    uu.print_log("  Assigning age categories")
-
     # Opens biomass tile
     with rasterio.open(model_extent) as model_extent_src: