Refactor make conditionals more declarative

.pylintrc

@@ -0,0 +1,5 @@
+# .pylintrc
+
+[MASTER]
+
+disable=line-too-long, redefined-outer-name, invalid-name

Dockerfile

@@ -1,8 +1,9 @@
-# Use osgeo GDAL image. It builds off Ubuntu 18.04 and uses GDAL 3.0.4
-FROM osgeo/gdal:ubuntu-small-3.0.4
+# Use osgeo GDAL image.
+#Ubuntu 20.04.4 LTS, Python 3.8.10, GDAL 3.4.2
+FROM osgeo/gdal:ubuntu-small-3.4.2
 
 # # Use this if downloading hdf files for burn year analysis
-# FROM osgeo/gdal:ubuntu-full-3.0.4
+# FROM osgeo/gdal:ubuntu-full-3.4.2
 
 ENV DIR=/usr/local/app
 ENV TMP=/usr/local/tmp
@@ -14,16 +15,17 @@ ENV SECRETS_PATH /usr/secrets
 RUN ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime
 
 # Install dependencies
+# PostGIS extension version based on https://computingforgeeks.com/how-to-install-postgis-on-ubuntu-linux/
 RUN apt-get update -y && apt-get install -y \
     make \
     automake \
     g++ \
     gcc \
     libpq-dev \
-    postgresql-10 \
-    postgresql-server-dev-10 \
-    postgresql-contrib-10 \
-    postgresql-10-postgis-2.4 \
+    postgresql-12 \
+    postgresql-server-dev-12 \
+    postgresql-contrib-12 \
+    postgresql-12-postgis-3 \
     python3-pip \
     wget \
     nano \
@@ -57,7 +59,7 @@ ENV PGDATABASE=ubuntu
 # Commented out the start/restart commands because even with running them, postgres isn't running when the container is created.
 # So there's no point in starting posgres here if it's not active when the instance opens.
 #######################################
-RUN cp pg_hba.conf /etc/postgresql/10/main/
+RUN cp pg_hba.conf /etc/postgresql/12/main/
 # RUN pg_ctlcluster 10 main start
 # RUN service postgresql restart
 
@@ -68,9 +70,9 @@ RUN pip3 install -r requirements.txt
 # Link gdal libraries
 RUN cd /usr/include && ln -s ./ gdal
 
-# Somehow, this makes gdal_calc.py accessible from anywhere in the Docker
-#https://www.continualintegration.com/miscellaneous-articles/all/how-do-you-troubleshoot-usr-bin-env-python-no-such-file-or-directory/
-RUN ln -s /usr/bin/python3 /usr/bin/python
+# # Somehow, this makes gdal_calc.py accessible from anywhere in the Docker
+# #https://www.continualintegration.com/miscellaneous-articles/all/how-do-you-troubleshoot-usr-bin-env-python-no-such-file-or-directory/
+# RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Enable ec2 to interact with GitHub
 RUN git config --global user.email [email protected]
@@ -82,7 +84,8 @@ RUN git config --global user.email [email protected]
 #RUN git pull origin model_v_1.2.2
 
 ## Compile C++ scripts
-#RUN g++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.exe -lgdal && \
+RUN g++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.exe -lgdal
+#    RUN g++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_generic.exe -lgdal && \
 #    g++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_soil_only.cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_soil_only.exe -lgdal && \
 #    g++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_no_shifting_ag.cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_no_shifting_ag.exe -lgdal && \
 #    g++ /usr/local/app/emissions/cpp_util/calc_gross_emissions_convert_to_grassland.cpp -o /usr/local/app/emissions/cpp_util/calc_gross_emissions_convert_to_grassland.exe -lgdal

analyses/aggregate_results_to_4_km.py

@@ -34,7 +34,7 @@
 # 0.1x0.1 degree resolution (approximately 10m in the tropics).
 # Each pixel in that raster is the sum of the 30m pixels converted to value/pixel (instead of value/ha).
 # The 0.1x0.1 degree tile is output.
-def aggregate(tile, thresh, sensit_type, no_upload):
+def aggregate(tile, thresh):
 
     # start time
     start = datetime.datetime.now()
@@ -45,11 +45,11 @@ def aggregate(tile, thresh, sensit_type, no_upload):
     xmin, ymin, xmax, ymax = uu.coords(tile_id)
 
     # Name of inputs
-    focal_tile_rewindow = '{0}_{1}_rewindow.tif'.format(tile_id, tile_type)
-    pixel_area_rewindow = '{0}_{1}.tif'.format(cn.pattern_pixel_area_rewindow, tile_id)
-    tcd_rewindow = '{0}_{1}.tif'.format(cn.pattern_tcd_rewindow, tile_id)
-    gain_rewindow = '{0}_{1}.tif'.format(cn.pattern_gain_rewindow, tile_id)
-    mangrove_rewindow = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000_rewindow)
+    focal_tile_rewindow = f'{tile_id}_{tile_type}_rewindow.tif'
+    pixel_area_rewindow = f'{cn.pattern_pixel_area_rewindow}_{tile_id}.tif'
+    tcd_rewindow = f'{cn.pattern_tcd_rewindow}_{tile_id}.tif'
+    gain_rewindow = f'{cn.pattern_gain_rewindow}_{tile_id}.tif'
+    mangrove_rewindow = f'{tile_id}_{cn.pattern_mangrove_biomass_2000_rewindow}.tif'
 
     # Opens input tiles for rasterio
     in_src = rasterio.open(focal_tile_rewindow)
@@ -59,19 +59,19 @@ def aggregate(tile, thresh, sensit_type, no_upload):
 
     try:
         mangrove_src = rasterio.open(mangrove_rewindow)
-        uu.print_log("    Mangrove tile found for {}".format(tile_id))
+        uu.print_log(f'    Mangrove tile found for {tile_id}')
     except:
-        uu.print_log("    No mangrove tile found for {}".format(tile_id))
+        uu.print_log(f'    No mangrove tile found for {tile_id}')
 
-    uu.print_log("  Converting {} to per-pixel values...".format(tile))
+    uu.print_log(f'  Converting {tile} to per-pixel values...')
 
     # Grabs the windows of the tile (stripes) in order to iterate over the entire tif without running out of memory
     windows = in_src.block_windows(1)
 
     #2D array in which the 0.04x0.04 deg aggregated sums will be stored
     sum_array = np.zeros([250,250], 'float32')
 
-    out_raster = "{0}_{1}_0_04deg.tif".format(tile_id, tile_type)
+    out_raster = f'{tile_id}_{tile_type}_0_04deg.tif'
 
     uu.check_memory()
 
@@ -129,7 +129,7 @@ def aggregate(tile, thresh, sensit_type, no_upload):
     if cn.pattern_net_flux in tile_type:
         sum_array = sum_array / cn.loss_years / cn.tonnes_to_megatonnes
 
-    uu.print_log("  Creating aggregated tile for {}...".format(tile))
+    uu.print_log(f'  Creating aggregated tile for {tile}...')
 
     # Converts array to the same output type as the raster that is created below
     sum_array = np.float32(sum_array)
@@ -148,7 +148,7 @@ def aggregate(tile, thresh, sensit_type, no_upload):
         # print(aggregated)
         # aggregated.update_tags(a="1")
         # print(aggregated.tags())
-        # uu.add_rasterio_tags(aggregated, sensit_type)
+        # uu.add_rasterio_tags(aggregated)
         # print(aggregated.tags())
         # if cn.pattern_annual_gain_AGC_all_types in tile_type:
         #     aggregated.update_tags(units='Mg aboveground carbon/pixel, where pixels are 0.04x0.04 degrees)',
@@ -185,12 +185,12 @@ def aggregate(tile, thresh, sensit_type, no_upload):
         # aggregated.close()
 
     # Prints information about the tile that was just processed
-    uu.end_of_fx_summary(start, tile_id, '{}_0_04deg'.format(tile_type), no_upload)
+    uu.end_of_fx_summary(start, tile_id, f'{tile_type}_0_04deg')
 
 
 # Calculates the percent difference between the standard model's net flux output
 # and the sensitivity model's net flux output
-def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload):
+def percent_diff(std_aggreg_flux, sensit_aggreg_flux):
 
     # start time
     start = datetime.datetime.now()
@@ -207,7 +207,7 @@ def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload):
     # fine for all the other analyses, though (including legal_Amazon_loss).
     # Maybe that divide by 0 is throwing off other values now.
     perc_diff_calc = '--calc=(A-B)/absolute(B)*100'
-    perc_diff_outfilename = '{0}_{1}_{2}.tif'.format(cn.pattern_aggreg_sensit_perc_diff, sensit_type, date_formatted)
+    perc_diff_outfilename = '{0}_{1}_{2}.tif'.format(cn.pattern_aggreg_sensit_perc_diff, cn.SENSIT_TYPE, date_formatted)
     perc_diff_outfilearg = '--outfile={}'.format(perc_diff_outfilename)
     # cmd = ['gdal_calc.py', '-A', sensit_aggreg_flux, '-B', std_aggreg_flux, perc_diff_calc, perc_diff_outfilearg,
     #        '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=DEFLATE', '--quiet']
@@ -216,11 +216,11 @@ def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload):
     uu.log_subprocess_output_full(cmd)
 
     # Prints information about the tile that was just processed
-    uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux, no_upload)
+    uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux)
 
 
 # Maps where the sources stay sources, sinks stay sinks, sources become sinks, and sinks become sources
-def sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload):
+def sign_change(std_aggreg_flux, sensit_aggreg_flux):
 
     # start time
     start = datetime.datetime.now()
@@ -240,14 +240,14 @@ def sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload):
         sensit_src = rasterio.open(sensit_aggreg_flux)
 
         # Creates the sign change raster
-        dst = rasterio.open('{0}_{1}_{2}.tif'.format(cn.pattern_aggreg_sensit_sign_change, sensit_type, date_formatted), 'w', **kwargs)
+        dst = rasterio.open('{0}_{1}_{2}.tif'.format(cn.pattern_aggreg_sensit_sign_change, cn.SENSIT_TYPE, date_formatted), 'w', **kwargs)
 
         # Adds metadata tags to the output raster
-        uu.add_rasterio_tags(dst, sensit_type)
+        uu.add_universal_metadata_rasterio(dst)
         dst.update_tags(
             key='1=stays net source. 2=stays net sink. 3=changes from net source to net sink. 4=changes from net sink to net source.')
         dst.update_tags(
-            source='Comparison of net flux at 0.04x0.04 degrees from standard model to net flux from {} sensitivity analysis'.format(sensit_type))
+            source='Comparison of net flux at 0.04x0.04 degrees from standard model to net flux from {} sensitivity analysis'.format(cn.SENSIT_TYPE))
         dst.update_tags(
             extent='Global')
 
@@ -273,4 +273,4 @@ def sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload):
 
 
     # Prints information about the tile that was just processed
-    uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux, no_upload)
+    uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux)

analyses/create_supplementary_outputs.py

@@ -24,7 +24,7 @@
 import constants_and_names as cn
 import universal_util as uu
 
-def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit_type, no_upload):
+def create_supplementary_outputs(tile_id, input_pattern, output_patterns):
 
     # start time
     start = datetime.datetime.now()
@@ -33,18 +33,18 @@ def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit
     tile_id = uu.get_tile_id(tile_id)
 
     # Names of inputs
-    focal_tile = '{0}_{1}.tif'.format(tile_id, input_pattern)
-    pixel_area = '{0}_{1}.tif'.format(cn.pattern_pixel_area, tile_id)
-    tcd = '{0}_{1}.tif'.format(cn.pattern_tcd, tile_id)
-    gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
-    mangrove = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
+    focal_tile = f'{tile_id}_{input_pattern}.tif'
+    pixel_area = f'{cn.pattern_pixel_area}_{tile_id}.tif'
+    tcd = f'{cn.pattern_tcd}_{tile_id}.tif'
+    gain = f'{cn.pattern_gain}_{tile_id}.tif'
+    mangrove = f'{tile_id}_{cn.pattern_mangrove_biomass_2000}.tif'
 
     # Names of outputs.
     # Requires that output patterns be listed in main script in the correct order for here
     # (currently, per pixel full extent, per hectare forest extent, per pixel forest extent).
-    per_pixel_full_extent = '{0}_{1}.tif'.format(tile_id, output_patterns[0])
-    per_hectare_forest_extent = '{0}_{1}.tif'.format(tile_id, output_patterns[1])
-    per_pixel_forest_extent = '{0}_{1}.tif'.format(tile_id, output_patterns[2])
+    per_pixel_full_extent = f'{tile_id}_{output_patterns[0]}.tif'
+    per_hectare_forest_extent = f'{tile_id}_{output_patterns[1]}.tif'
+    per_pixel_forest_extent = f'{tile_id}_{output_patterns[2]}.tif'
 
     # Opens input tiles for rasterio
     in_src = rasterio.open(focal_tile)
@@ -59,11 +59,11 @@ def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit
 
     try:
         mangrove_src = rasterio.open(mangrove)
-        uu.print_log("    Mangrove tile found for {}".format(tile_id))
+        uu.print_log(f'    Mangrove tile found for {tile_id}')
     except:
-        uu.print_log("    No mangrove tile found for {}".format(tile_id))
+        uu.print_log(f'    No mangrove tile found for {tile_id}')
 
-    uu.print_log("  Creating outputs for {}...".format(focal_tile))
+    uu.print_log(f'  Creating outputs for {focal_tile}...')
 
     kwargs.update(
         driver='GTiff',
@@ -80,25 +80,25 @@ def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit
 
     # Adds metadata tags to the output rasters
 
-    uu.add_rasterio_tags(per_pixel_full_extent_dst, sensit_type)
+    uu.add_universal_metadata_rasterio(per_pixel_full_extent_dst)
     per_pixel_full_extent_dst.update_tags(
-        units='Mg CO2e/pixel over model duration (2001-20{})'.format(cn.loss_years))
+        units=f'Mg CO2e/pixel over model duration (2001-20{cn.loss_years})')
     per_pixel_full_extent_dst.update_tags(
         source='per hectare full model extent tile')
     per_pixel_full_extent_dst.update_tags(
         extent='Full model extent: ((TCD2000>0 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations')
 
-    uu.add_rasterio_tags(per_hectare_forest_extent_dst, sensit_type)
+    uu.add_universal_metadata_rasterio(per_hectare_forest_extent_dst)
     per_hectare_forest_extent_dst.update_tags(
-        units='Mg CO2e/hectare over model duration (2001-20{})'.format(cn.loss_years))
+        units=f'Mg CO2e/hectare over model duration (2001-20{cn.loss_years})')
     per_hectare_forest_extent_dst.update_tags(
         source='per hectare full model extent tile')
     per_hectare_forest_extent_dst.update_tags(
         extent='Forest extent: ((TCD2000>30 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations')
 
-    uu.add_rasterio_tags(per_pixel_forest_extent_dst, sensit_type)
+    uu.add_universal_metadata_rasterio(per_pixel_forest_extent_dst)
     per_pixel_forest_extent_dst.update_tags(
-        units='Mg CO2e/pixel over model duration (2001-20{})'.format(cn.loss_years))
+        units=f'Mg CO2e/pixel over model duration (2001-20{cn.loss_years})')
     per_pixel_forest_extent_dst.update_tags(
         source='per hectare forest model extent tile')
     per_pixel_forest_extent_dst.update_tags(
@@ -143,7 +143,7 @@ def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit
         per_hectare_forest_extent_dst.write_band(1, dst_window_per_hectare_forest_extent, window=window)
         per_pixel_forest_extent_dst.write_band(1, dst_window_per_pixel_forest_extent, window=window)
 
-    uu.print_log("  Output tiles created for {}...".format(tile_id))
+    uu.print_log(f'  Output tiles created for {tile_id}...')
 
     # Prints information about the tile that was just processed
-    uu.end_of_fx_summary(start, tile_id, output_patterns[0], no_upload)
+    uu.end_of_fx_summary(start, tile_id, output_patterns[0])

analyses/download_tile_set.py

@@ -103,7 +103,7 @@ def download_tile_set(sensit_type, tile_id_list):
     parser = argparse.ArgumentParser(
         description='Download model outputs for specific tile')
     parser.add_argument('--model-type', '-t', required=True,
-                        help='{}'.format(cn.model_type_arg_help))
+                        help=f'{cn.model_type_arg_help}')
     parser.add_argument('--tile_id_list', '-l', required=True,
                         help='List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.')
     parser.add_argument('--run-date', '-d', required=False,
@@ -114,7 +114,7 @@ def download_tile_set(sensit_type, tile_id_list):
     run_date = args.run_date
 
     # Create the output log
-    uu.initiate_log(tile_id_list=tile_id_list, sensit_type=sensit_type, run_date=run_date)
+    uu.initiate_log(tile_id_list)
 
     # Checks whether the sensitivity analysis and tile_id_list arguments are valid
     uu.check_sensit_type(sensit_type)