python3.9 initial port

batch-setup/batch.py

@@ -164,9 +164,9 @@ def s3_policy(bucket_or_buckets, date_prefix, allow_write=False):
     for bucket in buckets:
         object_resources.extend([
             # allow access to objects under the date prefix
-            "arn:aws:s3:::%s/%s/*" % (bucket, date_prefix),
+            f"arn:aws:s3:::{bucket}/{date_prefix}/*",
             # and also objects under a hash + date prefix
-            "arn:aws:s3:::%s/*/%s/*" % (bucket, date_prefix),
+            f"arn:aws:s3:::{bucket}/*/{date_prefix}/*",
         ])
         bucket_resources.append("arn:aws:s3:::%s" % (bucket))
 
@@ -234,7 +234,7 @@ def kebab_to_camel(name):
 
 def ensure_job_role_arn(iam, run_id, name, buckets, date_prefixes):
     role_name = kebab_to_camel(
-        "batch-%s-%s" % (name, run_id))
+        f"batch-{name}-{run_id}")
 
     arn = None
     try:
@@ -285,7 +285,7 @@ def create_role(iam, image_name, role_name, buckets, date_prefixes):
         Description='Role to perform %s batch jobs in %s/%s environment.' %
         (image_name, date_prefixes.rawr, date_prefixes.meta))
 
-    class RolePolicies(object):
+    class RolePolicies:
         def __init__(self, iam, role_name):
             self.iam = iam
             self.role_name = role_name
@@ -349,7 +349,7 @@ def make_job_definitions(
         retry_value = retry_attempts[name] \
             if isinstance(retry_attempts, dict) else retry_attempts
 
-        job_name = "%s-%s" % (name, run_id)
+        job_name = f"{name}-{run_id}"
         definition = {
             'name': job_name,
             'job-role-arn': job_role_arn,

batch-setup/batch_setup.py

@@ -15,15 +15,15 @@ def get_or_create_role(boto_iam, role_name, role_document, role_path=None):
             RoleName=role_name,
         )
         role_arn = response['Role']['Arn']
-        print("Found role %s at %s" % (role_name, role_arn))
+        print(f"Found role {role_name} at {role_arn}")
     except boto_iam.exceptions.NoSuchEntityException:
         response = boto_iam.create_role(
             Path=role_path,
             RoleName=role_name,
             AssumeRolePolicyDocument=json.dumps(role_document),
         )
         role_arn = response['Role']['Arn']
-        print("Created role %s at %s" % (role_name, role_arn))
+        print(f"Created role {role_name} at {role_arn}")
 
     return role_arn
 
@@ -93,7 +93,7 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen
         ]
     )
     subnet_ids = [s['SubnetId'] for s in response['Subnets']]
-    print("Found subnets %s on VPC %s" % (subnet_ids, vpc_id))
+    print(f"Found subnets {subnet_ids} on VPC {vpc_id}")
 
     # Create the batch service role
     # https://docs.aws.amazon.com/batch/latest/userguide/service_IAM_role.html

batch-setup/cancel_all_jobs.py

@@ -9,5 +9,5 @@
                     "misconfiguration.")
 args = parser.parse_args()
 
-job_queue = 'job-queue-%s' % (args.date,)
+job_queue = f'job-queue-{args.date}'
 terminate_all_jobs(job_queue, args.reason)

batch-setup/cron.py

@@ -561,7 +561,7 @@ def bucket_name(arg_name, bucket_function):
 
         # otherwise, default to a value constructed from the bucket prefix.
         if args.bucket_prefix:
-            return '%s-%s-%s' % (args.bucket_prefix, bucket_function, region)
+            return f'{args.bucket_prefix}-{bucket_function}-{region}'
 
         # finally, error out if we can't figure out a value.
         raise RuntimeError('Must provide either --bucket-prefix or %s.'
@@ -597,7 +597,7 @@ def bucket_name(arg_name, bucket_function):
     smgr = boto3.client('secretsmanager')
     smgr_name = (args.db_password_secret_name or
                  ('TilesDatabasePassword' + args.run_id))
-    smgr_description = 'Tiles database password for %s import' % (args.run_id,)
+    smgr_description = f'Tiles database password for {args.run_id} import'
     db_password = generate_or_update_password(
         smgr, args.db_password, smgr_name, smgr_description)
 
@@ -625,7 +625,7 @@ def bucket_name(arg_name, bucket_function):
     )
 
     script_dir = os.path.dirname(os.path.realpath(__file__))
-    with open(os.path.join(script_dir, 'provision.sh'), 'r') as fh:
+    with open(os.path.join(script_dir, 'provision.sh')) as fh:
         provision = fh.read() % provision_params
         provision_base64 = b64encode(provision.encode('utf8'))
 
@@ -669,13 +669,13 @@ def bucket_name(arg_name, bucket_function):
     instance = response['Instances'][0]
     instance_id = instance['InstanceId']
 
-    print('reservation ID: %s' % (reservation_id,))
-    print('instance ID:    %s' % (instance_id,))
+    print(f'reservation ID: {reservation_id}')
+    print(f'instance ID:    {instance_id}')
 
     print('Waiting for instance to come up...')
     waiter = ec2.get_waiter('instance_status_ok')
     waiter.wait(InstanceIds=[instance_id])
 
     response = ec2.describe_instances(InstanceIds=[instance_id])
     instance = response['Reservations'][0]['Instances'][0]
-    print('public IP:      %r' % (instance.get('PublicIpAddress'),))
+    print('public IP:      {!r}'.format(instance.get('PublicIpAddress')))

batch-setup/ecr.py

@@ -39,7 +39,7 @@ def ensure_ecr(run_id):
 
     repo_uris = {}
     for repo_name in repo_names:
-        full_name = 'tilezen/%s-%s' % (repo_name, run_id)
+        full_name = f'tilezen/{repo_name}-{run_id}'
         repo_uris[repo_name] = ensure_repo(ecr, full_name)
 
     return repo_uris

batch-setup/make_meta_tiles.py

@@ -27,7 +27,7 @@
 MissingTiles = namedtuple('MissingTiles', 'low_zoom_file high_zoom_file')
 
 
-class MissingTileFinder(object):
+class MissingTileFinder:
     """
     Finds tiles missing from an S3 bucket and provides convenience methods to
     navigate them.
@@ -54,7 +54,7 @@ def __init__(self, missing_bucket, tile_bucket, src_date_prefix,
             assert self.region
             assert self.key_format_type is not None
 
-        with open(config, 'r') as fh:
+        with open(config) as fh:
             conf = yaml.load(fh.read())
             self.job_queue_name = conf['batch']['job-queue']
             self.job_definition = conf['batch']['job-definition']
@@ -132,8 +132,8 @@ def read_metas_to_file(self, filename, present=False, compress=False):
                '-bucket', self.missing_bucket,
                '-date-prefix', self.dst_date_prefix,
                '-region', self.region,
-               '-present=%r' % (bool(present),),
-               '-compress-output=%r' % (bool(compress),),
+               f'-present={bool(present)!r}',
+               f'-compress-output={bool(compress)!r}',
                '-max-zoom', str(self.max_zoom),
                stdout=filename)
 
@@ -237,7 +237,7 @@ def present_tiles(self):
             shutil.rmtree(tmpdir)
 
 
-class _JobSizer(object):
+class _JobSizer:
     """
     Individual instance of a callable which can evaluate the size of a job
     (i.e: grouped set of RAWR tiles).
@@ -340,7 +340,7 @@ def update_memory_request(cfg, mem_multiplier, mem_max):
 # adaptor class for MissingTiles to see just the high zoom parts, this is used
 # along with the LowZoomLense to loop over missing tiles generically but
 # separately.
-class HighZoomLense(object):
+class HighZoomLense:
     def __init__(self, config):
         self.config = config
         self.description = "high zoom tiles"
@@ -349,7 +349,7 @@ def missing_file(self, missing):
         return missing.high_zoom_file
 
 
-class LowZoomLense(object):
+class LowZoomLense:
     def __init__(self, config):
         self.config = config
         self.description = "low zoom tiles"
@@ -361,7 +361,7 @@ def missing_file(self, missing):
 # abstracts away the logic for a re-rendering loop, splitting between high and
 # low zoom tiles and stopping if all the tiles aren't rendered within a
 # certain number of retries.
-class TileRenderer(object):
+class TileRenderer:
 
     def __init__(self, tile_finder, big_jobs, split_zoom, zoom_max,
                  allowed_missing_tiles=0, tile_coords_generator=None):
@@ -489,10 +489,10 @@ def render(self, num_retries, lense):
     generator = None
     if args.use_tile_coords_generator:
         bboxes = args.tile_coords_generator_bbox.split(',')
-        assert len(bboxes) == 4, 'Seed config: custom bbox {} does not have exactly four elements!'.format(bboxes)
+        assert len(bboxes) == 4, f'Seed config: custom bbox {bboxes} does not have exactly four elements!'
         min_x, min_y, max_x, max_y = list(map(float, bboxes))
-        assert min_x < max_x, 'Invalid bbox. X: {} not less than {}'.format(min_x, max_x)
-        assert min_y < max_y, 'Invalid bbox. Y: {} not less than {}'.format(min_y, max_y)
+        assert min_x < max_x, f'Invalid bbox. X: {min_x} not less than {max_x}'
+        assert min_y < max_y, f'Invalid bbox. Y: {min_y} not less than {max_y}'
         generator = BoundingBoxTileCoordinatesGenerator(min_x=min_x,
                                                         min_y=min_y,
                                                         max_x=max_x,

batch-setup/make_rawr_tiles.py

@@ -44,8 +44,8 @@ def missing_tiles(missing_bucket, rawr_bucket, date_prefix, region,
                   key_format_type, config, zoom, tile_coords_generator):
     from make_meta_tiles import MissingTileFinder
     if bool(tile_coords_generator):
-        return set([c for c in tile_coords_generator.
-                   generate_tiles_coordinates([zoom])])
+        return {c for c in tile_coords_generator.
+                   generate_tiles_coordinates([zoom])}
     else:
         present = set()
         finder = MissingTileFinder(
@@ -79,7 +79,7 @@ def missing_jobs(missing_bucket, rawr_bucket, date_prefix, region, config,
 
     # the rawr tiles of `tile_zoom` is actually built by AWS batch jobs of
     # `job_zoom` so we need to do a zoomTo here to find the corresponding jobs
-    jobs = set(coord.zoomTo(job_zoom).container() for coord in tiles)
+    jobs = {coord.zoomTo(job_zoom).container() for coord in tiles}
 
     print("[make_rawr_tiles] Missing %d tiles (%d jobs)" % (len(tiles), len(jobs)))
 
@@ -103,7 +103,7 @@ def wc_line(filename):
     line utility `wc -l`.
     """
 
-    with open(filename, 'r') as fh:
+    with open(filename) as fh:
         count = sum(1 for _ in fh)
     return count
 
@@ -116,7 +116,7 @@ def head_lines(filename, n_lines):
 
     sample = []
 
-    with open(filename, 'r') as fh:
+    with open(filename) as fh:
         try:
             for _ in range(n_lines):
                 sample.append(next(fh).strip())
@@ -176,7 +176,7 @@ def make_rawr_tiles(rawr_config_file, missing_config_file, missing_bucket,
     """
 
     assert os.path.isfile(rawr_config_file), rawr_config_file
-    with open(rawr_config_file, 'r') as fh:
+    with open(rawr_config_file) as fh:
         config = yaml.load(fh.read())
         job_zoom = config['batch']['queue-zoom']
         logging_config = config['logging']['config']
@@ -259,10 +259,10 @@ def make_rawr_tiles(rawr_config_file, missing_config_file, missing_bucket,
     generator = None
     if args.use_tile_coords_generator:
         bboxes = args.tile_coords_generator_bbox.split(',')
-        assert len(bboxes) == 4, 'Seed config: custom bbox {} does not have exactly four elements!'.format(bboxes)
+        assert len(bboxes) == 4, f'Seed config: custom bbox {bboxes} does not have exactly four elements!'
         min_x, min_y, max_x, max_y = list(map(float, bboxes))
-        assert min_x < max_x, 'Invalid bbox. X: {} not less than {}'.format(min_x, max_x)
-        assert min_y < max_y, 'Invalid bbox. Y: {} not less than {}'.format(min_y, max_y)
+        assert min_x < max_x, f'Invalid bbox. X: {min_x} not less than {max_x}'
+        assert min_y < max_y, f'Invalid bbox. Y: {min_y} not less than {max_y}'
         generator = BoundingBoxTileCoordinatesGenerator(min_x=min_x,
                                                         min_y=min_y,
                                                         max_x=max_x,

batch-setup/provision.sh

@@ -28,36 +28,36 @@ done
 # a per-branch basis without clobbering the (singleton) file on S3.
 #
 cat >/usr/local/etc/py-requirements.txt <<EOF
-Jinja2==2.10.1
-MarkupSafe==1.0
+Jinja2==3.0.1
+MarkupSafe==2.0.1
 ModestMaps==1.4.7
 PyYAML==4.2b4
-Shapely==1.6.2.post1
+Shapely==1.7.1
 StreetNames==0.1.5
 Werkzeug==0.12.2
 appdirs==1.4.3
 argparse==1.4.0
 boto3==1.9.32
-boto==2.48.0
+boto==2.49.0
 edtf==2.6.0
 enum34==1.1.6
 future==0.16.0
 hiredis==0.2.0
-mapbox-vector-tile==1.2.0
+mapbox-vector-tile==1.2.1
 paramiko==2.4.2
 protobuf==3.4.0
-psycopg2==2.7.3.2
-pyclipper==1.0.6
+psycopg2==2.8.6
+pyclipper==1.2.1
 pycountry==17.9.23
-pyproj==2.1.0
+pyproj==3.0.1
 python-dateutil==2.6.1
 redis==2.10.6
-requests==2.20.1
+requests==2.25.1
 six==1.11.0
 statsd==3.2.1
 ujson==1.35
-wsgiref==0.1.2
 zope.dottedname==4.2
+mercantile==1.2.1
 EOF
 pip install --upgrade pip
 virtualenv /usr/local/venv