diff --git a/batch-setup/batch.py b/batch-setup/batch.py index 5831322..67d4497 100644 --- a/batch-setup/batch.py +++ b/batch-setup/batch.py @@ -164,9 +164,9 @@ def s3_policy(bucket_or_buckets, date_prefix, allow_write=False): for bucket in buckets: object_resources.extend([ # allow access to objects under the date prefix - "arn:aws:s3:::%s/%s/*" % (bucket, date_prefix), + f"arn:aws:s3:::{bucket}/{date_prefix}/*", # and also objects under a hash + date prefix - "arn:aws:s3:::%s/*/%s/*" % (bucket, date_prefix), + f"arn:aws:s3:::{bucket}/*/{date_prefix}/*", ]) bucket_resources.append("arn:aws:s3:::%s" % (bucket)) @@ -234,7 +234,7 @@ def kebab_to_camel(name): def ensure_job_role_arn(iam, run_id, name, buckets, date_prefixes): role_name = kebab_to_camel( - "batch-%s-%s" % (name, run_id)) + f"batch-{name}-{run_id}") arn = None try: @@ -285,7 +285,7 @@ def create_role(iam, image_name, role_name, buckets, date_prefixes): Description='Role to perform %s batch jobs in %s/%s environment.' % (image_name, date_prefixes.rawr, date_prefixes.meta)) - class RolePolicies(object): + class RolePolicies: def __init__(self, iam, role_name): self.iam = iam self.role_name = role_name @@ -349,7 +349,7 @@ def make_job_definitions( retry_value = retry_attempts[name] \ if isinstance(retry_attempts, dict) else retry_attempts - job_name = "%s-%s" % (name, run_id) + job_name = f"{name}-{run_id}" definition = { 'name': job_name, 'job-role-arn': job_role_arn, diff --git a/batch-setup/batch_setup.py b/batch-setup/batch_setup.py index daa9556..9a55667 100644 --- a/batch-setup/batch_setup.py +++ b/batch-setup/batch_setup.py @@ -15,7 +15,7 @@ def get_or_create_role(boto_iam, role_name, role_document, role_path=None): RoleName=role_name, ) role_arn = response['Role']['Arn'] - print("Found role %s at %s" % (role_name, role_arn)) + print(f"Found role {role_name} at {role_arn}") except boto_iam.exceptions.NoSuchEntityException: response = boto_iam.create_role( Path=role_path, @@ -23,7 +23,7 @@ def get_or_create_role(boto_iam, role_name, role_document, role_path=None): AssumeRolePolicyDocument=json.dumps(role_document), ) role_arn = response['Role']['Arn'] - print("Created role %s at %s" % (role_name, role_arn)) + print(f"Created role {role_name} at {role_arn}") return role_arn @@ -93,7 +93,7 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen ] ) subnet_ids = [s['SubnetId'] for s in response['Subnets']] - print("Found subnets %s on VPC %s" % (subnet_ids, vpc_id)) + print(f"Found subnets {subnet_ids} on VPC {vpc_id}") # Create the batch service role # https://docs.aws.amazon.com/batch/latest/userguide/service_IAM_role.html diff --git a/batch-setup/cancel_all_jobs.py b/batch-setup/cancel_all_jobs.py index af2cc01..cf6bce3 100644 --- a/batch-setup/cancel_all_jobs.py +++ b/batch-setup/cancel_all_jobs.py @@ -9,5 +9,5 @@ "misconfiguration.") args = parser.parse_args() -job_queue = 'job-queue-%s' % (args.date,) +job_queue = f'job-queue-{args.date}' terminate_all_jobs(job_queue, args.reason) diff --git a/batch-setup/cron.py b/batch-setup/cron.py index ed1fa7a..8ff9570 100644 --- a/batch-setup/cron.py +++ b/batch-setup/cron.py @@ -561,7 +561,7 @@ def bucket_name(arg_name, bucket_function): # otherwise, default to a value constructed from the bucket prefix. if args.bucket_prefix: - return '%s-%s-%s' % (args.bucket_prefix, bucket_function, region) + return f'{args.bucket_prefix}-{bucket_function}-{region}' # finally, error out if we can't figure out a value. raise RuntimeError('Must provide either --bucket-prefix or %s.' @@ -597,7 +597,7 @@ def bucket_name(arg_name, bucket_function): smgr = boto3.client('secretsmanager') smgr_name = (args.db_password_secret_name or ('TilesDatabasePassword' + args.run_id)) - smgr_description = 'Tiles database password for %s import' % (args.run_id,) + smgr_description = f'Tiles database password for {args.run_id} import' db_password = generate_or_update_password( smgr, args.db_password, smgr_name, smgr_description) @@ -625,7 +625,7 @@ def bucket_name(arg_name, bucket_function): ) script_dir = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(script_dir, 'provision.sh'), 'r') as fh: + with open(os.path.join(script_dir, 'provision.sh')) as fh: provision = fh.read() % provision_params provision_base64 = b64encode(provision.encode('utf8')) @@ -669,8 +669,8 @@ def bucket_name(arg_name, bucket_function): instance = response['Instances'][0] instance_id = instance['InstanceId'] - print('reservation ID: %s' % (reservation_id,)) - print('instance ID: %s' % (instance_id,)) + print(f'reservation ID: {reservation_id}') + print(f'instance ID: {instance_id}') print('Waiting for instance to come up...') waiter = ec2.get_waiter('instance_status_ok') @@ -678,4 +678,4 @@ def bucket_name(arg_name, bucket_function): response = ec2.describe_instances(InstanceIds=[instance_id]) instance = response['Reservations'][0]['Instances'][0] - print('public IP: %r' % (instance.get('PublicIpAddress'),)) + print('public IP: {!r}'.format(instance.get('PublicIpAddress'))) diff --git a/batch-setup/ecr.py b/batch-setup/ecr.py index 546032e..9cd35e7 100644 --- a/batch-setup/ecr.py +++ b/batch-setup/ecr.py @@ -39,7 +39,7 @@ def ensure_ecr(run_id): repo_uris = {} for repo_name in repo_names: - full_name = 'tilezen/%s-%s' % (repo_name, run_id) + full_name = f'tilezen/{repo_name}-{run_id}' repo_uris[repo_name] = ensure_repo(ecr, full_name) return repo_uris diff --git a/batch-setup/make_meta_tiles.py b/batch-setup/make_meta_tiles.py index 22302f2..818ce97 100644 --- a/batch-setup/make_meta_tiles.py +++ b/batch-setup/make_meta_tiles.py @@ -27,7 +27,7 @@ MissingTiles = namedtuple('MissingTiles', 'low_zoom_file high_zoom_file') -class MissingTileFinder(object): +class MissingTileFinder: """ Finds tiles missing from an S3 bucket and provides convenience methods to navigate them. @@ -54,7 +54,7 @@ def __init__(self, missing_bucket, tile_bucket, src_date_prefix, assert self.region assert self.key_format_type is not None - with open(config, 'r') as fh: + with open(config) as fh: conf = yaml.load(fh.read()) self.job_queue_name = conf['batch']['job-queue'] self.job_definition = conf['batch']['job-definition'] @@ -132,8 +132,8 @@ def read_metas_to_file(self, filename, present=False, compress=False): '-bucket', self.missing_bucket, '-date-prefix', self.dst_date_prefix, '-region', self.region, - '-present=%r' % (bool(present),), - '-compress-output=%r' % (bool(compress),), + f'-present={bool(present)!r}', + f'-compress-output={bool(compress)!r}', '-max-zoom', str(self.max_zoom), stdout=filename) @@ -237,7 +237,7 @@ def present_tiles(self): shutil.rmtree(tmpdir) -class _JobSizer(object): +class _JobSizer: """ Individual instance of a callable which can evaluate the size of a job (i.e: grouped set of RAWR tiles). @@ -340,7 +340,7 @@ def update_memory_request(cfg, mem_multiplier, mem_max): # adaptor class for MissingTiles to see just the high zoom parts, this is used # along with the LowZoomLense to loop over missing tiles generically but # separately. -class HighZoomLense(object): +class HighZoomLense: def __init__(self, config): self.config = config self.description = "high zoom tiles" @@ -349,7 +349,7 @@ def missing_file(self, missing): return missing.high_zoom_file -class LowZoomLense(object): +class LowZoomLense: def __init__(self, config): self.config = config self.description = "low zoom tiles" @@ -361,7 +361,7 @@ def missing_file(self, missing): # abstracts away the logic for a re-rendering loop, splitting between high and # low zoom tiles and stopping if all the tiles aren't rendered within a # certain number of retries. -class TileRenderer(object): +class TileRenderer: def __init__(self, tile_finder, big_jobs, split_zoom, zoom_max, allowed_missing_tiles=0, tile_coords_generator=None): @@ -489,10 +489,10 @@ def render(self, num_retries, lense): generator = None if args.use_tile_coords_generator: bboxes = args.tile_coords_generator_bbox.split(',') - assert len(bboxes) == 4, 'Seed config: custom bbox {} does not have exactly four elements!'.format(bboxes) + assert len(bboxes) == 4, f'Seed config: custom bbox {bboxes} does not have exactly four elements!' min_x, min_y, max_x, max_y = list(map(float, bboxes)) - assert min_x < max_x, 'Invalid bbox. X: {} not less than {}'.format(min_x, max_x) - assert min_y < max_y, 'Invalid bbox. Y: {} not less than {}'.format(min_y, max_y) + assert min_x < max_x, f'Invalid bbox. X: {min_x} not less than {max_x}' + assert min_y < max_y, f'Invalid bbox. Y: {min_y} not less than {max_y}' generator = BoundingBoxTileCoordinatesGenerator(min_x=min_x, min_y=min_y, max_x=max_x, diff --git a/batch-setup/make_rawr_tiles.py b/batch-setup/make_rawr_tiles.py index 63adafd..3f86b6f 100644 --- a/batch-setup/make_rawr_tiles.py +++ b/batch-setup/make_rawr_tiles.py @@ -44,8 +44,8 @@ def missing_tiles(missing_bucket, rawr_bucket, date_prefix, region, key_format_type, config, zoom, tile_coords_generator): from make_meta_tiles import MissingTileFinder if bool(tile_coords_generator): - return set([c for c in tile_coords_generator. - generate_tiles_coordinates([zoom])]) + return {c for c in tile_coords_generator. + generate_tiles_coordinates([zoom])} else: present = set() finder = MissingTileFinder( @@ -79,7 +79,7 @@ def missing_jobs(missing_bucket, rawr_bucket, date_prefix, region, config, # the rawr tiles of `tile_zoom` is actually built by AWS batch jobs of # `job_zoom` so we need to do a zoomTo here to find the corresponding jobs - jobs = set(coord.zoomTo(job_zoom).container() for coord in tiles) + jobs = {coord.zoomTo(job_zoom).container() for coord in tiles} print("[make_rawr_tiles] Missing %d tiles (%d jobs)" % (len(tiles), len(jobs))) @@ -103,7 +103,7 @@ def wc_line(filename): line utility `wc -l`. """ - with open(filename, 'r') as fh: + with open(filename) as fh: count = sum(1 for _ in fh) return count @@ -116,7 +116,7 @@ def head_lines(filename, n_lines): sample = [] - with open(filename, 'r') as fh: + with open(filename) as fh: try: for _ in range(n_lines): sample.append(next(fh).strip()) @@ -176,7 +176,7 @@ def make_rawr_tiles(rawr_config_file, missing_config_file, missing_bucket, """ assert os.path.isfile(rawr_config_file), rawr_config_file - with open(rawr_config_file, 'r') as fh: + with open(rawr_config_file) as fh: config = yaml.load(fh.read()) job_zoom = config['batch']['queue-zoom'] logging_config = config['logging']['config'] @@ -259,10 +259,10 @@ def make_rawr_tiles(rawr_config_file, missing_config_file, missing_bucket, generator = None if args.use_tile_coords_generator: bboxes = args.tile_coords_generator_bbox.split(',') - assert len(bboxes) == 4, 'Seed config: custom bbox {} does not have exactly four elements!'.format(bboxes) + assert len(bboxes) == 4, f'Seed config: custom bbox {bboxes} does not have exactly four elements!' min_x, min_y, max_x, max_y = list(map(float, bboxes)) - assert min_x < max_x, 'Invalid bbox. X: {} not less than {}'.format(min_x, max_x) - assert min_y < max_y, 'Invalid bbox. Y: {} not less than {}'.format(min_y, max_y) + assert min_x < max_x, f'Invalid bbox. X: {min_x} not less than {max_x}' + assert min_y < max_y, f'Invalid bbox. Y: {min_y} not less than {max_y}' generator = BoundingBoxTileCoordinatesGenerator(min_x=min_x, min_y=min_y, max_x=max_x, diff --git a/batch-setup/provision.sh b/batch-setup/provision.sh index 08cd021..bf8bdd2 100644 --- a/batch-setup/provision.sh +++ b/batch-setup/provision.sh @@ -28,36 +28,36 @@ done # a per-branch basis without clobbering the (singleton) file on S3. # cat >/usr/local/etc/py-requirements.txt <15}: {}".format(coord_str, msg['exception'])) trace = msg.get('stacktrace') if trace: print(" " + trace.replace("|", "\n ")) @@ -81,5 +81,5 @@ def print_failures(cwlogs, log_stream_name): name = job['jobName'] last_attempt = job['attempts'][-1] log_stream_name = last_attempt['container']['logStreamName'] - print("=== %s ===\n" % (name,)) + print(f"=== {name} ===\n") print_failures(cwlogs, log_stream_name) diff --git a/utils/meta-high-job-times.py b/utils/meta-high-job-times.py index f8de36f..41f52bc 100644 --- a/utils/meta-high-job-times.py +++ b/utils/meta-high-job-times.py @@ -11,7 +11,7 @@ args = parser.parse_args() job_queue = 'job-queue-' + args.date -prefix = 'meta-batch-%s-' % (args.date,) +prefix = f'meta-batch-{args.date}-' batch = boto3.client('batch') next_token = None diff --git a/utils/test_tiles.py b/utils/test_tiles.py index 5e2c05f..8fc358f 100644 --- a/utils/test_tiles.py +++ b/utils/test_tiles.py @@ -40,8 +40,8 @@ def test_tiles_within_bbox(): 48.760348) res = generator4.generate_tiles_coordinates([10]) - zoomedCoords = set(coord.zoomTo(7).container() for coord in res) + zoomedCoords = {coord.zoomTo(7).container() for coord in res} - assert set([Coordinate(44, 20, 7), Coordinate(44, 21, 7), Coordinate(45, 20, 7), Coordinate(45, 21, 7)]) == \ + assert {Coordinate(44, 20, 7), Coordinate(44, 21, 7), Coordinate(45, 20, 7), Coordinate(45, 21, 7)} == \ zoomedCoords diff --git a/utils/tiles.py b/utils/tiles.py index 4847218..f8fc249 100644 --- a/utils/tiles.py +++ b/utils/tiles.py @@ -5,7 +5,7 @@ from utils.constants import MIN_TILE_ZOOM -class TileCoordinatesGenerator(object): +class TileCoordinatesGenerator: """ Generate tiles EPSG:3857 coordinates from a list of zooms within a range """ @@ -56,7 +56,7 @@ def __init__(self, min_x, min_y, max_x, max_y, min_zoom=MIN_TILE_ZOOM, :param min_zoom: the minimum zoom(inclusive) it can generate tiles for :param max_zoom: the maximum zoom(inclusive) it can generate tiles for """ - super(BoundingBoxTileCoordinatesGenerator, self).__init__(min_zoom=min_zoom, max_zoom=max_zoom) + super().__init__(min_zoom=min_zoom, max_zoom=max_zoom) self.min_x = min_x self.min_y = min_y self.max_x = max_x