diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 2260fc12..37bef528 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -4,20 +4,8 @@ on: push jobs: - flake8: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - - run: | - python -m pip install --upgrade pip - python -m pip install flake8 flake8-import-order flake8-builtins # FIXME add flake8-blind-except - - run: flake8 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names metadata_construction verify + call-ruff-workflow: + uses: ASFHyP3/actions/.github/workflows/reusable-ruff.yml@v0.13.2 cfn-lint: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index e6f05ed9..3460dccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.0.2] +### Changed +- Replaced `flake8` with `ruff`. + ## [2.0.1] ### Changed - Upgraded all Lambda functions to Python 3.12 diff --git a/cmr-token/src/cmr_token.py b/cmr-token/src/cmr_token.py index 4bde5d36..e4b15ac3 100644 --- a/cmr-token/src/cmr_token.py +++ b/cmr-token/src/cmr_token.py @@ -6,6 +6,7 @@ import boto3 import requests_pkcs12 + log = getLogger() log.setLevel('INFO') s3 = boto3.client('s3') diff --git a/ingest/src/ingest.py b/ingest/src/ingest.py index 4d45d03e..f0aa17ab 100755 --- a/ingest/src/ingest.py +++ b/ingest/src/ingest.py @@ -51,7 +51,7 @@ def lambda_handler(event, context): 'Browse': { 'Bucket': config['browse_bucket'], 'Key': browse_output_key, - } + }, } log.info('Done processing %s', event['ProductName']) return output diff --git a/invoke/src/invoke.py b/invoke/src/invoke.py index abec94e2..37f9fb20 100644 --- a/invoke/src/invoke.py +++ b/invoke/src/invoke.py @@ -39,8 +39,9 @@ def invoke_ingest(config): log.warning('Processed %s of %s messages. Exiting.', messages_processed, config['max_messages_to_process']) break - messages = queue.receive_messages(MaxNumberOfMessages=config['max_messages_per_receive'], - WaitTimeSeconds=config['wait_time_in_seconds']) + messages = queue.receive_messages( + MaxNumberOfMessages=config['max_messages_per_receive'], WaitTimeSeconds=config['wait_time_in_seconds'] + ) if not messages: log.info('No messages found. Exiting.') break diff --git a/metadata-construction/src/metadata_construction.py b/metadata-construction/src/metadata_construction.py index 1137e11c..a8ec0e5c 100644 --- a/metadata-construction/src/metadata_construction.py +++ b/metadata-construction/src/metadata_construction.py @@ -5,6 +5,7 @@ import boto3 + log = getLogger() log.setLevel('INFO') CONFIG = json.loads(os.getenv('CONFIG')) @@ -51,7 +52,7 @@ def get_sds_metadata(obj): def format_polygon(polygon): coordinates = [] for long, lat in reversed(polygon): - coordinates.append({"Latitude": lat, "Longitude": long}) + coordinates.append({'Latitude': lat, 'Longitude': long}) return coordinates @@ -111,38 +112,32 @@ def render_granule_metadata(sds_metadata, config, product, browse) -> dict: 'Type': 'Update', }, ], - "DataGranule": { - "ArchiveAndDistributionInformation": [ - { - "Name": os.path.basename(product['Key']), - "SizeInBytes": get_s3_file_size(product) - } + 'DataGranule': { + 'ArchiveAndDistributionInformation': [ + {'Name': os.path.basename(product['Key']), 'SizeInBytes': get_s3_file_size(product)} ], - "DayNightFlag": "Unspecified", - "ProductionDateTime": sds_metadata['creation_timestamp'] + 'DayNightFlag': 'Unspecified', + 'ProductionDateTime': sds_metadata['creation_timestamp'], }, - "Platforms": [ - {"ShortName": platform} for platform in sorted(set(sds_metadata['metadata']['platform'])) - ], - "OrbitCalculatedSpatialDomains": [ - {"OrbitNumber": orbit} for orbit in sds_metadata['metadata']['orbit_number'] + 'Platforms': [{'ShortName': platform} for platform in sorted(set(sds_metadata['metadata']['platform']))], + 'OrbitCalculatedSpatialDomains': [{'OrbitNumber': orbit} for orbit in sds_metadata['metadata']['orbit_number']], + 'InputGranules': sds_metadata['metadata']['reference_scenes'] + sds_metadata['metadata']['secondary_scenes'], + 'AdditionalAttributes': [ + {'Name': 'ASCENDING_DESCENDING', 'Values': [sds_metadata['metadata']['orbit_direction']]}, + {'Name': 'BEAM_MODE', 'Values': [sds_metadata['metadata']['beam_mode']]}, + {'Name': 'POLARIZATION', 'Values': [sds_metadata['metadata']['polarization']]}, + {'Name': 'PERPENDICULAR_BASELINE', 'Values': [str(sds_metadata['metadata']['perpendicular_baseline'])]}, + {'Name': 'VERSION', 'Values': [sds_metadata['metadata']['version']]}, + {'Name': 'FRAME_NUMBER', 'Values': [str(sds_metadata['metadata']['frame_number'])]}, + {'Name': 'PATH_NUMBER', 'Values': [str(sds_metadata['metadata']['track_number'])]}, + {'Name': 'TEMPORAL_BASELINE_DAYS', 'Values': [str(sds_metadata['metadata']['temporal_baseline_days'])]}, ], - "InputGranules": sds_metadata['metadata']['reference_scenes'] + sds_metadata['metadata']['secondary_scenes'], - "AdditionalAttributes": [ - {"Name": "ASCENDING_DESCENDING", "Values": [sds_metadata['metadata']['orbit_direction']]}, - {"Name": "BEAM_MODE", "Values": [sds_metadata['metadata']['beam_mode']]}, - {"Name": "POLARIZATION", "Values": [sds_metadata['metadata']['polarization']]}, - {"Name": "PERPENDICULAR_BASELINE", "Values": [str(sds_metadata['metadata']['perpendicular_baseline'])]}, - {"Name": "VERSION", "Values": [sds_metadata['metadata']['version']]}, - {"Name": "FRAME_NUMBER", "Values": [str(sds_metadata['metadata']['frame_number'])]}, - {"Name": "PATH_NUMBER", "Values": [str(sds_metadata['metadata']['track_number'])]}, - {"Name": "TEMPORAL_BASELINE_DAYS", "Values": [str(sds_metadata['metadata']['temporal_baseline_days'])]} - ] } if 'weather_model' in sds_metadata['metadata']: - umm['AdditionalAttributes'].append({"Name": "WEATHER_MODEL", - "Values": sds_metadata['metadata']['weather_model']}) + umm['AdditionalAttributes'].append( + {'Name': 'WEATHER_MODEL', 'Values': sds_metadata['metadata']['weather_model']} + ) return umm diff --git a/metadata-to-cmr/src/cmr.py b/metadata-to-cmr/src/cmr.py index 92005879..88e9d5b4 100644 --- a/metadata-to-cmr/src/cmr.py +++ b/metadata-to-cmr/src/cmr.py @@ -9,6 +9,7 @@ import boto3 import requests + log = getLogger() diff --git a/metadata-to-cmr/src/daemon.py b/metadata-to-cmr/src/daemon.py index df498846..c335cdaf 100644 --- a/metadata-to-cmr/src/daemon.py +++ b/metadata-to-cmr/src/daemon.py @@ -5,6 +5,7 @@ import boto3 from botocore.client import Config from botocore.exceptions import ClientError + from cmr import get_session, process_task @@ -44,8 +45,11 @@ def daemon_loop(config, get_remaining_time_in_millis_fcn): sfn_client = get_sfn_client(config['sfn_connect_timeout']) while True: if get_remaining_time_in_millis_fcn() < config['max_task_time_in_millis']: - log.info('Remaining time %s less than max task time %s. Exiting.', get_remaining_time_in_millis_fcn(), - config['max_task_time_in_millis']) + log.info( + 'Remaining time %s less than max task time %s. Exiting.', + get_remaining_time_in_millis_fcn(), + config['max_task_time_in_millis'], + ) break task = get_task(sfn_client, config['activity']) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..9717bc2e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[project] +requires-python = "==3.12" + +[tool.ruff] +line-length = 120 +# The directories to consider when resolving first- vs. third-party imports. +# See: https://docs.astral.sh/ruff/settings/#src +src = ["**/src", "tests"] + +[tool.ruff.format] +indent-style = "space" +quote-style = "single" + +[tool.ruff.lint] +extend-select = [ + "I", # isort: https://docs.astral.sh/ruff/rules/#isort-i + "UP", # pyupgrade: https://docs.astral.sh/ruff/rules/#pyupgrade-up + + # TODO: uncomment the following extensions and address their warnings: + #"D", # pydocstyle: https://docs.astral.sh/ruff/rules/#pydocstyle-d + #"ANN", # annotations: https://docs.astral.sh/ruff/rules/#flake8-annotations-ann + #"PTH", # use-pathlib-pth: https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth +] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +case-sensitive = true +lines-after-imports = 2 diff --git a/tests/test_metadata_construction.py b/tests/test_metadata_construction.py index cbcecea0..487b4ffb 100644 --- a/tests/test_metadata_construction.py +++ b/tests/test_metadata_construction.py @@ -19,7 +19,7 @@ def test_get_file_content_from_s3(s3_stubber): s3_stubber.add_response( method='get_object', expected_params={'Bucket': 'myBucket', 'Key': 'myKey'}, - service_response={'Body': io.StringIO('myContent')} + service_response={'Body': io.StringIO('myContent')}, ) assert metadata_construction.get_file_content_from_s3('myBucket', 'myKey') == 'myContent' @@ -32,10 +32,7 @@ def test_write_to_file(tmp_path): def test_get_s3_file_size(s3_stubber): - obj = { - 'Bucket': 'myBucket', - 'Key': 'myKey' - } + obj = {'Bucket': 'myBucket', 'Key': 'myKey'} s3_stubber.add_response(method='head_object', expected_params=obj, service_response={'ContentLength': 123}) assert metadata_construction.get_s3_file_size(obj) == 123 @@ -43,7 +40,7 @@ def test_get_s3_file_size(s3_stubber): def test_get_sds_metadata(test_data_dir, s3_stubber): obj = { 'Bucket': 'ingest-test-aux', - 'Key': 'S1-GUNW-D-R-123-tops-20240212_20240107-032647-00038E_00036N-PP-2e78-v3_0_0' + 'Key': 'S1-GUNW-D-R-123-tops-20240212_20240107-032647-00038E_00036N-PP-2e78-v3_0_0', } sds_metadata_file = test_data_dir / 'granule1' / 'sds_metadata.json' @@ -55,7 +52,7 @@ def test_get_sds_metadata(test_data_dir, s3_stubber): def test_create_granule_metadata_in_s3_g1(test_data_dir, mocker): - sds_metadata =json.loads((test_data_dir / 'granule1'/ 'sds_metadata.json').read_text()) + sds_metadata = json.loads((test_data_dir / 'granule1' / 'sds_metadata.json').read_text()) inputs = json.loads((test_data_dir / 'granule1' / 'inputs.json').read_text()) config = json.loads((test_data_dir / 'granule1' / 'config.json').read_text()) @@ -80,7 +77,7 @@ def test_create_granule_metadata_in_s3_g1(test_data_dir, mocker): def test_create_granule_metadata_in_s3_g2(test_data_dir, mocker): - sds_metadata =json.loads((test_data_dir / 'granule2'/ 'sds_metadata.json').read_text()) + sds_metadata = json.loads((test_data_dir / 'granule2' / 'sds_metadata.json').read_text()) inputs = json.loads((test_data_dir / 'granule2' / 'inputs.json').read_text()) config = json.loads((test_data_dir / 'granule2' / 'config.json').read_text()) diff --git a/tests/test_verify.py b/tests/test_verify.py index 6d449a2c..76fe875d 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -17,7 +17,7 @@ def test_get_file_content_from_s3(s3_stubber): s3_stubber.add_response( method='get_object', expected_params={'Bucket': 'myBucket', 'Key': 'myKey'}, - service_response={'Body': io.StringIO('myContent')} + service_response={'Body': io.StringIO('myContent')}, ) assert verify.get_file_content_from_s3('myBucket', 'myKey') == 'myContent' diff --git a/verify/src/verify.py b/verify/src/verify.py index d5c0852a..b9a2c558 100644 --- a/verify/src/verify.py +++ b/verify/src/verify.py @@ -35,7 +35,7 @@ def get_file_content_from_s3(bucket, key): def get_json_from_file(filename): - with open(filename, 'r') as f: + with open(filename) as f: content = f.read() return json.loads(content)