From d84fd5c0c45dc8503941c9406b0a2fa20b5cffe2 Mon Sep 17 00:00:00 2001 From: Boris Date: Wed, 24 Apr 2024 14:16:08 -0500 Subject: [PATCH] Running black, and flake8, fixing flake8 errors --- .flake8 | 2 + .github/workflows/precommit.yaml | 2 +- .pre-commit-config.yaml | 23 ++- import_specifications/clients/authclient.py | 1 + import_specifications/clients/baseclient.py | 28 ++- .../clients/narrative_method_store_client.py | 65 +++++-- .../generate_import_template.py | 19 +- pyproject.toml | 9 - scripts/prune_acls.py | 6 +- staging_service/AutoDetectUtils.py | 5 +- staging_service/app.py | 12 +- staging_service/auth2Client.py | 1 + .../autodetect/GenerateMappings.py | 13 +- staging_service/autodetect/Mappings.py | 6 +- .../import_specifications/file_parser.py | 6 +- .../import_specifications/file_writers.py | 1 + .../individual_parsers.py | 26 ++- staging_service/utils.py | 4 +- .../import_specifications/test_file_parser.py | 88 +++++++--- .../test_file_writers.py | 54 +++++- .../test_individual_parsers.py | 165 +++++++++++++++--- tests/test_app.py | 28 ++- tests/test_app_error_formatter.py | 27 ++- tests/test_autodetect.py | 57 ++++-- tests/test_metadata.py | 10 +- tests/test_utils.py | 9 +- 26 files changed, 537 insertions(+), 130 deletions(-) create mode 100644 .flake8 delete mode 100644 pyproject.toml diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..a08b96a9 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +per-file-ignores = tests/test_app.py:F841 diff --git a/.github/workflows/precommit.yaml b/.github/workflows/precommit.yaml index 17cbea51..e9b30bb4 100644 --- a/.github/workflows/precommit.yaml +++ b/.github/workflows/precommit.yaml @@ -17,4 +17,4 @@ repos: rev: 6.1.0 hooks: - id: flake8 - args: [ --config, pyproject.toml ] + args: [ --config, .flake8 ] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 84e26733..81a30df3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,6 @@ repos: - id: end-of-file-fixer files: '\.py$' - id: check-yaml - # Keep this unchanged for YAML files - id: check-added-large-files files: '\.py$' @@ -15,14 +14,30 @@ repos: rev: 24.3.0 hooks: - id: black - language_version: python3.12 + language_version: python3.9 files: '\.py$' + args: + - --line-length=120 + - --exclude=staging_service/auth2Client.py + - --exclude=import_specifications/clients/baseclient\.py - repo: https://github.com/pycqa/flake8 rev: 7.0.0 hooks: - id: flake8 args: - - --config=server/pyproject.toml - - --ignore=E203 # Ignore E203 directly in the pre-commit configuration + - --ignore=E203,W503 + - --max-line-length=120 + - --config=.flake8 files: '\.py$' + additional_dependencies: [ flake8 ] + + +# - repo: https://github.com/astral-sh/ruff-pre-commit +# # Ruff version. +# rev: v0.4.1 +# hooks: +# # Run the linter. +# - id: ruff +# # Run the formatter. +# - id: ruff-format \ No newline at end of file diff --git a/import_specifications/clients/authclient.py b/import_specifications/clients/authclient.py index f2eb4de3..a7a354bb 100644 --- a/import_specifications/clients/authclient.py +++ b/import_specifications/clients/authclient.py @@ -5,6 +5,7 @@ @author: gaprice@lbl.gov """ + import hashlib import threading as _threading import time as _time diff --git a/import_specifications/clients/baseclient.py b/import_specifications/clients/baseclient.py index 0d551ce4..a79e4960 100644 --- a/import_specifications/clients/baseclient.py +++ b/import_specifications/clients/baseclient.py @@ -54,7 +54,7 @@ def _get_token(user_id, password, auth_svc): def _read_inifile( - file=_os.environ.get("KB_DEPLOYMENT_CONFIG", _os.environ["HOME"] + "/.kbase_config") + file=_os.environ.get("KB_DEPLOYMENT_CONFIG", _os.environ["HOME"] + "/.kbase_config"), ): # @ReservedAssignment # Another bandaid to read in the ~/.kbase_config file if one is present authdata = None @@ -65,7 +65,14 @@ def _read_inifile( # strip down whatever we read to only what is legit authdata = { x: config.get("authentication", x) if config.has_option("authentication", x) else None - for x in ("user_id", "token", "client_secret", "keyfile", "keyfile_passphrase", "password") + for x in ( + "user_id", + "token", + "client_secret", + "keyfile", + "keyfile_passphrase", + "password", + ) } except Exception as e: print("Error while reading INI file {}: {}".format(file, e)) @@ -165,7 +172,12 @@ def __init__( raise ValueError("Timeout value must be at least 1 second") def _call(self, url, method, params, context=None): - arg_hash = {"method": method, "params": params, "version": "1.1", "id": str(_random.random())[2:]} + arg_hash = { + "method": method, + "params": params, + "version": "1.1", + "id": str(_random.random())[2:], + } if context: if type(context) is not dict: raise ValueError("context is not type dict as required.") @@ -173,7 +185,11 @@ def _call(self, url, method, params, context=None): body = _json.dumps(arg_hash, cls=_JSONObjectEncoder) ret = _requests.post( - url, data=body, headers=self._headers, timeout=self.timeout, verify=not self.trust_all_ssl_certificates + url, + data=body, + headers=self._headers, + timeout=self.timeout, + verify=not self.trust_all_ssl_certificates, ) ret.encoding = "utf-8" if ret.status_code == 500: @@ -201,7 +217,9 @@ def _get_service_url(self, service_method, service_version): return self.url service, _ = service_method.split(".") service_status_ret = self._call( - self.url, "ServiceWizard.get_service_status", [{"module_name": service, "version": service_version}] + self.url, + "ServiceWizard.get_service_status", + [{"module_name": service, "version": service_version}], ) return service_status_ret["url"] diff --git a/import_specifications/clients/narrative_method_store_client.py b/import_specifications/clients/narrative_method_store_client.py index a2d00b03..0a131cd6 100644 --- a/import_specifications/clients/narrative_method_store_client.py +++ b/import_specifications/clients/narrative_method_store_client.py @@ -192,7 +192,10 @@ def list_methods_full_info(self, params, context=None): String, parameter "link" of type "url" """ return self._client.call_method( - "NarrativeMethodStore.list_methods_full_info", [params], self._service_ver, context + "NarrativeMethodStore.list_methods_full_info", + [params], + self._service_ver, + context, ) def list_methods_spec(self, params, context=None): @@ -541,7 +544,12 @@ def list_methods_spec(self, params, context=None): "target_property" of String, parameter "target_type_transform" of String, parameter "job_id_output_field" of String """ - return self._client.call_method("NarrativeMethodStore.list_methods_spec", [params], self._service_ver, context) + return self._client.call_method( + "NarrativeMethodStore.list_methods_spec", + [params], + self._service_ver, + context, + ) def list_method_ids_and_names(self, params, context=None): """ @@ -551,7 +559,10 @@ def list_method_ids_and_names(self, params, context=None): :returns: instance of mapping from String to String """ return self._client.call_method( - "NarrativeMethodStore.list_method_ids_and_names", [params], self._service_ver, context + "NarrativeMethodStore.list_method_ids_and_names", + [params], + self._service_ver, + context, ) def list_apps(self, params, context=None): @@ -598,7 +609,10 @@ def list_apps_full_info(self, params, context=None): parameter "url" of type "url" """ return self._client.call_method( - "NarrativeMethodStore.list_apps_full_info", [params], self._service_ver, context + "NarrativeMethodStore.list_apps_full_info", + [params], + self._service_ver, + context, ) def list_apps_spec(self, params, context=None): @@ -643,7 +657,12 @@ def list_app_ids_and_names(self, context=None): """ :returns: instance of mapping from String to String """ - return self._client.call_method("NarrativeMethodStore.list_app_ids_and_names", [], self._service_ver, context) + return self._client.call_method( + "NarrativeMethodStore.list_app_ids_and_names", + [], + self._service_ver, + context, + ) def list_types(self, params, context=None): """ @@ -692,7 +711,10 @@ def get_method_brief_info(self, params, context=None): parameter "app_type" of String """ return self._client.call_method( - "NarrativeMethodStore.get_method_brief_info", [params], self._service_ver, context + "NarrativeMethodStore.get_method_brief_info", + [params], + self._service_ver, + context, ) def get_method_full_info(self, params, context=None): @@ -731,7 +753,10 @@ def get_method_full_info(self, params, context=None): String, parameter "link" of type "url" """ return self._client.call_method( - "NarrativeMethodStore.get_method_full_info", [params], self._service_ver, context + "NarrativeMethodStore.get_method_full_info", + [params], + self._service_ver, + context, ) def get_method_spec(self, params, context=None): @@ -1091,7 +1116,12 @@ def get_app_brief_info(self, params, context=None): parameter "categories" of list of String, parameter "loading_error" of String """ - return self._client.call_method("NarrativeMethodStore.get_app_brief_info", [params], self._service_ver, context) + return self._client.call_method( + "NarrativeMethodStore.get_app_brief_info", + [params], + self._service_ver, + context, + ) def get_app_full_info(self, params, context=None): """ @@ -1112,7 +1142,12 @@ def get_app_full_info(self, params, context=None): parameter "screenshots" of list of type "ScreenShot" -> structure: parameter "url" of type "url" """ - return self._client.call_method("NarrativeMethodStore.get_app_full_info", [params], self._service_ver, context) + return self._client.call_method( + "NarrativeMethodStore.get_app_full_info", + [params], + self._service_ver, + context, + ) def get_app_spec(self, params, context=None): """ @@ -2463,7 +2498,10 @@ def load_widget_java_script(self, params, context=None): :returns: instance of String """ return self._client.call_method( - "NarrativeMethodStore.load_widget_java_script", [params], self._service_ver, context + "NarrativeMethodStore.load_widget_java_script", + [params], + self._service_ver, + context, ) def register_repo(self, params, context=None): @@ -2495,4 +2533,9 @@ def push_repo_to_tag(self, params, context=None): two values: 'beta' or 'release'.) -> structure: parameter "module_name" of String, parameter "tag" of String """ - return self._client.call_method("NarrativeMethodStore.push_repo_to_tag", [params], self._service_ver, context) + return self._client.call_method( + "NarrativeMethodStore.push_repo_to_tag", + [params], + self._service_ver, + context, + ) diff --git a/import_specifications/generate_import_template.py b/import_specifications/generate_import_template.py index caac72eb..37120b82 100755 --- a/import_specifications/generate_import_template.py +++ b/import_specifications/generate_import_template.py @@ -34,21 +34,30 @@ def parse_args(): parser = argparse.ArgumentParser(description="Generate a bulk import template for an app") parser.add_argument( - "app_id", help="The app ID to process, for example kb_uploadmethods/import_sra_as_reads_from_staging" + "app_id", + help="The app ID to process, for example kb_uploadmethods/import_sra_as_reads_from_staging", ) parser.add_argument( "data_type", - help="The datatype corresponding to the the app. This id is shared between the " - + "staging service and the narrative, for example sra_reads", + help="The datatype corresponding to the the app." + + "This id is shared between the staging service and the narrative, for example sra_reads", + ) + parser.add_argument( + "--tsv", + action="store_true", + help="Create a TSV file rather than a CSV file (the default)", ) - parser.add_argument("--tsv", action="store_true", help="Create a TSV file rather than a CSV file (the default)") parser.add_argument( "--env", choices=["prod", "appdev", "next", "ci"], default="prod", help="The KBase environment to query, default prod", ) - parser.add_argument("--print-spec", action="store_true", help="Print the input specification for the app to stderr") + parser.add_argument( + "--print-spec", + action="store_true", + help="Print the input specification for the app to stderr", + ) return parser.parse_args() diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index cd4cf93c..00000000 --- a/pyproject.toml +++ /dev/null @@ -1,9 +0,0 @@ -[tool.black] -line_length = 120 -multi_line_output = 3 -extend-exclude = '''Pipfile.lock''' - - -[flake8] -max-line-length = 120 -exclude = '''Pipfile.lock''' diff --git a/scripts/prune_acls.py b/scripts/prune_acls.py index 5bf4b42d..311e0f0b 100644 --- a/scripts/prune_acls.py +++ b/scripts/prune_acls.py @@ -3,6 +3,7 @@ """ Deletes ACLS from globus, and then clears out directories older than THRESHOLD (60) days """ + from __future__ import print_function # for python 2 import configparser @@ -20,7 +21,10 @@ current_time = time.time() THRESHOLD_DAYS = 60 -admin_acls = ["9cb619d0-4417-11e8-8e06-0a6d4e044368", "580118b2-dc53-11e6-9d02-22000a1e3b52"] +admin_acls = [ + "9cb619d0-4417-11e8-8e06-0a6d4e044368", + "580118b2-dc53-11e6-9d02-22000a1e3b52", +] admin_names = ["dolsonadmin", "dolson"] config = configparser.ConfigParser() diff --git a/staging_service/AutoDetectUtils.py b/staging_service/AutoDetectUtils.py index 98556da0..6047df69 100644 --- a/staging_service/AutoDetectUtils.py +++ b/staging_service/AutoDetectUtils.py @@ -2,6 +2,7 @@ This class is in charge of determining possible importers by determining the suffix of the filepath pulled in, and by looking up the appropriate mappings in the supported_apps_w_extensions.json file """ + from typing import Optional, Tuple, Dict @@ -9,7 +10,9 @@ class AutoDetectUtils: _MAPPINGS = None # expects to be set by config @staticmethod - def determine_possible_importers(filename: str) -> Tuple[Optional[list], Dict[str, object]]: + def determine_possible_importers( + filename: str, + ) -> Tuple[Optional[list], Dict[str, object]]: """ Given a filename, come up with a reference to all possible apps. :param filename: The filename to find applicable apps for diff --git a/staging_service/app.py b/staging_service/app.py index 122a5b85..b75ff5df 100644 --- a/staging_service/app.py +++ b/staging_service/app.py @@ -168,11 +168,13 @@ async def write_bulk_specification(request: web.Request) -> web.json_response: if request.content_type != _APP_JSON: # There should be a way to get aiohttp to handle this but I can't find it return _createJSONErrorResponse( - f"Required content-type is {_APP_JSON}", error_class=web.HTTPUnsupportedMediaType + f"Required content-type is {_APP_JSON}", + error_class=web.HTTPUnsupportedMediaType, ) if not request.content_length: return _createJSONErrorResponse( - "The content-length header is required and must be > 0", error_class=web.HTTPLengthRequired + "The content-length header is required and must be > 0", + error_class=web.HTTPLengthRequired, ) # No need to check the max content length; the server already does that. See tests data = await request.json() @@ -208,9 +210,9 @@ async def add_acl_concierge(request: web.Request): aclm = AclManager() result = aclm.add_acl_concierge(shared_directory=user_dir, concierge_path=concierge_path) result["msg"] = f"Requesting Globus Perms for the following globus dir: {concierge_path}" - result[ - "link" - ] = f"https://app.globus.org/file-manager?destination_id={aclm.endpoint_id}&destination_path={concierge_path}" + result["link"] = ( + f"https://app.globus.org/file-manager?destination_id={aclm.endpoint_id}&destination_path={concierge_path}" + ) return web.json_response(result) diff --git a/staging_service/auth2Client.py b/staging_service/auth2Client.py index 6d2978fa..414ba946 100644 --- a/staging_service/auth2Client.py +++ b/staging_service/auth2Client.py @@ -4,6 +4,7 @@ @author: gaprice@lbl.gov modified for python3 and authV2 """ + import hashlib import time as _time diff --git a/staging_service/autodetect/GenerateMappings.py b/staging_service/autodetect/GenerateMappings.py index c5954088..c660919e 100644 --- a/staging_service/autodetect/GenerateMappings.py +++ b/staging_service/autodetect/GenerateMappings.py @@ -22,6 +22,7 @@ * Note: We should serve the generated content from memory * Note: This doesn't handle if we want to have different output types based on file extensions feeding into the same app """ + from collections import defaultdict from staging_service.autodetect.Mappings import ( @@ -85,7 +86,10 @@ file_format_to_app_mapping = {} file_format_to_app_mapping[SRA] = [sra_reads_id] -file_format_to_app_mapping[FASTQ] = [fastq_reads_interleaved_id, fastq_reads_noninterleaved_id] +file_format_to_app_mapping[FASTQ] = [ + fastq_reads_interleaved_id, + fastq_reads_noninterleaved_id, +] file_format_to_app_mapping[FASTA] = [assembly_id, gff_genome_id, gff_metagenome_id] file_format_to_app_mapping[GENBANK] = [genbank_genome_id] file_format_to_app_mapping[GFF] = [gff_genome_id, gff_metagenome_id] @@ -100,7 +104,12 @@ phenotype_set_id, import_specification, ] -file_format_to_app_mapping[EXCEL] = [sample_set_id, media_id, fba_model_id, import_specification] +file_format_to_app_mapping[EXCEL] = [ + sample_set_id, + media_id, + fba_model_id, + import_specification, +] file_format_to_app_mapping[JSON] = [escher_map_id] file_format_to_app_mapping[SBML] = [fba_model_id] diff --git a/staging_service/autodetect/Mappings.py b/staging_service/autodetect/Mappings.py index 3a227fa3..51d27091 100644 --- a/staging_service/autodetect/Mappings.py +++ b/staging_service/autodetect/Mappings.py @@ -60,7 +60,11 @@ def _flatten(some_list): return list(itertools.chain.from_iterable(some_list)) -_COMPRESSION_EXT = ["", ".gz", ".gzip"] # empty string to keep the uncompressed extension +_COMPRESSION_EXT = [ + "", + ".gz", + ".gzip", +] # empty string to keep the uncompressed extension # longer term there's probably a better way to do this but this is quick diff --git a/staging_service/import_specifications/file_parser.py b/staging_service/import_specifications/file_parser.py index 8e7c6fd8..5cf8d214 100644 --- a/staging_service/import_specifications/file_parser.py +++ b/staging_service/import_specifications/file_parser.py @@ -56,7 +56,11 @@ def __post_init__(self): ErrorType.FILE_NOT_FOUND: (_ERR_SOURCE_1,), ErrorType.PARSE_FAIL: (_ERR_MESSAGE, _ERR_SOURCE_1), ErrorType.INCORRECT_COLUMN_COUNT: (_ERR_MESSAGE, _ERR_SOURCE_1), - ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE: (_ERR_MESSAGE, _ERR_SOURCE_1, _ERR_SOURCE_2), + ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE: ( + _ERR_MESSAGE, + _ERR_SOURCE_1, + _ERR_SOURCE_2, + ), ErrorType.NO_FILES_PROVIDED: tuple(), ErrorType.OTHER: (_ERR_MESSAGE,), } diff --git a/staging_service/import_specifications/file_writers.py b/staging_service/import_specifications/file_writers.py index ced1689a..45e37047 100644 --- a/staging_service/import_specifications/file_writers.py +++ b/staging_service/import_specifications/file_writers.py @@ -25,6 +25,7 @@ Leave the `data` list empty to write an empty template. :returns: A mapping of the data types to the files to which they were written. """ + # note that we can't use an f string here to interpolate the variables below, e.g. # order_and_display, etc. diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py index 7c36d928..3793d549 100644 --- a/staging_service/import_specifications/individual_parsers.py +++ b/staging_service/import_specifications/individual_parsers.py @@ -68,7 +68,11 @@ def _parse_header(header: str, spec_source: SpecificationSource, maximum_version match = _HEADER_REGEX.fullmatch(header) if not match: raise _ParseException( - Error(ErrorType.PARSE_FAIL, f'Invalid header; got "{header}", expected "{_EXPECTED_HEADER}"', spec_source) + Error( + ErrorType.PARSE_FAIL, + f'Invalid header; got "{header}", expected "{_EXPECTED_HEADER}"', + spec_source, + ) ) version = int(match[3]) if version > maximum_version: @@ -144,12 +148,20 @@ def _normalize_headers(headers: list[Any], line_number: int, spec_source: Specif for i, name in enumerate(ret, start=1): if not name: raise _ParseException( - Error(ErrorType.PARSE_FAIL, f"Missing header entry in row {line_number}, position {i}", spec_source) + Error( + ErrorType.PARSE_FAIL, + f"Missing header entry in row {line_number}, position {i}", + spec_source, + ) ) if name in seen: raise _ParseException( - Error(ErrorType.PARSE_FAIL, f"Duplicate header name in row {line_number}: {name}", spec_source) + Error( + ErrorType.PARSE_FAIL, + f"Duplicate header name in row {line_number}: {name}", + spec_source, + ) ) seen.add(name) return ret @@ -275,7 +287,13 @@ def parse_excel(path: Path) -> ParseResults: return _error(Error(ErrorType.PARSE_FAIL, "The given path is a directory", spcsrc)) except ValueError as e: if "Excel file format cannot be determined" in str(e): - return _error(Error(ErrorType.PARSE_FAIL, "Not a supported Excel file type", source_1=spcsrc)) + return _error( + Error( + ErrorType.PARSE_FAIL, + "Not a supported Excel file type", + source_1=spcsrc, + ) + ) raise e # bail out, not sure what's wrong, not sure how to test either if errors: return ParseResults(errors=tuple(errors)) diff --git a/staging_service/utils.py b/staging_service/utils.py index d0e4ebb4..201b4b23 100644 --- a/staging_service/utils.py +++ b/staging_service/utils.py @@ -151,7 +151,7 @@ def _add_acl(self, user_identity_id: str, shared_directory_basename: str): Attempt to add acl for the given user id and directory """ try: - resp = self.globus_transfer_client.add_endpoint_acl_rule( + self.globus_transfer_client.add_endpoint_acl_rule( self.endpoint_id, dict( DATA_TYPE="access", @@ -214,7 +214,7 @@ def _remove_acl(self, user_identity_id: str): } raise HTTPInternalServerError(text=json.dumps(response), content_type="application/json") - except globus_sdk.GlobusAPIError as error: + except globus_sdk.GlobusAPIError: response = { "success": False, "error_type": "GlobusAPIError", diff --git a/tests/import_specifications/test_file_parser.py b/tests/import_specifications/test_file_parser.py index cead1dae..a022b316 100644 --- a/tests/import_specifications/test_file_parser.py +++ b/tests/import_specifications/test_file_parser.py @@ -2,7 +2,7 @@ from collections.abc import Callable from pathlib import Path -from typing import Optional as O +from typing import Optional as Opt from unittest.mock import Mock, call from frozendict import frozendict @@ -21,7 +21,7 @@ from tests.test_utils import assert_exception_correct -def spcsrc(path: str, tab: O[str] = None): +def spcsrc(path: str, tab: Opt[str] = None): return SpecificationSource(Path(path), tab) @@ -44,17 +44,19 @@ def test_SpecificationSource_init_fail(): specificationSource_init_fail(None, ValueError("file is required")) -def specificationSource_init_fail(file_: O[str], expected: Exception): +def specificationSource_init_fail(file_: Opt[str], expected: Exception): with raises(Exception) as got: SpecificationSource(file_) assert_exception_correct(got.value, expected) def test_FileTypeResolution_init_w_parser_success(): - p = lambda path: ParseResults(errors=(Error(ErrorType.OTHER, "foo"),)) + def p(path): + return ParseResults(errors=(Error(ErrorType.OTHER, "foo"),)) + ftr = FileTypeResolution(p) - assert ftr.parser is p # Here only identity equality makes sense + assert ftr.parser is p # Check for identity equality assert ftr.unsupported_type is None @@ -73,7 +75,9 @@ def test_FileTypeResolution_init_fail(): def fileTypeResolution_init_fail( - parser: O[Callable[[Path], ParseResults]], unexpected_type: O[str], expected: Exception + parser: Opt[Callable[[Path], ParseResults]], + unexpected_type: Opt[str], + expected: Exception, ): with raises(Exception) as got: FileTypeResolution(parser, unexpected_type) @@ -117,7 +121,12 @@ def test_Error_init_w_INCORRECT_COLUMN_COUNT_success(): def test_Error_init_w_MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE_success(): - e = Error(ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE, "foo", spcsrc("foo2"), spcsrc("yay")) + e = Error( + ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE, + "foo", + spcsrc("foo2"), + spcsrc("yay"), + ) assert e.error == ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE assert e.message == "foo" @@ -156,7 +165,11 @@ def test_Error_init_fail(): # arguments are error type, message string, 1st source, 2nd source, exception error_init_fail(None, None, None, None, ValueError("error is required")) error_init_fail( - ErrorType.FILE_NOT_FOUND, None, None, None, ValueError("source_1 is required for a FILE_NOT_FOUND error") + ErrorType.FILE_NOT_FOUND, + None, + None, + None, + ValueError("source_1 is required for a FILE_NOT_FOUND error"), ) err = "message, source_1 is required for a PARSE_FAIL error" error_init_fail(ErrorType.PARSE_FAIL, None, spcsrc("wooo"), None, ValueError(err)) @@ -170,14 +183,20 @@ def test_Error_init_fail(): error_init_fail(ms, None, spcsrc("foo"), spcsrc("bar"), ValueError(err)) error_init_fail(ms, "msg", None, spcsrc("bar"), ValueError(err)) error_init_fail(ms, "msg", spcsrc("foo"), None, ValueError(err)) - error_init_fail(ErrorType.OTHER, None, None, None, ValueError("message is required for a OTHER error")) + error_init_fail( + ErrorType.OTHER, + None, + None, + None, + ValueError("message is required for a OTHER error"), + ) def error_init_fail( - errortype: O[ErrorType], - message: O[str], - source_1: O[SpecificationSource], - source_2: O[SpecificationSource], + errortype: Opt[ErrorType], + message: Opt[str], + source_1: Opt[SpecificationSource], + source_2: Opt[SpecificationSource], expected: Exception, ): with raises(Exception) as got: @@ -199,7 +218,9 @@ def test_ParseResult_init_fail(): def parseResult_init_fail( - source: O[SpecificationSource], result: O[tuple[frozendict[str, PRIMITIVE_TYPE], ...]], expected: Exception + source: Opt[SpecificationSource], + result: Opt[tuple[frozendict[str, PRIMITIVE_TYPE], ...]], + expected: Exception, ): with raises(Exception) as got: ParseResult(source, result) @@ -207,7 +228,12 @@ def parseResult_init_fail( PR_RESULTS = frozendict( - {"data_type": ParseResult(spcsrc("some_file", "tab"), (frozendict({"fasta_file": "foo.fa", "do_thing": 1}),))} + { + "data_type": ParseResult( + spcsrc("some_file", "tab"), + (frozendict({"fasta_file": "foo.fa", "do_thing": 1}),), + ) + } ) # make a tuple! PR_ERROR = ( @@ -242,7 +268,11 @@ def test_ParseResults_init_fail(): parseResults_init_fail(PR_RESULTS, PR_ERROR, ValueError(err)) -def parseResults_init_fail(results: O[frozendict[str, ParseResult]], errors: O[tuple[Error, ...]], expected: Exception): +def parseResults_init_fail( + results: Opt[frozendict[str, ParseResult]], + errors: Opt[tuple[Error, ...]], + expected: Exception, +): with raises(Exception) as got: ParseResults(results, errors) assert_exception_correct(got.value, expected) @@ -265,7 +295,8 @@ def test_parse_import_specifications_success(): frozendict( { "type1": ParseResult( - spcsrc("myfile.xlsx", "tab1"), (frozendict({"foo": "bar"}), frozendict({"baz": "bat"})) + spcsrc("myfile.xlsx", "tab1"), + (frozendict({"foo": "bar"}), frozendict({"baz": "bat"})), ), "type2": ParseResult(spcsrc("myfile.xlsx", "tab2"), (frozendict({"whee": "whoo"}),)), # tuple! } @@ -276,7 +307,8 @@ def test_parse_import_specifications_success(): frozendict( { "type_other": ParseResult( - spcsrc("somefile.csv"), (frozendict({"foo": "bar2"}), frozendict({"baz": "bat2"})) + spcsrc("somefile.csv"), + (frozendict({"foo": "bar2"}), frozendict({"baz": "bat2"})), ) } ) @@ -288,11 +320,13 @@ def test_parse_import_specifications_success(): frozendict( { "type1": ParseResult( - spcsrc("myfile.xlsx", "tab1"), (frozendict({"foo": "bar"}), frozendict({"baz": "bat"})) + spcsrc("myfile.xlsx", "tab1"), + (frozendict({"foo": "bar"}), frozendict({"baz": "bat"})), ), "type2": ParseResult(spcsrc("myfile.xlsx", "tab2"), (frozendict({"whee": "whoo"}),)), # tuple! "type_other": ParseResult( - spcsrc("somefile.csv"), (frozendict({"foo": "bar2"}), frozendict({"baz": "bat2"})) + spcsrc("somefile.csv"), + (frozendict({"foo": "bar2"}), frozendict({"baz": "bat2"})), ), } ) @@ -348,7 +382,12 @@ def test_parse_import_specification_unsupported_type_and_parser_error(): # check that other errors are also returned, and the results are ignored parser1.return_value = ParseResults( - errors=tuple([Error(ErrorType.OTHER, "foo"), Error(ErrorType.FILE_NOT_FOUND, source_1=spcsrc("foo.csv"))]) + errors=tuple( + [ + Error(ErrorType.OTHER, "foo"), + Error(ErrorType.FILE_NOT_FOUND, source_1=spcsrc("foo.csv")), + ] + ) ) parser2.return_value = ParseResults(frozendict({"foo": ParseResult(spcsrc("a"), tuple([frozendict({"a": "b"})]))})) @@ -396,7 +435,12 @@ def test_parse_import_specification_multiple_specs_and_parser_error(): # check that other errors are also returned, and the results are ignored parser1.return_value = ParseResults( - errors=tuple([Error(ErrorType.OTHER, "other"), Error(ErrorType.FILE_NOT_FOUND, source_1=spcsrc("myfile.xlsx"))]) + errors=tuple( + [ + Error(ErrorType.OTHER, "other"), + Error(ErrorType.FILE_NOT_FOUND, source_1=spcsrc("myfile.xlsx")), + ] + ) ) parser2.return_value = ParseResults( frozendict( diff --git a/tests/import_specifications/test_file_writers.py b/tests/import_specifications/test_file_writers.py index c212f15a..fe024653 100644 --- a/tests/import_specifications/test_file_writers.py +++ b/tests/import_specifications/test_file_writers.py @@ -189,22 +189,38 @@ def test_file_writers_fail(): file_writers_fail(None, {}, ValueError("The folder cannot be null")) file_writers_fail(p, None, E("The types value must be a mapping")) file_writers_fail(p, {}, E("At least one data type must be specified")) - file_writers_fail(p, {None: 1}, E("A data type cannot be a non-string or a whitespace only string")) - file_writers_fail(p, {" \t ": 1}, E("A data type cannot be a non-string or a whitespace only string")) + file_writers_fail( + p, + {None: 1}, + E("A data type cannot be a non-string or a whitespace only string"), + ) + file_writers_fail( + p, + {" \t ": 1}, + E("A data type cannot be a non-string or a whitespace only string"), + ) file_writers_fail(p, {"t": []}, E("The value for data type t must be a mapping")) file_writers_fail(p, {"t": 1}, E("The value for data type t must be a mapping")) file_writers_fail(p, {"t": {}}, E("Data type t missing order_and_display key")) file_writers_fail( - p, {"t": {"order_and_display": {}, "data": []}}, E("Data type t order_and_display value is not a list") + p, + {"t": {"order_and_display": {}, "data": []}}, + E("Data type t order_and_display value is not a list"), ) file_writers_fail( p, {"t": {"order_and_display": [], "data": []}}, E("At least one entry is required for order_and_display for type t"), ) - file_writers_fail(p, {"t": {"order_and_display": [["foo", "bar"]]}}, E("Data type t missing data key")) file_writers_fail( - p, {"t": {"order_and_display": [["foo", "bar"]], "data": "foo"}}, E("Data type t data value is not a list") + p, + {"t": {"order_and_display": [["foo", "bar"]]}}, + E("Data type t missing data key"), + ) + file_writers_fail( + p, + {"t": {"order_and_display": [["foo", "bar"]], "data": "foo"}}, + E("Data type t data value is not a list"), ) file_writers_fail( p, @@ -213,12 +229,22 @@ def test_file_writers_fail(): ) file_writers_fail( p, - {"t": {"order_and_display": [("foo", "bar"), ["whee", "whoo"], ["baz"]], "data": []}}, + { + "t": { + "order_and_display": [("foo", "bar"), ["whee", "whoo"], ["baz"]], + "data": [], + } + }, E("Invalid order_and_display entry for datatype t at index 2 - " + "expected 2 item list"), ) file_writers_fail( p, - {"t": {"order_and_display": [("foo", "bar", "whee"), ["whee", "whoo"]], "data": []}}, + { + "t": { + "order_and_display": [("foo", "bar", "whee"), ["whee", "whoo"]], + "data": [], + } + }, E("Invalid order_and_display entry for datatype t at index 0 - " + "expected 2 item list"), ) for parm in [None, " \t ", 1]: @@ -240,7 +266,12 @@ def test_file_writers_fail(): ) file_writers_fail( p, - {"t": {"order_and_display": [("bar", "foo"), ["whee", "whoo"]], "data": ["foo"]}}, + { + "t": { + "order_and_display": [("bar", "foo"), ["whee", "whoo"]], + "data": ["foo"], + } + }, E("Data type t data row 0 is not a mapping"), ) file_writers_fail( @@ -250,7 +281,12 @@ def test_file_writers_fail(): ) file_writers_fail( p, - {"t": {"order_and_display": [("foo", "bar"), ["whee", "whoo"]], "data": [{"foo": 1, "whee": 2}, {"foo": 2}]}}, + { + "t": { + "order_and_display": [("foo", "bar"), ["whee", "whoo"]], + "data": [{"foo": 1, "whee": 2}, {"foo": 2}], + } + }, E("Data type t data row 1 does not have the same keys as order_and_display"), ) file_writers_fail( diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py index fda77fb9..fb124d22 100644 --- a/tests/import_specifications/test_individual_parsers.py +++ b/tests/import_specifications/test_individual_parsers.py @@ -70,10 +70,38 @@ def _xsv_parse_success(temp_dir: Path, sep: str, parser: Callable[[Path], ParseR SpecificationSource(input_), tuple( [ - frozendict({"spec1": "val1", "spec2": "val2", "spec3": 7, "spec4": 3.2}), - frozendict({"spec1": "val3", "spec2": "val4", "spec3": 1, "spec4": 8.9}), - frozendict({"spec1": "val5", "spec2": None, "spec3": None, "spec4": 42.42}), - frozendict({"spec1": "val6", "spec2": None, "spec3": None, "spec4": 3.14}), + frozendict( + { + "spec1": "val1", + "spec2": "val2", + "spec3": 7, + "spec4": 3.2, + } + ), + frozendict( + { + "spec1": "val3", + "spec2": "val4", + "spec3": 1, + "spec4": 8.9, + } + ), + frozendict( + { + "spec1": "val5", + "spec2": None, + "spec3": None, + "spec4": 42.42, + } + ), + frozendict( + { + "spec1": "val6", + "spec2": None, + "spec3": None, + "spec4": 3.14, + } + ), ] ), ) @@ -115,9 +143,30 @@ def _xsv_parse_success_nan_inf(temp_dir: Path, sep: str, parser: Callable[[Path] SpecificationSource(input_), tuple( [ - frozendict({"spec1": "inf", "-Inf": "val2", "nan": "NaN", "inf": 3.2}), - frozendict({"spec1": "Inf", "-Inf": "val4", "nan": "-inf", "inf": 8.9}), - frozendict({"spec1": "val5", "-Inf": "-Inf", "nan": None, "inf": "nan"}), + frozendict( + { + "spec1": "inf", + "-Inf": "val2", + "nan": "NaN", + "inf": 3.2, + } + ), + frozendict( + { + "spec1": "Inf", + "-Inf": "val4", + "nan": "-inf", + "inf": 8.9, + } + ), + frozendict( + { + "spec1": "val5", + "-Inf": "-Inf", + "nan": None, + "inf": "nan", + } + ), ] ), ) @@ -204,8 +253,22 @@ def _xsv_parse_success_with_internal_and_trailing_empty_lines( SpecificationSource(input_), tuple( [ - frozendict({"spec1": "val3", "spec2": "val4", "spec3": 1, "spec4": 8.9}), - frozendict({"spec1": "val1", "spec2": "val2", "spec3": 7, "spec4": 3.2}), + frozendict( + { + "spec1": "val3", + "spec2": "val4", + "spec3": 1, + "spec4": 8.9, + } + ), + frozendict( + { + "spec1": "val1", + "spec2": "val2", + "spec3": 7, + "spec4": 3.2, + } + ), ] ), ) @@ -257,7 +320,15 @@ def test_xsv_parse_fail_directory(temp_dir: Path): res = parse_tsv(test_file) assert res == ParseResults( - errors=tuple([Error(ErrorType.PARSE_FAIL, "The given path is a directory", SpecificationSource(test_file))]) + errors=tuple( + [ + Error( + ErrorType.PARSE_FAIL, + "The given path is a directory", + SpecificationSource(test_file), + ) + ] + ) ) @@ -434,10 +505,12 @@ def test_excel_parse_success(): ), ), "type2": ParseResult( - SpecificationSource(ex, "tab2"), (frozendict({"h1": "golly gee", "2": 42, "h3": "super"}),) + SpecificationSource(ex, "tab2"), + (frozendict({"h1": "golly gee", "2": 42, "h3": "super"}),), ), "type3": ParseResult( - SpecificationSource(ex, "tab3"), (frozendict({"head1": "some data", "head2": 1}),) + SpecificationSource(ex, "tab3"), + (frozendict({"head1": "some data", "head2": 1}),), ), } ) @@ -503,7 +576,15 @@ def _excel_parse_fail(test_file: str, message: str = None, errors: list[Error] = assert res == ParseResults(errors=tuple(errors)) else: assert res == ParseResults( - errors=tuple([Error(ErrorType.PARSE_FAIL, message, source_1=SpecificationSource(test_file))]) + errors=tuple( + [ + Error( + ErrorType.PARSE_FAIL, + message, + source_1=SpecificationSource(test_file), + ) + ] + ) ) @@ -530,7 +611,13 @@ def test_excel_parse_fail_non_excel_file(temp_dir: Path): "Head 1, Head 2, Head 3\n", "1, 2, 3\n", ] - _xsv_parse_fail(temp_dir, lines, parse_excel, "Not a supported Excel file type", extension=".xlsx") + _xsv_parse_fail( + temp_dir, + lines, + parse_excel, + "Not a supported Excel file type", + extension=".xlsx", + ) def test_excel_parse_1emptytab(): @@ -558,28 +645,52 @@ def test_excel_parse_fail_headers_only(): _excel_parse_fail(f, "No non-header data in file") +def format_message(t): + return f"Found datatype {t} in multiple tabs" + + def test_excel_parse_fail_colliding_datatypes(): f = _get_test_file("testdatatypecollisions.xls") - l = lambda t: f"Found datatype {t} in multiple tabs" + msg = format_message + err = ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE _excel_parse_fail( f, errors=[ - Error(err, l("type2"), SpecificationSource(f, "dt2"), SpecificationSource(f, "dt2_2")), - Error(err, l("type3"), SpecificationSource(f, "dt3"), SpecificationSource(f, "dt3_2")), - Error(err, l("type2"), SpecificationSource(f, "dt2"), SpecificationSource(f, "dt2_3")), + Error( + err, + msg("type2"), + SpecificationSource(f, "dt2"), + SpecificationSource(f, "dt2_2"), + ), + Error( + err, + msg("type3"), + SpecificationSource(f, "dt3"), + SpecificationSource(f, "dt3_2"), + ), + Error( + err, + msg("type2"), + SpecificationSource(f, "dt2"), + SpecificationSource(f, "dt2_3"), + ), ], ) +def duplicate_header_message(h): + return f"Duplicate header name in row 2: {h}" + + def test_excel_parse_fail_duplicate_headers(): f = _get_test_file("testduplicateheaders.xlsx") - l = lambda h: f"Duplicate header name in row 2: {h}" + msg = duplicate_header_message _excel_parse_fail( f, errors=[ - Error(ErrorType.PARSE_FAIL, l("head1"), SpecificationSource(f, "dt2")), - Error(ErrorType.PARSE_FAIL, l("head2"), SpecificationSource(f, "dt3")), + Error(ErrorType.PARSE_FAIL, msg("head1"), SpecificationSource(f, "dt2")), + Error(ErrorType.PARSE_FAIL, msg("head2"), SpecificationSource(f, "dt3")), ], ) @@ -591,8 +702,16 @@ def test_excel_parse_fail_missing_header_item(): _excel_parse_fail( f, errors=[ - Error(ErrorType.PARSE_FAIL, err1, SpecificationSource(f, "missing header item error")), - Error(ErrorType.PARSE_FAIL, err2, SpecificationSource(f, "whitespace header item")), + Error( + ErrorType.PARSE_FAIL, + err1, + SpecificationSource(f, "missing header item error"), + ), + Error( + ErrorType.PARSE_FAIL, + err2, + SpecificationSource(f, "whitespace header item"), + ), ], ) diff --git a/tests/test_app.py b/tests/test_app.py index 7f9b5c9f..273bd72b 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1012,9 +1012,15 @@ async def test_bulk_specification_success(): }, "files": { "genomes": {"file": "testuser/genomes.tsv", "tab": None}, - "breakfastcereals": {"file": "testuser/somefolder/breakfastcereals.csv", "tab": None}, + "breakfastcereals": { + "file": "testuser/somefolder/breakfastcereals.csv", + "tab": None, + }, "fruit_bats": {"file": "testuser/importspec.xlsx", "tab": "bats"}, - "tree_sloths": {"file": "testuser/importspec.xlsx", "tab": "sloths"}, + "tree_sloths": { + "file": "testuser/importspec.xlsx", + "tab": "sloths", + }, }, } assert resp.status == 200 @@ -1384,7 +1390,10 @@ async def test_bulk_specification_fail_multiple_specs_per_type_excel(): ], }, "reads": { - "order_and_display": [["name", "Reads File Name"], ["inseam", "Reads inseam measurement in km"]], + "order_and_display": [ + ["name", "Reads File Name"], + ["inseam", "Reads inseam measurement in km"], + ], "data": [ {"name": "myreads.fa", "inseam": 0.1}, ], @@ -1408,7 +1417,10 @@ async def test_write_bulk_specification_success_csv(): js = await resp.json() assert js == { "output_file_type": "CSV", - "files_created": {"genome": "testuser/specs/genome.csv", "reads": "testuser/specs/reads.csv"}, + "files_created": { + "genome": "testuser/specs/genome.csv", + "reads": "testuser/specs/reads.csv", + }, } base = Path(fu.base_dir) / "testuser" check_file_contents( @@ -1451,7 +1463,10 @@ async def test_write_bulk_specification_success_tsv(): js = await resp.json() assert js == { "output_file_type": "TSV", - "files_created": {"genome": "testuser/tsvspecs/genome.tsv", "reads": "testuser/tsvspecs/reads.tsv"}, + "files_created": { + "genome": "testuser/tsvspecs/genome.tsv", + "reads": "testuser/tsvspecs/reads.tsv", + }, } base = Path(fu.base_dir) / "testuser" check_file_contents( @@ -1575,7 +1590,8 @@ async def test_write_bulk_specification_fail_no_file_type(): async def test_write_bulk_specification_fail_wrong_file_type(): await _write_bulk_specification_json_fail( - {"output_directory": "foo", "output_file_type": "XSV"}, "Invalid output_file_type: XSV" + {"output_directory": "foo", "output_file_type": "XSV"}, + "Invalid output_file_type: XSV", ) diff --git a/tests/test_app_error_formatter.py b/tests/test_app_error_formatter.py index 23883c24..ace418f0 100644 --- a/tests/test_app_error_formatter.py +++ b/tests/test_app_error_formatter.py @@ -1,7 +1,11 @@ from pathlib import Path from staging_service.app_error_formatter import format_import_spec_errors -from staging_service.import_specifications.file_parser import ErrorType, Error, SpecificationSource +from staging_service.import_specifications.file_parser import ( + ErrorType, + Error, + SpecificationSource, +) def _ss(file: str, tab: str = None) -> SpecificationSource: @@ -28,7 +32,12 @@ def test_format_import_spec_errors_all_the_errors_no_tabs(): Error(ErrorType.OTHER, "foobar1", _ss("file1")), Error(ErrorType.PARSE_FAIL, "foobar2", _ss("file2")), Error(ErrorType.INCORRECT_COLUMN_COUNT, "foobar3", _ss("file3")), - Error(ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE, "foobar4", _ss("file4"), _ss("file5")), + Error( + ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE, + "foobar4", + _ss("file4"), + _ss("file5"), + ), Error(ErrorType.NO_FILES_PROVIDED), Error(ErrorType.FILE_NOT_FOUND, source_1=_ss("file6")), ] @@ -73,7 +82,12 @@ def test_format_import_spec_errors_all_the_errors_with_tabs(): errors = [ Error(ErrorType.PARSE_FAIL, "foobar1", _ss("file1", "tab1")), Error(ErrorType.INCORRECT_COLUMN_COUNT, "foobar2", _ss("file2", "tab2")), - Error(ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE, "foobar3", _ss("file3", "tab3"), _ss("file4", "tab4")), + Error( + ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE, + "foobar3", + _ss("file3", "tab3"), + _ss("file4", "tab4"), + ), ] paths = { Path("file1"): Path("f1"), @@ -82,7 +96,12 @@ def test_format_import_spec_errors_all_the_errors_with_tabs(): Path("file4"): Path("f4"), } assert format_import_spec_errors(errors, paths) == [ - {"type": "cannot_parse_file", "message": "foobar1", "file": "f1", "tab": "tab1"}, + { + "type": "cannot_parse_file", + "message": "foobar1", + "file": "f1", + "tab": "tab1", + }, { "type": "incorrect_column_count", "message": "foobar2", diff --git a/tests/test_autodetect.py b/tests/test_autodetect.py index f8a501d5..76d4f687 100644 --- a/tests/test_autodetect.py +++ b/tests/test_autodetect.py @@ -49,7 +49,11 @@ def test_reasonable_filenames(): extensions = file_format_to_extension_mapping[heading] for extension in extensions: good_filenames.append( - (f"{heading}.{extension}", heading.count("."), extensions_mapping[extension]["file_ext_type"]) + ( + f"{heading}.{extension}", + heading.count("."), + extensions_mapping[extension]["file_ext_type"], + ) ) for filename, heading_dotcount, ext in good_filenames: @@ -77,9 +81,18 @@ def test_specific_filenames(): Test some made up filenames to check that multi-dot extensions are handled correctly """ test_data = [ - ("filename", (None, {"prefix": "filename", "suffix": None, "file_ext_type": []})), - ("file.name", (None, {"prefix": "file.name", "suffix": None, "file_ext_type": []})), - ("fil.en.ame", (None, {"prefix": "fil.en.ame", "suffix": None, "file_ext_type": []})), + ( + "filename", + (None, {"prefix": "filename", "suffix": None, "file_ext_type": []}), + ), + ( + "file.name", + (None, {"prefix": "file.name", "suffix": None, "file_ext_type": []}), + ), + ( + "fil.en.ame", + (None, {"prefix": "fil.en.ame", "suffix": None, "file_ext_type": []}), + ), ( "file.gZ", ( @@ -90,7 +103,11 @@ def test_specific_filenames(): "title": "Decompress/Unpack", } ], - {"prefix": "file", "suffix": "gZ", "file_ext_type": ["CompressedFileFormatArchive"]}, + { + "prefix": "file", + "suffix": "gZ", + "file_ext_type": ["CompressedFileFormatArchive"], + }, ), ), ( @@ -103,7 +120,11 @@ def test_specific_filenames(): "title": "Decompress/Unpack", } ], - {"prefix": "file.name", "suffix": "gZ", "file_ext_type": ["CompressedFileFormatArchive"]}, + { + "prefix": "file.name", + "suffix": "gZ", + "file_ext_type": ["CompressedFileFormatArchive"], + }, ), ), ( @@ -126,7 +147,11 @@ def test_specific_filenames(): "title": "GFF/FASTA MetaGenome", }, ], - {"prefix": "oscar_the_grouch_does_meth", "suffix": "FaStA.gz", "file_ext_type": ["FASTA"]}, + { + "prefix": "oscar_the_grouch_does_meth", + "suffix": "FaStA.gz", + "file_ext_type": ["FASTA"], + }, ), ), ( @@ -144,7 +169,11 @@ def test_specific_filenames(): "title": "GFF/FASTA MetaGenome", }, ], - {"prefix": "look.at.all.these.frigging.dots", "suffix": "gff2.gzip", "file_ext_type": ["GFF"]}, + { + "prefix": "look.at.all.these.frigging.dots", + "suffix": "gff2.gzip", + "file_ext_type": ["GFF"], + }, ), ), ] @@ -184,7 +213,11 @@ def test_zip_mappings(): "title": "Decompress/Unpack", } ] - assert fileinfo == {"prefix": "test", "suffix": "tar.gz", "file_ext_type": ["CompressedFileFormatArchive"]} + assert fileinfo == { + "prefix": "test", + "suffix": "tar.gz", + "file_ext_type": ["CompressedFileFormatArchive"], + } def test_get_mappings(): @@ -217,7 +250,11 @@ def test_get_mappings(): ], "fileinfo": [ {"prefix": "filename", "suffix": None, "file_ext_type": []}, - {"prefix": "file.name", "suffix": "Gz", "file_ext_type": ["CompressedFileFormatArchive"]}, + { + "prefix": "file.name", + "suffix": "Gz", + "file_ext_type": ["CompressedFileFormatArchive"], + }, {"prefix": "some.dots", "suffix": "gff3.gz", "file_ext_type": ["GFF"]}, ], } diff --git a/tests/test_metadata.py b/tests/test_metadata.py index aa8f0858..47777e8b 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,4 +1,4 @@ -""" Unit tests for the metadata handling routines. """ +"""Unit tests for the metadata handling routines.""" import json import uuid @@ -39,7 +39,13 @@ async def test_incomplete_metadata_file_update(temp_dir: Path): async def _incomplete_metadata_file_update(temp_dir, metadict, source): - target = Path(str(temp_dir / "full"), str(temp_dir / "meta"), "user_path", "myfilename", "super_fake_jgi_path") + target = Path( + str(temp_dir / "full"), + str(temp_dir / "meta"), + "user_path", + "myfilename", + "super_fake_jgi_path", + ) with open(target.full_path, "w") as p: p.writelines(make_test_lines(1, 6)) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1407cd51..c8b03754 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -34,7 +34,7 @@ def bootstrap_config(): def assert_exception_correct(got: Exception, expected: Exception): err = "".join(traceback.TracebackException.from_exception(got).format()) assert got.args == expected.args, err - assert type(got) == type(expected) + assert type(got) == type(expected) # noqa: E721 def check_file_contents(file: Path, lines: list[str]): @@ -42,7 +42,12 @@ def check_file_contents(file: Path, lines: list[str]): assert f.readlines() == lines -def check_excel_contents(wb: openpyxl.Workbook, sheetname: str, contents: list[list[Any]], column_widths: list[int]): +def check_excel_contents( + wb: openpyxl.Workbook, + sheetname: str, + contents: list[list[Any]], + column_widths: list[int], +): sheet = wb[sheetname] for i, row in enumerate(sheet.iter_rows()): assert [cell.value for cell in row] == contents[i]