From 183f5d8f8726cae586e15286b0231798b70abb48 Mon Sep 17 00:00:00 2001 From: Ricki <141253288+RickiJay-WMDE@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:22:00 +0100 Subject: [PATCH] chore: Lint Cleanup (#39) --- .github/workflows/backend.yml | 8 ----- data/database_connection.py | 2 +- .../wikibase_expectation_suite.json | 35 ++++++++++++++++--- ...roperty_usage_count_expectation_suite.json | 7 +++- ...se_software_version_expectation_suite.json | 7 +++- .../wikibase_url_expectation_suite.json | 7 +++- ...wikibase_user_group_expectation_suite.json | 7 +++- .../log_data/create_log_observation.py | 4 +-- .../api_data/log_data/fetch_log_data.py | 10 +++--- .../user_data/fetch_multiple_user_data.py | 2 +- ...reate_software_version_data_observation.py | 18 +++++----- .../connectivity_distance_dictionary.py | 4 +-- .../connectivity_link_dictionary.py | 4 +-- .../sparql_data/sparql_queries/item_links.py | 3 +- fetch_data/utils/counts.py | 3 +- model/database/wikibase_category_model.py | 3 +- model/database/wikibase_model.py | 16 ++++----- .../connectivity_observation_model.py | 6 ++-- .../wikibase_log_month_observation_model.py | 6 ++-- .../property/popularity_observation_model.py | 3 +- .../user/wikibase_user_group_model.py | 3 +- .../user/wikibase_user_observation_model.py | 4 +-- .../wikibase_version_observation_model.py | 3 +- .../wikibase_connectivity_observation.py | 6 ++-- .../log/wikibase_log_collection.py | 6 ++-- .../property_popularity/observation.py | 3 +- .../software_version_aggregate.py | 18 +++++----- .../software_version_observation.py | 9 +++-- .../user/wikibase_user_observation.py | 4 +-- .../observation/wikibase_observation_set.py | 6 ++-- model/strawberry/output/page.py | 6 ++-- model/strawberry/query.py | 3 +- pytest.ini | 1 + requirements-dev.txt | 4 +-- requirements.txt | 4 +-- resolvers/get_aggregate_created.py | 6 ++-- .../get_aggregate_property_popularity.py | 4 +-- resolvers/get_aggregate_quantity.py | 4 +-- resolvers/get_aggregate_software_version.py | 4 +-- resolvers/get_aggregate_statistics.py | 4 +-- resolvers/get_aggregate_users.py | 6 ++-- .../test_create_connectivity_observation.py | 3 +- .../test_create_log_observation.py | 14 ++++++-- .../__init__.py | 1 + .../test_constants.py | 5 +++ ...est_create_software_version_observation.py | 12 +++---- .../__init__.py | 1 + .../test_create_statistics_observation.py | 4 ++- .../test_aggregate_extensions_query.py | 28 ++------------- .../test_aggregate_libraries_query.py | 28 ++------------- .../test_aggregate_skins_query.py | 16 ++------- .../test_aggregate_software_query.py | 15 ++------ ...est_aggregate_property_popularity_query.py | 19 ++++------ tests/test_query/test_wikibase_list_query.py | 18 ++++------ tests/utils/__init__.py | 1 + tests/utils/assert_meta.py | 19 ++++++++++ tests/utils/mock_response.py | 12 ++++++- 57 files changed, 224 insertions(+), 235 deletions(-) create mode 100644 tests/test_create_observation/test_create_software_version_observation/__init__.py create mode 100644 tests/test_create_observation/test_create_software_version_observation/test_constants.py create mode 100644 tests/test_create_observation/test_create_statistics_observation/__init__.py create mode 100644 tests/utils/assert_meta.py diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index b7b44bfc..8c7c1f04 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -10,8 +10,6 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v3 - with: - python-version: "3.10.12" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -25,8 +23,6 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v3 - with: - python-version: "3.10.12" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -40,8 +36,6 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v3 - with: - python-version: "3.10.12" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -55,8 +49,6 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v3 - with: - python-version: "3.10.12" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/data/database_connection.py b/data/database_connection.py index f2fc521c..c659df3b 100644 --- a/data/database_connection.py +++ b/data/database_connection.py @@ -1,7 +1,7 @@ """Database Connection""" +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager -from typing import AsyncGenerator from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker diff --git a/data/gx/expectations/wikibase_expectation_suite.json b/data/gx/expectations/wikibase_expectation_suite.json index 500a17ec..4c99be5f 100644 --- a/data/gx/expectations/wikibase_expectation_suite.json +++ b/data/gx/expectations/wikibase_expectation_suite.json @@ -34,7 +34,12 @@ "id": "5342a046-55a7-4a61-908e-c461cd4a84bf", "kwargs": { "column": "wikibase_name", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -49,7 +54,12 @@ "id": "d4ffc143-e54a-4ad9-af88-b5c38eee60a4", "kwargs": { "column": "organization", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -64,7 +74,12 @@ "id": "43da03cf-e685-4bbd-979c-ffd8e16aeed7", "kwargs": { "column": "country", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -79,7 +94,12 @@ "id": "9ce6b575-0119-4dac-8d65-d1a210e6f423", "kwargs": { "column": "region", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -136,7 +156,12 @@ "id": "ca289730-c632-4991-81c5-f31eef1cee75", "kwargs": { "column": "description", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json b/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json index 07b52a34..bb2abd90 100644 --- a/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json +++ b/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json @@ -46,7 +46,12 @@ "id": "77b293b1-73c2-437a-a035-3adfb66294bd", "kwargs": { "column": "property_url", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/data/gx/expectations/wikibase_software_version_expectation_suite.json b/data/gx/expectations/wikibase_software_version_expectation_suite.json index f8f894f0..dfe39fc9 100644 --- a/data/gx/expectations/wikibase_software_version_expectation_suite.json +++ b/data/gx/expectations/wikibase_software_version_expectation_suite.json @@ -94,7 +94,12 @@ "id": "78894d21-05f1-470d-931b-3efe46132d2e", "kwargs": { "column": "version", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/data/gx/expectations/wikibase_url_expectation_suite.json b/data/gx/expectations/wikibase_url_expectation_suite.json index 5565f56b..b2052dfa 100644 --- a/data/gx/expectations/wikibase_url_expectation_suite.json +++ b/data/gx/expectations/wikibase_url_expectation_suite.json @@ -81,7 +81,12 @@ "id": "31ae4f4f-55f0-4ac7-8e0d-3f17fe4fe823", "kwargs": { "column": "url", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/data/gx/expectations/wikibase_user_group_expectation_suite.json b/data/gx/expectations/wikibase_user_group_expectation_suite.json index c0c2ce88..fe9883a7 100644 --- a/data/gx/expectations/wikibase_user_group_expectation_suite.json +++ b/data/gx/expectations/wikibase_user_group_expectation_suite.json @@ -40,7 +40,12 @@ "id": "67a3cb18-3559-471d-ba6c-ef4385a9b96b", "kwargs": { "column": "group_name", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/fetch_data/api_data/log_data/create_log_observation.py b/fetch_data/api_data/log_data/create_log_observation.py index b6dbc557..e3ffd0c1 100644 --- a/fetch_data/api_data/log_data/create_log_observation.py +++ b/fetch_data/api_data/log_data/create_log_observation.py @@ -1,8 +1,8 @@ """Create Log Observation""" +from collections.abc import Iterable from datetime import datetime from json.decoder import JSONDecodeError -from typing import List from requests.exceptions import ReadTimeout, SSLError from data import get_async_session from fetch_data.api_data.log_data.fetch_log_data import ( @@ -85,7 +85,7 @@ async def create_log_observation(wikibase_id: int) -> bool: async def create_log_month( - wikibase: WikibaseModel, log_list: List[WikibaseLogRecord] + wikibase: WikibaseModel, log_list: Iterable[WikibaseLogRecord] ) -> WikibaseLogMonthObservationModel: """Create Log Month""" diff --git a/fetch_data/api_data/log_data/fetch_log_data.py b/fetch_data/api_data/log_data/fetch_log_data.py index 27bc9830..5559b4a8 100644 --- a/fetch_data/api_data/log_data/fetch_log_data.py +++ b/fetch_data/api_data/log_data/fetch_log_data.py @@ -1,7 +1,7 @@ """Fetch Log Data""" from datetime import datetime -from typing import List, Optional +from typing import Optional from fetch_data.api_data.log_data.wikibase_log_record import WikibaseLogRecord from fetch_data.utils import dict_to_url, fetch_api_data @@ -24,10 +24,10 @@ def get_log_param_string( return dict_to_url(parameters) -def get_log_list_from_url(url: str) -> List[WikibaseLogRecord]: +def get_log_list_from_url(url: str) -> list[WikibaseLogRecord]: """Get Log List from URL""" - data = [] + data: list[WikibaseLogRecord] = [] query_data = fetch_api_data(url) for record in query_data["query"]["logevents"]: @@ -38,10 +38,10 @@ def get_log_list_from_url(url: str) -> List[WikibaseLogRecord]: def get_month_log_list( api_url: str, comparison_date: datetime, oldest: bool = False -) -> List[WikibaseLogRecord]: +) -> list[WikibaseLogRecord]: """Get Log List from api_url, limit to within 30 days of the comparison date""" - data: List[WikibaseLogRecord] = [] + data: list[WikibaseLogRecord] = [] limit = 500 should_query = True diff --git a/fetch_data/api_data/user_data/fetch_multiple_user_data.py b/fetch_data/api_data/user_data/fetch_multiple_user_data.py index 23305022..943ffb21 100644 --- a/fetch_data/api_data/user_data/fetch_multiple_user_data.py +++ b/fetch_data/api_data/user_data/fetch_multiple_user_data.py @@ -1,7 +1,7 @@ """Fetch Multiple User Data""" +from collections.abc import Iterable import json -from typing import Iterable import requests from fetch_data.api_data.user_data.user_data_url import user_url from model.database import WikibaseModel diff --git a/fetch_data/soup_data/create_software_version_data_observation.py b/fetch_data/soup_data/create_software_version_data_observation.py index 6c6bcc5c..aee6a8da 100644 --- a/fetch_data/soup_data/create_software_version_data_observation.py +++ b/fetch_data/soup_data/create_software_version_data_observation.py @@ -1,7 +1,7 @@ """Create Software Version Observation""" +from collections.abc import Iterable from datetime import datetime -from typing import List from urllib.error import HTTPError from bs4 import BeautifulSoup, Tag import requests @@ -61,7 +61,7 @@ async def create_software_version_observation(wikibase_id: int) -> bool: def compile_extension_versions( soup: BeautifulSoup, -) -> List[WikibaseSoftwareVersionModel]: +) -> list[WikibaseSoftwareVersionModel]: """Compile Extension Version List""" extensions_table = soup.find( @@ -79,12 +79,12 @@ def compile_extension_versions( def compile_installed_software_versions( soup: BeautifulSoup, -) -> List[WikibaseSoftwareVersionModel]: +) -> list[WikibaseSoftwareVersionModel]: """Compile Installed Software Version List""" installed_software_table: Tag = soup.find("table", attrs={"id": "sv-software"}) - software_versions: List[WikibaseSoftwareVersionModel] = [] + software_versions: list[WikibaseSoftwareVersionModel] = [] row: Tag for row in installed_software_table.find_all("tr"): if row.find("td"): @@ -124,12 +124,12 @@ def compile_installed_software_versions( return software_versions -def compile_library_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionModel]: +def compile_library_versions(soup: BeautifulSoup) -> list[WikibaseSoftwareVersionModel]: """Compile Library Version List""" libraries_table = soup.find("table", attrs={"id": "sv-libraries"}) - library_versions: List[WikibaseSoftwareVersionModel] = [] + library_versions: list[WikibaseSoftwareVersionModel] = [] row: Tag for row in libraries_table.find_all("tr"): if row.find("td"): @@ -149,7 +149,7 @@ def compile_library_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersio return unique_versions(library_versions) -def compile_skin_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionModel]: +def compile_skin_versions(soup: BeautifulSoup) -> list[WikibaseSoftwareVersionModel]: """Compile Skin Version List""" installed_skin_table: Tag = soup.find( @@ -214,8 +214,8 @@ def get_software_version_from_row( def unique_versions( - input_list: List[WikibaseSoftwareVersionModel], -) -> List[WikibaseSoftwareVersionModel]: + input_list: Iterable[WikibaseSoftwareVersionModel], +) -> list[WikibaseSoftwareVersionModel]: """Unique Version List""" temp: dict[str, WikibaseSoftwareVersionModel] = {} diff --git a/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py b/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py index f5a19288..109ee506 100644 --- a/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py +++ b/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py @@ -1,11 +1,11 @@ """Compile Connectivity Distance""" -from typing import Iterable, List +from collections.abc import Iterable from tqdm import tqdm def compile_distance_dict( - all_nodes: List[str], link_dict: dict[str, set[str]] + all_nodes: Iterable[str], link_dict: dict[str, set[str]] ) -> dict[str, dict[str, int]]: """Compile Distance Dictionary diff --git a/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py b/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py index 31867ca6..dea5abdf 100644 --- a/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py +++ b/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py @@ -1,11 +1,11 @@ """Compile Link Dictionary""" -from typing import List +from collections.abc import Iterable from fetch_data.sparql_data.sparql_queries import ItemLink def compile_link_dict( - clean_data: List[ItemLink], all_nodes: List[str], reverse: bool = False + clean_data: Iterable[ItemLink], all_nodes: Iterable[str], reverse: bool = False ) -> dict[str, set[str]]: """Compile Link Dictionary diff --git a/fetch_data/sparql_data/sparql_queries/item_links.py b/fetch_data/sparql_data/sparql_queries/item_links.py index 4409ac42..31cf0cb0 100644 --- a/fetch_data/sparql_data/sparql_queries/item_links.py +++ b/fetch_data/sparql_data/sparql_queries/item_links.py @@ -1,7 +1,6 @@ """Links between items""" import re -from typing import List ITEM_LINKS_QUERY = """SELECT ?item ?object WHERE { @@ -36,7 +35,7 @@ def clean_point(point: dict) -> ItemLink: ) -def clean_item_link_data(results: dict) -> List[ItemLink]: +def clean_item_link_data(results: dict) -> list[ItemLink]: """Query Results to list of data""" return [clean_point(p) for p in results["results"]["bindings"]] diff --git a/fetch_data/utils/counts.py b/fetch_data/utils/counts.py index 72d1b7d0..8d8c8824 100644 --- a/fetch_data/utils/counts.py +++ b/fetch_data/utils/counts.py @@ -1,6 +1,7 @@ """Counts""" -from typing import Iterable, TypeVar +from collections.abc import Iterable +from typing import TypeVar T = TypeVar("T") diff --git a/model/database/wikibase_category_model.py b/model/database/wikibase_category_model.py index c1827571..bf1d72f4 100644 --- a/model/database/wikibase_category_model.py +++ b/model/database/wikibase_category_model.py @@ -1,6 +1,5 @@ """Wikibase Category Table""" -from typing import List from sqlalchemy import Enum, Integer, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -18,7 +17,7 @@ class WikibaseCategoryModel(ModelBase): id: Mapped[int] = mapped_column("id", Integer, primary_key=True, autoincrement=True) """ID""" - wikibases: Mapped[List["WikibaseModel"]] = relationship( + wikibases: Mapped[list["WikibaseModel"]] = relationship( "WikibaseModel", lazy="selectin", back_populates="category" ) """Wikibases""" diff --git a/model/database/wikibase_model.py b/model/database/wikibase_model.py index 06f8d3e7..9478c5d9 100644 --- a/model/database/wikibase_model.py +++ b/model/database/wikibase_model.py @@ -1,6 +1,6 @@ """Wikibase Table""" -from typing import List, Optional +from typing import Optional from sqlalchemy import Boolean, ForeignKey, Integer, String, and_ from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -212,7 +212,7 @@ class WikibaseModel(ModelBase): ) """Special:Version URL""" - connectivity_observations: Mapped[List[WikibaseConnectivityObservationModel]] = ( + connectivity_observations: Mapped[list[WikibaseConnectivityObservationModel]] = ( relationship( "WikibaseConnectivityObservationModel", back_populates="wikibase", @@ -221,13 +221,13 @@ class WikibaseModel(ModelBase): ) """Connectivity Observations""" - log_observations: Mapped[List[WikibaseLogObservationModel]] = relationship( + log_observations: Mapped[list[WikibaseLogObservationModel]] = relationship( "WikibaseLogObservationModel", back_populates="wikibase", lazy="select" ) """Log Observations""" property_popularity_observations: Mapped[ - List[WikibasePropertyPopularityObservationModel] + list[WikibasePropertyPopularityObservationModel] ] = relationship( "WikibasePropertyPopularityObservationModel", back_populates="wikibase", @@ -235,7 +235,7 @@ class WikibaseModel(ModelBase): ) """Property Popularity Observations""" - quantity_observations: Mapped[List[WikibaseQuantityObservationModel]] = ( + quantity_observations: Mapped[list[WikibaseQuantityObservationModel]] = ( relationship( "WikibaseQuantityObservationModel", back_populates="wikibase", lazy="select" ) @@ -243,7 +243,7 @@ class WikibaseModel(ModelBase): """Quantity Observations""" software_version_observations: Mapped[ - List[WikibaseSoftwareVersionObservationModel] + list[WikibaseSoftwareVersionObservationModel] ] = relationship( "WikibaseSoftwareVersionObservationModel", back_populates="wikibase", @@ -251,7 +251,7 @@ class WikibaseModel(ModelBase): ) """Software Version Observations""" - statistics_observations: Mapped[List[WikibaseStatisticsObservationModel]] = ( + statistics_observations: Mapped[list[WikibaseStatisticsObservationModel]] = ( relationship( "WikibaseStatisticsObservationModel", back_populates="wikibase", @@ -260,7 +260,7 @@ class WikibaseModel(ModelBase): ) """Statistics Observations""" - user_observations: Mapped[List[WikibaseUserObservationModel]] = relationship( + user_observations: Mapped[list[WikibaseUserObservationModel]] = relationship( "WikibaseUserObservationModel", back_populates="wikibase", lazy="select" ) """User Observations""" diff --git a/model/database/wikibase_observation/connectivity/connectivity_observation_model.py b/model/database/wikibase_observation/connectivity/connectivity_observation_model.py index 4963c8bd..b37ca75a 100644 --- a/model/database/wikibase_observation/connectivity/connectivity_observation_model.py +++ b/model/database/wikibase_observation/connectivity/connectivity_observation_model.py @@ -1,6 +1,6 @@ """Wikibase Connectivity Observation Table""" -from typing import List, Optional +from typing import Optional from sqlalchemy import Double, Integer from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -37,7 +37,7 @@ class WikibaseConnectivityObservationModel(ModelBase, WikibaseObservationModel): """Average steps for item -> item connections, ignoring disconnected items""" item_relationship_count_observations: Mapped[ - List[WikibaseConnectivityObservationItemRelationshipCountModel] + list[WikibaseConnectivityObservationItemRelationshipCountModel] ] = relationship( "WikibaseConnectivityObservationItemRelationshipCountModel", back_populates="connectivity_observation", @@ -46,7 +46,7 @@ class WikibaseConnectivityObservationModel(ModelBase, WikibaseObservationModel): """Item / Relationship Count Observations""" object_relationship_count_observations: Mapped[ - List[WikibaseConnectivityObservationObjectRelationshipCountModel] + list[WikibaseConnectivityObservationObjectRelationshipCountModel] ] = relationship( "WikibaseConnectivityObservationObjectRelationshipCountModel", back_populates="connectivity_observation", diff --git a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py index 4b17115f..0ce45f87 100644 --- a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py +++ b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py @@ -1,7 +1,7 @@ """Wikibase Log Month Observation Table""" from datetime import datetime -from typing import List, Optional +from typing import Optional from sqlalchemy import DateTime, Integer from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -43,12 +43,12 @@ class WikibaseLogMonthObservationModel(ModelBase): ) """Number of Unique Users, Without Bots""" - log_type_records: Mapped[List[WikibaseLogMonthLogTypeObservationModel]] = ( + log_type_records: Mapped[list[WikibaseLogMonthLogTypeObservationModel]] = ( relationship("WikibaseLogMonthLogTypeObservationModel", lazy="selectin") ) """Log Type Observations""" - user_type_records: Mapped[List[WikibaseLogMonthUserTypeObservationModel]] = ( + user_type_records: Mapped[list[WikibaseLogMonthUserTypeObservationModel]] = ( relationship("WikibaseLogMonthUserTypeObservationModel", lazy="selectin") ) """User Type Observations""" diff --git a/model/database/wikibase_observation/property/popularity_observation_model.py b/model/database/wikibase_observation/property/popularity_observation_model.py index 132bdcd3..88fc1c6b 100644 --- a/model/database/wikibase_observation/property/popularity_observation_model.py +++ b/model/database/wikibase_observation/property/popularity_observation_model.py @@ -1,6 +1,5 @@ """Wikibase Property Popularity Observation Table""" -from typing import List from sqlalchemy.orm import Mapped, relationship from model.database.base import ModelBase @@ -17,7 +16,7 @@ class WikibasePropertyPopularityObservationModel(ModelBase, WikibaseObservationM __tablename__ = "wikibase_property_usage_observation" - property_count_observations: Mapped[List[WikibasePropertyPopularityCountModel]] = ( + property_count_observations: Mapped[list[WikibasePropertyPopularityCountModel]] = ( relationship( "WikibasePropertyPopularityCountModel", back_populates="wikibase_property_popularity_observation", diff --git a/model/database/wikibase_observation/user/wikibase_user_group_model.py b/model/database/wikibase_observation/user/wikibase_user_group_model.py index 5b56f715..c9d461f7 100644 --- a/model/database/wikibase_observation/user/wikibase_user_group_model.py +++ b/model/database/wikibase_observation/user/wikibase_user_group_model.py @@ -1,6 +1,5 @@ """Wikibase User Group Table""" -from typing import List from sqlalchemy import Boolean, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -27,7 +26,7 @@ class WikibaseUserGroupModel(ModelBase): ) """Wikibase Default?""" - user_group_observations: Mapped[List[WikibaseUserObservationGroupModel]] = ( + user_group_observations: Mapped[list[WikibaseUserObservationGroupModel]] = ( relationship( "WikibaseUserObservationGroupModel", back_populates="user_group", diff --git a/model/database/wikibase_observation/user/wikibase_user_observation_model.py b/model/database/wikibase_observation/user/wikibase_user_observation_model.py index 0f462d20..47f524bc 100644 --- a/model/database/wikibase_observation/user/wikibase_user_observation_model.py +++ b/model/database/wikibase_observation/user/wikibase_user_observation_model.py @@ -1,6 +1,6 @@ """Wikibase User Observation Table""" -from typing import List, Optional +from typing import Optional from sqlalchemy import Integer from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -23,7 +23,7 @@ class WikibaseUserObservationModel(ModelBase, WikibaseObservationModel): ) """Total Users""" - user_group_observations: Mapped[List[WikibaseUserObservationGroupModel]] = ( + user_group_observations: Mapped[list[WikibaseUserObservationGroupModel]] = ( relationship( "WikibaseUserObservationGroupModel", back_populates="user_observation", diff --git a/model/database/wikibase_observation/version/wikibase_version_observation_model.py b/model/database/wikibase_observation/version/wikibase_version_observation_model.py index 0ab57ec5..31fd2b3a 100644 --- a/model/database/wikibase_observation/version/wikibase_version_observation_model.py +++ b/model/database/wikibase_observation/version/wikibase_version_observation_model.py @@ -1,6 +1,5 @@ """Wikibase Software Version Observation Table""" -from typing import List from sqlalchemy.orm import Mapped, relationship from model.database.base import ModelBase @@ -17,7 +16,7 @@ class WikibaseSoftwareVersionObservationModel(ModelBase, WikibaseObservationMode __tablename__ = "wikibase_software_version_observation" - software_versions: Mapped[List[WikibaseSoftwareVersionModel]] = relationship( + software_versions: Mapped[list[WikibaseSoftwareVersionModel]] = relationship( "WikibaseSoftwareVersionModel", back_populates="wikibase_software_version_observation", lazy="selectin", diff --git a/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py b/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py index 8f1ea173..e0ce8606 100644 --- a/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py +++ b/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py @@ -1,6 +1,6 @@ """Wikibase Connectivity Data Observation Strawberry Model""" -from typing import List, Optional +from typing import Optional import strawberry from model.database import WikibaseConnectivityObservationModel @@ -32,10 +32,10 @@ class WikibaseConnectivityObservationStrawberryModel( graphql_type=Optional[BigInt], ) - relationship_item_counts: List[ + relationship_item_counts: list[ WikibaseConnectivityObservationItemRelationshipCountStrawberryModel ] = strawberry.field(description="Number of Items with Number of Relationships") - relationship_object_counts: List[ + relationship_object_counts: list[ WikibaseConnectivityObservationObjectRelationshipCountStrawberryModel ] = strawberry.field(description="Number of Items with Number of Relationships") diff --git a/model/strawberry/output/observation/log/wikibase_log_collection.py b/model/strawberry/output/observation/log/wikibase_log_collection.py index 2e13f61d..6f88044c 100644 --- a/model/strawberry/output/observation/log/wikibase_log_collection.py +++ b/model/strawberry/output/observation/log/wikibase_log_collection.py @@ -1,7 +1,7 @@ """Wikibase Log Collection Strawberry Models""" from datetime import datetime -from typing import List, Optional +from typing import Optional import strawberry from model.database import ( @@ -81,10 +81,10 @@ def marshal( class WikibaseLogMonthStrawberryModel(WikibaseLogCollectionStrawberryModel): """Wikibase Log Month""" - log_type_records: List[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field( + log_type_records: list[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field( description="Records of Each Type" ) - user_type_records: List[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field( + user_type_records: list[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field( description="Records of Each Type" ) human_users: int = strawberry.field( diff --git a/model/strawberry/output/observation/property_popularity/observation.py b/model/strawberry/output/observation/property_popularity/observation.py index d22ac911..6e51c9e8 100644 --- a/model/strawberry/output/observation/property_popularity/observation.py +++ b/model/strawberry/output/observation/property_popularity/observation.py @@ -1,6 +1,5 @@ """Wikibase Property Popularity Observation Strawberry Model""" -from typing import List import strawberry from model.database import WikibasePropertyPopularityObservationModel @@ -18,7 +17,7 @@ class WikibasePropertyPopularityObservationStrawberryModel( ): """Wikibase Property Popularity Observation""" - property_popularity_counts: List[WikibasePropertyPopularityCountStrawberryModel] = ( + property_popularity_counts: list[WikibasePropertyPopularityCountStrawberryModel] = ( strawberry.field(description="Number of Items with Number of Relationships") ) diff --git a/model/strawberry/output/observation/software_version/software_version_aggregate.py b/model/strawberry/output/observation/software_version/software_version_aggregate.py index 03920a67..5c26e60f 100644 --- a/model/strawberry/output/observation/software_version/software_version_aggregate.py +++ b/model/strawberry/output/observation/software_version/software_version_aggregate.py @@ -2,7 +2,7 @@ from datetime import datetime import re -from typing import List, Optional +from typing import Optional import strawberry from model.strawberry.output.semver import Semver @@ -55,7 +55,7 @@ class WikibaseSoftwareMidVersionAggregateStrawberryModel: version: Optional[str] = strawberry.field(description="Software Version") private_versions: strawberry.Private[ - List[WikibaseSoftwareVersionAggregateStrawberryModel] + list[WikibaseSoftwareVersionAggregateStrawberryModel] ] @strawberry.field(description="Wikibase Count") @@ -74,7 +74,7 @@ class WikibaseSoftwarePatchVersionAggregateStrawberryModel( @strawberry.field(description="Versions") def sub_versions( self, - ) -> Optional[List[WikibaseSoftwareVersionAggregateStrawberryModel]]: + ) -> Optional[list[WikibaseSoftwareVersionAggregateStrawberryModel]]: """Sub-Patch Versions""" if ( @@ -101,7 +101,7 @@ class WikibaseSoftwareMinorVersionAggregateStrawberryModel( @strawberry.field(description="Patch Versions") def patch_versions( self, - ) -> Optional[List[WikibaseSoftwarePatchVersionAggregateStrawberryModel]]: + ) -> Optional[list[WikibaseSoftwarePatchVersionAggregateStrawberryModel]]: """Patch Versions""" if self.version is None: @@ -135,7 +135,7 @@ class WikibaseSoftwareMajorVersionAggregateStrawberryModel( @strawberry.field(description="Minor Versions") def minor_versions( self, - ) -> Optional[List[WikibaseSoftwareMinorVersionAggregateStrawberryModel]]: + ) -> Optional[list[WikibaseSoftwareMinorVersionAggregateStrawberryModel]]: """Minor Versions""" if self.version is None: @@ -160,13 +160,13 @@ class WikibaseSoftwareVersionDoubleAggregateStrawberryModel: software_name: str = strawberry.field(description="Software Name") private_versions: strawberry.Private[ - List[WikibaseSoftwareVersionAggregateStrawberryModel] + list[WikibaseSoftwareVersionAggregateStrawberryModel] ] def __init__( self, software_name: str, - versions: List[WikibaseSoftwareVersionAggregateStrawberryModel], + versions: list[WikibaseSoftwareVersionAggregateStrawberryModel], ): self.software_name = software_name self.private_versions = versions @@ -180,7 +180,7 @@ def wikibase_count(self) -> int: @strawberry.field(description="Major Versions") def major_versions( self, - ) -> List[WikibaseSoftwareMajorVersionAggregateStrawberryModel]: + ) -> list[WikibaseSoftwareMajorVersionAggregateStrawberryModel]: """Major Versions""" temp: dict[ @@ -196,7 +196,7 @@ def major_versions( return sorted(temp.values(), key=lambda x: x.wikibase_count(), reverse=True) @strawberry.field(description="Version List") - def versions(self) -> List[WikibaseSoftwareVersionAggregateStrawberryModel]: + def versions(self) -> list[WikibaseSoftwareVersionAggregateStrawberryModel]: """Version List""" return sorted( diff --git a/model/strawberry/output/observation/software_version/software_version_observation.py b/model/strawberry/output/observation/software_version/software_version_observation.py index f1acbb6c..a20022e6 100644 --- a/model/strawberry/output/observation/software_version/software_version_observation.py +++ b/model/strawberry/output/observation/software_version/software_version_observation.py @@ -1,6 +1,5 @@ """Wikibase Software Version Observation Strawberry Model""" -from typing import List import strawberry from model.database import WikibaseSoftwareVersionObservationModel @@ -19,19 +18,19 @@ class WikibaseSoftwareVersionObservationStrawberryModel( ): """Wikibase Software Version Observation""" - installed_extensions: List[WikibaseSoftwareVersionStrawberryModel] = ( + installed_extensions: list[WikibaseSoftwareVersionStrawberryModel] = ( strawberry.field(description="Installed Extensions w/ Versions") ) - installed_libraries: List[WikibaseSoftwareVersionStrawberryModel] = ( + installed_libraries: list[WikibaseSoftwareVersionStrawberryModel] = ( strawberry.field(description="Installed Libraries w/ Versions") ) - installed_skins: List[WikibaseSoftwareVersionStrawberryModel] = strawberry.field( + installed_skins: list[WikibaseSoftwareVersionStrawberryModel] = strawberry.field( description="Installed Skins w/ Versions" ) - installed_software: List[WikibaseSoftwareVersionStrawberryModel] = strawberry.field( + installed_software: list[WikibaseSoftwareVersionStrawberryModel] = strawberry.field( description="Installed Software Versions" ) diff --git a/model/strawberry/output/observation/user/wikibase_user_observation.py b/model/strawberry/output/observation/user/wikibase_user_observation.py index 37c0573c..0618296f 100644 --- a/model/strawberry/output/observation/user/wikibase_user_observation.py +++ b/model/strawberry/output/observation/user/wikibase_user_observation.py @@ -1,6 +1,6 @@ """Wikibase User Data Observation Strawberry Model""" -from typing import List, Optional +from typing import Optional import strawberry from model.database import WikibaseUserObservationModel @@ -20,7 +20,7 @@ class WikibaseUserObservationStrawberryModel(WikibaseObservationStrawberryModel) total_users: Optional[int] = strawberry.field( description="Total Users", graphql_type=Optional[BigInt] ) - user_groups: List[WikibaseUserObservationGroupStrawberryModel] = strawberry.field( + user_groups: list[WikibaseUserObservationGroupStrawberryModel] = strawberry.field( description="User Groups and Counts" ) diff --git a/model/strawberry/output/observation/wikibase_observation_set.py b/model/strawberry/output/observation/wikibase_observation_set.py index b10a21c0..02f434b2 100644 --- a/model/strawberry/output/observation/wikibase_observation_set.py +++ b/model/strawberry/output/observation/wikibase_observation_set.py @@ -1,6 +1,6 @@ """Wikibase Observation Set Strawberry Model""" -from typing import Generic, List, Optional, TypeVar +from typing import Generic, Optional, TypeVar import strawberry from model.strawberry.output.observation.wikibase_observation import ( @@ -15,7 +15,7 @@ class WikibaseObservationSetStrawberryModel(Generic[T]): """Wikibase Observation Set""" - all_observations: List[T] = strawberry.field(description="All Observations") + all_observations: list[T] = strawberry.field(description="All Observations") @strawberry.field(description="Most Recent Observation that Returned Data") def most_recent(self) -> Optional[T]: @@ -29,7 +29,7 @@ def most_recent(self) -> Optional[T]: return None @classmethod - def marshal(cls, data: List[T]): + def marshal(cls, data: list[T]): """Coerce List into Set""" return cls(all_observations=data) diff --git a/model/strawberry/output/page.py b/model/strawberry/output/page.py index 8f026d54..a51d57a6 100644 --- a/model/strawberry/output/page.py +++ b/model/strawberry/output/page.py @@ -1,7 +1,7 @@ """Page""" from math import ceil -from typing import Annotated, Generic, List, TypeVar +from typing import Annotated, Generic, TypeVar import strawberry from model.strawberry.scalars import BigInt @@ -49,7 +49,7 @@ class Page(Generic[T]): """Page""" meta: PageMetadata = strawberry.field(description="Metadata") - data: List[T] = strawberry.field(description="Data") + data: list[T] = strawberry.field(description="Data") @classmethod def marshal( @@ -57,7 +57,7 @@ def marshal( page_number: PageNumberType, page_size: PageSizeType, total_count: int, - page_data: List[T], + page_data: list[T], ) -> "Page": """Marshal Data into Page""" return cls( diff --git a/model/strawberry/query.py b/model/strawberry/query.py index 2c07fd2c..7ce1ac50 100644 --- a/model/strawberry/query.py +++ b/model/strawberry/query.py @@ -1,6 +1,5 @@ """Query""" -from typing import List import strawberry from model.enum import WikibaseSoftwareType @@ -46,7 +45,7 @@ async def wikibase_list( @strawberry.field(description="Aggregated Year of First Log Date") async def aggregate_created( self, - ) -> List[WikibaseYearCreatedAggregateStrawberryModel]: + ) -> list[WikibaseYearCreatedAggregateStrawberryModel]: """Aggregated Year of First Log Date""" return await get_aggregate_created() diff --git a/pytest.ini b/pytest.ini index c7c2b50d..4d5f1fff 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,6 +8,7 @@ markers = log: log observation tests property: property popularity observation tests quantity: quantity observation tests + soup: soup-based observation tests sparql: sparql-based observation tests statistics: statistics observation tests user: user observation tests diff --git a/requirements-dev.txt b/requirements-dev.txt index 91a433e6..8bf271e7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,11 +1,11 @@ alembic>=1.13 -black>=24.8 +black>=24.10 freezegun>=1.5 great-expectations>=1.2 pylint>=3.3 pytest>=8.3 pytest-asyncio>=0.24 -pytest-cov>=5.0 +pytest-cov>=6.0 pytest-dependency>=0.6 pytest-mock>=3.14 pytest-order>=1.3 diff --git a/requirements.txt b/requirements.txt index 8d23556d..a3e02927 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,6 @@ numpy>=1.26,<2.0 requests>=2.32 SPARQLWrapper>=2.0 sqlalchemy[asyncio]>=2.0 -strawberry-graphql[fastapi]>=0.242 +strawberry-graphql[fastapi]>=0.247 tqdm>=4.66 -uvicorn[standard]>=0.30 +uvicorn[standard]>=0.32 diff --git a/resolvers/get_aggregate_created.py b/resolvers/get_aggregate_created.py index 765ec393..1933e681 100644 --- a/resolvers/get_aggregate_created.py +++ b/resolvers/get_aggregate_created.py @@ -1,7 +1,5 @@ """Get Aggregate Year Created""" -from typing import List, Tuple - from sqlalchemy import Select, and_, select, func from data import get_async_session @@ -9,7 +7,7 @@ from model.strawberry.output import WikibaseYearCreatedAggregateStrawberryModel -async def get_aggregate_created() -> List[WikibaseYearCreatedAggregateStrawberryModel]: +async def get_aggregate_created() -> list[WikibaseYearCreatedAggregateStrawberryModel]: """Get Aggregate Year Created""" total_quantity_query = get_created_query() @@ -24,7 +22,7 @@ async def get_aggregate_created() -> List[WikibaseYearCreatedAggregateStrawberry ] -def get_created_query() -> Select[Tuple[int, int]]: +def get_created_query() -> Select[tuple[int, int]]: """Get Year Created Query""" rank_subquery = ( diff --git a/resolvers/get_aggregate_property_popularity.py b/resolvers/get_aggregate_property_popularity.py index a35d12ab..66b40b50 100644 --- a/resolvers/get_aggregate_property_popularity.py +++ b/resolvers/get_aggregate_property_popularity.py @@ -1,7 +1,5 @@ """Get Aggregate Property Popularity""" -from typing import Tuple - from sqlalchemy import Select, and_, desc, select, func from data import get_async_session @@ -56,7 +54,7 @@ async def get_aggregate_property_popularity( ) -def get_unordered_query() -> Select[Tuple[int, str, int, int]]: +def get_unordered_query() -> Select[tuple[int, str, int, int]]: """Get Unordered Property Popularity Query""" rank_subquery = ( diff --git a/resolvers/get_aggregate_quantity.py b/resolvers/get_aggregate_quantity.py index ba3d41dd..5cddbb58 100644 --- a/resolvers/get_aggregate_quantity.py +++ b/resolvers/get_aggregate_quantity.py @@ -1,7 +1,5 @@ """Get Aggregate Quantity""" -from typing import Tuple - from sqlalchemy import Select, and_, select, func from data import get_async_session @@ -28,7 +26,7 @@ async def get_aggregate_quantity() -> WikibaseQuantityAggregateStrawberryModel: ) -def get_total_quantity_query() -> Select[Tuple[int, int, int, int, int]]: +def get_total_quantity_query() -> Select[tuple[int, int, int, int, int]]: """Get Total Quantity Query""" rank_subquery = ( diff --git a/resolvers/get_aggregate_software_version.py b/resolvers/get_aggregate_software_version.py index c8ded732..e8432efa 100644 --- a/resolvers/get_aggregate_software_version.py +++ b/resolvers/get_aggregate_software_version.py @@ -1,7 +1,7 @@ """Get Aggregate Software Version""" from datetime import datetime -from typing import Optional, Tuple +from typing import Optional from sqlalchemy import Select, and_, select, func @@ -71,7 +71,7 @@ async def get_aggregate_version( def get_query( software_type: WikibaseSoftwareType, -) -> Select[Tuple[str, Optional[str], Optional[datetime], Optional[str], int]]: +) -> Select[tuple[str, Optional[str], Optional[datetime], Optional[str], int]]: """Get Software Version Query""" rank_subquery = ( diff --git a/resolvers/get_aggregate_statistics.py b/resolvers/get_aggregate_statistics.py index 3759caa9..14ffde62 100644 --- a/resolvers/get_aggregate_statistics.py +++ b/resolvers/get_aggregate_statistics.py @@ -1,7 +1,5 @@ """Get Aggregate Statistics""" -from typing import Tuple - from sqlalchemy import Select, and_, select, func from data import get_async_session @@ -41,7 +39,7 @@ async def get_aggregate_statistics() -> WikibaseStatisticsAggregateStrawberryMod def get_total_statistics_query() -> ( - Select[Tuple[int, int, int, int, int, int, int, int, int]] + Select[tuple[int, int, int, int, int, int, int, int, int]] ): """Get Total Statistics Query""" diff --git a/resolvers/get_aggregate_users.py b/resolvers/get_aggregate_users.py index 4ff46bf8..df6f8b9a 100644 --- a/resolvers/get_aggregate_users.py +++ b/resolvers/get_aggregate_users.py @@ -1,7 +1,5 @@ """Get Aggregate Users""" -from typing import Tuple - from sqlalchemy import Select, and_, or_, select, func from data import get_async_session @@ -36,7 +34,7 @@ async def get_aggregate_users() -> WikibaseUserAggregateStrawberryModel: ) -def get_total_admin_query() -> Select[Tuple[int]]: +def get_total_admin_query() -> Select[tuple[int, int]]: """Get Total Admin Query""" rank_subquery = ( @@ -90,7 +88,7 @@ def get_total_admin_query() -> Select[Tuple[int]]: return query -def get_total_user_query() -> Select[Tuple[int, int]]: +def get_total_user_query() -> Select[tuple[int, int]]: """Get Total User Query""" rank_subquery = ( diff --git a/tests/test_create_observation/test_create_connectivity_observation.py b/tests/test_create_observation/test_create_connectivity_observation.py index 647f6a51..bc7603e0 100644 --- a/tests/test_create_observation/test_create_connectivity_observation.py +++ b/tests/test_create_observation/test_create_connectivity_observation.py @@ -1,6 +1,5 @@ """Test create_connectivity_observation""" -from typing import List from urllib.error import HTTPError import pytest from fetch_data import create_connectivity_observation @@ -33,7 +32,7 @@ ], ) async def test_create_connectivity_observation_success( - mocker, links: List[tuple[str, str]] + mocker, links: list[tuple[str, str]] ): """Test""" diff --git a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py index 7a4f3926..dc95f3d0 100644 --- a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py +++ b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py @@ -107,10 +107,13 @@ def mockery(*args, **kwargs): match (query.params.get("ledir"), query.params.get("lelimit")): case ("newer", 1): return MockResponse( - 200, json.dumps({"query": {"logevents": [oldest_mock_log]}}) + query.raw_url, + 200, + json.dumps({"query": {"logevents": [oldest_mock_log]}}), ) case ("newer", 500): return MockResponse( + query.raw_url, 200, json.dumps( { @@ -124,10 +127,13 @@ def mockery(*args, **kwargs): ) case ("older", 1): return MockResponse( - 200, json.dumps({"query": {"logevents": [newest_mock_log]}}) + query.raw_url, + 200, + json.dumps({"query": {"logevents": [newest_mock_log]}}), ) case ("older", 500): return MockResponse( + query.raw_url, 200, json.dumps( { @@ -149,7 +155,9 @@ def mockery(*args, **kwargs): users.append({"name": "User:B", "invalid": True}) if "User:C" in query.params.get("ususers"): users.append({"name": "User:C", "groups": ["*", "users", "bot"]}) - return MockResponse(200, json.dumps({"query": {"users": users}})) + return MockResponse( + query.raw_url, 200, json.dumps({"query": {"users": users}}) + ) raise NotImplementedError(query) mocker.patch( diff --git a/tests/test_create_observation/test_create_software_version_observation/__init__.py b/tests/test_create_observation/test_create_software_version_observation/__init__.py new file mode 100644 index 00000000..4632e8bc --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/__init__.py @@ -0,0 +1 @@ +"""Test Create Software Version Observation""" diff --git a/tests/test_create_observation/test_create_software_version_observation/test_constants.py b/tests/test_create_observation/test_create_software_version_observation/test_constants.py new file mode 100644 index 00000000..0b54bd95 --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/test_constants.py @@ -0,0 +1,5 @@ +"""Constants""" + +DATA_DIRECTORY = ( + "tests/test_create_observation/test_create_software_version_observation/data" +) diff --git a/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py b/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py index 687127da..b6f6bfe4 100644 --- a/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py +++ b/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py @@ -5,18 +5,17 @@ from urllib.error import HTTPError import pytest from fetch_data import create_software_version_observation -from tests.utils import MockResponse - - -DATA_DIRECTORY = ( - "tests/test_create_observation/test_create_software_version_observation/data" +from tests.test_create_observation.test_create_software_version_observation.test_constants import ( + DATA_DIRECTORY, ) +from tests.utils import MockResponse @pytest.mark.asyncio @pytest.mark.dependency( name="software-version-success", depends=["add-wikibase"], scope="session" ) +@pytest.mark.soup @pytest.mark.version async def test_create_software_version_observation_success(mocker): """Test Data Returned Scenario""" @@ -29,7 +28,7 @@ async def test_create_software_version_observation_success(mocker): mocker.patch( "fetch_data.soup_data.create_software_version_data_observation.requests.get", - side_effect=[MockResponse(200, version_html.read())], + side_effect=[MockResponse("", 200, version_html.read())], ) success = await create_software_version_observation(1) assert success @@ -39,6 +38,7 @@ async def test_create_software_version_observation_success(mocker): @pytest.mark.dependency( name="software-version-failure", depends=["software-version-success"] ) +@pytest.mark.soup @pytest.mark.version async def test_create_software_version_observation_failure(mocker): """Test Failure Scenario""" diff --git a/tests/test_create_observation/test_create_statistics_observation/__init__.py b/tests/test_create_observation/test_create_statistics_observation/__init__.py new file mode 100644 index 00000000..83e8fe5d --- /dev/null +++ b/tests/test_create_observation/test_create_statistics_observation/__init__.py @@ -0,0 +1 @@ +"""Test Create Statistics Observation""" diff --git a/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py b/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py index d0ee7168..75084f1d 100644 --- a/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py +++ b/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py @@ -15,6 +15,7 @@ @pytest.mark.dependency( name="statistics-success", depends=["add-wikibase"], scope="session" ) +@pytest.mark.soup @pytest.mark.statistics async def test_create_statistics_observation_success(mocker): """Test Data Returned Scenario""" @@ -25,7 +26,7 @@ async def test_create_statistics_observation_success(mocker): mocker.patch( "fetch_data.soup_data.create_statistics_data_observation.requests.get", - side_effect=[MockResponse(200, version_html.read())], + side_effect=[MockResponse("", 200, version_html.read())], ) success = await create_special_statistics_observation(1) assert success @@ -33,6 +34,7 @@ async def test_create_statistics_observation_success(mocker): @pytest.mark.asyncio @pytest.mark.dependency(name="statistics-failure", depends=["statistics-success"]) +@pytest.mark.soup @pytest.mark.statistics async def test_create_statistics_observation_failure(mocker): """Test Failure Scenario""" diff --git a/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py b/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py index 073a143d..870a402f 100644 --- a/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py +++ b/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py @@ -9,7 +9,7 @@ SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT, ) from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import assert_layered_property_count, assert_page_meta AGGREGATE_EXTENSIONS_QUERY = ( @@ -39,18 +39,7 @@ async def test_aggregate_extensions_query_page_one(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "pageSize"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "totalCount"], 10 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "totalPages"], 2 - ) + assert_page_meta(result.data["aggregateExtensionPopularity"], 1, 5, 10, 2) assert_layered_property_count( result.data, ["aggregateExtensionPopularity", "data"], 5 ) @@ -113,18 +102,7 @@ async def test_aggregate_extensions_query_page_two(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "pageNumber"], 2 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "pageSize"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "totalCount"], 10 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "totalPages"], 2 - ) + assert_page_meta(result.data["aggregateExtensionPopularity"], 2, 5, 10, 2) assert_layered_property_count( result.data, ["aggregateExtensionPopularity", "data"], 5 ) diff --git a/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py b/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py index e7741bd7..d7bee490 100644 --- a/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py +++ b/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py @@ -8,7 +8,7 @@ SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT, ) from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import assert_layered_property_count, assert_page_meta AGGREGATE_LIBRARIES_QUERY = ( @@ -38,18 +38,7 @@ async def test_aggregate_libraries_query_page_one(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "pageSize"], 30 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "totalCount"], 59 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "totalPages"], 2 - ) + assert_page_meta(result.data["aggregateLibraryPopularity"], 1, 30, 59, 2) assert_layered_property_count( result.data, ["aggregateLibraryPopularity", "data"], 30 ) @@ -117,18 +106,7 @@ async def test_aggregate_libraries_query_page_two(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "pageNumber"], 2 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "pageSize"], 30 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "totalCount"], 59 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "totalPages"], 2 - ) + assert_page_meta(result.data["aggregateLibraryPopularity"], 2, 30, 59, 2) assert_layered_property_count( result.data, ["aggregateLibraryPopularity", "data"], 29 ) diff --git a/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py b/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py index fe23d216..0e0032c2 100644 --- a/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py +++ b/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py @@ -1,6 +1,7 @@ """Test Aggregated Skins Query""" import pytest + from tests.test_query.aggregation.software_version.assert_software_version_aggregate import ( assert_software_version_aggregate, ) @@ -8,7 +9,7 @@ SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT, ) from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import assert_layered_property_count, assert_page_meta AGGREGATE_SKINS_QUERY = ( @@ -38,18 +39,7 @@ async def test_aggregate_skins_query_page_one(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateSkinPopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregateSkinPopularity", "meta", "pageSize"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateSkinPopularity", "meta", "totalCount"], 3 - ) - assert_layered_property_value( - result.data, ["aggregateSkinPopularity", "meta", "totalPages"], 1 - ) + assert_page_meta(result.data["aggregateSkinPopularity"], 1, 5, 3, 1) assert_layered_property_count(result.data, ["aggregateSkinPopularity", "data"], 3) for index, ( diff --git a/tests/test_query/aggregation/software_version/test_aggregate_software_query.py b/tests/test_query/aggregation/software_version/test_aggregate_software_query.py index e388859c..ba7d3d2a 100644 --- a/tests/test_query/aggregation/software_version/test_aggregate_software_query.py +++ b/tests/test_query/aggregation/software_version/test_aggregate_software_query.py @@ -9,7 +9,7 @@ SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT, ) from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import assert_layered_property_count, assert_page_meta AGGREGATE_SOFTWARE_QUERY = ( @@ -39,18 +39,7 @@ async def test_aggregate_software_query(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateSoftwarePopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregateSoftwarePopularity", "meta", "pageSize"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateSoftwarePopularity", "meta", "totalCount"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateSoftwarePopularity", "meta", "totalPages"], 1 - ) + assert_page_meta(result.data["aggregateSoftwarePopularity"], 1, 5, 5, 1) assert_layered_property_count( result.data, ["aggregateSoftwarePopularity", "data"], 5 ) diff --git a/tests/test_query/aggregation/test_aggregate_property_popularity_query.py b/tests/test_query/aggregation/test_aggregate_property_popularity_query.py index e591e678..4fdfb76a 100644 --- a/tests/test_query/aggregation/test_aggregate_property_popularity_query.py +++ b/tests/test_query/aggregation/test_aggregate_property_popularity_query.py @@ -2,7 +2,11 @@ import pytest from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import ( + assert_layered_property_count, + assert_layered_property_value, + assert_page_meta, +) AGGREGATED_PROPERTY_POPULARITY_QUERY = """ @@ -41,18 +45,7 @@ async def test_aggregate_property_popularity_query(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregatePropertyPopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregatePropertyPopularity", "meta", "pageSize"], 30 - ) - assert_layered_property_value( - result.data, ["aggregatePropertyPopularity", "meta", "totalCount"], 2 - ) - assert_layered_property_value( - result.data, ["aggregatePropertyPopularity", "meta", "totalPages"], 1 - ) + assert_page_meta(result.data["aggregatePropertyPopularity"], 1, 30, 2, 1) assert_layered_property_count( result.data, ["aggregatePropertyPopularity", "data"], 2 diff --git a/tests/test_query/test_wikibase_list_query.py b/tests/test_query/test_wikibase_list_query.py index 02f95101..1e1d939e 100644 --- a/tests/test_query/test_wikibase_list_query.py +++ b/tests/test_query/test_wikibase_list_query.py @@ -2,7 +2,11 @@ import pytest from tests.test_schema import test_schema -from tests.utils import assert_layered_property_value, assert_property_value +from tests.utils import ( + assert_layered_property_value, + assert_page_meta, + assert_property_value, +) WIKIBASE_LIST_QUERY = """ @@ -81,17 +85,7 @@ async def test_wikibase_list_query(): assert result.errors is None assert result.data is not None assert "wikibaseList" in result.data - assert "meta" in result.data["wikibaseList"] - assert_layered_property_value( - result.data, ["wikibaseList", "meta", "pageNumber"], 1 - ) - assert_layered_property_value(result.data, ["wikibaseList", "meta", "pageSize"], 1) - assert_layered_property_value( - result.data, ["wikibaseList", "meta", "totalCount"], 1 - ) - assert_layered_property_value( - result.data, ["wikibaseList", "meta", "totalPages"], 1 - ) + assert_page_meta(result.data["wikibaseList"], 1, 1, 1, 1) assert "data" in result.data["wikibaseList"] assert len(result.data["wikibaseList"]["data"]) == 1 result_datum = result.data["wikibaseList"]["data"][0] diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 5279dece..ea7e951a 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,5 +1,6 @@ """Test Utilities""" +from tests.utils.assert_meta import assert_page_meta from tests.utils.assert_property_value import ( assert_layered_property_count, assert_layered_property_value, diff --git a/tests/utils/assert_meta.py b/tests/utils/assert_meta.py new file mode 100644 index 00000000..af984182 --- /dev/null +++ b/tests/utils/assert_meta.py @@ -0,0 +1,19 @@ +"""Assert Page Metadata""" + +from tests.utils.assert_property_value import assert_layered_property_value + + +def assert_page_meta( + page: dict, + expected_page_number: int, + expected_page_size: int, + expected_total_count: int, + expected_total_pages: int, +): + """Assert Page Metadata""" + + assert "meta" in page + assert_layered_property_value(page, ["meta", "pageNumber"], expected_page_number) + assert_layered_property_value(page, ["meta", "pageSize"], expected_page_size) + assert_layered_property_value(page, ["meta", "totalCount"], expected_total_count) + assert_layered_property_value(page, ["meta", "totalPages"], expected_total_pages) diff --git a/tests/utils/mock_response.py b/tests/utils/mock_response.py index 6123a33f..ffd8bf85 100644 --- a/tests/utils/mock_response.py +++ b/tests/utils/mock_response.py @@ -1,12 +1,22 @@ """Mock HTTP Response""" +from typing import Optional + class MockResponse: """Mock HTTP Response""" status_code: int content: bytes + url: str + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + pass - def __init__(self, status_code: int, content: bytes): + def __init__(self, url: str, status_code: int, content: Optional[bytes] = None): + self.url = url self.content = content self.status_code = status_code