diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index b7b44bfc..8c7c1f04 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -10,8 +10,6 @@ jobs:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v3
- with:
- python-version: "3.10.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
@@ -25,8 +23,6 @@ jobs:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v3
- with:
- python-version: "3.10.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
@@ -40,8 +36,6 @@ jobs:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v3
- with:
- python-version: "3.10.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
@@ -55,8 +49,6 @@ jobs:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v3
- with:
- python-version: "3.10.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
diff --git a/alembic/versions/0833cb54870f_software_model.py b/alembic/versions/0833cb54870f_software_model.py
new file mode 100644
index 00000000..92f1304a
--- /dev/null
+++ b/alembic/versions/0833cb54870f_software_model.py
@@ -0,0 +1,107 @@
+"""Software Model
+
+Revision ID: 0833cb54870f
+Revises: e3ecd40e3851
+Create Date: 2024-10-16 23:03:09.901779
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "0833cb54870f"
+down_revision: Union[str, None] = "e3ecd40e3851"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.create_table(
+ "wikibase_software",
+ sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+ sa.Column(
+ "software_type",
+ sa.Enum(
+ "SOFTWARE", "SKIN", "EXTENSION", "LIBRARY", name="wikibasesoftwaretype"
+ ),
+ nullable=False,
+ ),
+ sa.Column("software_name", sa.String(), nullable=False),
+ sa.Column("url", sa.String(), nullable=True),
+ sa.Column("description", sa.String(), nullable=True),
+ sa.Column("latest_version", sa.String(), nullable=True),
+ sa.Column("quarterly_download_count", sa.Integer(), nullable=True),
+ sa.Column("public_wiki_count", sa.Integer(), nullable=True),
+ sa.Column("mw_bundled", sa.Boolean(), nullable=True),
+ sa.PrimaryKeyConstraint("id"),
+ sa.UniqueConstraint(
+ "software_type", "software_name", name="unique_software_type_name"
+ ),
+ )
+
+ op.create_table(
+ "wikibase_software_tag",
+ sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+ sa.Column("tag", sa.String(), nullable=False),
+ sa.PrimaryKeyConstraint("id"),
+ )
+
+ op.create_table(
+ "wikibase_software_tag_xref",
+ sa.Column("wikibase_software_id", sa.Integer(), nullable=False),
+ sa.Column("wikibase_software_tag_id", sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(
+ ["wikibase_software_id"],
+ ["wikibase_software.id"],
+ name="foreign_wikibase_software_id",
+ ),
+ sa.ForeignKeyConstraint(
+ ["wikibase_software_tag_id"],
+ ["wikibase_software_tag.id"],
+ name="foreign_wikibase_software_tag_id",
+ ),
+ sa.PrimaryKeyConstraint("wikibase_software_id", "wikibase_software_tag_id"),
+ )
+
+ with op.batch_alter_table("wikibase_software_version") as batch_op:
+ batch_op.add_column(
+ sa.Column("wikibase_software_id", sa.Integer(), nullable=True)
+ )
+ batch_op.create_foreign_key(
+ "software", "wikibase_software", ["wikibase_software_id"], ["id"]
+ )
+ with op.get_bind() as conn:
+ conn.execute(
+ sa.text(
+ """INSERT INTO wikibase_software (software_type, software_name)
+SELECT DISTINCT software_type, software_name
+FROM wikibase_software_version
+ORDER BY software_type, software_name"""
+ )
+ )
+ conn.execute(
+ sa.text(
+ """UPDATE wikibase_software_version
+SET wikibase_software_id = wikibase_software.id
+FROM wikibase_software
+WHERE wikibase_software_version.software_name = wikibase_software.software_name AND wikibase_software_version.software_type = wikibase_software.software_type"""
+ )
+ )
+ conn.commit()
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("wikibase_software_version") as batch_op:
+ batch_op.drop_constraint("software", type_="foreignkey")
+ batch_op.drop_column("wikibase_software_id")
+ op.drop_table("wikibase_software_tag_xref")
+ op.drop_table("wikibase_software_tag")
+ op.drop_table("wikibase_software")
+ # ### end Alembic commands ###
diff --git a/alembic/versions/3b5538b2faec_fetched.py b/alembic/versions/3b5538b2faec_fetched.py
new file mode 100644
index 00000000..e49d4341
--- /dev/null
+++ b/alembic/versions/3b5538b2faec_fetched.py
@@ -0,0 +1,34 @@
+"""Fetched
+
+Revision ID: 3b5538b2faec
+Revises: 7fd538ca8735
+Create Date: 2024-10-17 12:36:32.239393
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "3b5538b2faec"
+down_revision: Union[str, None] = "7fd538ca8735"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column(
+ "wikibase_software",
+ sa.Column("fetched", sa.DateTime(timezone=True), nullable=True),
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("wikibase_software", "fetched")
+ # ### end Alembic commands ###
diff --git a/alembic/versions/7fd538ca8735_no_name_or_type.py b/alembic/versions/7fd538ca8735_no_name_or_type.py
new file mode 100644
index 00000000..937aabc6
--- /dev/null
+++ b/alembic/versions/7fd538ca8735_no_name_or_type.py
@@ -0,0 +1,53 @@
+"""No Name or Type
+
+Revision ID: 7fd538ca8735
+Revises: bf635fa3a7ce
+Create Date: 2024-10-17 00:13:58.311963
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "7fd538ca8735"
+down_revision: Union[str, None] = "bf635fa3a7ce"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("wikibase_software_version") as batch_op:
+ batch_op.drop_constraint(
+ "unique_observation_software_type_name", type_="unique"
+ )
+ batch_op.create_unique_constraint(
+ "unique_observation_software_id",
+ ["wikibase_software_version_observation_id", "wikibase_software_id"],
+ )
+ batch_op.drop_column("software_type")
+ batch_op.drop_column("software_name")
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("wikibase_software_version") as batch_op:
+ batch_op.add_column(sa.Column("software_name", sa.VARCHAR(), nullable=False))
+ batch_op.add_column(
+ sa.Column("software_type", sa.VARCHAR(length=9), nullable=False)
+ )
+ batch_op.drop_constraint("unique_observation_software_id", type_="unique")
+ batch_op.create_unique_constraint(
+ "unique_observation_software_type_name",
+ [
+ "wikibase_software_version_observation_id",
+ "software_type",
+ "software_name",
+ ],
+ )
+ # ### end Alembic commands ###
diff --git a/alembic/versions/bf635fa3a7ce_mandatory_software_relationship.py b/alembic/versions/bf635fa3a7ce_mandatory_software_relationship.py
new file mode 100644
index 00000000..79522c70
--- /dev/null
+++ b/alembic/versions/bf635fa3a7ce_mandatory_software_relationship.py
@@ -0,0 +1,37 @@
+"""Mandatory Software Relationship
+
+Revision ID: bf635fa3a7ce
+Revises: 0833cb54870f
+Create Date: 2024-10-16 23:57:48.573469
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "bf635fa3a7ce"
+down_revision: Union[str, None] = "0833cb54870f"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("wikibase_software_version") as batch_op:
+ batch_op.alter_column(
+ "wikibase_software_id", existing_type=sa.INTEGER(), nullable=False
+ )
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("wikibase_software_version") as batch_op:
+ batch_op.alter_column(
+ "wikibase_software_id", existing_type=sa.INTEGER(), nullable=True
+ )
+ # ### end Alembic commands ###
diff --git a/alembic/versions/ced691eaf66e_archived.py b/alembic/versions/ced691eaf66e_archived.py
new file mode 100644
index 00000000..04907b78
--- /dev/null
+++ b/alembic/versions/ced691eaf66e_archived.py
@@ -0,0 +1,37 @@
+"""Archived
+
+Revision ID: ced691eaf66e
+Revises: 3b5538b2faec
+Create Date: 2024-10-17 16:19:32.355862
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "ced691eaf66e"
+down_revision: Union[str, None] = "3b5538b2faec"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("wikibase_software") as batch_op:
+ batch_op.add_column(sa.Column("archived", sa.Boolean(), nullable=True))
+ with op.batch_alter_table("wikibase_software_tag") as batch_op:
+ batch_op.create_unique_constraint("unique_tag", ["tag"])
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ with op.batch_alter_table("wikibase_software_tag") as batch_op:
+ batch_op.drop_constraint("unique_tag", type_="unique")
+ with op.batch_alter_table("wikibase_software") as batch_op:
+ batch_op.drop_column("archived")
+ # ### end Alembic commands ###
diff --git a/data/database_connection.py b/data/database_connection.py
index f2fc521c..c659df3b 100644
--- a/data/database_connection.py
+++ b/data/database_connection.py
@@ -1,7 +1,7 @@
"""Database Connection"""
+from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
-from typing import AsyncGenerator
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import sessionmaker
diff --git a/data/gx/checkpoints/alembic_version_checkpoint b/data/gx/checkpoints/alembic_version_checkpoint.json
similarity index 100%
rename from data/gx/checkpoints/alembic_version_checkpoint
rename to data/gx/checkpoints/alembic_version_checkpoint.json
diff --git a/data/gx/checkpoints/wikibase_checkpoint b/data/gx/checkpoints/wikibase_checkpoint.json
similarity index 100%
rename from data/gx/checkpoints/wikibase_checkpoint
rename to data/gx/checkpoints/wikibase_checkpoint.json
diff --git a/data/gx/checkpoints/wikibase_connectivity_observation_checkpoint b/data/gx/checkpoints/wikibase_connectivity_observation_checkpoint.json
similarity index 100%
rename from data/gx/checkpoints/wikibase_connectivity_observation_checkpoint
rename to data/gx/checkpoints/wikibase_connectivity_observation_checkpoint.json
diff --git a/data/gx/checkpoints/wikibase_log_observation_checkpoint b/data/gx/checkpoints/wikibase_log_observation_checkpoint.json
similarity index 100%
rename from data/gx/checkpoints/wikibase_log_observation_checkpoint
rename to data/gx/checkpoints/wikibase_log_observation_checkpoint.json
diff --git a/data/gx/checkpoints/wikibase_property_usage_observation_checkpoint b/data/gx/checkpoints/wikibase_property_usage_observation_checkpoint.json
similarity index 100%
rename from data/gx/checkpoints/wikibase_property_usage_observation_checkpoint
rename to data/gx/checkpoints/wikibase_property_usage_observation_checkpoint.json
diff --git a/data/gx/checkpoints/wikibase_quantity_observation_checkpoint b/data/gx/checkpoints/wikibase_quantity_observation_checkpoint.json
similarity index 100%
rename from data/gx/checkpoints/wikibase_quantity_observation_checkpoint
rename to data/gx/checkpoints/wikibase_quantity_observation_checkpoint.json
diff --git a/data/gx/checkpoints/wikibase_software_checkpoint.json b/data/gx/checkpoints/wikibase_software_checkpoint.json
new file mode 100644
index 00000000..82fc4d41
--- /dev/null
+++ b/data/gx/checkpoints/wikibase_software_checkpoint.json
@@ -0,0 +1,30 @@
+{
+ "actions": [
+ {
+ "name": "update_all_data_docs",
+ "site_names": [],
+ "type": "update_data_docs"
+ }
+ ],
+ "id": "ff6c1445-1da6-4b3e-af20-6947ce253bd2",
+ "name": "wikibase_software_checkpoint",
+ "result_format": { "result_format": "COMPLETE" },
+ "validation_definitions": [
+ {
+ "id": "76869a4a-b823-4ccd-b091-2e4dff6befee",
+ "name": "wikibase_software_validation_definition"
+ },
+ {
+ "id": "fa8b2a52-4651-4bec-9e42-7af94e5faca7",
+ "name": "wikibase_software_tag_validation_definition"
+ },
+ {
+ "id": "0fbc7263-c0d9-4390-884d-445bf59a0647",
+ "name": "wikibase_software_tag_xref_validation_definition"
+ },
+ {
+ "id": "8768e1d1-11d5-44a6-8f5d-0a03d1d535d4",
+ "name": "populated_wikibase_software_validation_definition"
+ }
+ ]
+}
diff --git a/data/gx/checkpoints/wikibase_software_version_observation_checkpoint b/data/gx/checkpoints/wikibase_software_version_observation_checkpoint.json
similarity index 100%
rename from data/gx/checkpoints/wikibase_software_version_observation_checkpoint
rename to data/gx/checkpoints/wikibase_software_version_observation_checkpoint.json
diff --git a/data/gx/checkpoints/wikibase_statistics_observation_checkpoint b/data/gx/checkpoints/wikibase_statistics_observation_checkpoint
deleted file mode 100644
index e816e1a9..00000000
--- a/data/gx/checkpoints/wikibase_statistics_observation_checkpoint
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "actions": [
- {
- "name": "update_all_data_docs",
- "site_names": [],
- "type": "update_data_docs"
- }
- ],
- "id": "921797b3-b57f-47e2-bb02-e178b4e06cb5",
- "name": "wikibase_statistics_observation_checkpoint",
- "result_format": {
- "result_format": "COMPLETE"
- },
- "validation_definitions": [
- {
- "id": "7212ff2d-cc83-493a-a983-1c69dfff7c12",
- "name": "obs_wikibase_statistics_observation_validation_definition"
- },
- {
- "id": "9ea78ef4-0641-485e-a972-219b43b24e88",
- "name": "wikibase_statistics_observation_validation_definition"
- },
- {
- "id": "d8fba0be-ef0d-472e-823d-a1da6c7ff97a",
- "name": "valid_wikibase_statistics_observation_validation_definition"
- }
- ]
-}
\ No newline at end of file
diff --git a/data/gx/checkpoints/wikibase_statistics_observation_checkpoint.json b/data/gx/checkpoints/wikibase_statistics_observation_checkpoint.json
new file mode 100644
index 00000000..a6dbff86
--- /dev/null
+++ b/data/gx/checkpoints/wikibase_statistics_observation_checkpoint.json
@@ -0,0 +1,26 @@
+{
+ "actions": [
+ {
+ "name": "update_all_data_docs",
+ "site_names": [],
+ "type": "update_data_docs"
+ }
+ ],
+ "id": "921797b3-b57f-47e2-bb02-e178b4e06cb5",
+ "name": "wikibase_statistics_observation_checkpoint",
+ "result_format": { "result_format": "COMPLETE" },
+ "validation_definitions": [
+ {
+ "id": "7212ff2d-cc83-493a-a983-1c69dfff7c12",
+ "name": "obs_wikibase_statistics_observation_validation_definition"
+ },
+ {
+ "id": "9ea78ef4-0641-485e-a972-219b43b24e88",
+ "name": "wikibase_statistics_observation_validation_definition"
+ },
+ {
+ "id": "d8fba0be-ef0d-472e-823d-a1da6c7ff97a",
+ "name": "valid_wikibase_statistics_observation_validation_definition"
+ }
+ ]
+}
diff --git a/data/gx/checkpoints/wikibase_user_observation_checkpoint b/data/gx/checkpoints/wikibase_user_observation_checkpoint.json
similarity index 100%
rename from data/gx/checkpoints/wikibase_user_observation_checkpoint
rename to data/gx/checkpoints/wikibase_user_observation_checkpoint.json
diff --git a/data/gx/expectations/populated_wikibase_software_expetation_suite.json b/data/gx/expectations/populated_wikibase_software_expetation_suite.json
new file mode 100644
index 00000000..63a56b62
--- /dev/null
+++ b/data/gx/expectations/populated_wikibase_software_expetation_suite.json
@@ -0,0 +1,158 @@
+{
+ "expectations": [
+ {
+ "id": "5186d0bf-cd58-4e83-8ac9-cf1cc2274385",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "2436ef29-36c5-4559-93da-8b926017a00a",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "6ca95d62-9eeb-4435-9817-21a007dd2368",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_values_to_be_unique"
+ },
+ {
+ "id": "a04ae020-0d60-4a80-ae92-5ed52885870b",
+ "kwargs": { "column": "software_type" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "17ac7195-315c-431f-b328-5d758113a7d3",
+ "kwargs": { "column": "software_type" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "da8f1e6a-66e2-41fd-9b5f-6fdd158a4717",
+ "kwargs": { "column": "software_name" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "32acaecc-fa3b-4cfb-a5c9-aca71ad2b345",
+ "kwargs": { "column": "software_name" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "f316d5dd-3974-4007-9865-2fbe51edcd11",
+ "kwargs": { "column_list": ["software_type", "software_name"] },
+ "meta": {},
+ "type": "expect_compound_columns_to_be_unique"
+ },
+ {
+ "id": "2bf6c7a8-44f9-4566-9e57-f0459ab758f4",
+ "kwargs": { "column": "url" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "be18048d-ff1a-4d83-9389-2b4be7cd9b90",
+ "kwargs": { "column": "url" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "fc428b2f-bfbd-4501-9058-fb8899685639",
+ "kwargs": { "column": "url" },
+ "meta": {},
+ "type": "expect_column_values_to_be_unique"
+ },
+ {
+ "id": "9182c8f3-080e-43fb-bb24-f25c352ec709",
+ "kwargs": { "column": "description" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "1eab6823-0c11-4606-b98a-24cfddd0fa21",
+ "kwargs": { "column": "description", "mostly": 0.7 },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "0e19eae2-39aa-4b09-bc33-444ba2475d6b",
+ "kwargs": { "column": "latest_version" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "0f0a53ab-916c-4a52-b6bf-dd787bfe7c71",
+ "kwargs": { "column": "latest_version", "mostly": 0.6 },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "5da516dc-5044-42ec-9d62-33f897d20f30",
+ "kwargs": { "column": "quarterly_download_count" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "479da3dc-7d7d-41f1-88ac-aaf89a7fece1",
+ "kwargs": { "column": "quarterly_download_count", "mostly": 0.25 },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "9862d304-019a-432b-bcaa-8fe914225c5a",
+ "kwargs": { "column": "public_wiki_count" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "c9aaffa5-b706-463e-b00d-02ba8ccf310c",
+ "kwargs": { "column": "public_wiki_count", "mostly": 0.25 },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "e0e7e43b-9df0-4f3c-9e95-ea6813f45ddd",
+ "kwargs": { "column": "mw_bundled" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "677459aa-3f9c-46c4-9d79-e711adeb2f88",
+ "kwargs": { "column": "mw_bundled", "mostly": 0.75 },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "4fc45491-e0b9-4a74-bf70-9886614821ba",
+ "kwargs": { "column": "fetched" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "ef8dc31f-5b13-4abe-8b97-d9ecc54332fa",
+ "kwargs": { "column": "fetched" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "bfd95bd5-0756-4133-8d4a-8ccd58ca5107",
+ "kwargs": { "column": "archived" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "bf643ee9-b0eb-4500-92ea-c49d22adccf6",
+ "kwargs": { "column": "archived", "mostly": 0.75 },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ }
+ ],
+ "id": "a4fe5870-c78d-433d-af64-76b02603e528",
+ "meta": { "great_expectations_version": "1.2.0" },
+ "name": "populated_wikibase_software_expetation_suite",
+ "notes": null
+}
diff --git a/data/gx/expectations/wikibase_expectation_suite.json b/data/gx/expectations/wikibase_expectation_suite.json
index 500a17ec..4c99be5f 100644
--- a/data/gx/expectations/wikibase_expectation_suite.json
+++ b/data/gx/expectations/wikibase_expectation_suite.json
@@ -34,7 +34,12 @@
"id": "5342a046-55a7-4a61-908e-c461cd4a84bf",
"kwargs": {
"column": "wikibase_name",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
@@ -49,7 +54,12 @@
"id": "d4ffc143-e54a-4ad9-af88-b5c38eee60a4",
"kwargs": {
"column": "organization",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
@@ -64,7 +74,12 @@
"id": "43da03cf-e685-4bbd-979c-ffd8e16aeed7",
"kwargs": {
"column": "country",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
@@ -79,7 +94,12 @@
"id": "9ce6b575-0119-4dac-8d65-d1a210e6f423",
"kwargs": {
"column": "region",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
@@ -136,7 +156,12 @@
"id": "ca289730-c632-4991-81c5-f31eef1cee75",
"kwargs": {
"column": "description",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
diff --git a/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json b/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json
index 07b52a34..bb2abd90 100644
--- a/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json
+++ b/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json
@@ -46,7 +46,12 @@
"id": "77b293b1-73c2-437a-a035-3adfb66294bd",
"kwargs": {
"column": "property_url",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
diff --git a/data/gx/expectations/wikibase_software_expectation_suite.json b/data/gx/expectations/wikibase_software_expectation_suite.json
new file mode 100644
index 00000000..d1fdffa9
--- /dev/null
+++ b/data/gx/expectations/wikibase_software_expectation_suite.json
@@ -0,0 +1,181 @@
+{
+ "expectations": [
+ {
+ "id": "e388d561-3614-4994-b20b-a1d35702c1c1",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "296c30ba-224d-4563-a052-9c210044b78a",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "49df4432-ef6f-4e00-a415-c7f15bebf3f6",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_values_to_be_unique"
+ },
+ {
+ "id": "e6289c50-751a-4a0c-a9f5-9ac59402c297",
+ "kwargs": { "column": "software_type" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "1016719e-6730-47e6-93c8-d2ad8d27abfd",
+ "kwargs": { "column": "software_type" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "f1829a39-90d6-4b1d-bc3b-8c7c16775258",
+ "kwargs": {
+ "column": "software_type",
+ "value_set": ["SOFTWARE", "SKIN", "EXTENSION", "LIBRARY"]
+ },
+ "meta": {},
+ "type": "expect_column_distinct_values_to_be_in_set"
+ },
+ {
+ "id": "a4985482-eaa0-47ae-876b-f87a80f9fb2e",
+ "kwargs": { "column": "software_name" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "0c1a773f-7e7f-4ad2-8faa-c3c8541abf3f",
+ "kwargs": { "column": "software_name" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "5c029502-61a0-4af2-b963-7754ea0ebf75",
+ "kwargs": {
+ "column": "software_name",
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
+ },
+ "meta": {},
+ "type": "expect_column_values_to_not_match_regex_list"
+ },
+ {
+ "id": "f3c3a1c5-06d5-46a7-bc2e-40ebdee39146",
+ "kwargs": { "column_list": ["software_type", "software_name"] },
+ "meta": {},
+ "type": "expect_compound_columns_to_be_unique"
+ },
+ {
+ "id": "1ea89a08-334a-4e8a-a75a-c4c6bec61b92",
+ "kwargs": { "column": "url" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "fb215a29-24be-46ff-936e-5c40d59b386a",
+ "kwargs": {
+ "column": "url",
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
+ },
+ "meta": {},
+ "type": "expect_column_values_to_not_match_regex_list"
+ },
+ {
+ "id": "22e597ee-01d9-4486-bebe-b2e977a10f0a",
+ "kwargs": { "column": "description" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "aeaf5287-bbae-4a3b-9a2f-761b7e052629",
+ "kwargs": {
+ "column": "description",
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
+ },
+ "meta": {},
+ "type": "expect_column_values_to_not_match_regex_list"
+ },
+ {
+ "id": "e7002ac0-cd25-4a4c-ad65-d56ae2c894fd",
+ "kwargs": { "column": "latest_version" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "54d43eee-b079-4448-a90c-2162aa5dfc71",
+ "kwargs": {
+ "column": "latest_version",
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
+ },
+ "meta": {},
+ "type": "expect_column_values_to_not_match_regex_list"
+ },
+ {
+ "id": "4053d987-b652-4211-9cf4-10ec31958c33",
+ "kwargs": { "column": "quarterly_download_count" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "ed02306d-f11f-40b5-b85f-54ac7fbc1181",
+ "kwargs": { "column": "public_wiki_count" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "4ade6cb9-8dc3-4c93-b3cc-93b1163201a5",
+ "kwargs": { "column": "mw_bundled" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "7602df3c-485c-401d-8cb2-b703c511a8e1",
+ "kwargs": { "column": "mw_bundled", "value_set": [false, true] },
+ "meta": {},
+ "type": "expect_column_values_to_be_in_set"
+ },
+ {
+ "id": "a14ce5ac-562d-405d-85bc-7da4a5e07888",
+ "kwargs": { "column": "fetched" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "fc19a549-a4ef-4aba-847b-f0c3efed2b77",
+ "kwargs": { "column": "archived" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "23a2d72e-5309-49c1-af1c-e495022580ca",
+ "kwargs": { "column": "archived", "value_set": [false, true] },
+ "meta": {},
+ "type": "expect_column_values_to_be_in_set"
+ }
+ ],
+ "id": "072a1275-09e6-4835-85f7-8f55e99dcc3e",
+ "meta": { "great_expectations_version": "1.1.3" },
+ "name": "wikibase_software_expectation_suite",
+ "notes": null
+}
diff --git a/data/gx/expectations/wikibase_software_tag_expetation_suite.json b/data/gx/expectations/wikibase_software_tag_expetation_suite.json
new file mode 100644
index 00000000..fe7ffa90
--- /dev/null
+++ b/data/gx/expectations/wikibase_software_tag_expetation_suite.json
@@ -0,0 +1,58 @@
+{
+ "expectations": [
+ {
+ "id": "6d2611be-976c-436c-955f-443992875712",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "24d9c05f-8a84-4aa8-bcba-89e4d1c78cb1",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "d858a203-8efe-4b41-b4fc-b9fe62cee442",
+ "kwargs": { "column": "id" },
+ "meta": {},
+ "type": "expect_column_values_to_be_unique"
+ },
+ {
+ "id": "2747ff12-3259-4071-825d-21330e6db6fa",
+ "kwargs": { "column": "tag" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "c322e1c0-b923-4dd7-95e8-464d2b05edd0",
+ "kwargs": { "column": "tag" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "5d4645fd-e469-43da-a7f5-ab7e4222fdf7",
+ "kwargs": { "column": "tag" },
+ "meta": {},
+ "type": "expect_column_values_to_be_unique"
+ },
+ {
+ "id": "5baeb31e-aa99-46b2-bd6e-035155afcbfc",
+ "kwargs": {
+ "column": "tag",
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
+ },
+ "meta": {},
+ "type": "expect_column_values_to_not_match_regex_list"
+ }
+ ],
+ "id": "920eb0d9-1590-45e4-ad9b-29c41fdcfce0",
+ "meta": { "great_expectations_version": "1.1.3" },
+ "name": "wikibase_software_tag_expetation_suite",
+ "notes": null
+}
diff --git a/data/gx/expectations/wikibase_software_tag_xref_expetation_suite.json b/data/gx/expectations/wikibase_software_tag_xref_expetation_suite.json
new file mode 100644
index 00000000..f264161f
--- /dev/null
+++ b/data/gx/expectations/wikibase_software_tag_xref_expetation_suite.json
@@ -0,0 +1,50 @@
+{
+ "expectations": [
+ {
+ "id": "acdb75b6-decb-444e-8a79-ea85b911cb1c",
+ "kwargs": {
+ "column": "wikibase_software_id"
+ },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "2c8375c1-d080-498f-8705-088b75976607",
+ "kwargs": {
+ "column": "wikibase_software_id"
+ },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "0d2c8f51-e24b-4d38-be35-6605062c0a54",
+ "kwargs": {
+ "column": "wikibase_software_tag_id"
+ },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "86d99958-557a-41fe-a78f-761397525dc2",
+ "kwargs": {
+ "column": "wikibase_software_tag_id"
+ },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "96879ea2-1933-401b-be26-55b1a3517013",
+ "kwargs": {
+ "column_list": ["wikibase_software_id", "wikibase_software_tag_id"]
+ },
+ "meta": {},
+ "type": "expect_compound_columns_to_be_unique"
+ }
+ ],
+ "id": "e9bf1186-f8b1-4f4a-835f-14e6d1ee6bb5",
+ "meta": {
+ "great_expectations_version": "1.1.3"
+ },
+ "name": "wikibase_software_tag_xref_expetation_suite",
+ "notes": null
+}
diff --git a/data/gx/expectations/wikibase_software_version_expectation_suite.json b/data/gx/expectations/wikibase_software_version_expectation_suite.json
index f8f894f0..d3a3fc24 100644
--- a/data/gx/expectations/wikibase_software_version_expectation_suite.json
+++ b/data/gx/expectations/wikibase_software_version_expectation_suite.json
@@ -30,60 +30,6 @@
"meta": {},
"type": "expect_column_values_to_not_be_null"
},
- {
- "id": "c4616471-93e0-4df0-abbe-9bba1619592f",
- "kwargs": { "column": "software_type" },
- "meta": {},
- "type": "expect_column_to_exist"
- },
- {
- "id": "a96e0d04-a0aa-4b24-b27a-c006afcb2012",
- "kwargs": { "column": "software_type" },
- "meta": {},
- "type": "expect_column_values_to_not_be_null"
- },
- {
- "id": "3fa3ac0b-0a7e-4132-81bd-3e998f35be3b",
- "kwargs": {
- "column": "software_type",
- "value_set": ["SOFTWARE", "SKIN", "EXTENSION", "LIBRARY"]
- },
- "meta": {},
- "type": "expect_column_distinct_values_to_be_in_set"
- },
- {
- "id": "e5cc3f78-9cd7-4087-a880-b2c9616811d2",
- "kwargs": { "column": "software_name" },
- "meta": {},
- "type": "expect_column_to_exist"
- },
- {
- "id": "daef9801-92bb-41af-a081-6d6ba2c239de",
- "kwargs": { "column": "software_name" },
- "meta": {},
- "type": "expect_column_values_to_not_be_null"
- },
- {
- "id": "b852e073-ad49-4028-a42a-217448fde29b",
- "kwargs": {
- "column": "software_name",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
- },
- "meta": {},
- "type": "expect_column_values_to_not_match_regex_list"
- },
- {
- "id": "1a591246-4126-4545-8a35-74960762bca8",
- "kwargs": {
- "column_list": [
- "wikibase_software_version_observation_id",
- "software_type",
- "software_name"
- ]
- },
- "meta": {},
- "type": "expect_compound_columns_to_be_unique"
- },
{
"id": "755e9f43-a29d-4cd7-a290-6c302d4f0fb9",
"kwargs": { "column": "version" },
@@ -94,7 +40,12 @@
"id": "78894d21-05f1-470d-931b-3efe46132d2e",
"kwargs": {
"column": "version",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
@@ -116,6 +67,29 @@
"kwargs": { "column": "version_date" },
"meta": {},
"type": "expect_column_to_exist"
+ },
+ {
+ "id": "db3ceb44-e8bc-4bb6-ad37-64ba3b6c87d3",
+ "kwargs": { "column": "wikibase_software_id" },
+ "meta": {},
+ "type": "expect_column_to_exist"
+ },
+ {
+ "id": "9277e198-b784-47cd-88a2-576d547a99b9",
+ "kwargs": { "column": "wikibase_software_id" },
+ "meta": {},
+ "type": "expect_column_values_to_not_be_null"
+ },
+ {
+ "id": "e7f9a0eb-32f5-4f89-a3b7-3782afe81433",
+ "kwargs": {
+ "column_list": [
+ "wikibase_software_id",
+ "wikibase_software_version_observation_id"
+ ]
+ },
+ "meta": {},
+ "type": "expect_compound_columns_to_be_unique"
}
],
"id": "dc5cb675-5725-46cb-940d-df7a04a56257",
diff --git a/data/gx/expectations/wikibase_url_expectation_suite.json b/data/gx/expectations/wikibase_url_expectation_suite.json
index 5565f56b..b2052dfa 100644
--- a/data/gx/expectations/wikibase_url_expectation_suite.json
+++ b/data/gx/expectations/wikibase_url_expectation_suite.json
@@ -81,7 +81,12 @@
"id": "31ae4f4f-55f0-4ac7-8e0d-3f17fe4fe823",
"kwargs": {
"column": "url",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
diff --git a/data/gx/expectations/wikibase_user_group_expectation_suite.json b/data/gx/expectations/wikibase_user_group_expectation_suite.json
index c0c2ce88..fe9883a7 100644
--- a/data/gx/expectations/wikibase_user_group_expectation_suite.json
+++ b/data/gx/expectations/wikibase_user_group_expectation_suite.json
@@ -40,7 +40,12 @@
"id": "67a3cb18-3559-471d-ba6c-ef4385a9b96b",
"kwargs": {
"column": "group_name",
- "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"]
+ "regex_list": [
+ "^[ \t\r\n]*$",
+ "^[ \t\r\n]+",
+ "[ \t\r\n]+$",
+ "[ \t\r\n]{2,}"
+ ]
},
"meta": {},
"type": "expect_column_values_to_not_match_regex_list"
diff --git a/data/gx/great_expectations.yml b/data/gx/great_expectations.yml
index f04b6c67..e2977c13 100644
--- a/data/gx/great_expectations.yml
+++ b/data/gx/great_expectations.yml
@@ -360,5 +360,44 @@ fluent_datasources:
id: 1ae4e957-b4b4-42fc-b22a-381ea6e8bceb
partitioner:
query: SELECT * FROM wikibase_log_observation_month WHERE log_count > 0
+ wikibase_software_table:
+ type: table
+ id: c93da63e-392d-443c-879e-902c2692b41e
+ batch_metadata: {}
+ batch_definitions:
+ FULL_TABLE:
+ id: 550b72d7-9aca-40cc-9d2a-7f94fcb44828
+ partitioner:
+ table_name: wikibase_software
+ schema_name:
+ wikibase_software_tag_table:
+ type: table
+ id: d27a75dc-0e2e-43b2-91da-a1f2e9e0590a
+ batch_metadata: {}
+ batch_definitions:
+ FULL_TABLE:
+ id: 3b69d250-8f9a-4d11-b5a2-82463d829103
+ partitioner:
+ table_name: wikibase_software_tag
+ schema_name:
+ wikibase_software_tag_xref_table:
+ type: table
+ id: 7f899383-9ce6-4afa-8dbb-0e49c30f3fd5
+ batch_metadata: {}
+ batch_definitions:
+ FULL_TABLE:
+ id: ce729967-02a8-4e8f-b43a-c0d4ac078e44
+ partitioner:
+ table_name: wikibase_software_tag_xref
+ schema_name:
+ populated_wikibase_software_records:
+ type: query
+ id: 97690d4d-e26a-4c89-807c-98e2a760e648
+ batch_metadata: {}
+ batch_definitions:
+ FULL_TABLE:
+ id: 17906408-5608-45ff-95fe-45ae29b85630
+ partitioner:
+ query: SELECT * FROM wikibase_software WHERE url IS NOT NULL
connection_string: sqlite:///data/wikibase-data.db
data_context_id: 14fd50bc-47c8-4033-9643-e60c471dc6c5
diff --git a/data/gx/validation_definitions/alembic_version_validation_definition b/data/gx/validation_definitions/alembic_version_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/alembic_version_validation_definition
rename to data/gx/validation_definitions/alembic_version_validation_definition.json
diff --git a/data/gx/validation_definitions/nonzero_wikibase_connectivity_observation_validation_definition b/data/gx/validation_definitions/nonzero_wikibase_connectivity_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/nonzero_wikibase_connectivity_observation_validation_definition
rename to data/gx/validation_definitions/nonzero_wikibase_connectivity_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/nonzero_wikibase_log_observation_month_validation_definition b/data/gx/validation_definitions/nonzero_wikibase_log_observation_month_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/nonzero_wikibase_log_observation_month_validation_definition
rename to data/gx/validation_definitions/nonzero_wikibase_log_observation_month_validation_definition.json
diff --git a/data/gx/validation_definitions/obs_wikibase_connectivity_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_connectivity_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/obs_wikibase_connectivity_observation_validation_definition
rename to data/gx/validation_definitions/obs_wikibase_connectivity_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition
rename to data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/obs_wikibase_property_usage_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_property_usage_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/obs_wikibase_property_usage_observation_validation_definition
rename to data/gx/validation_definitions/obs_wikibase_property_usage_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/obs_wikibase_quantity_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_quantity_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/obs_wikibase_quantity_observation_validation_definition
rename to data/gx/validation_definitions/obs_wikibase_quantity_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/obs_wikibase_software_version_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_software_version_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/obs_wikibase_software_version_observation_validation_definition
rename to data/gx/validation_definitions/obs_wikibase_software_version_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition
deleted file mode 100644
index a6e80860..00000000
--- a/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "data": {
- "asset": {
- "id": "05cc691f-d972-4cdd-8172-86dc600cf18e",
- "name": "wikibase_statistics_observation_table"
- },
- "batch_definition": {
- "id": "96994a6a-aa72-4eb2-96b8-c285a7291b69",
- "name": "FULL_TABLE"
- },
- "datasource": {
- "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
- "name": "wikibase_datasource"
- }
- },
- "id": "7212ff2d-cc83-493a-a983-1c69dfff7c12",
- "name": "obs_wikibase_statistics_observation_validation_definition",
- "suite": {
- "id": "a5f967b2-e9e0-4b2c-8dd3-46acbbb2401a",
- "name": "wikibase_observation_expectation_suite"
- }
-}
\ No newline at end of file
diff --git a/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition.json b/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition.json
new file mode 100644
index 00000000..f1e67f91
--- /dev/null
+++ b/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition.json
@@ -0,0 +1,22 @@
+{
+ "data": {
+ "asset": {
+ "id": "05cc691f-d972-4cdd-8172-86dc600cf18e",
+ "name": "wikibase_statistics_observation_table"
+ },
+ "batch_definition": {
+ "id": "96994a6a-aa72-4eb2-96b8-c285a7291b69",
+ "name": "FULL_TABLE"
+ },
+ "datasource": {
+ "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
+ "name": "wikibase_datasource"
+ }
+ },
+ "id": "7212ff2d-cc83-493a-a983-1c69dfff7c12",
+ "name": "obs_wikibase_statistics_observation_validation_definition",
+ "suite": {
+ "id": "a5f967b2-e9e0-4b2c-8dd3-46acbbb2401a",
+ "name": "wikibase_observation_expectation_suite"
+ }
+}
diff --git a/data/gx/validation_definitions/obs_wikibase_user_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_user_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/obs_wikibase_user_observation_validation_definition
rename to data/gx/validation_definitions/obs_wikibase_user_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/populated_wikibase_software_validation_definition.json b/data/gx/validation_definitions/populated_wikibase_software_validation_definition.json
new file mode 100644
index 00000000..447856bb
--- /dev/null
+++ b/data/gx/validation_definitions/populated_wikibase_software_validation_definition.json
@@ -0,0 +1,22 @@
+{
+ "data": {
+ "asset": {
+ "id": "97690d4d-e26a-4c89-807c-98e2a760e648",
+ "name": "populated_wikibase_software_records"
+ },
+ "batch_definition": {
+ "id": "17906408-5608-45ff-95fe-45ae29b85630",
+ "name": "FULL_TABLE"
+ },
+ "datasource": {
+ "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
+ "name": "wikibase_datasource"
+ }
+ },
+ "id": "8768e1d1-11d5-44a6-8f5d-0a03d1d535d4",
+ "name": "populated_wikibase_software_validation_definition",
+ "suite": {
+ "id": "a4fe5870-c78d-433d-af64-76b02603e528",
+ "name": "populated_wikibase_software_expetation_suite"
+ }
+}
diff --git a/data/gx/validation_definitions/valid_wikibase_connectivity_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_connectivity_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/valid_wikibase_connectivity_observation_validation_definition
rename to data/gx/validation_definitions/valid_wikibase_connectivity_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition
rename to data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/valid_wikibase_quantity_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_quantity_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/valid_wikibase_quantity_observation_validation_definition
rename to data/gx/validation_definitions/valid_wikibase_quantity_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition
deleted file mode 100644
index 23e93f9a..00000000
--- a/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "data": {
- "asset": {
- "id": "d984b58f-385e-4939-aad2-d45f5432face",
- "name": "valid_wikibase_statistics_observations"
- },
- "batch_definition": {
- "id": "5c3e8bee-2993-4a54-94df-0ea1dbfb945f",
- "name": "FULL_TABLE"
- },
- "datasource": {
- "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
- "name": "wikibase_datasource"
- }
- },
- "id": "d8fba0be-ef0d-472e-823d-a1da6c7ff97a",
- "name": "valid_wikibase_statistics_observation_validation_definition",
- "suite": {
- "id": "ae836f37-1896-4b7f-a50a-9e2906c9592d",
- "name": "valid_wikibase_statistics_observation_expectation_suite"
- }
-}
\ No newline at end of file
diff --git a/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition.json b/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition.json
new file mode 100644
index 00000000..74da8d93
--- /dev/null
+++ b/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition.json
@@ -0,0 +1,22 @@
+{
+ "data": {
+ "asset": {
+ "id": "d984b58f-385e-4939-aad2-d45f5432face",
+ "name": "valid_wikibase_statistics_observations"
+ },
+ "batch_definition": {
+ "id": "5c3e8bee-2993-4a54-94df-0ea1dbfb945f",
+ "name": "FULL_TABLE"
+ },
+ "datasource": {
+ "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
+ "name": "wikibase_datasource"
+ }
+ },
+ "id": "d8fba0be-ef0d-472e-823d-a1da6c7ff97a",
+ "name": "valid_wikibase_statistics_observation_validation_definition",
+ "suite": {
+ "id": "ae836f37-1896-4b7f-a50a-9e2906c9592d",
+ "name": "valid_wikibase_statistics_observation_expectation_suite"
+ }
+}
diff --git a/data/gx/validation_definitions/valid_wikibase_user_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_user_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/valid_wikibase_user_observation_validation_definition
rename to data/gx/validation_definitions/valid_wikibase_user_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/valid_wikibase_validation_definition b/data/gx/validation_definitions/valid_wikibase_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/valid_wikibase_validation_definition
rename to data/gx/validation_definitions/valid_wikibase_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_category_validation_definition b/data/gx/validation_definitions/wikibase_category_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_category_validation_definition
rename to data/gx/validation_definitions/wikibase_category_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_connectivity_observation_item_relationship_count_validation_definition b/data/gx/validation_definitions/wikibase_connectivity_observation_item_relationship_count_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_connectivity_observation_item_relationship_count_validation_definition
rename to data/gx/validation_definitions/wikibase_connectivity_observation_item_relationship_count_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_connectivity_observation_object_relationship_count_validation_definition b/data/gx/validation_definitions/wikibase_connectivity_observation_object_relationship_count_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_connectivity_observation_object_relationship_count_validation_definition
rename to data/gx/validation_definitions/wikibase_connectivity_observation_object_relationship_count_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_connectivity_observation_validation_definition b/data/gx/validation_definitions/wikibase_connectivity_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_connectivity_observation_validation_definition
rename to data/gx/validation_definitions/wikibase_connectivity_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_log_observation_month_type_validation_definition b/data/gx/validation_definitions/wikibase_log_observation_month_type_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_log_observation_month_type_validation_definition
rename to data/gx/validation_definitions/wikibase_log_observation_month_type_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_log_observation_month_user_validation_definition b/data/gx/validation_definitions/wikibase_log_observation_month_user_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_log_observation_month_user_validation_definition
rename to data/gx/validation_definitions/wikibase_log_observation_month_user_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_log_observation_month_validation_definition b/data/gx/validation_definitions/wikibase_log_observation_month_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_log_observation_month_validation_definition
rename to data/gx/validation_definitions/wikibase_log_observation_month_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_log_observation_validation_definition b/data/gx/validation_definitions/wikibase_log_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_log_observation_validation_definition
rename to data/gx/validation_definitions/wikibase_log_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_property_usage_count_validation_definition b/data/gx/validation_definitions/wikibase_property_usage_count_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_property_usage_count_validation_definition
rename to data/gx/validation_definitions/wikibase_property_usage_count_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_quantity_observation_validation_definition b/data/gx/validation_definitions/wikibase_quantity_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_quantity_observation_validation_definition
rename to data/gx/validation_definitions/wikibase_quantity_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_software_tag_validation_definition.json b/data/gx/validation_definitions/wikibase_software_tag_validation_definition.json
new file mode 100644
index 00000000..217f5198
--- /dev/null
+++ b/data/gx/validation_definitions/wikibase_software_tag_validation_definition.json
@@ -0,0 +1,22 @@
+{
+ "data": {
+ "asset": {
+ "id": "d27a75dc-0e2e-43b2-91da-a1f2e9e0590a",
+ "name": "wikibase_software_tag_table"
+ },
+ "batch_definition": {
+ "id": "3b69d250-8f9a-4d11-b5a2-82463d829103",
+ "name": "FULL_TABLE"
+ },
+ "datasource": {
+ "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
+ "name": "wikibase_datasource"
+ }
+ },
+ "id": "fa8b2a52-4651-4bec-9e42-7af94e5faca7",
+ "name": "wikibase_software_tag_validation_definition",
+ "suite": {
+ "id": "920eb0d9-1590-45e4-ad9b-29c41fdcfce0",
+ "name": "wikibase_software_tag_expetation_suite"
+ }
+}
diff --git a/data/gx/validation_definitions/wikibase_software_tag_xref_validation_definition.json b/data/gx/validation_definitions/wikibase_software_tag_xref_validation_definition.json
new file mode 100644
index 00000000..20128db7
--- /dev/null
+++ b/data/gx/validation_definitions/wikibase_software_tag_xref_validation_definition.json
@@ -0,0 +1,22 @@
+{
+ "data": {
+ "asset": {
+ "id": "7f899383-9ce6-4afa-8dbb-0e49c30f3fd5",
+ "name": "wikibase_software_tag_xref_table"
+ },
+ "batch_definition": {
+ "id": "ce729967-02a8-4e8f-b43a-c0d4ac078e44",
+ "name": "FULL_TABLE"
+ },
+ "datasource": {
+ "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
+ "name": "wikibase_datasource"
+ }
+ },
+ "id": "0fbc7263-c0d9-4390-884d-445bf59a0647",
+ "name": "wikibase_software_tag_xref_validation_definition",
+ "suite": {
+ "id": "e9bf1186-f8b1-4f4a-835f-14e6d1ee6bb5",
+ "name": "wikibase_software_tag_xref_expetation_suite"
+ }
+}
diff --git a/data/gx/validation_definitions/wikibase_software_validation_definition.json b/data/gx/validation_definitions/wikibase_software_validation_definition.json
new file mode 100644
index 00000000..a1998e89
--- /dev/null
+++ b/data/gx/validation_definitions/wikibase_software_validation_definition.json
@@ -0,0 +1,22 @@
+{
+ "data": {
+ "asset": {
+ "id": "c93da63e-392d-443c-879e-902c2692b41e",
+ "name": "wikibase_software_table"
+ },
+ "batch_definition": {
+ "id": "550b72d7-9aca-40cc-9d2a-7f94fcb44828",
+ "name": "FULL_TABLE"
+ },
+ "datasource": {
+ "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
+ "name": "wikibase_datasource"
+ }
+ },
+ "id": "76869a4a-b823-4ccd-b091-2e4dff6befee",
+ "name": "wikibase_software_validation_definition",
+ "suite": {
+ "id": "072a1275-09e6-4835-85f7-8f55e99dcc3e",
+ "name": "wikibase_software_expectation_suite"
+ }
+}
diff --git a/data/gx/validation_definitions/wikibase_software_version_validation_definition b/data/gx/validation_definitions/wikibase_software_version_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_software_version_validation_definition
rename to data/gx/validation_definitions/wikibase_software_version_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition b/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition
deleted file mode 100644
index 20caee4c..00000000
--- a/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "data": {
- "asset": {
- "id": "05cc691f-d972-4cdd-8172-86dc600cf18e",
- "name": "wikibase_statistics_observation_table"
- },
- "batch_definition": {
- "id": "96994a6a-aa72-4eb2-96b8-c285a7291b69",
- "name": "FULL_TABLE"
- },
- "datasource": {
- "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
- "name": "wikibase_datasource"
- }
- },
- "id": "9ea78ef4-0641-485e-a972-219b43b24e88",
- "name": "wikibase_statistics_observation_validation_definition",
- "suite": {
- "id": "3b148b36-c190-4dc0-8504-75d670d6203d",
- "name": "wikibase_statistics_observation_expectation_suite"
- }
-}
\ No newline at end of file
diff --git a/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition.json b/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition.json
new file mode 100644
index 00000000..0ecebd21
--- /dev/null
+++ b/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition.json
@@ -0,0 +1,22 @@
+{
+ "data": {
+ "asset": {
+ "id": "05cc691f-d972-4cdd-8172-86dc600cf18e",
+ "name": "wikibase_statistics_observation_table"
+ },
+ "batch_definition": {
+ "id": "96994a6a-aa72-4eb2-96b8-c285a7291b69",
+ "name": "FULL_TABLE"
+ },
+ "datasource": {
+ "id": "4a58b404-1ee3-4e91-a373-78d1b365f987",
+ "name": "wikibase_datasource"
+ }
+ },
+ "id": "9ea78ef4-0641-485e-a972-219b43b24e88",
+ "name": "wikibase_statistics_observation_validation_definition",
+ "suite": {
+ "id": "3b148b36-c190-4dc0-8504-75d670d6203d",
+ "name": "wikibase_statistics_observation_expectation_suite"
+ }
+}
diff --git a/data/gx/validation_definitions/wikibase_url_validation_definition b/data/gx/validation_definitions/wikibase_url_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_url_validation_definition
rename to data/gx/validation_definitions/wikibase_url_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_user_group_validation_definition b/data/gx/validation_definitions/wikibase_user_group_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_user_group_validation_definition
rename to data/gx/validation_definitions/wikibase_user_group_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_user_observation_group_validation_definition b/data/gx/validation_definitions/wikibase_user_observation_group_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_user_observation_group_validation_definition
rename to data/gx/validation_definitions/wikibase_user_observation_group_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_user_observation_validation_definition b/data/gx/validation_definitions/wikibase_user_observation_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_user_observation_validation_definition
rename to data/gx/validation_definitions/wikibase_user_observation_validation_definition.json
diff --git a/data/gx/validation_definitions/wikibase_validation_definition b/data/gx/validation_definitions/wikibase_validation_definition.json
similarity index 100%
rename from data/gx/validation_definitions/wikibase_validation_definition
rename to data/gx/validation_definitions/wikibase_validation_definition.json
diff --git a/data/wikibase-data.db b/data/wikibase-data.db
index 94e302ac..202e9624 100644
Binary files a/data/wikibase-data.db and b/data/wikibase-data.db differ
diff --git a/data/wikibase-test-data.db b/data/wikibase-test-data.db
index 9117fa0d..0f9d6540 100644
Binary files a/data/wikibase-test-data.db and b/data/wikibase-test-data.db differ
diff --git a/fetch_data/__init__.py b/fetch_data/__init__.py
index 35b47b56..2aadbab2 100644
--- a/fetch_data/__init__.py
+++ b/fetch_data/__init__.py
@@ -9,4 +9,5 @@
from fetch_data.soup_data import (
create_software_version_observation,
create_special_statistics_observation,
+ update_software_data,
)
diff --git a/fetch_data/api_data/log_data/create_log_observation.py b/fetch_data/api_data/log_data/create_log_observation.py
index b6dbc557..e3ffd0c1 100644
--- a/fetch_data/api_data/log_data/create_log_observation.py
+++ b/fetch_data/api_data/log_data/create_log_observation.py
@@ -1,8 +1,8 @@
"""Create Log Observation"""
+from collections.abc import Iterable
from datetime import datetime
from json.decoder import JSONDecodeError
-from typing import List
from requests.exceptions import ReadTimeout, SSLError
from data import get_async_session
from fetch_data.api_data.log_data.fetch_log_data import (
@@ -85,7 +85,7 @@ async def create_log_observation(wikibase_id: int) -> bool:
async def create_log_month(
- wikibase: WikibaseModel, log_list: List[WikibaseLogRecord]
+ wikibase: WikibaseModel, log_list: Iterable[WikibaseLogRecord]
) -> WikibaseLogMonthObservationModel:
"""Create Log Month"""
diff --git a/fetch_data/api_data/log_data/fetch_log_data.py b/fetch_data/api_data/log_data/fetch_log_data.py
index 27bc9830..5559b4a8 100644
--- a/fetch_data/api_data/log_data/fetch_log_data.py
+++ b/fetch_data/api_data/log_data/fetch_log_data.py
@@ -1,7 +1,7 @@
"""Fetch Log Data"""
from datetime import datetime
-from typing import List, Optional
+from typing import Optional
from fetch_data.api_data.log_data.wikibase_log_record import WikibaseLogRecord
from fetch_data.utils import dict_to_url, fetch_api_data
@@ -24,10 +24,10 @@ def get_log_param_string(
return dict_to_url(parameters)
-def get_log_list_from_url(url: str) -> List[WikibaseLogRecord]:
+def get_log_list_from_url(url: str) -> list[WikibaseLogRecord]:
"""Get Log List from URL"""
- data = []
+ data: list[WikibaseLogRecord] = []
query_data = fetch_api_data(url)
for record in query_data["query"]["logevents"]:
@@ -38,10 +38,10 @@ def get_log_list_from_url(url: str) -> List[WikibaseLogRecord]:
def get_month_log_list(
api_url: str, comparison_date: datetime, oldest: bool = False
-) -> List[WikibaseLogRecord]:
+) -> list[WikibaseLogRecord]:
"""Get Log List from api_url, limit to within 30 days of the comparison date"""
- data: List[WikibaseLogRecord] = []
+ data: list[WikibaseLogRecord] = []
limit = 500
should_query = True
diff --git a/fetch_data/api_data/user_data/fetch_multiple_user_data.py b/fetch_data/api_data/user_data/fetch_multiple_user_data.py
index 23305022..943ffb21 100644
--- a/fetch_data/api_data/user_data/fetch_multiple_user_data.py
+++ b/fetch_data/api_data/user_data/fetch_multiple_user_data.py
@@ -1,7 +1,7 @@
"""Fetch Multiple User Data"""
+from collections.abc import Iterable
import json
-from typing import Iterable
import requests
from fetch_data.api_data.user_data.user_data_url import user_url
from model.database import WikibaseModel
diff --git a/fetch_data/soup_data/__init__.py b/fetch_data/soup_data/__init__.py
index d736a837..4c62f5c6 100644
--- a/fetch_data/soup_data/__init__.py
+++ b/fetch_data/soup_data/__init__.py
@@ -1,8 +1,9 @@
"""Data from Parsing Webpages with Beautiful Soup"""
-from fetch_data.soup_data.create_software_version_data_observation import (
- create_software_version_observation,
-)
from fetch_data.soup_data.create_statistics_data_observation import (
create_special_statistics_observation,
)
+from fetch_data.soup_data.software import (
+ create_software_version_observation,
+ update_software_data,
+)
diff --git a/fetch_data/soup_data/software/__init__.py b/fetch_data/soup_data/software/__init__.py
new file mode 100644
index 00000000..974616f7
--- /dev/null
+++ b/fetch_data/soup_data/software/__init__.py
@@ -0,0 +1,6 @@
+"""Software Observations & Data"""
+
+from fetch_data.soup_data.software.create_software_version_data_observation import (
+ create_software_version_observation,
+)
+from fetch_data.soup_data.software.get_update_software_data import update_software_data
diff --git a/fetch_data/soup_data/create_software_version_data_observation.py b/fetch_data/soup_data/software/create_software_version_data_observation.py
similarity index 71%
rename from fetch_data/soup_data/create_software_version_data_observation.py
rename to fetch_data/soup_data/software/create_software_version_data_observation.py
index 6c6bcc5c..1db13db7 100644
--- a/fetch_data/soup_data/create_software_version_data_observation.py
+++ b/fetch_data/soup_data/software/create_software_version_data_observation.py
@@ -1,12 +1,18 @@
"""Create Software Version Observation"""
+from collections.abc import Iterable
from datetime import datetime
-from typing import List
from urllib.error import HTTPError
from bs4 import BeautifulSoup, Tag
import requests
from requests.exceptions import SSLError
+from sqlalchemy.ext.asyncio import AsyncSession
+import strawberry
from data import get_async_session
+from fetch_data.soup_data.software.get_software_model import (
+ get_or_create_software_model,
+)
+from fetch_data.soup_data.software.get_update_software_data import update_software_data
from fetch_data.utils import get_wikibase_from_database, parse_datetime
from model.database import (
WikibaseModel,
@@ -16,9 +22,13 @@
from model.enum import WikibaseSoftwareType
-async def create_software_version_observation(wikibase_id: int) -> bool:
+async def create_software_version_observation(
+ wikibase_id: int, info: strawberry.Info
+) -> bool:
"""Create Software Version Observation"""
+ info.context["background_tasks"].add_task(update_software_data)
+
async with get_async_session() as async_session:
wikibase: WikibaseModel = await get_wikibase_from_database(
async_session=async_session,
@@ -39,16 +49,18 @@ async def create_software_version_observation(wikibase_id: int) -> bool:
observation.returned_data = True
- installed_software_versions = compile_installed_software_versions(soup)
+ installed_software_versions = await compile_installed_software_versions(
+ async_session, soup
+ )
observation.software_versions.extend(installed_software_versions)
- skin_versions = compile_skin_versions(soup)
+ skin_versions = await compile_skin_versions(async_session, soup)
observation.software_versions.extend(skin_versions)
- extensions_versions = compile_extension_versions(soup)
+ extensions_versions = await compile_extension_versions(async_session, soup)
observation.software_versions.extend(extensions_versions)
- library_versions = compile_library_versions(soup)
+ library_versions = await compile_library_versions(async_session, soup)
observation.software_versions.extend(library_versions)
except (HTTPError, SSLError):
observation.returned_data = False
@@ -59,9 +71,9 @@ async def create_software_version_observation(wikibase_id: int) -> bool:
return observation.returned_data
-def compile_extension_versions(
- soup: BeautifulSoup,
-) -> List[WikibaseSoftwareVersionModel]:
+async def compile_extension_versions(
+ async_session: AsyncSession, soup: BeautifulSoup
+) -> list[WikibaseSoftwareVersionModel]:
"""Compile Extension Version List"""
extensions_table = soup.find(
@@ -69,7 +81,9 @@ def compile_extension_versions(
)
return unique_versions(
[
- get_software_version_from_row(row, WikibaseSoftwareType.EXTENSION)
+ await get_software_version_from_row(
+ async_session, row, WikibaseSoftwareType.EXTENSION
+ )
for row in extensions_table.find_all(
"tr", attrs={"class": "mw-version-ext"}
)
@@ -77,14 +91,14 @@ def compile_extension_versions(
)
-def compile_installed_software_versions(
- soup: BeautifulSoup,
-) -> List[WikibaseSoftwareVersionModel]:
+async def compile_installed_software_versions(
+ async_session: AsyncSession, soup: BeautifulSoup
+) -> list[WikibaseSoftwareVersionModel]:
"""Compile Installed Software Version List"""
installed_software_table: Tag = soup.find("table", attrs={"id": "sv-software"})
- software_versions: List[WikibaseSoftwareVersionModel] = []
+ software_versions: list[WikibaseSoftwareVersionModel] = []
row: Tag
for row in installed_software_table.find_all("tr"):
if row.find("td"):
@@ -113,8 +127,9 @@ def compile_installed_software_versions(
software_versions.append(
WikibaseSoftwareVersionModel(
- software_type=WikibaseSoftwareType.SOFTWARE,
- software_name=software_name,
+ software=await get_or_create_software_model(
+ async_session, WikibaseSoftwareType.SOFTWARE, software_name
+ ),
version=version,
version_hash=version_hash,
version_date=version_date,
@@ -124,12 +139,14 @@ def compile_installed_software_versions(
return software_versions
-def compile_library_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionModel]:
+async def compile_library_versions(
+ async_session: AsyncSession, soup: BeautifulSoup
+) -> list[WikibaseSoftwareVersionModel]:
"""Compile Library Version List"""
libraries_table = soup.find("table", attrs={"id": "sv-libraries"})
- library_versions: List[WikibaseSoftwareVersionModel] = []
+ library_versions: list[WikibaseSoftwareVersionModel] = []
row: Tag
for row in libraries_table.find_all("tr"):
if row.find("td"):
@@ -140,8 +157,9 @@ def compile_library_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersio
version = row.find_all("td")[1].string
library_versions.append(
WikibaseSoftwareVersionModel(
- software_type=WikibaseSoftwareType.LIBRARY,
- software_name=software_name,
+ software=await get_or_create_software_model(
+ async_session, WikibaseSoftwareType.LIBRARY, software_name
+ ),
version=version,
)
)
@@ -149,7 +167,9 @@ def compile_library_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersio
return unique_versions(library_versions)
-def compile_skin_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionModel]:
+async def compile_skin_versions(
+ async_session: AsyncSession, soup: BeautifulSoup
+) -> list[WikibaseSoftwareVersionModel]:
"""Compile Skin Version List"""
installed_skin_table: Tag = soup.find(
@@ -157,7 +177,9 @@ def compile_skin_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionMo
)
return unique_versions(
[
- get_software_version_from_row(row, WikibaseSoftwareType.SKIN)
+ await get_software_version_from_row(
+ async_session, row, WikibaseSoftwareType.SKIN
+ )
for row in installed_skin_table.find_all(
"tr", attrs={"class": "mw-version-ext"}
)
@@ -165,8 +187,8 @@ def compile_skin_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionMo
)
-def get_software_version_from_row(
- row: Tag, software_type: WikibaseSoftwareType
+async def get_software_version_from_row(
+ async_session: AsyncSession, row: Tag, software_type: WikibaseSoftwareType
) -> WikibaseSoftwareVersionModel:
"""Parse Software Version from Table Row"""
@@ -205,8 +227,9 @@ def get_software_version_from_row(
version_date = parse_datetime(date_tag.string)
return WikibaseSoftwareVersionModel(
- software_type=software_type,
- software_name=software_name,
+ software=await get_or_create_software_model(
+ async_session, software_type, software_name
+ ),
version=version,
version_hash=version_hash,
version_date=version_date,
@@ -214,8 +237,8 @@ def get_software_version_from_row(
def unique_versions(
- input_list: List[WikibaseSoftwareVersionModel],
-) -> List[WikibaseSoftwareVersionModel]:
+ input_list: Iterable[WikibaseSoftwareVersionModel],
+) -> list[WikibaseSoftwareVersionModel]:
"""Unique Version List"""
temp: dict[str, WikibaseSoftwareVersionModel] = {}
diff --git a/fetch_data/soup_data/software/get_software_model.py b/fetch_data/soup_data/software/get_software_model.py
new file mode 100644
index 00000000..3c14aab9
--- /dev/null
+++ b/fetch_data/soup_data/software/get_software_model.py
@@ -0,0 +1,87 @@
+"""Get or Create Software Model"""
+
+import re
+from typing import Optional
+from sqlalchemy import and_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from model.database import WikibaseSoftwareModel
+from model.enum import WikibaseSoftwareType
+
+EXTENSIONNAME_PATTERN = r"⧼([a-z]+)-extensionname⧽"
+
+
+async def get_or_create_software_model(
+ async_session: AsyncSession, software_type: WikibaseSoftwareType, software_name: str
+) -> WikibaseSoftwareModel:
+ """Fetch or Create Software Model"""
+
+ if re.match(EXTENSIONNAME_PATTERN, software_name):
+ software_name = re.sub(EXTENSIONNAME_PATTERN, r"\1", software_name)
+
+ existing = await get_existing_software_model(
+ async_session, software_type, software_name
+ )
+ if existing is not None:
+ return existing
+
+ nearby = await get_nearby_software_model(
+ async_session, software_type, software_name
+ )
+ if nearby is not None:
+ return nearby
+
+ creating = WikibaseSoftwareModel(
+ software_type=software_type, software_name=software_name
+ )
+ async_session.add(creating)
+ await async_session.flush()
+ await async_session.refresh(creating)
+ return creating
+
+
+async def get_existing_software_model(
+ async_session: AsyncSession, software_type: WikibaseSoftwareType, software_name: str
+) -> Optional[WikibaseSoftwareModel]:
+ """Return Existing Software, If Existent"""
+
+ return (
+ await async_session.scalars(
+ select(WikibaseSoftwareModel).where(
+ and_(
+ WikibaseSoftwareModel.software_type == software_type,
+ WikibaseSoftwareModel.software_name == software_name,
+ )
+ )
+ )
+ ).one_or_none()
+
+
+async def get_nearby_software_model(
+ async_session: AsyncSession, software_type: WikibaseSoftwareType, software_name: str
+) -> Optional[WikibaseSoftwareModel]:
+ """
+ Return Matching Software, if found
+
+ Only allowable differences: whitespace and capitalization
+
+ Assert only one or zero matches
+ """
+
+ all_software_of_type = (
+ await async_session.scalars(
+ select(WikibaseSoftwareModel).where(
+ WikibaseSoftwareModel.software_type == software_type
+ )
+ )
+ ).all()
+ nearby: Optional[WikibaseSoftwareModel] = None
+ nearby_count = 0
+ for s in all_software_of_type:
+ if (
+ s.software_name.replace(" ", "").lower()
+ == software_name.replace(" ", "").lower()
+ ):
+ nearby = s
+ nearby_count += 1
+ assert nearby_count <= 1
+ return nearby
diff --git a/fetch_data/soup_data/software/get_update_software_data.py b/fetch_data/soup_data/software/get_update_software_data.py
new file mode 100644
index 00000000..6ed88c8e
--- /dev/null
+++ b/fetch_data/soup_data/software/get_update_software_data.py
@@ -0,0 +1,232 @@
+"""Update Software Data"""
+
+from datetime import datetime, timedelta, timezone
+import os
+import re
+from typing import Iterable, List, Optional
+from bs4 import BeautifulSoup
+import requests
+from sqlalchemy import Select, and_, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from data import get_async_session
+from model.database import WikibaseSoftwareModel, WikibaseSoftwareTagModel
+from model.enum import WikibaseSoftwareType
+
+
+async def update_software_data():
+ """Fetch Software Info"""
+
+ carry_on = True
+ while carry_on:
+
+ async with get_async_session() as async_session:
+ os.makedirs("dump", exist_ok=True)
+
+ unfound_extensions: Iterable[WikibaseSoftwareModel] = (
+ await async_session.scalars(get_update_extension_query())
+ ).all()
+ carry_on = len(unfound_extensions) > 0
+
+ for ext in unfound_extensions:
+ if ext.url is None:
+ ext.url = (
+ f"https://www.mediawiki.org/wiki/Extension:{ext.software_name}"
+ )
+
+ await compile_data_from_url(async_session, ext)
+ await async_session.flush()
+ await async_session.commit()
+
+
+async def compile_data_from_url(
+ async_session: AsyncSession,
+ ext: WikibaseSoftwareModel,
+ override_url: Optional[str] = None,
+ archived: bool = False,
+):
+ """Compile Software Data from URL"""
+
+ with requests.get(
+ override_url or ext.url, timeout=10, allow_redirects=True
+ ) as response:
+
+ ext.data_fetched = datetime.now(timezone.utc)
+ print(f"{response.url}: {response.status_code}")
+
+ if response.status_code == 200:
+
+ if override_url is None and response.url != ext.url:
+ ext.url = response.url
+
+ soup = BeautifulSoup(response.content, features="html.parser")
+
+ page_archived = (
+ soup.find("b", string="This extension has been archived.") is not None
+ )
+ ext.archived = archived or page_archived
+ if page_archived:
+ permanent_link_tag = soup.find(
+ "a", string="To see the page before archival, click here."
+ )
+ return await compile_data_from_url(
+ async_session,
+ ext,
+ override_url=f"https://www.mediawiki.org{permanent_link_tag['href']}",
+ archived=True,
+ )
+
+ ext.tags = await compile_tag_list(async_session, soup)
+ ext.description = compile_description(soup)
+ ext.latest_version = compile_latest_version(soup)
+ ext.quarterly_download_count = compile_quarterly_count(soup)
+ ext.public_wiki_count = compile_wiki_count(soup)
+ ext.mediawiki_bundled = compile_bundled(soup)
+
+
+def get_update_extension_query() -> Select[WikibaseSoftwareModel]:
+ """Update Extension List Query"""
+
+ return (
+ select(WikibaseSoftwareModel)
+ .where(
+ and_(
+ or_(
+ # pylint: disable=singleton-comparison
+ WikibaseSoftwareModel.data_fetched == None,
+ WikibaseSoftwareModel.data_fetched
+ < (datetime.today() - timedelta(days=30)),
+ ),
+ WikibaseSoftwareModel.software_type == WikibaseSoftwareType.EXTENSION,
+ or_(
+ # pylint: disable=singleton-comparison
+ WikibaseSoftwareModel.archived == False,
+ WikibaseSoftwareModel.archived == None,
+ ),
+ )
+ )
+ .limit(5)
+ )
+
+
+async def compile_tag_list(
+ async_session: AsyncSession, soup: BeautifulSoup
+) -> List[WikibaseSoftwareTagModel]:
+ """Compile Tag List"""
+
+ type_title_tag = soup.find(
+ "a", attrs={"href": "/wiki/Special:MyLanguage/Template:Extension#type"}
+ )
+ if type_title_tag is None:
+ return []
+
+ type_tag = type_title_tag.find_parent("td").find_next_sibling("td")
+ assert type_tag is not None
+
+ tag_list: list[str] = []
+ if len(list(type_tag.children)) == 1:
+ tag_list = [
+ s.strip()
+ for t_string in type_tag.stripped_strings
+ for s in t_string.split(",")
+ ]
+ else:
+ tag_list = [
+ stripped
+ for t in type_tag.find_all("a")
+ if t.string is not None
+ for s in t.string.split(",")
+ if (stripped := (s).strip()) != ""
+ ]
+ all_tags = await fetch_or_create_tags(async_session, tag_list)
+
+ return all_tags
+
+
+async def fetch_or_create_tags(
+ async_session: AsyncSession, tag_list: List[str]
+) -> List[WikibaseSoftwareTagModel]:
+ """Fetch or Create Tags"""
+
+ existing_tags = (
+ await async_session.scalars(
+ select(WikibaseSoftwareTagModel).where(
+ WikibaseSoftwareTagModel.tag.in_(tag_list)
+ )
+ )
+ ).all()
+ existing_tag_strs = {t.tag for t in existing_tags}
+ all_tags = [*existing_tags]
+ for tag in tag_list:
+ if tag not in existing_tag_strs:
+ all_tags.append(WikibaseSoftwareTagModel(tag))
+ return all_tags
+
+
+def compile_description(soup: BeautifulSoup) -> Optional[str]:
+ """Compile Description"""
+
+ description_title_tag = soup.find(
+ "a", attrs={"href": "/wiki/Special:MyLanguage/Template:Extension#description"}
+ )
+ if description_title_tag is None:
+ return None
+
+ description_tag = description_title_tag.find_parent("td").find_next_sibling("td")
+ assert description_tag is not None
+ result = re.sub(r"[ ]{2,}", r" ", "".join(description_tag.strings).strip())
+ return result[0].upper() + result[1:]
+
+
+def compile_latest_version(soup: BeautifulSoup) -> Optional[str]:
+ """Compile Latest Version"""
+
+ version_title_tag = soup.find(
+ "a", attrs={"href": "/wiki/Special:MyLanguage/Template:Extension#version"}
+ )
+ if version_title_tag is None:
+ return None
+
+ version_tag = version_title_tag.find_parent("td").find_next_sibling("td")
+ assert version_tag is not None, f"{version_title_tag.prettify()}"
+ return re.sub(r" ", r" ", "".join(version_tag.strings).strip())
+
+
+def compile_quarterly_count(soup: BeautifulSoup) -> Optional[int]:
+ """Compile Quarterly Download Count"""
+
+ qd_title_tag = soup.find("b", string="Quarterly downloads")
+ if qd_title_tag is None:
+ return None
+
+ qd_tag = qd_title_tag.find_parent("td").find_next_sibling("td")
+ assert qd_tag is not None
+ attempt = re.sub(r"^(\d+(,\d+)*) .*$", r"\1", "".join(qd_tag.strings))
+ return int(attempt.replace(",", ""))
+
+
+def compile_wiki_count(soup: BeautifulSoup) -> Optional[int]:
+ """Compile Public Wiki Count"""
+
+ pw_title_tag = soup.find("b", string="Public wikis using")
+ if pw_title_tag is None:
+ return None
+
+ pw_tag = pw_title_tag.find_parent("td").find_next_sibling("td")
+ assert pw_tag is not None
+ attempt = re.sub(r"^(\d+(,\d+)*) .*$", r"\1", "".join(pw_tag.strings))
+ return int(attempt.replace(",", ""))
+
+
+def compile_bundled(soup: BeautifulSoup) -> Optional[bool]:
+ """Compile Bundled with MediaWiki"""
+
+ category_list_tag = soup.find("div", attrs={"id": "mw-normal-catlinks"})
+ if category_list_tag is None:
+ return None
+ for a_tag in category_list_tag.find_all("a"):
+ if re.match(
+ r"^/wiki/Category:Extensions_bundled_with_MediaWiki_\d+\.\d+$",
+ a_tag["href"],
+ ):
+ return True
+ return False
diff --git a/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py b/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py
index f5a19288..109ee506 100644
--- a/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py
+++ b/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py
@@ -1,11 +1,11 @@
"""Compile Connectivity Distance"""
-from typing import Iterable, List
+from collections.abc import Iterable
from tqdm import tqdm
def compile_distance_dict(
- all_nodes: List[str], link_dict: dict[str, set[str]]
+ all_nodes: Iterable[str], link_dict: dict[str, set[str]]
) -> dict[str, dict[str, int]]:
"""Compile Distance Dictionary
diff --git a/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py b/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py
index 31867ca6..dea5abdf 100644
--- a/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py
+++ b/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py
@@ -1,11 +1,11 @@
"""Compile Link Dictionary"""
-from typing import List
+from collections.abc import Iterable
from fetch_data.sparql_data.sparql_queries import ItemLink
def compile_link_dict(
- clean_data: List[ItemLink], all_nodes: List[str], reverse: bool = False
+ clean_data: Iterable[ItemLink], all_nodes: Iterable[str], reverse: bool = False
) -> dict[str, set[str]]:
"""Compile Link Dictionary
diff --git a/fetch_data/sparql_data/sparql_queries/item_links.py b/fetch_data/sparql_data/sparql_queries/item_links.py
index 4409ac42..31cf0cb0 100644
--- a/fetch_data/sparql_data/sparql_queries/item_links.py
+++ b/fetch_data/sparql_data/sparql_queries/item_links.py
@@ -1,7 +1,6 @@
"""Links between items"""
import re
-from typing import List
ITEM_LINKS_QUERY = """SELECT ?item ?object WHERE {
@@ -36,7 +35,7 @@ def clean_point(point: dict) -> ItemLink:
)
-def clean_item_link_data(results: dict) -> List[ItemLink]:
+def clean_item_link_data(results: dict) -> list[ItemLink]:
"""Query Results to list of data"""
return [clean_point(p) for p in results["results"]["bindings"]]
diff --git a/fetch_data/utils/counts.py b/fetch_data/utils/counts.py
index 72d1b7d0..8d8c8824 100644
--- a/fetch_data/utils/counts.py
+++ b/fetch_data/utils/counts.py
@@ -1,6 +1,7 @@
"""Counts"""
-from typing import Iterable, TypeVar
+from collections.abc import Iterable
+from typing import TypeVar
T = TypeVar("T")
diff --git a/model/database/__init__.py b/model/database/__init__.py
index ac44ab96..1d6c57d5 100644
--- a/model/database/__init__.py
+++ b/model/database/__init__.py
@@ -21,4 +21,8 @@
WikibaseUserObservationGroupModel,
WikibaseUserObservationModel,
)
+from model.database.wikibase_software import (
+ WikibaseSoftwareModel,
+ WikibaseSoftwareTagModel,
+)
from model.database.wikibase_url_model import WikibaseURLModel
diff --git a/model/database/wikibase_category_model.py b/model/database/wikibase_category_model.py
index c1827571..bf1d72f4 100644
--- a/model/database/wikibase_category_model.py
+++ b/model/database/wikibase_category_model.py
@@ -1,6 +1,5 @@
"""Wikibase Category Table"""
-from typing import List
from sqlalchemy import Enum, Integer, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -18,7 +17,7 @@ class WikibaseCategoryModel(ModelBase):
id: Mapped[int] = mapped_column("id", Integer, primary_key=True, autoincrement=True)
"""ID"""
- wikibases: Mapped[List["WikibaseModel"]] = relationship(
+ wikibases: Mapped[list["WikibaseModel"]] = relationship(
"WikibaseModel", lazy="selectin", back_populates="category"
)
"""Wikibases"""
diff --git a/model/database/wikibase_model.py b/model/database/wikibase_model.py
index 06f8d3e7..9478c5d9 100644
--- a/model/database/wikibase_model.py
+++ b/model/database/wikibase_model.py
@@ -1,6 +1,6 @@
"""Wikibase Table"""
-from typing import List, Optional
+from typing import Optional
from sqlalchemy import Boolean, ForeignKey, Integer, String, and_
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -212,7 +212,7 @@ class WikibaseModel(ModelBase):
)
"""Special:Version URL"""
- connectivity_observations: Mapped[List[WikibaseConnectivityObservationModel]] = (
+ connectivity_observations: Mapped[list[WikibaseConnectivityObservationModel]] = (
relationship(
"WikibaseConnectivityObservationModel",
back_populates="wikibase",
@@ -221,13 +221,13 @@ class WikibaseModel(ModelBase):
)
"""Connectivity Observations"""
- log_observations: Mapped[List[WikibaseLogObservationModel]] = relationship(
+ log_observations: Mapped[list[WikibaseLogObservationModel]] = relationship(
"WikibaseLogObservationModel", back_populates="wikibase", lazy="select"
)
"""Log Observations"""
property_popularity_observations: Mapped[
- List[WikibasePropertyPopularityObservationModel]
+ list[WikibasePropertyPopularityObservationModel]
] = relationship(
"WikibasePropertyPopularityObservationModel",
back_populates="wikibase",
@@ -235,7 +235,7 @@ class WikibaseModel(ModelBase):
)
"""Property Popularity Observations"""
- quantity_observations: Mapped[List[WikibaseQuantityObservationModel]] = (
+ quantity_observations: Mapped[list[WikibaseQuantityObservationModel]] = (
relationship(
"WikibaseQuantityObservationModel", back_populates="wikibase", lazy="select"
)
@@ -243,7 +243,7 @@ class WikibaseModel(ModelBase):
"""Quantity Observations"""
software_version_observations: Mapped[
- List[WikibaseSoftwareVersionObservationModel]
+ list[WikibaseSoftwareVersionObservationModel]
] = relationship(
"WikibaseSoftwareVersionObservationModel",
back_populates="wikibase",
@@ -251,7 +251,7 @@ class WikibaseModel(ModelBase):
)
"""Software Version Observations"""
- statistics_observations: Mapped[List[WikibaseStatisticsObservationModel]] = (
+ statistics_observations: Mapped[list[WikibaseStatisticsObservationModel]] = (
relationship(
"WikibaseStatisticsObservationModel",
back_populates="wikibase",
@@ -260,7 +260,7 @@ class WikibaseModel(ModelBase):
)
"""Statistics Observations"""
- user_observations: Mapped[List[WikibaseUserObservationModel]] = relationship(
+ user_observations: Mapped[list[WikibaseUserObservationModel]] = relationship(
"WikibaseUserObservationModel", back_populates="wikibase", lazy="select"
)
"""User Observations"""
diff --git a/model/database/wikibase_observation/connectivity/connectivity_observation_model.py b/model/database/wikibase_observation/connectivity/connectivity_observation_model.py
index 4963c8bd..b37ca75a 100644
--- a/model/database/wikibase_observation/connectivity/connectivity_observation_model.py
+++ b/model/database/wikibase_observation/connectivity/connectivity_observation_model.py
@@ -1,6 +1,6 @@
"""Wikibase Connectivity Observation Table"""
-from typing import List, Optional
+from typing import Optional
from sqlalchemy import Double, Integer
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -37,7 +37,7 @@ class WikibaseConnectivityObservationModel(ModelBase, WikibaseObservationModel):
"""Average steps for item -> item connections, ignoring disconnected items"""
item_relationship_count_observations: Mapped[
- List[WikibaseConnectivityObservationItemRelationshipCountModel]
+ list[WikibaseConnectivityObservationItemRelationshipCountModel]
] = relationship(
"WikibaseConnectivityObservationItemRelationshipCountModel",
back_populates="connectivity_observation",
@@ -46,7 +46,7 @@ class WikibaseConnectivityObservationModel(ModelBase, WikibaseObservationModel):
"""Item / Relationship Count Observations"""
object_relationship_count_observations: Mapped[
- List[WikibaseConnectivityObservationObjectRelationshipCountModel]
+ list[WikibaseConnectivityObservationObjectRelationshipCountModel]
] = relationship(
"WikibaseConnectivityObservationObjectRelationshipCountModel",
back_populates="connectivity_observation",
diff --git a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py
index 4b17115f..0ce45f87 100644
--- a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py
+++ b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py
@@ -1,7 +1,7 @@
"""Wikibase Log Month Observation Table"""
from datetime import datetime
-from typing import List, Optional
+from typing import Optional
from sqlalchemy import DateTime, Integer
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -43,12 +43,12 @@ class WikibaseLogMonthObservationModel(ModelBase):
)
"""Number of Unique Users, Without Bots"""
- log_type_records: Mapped[List[WikibaseLogMonthLogTypeObservationModel]] = (
+ log_type_records: Mapped[list[WikibaseLogMonthLogTypeObservationModel]] = (
relationship("WikibaseLogMonthLogTypeObservationModel", lazy="selectin")
)
"""Log Type Observations"""
- user_type_records: Mapped[List[WikibaseLogMonthUserTypeObservationModel]] = (
+ user_type_records: Mapped[list[WikibaseLogMonthUserTypeObservationModel]] = (
relationship("WikibaseLogMonthUserTypeObservationModel", lazy="selectin")
)
"""User Type Observations"""
diff --git a/model/database/wikibase_observation/property/popularity_observation_model.py b/model/database/wikibase_observation/property/popularity_observation_model.py
index 132bdcd3..88fc1c6b 100644
--- a/model/database/wikibase_observation/property/popularity_observation_model.py
+++ b/model/database/wikibase_observation/property/popularity_observation_model.py
@@ -1,6 +1,5 @@
"""Wikibase Property Popularity Observation Table"""
-from typing import List
from sqlalchemy.orm import Mapped, relationship
from model.database.base import ModelBase
@@ -17,7 +16,7 @@ class WikibasePropertyPopularityObservationModel(ModelBase, WikibaseObservationM
__tablename__ = "wikibase_property_usage_observation"
- property_count_observations: Mapped[List[WikibasePropertyPopularityCountModel]] = (
+ property_count_observations: Mapped[list[WikibasePropertyPopularityCountModel]] = (
relationship(
"WikibasePropertyPopularityCountModel",
back_populates="wikibase_property_popularity_observation",
diff --git a/model/database/wikibase_observation/user/wikibase_user_group_model.py b/model/database/wikibase_observation/user/wikibase_user_group_model.py
index 5b56f715..c9d461f7 100644
--- a/model/database/wikibase_observation/user/wikibase_user_group_model.py
+++ b/model/database/wikibase_observation/user/wikibase_user_group_model.py
@@ -1,6 +1,5 @@
"""Wikibase User Group Table"""
-from typing import List
from sqlalchemy import Boolean, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -27,7 +26,7 @@ class WikibaseUserGroupModel(ModelBase):
)
"""Wikibase Default?"""
- user_group_observations: Mapped[List[WikibaseUserObservationGroupModel]] = (
+ user_group_observations: Mapped[list[WikibaseUserObservationGroupModel]] = (
relationship(
"WikibaseUserObservationGroupModel",
back_populates="user_group",
diff --git a/model/database/wikibase_observation/user/wikibase_user_observation_model.py b/model/database/wikibase_observation/user/wikibase_user_observation_model.py
index 0f462d20..47f524bc 100644
--- a/model/database/wikibase_observation/user/wikibase_user_observation_model.py
+++ b/model/database/wikibase_observation/user/wikibase_user_observation_model.py
@@ -1,6 +1,6 @@
"""Wikibase User Observation Table"""
-from typing import List, Optional
+from typing import Optional
from sqlalchemy import Integer
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -23,7 +23,7 @@ class WikibaseUserObservationModel(ModelBase, WikibaseObservationModel):
)
"""Total Users"""
- user_group_observations: Mapped[List[WikibaseUserObservationGroupModel]] = (
+ user_group_observations: Mapped[list[WikibaseUserObservationGroupModel]] = (
relationship(
"WikibaseUserObservationGroupModel",
back_populates="user_observation",
diff --git a/model/database/wikibase_observation/version/software_version_model.py b/model/database/wikibase_observation/version/software_version_model.py
index 5c4adb50..47bfb1e9 100644
--- a/model/database/wikibase_observation/version/software_version_model.py
+++ b/model/database/wikibase_observation/version/software_version_model.py
@@ -2,11 +2,11 @@
from datetime import datetime
from typing import Optional
-from sqlalchemy import DateTime, Enum, ForeignKey, Integer, String, UniqueConstraint
+from sqlalchemy import DateTime, ForeignKey, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
from model.database.base import ModelBase
-from model.enum import WikibaseSoftwareType
+from model.database.wikibase_software import WikibaseSoftwareModel
class WikibaseSoftwareVersionModel(ModelBase):
@@ -17,9 +17,8 @@ class WikibaseSoftwareVersionModel(ModelBase):
__table_args__ = (
UniqueConstraint(
"wikibase_software_version_observation_id",
- "software_type",
- "software_name",
- name="unique_observation_software_type_name",
+ "wikibase_software_id",
+ name="unique_observation_software_id",
),
)
@@ -44,13 +43,19 @@ class WikibaseSoftwareVersionModel(ModelBase):
)
"""Software Version Observation"""
- software_type: Mapped[WikibaseSoftwareType] = mapped_column(
- "software_type", Enum(WikibaseSoftwareType), nullable=False
+ software_id: Mapped[int] = mapped_column(
+ "wikibase_software_id",
+ ForeignKey(column="wikibase_software.id", name="software"),
+ nullable=False,
)
- """Software Type"""
+ """Software Id"""
- software_name: Mapped[str] = mapped_column("software_name", String, nullable=False)
- """Software Name"""
+ software: Mapped[WikibaseSoftwareModel] = relationship(
+ "WikibaseSoftwareModel",
+ # back_populates="software_versions",
+ lazy="selectin",
+ )
+ """Software"""
version: Mapped[Optional[str]] = mapped_column("version", String, nullable=True)
"""Version"""
@@ -68,14 +73,13 @@ class WikibaseSoftwareVersionModel(ModelBase):
# pylint: disable=too-many-arguments,too-many-positional-arguments
def __init__(
self,
- software_type: WikibaseSoftwareType,
- software_name: str,
+ software: WikibaseSoftwareModel,
version: str,
version_hash: Optional[str] = None,
version_date: Optional[datetime] = None,
):
- self.software_type = software_type
- self.software_name = software_name
+ self.software = software
+
self.version_hash = (
None
if version_hash is None
@@ -87,8 +91,8 @@ def __init__(
def __str__(self) -> str:
return (
"WikibaseSoftwareVersionModel("
- + f"software_type={self.software_type}, "
- + f"software_name={self.software_name}, "
+ + f"software_type={self.software.software_type}, "
+ + f"software_name={self.software.software_name}, "
+ f"version={self.version}, "
+ f"version_date={self.version_date}, "
+ f"version_hash={self.version_hash})"
diff --git a/model/database/wikibase_observation/version/wikibase_version_observation_model.py b/model/database/wikibase_observation/version/wikibase_version_observation_model.py
index 0ab57ec5..31fd2b3a 100644
--- a/model/database/wikibase_observation/version/wikibase_version_observation_model.py
+++ b/model/database/wikibase_observation/version/wikibase_version_observation_model.py
@@ -1,6 +1,5 @@
"""Wikibase Software Version Observation Table"""
-from typing import List
from sqlalchemy.orm import Mapped, relationship
from model.database.base import ModelBase
@@ -17,7 +16,7 @@ class WikibaseSoftwareVersionObservationModel(ModelBase, WikibaseObservationMode
__tablename__ = "wikibase_software_version_observation"
- software_versions: Mapped[List[WikibaseSoftwareVersionModel]] = relationship(
+ software_versions: Mapped[list[WikibaseSoftwareVersionModel]] = relationship(
"WikibaseSoftwareVersionModel",
back_populates="wikibase_software_version_observation",
lazy="selectin",
diff --git a/model/database/wikibase_software/__init__.py b/model/database/wikibase_software/__init__.py
new file mode 100644
index 00000000..220f8a77
--- /dev/null
+++ b/model/database/wikibase_software/__init__.py
@@ -0,0 +1,4 @@
+"""Wikibase Software"""
+
+from model.database.wikibase_software.software_model import WikibaseSoftwareModel
+from model.database.wikibase_software.software_tag_model import WikibaseSoftwareTagModel
diff --git a/model/database/wikibase_software/software_model.py b/model/database/wikibase_software/software_model.py
new file mode 100644
index 00000000..8b6e1c87
--- /dev/null
+++ b/model/database/wikibase_software/software_model.py
@@ -0,0 +1,87 @@
+"""Wikibase Software Table"""
+
+from datetime import datetime
+from typing import List, Optional
+from sqlalchemy import Boolean, DateTime, Enum, Integer, String, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from model.database.base import ModelBase
+from model.database.wikibase_software.software_tag_model import WikibaseSoftwareTagModel
+from model.database.wikibase_software.software_tag_xref_model import (
+ software_tag_xref_table,
+)
+from model.enum import WikibaseSoftwareType
+
+
+class WikibaseSoftwareModel(ModelBase):
+ """Wikibase Software Table"""
+
+ __tablename__ = "wikibase_software"
+
+ __table_args__ = (
+ UniqueConstraint(
+ "software_type", "software_name", name="unique_software_type_name"
+ ),
+ )
+
+ id: Mapped[int] = mapped_column("id", Integer, primary_key=True, autoincrement=True)
+ """ID"""
+
+ software_type: Mapped[WikibaseSoftwareType] = mapped_column(
+ "software_type", Enum(WikibaseSoftwareType), nullable=False
+ )
+ """Software Type"""
+
+ software_name: Mapped[str] = mapped_column("software_name", String, nullable=False)
+ """Software Name"""
+
+ url: Mapped[Optional[str]] = mapped_column("url", String, nullable=True)
+ """Reference URL"""
+
+ data_fetched: Mapped[Optional[datetime]] = mapped_column(
+ "fetched", DateTime(timezone=True), nullable=True
+ )
+
+ tags: Mapped[List[WikibaseSoftwareTagModel]] = relationship(
+ secondary=software_tag_xref_table, lazy="selectin"
+ )
+
+ description: Mapped[Optional[str]] = mapped_column(
+ "description", String, nullable=True
+ )
+ """Description"""
+
+ latest_version: Mapped[Optional[str]] = mapped_column(
+ "latest_version", String, nullable=True
+ )
+ """Latest Version"""
+
+ quarterly_download_count: Mapped[Optional[int]] = mapped_column(
+ "quarterly_download_count", Integer, nullable=True
+ )
+ """Quarterly Downloads"""
+
+ public_wiki_count: Mapped[Optional[int]] = mapped_column(
+ "public_wiki_count", Integer, nullable=True
+ )
+ """Public Wikis Using"""
+
+ mediawiki_bundled: Mapped[Optional[bool]] = mapped_column(
+ "mw_bundled", Boolean, nullable=True
+ )
+ """Bundled with Mediawiki"""
+
+ archived: Mapped[Optional[bool]] = mapped_column("archived", Boolean, nullable=True)
+ """Archived Extension"""
+
+ def __init__(self, software_type: WikibaseSoftwareType, software_name: str):
+ self.software_type = software_type
+ self.software_name = software_name
+
+ def __str__(self) -> str:
+ return (
+ "WikibaseSoftwareModel("
+ + f"id={self.id}, "
+ + f"software_type={self.software_type}, "
+ + f"software_name={self.software_name})"
+ )
diff --git a/model/database/wikibase_software/software_tag_model.py b/model/database/wikibase_software/software_tag_model.py
new file mode 100644
index 00000000..13cab75c
--- /dev/null
+++ b/model/database/wikibase_software/software_tag_model.py
@@ -0,0 +1,24 @@
+"""Wikibase Software Version Table"""
+
+from sqlalchemy import Integer, String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from model.database.base import ModelBase
+
+
+class WikibaseSoftwareTagModel(ModelBase):
+ """Wikibase Software Tag Table"""
+
+ __tablename__ = "wikibase_software_tag"
+
+ id: Mapped[int] = mapped_column("id", Integer, primary_key=True, autoincrement=True)
+ """ID"""
+
+ tag: Mapped[str] = mapped_column("tag", String, nullable=False, unique=True)
+ """Software Tag"""
+
+ def __init__(self, tag: str):
+ self.tag = tag
+
+ def __str__(self) -> str:
+ return f"WikibaseSoftwareTagModel(id={self.id}, tag={self.tag})"
diff --git a/model/database/wikibase_software/software_tag_xref_model.py b/model/database/wikibase_software/software_tag_xref_model.py
new file mode 100644
index 00000000..e385384e
--- /dev/null
+++ b/model/database/wikibase_software/software_tag_xref_model.py
@@ -0,0 +1,24 @@
+"""Software/Tag XRef Table"""
+
+from sqlalchemy import Column, ForeignKey, Table
+
+from model.database.base import ModelBase
+
+
+software_tag_xref_table = Table(
+ "wikibase_software_tag_xref",
+ ModelBase.metadata,
+ Column(
+ "wikibase_software_id",
+ ForeignKey("wikibase_software.id", None, False, "foreign_wikibase_software_id"),
+ primary_key=True,
+ ),
+ Column(
+ "wikibase_software_tag_id",
+ ForeignKey(
+ "wikibase_software_tag.id", None, False, "foreign_wikibase_software_tag_id"
+ ),
+ primary_key=True,
+ ),
+)
+"""Software/Tag XRef Table"""
diff --git a/model/strawberry/output/__init__.py b/model/strawberry/output/__init__.py
index 77820df3..af443b7a 100644
--- a/model/strawberry/output/__init__.py
+++ b/model/strawberry/output/__init__.py
@@ -11,3 +11,4 @@
)
from model.strawberry.output.page import Page, PageNumberType, PageSizeType
from model.strawberry.output.wikibase import WikibaseStrawberryModel
+from model.strawberry.output.wikibase_software import WikibaseSoftwareStrawberryModel
diff --git a/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py b/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py
index 8f1ea173..e0ce8606 100644
--- a/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py
+++ b/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py
@@ -1,6 +1,6 @@
"""Wikibase Connectivity Data Observation Strawberry Model"""
-from typing import List, Optional
+from typing import Optional
import strawberry
from model.database import WikibaseConnectivityObservationModel
@@ -32,10 +32,10 @@ class WikibaseConnectivityObservationStrawberryModel(
graphql_type=Optional[BigInt],
)
- relationship_item_counts: List[
+ relationship_item_counts: list[
WikibaseConnectivityObservationItemRelationshipCountStrawberryModel
] = strawberry.field(description="Number of Items with Number of Relationships")
- relationship_object_counts: List[
+ relationship_object_counts: list[
WikibaseConnectivityObservationObjectRelationshipCountStrawberryModel
] = strawberry.field(description="Number of Items with Number of Relationships")
diff --git a/model/strawberry/output/observation/log/wikibase_log_collection.py b/model/strawberry/output/observation/log/wikibase_log_collection.py
index 2e13f61d..6f88044c 100644
--- a/model/strawberry/output/observation/log/wikibase_log_collection.py
+++ b/model/strawberry/output/observation/log/wikibase_log_collection.py
@@ -1,7 +1,7 @@
"""Wikibase Log Collection Strawberry Models"""
from datetime import datetime
-from typing import List, Optional
+from typing import Optional
import strawberry
from model.database import (
@@ -81,10 +81,10 @@ def marshal(
class WikibaseLogMonthStrawberryModel(WikibaseLogCollectionStrawberryModel):
"""Wikibase Log Month"""
- log_type_records: List[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field(
+ log_type_records: list[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field(
description="Records of Each Type"
)
- user_type_records: List[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field(
+ user_type_records: list[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field(
description="Records of Each Type"
)
human_users: int = strawberry.field(
diff --git a/model/strawberry/output/observation/property_popularity/observation.py b/model/strawberry/output/observation/property_popularity/observation.py
index d22ac911..6e51c9e8 100644
--- a/model/strawberry/output/observation/property_popularity/observation.py
+++ b/model/strawberry/output/observation/property_popularity/observation.py
@@ -1,6 +1,5 @@
"""Wikibase Property Popularity Observation Strawberry Model"""
-from typing import List
import strawberry
from model.database import WikibasePropertyPopularityObservationModel
@@ -18,7 +17,7 @@ class WikibasePropertyPopularityObservationStrawberryModel(
):
"""Wikibase Property Popularity Observation"""
- property_popularity_counts: List[WikibasePropertyPopularityCountStrawberryModel] = (
+ property_popularity_counts: list[WikibasePropertyPopularityCountStrawberryModel] = (
strawberry.field(description="Number of Items with Number of Relationships")
)
diff --git a/model/strawberry/output/observation/software_version/software_version.py b/model/strawberry/output/observation/software_version/software_version.py
index c6c19fe5..458c001e 100644
--- a/model/strawberry/output/observation/software_version/software_version.py
+++ b/model/strawberry/output/observation/software_version/software_version.py
@@ -5,6 +5,7 @@
import strawberry
from model.database import WikibaseSoftwareVersionModel
+from model.strawberry.output.wikibase_software import WikibaseSoftwareStrawberryModel
@strawberry.type
@@ -12,7 +13,10 @@ class WikibaseSoftwareVersionStrawberryModel:
"""Wikibase Software Version"""
id: strawberry.ID
- software_name: str = strawberry.field(description="Software Name")
+ software: WikibaseSoftwareStrawberryModel = strawberry.field(description="Software")
+ software_name: str = strawberry.field(
+ description="Software Name", deprecation_reason="Use software/softwareName"
+ )
version: Optional[str] = strawberry.field(description="Software Version")
version_date: Optional[datetime] = strawberry.field(
description="Software Version Release Date"
@@ -29,7 +33,8 @@ def marshal(
return cls(
id=strawberry.ID(model.id),
- software_name=model.software_name,
+ software=WikibaseSoftwareStrawberryModel.marshal(model.software),
+ software_name=model.software.software_name,
version=model.version,
version_date=model.version_date,
version_hash=model.version_hash,
diff --git a/model/strawberry/output/observation/software_version/software_version_aggregate.py b/model/strawberry/output/observation/software_version/software_version_aggregate.py
index 03920a67..5c26e60f 100644
--- a/model/strawberry/output/observation/software_version/software_version_aggregate.py
+++ b/model/strawberry/output/observation/software_version/software_version_aggregate.py
@@ -2,7 +2,7 @@
from datetime import datetime
import re
-from typing import List, Optional
+from typing import Optional
import strawberry
from model.strawberry.output.semver import Semver
@@ -55,7 +55,7 @@ class WikibaseSoftwareMidVersionAggregateStrawberryModel:
version: Optional[str] = strawberry.field(description="Software Version")
private_versions: strawberry.Private[
- List[WikibaseSoftwareVersionAggregateStrawberryModel]
+ list[WikibaseSoftwareVersionAggregateStrawberryModel]
]
@strawberry.field(description="Wikibase Count")
@@ -74,7 +74,7 @@ class WikibaseSoftwarePatchVersionAggregateStrawberryModel(
@strawberry.field(description="Versions")
def sub_versions(
self,
- ) -> Optional[List[WikibaseSoftwareVersionAggregateStrawberryModel]]:
+ ) -> Optional[list[WikibaseSoftwareVersionAggregateStrawberryModel]]:
"""Sub-Patch Versions"""
if (
@@ -101,7 +101,7 @@ class WikibaseSoftwareMinorVersionAggregateStrawberryModel(
@strawberry.field(description="Patch Versions")
def patch_versions(
self,
- ) -> Optional[List[WikibaseSoftwarePatchVersionAggregateStrawberryModel]]:
+ ) -> Optional[list[WikibaseSoftwarePatchVersionAggregateStrawberryModel]]:
"""Patch Versions"""
if self.version is None:
@@ -135,7 +135,7 @@ class WikibaseSoftwareMajorVersionAggregateStrawberryModel(
@strawberry.field(description="Minor Versions")
def minor_versions(
self,
- ) -> Optional[List[WikibaseSoftwareMinorVersionAggregateStrawberryModel]]:
+ ) -> Optional[list[WikibaseSoftwareMinorVersionAggregateStrawberryModel]]:
"""Minor Versions"""
if self.version is None:
@@ -160,13 +160,13 @@ class WikibaseSoftwareVersionDoubleAggregateStrawberryModel:
software_name: str = strawberry.field(description="Software Name")
private_versions: strawberry.Private[
- List[WikibaseSoftwareVersionAggregateStrawberryModel]
+ list[WikibaseSoftwareVersionAggregateStrawberryModel]
]
def __init__(
self,
software_name: str,
- versions: List[WikibaseSoftwareVersionAggregateStrawberryModel],
+ versions: list[WikibaseSoftwareVersionAggregateStrawberryModel],
):
self.software_name = software_name
self.private_versions = versions
@@ -180,7 +180,7 @@ def wikibase_count(self) -> int:
@strawberry.field(description="Major Versions")
def major_versions(
self,
- ) -> List[WikibaseSoftwareMajorVersionAggregateStrawberryModel]:
+ ) -> list[WikibaseSoftwareMajorVersionAggregateStrawberryModel]:
"""Major Versions"""
temp: dict[
@@ -196,7 +196,7 @@ def major_versions(
return sorted(temp.values(), key=lambda x: x.wikibase_count(), reverse=True)
@strawberry.field(description="Version List")
- def versions(self) -> List[WikibaseSoftwareVersionAggregateStrawberryModel]:
+ def versions(self) -> list[WikibaseSoftwareVersionAggregateStrawberryModel]:
"""Version List"""
return sorted(
diff --git a/model/strawberry/output/observation/software_version/software_version_observation.py b/model/strawberry/output/observation/software_version/software_version_observation.py
index f1acbb6c..d112dcc9 100644
--- a/model/strawberry/output/observation/software_version/software_version_observation.py
+++ b/model/strawberry/output/observation/software_version/software_version_observation.py
@@ -1,6 +1,5 @@
"""Wikibase Software Version Observation Strawberry Model"""
-from typing import List
import strawberry
from model.database import WikibaseSoftwareVersionObservationModel
@@ -19,19 +18,19 @@ class WikibaseSoftwareVersionObservationStrawberryModel(
):
"""Wikibase Software Version Observation"""
- installed_extensions: List[WikibaseSoftwareVersionStrawberryModel] = (
+ installed_extensions: list[WikibaseSoftwareVersionStrawberryModel] = (
strawberry.field(description="Installed Extensions w/ Versions")
)
- installed_libraries: List[WikibaseSoftwareVersionStrawberryModel] = (
+ installed_libraries: list[WikibaseSoftwareVersionStrawberryModel] = (
strawberry.field(description="Installed Libraries w/ Versions")
)
- installed_skins: List[WikibaseSoftwareVersionStrawberryModel] = strawberry.field(
+ installed_skins: list[WikibaseSoftwareVersionStrawberryModel] = strawberry.field(
description="Installed Skins w/ Versions"
)
- installed_software: List[WikibaseSoftwareVersionStrawberryModel] = strawberry.field(
+ installed_software: list[WikibaseSoftwareVersionStrawberryModel] = strawberry.field(
description="Installed Software Versions"
)
@@ -49,7 +48,7 @@ def marshal(
[
WikibaseSoftwareVersionStrawberryModel.marshal(o)
for o in model.software_versions
- if o.software_type == WikibaseSoftwareType.EXTENSION
+ if o.software.software_type == WikibaseSoftwareType.EXTENSION
],
key=lambda x: x.software_name,
),
@@ -57,7 +56,7 @@ def marshal(
[
WikibaseSoftwareVersionStrawberryModel.marshal(o)
for o in model.software_versions
- if o.software_type == WikibaseSoftwareType.LIBRARY
+ if o.software.software_type == WikibaseSoftwareType.LIBRARY
],
key=lambda x: x.software_name,
),
@@ -65,7 +64,7 @@ def marshal(
[
WikibaseSoftwareVersionStrawberryModel.marshal(o)
for o in model.software_versions
- if o.software_type == WikibaseSoftwareType.SKIN
+ if o.software.software_type == WikibaseSoftwareType.SKIN
],
key=lambda x: x.software_name,
),
@@ -73,7 +72,7 @@ def marshal(
[
WikibaseSoftwareVersionStrawberryModel.marshal(o)
for o in model.software_versions
- if o.software_type == WikibaseSoftwareType.SOFTWARE
+ if o.software.software_type == WikibaseSoftwareType.SOFTWARE
],
key=lambda x: x.software_name,
),
diff --git a/model/strawberry/output/observation/user/wikibase_user_observation.py b/model/strawberry/output/observation/user/wikibase_user_observation.py
index 37c0573c..0618296f 100644
--- a/model/strawberry/output/observation/user/wikibase_user_observation.py
+++ b/model/strawberry/output/observation/user/wikibase_user_observation.py
@@ -1,6 +1,6 @@
"""Wikibase User Data Observation Strawberry Model"""
-from typing import List, Optional
+from typing import Optional
import strawberry
from model.database import WikibaseUserObservationModel
@@ -20,7 +20,7 @@ class WikibaseUserObservationStrawberryModel(WikibaseObservationStrawberryModel)
total_users: Optional[int] = strawberry.field(
description="Total Users", graphql_type=Optional[BigInt]
)
- user_groups: List[WikibaseUserObservationGroupStrawberryModel] = strawberry.field(
+ user_groups: list[WikibaseUserObservationGroupStrawberryModel] = strawberry.field(
description="User Groups and Counts"
)
diff --git a/model/strawberry/output/observation/wikibase_observation_set.py b/model/strawberry/output/observation/wikibase_observation_set.py
index b10a21c0..02f434b2 100644
--- a/model/strawberry/output/observation/wikibase_observation_set.py
+++ b/model/strawberry/output/observation/wikibase_observation_set.py
@@ -1,6 +1,6 @@
"""Wikibase Observation Set Strawberry Model"""
-from typing import Generic, List, Optional, TypeVar
+from typing import Generic, Optional, TypeVar
import strawberry
from model.strawberry.output.observation.wikibase_observation import (
@@ -15,7 +15,7 @@
class WikibaseObservationSetStrawberryModel(Generic[T]):
"""Wikibase Observation Set"""
- all_observations: List[T] = strawberry.field(description="All Observations")
+ all_observations: list[T] = strawberry.field(description="All Observations")
@strawberry.field(description="Most Recent Observation that Returned Data")
def most_recent(self) -> Optional[T]:
@@ -29,7 +29,7 @@ def most_recent(self) -> Optional[T]:
return None
@classmethod
- def marshal(cls, data: List[T]):
+ def marshal(cls, data: list[T]):
"""Coerce List into Set"""
return cls(all_observations=data)
diff --git a/model/strawberry/output/page.py b/model/strawberry/output/page.py
index 8f026d54..a51d57a6 100644
--- a/model/strawberry/output/page.py
+++ b/model/strawberry/output/page.py
@@ -1,7 +1,7 @@
"""Page"""
from math import ceil
-from typing import Annotated, Generic, List, TypeVar
+from typing import Annotated, Generic, TypeVar
import strawberry
from model.strawberry.scalars import BigInt
@@ -49,7 +49,7 @@ class Page(Generic[T]):
"""Page"""
meta: PageMetadata = strawberry.field(description="Metadata")
- data: List[T] = strawberry.field(description="Data")
+ data: list[T] = strawberry.field(description="Data")
@classmethod
def marshal(
@@ -57,7 +57,7 @@ def marshal(
page_number: PageNumberType,
page_size: PageSizeType,
total_count: int,
- page_data: List[T],
+ page_data: list[T],
) -> "Page":
"""Marshal Data into Page"""
return cls(
diff --git a/model/strawberry/output/wikibase_software.py b/model/strawberry/output/wikibase_software.py
new file mode 100644
index 00000000..8be6a05f
--- /dev/null
+++ b/model/strawberry/output/wikibase_software.py
@@ -0,0 +1,64 @@
+"""Wikibase Instance Strawberry Model"""
+
+from datetime import datetime
+from typing import Optional
+import strawberry
+
+from model.database import WikibaseSoftwareModel
+from model.enum.wikibase_software_type_enum import WikibaseSoftwareType
+
+
+@strawberry.type
+class WikibaseSoftwareStrawberryModel:
+ """Wikibase Software"""
+
+ id: strawberry.ID
+ software_type: WikibaseSoftwareType = strawberry.field(
+ description="Wikibase Software Type"
+ )
+ software_name: str = strawberry.field(description="Wikibase Software Name")
+
+ url: Optional[str] = strawberry.field(description="Reference URL")
+
+ fetched: Optional[datetime] = strawberry.field(
+ description="Date Fetched from MediaWiki"
+ )
+
+ tags: list[str] = strawberry.field(description="Tag List")
+
+ description: Optional[str] = strawberry.field(description="Description")
+
+ latest_version: Optional[str] = strawberry.field(description="Latest Version")
+
+ quarterly_download_count: Optional[int] = strawberry.field(
+ description="Quarterly Downloads"
+ )
+
+ public_wiki_count: Optional[int] = strawberry.field(
+ description="Public Wikis Using"
+ )
+
+ mediawiki_bundled: Optional[bool] = strawberry.field(
+ description="Bundled with MediaWiki"
+ )
+
+ archived: Optional[bool] = strawberry.field(description="Archived Extension")
+
+ @classmethod
+ def marshal(cls, model: WikibaseSoftwareModel) -> "WikibaseSoftwareStrawberryModel":
+ """Coerce Database Model to Strawberry Model"""
+
+ return cls(
+ id=strawberry.ID(model.id),
+ software_type=model.software_type,
+ software_name=model.software_name,
+ url=model.url,
+ fetched=model.data_fetched,
+ tags=[a.tag for a in model.tags],
+ description=model.description,
+ latest_version=model.latest_version,
+ quarterly_download_count=model.quarterly_download_count,
+ public_wiki_count=model.public_wiki_count,
+ mediawiki_bundled=model.mediawiki_bundled,
+ archived=model.archived,
+ )
diff --git a/model/strawberry/query.py b/model/strawberry/query.py
index 2c07fd2c..3b999d09 100644
--- a/model/strawberry/query.py
+++ b/model/strawberry/query.py
@@ -1,6 +1,5 @@
"""Query"""
-from typing import List
import strawberry
from model.enum import WikibaseSoftwareType
@@ -11,6 +10,7 @@
WikibasePropertyPopularityAggregateCountStrawberryModel,
WikibaseQuantityAggregateStrawberryModel,
WikibaseSoftwareVersionDoubleAggregateStrawberryModel,
+ WikibaseSoftwareStrawberryModel,
WikibaseStatisticsAggregateStrawberryModel,
WikibaseStrawberryModel,
WikibaseUserAggregateStrawberryModel,
@@ -23,6 +23,7 @@
get_aggregate_statistics,
get_aggregate_users,
get_aggregate_version,
+ get_software_list,
get_wikibase,
get_wikibase_list,
)
@@ -43,10 +44,20 @@ async def wikibase_list(
return await get_wikibase_list(page_number, page_size)
+ @strawberry.field(description="List of Extensions")
+ async def extension_list(
+ self, page_number: PageNumberType, page_size: PageSizeType
+ ) -> Page[WikibaseSoftwareStrawberryModel]:
+ """List of Wikibases"""
+
+ return await get_software_list(
+ page_number, page_size, WikibaseSoftwareType.EXTENSION
+ )
+
@strawberry.field(description="Aggregated Year of First Log Date")
async def aggregate_created(
self,
- ) -> List[WikibaseYearCreatedAggregateStrawberryModel]:
+ ) -> list[WikibaseYearCreatedAggregateStrawberryModel]:
"""Aggregated Year of First Log Date"""
return await get_aggregate_created()
diff --git a/pytest.ini b/pytest.ini
index c7c2b50d..4d5f1fff 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -8,6 +8,7 @@ markers =
log: log observation tests
property: property popularity observation tests
quantity: quantity observation tests
+ soup: soup-based observation tests
sparql: sparql-based observation tests
statistics: statistics observation tests
user: user observation tests
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 8567c6cb..8bf271e7 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,11 +1,11 @@
alembic>=1.13
-black>=24.8
+black>=24.10
freezegun>=1.5
-great-expectations>=1.0,<1.2
+great-expectations>=1.2
pylint>=3.3
pytest>=8.3
pytest-asyncio>=0.24
-pytest-cov>=5.0
+pytest-cov>=6.0
pytest-dependency>=0.6
pytest-mock>=3.14
pytest-order>=1.3
diff --git a/requirements.txt b/requirements.txt
index 8d23556d..a3e02927 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,6 +7,6 @@ numpy>=1.26,<2.0
requests>=2.32
SPARQLWrapper>=2.0
sqlalchemy[asyncio]>=2.0
-strawberry-graphql[fastapi]>=0.242
+strawberry-graphql[fastapi]>=0.247
tqdm>=4.66
-uvicorn[standard]>=0.30
+uvicorn[standard]>=0.32
diff --git a/resolvers/__init__.py b/resolvers/__init__.py
index bef7e4ab..1ede1f26 100644
--- a/resolvers/__init__.py
+++ b/resolvers/__init__.py
@@ -9,5 +9,6 @@
from resolvers.get_aggregate_software_version import get_aggregate_version
from resolvers.get_aggregate_statistics import get_aggregate_statistics
from resolvers.get_aggregate_users import get_aggregate_users
+from resolvers.get_software_list import get_software_list
from resolvers.get_wikibase import get_wikibase
from resolvers.get_wikibase_list import get_wikibase_list
diff --git a/resolvers/get_aggregate_created.py b/resolvers/get_aggregate_created.py
index 765ec393..1933e681 100644
--- a/resolvers/get_aggregate_created.py
+++ b/resolvers/get_aggregate_created.py
@@ -1,7 +1,5 @@
"""Get Aggregate Year Created"""
-from typing import List, Tuple
-
from sqlalchemy import Select, and_, select, func
from data import get_async_session
@@ -9,7 +7,7 @@
from model.strawberry.output import WikibaseYearCreatedAggregateStrawberryModel
-async def get_aggregate_created() -> List[WikibaseYearCreatedAggregateStrawberryModel]:
+async def get_aggregate_created() -> list[WikibaseYearCreatedAggregateStrawberryModel]:
"""Get Aggregate Year Created"""
total_quantity_query = get_created_query()
@@ -24,7 +22,7 @@ async def get_aggregate_created() -> List[WikibaseYearCreatedAggregateStrawberry
]
-def get_created_query() -> Select[Tuple[int, int]]:
+def get_created_query() -> Select[tuple[int, int]]:
"""Get Year Created Query"""
rank_subquery = (
diff --git a/resolvers/get_aggregate_property_popularity.py b/resolvers/get_aggregate_property_popularity.py
index a35d12ab..66b40b50 100644
--- a/resolvers/get_aggregate_property_popularity.py
+++ b/resolvers/get_aggregate_property_popularity.py
@@ -1,7 +1,5 @@
"""Get Aggregate Property Popularity"""
-from typing import Tuple
-
from sqlalchemy import Select, and_, desc, select, func
from data import get_async_session
@@ -56,7 +54,7 @@ async def get_aggregate_property_popularity(
)
-def get_unordered_query() -> Select[Tuple[int, str, int, int]]:
+def get_unordered_query() -> Select[tuple[int, str, int, int]]:
"""Get Unordered Property Popularity Query"""
rank_subquery = (
diff --git a/resolvers/get_aggregate_quantity.py b/resolvers/get_aggregate_quantity.py
index ba3d41dd..5cddbb58 100644
--- a/resolvers/get_aggregate_quantity.py
+++ b/resolvers/get_aggregate_quantity.py
@@ -1,7 +1,5 @@
"""Get Aggregate Quantity"""
-from typing import Tuple
-
from sqlalchemy import Select, and_, select, func
from data import get_async_session
@@ -28,7 +26,7 @@ async def get_aggregate_quantity() -> WikibaseQuantityAggregateStrawberryModel:
)
-def get_total_quantity_query() -> Select[Tuple[int, int, int, int, int]]:
+def get_total_quantity_query() -> Select[tuple[int, int, int, int, int]]:
"""Get Total Quantity Query"""
rank_subquery = (
diff --git a/resolvers/get_aggregate_software_version.py b/resolvers/get_aggregate_software_version.py
index c8ded732..6d591e96 100644
--- a/resolvers/get_aggregate_software_version.py
+++ b/resolvers/get_aggregate_software_version.py
@@ -1,13 +1,14 @@
"""Get Aggregate Software Version"""
from datetime import datetime
-from typing import Optional, Tuple
+from typing import Optional
from sqlalchemy import Select, and_, select, func
from data import get_async_session
from model.database import (
WikibaseModel,
+ WikibaseSoftwareModel,
WikibaseSoftwareVersionModel,
WikibaseSoftwareVersionObservationModel,
)
@@ -71,7 +72,7 @@ async def get_aggregate_version(
def get_query(
software_type: WikibaseSoftwareType,
-) -> Select[Tuple[str, Optional[str], Optional[datetime], Optional[str], int]]:
+) -> Select[tuple[str, Optional[str], Optional[datetime], Optional[str], int]]:
"""Get Software Version Query"""
rank_subquery = (
@@ -95,15 +96,15 @@ def get_query(
)
.subquery()
)
- query = (
+
+ next_subquery = (
select(
- WikibaseSoftwareVersionModel.software_name,
WikibaseSoftwareVersionModel.version,
WikibaseSoftwareVersionModel.version_date,
WikibaseSoftwareVersionModel.version_hash,
- # pylint: disable=not-callable
- func.count().label("wikibase_count"),
+ WikibaseSoftwareModel.software_name,
)
+ .join(WikibaseSoftwareModel)
.join(
rank_subquery,
onclause=and_(
@@ -112,13 +113,20 @@ def get_query(
rank_subquery.c.rank == 1,
),
)
- .where(WikibaseSoftwareVersionModel.software_type == software_type)
- .group_by(
- WikibaseSoftwareVersionModel.software_name,
- WikibaseSoftwareVersionModel.version,
- WikibaseSoftwareVersionModel.version_date,
- WikibaseSoftwareVersionModel.version_hash,
- )
- .order_by("id")
+ .where(WikibaseSoftwareModel.software_type == software_type)
+ .subquery()
+ )
+
+ query = select(
+ next_subquery.c.software_name,
+ next_subquery.c.version,
+ next_subquery.c.version_date,
+ next_subquery.c.version_hash,
+ func.count().label("wikibase_count"), # pylint: disable=not-callable
+ ).group_by(
+ next_subquery.c.software_name,
+ next_subquery.c.version,
+ next_subquery.c.version_date,
+ next_subquery.c.version_hash,
)
return query
diff --git a/resolvers/get_aggregate_statistics.py b/resolvers/get_aggregate_statistics.py
index 3759caa9..14ffde62 100644
--- a/resolvers/get_aggregate_statistics.py
+++ b/resolvers/get_aggregate_statistics.py
@@ -1,7 +1,5 @@
"""Get Aggregate Statistics"""
-from typing import Tuple
-
from sqlalchemy import Select, and_, select, func
from data import get_async_session
@@ -41,7 +39,7 @@ async def get_aggregate_statistics() -> WikibaseStatisticsAggregateStrawberryMod
def get_total_statistics_query() -> (
- Select[Tuple[int, int, int, int, int, int, int, int, int]]
+ Select[tuple[int, int, int, int, int, int, int, int, int]]
):
"""Get Total Statistics Query"""
diff --git a/resolvers/get_aggregate_users.py b/resolvers/get_aggregate_users.py
index 4ff46bf8..df6f8b9a 100644
--- a/resolvers/get_aggregate_users.py
+++ b/resolvers/get_aggregate_users.py
@@ -1,7 +1,5 @@
"""Get Aggregate Users"""
-from typing import Tuple
-
from sqlalchemy import Select, and_, or_, select, func
from data import get_async_session
@@ -36,7 +34,7 @@ async def get_aggregate_users() -> WikibaseUserAggregateStrawberryModel:
)
-def get_total_admin_query() -> Select[Tuple[int]]:
+def get_total_admin_query() -> Select[tuple[int, int]]:
"""Get Total Admin Query"""
rank_subquery = (
@@ -90,7 +88,7 @@ def get_total_admin_query() -> Select[Tuple[int]]:
return query
-def get_total_user_query() -> Select[Tuple[int, int]]:
+def get_total_user_query() -> Select[tuple[int, int]]:
"""Get Total User Query"""
rank_subquery = (
diff --git a/resolvers/get_software_list.py b/resolvers/get_software_list.py
new file mode 100644
index 00000000..2f9f1d2a
--- /dev/null
+++ b/resolvers/get_software_list.py
@@ -0,0 +1,47 @@
+"""Get Wikibase List"""
+
+from sqlalchemy import func, select
+
+from data import get_async_session
+from model.database import WikibaseSoftwareModel
+from model.enum import WikibaseSoftwareType
+from model.strawberry.output import (
+ Page,
+ PageNumberType,
+ PageSizeType,
+ WikibaseSoftwareStrawberryModel,
+)
+
+
+async def get_software_list(
+ page_number: PageNumberType,
+ page_size: PageSizeType,
+ software_type: WikibaseSoftwareType,
+) -> Page[WikibaseSoftwareStrawberryModel]:
+ """Get Wikibase List"""
+
+ async with get_async_session() as async_session:
+ total_count = await async_session.scalar(
+ # pylint: disable=not-callable
+ select(func.count())
+ .select_from(WikibaseSoftwareModel)
+ .where(WikibaseSoftwareModel.software_type == software_type)
+ )
+ results = (
+ await async_session.scalars(
+ select(WikibaseSoftwareModel)
+ .where(WikibaseSoftwareModel.software_type == software_type)
+ .order_by(
+ WikibaseSoftwareModel.software_type,
+ WikibaseSoftwareModel.software_name,
+ )
+ .offset((page_number - 1) * page_size)
+ .limit(page_size)
+ )
+ ).all()
+ return Page.marshal(
+ page_number,
+ page_size,
+ total_count,
+ [WikibaseSoftwareStrawberryModel.marshal(c) for c in results],
+ )
diff --git a/tests/mock_info.py b/tests/mock_info.py
new file mode 100644
index 00000000..aaeca2fd
--- /dev/null
+++ b/tests/mock_info.py
@@ -0,0 +1,26 @@
+"""MockInfo"""
+
+from typing import List
+
+
+class MockBackgroundClassList:
+ """Mock Backround Class List"""
+
+ tasks: List
+
+ def add_task(self, task):
+ """Add Background Task"""
+
+ self.tasks.append(task)
+
+ def __init__(self):
+ self.tasks = []
+
+
+class MockInfo:
+ """Mock StrawberryInfo"""
+
+ context: dict
+
+ def __init__(self, context: dict):
+ self.context = context
diff --git a/tests/test_create_observation/test_create_connectivity_observation.py b/tests/test_create_observation/test_create_connectivity_observation.py
index 647f6a51..bc7603e0 100644
--- a/tests/test_create_observation/test_create_connectivity_observation.py
+++ b/tests/test_create_observation/test_create_connectivity_observation.py
@@ -1,6 +1,5 @@
"""Test create_connectivity_observation"""
-from typing import List
from urllib.error import HTTPError
import pytest
from fetch_data import create_connectivity_observation
@@ -33,7 +32,7 @@
],
)
async def test_create_connectivity_observation_success(
- mocker, links: List[tuple[str, str]]
+ mocker, links: list[tuple[str, str]]
):
"""Test"""
diff --git a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py
index 7a4f3926..dc95f3d0 100644
--- a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py
+++ b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py
@@ -107,10 +107,13 @@ def mockery(*args, **kwargs):
match (query.params.get("ledir"), query.params.get("lelimit")):
case ("newer", 1):
return MockResponse(
- 200, json.dumps({"query": {"logevents": [oldest_mock_log]}})
+ query.raw_url,
+ 200,
+ json.dumps({"query": {"logevents": [oldest_mock_log]}}),
)
case ("newer", 500):
return MockResponse(
+ query.raw_url,
200,
json.dumps(
{
@@ -124,10 +127,13 @@ def mockery(*args, **kwargs):
)
case ("older", 1):
return MockResponse(
- 200, json.dumps({"query": {"logevents": [newest_mock_log]}})
+ query.raw_url,
+ 200,
+ json.dumps({"query": {"logevents": [newest_mock_log]}}),
)
case ("older", 500):
return MockResponse(
+ query.raw_url,
200,
json.dumps(
{
@@ -149,7 +155,9 @@ def mockery(*args, **kwargs):
users.append({"name": "User:B", "invalid": True})
if "User:C" in query.params.get("ususers"):
users.append({"name": "User:C", "groups": ["*", "users", "bot"]})
- return MockResponse(200, json.dumps({"query": {"users": users}}))
+ return MockResponse(
+ query.raw_url, 200, json.dumps({"query": {"users": users}})
+ )
raise NotImplementedError(query)
mocker.patch(
diff --git a/tests/test_create_observation/test_create_software_version_observation/__init__.py b/tests/test_create_observation/test_create_software_version_observation/__init__.py
new file mode 100644
index 00000000..4632e8bc
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/__init__.py
@@ -0,0 +1 @@
+"""Test Create Software Version Observation"""
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Babel.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Babel.html
new file mode 100644
index 00000000..41e72532
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Babel.html
@@ -0,0 +1,1136 @@
+
+
+
+
+Extension:Babel - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : Babel
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The Babel extension adds a parser function to replace the old Babel system that completely relied on templates. If an unrecognized language parameter is specified, it will see if there is an existing template with the name and include that.
+
On Wikimedia projects, the noun Babel (in reference to the Tower of Babel ) refers to the texts on user pages aiding multilingual communication by making it easier to contact someone who speaks a certain language. The idea originated on the Wikimedia Commons and has also been implemented on many other wikis.
+
+
+
Installation
+
Download and move the extracted Babel
folder to your extensions/
directory. Developers and code contributors should install the extension from Git instead, using:cd extensions/ git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Babel
+Add the following code at the bottom of your LocalSettings.php file: wfLoadExtension ( 'Babel' );
+
+Run the update script which will automatically create the necessary database tables that this extension needs.
+Configure as required.
+ Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+
+
The CSS is located in the file resources/ext.babel.css
. You can change the style as desired by overriding them in the page MediaWiki:Common.css
.
+If the CLDR extension is found language names are taken from that (where translations are unavailable), otherwise built in MediaWiki language names and English defaults are used.
+
Usage
+
Syntax for the #babel
parser function is as follows:
+
+
{{ #babel : babelcode1 | babelcode2 | ... }}
+
+
Add one of the following codes for each language you speak or understand, separated by |
, where xx
is the MediaWiki language code , ISO 639-1 code, or ISO 639-3 code for the language. The general usage of each code level is as follows:
+
+
xx-0
+If you don't understand the language at all.
+xx-1
+Basic ability—enough to understand written material or simple questions in this language.
+xx-2
+Intermediate ability—enough for editing or discussions.
+xx-3
+Advanced level—though you can write in this language with no problem, some small errors might occur.
+xx-4
+"Near-native" level—although it's not your first language from birth, your ability is something like that of a native speaker.
+xx-5
+Professional proficiency.
+xx
or xx-N
+Native speakers who use a language every day and have a thorough grasp of it, including colloquialisms and idioms.
+
To include any other template, add the name of the template, e.g., add User CSS
if you want to include Template:User CSS
. A prefix or suffix may be added to template names (e.g., User
at the beginning) depending on the local configuration. This can be used to restrict the selection and reduce the length of parameters; for example, CSS
could include Template:User CSS
if configured in such a way.
+
+
Parameters
+
To remove the header and footer, use plain=1
as the first parameter, e.g., {{ #babel : plain=1 | babelcode1 | babelcode2 | ... }}
.
+This makes it easier to use Babel with other userboxes.
+
To hide categories, use the nocat=1
parameter as the first parameter, e.g., {{ #babel : nocat=1 | babelcode1 | babelcode2 | ... }}
.
+
Please note that only one of the parameters above is allowed.
+At the moment, it is not possible to use both parameters; for example, {{ #babel : nocat=1 | plain = 1 | babelcode1 | babelcode2 | ... }}
will not work.
+
+
Categorization
+
If categorization is enabled, the extension creates categories using the Babel AutoCreate bot with the text specified in MediaWiki:babel-autocreate-text-levels and MediaWiki:babel-autocreate-text-main . With basic settings, the categories that the bot creates are not categorized, and to fix this, it is recommended to do the following:
+
+
Create a template {{Babel category }} that will generate categories.
+Replacing text on MediaWiki:babel-autocreate-text-levels with {{ Babel category | level = $1| language = $2| ISO = $3}}
+
+Replacing text on MediaWiki:babel-autocreate-text-main with {{ Babel category | language = $1| ISO = $2}}
+
+
This will allow you to categorize categories automatically, and if something happens, you can simultaneously replace the categorization and text in all categories.
+
+
Configuration
+
Configuration parameters
+
Babel has several configuration parameters which can be modified in LocalSettings.php
.
+
+
$wgBabelLanguageCodesCdb
+(string) the path of the language code database file, the default should suffice.
+$wgBabelLanguageNamesCdb
+(string) the path of the language name database file, the default should suffice.
+$wgBabelCategoryNames
+(array of string or boolean, indexed by the strings "1", "2", … "5", "N") where each entry is the name of a category for the skill level indicated by its index, possible variable elements are: %code%
(language code), %wikiname%
(the name of the language in the wiki's content language), and %nativename%
(the name of the language in its language). To disable adding a category for a particular level, set the corresponding value to false.
+
For example:
+
$wgBabelCategoryNames = [
+ '0' => 'User %code%-0' ,
+ '1' => 'User %code%-1' ,
+ '2' => 'User %code%-2' ,
+ '3' => 'User %code%-3' ,
+ '4' => 'User %code%-4' ,
+ '5' => 'User %code%-5' ,
+ 'N' => 'User %code%-N' ,
+];
+
+
will use categories like "Category:User en-0" and "Category:User fr-N". The default is just "Category:Fr-N" and so on.
+
$wgBabelMainCategory
+(string) Name of the main (non-level) category for each language to which all users of that language are added. Set to false to disable; defaults to format "Category:Fr". It accepts the same format as $wgBabelCategoryNames
above. Example:
+$wgBabelMainCategory = 'User %code%' ;
+
$wgBabelDefaultLevel
+(string) Default ability level to use when none is specified, should be an index from $wgBabelCategoryNames
, that is one of the strings "1", "2", … "5", "N". Default is "N".
+$wgBabelUseUserLanguage
+(boolean) Whether to use the user interface language for the header and footer message. If false (default), it will be in the page content language. This is because using the user interface language may fragment the parser cache.
+$wgBabelCategorizeNamespaces
+Array of namespaces to only add automatic categorization to. For example, if $wgBabelCategorizeNamespaces = [ NS_USER ];
, then Babel will only add categories to pages in the user namespace. The default is null, which means categorizing all namespaces.
+$wgBabelCategoryOverride
+Whether to allow Babel categories to be overridden on the wiki using MediaWiki:Babel-category-override
+$wgBabelAutoCreate
+Whether to auto-create categories.
+
System messages
+
Several customizations can also be made using MediaWiki namespace messages.
+
+
MediaWiki:babel-template "Template:User $1
"
+The format of template names when one is being included.
+MediaWiki:babel-portal "
"
+The format of the link's target from the language code. Set to the empty string to not link the language code.
+MediaWiki:Babel-autocreate-user "Babel AutoCreate
"
+Username to be used for auto-creation of Babel related categories
+MediaWiki:babel-autocreate-text-levels "Users in this category indicate they have skill level $1 for language $2.
"
+Text to insert into auto-created categories for different language levels. You have to change this if you want them to be auto-categorized in the main category of the respective language ($wgBabelMainCategory
).
+MediaWiki:babel-autocreate-text-main "Users in this category indicate their knowledge of language $1.
"
+Text to insert into auto-created categories for non-level categories. You have to change this if you want them to be auto-categorized in a parent category for all languages.
+MediaWiki:babel "Babel user information
"
+The header of the babel box. Set to -
to not display a header.
+MediaWiki:babel-url "m:User language
"
+The page name where information on the babel extension can be found. Set to -
to display no link in the header.
+MediaWiki:Babel-footer "
"
+The footer of the babel box. Set to -
to not display a footer.
+MediaWiki:babel-footer-url "
+:Category:Babel - Users by language
"
+The page to link to in the footer of the babel box
+MediaWiki:Babel-category-override "$1
"
+Overrides any automatically-generated Babel categories. Parameters: $1 = the category that would be generated normally. $2 = the language code $3 = the babel level. Any categories overridden using this method will not be auto-created to reduce the risk of vandalism or mistaken edits to that page.
+
API
+
+
+
Get information about what languages the user knows
+
+
Specific parameter:
babuser User to get information about
+
This parameter is required. Type: user, by any of username, IP, Temporary user, IP range and interwiki name (e.g. "prefix>ExampleName")
+
+
+
+
This extension is being used on one or more Wikimedia projects . This probably means that the extension is stable and works well enough to be used by such high-traffic websites. Look for this extension's name in Wikimedia's CommonSettings.php and InitialiseSettings.php configuration files to see where it's installed. A full list of the extensions installed on a particular wiki can be seen on the wiki's Special:Version page.
+
This extension is included in the following wiki farms/hosts and/or packages:
+This is not an authoritative list. Some wiki farms/hosts and/or packages may contain this extension even if they are not listed here. Always check with your wiki farms/hosts or bundle to confirm.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_GoogleAnalyticsIntegration.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_GoogleAnalyticsIntegration.html
new file mode 100644
index 00000000..14062f2a
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_GoogleAnalyticsIntegration.html
@@ -0,0 +1,850 @@
+
+
+
+
+Extension:Google Analytics Integration - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : Google Analytics Integration
+
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The Google Analytics Integration extension inserts Google Universal Analytics (and/or other web analytics) tracking code in every page viewed.
+Exclusion of specific pages, namespaces, special pages and all pages for specific groups of users is configurable.
+
This extension may be outdated for your needs. If you are using the new Global Site Tag for Google Analytics, try using Extension:HeadScript instead.
+
Alternatively you can put Google Analytics tag directly into MediaWiki:Common.js of your wiki.
+
+
+
Download and move the extracted googleAnalytics
folder to your extensions/
directory. Developers and code contributors should install the extension from Git instead, using:cd extensions/ git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/googleAnalytics
+Add the following code at the bottom of your LocalSettings.php file: require_once " $IP /extensions/googleAnalytics/googleAnalytics.php" ;
+// Replace xxxxxxx-x with YOUR GoogleAnalytics UA number
+$wgGoogleAnalyticsAccount = 'UA-xxxxxxx-x' ;
+
+// Optional configuration (for defaults see googleAnalytics.php)
+
+// Add HTML code for any additional web analytics (can be used alone or with $wgGoogleAnalyticsAccount)
+$wgGoogleAnalyticsOtherCode = '<script type="text/javascript" src="https://analytics.example.com/tracking.js"></script>' ;
+
+// Store full IP address in Google Universal Analytics (see https://support.google.com/analytics/answer/2763052?hl=en for details)
+$wgGoogleAnalyticsAnonymizeIP = true ;
+
+// Array with NUMERIC namespace IDs where web analytics code should NOT be included.
+$wgGoogleAnalyticsIgnoreNsIDs = [
+ 500
+ ];
+
+// Array with page names (see magic word {{FULLPAGENAME}}) where web analytics code should NOT be included.
+$wgGoogleAnalyticsIgnorePages = [
+ 'PageName' ,
+ 'NamespaceName:PageName'
+ ];
+
+// Array with special pages where web analytics code should NOT be included.
+$wgGoogleAnalyticsIgnoreSpecials = [
+ 'Userlogin' ,
+ 'Userlogout' ,
+ 'Preferences' ,
+ 'ChangePassword' ,
+ 'OATH'
+ ];
+
+// Use 'noanalytics' permission to exclude specific user groups from web analytics, e.g.
+$wgGroupPermissions [ 'sysop' ][ 'noanalytics' ] = true ;
+$wgGroupPermissions [ 'bot' ][ 'noanalytics' ] = true ;
+
+// To exclude all logged in users, give 'noanalytics' permission to the 'user' group, i.e.
+$wgGroupPermissions [ 'user' ][ 'noanalytics' ] = true ;
+
+ Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+
The following options were removed in version 3.0.0:
+
$wgGoogleAnalyticsAddASAC
+$wgGoogleAnalyticsIgnoreSysops
+$wgGoogleAnalyticsIgnoreBots
+
+
Create a Google Analytics account.
+Locate your UA number.
+For the legacy code block, it can be found on the following line:
+For the new ga.js code block, it can be found on the following line: var pageTracker = _gat . _getTracker ( "UA-xxxxxxx-x" );
+
+Follow installation instructions .
+Google Analytics stats should start populating within 24-48 hours.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_LabeledSectionTransclusion.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_LabeledSectionTransclusion.html
new file mode 100644
index 00000000..ce854763
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_LabeledSectionTransclusion.html
@@ -0,0 +1,1214 @@
+
+
+
+
+Extension:Labeled Section Transclusion - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : Labeled Section Transclusion
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The Labeled Section Transclusion extension allows selective transclusion of marked-off sections of text, parsing wikitext as normal. Its functionality is similar to an enhanced version of the < onlyinclude >
tag with normal wiki transclusion, which selects sections for inclusion. It is enabled on all Wikimedia wikis.
+
While normal transclusion is primarily intended to transclude large portions of small templates, labeled section transclusion is intended for small portions of large pages.
+
However, there are some differences. In the native template transclusion, sections are marked by behavior; thus you can have only one (possibly non-contiguous) section to be included or skipped.
+
Here, sections are marked by name, and behavior is chosen by the caller, which can include or skip sections as needed. Different pages can include or exclude selected sections; there can be arbitrary numbers of sections, which can also overlap arbitrarily.
+
Marking sections by name rather than behavior allows edit section links to be rendered more appropriately for getting excerpts from larger texts, since the extension can now account for sections that are skipped in the beginning of the page, allowing transcluded sections to be offset appropriately.
+
+
+
+
Download and move the extracted LabeledSectionTransclusion
folder to your extensions/
directory. Developers and code contributors should install the extension from Git instead, using:cd extensions/ git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/LabeledSectionTransclusion
+Add the following code at the bottom of your LocalSettings.php file: wfLoadExtension ( 'LabeledSectionTransclusion' );
+
+ Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+
There is also a Gadget in use on Wikisource.org wikis that makes it possible to define sections with a simplified ## label ##
syntax.
+Its code can be found at Wikisource:MediaWiki:Gadget-Easy_LST.js .
+
+
+
Transclude any marked part [ edit ]
+
Step 1: Mark off sections [ edit ]
+
Mark off sections in the text using < section >
tags like this:
+
+
< section begin = "chapter1" /> this is chapter 1< section end = "chapter1" />
+
+
Note that these tags are not HTML/XML, and do not use the normal attribute syntax.
+For this reason, and because the begin
and end
markers are individual, rather than normal XML open/close tags, this allows nested or overlapping sections. This allows you to insert section tags without worrying about interfering with other sections.
+
+
+
Step 2a: Transclude the section [ edit ]
+
Call the parser function #lst to transclude it, i.e. to transclude a section called chapter1 from a page called articleX :
+
+
{{#lst:articleX|chapter1}}
+
+
The target article defines the location of the section; its behavior is determined by the parser function.
+
+
Step 2b: Transclude the page but excluding the section [ edit ]
+
To transclude a page, but exclude a specified section, use the #lstx function:
+
+
{{#lstx:articleX|chapter1}}
+
+
Optionally, you may add replacement text to the excluded section.
+
+
{{#lstx:articleX|chapter1|replacement_text}}
+
+
Example:
+
+
{{#lstx:articleX|chapter1|See chapter 1 in [[articleX]].}}
+
+
The replacement text will appear in the area where the section is skipped (excluded).
+
+
Discontiguous sections [ edit ]
+
It is possible to have multiple sections with the same name; in this case, every section with that name will be included/excluded. This is especially useful to mark various discussions.
+
+
+
These functions have an additional, optional argument to specify a section range; i.e. {{#lst:articleX|chapter1|chapter3}}, to include everything from the beginning of chapter 1 to the end of chapter 3. This allows using empty marker pairs to mark one end of the section, possibly in a template. A similar mechanism is currently used at the French Wikisource.
+
+
+
This also works with substitution; it's even possible for an article to substitute a section of itself. One use of this provides a neat way to archive talk pages: Mark the text to be archived using < section begin = archive />
, etc. Then create an archive page with the text, using {{subst:#lst:talk_page|archive}}, which copies archived sections. Lastly, replace the contents of talk_page with {{subst:#lstx:talk_page|archive}} to remove those sections.
+
There is optional support for transcluding sections of text marked with the normal headings, i.e. ==this section==
. If installed, this is done with the lsth function.
+
+
Transclude before the first heading [ edit ]
+
To transclude the introduction of a page (i.e. the content before the first heading), use
+
+
{{#lsth:pagename}}
+
+
Transclude a specific section [ edit ]
+
You can also transclude the whole content of the sectionX (which includes all its sub-sections but excludes the heading of sectionX itself).
+
+
{{#lsth:pagename|sectionX}}
+
+
Things to note:
+
+
Only the first occurrence of the sectionX is transcluded if you have more than one section with the same name.
+Make sure you type what the heading of sectionX is in wikitext , not how it is displayed. For example if the heading of the section is ==List of [[Extension]]==
, you should type "List of [[Extension]]
" not "List of Extension
".
+When transcluding a section from a page marked for translation using the translate extension, transclude from the language-specific version. E.g. from pagename/en rather than from pagename .
+The matching is case in sensitive, to prevent links from breaking due to case changes.
+
Transclude multiple sections [ edit ]
+
You can also transclude from the first occurrence of sectionX (excluding the heading of sectionX itself) until it reaches the next occurrence of sectionY . Note that sectionY acts as a stop point so the transclusion doesn't contain the content of sectionY .
+
+
{{#lsth:pagename|sectionX|sectionY}}
+
+
Notes about skipped headings [ edit ]
+
Since the traditional transclusion in MediaWiki isn't intended to transclude sections, it doesn't account for skipped headings. As a result, if you were to transclude a template with multiple headings, and skip the first heading, then all of the edit sections links would point to the wrong section in the template.
+
When this extension is used (with MediaWiki 1.9 or later), the #lst and #lsth functions count headings in the "skipped" beginning part, and offset transcluded headings appropriately. This will allow these links to point to the correct section in the simple case.
+
Note that #lstx does not count skipped headings, and that skipped headings within discontiguous sections are not offset. But it seems it has been fixed now (likely when ported to MediaWiki's new preprocessor). The transcluded headings can be linked to the correct sections .
+
+
+
Internally, the parser functions all use the lst prefix, for consistency with the name of the extension. Since this acronym may be confusing to non-developers, readable English variants have been introduced, so the functions can currently be called from either name.
+
+
+
+function
+
+English
+
+German
+
+Hebrew (RTL)
+
+Portuguese
+
+
+#lst
+#section
+#Abschnitt
+#קטע
+#trecho
+
+
+#lstx
+#section-x
+#Abschnitt-x
+#בלי קטע
+#trecho-x
+
+
+#lsth
+#section-h
+
+
+
+
+
Additionally, the tag can now be localised; currently: English, German, Hebrew, Portuguese; i.e.:
+
+
English
+< section begin = x/ > ... < section end = x/ >
+German
+< Abschnitt Anfang = x/ > ... < Abschnitt Ende = x/ >
+Hebrew (RTL)
+< קטע התחלה = א > ... < קטע סוף = א >
("start" code to the right and "end" code to the left)
+Portuguese
+< trecho começo = x/ > ... < trecho fim = x/ >
+
Each localization is enabled only if the page matches the respective content language.
+
+
+
{{#lsth:pagename|sectionX}} only works on the first section if multiple sections have name sectionX . Only the first occurrence of sectionX is transcluded if an article has more than one section with the same name.
+While it is possible to use this extension across namespaces, interwiki references are not resolved. It is not yet possible, for example, to include part of a Wikisource page into a remote MediaWiki installation.
+Section tags cannot themselves be transcluded in order to work on other pages. {{#lst:}}
and {{#lstx:}}
work only if section tags appear directly in the wikitext of the transcluded page. This means, for instance, that these tags cannot be embedded in a template using template parameters and parser functions . The #tag
magic word does not work with section tags.
+As of 2014, section tags don't have any effect when used inside a template parameter. If page A contains a text {{B|X}}, there's no way {{#lst:A|...}} can access X.
+
+
+
+
+
This extension is being used on one or more Wikimedia projects . This probably means that the extension is stable and works well enough to be used by such high-traffic websites. Look for this extension's name in Wikimedia's CommonSettings.php and InitialiseSettings.php configuration files to see where it's installed. A full list of the extensions installed on a particular wiki can be seen on the wiki's Special:Version page.
+
This extension is included in the following wiki farms/hosts and/or packages:
+This is not an authoritative list. Some wiki farms/hosts and/or packages may contain this extension even if they are not listed here. Always check with your wiki farms/hosts or bundle to confirm.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_ProofreadPage.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_ProofreadPage.html
new file mode 100644
index 00000000..6474d8c5
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_ProofreadPage.html
@@ -0,0 +1,1205 @@
+
+
+
+
+Extension:Proofread Page - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : Proofread Page
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+The Proofread Page extension creates a book either:
+
+
as a column of OCR text beside a column of scanned images, or
+broken into chapters or poems. The content of a document appears in the MediaWiki page (via transclusion).
+
The extension is intended to allow easy comparison of text to the original digitization.
+
This extension shows the text in several ways without actually duplicating the original text.[ 1]
+
+
+
+
The extension is installed on all Wikisource wikis.
+For the syntax, see the Wikisource Proofread Page documentation .
+It was previously also used on Bibliowiki .
+
+
Requirements and recommendations [ edit ]
+
Access to the command line is required if running the update script (maintenance/update.php) from the web browser fails (see Upgrade documentation and Update.php documentation ).
+If you want to use DjVu files (optional but recommended), a native DjVu handler needs to be available for configuration. See also Manual:How to use DjVu with MediaWiki .
+In addition, use of ProofreadPage is highly improved by the use of the following extensions:
+
+
+
+
+
+
Download and move the extracted ProofreadPage
folder to your extensions/
directory. Developers and code contributors should install the extension from Git instead, using:cd extensions/ git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/ProofreadPage
+Add the following code at the bottom of your LocalSettings.php file: wfLoadExtension ( 'ProofreadPage' );
+
+Run the update script which will automatically create the necessary database tables that this extension needs.
+ Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+
+
The extension links directly to image thumbnails which often don't exist.
+You must catch 404 errors and generate the missing thumbnails.
+You can do this with any one of these solutions:
+
+
Set an Apache RewriteRule in .htaccess to thumb.php for missing thumbnails:
+
RewriteEngine On
+ RewriteCond %{REQUEST_FILENAME} !-f
+ RewriteCond %{REQUEST_FILENAME} !-d
+ RewriteRule ^/w/images/thumb/[0-9a-f]/[0-9a-f][0-9a-f]/([^/]+)/page([0-9]+)-?([0-9]+)px-.*$ /w/thumb.php ?f=$1&p=$2&w=$3 [L,QSA]
+
+
or set the Apache 404 handler to Wikimedia's thumb-handler . This is a general-purpose 404 handler with Wikimedia-specific code, not simply a thumbnail generator.
+
ErrorDocument 404 /w/extensions/upload-scripts/404.php
+
+
For MediaWiki >= 1.20, you can simply redirect to thumb_handler.php:
+
RewriteEngine On
+ RewriteCond %{REQUEST_FILENAME} !-f
+ RewriteCond %{REQUEST_FILENAME} !-d
+ RewriteRule ^/w/images/thumb/[0-9a-f]/[0-9a-f][0-9a-f]/([^/]+)/page([0-9]+)-?([0-9]+)px-.*$ /w/thumb_handler.php [L,QSA]
+
+
+
ErrorDocument 404 /w/thumb_handler.php
+
+
Warning: There is an .htaccess file in the images directory that may interfere with any .htaccess rules you install.
+
If you encounter a problem similar to the following:
+
+
phab:T301291 – PDF and DjVu files on Commons failed to be processed (no thumbnails, zero pages) but otherwise valid
+phab:T298417 – Undeleted DjVu files show incorrect metadata: 0x0 size, no page number info
+phab:T299521 – PDF file has 0x0 image size in Commons after uploading a new version while the page number is correct
+
Try next steps:
+
+
+repair thumbnails for DjVu files of the core MediaWiki (for PDF use mimetype application/pdf
)
+
+ php maintenance/refreshImageMetadata.php --verbose --mime image/vnd.djvu --force
+
+
+
+needed for actualization info about the pages counts of the Special:IndexPages
+
+ php maintenance/refreshLinks.php --namespace 252
+
+
+
+
+
ProofreadPage create by default two custom namespaces named "Page" and "Index" in English with respectively ids 250 and 252.
+
Their names are translated if your wiki use another language.
+Full list .
+
You can customize their name or their ID: Create namespaces by hand and set their IDs in Manual:LocalSettings.php using $wgProofreadPageNamespaceIds global.
+You will do something like:
+
+
define ( 'NS_PROOFREAD_PAGE' , 250 );
+define ( 'NS_PROOFREAD_PAGE_TALK' , 251 );
+define ( 'NS_PROOFREAD_INDEX' , 252 );
+define ( 'NS_PROOFREAD_INDEX_TALK' , 253 );
+$wgExtraNamespaces [ NS_PROOFREAD_PAGE ] = 'Page' ;
+$wgExtraNamespaces [ NS_PROOFREAD_PAGE_TALK ] = 'Page_talk' ;
+$wgExtraNamespaces [ NS_PROOFREAD_INDEX ] = 'Index' ;
+$wgExtraNamespaces [ NS_PROOFREAD_INDEX_TALK ] = 'Index_talk' ;
+$wgProofreadPageNamespaceIds = array (
+ 'index' => NS_PROOFREAD_INDEX ,
+ 'page' => NS_PROOFREAD_PAGE
+);
+
+
Namespace id customization is not recommended and might not be supported in the future.
+
+
+
+
Configuration of index namespace [ edit ]
+
For more details, see Extension:Proofread Page/Index data configuration
+
+
+
The configuration is a JSON array of properties.
+Here is the structure of a property in the array, all the parameters are optional, the default value are set:
+
+
{
+ "ID" : { //id of the metadata (first parameter of proofreadpage_index_attributes)
+ "type" : "string" , //the property type (for compatibility reasons the values have not to be of this type). Possibles values: string, number, page. If set, the newly set values should be valid according to the type (e.g. for a number a valid number, for a page an existing wiki page...)
+ "size" : 1 , //only for the type string : number of lines of the input (third parameter of proofreadpage_index_attributes)
+ "values" : { "a" : "A" , "b" : "B" , "c" : "C" , "d" : "D" }, //an array values : label that list the possible values (for compatibility reasons the stored values have not to be one of these)
+ "default" : "" , //the default value
+ "header" : false , //add the property to MediaWiki:Proofreadpage_header_template template (true is equivalent to being listed in proofreadpage_js_attributes)
+ "label" : "ID" , //the label in the form (second parameter of proofreadpage_index_attributes)
+ "help" : "" , //a short help text
+ "delimiter" : [], //list of delimiters between two part of values. By example ["; ", " and "] for strings like "J. M. Dent; E. P. Dutton and A. D. Robert"
+ "data" : "" //proofreadpage's metadata type that the property is equivalent to
+ }
+}
+
+
The data parameter can have for value:
+"type", "language", "title", "author", "translator", "illustrator", "editor", "school", "year", "publisher", "place", "progress"
+
+
+
The extension puts a separator between every transcluded page and the next, which is defined by wgProofreadPagePageSeparator
.
+The default value is  
(a whitespace).
+Set wgProofreadPagePageSeparator = ""
to suppress the separator.
+
+
Join hyphenated words across pages [ edit ]
+
When a word is hyphenated between a page and the next, the extension joins together the two halves of the word.
+Example: his- and tory becomes history .
+The "joiner" character is defined by wgProofreadPagePageJoiner
and defaults to '-' (the ASCII hyphen character).
+
+
+
See Change tagging to set up change tags.
+
+
+
Creating your first page (example with DjVu)[ edit ]
+
Before following these steps ensure you have followed the instructions in Manual:How to use DjVu with MediaWiki .
+(when and in which namespace is the DjVu file itself uploaded?)
+Create a page in the "Page" namespace (or the internationalized name if you use an not-English wiki). For example if your namespace is 'Page' create Page:Carroll - Alice's Adventures in Wonderland.djvu
+Create the corresponding file for this page commons:File:Carroll - Alice's Adventures in Wonderland.djvu (or set Manual:$wgUseInstantCommons to true
).
+Create the index page Index:Carroll - Alice's Adventures in Wonderland.djvu
+Insert the tag < pagelist />
in the Pages field to visualize the page list
+To edit page 5 of the book navigate to 'Page:Carroll - Alice's Adventures in Wonderland/5' and click edit
+
+
This extension introduces the following tags:
+< pages >
, < pagelist >
+
+
+
+↑
+Because the pages are not in the main namespace, they are not included in the statistical count of text units.
+
+
+
+
+
This extension is being used on one or more Wikimedia projects . This probably means that the extension is stable and works well enough to be used by such high-traffic websites. Look for this extension's name in Wikimedia's CommonSettings.php and InitialiseSettings.php configuration files to see where it's installed. A full list of the extensions installed on a particular wiki can be seen on the wiki's Special:Version page.
+
This extension is included in the following wiki farms/hosts and/or packages:
+This is not an authoritative list. Some wiki farms/hosts and/or packages may contain this extension even if they are not listed here. Always check with your wiki farms/hosts or bundle to confirm.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Scribunto.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Scribunto.html
new file mode 100644
index 00000000..8d1b192a
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Scribunto.html
@@ -0,0 +1,1597 @@
+
+
+
+
+Extension:Scribunto - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : Scribunto
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
This extension
comes with MediaWiki 1.34 and above. Thus you do not have to download it again. However, you still need to follow the other instructions provided.
+
This extension runs on top of an executable. You must have permission to run executables on your host in order for this extension to work.
+
+
+
The Scribunto (Latin : "they shall write/let them write (in the future) ") extension allows for embedding scripting languages in MediaWiki.
+
Currently the only supported scripting language is Lua .
+Scribunto Lua scripts go in a namespace called Module .
+Modules are run on normal wiki pages using the #invoke parser function and each module has a collection of functions , which can be called using wikitext syntax such as:
+
+
{{#invoke: Module_name | function_name | arg1 | arg2 | arg3 ... }}
+
+
+
+
This extension contains code licensed GNU General Public License v2.0 or later (GPL-2.0+) as well as code licensed MIT License (MIT).
+
+
+
PCRE version compatibility [ edit ]
+
PCRE 8.33+ is recommended.
+PCRE 8.33 was released in May 2013.
+You can see the version of PCRE used by PHP by viewing a phpinfo() web page, or from the command line with the following command:
+
+
php -r 'echo "pcre: " . ( extension_loaded( "pcre" ) ? PCRE_VERSION : "no" ) . "\n";'
+
+
Scribunto will not work with versions of PCRE lower than 8.10.
+PCRE 8.32 has a bug that will cause it to reject certain non-character codepoints , which will cause errors in the mw.html module.
+
CentOS 6 and RHEL 6 are stuck on PCRE 7 and need to be upgraded.
+
Updating to 8.33 on a server with an older version may be relatively complicated.
+See Updating to PCRE 8.33 or Higher for details.
+
+
+
+
+
+MediaWiki versions:
+ 1.25 – 1.28
+
+
Scribunto versions for MediaWiki 1.25 to 1.28 required PHP's pcntl extension , which is only available on Unix/Linux platforms, if you want to use "LuaStandalone" (i.e. running in a separate child process).
+This requirement was removed in Scribunto for MediaWiki 1.29.
+
You can check whether pcntl support is enabled by viewing a phpinfo() web page, or from the command line with the following command:
+
+
php -r 'echo "pcntl: " . ( extension_loaded( "pcntl" ) ? "yes" : "no" ) . "\n";'
+
+
PHP mbstring extension [ edit ]
+
PHP needs to have the mbstring extension enabled.
+
You can check whether mbstring support is enabled by viewing a phpinfo() web page, or from the command line with the following command:
+
+
php -r 'echo "mbstring: " . ( extension_loaded( "mbstring" ) ? "yes" : "no" ) . "\n";'
+
+
+
+
Scribunto comes bundled with Lua binary distributions for Linux (x86 and x86-64), Mac OS X Lion, and Windows (32- and 64-bit).
+
Scribunto should work for you out of the box if:
+
+
Your web server is run on one of the above platforms.
+PHP's proc_open
function is not restricted.[ 1]
+proc_terminate
and shell_exec
are not disabled in PHP.
+Your web server is configured to allow the execution of binary files in the MediaWiki tree.
+
Note: Execute permissions may need to be set; for example, in Linux use:
+ chmod 755 /path/to/extensions/Scribunto/includes/Engines/LuaStandalone/binaries/lua5_1_5_linux_64_generic/lua
+ If you are using SELinux in "Enforcing" mode on your server, you might need to set a proper context for the binaries. Example for RHEL/CentOS 7: chcon -t httpd_sys_script_exec_t /path/to/extensions/Scribunto/includes/Engines/LuaStandalone/binaries/lua5_1_5_linux_64_generic/lua
+
+
P.S. Check your version of the extension to see if the name of the engines folder is capitalised or fully lowercase.[ 2]
+
+
Additional binaries [ edit ]
+
Additional Lua binary distributions, which may be needed for your web server if its operating system is not in the list above, can be obtained from http://luabinaries.sourceforge.net/ or from your Linux distribution.
+
Only binary files for Lua 5.1.x are supported.
+
Once you've installed the appropriate binary file on your web server, configure the location of the file with:
+
+
# Where Lua is the name of the binary file
+# e.g. SourceForge LuaBinaries 5.1.5 - Release 2 name the binary file lua5.1
+$wgScribuntoEngineConf [ 'luastandalone' ][ 'luaPath' ] = '/path/to/binaries/lua5.1' ;
+
+
Note that you should not add the above line unless you've confirmed that Scribunto's built-in binaries don't work for you.
+
LuaJIT, although theoretically compatible, is not supported.
+
The support was removed due to Spectre and bitrot concerns (phab:T184156 ).
+
+
+
Download and move the extracted Scribunto
folder to your extensions/
directory. Developers and code contributors should install the extension from Git instead, using:cd extensions/ git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Scribunto
+Add the following code at the bottom of your LocalSettings.php file: wfLoadExtension ( 'Scribunto' );
+$wgScribuntoDefaultEngine = 'luastandalone' ;
+
+Set execute permissions for the Lua binaries bundled with this extension:
+
chmod a+x /path/to/extensions/Scribunto/includes/Engines/LuaStandalone/binaries/yourOS/lua
+[ 2]
+
Set type to httpd_sys_script_exec_t
if SELinux is enforced:
+
chcon -t httpd_sys_script_exec_t /path/to/extensions/Scribunto/includes/Engines/LuaStandalone/binaries/yourOS/lua
+[ 2]
+
Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+
+Vagrant installation:
+
+
If using Vagrant , install with vagrant roles enable scribunto --provision
+
+
+
Optional installation [ edit ]
+
Integrating extensions [ edit ]
+
For a more pleasant user interface, with syntax highlighting and a code editor with autoindent, install the following extensions:
+
+
+
+
+MediaWiki version:
+≤ 1.30
+
+
Then in your LocalSettings.php
after all the extension registrations, add:
+
+
$wgScribuntoUseGeSHi = true ;
+$wgScribuntoUseCodeEditor = true ;
+
+
These settings are not necessary in MediaWiki versions after 1.30, where the code editor and syntax hightlighting should be used in the appropriate places automatically once the extensions are loaded. However, other settings may be desired. See the documentation for each extension.
+
+
+
We have developed a PHP extension written in C called LuaSandbox.
+It can be used as an alternative to the standalone binary, and will provide improved performance.
+See LuaSandbox for details and installation instructions.
+
If you initially installed the extension to use the Lua standalone binary, be sure to update LocalSettings.php
with the following configuration setting:
+
+
$wgScribuntoDefaultEngine = 'luasandbox' ;
+
+
+
The following configuration variables are available:
+
+
$wgScribuntoDefaultEngine
+Select the engine. Valid values are the keys in $wgScribuntoEngineConf
, which by default are 'luasandbox'
or 'luastandalone'
.
+$wgScribuntoUseGeSHi
+When Extension:SyntaxHighlight is installed, set this true to use it when displaying Module pages. (MediaWiki 1.30 or earlier.)
+$wgScribuntoUseCodeEditor
+When Extension:CodeEditor is installed, set this true
to use it when editing Module pages. (MediaWiki 1.30 or earlier.)
+$wgScribuntoEngineConf
+An associative array for engine configuration. Keys are the valid values for $wgScribuntoDefaultEngine
, and values are associative arrays of configuration data. Each configuration array must contain a 'class'
key naming the ScribuntoEngineBase
subclass to use.
+
+
The following keys are used in $wgScribuntoEngineConf
for Scribunto_LuaStandaloneEngine
.
+Generally you'd set these as something like
+
+
$wgScribuntoEngineConf [ 'luastandalone' ][ 'key' ] = 'value' ;
+
+
luaPath
+Specify the path to a Lua interpreter.
+errorFile
+Specify the path to a file, writable by the web server user, where the error and debugging output from the standalone interpreter will be logged.
+Error output produced by the standalone interpreter are not logged by default. Configure logging with:
+
$wgScribuntoEngineConf [ 'luastandalone' ][ 'errorFile' ] = '/path/to/file.log' ;
+
+
memoryLimit
+Specify the memory limit in bytes. Default 52428800 (50MB) (enforced using ulimit).
+cpuLimit
+Specify the CPU time limit in seconds (enforced using ulimit).
+allowEnvFuncs
+Set true to allow use of setfenv and getfenv in modules.
+
+
The following keys are used in $wgScribuntoEngineConf
for Scribunto_LuaSandboxEngine
.
+Generally you'd set these as something like
+
+
$wgScribuntoEngineConf [ 'luasandbox' ][ 'key' ] = 'value' ;
+
+
memoryLimit
+Specify the memory limit in bytes. Default 52428800 (50MB)
+cpuLimit
+Specify the CPU time limit in seconds.
+profilerPeriod
+Specify the time between polls in sections for the Lua profiler.
+allowEnvFuncs
+Set true to allow use of setfenv and getfenv in modules.
+
+
Scripts go in a new namespace called Module .
+Each module has a collection of functions, which can be called using wikitext syntax such as:
+
+
{{#invoke: Module_name | function_name | arg1 | arg2 | arg3 ... }}
+
+
+
+
Lua is a simple programming language intended to be accessible to beginners.
+For a quick crash-course on Lua, try Learn Lua in 15 Minutes .
+
The best comprehensive introduction to Lua is the book Programming in Lua .
+The first edition (for Lua 5.0) is available online and is mostly relevant to Lua 5.1, the version used by Scribunto:
+
+
+
The reference manual is also useful:
+
+
+
+
In Lua, the set of all global variables and functions is called an environment.
+
Each {{#invoke:}}
call runs in a separate environment.
+Variables defined in one {{#invoke:}}
will not be available from another.
+This restriction was necessary to maintain flexibility in the wikitext parser implementation.
+
+
+
+
Debug console usage example
+
See also: Extension:Scribunto/Debug console
+
When editing a Lua module a so-called "debug console" can be found underneath the edit form.
+In this debug console Lua code can be executed without having to save or even create the Lua module in question.
+
+
+
Troubleshooting using the clickable "Script error" link.
+
Note that red Script error messages are clickable and will provide more detailed information.
+
+
Lua error: Internal error: The interpreter exited with status 1. [ edit ]
+
When using the LuaStandalone engine (this is the default), errors along the lines of "Lua error: Internal error: The interpreter exited with status 1." may be generated if the standalone Lua interpreter cannot be executed or runs into various runtime errors.
+To obtain more information, assign a file path to $wgScribuntoEngineConf [ 'luastandalone' ][ 'errorFile' ]
.
+The interpreter's error output will be logged to the specified file, which should prove more helpful in tracking down the issue.
+The information in the debug log includes debugging information, which is why there is so much of it.
+You should be able to ignore any line beginning with "TX" or "RX".
+
If you're setting up Scribunto and are using IIS/Windows, this appears to be solved by commenting out a particular line .
+
+
Lua error: Internal error: The interpreter exited with status 2. [ edit ]
+
When using the LuaStandalone engine (this is the default), status 2 suggests memory allocation errors, probably caused by settings that allocate inadequate memory space for PHP or Lua, or both.
+Assigning a file path to $wgScribuntoEngineConf [ 'luastandalone' ][ 'errorFile' ]
and examining that output can be valuable in diagnosing memory allocation errors.
+
Increase PHP allocation in your PHP configuration; add the line memory_limit = 200M
.
+This allocation of 200MB is often sufficient (as of MediaWiki 1.24) but can be increased as required.
+Set Scribunto's memory allocation in LocalSettings.php
as a line:
+
+
$wgScribuntoEngineConf [ 'luastandalone' ][ 'memoryLimit' ] = 209715200 ; # bytes
+
+
Finally, depending on the server configuration, some installations may be helped by adding another LocalSettings.php
line
+
+
$wgMaxShellMemory = 204800 ; # in KB
+
+
Note that all 3 memory limits are given in different units.
+
+
Lua error: Internal error: 2. on ARM architecture [ edit ]
+
If you're using an ARM architecture processor like on a RaspberryPi you'll face the error Lua error: Internal error: The interpreter exited with status 2.
due to wrong delivered binary format of the Lua interpreter.
+
Check your Lua interpreter in:
+
+
/path/to/webdir/Scribunto/includes/Engines/LuaStandalone/binaries/lua5_1_5_linux_32_generic
+
+
Check the interpreter by using:
+
+
file lua
+
+
The result should look like :
+
+
lua: ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0
+
+
The installed default Lua interpreter shows:
+
+
lua: ELF 32-bit LSB pie executable, Intel 80386, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.2, for GNU/Linux 2.6.9,
+
+
look at the "Intel 80386" part what definitely is not correct.
+
Check in /usr/bin
what version of Lua is installed on your system. If you have lua5.1 installed, you can either copy the interpreter to your lua5_1_5_linux_32_generic
directory or set in your LocalSettings.php:
+
+
$wgScribuntoEngineConf [ 'luastandalone' ][ 'luaPath' ] = '/usr/bin/lua5.1' ;
+
+
At present don't set wgScribuntoEngineConf
to /usr/bin/lua5.3, it'll lead to the "Internal error 1".
+
+
Lua error: Internal error: The interpreter exited with status 24. [ edit ]
+
When using the LuaStandalone engine (this is the default), status 24 suggests CPU time limit errors, although those should be generating a "The time allocated for running scripts has expired" message instead.
+It would be useful to file a task in Phabricator and participate in determining why the XCPU signal isn't being caught.
+
+
Lua error: Internal error: The interpreter exited with status 126. [ edit ]
+
When using the LuaStandalone engine (this is the default), errors along the lines of "Lua error: Internal error: The interpreter exited with status 126." may be generated if the standalone Lua interpreter cannot be executed.
+This generally arises from either of two causes:
+
+
The Lua executable file's permissions do not include Execute. Set permissions as described under #Installation .
+The server does not allow execution of files from the place where the executable is installed, e.g. the filesystem is mounted with the 'noexec'
flag. This often occurs with shared hosted servers. Remedies include adjusting $wgScribuntoEngineConf [ 'luastandalone' ][ 'luaPath' ]
to point to a Lua 5.1 binary installed in an executable location, or adjusting or convincing the shared host to adjust the setting preventing execution.
+
Error condition such as: Fatal exception of type MWException [ edit ]
+
Check the MediaWiki, PHP, or webserver logs for more details on the exception, or temporarily set $wgShowExceptionDetails to true
.
+
+
version 'GLIBC_2.11' not found[ edit ]
+
If the above gives you errors such as "version 'GLIBC_2.11' not found", it means the version of the standard C library on your system is too old for the binaries provided with Scribunto.
+You should upgrade your C library, or use a version of Lua 5.1 compiled for the C library you do have installed.
+To upgrade your C library, your best option is usually to follow your distribution's instructions for upgrading packages (or for upgrading to a new release of the distribution, if applicable).
+
If you copy the lua binaries from Scribunto master (or from gerrit:77905 ), that should suffice, if you can't or don't want to upgrade your C library.
+The distributed binaries were recently recompiled against an older version of glibc, so the minimum is now 2.3 rather than 2.11.
+
+
Lua errors in Scribunto files [ edit ]
+
Errors here include:
+
+
attempt to index field 'text' (a nil value)
+Lua error in mw.html.lua at line 253: Invalid class given:
+
If you are getting errors such these when attempting to use modules imported from WMF wikis, most likely your version of Scribunto is out of date.
+
Upgrade if possible; for advanced users, you might also try to identify the needed newer commits and cherry-pick them into your local installation.
+
+
preg_replace_callback(): Compilation failed: unknown property name after \P or \p at offset 7[ edit ]
+
preg_replace_callback(): Compilation failed: unknown property name after \P or \p at offset 7
+
+
this usually indicates an incompatible version of PCRE; you'll need to update to >= 8.10
+@todo: link to instructions on how to upgrade
+
+
If you copy templates from Wikipedia and then get big red "Lua error: x" messages where the Scribunto invocation (e.g. the template that uses {{#invoke:}}
) should be, that probably means that you didn't import everything you needed. Make sure that you tick the "Include templates" box at w:Special:Export when you export.
+
When importing pages from another wiki, it is also possible for templates or modules in the imported data to overwrite existing templates or modules with the same title, which may break existing pages, templates, and modules that depend on the overwritten versions.
+
+
+
Make sure your extension version is applicable to your MediaWiki version.
+
+
+
+
+
+
+
General
+
Lua Wikibase client - functionality for the Scribunto extension.
+Commons:Lua - there may be specific notes for using Lua modules on Wikimedia Commons, including additional Lua extensions installed (e.g. for local support of internationalization and for parsing or playing medias). Some general purpose modules may be reused in other wikis in various languages (except specific tunings for policies, namespaces or project/maintenance pages with dedicated names). If possible, modules that could be widely reused across wikis should be tested and internationalized on Wikimedia Commons.
+w:Help:Lua - there may be specific notes for using Lua modules on Wikipedia, including additional Lua extensions installed (including for integrating Wikidata and Wikimedia Commons contents, generating complex infoboxes and navigation boxes, or to facilitate the general administration/maintenance of the wiki contents under applicable policies). Some other localized Wikipedia editions (or other projects such Wiktionnary, Wikisource or Wikinews) may also have their own needs and Lua modules.
+d:Help:Lua - there may be specific notes for using Lua modules on Wikidata, including additional Lua extensions installed (e.g. for local support of internationalization and for database queries)
+
Extensions
+
+
+
+
+
+
+↑
+i.e. Scribunto will not work if proc_open
is listed in the disable_functions
array in your server's "php.ini" file. If it is, you may see an error message like proc_open(): open_basedir restriction in effect. File(/dev/null) is not within the allowed path(s):
. If you are using Plesk and have been granted sufficient permissions, you may be able to set open_basedir
in the PHP settings for your domain or subdomain. Try changing {WEBSPACEROOT}{/}{:}{TMP}{/}
to {WEBSPACEROOT}{/}{:}{TMP}{/}{:}/dev/null{:}/bin/bash
.
+
+↑ 2.0 2.1 2.2 The name of the engines folder changed from lowercase to capitalised in 2022 .
+
+
+
This extension is being used on one or more Wikimedia projects . This probably means that the extension is stable and works well enough to be used by such high-traffic websites. Look for this extension's name in Wikimedia's CommonSettings.php and InitialiseSettings.php configuration files to see where it's installed. A full list of the extensions installed on a particular wiki can be seen on the wiki's Special:Version page.
+
This extension is included in the following wiki farms/hosts and/or packages:
+This is not an authoritative list. Some wiki farms/hosts and/or packages may contain this extension even if they are not listed here. Always check with your wiki farms/hosts or bundle to confirm.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_UniversalLanguageSelector.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_UniversalLanguageSelector.html
new file mode 100644
index 00000000..fe4fb26a
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_UniversalLanguageSelector.html
@@ -0,0 +1,1178 @@
+
+
+
+
+Extension:UniversalLanguageSelector - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : UniversalLanguageSelector
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The Universal Language Selector is a tool that allows users to select a language and easily configure its support. Where used, it also ships the functionality of both the former WebFonts and Narayam extensions (both of which have been deprecated in favor of Universal Language Selector).
+See Universal Language Selector for background and additional information.
+
+
+
Usage
+
The primary aim is to allow users to select a language and easily configure its support.
+
The extension provides the following features:
+
+
Flexible and easy way to select a language from a large set of languages.
+Selection can be based on geographical region-based browsing, and searching.
+Search can be based on ISO language code, language name written in current user interface language or in its own script(autonym)
+Cross-language search - search language names using any script
+Autocompletion of language names
+Auto correction of spelling mistakes in search strings
+Geo IP based language suggestions
+Language selection based on users browser/OS language
+Input methods
+See Help:Extension:UniversalLanguageSelector/Input methods for complete instructions.
+An easily selectable input method collection, readily available in every editable field
+Provides a large set of input methods for a wide range of languages
+
+Per-language input method preferences
+Webfonts
+A large collection of fonts to choose for each language to use as an embedded font for the page
+
+Per-language font preferences
+
Universal Language Selector with geoip based language suggestion for a user from India
+
Language settings allow a registered user accessing English Wikipedia to change the UI to their native language.
+
A Bengali user is able to read content from the Bengali Wikipedia despite the lack of fonts in their computer. Web fonts are provided automatically for non-Latin scripts for which an open-source font is available. The user can decide to use their system fonts on a per-language basis.
+
A Hindi speaker without a Hindi keyboard configures input methods so that they can type in their language.
+
When searching a user can switch between English and Hindi.
+
Adding fonts
+
Supporting more languages is only a matter of including the proper fonts in the code. However, please note that we will only add support for open-source licensed fonts, such as those licensed under GNU GPL, SIL OFL, etc. An example directory of such open-source fonts is Google Fonts [1] (not yet fully examined/exploited by the authors of this extension); see also the Open Font Library .
+
First of all, you need to find or produce such a open-source font (this is the most essential part, and you have to do it yourself); then, it has to be converted to the woff2 format, you can file a request in Phabricator for the font to be added to the extension.
+
#Preparing webfonts below explains how to convert the fonts: basic knowledge about GNU/Linux-based operating system is required; if you have difficulty in doing this, you can skip this step and ask someone else to do it for you on the same Phabricator request (of course this will slow down the process).
+
+
Preparing webfonts
+
Creating .woff2:
+
Use https://github.com/google/woff2 to generate woff2 from ttf.
+This will produce a compressed woff2 file. Modern browsers support this format.
+
Create a font.ini file. Here's an example:
+
+
[AbyssinicaSIL]
+languages = am*, ti*
+version = 1.200
+license = OFL 1.1
+licensefile = OFL.txt
+url = http://scripts.sil.org/AbyssinicaSIL
+request-url = https://phabricator.wikimedia.org/[Task Number]
+woff2 = AbyssinicaSIL.woff2
+bold = AbyssinicaSIL Bold
+
+[AbyssinicaSIL Bold]
+woff2 = AbyssinicaSIL-Bold.woff2
+fontweight = bold
+
+
An asterisk (*
) after a language code means that this font will be the default font for that language. Don't use the asterisk if you want the option to use this font for that language.
+
After creating the files, do the following:
+
+
Create a directory for the font under data/fontrepo/fonts
.
+Put the woff2 and font.ini files in that directory and add them to the source repository (git add
).
+Go to the scripts/
directory and run php compile-font-repo.php
.
+Commit the changes to the repository (git commit -a
) and submit them according to the Git workflow .
+
+
Follow the instructions on the jquery.ime github wiki , but file requests in the Wikimedia-extensions-UniversalLanguageSelector Phabricator product.
+
+
Installation
+
Download and move the extracted UniversalLanguageSelector
folder to your extensions/
directory. Developers and code contributors should install the extension from Git instead, using:cd extensions/ git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/UniversalLanguageSelector
+Add the following code at the bottom of your LocalSettings.php file: wfLoadExtension ( 'UniversalLanguageSelector' );
+
+ Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+
+Vagrant installation:
+
+
If using Vagrant , install with vagrant roles enable uls --provision
+
Updating LanguageNameIndex
+
For performing cross language search, searching autonyms , language data needs to be populated.
+ULS comes with a pre-populated language name index(data/langnames.ser).
+In case you want to update it, install Extension:CLDR and update the data with the following command.
+
+
php UniversalLanguageSelector/data/LanguageNameIndexer.php
+
and verify that langnames.ser file gets generated in ULS/data/ folder.
+
+
Configuration
+
The following variables are created automatically during initialization and can be used from JavaScript using mw.config.get( NAME ):
+
+
wgULSLanguages
- an associative array where the keys are language codes and the values are language names in English.
+wgULSAcceptLanguageList
- an array of language codes from the user's Accept-Language value. These are the languages selected in the user's browser preferences.
+
For serving fonts, you might want to add the following MIME types to your webserver if not already there. This guide might help.
+
+
AddType font/woff2 .woff2
+
+
The following variables can also be configured:
+
+
$wgULSGeoService
- ULS can use geolocation services to suggest languages based on the country the user is visiting from. Setting this to false will prevent built-in geolocation from being used. You can provide your own geolocation by setting window. Geo to object which has key 'country_code' or 'country'. If set to true, it will query Wikimedia's geoip service. The service should return jsonp that uses the supplied callback parameter. Defaults to http://freegeoip.net/json/ (warning : this website has shut down its API) and expects the same format.
+$wgULSEnable
- Enable language selection, compact language links, input methods, and web fonts for everyone unless the behavior is overridden by the configuration variables below. Even if false, the classes and resource loader modules are registered for using other extensions. Language changing via cookie or setlang query parameter is not possible.
+$wgULSAnonCanChangeLanguage
- Allow anonymous users to change the language with cookie and setlang query param. Do not use it if you are caching anonymous page views without taking cookies into account. It does not have any effect if either $wgULSEnable
or $wgULSEnableAnon
is set to false
.
+$wgULSIMEEnabled
- Disable the input methods feature for all users by default. The user can still enable it manually.
+$wgULSPosition
- The location and the form of the language selection trigger. The possible values are: personal
: as a link near the username or the login link in the personal toolbar (default). interlanguage
: as an icon near the header of the list of interlanguage links in the sidebar.
+$wgULSNoImeSelectors
- Array of jQuery selectors of elements on which IME must not be enabled. eg: [ '#wpCaptchaWord' ];
+$wgULSLanguageDetection
- Whether to automatically detect the user's language from the Accept-Language header.
+
Position of ULS trigger
+
$wgULSPosition
- The location and the form of the language selection trigger. The possible values are: personal
: as a link near the username or the login link in the personal toolbar (default). interlanguage
: as an icon near the header of the list of interlanguage links in the sidebar.
+
It is also possible to have a ULS trigger anywhere on the screen. An element with uls-settings-trigger
will act as a ULS trigger.
+
+
Overriding default fonts
+
ULS has a large font repository to serve as webfonts.
+Sometimes, there are multiple fonts for a language, and there is a default font for each language/script.
+The order of fonts or default font can be overridden as follows using global scripts (MediaWiki:Common.js) or personal scripts (Special:MyPage/common.js):
+
+
$ . webfonts . repository . languages . languageCode = [ "system" , "FontA" , "FontB" ];
+
+
Here, languageCode should be a valid langauge code(eg: en, hi, nl). FontA and FontB are fonts available in font repository. In the above example for languageCode, we set a font available in local computer as default font. ie No default webfont.
+
+
Caching configuration
+
To ensure that the web fonts files are cached on the clients' machines, font file types must be added to the web server configuration. In Apache2 this consists of:
+
+
Adding font file extensions to the FileTimes regex at FilesMatch for the relevant directory, example:
+
<FilesMatch "\.(gif|jpe?g|png|css|js|woff2|svg)$" >
+
+
Adding ExpiresByType values to the relevant MIME types, similarly to image MIME types.
+Note that there's no standard MIME type for TTF. application/x-font-ttf is used for Wikimedia.
+Adding the MIME types:
+
AddType font/woff2 .woff2
+
+
For a full example see the caching configuration update done for the Wikimedia cluster .
+
+
Page translation
+
UniversalLanguageSelector is one of the dependencies of the Translate extension, which uses it for several language selection features. One of it is the MyLanguage system for links, which depends on the interface language of the user, but more can be configured: see Page translation feature .
+
+
Using Webfonts
+
Users can choose web fonts for a language from the Language settings -> Display settings. The first font in that menu will be applied to the wiki by default. A user can change the font to be remembered across the pages. Optionally, the user can disable the font embedding by selecting the system font.
+
If the font is available in the user's local system, the font will not be downloaded from the MediaWiki server. It will be taken from the user's computer. Otherwise, the font will be downloaded from the server only once, when the user selects the font the first time. From then on, the font will be taken from the local cache.
+
+
Alternate ways to load fonts
+
By specifying font-family
+
Inside the wiki text <span style="font-family:'YourFontName';">YourText</span>
, webfonts extension will check whether the font is available with the extension, if so it will download it to the client.
+So the reader will not face any difficulty reading the text even if the specified font is not available on their computer.
+
+
By specifying language
+
Inside the wiki text <span lang="my">YourText</span>
, the web fonts extension will check whether any font is available for the given language with the extension and if so, it will download it to the client.
+So the reader will not face any difficulty reading the text even if the specified font is not available on their computer.
+The default font will be used if there are multiple fonts for the language.
+If the default font is not preferred, use the font-family approach to specify the font.
+If the tag has both lang and font-family definitions, font-family gets precedence.
+
Example:
+
<span lang=sux>𒄖𒉈𒅁𒌨𒅎</span>
+
gives the text rendered in Cuneiform using Akkadian font
+
𒄖𒉈𒅁𒌨𒅎
+
+
See also
+
+
This extension is being used on one or more Wikimedia projects . This probably means that the extension is stable and works well enough to be used by such high-traffic websites. Look for this extension's name in Wikimedia's CommonSettings.php and InitialiseSettings.php configuration files to see where it's installed. A full list of the extensions installed on a particular wiki can be seen on the wiki's Special:Version page.
+
This extension is included in the following wiki farms/hosts and/or packages:
+This is not an authoritative list. Some wiki farms/hosts and/or packages may contain this extension even if they are not listed here. Always check with your wiki farms/hosts or bundle to confirm.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseClient.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseClient.html
new file mode 100644
index 00000000..44835eda
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseClient.html
@@ -0,0 +1,836 @@
+
+
+
+
+Extension:Wikibase Client - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : Wikibase Client
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Wikibase Client is part of Wikibase .
+Wikibase Client is an extension that acts as a client to the Wikibase Repository extension.
+Its development is part of the Wikidata project.
+
It allows to use and display data from a Wikibase Repository via Lua modules (mw.wikibase
) or parser functions (#statements
, #property
).
+Clients can also use centralized language links or article placeholders .
+
+
+
+
See basic installation instructions .
+For experienced configuration options see Advanced Configuration .
+
A complete documentation exists in the docs folder for Wikibase, and is published to the Wikimedia documentation site .
+
+
+
For a list of features and how to use them see Advanced configuration .
+
+
+
The documentation of the available PHP and JavaScript hooks can be found in these pages:
+
+
+
For more information see Wikibase/Developing extensions .
+
+
+
+
This extension is being used on one or more Wikimedia projects . This probably means that the extension is stable and works well enough to be used by such high-traffic websites. Look for this extension's name in Wikimedia's CommonSettings.php and InitialiseSettings.php configuration files to see where it's installed. A full list of the extensions installed on a particular wiki can be seen on the wiki's Special:Version page.
+
This extension is included in the following wiki farms/hosts and/or packages:
+This is not an authoritative list. Some wiki farms/hosts and/or packages may contain this extension even if they are not listed here. Always check with your wiki farms/hosts or bundle to confirm.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib.html
new file mode 100644
index 00000000..df6d68de
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib.html
@@ -0,0 +1,602 @@
+
+
+
+
+Extension:WikibaseLib - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+ Extension : WikibaseLib
+
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
This extension has been archived.
+This extension has not been maintained for some time, and no longer supports recent releases of MediaWiki.
+
+The following other choices are available:
+
+
+To see the page before archival, click here.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib_Archived.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib_Archived.html
new file mode 100644
index 00000000..13cd4fdc
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib_Archived.html
@@ -0,0 +1,911 @@
+
+
+
+
+Extension:WikibaseLib - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : WikibaseLib
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
WikibaseLib provides common Wikibase functionality for Wikibase Repository and Wikibase Client .
+
+
+
Feature overview
+
TODO
+
+
Requirements
+
WikibaseLib requires:
+
+
+
Some components might be dependent on one of the following:
+
+
+
Download
+
WikibaseLib is in the lib directory of the Wikibase git repo on gerrit.
+
The extension can be retrieved directly from Git [? ] :
+
+
Browse code
+Some extensions have tags for stable releases.
+
+Each branch is associated with a past MediaWiki release. There is also a "master" branch containing the latest alpha version (might require an alpha version of MediaWiki).
+
+
Extract the snapshot and place it in the extensions/Wikibase/ directory of your MediaWiki installation.
+
If you are familiar with Git and have shell access to your server, you can also obtain the extension as follows:
+
+
+
cd extensions/
+git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Wikibase.git
+
+
+
Installation
+
Once you have downloaded the code, place the WikibaseLib directory within your MediaWiki 'extensions' directory.
+Then add the following code to your LocalSettings.php file:
+
+
# WikibaseLib
+require_once ( " $IP /extensions/Wikibase/lib/WikibaseLib.php" );
+
+# DataValues
+require_once ( " $IP /extensions/DataValues/DataValues.php" );
+
+# DataTypes
+require_once ( " $IP /extensions/DataTypes/DataTypes.php" );
+
+
Version
+
This is a copy of the release notes file on Git , which might be more up to date than this page.
+
+
Internationalization
+
WikibaseLib is fully internationalized. Translation of WikibaseLib messages is done through translatewiki.net . The translation for this extension can be found here . To add language values or change existing ones, you should create an account on translatewiki.net, then request permission from the administrators to translate a certain language or languages on this page (this is a very simple process). Once you have permission for a given language, you can log in and add or edit whatever messages you want to in that language.
+
+
Screenshots
+
+
See also
+
+
External links
+
+
This extension is being used on one or more Wikimedia projects . This probably means that the extension is stable and works well enough to be used by such high-traffic websites. Look for this extension's name in Wikimedia's CommonSettings.php and InitialiseSettings.php configuration files to see where it's installed. A full list of the extensions installed on a particular wiki can be seen on the wiki's Special:Version page.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseRepository.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseRepository.html
new file mode 100644
index 00000000..3b7afbbc
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseRepository.html
@@ -0,0 +1,1012 @@
+
+
+
+
+Extension:Wikibase Repository - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Toggle the table of contents
+
+
+
+
+
+ Extension : Wikibase Repository
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Wikibase Repository is part of Wikibase .
+Wikibase Repository allows you to use your wiki as a structured data repository.
+Its development is part of the Wikidata project.
+The data can in turn be used in a wiki using the Wikibase Client extension.
+
+
+
+
See the basic installation instructions .
+For experienced configuration options see Advanced Configuration .
+
A complete documentation exists in the docs folder for Wikibase, and is published to the Wikimedia documentation site .
+
+
Integration with other extensions [ edit ]
+
Wikibase makes use of the following if they are installed:
+
+
+
If installed and JavaScript is supported by the user client / browser, labels, aliases and descriptions will be shown to the user in frequently used languages in addition to the user interface language. (These languages are acquired from mw.uls.getFrequentLanguageList()
.)
+
+
If installed, logged-in users are able to define additional languages that labels, aliases and descriptions are shown in by specifying languages on their user pages using the Babel syntax.
+
+
The documentation of the available PHP and JavaScript hooks can be found in here:
+
+
+
For more information see Wikibase/Developing extensions .
+
+
+
Term store wbit_id BIGINT wbit_item_id INT wbit_term_in_lang_id INT wbpt_id INT wbpt_property_id INT wbpt_term_in_lang_id INT wbtl_id INT wbtl_type_id INT wbtl_text_in_lang_id INT wbxl_id INT wbxl_language VARBINARY(20) wbxl_text_id INT wbx_id INT wbx_text VARBINARY(255) wby_id INT wby_name VARBINARY(45)
Misc id_value INT id_type VARBINARY(32) ips_row_id BIGINT ips_item_id INT ips_site_id VARBINARY(32) ips_site_page VARCHAR(310) pi_property_id INT pi_type VARBINARY(32) pi_info BLOB
+
See the automated documentation
+
+
+
+
This extension is being used on one or more Wikimedia projects . This probably means that the extension is stable and works well enough to be used by such high-traffic websites. Look for this extension's name in Wikimedia's CommonSettings.php and InitialiseSettings.php configuration files to see where it's installed. A full list of the extensions installed on a particular wiki can be seen on the wiki's Special:Version page.
+
This extension is included in the following wiki farms/hosts and/or packages:
+This is not an authoritative list. Some wiki farms/hosts and/or packages may contain this extension even if they are not listed here. Always check with your wiki farms/hosts or bundle to confirm.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseView.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseView.html
new file mode 100644
index 00000000..73c8c840
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseView.html
@@ -0,0 +1,549 @@
+
+
+
+
+Extension:WikibaseView - MediaWiki
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Jump to content
+
+
+
+
+
+
+
+
+ Extension : WikibaseView
+
+
+
+
+
+
+
+
+
+
+
+
+
+
English
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Tools
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
There is currently no text in this page.
+You can search for this page title in other pages,
+search the related logs ,
+or create this page .
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Special_Version.html b/tests/test_create_observation/test_create_software_version_observation/data/Special_Version.html
index 49856a80..f42c07b2 100644
--- a/tests/test_create_observation/test_create_software_version_observation/data/Special_Version.html
+++ b/tests/test_create_observation/test_create_software_version_observation/data/Special_Version.html
@@ -63,7 +63,7 @@ I
Lua
5.1.5
-Installed skins Skin Version License Description Authors MonoBook – GPL-2.0-or-later The classic MediaWiki skin since 2004, named after the black-and-white photo of a book in the page background Gabriel Wicke, Isarra Yos and others Timeless 0.8.9 GPL-2.0-or-later A timeless skin designed after the Winter prototype by Brandon Harris, and various styles by the Wikimedia Foundation Isarra Yos Vector – GPL-2.0-or-later Modern version of MonoBook with fresh look and many usability improvements Trevor Parscal, Roan Kattouw and others
Installed extensions Parser hooks Extension Version License Description Authors Babel 1.11.1 MLEB 2020.01 GPL-2.0-or-later Adds the #babel
parser function to allow automated generation of a babel userbox column with the ability to include custom templates Robert Leverington, Thiemo Kreuz, Legoktm, Nikerabbit, Amir Aharoni and Ricordisamoa LabeledSectionTransclusion – (f621799) 14:52, 29 January 2020 GPL-2.0-or-later Adds #lst
and #lstx
functions and <section>
tag, enables marked sections of text to be transcluded Steve Sanbeg Scribunto – GPL-2.0-or-later AND MIT Framework for embedding scripting languages into MediaWiki pages Victor Vasiliev, Tim Starling and Brad Jorsch Wikibase Extension Version License Description Authors WikibaseClient – (dbbcdd8) 12:52, 10 December 2019 GPL-2.0-or-later Client for the Wikibase extension The Wikidata team and others WikibaseLib – (dbbcdd8) 12:52, 10 December 2019 GPL-2.0-or-later Holds common functionality for the Wikibase and Wikibase Client extensions The Wikidata team and others WikibaseRepository – (dbbcdd8) 12:52, 10 December 2019 GPL-2.0-or-later Structured data repository The Wikidata team and others WikibaseView – (dbbcdd8) 12:52, 10 December 2019 GPL-2.0-or-later View component for the Wikibase Repository The Wikidata team and others Other Extension Version License Description Authors Google Analytics Integration 3.0.1 (6441403) 09:12, 6 August 2019 GPL-2.0-or-later Inserts Google Universal Analytics (and/or other web analytics) scripts into MediaWiki pages for tracking Tim Laqua and Davis Mosenkovs ProofreadPage – (cb0a218) 09:20, 30 September 2019 GPL-2.0-or-later Allow easy comparison of text to the original scan ThomasV and Thomas Pellissier Tanon UniversalLanguageSelector 2020-01-23 MLEB 2020.01 (61f1a98) 13:38, 3 March 2020 GPL-2.0-or-later Gives the user several ways to select a language and to adjust language settings Alolita Sharma, Amir Aharoni, Arun Ganesh, Brandon Harris, Kartik Mistry, Niharika Kohli, Niklas Laxström, Pau Giner, Santhosh Thottingal, Siebrand Mazeland and others
Installed libraries Library Version License Description Authors composer/installers 1.8.0 MIT A multi-framework Composer library installer Kyle Robinson Young composer/semver 1.5.0 MIT Semver library that offers utilities, version constraint parsing and validation. Nils Adermann , Jordi Boggiano and Rob Bast cssjanus/cssjanus 1.3.0 Apache-2.0 Convert CSS stylesheets between left-to-right and right-to-left. Trevor Parscal, Roan Kattouw and Timo Tijhof data-values/common 0.4.3 GPL-2.0+ Contains common implementations of the interfaces defined by DataValuesInterfaces Jeroen De Dauw data-values/data-values 2.3.0 GPL-2.0-or-later Defines the DataValue interface and some trivial implementations Jeroen De Dauw data-values/geo 3.0.1 GPL-2.0-or-later Geographical value objects, parsers and formatters Jeroen De Dauw and The Wikidata teamdata-values/interfaces 0.2.5 GPL-2.0+ Defines interfaces for ValueParsers, ValueFormatters and ValueValidators Jeroen De Dauw data-values/number 0.10.1 GPL-2.0-or-later Numerical value objects, parsers and formatters Daniel Kinzler and Thiemo Kreuz data-values/serialization 1.2.3 GPL-2.0-or-later Serializers and deserializers for DataValue implementations Jeroen De Dauw data-values/time 1.0.1 GPL-2.0-or-later Time value objects, parsers and formatters The Wikidata team diff/diff 2.3.0 GPL-2.0+ Small standalone library for representing differences between data structures, computing such differences, and applying them as patches Jeroen De Dauw guzzlehttp/guzzle 6.3.3 MIT Guzzle is a PHP HTTP client library Michael Dowling guzzlehttp/promises 1.3.1 MIT Guzzle promises library Michael Dowling guzzlehttp/psr7 1.6.1 MIT PSR-7 message implementation that also provides common utility methods Michael Dowling and Tobias Schultze liuggio/statsd-php-client 1.0.18 MIT Statsd (Object Oriented) client library for PHP Giulio De Donato onoi/message-reporter 1.4.1 GPL-2.0-or-later An interface to report and relay arbitrary messages to registered handlers Jeroen De Dauw and James Hong Kong oojs/oojs-ui 0.34.1 MIT Provides library of common widgets, layouts, and windows. Bartosz Dziewoński, Ed Sanders, James D. Forrester, Kirsten Menger-Anderson, Kunal Mehta, Prateek Saxena, Roan Kattouw, Rob Moen, Timo Tijhof and Trevor Parscal pear/console_getopt 1.4.3 BSD-2-Clause More info available on: http://pear.php.net/package/Console_Getopt Andrei Zmievski, Stig Bakken and Greg Beaver pear/mail 1.4.1 BSD-2-Clause Class that provides multiple interfaces for sending emails. Chuck Hagenbuch, Richard Heyes and Aleksander Machniak pear/mail_mime 1.10.2 BSD-3-clause Mail_Mime provides classes to create MIME messages Cipriano Groenendal and Aleksander Machniak pear/net_smtp 1.8.1 BSD-2-Clause An implementation of the SMTP protocol Chuck Hagenbuch and Jon Parise pear/net_socket 1.2.2 PHP License More info available on: http://pear.php.net/package/Net_Socket Chuck Hagenbuch, Aleksander Machniak and Stig Bakken pear/pear-core-minimal 1.10.10 BSD-3-Clause Minimal set of PEAR core files to be used as composer dependency Christian Weiske pear/pear_exception 1.0.1 BSD-2-Clause The PEAR Exception base class. Helgi Thormar and Greg Beaver pleonasm/bloom-filter 1.0.2 BSD-2-Clause A pure PHP implementation of a Bloom Filter Matthew Nagi psr/container 1.0.0 MIT Common Container Interface (PHP FIG PSR-11) PHP-FIG psr/http-message 1.0.1 MIT Common interface for HTTP messages PHP-FIG psr/log 1.0.2 MIT Common interface for logging libraries PHP-FIG psr/simple-cache 1.0.1 MIT Common interfaces for simple caching PHP-FIG ralouphie/getallheaders 3.0.3 MIT A polyfill for getallheaders. Ralph Khattar serialization/serialization 4.0.0 GPL-2.0+ Library defining a Serializer and a Deserializer interface and basic utilities Jeroen De Dauw wikibase/data-model 9.2.0 GPL-2.0-or-later PHP implementation of the Wikibase DataModel Jeroen De Dauw and Thiemo Kreuzwikibase/data-model-serialization 2.9.1 GPL-2.0-or-later Serializers and deserializers for the Wikibase DataModel Thomas PT and Jeroen De Dauw wikibase/data-model-services 3.15.0 GPL-2.0-or-later Services around the Wikibase DataModel The Wikidata team and Jeroen De Dauw wikibase/internal-serialization 2.10.0 GPL-2.0-or-later Serializers and deserializers for the data access layer of Wikibase Repository Jeroen De Dauw wikibase/term-store 1.0.4 BSD-3-Clause Tiny Wikibase library that defines interfaces for persisting terms of Items and Properties The Wikidata team wikimedia/assert 0.2.2 MIT Provides runtime assertions Daniel Kinzler wikimedia/at-ease 2.0.0 GPL-2.0-or-later Safe replacement to @ for suppressing warnings. Tim Starling and MediaWiki developers wikimedia/base-convert 2.0.0 GPL-2.0-or-later Convert an arbitrarily-long string from one numeric base to another, optionally zero-padding to a minimum column width. Brion Vibber and Tyler Romeo wikimedia/cdb 1.4.1 GPL-2.0+ Constant Database (CDB) wrapper library for PHP. Provides pure-PHP fallback when dba_* functions are absent. Daniel Kinzler, Tim Starling, Chad Horohoe and Ori Livneh wikimedia/cldr-plural-rule-parser 1.0.0 GPL-2.0+ Evaluates plural rules specified in the CLDR project notation. Tim Starling and Niklas Laxström wikimedia/composer-merge-plugin 1.4.1 MIT Composer plugin to merge multiple composer.json files Bryan Davis wikimedia/html-formatter 1.0.2 GPL-2.0-or-later Performs transformations of HTML by wrapping around libxml2 and working around its countless bugs. MediaWiki contributors wikimedia/ip-set 2.1.0 GPL-2.0-or-later Efficiently match IP addresses against a set of CIDR specifications. Brandon Black wikimedia/less.php 1.8.0 Apache-2.0 PHP port of the Javascript version of LESS http://lesscss.org (Originally maintained by Josh Schmidt) Matt Agar , Martin Jantošovič and Josh Schmidt wikimedia/object-factory 2.1.0 GPL-2.0-or-later Construct objects from configuration instructions Bryan Davis wikimedia/password-blacklist 0.1.4 MIT PasswordBlacklist for the 100,000 most used passwords Sam Reed wikimedia/php-session-serializer 1.0.7 GPL-2.0-or-later Provides methods like PHP's session_encode and session_decode that don't mess with $_SESSION Brad Jorsch wikimedia/purtle 1.0.7 GPL-2.0-or-later Fast streaming RDF serializer Daniel Kinzler, Stanislav Malyshev, C. Scott Ananian and Thiemo Kreuz wikimedia/relpath 2.1.1 MIT Compute a relative filepath between two paths. Ori Livneh wikimedia/remex-html 2.1.0 MIT Fast HTML 5 parser Tim Starling wikimedia/running-stat 1.2.1 GPL-2.0+ PHP implementations of online statistical algorithms Ori Livneh wikimedia/scoped-callback 3.0.0 GPL-2.0-or-later Class for asserting that a callback happens when a dummy object leaves scope Aaron Schulz wikimedia/timestamp 3.0.0 GPL-2.0-or-later Creation, parsing, and conversion of timestamps Tyler Romeo wikimedia/utfnormal 2.0.0 GPL-2.0-or-later Contains Unicode normalization routines, including both pure PHP implementations and automatic use of the 'intl' PHP extension when present Brion Vibber wikimedia/wait-condition-loop 1.0.1 GPL-2.0+ Wait loop that reaches a condition or times out Aaron Schulz wikimedia/wrappedstring 3.0.1 MIT Automatically compact sequentially-outputted strings that share a common prefix / suffix pair. Timo Tijhof wikimedia/xmp-reader 0.6.3 GPL-2.0-or-later Reader for XMP data containing properties relevant to images Brian Wolff zordius/lightncandy 0.23 MIT An extremely fast PHP implementation of handlebars ( http://handlebarsjs.com/ ) and mustache ( http://mustache.github.io/ ). Zordius Chen
<gallery> , <indicator> , <nowiki> , <pagelist> , <pagequality> , <pages> , <pre> and <section> anchorencode, babel, basepagename, basepagenamee, bidi, canonicalurl, canonicalurle, cascadingsources, defaultsort, displaytitle, filepath, formatdate, formatnum, fullpagename, fullpagenamee, fullurl, fullurle, gender, grammar, int, invoke, language, lc, lcfirst, localurl, localurle, lst, lsth, lstx, namespace, namespacee, namespacenumber, noexternallanglinks, ns, nse, numberingroup, numberofactiveusers, numberofadmins, numberofarticles, numberofedits, numberoffiles, numberofpages, numberofusers, padleft, padright, pageid, pagename, pagenamee, pagesincategory, pagesize, plural, property, protectionexpiry, protectionlevel, revisionday, revisionday2, revisionid, revisionmonth, revisionmonth1, revisiontimestamp, revisionuser, revisionyear, rootpagename, rootpagenamee, special, speciale, statements, subjectpagename, subjectpagenamee, subjectspace, subjectspacee, subpagename, subpagenamee, tag, talkpagename, talkpagenamee, talkspace, talkspacee, uc, ucfirst and urlencode
+Installed skins Skin Version License Description Authors MonoBook – GPL-2.0-or-later The classic MediaWiki skin since 2004, named after the black-and-white photo of a book in the page background Gabriel Wicke, Isarra Yos and others Timeless 0.8.9 GPL-2.0-or-later A timeless skin designed after the Winter prototype by Brandon Harris, and various styles by the Wikimedia Foundation Isarra Yos Vector – GPL-2.0-or-later Modern version of MonoBook with fresh look and many usability improvements Trevor Parscal, Roan Kattouw and others
Installed extensions Parser hooks Extension Version License Description Authors Babel 1.11.1 MLEB 2020.01 GPL-2.0-or-later Adds the #babel
parser function to allow automated generation of a babel userbox column with the ability to include custom templates Robert Leverington, Thiemo Kreuz, Legoktm, Nikerabbit, Amir Aharoni and Ricordisamoa LabeledSectionTransclusion – (f621799) 14:52, 29 January 2020 GPL-2.0-or-later Adds #lst
and #lstx
functions and <section>
tag, enables marked sections of text to be transcluded Steve Sanbeg Scribunto – GPL-2.0-or-later AND MIT Framework for embedding scripting languages into MediaWiki pages Victor Vasiliev, Tim Starling and Brad Jorsch Wikibase Extension Version License Description Authors WikibaseClient – (dbbcdd8) 12:52, 10 December 2019 GPL-2.0-or-later Client for the Wikibase extension The Wikidata team and others WikibaseLib – (dbbcdd8) 12:52, 10 December 2019 GPL-2.0-or-later Holds common functionality for the Wikibase and Wikibase Client extensions The Wikidata team and others WikibaseRepository – (dbbcdd8) 12:52, 10 December 2019 GPL-2.0-or-later Structured data repository The Wikidata team and others WikibaseView – (dbbcdd8) 12:52, 10 December 2019 GPL-2.0-or-later View component for the Wikibase Repository The Wikidata team and others Other Extension Version License Description Authors Google Analytics Integration 3.0.1 (6441403) 09:12, 6 August 2019 GPL-2.0-or-later Inserts Google Universal Analytics (and/or other web analytics) scripts into MediaWiki pages for tracking Tim Laqua and Davis Mosenkovs ProofreadPage – (cb0a218) 09:20, 30 September 2019 GPL-2.0-or-later Allow easy comparison of text to the original scan ThomasV and Thomas Pellissier Tanon UniversalLanguageSelector 2020-01-23 MLEB 2020.01 (61f1a98) 13:38, 3 March 2020 GPL-2.0-or-later Gives the user several ways to select a language and to adjust language settings Alolita Sharma, Amir Aharoni, Arun Ganesh, Brandon Harris, Kartik Mistry, Niharika Kohli, Niklas Laxström, Pau Giner, Santhosh Thottingal, Siebrand Mazeland and others ⧼mirahezemagic-extensionname⧽ – (e742444) 15:21 17 oct 2024 GPL-3.0-or-later Mensajes de MediaWiki, scripts y ganchos específicos para Miraheze Agent Isai, John Lewis, Labster, MacFan4000, Reception123, Revi, Paladox, Southparkfan y Universal Omega
Installed libraries Library Version License Description Authors composer/installers 1.8.0 MIT A multi-framework Composer library installer Kyle Robinson Young composer/semver 1.5.0 MIT Semver library that offers utilities, version constraint parsing and validation. Nils Adermann , Jordi Boggiano and Rob Bast cssjanus/cssjanus 1.3.0 Apache-2.0 Convert CSS stylesheets between left-to-right and right-to-left. Trevor Parscal, Roan Kattouw and Timo Tijhof data-values/common 0.4.3 GPL-2.0+ Contains common implementations of the interfaces defined by DataValuesInterfaces Jeroen De Dauw data-values/data-values 2.3.0 GPL-2.0-or-later Defines the DataValue interface and some trivial implementations Jeroen De Dauw data-values/geo 3.0.1 GPL-2.0-or-later Geographical value objects, parsers and formatters Jeroen De Dauw and The Wikidata teamdata-values/interfaces 0.2.5 GPL-2.0+ Defines interfaces for ValueParsers, ValueFormatters and ValueValidators Jeroen De Dauw data-values/number 0.10.1 GPL-2.0-or-later Numerical value objects, parsers and formatters Daniel Kinzler and Thiemo Kreuz data-values/serialization 1.2.3 GPL-2.0-or-later Serializers and deserializers for DataValue implementations Jeroen De Dauw data-values/time 1.0.1 GPL-2.0-or-later Time value objects, parsers and formatters The Wikidata team diff/diff 2.3.0 GPL-2.0+ Small standalone library for representing differences between data structures, computing such differences, and applying them as patches Jeroen De Dauw guzzlehttp/guzzle 6.3.3 MIT Guzzle is a PHP HTTP client library Michael Dowling guzzlehttp/promises 1.3.1 MIT Guzzle promises library Michael Dowling guzzlehttp/psr7 1.6.1 MIT PSR-7 message implementation that also provides common utility methods Michael Dowling and Tobias Schultze liuggio/statsd-php-client 1.0.18 MIT Statsd (Object Oriented) client library for PHP Giulio De Donato onoi/message-reporter 1.4.1 GPL-2.0-or-later An interface to report and relay arbitrary messages to registered handlers Jeroen De Dauw and James Hong Kong oojs/oojs-ui 0.34.1 MIT Provides library of common widgets, layouts, and windows. Bartosz Dziewoński, Ed Sanders, James D. Forrester, Kirsten Menger-Anderson, Kunal Mehta, Prateek Saxena, Roan Kattouw, Rob Moen, Timo Tijhof and Trevor Parscal pear/console_getopt 1.4.3 BSD-2-Clause More info available on: http://pear.php.net/package/Console_Getopt Andrei Zmievski, Stig Bakken and Greg Beaver pear/mail 1.4.1 BSD-2-Clause Class that provides multiple interfaces for sending emails. Chuck Hagenbuch, Richard Heyes and Aleksander Machniak pear/mail_mime 1.10.2 BSD-3-clause Mail_Mime provides classes to create MIME messages Cipriano Groenendal and Aleksander Machniak pear/net_smtp 1.8.1 BSD-2-Clause An implementation of the SMTP protocol Chuck Hagenbuch and Jon Parise pear/net_socket 1.2.2 PHP License More info available on: http://pear.php.net/package/Net_Socket Chuck Hagenbuch, Aleksander Machniak and Stig Bakken pear/pear-core-minimal 1.10.10 BSD-3-Clause Minimal set of PEAR core files to be used as composer dependency Christian Weiske pear/pear_exception 1.0.1 BSD-2-Clause The PEAR Exception base class. Helgi Thormar and Greg Beaver pleonasm/bloom-filter 1.0.2 BSD-2-Clause A pure PHP implementation of a Bloom Filter Matthew Nagi psr/container 1.0.0 MIT Common Container Interface (PHP FIG PSR-11) PHP-FIG psr/http-message 1.0.1 MIT Common interface for HTTP messages PHP-FIG psr/log 1.0.2 MIT Common interface for logging libraries PHP-FIG psr/simple-cache 1.0.1 MIT Common interfaces for simple caching PHP-FIG ralouphie/getallheaders 3.0.3 MIT A polyfill for getallheaders. Ralph Khattar serialization/serialization 4.0.0 GPL-2.0+ Library defining a Serializer and a Deserializer interface and basic utilities Jeroen De Dauw wikibase/data-model 9.2.0 GPL-2.0-or-later PHP implementation of the Wikibase DataModel Jeroen De Dauw and Thiemo Kreuzwikibase/data-model-serialization 2.9.1 GPL-2.0-or-later Serializers and deserializers for the Wikibase DataModel Thomas PT and Jeroen De Dauw wikibase/data-model-services 3.15.0 GPL-2.0-or-later Services around the Wikibase DataModel The Wikidata team and Jeroen De Dauw wikibase/internal-serialization 2.10.0 GPL-2.0-or-later Serializers and deserializers for the data access layer of Wikibase Repository Jeroen De Dauw wikibase/term-store 1.0.4 BSD-3-Clause Tiny Wikibase library that defines interfaces for persisting terms of Items and Properties The Wikidata team wikimedia/assert 0.2.2 MIT Provides runtime assertions Daniel Kinzler wikimedia/at-ease 2.0.0 GPL-2.0-or-later Safe replacement to @ for suppressing warnings. Tim Starling and MediaWiki developers wikimedia/base-convert 2.0.0 GPL-2.0-or-later Convert an arbitrarily-long string from one numeric base to another, optionally zero-padding to a minimum column width. Brion Vibber and Tyler Romeo wikimedia/cdb 1.4.1 GPL-2.0+ Constant Database (CDB) wrapper library for PHP. Provides pure-PHP fallback when dba_* functions are absent. Daniel Kinzler, Tim Starling, Chad Horohoe and Ori Livneh wikimedia/cldr-plural-rule-parser 1.0.0 GPL-2.0+ Evaluates plural rules specified in the CLDR project notation. Tim Starling and Niklas Laxström wikimedia/composer-merge-plugin 1.4.1 MIT Composer plugin to merge multiple composer.json files Bryan Davis wikimedia/html-formatter 1.0.2 GPL-2.0-or-later Performs transformations of HTML by wrapping around libxml2 and working around its countless bugs. MediaWiki contributors wikimedia/ip-set 2.1.0 GPL-2.0-or-later Efficiently match IP addresses against a set of CIDR specifications. Brandon Black wikimedia/less.php 1.8.0 Apache-2.0 PHP port of the Javascript version of LESS http://lesscss.org (Originally maintained by Josh Schmidt) Matt Agar , Martin Jantošovič and Josh Schmidt wikimedia/object-factory 2.1.0 GPL-2.0-or-later Construct objects from configuration instructions Bryan Davis wikimedia/password-blacklist 0.1.4 MIT PasswordBlacklist for the 100,000 most used passwords Sam Reed wikimedia/php-session-serializer 1.0.7 GPL-2.0-or-later Provides methods like PHP's session_encode and session_decode that don't mess with $_SESSION Brad Jorsch wikimedia/purtle 1.0.7 GPL-2.0-or-later Fast streaming RDF serializer Daniel Kinzler, Stanislav Malyshev, C. Scott Ananian and Thiemo Kreuz wikimedia/relpath 2.1.1 MIT Compute a relative filepath between two paths. Ori Livneh wikimedia/remex-html 2.1.0 MIT Fast HTML 5 parser Tim Starling wikimedia/running-stat 1.2.1 GPL-2.0+ PHP implementations of online statistical algorithms Ori Livneh wikimedia/scoped-callback 3.0.0 GPL-2.0-or-later Class for asserting that a callback happens when a dummy object leaves scope Aaron Schulz wikimedia/timestamp 3.0.0 GPL-2.0-or-later Creation, parsing, and conversion of timestamps Tyler Romeo wikimedia/utfnormal 2.0.0 GPL-2.0-or-later Contains Unicode normalization routines, including both pure PHP implementations and automatic use of the 'intl' PHP extension when present Brion Vibber wikimedia/wait-condition-loop 1.0.1 GPL-2.0+ Wait loop that reaches a condition or times out Aaron Schulz wikimedia/wrappedstring 3.0.1 MIT Automatically compact sequentially-outputted strings that share a common prefix / suffix pair. Timo Tijhof wikimedia/xmp-reader 0.6.3 GPL-2.0-or-later Reader for XMP data containing properties relevant to images Brian Wolff zordius/lightncandy 0.23 MIT An extremely fast PHP implementation of handlebars ( http://handlebarsjs.com/ ) and mustache ( http://mustache.github.io/ ). Zordius Chen
<gallery> , <indicator> , <nowiki> , <pagelist> , <pagequality> , <pages> , <pre> and <section> anchorencode, babel, basepagename, basepagenamee, bidi, canonicalurl, canonicalurle, cascadingsources, defaultsort, displaytitle, filepath, formatdate, formatnum, fullpagename, fullpagenamee, fullurl, fullurle, gender, grammar, int, invoke, language, lc, lcfirst, localurl, localurle, lst, lsth, lstx, namespace, namespacee, namespacenumber, noexternallanglinks, ns, nse, numberingroup, numberofactiveusers, numberofadmins, numberofarticles, numberofedits, numberoffiles, numberofpages, numberofusers, padleft, padright, pageid, pagename, pagenamee, pagesincategory, pagesize, plural, property, protectionexpiry, protectionlevel, revisionday, revisionday2, revisionid, revisionmonth, revisionmonth1, revisiontimestamp, revisionuser, revisionyear, rootpagename, rootpagenamee, special, speciale, statements, subjectpagename, subjectpagenamee, subjectspace, subjectspacee, subpagename, subpagenamee, tag, talkpagename, talkpagenamee, talkspace, talkspacee, uc, ucfirst and urlencode
diff --git a/tests/test_create_observation/test_create_software_version_observation/test_constants.py b/tests/test_create_observation/test_create_software_version_observation/test_constants.py
new file mode 100644
index 00000000..0b54bd95
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/test_constants.py
@@ -0,0 +1,5 @@
+"""Constants"""
+
+DATA_DIRECTORY = (
+ "tests/test_create_observation/test_create_software_version_observation/data"
+)
diff --git a/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py b/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py
index 687127da..4d842e75 100644
--- a/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py
+++ b/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py
@@ -5,18 +5,18 @@
from urllib.error import HTTPError
import pytest
from fetch_data import create_software_version_observation
-from tests.utils import MockResponse
-
-
-DATA_DIRECTORY = (
- "tests/test_create_observation/test_create_software_version_observation/data"
+from tests.mock_info import MockBackgroundClassList, MockInfo
+from tests.test_create_observation.test_create_software_version_observation.test_constants import (
+ DATA_DIRECTORY,
)
+from tests.utils import MockResponse
@pytest.mark.asyncio
@pytest.mark.dependency(
name="software-version-success", depends=["add-wikibase"], scope="session"
)
+@pytest.mark.soup
@pytest.mark.version
async def test_create_software_version_observation_success(mocker):
"""Test Data Returned Scenario"""
@@ -28,17 +28,20 @@ async def test_create_software_version_observation_success(mocker):
) as version_html:
mocker.patch(
- "fetch_data.soup_data.create_software_version_data_observation.requests.get",
- side_effect=[MockResponse(200, version_html.read())],
+ "fetch_data.soup_data.software.create_software_version_data_observation.requests.get",
+ side_effect=[MockResponse("", 200, version_html.read())],
)
- success = await create_software_version_observation(1)
+ mock_info = MockInfo(context={"background_tasks": MockBackgroundClassList()})
+ success = await create_software_version_observation(1, mock_info)
assert success
+ assert len(mock_info.context["background_tasks"].tasks) == 1
@pytest.mark.asyncio
@pytest.mark.dependency(
name="software-version-failure", depends=["software-version-success"]
)
+@pytest.mark.soup
@pytest.mark.version
async def test_create_software_version_observation_failure(mocker):
"""Test Failure Scenario"""
@@ -46,7 +49,7 @@ async def test_create_software_version_observation_failure(mocker):
time.sleep(1)
mocker.patch(
- "fetch_data.soup_data.create_software_version_data_observation.requests.get",
+ "fetch_data.soup_data.software.create_software_version_data_observation.requests.get",
side_effect=[
HTTPError(
url="example.com/wiki/Special:Version",
@@ -57,5 +60,7 @@ async def test_create_software_version_observation_failure(mocker):
)
],
)
- success = await create_software_version_observation(1)
+ mock_info = MockInfo(context={"background_tasks": MockBackgroundClassList()})
+ success = await create_software_version_observation(1, mock_info)
assert success is False
+ assert len(mock_info.context["background_tasks"].tasks) == 1
diff --git a/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py b/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py
new file mode 100644
index 00000000..4dbd69e0
--- /dev/null
+++ b/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py
@@ -0,0 +1,137 @@
+"""Test Extension Data"""
+
+from freezegun import freeze_time
+import pytest
+from fetch_data import update_software_data
+from fetch_data.soup_data.software.get_update_software_data import (
+ get_update_extension_query,
+)
+from tests.test_create_observation.test_create_software_version_observation.test_constants import (
+ DATA_DIRECTORY,
+)
+from tests.utils import MockResponse
+
+
+@freeze_time("2024-03-01")
+@pytest.mark.asyncio
+@pytest.mark.dependency(
+ name="update-software-data", depends=["software-version-success"], scope="session"
+)
+@pytest.mark.soup
+@pytest.mark.version
+async def test_update_software_data(mocker):
+ """Test Update Software Data"""
+
+ # pylint: disable=unused-argument,too-many-return-statements
+ def mockery(*args, **kwargs):
+ print(args)
+ query = args[0]
+ match query:
+ case "https://www.mediawiki.org/wiki/Extension:Babel":
+ with open(f"{DATA_DIRECTORY}/Mediawiki_Babel.html", mode="rb") as data:
+ return MockResponse(query, 200, data.read())
+ case (
+ "https://www.mediawiki.org/wiki/Extension:Google Analytics Integration"
+ ):
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_GoogleAnalyticsIntegration.html",
+ mode="rb",
+ ) as data:
+ return MockResponse(
+ "https://www.mediawiki.org/wiki/Extension:Google_Analytics_Integration",
+ 200,
+ data.read(),
+ )
+ case "https://www.mediawiki.org/wiki/Extension:LabeledSectionTransclusion":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_LabeledSectionTransclusion.html",
+ mode="rb",
+ ) as data:
+ return MockResponse(
+ "https://www.mediawiki.org/wiki/Extension:Labeled_Section_Transclusion",
+ 200,
+ data.read(),
+ )
+ case "https://www.mediawiki.org/wiki/Extension:MirahezeMagic":
+ return MockResponse(
+ "https://www.mediawiki.org/wiki/Extension:MirahezeMagic", 404
+ )
+ case "https://www.mediawiki.org/wiki/Extension:ProofreadPage":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_ProofreadPage.html", mode="rb"
+ ) as data:
+ return MockResponse(
+ "https://www.mediawiki.org/wiki/Extension:Proofread_Page",
+ 200,
+ data.read(),
+ )
+ case "https://www.mediawiki.org/wiki/Extension:Scribunto":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_Scribunto.html", mode="rb"
+ ) as data:
+ return MockResponse(query, 200, data.read())
+ case "https://www.mediawiki.org/wiki/Extension:UniversalLanguageSelector":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_UniversalLanguageSelector.html",
+ mode="rb",
+ ) as data:
+ return MockResponse(query, 200, data.read())
+ case "https://www.mediawiki.org/wiki/Extension:WikibaseClient":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_WikibaseClient.html", mode="rb"
+ ) as data:
+ return MockResponse(
+ "https://www.mediawiki.org/wiki/Extension:Wikibase_Client",
+ 200,
+ data.read(),
+ )
+ case "https://www.mediawiki.org/wiki/Extension:WikibaseLib":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_WikibaseLib.html", mode="rb"
+ ) as data:
+ return MockResponse(query, 200, data.read())
+ case "https://www.mediawiki.org/wiki/Extension:WikibaseRepository":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_WikibaseRepository.html", mode="rb"
+ ) as data:
+ return MockResponse(
+ "https://www.mediawiki.org/wiki/Extension:Wikibase_Repository",
+ 200,
+ data.read(),
+ )
+ case "https://www.mediawiki.org/wiki/Extension:WikibaseView":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_WikibaseView.html", mode="rb"
+ ) as data:
+ return MockResponse(query, 200, data.read())
+ case "https://www.mediawiki.org/wiki/Special:PermanentLink/3981869":
+ with open(
+ f"{DATA_DIRECTORY}/Mediawiki_WikibaseLib_Archived.html", mode="rb"
+ ) as data:
+ return MockResponse(
+ "https://www.mediawiki.org/w/index.php?oldid=3981869",
+ 200,
+ data.read(),
+ )
+ raise NotImplementedError(query)
+
+ mocker.patch(
+ "fetch_data.soup_data.software.get_update_software_data.requests.get",
+ side_effect=mockery,
+ )
+
+ await update_software_data()
+
+
+@pytest.mark.version
+def test_get_update_extension_query():
+ """Test Update Extension Query"""
+
+ query = get_update_extension_query()
+ assert str(query) == (
+ # pylint: disable=line-too-long
+ """SELECT wikibase_software.id, wikibase_software.software_type, wikibase_software.software_name, wikibase_software.url, wikibase_software.fetched, wikibase_software.description, wikibase_software.latest_version, wikibase_software.quarterly_download_count, wikibase_software.public_wiki_count, wikibase_software.mw_bundled, wikibase_software.archived
+FROM wikibase_software
+WHERE (wikibase_software.fetched IS NULL OR wikibase_software.fetched < :fetched_1) AND wikibase_software.software_type = :software_type_1 AND (wikibase_software.archived = false OR wikibase_software.archived IS NULL)
+ LIMIT :param_1"""
+ )
diff --git a/tests/test_create_observation/test_create_statistics_observation/__init__.py b/tests/test_create_observation/test_create_statistics_observation/__init__.py
new file mode 100644
index 00000000..83e8fe5d
--- /dev/null
+++ b/tests/test_create_observation/test_create_statistics_observation/__init__.py
@@ -0,0 +1 @@
+"""Test Create Statistics Observation"""
diff --git a/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py b/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py
index d0ee7168..75084f1d 100644
--- a/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py
+++ b/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py
@@ -15,6 +15,7 @@
@pytest.mark.dependency(
name="statistics-success", depends=["add-wikibase"], scope="session"
)
+@pytest.mark.soup
@pytest.mark.statistics
async def test_create_statistics_observation_success(mocker):
"""Test Data Returned Scenario"""
@@ -25,7 +26,7 @@ async def test_create_statistics_observation_success(mocker):
mocker.patch(
"fetch_data.soup_data.create_statistics_data_observation.requests.get",
- side_effect=[MockResponse(200, version_html.read())],
+ side_effect=[MockResponse("", 200, version_html.read())],
)
success = await create_special_statistics_observation(1)
assert success
@@ -33,6 +34,7 @@ async def test_create_statistics_observation_success(mocker):
@pytest.mark.asyncio
@pytest.mark.dependency(name="statistics-failure", depends=["statistics-success"])
+@pytest.mark.soup
@pytest.mark.statistics
async def test_create_statistics_observation_failure(mocker):
"""Test Failure Scenario"""
diff --git a/tests/test_data_expectations.py b/tests/test_data_expectations.py
index b003dc91..571ea3d0 100644
--- a/tests/test_data_expectations.py
+++ b/tests/test_data_expectations.py
@@ -1,6 +1,7 @@
"""Test Data Expectations with Great Expectations"""
import os
+from pathlib import Path
import pytest
import great_expectations as gx
@@ -15,7 +16,11 @@ def pytest_generate_tests(metafunc):
if "checkpoint_name" in metafunc.fixturenames:
# Generate test cases based on the test_data list
metafunc.parametrize(
- "checkpoint_name", sorted(os.listdir(CHECKPOINT_DIRECTORY))
+ "checkpoint_name",
+ [
+ Path(checkpoint_filename).stem
+ for checkpoint_filename in sorted(os.listdir(CHECKPOINT_DIRECTORY))
+ ],
)
diff --git a/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py b/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py
index 073a143d..8333f839 100644
--- a/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py
+++ b/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py
@@ -9,7 +9,7 @@
SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT,
)
from tests.test_schema import test_schema
-from tests.utils import assert_layered_property_count, assert_layered_property_value
+from tests.utils import assert_layered_property_count, assert_page_meta
AGGREGATE_EXTENSIONS_QUERY = (
@@ -39,18 +39,7 @@ async def test_aggregate_extensions_query_page_one():
assert result.errors is None
assert result.data is not None
- assert_layered_property_value(
- result.data, ["aggregateExtensionPopularity", "meta", "pageNumber"], 1
- )
- assert_layered_property_value(
- result.data, ["aggregateExtensionPopularity", "meta", "pageSize"], 5
- )
- assert_layered_property_value(
- result.data, ["aggregateExtensionPopularity", "meta", "totalCount"], 10
- )
- assert_layered_property_value(
- result.data, ["aggregateExtensionPopularity", "meta", "totalPages"], 2
- )
+ assert_page_meta(result.data["aggregateExtensionPopularity"], 1, 5, 11, 3)
assert_layered_property_count(
result.data, ["aggregateExtensionPopularity", "data"], 5
)
@@ -78,6 +67,13 @@ async def test_aggregate_extensions_query_page_one():
datetime(2020, 1, 29, 14, 52),
"f621799",
),
+ (
+ "Miraheze Magic",
+ "e742444",
+ (None, None, None),
+ datetime(2024, 10, 17, 15, 21),
+ "e742444",
+ ),
(
"ProofreadPage",
"cb0a218",
@@ -85,7 +81,6 @@ async def test_aggregate_extensions_query_page_one():
datetime(2019, 9, 30, 9, 20),
"cb0a218",
),
- ("Scribunto", None, (None, None, None), None, None),
]
):
@@ -113,18 +108,7 @@ async def test_aggregate_extensions_query_page_two():
assert result.errors is None
assert result.data is not None
- assert_layered_property_value(
- result.data, ["aggregateExtensionPopularity", "meta", "pageNumber"], 2
- )
- assert_layered_property_value(
- result.data, ["aggregateExtensionPopularity", "meta", "pageSize"], 5
- )
- assert_layered_property_value(
- result.data, ["aggregateExtensionPopularity", "meta", "totalCount"], 10
- )
- assert_layered_property_value(
- result.data, ["aggregateExtensionPopularity", "meta", "totalPages"], 2
- )
+ assert_page_meta(result.data["aggregateExtensionPopularity"], 2, 5, 11, 3)
assert_layered_property_count(
result.data, ["aggregateExtensionPopularity", "data"], 5
)
@@ -137,6 +121,7 @@ async def test_aggregate_extensions_query_page_two():
expected_version_hash,
) in enumerate(
[
+ ("Scribunto", None, (None, None, None), None, None),
(
"UniversalLanguageSelector",
"2020-01-23",
@@ -165,6 +150,46 @@ async def test_aggregate_extensions_query_page_two():
datetime(2019, 12, 10, 12, 52),
"dbbcdd8",
),
+ ]
+ ):
+
+ assert_software_version_aggregate(
+ result.data["aggregateExtensionPopularity"]["data"][index],
+ expected_software_name,
+ expected_version_string,
+ expected_version_semver,
+ expected_version_date,
+ expected_version_hash,
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.agg
+@pytest.mark.dependency(depends=["software-version-success"], scope="session")
+@pytest.mark.query
+@pytest.mark.version
+async def test_aggregate_extensions_query_page_three():
+ """Test Aggregated Extensions Query - 11"""
+
+ result = await test_schema.execute(
+ AGGREGATE_EXTENSIONS_QUERY, variable_values={"pageNumber": 3, "pageSize": 5}
+ )
+
+ assert result.errors is None
+ assert result.data is not None
+ assert_page_meta(result.data["aggregateExtensionPopularity"], 3, 5, 11, 3)
+ assert_layered_property_count(
+ result.data, ["aggregateExtensionPopularity", "data"], 1
+ )
+
+ for index, (
+ expected_software_name,
+ expected_version_string,
+ expected_version_semver,
+ expected_version_date,
+ expected_version_hash,
+ ) in enumerate(
+ [
(
"WikibaseView",
"dbbcdd8",
diff --git a/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py b/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py
index e7741bd7..d7bee490 100644
--- a/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py
+++ b/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py
@@ -8,7 +8,7 @@
SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT,
)
from tests.test_schema import test_schema
-from tests.utils import assert_layered_property_count, assert_layered_property_value
+from tests.utils import assert_layered_property_count, assert_page_meta
AGGREGATE_LIBRARIES_QUERY = (
@@ -38,18 +38,7 @@ async def test_aggregate_libraries_query_page_one():
assert result.errors is None
assert result.data is not None
- assert_layered_property_value(
- result.data, ["aggregateLibraryPopularity", "meta", "pageNumber"], 1
- )
- assert_layered_property_value(
- result.data, ["aggregateLibraryPopularity", "meta", "pageSize"], 30
- )
- assert_layered_property_value(
- result.data, ["aggregateLibraryPopularity", "meta", "totalCount"], 59
- )
- assert_layered_property_value(
- result.data, ["aggregateLibraryPopularity", "meta", "totalPages"], 2
- )
+ assert_page_meta(result.data["aggregateLibraryPopularity"], 1, 30, 59, 2)
assert_layered_property_count(
result.data, ["aggregateLibraryPopularity", "data"], 30
)
@@ -117,18 +106,7 @@ async def test_aggregate_libraries_query_page_two():
assert result.errors is None
assert result.data is not None
- assert_layered_property_value(
- result.data, ["aggregateLibraryPopularity", "meta", "pageNumber"], 2
- )
- assert_layered_property_value(
- result.data, ["aggregateLibraryPopularity", "meta", "pageSize"], 30
- )
- assert_layered_property_value(
- result.data, ["aggregateLibraryPopularity", "meta", "totalCount"], 59
- )
- assert_layered_property_value(
- result.data, ["aggregateLibraryPopularity", "meta", "totalPages"], 2
- )
+ assert_page_meta(result.data["aggregateLibraryPopularity"], 2, 30, 59, 2)
assert_layered_property_count(
result.data, ["aggregateLibraryPopularity", "data"], 29
)
diff --git a/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py b/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py
index fe23d216..0e0032c2 100644
--- a/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py
+++ b/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py
@@ -1,6 +1,7 @@
"""Test Aggregated Skins Query"""
import pytest
+
from tests.test_query.aggregation.software_version.assert_software_version_aggregate import (
assert_software_version_aggregate,
)
@@ -8,7 +9,7 @@
SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT,
)
from tests.test_schema import test_schema
-from tests.utils import assert_layered_property_count, assert_layered_property_value
+from tests.utils import assert_layered_property_count, assert_page_meta
AGGREGATE_SKINS_QUERY = (
@@ -38,18 +39,7 @@ async def test_aggregate_skins_query_page_one():
assert result.errors is None
assert result.data is not None
- assert_layered_property_value(
- result.data, ["aggregateSkinPopularity", "meta", "pageNumber"], 1
- )
- assert_layered_property_value(
- result.data, ["aggregateSkinPopularity", "meta", "pageSize"], 5
- )
- assert_layered_property_value(
- result.data, ["aggregateSkinPopularity", "meta", "totalCount"], 3
- )
- assert_layered_property_value(
- result.data, ["aggregateSkinPopularity", "meta", "totalPages"], 1
- )
+ assert_page_meta(result.data["aggregateSkinPopularity"], 1, 5, 3, 1)
assert_layered_property_count(result.data, ["aggregateSkinPopularity", "data"], 3)
for index, (
diff --git a/tests/test_query/aggregation/software_version/test_aggregate_software_query.py b/tests/test_query/aggregation/software_version/test_aggregate_software_query.py
index e388859c..ba7d3d2a 100644
--- a/tests/test_query/aggregation/software_version/test_aggregate_software_query.py
+++ b/tests/test_query/aggregation/software_version/test_aggregate_software_query.py
@@ -9,7 +9,7 @@
SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT,
)
from tests.test_schema import test_schema
-from tests.utils import assert_layered_property_count, assert_layered_property_value
+from tests.utils import assert_layered_property_count, assert_page_meta
AGGREGATE_SOFTWARE_QUERY = (
@@ -39,18 +39,7 @@ async def test_aggregate_software_query():
assert result.errors is None
assert result.data is not None
- assert_layered_property_value(
- result.data, ["aggregateSoftwarePopularity", "meta", "pageNumber"], 1
- )
- assert_layered_property_value(
- result.data, ["aggregateSoftwarePopularity", "meta", "pageSize"], 5
- )
- assert_layered_property_value(
- result.data, ["aggregateSoftwarePopularity", "meta", "totalCount"], 5
- )
- assert_layered_property_value(
- result.data, ["aggregateSoftwarePopularity", "meta", "totalPages"], 1
- )
+ assert_page_meta(result.data["aggregateSoftwarePopularity"], 1, 5, 5, 1)
assert_layered_property_count(
result.data, ["aggregateSoftwarePopularity", "data"], 5
)
diff --git a/tests/test_query/aggregation/test_aggregate_property_popularity_query.py b/tests/test_query/aggregation/test_aggregate_property_popularity_query.py
index e591e678..4fdfb76a 100644
--- a/tests/test_query/aggregation/test_aggregate_property_popularity_query.py
+++ b/tests/test_query/aggregation/test_aggregate_property_popularity_query.py
@@ -2,7 +2,11 @@
import pytest
from tests.test_schema import test_schema
-from tests.utils import assert_layered_property_count, assert_layered_property_value
+from tests.utils import (
+ assert_layered_property_count,
+ assert_layered_property_value,
+ assert_page_meta,
+)
AGGREGATED_PROPERTY_POPULARITY_QUERY = """
@@ -41,18 +45,7 @@ async def test_aggregate_property_popularity_query():
assert result.errors is None
assert result.data is not None
- assert_layered_property_value(
- result.data, ["aggregatePropertyPopularity", "meta", "pageNumber"], 1
- )
- assert_layered_property_value(
- result.data, ["aggregatePropertyPopularity", "meta", "pageSize"], 30
- )
- assert_layered_property_value(
- result.data, ["aggregatePropertyPopularity", "meta", "totalCount"], 2
- )
- assert_layered_property_value(
- result.data, ["aggregatePropertyPopularity", "meta", "totalPages"], 1
- )
+ assert_page_meta(result.data["aggregatePropertyPopularity"], 1, 30, 2, 1)
assert_layered_property_count(
result.data, ["aggregatePropertyPopularity", "data"], 2
diff --git a/tests/test_query/test_extension_list_query.py b/tests/test_query/test_extension_list_query.py
new file mode 100644
index 00000000..18695d97
--- /dev/null
+++ b/tests/test_query/test_extension_list_query.py
@@ -0,0 +1,309 @@
+"""Test Wikibase List"""
+
+from datetime import datetime
+from typing import Optional
+import pytest
+from tests.test_schema import test_schema
+from tests.utils import assert_layered_property_value, assert_page_meta
+
+
+EXTENSION_LIST_QUERY = """
+query MyQuery($pageNumber: Int!, $pageSize: Int!) {
+ extensionList(pageNumber: $pageNumber, pageSize: $pageSize) {
+ meta {
+ pageNumber
+ pageSize
+ totalCount
+ totalPages
+ }
+ data {
+ id
+ softwareName
+ softwareType
+ url
+ archived
+ description
+ fetched
+ latestVersion
+ mediawikiBundled
+ publicWikiCount
+ quarterlyDownloadCount
+ tags
+ }
+ }
+}"""
+
+
+@pytest.mark.asyncio
+@pytest.mark.query
+@pytest.mark.version
+@pytest.mark.dependency(depends=["update-software-data"], scope="session")
+async def test_extension_list_query():
+ """Test Extension List"""
+
+ result = await test_schema.execute(
+ EXTENSION_LIST_QUERY, variable_values={"pageNumber": 1, "pageSize": 10}
+ )
+
+ assert result.errors is None
+ assert result.data is not None
+ assert "extensionList" in result.data
+ assert_page_meta(result.data["extensionList"], 1, 10, 11, 2)
+ assert "data" in result.data["extensionList"]
+ assert len(result.data["extensionList"]["data"]) == 10
+
+
+@pytest.mark.asyncio
+@pytest.mark.query
+@pytest.mark.version
+@pytest.mark.dependency(depends=["update-software-data"], scope="session")
+@pytest.mark.parametrize(
+ [
+ "idx",
+ "expected_id",
+ "expected_name",
+ "expected_url",
+ "expected_archived",
+ "expected_description",
+ "expected_fetched",
+ "expected_latest_version",
+ "expected_mediawiki_bundled",
+ "expected_public_wiki_count",
+ "expected_quarterly_download_count",
+ "expected_tags",
+ ],
+ [
+ (
+ 0,
+ "2",
+ "Babel",
+ "Babel",
+ False,
+ # pylint: disable=line-too-long
+ "Adds a parser function to inform other users about language proficiency and categorize users of the same levels and languages.",
+ datetime(2024, 3, 1),
+ "Continuous updates",
+ False,
+ 2416,
+ 63,
+ ["Parser function"],
+ ),
+ (
+ 1,
+ "17",
+ "Google Analytics Integration",
+ "Google_Analytics_Integration",
+ False,
+ # pylint: disable=line-too-long
+ "Automatically inserts Google Universal Analytics (and/or other web analytics) tracking code at the bottom of MediaWiki pages",
+ datetime(2024, 3, 1),
+ "3.0.1 (2017-10-29)",
+ False,
+ 1302,
+ None,
+ ["User activity", "Hook"],
+ ),
+ (
+ 2,
+ "11",
+ "LabeledSectionTransclusion",
+ "Labeled_Section_Transclusion",
+ False,
+ "Enables marked sections of text to be transcluded",
+ datetime(2024, 3, 1),
+ None,
+ False,
+ 6919,
+ None,
+ ["Parser function", "Tag"],
+ ),
+ (
+ 3,
+ "1",
+ "Miraheze Magic",
+ "MirahezeMagic",
+ None,
+ None,
+ datetime(2024, 3, 1),
+ None,
+ None,
+ None,
+ None,
+ [],
+ ),
+ (
+ 4,
+ "18",
+ "ProofreadPage",
+ "Proofread_Page",
+ False,
+ # pylint: disable=line-too-long
+ "The Proofread Page extension can render a book either as a column of OCR text beside a column of scanned images, or broken into its logical organization (such as chapters or poems) using transclusion.",
+ datetime(2024, 3, 1),
+ "continuous updates",
+ False,
+ None,
+ None,
+ ["Tag", "Page action", "ContentHandler", "API", "Database"],
+ ),
+ (
+ 5,
+ "12",
+ "Scribunto",
+ "Scribunto",
+ False,
+ "Provides a framework for embedding scripting languages into MediaWiki pages",
+ datetime(2024, 3, 1),
+ "Continuous updates",
+ True,
+ 8789,
+ 450,
+ ["Parser extension"],
+ ),
+ (
+ 6,
+ "19",
+ "UniversalLanguageSelector",
+ "UniversalLanguageSelector",
+ False,
+ "Tool that allows users to select a language and configure its support in an easy way.",
+ datetime(2024, 3, 1),
+ "2024-07-16",
+ False,
+ 1237,
+ 243,
+ ["Skin", "Beta Feature"],
+ ),
+ (
+ 7,
+ "13",
+ "WikibaseClient",
+ "Wikibase_Client",
+ False,
+ "Client for structured data repository",
+ datetime(2024, 3, 1),
+ None,
+ False,
+ None,
+ None,
+ ["Parser function", "Ajax"],
+ ),
+ (
+ 8,
+ "14",
+ "WikibaseLib",
+ "WikibaseLib",
+ True,
+ "Provides common Wikibase functionality for Wikibase Repository and Wikibase Client",
+ datetime(2024, 3, 1),
+ "continuous updates",
+ False,
+ None,
+ None,
+ [],
+ ),
+ (
+ 9,
+ "15",
+ "WikibaseRepository",
+ "Wikibase_Repository",
+ False,
+ "Structured data repository",
+ datetime(2024, 3, 1),
+ "continuous updates",
+ False,
+ None,
+ None,
+ ["ContentHandler", "API", "Ajax"],
+ ),
+ (
+ 10,
+ "16",
+ "WikibaseView",
+ "WikibaseView",
+ False,
+ None,
+ datetime(2024, 3, 1),
+ None,
+ None,
+ None,
+ None,
+ [],
+ ),
+ ],
+)
+# pylint: disable=too-many-arguments,too-many-positional-arguments
+async def test_extension_list_query_parameterized(
+ idx: int,
+ expected_id: str,
+ expected_name: str,
+ expected_url: str,
+ expected_archived: bool,
+ expected_description: Optional[str],
+ expected_fetched: datetime,
+ expected_latest_version: Optional[str],
+ expected_mediawiki_bundled: Optional[bool],
+ expected_public_wiki_count: Optional[int],
+ expected_quarterly_download_count: Optional[int],
+ expected_tags: list[str],
+):
+ """Test Extension List"""
+
+ result = await test_schema.execute(
+ EXTENSION_LIST_QUERY, variable_values={"pageNumber": 1, "pageSize": 100}
+ )
+
+ assert result.errors is None
+ assert result.data is not None
+ assert "extensionList" in result.data
+ assert_page_meta(result.data["extensionList"], 1, 100, 11, 1)
+ assert "data" in result.data["extensionList"]
+ assert len(result.data["extensionList"]["data"]) == 11
+ assert_layered_property_value(
+ result.data, ["extensionList", "data", idx, "id"], expected_id
+ )
+ assert_layered_property_value(
+ result.data, ["extensionList", "data", idx, "softwareName"], expected_name
+ )
+ assert_layered_property_value(
+ result.data, ["extensionList", "data", idx, "softwareType"], "EXTENSION"
+ )
+ assert_layered_property_value(
+ result.data,
+ ["extensionList", "data", idx, "url"],
+ f"https://www.mediawiki.org/wiki/Extension:{expected_url}",
+ )
+ assert_layered_property_value(
+ result.data, ["extensionList", "data", idx, "archived"], expected_archived
+ )
+ assert_layered_property_value(
+ result.data, ["extensionList", "data", idx, "description"], expected_description
+ )
+ assert_layered_property_value(
+ result.data,
+ ["extensionList", "data", idx, "fetched"],
+ expected_fetched.strftime("%Y-%m-%dT%H:%M:%S"),
+ )
+ assert_layered_property_value(
+ result.data,
+ ["extensionList", "data", idx, "latestVersion"],
+ expected_latest_version,
+ )
+ assert_layered_property_value(
+ result.data,
+ ["extensionList", "data", idx, "mediawikiBundled"],
+ expected_mediawiki_bundled,
+ )
+ assert_layered_property_value(
+ result.data,
+ ["extensionList", "data", idx, "publicWikiCount"],
+ expected_public_wiki_count,
+ )
+ assert_layered_property_value(
+ result.data,
+ ["extensionList", "data", idx, "quarterlyDownloadCount"],
+ expected_quarterly_download_count,
+ )
+ assert_layered_property_value(
+ result.data, ["extensionList", "data", idx, "tags"], expected_tags
+ )
diff --git a/tests/test_query/test_wikibase_list_query.py b/tests/test_query/test_wikibase_list_query.py
index 02f95101..1e1d939e 100644
--- a/tests/test_query/test_wikibase_list_query.py
+++ b/tests/test_query/test_wikibase_list_query.py
@@ -2,7 +2,11 @@
import pytest
from tests.test_schema import test_schema
-from tests.utils import assert_layered_property_value, assert_property_value
+from tests.utils import (
+ assert_layered_property_value,
+ assert_page_meta,
+ assert_property_value,
+)
WIKIBASE_LIST_QUERY = """
@@ -81,17 +85,7 @@ async def test_wikibase_list_query():
assert result.errors is None
assert result.data is not None
assert "wikibaseList" in result.data
- assert "meta" in result.data["wikibaseList"]
- assert_layered_property_value(
- result.data, ["wikibaseList", "meta", "pageNumber"], 1
- )
- assert_layered_property_value(result.data, ["wikibaseList", "meta", "pageSize"], 1)
- assert_layered_property_value(
- result.data, ["wikibaseList", "meta", "totalCount"], 1
- )
- assert_layered_property_value(
- result.data, ["wikibaseList", "meta", "totalPages"], 1
- )
+ assert_page_meta(result.data["wikibaseList"], 1, 1, 1, 1)
assert "data" in result.data["wikibaseList"]
assert len(result.data["wikibaseList"]["data"]) == 1
result_datum = result.data["wikibaseList"]["data"][0]
diff --git a/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_extensions_query.py b/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_extensions_query.py
index 8227b38e..5307d394 100644
--- a/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_extensions_query.py
+++ b/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_extensions_query.py
@@ -60,7 +60,7 @@ async def test_wikibase_software_version_most_recent_observation_extensions_quer
assert "observationDate" in most_recent
assert_property_value(most_recent, "returnedData", True)
- assert_layered_property_count(most_recent, ["installedExtensions"], 10)
+ assert_layered_property_count(most_recent, ["installedExtensions"], 11)
for index, (
expected_id,
expected_name,
@@ -84,6 +84,13 @@ async def test_wikibase_software_version_most_recent_observation_extensions_quer
datetime(2020, 1, 29, 14, 52),
"f621799",
),
+ (
+ "19",
+ "Miraheze Magic",
+ "e742444",
+ datetime(2024, 10, 17, 15, 21),
+ "e742444",
+ ),
("17", "ProofreadPage", "cb0a218", datetime(2019, 9, 30, 9, 20), "cb0a218"),
("11", "Scribunto", None, None, None),
(
diff --git a/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_libraries_query.py b/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_libraries_query.py
index bfa1ab2c..dfda6f47 100644
--- a/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_libraries_query.py
+++ b/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_libraries_query.py
@@ -68,65 +68,65 @@ async def test_wikibase_software_version_most_recent_observation_libraries_query
expected_version_hash,
) in enumerate(
[
- ("19", "composer/installers", "1.8.0", None, None),
- ("20", "composer/semver", "1.5.0", None, None),
- ("21", "cssjanus/cssjanus", "1.3.0", None, None),
- ("22", "data-values/common", "0.4.3", None, None),
- ("23", "data-values/data-values", "2.3.0", None, None),
- ("24", "data-values/geo", "3.0.1", None, None),
- ("25", "data-values/interfaces", "0.2.5", None, None),
- ("26", "data-values/number", "0.10.1", None, None),
- ("27", "data-values/serialization", "1.2.3", None, None),
- ("28", "data-values/time", "1.0.1", None, None),
- ("29", "diff/diff", "2.3.0", None, None),
- ("30", "guzzlehttp/guzzle", "6.3.3", None, None),
- ("31", "guzzlehttp/promises", "1.3.1", None, None),
- ("32", "guzzlehttp/psr7", "1.6.1", None, None),
- ("33", "liuggio/statsd-php-client", "1.0.18", None, None),
- ("34", "onoi/message-reporter", "1.4.1", None, None),
- ("35", "oojs/oojs-ui", "0.34.1", None, None),
- ("36", "pear/console_getopt", "1.4.3", None, None),
- ("37", "pear/mail", "1.4.1", None, None),
- ("38", "pear/mail_mime", "1.10.2", None, None),
- ("39", "pear/net_smtp", "1.8.1", None, None),
- ("40", "pear/net_socket", "1.2.2", None, None),
- ("41", "pear/pear-core-minimal", "1.10.10", None, None),
- ("42", "pear/pear_exception", "1.0.1", None, None),
- ("43", "pleonasm/bloom-filter", "1.0.2", None, None),
- ("44", "psr/container", "1.0.0", None, None),
- ("45", "psr/http-message", "1.0.1", None, None),
- ("46", "psr/log", "1.0.2", None, None),
- ("47", "psr/simple-cache", "1.0.1", None, None),
- ("48", "ralouphie/getallheaders", "3.0.3", None, None),
- ("49", "serialization/serialization", "4.0.0", None, None),
- ("50", "wikibase/data-model", "9.2.0", None, None),
- ("51", "wikibase/data-model-serialization", "2.9.1", None, None),
- ("52", "wikibase/data-model-services", "3.15.0", None, None),
- ("53", "wikibase/internal-serialization", "2.10.0", None, None),
- ("54", "wikibase/term-store", "1.0.4", None, None),
- ("55", "wikimedia/assert", "0.2.2", None, None),
- ("56", "wikimedia/at-ease", "2.0.0", None, None),
- ("57", "wikimedia/base-convert", "2.0.0", None, None),
- ("58", "wikimedia/cdb", "1.4.1", None, None),
- ("59", "wikimedia/cldr-plural-rule-parser", "1.0.0", None, None),
- ("60", "wikimedia/composer-merge-plugin", "1.4.1", None, None),
- ("61", "wikimedia/html-formatter", "1.0.2", None, None),
- ("62", "wikimedia/ip-set", "2.1.0", None, None),
- ("63", "wikimedia/less.php", "1.8.0", None, None),
- ("64", "wikimedia/object-factory", "2.1.0", None, None),
- ("65", "wikimedia/password-blacklist", "0.1.4", None, None),
- ("66", "wikimedia/php-session-serializer", "1.0.7", None, None),
- ("67", "wikimedia/purtle", "1.0.7", None, None),
- ("68", "wikimedia/relpath", "2.1.1", None, None),
- ("69", "wikimedia/remex-html", "2.1.0", None, None),
- ("70", "wikimedia/running-stat", "1.2.1", None, None),
- ("71", "wikimedia/scoped-callback", "3.0.0", None, None),
- ("72", "wikimedia/timestamp", "3.0.0", None, None),
- ("73", "wikimedia/utfnormal", "2.0.0", None, None),
- ("74", "wikimedia/wait-condition-loop", "1.0.1", None, None),
- ("75", "wikimedia/wrappedstring", "3.0.1", None, None),
- ("76", "wikimedia/xmp-reader", "0.6.3", None, None),
- ("77", "zordius/lightncandy", "0.23", None, None),
+ ("20", "composer/installers", "1.8.0", None, None),
+ ("21", "composer/semver", "1.5.0", None, None),
+ ("22", "cssjanus/cssjanus", "1.3.0", None, None),
+ ("23", "data-values/common", "0.4.3", None, None),
+ ("24", "data-values/data-values", "2.3.0", None, None),
+ ("25", "data-values/geo", "3.0.1", None, None),
+ ("26", "data-values/interfaces", "0.2.5", None, None),
+ ("27", "data-values/number", "0.10.1", None, None),
+ ("28", "data-values/serialization", "1.2.3", None, None),
+ ("29", "data-values/time", "1.0.1", None, None),
+ ("30", "diff/diff", "2.3.0", None, None),
+ ("31", "guzzlehttp/guzzle", "6.3.3", None, None),
+ ("32", "guzzlehttp/promises", "1.3.1", None, None),
+ ("33", "guzzlehttp/psr7", "1.6.1", None, None),
+ ("34", "liuggio/statsd-php-client", "1.0.18", None, None),
+ ("35", "onoi/message-reporter", "1.4.1", None, None),
+ ("36", "oojs/oojs-ui", "0.34.1", None, None),
+ ("37", "pear/console_getopt", "1.4.3", None, None),
+ ("38", "pear/mail", "1.4.1", None, None),
+ ("39", "pear/mail_mime", "1.10.2", None, None),
+ ("40", "pear/net_smtp", "1.8.1", None, None),
+ ("41", "pear/net_socket", "1.2.2", None, None),
+ ("42", "pear/pear-core-minimal", "1.10.10", None, None),
+ ("43", "pear/pear_exception", "1.0.1", None, None),
+ ("44", "pleonasm/bloom-filter", "1.0.2", None, None),
+ ("45", "psr/container", "1.0.0", None, None),
+ ("46", "psr/http-message", "1.0.1", None, None),
+ ("47", "psr/log", "1.0.2", None, None),
+ ("48", "psr/simple-cache", "1.0.1", None, None),
+ ("49", "ralouphie/getallheaders", "3.0.3", None, None),
+ ("50", "serialization/serialization", "4.0.0", None, None),
+ ("51", "wikibase/data-model", "9.2.0", None, None),
+ ("52", "wikibase/data-model-serialization", "2.9.1", None, None),
+ ("53", "wikibase/data-model-services", "3.15.0", None, None),
+ ("54", "wikibase/internal-serialization", "2.10.0", None, None),
+ ("55", "wikibase/term-store", "1.0.4", None, None),
+ ("56", "wikimedia/assert", "0.2.2", None, None),
+ ("57", "wikimedia/at-ease", "2.0.0", None, None),
+ ("58", "wikimedia/base-convert", "2.0.0", None, None),
+ ("59", "wikimedia/cdb", "1.4.1", None, None),
+ ("60", "wikimedia/cldr-plural-rule-parser", "1.0.0", None, None),
+ ("61", "wikimedia/composer-merge-plugin", "1.4.1", None, None),
+ ("62", "wikimedia/html-formatter", "1.0.2", None, None),
+ ("63", "wikimedia/ip-set", "2.1.0", None, None),
+ ("64", "wikimedia/less.php", "1.8.0", None, None),
+ ("65", "wikimedia/object-factory", "2.1.0", None, None),
+ ("66", "wikimedia/password-blacklist", "0.1.4", None, None),
+ ("67", "wikimedia/php-session-serializer", "1.0.7", None, None),
+ ("68", "wikimedia/purtle", "1.0.7", None, None),
+ ("69", "wikimedia/relpath", "2.1.1", None, None),
+ ("70", "wikimedia/remex-html", "2.1.0", None, None),
+ ("71", "wikimedia/running-stat", "1.2.1", None, None),
+ ("72", "wikimedia/scoped-callback", "3.0.0", None, None),
+ ("73", "wikimedia/timestamp", "3.0.0", None, None),
+ ("74", "wikimedia/utfnormal", "2.0.0", None, None),
+ ("75", "wikimedia/wait-condition-loop", "1.0.1", None, None),
+ ("76", "wikimedia/wrappedstring", "3.0.1", None, None),
+ ("77", "wikimedia/xmp-reader", "0.6.3", None, None),
+ ("78", "zordius/lightncandy", "0.23", None, None),
]
):
assert_software_version(
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
index 5279dece..ea7e951a 100644
--- a/tests/utils/__init__.py
+++ b/tests/utils/__init__.py
@@ -1,5 +1,6 @@
"""Test Utilities"""
+from tests.utils.assert_meta import assert_page_meta
from tests.utils.assert_property_value import (
assert_layered_property_count,
assert_layered_property_value,
diff --git a/tests/utils/assert_meta.py b/tests/utils/assert_meta.py
new file mode 100644
index 00000000..af984182
--- /dev/null
+++ b/tests/utils/assert_meta.py
@@ -0,0 +1,19 @@
+"""Assert Page Metadata"""
+
+from tests.utils.assert_property_value import assert_layered_property_value
+
+
+def assert_page_meta(
+ page: dict,
+ expected_page_number: int,
+ expected_page_size: int,
+ expected_total_count: int,
+ expected_total_pages: int,
+):
+ """Assert Page Metadata"""
+
+ assert "meta" in page
+ assert_layered_property_value(page, ["meta", "pageNumber"], expected_page_number)
+ assert_layered_property_value(page, ["meta", "pageSize"], expected_page_size)
+ assert_layered_property_value(page, ["meta", "totalCount"], expected_total_count)
+ assert_layered_property_value(page, ["meta", "totalPages"], expected_total_pages)
diff --git a/tests/utils/mock_response.py b/tests/utils/mock_response.py
index 6123a33f..ffd8bf85 100644
--- a/tests/utils/mock_response.py
+++ b/tests/utils/mock_response.py
@@ -1,12 +1,22 @@
"""Mock HTTP Response"""
+from typing import Optional
+
class MockResponse:
"""Mock HTTP Response"""
status_code: int
content: bytes
+ url: str
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args, **kwargs):
+ pass
- def __init__(self, status_code: int, content: bytes):
+ def __init__(self, url: str, status_code: int, content: Optional[bytes] = None):
+ self.url = url
self.content = content
self.status_code = status_code