diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index b7b44bfc..8c7c1f04 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -10,8 +10,6 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v3 - with: - python-version: "3.10.12" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -25,8 +23,6 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v3 - with: - python-version: "3.10.12" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -40,8 +36,6 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v3 - with: - python-version: "3.10.12" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -55,8 +49,6 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v3 - with: - python-version: "3.10.12" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/alembic/versions/0833cb54870f_software_model.py b/alembic/versions/0833cb54870f_software_model.py new file mode 100644 index 00000000..92f1304a --- /dev/null +++ b/alembic/versions/0833cb54870f_software_model.py @@ -0,0 +1,107 @@ +"""Software Model + +Revision ID: 0833cb54870f +Revises: e3ecd40e3851 +Create Date: 2024-10-16 23:03:09.901779 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "0833cb54870f" +down_revision: Union[str, None] = "e3ecd40e3851" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "wikibase_software", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "software_type", + sa.Enum( + "SOFTWARE", "SKIN", "EXTENSION", "LIBRARY", name="wikibasesoftwaretype" + ), + nullable=False, + ), + sa.Column("software_name", sa.String(), nullable=False), + sa.Column("url", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("latest_version", sa.String(), nullable=True), + sa.Column("quarterly_download_count", sa.Integer(), nullable=True), + sa.Column("public_wiki_count", sa.Integer(), nullable=True), + sa.Column("mw_bundled", sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "software_type", "software_name", name="unique_software_type_name" + ), + ) + + op.create_table( + "wikibase_software_tag", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("tag", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + + op.create_table( + "wikibase_software_tag_xref", + sa.Column("wikibase_software_id", sa.Integer(), nullable=False), + sa.Column("wikibase_software_tag_id", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["wikibase_software_id"], + ["wikibase_software.id"], + name="foreign_wikibase_software_id", + ), + sa.ForeignKeyConstraint( + ["wikibase_software_tag_id"], + ["wikibase_software_tag.id"], + name="foreign_wikibase_software_tag_id", + ), + sa.PrimaryKeyConstraint("wikibase_software_id", "wikibase_software_tag_id"), + ) + + with op.batch_alter_table("wikibase_software_version") as batch_op: + batch_op.add_column( + sa.Column("wikibase_software_id", sa.Integer(), nullable=True) + ) + batch_op.create_foreign_key( + "software", "wikibase_software", ["wikibase_software_id"], ["id"] + ) + with op.get_bind() as conn: + conn.execute( + sa.text( + """INSERT INTO wikibase_software (software_type, software_name) +SELECT DISTINCT software_type, software_name +FROM wikibase_software_version +ORDER BY software_type, software_name""" + ) + ) + conn.execute( + sa.text( + """UPDATE wikibase_software_version +SET wikibase_software_id = wikibase_software.id +FROM wikibase_software +WHERE wikibase_software_version.software_name = wikibase_software.software_name AND wikibase_software_version.software_type = wikibase_software.software_type""" + ) + ) + conn.commit() + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_software_version") as batch_op: + batch_op.drop_constraint("software", type_="foreignkey") + batch_op.drop_column("wikibase_software_id") + op.drop_table("wikibase_software_tag_xref") + op.drop_table("wikibase_software_tag") + op.drop_table("wikibase_software") + # ### end Alembic commands ### diff --git a/alembic/versions/3b5538b2faec_fetched.py b/alembic/versions/3b5538b2faec_fetched.py new file mode 100644 index 00000000..e49d4341 --- /dev/null +++ b/alembic/versions/3b5538b2faec_fetched.py @@ -0,0 +1,34 @@ +"""Fetched + +Revision ID: 3b5538b2faec +Revises: 7fd538ca8735 +Create Date: 2024-10-17 12:36:32.239393 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "3b5538b2faec" +down_revision: Union[str, None] = "7fd538ca8735" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "wikibase_software", + sa.Column("fetched", sa.DateTime(timezone=True), nullable=True), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("wikibase_software", "fetched") + # ### end Alembic commands ### diff --git a/alembic/versions/7fd538ca8735_no_name_or_type.py b/alembic/versions/7fd538ca8735_no_name_or_type.py new file mode 100644 index 00000000..937aabc6 --- /dev/null +++ b/alembic/versions/7fd538ca8735_no_name_or_type.py @@ -0,0 +1,53 @@ +"""No Name or Type + +Revision ID: 7fd538ca8735 +Revises: bf635fa3a7ce +Create Date: 2024-10-17 00:13:58.311963 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "7fd538ca8735" +down_revision: Union[str, None] = "bf635fa3a7ce" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_software_version") as batch_op: + batch_op.drop_constraint( + "unique_observation_software_type_name", type_="unique" + ) + batch_op.create_unique_constraint( + "unique_observation_software_id", + ["wikibase_software_version_observation_id", "wikibase_software_id"], + ) + batch_op.drop_column("software_type") + batch_op.drop_column("software_name") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_software_version") as batch_op: + batch_op.add_column(sa.Column("software_name", sa.VARCHAR(), nullable=False)) + batch_op.add_column( + sa.Column("software_type", sa.VARCHAR(length=9), nullable=False) + ) + batch_op.drop_constraint("unique_observation_software_id", type_="unique") + batch_op.create_unique_constraint( + "unique_observation_software_type_name", + [ + "wikibase_software_version_observation_id", + "software_type", + "software_name", + ], + ) + # ### end Alembic commands ### diff --git a/alembic/versions/bf635fa3a7ce_mandatory_software_relationship.py b/alembic/versions/bf635fa3a7ce_mandatory_software_relationship.py new file mode 100644 index 00000000..79522c70 --- /dev/null +++ b/alembic/versions/bf635fa3a7ce_mandatory_software_relationship.py @@ -0,0 +1,37 @@ +"""Mandatory Software Relationship + +Revision ID: bf635fa3a7ce +Revises: 0833cb54870f +Create Date: 2024-10-16 23:57:48.573469 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "bf635fa3a7ce" +down_revision: Union[str, None] = "0833cb54870f" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_software_version") as batch_op: + batch_op.alter_column( + "wikibase_software_id", existing_type=sa.INTEGER(), nullable=False + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_software_version") as batch_op: + batch_op.alter_column( + "wikibase_software_id", existing_type=sa.INTEGER(), nullable=True + ) + # ### end Alembic commands ### diff --git a/alembic/versions/ced691eaf66e_archived.py b/alembic/versions/ced691eaf66e_archived.py new file mode 100644 index 00000000..04907b78 --- /dev/null +++ b/alembic/versions/ced691eaf66e_archived.py @@ -0,0 +1,37 @@ +"""Archived + +Revision ID: ced691eaf66e +Revises: 3b5538b2faec +Create Date: 2024-10-17 16:19:32.355862 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "ced691eaf66e" +down_revision: Union[str, None] = "3b5538b2faec" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_software") as batch_op: + batch_op.add_column(sa.Column("archived", sa.Boolean(), nullable=True)) + with op.batch_alter_table("wikibase_software_tag") as batch_op: + batch_op.create_unique_constraint("unique_tag", ["tag"]) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("wikibase_software_tag") as batch_op: + batch_op.drop_constraint("unique_tag", type_="unique") + with op.batch_alter_table("wikibase_software") as batch_op: + batch_op.drop_column("archived") + # ### end Alembic commands ### diff --git a/data/database_connection.py b/data/database_connection.py index f2fc521c..c659df3b 100644 --- a/data/database_connection.py +++ b/data/database_connection.py @@ -1,7 +1,7 @@ """Database Connection""" +from collections.abc import AsyncGenerator from contextlib import asynccontextmanager -from typing import AsyncGenerator from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker diff --git a/data/gx/checkpoints/alembic_version_checkpoint b/data/gx/checkpoints/alembic_version_checkpoint.json similarity index 100% rename from data/gx/checkpoints/alembic_version_checkpoint rename to data/gx/checkpoints/alembic_version_checkpoint.json diff --git a/data/gx/checkpoints/wikibase_checkpoint b/data/gx/checkpoints/wikibase_checkpoint.json similarity index 100% rename from data/gx/checkpoints/wikibase_checkpoint rename to data/gx/checkpoints/wikibase_checkpoint.json diff --git a/data/gx/checkpoints/wikibase_connectivity_observation_checkpoint b/data/gx/checkpoints/wikibase_connectivity_observation_checkpoint.json similarity index 100% rename from data/gx/checkpoints/wikibase_connectivity_observation_checkpoint rename to data/gx/checkpoints/wikibase_connectivity_observation_checkpoint.json diff --git a/data/gx/checkpoints/wikibase_log_observation_checkpoint b/data/gx/checkpoints/wikibase_log_observation_checkpoint.json similarity index 100% rename from data/gx/checkpoints/wikibase_log_observation_checkpoint rename to data/gx/checkpoints/wikibase_log_observation_checkpoint.json diff --git a/data/gx/checkpoints/wikibase_property_usage_observation_checkpoint b/data/gx/checkpoints/wikibase_property_usage_observation_checkpoint.json similarity index 100% rename from data/gx/checkpoints/wikibase_property_usage_observation_checkpoint rename to data/gx/checkpoints/wikibase_property_usage_observation_checkpoint.json diff --git a/data/gx/checkpoints/wikibase_quantity_observation_checkpoint b/data/gx/checkpoints/wikibase_quantity_observation_checkpoint.json similarity index 100% rename from data/gx/checkpoints/wikibase_quantity_observation_checkpoint rename to data/gx/checkpoints/wikibase_quantity_observation_checkpoint.json diff --git a/data/gx/checkpoints/wikibase_software_checkpoint.json b/data/gx/checkpoints/wikibase_software_checkpoint.json new file mode 100644 index 00000000..82fc4d41 --- /dev/null +++ b/data/gx/checkpoints/wikibase_software_checkpoint.json @@ -0,0 +1,30 @@ +{ + "actions": [ + { + "name": "update_all_data_docs", + "site_names": [], + "type": "update_data_docs" + } + ], + "id": "ff6c1445-1da6-4b3e-af20-6947ce253bd2", + "name": "wikibase_software_checkpoint", + "result_format": { "result_format": "COMPLETE" }, + "validation_definitions": [ + { + "id": "76869a4a-b823-4ccd-b091-2e4dff6befee", + "name": "wikibase_software_validation_definition" + }, + { + "id": "fa8b2a52-4651-4bec-9e42-7af94e5faca7", + "name": "wikibase_software_tag_validation_definition" + }, + { + "id": "0fbc7263-c0d9-4390-884d-445bf59a0647", + "name": "wikibase_software_tag_xref_validation_definition" + }, + { + "id": "8768e1d1-11d5-44a6-8f5d-0a03d1d535d4", + "name": "populated_wikibase_software_validation_definition" + } + ] +} diff --git a/data/gx/checkpoints/wikibase_software_version_observation_checkpoint b/data/gx/checkpoints/wikibase_software_version_observation_checkpoint.json similarity index 100% rename from data/gx/checkpoints/wikibase_software_version_observation_checkpoint rename to data/gx/checkpoints/wikibase_software_version_observation_checkpoint.json diff --git a/data/gx/checkpoints/wikibase_statistics_observation_checkpoint b/data/gx/checkpoints/wikibase_statistics_observation_checkpoint deleted file mode 100644 index e816e1a9..00000000 --- a/data/gx/checkpoints/wikibase_statistics_observation_checkpoint +++ /dev/null @@ -1,28 +0,0 @@ -{ - "actions": [ - { - "name": "update_all_data_docs", - "site_names": [], - "type": "update_data_docs" - } - ], - "id": "921797b3-b57f-47e2-bb02-e178b4e06cb5", - "name": "wikibase_statistics_observation_checkpoint", - "result_format": { - "result_format": "COMPLETE" - }, - "validation_definitions": [ - { - "id": "7212ff2d-cc83-493a-a983-1c69dfff7c12", - "name": "obs_wikibase_statistics_observation_validation_definition" - }, - { - "id": "9ea78ef4-0641-485e-a972-219b43b24e88", - "name": "wikibase_statistics_observation_validation_definition" - }, - { - "id": "d8fba0be-ef0d-472e-823d-a1da6c7ff97a", - "name": "valid_wikibase_statistics_observation_validation_definition" - } - ] -} \ No newline at end of file diff --git a/data/gx/checkpoints/wikibase_statistics_observation_checkpoint.json b/data/gx/checkpoints/wikibase_statistics_observation_checkpoint.json new file mode 100644 index 00000000..a6dbff86 --- /dev/null +++ b/data/gx/checkpoints/wikibase_statistics_observation_checkpoint.json @@ -0,0 +1,26 @@ +{ + "actions": [ + { + "name": "update_all_data_docs", + "site_names": [], + "type": "update_data_docs" + } + ], + "id": "921797b3-b57f-47e2-bb02-e178b4e06cb5", + "name": "wikibase_statistics_observation_checkpoint", + "result_format": { "result_format": "COMPLETE" }, + "validation_definitions": [ + { + "id": "7212ff2d-cc83-493a-a983-1c69dfff7c12", + "name": "obs_wikibase_statistics_observation_validation_definition" + }, + { + "id": "9ea78ef4-0641-485e-a972-219b43b24e88", + "name": "wikibase_statistics_observation_validation_definition" + }, + { + "id": "d8fba0be-ef0d-472e-823d-a1da6c7ff97a", + "name": "valid_wikibase_statistics_observation_validation_definition" + } + ] +} diff --git a/data/gx/checkpoints/wikibase_user_observation_checkpoint b/data/gx/checkpoints/wikibase_user_observation_checkpoint.json similarity index 100% rename from data/gx/checkpoints/wikibase_user_observation_checkpoint rename to data/gx/checkpoints/wikibase_user_observation_checkpoint.json diff --git a/data/gx/expectations/populated_wikibase_software_expetation_suite.json b/data/gx/expectations/populated_wikibase_software_expetation_suite.json new file mode 100644 index 00000000..63a56b62 --- /dev/null +++ b/data/gx/expectations/populated_wikibase_software_expetation_suite.json @@ -0,0 +1,158 @@ +{ + "expectations": [ + { + "id": "5186d0bf-cd58-4e83-8ac9-cf1cc2274385", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "2436ef29-36c5-4559-93da-8b926017a00a", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "6ca95d62-9eeb-4435-9817-21a007dd2368", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_values_to_be_unique" + }, + { + "id": "a04ae020-0d60-4a80-ae92-5ed52885870b", + "kwargs": { "column": "software_type" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "17ac7195-315c-431f-b328-5d758113a7d3", + "kwargs": { "column": "software_type" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "da8f1e6a-66e2-41fd-9b5f-6fdd158a4717", + "kwargs": { "column": "software_name" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "32acaecc-fa3b-4cfb-a5c9-aca71ad2b345", + "kwargs": { "column": "software_name" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "f316d5dd-3974-4007-9865-2fbe51edcd11", + "kwargs": { "column_list": ["software_type", "software_name"] }, + "meta": {}, + "type": "expect_compound_columns_to_be_unique" + }, + { + "id": "2bf6c7a8-44f9-4566-9e57-f0459ab758f4", + "kwargs": { "column": "url" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "be18048d-ff1a-4d83-9389-2b4be7cd9b90", + "kwargs": { "column": "url" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "fc428b2f-bfbd-4501-9058-fb8899685639", + "kwargs": { "column": "url" }, + "meta": {}, + "type": "expect_column_values_to_be_unique" + }, + { + "id": "9182c8f3-080e-43fb-bb24-f25c352ec709", + "kwargs": { "column": "description" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "1eab6823-0c11-4606-b98a-24cfddd0fa21", + "kwargs": { "column": "description", "mostly": 0.7 }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "0e19eae2-39aa-4b09-bc33-444ba2475d6b", + "kwargs": { "column": "latest_version" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "0f0a53ab-916c-4a52-b6bf-dd787bfe7c71", + "kwargs": { "column": "latest_version", "mostly": 0.6 }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "5da516dc-5044-42ec-9d62-33f897d20f30", + "kwargs": { "column": "quarterly_download_count" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "479da3dc-7d7d-41f1-88ac-aaf89a7fece1", + "kwargs": { "column": "quarterly_download_count", "mostly": 0.25 }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "9862d304-019a-432b-bcaa-8fe914225c5a", + "kwargs": { "column": "public_wiki_count" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "c9aaffa5-b706-463e-b00d-02ba8ccf310c", + "kwargs": { "column": "public_wiki_count", "mostly": 0.25 }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "e0e7e43b-9df0-4f3c-9e95-ea6813f45ddd", + "kwargs": { "column": "mw_bundled" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "677459aa-3f9c-46c4-9d79-e711adeb2f88", + "kwargs": { "column": "mw_bundled", "mostly": 0.75 }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "4fc45491-e0b9-4a74-bf70-9886614821ba", + "kwargs": { "column": "fetched" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "ef8dc31f-5b13-4abe-8b97-d9ecc54332fa", + "kwargs": { "column": "fetched" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "bfd95bd5-0756-4133-8d4a-8ccd58ca5107", + "kwargs": { "column": "archived" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "bf643ee9-b0eb-4500-92ea-c49d22adccf6", + "kwargs": { "column": "archived", "mostly": 0.75 }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + } + ], + "id": "a4fe5870-c78d-433d-af64-76b02603e528", + "meta": { "great_expectations_version": "1.2.0" }, + "name": "populated_wikibase_software_expetation_suite", + "notes": null +} diff --git a/data/gx/expectations/wikibase_expectation_suite.json b/data/gx/expectations/wikibase_expectation_suite.json index 500a17ec..4c99be5f 100644 --- a/data/gx/expectations/wikibase_expectation_suite.json +++ b/data/gx/expectations/wikibase_expectation_suite.json @@ -34,7 +34,12 @@ "id": "5342a046-55a7-4a61-908e-c461cd4a84bf", "kwargs": { "column": "wikibase_name", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -49,7 +54,12 @@ "id": "d4ffc143-e54a-4ad9-af88-b5c38eee60a4", "kwargs": { "column": "organization", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -64,7 +74,12 @@ "id": "43da03cf-e685-4bbd-979c-ffd8e16aeed7", "kwargs": { "column": "country", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -79,7 +94,12 @@ "id": "9ce6b575-0119-4dac-8d65-d1a210e6f423", "kwargs": { "column": "region", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -136,7 +156,12 @@ "id": "ca289730-c632-4991-81c5-f31eef1cee75", "kwargs": { "column": "description", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json b/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json index 07b52a34..bb2abd90 100644 --- a/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json +++ b/data/gx/expectations/wikibase_property_usage_count_expectation_suite.json @@ -46,7 +46,12 @@ "id": "77b293b1-73c2-437a-a035-3adfb66294bd", "kwargs": { "column": "property_url", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/data/gx/expectations/wikibase_software_expectation_suite.json b/data/gx/expectations/wikibase_software_expectation_suite.json new file mode 100644 index 00000000..d1fdffa9 --- /dev/null +++ b/data/gx/expectations/wikibase_software_expectation_suite.json @@ -0,0 +1,181 @@ +{ + "expectations": [ + { + "id": "e388d561-3614-4994-b20b-a1d35702c1c1", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "296c30ba-224d-4563-a052-9c210044b78a", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "49df4432-ef6f-4e00-a415-c7f15bebf3f6", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_values_to_be_unique" + }, + { + "id": "e6289c50-751a-4a0c-a9f5-9ac59402c297", + "kwargs": { "column": "software_type" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "1016719e-6730-47e6-93c8-d2ad8d27abfd", + "kwargs": { "column": "software_type" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "f1829a39-90d6-4b1d-bc3b-8c7c16775258", + "kwargs": { + "column": "software_type", + "value_set": ["SOFTWARE", "SKIN", "EXTENSION", "LIBRARY"] + }, + "meta": {}, + "type": "expect_column_distinct_values_to_be_in_set" + }, + { + "id": "a4985482-eaa0-47ae-876b-f87a80f9fb2e", + "kwargs": { "column": "software_name" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "0c1a773f-7e7f-4ad2-8faa-c3c8541abf3f", + "kwargs": { "column": "software_name" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "5c029502-61a0-4af2-b963-7754ea0ebf75", + "kwargs": { + "column": "software_name", + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] + }, + "meta": {}, + "type": "expect_column_values_to_not_match_regex_list" + }, + { + "id": "f3c3a1c5-06d5-46a7-bc2e-40ebdee39146", + "kwargs": { "column_list": ["software_type", "software_name"] }, + "meta": {}, + "type": "expect_compound_columns_to_be_unique" + }, + { + "id": "1ea89a08-334a-4e8a-a75a-c4c6bec61b92", + "kwargs": { "column": "url" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "fb215a29-24be-46ff-936e-5c40d59b386a", + "kwargs": { + "column": "url", + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] + }, + "meta": {}, + "type": "expect_column_values_to_not_match_regex_list" + }, + { + "id": "22e597ee-01d9-4486-bebe-b2e977a10f0a", + "kwargs": { "column": "description" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "aeaf5287-bbae-4a3b-9a2f-761b7e052629", + "kwargs": { + "column": "description", + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] + }, + "meta": {}, + "type": "expect_column_values_to_not_match_regex_list" + }, + { + "id": "e7002ac0-cd25-4a4c-ad65-d56ae2c894fd", + "kwargs": { "column": "latest_version" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "54d43eee-b079-4448-a90c-2162aa5dfc71", + "kwargs": { + "column": "latest_version", + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] + }, + "meta": {}, + "type": "expect_column_values_to_not_match_regex_list" + }, + { + "id": "4053d987-b652-4211-9cf4-10ec31958c33", + "kwargs": { "column": "quarterly_download_count" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "ed02306d-f11f-40b5-b85f-54ac7fbc1181", + "kwargs": { "column": "public_wiki_count" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "4ade6cb9-8dc3-4c93-b3cc-93b1163201a5", + "kwargs": { "column": "mw_bundled" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "7602df3c-485c-401d-8cb2-b703c511a8e1", + "kwargs": { "column": "mw_bundled", "value_set": [false, true] }, + "meta": {}, + "type": "expect_column_values_to_be_in_set" + }, + { + "id": "a14ce5ac-562d-405d-85bc-7da4a5e07888", + "kwargs": { "column": "fetched" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "fc19a549-a4ef-4aba-847b-f0c3efed2b77", + "kwargs": { "column": "archived" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "23a2d72e-5309-49c1-af1c-e495022580ca", + "kwargs": { "column": "archived", "value_set": [false, true] }, + "meta": {}, + "type": "expect_column_values_to_be_in_set" + } + ], + "id": "072a1275-09e6-4835-85f7-8f55e99dcc3e", + "meta": { "great_expectations_version": "1.1.3" }, + "name": "wikibase_software_expectation_suite", + "notes": null +} diff --git a/data/gx/expectations/wikibase_software_tag_expetation_suite.json b/data/gx/expectations/wikibase_software_tag_expetation_suite.json new file mode 100644 index 00000000..fe7ffa90 --- /dev/null +++ b/data/gx/expectations/wikibase_software_tag_expetation_suite.json @@ -0,0 +1,58 @@ +{ + "expectations": [ + { + "id": "6d2611be-976c-436c-955f-443992875712", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "24d9c05f-8a84-4aa8-bcba-89e4d1c78cb1", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "d858a203-8efe-4b41-b4fc-b9fe62cee442", + "kwargs": { "column": "id" }, + "meta": {}, + "type": "expect_column_values_to_be_unique" + }, + { + "id": "2747ff12-3259-4071-825d-21330e6db6fa", + "kwargs": { "column": "tag" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "c322e1c0-b923-4dd7-95e8-464d2b05edd0", + "kwargs": { "column": "tag" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "5d4645fd-e469-43da-a7f5-ab7e4222fdf7", + "kwargs": { "column": "tag" }, + "meta": {}, + "type": "expect_column_values_to_be_unique" + }, + { + "id": "5baeb31e-aa99-46b2-bd6e-035155afcbfc", + "kwargs": { + "column": "tag", + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] + }, + "meta": {}, + "type": "expect_column_values_to_not_match_regex_list" + } + ], + "id": "920eb0d9-1590-45e4-ad9b-29c41fdcfce0", + "meta": { "great_expectations_version": "1.1.3" }, + "name": "wikibase_software_tag_expetation_suite", + "notes": null +} diff --git a/data/gx/expectations/wikibase_software_tag_xref_expetation_suite.json b/data/gx/expectations/wikibase_software_tag_xref_expetation_suite.json new file mode 100644 index 00000000..f264161f --- /dev/null +++ b/data/gx/expectations/wikibase_software_tag_xref_expetation_suite.json @@ -0,0 +1,50 @@ +{ + "expectations": [ + { + "id": "acdb75b6-decb-444e-8a79-ea85b911cb1c", + "kwargs": { + "column": "wikibase_software_id" + }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "2c8375c1-d080-498f-8705-088b75976607", + "kwargs": { + "column": "wikibase_software_id" + }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "0d2c8f51-e24b-4d38-be35-6605062c0a54", + "kwargs": { + "column": "wikibase_software_tag_id" + }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "86d99958-557a-41fe-a78f-761397525dc2", + "kwargs": { + "column": "wikibase_software_tag_id" + }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "96879ea2-1933-401b-be26-55b1a3517013", + "kwargs": { + "column_list": ["wikibase_software_id", "wikibase_software_tag_id"] + }, + "meta": {}, + "type": "expect_compound_columns_to_be_unique" + } + ], + "id": "e9bf1186-f8b1-4f4a-835f-14e6d1ee6bb5", + "meta": { + "great_expectations_version": "1.1.3" + }, + "name": "wikibase_software_tag_xref_expetation_suite", + "notes": null +} diff --git a/data/gx/expectations/wikibase_software_version_expectation_suite.json b/data/gx/expectations/wikibase_software_version_expectation_suite.json index f8f894f0..d3a3fc24 100644 --- a/data/gx/expectations/wikibase_software_version_expectation_suite.json +++ b/data/gx/expectations/wikibase_software_version_expectation_suite.json @@ -30,60 +30,6 @@ "meta": {}, "type": "expect_column_values_to_not_be_null" }, - { - "id": "c4616471-93e0-4df0-abbe-9bba1619592f", - "kwargs": { "column": "software_type" }, - "meta": {}, - "type": "expect_column_to_exist" - }, - { - "id": "a96e0d04-a0aa-4b24-b27a-c006afcb2012", - "kwargs": { "column": "software_type" }, - "meta": {}, - "type": "expect_column_values_to_not_be_null" - }, - { - "id": "3fa3ac0b-0a7e-4132-81bd-3e998f35be3b", - "kwargs": { - "column": "software_type", - "value_set": ["SOFTWARE", "SKIN", "EXTENSION", "LIBRARY"] - }, - "meta": {}, - "type": "expect_column_distinct_values_to_be_in_set" - }, - { - "id": "e5cc3f78-9cd7-4087-a880-b2c9616811d2", - "kwargs": { "column": "software_name" }, - "meta": {}, - "type": "expect_column_to_exist" - }, - { - "id": "daef9801-92bb-41af-a081-6d6ba2c239de", - "kwargs": { "column": "software_name" }, - "meta": {}, - "type": "expect_column_values_to_not_be_null" - }, - { - "id": "b852e073-ad49-4028-a42a-217448fde29b", - "kwargs": { - "column": "software_name", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] - }, - "meta": {}, - "type": "expect_column_values_to_not_match_regex_list" - }, - { - "id": "1a591246-4126-4545-8a35-74960762bca8", - "kwargs": { - "column_list": [ - "wikibase_software_version_observation_id", - "software_type", - "software_name" - ] - }, - "meta": {}, - "type": "expect_compound_columns_to_be_unique" - }, { "id": "755e9f43-a29d-4cd7-a290-6c302d4f0fb9", "kwargs": { "column": "version" }, @@ -94,7 +40,12 @@ "id": "78894d21-05f1-470d-931b-3efe46132d2e", "kwargs": { "column": "version", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" @@ -116,6 +67,29 @@ "kwargs": { "column": "version_date" }, "meta": {}, "type": "expect_column_to_exist" + }, + { + "id": "db3ceb44-e8bc-4bb6-ad37-64ba3b6c87d3", + "kwargs": { "column": "wikibase_software_id" }, + "meta": {}, + "type": "expect_column_to_exist" + }, + { + "id": "9277e198-b784-47cd-88a2-576d547a99b9", + "kwargs": { "column": "wikibase_software_id" }, + "meta": {}, + "type": "expect_column_values_to_not_be_null" + }, + { + "id": "e7f9a0eb-32f5-4f89-a3b7-3782afe81433", + "kwargs": { + "column_list": [ + "wikibase_software_id", + "wikibase_software_version_observation_id" + ] + }, + "meta": {}, + "type": "expect_compound_columns_to_be_unique" } ], "id": "dc5cb675-5725-46cb-940d-df7a04a56257", diff --git a/data/gx/expectations/wikibase_url_expectation_suite.json b/data/gx/expectations/wikibase_url_expectation_suite.json index 5565f56b..b2052dfa 100644 --- a/data/gx/expectations/wikibase_url_expectation_suite.json +++ b/data/gx/expectations/wikibase_url_expectation_suite.json @@ -81,7 +81,12 @@ "id": "31ae4f4f-55f0-4ac7-8e0d-3f17fe4fe823", "kwargs": { "column": "url", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/data/gx/expectations/wikibase_user_group_expectation_suite.json b/data/gx/expectations/wikibase_user_group_expectation_suite.json index c0c2ce88..fe9883a7 100644 --- a/data/gx/expectations/wikibase_user_group_expectation_suite.json +++ b/data/gx/expectations/wikibase_user_group_expectation_suite.json @@ -40,7 +40,12 @@ "id": "67a3cb18-3559-471d-ba6c-ef4385a9b96b", "kwargs": { "column": "group_name", - "regex_list": ["^[ \t\r\n]*$", "^[ \t\r\n]+", "[ \t\r\n]+$"] + "regex_list": [ + "^[ \t\r\n]*$", + "^[ \t\r\n]+", + "[ \t\r\n]+$", + "[ \t\r\n]{2,}" + ] }, "meta": {}, "type": "expect_column_values_to_not_match_regex_list" diff --git a/data/gx/great_expectations.yml b/data/gx/great_expectations.yml index f04b6c67..e2977c13 100644 --- a/data/gx/great_expectations.yml +++ b/data/gx/great_expectations.yml @@ -360,5 +360,44 @@ fluent_datasources: id: 1ae4e957-b4b4-42fc-b22a-381ea6e8bceb partitioner: query: SELECT * FROM wikibase_log_observation_month WHERE log_count > 0 + wikibase_software_table: + type: table + id: c93da63e-392d-443c-879e-902c2692b41e + batch_metadata: {} + batch_definitions: + FULL_TABLE: + id: 550b72d7-9aca-40cc-9d2a-7f94fcb44828 + partitioner: + table_name: wikibase_software + schema_name: + wikibase_software_tag_table: + type: table + id: d27a75dc-0e2e-43b2-91da-a1f2e9e0590a + batch_metadata: {} + batch_definitions: + FULL_TABLE: + id: 3b69d250-8f9a-4d11-b5a2-82463d829103 + partitioner: + table_name: wikibase_software_tag + schema_name: + wikibase_software_tag_xref_table: + type: table + id: 7f899383-9ce6-4afa-8dbb-0e49c30f3fd5 + batch_metadata: {} + batch_definitions: + FULL_TABLE: + id: ce729967-02a8-4e8f-b43a-c0d4ac078e44 + partitioner: + table_name: wikibase_software_tag_xref + schema_name: + populated_wikibase_software_records: + type: query + id: 97690d4d-e26a-4c89-807c-98e2a760e648 + batch_metadata: {} + batch_definitions: + FULL_TABLE: + id: 17906408-5608-45ff-95fe-45ae29b85630 + partitioner: + query: SELECT * FROM wikibase_software WHERE url IS NOT NULL connection_string: sqlite:///data/wikibase-data.db data_context_id: 14fd50bc-47c8-4033-9643-e60c471dc6c5 diff --git a/data/gx/validation_definitions/alembic_version_validation_definition b/data/gx/validation_definitions/alembic_version_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/alembic_version_validation_definition rename to data/gx/validation_definitions/alembic_version_validation_definition.json diff --git a/data/gx/validation_definitions/nonzero_wikibase_connectivity_observation_validation_definition b/data/gx/validation_definitions/nonzero_wikibase_connectivity_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/nonzero_wikibase_connectivity_observation_validation_definition rename to data/gx/validation_definitions/nonzero_wikibase_connectivity_observation_validation_definition.json diff --git a/data/gx/validation_definitions/nonzero_wikibase_log_observation_month_validation_definition b/data/gx/validation_definitions/nonzero_wikibase_log_observation_month_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/nonzero_wikibase_log_observation_month_validation_definition rename to data/gx/validation_definitions/nonzero_wikibase_log_observation_month_validation_definition.json diff --git a/data/gx/validation_definitions/obs_wikibase_connectivity_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_connectivity_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/obs_wikibase_connectivity_observation_validation_definition rename to data/gx/validation_definitions/obs_wikibase_connectivity_observation_validation_definition.json diff --git a/data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition rename to data/gx/validation_definitions/obs_wikibase_log_observation_validation_definition.json diff --git a/data/gx/validation_definitions/obs_wikibase_property_usage_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_property_usage_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/obs_wikibase_property_usage_observation_validation_definition rename to data/gx/validation_definitions/obs_wikibase_property_usage_observation_validation_definition.json diff --git a/data/gx/validation_definitions/obs_wikibase_quantity_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_quantity_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/obs_wikibase_quantity_observation_validation_definition rename to data/gx/validation_definitions/obs_wikibase_quantity_observation_validation_definition.json diff --git a/data/gx/validation_definitions/obs_wikibase_software_version_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_software_version_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/obs_wikibase_software_version_observation_validation_definition rename to data/gx/validation_definitions/obs_wikibase_software_version_observation_validation_definition.json diff --git a/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition deleted file mode 100644 index a6e80860..00000000 --- a/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition +++ /dev/null @@ -1,22 +0,0 @@ -{ - "data": { - "asset": { - "id": "05cc691f-d972-4cdd-8172-86dc600cf18e", - "name": "wikibase_statistics_observation_table" - }, - "batch_definition": { - "id": "96994a6a-aa72-4eb2-96b8-c285a7291b69", - "name": "FULL_TABLE" - }, - "datasource": { - "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", - "name": "wikibase_datasource" - } - }, - "id": "7212ff2d-cc83-493a-a983-1c69dfff7c12", - "name": "obs_wikibase_statistics_observation_validation_definition", - "suite": { - "id": "a5f967b2-e9e0-4b2c-8dd3-46acbbb2401a", - "name": "wikibase_observation_expectation_suite" - } -} \ No newline at end of file diff --git a/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition.json b/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition.json new file mode 100644 index 00000000..f1e67f91 --- /dev/null +++ b/data/gx/validation_definitions/obs_wikibase_statistics_observation_validation_definition.json @@ -0,0 +1,22 @@ +{ + "data": { + "asset": { + "id": "05cc691f-d972-4cdd-8172-86dc600cf18e", + "name": "wikibase_statistics_observation_table" + }, + "batch_definition": { + "id": "96994a6a-aa72-4eb2-96b8-c285a7291b69", + "name": "FULL_TABLE" + }, + "datasource": { + "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", + "name": "wikibase_datasource" + } + }, + "id": "7212ff2d-cc83-493a-a983-1c69dfff7c12", + "name": "obs_wikibase_statistics_observation_validation_definition", + "suite": { + "id": "a5f967b2-e9e0-4b2c-8dd3-46acbbb2401a", + "name": "wikibase_observation_expectation_suite" + } +} diff --git a/data/gx/validation_definitions/obs_wikibase_user_observation_validation_definition b/data/gx/validation_definitions/obs_wikibase_user_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/obs_wikibase_user_observation_validation_definition rename to data/gx/validation_definitions/obs_wikibase_user_observation_validation_definition.json diff --git a/data/gx/validation_definitions/populated_wikibase_software_validation_definition.json b/data/gx/validation_definitions/populated_wikibase_software_validation_definition.json new file mode 100644 index 00000000..447856bb --- /dev/null +++ b/data/gx/validation_definitions/populated_wikibase_software_validation_definition.json @@ -0,0 +1,22 @@ +{ + "data": { + "asset": { + "id": "97690d4d-e26a-4c89-807c-98e2a760e648", + "name": "populated_wikibase_software_records" + }, + "batch_definition": { + "id": "17906408-5608-45ff-95fe-45ae29b85630", + "name": "FULL_TABLE" + }, + "datasource": { + "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", + "name": "wikibase_datasource" + } + }, + "id": "8768e1d1-11d5-44a6-8f5d-0a03d1d535d4", + "name": "populated_wikibase_software_validation_definition", + "suite": { + "id": "a4fe5870-c78d-433d-af64-76b02603e528", + "name": "populated_wikibase_software_expetation_suite" + } +} diff --git a/data/gx/validation_definitions/valid_wikibase_connectivity_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_connectivity_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/valid_wikibase_connectivity_observation_validation_definition rename to data/gx/validation_definitions/valid_wikibase_connectivity_observation_validation_definition.json diff --git a/data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition rename to data/gx/validation_definitions/valid_wikibase_log_observation_validation_definition.json diff --git a/data/gx/validation_definitions/valid_wikibase_quantity_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_quantity_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/valid_wikibase_quantity_observation_validation_definition rename to data/gx/validation_definitions/valid_wikibase_quantity_observation_validation_definition.json diff --git a/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition deleted file mode 100644 index 23e93f9a..00000000 --- a/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition +++ /dev/null @@ -1,22 +0,0 @@ -{ - "data": { - "asset": { - "id": "d984b58f-385e-4939-aad2-d45f5432face", - "name": "valid_wikibase_statistics_observations" - }, - "batch_definition": { - "id": "5c3e8bee-2993-4a54-94df-0ea1dbfb945f", - "name": "FULL_TABLE" - }, - "datasource": { - "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", - "name": "wikibase_datasource" - } - }, - "id": "d8fba0be-ef0d-472e-823d-a1da6c7ff97a", - "name": "valid_wikibase_statistics_observation_validation_definition", - "suite": { - "id": "ae836f37-1896-4b7f-a50a-9e2906c9592d", - "name": "valid_wikibase_statistics_observation_expectation_suite" - } -} \ No newline at end of file diff --git a/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition.json b/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition.json new file mode 100644 index 00000000..74da8d93 --- /dev/null +++ b/data/gx/validation_definitions/valid_wikibase_statistics_observation_validation_definition.json @@ -0,0 +1,22 @@ +{ + "data": { + "asset": { + "id": "d984b58f-385e-4939-aad2-d45f5432face", + "name": "valid_wikibase_statistics_observations" + }, + "batch_definition": { + "id": "5c3e8bee-2993-4a54-94df-0ea1dbfb945f", + "name": "FULL_TABLE" + }, + "datasource": { + "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", + "name": "wikibase_datasource" + } + }, + "id": "d8fba0be-ef0d-472e-823d-a1da6c7ff97a", + "name": "valid_wikibase_statistics_observation_validation_definition", + "suite": { + "id": "ae836f37-1896-4b7f-a50a-9e2906c9592d", + "name": "valid_wikibase_statistics_observation_expectation_suite" + } +} diff --git a/data/gx/validation_definitions/valid_wikibase_user_observation_validation_definition b/data/gx/validation_definitions/valid_wikibase_user_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/valid_wikibase_user_observation_validation_definition rename to data/gx/validation_definitions/valid_wikibase_user_observation_validation_definition.json diff --git a/data/gx/validation_definitions/valid_wikibase_validation_definition b/data/gx/validation_definitions/valid_wikibase_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/valid_wikibase_validation_definition rename to data/gx/validation_definitions/valid_wikibase_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_category_validation_definition b/data/gx/validation_definitions/wikibase_category_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_category_validation_definition rename to data/gx/validation_definitions/wikibase_category_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_connectivity_observation_item_relationship_count_validation_definition b/data/gx/validation_definitions/wikibase_connectivity_observation_item_relationship_count_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_connectivity_observation_item_relationship_count_validation_definition rename to data/gx/validation_definitions/wikibase_connectivity_observation_item_relationship_count_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_connectivity_observation_object_relationship_count_validation_definition b/data/gx/validation_definitions/wikibase_connectivity_observation_object_relationship_count_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_connectivity_observation_object_relationship_count_validation_definition rename to data/gx/validation_definitions/wikibase_connectivity_observation_object_relationship_count_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_connectivity_observation_validation_definition b/data/gx/validation_definitions/wikibase_connectivity_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_connectivity_observation_validation_definition rename to data/gx/validation_definitions/wikibase_connectivity_observation_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_log_observation_month_type_validation_definition b/data/gx/validation_definitions/wikibase_log_observation_month_type_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_log_observation_month_type_validation_definition rename to data/gx/validation_definitions/wikibase_log_observation_month_type_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_log_observation_month_user_validation_definition b/data/gx/validation_definitions/wikibase_log_observation_month_user_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_log_observation_month_user_validation_definition rename to data/gx/validation_definitions/wikibase_log_observation_month_user_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_log_observation_month_validation_definition b/data/gx/validation_definitions/wikibase_log_observation_month_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_log_observation_month_validation_definition rename to data/gx/validation_definitions/wikibase_log_observation_month_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_log_observation_validation_definition b/data/gx/validation_definitions/wikibase_log_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_log_observation_validation_definition rename to data/gx/validation_definitions/wikibase_log_observation_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_property_usage_count_validation_definition b/data/gx/validation_definitions/wikibase_property_usage_count_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_property_usage_count_validation_definition rename to data/gx/validation_definitions/wikibase_property_usage_count_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_quantity_observation_validation_definition b/data/gx/validation_definitions/wikibase_quantity_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_quantity_observation_validation_definition rename to data/gx/validation_definitions/wikibase_quantity_observation_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_software_tag_validation_definition.json b/data/gx/validation_definitions/wikibase_software_tag_validation_definition.json new file mode 100644 index 00000000..217f5198 --- /dev/null +++ b/data/gx/validation_definitions/wikibase_software_tag_validation_definition.json @@ -0,0 +1,22 @@ +{ + "data": { + "asset": { + "id": "d27a75dc-0e2e-43b2-91da-a1f2e9e0590a", + "name": "wikibase_software_tag_table" + }, + "batch_definition": { + "id": "3b69d250-8f9a-4d11-b5a2-82463d829103", + "name": "FULL_TABLE" + }, + "datasource": { + "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", + "name": "wikibase_datasource" + } + }, + "id": "fa8b2a52-4651-4bec-9e42-7af94e5faca7", + "name": "wikibase_software_tag_validation_definition", + "suite": { + "id": "920eb0d9-1590-45e4-ad9b-29c41fdcfce0", + "name": "wikibase_software_tag_expetation_suite" + } +} diff --git a/data/gx/validation_definitions/wikibase_software_tag_xref_validation_definition.json b/data/gx/validation_definitions/wikibase_software_tag_xref_validation_definition.json new file mode 100644 index 00000000..20128db7 --- /dev/null +++ b/data/gx/validation_definitions/wikibase_software_tag_xref_validation_definition.json @@ -0,0 +1,22 @@ +{ + "data": { + "asset": { + "id": "7f899383-9ce6-4afa-8dbb-0e49c30f3fd5", + "name": "wikibase_software_tag_xref_table" + }, + "batch_definition": { + "id": "ce729967-02a8-4e8f-b43a-c0d4ac078e44", + "name": "FULL_TABLE" + }, + "datasource": { + "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", + "name": "wikibase_datasource" + } + }, + "id": "0fbc7263-c0d9-4390-884d-445bf59a0647", + "name": "wikibase_software_tag_xref_validation_definition", + "suite": { + "id": "e9bf1186-f8b1-4f4a-835f-14e6d1ee6bb5", + "name": "wikibase_software_tag_xref_expetation_suite" + } +} diff --git a/data/gx/validation_definitions/wikibase_software_validation_definition.json b/data/gx/validation_definitions/wikibase_software_validation_definition.json new file mode 100644 index 00000000..a1998e89 --- /dev/null +++ b/data/gx/validation_definitions/wikibase_software_validation_definition.json @@ -0,0 +1,22 @@ +{ + "data": { + "asset": { + "id": "c93da63e-392d-443c-879e-902c2692b41e", + "name": "wikibase_software_table" + }, + "batch_definition": { + "id": "550b72d7-9aca-40cc-9d2a-7f94fcb44828", + "name": "FULL_TABLE" + }, + "datasource": { + "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", + "name": "wikibase_datasource" + } + }, + "id": "76869a4a-b823-4ccd-b091-2e4dff6befee", + "name": "wikibase_software_validation_definition", + "suite": { + "id": "072a1275-09e6-4835-85f7-8f55e99dcc3e", + "name": "wikibase_software_expectation_suite" + } +} diff --git a/data/gx/validation_definitions/wikibase_software_version_validation_definition b/data/gx/validation_definitions/wikibase_software_version_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_software_version_validation_definition rename to data/gx/validation_definitions/wikibase_software_version_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition b/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition deleted file mode 100644 index 20caee4c..00000000 --- a/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition +++ /dev/null @@ -1,22 +0,0 @@ -{ - "data": { - "asset": { - "id": "05cc691f-d972-4cdd-8172-86dc600cf18e", - "name": "wikibase_statistics_observation_table" - }, - "batch_definition": { - "id": "96994a6a-aa72-4eb2-96b8-c285a7291b69", - "name": "FULL_TABLE" - }, - "datasource": { - "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", - "name": "wikibase_datasource" - } - }, - "id": "9ea78ef4-0641-485e-a972-219b43b24e88", - "name": "wikibase_statistics_observation_validation_definition", - "suite": { - "id": "3b148b36-c190-4dc0-8504-75d670d6203d", - "name": "wikibase_statistics_observation_expectation_suite" - } -} \ No newline at end of file diff --git a/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition.json b/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition.json new file mode 100644 index 00000000..0ecebd21 --- /dev/null +++ b/data/gx/validation_definitions/wikibase_statistics_observation_validation_definition.json @@ -0,0 +1,22 @@ +{ + "data": { + "asset": { + "id": "05cc691f-d972-4cdd-8172-86dc600cf18e", + "name": "wikibase_statistics_observation_table" + }, + "batch_definition": { + "id": "96994a6a-aa72-4eb2-96b8-c285a7291b69", + "name": "FULL_TABLE" + }, + "datasource": { + "id": "4a58b404-1ee3-4e91-a373-78d1b365f987", + "name": "wikibase_datasource" + } + }, + "id": "9ea78ef4-0641-485e-a972-219b43b24e88", + "name": "wikibase_statistics_observation_validation_definition", + "suite": { + "id": "3b148b36-c190-4dc0-8504-75d670d6203d", + "name": "wikibase_statistics_observation_expectation_suite" + } +} diff --git a/data/gx/validation_definitions/wikibase_url_validation_definition b/data/gx/validation_definitions/wikibase_url_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_url_validation_definition rename to data/gx/validation_definitions/wikibase_url_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_user_group_validation_definition b/data/gx/validation_definitions/wikibase_user_group_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_user_group_validation_definition rename to data/gx/validation_definitions/wikibase_user_group_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_user_observation_group_validation_definition b/data/gx/validation_definitions/wikibase_user_observation_group_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_user_observation_group_validation_definition rename to data/gx/validation_definitions/wikibase_user_observation_group_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_user_observation_validation_definition b/data/gx/validation_definitions/wikibase_user_observation_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_user_observation_validation_definition rename to data/gx/validation_definitions/wikibase_user_observation_validation_definition.json diff --git a/data/gx/validation_definitions/wikibase_validation_definition b/data/gx/validation_definitions/wikibase_validation_definition.json similarity index 100% rename from data/gx/validation_definitions/wikibase_validation_definition rename to data/gx/validation_definitions/wikibase_validation_definition.json diff --git a/data/wikibase-data.db b/data/wikibase-data.db index 94e302ac..202e9624 100644 Binary files a/data/wikibase-data.db and b/data/wikibase-data.db differ diff --git a/data/wikibase-test-data.db b/data/wikibase-test-data.db index 9117fa0d..0f9d6540 100644 Binary files a/data/wikibase-test-data.db and b/data/wikibase-test-data.db differ diff --git a/fetch_data/__init__.py b/fetch_data/__init__.py index 35b47b56..2aadbab2 100644 --- a/fetch_data/__init__.py +++ b/fetch_data/__init__.py @@ -9,4 +9,5 @@ from fetch_data.soup_data import ( create_software_version_observation, create_special_statistics_observation, + update_software_data, ) diff --git a/fetch_data/api_data/log_data/create_log_observation.py b/fetch_data/api_data/log_data/create_log_observation.py index b6dbc557..e3ffd0c1 100644 --- a/fetch_data/api_data/log_data/create_log_observation.py +++ b/fetch_data/api_data/log_data/create_log_observation.py @@ -1,8 +1,8 @@ """Create Log Observation""" +from collections.abc import Iterable from datetime import datetime from json.decoder import JSONDecodeError -from typing import List from requests.exceptions import ReadTimeout, SSLError from data import get_async_session from fetch_data.api_data.log_data.fetch_log_data import ( @@ -85,7 +85,7 @@ async def create_log_observation(wikibase_id: int) -> bool: async def create_log_month( - wikibase: WikibaseModel, log_list: List[WikibaseLogRecord] + wikibase: WikibaseModel, log_list: Iterable[WikibaseLogRecord] ) -> WikibaseLogMonthObservationModel: """Create Log Month""" diff --git a/fetch_data/api_data/log_data/fetch_log_data.py b/fetch_data/api_data/log_data/fetch_log_data.py index 27bc9830..5559b4a8 100644 --- a/fetch_data/api_data/log_data/fetch_log_data.py +++ b/fetch_data/api_data/log_data/fetch_log_data.py @@ -1,7 +1,7 @@ """Fetch Log Data""" from datetime import datetime -from typing import List, Optional +from typing import Optional from fetch_data.api_data.log_data.wikibase_log_record import WikibaseLogRecord from fetch_data.utils import dict_to_url, fetch_api_data @@ -24,10 +24,10 @@ def get_log_param_string( return dict_to_url(parameters) -def get_log_list_from_url(url: str) -> List[WikibaseLogRecord]: +def get_log_list_from_url(url: str) -> list[WikibaseLogRecord]: """Get Log List from URL""" - data = [] + data: list[WikibaseLogRecord] = [] query_data = fetch_api_data(url) for record in query_data["query"]["logevents"]: @@ -38,10 +38,10 @@ def get_log_list_from_url(url: str) -> List[WikibaseLogRecord]: def get_month_log_list( api_url: str, comparison_date: datetime, oldest: bool = False -) -> List[WikibaseLogRecord]: +) -> list[WikibaseLogRecord]: """Get Log List from api_url, limit to within 30 days of the comparison date""" - data: List[WikibaseLogRecord] = [] + data: list[WikibaseLogRecord] = [] limit = 500 should_query = True diff --git a/fetch_data/api_data/user_data/fetch_multiple_user_data.py b/fetch_data/api_data/user_data/fetch_multiple_user_data.py index 23305022..943ffb21 100644 --- a/fetch_data/api_data/user_data/fetch_multiple_user_data.py +++ b/fetch_data/api_data/user_data/fetch_multiple_user_data.py @@ -1,7 +1,7 @@ """Fetch Multiple User Data""" +from collections.abc import Iterable import json -from typing import Iterable import requests from fetch_data.api_data.user_data.user_data_url import user_url from model.database import WikibaseModel diff --git a/fetch_data/soup_data/__init__.py b/fetch_data/soup_data/__init__.py index d736a837..4c62f5c6 100644 --- a/fetch_data/soup_data/__init__.py +++ b/fetch_data/soup_data/__init__.py @@ -1,8 +1,9 @@ """Data from Parsing Webpages with Beautiful Soup""" -from fetch_data.soup_data.create_software_version_data_observation import ( - create_software_version_observation, -) from fetch_data.soup_data.create_statistics_data_observation import ( create_special_statistics_observation, ) +from fetch_data.soup_data.software import ( + create_software_version_observation, + update_software_data, +) diff --git a/fetch_data/soup_data/software/__init__.py b/fetch_data/soup_data/software/__init__.py new file mode 100644 index 00000000..974616f7 --- /dev/null +++ b/fetch_data/soup_data/software/__init__.py @@ -0,0 +1,6 @@ +"""Software Observations & Data""" + +from fetch_data.soup_data.software.create_software_version_data_observation import ( + create_software_version_observation, +) +from fetch_data.soup_data.software.get_update_software_data import update_software_data diff --git a/fetch_data/soup_data/create_software_version_data_observation.py b/fetch_data/soup_data/software/create_software_version_data_observation.py similarity index 71% rename from fetch_data/soup_data/create_software_version_data_observation.py rename to fetch_data/soup_data/software/create_software_version_data_observation.py index 6c6bcc5c..1db13db7 100644 --- a/fetch_data/soup_data/create_software_version_data_observation.py +++ b/fetch_data/soup_data/software/create_software_version_data_observation.py @@ -1,12 +1,18 @@ """Create Software Version Observation""" +from collections.abc import Iterable from datetime import datetime -from typing import List from urllib.error import HTTPError from bs4 import BeautifulSoup, Tag import requests from requests.exceptions import SSLError +from sqlalchemy.ext.asyncio import AsyncSession +import strawberry from data import get_async_session +from fetch_data.soup_data.software.get_software_model import ( + get_or_create_software_model, +) +from fetch_data.soup_data.software.get_update_software_data import update_software_data from fetch_data.utils import get_wikibase_from_database, parse_datetime from model.database import ( WikibaseModel, @@ -16,9 +22,13 @@ from model.enum import WikibaseSoftwareType -async def create_software_version_observation(wikibase_id: int) -> bool: +async def create_software_version_observation( + wikibase_id: int, info: strawberry.Info +) -> bool: """Create Software Version Observation""" + info.context["background_tasks"].add_task(update_software_data) + async with get_async_session() as async_session: wikibase: WikibaseModel = await get_wikibase_from_database( async_session=async_session, @@ -39,16 +49,18 @@ async def create_software_version_observation(wikibase_id: int) -> bool: observation.returned_data = True - installed_software_versions = compile_installed_software_versions(soup) + installed_software_versions = await compile_installed_software_versions( + async_session, soup + ) observation.software_versions.extend(installed_software_versions) - skin_versions = compile_skin_versions(soup) + skin_versions = await compile_skin_versions(async_session, soup) observation.software_versions.extend(skin_versions) - extensions_versions = compile_extension_versions(soup) + extensions_versions = await compile_extension_versions(async_session, soup) observation.software_versions.extend(extensions_versions) - library_versions = compile_library_versions(soup) + library_versions = await compile_library_versions(async_session, soup) observation.software_versions.extend(library_versions) except (HTTPError, SSLError): observation.returned_data = False @@ -59,9 +71,9 @@ async def create_software_version_observation(wikibase_id: int) -> bool: return observation.returned_data -def compile_extension_versions( - soup: BeautifulSoup, -) -> List[WikibaseSoftwareVersionModel]: +async def compile_extension_versions( + async_session: AsyncSession, soup: BeautifulSoup +) -> list[WikibaseSoftwareVersionModel]: """Compile Extension Version List""" extensions_table = soup.find( @@ -69,7 +81,9 @@ def compile_extension_versions( ) return unique_versions( [ - get_software_version_from_row(row, WikibaseSoftwareType.EXTENSION) + await get_software_version_from_row( + async_session, row, WikibaseSoftwareType.EXTENSION + ) for row in extensions_table.find_all( "tr", attrs={"class": "mw-version-ext"} ) @@ -77,14 +91,14 @@ def compile_extension_versions( ) -def compile_installed_software_versions( - soup: BeautifulSoup, -) -> List[WikibaseSoftwareVersionModel]: +async def compile_installed_software_versions( + async_session: AsyncSession, soup: BeautifulSoup +) -> list[WikibaseSoftwareVersionModel]: """Compile Installed Software Version List""" installed_software_table: Tag = soup.find("table", attrs={"id": "sv-software"}) - software_versions: List[WikibaseSoftwareVersionModel] = [] + software_versions: list[WikibaseSoftwareVersionModel] = [] row: Tag for row in installed_software_table.find_all("tr"): if row.find("td"): @@ -113,8 +127,9 @@ def compile_installed_software_versions( software_versions.append( WikibaseSoftwareVersionModel( - software_type=WikibaseSoftwareType.SOFTWARE, - software_name=software_name, + software=await get_or_create_software_model( + async_session, WikibaseSoftwareType.SOFTWARE, software_name + ), version=version, version_hash=version_hash, version_date=version_date, @@ -124,12 +139,14 @@ def compile_installed_software_versions( return software_versions -def compile_library_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionModel]: +async def compile_library_versions( + async_session: AsyncSession, soup: BeautifulSoup +) -> list[WikibaseSoftwareVersionModel]: """Compile Library Version List""" libraries_table = soup.find("table", attrs={"id": "sv-libraries"}) - library_versions: List[WikibaseSoftwareVersionModel] = [] + library_versions: list[WikibaseSoftwareVersionModel] = [] row: Tag for row in libraries_table.find_all("tr"): if row.find("td"): @@ -140,8 +157,9 @@ def compile_library_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersio version = row.find_all("td")[1].string library_versions.append( WikibaseSoftwareVersionModel( - software_type=WikibaseSoftwareType.LIBRARY, - software_name=software_name, + software=await get_or_create_software_model( + async_session, WikibaseSoftwareType.LIBRARY, software_name + ), version=version, ) ) @@ -149,7 +167,9 @@ def compile_library_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersio return unique_versions(library_versions) -def compile_skin_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionModel]: +async def compile_skin_versions( + async_session: AsyncSession, soup: BeautifulSoup +) -> list[WikibaseSoftwareVersionModel]: """Compile Skin Version List""" installed_skin_table: Tag = soup.find( @@ -157,7 +177,9 @@ def compile_skin_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionMo ) return unique_versions( [ - get_software_version_from_row(row, WikibaseSoftwareType.SKIN) + await get_software_version_from_row( + async_session, row, WikibaseSoftwareType.SKIN + ) for row in installed_skin_table.find_all( "tr", attrs={"class": "mw-version-ext"} ) @@ -165,8 +187,8 @@ def compile_skin_versions(soup: BeautifulSoup) -> List[WikibaseSoftwareVersionMo ) -def get_software_version_from_row( - row: Tag, software_type: WikibaseSoftwareType +async def get_software_version_from_row( + async_session: AsyncSession, row: Tag, software_type: WikibaseSoftwareType ) -> WikibaseSoftwareVersionModel: """Parse Software Version from Table Row""" @@ -205,8 +227,9 @@ def get_software_version_from_row( version_date = parse_datetime(date_tag.string) return WikibaseSoftwareVersionModel( - software_type=software_type, - software_name=software_name, + software=await get_or_create_software_model( + async_session, software_type, software_name + ), version=version, version_hash=version_hash, version_date=version_date, @@ -214,8 +237,8 @@ def get_software_version_from_row( def unique_versions( - input_list: List[WikibaseSoftwareVersionModel], -) -> List[WikibaseSoftwareVersionModel]: + input_list: Iterable[WikibaseSoftwareVersionModel], +) -> list[WikibaseSoftwareVersionModel]: """Unique Version List""" temp: dict[str, WikibaseSoftwareVersionModel] = {} diff --git a/fetch_data/soup_data/software/get_software_model.py b/fetch_data/soup_data/software/get_software_model.py new file mode 100644 index 00000000..3c14aab9 --- /dev/null +++ b/fetch_data/soup_data/software/get_software_model.py @@ -0,0 +1,87 @@ +"""Get or Create Software Model""" + +import re +from typing import Optional +from sqlalchemy import and_, select +from sqlalchemy.ext.asyncio import AsyncSession +from model.database import WikibaseSoftwareModel +from model.enum import WikibaseSoftwareType + +EXTENSIONNAME_PATTERN = r"⧼([a-z]+)-extensionname⧽" + + +async def get_or_create_software_model( + async_session: AsyncSession, software_type: WikibaseSoftwareType, software_name: str +) -> WikibaseSoftwareModel: + """Fetch or Create Software Model""" + + if re.match(EXTENSIONNAME_PATTERN, software_name): + software_name = re.sub(EXTENSIONNAME_PATTERN, r"\1", software_name) + + existing = await get_existing_software_model( + async_session, software_type, software_name + ) + if existing is not None: + return existing + + nearby = await get_nearby_software_model( + async_session, software_type, software_name + ) + if nearby is not None: + return nearby + + creating = WikibaseSoftwareModel( + software_type=software_type, software_name=software_name + ) + async_session.add(creating) + await async_session.flush() + await async_session.refresh(creating) + return creating + + +async def get_existing_software_model( + async_session: AsyncSession, software_type: WikibaseSoftwareType, software_name: str +) -> Optional[WikibaseSoftwareModel]: + """Return Existing Software, If Existent""" + + return ( + await async_session.scalars( + select(WikibaseSoftwareModel).where( + and_( + WikibaseSoftwareModel.software_type == software_type, + WikibaseSoftwareModel.software_name == software_name, + ) + ) + ) + ).one_or_none() + + +async def get_nearby_software_model( + async_session: AsyncSession, software_type: WikibaseSoftwareType, software_name: str +) -> Optional[WikibaseSoftwareModel]: + """ + Return Matching Software, if found + + Only allowable differences: whitespace and capitalization + + Assert only one or zero matches + """ + + all_software_of_type = ( + await async_session.scalars( + select(WikibaseSoftwareModel).where( + WikibaseSoftwareModel.software_type == software_type + ) + ) + ).all() + nearby: Optional[WikibaseSoftwareModel] = None + nearby_count = 0 + for s in all_software_of_type: + if ( + s.software_name.replace(" ", "").lower() + == software_name.replace(" ", "").lower() + ): + nearby = s + nearby_count += 1 + assert nearby_count <= 1 + return nearby diff --git a/fetch_data/soup_data/software/get_update_software_data.py b/fetch_data/soup_data/software/get_update_software_data.py new file mode 100644 index 00000000..6ed88c8e --- /dev/null +++ b/fetch_data/soup_data/software/get_update_software_data.py @@ -0,0 +1,232 @@ +"""Update Software Data""" + +from datetime import datetime, timedelta, timezone +import os +import re +from typing import Iterable, List, Optional +from bs4 import BeautifulSoup +import requests +from sqlalchemy import Select, and_, or_, select +from sqlalchemy.ext.asyncio import AsyncSession +from data import get_async_session +from model.database import WikibaseSoftwareModel, WikibaseSoftwareTagModel +from model.enum import WikibaseSoftwareType + + +async def update_software_data(): + """Fetch Software Info""" + + carry_on = True + while carry_on: + + async with get_async_session() as async_session: + os.makedirs("dump", exist_ok=True) + + unfound_extensions: Iterable[WikibaseSoftwareModel] = ( + await async_session.scalars(get_update_extension_query()) + ).all() + carry_on = len(unfound_extensions) > 0 + + for ext in unfound_extensions: + if ext.url is None: + ext.url = ( + f"https://www.mediawiki.org/wiki/Extension:{ext.software_name}" + ) + + await compile_data_from_url(async_session, ext) + await async_session.flush() + await async_session.commit() + + +async def compile_data_from_url( + async_session: AsyncSession, + ext: WikibaseSoftwareModel, + override_url: Optional[str] = None, + archived: bool = False, +): + """Compile Software Data from URL""" + + with requests.get( + override_url or ext.url, timeout=10, allow_redirects=True + ) as response: + + ext.data_fetched = datetime.now(timezone.utc) + print(f"{response.url}: {response.status_code}") + + if response.status_code == 200: + + if override_url is None and response.url != ext.url: + ext.url = response.url + + soup = BeautifulSoup(response.content, features="html.parser") + + page_archived = ( + soup.find("b", string="This extension has been archived.") is not None + ) + ext.archived = archived or page_archived + if page_archived: + permanent_link_tag = soup.find( + "a", string="To see the page before archival, click here." + ) + return await compile_data_from_url( + async_session, + ext, + override_url=f"https://www.mediawiki.org{permanent_link_tag['href']}", + archived=True, + ) + + ext.tags = await compile_tag_list(async_session, soup) + ext.description = compile_description(soup) + ext.latest_version = compile_latest_version(soup) + ext.quarterly_download_count = compile_quarterly_count(soup) + ext.public_wiki_count = compile_wiki_count(soup) + ext.mediawiki_bundled = compile_bundled(soup) + + +def get_update_extension_query() -> Select[WikibaseSoftwareModel]: + """Update Extension List Query""" + + return ( + select(WikibaseSoftwareModel) + .where( + and_( + or_( + # pylint: disable=singleton-comparison + WikibaseSoftwareModel.data_fetched == None, + WikibaseSoftwareModel.data_fetched + < (datetime.today() - timedelta(days=30)), + ), + WikibaseSoftwareModel.software_type == WikibaseSoftwareType.EXTENSION, + or_( + # pylint: disable=singleton-comparison + WikibaseSoftwareModel.archived == False, + WikibaseSoftwareModel.archived == None, + ), + ) + ) + .limit(5) + ) + + +async def compile_tag_list( + async_session: AsyncSession, soup: BeautifulSoup +) -> List[WikibaseSoftwareTagModel]: + """Compile Tag List""" + + type_title_tag = soup.find( + "a", attrs={"href": "/wiki/Special:MyLanguage/Template:Extension#type"} + ) + if type_title_tag is None: + return [] + + type_tag = type_title_tag.find_parent("td").find_next_sibling("td") + assert type_tag is not None + + tag_list: list[str] = [] + if len(list(type_tag.children)) == 1: + tag_list = [ + s.strip() + for t_string in type_tag.stripped_strings + for s in t_string.split(",") + ] + else: + tag_list = [ + stripped + for t in type_tag.find_all("a") + if t.string is not None + for s in t.string.split(",") + if (stripped := (s).strip()) != "" + ] + all_tags = await fetch_or_create_tags(async_session, tag_list) + + return all_tags + + +async def fetch_or_create_tags( + async_session: AsyncSession, tag_list: List[str] +) -> List[WikibaseSoftwareTagModel]: + """Fetch or Create Tags""" + + existing_tags = ( + await async_session.scalars( + select(WikibaseSoftwareTagModel).where( + WikibaseSoftwareTagModel.tag.in_(tag_list) + ) + ) + ).all() + existing_tag_strs = {t.tag for t in existing_tags} + all_tags = [*existing_tags] + for tag in tag_list: + if tag not in existing_tag_strs: + all_tags.append(WikibaseSoftwareTagModel(tag)) + return all_tags + + +def compile_description(soup: BeautifulSoup) -> Optional[str]: + """Compile Description""" + + description_title_tag = soup.find( + "a", attrs={"href": "/wiki/Special:MyLanguage/Template:Extension#description"} + ) + if description_title_tag is None: + return None + + description_tag = description_title_tag.find_parent("td").find_next_sibling("td") + assert description_tag is not None + result = re.sub(r"[ ]{2,}", r" ", "".join(description_tag.strings).strip()) + return result[0].upper() + result[1:] + + +def compile_latest_version(soup: BeautifulSoup) -> Optional[str]: + """Compile Latest Version""" + + version_title_tag = soup.find( + "a", attrs={"href": "/wiki/Special:MyLanguage/Template:Extension#version"} + ) + if version_title_tag is None: + return None + + version_tag = version_title_tag.find_parent("td").find_next_sibling("td") + assert version_tag is not None, f"{version_title_tag.prettify()}" + return re.sub(r" ", r" ", "".join(version_tag.strings).strip()) + + +def compile_quarterly_count(soup: BeautifulSoup) -> Optional[int]: + """Compile Quarterly Download Count""" + + qd_title_tag = soup.find("b", string="Quarterly downloads") + if qd_title_tag is None: + return None + + qd_tag = qd_title_tag.find_parent("td").find_next_sibling("td") + assert qd_tag is not None + attempt = re.sub(r"^(\d+(,\d+)*) .*$", r"\1", "".join(qd_tag.strings)) + return int(attempt.replace(",", "")) + + +def compile_wiki_count(soup: BeautifulSoup) -> Optional[int]: + """Compile Public Wiki Count""" + + pw_title_tag = soup.find("b", string="Public wikis using") + if pw_title_tag is None: + return None + + pw_tag = pw_title_tag.find_parent("td").find_next_sibling("td") + assert pw_tag is not None + attempt = re.sub(r"^(\d+(,\d+)*) .*$", r"\1", "".join(pw_tag.strings)) + return int(attempt.replace(",", "")) + + +def compile_bundled(soup: BeautifulSoup) -> Optional[bool]: + """Compile Bundled with MediaWiki""" + + category_list_tag = soup.find("div", attrs={"id": "mw-normal-catlinks"}) + if category_list_tag is None: + return None + for a_tag in category_list_tag.find_all("a"): + if re.match( + r"^/wiki/Category:Extensions_bundled_with_MediaWiki_\d+\.\d+$", + a_tag["href"], + ): + return True + return False diff --git a/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py b/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py index f5a19288..109ee506 100644 --- a/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py +++ b/fetch_data/sparql_data/connectivity_math/connectivity_distance_dictionary.py @@ -1,11 +1,11 @@ """Compile Connectivity Distance""" -from typing import Iterable, List +from collections.abc import Iterable from tqdm import tqdm def compile_distance_dict( - all_nodes: List[str], link_dict: dict[str, set[str]] + all_nodes: Iterable[str], link_dict: dict[str, set[str]] ) -> dict[str, dict[str, int]]: """Compile Distance Dictionary diff --git a/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py b/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py index 31867ca6..dea5abdf 100644 --- a/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py +++ b/fetch_data/sparql_data/connectivity_math/connectivity_link_dictionary.py @@ -1,11 +1,11 @@ """Compile Link Dictionary""" -from typing import List +from collections.abc import Iterable from fetch_data.sparql_data.sparql_queries import ItemLink def compile_link_dict( - clean_data: List[ItemLink], all_nodes: List[str], reverse: bool = False + clean_data: Iterable[ItemLink], all_nodes: Iterable[str], reverse: bool = False ) -> dict[str, set[str]]: """Compile Link Dictionary diff --git a/fetch_data/sparql_data/sparql_queries/item_links.py b/fetch_data/sparql_data/sparql_queries/item_links.py index 4409ac42..31cf0cb0 100644 --- a/fetch_data/sparql_data/sparql_queries/item_links.py +++ b/fetch_data/sparql_data/sparql_queries/item_links.py @@ -1,7 +1,6 @@ """Links between items""" import re -from typing import List ITEM_LINKS_QUERY = """SELECT ?item ?object WHERE { @@ -36,7 +35,7 @@ def clean_point(point: dict) -> ItemLink: ) -def clean_item_link_data(results: dict) -> List[ItemLink]: +def clean_item_link_data(results: dict) -> list[ItemLink]: """Query Results to list of data""" return [clean_point(p) for p in results["results"]["bindings"]] diff --git a/fetch_data/utils/counts.py b/fetch_data/utils/counts.py index 72d1b7d0..8d8c8824 100644 --- a/fetch_data/utils/counts.py +++ b/fetch_data/utils/counts.py @@ -1,6 +1,7 @@ """Counts""" -from typing import Iterable, TypeVar +from collections.abc import Iterable +from typing import TypeVar T = TypeVar("T") diff --git a/model/database/__init__.py b/model/database/__init__.py index ac44ab96..1d6c57d5 100644 --- a/model/database/__init__.py +++ b/model/database/__init__.py @@ -21,4 +21,8 @@ WikibaseUserObservationGroupModel, WikibaseUserObservationModel, ) +from model.database.wikibase_software import ( + WikibaseSoftwareModel, + WikibaseSoftwareTagModel, +) from model.database.wikibase_url_model import WikibaseURLModel diff --git a/model/database/wikibase_category_model.py b/model/database/wikibase_category_model.py index c1827571..bf1d72f4 100644 --- a/model/database/wikibase_category_model.py +++ b/model/database/wikibase_category_model.py @@ -1,6 +1,5 @@ """Wikibase Category Table""" -from typing import List from sqlalchemy import Enum, Integer, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -18,7 +17,7 @@ class WikibaseCategoryModel(ModelBase): id: Mapped[int] = mapped_column("id", Integer, primary_key=True, autoincrement=True) """ID""" - wikibases: Mapped[List["WikibaseModel"]] = relationship( + wikibases: Mapped[list["WikibaseModel"]] = relationship( "WikibaseModel", lazy="selectin", back_populates="category" ) """Wikibases""" diff --git a/model/database/wikibase_model.py b/model/database/wikibase_model.py index 06f8d3e7..9478c5d9 100644 --- a/model/database/wikibase_model.py +++ b/model/database/wikibase_model.py @@ -1,6 +1,6 @@ """Wikibase Table""" -from typing import List, Optional +from typing import Optional from sqlalchemy import Boolean, ForeignKey, Integer, String, and_ from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -212,7 +212,7 @@ class WikibaseModel(ModelBase): ) """Special:Version URL""" - connectivity_observations: Mapped[List[WikibaseConnectivityObservationModel]] = ( + connectivity_observations: Mapped[list[WikibaseConnectivityObservationModel]] = ( relationship( "WikibaseConnectivityObservationModel", back_populates="wikibase", @@ -221,13 +221,13 @@ class WikibaseModel(ModelBase): ) """Connectivity Observations""" - log_observations: Mapped[List[WikibaseLogObservationModel]] = relationship( + log_observations: Mapped[list[WikibaseLogObservationModel]] = relationship( "WikibaseLogObservationModel", back_populates="wikibase", lazy="select" ) """Log Observations""" property_popularity_observations: Mapped[ - List[WikibasePropertyPopularityObservationModel] + list[WikibasePropertyPopularityObservationModel] ] = relationship( "WikibasePropertyPopularityObservationModel", back_populates="wikibase", @@ -235,7 +235,7 @@ class WikibaseModel(ModelBase): ) """Property Popularity Observations""" - quantity_observations: Mapped[List[WikibaseQuantityObservationModel]] = ( + quantity_observations: Mapped[list[WikibaseQuantityObservationModel]] = ( relationship( "WikibaseQuantityObservationModel", back_populates="wikibase", lazy="select" ) @@ -243,7 +243,7 @@ class WikibaseModel(ModelBase): """Quantity Observations""" software_version_observations: Mapped[ - List[WikibaseSoftwareVersionObservationModel] + list[WikibaseSoftwareVersionObservationModel] ] = relationship( "WikibaseSoftwareVersionObservationModel", back_populates="wikibase", @@ -251,7 +251,7 @@ class WikibaseModel(ModelBase): ) """Software Version Observations""" - statistics_observations: Mapped[List[WikibaseStatisticsObservationModel]] = ( + statistics_observations: Mapped[list[WikibaseStatisticsObservationModel]] = ( relationship( "WikibaseStatisticsObservationModel", back_populates="wikibase", @@ -260,7 +260,7 @@ class WikibaseModel(ModelBase): ) """Statistics Observations""" - user_observations: Mapped[List[WikibaseUserObservationModel]] = relationship( + user_observations: Mapped[list[WikibaseUserObservationModel]] = relationship( "WikibaseUserObservationModel", back_populates="wikibase", lazy="select" ) """User Observations""" diff --git a/model/database/wikibase_observation/connectivity/connectivity_observation_model.py b/model/database/wikibase_observation/connectivity/connectivity_observation_model.py index 4963c8bd..b37ca75a 100644 --- a/model/database/wikibase_observation/connectivity/connectivity_observation_model.py +++ b/model/database/wikibase_observation/connectivity/connectivity_observation_model.py @@ -1,6 +1,6 @@ """Wikibase Connectivity Observation Table""" -from typing import List, Optional +from typing import Optional from sqlalchemy import Double, Integer from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -37,7 +37,7 @@ class WikibaseConnectivityObservationModel(ModelBase, WikibaseObservationModel): """Average steps for item -> item connections, ignoring disconnected items""" item_relationship_count_observations: Mapped[ - List[WikibaseConnectivityObservationItemRelationshipCountModel] + list[WikibaseConnectivityObservationItemRelationshipCountModel] ] = relationship( "WikibaseConnectivityObservationItemRelationshipCountModel", back_populates="connectivity_observation", @@ -46,7 +46,7 @@ class WikibaseConnectivityObservationModel(ModelBase, WikibaseObservationModel): """Item / Relationship Count Observations""" object_relationship_count_observations: Mapped[ - List[WikibaseConnectivityObservationObjectRelationshipCountModel] + list[WikibaseConnectivityObservationObjectRelationshipCountModel] ] = relationship( "WikibaseConnectivityObservationObjectRelationshipCountModel", back_populates="connectivity_observation", diff --git a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py index 4b17115f..0ce45f87 100644 --- a/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py +++ b/model/database/wikibase_observation/log/wikibase_log_month_observation_model.py @@ -1,7 +1,7 @@ """Wikibase Log Month Observation Table""" from datetime import datetime -from typing import List, Optional +from typing import Optional from sqlalchemy import DateTime, Integer from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -43,12 +43,12 @@ class WikibaseLogMonthObservationModel(ModelBase): ) """Number of Unique Users, Without Bots""" - log_type_records: Mapped[List[WikibaseLogMonthLogTypeObservationModel]] = ( + log_type_records: Mapped[list[WikibaseLogMonthLogTypeObservationModel]] = ( relationship("WikibaseLogMonthLogTypeObservationModel", lazy="selectin") ) """Log Type Observations""" - user_type_records: Mapped[List[WikibaseLogMonthUserTypeObservationModel]] = ( + user_type_records: Mapped[list[WikibaseLogMonthUserTypeObservationModel]] = ( relationship("WikibaseLogMonthUserTypeObservationModel", lazy="selectin") ) """User Type Observations""" diff --git a/model/database/wikibase_observation/property/popularity_observation_model.py b/model/database/wikibase_observation/property/popularity_observation_model.py index 132bdcd3..88fc1c6b 100644 --- a/model/database/wikibase_observation/property/popularity_observation_model.py +++ b/model/database/wikibase_observation/property/popularity_observation_model.py @@ -1,6 +1,5 @@ """Wikibase Property Popularity Observation Table""" -from typing import List from sqlalchemy.orm import Mapped, relationship from model.database.base import ModelBase @@ -17,7 +16,7 @@ class WikibasePropertyPopularityObservationModel(ModelBase, WikibaseObservationM __tablename__ = "wikibase_property_usage_observation" - property_count_observations: Mapped[List[WikibasePropertyPopularityCountModel]] = ( + property_count_observations: Mapped[list[WikibasePropertyPopularityCountModel]] = ( relationship( "WikibasePropertyPopularityCountModel", back_populates="wikibase_property_popularity_observation", diff --git a/model/database/wikibase_observation/user/wikibase_user_group_model.py b/model/database/wikibase_observation/user/wikibase_user_group_model.py index 5b56f715..c9d461f7 100644 --- a/model/database/wikibase_observation/user/wikibase_user_group_model.py +++ b/model/database/wikibase_observation/user/wikibase_user_group_model.py @@ -1,6 +1,5 @@ """Wikibase User Group Table""" -from typing import List from sqlalchemy import Boolean, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -27,7 +26,7 @@ class WikibaseUserGroupModel(ModelBase): ) """Wikibase Default?""" - user_group_observations: Mapped[List[WikibaseUserObservationGroupModel]] = ( + user_group_observations: Mapped[list[WikibaseUserObservationGroupModel]] = ( relationship( "WikibaseUserObservationGroupModel", back_populates="user_group", diff --git a/model/database/wikibase_observation/user/wikibase_user_observation_model.py b/model/database/wikibase_observation/user/wikibase_user_observation_model.py index 0f462d20..47f524bc 100644 --- a/model/database/wikibase_observation/user/wikibase_user_observation_model.py +++ b/model/database/wikibase_observation/user/wikibase_user_observation_model.py @@ -1,6 +1,6 @@ """Wikibase User Observation Table""" -from typing import List, Optional +from typing import Optional from sqlalchemy import Integer from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -23,7 +23,7 @@ class WikibaseUserObservationModel(ModelBase, WikibaseObservationModel): ) """Total Users""" - user_group_observations: Mapped[List[WikibaseUserObservationGroupModel]] = ( + user_group_observations: Mapped[list[WikibaseUserObservationGroupModel]] = ( relationship( "WikibaseUserObservationGroupModel", back_populates="user_observation", diff --git a/model/database/wikibase_observation/version/software_version_model.py b/model/database/wikibase_observation/version/software_version_model.py index 5c4adb50..47bfb1e9 100644 --- a/model/database/wikibase_observation/version/software_version_model.py +++ b/model/database/wikibase_observation/version/software_version_model.py @@ -2,11 +2,11 @@ from datetime import datetime from typing import Optional -from sqlalchemy import DateTime, Enum, ForeignKey, Integer, String, UniqueConstraint +from sqlalchemy import DateTime, ForeignKey, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from model.database.base import ModelBase -from model.enum import WikibaseSoftwareType +from model.database.wikibase_software import WikibaseSoftwareModel class WikibaseSoftwareVersionModel(ModelBase): @@ -17,9 +17,8 @@ class WikibaseSoftwareVersionModel(ModelBase): __table_args__ = ( UniqueConstraint( "wikibase_software_version_observation_id", - "software_type", - "software_name", - name="unique_observation_software_type_name", + "wikibase_software_id", + name="unique_observation_software_id", ), ) @@ -44,13 +43,19 @@ class WikibaseSoftwareVersionModel(ModelBase): ) """Software Version Observation""" - software_type: Mapped[WikibaseSoftwareType] = mapped_column( - "software_type", Enum(WikibaseSoftwareType), nullable=False + software_id: Mapped[int] = mapped_column( + "wikibase_software_id", + ForeignKey(column="wikibase_software.id", name="software"), + nullable=False, ) - """Software Type""" + """Software Id""" - software_name: Mapped[str] = mapped_column("software_name", String, nullable=False) - """Software Name""" + software: Mapped[WikibaseSoftwareModel] = relationship( + "WikibaseSoftwareModel", + # back_populates="software_versions", + lazy="selectin", + ) + """Software""" version: Mapped[Optional[str]] = mapped_column("version", String, nullable=True) """Version""" @@ -68,14 +73,13 @@ class WikibaseSoftwareVersionModel(ModelBase): # pylint: disable=too-many-arguments,too-many-positional-arguments def __init__( self, - software_type: WikibaseSoftwareType, - software_name: str, + software: WikibaseSoftwareModel, version: str, version_hash: Optional[str] = None, version_date: Optional[datetime] = None, ): - self.software_type = software_type - self.software_name = software_name + self.software = software + self.version_hash = ( None if version_hash is None @@ -87,8 +91,8 @@ def __init__( def __str__(self) -> str: return ( "WikibaseSoftwareVersionModel(" - + f"software_type={self.software_type}, " - + f"software_name={self.software_name}, " + + f"software_type={self.software.software_type}, " + + f"software_name={self.software.software_name}, " + f"version={self.version}, " + f"version_date={self.version_date}, " + f"version_hash={self.version_hash})" diff --git a/model/database/wikibase_observation/version/wikibase_version_observation_model.py b/model/database/wikibase_observation/version/wikibase_version_observation_model.py index 0ab57ec5..31fd2b3a 100644 --- a/model/database/wikibase_observation/version/wikibase_version_observation_model.py +++ b/model/database/wikibase_observation/version/wikibase_version_observation_model.py @@ -1,6 +1,5 @@ """Wikibase Software Version Observation Table""" -from typing import List from sqlalchemy.orm import Mapped, relationship from model.database.base import ModelBase @@ -17,7 +16,7 @@ class WikibaseSoftwareVersionObservationModel(ModelBase, WikibaseObservationMode __tablename__ = "wikibase_software_version_observation" - software_versions: Mapped[List[WikibaseSoftwareVersionModel]] = relationship( + software_versions: Mapped[list[WikibaseSoftwareVersionModel]] = relationship( "WikibaseSoftwareVersionModel", back_populates="wikibase_software_version_observation", lazy="selectin", diff --git a/model/database/wikibase_software/__init__.py b/model/database/wikibase_software/__init__.py new file mode 100644 index 00000000..220f8a77 --- /dev/null +++ b/model/database/wikibase_software/__init__.py @@ -0,0 +1,4 @@ +"""Wikibase Software""" + +from model.database.wikibase_software.software_model import WikibaseSoftwareModel +from model.database.wikibase_software.software_tag_model import WikibaseSoftwareTagModel diff --git a/model/database/wikibase_software/software_model.py b/model/database/wikibase_software/software_model.py new file mode 100644 index 00000000..8b6e1c87 --- /dev/null +++ b/model/database/wikibase_software/software_model.py @@ -0,0 +1,87 @@ +"""Wikibase Software Table""" + +from datetime import datetime +from typing import List, Optional +from sqlalchemy import Boolean, DateTime, Enum, Integer, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from model.database.base import ModelBase +from model.database.wikibase_software.software_tag_model import WikibaseSoftwareTagModel +from model.database.wikibase_software.software_tag_xref_model import ( + software_tag_xref_table, +) +from model.enum import WikibaseSoftwareType + + +class WikibaseSoftwareModel(ModelBase): + """Wikibase Software Table""" + + __tablename__ = "wikibase_software" + + __table_args__ = ( + UniqueConstraint( + "software_type", "software_name", name="unique_software_type_name" + ), + ) + + id: Mapped[int] = mapped_column("id", Integer, primary_key=True, autoincrement=True) + """ID""" + + software_type: Mapped[WikibaseSoftwareType] = mapped_column( + "software_type", Enum(WikibaseSoftwareType), nullable=False + ) + """Software Type""" + + software_name: Mapped[str] = mapped_column("software_name", String, nullable=False) + """Software Name""" + + url: Mapped[Optional[str]] = mapped_column("url", String, nullable=True) + """Reference URL""" + + data_fetched: Mapped[Optional[datetime]] = mapped_column( + "fetched", DateTime(timezone=True), nullable=True + ) + + tags: Mapped[List[WikibaseSoftwareTagModel]] = relationship( + secondary=software_tag_xref_table, lazy="selectin" + ) + + description: Mapped[Optional[str]] = mapped_column( + "description", String, nullable=True + ) + """Description""" + + latest_version: Mapped[Optional[str]] = mapped_column( + "latest_version", String, nullable=True + ) + """Latest Version""" + + quarterly_download_count: Mapped[Optional[int]] = mapped_column( + "quarterly_download_count", Integer, nullable=True + ) + """Quarterly Downloads""" + + public_wiki_count: Mapped[Optional[int]] = mapped_column( + "public_wiki_count", Integer, nullable=True + ) + """Public Wikis Using""" + + mediawiki_bundled: Mapped[Optional[bool]] = mapped_column( + "mw_bundled", Boolean, nullable=True + ) + """Bundled with Mediawiki""" + + archived: Mapped[Optional[bool]] = mapped_column("archived", Boolean, nullable=True) + """Archived Extension""" + + def __init__(self, software_type: WikibaseSoftwareType, software_name: str): + self.software_type = software_type + self.software_name = software_name + + def __str__(self) -> str: + return ( + "WikibaseSoftwareModel(" + + f"id={self.id}, " + + f"software_type={self.software_type}, " + + f"software_name={self.software_name})" + ) diff --git a/model/database/wikibase_software/software_tag_model.py b/model/database/wikibase_software/software_tag_model.py new file mode 100644 index 00000000..13cab75c --- /dev/null +++ b/model/database/wikibase_software/software_tag_model.py @@ -0,0 +1,24 @@ +"""Wikibase Software Version Table""" + +from sqlalchemy import Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from model.database.base import ModelBase + + +class WikibaseSoftwareTagModel(ModelBase): + """Wikibase Software Tag Table""" + + __tablename__ = "wikibase_software_tag" + + id: Mapped[int] = mapped_column("id", Integer, primary_key=True, autoincrement=True) + """ID""" + + tag: Mapped[str] = mapped_column("tag", String, nullable=False, unique=True) + """Software Tag""" + + def __init__(self, tag: str): + self.tag = tag + + def __str__(self) -> str: + return f"WikibaseSoftwareTagModel(id={self.id}, tag={self.tag})" diff --git a/model/database/wikibase_software/software_tag_xref_model.py b/model/database/wikibase_software/software_tag_xref_model.py new file mode 100644 index 00000000..e385384e --- /dev/null +++ b/model/database/wikibase_software/software_tag_xref_model.py @@ -0,0 +1,24 @@ +"""Software/Tag XRef Table""" + +from sqlalchemy import Column, ForeignKey, Table + +from model.database.base import ModelBase + + +software_tag_xref_table = Table( + "wikibase_software_tag_xref", + ModelBase.metadata, + Column( + "wikibase_software_id", + ForeignKey("wikibase_software.id", None, False, "foreign_wikibase_software_id"), + primary_key=True, + ), + Column( + "wikibase_software_tag_id", + ForeignKey( + "wikibase_software_tag.id", None, False, "foreign_wikibase_software_tag_id" + ), + primary_key=True, + ), +) +"""Software/Tag XRef Table""" diff --git a/model/strawberry/output/__init__.py b/model/strawberry/output/__init__.py index 77820df3..af443b7a 100644 --- a/model/strawberry/output/__init__.py +++ b/model/strawberry/output/__init__.py @@ -11,3 +11,4 @@ ) from model.strawberry.output.page import Page, PageNumberType, PageSizeType from model.strawberry.output.wikibase import WikibaseStrawberryModel +from model.strawberry.output.wikibase_software import WikibaseSoftwareStrawberryModel diff --git a/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py b/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py index 8f1ea173..e0ce8606 100644 --- a/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py +++ b/model/strawberry/output/observation/connectivity/wikibase_connectivity_observation.py @@ -1,6 +1,6 @@ """Wikibase Connectivity Data Observation Strawberry Model""" -from typing import List, Optional +from typing import Optional import strawberry from model.database import WikibaseConnectivityObservationModel @@ -32,10 +32,10 @@ class WikibaseConnectivityObservationStrawberryModel( graphql_type=Optional[BigInt], ) - relationship_item_counts: List[ + relationship_item_counts: list[ WikibaseConnectivityObservationItemRelationshipCountStrawberryModel ] = strawberry.field(description="Number of Items with Number of Relationships") - relationship_object_counts: List[ + relationship_object_counts: list[ WikibaseConnectivityObservationObjectRelationshipCountStrawberryModel ] = strawberry.field(description="Number of Items with Number of Relationships") diff --git a/model/strawberry/output/observation/log/wikibase_log_collection.py b/model/strawberry/output/observation/log/wikibase_log_collection.py index 2e13f61d..6f88044c 100644 --- a/model/strawberry/output/observation/log/wikibase_log_collection.py +++ b/model/strawberry/output/observation/log/wikibase_log_collection.py @@ -1,7 +1,7 @@ """Wikibase Log Collection Strawberry Models""" from datetime import datetime -from typing import List, Optional +from typing import Optional import strawberry from model.database import ( @@ -81,10 +81,10 @@ def marshal( class WikibaseLogMonthStrawberryModel(WikibaseLogCollectionStrawberryModel): """Wikibase Log Month""" - log_type_records: List[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field( + log_type_records: list[WikibaseLogMonthLogTypeStrawberryModel] = strawberry.field( description="Records of Each Type" ) - user_type_records: List[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field( + user_type_records: list[WikibaseLogMonthUserTypeStrawberryModel] = strawberry.field( description="Records of Each Type" ) human_users: int = strawberry.field( diff --git a/model/strawberry/output/observation/property_popularity/observation.py b/model/strawberry/output/observation/property_popularity/observation.py index d22ac911..6e51c9e8 100644 --- a/model/strawberry/output/observation/property_popularity/observation.py +++ b/model/strawberry/output/observation/property_popularity/observation.py @@ -1,6 +1,5 @@ """Wikibase Property Popularity Observation Strawberry Model""" -from typing import List import strawberry from model.database import WikibasePropertyPopularityObservationModel @@ -18,7 +17,7 @@ class WikibasePropertyPopularityObservationStrawberryModel( ): """Wikibase Property Popularity Observation""" - property_popularity_counts: List[WikibasePropertyPopularityCountStrawberryModel] = ( + property_popularity_counts: list[WikibasePropertyPopularityCountStrawberryModel] = ( strawberry.field(description="Number of Items with Number of Relationships") ) diff --git a/model/strawberry/output/observation/software_version/software_version.py b/model/strawberry/output/observation/software_version/software_version.py index c6c19fe5..458c001e 100644 --- a/model/strawberry/output/observation/software_version/software_version.py +++ b/model/strawberry/output/observation/software_version/software_version.py @@ -5,6 +5,7 @@ import strawberry from model.database import WikibaseSoftwareVersionModel +from model.strawberry.output.wikibase_software import WikibaseSoftwareStrawberryModel @strawberry.type @@ -12,7 +13,10 @@ class WikibaseSoftwareVersionStrawberryModel: """Wikibase Software Version""" id: strawberry.ID - software_name: str = strawberry.field(description="Software Name") + software: WikibaseSoftwareStrawberryModel = strawberry.field(description="Software") + software_name: str = strawberry.field( + description="Software Name", deprecation_reason="Use software/softwareName" + ) version: Optional[str] = strawberry.field(description="Software Version") version_date: Optional[datetime] = strawberry.field( description="Software Version Release Date" @@ -29,7 +33,8 @@ def marshal( return cls( id=strawberry.ID(model.id), - software_name=model.software_name, + software=WikibaseSoftwareStrawberryModel.marshal(model.software), + software_name=model.software.software_name, version=model.version, version_date=model.version_date, version_hash=model.version_hash, diff --git a/model/strawberry/output/observation/software_version/software_version_aggregate.py b/model/strawberry/output/observation/software_version/software_version_aggregate.py index 03920a67..5c26e60f 100644 --- a/model/strawberry/output/observation/software_version/software_version_aggregate.py +++ b/model/strawberry/output/observation/software_version/software_version_aggregate.py @@ -2,7 +2,7 @@ from datetime import datetime import re -from typing import List, Optional +from typing import Optional import strawberry from model.strawberry.output.semver import Semver @@ -55,7 +55,7 @@ class WikibaseSoftwareMidVersionAggregateStrawberryModel: version: Optional[str] = strawberry.field(description="Software Version") private_versions: strawberry.Private[ - List[WikibaseSoftwareVersionAggregateStrawberryModel] + list[WikibaseSoftwareVersionAggregateStrawberryModel] ] @strawberry.field(description="Wikibase Count") @@ -74,7 +74,7 @@ class WikibaseSoftwarePatchVersionAggregateStrawberryModel( @strawberry.field(description="Versions") def sub_versions( self, - ) -> Optional[List[WikibaseSoftwareVersionAggregateStrawberryModel]]: + ) -> Optional[list[WikibaseSoftwareVersionAggregateStrawberryModel]]: """Sub-Patch Versions""" if ( @@ -101,7 +101,7 @@ class WikibaseSoftwareMinorVersionAggregateStrawberryModel( @strawberry.field(description="Patch Versions") def patch_versions( self, - ) -> Optional[List[WikibaseSoftwarePatchVersionAggregateStrawberryModel]]: + ) -> Optional[list[WikibaseSoftwarePatchVersionAggregateStrawberryModel]]: """Patch Versions""" if self.version is None: @@ -135,7 +135,7 @@ class WikibaseSoftwareMajorVersionAggregateStrawberryModel( @strawberry.field(description="Minor Versions") def minor_versions( self, - ) -> Optional[List[WikibaseSoftwareMinorVersionAggregateStrawberryModel]]: + ) -> Optional[list[WikibaseSoftwareMinorVersionAggregateStrawberryModel]]: """Minor Versions""" if self.version is None: @@ -160,13 +160,13 @@ class WikibaseSoftwareVersionDoubleAggregateStrawberryModel: software_name: str = strawberry.field(description="Software Name") private_versions: strawberry.Private[ - List[WikibaseSoftwareVersionAggregateStrawberryModel] + list[WikibaseSoftwareVersionAggregateStrawberryModel] ] def __init__( self, software_name: str, - versions: List[WikibaseSoftwareVersionAggregateStrawberryModel], + versions: list[WikibaseSoftwareVersionAggregateStrawberryModel], ): self.software_name = software_name self.private_versions = versions @@ -180,7 +180,7 @@ def wikibase_count(self) -> int: @strawberry.field(description="Major Versions") def major_versions( self, - ) -> List[WikibaseSoftwareMajorVersionAggregateStrawberryModel]: + ) -> list[WikibaseSoftwareMajorVersionAggregateStrawberryModel]: """Major Versions""" temp: dict[ @@ -196,7 +196,7 @@ def major_versions( return sorted(temp.values(), key=lambda x: x.wikibase_count(), reverse=True) @strawberry.field(description="Version List") - def versions(self) -> List[WikibaseSoftwareVersionAggregateStrawberryModel]: + def versions(self) -> list[WikibaseSoftwareVersionAggregateStrawberryModel]: """Version List""" return sorted( diff --git a/model/strawberry/output/observation/software_version/software_version_observation.py b/model/strawberry/output/observation/software_version/software_version_observation.py index f1acbb6c..d112dcc9 100644 --- a/model/strawberry/output/observation/software_version/software_version_observation.py +++ b/model/strawberry/output/observation/software_version/software_version_observation.py @@ -1,6 +1,5 @@ """Wikibase Software Version Observation Strawberry Model""" -from typing import List import strawberry from model.database import WikibaseSoftwareVersionObservationModel @@ -19,19 +18,19 @@ class WikibaseSoftwareVersionObservationStrawberryModel( ): """Wikibase Software Version Observation""" - installed_extensions: List[WikibaseSoftwareVersionStrawberryModel] = ( + installed_extensions: list[WikibaseSoftwareVersionStrawberryModel] = ( strawberry.field(description="Installed Extensions w/ Versions") ) - installed_libraries: List[WikibaseSoftwareVersionStrawberryModel] = ( + installed_libraries: list[WikibaseSoftwareVersionStrawberryModel] = ( strawberry.field(description="Installed Libraries w/ Versions") ) - installed_skins: List[WikibaseSoftwareVersionStrawberryModel] = strawberry.field( + installed_skins: list[WikibaseSoftwareVersionStrawberryModel] = strawberry.field( description="Installed Skins w/ Versions" ) - installed_software: List[WikibaseSoftwareVersionStrawberryModel] = strawberry.field( + installed_software: list[WikibaseSoftwareVersionStrawberryModel] = strawberry.field( description="Installed Software Versions" ) @@ -49,7 +48,7 @@ def marshal( [ WikibaseSoftwareVersionStrawberryModel.marshal(o) for o in model.software_versions - if o.software_type == WikibaseSoftwareType.EXTENSION + if o.software.software_type == WikibaseSoftwareType.EXTENSION ], key=lambda x: x.software_name, ), @@ -57,7 +56,7 @@ def marshal( [ WikibaseSoftwareVersionStrawberryModel.marshal(o) for o in model.software_versions - if o.software_type == WikibaseSoftwareType.LIBRARY + if o.software.software_type == WikibaseSoftwareType.LIBRARY ], key=lambda x: x.software_name, ), @@ -65,7 +64,7 @@ def marshal( [ WikibaseSoftwareVersionStrawberryModel.marshal(o) for o in model.software_versions - if o.software_type == WikibaseSoftwareType.SKIN + if o.software.software_type == WikibaseSoftwareType.SKIN ], key=lambda x: x.software_name, ), @@ -73,7 +72,7 @@ def marshal( [ WikibaseSoftwareVersionStrawberryModel.marshal(o) for o in model.software_versions - if o.software_type == WikibaseSoftwareType.SOFTWARE + if o.software.software_type == WikibaseSoftwareType.SOFTWARE ], key=lambda x: x.software_name, ), diff --git a/model/strawberry/output/observation/user/wikibase_user_observation.py b/model/strawberry/output/observation/user/wikibase_user_observation.py index 37c0573c..0618296f 100644 --- a/model/strawberry/output/observation/user/wikibase_user_observation.py +++ b/model/strawberry/output/observation/user/wikibase_user_observation.py @@ -1,6 +1,6 @@ """Wikibase User Data Observation Strawberry Model""" -from typing import List, Optional +from typing import Optional import strawberry from model.database import WikibaseUserObservationModel @@ -20,7 +20,7 @@ class WikibaseUserObservationStrawberryModel(WikibaseObservationStrawberryModel) total_users: Optional[int] = strawberry.field( description="Total Users", graphql_type=Optional[BigInt] ) - user_groups: List[WikibaseUserObservationGroupStrawberryModel] = strawberry.field( + user_groups: list[WikibaseUserObservationGroupStrawberryModel] = strawberry.field( description="User Groups and Counts" ) diff --git a/model/strawberry/output/observation/wikibase_observation_set.py b/model/strawberry/output/observation/wikibase_observation_set.py index b10a21c0..02f434b2 100644 --- a/model/strawberry/output/observation/wikibase_observation_set.py +++ b/model/strawberry/output/observation/wikibase_observation_set.py @@ -1,6 +1,6 @@ """Wikibase Observation Set Strawberry Model""" -from typing import Generic, List, Optional, TypeVar +from typing import Generic, Optional, TypeVar import strawberry from model.strawberry.output.observation.wikibase_observation import ( @@ -15,7 +15,7 @@ class WikibaseObservationSetStrawberryModel(Generic[T]): """Wikibase Observation Set""" - all_observations: List[T] = strawberry.field(description="All Observations") + all_observations: list[T] = strawberry.field(description="All Observations") @strawberry.field(description="Most Recent Observation that Returned Data") def most_recent(self) -> Optional[T]: @@ -29,7 +29,7 @@ def most_recent(self) -> Optional[T]: return None @classmethod - def marshal(cls, data: List[T]): + def marshal(cls, data: list[T]): """Coerce List into Set""" return cls(all_observations=data) diff --git a/model/strawberry/output/page.py b/model/strawberry/output/page.py index 8f026d54..a51d57a6 100644 --- a/model/strawberry/output/page.py +++ b/model/strawberry/output/page.py @@ -1,7 +1,7 @@ """Page""" from math import ceil -from typing import Annotated, Generic, List, TypeVar +from typing import Annotated, Generic, TypeVar import strawberry from model.strawberry.scalars import BigInt @@ -49,7 +49,7 @@ class Page(Generic[T]): """Page""" meta: PageMetadata = strawberry.field(description="Metadata") - data: List[T] = strawberry.field(description="Data") + data: list[T] = strawberry.field(description="Data") @classmethod def marshal( @@ -57,7 +57,7 @@ def marshal( page_number: PageNumberType, page_size: PageSizeType, total_count: int, - page_data: List[T], + page_data: list[T], ) -> "Page": """Marshal Data into Page""" return cls( diff --git a/model/strawberry/output/wikibase_software.py b/model/strawberry/output/wikibase_software.py new file mode 100644 index 00000000..8be6a05f --- /dev/null +++ b/model/strawberry/output/wikibase_software.py @@ -0,0 +1,64 @@ +"""Wikibase Instance Strawberry Model""" + +from datetime import datetime +from typing import Optional +import strawberry + +from model.database import WikibaseSoftwareModel +from model.enum.wikibase_software_type_enum import WikibaseSoftwareType + + +@strawberry.type +class WikibaseSoftwareStrawberryModel: + """Wikibase Software""" + + id: strawberry.ID + software_type: WikibaseSoftwareType = strawberry.field( + description="Wikibase Software Type" + ) + software_name: str = strawberry.field(description="Wikibase Software Name") + + url: Optional[str] = strawberry.field(description="Reference URL") + + fetched: Optional[datetime] = strawberry.field( + description="Date Fetched from MediaWiki" + ) + + tags: list[str] = strawberry.field(description="Tag List") + + description: Optional[str] = strawberry.field(description="Description") + + latest_version: Optional[str] = strawberry.field(description="Latest Version") + + quarterly_download_count: Optional[int] = strawberry.field( + description="Quarterly Downloads" + ) + + public_wiki_count: Optional[int] = strawberry.field( + description="Public Wikis Using" + ) + + mediawiki_bundled: Optional[bool] = strawberry.field( + description="Bundled with MediaWiki" + ) + + archived: Optional[bool] = strawberry.field(description="Archived Extension") + + @classmethod + def marshal(cls, model: WikibaseSoftwareModel) -> "WikibaseSoftwareStrawberryModel": + """Coerce Database Model to Strawberry Model""" + + return cls( + id=strawberry.ID(model.id), + software_type=model.software_type, + software_name=model.software_name, + url=model.url, + fetched=model.data_fetched, + tags=[a.tag for a in model.tags], + description=model.description, + latest_version=model.latest_version, + quarterly_download_count=model.quarterly_download_count, + public_wiki_count=model.public_wiki_count, + mediawiki_bundled=model.mediawiki_bundled, + archived=model.archived, + ) diff --git a/model/strawberry/query.py b/model/strawberry/query.py index 2c07fd2c..3b999d09 100644 --- a/model/strawberry/query.py +++ b/model/strawberry/query.py @@ -1,6 +1,5 @@ """Query""" -from typing import List import strawberry from model.enum import WikibaseSoftwareType @@ -11,6 +10,7 @@ WikibasePropertyPopularityAggregateCountStrawberryModel, WikibaseQuantityAggregateStrawberryModel, WikibaseSoftwareVersionDoubleAggregateStrawberryModel, + WikibaseSoftwareStrawberryModel, WikibaseStatisticsAggregateStrawberryModel, WikibaseStrawberryModel, WikibaseUserAggregateStrawberryModel, @@ -23,6 +23,7 @@ get_aggregate_statistics, get_aggregate_users, get_aggregate_version, + get_software_list, get_wikibase, get_wikibase_list, ) @@ -43,10 +44,20 @@ async def wikibase_list( return await get_wikibase_list(page_number, page_size) + @strawberry.field(description="List of Extensions") + async def extension_list( + self, page_number: PageNumberType, page_size: PageSizeType + ) -> Page[WikibaseSoftwareStrawberryModel]: + """List of Wikibases""" + + return await get_software_list( + page_number, page_size, WikibaseSoftwareType.EXTENSION + ) + @strawberry.field(description="Aggregated Year of First Log Date") async def aggregate_created( self, - ) -> List[WikibaseYearCreatedAggregateStrawberryModel]: + ) -> list[WikibaseYearCreatedAggregateStrawberryModel]: """Aggregated Year of First Log Date""" return await get_aggregate_created() diff --git a/pytest.ini b/pytest.ini index c7c2b50d..4d5f1fff 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,6 +8,7 @@ markers = log: log observation tests property: property popularity observation tests quantity: quantity observation tests + soup: soup-based observation tests sparql: sparql-based observation tests statistics: statistics observation tests user: user observation tests diff --git a/requirements-dev.txt b/requirements-dev.txt index 8567c6cb..8bf271e7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,11 +1,11 @@ alembic>=1.13 -black>=24.8 +black>=24.10 freezegun>=1.5 -great-expectations>=1.0,<1.2 +great-expectations>=1.2 pylint>=3.3 pytest>=8.3 pytest-asyncio>=0.24 -pytest-cov>=5.0 +pytest-cov>=6.0 pytest-dependency>=0.6 pytest-mock>=3.14 pytest-order>=1.3 diff --git a/requirements.txt b/requirements.txt index 8d23556d..a3e02927 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,6 @@ numpy>=1.26,<2.0 requests>=2.32 SPARQLWrapper>=2.0 sqlalchemy[asyncio]>=2.0 -strawberry-graphql[fastapi]>=0.242 +strawberry-graphql[fastapi]>=0.247 tqdm>=4.66 -uvicorn[standard]>=0.30 +uvicorn[standard]>=0.32 diff --git a/resolvers/__init__.py b/resolvers/__init__.py index bef7e4ab..1ede1f26 100644 --- a/resolvers/__init__.py +++ b/resolvers/__init__.py @@ -9,5 +9,6 @@ from resolvers.get_aggregate_software_version import get_aggregate_version from resolvers.get_aggregate_statistics import get_aggregate_statistics from resolvers.get_aggregate_users import get_aggregate_users +from resolvers.get_software_list import get_software_list from resolvers.get_wikibase import get_wikibase from resolvers.get_wikibase_list import get_wikibase_list diff --git a/resolvers/get_aggregate_created.py b/resolvers/get_aggregate_created.py index 765ec393..1933e681 100644 --- a/resolvers/get_aggregate_created.py +++ b/resolvers/get_aggregate_created.py @@ -1,7 +1,5 @@ """Get Aggregate Year Created""" -from typing import List, Tuple - from sqlalchemy import Select, and_, select, func from data import get_async_session @@ -9,7 +7,7 @@ from model.strawberry.output import WikibaseYearCreatedAggregateStrawberryModel -async def get_aggregate_created() -> List[WikibaseYearCreatedAggregateStrawberryModel]: +async def get_aggregate_created() -> list[WikibaseYearCreatedAggregateStrawberryModel]: """Get Aggregate Year Created""" total_quantity_query = get_created_query() @@ -24,7 +22,7 @@ async def get_aggregate_created() -> List[WikibaseYearCreatedAggregateStrawberry ] -def get_created_query() -> Select[Tuple[int, int]]: +def get_created_query() -> Select[tuple[int, int]]: """Get Year Created Query""" rank_subquery = ( diff --git a/resolvers/get_aggregate_property_popularity.py b/resolvers/get_aggregate_property_popularity.py index a35d12ab..66b40b50 100644 --- a/resolvers/get_aggregate_property_popularity.py +++ b/resolvers/get_aggregate_property_popularity.py @@ -1,7 +1,5 @@ """Get Aggregate Property Popularity""" -from typing import Tuple - from sqlalchemy import Select, and_, desc, select, func from data import get_async_session @@ -56,7 +54,7 @@ async def get_aggregate_property_popularity( ) -def get_unordered_query() -> Select[Tuple[int, str, int, int]]: +def get_unordered_query() -> Select[tuple[int, str, int, int]]: """Get Unordered Property Popularity Query""" rank_subquery = ( diff --git a/resolvers/get_aggregate_quantity.py b/resolvers/get_aggregate_quantity.py index ba3d41dd..5cddbb58 100644 --- a/resolvers/get_aggregate_quantity.py +++ b/resolvers/get_aggregate_quantity.py @@ -1,7 +1,5 @@ """Get Aggregate Quantity""" -from typing import Tuple - from sqlalchemy import Select, and_, select, func from data import get_async_session @@ -28,7 +26,7 @@ async def get_aggregate_quantity() -> WikibaseQuantityAggregateStrawberryModel: ) -def get_total_quantity_query() -> Select[Tuple[int, int, int, int, int]]: +def get_total_quantity_query() -> Select[tuple[int, int, int, int, int]]: """Get Total Quantity Query""" rank_subquery = ( diff --git a/resolvers/get_aggregate_software_version.py b/resolvers/get_aggregate_software_version.py index c8ded732..6d591e96 100644 --- a/resolvers/get_aggregate_software_version.py +++ b/resolvers/get_aggregate_software_version.py @@ -1,13 +1,14 @@ """Get Aggregate Software Version""" from datetime import datetime -from typing import Optional, Tuple +from typing import Optional from sqlalchemy import Select, and_, select, func from data import get_async_session from model.database import ( WikibaseModel, + WikibaseSoftwareModel, WikibaseSoftwareVersionModel, WikibaseSoftwareVersionObservationModel, ) @@ -71,7 +72,7 @@ async def get_aggregate_version( def get_query( software_type: WikibaseSoftwareType, -) -> Select[Tuple[str, Optional[str], Optional[datetime], Optional[str], int]]: +) -> Select[tuple[str, Optional[str], Optional[datetime], Optional[str], int]]: """Get Software Version Query""" rank_subquery = ( @@ -95,15 +96,15 @@ def get_query( ) .subquery() ) - query = ( + + next_subquery = ( select( - WikibaseSoftwareVersionModel.software_name, WikibaseSoftwareVersionModel.version, WikibaseSoftwareVersionModel.version_date, WikibaseSoftwareVersionModel.version_hash, - # pylint: disable=not-callable - func.count().label("wikibase_count"), + WikibaseSoftwareModel.software_name, ) + .join(WikibaseSoftwareModel) .join( rank_subquery, onclause=and_( @@ -112,13 +113,20 @@ def get_query( rank_subquery.c.rank == 1, ), ) - .where(WikibaseSoftwareVersionModel.software_type == software_type) - .group_by( - WikibaseSoftwareVersionModel.software_name, - WikibaseSoftwareVersionModel.version, - WikibaseSoftwareVersionModel.version_date, - WikibaseSoftwareVersionModel.version_hash, - ) - .order_by("id") + .where(WikibaseSoftwareModel.software_type == software_type) + .subquery() + ) + + query = select( + next_subquery.c.software_name, + next_subquery.c.version, + next_subquery.c.version_date, + next_subquery.c.version_hash, + func.count().label("wikibase_count"), # pylint: disable=not-callable + ).group_by( + next_subquery.c.software_name, + next_subquery.c.version, + next_subquery.c.version_date, + next_subquery.c.version_hash, ) return query diff --git a/resolvers/get_aggregate_statistics.py b/resolvers/get_aggregate_statistics.py index 3759caa9..14ffde62 100644 --- a/resolvers/get_aggregate_statistics.py +++ b/resolvers/get_aggregate_statistics.py @@ -1,7 +1,5 @@ """Get Aggregate Statistics""" -from typing import Tuple - from sqlalchemy import Select, and_, select, func from data import get_async_session @@ -41,7 +39,7 @@ async def get_aggregate_statistics() -> WikibaseStatisticsAggregateStrawberryMod def get_total_statistics_query() -> ( - Select[Tuple[int, int, int, int, int, int, int, int, int]] + Select[tuple[int, int, int, int, int, int, int, int, int]] ): """Get Total Statistics Query""" diff --git a/resolvers/get_aggregate_users.py b/resolvers/get_aggregate_users.py index 4ff46bf8..df6f8b9a 100644 --- a/resolvers/get_aggregate_users.py +++ b/resolvers/get_aggregate_users.py @@ -1,7 +1,5 @@ """Get Aggregate Users""" -from typing import Tuple - from sqlalchemy import Select, and_, or_, select, func from data import get_async_session @@ -36,7 +34,7 @@ async def get_aggregate_users() -> WikibaseUserAggregateStrawberryModel: ) -def get_total_admin_query() -> Select[Tuple[int]]: +def get_total_admin_query() -> Select[tuple[int, int]]: """Get Total Admin Query""" rank_subquery = ( @@ -90,7 +88,7 @@ def get_total_admin_query() -> Select[Tuple[int]]: return query -def get_total_user_query() -> Select[Tuple[int, int]]: +def get_total_user_query() -> Select[tuple[int, int]]: """Get Total User Query""" rank_subquery = ( diff --git a/resolvers/get_software_list.py b/resolvers/get_software_list.py new file mode 100644 index 00000000..2f9f1d2a --- /dev/null +++ b/resolvers/get_software_list.py @@ -0,0 +1,47 @@ +"""Get Wikibase List""" + +from sqlalchemy import func, select + +from data import get_async_session +from model.database import WikibaseSoftwareModel +from model.enum import WikibaseSoftwareType +from model.strawberry.output import ( + Page, + PageNumberType, + PageSizeType, + WikibaseSoftwareStrawberryModel, +) + + +async def get_software_list( + page_number: PageNumberType, + page_size: PageSizeType, + software_type: WikibaseSoftwareType, +) -> Page[WikibaseSoftwareStrawberryModel]: + """Get Wikibase List""" + + async with get_async_session() as async_session: + total_count = await async_session.scalar( + # pylint: disable=not-callable + select(func.count()) + .select_from(WikibaseSoftwareModel) + .where(WikibaseSoftwareModel.software_type == software_type) + ) + results = ( + await async_session.scalars( + select(WikibaseSoftwareModel) + .where(WikibaseSoftwareModel.software_type == software_type) + .order_by( + WikibaseSoftwareModel.software_type, + WikibaseSoftwareModel.software_name, + ) + .offset((page_number - 1) * page_size) + .limit(page_size) + ) + ).all() + return Page.marshal( + page_number, + page_size, + total_count, + [WikibaseSoftwareStrawberryModel.marshal(c) for c in results], + ) diff --git a/tests/mock_info.py b/tests/mock_info.py new file mode 100644 index 00000000..aaeca2fd --- /dev/null +++ b/tests/mock_info.py @@ -0,0 +1,26 @@ +"""MockInfo""" + +from typing import List + + +class MockBackgroundClassList: + """Mock Backround Class List""" + + tasks: List + + def add_task(self, task): + """Add Background Task""" + + self.tasks.append(task) + + def __init__(self): + self.tasks = [] + + +class MockInfo: + """Mock StrawberryInfo""" + + context: dict + + def __init__(self, context: dict): + self.context = context diff --git a/tests/test_create_observation/test_create_connectivity_observation.py b/tests/test_create_observation/test_create_connectivity_observation.py index 647f6a51..bc7603e0 100644 --- a/tests/test_create_observation/test_create_connectivity_observation.py +++ b/tests/test_create_observation/test_create_connectivity_observation.py @@ -1,6 +1,5 @@ """Test create_connectivity_observation""" -from typing import List from urllib.error import HTTPError import pytest from fetch_data import create_connectivity_observation @@ -33,7 +32,7 @@ ], ) async def test_create_connectivity_observation_success( - mocker, links: List[tuple[str, str]] + mocker, links: list[tuple[str, str]] ): """Test""" diff --git a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py index 7a4f3926..dc95f3d0 100644 --- a/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py +++ b/tests/test_create_observation/test_create_log_observation/test_create_log_observation.py @@ -107,10 +107,13 @@ def mockery(*args, **kwargs): match (query.params.get("ledir"), query.params.get("lelimit")): case ("newer", 1): return MockResponse( - 200, json.dumps({"query": {"logevents": [oldest_mock_log]}}) + query.raw_url, + 200, + json.dumps({"query": {"logevents": [oldest_mock_log]}}), ) case ("newer", 500): return MockResponse( + query.raw_url, 200, json.dumps( { @@ -124,10 +127,13 @@ def mockery(*args, **kwargs): ) case ("older", 1): return MockResponse( - 200, json.dumps({"query": {"logevents": [newest_mock_log]}}) + query.raw_url, + 200, + json.dumps({"query": {"logevents": [newest_mock_log]}}), ) case ("older", 500): return MockResponse( + query.raw_url, 200, json.dumps( { @@ -149,7 +155,9 @@ def mockery(*args, **kwargs): users.append({"name": "User:B", "invalid": True}) if "User:C" in query.params.get("ususers"): users.append({"name": "User:C", "groups": ["*", "users", "bot"]}) - return MockResponse(200, json.dumps({"query": {"users": users}})) + return MockResponse( + query.raw_url, 200, json.dumps({"query": {"users": users}}) + ) raise NotImplementedError(query) mocker.patch( diff --git a/tests/test_create_observation/test_create_software_version_observation/__init__.py b/tests/test_create_observation/test_create_software_version_observation/__init__.py new file mode 100644 index 00000000..4632e8bc --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/__init__.py @@ -0,0 +1 @@ +"""Test Create Software Version Observation""" diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Babel.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Babel.html new file mode 100644 index 00000000..41e72532 --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Babel.html @@ -0,0 +1,1136 @@ + + + + +Extension:Babel - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Extension:Babel

+
+
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
Babel
+Release status: stable
Implementation Parser function +
Description Adds a parser function to inform other users about language proficiency and categorize users of the same levels and languages.
Author(s) Robert Leverington (RobertLtalk)
Latest version Continuous updates
Compatibility policy Master maintains backward compatibility.
MediaWiki >= 1.41.0
Database changes Yes
Composer mediawiki/babel
Tables babel
License GNU General Public License 2.0 or later
DownloadIncluded in Language Extension Bundle +
Example Translatewiki.net
+ +
+
  • $wgBabelMainCategory
  • +
  • $wgBabelAutoCreate
  • +
  • $wgBabelDefaultLevel
  • +
  • $wgBabelCategorizeNamespaces
  • +
  • $wgBabelUseUserLanguage
  • +
  • $wgBabelCentralDb
  • +
  • $wgBabelCategoryNames
  • +
  • $wgBabelUseCommunityConfiguration
  • +
  • $wgBabelAllowOverride
+ + +
Quarterly downloads63 (Ranked 74th)
Public wikis using2,416 (Ranked 192nd)
+Translate the Babel extension if it is available at translatewiki.net
+Issues Open tasks · Report a bug
+

The Babel extension adds a parser function to replace the old Babel system that completely relied on templates. If an unrecognized language parameter is specified, it will see if there is an existing template with the name and include that. +

On Wikimedia projects, the noun Babel (in reference to the Tower of Babel) refers to the texts on user pages aiding multilingual communication by making it easier to contact someone who speaks a certain language. The idea originated on the Wikimedia Commons and has also been implemented on many other wikis. +

+ +

Installation

+
  • Download and move the extracted Babel folder to your extensions/ directory.
    Developers and code contributors should install the extension from Git instead, using:cd extensions/
    git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Babel
  • +
  • Add the following code at the bottom of your LocalSettings.php file:
    wfLoadExtension( 'Babel' );
    +
  • +
  • Run the update script which will automatically create the necessary database tables that this extension needs.
  • +
  • Configure as required.
  • +
  • Yes Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+
+
  • The CSS is located in the file resources/ext.babel.css. You can change the style as desired by overriding them in the page MediaWiki:Common.css.
  • +
  • If the CLDR extension is found language names are taken from that (where translations are unavailable), otherwise built in MediaWiki language names and English defaults are used.
+

Usage

+

Syntax for the #babel parser function is as follows: +

+
{{#babel: babelcode1 | babelcode2 | ... }}
+
+

Add one of the following codes for each language you speak or understand, separated by |, where xx is the MediaWiki language code, ISO 639-1 code, or ISO 639-3 code for the language. The general usage of each code level is as follows: +

+
xx-0
+
If you don't understand the language at all.
+
xx-1
+
Basic ability—enough to understand written material or simple questions in this language.
+
xx-2
+
Intermediate ability—enough for editing or discussions.
+
xx-3
+
Advanced level—though you can write in this language with no problem, some small errors might occur.
+
xx-4
+
"Near-native" level—although it's not your first language from birth, your ability is something like that of a native speaker.
+
xx-5
+
Professional proficiency.
+
xx or xx-N
+
Native speakers who use a language every day and have a thorough grasp of it, including colloquialisms and idioms.
+

To include any other template, add the name of the template, e.g., add User CSS if you want to include Template:User CSS. A prefix or suffix may be added to template names (e.g., User at the beginning) depending on the local configuration. This can be used to restrict the selection and reduce the length of parameters; for example, CSS could include Template:User CSS if configured in such a way. +

+

Parameters

+

To remove the header and footer, use plain=1 as the first parameter, e.g., {{#babel: plain=1 | babelcode1 | babelcode2 | ... }}. +This makes it easier to use Babel with other userboxes. +

To hide categories, use the nocat=1 parameter as the first parameter, e.g., {{#babel: nocat=1 | babelcode1 | babelcode2 | ... }}. +

Please note that only one of the parameters above is allowed. +At the moment, it is not possible to use both parameters; for example, {{#babel: nocat=1 | plain=1 | babelcode1 | babelcode2 | ... }} will not work. +

+

Categorization

+

If categorization is enabled, the extension creates categories using the Babel AutoCreate bot with the text specified in MediaWiki:babel-autocreate-text-levels and MediaWiki:babel-autocreate-text-main. With basic settings, the categories that the bot creates are not categorized, and to fix this, it is recommended to do the following: +

+
  1. Create a template {{Babel category }} that will generate categories.
  2. +
  3. Replacing text on MediaWiki:babel-autocreate-text-levels with
    {{Babel category|level=$1|language=$2|ISO=$3}}
    +
  4. +
  5. Replacing text on MediaWiki:babel-autocreate-text-main with
    {{Babel category|language=$1|ISO=$2}}
    +
+

This will allow you to categorize categories automatically, and if something happens, you can simultaneously replace the categorization and text in all categories. +

+

Configuration

+

Configuration parameters

+

Babel has several configuration parameters which can be modified in LocalSettings.php . +

+
$wgBabelLanguageCodesCdb
+
(string) the path of the language code database file, the default should suffice.
+
$wgBabelLanguageNamesCdb
+
(string) the path of the language name database file, the default should suffice.
+
$wgBabelCategoryNames
+
(array of string or boolean, indexed by the strings "1", "2", … "5", "N") where each entry is the name of a category for the skill level indicated by its index, possible variable elements are: %code% (language code), %wikiname% (the name of the language in the wiki's content language), and %nativename% (the name of the language in its language). To disable adding a category for a particular level, set the corresponding value to false.
+
For example:
+
$wgBabelCategoryNames = [
+	'0' => 'User %code%-0',
+	'1' => 'User %code%-1',
+	'2' => 'User %code%-2',
+	'3' => 'User %code%-3',
+	'4' => 'User %code%-4',
+	'5' => 'User %code%-5',
+	'N' => 'User %code%-N',
+];
+
+
will use categories like "Category:User en-0" and "Category:User fr-N". The default is just "Category:Fr-N" and so on.
+
$wgBabelMainCategory
+
(string) Name of the main (non-level) category for each language to which all users of that language are added. Set to false to disable; defaults to format "Category:Fr". It accepts the same format as $wgBabelCategoryNames above. Example: +
$wgBabelMainCategory = 'User %code%';
+
$wgBabelDefaultLevel
+
(string) Default ability level to use when none is specified, should be an index from $wgBabelCategoryNames, that is one of the strings "1", "2", … "5", "N". Default is "N".
+
$wgBabelUseUserLanguage
+
(boolean) Whether to use the user interface language for the header and footer message. If false (default), it will be in the page content language. This is because using the user interface language may fragment the parser cache.
+
$wgBabelCategorizeNamespaces
+
Array of namespaces to only add automatic categorization to. For example, if $wgBabelCategorizeNamespaces = [ NS_USER ];, then Babel will only add categories to pages in the user namespace. The default is null, which means categorizing all namespaces.
+
$wgBabelCategoryOverride
+
Whether to allow Babel categories to be overridden on the wiki using MediaWiki:Babel-category-override
+
$wgBabelAutoCreate
+
Whether to auto-create categories.
+

System messages

+

Several customizations can also be made using MediaWiki namespace messages. +

+
MediaWiki:babel-template "Template:User $1"
+
The format of template names when one is being included.
+
MediaWiki:babel-portal ""
+
The format of the link's target from the language code. Set to the empty string to not link the language code.
+
MediaWiki:Babel-autocreate-user "Babel AutoCreate"
+
Username to be used for auto-creation of Babel related categories
+
MediaWiki:babel-autocreate-text-levels "Users in this category indicate they have skill level $1 for language $2."
+
Text to insert into auto-created categories for different language levels. You have to change this if you want them to be auto-categorized in the main category of the respective language ($wgBabelMainCategory).
+
MediaWiki:babel-autocreate-text-main "Users in this category indicate their knowledge of language $1."
+
Text to insert into auto-created categories for non-level categories. You have to change this if you want them to be auto-categorized in a parent category for all languages.
+
MediaWiki:babel "Babel user information"
+
The header of the babel box. Set to - to not display a header.
+
MediaWiki:babel-url "m:User language"
+
The page name where information on the babel extension can be found. Set to - to display no link in the header.
+
MediaWiki:Babel-footer ""
+
The footer of the babel box. Set to - to not display a footer.
+
MediaWiki:babel-footer-url " +:Category:Babel - Users by language"
+
The page to link to in the footer of the babel box
+
MediaWiki:Babel-category-override "$1"
+
Overrides any automatically-generated Babel categories. Parameters:
$1 = the category that would be generated normally.
$2 = the language code
$3 = the babel level.
Any categories overridden using this method will not be auto-created to reduce the risk of vandalism or mistaken edits to that page.
+

API

+

meta=babel (bab)

(main | query | babel)
+
+

Get information about what languages the user knows +

+
Specific parameter:
Other general parameters are available.
babuser

User to get information about +

This parameter is required.
Type: user, by any of username, IP, Temporary user, IP range and interwiki name (e.g. "prefix>ExampleName")
+
Example:
Get the Babel information for user Example
api.php?action=query&meta=babel&babuser=Example [open in sandbox]
+


+

+ + + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_GoogleAnalyticsIntegration.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_GoogleAnalyticsIntegration.html new file mode 100644 index 00000000..14062f2a --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_GoogleAnalyticsIntegration.html @@ -0,0 +1,850 @@ + + + + +Extension:Google Analytics Integration - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Extension:Google Analytics Integration

+
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
Google Analytics Integration
+Release status: stable
Implementation User activity , Hook +
Description Automatically inserts Google Universal Analytics (and/or other web analytics) tracking code at the bottom of MediaWiki pages
Author(s) Tim Laqua, Dāvis Mošenkovs
Latest version 3.0.1 (2017-10-29)
MediaWiki 1.23+
PHP 5.4+
Database changes No
License GNU General Public License 2.0 or later
Download
Example Projects by Davis Mosenkovs (view page source at the bottom)
+ +
+

+

+
  • $wgGoogleAnalyticsAccount
  • +
  • $wgGoogleAnalyticsOtherCode
  • +
  • $wgGoogleAnalyticsAnonymizeIP
  • +
  • $wgGoogleAnalyticsIgnoreNsIDs
  • +
  • $wgGoogleAnalyticsIgnorePages
  • +
  • $wgGoogleAnalyticsIgnoreSpecials
+ +
+
  • noanalytics
+ + +
Public wikis using1,302 (Ranked 194th)
+Translate the Google Analytics Integration extension
+

The Google Analytics Integration extension inserts Google Universal Analytics (and/or other web analytics) tracking code in every page viewed. +Exclusion of specific pages, namespaces, special pages and all pages for specific groups of users is configurable. +

This extension may be outdated for your needs. If you are using the new Global Site Tag for Google Analytics, try using Extension:HeadScript instead. +

Alternatively you can put Google Analytics tag directly into MediaWiki:Common.js of your wiki. +

+

Installation

[edit]
+
  • Download and move the extracted googleAnalytics folder to your extensions/ directory.
    Developers and code contributors should install the extension from Git instead, using:cd extensions/
    git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/googleAnalytics
  • +
  • Add the following code at the bottom of your LocalSettings.php file:
    require_once "$IP/extensions/googleAnalytics/googleAnalytics.php";
    +// Replace xxxxxxx-x with YOUR GoogleAnalytics UA number
    +$wgGoogleAnalyticsAccount = 'UA-xxxxxxx-x'; 
    +
    +// Optional configuration (for defaults see googleAnalytics.php)
    +
    +// Add HTML code for any additional web analytics (can be used alone or with $wgGoogleAnalyticsAccount)
    +$wgGoogleAnalyticsOtherCode = '<script type="text/javascript" src="https://analytics.example.com/tracking.js"></script>';
    +
    +// Store full IP address in Google Universal Analytics (see https://support.google.com/analytics/answer/2763052?hl=en for details)
    +$wgGoogleAnalyticsAnonymizeIP = true; 
    +
    +// Array with NUMERIC namespace IDs where web analytics code should NOT be included.
    +$wgGoogleAnalyticsIgnoreNsIDs = [
    +    500
    +    ];
    +
    +// Array with page names (see magic word {{FULLPAGENAME}}) where web analytics code should NOT be included.
    +$wgGoogleAnalyticsIgnorePages = [
    +    'PageName',
    +    'NamespaceName:PageName'
    +    ];
    +    
    +// Array with special pages where web analytics code should NOT be included.
    +$wgGoogleAnalyticsIgnoreSpecials = [
    +    'Userlogin',
    +    'Userlogout',
    +    'Preferences',
    +    'ChangePassword',
    +    'OATH'
    +    ];
    +    
    +// Use 'noanalytics' permission to exclude specific user groups from web analytics, e.g.
    +$wgGroupPermissions['sysop']['noanalytics'] = true;
    +$wgGroupPermissions['bot']['noanalytics'] = true;
    +
    +// To exclude all logged in users, give 'noanalytics' permission to the 'user' group, i.e.
    +$wgGroupPermissions['user']['noanalytics'] = true;
    +
  • +
  • Yes Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+
The following options were removed in version 3.0.0: +
  • $wgGoogleAnalyticsAddASAC
  • +
  • $wgGoogleAnalyticsIgnoreSysops
  • +
  • $wgGoogleAnalyticsIgnoreBots
+

Usage

[edit]
+
  1. Create a Google Analytics account.
  2. +
  3. Locate your UA number. +
    • For the legacy code block, it can be found on the following line:
      _uacct="UA-xxxxxxx-x";
      +
    • +
    • For the new ga.js code block, it can be found on the following line:
      var pageTracker = _gat._getTracker("UA-xxxxxxx-x");
      +
  4. +
  5. Follow installation instructions.
  6. +
  7. Google Analytics stats should start populating within 24-48 hours.
+

See also

[edit]
+ + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_LabeledSectionTransclusion.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_LabeledSectionTransclusion.html new file mode 100644 index 00000000..ce854763 --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_LabeledSectionTransclusion.html @@ -0,0 +1,1214 @@ + + + + +Extension:Labeled Section Transclusion - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+ +
+
+
+ +

Extension:Labeled Section Transclusion

+
+
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
Labeled Section Transclusion
+Release status: stable
Implementation Tag , Parser function +
Description Enables marked sections of text to be transcluded
Author(s) Steve Sanbeg
Compatibility policy Snapshots releases along with MediaWiki. Master is not backward compatible.
MediaWiki 1.19+
Database changes No
License GNU General Public License 2.0 or later
Download
+ + +
Public wikis using6,919 (Ranked 26th)
+Translate the Labeled Section Transclusion extension if it is available at translatewiki.net
+Issues Open tasks · Report a bug
+

The Labeled Section Transclusion extension allows selective transclusion of marked-off sections of text, parsing wikitext as normal. Its functionality is similar to an enhanced version of the ‎<onlyinclude> tag with normal wiki transclusion, which selects sections for inclusion. It is enabled on all Wikimedia wikis. +

While normal transclusion is primarily intended to transclude large portions of small templates, labeled section transclusion is intended for small portions of large pages. +

However, there are some differences. In the native template transclusion, sections are marked by behavior; thus you can have only one (possibly non-contiguous) section to be included or skipped. +

Here, sections are marked by name, and behavior is chosen by the caller, which can include or skip sections as needed. Different pages can include or exclude selected sections; there can be arbitrary numbers of sections, which can also overlap arbitrarily. +

Marking sections by name rather than behavior allows edit section links to be rendered more appropriately for getting excerpts from larger texts, since the extension can now account for sections that are skipped in the beginning of the page, allowing transcluded sections to be offset appropriately. +

+ +

Installation

[edit]
+
  • Download and move the extracted LabeledSectionTransclusion folder to your extensions/ directory.
    Developers and code contributors should install the extension from Git instead, using:cd extensions/
    git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/LabeledSectionTransclusion
  • +
  • Add the following code at the bottom of your LocalSettings.php file:
    wfLoadExtension( 'LabeledSectionTransclusion' );
    +
  • +
  • Yes Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+

There is also a Gadget in use on Wikisource.org wikis that makes it possible to define sections with a simplified ## label ## syntax. +Its code can be found at Wikisource:MediaWiki:Gadget-Easy_LST.js. +

+

Functions

[edit]
+

Transclude any marked part

[edit]
+

Step 1: Mark off sections

[edit]
+

Mark off sections in the text using ‎<section> tags like this: +

+
<section begin="chapter1" />this is chapter 1<section end="chapter1" />
+
+

Note that these tags are not HTML/XML, and do not use the normal attribute syntax. +For this reason, and because the begin and end markers are individual, rather than normal XML open/close tags, this allows nested or overlapping sections. This allows you to insert section tags without worrying about interfering with other sections. +


+

+

Step 2a: Transclude the section

[edit]
+

Call the parser function #lst to transclude it, i.e. to transclude a section called chapter1 from a page called articleX: +

+
{{#lst:articleX|chapter1}}
+
+

The target article defines the location of the section; its behavior is determined by the parser function. +

+

Step 2b: Transclude the page but excluding the section

[edit]
+

To transclude a page, but exclude a specified section, use the #lstx function: +

+
{{#lstx:articleX|chapter1}}
+
+

Optionally, you may add replacement text to the excluded section. +

+
{{#lstx:articleX|chapter1|replacement_text}}
+
+

Example: +

+
{{#lstx:articleX|chapter1|See chapter 1 in [[articleX]].}}
+
+

The replacement text will appear in the area where the section is skipped (excluded). +

+

Discontiguous sections

[edit]
+

It is possible to have multiple sections with the same name; in this case, every section with that name will be included/excluded. This is especially useful to mark various discussions. +

+

Section ranges

[edit]
+

These functions have an additional, optional argument to specify a section range; i.e. {{#lst:articleX|chapter1|chapter3}}, to include everything from the beginning of chapter 1 to the end of chapter 3. This allows using empty marker pairs to mark one end of the section, possibly in a template. A similar mechanism is currently used at the French Wikisource. +

+

Substitution

[edit]
+

This also works with substitution; it's even possible for an article to substitute a section of itself. One use of this provides a neat way to archive talk pages: Mark the text to be archived using <section begin=archive />, etc. Then create an archive page with the text, using {{subst:#lst:talk_page|archive}}, which copies archived sections. Lastly, replace the contents of talk_page with {{subst:#lstx:talk_page|archive}} to remove those sections. +

There is optional support for transcluding sections of text marked with the normal headings, i.e. ==this section==. If installed, this is done with the lsth function. +

+

Transclude before the first heading

[edit]
+

To transclude the introduction of a page (i.e. the content before the first heading), use +

+
{{#lsth:pagename}}
+
+

Transclude a specific section

[edit]
+

You can also transclude the whole content of the sectionX (which includes all its sub-sections but excludes the heading of sectionX itself). +

+
{{#lsth:pagename|sectionX}}
+
+

Things to note: +

+
  1. Only the first occurrence of the sectionX is transcluded if you have more than one section with the same name.
  2. +
  3. Make sure you type what the heading of sectionX is in wikitext, not how it is displayed. For example if the heading of the section is ==List of [[Extension]]==, you should type "List of [[Extension]]" not "List of Extension".
  4. +
  5. When transcluding a section from a page marked for translation using the translate extension, transclude from the language-specific version. E.g. from pagename/en rather than from pagename.
  6. +
  7. The matching is case insensitive, to prevent links from breaking due to case changes.
+

Transclude multiple sections

[edit]
+

You can also transclude from the first occurrence of sectionX (excluding the heading of sectionX itself) until it reaches the next occurrence of sectionY. Note that sectionY acts as a stop point so the transclusion doesn't contain the content of sectionY. +

+
{{#lsth:pagename|sectionX|sectionY}}
+
+

Notes about skipped headings

[edit]
+

Since the traditional transclusion in MediaWiki isn't intended to transclude sections, it doesn't account for skipped headings. As a result, if you were to transclude a template with multiple headings, and skip the first heading, then all of the edit sections links would point to the wrong section in the template. +

When this extension is used (with MediaWiki 1.9 or later), the #lst and #lsth functions count headings in the "skipped" beginning part, and offset transcluded headings appropriately. This will allow these links to point to the correct section in the simple case. +

Note that #lstx does not count skipped headings, and that skipped headings within discontiguous sections are not offset. But it seems it has been fixed now (likely when ported to MediaWiki's new preprocessor). The transcluded headings can be linked to the correct sections. +

+

Localisation

[edit]
+

Internally, the parser functions all use the lst prefix, for consistency with the name of the extension. Since this acronym may be confusing to non-developers, readable English variants have been introduced, so the functions can currently be called from either name. +

+ + + + + + + + + + + + + + + + + + + + + + + + +
function +English +German +Hebrew (RTL) +Portuguese +
#lst#section#Abschnitt
#קטע
#trecho +
#lstx#section-x#Abschnitt-x
#בלי קטע
#trecho-x +
#lsth#section-h +
+

Additionally, the tag can now be localised; currently: English, German, Hebrew, Portuguese; i.e.: +

+
English
+
<section begin=x/> ... <section end=x/>
+
German
+
<Abschnitt Anfang=x/> ... <Abschnitt Ende=x/>
+
Hebrew (RTL)
+
<קטע התחלה=א> ... <קטע סוף=א> ("start" code to the right and "end" code to the left)
+
Portuguese
+
<trecho começo=x/> ... <trecho fim=x/>
+

Each localization is enabled only if the page matches the respective content language. +

+

Limitations

[edit]
+
  • {{#lsth:pagename|sectionX}} only works on the first section if multiple sections have name sectionX. Only the first occurrence of sectionX is transcluded if an article has more than one section with the same name.
  • +
  • While it is possible to use this extension across namespaces, interwiki references are not resolved. It is not yet possible, for example, to include part of a Wikisource page into a remote MediaWiki installation.
  • +
  • Section tags cannot themselves be transcluded in order to work on other pages. {{#lst:}} and {{#lstx:}} work only if section tags appear directly in the wikitext of the transcluded page. This means, for instance, that these tags cannot be embedded in a template using template parameters and parser functions. The #tag magic word does not work with section tags.
  • +
  • As of 2014, section tags don't have any effect when used inside a template parameter. If page A contains a text {{B|X}}, there's no way {{#lst:A|...}} can access X.
+

Examples

[edit]
+ +

See also

[edit]
+ + + + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_ProofreadPage.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_ProofreadPage.html new file mode 100644 index 00000000..6474d8c5 --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_ProofreadPage.html @@ -0,0 +1,1205 @@ + + + + +Extension:Proofread Page - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Extension:Proofread Page

+
+
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
Proofread Page
+Release status: stable
Implementation Page action , ContentHandler , Tag , API , Database +
Description The Proofread Page extension can render a book either as a column of OCR text beside a column of scanned images, or broken into its logical organization (such as chapters or poems) using transclusion.
Author(s) ThomasV (original author)
Tpt (current maintainer)
Latest version continuous updates
Compatibility policy Snapshots releases along with MediaWiki. Master is not backward compatible.
MediaWiki current master
PHP 7.0+
Database changes Yes
Composer mediawiki/proofread-page
Tables pr_index
License GNU General Public License 2.0 or later
Download
Help Help:Extension:ProofreadPage
Example s:Index:Wind in the Willows (1913).djvu
+ +
+Page, Index
+ +
+
  • $wgProofreadPagePageJoiner
  • +
  • $wgProofreadPagePageSeparator
  • +
  • $wgProofreadPageNamespaceIds
  • +
  • $wgProofreadPageEnableEditInSequence
  • +
  • $wgProofreadPageBookNamespaces
  • +
  • $wgProofreadPageUseStatusChangeTags
  • +
  • $wgProofreadPagePageSeparatorPlaceholder
+ +
+

+

+
  • pagequality
  • +
  • pagequality-admin
  • +
  • pagequality-validate
+ + +
+Translate the Proofread Page extension if it is available at translatewiki.net
+Issues Open tasks · Report a bug
+
+
+
+

Proofread Page extension

+

+2020 Coolest Tool
Award Winner
+

+

+in the category +
+Impact +

+
+
+


+The Proofread Page extension creates a book either: +

+
  1. as a column of OCR text beside a column of scanned images, or
  2. +
  3. broken into chapters or poems. The content of a document appears in the MediaWiki page (via transclusion).
+

The extension is intended to allow easy comparison of text to the original digitization. +

This extension shows the text in several ways without actually duplicating the original text.[1] +

+ +

Use

[edit]
+

The extension is installed on all Wikisource wikis. +For the syntax, see the Wikisource Proofread Page documentation. +It was previously also used on Bibliowiki. +

+

Requirements and recommendations

[edit]
+ +


+

+

Installation

[edit]
+

Extension

[edit]
+
  • Download and move the extracted ProofreadPage folder to your extensions/ directory.
    Developers and code contributors should install the extension from Git instead, using:cd extensions/
    git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/ProofreadPage
  • +
  • Add the following code at the bottom of your LocalSettings.php file:
    wfLoadExtension( 'ProofreadPage' );
    +
  • +
  • Run the update script which will automatically create the necessary database tables that this extension needs.
  • +
  • Yes Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+

Thumbnailing

[edit]
+

The extension links directly to image thumbnails which often don't exist. +You must catch 404 errors and generate the missing thumbnails. +You can do this with any one of these solutions: +

+
  • Set an Apache RewriteRule in .htaccess to thumb.php for missing thumbnails:
+
    RewriteEngine On
+    RewriteCond %{REQUEST_FILENAME} !-f
+    RewriteCond %{REQUEST_FILENAME} !-d
+    RewriteRule ^/w/images/thumb/[0-9a-f]/[0-9a-f][0-9a-f]/([^/]+)/page([0-9]+)-?([0-9]+)px-.*$ /w/thumb.php?f=$1&p=$2&w=$3 [L,QSA]
+
+
  • or set the Apache 404 handler to Wikimedia's thumb-handler. This is a general-purpose 404 handler with Wikimedia-specific code, not simply a thumbnail generator.
+
    ErrorDocument 404 /w/extensions/upload-scripts/404.php
+
+
  • For MediaWiki >= 1.20, you can simply redirect to thumb_handler.php:
+
    RewriteEngine On
+    RewriteCond %{REQUEST_FILENAME} !-f
+    RewriteCond %{REQUEST_FILENAME} !-d
+    RewriteRule ^/w/images/thumb/[0-9a-f]/[0-9a-f][0-9a-f]/([^/]+)/page([0-9]+)-?([0-9]+)px-.*$ /w/thumb_handler.php [L,QSA]
+
+
  • or in apache2.conf:
+
    ErrorDocument 404 /w/thumb_handler.php
+
+
Warning Warning: There is an .htaccess file in the images directory that may interfere with any .htaccess rules you install.
+

If you encounter a problem similar to the following: +

+
  • phab:T301291 – PDF and DjVu files on Commons failed to be processed (no thumbnails, zero pages) but otherwise valid
  • +
  • phab:T298417 – Undeleted DjVu files show incorrect metadata: 0x0 size, no page number info
  • +
  • phab:T299521 – PDF file has 0x0 image size in Commons after uploading a new version while the page number is correct
+

Try next steps: +

+
    +
  1. repair thumbnails for DjVu files of the core MediaWiki (for PDF use mimetype application/pdf ) + +
    php maintenance/refreshImageMetadata.php --verbose --mime image/vnd.djvu --force
    +
    +
  2. + +
  3. needed for actualization info about the pages counts of the Special:IndexPages + +
    php maintenance/refreshLinks.php --namespace 252
    +
    +
  4. +
+

Namespaces

[edit]
+

ProofreadPage create by default two custom namespaces named "Page" and "Index" in English with respectively ids 250 and 252. +

Their names are translated if your wiki use another language. +Full list. +

You can customize their name or their ID: Create namespaces by hand and set their IDs in Manual:LocalSettings.php using $wgProofreadPageNamespaceIds global. +You will do something like: +

+
define( 'NS_PROOFREAD_PAGE', 250);
+define( 'NS_PROOFREAD_PAGE_TALK', 251);
+define( 'NS_PROOFREAD_INDEX', 252);
+define( 'NS_PROOFREAD_INDEX_TALK', 253);
+$wgExtraNamespaces[NS_PROOFREAD_PAGE] = 'Page';
+$wgExtraNamespaces[NS_PROOFREAD_PAGE_TALK] = 'Page_talk';
+$wgExtraNamespaces[NS_PROOFREAD_INDEX] = 'Index';
+$wgExtraNamespaces[NS_PROOFREAD_INDEX_TALK] = 'Index_talk';
+$wgProofreadPageNamespaceIds = array(
+    'index' => NS_PROOFREAD_INDEX,
+    'page' => NS_PROOFREAD_PAGE
+);
+
+

Namespace id customization is not recommended and might not be supported in the future. +

+

Configuration

[edit]
+ +

Configuration of index namespace

[edit]
+

For more details, see Extension:Proofread Page/Index data configuration +

+ +

The configuration is a JSON array of properties. +Here is the structure of a property in the array, all the parameters are optional, the default value are set: +

+
{
+  "ID": { //id of the metadata (first parameter of proofreadpage_index_attributes)
+    "type": "string", //the property type (for compatibility reasons the values have not to be of this type). Possibles values: string, number, page. If set, the newly set values should be valid according to the type (e.g. for a number a valid number, for a page an existing wiki page...)
+    "size": 1, //only for the type string : number of lines of the input (third parameter of proofreadpage_index_attributes)
+    "values":  {"a":"A", "b":"B","c":"C", "d":"D"}, //an array values : label that list the possible values (for compatibility reasons the stored values have not to be one of these)
+    "default": "", //the default value
+    "header": false, //add the property to MediaWiki:Proofreadpage_header_template template (true is equivalent to being listed in proofreadpage_js_attributes)
+    "label": "ID", //the label in the form (second parameter of proofreadpage_index_attributes)
+    "help": "", //a short help text
+    "delimiter": [], //list of delimiters between two part of values. By example ["; ", " and "] for strings like "J. M. Dent; E. P. Dutton and A. D. Robert"
+    "data": "" //proofreadpage's metadata type that the property is equivalent to
+  }
+}
+
+

The data parameter can have for value: +"type", "language", "title", "author", "translator", "illustrator", "editor", "school", "year", "publisher", "place", "progress" +

+

Page separator

[edit]
+

The extension puts a separator between every transcluded page and the next, which is defined by wgProofreadPagePageSeparator. +The default value is &#32; (a whitespace). +Set wgProofreadPagePageSeparator = "" to suppress the separator. +

+

Join hyphenated words across pages

[edit]
+

When a word is hyphenated between a page and the next, the extension joins together the two halves of the word. +Example: his- and tory becomes history. +The "joiner" character is defined by wgProofreadPagePageJoiner and defaults to '-' (the ASCII hyphen character). +

+

Configure change tagging (optional)

[edit]
+

See Change tagging to set up change tags. +

+

Usage

[edit]
+

Creating your first page (example with DjVu)

[edit]
+
  • Before following these steps ensure you have followed the instructions in Manual:How to use DjVu with MediaWiki .
  • +
  • (when and in which namespace is the DjVu file itself uploaded?)
  • +
  • Create a page in the "Page" namespace (or the internationalized name if you use an not-English wiki). For example if your namespace is 'Page' create Page:Carroll - Alice's Adventures in Wonderland.djvu
  • +
  • Create the corresponding file for this page commons:File:Carroll - Alice's Adventures in Wonderland.djvu (or set Manual:$wgUseInstantCommons to true).
  • +
  • Create the index page Index:Carroll - Alice's Adventures in Wonderland.djvu +
    • Insert the tag ‎<pagelist /> in the Pages field to visualize the page list
  • +
  • To edit page 5 of the book navigate to 'Page:Carroll - Alice's Adventures in Wonderland/5' and click edit
+

Syntax

[edit]
+

This extension introduces the following tags: +‎<pages>, ‎<pagelist> +

+

Notes

[edit]
+
    +
  1. +Because the pages are not in the main namespace, they are not included in the statistical count of text units. +
  2. +
+

See also

[edit]
+ + + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Scribunto.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Scribunto.html new file mode 100644 index 00000000..8d1b192a --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_Scribunto.html @@ -0,0 +1,1597 @@ + + + + +Extension:Scribunto - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Extension:Scribunto

+
+
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+
This extension comes with MediaWiki 1.34 and above. Thus you do not have to download it again. However, you still need to follow the other instructions provided.
+
This extension runs on top of an executable. You must have permission to run executables on your host in order for this extension to work.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
Scribunto
+Release status: stable
Implementation Parser extension +
Description Provides a framework for embedding scripting languages into MediaWiki pages
Author(s) +
  • Victor Vasiliev
  • +
  • Tim Starling
+
and others
Latest version Continuous updates
Compatibility policy Snapshots releases along with MediaWiki. Master is not backward compatible.
MediaWiki >= 1.43
PHP 5.5+
License GPL-2.0-or-later AND MIT
Download
+ +
+Module (ns:828), Talk_Module (ns:829)
+ +
+
  • $wgScribuntoDefaultEngine
  • +
  • $wgScribuntoSlowFunctionThreshold
  • +
  • $wgScribuntoGatherFunctionStats
  • +
  • $wgScribuntoUseGeSHi
  • +
  • $wgScribuntoUseCodeEditor
  • +
  • $wgScribuntoEngineConf
+ + +
+ + +
Quarterly downloads450 (Ranked 7th)
Public wikis using8,789 (Ranked 30th)
+Translate the Scribunto extension if it is available at translatewiki.net
Vagrant role scribunto
+Issues Open tasks · Report a bug
+

The Scribunto (Latin: "they shall write/let them write (in the future)") extension allows for embedding scripting languages in MediaWiki. +

Currently the only supported scripting language is Lua. +Scribunto Lua scripts go in a namespace called Module. +Modules are run on normal wiki pages using the #invoke parser function and each module has a collection of functions, which can be called using wikitext syntax such as: +

+
{{#invoke: Module_name | function_name | arg1 | arg2 | arg3 ... }}
+
+ +

License

[edit]
+

This extension contains code licensed GNU General Public License v2.0 or later (GPL-2.0+) as well as code licensed MIT License (MIT). +

+

Requirements

[edit]
+

PCRE version compatibility

[edit]
+

PCRE 8.33+ is recommended. +PCRE 8.33 was released in May 2013. +You can see the version of PCRE used by PHP by viewing a phpinfo() web page, or from the command line with the following command: +

+
php -r 'echo "pcre: " . ( extension_loaded( "pcre" ) ? PCRE_VERSION : "no" ) . "\n";'
+
+
  • Scribunto will not work with versions of PCRE lower than 8.10.
  • +
  • PCRE 8.32 has a bug that will cause it to reject certain non-character codepoints, which will cause errors in the mw.html module.
+

CentOS 6 and RHEL 6 are stuck on PCRE 7 and need to be upgraded. +

Updating to 8.33 on a server with an older version may be relatively complicated. +See Updating to PCRE 8.33 or Higher for details. +


+

+

PHP pcntl (LTS)

[edit]
+ + + + +
MediaWiki versions:
1.25 – 1.28
+

Scribunto versions for MediaWiki 1.25 to 1.28 required PHP's pcntl extension, which is only available on Unix/Linux platforms, if you want to use "LuaStandalone" (i.e. running in a separate child process). +This requirement was removed in Scribunto for MediaWiki 1.29. +

You can check whether pcntl support is enabled by viewing a phpinfo() web page, or from the command line with the following command: +

+
php -r 'echo "pcntl: " . ( extension_loaded( "pcntl" ) ? "yes" : "no" ) . "\n";'
+
+

PHP mbstring extension

[edit]
+

PHP needs to have the mbstring extension enabled. +

You can check whether mbstring support is enabled by viewing a phpinfo() web page, or from the command line with the following command: +

+
php -r 'echo "mbstring: " . ( extension_loaded( "mbstring" ) ? "yes" : "no" ) . "\n";'
+
+

Lua binary

[edit]
+

Bundled binaries

[edit]
+

Scribunto comes bundled with Lua binary distributions for Linux (x86 and x86-64), Mac OS X Lion, and Windows (32- and 64-bit). +

Scribunto should work for you out of the box if: +

+
  1. Your web server is run on one of the above platforms.
  2. +
  3. PHP's proc_open function is not restricted.[1]
  4. +
  5. proc_terminate and shell_exec are not disabled in PHP.
  6. +
  7. Your web server is configured to allow the execution of binary files in the MediaWiki tree.
+
Note Note: Execute permissions may need to be set; for example, in Linux use: +
chmod 755 /path/to/extensions/Scribunto/includes/Engines/LuaStandalone/binaries/lua5_1_5_linux_64_generic/lua
+
If you are using SELinux in "Enforcing" mode on your server, you might need to set a proper context for the binaries. Example for RHEL/CentOS 7:
chcon -t httpd_sys_script_exec_t /path/to/extensions/Scribunto/includes/Engines/LuaStandalone/binaries/lua5_1_5_linux_64_generic/lua
+
+

P.S. Check your version of the extension to see if the name of the engines folder is capitalised or fully lowercase.[2] +

+

Additional binaries

[edit]
+

Additional Lua binary distributions, which may be needed for your web server if its operating system is not in the list above, can be obtained from http://luabinaries.sourceforge.net/ or from your Linux distribution. +

Only binary files for Lua 5.1.x are supported. +

Once you've installed the appropriate binary file on your web server, configure the location of the file with: +

+
# Where Lua is the name of the binary file
+# e.g. SourceForge LuaBinaries 5.1.5 - Release 2 name the binary file lua5.1
+$wgScribuntoEngineConf['luastandalone']['luaPath'] = '/path/to/binaries/lua5.1';
+
+

Note that you should not add the above line unless you've confirmed that Scribunto's built-in binaries don't work for you. +

LuaJIT, although theoretically compatible, is not supported. +

The support was removed due to Spectre and bitrot concerns (phab:T184156). +

+

Installation

[edit]
+
  • Download and move the extracted Scribunto folder to your extensions/ directory.
    Developers and code contributors should install the extension from Git instead, using:cd extensions/
    git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Scribunto
  • +
  • Add the following code at the bottom of your LocalSettings.php file:
    wfLoadExtension( 'Scribunto' );
    +$wgScribuntoDefaultEngine = 'luastandalone';
    +
  • +
  • Set execute permissions for the Lua binaries bundled with this extension:
+
chmod a+x /path/to/extensions/Scribunto/includes/Engines/LuaStandalone/binaries/yourOS/lua
+

[2] +

  • Set type to httpd_sys_script_exec_t if SELinux is enforced:
+
chcon -t httpd_sys_script_exec_t /path/to/extensions/Scribunto/includes/Engines/LuaStandalone/binaries/yourOS/lua
+

[2] +

  • Yes Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+


+Vagrant installation: +

+
  • If using Vagrant , install with vagrant roles enable scribunto --provision
+


+

+

Optional installation

[edit]
+

Integrating extensions

[edit]
+

For a more pleasant user interface, with syntax highlighting and a code editor with autoindent, install the following extensions: +

+ + + + + +
MediaWiki version:
1.30
+

Then in your LocalSettings.php after all the extension registrations, add: +

+
$wgScribuntoUseGeSHi = true;
+$wgScribuntoUseCodeEditor = true;
+
+

These settings are not necessary in MediaWiki versions after 1.30, where the code editor and syntax hightlighting should be used in the appropriate places automatically once the extensions are loaded. However, other settings may be desired. See the documentation for each extension. +

+

LuaSandbox

[edit]
+

We have developed a PHP extension written in C called LuaSandbox. +It can be used as an alternative to the standalone binary, and will provide improved performance. +See LuaSandbox for details and installation instructions. +

If you initially installed the extension to use the Lua standalone binary, be sure to update LocalSettings.php with the following configuration setting: +

+
$wgScribuntoDefaultEngine = 'luasandbox';
+
+

Configuration

[edit]
+

The following configuration variables are available: +

+
$wgScribuntoDefaultEngine
+
Select the engine. Valid values are the keys in $wgScribuntoEngineConf, which by default are 'luasandbox' or 'luastandalone'.
+
$wgScribuntoUseGeSHi
+
When Extension:SyntaxHighlight is installed, set this true to use it when displaying Module pages. (MediaWiki 1.30 or earlier.)
+
$wgScribuntoUseCodeEditor
+
When Extension:CodeEditor is installed, set this true to use it when editing Module pages. (MediaWiki 1.30 or earlier.)
+
$wgScribuntoEngineConf
+
An associative array for engine configuration. Keys are the valid values for $wgScribuntoDefaultEngine, and values are associative arrays of configuration data. Each configuration array must contain a 'class' key naming the ScribuntoEngineBase subclass to use.
+

LuaStandalone

[edit]
+

The following keys are used in $wgScribuntoEngineConf for Scribunto_LuaStandaloneEngine. +Generally you'd set these as something like +

+
$wgScribuntoEngineConf['luastandalone']['key'] = 'value';
+
+
luaPath
+
Specify the path to a Lua interpreter.
+
errorFile
+
Specify the path to a file, writable by the web server user, where the error and debugging output from the standalone interpreter will be logged.
+
Error output produced by the standalone interpreter are not logged by default. Configure logging with:
+
$wgScribuntoEngineConf['luastandalone']['errorFile'] = '/path/to/file.log';
+
+
memoryLimit
+
Specify the memory limit in bytes. Default 52428800 (50MB) (enforced using ulimit).
+
cpuLimit
+
Specify the CPU time limit in seconds (enforced using ulimit).
+
allowEnvFuncs
+
Set true to allow use of setfenv and getfenv in modules.
+

LuaSandbox

[edit]
+

The following keys are used in $wgScribuntoEngineConf for Scribunto_LuaSandboxEngine. +Generally you'd set these as something like +

+
$wgScribuntoEngineConf['luasandbox']['key'] = 'value';
+
+
memoryLimit
+
Specify the memory limit in bytes. Default 52428800 (50MB)
+
cpuLimit
+
Specify the CPU time limit in seconds.
+
profilerPeriod
+
Specify the time between polls in sections for the Lua profiler.
+
allowEnvFuncs
+
Set true to allow use of setfenv and getfenv in modules.
+

Usage

[edit]
+

Scripts go in a new namespace called Module. +Each module has a collection of functions, which can be called using wikitext syntax such as: +

+
{{#invoke: Module_name | function_name | arg1 | arg2 | arg3 ... }}
+
+

Lua

[edit]
+

Learning Lua

[edit]
+

Lua is a simple programming language intended to be accessible to beginners. +For a quick crash-course on Lua, try Learn Lua in 15 Minutes. +

The best comprehensive introduction to Lua is the book Programming in Lua. +The first edition (for Lua 5.0) is available online and is mostly relevant to Lua 5.1, the version used by Scribunto: +

+ +

The reference manual is also useful: +

+ +

Lua environment

[edit]
+

In Lua, the set of all global variables and functions is called an environment. +

Each {{#invoke:}} call runs in a separate environment. +Variables defined in one {{#invoke:}} will not be available from another. +This restriction was necessary to maintain flexibility in the wikitext parser implementation. +

+
The environment which scripts run in is not quite the same as in standard Lua. These differences are noted in Extension:Scribunto/Lua reference manual.
+

Debug console

[edit]
+
Debug console usage example
+
See also: Extension:Scribunto/Debug console
+

When editing a Lua module a so-called "debug console" can be found underneath the edit form. +In this debug console Lua code can be executed without having to save or even create the Lua module in question. +

+

Troubleshooting

[edit]
+
Troubleshooting using the clickable "Script error" link.
+

Note that red Script error messages are clickable and will provide more detailed information. +

+

Lua error: Internal error: The interpreter exited with status 1.

[edit]
+

When using the LuaStandalone engine (this is the default), errors along the lines of "Lua error: Internal error: The interpreter exited with status 1." may be generated if the standalone Lua interpreter cannot be executed or runs into various runtime errors. +To obtain more information, assign a file path to $wgScribuntoEngineConf['luastandalone']['errorFile']. +The interpreter's error output will be logged to the specified file, which should prove more helpful in tracking down the issue. +The information in the debug log includes debugging information, which is why there is so much of it. +You should be able to ignore any line beginning with "TX" or "RX". +

If you're setting up Scribunto and are using IIS/Windows, this appears to be solved by commenting out a particular line. +

+

Lua error: Internal error: The interpreter exited with status 2.

[edit]
+

When using the LuaStandalone engine (this is the default), status 2 suggests memory allocation errors, probably caused by settings that allocate inadequate memory space for PHP or Lua, or both. +Assigning a file path to $wgScribuntoEngineConf['luastandalone']['errorFile'] and examining that output can be valuable in diagnosing memory allocation errors. +

Increase PHP allocation in your PHP configuration; add the line memory_limit = 200M. +This allocation of 200MB is often sufficient (as of MediaWiki 1.24) but can be increased as required. +Set Scribunto's memory allocation in LocalSettings.php as a line: +

+
$wgScribuntoEngineConf['luastandalone']['memoryLimit'] = 209715200; # bytes
+
+

Finally, depending on the server configuration, some installations may be helped by adding another LocalSettings.php line +

+
$wgMaxShellMemory = 204800; # in KB
+
+

Note that all 3 memory limits are given in different units. +

+

Lua error: Internal error: 2. on ARM architecture

[edit]
+

If you're using an ARM architecture processor like on a RaspberryPi you'll face the error Lua error: Internal error: The interpreter exited with status 2. due to wrong delivered binary format of the Lua interpreter. +

Check your Lua interpreter in: +

+
/path/to/webdir/Scribunto/includes/Engines/LuaStandalone/binaries/lua5_1_5_linux_32_generic
+
+

Check the interpreter by using: +

+
file lua 
+
+

The result should look like : +

+
lua: ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0
+
+

The installed default Lua interpreter shows: +

+
lua: ELF 32-bit LSB pie executable, Intel 80386, version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.2, for GNU/Linux 2.6.9,
+
+

look at the "Intel 80386" part what definitely is not correct. +

Check in /usr/bin what version of Lua is installed on your system. If you have lua5.1 installed, you can either copy the interpreter to your lua5_1_5_linux_32_generic directory or set in your LocalSettings.php: +

+
$wgScribuntoEngineConf['luastandalone']['luaPath'] = '/usr/bin/lua5.1';
+
+

At present don't set wgScribuntoEngineConf to /usr/bin/lua5.3, it'll lead to the "Internal error 1". +

+

Lua error: Internal error: The interpreter exited with status 24.

[edit]
+

When using the LuaStandalone engine (this is the default), status 24 suggests CPU time limit errors, although those should be generating a "The time allocated for running scripts has expired" message instead. +It would be useful to file a task in Phabricator and participate in determining why the XCPU signal isn't being caught. +

+

Lua error: Internal error: The interpreter exited with status 126.

[edit]
+

When using the LuaStandalone engine (this is the default), errors along the lines of "Lua error: Internal error: The interpreter exited with status 126." may be generated if the standalone Lua interpreter cannot be executed. +This generally arises from either of two causes: +

+
  • The Lua executable file's permissions do not include Execute. Set permissions as described under #Installation.
  • +
  • The server does not allow execution of files from the place where the executable is installed, e.g. the filesystem is mounted with the 'noexec' flag. This often occurs with shared hosted servers. Remedies include adjusting $wgScribuntoEngineConf['luastandalone']['luaPath'] to point to a Lua 5.1 binary installed in an executable location, or adjusting or convincing the shared host to adjust the setting preventing execution.
+

Error condition such as: Fatal exception of type MWException

[edit]
+

Check the MediaWiki, PHP, or webserver logs for more details on the exception, or temporarily set $wgShowExceptionDetails to true. +

+

version 'GLIBC_2.11' not found

[edit]
+

If the above gives you errors such as "version 'GLIBC_2.11' not found", it means the version of the standard C library on your system is too old for the binaries provided with Scribunto. +You should upgrade your C library, or use a version of Lua 5.1 compiled for the C library you do have installed. +To upgrade your C library, your best option is usually to follow your distribution's instructions for upgrading packages (or for upgrading to a new release of the distribution, if applicable). +

If you copy the lua binaries from Scribunto master (or from gerrit:77905), that should suffice, if you can't or don't want to upgrade your C library. +The distributed binaries were recently recompiled against an older version of glibc, so the minimum is now 2.3 rather than 2.11. +

+

Lua errors in Scribunto files

[edit]
+

Errors here include: +

+
  • attempt to index field 'text' (a nil value)
  • +
  • Lua error in mw.html.lua at line 253: Invalid class given:
+

If you are getting errors such these when attempting to use modules imported from WMF wikis, most likely your version of Scribunto is out of date. +

Upgrade if possible; for advanced users, you might also try to identify the needed newer commits and cherry-pick them into your local installation. +

+

preg_replace_callback(): Compilation failed: unknown property name after \P or \p at offset 7

[edit]
+

preg_replace_callback(): Compilation failed: unknown property name after \P or \p at offset 7 +

+
  • this usually indicates an incompatible version of PCRE; you'll need to update to >= 8.10
  • +
  • @todo: link to instructions on how to upgrade
+

Lua error

[edit]
+

If you copy templates from Wikipedia and then get big red "Lua error: x" messages where the Scribunto invocation (e.g. the template that uses {{#invoke:}}) should be, that probably means that you didn't import everything you needed. Make sure that you tick the "Include templates" box at w:Special:Export when you export. +

When importing pages from another wiki, it is also possible for templates or modules in the imported data to overwrite existing templates or modules with the same title, which may break existing pages, templates, and modules that depend on the overwritten versions. +

+

Blank screen

[edit]
+

Make sure your extension version is applicable to your MediaWiki version. +

+

Design documents

[edit]
+ +

Other pages

[edit]
+ +

See also

[edit]
+
General
+
  • Lua Wikibase client - functionality for the Scribunto extension.
  • +
  • Commons:Lua - there may be specific notes for using Lua modules on Wikimedia Commons, including additional Lua extensions installed (e.g. for local support of internationalization and for parsing or playing medias). Some general purpose modules may be reused in other wikis in various languages (except specific tunings for policies, namespaces or project/maintenance pages with dedicated names). If possible, modules that could be widely reused across wikis should be tested and internationalized on Wikimedia Commons.
  • +
  • w:Help:Lua - there may be specific notes for using Lua modules on Wikipedia, including additional Lua extensions installed (including for integrating Wikidata and Wikimedia Commons contents, generating complex infoboxes and navigation boxes, or to facilitate the general administration/maintenance of the wiki contents under applicable policies). Some other localized Wikipedia editions (or other projects such Wiktionnary, Wikisource or Wikinews) may also have their own needs and Lua modules.
  • +
  • d:Help:Lua - there may be specific notes for using Lua modules on Wikidata, including additional Lua extensions installed (e.g. for local support of internationalization and for database queries)
+
Extensions
+ + +
[edit]
+ +

Notes

[edit]
+
    +
  1. +i.e. Scribunto will not work if proc_open is listed in the disable_functions array in your server's "php.ini" file. If it is, you may see an error message like proc_open(): open_basedir restriction in effect. File(/dev/null) is not within the allowed path(s):. If you are using Plesk and have been granted sufficient permissions, you may be able to set open_basedir in the PHP settings for your domain or subdomain. Try changing {WEBSPACEROOT}{/}{:}{TMP}{/} to {WEBSPACEROOT}{/}{:}{TMP}{/}{:}/dev/null{:}/bin/bash. +
  2. +
  3. 2.0 2.1 2.2 The name of the engines folder changed from lowercase to capitalised in 2022. +
  4. +
+ + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_UniversalLanguageSelector.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_UniversalLanguageSelector.html new file mode 100644 index 00000000..fe4fb26a --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_UniversalLanguageSelector.html @@ -0,0 +1,1178 @@ + + + + +Extension:UniversalLanguageSelector - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Extension:UniversalLanguageSelector

+
+
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
UniversalLanguageSelector
+Release status: stable
Implementation User interface, Skin , Beta Feature +
Description Tool that allows users to select a language and configure its support in an easy way.
Author(s) Wikimedia Language team
Latest version 2024-07-16
Compatibility policy Master maintains backward compatibility.
MediaWiki >= 1.41.0
Composer mediawiki/universal-language-selector
License +
Download +Included in Language Extension Bundle
Help Help:Extension:UniversalLanguageSelector
+ +
+
  • $wgULSWebfontsEnabled
  • +
  • $wgULSGeoService
  • +
  • $wgULSLanguageDetection
  • +
  • $wgULSAnonCanChangeLanguage
  • +
  • $wgULSEnable
  • +
  • $wgULSImeSelectors
  • +
  • $wgULSIMEEnabled
  • +
  • $wgULSNoWebfontsSelectors
  • +
  • $wgULSFontRepositoryBasePath
  • +
  • $wgULSCompactLanguageLinksBetaFeature
  • +
  • $wgULSNoImeSelectors
  • +
  • $wgULSPosition
+ + +
+Not compatible with Internet Explorer 8 or lower.
Quarterly downloads243 (Ranked 26th)
Public wikis using1,237 (Ranked 214th)
+Translate the UniversalLanguageSelector extension if it is available at translatewiki.net
+Issues Open tasks · Report a bug
+

The Universal Language Selector is a tool that allows users to select a language and easily configure its support. Where used, it also ships the functionality of both the former WebFonts and Narayam extensions (both of which have been deprecated in favor of Universal Language Selector). +See Universal Language Selector for background and additional information. +

+ +

Usage

+

The primary aim is to allow users to select a language and easily configure its support. +

The extension provides the following features: +

+
  1. Flexible and easy way to select a language from a large set of languages. +
    1. Selection can be based on geographical region-based browsing, and searching.
    2. +
    3. Search can be based on ISO language code, language name written in current user interface language or in its own script(autonym)
    4. +
    5. Cross-language search - search language names using any script
    6. +
    7. Autocompletion of language names
    8. +
    9. Auto correction of spelling mistakes in search strings
    10. +
    11. Geo IP based language suggestions
    12. +
    13. Language selection based on users browser/OS language
  2. +
  3. Input methods +
    See Help:Extension:UniversalLanguageSelector/Input methods for complete instructions.
    +
    1. An easily selectable input method collection, readily available in every editable field
    2. +
    3. Provides a large set of input methods for a wide range of languages +
    4. +
    5. Per-language input method preferences
  4. +
  5. Webfonts +
    1. A large collection of fonts to choose for each language to use as an embedded font for the page +
    2. +
    3. Per-language font preferences
+
Universal Language Selector with geoip based language suggestion for a user from India
+
Language settings allow a registered user accessing English Wikipedia to change the UI to their native language.
+
A Bengali user is able to read content from the Bengali Wikipedia despite the lack of fonts in their computer. Web fonts are provided automatically for non-Latin scripts for which an open-source font is available. The user can decide to use their system fonts on a per-language basis.
+
A Hindi speaker without a Hindi keyboard configures input methods so that they can type in their language.
+
When searching a user can switch between English and Hindi.
+

Adding fonts

+

Supporting more languages is only a matter of including the proper fonts in the code. However, please note that we will only add support for open-source licensed fonts, such as those licensed under GNU GPL, SIL OFL, etc. An example directory of such open-source fonts is Google Fonts [1] (not yet fully examined/exploited by the authors of this extension); see also the Open Font Library. +

First of all, you need to find or produce such a open-source font (this is the most essential part, and you have to do it yourself); then, it has to be converted to the woff2 format, you can file a request in Phabricator for the font to be added to the extension. +

#Preparing webfonts below explains how to convert the fonts: basic knowledge about GNU/Linux-based operating system is required; if you have difficulty in doing this, you can skip this step and ask someone else to do it for you on the same Phabricator request (of course this will slow down the process). +

+

Preparing webfonts

+

Creating .woff2: +

Use https://github.com/google/woff2 to generate woff2 from ttf. +This will produce a compressed woff2 file. Modern browsers support this format. +

Create a font.ini file. Here's an example: +

+
[AbyssinicaSIL]
+languages=am*, ti*
+version=1.200
+license=OFL 1.1
+licensefile=OFL.txt
+url=http://scripts.sil.org/AbyssinicaSIL
+request-url=https://phabricator.wikimedia.org/[Task Number]
+woff2=AbyssinicaSIL.woff2
+bold=AbyssinicaSIL Bold
+
+[AbyssinicaSIL Bold]
+woff2=AbyssinicaSIL-Bold.woff2
+fontweight=bold
+
+

An asterisk (*) after a language code means that this font will be the default font for that language. Don't use the asterisk if you want the option to use this font for that language. +

After creating the files, do the following: +

+
  1. Create a directory for the font under data/fontrepo/fonts.
  2. +
  3. Put the woff2 and font.ini files in that directory and add them to the source repository (git add).
  4. +
  5. Go to the scripts/ directory and run php compile-font-repo.php.
  6. +
  7. Commit the changes to the repository (git commit -a) and submit them according to the Git workflow.
+

Adding support for a new key mapping (input method)

+

Follow the instructions on the jquery.ime github wiki, but file requests in the Wikimedia-extensions-UniversalLanguageSelector Phabricator product. +

+

Installation

+
  • Download and move the extracted UniversalLanguageSelector folder to your extensions/ directory.
    Developers and code contributors should install the extension from Git instead, using:cd extensions/
    git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/UniversalLanguageSelector
  • +
  • Add the following code at the bottom of your LocalSettings.php file:
    wfLoadExtension( 'UniversalLanguageSelector' );
    +
  • +
  • Yes Done – Navigate to Special:Version on your wiki to verify that the extension is successfully installed.
+


+Vagrant installation: +

+
  • If using Vagrant , install with vagrant roles enable uls --provision
+

Updating LanguageNameIndex

+

For performing cross language search, searching autonyms, language data needs to be populated. +ULS comes with a pre-populated language name index(data/langnames.ser). +In case you want to update it, install Extension:CLDR and update the data with the following command. +

+
php UniversalLanguageSelector/data/LanguageNameIndexer.php
+

and verify that langnames.ser file gets generated in ULS/data/ folder. +

+

Configuration

+

The following variables are created automatically during initialization and can be used from JavaScript using mw.config.get( NAME ): +

+
  • wgULSLanguages - an associative array where the keys are language codes and the values are language names in English.
  • +
  • wgULSAcceptLanguageList - an array of language codes from the user's Accept-Language value. These are the languages selected in the user's browser preferences.
+

For serving fonts, you might want to add the following MIME types to your webserver if not already there. This guide might help. +

+
AddType font/woff2 .woff2
+
+

The following variables can also be configured: +

+
  • $wgULSGeoService - ULS can use geolocation services to suggest languages based on the country the user is visiting from. Setting this to false will prevent built-in geolocation from being used. You can provide your own geolocation by setting window. Geo to object which has key 'country_code' or 'country'. If set to true, it will query Wikimedia's geoip service. The service should return jsonp that uses the supplied callback parameter. Defaults to http://freegeoip.net/json/ (warning: this website has shut down its API) and expects the same format.
  • +
  • $wgULSEnable - Enable language selection, compact language links, input methods, and web fonts for everyone unless the behavior is overridden by the configuration variables below. Even if false, the classes and resource loader modules are registered for using other extensions. Language changing via cookie or setlang query parameter is not possible.
  • +
  • $wgULSAnonCanChangeLanguage - Allow anonymous users to change the language with cookie and setlang query param. Do not use it if you are caching anonymous page views without taking cookies into account. It does not have any effect if either $wgULSEnable or $wgULSEnableAnon is set to false.
  • +
  • $wgULSIMEEnabled - Disable the input methods feature for all users by default. The user can still enable it manually.
  • +
  • $wgULSPosition - The location and the form of the language selection trigger. The possible values are: personal: as a link near the username or the login link in the personal toolbar (default). interlanguage: as an icon near the header of the list of interlanguage links in the sidebar.
  • +
  • $wgULSNoImeSelectors - Array of jQuery selectors of elements on which IME must not be enabled. eg: [ '#wpCaptchaWord' ];
  • +
  • $wgULSLanguageDetection - Whether to automatically detect the user's language from the Accept-Language header.
+

Position of ULS trigger

+
  • $wgULSPosition - The location and the form of the language selection trigger. The possible values are: personal: as a link near the username or the login link in the personal toolbar (default). interlanguage: as an icon near the header of the list of interlanguage links in the sidebar.
+

It is also possible to have a ULS trigger anywhere on the screen. An element with uls-settings-trigger will act as a ULS trigger. +

+

Overriding default fonts

+

ULS has a large font repository to serve as webfonts. +Sometimes, there are multiple fonts for a language, and there is a default font for each language/script. +The order of fonts or default font can be overridden as follows using global scripts (MediaWiki:Common.js) or personal scripts (Special:MyPage/common.js): +

+
$.webfonts.repository.languages.languageCode = ["system", "FontA", "FontB"];
+
+

Here, languageCode should be a valid langauge code(eg: en, hi, nl). FontA and FontB are fonts available in font repository. In the above example for languageCode, we set a font available in local computer as default font. ie No default webfont. +

+

Caching configuration

+

To ensure that the web fonts files are cached on the clients' machines, font file types must be added to the web server configuration. In Apache2 this consists of: +

+
  • Adding font file extensions to the FileTimes regex at FilesMatch for the relevant directory, example:
+
<FilesMatch "\.(gif|jpe?g|png|css|js|woff2|svg)$">
+
+
  • Adding ExpiresByType values to the relevant MIME types, similarly to image MIME types. +
    • Note that there's no standard MIME type for TTF. application/x-font-ttf is used for Wikimedia.
  • +
  • Adding the MIME types:
+
AddType font/woff2 .woff2
+
+

For a full example see the caching configuration update done for the Wikimedia cluster. +

+

Page translation

+

UniversalLanguageSelector is one of the dependencies of the Translate extension, which uses it for several language selection features. One of it is the MyLanguage system for links, which depends on the interface language of the user, but more can be configured: see Page translation feature. +

+

Using Webfonts

+

Users can choose web fonts for a language from the Language settings -> Display settings. The first font in that menu will be applied to the wiki by default. A user can change the font to be remembered across the pages. Optionally, the user can disable the font embedding by selecting the system font. +

If the font is available in the user's local system, the font will not be downloaded from the MediaWiki server. It will be taken from the user's computer. Otherwise, the font will be downloaded from the server only once, when the user selects the font the first time. From then on, the font will be taken from the local cache. +

+

Alternate ways to load fonts

+

By specifying font-family

+

Inside the wiki text <span style="font-family:'YourFontName';">YourText</span>, webfonts extension will check whether the font is available with the extension, if so it will download it to the client. +So the reader will not face any difficulty reading the text even if the specified font is not available on their computer. +

+

By specifying language

+

Inside the wiki text <span lang="my">YourText</span>, the web fonts extension will check whether any font is available for the given language with the extension and if so, it will download it to the client. +So the reader will not face any difficulty reading the text even if the specified font is not available on their computer. +The default font will be used if there are multiple fonts for the language. +If the default font is not preferred, use the font-family approach to specify the font. +If the tag has both lang and font-family definitions, font-family gets precedence. +

Example: +

<span lang=sux>𒄖𒉈𒅁𒌨𒅎</span> +

gives the text rendered in Cuneiform using Akkadian font +

𒄖𒉈𒅁𒌨𒅎 +

+

See also

+ + + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseClient.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseClient.html new file mode 100644 index 00000000..44835eda --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseClient.html @@ -0,0 +1,836 @@ + + + + +Extension:Wikibase Client - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Extension:Wikibase Client

+
+
+
shortcut: WBC
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
Wikibase Client
+Release status: stable
Implementation Ajax, User interface, Data extraction, Parser function +
Description Client for structured data repository
Author(s) The Wikidata team
(contributors list)
Compatibility policy Snapshots releases along with MediaWiki. Master is not backward compatible.
MediaWiki master
Database changes Yes
Tables wbc_entity_usage
License GNU General Public License 2.0 or later
Download
readme
+ + +
+ + +
+Translate the Wikibase Client extension
+Issues Open tasks · Report a bug
+

Wikibase Client is part of Wikibase . +Wikibase Client is an extension that acts as a client to the Wikibase Repository extension. +Its development is part of the Wikidata project. +

It allows to use and display data from a Wikibase Repository via Lua modules (mw.wikibase) or parser functions (#statements, #property). +Clients can also use centralized language links or article placeholders . +

+ +

Installation

[edit]
+

See basic installation instructions . +For experienced configuration options see Advanced Configuration . +

A complete documentation exists in the docs folder for Wikibase, and is published to the Wikimedia documentation site. +

+

Features & Usage

[edit]
+

For a list of features and how to use them see Advanced configuration . +

+

Available hooks

[edit]
+

The documentation of the available PHP and JavaScript hooks can be found in these pages: +

+ +

For more information see Wikibase/Developing extensions . +

+

See also

[edit]
+ + + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib.html new file mode 100644 index 00000000..df6d68de --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib.html @@ -0,0 +1,602 @@ + + + + +Extension:WikibaseLib - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+

Extension:WikibaseLib

+
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+ + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib_Archived.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib_Archived.html new file mode 100644 index 00000000..13cd4fdc --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseLib_Archived.html @@ -0,0 +1,911 @@ + + + + +Extension:WikibaseLib - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Extension:WikibaseLib

+
+
+
shortcut: WBL
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
WikibaseLib
+Release status: stable
Description provides common Wikibase functionality for Wikibase Repository and Wikibase Client
Author(s) The Wikidata team
(contributors list)
Latest version continuous updates
Compatibility policy Snapshots releases along with MediaWiki. Master is not backward compatible.
MediaWiki master
Database changes Yes
License GNU General Public License 2.0 or later
Download
readme
+ + +
+ + +
+Translate the WikibaseLib extension
+Issues Open tasks · Report a bug
+

WikibaseLib provides common Wikibase functionality for Wikibase Repository and Wikibase Client . +

+ +

Feature overview

+

TODO +

+

Requirements

+

WikibaseLib requires: +

+ +

Some components might be dependent on one of the following: +

+ +

Download

+

WikibaseLib is in the lib directory of the Wikibase git repo on gerrit. +

The extension can be retrieved directly from Git [?]: +

+
  • Browse code
  • +
  • Some extensions have tags for stable releases. +
  • +
  • Each branch is associated with a past MediaWiki release. There is also a "master" branch containing the latest alpha version (might require an alpha version of MediaWiki). +
+

Extract the snapshot and place it in the extensions/Wikibase/ directory of your MediaWiki installation. +

If you are familiar with Git and have shell access to your server, you can also obtain the extension as follows: +

+
+

cd extensions/ +git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Wikibase.git + +

+

Installation

+

Once you have downloaded the code, place the WikibaseLib directory within your MediaWiki 'extensions' directory. +Then add the following code to your LocalSettings.php file: +

+
# WikibaseLib
+require_once( "$IP/extensions/Wikibase/lib/WikibaseLib.php" );
+
+# DataValues
+require_once( "$IP/extensions/DataValues/DataValues.php" );
+
+# DataTypes
+require_once( "$IP/extensions/DataTypes/DataTypes.php" );
+
+

Version

+

This is a copy of the release notes file on Git, which might be more up to date than this page. +

+

Internationalization

+

WikibaseLib is fully internationalized. Translation of WikibaseLib messages is done through translatewiki.net. The translation for this extension can be found here. To add language values or change existing ones, you should create an account on translatewiki.net, then request permission from the administrators to translate a certain language or languages on this page (this is a very simple process). Once you have permission for a given language, you can log in and add or edit whatever messages you want to in that language. +

+

Screenshots

+ +

See also

+ +
+ + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseRepository.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseRepository.html new file mode 100644 index 00000000..3b7afbbc --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseRepository.html @@ -0,0 +1,1012 @@ + + + + +Extension:Wikibase Repository - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+
+
+
+
+ +

Extension:Wikibase Repository

+
+
+
shortcut: WBR
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MediaWiki extensions manual
Wikibase Repository
+Release status: stable
Implementation API , Ajax, User interface, ContentHandler +
Description Structured data repository
Author(s) The Wikidata team
(contributors list)
Latest version continuous updates
Compatibility policy Snapshots releases along with MediaWiki. Master is not backward compatible.
MediaWiki master
Database changes Yes
Tables wbt_item_terms
wbt_property_terms
wbt_term_in_lang
wbt_text_in_lang
wbt_text
wbt_type
wb_changes
wb_changes_subscriptions
wb_id_counters
wb_items_per_site
wb_property_info
License GNU General Public License 2.0 or later
Download
readme
+ +
+

+

+
  • item-term
  • +
  • property-term
  • +
  • item-merge
  • +
  • item-redirect
  • +
  • property-create
+ + +
+ + +
+Translate the Wikibase Repository extension
+Issues Open tasks · Report a bug
+

Wikibase Repository is part of Wikibase . +Wikibase Repository allows you to use your wiki as a structured data repository. +Its development is part of the Wikidata project. +The data can in turn be used in a wiki using the Wikibase Client extension. +

+ +

Installation

[edit]
+

See the basic installation instructions. +For experienced configuration options see Advanced Configuration. +

A complete documentation exists in the docs folder for Wikibase, and is published to the Wikimedia documentation site. +

+

Integration with other extensions

[edit]
+

Wikibase makes use of the following if they are installed: +

+ +
If installed and JavaScript is supported by the user client / browser, labels, aliases and descriptions will be shown to the user in frequently used languages in addition to the user interface language. (These languages are acquired from mw.uls.getFrequentLanguageList().)
+ +
If installed, logged-in users are able to define additional languages that labels, aliases and descriptions are shown in by specifying languages on their user pages using the Babel syntax.
+

Available hooks

[edit]
+

The documentation of the available PHP and JavaScript hooks can be found in here: +

+ +

For more information see Wikibase/Developing extensions . +

+

Database

[edit]
+

Term store

  • wbit_id BIGINT
  • wbit_item_id INT
  • wbit_term_in_lang_id INT
  • wbpt_id INT
  • wbpt_property_id INT
  • wbpt_term_in_lang_id INT
  • wbtl_id INT
  • wbtl_type_id INT
  • wbtl_text_in_lang_id INT
  • wbxl_id INT
  • wbxl_language VARBINARY(20)
  • wbxl_text_id INT
  • wbx_id INT
  • wbx_text VARBINARY(255)
  • wby_id INT
  • wby_name VARBINARY(45)

Misc

  • id_value INT
  • id_type VARBINARY(32)
  • ips_row_id BIGINT
  • ips_item_id INT
  • ips_site_id VARBINARY(32)
  • ips_site_page VARCHAR(310)
  • pi_property_id INT
  • pi_type VARBINARY(32)
  • pi_info BLOB
+

See the automated documentation +

+

See also

[edit]
+ + + + + + + +
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseView.html b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseView.html new file mode 100644 index 00000000..73c8c840 --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/data/Mediawiki_WikibaseView.html @@ -0,0 +1,549 @@ + + + + +Extension:WikibaseView - MediaWiki + + + + + + + + + + + + + + + + + + + + + + + + + + + +Jump to content +
+
+
+ + + + +
+
+ + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+

Extension:WikibaseView

+
+
+
+
+
+
+ +
+
+ + + +
+
+
+
+
+ + +
+
+
+
+ +
From mediawiki.org
+
+
+ + +
+

There is currently no text in this page. +You can search for this page title in other pages, +search the related logs, +or create this page. +

+
+
+ +
+
+ +
+ +
+
+
+ +
+ + \ No newline at end of file diff --git a/tests/test_create_observation/test_create_software_version_observation/data/Special_Version.html b/tests/test_create_observation/test_create_software_version_observation/data/Special_Version.html index 49856a80..f42c07b2 100644 --- a/tests/test_create_observation/test_create_software_version_observation/data/Special_Version.html +++ b/tests/test_create_observation/test_create_software_version_observation/data/Special_Version.html @@ -63,7 +63,7 @@

I Lua 5.1.5 -

Installed skins

Installed extensions

Installed libraries

<gallery>, <indicator>, <nowiki>, <pagelist>, <pagequality>, <pages>, <pre> and <section>anchorencode, babel, basepagename, basepagenamee, bidi, canonicalurl, canonicalurle, cascadingsources, defaultsort, displaytitle, filepath, formatdate, formatnum, fullpagename, fullpagenamee, fullurl, fullurle, gender, grammar, int, invoke, language, lc, lcfirst, localurl, localurle, lst, lsth, lstx, namespace, namespacee, namespacenumber, noexternallanglinks, ns, nse, numberingroup, numberofactiveusers, numberofadmins, numberofarticles, numberofedits, numberoffiles, numberofpages, numberofusers, padleft, padright, pageid, pagename, pagenamee, pagesincategory, pagesize, plural, property, protectionexpiry, protectionlevel, revisionday, revisionday2, revisionid, revisionmonth, revisionmonth1, revisiontimestamp, revisionuser, revisionyear, rootpagename, rootpagenamee, special, speciale, statements, subjectpagename, subjectpagenamee, subjectspace, subjectspacee, subpagename, subpagenamee, tag, talkpagename, talkpagenamee, talkspace, talkspacee, uc, ucfirst and urlencode +

Installed skins

Installed extensions

Installed libraries

<gallery>, <indicator>, <nowiki>, <pagelist>, <pagequality>, <pages>, <pre> and <section>anchorencode, babel, basepagename, basepagenamee, bidi, canonicalurl, canonicalurle, cascadingsources, defaultsort, displaytitle, filepath, formatdate, formatnum, fullpagename, fullpagenamee, fullurl, fullurle, gender, grammar, int, invoke, language, lc, lcfirst, localurl, localurle, lst, lsth, lstx, namespace, namespacee, namespacenumber, noexternallanglinks, ns, nse, numberingroup, numberofactiveusers, numberofadmins, numberofarticles, numberofedits, numberoffiles, numberofpages, numberofusers, padleft, padright, pageid, pagename, pagenamee, pagesincategory, pagesize, plural, property, protectionexpiry, protectionlevel, revisionday, revisionday2, revisionid, revisionmonth, revisionmonth1, revisiontimestamp, revisionuser, revisionyear, rootpagename, rootpagenamee, special, speciale, statements, subjectpagename, subjectpagenamee, subjectspace, subjectspacee, subpagename, subpagenamee, tag, talkpagename, talkpagenamee, talkspace, talkspacee, uc, ucfirst and urlencode diff --git a/tests/test_create_observation/test_create_software_version_observation/test_constants.py b/tests/test_create_observation/test_create_software_version_observation/test_constants.py new file mode 100644 index 00000000..0b54bd95 --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/test_constants.py @@ -0,0 +1,5 @@ +"""Constants""" + +DATA_DIRECTORY = ( + "tests/test_create_observation/test_create_software_version_observation/data" +) diff --git a/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py b/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py index 687127da..4d842e75 100644 --- a/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py +++ b/tests/test_create_observation/test_create_software_version_observation/test_create_software_version_observation.py @@ -5,18 +5,18 @@ from urllib.error import HTTPError import pytest from fetch_data import create_software_version_observation -from tests.utils import MockResponse - - -DATA_DIRECTORY = ( - "tests/test_create_observation/test_create_software_version_observation/data" +from tests.mock_info import MockBackgroundClassList, MockInfo +from tests.test_create_observation.test_create_software_version_observation.test_constants import ( + DATA_DIRECTORY, ) +from tests.utils import MockResponse @pytest.mark.asyncio @pytest.mark.dependency( name="software-version-success", depends=["add-wikibase"], scope="session" ) +@pytest.mark.soup @pytest.mark.version async def test_create_software_version_observation_success(mocker): """Test Data Returned Scenario""" @@ -28,17 +28,20 @@ async def test_create_software_version_observation_success(mocker): ) as version_html: mocker.patch( - "fetch_data.soup_data.create_software_version_data_observation.requests.get", - side_effect=[MockResponse(200, version_html.read())], + "fetch_data.soup_data.software.create_software_version_data_observation.requests.get", + side_effect=[MockResponse("", 200, version_html.read())], ) - success = await create_software_version_observation(1) + mock_info = MockInfo(context={"background_tasks": MockBackgroundClassList()}) + success = await create_software_version_observation(1, mock_info) assert success + assert len(mock_info.context["background_tasks"].tasks) == 1 @pytest.mark.asyncio @pytest.mark.dependency( name="software-version-failure", depends=["software-version-success"] ) +@pytest.mark.soup @pytest.mark.version async def test_create_software_version_observation_failure(mocker): """Test Failure Scenario""" @@ -46,7 +49,7 @@ async def test_create_software_version_observation_failure(mocker): time.sleep(1) mocker.patch( - "fetch_data.soup_data.create_software_version_data_observation.requests.get", + "fetch_data.soup_data.software.create_software_version_data_observation.requests.get", side_effect=[ HTTPError( url="example.com/wiki/Special:Version", @@ -57,5 +60,7 @@ async def test_create_software_version_observation_failure(mocker): ) ], ) - success = await create_software_version_observation(1) + mock_info = MockInfo(context={"background_tasks": MockBackgroundClassList()}) + success = await create_software_version_observation(1, mock_info) assert success is False + assert len(mock_info.context["background_tasks"].tasks) == 1 diff --git a/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py b/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py new file mode 100644 index 00000000..4dbd69e0 --- /dev/null +++ b/tests/test_create_observation/test_create_software_version_observation/test_update_software_data.py @@ -0,0 +1,137 @@ +"""Test Extension Data""" + +from freezegun import freeze_time +import pytest +from fetch_data import update_software_data +from fetch_data.soup_data.software.get_update_software_data import ( + get_update_extension_query, +) +from tests.test_create_observation.test_create_software_version_observation.test_constants import ( + DATA_DIRECTORY, +) +from tests.utils import MockResponse + + +@freeze_time("2024-03-01") +@pytest.mark.asyncio +@pytest.mark.dependency( + name="update-software-data", depends=["software-version-success"], scope="session" +) +@pytest.mark.soup +@pytest.mark.version +async def test_update_software_data(mocker): + """Test Update Software Data""" + + # pylint: disable=unused-argument,too-many-return-statements + def mockery(*args, **kwargs): + print(args) + query = args[0] + match query: + case "https://www.mediawiki.org/wiki/Extension:Babel": + with open(f"{DATA_DIRECTORY}/Mediawiki_Babel.html", mode="rb") as data: + return MockResponse(query, 200, data.read()) + case ( + "https://www.mediawiki.org/wiki/Extension:Google Analytics Integration" + ): + with open( + f"{DATA_DIRECTORY}/Mediawiki_GoogleAnalyticsIntegration.html", + mode="rb", + ) as data: + return MockResponse( + "https://www.mediawiki.org/wiki/Extension:Google_Analytics_Integration", + 200, + data.read(), + ) + case "https://www.mediawiki.org/wiki/Extension:LabeledSectionTransclusion": + with open( + f"{DATA_DIRECTORY}/Mediawiki_LabeledSectionTransclusion.html", + mode="rb", + ) as data: + return MockResponse( + "https://www.mediawiki.org/wiki/Extension:Labeled_Section_Transclusion", + 200, + data.read(), + ) + case "https://www.mediawiki.org/wiki/Extension:MirahezeMagic": + return MockResponse( + "https://www.mediawiki.org/wiki/Extension:MirahezeMagic", 404 + ) + case "https://www.mediawiki.org/wiki/Extension:ProofreadPage": + with open( + f"{DATA_DIRECTORY}/Mediawiki_ProofreadPage.html", mode="rb" + ) as data: + return MockResponse( + "https://www.mediawiki.org/wiki/Extension:Proofread_Page", + 200, + data.read(), + ) + case "https://www.mediawiki.org/wiki/Extension:Scribunto": + with open( + f"{DATA_DIRECTORY}/Mediawiki_Scribunto.html", mode="rb" + ) as data: + return MockResponse(query, 200, data.read()) + case "https://www.mediawiki.org/wiki/Extension:UniversalLanguageSelector": + with open( + f"{DATA_DIRECTORY}/Mediawiki_UniversalLanguageSelector.html", + mode="rb", + ) as data: + return MockResponse(query, 200, data.read()) + case "https://www.mediawiki.org/wiki/Extension:WikibaseClient": + with open( + f"{DATA_DIRECTORY}/Mediawiki_WikibaseClient.html", mode="rb" + ) as data: + return MockResponse( + "https://www.mediawiki.org/wiki/Extension:Wikibase_Client", + 200, + data.read(), + ) + case "https://www.mediawiki.org/wiki/Extension:WikibaseLib": + with open( + f"{DATA_DIRECTORY}/Mediawiki_WikibaseLib.html", mode="rb" + ) as data: + return MockResponse(query, 200, data.read()) + case "https://www.mediawiki.org/wiki/Extension:WikibaseRepository": + with open( + f"{DATA_DIRECTORY}/Mediawiki_WikibaseRepository.html", mode="rb" + ) as data: + return MockResponse( + "https://www.mediawiki.org/wiki/Extension:Wikibase_Repository", + 200, + data.read(), + ) + case "https://www.mediawiki.org/wiki/Extension:WikibaseView": + with open( + f"{DATA_DIRECTORY}/Mediawiki_WikibaseView.html", mode="rb" + ) as data: + return MockResponse(query, 200, data.read()) + case "https://www.mediawiki.org/wiki/Special:PermanentLink/3981869": + with open( + f"{DATA_DIRECTORY}/Mediawiki_WikibaseLib_Archived.html", mode="rb" + ) as data: + return MockResponse( + "https://www.mediawiki.org/w/index.php?oldid=3981869", + 200, + data.read(), + ) + raise NotImplementedError(query) + + mocker.patch( + "fetch_data.soup_data.software.get_update_software_data.requests.get", + side_effect=mockery, + ) + + await update_software_data() + + +@pytest.mark.version +def test_get_update_extension_query(): + """Test Update Extension Query""" + + query = get_update_extension_query() + assert str(query) == ( + # pylint: disable=line-too-long + """SELECT wikibase_software.id, wikibase_software.software_type, wikibase_software.software_name, wikibase_software.url, wikibase_software.fetched, wikibase_software.description, wikibase_software.latest_version, wikibase_software.quarterly_download_count, wikibase_software.public_wiki_count, wikibase_software.mw_bundled, wikibase_software.archived +FROM wikibase_software +WHERE (wikibase_software.fetched IS NULL OR wikibase_software.fetched < :fetched_1) AND wikibase_software.software_type = :software_type_1 AND (wikibase_software.archived = false OR wikibase_software.archived IS NULL) + LIMIT :param_1""" + ) diff --git a/tests/test_create_observation/test_create_statistics_observation/__init__.py b/tests/test_create_observation/test_create_statistics_observation/__init__.py new file mode 100644 index 00000000..83e8fe5d --- /dev/null +++ b/tests/test_create_observation/test_create_statistics_observation/__init__.py @@ -0,0 +1 @@ +"""Test Create Statistics Observation""" diff --git a/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py b/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py index d0ee7168..75084f1d 100644 --- a/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py +++ b/tests/test_create_observation/test_create_statistics_observation/test_create_statistics_observation.py @@ -15,6 +15,7 @@ @pytest.mark.dependency( name="statistics-success", depends=["add-wikibase"], scope="session" ) +@pytest.mark.soup @pytest.mark.statistics async def test_create_statistics_observation_success(mocker): """Test Data Returned Scenario""" @@ -25,7 +26,7 @@ async def test_create_statistics_observation_success(mocker): mocker.patch( "fetch_data.soup_data.create_statistics_data_observation.requests.get", - side_effect=[MockResponse(200, version_html.read())], + side_effect=[MockResponse("", 200, version_html.read())], ) success = await create_special_statistics_observation(1) assert success @@ -33,6 +34,7 @@ async def test_create_statistics_observation_success(mocker): @pytest.mark.asyncio @pytest.mark.dependency(name="statistics-failure", depends=["statistics-success"]) +@pytest.mark.soup @pytest.mark.statistics async def test_create_statistics_observation_failure(mocker): """Test Failure Scenario""" diff --git a/tests/test_data_expectations.py b/tests/test_data_expectations.py index b003dc91..571ea3d0 100644 --- a/tests/test_data_expectations.py +++ b/tests/test_data_expectations.py @@ -1,6 +1,7 @@ """Test Data Expectations with Great Expectations""" import os +from pathlib import Path import pytest import great_expectations as gx @@ -15,7 +16,11 @@ def pytest_generate_tests(metafunc): if "checkpoint_name" in metafunc.fixturenames: # Generate test cases based on the test_data list metafunc.parametrize( - "checkpoint_name", sorted(os.listdir(CHECKPOINT_DIRECTORY)) + "checkpoint_name", + [ + Path(checkpoint_filename).stem + for checkpoint_filename in sorted(os.listdir(CHECKPOINT_DIRECTORY)) + ], ) diff --git a/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py b/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py index 073a143d..8333f839 100644 --- a/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py +++ b/tests/test_query/aggregation/software_version/test_aggregate_extensions_query.py @@ -9,7 +9,7 @@ SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT, ) from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import assert_layered_property_count, assert_page_meta AGGREGATE_EXTENSIONS_QUERY = ( @@ -39,18 +39,7 @@ async def test_aggregate_extensions_query_page_one(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "pageSize"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "totalCount"], 10 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "totalPages"], 2 - ) + assert_page_meta(result.data["aggregateExtensionPopularity"], 1, 5, 11, 3) assert_layered_property_count( result.data, ["aggregateExtensionPopularity", "data"], 5 ) @@ -78,6 +67,13 @@ async def test_aggregate_extensions_query_page_one(): datetime(2020, 1, 29, 14, 52), "f621799", ), + ( + "Miraheze Magic", + "e742444", + (None, None, None), + datetime(2024, 10, 17, 15, 21), + "e742444", + ), ( "ProofreadPage", "cb0a218", @@ -85,7 +81,6 @@ async def test_aggregate_extensions_query_page_one(): datetime(2019, 9, 30, 9, 20), "cb0a218", ), - ("Scribunto", None, (None, None, None), None, None), ] ): @@ -113,18 +108,7 @@ async def test_aggregate_extensions_query_page_two(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "pageNumber"], 2 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "pageSize"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "totalCount"], 10 - ) - assert_layered_property_value( - result.data, ["aggregateExtensionPopularity", "meta", "totalPages"], 2 - ) + assert_page_meta(result.data["aggregateExtensionPopularity"], 2, 5, 11, 3) assert_layered_property_count( result.data, ["aggregateExtensionPopularity", "data"], 5 ) @@ -137,6 +121,7 @@ async def test_aggregate_extensions_query_page_two(): expected_version_hash, ) in enumerate( [ + ("Scribunto", None, (None, None, None), None, None), ( "UniversalLanguageSelector", "2020-01-23", @@ -165,6 +150,46 @@ async def test_aggregate_extensions_query_page_two(): datetime(2019, 12, 10, 12, 52), "dbbcdd8", ), + ] + ): + + assert_software_version_aggregate( + result.data["aggregateExtensionPopularity"]["data"][index], + expected_software_name, + expected_version_string, + expected_version_semver, + expected_version_date, + expected_version_hash, + ) + + +@pytest.mark.asyncio +@pytest.mark.agg +@pytest.mark.dependency(depends=["software-version-success"], scope="session") +@pytest.mark.query +@pytest.mark.version +async def test_aggregate_extensions_query_page_three(): + """Test Aggregated Extensions Query - 11""" + + result = await test_schema.execute( + AGGREGATE_EXTENSIONS_QUERY, variable_values={"pageNumber": 3, "pageSize": 5} + ) + + assert result.errors is None + assert result.data is not None + assert_page_meta(result.data["aggregateExtensionPopularity"], 3, 5, 11, 3) + assert_layered_property_count( + result.data, ["aggregateExtensionPopularity", "data"], 1 + ) + + for index, ( + expected_software_name, + expected_version_string, + expected_version_semver, + expected_version_date, + expected_version_hash, + ) in enumerate( + [ ( "WikibaseView", "dbbcdd8", diff --git a/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py b/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py index e7741bd7..d7bee490 100644 --- a/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py +++ b/tests/test_query/aggregation/software_version/test_aggregate_libraries_query.py @@ -8,7 +8,7 @@ SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT, ) from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import assert_layered_property_count, assert_page_meta AGGREGATE_LIBRARIES_QUERY = ( @@ -38,18 +38,7 @@ async def test_aggregate_libraries_query_page_one(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "pageSize"], 30 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "totalCount"], 59 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "totalPages"], 2 - ) + assert_page_meta(result.data["aggregateLibraryPopularity"], 1, 30, 59, 2) assert_layered_property_count( result.data, ["aggregateLibraryPopularity", "data"], 30 ) @@ -117,18 +106,7 @@ async def test_aggregate_libraries_query_page_two(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "pageNumber"], 2 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "pageSize"], 30 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "totalCount"], 59 - ) - assert_layered_property_value( - result.data, ["aggregateLibraryPopularity", "meta", "totalPages"], 2 - ) + assert_page_meta(result.data["aggregateLibraryPopularity"], 2, 30, 59, 2) assert_layered_property_count( result.data, ["aggregateLibraryPopularity", "data"], 29 ) diff --git a/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py b/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py index fe23d216..0e0032c2 100644 --- a/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py +++ b/tests/test_query/aggregation/software_version/test_aggregate_skins_query.py @@ -1,6 +1,7 @@ """Test Aggregated Skins Query""" import pytest + from tests.test_query.aggregation.software_version.assert_software_version_aggregate import ( assert_software_version_aggregate, ) @@ -8,7 +9,7 @@ SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT, ) from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import assert_layered_property_count, assert_page_meta AGGREGATE_SKINS_QUERY = ( @@ -38,18 +39,7 @@ async def test_aggregate_skins_query_page_one(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateSkinPopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregateSkinPopularity", "meta", "pageSize"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateSkinPopularity", "meta", "totalCount"], 3 - ) - assert_layered_property_value( - result.data, ["aggregateSkinPopularity", "meta", "totalPages"], 1 - ) + assert_page_meta(result.data["aggregateSkinPopularity"], 1, 5, 3, 1) assert_layered_property_count(result.data, ["aggregateSkinPopularity", "data"], 3) for index, ( diff --git a/tests/test_query/aggregation/software_version/test_aggregate_software_query.py b/tests/test_query/aggregation/software_version/test_aggregate_software_query.py index e388859c..ba7d3d2a 100644 --- a/tests/test_query/aggregation/software_version/test_aggregate_software_query.py +++ b/tests/test_query/aggregation/software_version/test_aggregate_software_query.py @@ -9,7 +9,7 @@ SOFTWARE_VERSION_DOUBLE_AGGREGATE_FRAGMENT, ) from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import assert_layered_property_count, assert_page_meta AGGREGATE_SOFTWARE_QUERY = ( @@ -39,18 +39,7 @@ async def test_aggregate_software_query(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregateSoftwarePopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregateSoftwarePopularity", "meta", "pageSize"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateSoftwarePopularity", "meta", "totalCount"], 5 - ) - assert_layered_property_value( - result.data, ["aggregateSoftwarePopularity", "meta", "totalPages"], 1 - ) + assert_page_meta(result.data["aggregateSoftwarePopularity"], 1, 5, 5, 1) assert_layered_property_count( result.data, ["aggregateSoftwarePopularity", "data"], 5 ) diff --git a/tests/test_query/aggregation/test_aggregate_property_popularity_query.py b/tests/test_query/aggregation/test_aggregate_property_popularity_query.py index e591e678..4fdfb76a 100644 --- a/tests/test_query/aggregation/test_aggregate_property_popularity_query.py +++ b/tests/test_query/aggregation/test_aggregate_property_popularity_query.py @@ -2,7 +2,11 @@ import pytest from tests.test_schema import test_schema -from tests.utils import assert_layered_property_count, assert_layered_property_value +from tests.utils import ( + assert_layered_property_count, + assert_layered_property_value, + assert_page_meta, +) AGGREGATED_PROPERTY_POPULARITY_QUERY = """ @@ -41,18 +45,7 @@ async def test_aggregate_property_popularity_query(): assert result.errors is None assert result.data is not None - assert_layered_property_value( - result.data, ["aggregatePropertyPopularity", "meta", "pageNumber"], 1 - ) - assert_layered_property_value( - result.data, ["aggregatePropertyPopularity", "meta", "pageSize"], 30 - ) - assert_layered_property_value( - result.data, ["aggregatePropertyPopularity", "meta", "totalCount"], 2 - ) - assert_layered_property_value( - result.data, ["aggregatePropertyPopularity", "meta", "totalPages"], 1 - ) + assert_page_meta(result.data["aggregatePropertyPopularity"], 1, 30, 2, 1) assert_layered_property_count( result.data, ["aggregatePropertyPopularity", "data"], 2 diff --git a/tests/test_query/test_extension_list_query.py b/tests/test_query/test_extension_list_query.py new file mode 100644 index 00000000..18695d97 --- /dev/null +++ b/tests/test_query/test_extension_list_query.py @@ -0,0 +1,309 @@ +"""Test Wikibase List""" + +from datetime import datetime +from typing import Optional +import pytest +from tests.test_schema import test_schema +from tests.utils import assert_layered_property_value, assert_page_meta + + +EXTENSION_LIST_QUERY = """ +query MyQuery($pageNumber: Int!, $pageSize: Int!) { + extensionList(pageNumber: $pageNumber, pageSize: $pageSize) { + meta { + pageNumber + pageSize + totalCount + totalPages + } + data { + id + softwareName + softwareType + url + archived + description + fetched + latestVersion + mediawikiBundled + publicWikiCount + quarterlyDownloadCount + tags + } + } +}""" + + +@pytest.mark.asyncio +@pytest.mark.query +@pytest.mark.version +@pytest.mark.dependency(depends=["update-software-data"], scope="session") +async def test_extension_list_query(): + """Test Extension List""" + + result = await test_schema.execute( + EXTENSION_LIST_QUERY, variable_values={"pageNumber": 1, "pageSize": 10} + ) + + assert result.errors is None + assert result.data is not None + assert "extensionList" in result.data + assert_page_meta(result.data["extensionList"], 1, 10, 11, 2) + assert "data" in result.data["extensionList"] + assert len(result.data["extensionList"]["data"]) == 10 + + +@pytest.mark.asyncio +@pytest.mark.query +@pytest.mark.version +@pytest.mark.dependency(depends=["update-software-data"], scope="session") +@pytest.mark.parametrize( + [ + "idx", + "expected_id", + "expected_name", + "expected_url", + "expected_archived", + "expected_description", + "expected_fetched", + "expected_latest_version", + "expected_mediawiki_bundled", + "expected_public_wiki_count", + "expected_quarterly_download_count", + "expected_tags", + ], + [ + ( + 0, + "2", + "Babel", + "Babel", + False, + # pylint: disable=line-too-long + "Adds a parser function to inform other users about language proficiency and categorize users of the same levels and languages.", + datetime(2024, 3, 1), + "Continuous updates", + False, + 2416, + 63, + ["Parser function"], + ), + ( + 1, + "17", + "Google Analytics Integration", + "Google_Analytics_Integration", + False, + # pylint: disable=line-too-long + "Automatically inserts Google Universal Analytics (and/or other web analytics) tracking code at the bottom of MediaWiki pages", + datetime(2024, 3, 1), + "3.0.1 (2017-10-29)", + False, + 1302, + None, + ["User activity", "Hook"], + ), + ( + 2, + "11", + "LabeledSectionTransclusion", + "Labeled_Section_Transclusion", + False, + "Enables marked sections of text to be transcluded", + datetime(2024, 3, 1), + None, + False, + 6919, + None, + ["Parser function", "Tag"], + ), + ( + 3, + "1", + "Miraheze Magic", + "MirahezeMagic", + None, + None, + datetime(2024, 3, 1), + None, + None, + None, + None, + [], + ), + ( + 4, + "18", + "ProofreadPage", + "Proofread_Page", + False, + # pylint: disable=line-too-long + "The Proofread Page extension can render a book either as a column of OCR text beside a column of scanned images, or broken into its logical organization (such as chapters or poems) using transclusion.", + datetime(2024, 3, 1), + "continuous updates", + False, + None, + None, + ["Tag", "Page action", "ContentHandler", "API", "Database"], + ), + ( + 5, + "12", + "Scribunto", + "Scribunto", + False, + "Provides a framework for embedding scripting languages into MediaWiki pages", + datetime(2024, 3, 1), + "Continuous updates", + True, + 8789, + 450, + ["Parser extension"], + ), + ( + 6, + "19", + "UniversalLanguageSelector", + "UniversalLanguageSelector", + False, + "Tool that allows users to select a language and configure its support in an easy way.", + datetime(2024, 3, 1), + "2024-07-16", + False, + 1237, + 243, + ["Skin", "Beta Feature"], + ), + ( + 7, + "13", + "WikibaseClient", + "Wikibase_Client", + False, + "Client for structured data repository", + datetime(2024, 3, 1), + None, + False, + None, + None, + ["Parser function", "Ajax"], + ), + ( + 8, + "14", + "WikibaseLib", + "WikibaseLib", + True, + "Provides common Wikibase functionality for Wikibase Repository and Wikibase Client", + datetime(2024, 3, 1), + "continuous updates", + False, + None, + None, + [], + ), + ( + 9, + "15", + "WikibaseRepository", + "Wikibase_Repository", + False, + "Structured data repository", + datetime(2024, 3, 1), + "continuous updates", + False, + None, + None, + ["ContentHandler", "API", "Ajax"], + ), + ( + 10, + "16", + "WikibaseView", + "WikibaseView", + False, + None, + datetime(2024, 3, 1), + None, + None, + None, + None, + [], + ), + ], +) +# pylint: disable=too-many-arguments,too-many-positional-arguments +async def test_extension_list_query_parameterized( + idx: int, + expected_id: str, + expected_name: str, + expected_url: str, + expected_archived: bool, + expected_description: Optional[str], + expected_fetched: datetime, + expected_latest_version: Optional[str], + expected_mediawiki_bundled: Optional[bool], + expected_public_wiki_count: Optional[int], + expected_quarterly_download_count: Optional[int], + expected_tags: list[str], +): + """Test Extension List""" + + result = await test_schema.execute( + EXTENSION_LIST_QUERY, variable_values={"pageNumber": 1, "pageSize": 100} + ) + + assert result.errors is None + assert result.data is not None + assert "extensionList" in result.data + assert_page_meta(result.data["extensionList"], 1, 100, 11, 1) + assert "data" in result.data["extensionList"] + assert len(result.data["extensionList"]["data"]) == 11 + assert_layered_property_value( + result.data, ["extensionList", "data", idx, "id"], expected_id + ) + assert_layered_property_value( + result.data, ["extensionList", "data", idx, "softwareName"], expected_name + ) + assert_layered_property_value( + result.data, ["extensionList", "data", idx, "softwareType"], "EXTENSION" + ) + assert_layered_property_value( + result.data, + ["extensionList", "data", idx, "url"], + f"https://www.mediawiki.org/wiki/Extension:{expected_url}", + ) + assert_layered_property_value( + result.data, ["extensionList", "data", idx, "archived"], expected_archived + ) + assert_layered_property_value( + result.data, ["extensionList", "data", idx, "description"], expected_description + ) + assert_layered_property_value( + result.data, + ["extensionList", "data", idx, "fetched"], + expected_fetched.strftime("%Y-%m-%dT%H:%M:%S"), + ) + assert_layered_property_value( + result.data, + ["extensionList", "data", idx, "latestVersion"], + expected_latest_version, + ) + assert_layered_property_value( + result.data, + ["extensionList", "data", idx, "mediawikiBundled"], + expected_mediawiki_bundled, + ) + assert_layered_property_value( + result.data, + ["extensionList", "data", idx, "publicWikiCount"], + expected_public_wiki_count, + ) + assert_layered_property_value( + result.data, + ["extensionList", "data", idx, "quarterlyDownloadCount"], + expected_quarterly_download_count, + ) + assert_layered_property_value( + result.data, ["extensionList", "data", idx, "tags"], expected_tags + ) diff --git a/tests/test_query/test_wikibase_list_query.py b/tests/test_query/test_wikibase_list_query.py index 02f95101..1e1d939e 100644 --- a/tests/test_query/test_wikibase_list_query.py +++ b/tests/test_query/test_wikibase_list_query.py @@ -2,7 +2,11 @@ import pytest from tests.test_schema import test_schema -from tests.utils import assert_layered_property_value, assert_property_value +from tests.utils import ( + assert_layered_property_value, + assert_page_meta, + assert_property_value, +) WIKIBASE_LIST_QUERY = """ @@ -81,17 +85,7 @@ async def test_wikibase_list_query(): assert result.errors is None assert result.data is not None assert "wikibaseList" in result.data - assert "meta" in result.data["wikibaseList"] - assert_layered_property_value( - result.data, ["wikibaseList", "meta", "pageNumber"], 1 - ) - assert_layered_property_value(result.data, ["wikibaseList", "meta", "pageSize"], 1) - assert_layered_property_value( - result.data, ["wikibaseList", "meta", "totalCount"], 1 - ) - assert_layered_property_value( - result.data, ["wikibaseList", "meta", "totalPages"], 1 - ) + assert_page_meta(result.data["wikibaseList"], 1, 1, 1, 1) assert "data" in result.data["wikibaseList"] assert len(result.data["wikibaseList"]["data"]) == 1 result_datum = result.data["wikibaseList"]["data"][0] diff --git a/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_extensions_query.py b/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_extensions_query.py index 8227b38e..5307d394 100644 --- a/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_extensions_query.py +++ b/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_extensions_query.py @@ -60,7 +60,7 @@ async def test_wikibase_software_version_most_recent_observation_extensions_quer assert "observationDate" in most_recent assert_property_value(most_recent, "returnedData", True) - assert_layered_property_count(most_recent, ["installedExtensions"], 10) + assert_layered_property_count(most_recent, ["installedExtensions"], 11) for index, ( expected_id, expected_name, @@ -84,6 +84,13 @@ async def test_wikibase_software_version_most_recent_observation_extensions_quer datetime(2020, 1, 29, 14, 52), "f621799", ), + ( + "19", + "Miraheze Magic", + "e742444", + datetime(2024, 10, 17, 15, 21), + "e742444", + ), ("17", "ProofreadPage", "cb0a218", datetime(2019, 9, 30, 9, 20), "cb0a218"), ("11", "Scribunto", None, None, None), ( diff --git a/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_libraries_query.py b/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_libraries_query.py index bfa1ab2c..dfda6f47 100644 --- a/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_libraries_query.py +++ b/tests/test_query/wikibase/software_version_obs/test_wikibase_software_version_most_recent_observation_libraries_query.py @@ -68,65 +68,65 @@ async def test_wikibase_software_version_most_recent_observation_libraries_query expected_version_hash, ) in enumerate( [ - ("19", "composer/installers", "1.8.0", None, None), - ("20", "composer/semver", "1.5.0", None, None), - ("21", "cssjanus/cssjanus", "1.3.0", None, None), - ("22", "data-values/common", "0.4.3", None, None), - ("23", "data-values/data-values", "2.3.0", None, None), - ("24", "data-values/geo", "3.0.1", None, None), - ("25", "data-values/interfaces", "0.2.5", None, None), - ("26", "data-values/number", "0.10.1", None, None), - ("27", "data-values/serialization", "1.2.3", None, None), - ("28", "data-values/time", "1.0.1", None, None), - ("29", "diff/diff", "2.3.0", None, None), - ("30", "guzzlehttp/guzzle", "6.3.3", None, None), - ("31", "guzzlehttp/promises", "1.3.1", None, None), - ("32", "guzzlehttp/psr7", "1.6.1", None, None), - ("33", "liuggio/statsd-php-client", "1.0.18", None, None), - ("34", "onoi/message-reporter", "1.4.1", None, None), - ("35", "oojs/oojs-ui", "0.34.1", None, None), - ("36", "pear/console_getopt", "1.4.3", None, None), - ("37", "pear/mail", "1.4.1", None, None), - ("38", "pear/mail_mime", "1.10.2", None, None), - ("39", "pear/net_smtp", "1.8.1", None, None), - ("40", "pear/net_socket", "1.2.2", None, None), - ("41", "pear/pear-core-minimal", "1.10.10", None, None), - ("42", "pear/pear_exception", "1.0.1", None, None), - ("43", "pleonasm/bloom-filter", "1.0.2", None, None), - ("44", "psr/container", "1.0.0", None, None), - ("45", "psr/http-message", "1.0.1", None, None), - ("46", "psr/log", "1.0.2", None, None), - ("47", "psr/simple-cache", "1.0.1", None, None), - ("48", "ralouphie/getallheaders", "3.0.3", None, None), - ("49", "serialization/serialization", "4.0.0", None, None), - ("50", "wikibase/data-model", "9.2.0", None, None), - ("51", "wikibase/data-model-serialization", "2.9.1", None, None), - ("52", "wikibase/data-model-services", "3.15.0", None, None), - ("53", "wikibase/internal-serialization", "2.10.0", None, None), - ("54", "wikibase/term-store", "1.0.4", None, None), - ("55", "wikimedia/assert", "0.2.2", None, None), - ("56", "wikimedia/at-ease", "2.0.0", None, None), - ("57", "wikimedia/base-convert", "2.0.0", None, None), - ("58", "wikimedia/cdb", "1.4.1", None, None), - ("59", "wikimedia/cldr-plural-rule-parser", "1.0.0", None, None), - ("60", "wikimedia/composer-merge-plugin", "1.4.1", None, None), - ("61", "wikimedia/html-formatter", "1.0.2", None, None), - ("62", "wikimedia/ip-set", "2.1.0", None, None), - ("63", "wikimedia/less.php", "1.8.0", None, None), - ("64", "wikimedia/object-factory", "2.1.0", None, None), - ("65", "wikimedia/password-blacklist", "0.1.4", None, None), - ("66", "wikimedia/php-session-serializer", "1.0.7", None, None), - ("67", "wikimedia/purtle", "1.0.7", None, None), - ("68", "wikimedia/relpath", "2.1.1", None, None), - ("69", "wikimedia/remex-html", "2.1.0", None, None), - ("70", "wikimedia/running-stat", "1.2.1", None, None), - ("71", "wikimedia/scoped-callback", "3.0.0", None, None), - ("72", "wikimedia/timestamp", "3.0.0", None, None), - ("73", "wikimedia/utfnormal", "2.0.0", None, None), - ("74", "wikimedia/wait-condition-loop", "1.0.1", None, None), - ("75", "wikimedia/wrappedstring", "3.0.1", None, None), - ("76", "wikimedia/xmp-reader", "0.6.3", None, None), - ("77", "zordius/lightncandy", "0.23", None, None), + ("20", "composer/installers", "1.8.0", None, None), + ("21", "composer/semver", "1.5.0", None, None), + ("22", "cssjanus/cssjanus", "1.3.0", None, None), + ("23", "data-values/common", "0.4.3", None, None), + ("24", "data-values/data-values", "2.3.0", None, None), + ("25", "data-values/geo", "3.0.1", None, None), + ("26", "data-values/interfaces", "0.2.5", None, None), + ("27", "data-values/number", "0.10.1", None, None), + ("28", "data-values/serialization", "1.2.3", None, None), + ("29", "data-values/time", "1.0.1", None, None), + ("30", "diff/diff", "2.3.0", None, None), + ("31", "guzzlehttp/guzzle", "6.3.3", None, None), + ("32", "guzzlehttp/promises", "1.3.1", None, None), + ("33", "guzzlehttp/psr7", "1.6.1", None, None), + ("34", "liuggio/statsd-php-client", "1.0.18", None, None), + ("35", "onoi/message-reporter", "1.4.1", None, None), + ("36", "oojs/oojs-ui", "0.34.1", None, None), + ("37", "pear/console_getopt", "1.4.3", None, None), + ("38", "pear/mail", "1.4.1", None, None), + ("39", "pear/mail_mime", "1.10.2", None, None), + ("40", "pear/net_smtp", "1.8.1", None, None), + ("41", "pear/net_socket", "1.2.2", None, None), + ("42", "pear/pear-core-minimal", "1.10.10", None, None), + ("43", "pear/pear_exception", "1.0.1", None, None), + ("44", "pleonasm/bloom-filter", "1.0.2", None, None), + ("45", "psr/container", "1.0.0", None, None), + ("46", "psr/http-message", "1.0.1", None, None), + ("47", "psr/log", "1.0.2", None, None), + ("48", "psr/simple-cache", "1.0.1", None, None), + ("49", "ralouphie/getallheaders", "3.0.3", None, None), + ("50", "serialization/serialization", "4.0.0", None, None), + ("51", "wikibase/data-model", "9.2.0", None, None), + ("52", "wikibase/data-model-serialization", "2.9.1", None, None), + ("53", "wikibase/data-model-services", "3.15.0", None, None), + ("54", "wikibase/internal-serialization", "2.10.0", None, None), + ("55", "wikibase/term-store", "1.0.4", None, None), + ("56", "wikimedia/assert", "0.2.2", None, None), + ("57", "wikimedia/at-ease", "2.0.0", None, None), + ("58", "wikimedia/base-convert", "2.0.0", None, None), + ("59", "wikimedia/cdb", "1.4.1", None, None), + ("60", "wikimedia/cldr-plural-rule-parser", "1.0.0", None, None), + ("61", "wikimedia/composer-merge-plugin", "1.4.1", None, None), + ("62", "wikimedia/html-formatter", "1.0.2", None, None), + ("63", "wikimedia/ip-set", "2.1.0", None, None), + ("64", "wikimedia/less.php", "1.8.0", None, None), + ("65", "wikimedia/object-factory", "2.1.0", None, None), + ("66", "wikimedia/password-blacklist", "0.1.4", None, None), + ("67", "wikimedia/php-session-serializer", "1.0.7", None, None), + ("68", "wikimedia/purtle", "1.0.7", None, None), + ("69", "wikimedia/relpath", "2.1.1", None, None), + ("70", "wikimedia/remex-html", "2.1.0", None, None), + ("71", "wikimedia/running-stat", "1.2.1", None, None), + ("72", "wikimedia/scoped-callback", "3.0.0", None, None), + ("73", "wikimedia/timestamp", "3.0.0", None, None), + ("74", "wikimedia/utfnormal", "2.0.0", None, None), + ("75", "wikimedia/wait-condition-loop", "1.0.1", None, None), + ("76", "wikimedia/wrappedstring", "3.0.1", None, None), + ("77", "wikimedia/xmp-reader", "0.6.3", None, None), + ("78", "zordius/lightncandy", "0.23", None, None), ] ): assert_software_version( diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 5279dece..ea7e951a 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,5 +1,6 @@ """Test Utilities""" +from tests.utils.assert_meta import assert_page_meta from tests.utils.assert_property_value import ( assert_layered_property_count, assert_layered_property_value, diff --git a/tests/utils/assert_meta.py b/tests/utils/assert_meta.py new file mode 100644 index 00000000..af984182 --- /dev/null +++ b/tests/utils/assert_meta.py @@ -0,0 +1,19 @@ +"""Assert Page Metadata""" + +from tests.utils.assert_property_value import assert_layered_property_value + + +def assert_page_meta( + page: dict, + expected_page_number: int, + expected_page_size: int, + expected_total_count: int, + expected_total_pages: int, +): + """Assert Page Metadata""" + + assert "meta" in page + assert_layered_property_value(page, ["meta", "pageNumber"], expected_page_number) + assert_layered_property_value(page, ["meta", "pageSize"], expected_page_size) + assert_layered_property_value(page, ["meta", "totalCount"], expected_total_count) + assert_layered_property_value(page, ["meta", "totalPages"], expected_total_pages) diff --git a/tests/utils/mock_response.py b/tests/utils/mock_response.py index 6123a33f..ffd8bf85 100644 --- a/tests/utils/mock_response.py +++ b/tests/utils/mock_response.py @@ -1,12 +1,22 @@ """Mock HTTP Response""" +from typing import Optional + class MockResponse: """Mock HTTP Response""" status_code: int content: bytes + url: str + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + pass - def __init__(self, status_code: int, content: bytes): + def __init__(self, url: str, status_code: int, content: Optional[bytes] = None): + self.url = url self.content = content self.status_code = status_code