From 4247d2785d1f5af4592c15fe292b614993f0e3e8 Mon Sep 17 00:00:00 2001 From: PascalEgn Date: Thu, 8 Aug 2024 16:25:45 +0200 Subject: [PATCH] global: add pre-commit with ruff --- .pre-commit-config.yaml | 31 ++--- backend/gunicorn_config.py | 1 - backend/inspirehep/__init__.py | 1 - backend/inspirehep/access_control.py | 1 - backend/inspirehep/accounts/__init__.py | 1 - backend/inspirehep/accounts/api.py | 5 +- backend/inspirehep/accounts/cli.py | 1 - backend/inspirehep/accounts/config.py | 13 +- backend/inspirehep/accounts/decorators.py | 1 - backend/inspirehep/accounts/fixtures.py | 1 - backend/inspirehep/accounts/handlers.py | 6 +- backend/inspirehep/accounts/roles.py | 1 - backend/inspirehep/accounts/views.py | 8 +- ...update_primary_key_in_students_advisors.py | 3 +- ...dd_authors_to_records_table_and_update_.py | 4 +- .../7be4c8b5c5e8_inspirehep_new_migrations.py | 2 - ...add_indexed_to_records_citations_on_is_.py | 1 - ...22b_add_recid_to_recordsauthors_id_type.py | 1 + ...7b2_change_record_citations_to_records_.py | 1 - ...ad8f861_add_index_for_objectversion_key.py | 1 - backend/inspirehep/assign/tasks.py | 13 +- backend/inspirehep/assign/utils.py | 8 +- backend/inspirehep/assign/views.py | 1 - backend/inspirehep/cds/api.py | 3 +- backend/inspirehep/cds/cli.py | 5 +- backend/inspirehep/cds/config.py | 1 - backend/inspirehep/cds/errors.py | 1 - backend/inspirehep/cds/models.py | 1 - backend/inspirehep/celery.py | 3 +- backend/inspirehep/config.py | 1 - backend/inspirehep/curation/__init__.py | 5 + backend/inspirehep/curation/api.py | 13 +- backend/inspirehep/curation/cli.py | 2 +- backend/inspirehep/curation/errors.py | 1 - .../curation/search_check_do/__init__.py | 2 +- .../curation/search_check_do/examples.py | 5 +- backend/inspirehep/curation/utils.py | 12 +- backend/inspirehep/curation/views.py | 10 +- backend/inspirehep/disambiguation/cli.py | 8 +- backend/inspirehep/disambiguation/loaders.py | 1 - backend/inspirehep/disambiguation/tasks.py | 7 +- backend/inspirehep/disambiguation/utils.py | 9 +- backend/inspirehep/editor/authorlist_utils.py | 10 +- backend/inspirehep/editor/config.py | 1 - backend/inspirehep/editor/editor_soft_lock.py | 2 +- backend/inspirehep/editor/errors.py | 1 - backend/inspirehep/editor/views.py | 26 ++-- backend/inspirehep/errors.py | 3 +- backend/inspirehep/factory.py | 3 +- backend/inspirehep/files/api/__init__.py | 2 +- backend/inspirehep/files/api/s3.py | 1 - backend/inspirehep/files/cli.py | 4 +- backend/inspirehep/files/config.py | 1 - backend/inspirehep/files/ext.py | 3 +- backend/inspirehep/files/proxies.py | 1 - backend/inspirehep/fixtures.py | 1 - backend/inspirehep/hal/api.py | 1 - backend/inspirehep/hal/config.py | 1 - backend/inspirehep/hal/core/sword.py | 1 - backend/inspirehep/hal/core/tei.py | 1 - backend/inspirehep/hal/errors.py | 1 - backend/inspirehep/hal/tasks.py | 3 +- backend/inspirehep/hal/templates/hal/art.xml | 2 +- backend/inspirehep/hal/templates/hal/base.xml | 2 +- backend/inspirehep/hal/templates/hal/comm.xml | 2 +- .../inspirehep/hal/templates/hal/preprint.xml | 2 +- backend/inspirehep/hal/utils.py | 1 - backend/inspirehep/hal/views.py | 1 - backend/inspirehep/hepdata/cli.py | 8 +- backend/inspirehep/indexer/api.py | 1 - backend/inspirehep/indexer/base.py | 12 +- backend/inspirehep/indexer/cli.py | 3 +- backend/inspirehep/indexer/tasks.py | 2 - backend/inspirehep/legacy/views.py | 1 - backend/inspirehep/logger/__init__.py | 3 +- backend/inspirehep/logger/config.py | 4 +- backend/inspirehep/logger/ext.py | 1 - backend/inspirehep/mailing/__init__.py | 3 +- backend/inspirehep/mailing/api/__init__.py | 5 + backend/inspirehep/mailing/api/authors.py | 1 - backend/inspirehep/mailing/api/conferences.py | 1 - backend/inspirehep/mailing/api/jobs.py | 6 +- backend/inspirehep/mailing/api/seminars.py | 1 - backend/inspirehep/mailing/cli.py | 1 - backend/inspirehep/mailing/config.py | 1 - backend/inspirehep/mailing/ext.py | 9 +- backend/inspirehep/mailing/loaders.py | 1 - .../inspirehep/mailing/providers/__init__.py | 1 - .../mailing/providers/flask_mail.py | 1 - .../inspirehep/mailing/providers/mailtrain.py | 1 - backend/inspirehep/mailing/utils.py | 1 - backend/inspirehep/mailing/views.py | 4 +- backend/inspirehep/matcher/__init__.py | 1 - backend/inspirehep/matcher/api.py | 14 +- backend/inspirehep/matcher/cli.py | 4 +- backend/inspirehep/matcher/config.py | 3 +- backend/inspirehep/matcher/parsers.py | 7 +- backend/inspirehep/matcher/tasks.py | 8 +- backend/inspirehep/matcher/utils.py | 10 +- backend/inspirehep/matcher/validators.py | 1 - backend/inspirehep/oai/config.py | 1 - backend/inspirehep/oai/serializers.py | 1 - backend/inspirehep/oai/utils.py | 17 ++- backend/inspirehep/orcid/__init__.py | 8 +- backend/inspirehep/orcid/api.py | 1 - backend/inspirehep/orcid/builder.py | 19 ++- backend/inspirehep/orcid/cache.py | 11 +- backend/inspirehep/orcid/cli.py | 1 - backend/inspirehep/orcid/config.py | 1 - backend/inspirehep/orcid/converter.py | 8 +- backend/inspirehep/orcid/domain_models.py | 55 ++++---- backend/inspirehep/orcid/exceptions.py | 9 +- backend/inspirehep/orcid/ext.py | 4 +- .../inspirehep/orcid/push_access_tokens.py | 7 +- backend/inspirehep/orcid/putcode_getter.py | 14 +- backend/inspirehep/orcid/tasks.py | 28 ++-- backend/inspirehep/orcid/utils.py | 6 +- backend/inspirehep/pidstore/api/authors.py | 9 +- backend/inspirehep/pidstore/api/base.py | 3 +- .../inspirehep/pidstore/api/conferences.py | 2 - backend/inspirehep/pidstore/api/data.py | 6 +- .../inspirehep/pidstore/api/experiments.py | 6 +- .../inspirehep/pidstore/api/institutions.py | 6 +- backend/inspirehep/pidstore/api/jobs.py | 6 +- backend/inspirehep/pidstore/api/journals.py | 6 +- backend/inspirehep/pidstore/api/literature.py | 2 - backend/inspirehep/pidstore/api/seminars.py | 6 +- backend/inspirehep/pidstore/cli.py | 12 +- backend/inspirehep/pidstore/converters.py | 3 +- backend/inspirehep/pidstore/errors.py | 1 - backend/inspirehep/pidstore/minters/arxiv.py | 3 +- backend/inspirehep/pidstore/minters/bai.py | 16 ++- backend/inspirehep/pidstore/minters/base.py | 11 +- backend/inspirehep/pidstore/minters/cnum.py | 4 +- .../pidstore/minters/control_number.py | 3 +- backend/inspirehep/pidstore/minters/doi.py | 3 +- backend/inspirehep/pidstore/minters/orcid.py | 3 +- backend/inspirehep/pidstore/minters/texkey.py | 1 - backend/inspirehep/pidstore/models.py | 3 +- backend/inspirehep/pidstore/providers/bai.py | 1 - backend/inspirehep/pidstore/providers/base.py | 1 - backend/inspirehep/pidstore/providers/cnum.py | 6 +- .../inspirehep/pidstore/providers/external.py | 2 - .../inspirehep/pidstore/providers/recid.py | 1 - .../inspirehep/pidstore/providers/texkey.py | 8 +- backend/inspirehep/pidstore/resolvers.py | 6 +- backend/inspirehep/records/__init__.py | 1 - backend/inspirehep/records/api/__init__.py | 21 ++- backend/inspirehep/records/api/authors.py | 10 +- backend/inspirehep/records/api/base.py | 13 +- backend/inspirehep/records/api/conferences.py | 1 - backend/inspirehep/records/api/data.py | 6 +- backend/inspirehep/records/api/experiments.py | 8 +- .../inspirehep/records/api/institutions.py | 8 +- backend/inspirehep/records/api/jobs.py | 1 - backend/inspirehep/records/api/journals.py | 6 +- backend/inspirehep/records/api/literature.py | 4 +- backend/inspirehep/records/api/mixins.py | 9 +- backend/inspirehep/records/api/seminars.py | 7 +- backend/inspirehep/records/cli.py | 5 +- backend/inspirehep/records/config.py | 1 - backend/inspirehep/records/errors.py | 1 - backend/inspirehep/records/ext.py | 5 +- backend/inspirehep/records/links.py | 5 +- .../records/marshmallow/authors/base.py | 6 +- .../marshmallow/authors/common/advisor.py | 1 - .../marshmallow/authors/common/position.py | 7 +- .../records/marshmallow/authors/es.py | 6 +- .../records/marshmallow/authors/ui.py | 13 +- .../inspirehep/records/marshmallow/base.py | 1 - .../common/accelerator_experiment.py | 9 +- .../records/marshmallow/common/mixins.py | 1 - .../records/marshmallow/conferences/base.py | 1 - .../common/proceeding_info_item.py | 1 - .../records/marshmallow/conferences/es.py | 1 - .../records/marshmallow/conferences/ui.py | 1 - .../records/marshmallow/data/base.py | 1 - .../inspirehep/records/marshmallow/data/es.py | 5 +- .../records/marshmallow/experiments/base.py | 1 - .../experiments/common/related_records.py | 1 - .../records/marshmallow/experiments/es.py | 6 +- .../records/marshmallow/experiments/ui.py | 1 - .../marshmallow/fields/list_with_limit.py | 5 +- .../fields/nested_without_empty_objects.py | 3 +- .../records/marshmallow/fields/non_hidden.py | 1 - .../records/marshmallow/institutions/base.py | 1 - .../institutions/common/__init__.py | 1 - .../institutions/common/related_records.py | 1 - .../records/marshmallow/institutions/es.py | 5 +- .../records/marshmallow/institutions/ui.py | 1 - .../records/marshmallow/jobs/base.py | 8 +- .../inspirehep/records/marshmallow/jobs/es.py | 5 +- .../inspirehep/records/marshmallow/jobs/ui.py | 14 +- .../records/marshmallow/journals/base.py | 1 - .../records/marshmallow/journals/es.py | 5 +- .../records/marshmallow/journals/ui.py | 8 +- .../records/marshmallow/literature/authors.py | 7 +- .../records/marshmallow/literature/base.py | 9 +- .../records/marshmallow/literature/bibtex.py | 8 +- .../marshmallow/literature/common/abstract.py | 1 - .../marshmallow/literature/common/author.py | 2 - .../literature/common/citation_item.py | 17 ++- .../literature/common/collaboration.py | 1 - .../common/collaboration_with_suffix.py | 5 +- .../literature/common/conference_info_item.py | 1 - .../marshmallow/literature/common/doi.py | 1 - .../common/external_system_identifier.py | 5 +- .../marshmallow/literature/common/isbn.py | 1 - .../common/publication_info_item.py | 1 - .../literature/common/reference_item.py | 16 ++- .../literature/common/thesis_info.py | 1 - .../records/marshmallow/literature/cv.py | 1 - .../records/marshmallow/literature/es.py | 13 +- .../records/marshmallow/literature/latex.py | 10 +- .../marshmallow/literature/references.py | 5 +- .../records/marshmallow/literature/ui.py | 32 ++--- .../records/marshmallow/literature/utils.py | 1 - .../records/marshmallow/seminars/base.py | 2 - .../seminars/common/literature_record.py | 1 - .../marshmallow/seminars/common/speaker.py | 2 - .../records/marshmallow/seminars/es.py | 5 +- .../records/marshmallow/seminars/ui.py | 4 +- .../inspirehep/records/marshmallow/utils.py | 3 +- backend/inspirehep/records/models.py | 5 +- backend/inspirehep/records/receivers.py | 30 +++-- .../inspirehep/records/serializers/bibtex.py | 6 +- backend/inspirehep/records/serializers/cv.py | 6 +- .../inspirehep/records/serializers/jinja.py | 1 - .../records/serializers/json/__init__.py | 2 +- .../records/serializers/json/authors.py | 1 - .../records/serializers/json/conferences.py | 1 - .../records/serializers/json/experiments.py | 1 - .../records/serializers/json/institutions.py | 1 - .../records/serializers/json/jobs.py | 1 - .../records/serializers/json/journals.py | 1 - .../records/serializers/json/literature.py | 1 - .../records/serializers/json/raw.py | 1 - .../records/serializers/json/seminars.py | 1 - .../inspirehep/records/serializers/latex.py | 8 +- backend/inspirehep/records/serializers/oai.py | 1 - backend/inspirehep/records/tasks.py | 1 - backend/inspirehep/records/utils.py | 8 +- backend/inspirehep/records/views.py | 20 +-- backend/inspirehep/search/aggregations.py | 2 - backend/inspirehep/search/api.py | 21 ++- backend/inspirehep/search/config.py | 1 - backend/inspirehep/search/errors.py | 1 - backend/inspirehep/search/facets.py | 2 - .../inspirehep/search/factories/__init__.py | 1 - backend/inspirehep/search/factories/facet.py | 3 +- backend/inspirehep/search/factories/filter.py | 3 +- backend/inspirehep/search/factories/query.py | 10 +- backend/inspirehep/search/factories/search.py | 11 +- .../mappings/os-v2/records/seminars.json | 2 +- .../search/mappings/v7/records/seminars.json | 2 +- backend/inspirehep/search/serializers.py | 11 +- backend/inspirehep/search/utils.py | 1 - backend/inspirehep/search/views.py | 1 - backend/inspirehep/serializers.py | 16 +-- backend/inspirehep/sitemap/cli.py | 3 +- backend/inspirehep/sitemap/collections.py | 1 - backend/inspirehep/sitemap/config.py | 1 - backend/inspirehep/sitemap/sitemap.py | 7 +- backend/inspirehep/sitemap/tasks.py | 9 +- .../sitemap/templates/sitemap/index.xml | 2 +- .../sitemap/templates/sitemap/page.xml | 2 +- backend/inspirehep/sitemap/utils.py | 1 - backend/inspirehep/sitemap/views.py | 1 - backend/inspirehep/snow/api.py | 37 +++--- backend/inspirehep/snow/config.py | 1 - backend/inspirehep/snow/errors.py | 1 - backend/inspirehep/snow/utils.py | 1 - backend/inspirehep/submissions/errors.py | 3 +- backend/inspirehep/submissions/loaders.py | 6 +- .../submissions/marshmallow/author.py | 1 - .../submissions/marshmallow/conferences.py | 1 - .../submissions/marshmallow/experiments.py | 2 - .../submissions/marshmallow/institutions.py | 3 - .../inspirehep/submissions/marshmallow/job.py | 1 - .../submissions/marshmallow/journals.py | 1 - .../submissions/marshmallow/literature.py | 2 +- .../submissions/marshmallow/seminars.py | 5 +- backend/inspirehep/submissions/serializers.py | 8 +- backend/inspirehep/submissions/tasks.py | 1 - backend/inspirehep/submissions/utils.py | 1 - backend/inspirehep/submissions/views.py | 39 +++--- backend/inspirehep/tools/errors.py | 1 - backend/inspirehep/tools/utils.py | 5 +- backend/inspirehep/tools/views.py | 8 +- backend/inspirehep/utils.py | 11 +- backend/inspirehep/version.py | 2 - backend/inspirehep/views.py | 1 - backend/inspirehep/wsgi.py | 1 - backend/tests/conftest.py | 3 +- backend/tests/helpers/cleanups.py | 12 +- backend/tests/helpers/compare.py | 1 - backend/tests/helpers/factories/db/base.py | 3 +- .../helpers/factories/db/invenio_accounts.py | 6 +- .../factories/db/invenio_oauthclient.py | 21 +-- .../helpers/factories/db/invenio_pidstore.py | 8 +- .../helpers/factories/db/invenio_records.py | 17 +-- .../tests/helpers/factories/models/base.py | 2 - .../helpers/factories/models/pidstore.py | 2 - .../tests/helpers/factories/models/records.py | 2 - .../factories/models/user_access_token.py | 10 +- backend/tests/helpers/providers/faker.py | 2 - .../helpers/providers/record_provider.py | 27 ++-- backend/tests/helpers/utils.py | 20 +-- .../integration-async/accounts/test_views.py | 1 - .../integration-async/assign/test_tasks.py | 7 +- .../integration-async/assign/test_views.py | 9 +- .../assign/test_views/2171912.json | 2 +- .../tests/integration-async/cds/conftest.py | 5 +- .../tests/integration-async/cds/test_api.py | 4 +- .../tests/integration-async/cds/test_cli.py | 4 +- .../integration-async/cds/test_models.py | 4 +- backend/tests/integration-async/conftest.py | 47 ++++--- .../curation/test_curation_cli.py | 7 +- .../disambiguation/test_disambiguation_cli.py | 4 +- .../test_disambiguation_tasks.py | 52 +++----- .../editor/test_editor_api.py | 11 +- .../integration-async/hal/test_hal_tasks.py | 6 +- .../integration-async/matcher/test_cli.py | 4 +- .../tests/integration-async/oai/test_views.py | 4 +- .../pidstore/minters/test_minter_cnum.py | 6 +- .../pidstore/minters/test_minter_texkey.py | 3 +- .../pidstore/providers/test_bai_provider.py | 3 +- .../integration-async/pidstore/test_cli.py | 21 +-- .../integration-async/pidstore/test_models.py | 4 +- .../integration-async/records/api/__init__.py | 1 - .../records/api/test_authors.py | 3 +- .../records/api/test_base.py | 6 +- .../records/api/test_literature.py | 6 +- .../records/indexer/__init__.py | 1 - .../records/indexer/test_authors.py | 6 +- .../records/indexer/test_conferences.py | 8 +- .../records/indexer/test_data.py | 6 +- .../records/indexer/test_experiments.py | 8 +- .../records/indexer/test_fulltext.py | 13 +- .../records/indexer/test_indexer_cli.py | 30 +++-- .../records/indexer/test_institutions.py | 8 +- .../records/indexer/test_jobs.py | 6 +- .../records/indexer/test_journals.py | 6 +- .../records/indexer/test_literature.py | 16 +-- .../records/indexer/test_seminars.py | 6 +- .../records/test_fulltext.py | 9 +- .../records/test_record_views.py | 26 ++-- .../records/test_records_cli.py | 10 +- .../integration-async/records/test_tasks.py | 17 +-- backend/tests/integration/__init__.py | 1 - .../tests/integration/accounts/__init__.py | 1 - .../accounts/test_account_handlers.py | 2 - .../tests/integration/accounts/test_cli.py | 1 - .../integration/accounts/test_decorators.py | 20 +-- .../tests/integration/accounts/test_views.py | 1 - backend/tests/integration/alembic/conftest.py | 1 - .../tests/integration/alembic/test_alembic.py | 3 +- .../tests/integration/assign/test_tasks.py | 2 - .../tests/integration/assign/test_views.py | 10 +- backend/tests/integration/cds/test_api.py | 8 +- .../config/test_views_suggesters.py | 11 +- backend/tests/integration/conftest.py | 64 +++++----- .../tests/integration/curation/test_api.py | 69 +++++----- .../literature/literature_1459277.json | 2 +- .../literature/literature_1800446.json | 2 +- .../literature/literature_1836272.json | 2 +- .../literature/literature_1862822.json | 2 +- .../literature/literature_1863053.json | 2 +- .../curation/test_curation_views.py | 4 +- .../curation/test_search_check_do_base.py | 18 +-- .../tests/integration/curation/test_utils.py | 2 - .../integration/disambiguation/__init__.py | 1 - .../integration/disambiguation/test_cli.py | 2 - .../integration/disambiguation/test_utils.py | 2 - backend/tests/integration/editor/conftest.py | 5 +- .../integration/editor/test_editor_views.py | 120 +++++++++--------- backend/tests/integration/files/__init__.py | 1 - .../tests/integration/files/test_api_s3.py | 4 +- backend/tests/integration/files/test_cli.py | 1 - backend/tests/integration/files/test_ext.py | 2 - .../tests/integration/fixtures/1663923.xml | 2 +- .../tests/integration/fixtures/1663924.xml | 2 +- backend/tests/integration/hal/conftest.py | 3 +- .../integration/hal/data/_get_hal_id_map.json | 2 +- .../hal/data/_get_hal_id_map_institution.json | 2 +- backend/tests/integration/hal/data/aofr.xsd | 24 ++-- .../integration/hal/data/convert_to_tei.json | 2 +- .../convert_to_tei_handles_preprints.json | 2 +- .../hal/data/expected_conference_record.json | 2 +- .../hal/data/get_conference_record.json | 2 +- .../hal/data/hal_preprod_institute.json | 2 +- .../hal/data/hal_preprod_record.json | 2 +- .../data/hal_preprod_unicode_institute.json | 2 +- .../hal/data/hal_preprod_unicode_record.json | 2 +- .../tests/integration/hal/test_core_sword.py | 10 +- .../tests/integration/hal/test_core_tei.py | 4 +- backend/tests/integration/hal/test_hal_api.py | 2 - backend/tests/integration/hal/test_utils.py | 2 - backend/tests/integration/hepdata/__init__.py | 1 - backend/tests/integration/hepdata/test_cli.py | 1 - backend/tests/integration/indexer/test_cli.py | 12 +- .../indexer/test_indexer_authors.py | 1 - .../indexer/test_indexer_conferences.py | 4 +- .../integration/indexer/test_indexer_data.py | 1 - .../indexer/test_indexer_experiments.py | 1 - .../indexer/test_indexer_institutions.py | 4 +- .../integration/indexer/test_indexer_jobs.py | 1 - .../indexer/test_indexer_journals.py | 1 - .../indexer/test_indexer_literature.py | 3 - .../indexer/test_indexer_seminars.py | 4 +- .../indexer/test_indexer_seminars/1.json | 2 +- backend/tests/integration/legacy/__init__.py | 1 - .../tests/integration/legacy/test_views.py | 1 - backend/tests/integration/mailing/__init__.py | 1 - backend/tests/integration/mailing/conftest.py | 10 +- .../integration/mailing/data/1234567.json | 2 +- .../integration/mailing/data/1234568.json | 2 +- .../tests/integration/mailing/data/rss.xml | 2 +- .../mailing/test_api_conferences.py | 3 +- .../integration/mailing/test_api_jobs.py | 12 +- .../integration/mailing/test_api_seminars.py | 3 +- backend/tests/integration/mailing/test_cli.py | 3 +- backend/tests/integration/mailing/test_ext.py | 1 - .../mailing/test_providers_mailtrain.py | 2 - .../tests/integration/mailing/test_views.py | 3 +- backend/tests/integration/matcher/__init__.py | 1 - backend/tests/integration/matcher/test_api.py | 4 +- .../tests/integration/matcher/test_tasks.py | 4 +- .../tests/integration/matcher/test_utils.py | 2 - .../integration/matcher/test_validators.py | 1 - .../tests/integration/matcher/test_views.py | 3 +- ...t_get_all_inspire_putcodes_happy_flow.yaml | 10 +- ..._get_all_inspire_putcodes_with_recids.yaml | 4 +- ...tcodes_and_recids_by_identifiers_iter.yaml | 4 +- ...dPutcodeGetter.test_putcode_not_found.yaml | 4 +- backend/tests/integration/orcid/conftest.py | 14 +- .../integration/orcid/fixtures/1608652.xml | 3 +- .../orcid/fixtures/common_2.0/common-2.0.xsd | 56 ++++---- .../orcid/fixtures/record_2.0/error-2.0.xsd | 2 +- .../orcid/fixtures/record_2.0/work-2.0.xsd | 6 +- .../tests/integration/orcid/test_orcid_api.py | 71 ++++++----- .../integration/orcid/test_orcid_cache.py | 12 +- .../test_orcid_cli_import_legacy_tokens.py | 85 ++++++------- .../integration/orcid/test_orcid_converter.py | 4 +- .../orcid/test_orcid_domain_models.py | 109 +++++++++------- .../orcid/test_orcid_push_access_tokens.py | 22 ++-- .../orcid/test_orcid_putcode_getter.py | 13 +- .../integration/orcid/test_orcid_tasks.py | 16 +-- .../orcid/test_orcid_tasks_orcid_push.py | 106 +++++++++------- .../integration/orcid/test_orcid_utils.py | 29 +++-- .../pidstore/minters/test_minters_arxiv.py | 4 +- .../pidstore/minters/test_minters_bai.py | 15 +-- .../pidstore/minters/test_minters_cnum.py | 10 +- .../minters/test_minters_control_number.py | 6 +- .../pidstore/minters/test_minters_doi.py | 4 +- .../pidstore/minters/test_minters_orcid.py | 4 +- .../pidstore/minters/test_minters_texkey.py | 4 +- .../providers/pid_from_legacy/conftest.py | 2 - .../test_get_pid_from_legacy.py | 9 +- .../providers/test_providers_recid.py | 13 +- .../tests/integration/pidstore/test_models.py | 4 +- .../pidstore/test_pidstore_api_base.py | 3 +- .../tests/integration/records/api/__init__.py | 1 - .../records/api/test_api_authors.py | 4 +- .../integration/records/api/test_api_base.py | 41 +++--- .../records/api/test_api_conferences.py | 9 +- .../integration/records/api/test_api_data.py | 6 +- .../records/api/test_api_experiments.py | 9 +- .../records/api/test_api_institutions.py | 9 +- .../integration/records/api/test_api_jobs.py | 7 +- .../records/api/test_api_journals.py | 7 +- .../records/api/test_api_literature.py | 34 +++-- .../records/api/test_api_mixins.py | 5 +- .../records/api/test_api_seminars.py | 6 +- .../data/cv_with_author_with_editor_role.html | 2 +- ...ith_author_with_multiple_affiliations.html | 2 +- .../records/serializers/json/test_authors.py | 4 +- .../serializers/json/test_conferences.py | 4 +- .../records/serializers/json/test_data.py | 1 - .../serializers/json/test_experiments.py | 4 +- .../serializers/json/test_institutions.py | 8 +- .../records/serializers/json/test_jobs.py | 6 +- .../records/serializers/json/test_journals.py | 1 - .../serializers/json/test_literature.py | 6 +- .../records/serializers/json/test_seminars.py | 10 +- .../serializers/json/test_seminars/1.json | 2 +- .../records/serializers/test_bibtex.py | 20 +-- .../records/serializers/test_cv.py | 3 - .../records/serializers/test_latex.py | 12 +- backend/tests/integration/records/test_cli.py | 4 +- .../tests/integration/records/test_links.py | 6 +- .../tests/integration/records/test_tasks.py | 2 - .../tests/integration/records/test_utils.py | 8 +- .../integration/records/views/test_views.py | 14 +- .../records/views/test_views_article.py | 1 - .../records/views/test_views_authors.py | 14 +- .../records/views/test_views_citations.py | 1 - .../records/views/test_views_conferences.py | 6 +- .../records/views/test_views_data.py | 1 - .../records/views/test_views_experiments.py | 2 - .../records/views/test_views_institutions.py | 6 +- .../records/views/test_views_jobs.py | 10 +- .../records/views/test_views_journals.py | 2 - .../records/views/test_views_literature.py | 30 ++--- .../records/views/test_views_seminars.py | 1 - backend/tests/integration/search/__init__.py | 1 - backend/tests/integration/search/conftest.py | 5 +- .../integration/search/test_aggregations.py | 3 +- backend/tests/integration/search/test_api.py | 52 +++----- .../search/test_api/conferences/1305234.json | 2 +- .../search/test_api/conferences/1388546.json | 2 +- .../search/test_api/conferences/1505770.json | 2 +- .../search/test_api/conferences/1638643.json | 2 +- .../search/test_api/conferences/1643455.json | 2 +- .../search/test_api/conferences/1685062.json | 2 +- .../search/test_api/conferences/1732179.json | 2 +- .../search/test_api/conferences/1754131.json | 2 +- .../search/test_api/conferences/1830349.json | 2 +- .../search/test_api/conferences/1838279.json | 2 +- .../search/test_api/conferences/1844340.json | 2 +- .../search/test_api/conferences/1848328.json | 2 +- .../search/test_api/conferences/1852015.json | 2 +- .../search/test_api/conferences/965775.json | 2 +- .../search/test_api/conferences/966168.json | 2 +- .../search/test_api/conferences/968851.json | 2 +- .../search/test_api/conferences/978211.json | 2 +- .../search/test_api/conferences/978212.json | 2 +- .../search/test_api/conferences/978213.json | 2 +- .../tests/integration/search/test_facets.py | 9 +- .../search/test_factories_facet.py | 26 ++-- .../search/test_factories_filter.py | 18 +-- .../search/test_factories_query.py | 6 +- .../search/test_factories_search.py | 10 +- .../tests/integration/search/test_utils.py | 60 +++++---- .../tests/integration/search/test_views.py | 1 - backend/tests/integration/sitemap/test_cli.py | 4 +- backend/tests/integration/snow/test_snow.py | 43 ++++--- .../tests/integration/snow/test_snow_views.py | 75 +++++------ .../submissions/test_submissions_views.py | 36 +++--- backend/tests/integration/test_celery.py | 1 - backend/tests/integration/test_factory.py | 1 - backend/tests/integration/test_serializers.py | 8 +- backend/tests/integration/test_views.py | 1 - backend/tests/integration/tools/__init__.py | 1 - backend/tests/integration/tools/test_views.py | 18 +-- .../bibliography_generator_test.tex | 2 +- .../tests/unit/assign/test_assign_utils.py | 6 +- .../tests/unit/assign/test_assign_views.py | 1 - backend/tests/unit/curation/test_utils.py | 2 - backend/tests/unit/disambiguation/__init__.py | 1 - .../tests/unit/disambiguation/test_utils.py | 4 +- .../unit/editor/test_authorlist_utils.py | 15 +-- backend/tests/unit/files/test_s3.py | 17 ++- backend/tests/unit/hal/test_core_tei.py | 2 - backend/tests/unit/hal/test_utils.py | 2 - backend/tests/unit/logger/test_ext.py | 2 - backend/tests/unit/logger/test_logger.py | 2 - backend/tests/unit/mailing/__init__.py | 1 - backend/tests/unit/mailing/test_utils.py | 4 +- backend/tests/unit/matcher/test_parsers.py | 2 - backend/tests/unit/matcher/test_utils.py | 22 ++-- backend/tests/unit/oai/test_utils.py | 30 ++--- .../unit/pidstore/api/test_pidstore_base.py | 15 +-- .../tests/unit/pidstore/test_minters_base.py | 1 - .../test_providers/test_bai_provider.py | 58 ++++----- .../test_providers/test_texkey_provider.py | 46 +++---- backend/tests/unit/records/api/test_base.py | 2 - .../tests/unit/records/api/test_literature.py | 1 - .../tests/unit/records/api/test_subclasses.py | 1 - .../tests/unit/records/indexer/test_base.py | 23 ++-- .../unit/records/indexer/test_index_record.py | 4 +- .../records/marshmallow/author/__init__.py | 1 - .../author/common/test_position.py | 2 - .../records/marshmallow/author/test_base.py | 2 - .../records/marshmallow/author/test_ui.py | 2 - .../common/test_accelerator_experiment.py | 2 - .../marshmallow/conferences/__init__.py | 1 - .../common/test_proceeding_info_item.py | 1 - .../marshmallow/conferences/test_base.py | 2 - .../marshmallow/conferences/test_ui.py | 1 - .../unit/records/marshmallow/data/__init__.py | 1 - .../marshmallow/experiments/__init__.py | 1 - .../marshmallow/experiments/test_base.py | 1 - .../marshmallow/experiments/test_es.py | 2 - .../fields/test_list_with_limit.py | 4 +- .../test_nested_without_empty_objects.py | 4 +- .../marshmallow/fields/test_non_hidden.py | 4 +- .../marshmallow/institutions/__init__.py | 1 - .../marshmallow/institutions/test_base.py | 1 - .../marshmallow/institutions/test_es.py | 2 - .../unit/records/marshmallow/jobs/__init__.py | 1 - .../records/marshmallow/jobs/test_base.py | 2 - .../records/marshmallow/journals/__init__.py | 1 - .../records/marshmallow/journals/test_base.py | 1 - .../records/marshmallow/journals/test_es.py | 2 - .../records/marshmallow/journals/test_ui.py | 1 - .../literature/common/test_abstract_source.py | 10 +- .../literature/common/test_author.py | 2 - .../literature/common/test_citation_item.py | 4 - .../literature/common/test_collaboration.py | 2 - .../common/test_conference_info_item.py | 4 +- .../marshmallow/literature/common/test_doi.py | 4 +- .../common/test_external_system_identifier.py | 4 +- .../literature/common/test_isbn.py | 2 - .../common/test_publication_info_item.py | 2 - .../literature/common/test_reference_item.py | 14 +- .../literature/common/test_thesis_info.py | 2 - .../literature/test_authors_schema.py | 2 - .../literature/test_base_schema.py | 2 - .../literature/test_bibtex_schema.py | 6 +- .../marshmallow/literature/test_es_schema.py | 6 +- .../literature/test_latex_schema.py | 4 +- .../test_literature_marshmallow_utils.py | 1 - .../literature/test_references_schema.py | 2 - .../records/marshmallow/literature/test_ui.py | 4 +- .../seminars/common/test_speaker.py | 2 - .../records/marshmallow/seminars/test_base.py | 1 - .../marshmallow/test_marshmallow_base.py | 4 +- .../tests/unit/submissions/test_authors.py | 2 - backend/tests/unit/submissions/test_jobs.py | 1 - .../tests/unit/submissions/test_literature.py | 6 +- backend/tests/unit/test_serializers.py | 5 +- ruff.toml | 39 ++++++ 623 files changed, 1814 insertions(+), 2564 deletions(-) create mode 100644 backend/inspirehep/curation/__init__.py create mode 100644 backend/inspirehep/mailing/api/__init__.py create mode 100644 ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f9cf3c260..a9ee68532e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,19 @@ +files: ^backend/ repos: - - repo: https://github.com/psf/black - rev: '24.2.0' + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 hooks: - - id: black - - repo: https://github.com/pre-commit/mirrors-prettier - rev: 'v3.1.0' + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: fix-byte-order-marker + - id: mixed-line-ending + - id: name-tests-test + args: [ --pytest-test-first ] + exclude: '^(?!factories/)' + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.6 hooks: - - id: prettier - - repo: https://github.com/pycqa/isort - rev: '5.13.2' - hooks: - - id: isort - - repo: https://github.com/pycqa/flake8 - rev: '7.0.0' - hooks: - - id: flake8 - args: ['--config=backend/setup.cfg'] + - id: ruff + args: [ --fix ] + - id: ruff-format diff --git a/backend/gunicorn_config.py b/backend/gunicorn_config.py index 55369fec9b..8320d8a4e6 100644 --- a/backend/gunicorn_config.py +++ b/backend/gunicorn_config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/__init__.py b/backend/inspirehep/__init__.py index 11549a6294..978129675d 100644 --- a/backend/inspirehep/__init__.py +++ b/backend/inspirehep/__init__.py @@ -8,7 +8,6 @@ """inspirehep.""" - from .version import __version__ __all__ = ("__version__",) diff --git a/backend/inspirehep/access_control.py b/backend/inspirehep/access_control.py index d38bbe60e4..7e7ca3c962 100644 --- a/backend/inspirehep/access_control.py +++ b/backend/inspirehep/access_control.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/accounts/__init__.py b/backend/inspirehep/accounts/__init__.py index 71f4302bdb..ffa3e31145 100644 --- a/backend/inspirehep/accounts/__init__.py +++ b/backend/inspirehep/accounts/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/accounts/api.py b/backend/inspirehep/accounts/api.py index 9e89a24fdf..f5047c318d 100644 --- a/backend/inspirehep/accounts/api.py +++ b/backend/inspirehep/accounts/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -84,8 +83,8 @@ def get_current_user_remote_orcid_account(): def get_allowed_roles_for_collections(collections, read_only=True): - private_collections = set(collections) - ( - current_app.config["NON_PRIVATE_LITERATURE_COLLECTIONS"] + private_collections = ( + set(collections) - (current_app.config["NON_PRIVATE_LITERATURE_COLLECTIONS"]) ) roles = set() for col in private_collections: diff --git a/backend/inspirehep/accounts/cli.py b/backend/inspirehep/accounts/cli.py index 736d4686fc..1874c40937 100644 --- a/backend/inspirehep/accounts/cli.py +++ b/backend/inspirehep/accounts/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # diff --git a/backend/inspirehep/accounts/config.py b/backend/inspirehep/accounts/config.py index 89c5977c4b..8028bc9f06 100644 --- a/backend/inspirehep/accounts/config.py +++ b/backend/inspirehep/accounts/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -24,9 +23,9 @@ "scope": " ".join(["/read-limited", "/activities/update", "/person/update"]), "show_login": "true", } -REMOTE_SANDBOX_REST_APP["signup_handler"][ - "setup" -] = "inspirehep.accounts.handlers:account_setup_handler" +REMOTE_SANDBOX_REST_APP["signup_handler"]["setup"] = ( + "inspirehep.accounts.handlers:account_setup_handler" +) # ============================================================================== # ORCID production ============================================================= @@ -37,9 +36,9 @@ "scope": " ".join(["/read-limited", "/activities/update", "/person/update"]), "show_login": "true", } -REMOTE_REST_APP["signup_handler"][ - "setup" -] = "inspirehep.accounts.handlers:account_setup_handler" +REMOTE_REST_APP["signup_handler"]["setup"] = ( + "inspirehep.accounts.handlers:account_setup_handler" +) REMOTE_REST_APP["params"]["access_token_url"] = "https://api.orcid.org/oauth/token" REMOTE_REST_APP["params"]["authorize_url"] = "https://orcid.org/oauth/authorize" # ============================================================================== diff --git a/backend/inspirehep/accounts/decorators.py b/backend/inspirehep/accounts/decorators.py index 598dbc757f..c3a276747b 100644 --- a/backend/inspirehep/accounts/decorators.py +++ b/backend/inspirehep/accounts/decorators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/accounts/fixtures.py b/backend/inspirehep/accounts/fixtures.py index a084e5670f..369932104a 100644 --- a/backend/inspirehep/accounts/fixtures.py +++ b/backend/inspirehep/accounts/fixtures.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/accounts/handlers.py b/backend/inspirehep/accounts/handlers.py index 0d9aec8ce3..ef07dd9061 100644 --- a/backend/inspirehep/accounts/handlers.py +++ b/backend/inspirehep/accounts/handlers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -12,9 +11,10 @@ from invenio_oauthclient.utils import oauth_link_external_id from invenio_pidstore.models import PersistentIdentifier -from .api import get_current_user_remote_orcid_account +from inspirehep.accounts.api import get_current_user_remote_orcid_account from inspirehep.records.utils import get_pid_for_pid + def get_current_user_data(): remote_orcid_account = get_current_user_remote_orcid_account() orcid_account_extra_data = ( @@ -26,7 +26,7 @@ def get_current_user_data(): "email": current_user.email, "roles": [role.name for role in current_user.roles], "orcid": orcid, - "profile_control_number": get_pid_for_pid('orcid', orcid, 'recid'), + "profile_control_number": get_pid_for_pid("orcid", orcid, "recid"), "allow_orcid_push": orcid_account_extra_data.get("allow_push"), } } diff --git a/backend/inspirehep/accounts/roles.py b/backend/inspirehep/accounts/roles.py index 869320a452..c36c3775db 100644 --- a/backend/inspirehep/accounts/roles.py +++ b/backend/inspirehep/accounts/roles.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/accounts/views.py b/backend/inspirehep/accounts/views.py index 7bc175c788..af769cc31d 100644 --- a/backend/inspirehep/accounts/views.py +++ b/backend/inspirehep/accounts/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -15,13 +14,12 @@ from webargs import fields from webargs.flaskparser import FlaskParser +from inspirehep.accounts.api import get_current_user_remote_orcid_account +from inspirehep.accounts.decorators import login_required +from inspirehep.accounts.handlers import get_current_user_data from inspirehep.orcid.tasks import push_account_literature_to_orcid from inspirehep.serializers import jsonify -from .api import get_current_user_remote_orcid_account -from .decorators import login_required -from .handlers import get_current_user_data - parser = FlaskParser() blueprint = Blueprint( diff --git a/backend/inspirehep/alembic/35ba3d715114_update_primary_key_in_students_advisors.py b/backend/inspirehep/alembic/35ba3d715114_update_primary_key_in_students_advisors.py index b52cd8a131..b36f43ad1f 100644 --- a/backend/inspirehep/alembic/35ba3d715114_update_primary_key_in_students_advisors.py +++ b/backend/inspirehep/alembic/35ba3d715114_update_primary_key_in_students_advisors.py @@ -5,8 +5,7 @@ # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. -"""update primary key in students advisors -""" +"""update primary key in students advisors""" import sqlalchemy as sa from alembic import op diff --git a/backend/inspirehep/alembic/595c36d68964_add_authors_to_records_table_and_update_.py b/backend/inspirehep/alembic/595c36d68964_add_authors_to_records_table_and_update_.py index c4305f2f5b..09b3ae9732 100644 --- a/backend/inspirehep/alembic/595c36d68964_add_authors_to_records_table_and_update_.py +++ b/backend/inspirehep/alembic/595c36d68964_add_authors_to_records_table_and_update_.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -9,11 +8,10 @@ import sqlalchemy as sa from alembic import op +from inspirehep.records.models import AuthorSchemaType from sqlalchemy.dialects.postgresql import ENUM from sqlalchemy_utils import UUIDType -from inspirehep.records.models import AuthorSchemaType - revision = "595c36d68964" down_revision = "cea5fa2e5d2c" branch_labels = () diff --git a/backend/inspirehep/alembic/7be4c8b5c5e8_inspirehep_new_migrations.py b/backend/inspirehep/alembic/7be4c8b5c5e8_inspirehep_new_migrations.py index 91abf4edca..79c6ab523a 100644 --- a/backend/inspirehep/alembic/7be4c8b5c5e8_inspirehep_new_migrations.py +++ b/backend/inspirehep/alembic/7be4c8b5c5e8_inspirehep_new_migrations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,7 +7,6 @@ """Inspirehep initial revision of migrations which makes db identical like in inspire-next""" - # revision identifiers, used by Alembic. revision = "7be4c8b5c5e8" down_revision = "b5be5fda2ee7" diff --git a/backend/inspirehep/alembic/8ba47044154a_add_indexed_to_records_citations_on_is_.py b/backend/inspirehep/alembic/8ba47044154a_add_indexed_to_records_citations_on_is_.py index 95a94235a9..6642e66e79 100644 --- a/backend/inspirehep/alembic/8ba47044154a_add_indexed_to_records_citations_on_is_.py +++ b/backend/inspirehep/alembic/8ba47044154a_add_indexed_to_records_citations_on_is_.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/alembic/b495825c322b_add_recid_to_recordsauthors_id_type.py b/backend/inspirehep/alembic/b495825c322b_add_recid_to_recordsauthors_id_type.py index 9ae35ddad8..6dbd90f81d 100644 --- a/backend/inspirehep/alembic/b495825c322b_add_recid_to_recordsauthors_id_type.py +++ b/backend/inspirehep/alembic/b495825c322b_add_recid_to_recordsauthors_id_type.py @@ -6,6 +6,7 @@ # under the terms of the MIT License; see LICENSE file for more details. """add recid to RecordsAuthors id_type""" + from alembic import op # revision identifiers, used by Alembic. diff --git a/backend/inspirehep/alembic/c6570e49b7b2_change_record_citations_to_records_.py b/backend/inspirehep/alembic/c6570e49b7b2_change_record_citations_to_records_.py index 6346509163..28ce5eca57 100644 --- a/backend/inspirehep/alembic/c6570e49b7b2_change_record_citations_to_records_.py +++ b/backend/inspirehep/alembic/c6570e49b7b2_change_record_citations_to_records_.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/alembic/e5e43ad8f861_add_index_for_objectversion_key.py b/backend/inspirehep/alembic/e5e43ad8f861_add_index_for_objectversion_key.py index af85e79ba6..b614c37957 100644 --- a/backend/inspirehep/alembic/e5e43ad8f861_add_index_for_objectversion_key.py +++ b/backend/inspirehep/alembic/e5e43ad8f861_add_index_for_objectversion_key.py @@ -7,7 +7,6 @@ """Add index for ObjectVersion.key""" - # revision identifiers, used by Alembic. revision = "e5e43ad8f861" down_revision = "788a3a61a635" diff --git a/backend/inspirehep/assign/tasks.py b/backend/inspirehep/assign/tasks.py index 6ba491ad7f..dff854d5ac 100644 --- a/backend/inspirehep/assign/tasks.py +++ b/backend/inspirehep/assign/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -117,9 +116,9 @@ def assign_conference(record, conference_ref, cnum): cnum=cnum, ) elif cnum == record_cnum: - builder.record["publication_info"][idx][ - "conference_record" - ] = conference_ref + builder.record["publication_info"][idx]["conference_record"] = ( + conference_ref + ) LOGGER.warning( "conference cnum already assigned to paper without ref.", recid=record.control_number, @@ -226,9 +225,9 @@ def create_rt_ticket_for_claiming_action( from_author_recid, lit_record.get("authors", []) ) if matched_author_name: - incompatibile_names_papers[ - f"{INSPIREHEP_URL}/literature/{paper_recid}" - ] = matched_author_name + incompatibile_names_papers[f"{INSPIREHEP_URL}/literature/{paper_recid}"] = ( + matched_author_name + ) to_author_names = [] to_author_names.extend(get_value(to_author_record, "name.name_variants", [])) diff --git a/backend/inspirehep/assign/utils.py b/backend/inspirehep/assign/utils.py index 565e7f6fe3..397b91808d 100644 --- a/backend/inspirehep/assign/utils.py +++ b/backend/inspirehep/assign/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -117,10 +116,7 @@ def get_last_names(name): return {unidecode(parsed_name.first)} # corner case for full names without comma, # we are treating them as last names (ie. "Smith Davis") - if "," not in name: - names = name.split() - else: - names = parsed_name.last_list + names = name.split() if "," not in name else parsed_name.last_list return {unidecode(name) for name in names} @@ -148,7 +144,7 @@ def _check_names_compability(lit_record, author_parsed_name, last_names_only=Fal ) matched_authors_recids = [ recid - for recid in lit_authors_names_recids.keys() + for recid in lit_authors_names_recids if lit_authors_names_recids[recid] == author_name_to_compare ] if len(matched_authors_recids) == 1: diff --git a/backend/inspirehep/assign/views.py b/backend/inspirehep/assign/views.py index 35eca1d9d4..9c5b5197ee 100644 --- a/backend/inspirehep/assign/views.py +++ b/backend/inspirehep/assign/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/cds/api.py b/backend/inspirehep/cds/api.py index dddeff80e8..9f0e02e1c1 100644 --- a/backend/inspirehep/cds/api.py +++ b/backend/inspirehep/cds/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -8,13 +7,13 @@ import requests import structlog -from opensearch_dsl import Q from flask import current_app from idutils import is_arxiv from inspire_schemas.builders import LiteratureBuilder from inspire_utils.record import get_value, get_values_for_schema from invenio_db import db from invenio_pidstore.errors import PIDDoesNotExistError +from opensearch_dsl import Q from sqlalchemy.exc import SQLAlchemyError from inspirehep.cds.errors import CDSSyncError, MissingCDSServerConfig diff --git a/backend/inspirehep/cds/cli.py b/backend/inspirehep/cds/cli.py index b52bd8ef5e..bb05ca9c79 100644 --- a/backend/inspirehep/cds/cli.py +++ b/backend/inspirehep/cds/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -42,10 +41,10 @@ def sync(since=None): elif not isinstance(since, date): try: since = datetime.strptime(since, "%Y-%m-%d").date() - except ValueError: + except ValueError as e: raise WrongDateFormat( f"`since`: {since} is in wrong format. Should be in ISO format: YYYY-MM-DD." - ) + ) from e try: click.echo("Starting CDS Sync.") sync_identifiers(since) diff --git a/backend/inspirehep/cds/config.py b/backend/inspirehep/cds/config.py index 8a0790cfe3..90ced4cf7d 100644 --- a/backend/inspirehep/cds/config.py +++ b/backend/inspirehep/cds/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # diff --git a/backend/inspirehep/cds/errors.py b/backend/inspirehep/cds/errors.py index 1d57922af9..0054de1ca7 100644 --- a/backend/inspirehep/cds/errors.py +++ b/backend/inspirehep/cds/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # diff --git a/backend/inspirehep/cds/models.py b/backend/inspirehep/cds/models.py index 77d8405f3a..e301af5c96 100644 --- a/backend/inspirehep/cds/models.py +++ b/backend/inspirehep/cds/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/celery.py b/backend/inspirehep/celery.py index 1ae54edd99..4b7fd63717 100644 --- a/backend/inspirehep/celery.py +++ b/backend/inspirehep/celery.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -26,7 +25,7 @@ class CeleryTask(AppContextTask): def on_failure(self, exc, task_id, args, kwargs, einfo): if isinstance( - exc, (InvalidRequestError, OperationalError, Psycopg2OperationalError) + exc, InvalidRequestError | OperationalError | Psycopg2OperationalError ): LOGGER.exception("Shutting down celery process") try: diff --git a/backend/inspirehep/config.py b/backend/inspirehep/config.py index d8879ea41b..bd2b11d3c5 100644 --- a/backend/inspirehep/config.py +++ b/backend/inspirehep/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/curation/__init__.py b/backend/inspirehep/curation/__init__.py new file mode 100644 index 0000000000..e95765a35c --- /dev/null +++ b/backend/inspirehep/curation/__init__.py @@ -0,0 +1,5 @@ +# +# Copyright (C) 2021 CERN. +# +# inspirehep is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. diff --git a/backend/inspirehep/curation/api.py b/backend/inspirehep/curation/api.py index 3164a25275..22a88ad999 100644 --- a/backend/inspirehep/curation/api.py +++ b/backend/inspirehep/curation/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # @@ -9,7 +8,7 @@ from inspire_utils.dedupers import dedupe_list from inspire_utils.record import get_value -from .utils import ( +from inspirehep.curation.utils import ( assign_institution, collaboration_multi_search_query, create_accelerator_experiment_from_collaboration_match, @@ -33,7 +32,7 @@ def normalize_collaborations(collaborations, wf_id): workflow_id=wf_id, ) return - if not len(search_responses) == len(collaborations) * 2: + if len(search_responses) != len(collaborations) * 2: LOGGER.exception( "Results count does not match collaborations count", record_collaboration_number=len(collaborations), @@ -43,7 +42,7 @@ def normalize_collaborations(collaborations, wf_id): return for collaboration, collaboration_response, subgroup_response in zip( - collaborations, search_responses[::2], search_responses[1::2] + collaborations, search_responses[::2], search_responses[1::2], strict=False ): if "record" in collaboration: continue @@ -121,6 +120,6 @@ def assign_institution_reference_to_affiliations( else: complete_affiliation = assign_institution(affiliation) if complete_affiliation: - already_matched_affiliations_refs[ - complete_affiliation["value"] - ] = complete_affiliation["record"] + already_matched_affiliations_refs[complete_affiliation["value"]] = ( + complete_affiliation["record"] + ) diff --git a/backend/inspirehep/curation/cli.py b/backend/inspirehep/curation/cli.py index 01289c0619..e1e9eab474 100644 --- a/backend/inspirehep/curation/cli.py +++ b/backend/inspirehep/curation/cli.py @@ -1,7 +1,7 @@ import click import requests -from opensearch_dsl import Q from flask.cli import with_appcontext +from opensearch_dsl import Q from inspirehep.curation.tasks import update_pdg_keywords_in_records from inspirehep.search.api import LiteratureSearch diff --git a/backend/inspirehep/curation/errors.py b/backend/inspirehep/curation/errors.py index 47e8c936f4..10a81c8e7a 100644 --- a/backend/inspirehep/curation/errors.py +++ b/backend/inspirehep/curation/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # diff --git a/backend/inspirehep/curation/search_check_do/__init__.py b/backend/inspirehep/curation/search_check_do/__init__.py index 11bc1f7c54..c153a65d03 100644 --- a/backend/inspirehep/curation/search_check_do/__init__.py +++ b/backend/inspirehep/curation/search_check_do/__init__.py @@ -3,4 +3,4 @@ # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -from .base import SearchCheckDo # noqa: F401 +from .base import SearchCheckDo # noqa F401 diff --git a/backend/inspirehep/curation/search_check_do/examples.py b/backend/inspirehep/curation/search_check_do/examples.py index 28b0a6b7ff..83fe1063b7 100644 --- a/backend/inspirehep/curation/search_check_do/examples.py +++ b/backend/inspirehep/curation/search_check_do/examples.py @@ -8,14 +8,13 @@ from inspire_utils.record import get_values_for_schema from invenio_db import db from invenio_pidstore.models import PersistentIdentifier +from search_check_do import SearchCheckDo from sqlalchemy.orm import aliased from inspirehep.records.utils import get_ref_from_pid, remove_author_bai_from_id_list from inspirehep.search.api import AuthorsSearch from inspirehep.utils import chunker, flatten_list -from . import SearchCheckDo - class SciPostSetRefereed(SearchCheckDo): """Set all SciPost Phys. papers as refereed.""" @@ -67,7 +66,7 @@ def do(record, logger, state): AuthorsSearch().query_from_iq(f"ids.value:{inspire_id}").execute().hits ) recids = [hit.control_number for hit in hits] - if not len(recids) == 1: + if len(recids) != 1: logger.warning( "No unique match for INSPIRE ID, skipping.", inspire_id=inspire_id, diff --git a/backend/inspirehep/curation/utils.py b/backend/inspirehep/curation/utils.py index e01eacbb3d..18e299126f 100644 --- a/backend/inspirehep/curation/utils.py +++ b/backend/inspirehep/curation/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,7 +7,6 @@ import re import structlog -from opensearch_dsl import MultiSearch, Q, Search from inspire_dojson.utils import get_recid_from_ref from inspire_schemas.utils import normalize_collaboration_name from inspire_utils.record import get_value @@ -16,12 +14,12 @@ from invenio_pidstore.errors import PIDDoesNotExistError from invenio_search import current_search_client from invenio_search.utils import prefix_index +from opensearch_dsl import MultiSearch, Q, Search +from inspirehep.curation.errors import SubGroupNotFound from inspirehep.records.api import JournalsRecord from inspirehep.search.api import InstitutionsSearch, LiteratureSearch -from .errors import SubGroupNotFound - LOGGER = structlog.getLogger() @@ -157,7 +155,9 @@ def find_subgroup(subgroup, experiment): normalize_collaboration_name(clean_special_characters.sub(" ", element.lower())) for element in subgroups ] - for subgroup, normalized_subgroup_from_list in zip(subgroups, normalized_subgroups): + for subgroup, normalized_subgroup_from_list in zip( + subgroups, normalized_subgroups, strict=False + ): if normalized_subgroup_from_list == normalized_subgroup: return subgroup raise SubGroupNotFound(experiment["control_number"], subgroup) @@ -289,7 +289,7 @@ def extract_matched_aff_from_highlight( extracted_raw_aff = re.sub( "|", "", highlighted_raw_affs[longest_highlight_idx] ) - for raw_aff, aff in zip(author_raw_affs, author_affs): + for raw_aff, aff in zip(author_raw_affs, author_affs, strict=False): if raw_aff["value"] == extracted_raw_aff: return [aff] diff --git a/backend/inspirehep/curation/views.py b/backend/inspirehep/curation/views.py index 1fced4eaa5..2cb140a147 100644 --- a/backend/inspirehep/curation/views.py +++ b/backend/inspirehep/curation/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,15 +12,14 @@ from inspirehep.accounts.decorators import login_required_with_roles from inspirehep.accounts.roles import Roles -from inspirehep.records.api import LiteratureRecord -from inspirehep.search.api import JournalsSearch -from inspirehep.serializers import jsonify - -from .api import ( +from inspirehep.curation.api import ( assign_institution_reference_to_affiliations, normalize_affiliations, normalize_collaborations, ) +from inspirehep.records.api import LiteratureRecord +from inspirehep.search.api import JournalsSearch +from inspirehep.serializers import jsonify blueprint = Blueprint("inspirehep_curation", __name__, url_prefix="/curation") parser = FlaskParser() diff --git a/backend/inspirehep/disambiguation/cli.py b/backend/inspirehep/disambiguation/cli.py index 574acdcb11..54be3477ee 100644 --- a/backend/inspirehep/disambiguation/cli.py +++ b/backend/inspirehep/disambiguation/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -9,12 +8,8 @@ import click import structlog -from opensearch_dsl import Q from flask.cli import with_appcontext from flask_celeryext.app import current_celery_app -from invenio_db import db -from sqlalchemy.orm.exc import NoResultFound - from inspirehep.disambiguation.tasks import disambiguate_authors from inspirehep.errors import DB_TASK_EXCEPTIONS from inspirehep.records.api import AuthorsRecord @@ -22,6 +17,9 @@ from inspirehep.records.models import RecordsAuthors from inspirehep.search.api import AuthorsSearch, LiteratureSearch from inspirehep.utils import chunker +from invenio_db import db +from opensearch_dsl import Q +from sqlalchemy.orm.exc import NoResultFound MAX_INDEXER_QUEUE_LEN = 100000 MAX_DISAMBIGUATION_QUEUE_LEN = 10000 diff --git a/backend/inspirehep/disambiguation/loaders.py b/backend/inspirehep/disambiguation/loaders.py index f14c7b5bc7..acf479d076 100644 --- a/backend/inspirehep/disambiguation/loaders.py +++ b/backend/inspirehep/disambiguation/loaders.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/disambiguation/tasks.py b/backend/inspirehep/disambiguation/tasks.py index ea37c14dd9..8532035073 100644 --- a/backend/inspirehep/disambiguation/tasks.py +++ b/backend/inspirehep/disambiguation/tasks.py @@ -5,9 +5,6 @@ from inspire_matcher.api import match from inspire_utils.name import ParsedName from inspire_utils.record import get_value, get_values_for_schema -from invenio_db import db -from sqlalchemy.orm.exc import StaleDataError - from inspirehep.disambiguation.utils import ( create_new_stub_author, reorder_lit_author_names, @@ -20,6 +17,8 @@ ) from inspirehep.records.api import InspireRecord from inspirehep.records.api.authors import AuthorsRecord +from invenio_db import db +from sqlalchemy.orm.exc import StaleDataError LOGGER = structlog.getLogger() @@ -74,7 +73,7 @@ def match_literature_author(author, record): } matched_authors_references = [] - for config, validator in zip(configs, validators): + for config, validator in zip(configs, validators, strict=False): matched_records = match_literature_author_with_config( author_matcher_data, config ) diff --git a/backend/inspirehep/disambiguation/utils.py b/backend/inspirehep/disambiguation/utils.py index 308b7add96..421fc95396 100644 --- a/backend/inspirehep/disambiguation/utils.py +++ b/backend/inspirehep/disambiguation/utils.py @@ -4,11 +4,10 @@ import structlog from flask import url_for from inspire_dojson.utils import get_record_ref +from inspirehep.records.api.authors import AuthorsRecord from prometheus_client import Counter as metrics_counter from unidecode import unidecode -from inspirehep.records.api.authors import AuthorsRecord - LOGGER = structlog.getLogger() disambiguation_changed_signatures = metrics_counter( @@ -139,7 +138,9 @@ def reorder_lit_author_names(lit_author_name, author_name): unidecode(name).lower() for name in lit_author_first_names.split(" ") ) + last_idx = 0 for nb, name in enumerate(lit_author_first_last_name_tokens): + last_idx = nb normalized_name = unidecode(name).lower() name_only_in_author_last_names = ( normalized_name in author_last_names_normalized @@ -160,6 +161,6 @@ def reorder_lit_author_names(lit_author_name, author_name): and name_only_in_author_last_names ): break - lit_author_first_names = " ".join(lit_author_first_last_name_tokens[:nb]) - lit_author_last_names = " ".join(lit_author_first_last_name_tokens[nb:]) + lit_author_first_names = " ".join(lit_author_first_last_name_tokens[:last_idx]) + lit_author_last_names = " ".join(lit_author_first_last_name_tokens[last_idx:]) return f"{lit_author_last_names}, {lit_author_first_names}".strip(" ,") diff --git a/backend/inspirehep/editor/authorlist_utils.py b/backend/inspirehep/editor/authorlist_utils.py index 1ad6fe6ad6..19aba955a8 100644 --- a/backend/inspirehep/editor/authorlist_utils.py +++ b/backend/inspirehep/editor/authorlist_utils.py @@ -78,7 +78,7 @@ def create_author_fullname(author_names): warnings = [] fullname = " ".join(author_names) if len(author_names) == 1: - warnings.append("Author without firstname: %s" % fullname) + warnings.append(f"Author without firstname: {fullname}") return fullname, warnings @@ -97,7 +97,7 @@ def handle_affiliations( added_affiliation = add_author_affiliation( affiliations, word_in_processing, author_affs ) - if not added_affiliation and not word_in_processing == ",": + if not added_affiliation and word_in_processing != ",": for aff_key in split_id(word_in_processing): added_affiliation = add_author_affiliation( affiliations, aff_key, author_affs @@ -183,7 +183,6 @@ def parse_authors(text, affiliations): def add_author_to_authors(fullname, authors, author_affs): - if not fullname: return authors @@ -226,9 +225,8 @@ def determine_aff_type_character(char_list): if aff_type == "alpha": if not char.isalpha(): return None - elif aff_type == "digit": - if not char.isdigit(): - return None + elif aff_type == "digit" and not char.isdigit(): + return None else: if char.isalpha(): aff_type = "alpha" diff --git a/backend/inspirehep/editor/config.py b/backend/inspirehep/editor/config.py index 7904d0069d..2e3aab5765 100644 --- a/backend/inspirehep/editor/config.py +++ b/backend/inspirehep/editor/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/editor/editor_soft_lock.py b/backend/inspirehep/editor/editor_soft_lock.py index 8b5abb47d7..d8e20449a1 100644 --- a/backend/inspirehep/editor/editor_soft_lock.py +++ b/backend/inspirehep/editor/editor_soft_lock.py @@ -9,7 +9,7 @@ LOGGER = structlog.getLogger() -class EditorSoftLock(object): +class EditorSoftLock: def __init__(self, recid, record_version, user_email=None, task_name=None): if not (task_name or user_email): raise ValueError("User mail or task name must be passed") diff --git a/backend/inspirehep/editor/errors.py b/backend/inspirehep/editor/errors.py index 5c14900219..94e7e03abf 100644 --- a/backend/inspirehep/editor/errors.py +++ b/backend/inspirehep/editor/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/editor/views.py b/backend/inspirehep/editor/views.py index dce7da130b..96de49879b 100644 --- a/backend/inspirehep/editor/views.py +++ b/backend/inspirehep/editor/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -30,6 +29,9 @@ assign_institution_reference_to_affiliations, normalize_affiliations, ) +from inspirehep.editor.authorlist_utils import authorlist +from inspirehep.editor.editor_soft_lock import EditorSoftLock +from inspirehep.editor.errors import EditorGetRevisionError, EditorRevertToRevisionError from inspirehep.files.api import current_s3_instance from inspirehep.matcher.api import get_affiliations_from_pdf, match_references from inspirehep.matcher.utils import create_journal_dict, map_refextract_to_schema @@ -40,10 +42,6 @@ from inspirehep.snow.errors import EditTicketException from inspirehep.utils import hash_data -from .authorlist_utils import authorlist -from .editor_soft_lock import EditorSoftLock -from .errors import EditorGetRevisionError, EditorRevertToRevisionError - blueprint = Blueprint("inspirehep_editor", __name__, url_prefix="/editor") LOGGER = structlog.getLogger() @@ -67,8 +65,8 @@ def revert_to_revision(endpoint, pid_value): record.revert(revision_id) db.session.commit() return jsonify(success=True) - except Exception: - raise EditorRevertToRevisionError + except Exception as e: + raise EditorRevertToRevisionError from e @blueprint.route("//", methods=["GET"]) @@ -121,10 +119,7 @@ def get_revisions(endpoint, pid_value): transaction_id = revision.model.transaction_id user = Transaction.query.filter(Transaction.id == transaction_id).one().user - if user: - user_email = user.email - else: - user_email = "system" + user_email = user.email if user else "system" revisions.append( { @@ -136,8 +131,8 @@ def get_revisions(endpoint, pid_value): } ) return jsonify(revisions) - except Exception: - raise EditorGetRevisionError + except Exception as e: + raise EditorGetRevisionError from e @blueprint.route("/revisions//", methods=["GET"]) @@ -156,8 +151,8 @@ def get_revision(transaction_id, rec_uuid): .one() ) return jsonify(revision.json) - except Exception: - raise EditorGetRevisionError + except Exception as e: + raise EditorGetRevisionError from e # TODO: change endpoint name @@ -388,6 +383,7 @@ def normalize_affiliations_for_authors(parsed_authors): for author, normalized_affiliation in zip( parsed_authors.get("authors", []), normalized_affiliations_result["normalized_affiliations"], + strict=False, ): if "affiliations" in author: continue diff --git a/backend/inspirehep/errors.py b/backend/inspirehep/errors.py index 5e8620cd45..1e6b5c2550 100644 --- a/backend/inspirehep/errors.py +++ b/backend/inspirehep/errors.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +from invenio_rest.errors import RESTException from opensearchpy import ( ConflictError, ConnectionError, @@ -13,7 +13,6 @@ RequestError, TransportError, ) -from invenio_rest.errors import RESTException from sqlalchemy.exc import ( DisconnectionError, IntegrityError, diff --git a/backend/inspirehep/factory.py b/backend/inspirehep/factory.py index a324e06ae5..6ea128683c 100644 --- a/backend/inspirehep/factory.py +++ b/backend/inspirehep/factory.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,7 +10,7 @@ from invenio_config import create_config_loader from werkzeug.middleware.dispatcher import DispatcherMiddleware -from . import config +from inspirehep import config env_prefix = "INVENIO" diff --git a/backend/inspirehep/files/api/__init__.py b/backend/inspirehep/files/api/__init__.py index 036d2fc3e1..42f7a9d023 100644 --- a/backend/inspirehep/files/api/__init__.py +++ b/backend/inspirehep/files/api/__init__.py @@ -6,4 +6,4 @@ # the terms of the MIT License; see LICENSE file for more details. # flake8: noqa -from ..proxies import current_s3_instance # noqa: F401 +from ..proxies import current_s3_instance # noqa F401 diff --git a/backend/inspirehep/files/api/s3.py b/backend/inspirehep/files/api/s3.py index d8f4637f03..d064537e5f 100644 --- a/backend/inspirehep/files/api/s3.py +++ b/backend/inspirehep/files/api/s3.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/files/cli.py b/backend/inspirehep/files/cli.py index 3a5b03b201..37b5840876 100644 --- a/backend/inspirehep/files/cli.py +++ b/backend/inspirehep/files/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -9,7 +8,7 @@ from flask import current_app from flask.cli import with_appcontext -from .proxies import current_s3_instance +from inspirehep.files.proxies import current_s3_instance @click.group() @@ -40,7 +39,6 @@ def files(): @files.command(help="Creates S3 buckets for [0-9] and [a-f]") @with_appcontext def create_buckets(): - click.secho("Creating buckets") if not current_app.config["FEATURE_FLAG_ENABLE_FILES"]: diff --git a/backend/inspirehep/files/config.py b/backend/inspirehep/files/config.py index 355b3e3408..a26a7e37df 100644 --- a/backend/inspirehep/files/config.py +++ b/backend/inspirehep/files/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/files/ext.py b/backend/inspirehep/files/ext.py index 9b0d1132c2..980570e79d 100644 --- a/backend/inspirehep/files/ext.py +++ b/backend/inspirehep/files/ext.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,7 +12,7 @@ from inspirehep.files.api.s3 import S3 -class InspireS3(object): +class InspireS3: """Store boto3 connectors inside Flask's application context for easier handling. """ diff --git a/backend/inspirehep/files/proxies.py b/backend/inspirehep/files/proxies.py index 699b67faf7..e0cf4bef8d 100644 --- a/backend/inspirehep/files/proxies.py +++ b/backend/inspirehep/files/proxies.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/fixtures.py b/backend/inspirehep/fixtures.py index b1c6a9a900..ee04be818f 100644 --- a/backend/inspirehep/fixtures.py +++ b/backend/inspirehep/fixtures.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/hal/api.py b/backend/inspirehep/hal/api.py index 7e420ae845..5701c093d7 100644 --- a/backend/inspirehep/hal/api.py +++ b/backend/inspirehep/hal/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/hal/config.py b/backend/inspirehep/hal/config.py index ffd9070eab..aeb448f434 100644 --- a/backend/inspirehep/hal/config.py +++ b/backend/inspirehep/hal/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of INSPIRE. # Copyright (C) 2014-2017 CERN. diff --git a/backend/inspirehep/hal/core/sword.py b/backend/inspirehep/hal/core/sword.py index 2a927c11fd..c392b38251 100644 --- a/backend/inspirehep/hal/core/sword.py +++ b/backend/inspirehep/hal/core/sword.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of INSPIRE. # Copyright (C) 2014-2019 CERN. diff --git a/backend/inspirehep/hal/core/tei.py b/backend/inspirehep/hal/core/tei.py index d3a69f9b2e..70b5cddc90 100644 --- a/backend/inspirehep/hal/core/tei.py +++ b/backend/inspirehep/hal/core/tei.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of INSPIRE. # Copyright (C) 2014-2019 CERN. diff --git a/backend/inspirehep/hal/errors.py b/backend/inspirehep/hal/errors.py index ae9f468c1a..b1bc07dcc3 100644 --- a/backend/inspirehep/hal/errors.py +++ b/backend/inspirehep/hal/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/hal/tasks.py b/backend/inspirehep/hal/tasks.py index 9662df6793..e335dc701d 100644 --- a/backend/inspirehep/hal/tasks.py +++ b/backend/inspirehep/hal/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -110,7 +109,7 @@ def _hal_create(tei, record): hal_id = re.findall("hal-[0-9]{8}", message)[0] receipt = _hal_update(tei, hal_id, record) else: - raise HALCreateException(message) + raise HALCreateException(message) from e return receipt diff --git a/backend/inspirehep/hal/templates/hal/art.xml b/backend/inspirehep/hal/templates/hal/art.xml index e4edc0675f..dafb245cd0 100644 --- a/backend/inspirehep/hal/templates/hal/art.xml +++ b/backend/inspirehep/hal/templates/hal/art.xml @@ -77,4 +77,4 @@ {% endif %} -{% endblock profileDesc %} \ No newline at end of file +{% endblock profileDesc %} diff --git a/backend/inspirehep/hal/templates/hal/base.xml b/backend/inspirehep/hal/templates/hal/base.xml index 7add089a92..9d93da84aa 100644 --- a/backend/inspirehep/hal/templates/hal/base.xml +++ b/backend/inspirehep/hal/templates/hal/base.xml @@ -11,4 +11,4 @@ - \ No newline at end of file + diff --git a/backend/inspirehep/hal/templates/hal/comm.xml b/backend/inspirehep/hal/templates/hal/comm.xml index 0bb7e38ad6..0dc0352936 100644 --- a/backend/inspirehep/hal/templates/hal/comm.xml +++ b/backend/inspirehep/hal/templates/hal/comm.xml @@ -90,4 +90,4 @@ {% endif %} -{% endblock profileDesc %} \ No newline at end of file +{% endblock profileDesc %} diff --git a/backend/inspirehep/hal/templates/hal/preprint.xml b/backend/inspirehep/hal/templates/hal/preprint.xml index a0096ed15f..53a1527d4d 100644 --- a/backend/inspirehep/hal/templates/hal/preprint.xml +++ b/backend/inspirehep/hal/templates/hal/preprint.xml @@ -64,4 +64,4 @@ {% endif %} -{% endblock profileDesc %} \ No newline at end of file +{% endblock profileDesc %} diff --git a/backend/inspirehep/hal/utils.py b/backend/inspirehep/hal/utils.py index e852081615..c1542e623b 100644 --- a/backend/inspirehep/hal/utils.py +++ b/backend/inspirehep/hal/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/hal/views.py b/backend/inspirehep/hal/views.py index c0c8a5557f..3f221f69cc 100644 --- a/backend/inspirehep/hal/views.py +++ b/backend/inspirehep/hal/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/hepdata/cli.py b/backend/inspirehep/hepdata/cli.py index bf96b33b32..96c7188215 100644 --- a/backend/inspirehep/hepdata/cli.py +++ b/backend/inspirehep/hepdata/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -11,9 +10,8 @@ import structlog from flask.cli import with_appcontext from inspire_utils.record import get_values_for_schema -from invenio_db import db - from inspirehep.records.api import LiteratureRecord +from invenio_db import db LOGGER = structlog.getLogger() HEPDATA_URL = "https://www.hepdata.net/search/ids" @@ -37,10 +35,10 @@ def harvest(since): if since: try: since = datetime.strptime(since, "%Y-%m-%d").date() - except ValueError: + except ValueError as e: raise ValueError( f"`since`: {since} is in wrong format. Should be in ISO format: YYYY-MM-DD." - ) + ) from e else: since = (datetime.now() - timedelta(1)).strftime("%Y-%m-%d") payload = {"inspire_ids": True, "last_updated": since, "sort_by": "latest"} diff --git a/backend/inspirehep/indexer/api.py b/backend/inspirehep/indexer/api.py index f3ffd71456..0df958ca63 100644 --- a/backend/inspirehep/indexer/api.py +++ b/backend/inspirehep/indexer/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # diff --git a/backend/inspirehep/indexer/base.py b/backend/inspirehep/indexer/base.py index a13e85e2b0..bd09b442b6 100644 --- a/backend/inspirehep/indexer/base.py +++ b/backend/inspirehep/indexer/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,15 +5,15 @@ # the terms of the MIT License; see LICENSE file for more details. import structlog -from opensearchpy import ConflictError, NotFoundError, RequestError, TransportError -from opensearchpy.helpers import streaming_bulk from flask import current_app from inspire_utils.record import get_value from invenio_indexer.api import RecordIndexer from invenio_indexer.signals import before_record_index from invenio_search import current_search_client as es -from invenio_search.engine import dsl, search +from invenio_search.engine import search from kombu.exceptions import EncodeError +from opensearchpy import ConflictError, NotFoundError, RequestError, TransportError +from opensearchpy.helpers import streaming_bulk from sqlalchemy.orm.exc import NoResultFound LOGGER = structlog.getLogger() @@ -65,6 +64,7 @@ def _process_bulk_record_for_index( """ from inspirehep.records.api import LiteratureRecord + index_from_record = self.record_to_index(record) if not index: index = index_from_record @@ -174,7 +174,9 @@ def _get_indexing_arguments(self, fulltext=False): arguments = { "pipeline": current_app.config["ES_FULLTEXT_PIPELINE_NAME"], # TODO: when opensearch 2.2.1 is released, update package & change arg name to `timeout` - "request_timeout": int(current_app.config["FULLLTEXT_INDEXER_REQUEST_TIMEOUT"]), + "request_timeout": int( + current_app.config["FULLLTEXT_INDEXER_REQUEST_TIMEOUT"] + ), } return arguments diff --git a/backend/inspirehep/indexer/cli.py b/backend/inspirehep/indexer/cli.py index 97c16f6f50..3c8865d5ad 100644 --- a/backend/inspirehep/indexer/cli.py +++ b/backend/inspirehep/indexer/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -219,7 +218,7 @@ def reindex_records( created_tasks = dispatch_indexing_task(items, batch_size, queue_name) all_tasks.extend(created_tasks) - click.secho("Created {} bulk-indexing tasks.".format(len(all_tasks)), fg="green") + click.secho(f"Created {len(all_tasks)} bulk-indexing tasks.", fg="green") with click.progressbar( length=len(all_tasks), label="Indexing records" diff --git a/backend/inspirehep/indexer/tasks.py b/backend/inspirehep/indexer/tasks.py index d9a86a89ac..7fd29a5b42 100644 --- a/backend/inspirehep/indexer/tasks.py +++ b/backend/inspirehep/indexer/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,6 @@ import structlog from celery import shared_task -from flask import current_app from inspirehep.errors import DB_TASK_EXCEPTIONS, ES_TASK_EXCEPTIONS from inspirehep.indexer.api import get_references_to_update diff --git a/backend/inspirehep/legacy/views.py b/backend/inspirehep/legacy/views.py index bd734bd55c..ae403271be 100644 --- a/backend/inspirehep/legacy/views.py +++ b/backend/inspirehep/legacy/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/logger/__init__.py b/backend/inspirehep/logger/__init__.py index 5400595880..fe6b0ca52b 100644 --- a/backend/inspirehep/logger/__init__.py +++ b/backend/inspirehep/logger/__init__.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -from .ext import InspireLogger # noqa: F401 +from inspirehep.logger.ext import InspireLogger # noqa F401 diff --git a/backend/inspirehep/logger/config.py b/backend/inspirehep/logger/config.py index b23da7060b..848ddb01cf 100644 --- a/backend/inspirehep/logger/config.py +++ b/backend/inspirehep/logger/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -72,6 +71,7 @@ root_logger.addHandler(handler) root_logger.setLevel(logging.INFO) + # Celery logging # ============== @task_prerun.connect @@ -98,7 +98,7 @@ def log_error( traceback=None, einfo=None, *signal_args, - **signal_kwargs + **signal_kwargs, ): logger = structlog.get_logger() logger.error( diff --git a/backend/inspirehep/logger/ext.py b/backend/inspirehep/logger/ext.py index 0b8eda4b3a..b0dda24831 100644 --- a/backend/inspirehep/logger/ext.py +++ b/backend/inspirehep/logger/ext.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/__init__.py b/backend/inspirehep/mailing/__init__.py index 8b646e0482..91cf75c36a 100644 --- a/backend/inspirehep/mailing/__init__.py +++ b/backend/inspirehep/mailing/__init__.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -from .ext import InspireMailing # noqa: F401 +from inspirehep.mailing.ext import InspireMailing # noqa F401 diff --git a/backend/inspirehep/mailing/api/__init__.py b/backend/inspirehep/mailing/api/__init__.py new file mode 100644 index 0000000000..ffa3e31145 --- /dev/null +++ b/backend/inspirehep/mailing/api/__init__.py @@ -0,0 +1,5 @@ +# +# Copyright (C) 2019 CERN. +# +# inspirehep is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. diff --git a/backend/inspirehep/mailing/api/authors.py b/backend/inspirehep/mailing/api/authors.py index 0e54145528..3fe97426f8 100644 --- a/backend/inspirehep/mailing/api/authors.py +++ b/backend/inspirehep/mailing/api/authors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/api/conferences.py b/backend/inspirehep/mailing/api/conferences.py index a2ccf89bd5..3acd9f6d54 100644 --- a/backend/inspirehep/mailing/api/conferences.py +++ b/backend/inspirehep/mailing/api/conferences.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/api/jobs.py b/backend/inspirehep/mailing/api/jobs.py index 74d6f49ed3..b5b52a50f5 100644 --- a/backend/inspirehep/mailing/api/jobs.py +++ b/backend/inspirehep/mailing/api/jobs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,16 +5,15 @@ # the terms of the MIT License; see LICENSE file for more details. import structlog -from opensearch_dsl.query import Q from flask import current_app, render_template from inspire_utils.record import get_value from invenio_oauthclient.models import UserIdentity +from opensearch_dsl.query import Q from inspirehep.mailing.providers.flask_mail import send_email +from inspirehep.mailing.providers.mailtrain import mailtrain_subscribe_user_to_list from inspirehep.search.api import JobsSearch -from ..providers.mailtrain import mailtrain_subscribe_user_to_list - LOGGER = structlog.getLogger() diff --git a/backend/inspirehep/mailing/api/seminars.py b/backend/inspirehep/mailing/api/seminars.py index b3a2ce5d30..ffbd3c69b7 100644 --- a/backend/inspirehep/mailing/api/seminars.py +++ b/backend/inspirehep/mailing/api/seminars.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/cli.py b/backend/inspirehep/mailing/cli.py index b58c3c6e25..8441647a2b 100644 --- a/backend/inspirehep/mailing/cli.py +++ b/backend/inspirehep/mailing/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/config.py b/backend/inspirehep/mailing/config.py index 4d05740d37..92b2971d79 100644 --- a/backend/inspirehep/mailing/config.py +++ b/backend/inspirehep/mailing/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/ext.py b/backend/inspirehep/mailing/ext.py index a312050a2f..88497cd002 100644 --- a/backend/inspirehep/mailing/ext.py +++ b/backend/inspirehep/mailing/ext.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,7 @@ from flask_mail import Mail -from .utils import humanize_date_to_natural_time +from inspirehep.mailing.utils import humanize_date_to_natural_time class InspireMailing: @@ -16,9 +15,9 @@ def __init__(self, app=None): self.init_app(app) def init_app(self, app): - app.jinja_env.filters[ - "humanizeDateToNaturalTime" - ] = humanize_date_to_natural_time + app.jinja_env.filters["humanizeDateToNaturalTime"] = ( + humanize_date_to_natural_time + ) app.extensions["inspirehep-mailing"] = self mail = Mail() diff --git a/backend/inspirehep/mailing/loaders.py b/backend/inspirehep/mailing/loaders.py index 427046354e..90b66ead22 100644 --- a/backend/inspirehep/mailing/loaders.py +++ b/backend/inspirehep/mailing/loaders.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/providers/__init__.py b/backend/inspirehep/mailing/providers/__init__.py index 71f4302bdb..ffa3e31145 100644 --- a/backend/inspirehep/mailing/providers/__init__.py +++ b/backend/inspirehep/mailing/providers/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/providers/flask_mail.py b/backend/inspirehep/mailing/providers/flask_mail.py index 0bbd7d9ce6..1a6519a043 100644 --- a/backend/inspirehep/mailing/providers/flask_mail.py +++ b/backend/inspirehep/mailing/providers/flask_mail.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/providers/mailtrain.py b/backend/inspirehep/mailing/providers/mailtrain.py index 6d19bcbd98..ee70ca66cb 100644 --- a/backend/inspirehep/mailing/providers/mailtrain.py +++ b/backend/inspirehep/mailing/providers/mailtrain.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/utils.py b/backend/inspirehep/mailing/utils.py index ec97d1f50f..c6d3f45736 100644 --- a/backend/inspirehep/mailing/utils.py +++ b/backend/inspirehep/mailing/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/mailing/views.py b/backend/inspirehep/mailing/views.py index 4806d2022b..2ccd6da5bb 100644 --- a/backend/inspirehep/mailing/views.py +++ b/backend/inspirehep/mailing/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -14,10 +13,9 @@ from redis import StrictRedis from inspirehep.mailing.api.jobs import subscribe_to_jobs_weekly_list +from inspirehep.mailing.loaders import JobsWeeklySubscribeSchema from inspirehep.serializers import jsonify -from .loaders import JobsWeeklySubscribeSchema - LOGGER = structlog.getLogger() blueprint = Blueprint( diff --git a/backend/inspirehep/matcher/__init__.py b/backend/inspirehep/matcher/__init__.py index d764bdcdea..5724951e13 100644 --- a/backend/inspirehep/matcher/__init__.py +++ b/backend/inspirehep/matcher/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/matcher/api.py b/backend/inspirehep/matcher/api.py index 9795157381..1365888d87 100644 --- a/backend/inspirehep/matcher/api.py +++ b/backend/inspirehep/matcher/api.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +import contextlib import re from copy import copy, deepcopy from urllib.parse import urljoin @@ -16,11 +16,9 @@ from inspire_utils.dedupers import dedupe_list from inspire_utils.record import get_value -from inspirehep.matcher.parsers import GrobidAuthors +from inspirehep.matcher.parsers import GrobidAuthors, GrobidReferenceParser from inspirehep.matcher.serializers import LiteratureSummary -from .parsers import GrobidReferenceParser - def get_reference_from_grobid(query): data = {"citations": query} @@ -49,12 +47,10 @@ def match_reference_with_config(reference, config, previous_matched_recid=None): dict: the matched reference. """ # XXX: avoid this type casting. - try: + with contextlib.suppress(KeyError, ValueError): reference["reference"]["publication_info"]["year"] = str( reference["reference"]["publication_info"]["year"] ) - except (KeyError, ValueError): - pass matched_recids = [ matched_record["_source"]["control_number"] @@ -71,12 +67,10 @@ def match_reference_with_config(reference, config, previous_matched_recid=None): _add_match_to_reference(reference, previous_matched_recid, config["index"]) # XXX: avoid this type casting. - try: + with contextlib.suppress(KeyError): reference["reference"]["publication_info"]["year"] = int( reference["reference"]["publication_info"]["year"] ) - except KeyError: - pass return reference diff --git a/backend/inspirehep/matcher/cli.py b/backend/inspirehep/matcher/cli.py index 1a71c503c2..3f2db9bfd5 100644 --- a/backend/inspirehep/matcher/cli.py +++ b/backend/inspirehep/matcher/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,10 +10,9 @@ from flask.cli import with_appcontext from invenio_pidstore.models import PersistentIdentifier, PIDStatus +from inspirehep.matcher.tasks import match_references_by_uuids from inspirehep.utils import chunker -from .tasks import match_references_by_uuids - LOGGER = structlog.getLogger() diff --git a/backend/inspirehep/matcher/config.py b/backend/inspirehep/matcher/config.py index ba989dc681..acd9171637 100644 --- a/backend/inspirehep/matcher/config.py +++ b/backend/inspirehep/matcher/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -9,7 +8,7 @@ from inspire_matcher.config import MATCHER_DEFAULT_CONFIGURATION as exact_match -from .validators import authors_validator +from inspirehep.matcher.validators import authors_validator GROBID_URL = "https://grobid.inspirebeta.net" diff --git a/backend/inspirehep/matcher/parsers.py b/backend/inspirehep/matcher/parsers.py index ba6b2e7254..23ba73cab8 100644 --- a/backend/inspirehep/matcher/parsers.py +++ b/backend/inspirehep/matcher/parsers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -89,7 +88,7 @@ def year(self): ).get() -class GrobidAuthors(object): +class GrobidAuthors: def __init__(self, xml_text): self._xml = Selector(text=xml_text, type="xml") self._xml.remove_namespaces() @@ -122,7 +121,7 @@ def parse_all(self): return list(self.parse_one()) -class GrobidAuthor(object): +class GrobidAuthor: def __init__(self, author_selector): self._author = author_selector @@ -130,7 +129,7 @@ def __init__(self, author_selector): def _extract(source, path, type=None, text=False): path += "[string-length(normalize-space()) > 0]" if type: - path += "[@type='{type}']".format(type=type) + path += f"[@type='{type}']" if text: path += "/text()" return source.xpath(path) diff --git a/backend/inspirehep/matcher/tasks.py b/backend/inspirehep/matcher/tasks.py index 43237a8021..48e0fe1423 100644 --- a/backend/inspirehep/matcher/tasks.py +++ b/backend/inspirehep/matcher/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -14,10 +13,9 @@ from sqlalchemy.dialects.postgresql import JSONB from inspirehep.errors import DB_TASK_EXCEPTIONS, ES_TASK_EXCEPTIONS +from inspirehep.matcher.api import match_references from inspirehep.records.api import LiteratureRecord -from .api import match_references - LOGGER = structlog.getLogger() RETRY_BACKOFF = 10 @@ -34,10 +32,10 @@ ) def match_references_by_uuids(literature_uuids): record_json = type_coerce(RecordMetadata.json, JSONB) - has_references = record_json.has_key("references") # noqa: W601 + has_references = record_json.has_key("references") # noqa W601 selected_uuids = RecordMetadata.id.in_(literature_uuids) not_deleted = or_( # exclude deleted records incase some are deleted after uuids are fetched by the callee - not_(record_json.has_key("deleted")), # noqa: W601 + not_(record_json.has_key("deleted")), # noqa W601 not_(record_json["deleted"] == cast(True, JSONB)), ) with_references_query = RecordMetadata.query.filter( diff --git a/backend/inspirehep/matcher/utils.py b/backend/inspirehep/matcher/utils.py index ed825c3aee..e6eaec8ac5 100644 --- a/backend/inspirehep/matcher/utils.py +++ b/backend/inspirehep/matcher/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -34,7 +33,10 @@ def map_refextract_to_schema(extracted_references, source=None): ("linemarker", rb.set_label), ("misc", rb.add_misc), ("publisher", rb.set_publisher), - ("raw_ref", lambda raw_ref: rb.add_raw_reference(raw_ref, source=source)), + ( + "raw_ref", + lambda raw_ref, rb=rb: rb.add_raw_reference(raw_ref, source=source), + ), ("reportnumber", rb.add_report_number), ("texkey", rb.set_texkey), ("title", rb.add_title), @@ -92,11 +94,11 @@ def create_journal_dict(): title_dict = {} - for (short_title, journal_title) in titles_query.all(): + for short_title, journal_title in titles_query.all(): title_dict[normalize_title(short_title)] = short_title title_dict[normalize_title(journal_title)] = short_title - for (short_title, title_variants) in title_variants_query.all(): + for short_title, title_variants in title_variants_query.all(): if title_variants is None: continue diff --git a/backend/inspirehep/matcher/validators.py b/backend/inspirehep/matcher/validators.py index 7a22590fa0..65c2201e05 100644 --- a/backend/inspirehep/matcher/validators.py +++ b/backend/inspirehep/matcher/validators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/oai/config.py b/backend/inspirehep/oai/config.py index d9eedc4db6..dbc1a0735c 100644 --- a/backend/inspirehep/oai/config.py +++ b/backend/inspirehep/oai/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/oai/serializers.py b/backend/inspirehep/oai/serializers.py index fac7d53c78..a483ad8d15 100644 --- a/backend/inspirehep/oai/serializers.py +++ b/backend/inspirehep/oai/serializers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/oai/utils.py b/backend/inspirehep/oai/utils.py index 786471c60d..3d83c6cbd1 100644 --- a/backend/inspirehep/oai/utils.py +++ b/backend/inspirehep/oai/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -38,14 +37,14 @@ def has_cern_affiliation(record): affiliations_values = get_value(record, "authors.affiliations.value", default=[]) affiliations = chain.from_iterable(affiliations_values) - if any( - affiliation - for affiliation in affiliations - if affiliation.lower().startswith("cern") - ) and record.get("curated"): - return True - - return False + return bool( + any( + affiliation + for affiliation in affiliations + if affiliation.lower().startswith("cern") + ) + and record.get("curated") + ) def has_cern_collaboration(record): diff --git a/backend/inspirehep/orcid/__init__.py b/backend/inspirehep/orcid/__init__.py index 6dfa3ee0d4..9df1e9ea14 100644 --- a/backend/inspirehep/orcid/__init__.py +++ b/backend/inspirehep/orcid/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of INSPIRE. # Copyright (C) 2017-2018 CERN. @@ -22,7 +21,6 @@ """ORCID integration module.""" - -from .builder import OrcidBuilder # noqa: F401 -from .converter import OrcidConverter # noqa: F401 -from .ext import InspireOrcid # noqa: F401 +from inspirehep.orcid.builder import OrcidBuilder # noqa F401 +from inspirehep.orcid.converter import OrcidConverter # noqa F401 +from inspirehep.orcid.ext import InspireOrcid # noqa F401 diff --git a/backend/inspirehep/orcid/api.py b/backend/inspirehep/orcid/api.py index 7dfb8a135e..3ccde46993 100644 --- a/backend/inspirehep/orcid/api.py +++ b/backend/inspirehep/orcid/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/orcid/builder.py b/backend/inspirehep/orcid/builder.py index 10748ffd3a..62e3ab5a96 100644 --- a/backend/inspirehep/orcid/builder.py +++ b/backend/inspirehep/orcid/builder.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -20,7 +19,7 @@ _ELEMENT_MAKERS = {"work": _WORK, "common": _COMMON} -class OrcidBuilder(object): +class OrcidBuilder: """Class used to build ORCID-compatible work records in JSON.""" def __init__(self): @@ -87,13 +86,13 @@ def add_publication_date(self, partial_date): """ publication_date = _COMMON("publication-date") - publication_date.append(_COMMON.year("{:04d}".format(partial_date.year))) + publication_date.append(_COMMON.year(f"{partial_date.year:04d}")) if partial_date.month: - publication_date.append(_COMMON.month("{:02d}".format(partial_date.month))) + publication_date.append(_COMMON.month(f"{partial_date.month:02d}")) if partial_date.day: - publication_date.append(_COMMON.day("{:02d}".format(partial_date.day))) + publication_date.append(_COMMON.day(f"{partial_date.day:02d}")) self.record.append(publication_date) @@ -151,9 +150,7 @@ def add_doi(self, value, relationship=None): value (string): the identifier itself relationship (string): either "part-of" or "self", optional, see `OrcidBuilder._make_external_id_field` """ - self.add_external_id( - "doi", value, "http://dx.doi.org/{}".format(value), relationship - ) + self.add_external_id("doi", value, f"http://dx.doi.org/{value}", relationship) def add_arxiv(self, value, relationship=None): """Add arXiv identifier to the record. @@ -163,7 +160,7 @@ def add_arxiv(self, value, relationship=None): relationship (string): either "part-of" or "self", optional, see `OrcidBuilder._make_external_id_field` """ self.add_external_id( - "arxiv", value, "http://arxiv.org/abs/{}".format(value), relationship + "arxiv", value, f"http://arxiv.org/abs/{value}", relationship ) def add_citation(self, _type, value): @@ -284,7 +281,7 @@ def _make_contributor_orcid_field(self, reference): """ return _COMMON( "contributor-orcid", - _COMMON.uri("http://orcid.org/{}".format(reference)), + _COMMON.uri(f"http://orcid.org/{reference}"), _COMMON.path(reference), _COMMON.host("orcid.org"), ) @@ -302,7 +299,7 @@ def _get_or_make_field(self, root, field_tag): namespace, relative_tag = tuple(field_tag.split(":")) element_maker = _ELEMENT_MAKERS[namespace] try: - field = root.xpath("/*/{}".format(field_tag), namespaces=_NAMESPACES)[0] + field = root.xpath(f"/*/{field_tag}", namespaces=_NAMESPACES)[0] except IndexError: field = element_maker(relative_tag) root.append(field) diff --git a/backend/inspirehep/orcid/cache.py b/backend/inspirehep/orcid/cache.py index 44bd37e96d..556d03424b 100644 --- a/backend/inspirehep/orcid/cache.py +++ b/backend/inspirehep/orcid/cache.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2016-2018 CERN. @@ -12,12 +11,12 @@ from flask import current_app as app from redis import StrictRedis -from .converter import OrcidConverter +from inspirehep.orcid.converter import OrcidConverter CACHE_PREFIX = None -class OrcidCache(object): +class OrcidCache: def __init__(self, orcid, recid): """ Orcid cached data. @@ -44,8 +43,8 @@ def _key(self): """Return the string '`CACHE_PREFIX`:orcidcache:`orcid_value`:`recid`'""" prefix = "" if CACHE_PREFIX: - prefix = "{}:".format(CACHE_PREFIX) - return "{}orcidcache:{}:{}".format(prefix, self.orcid, self.recid) + prefix = f"{CACHE_PREFIX}:" + return f"{prefix}orcidcache:{self.orcid}:{self.recid}" def write_work_putcode(self, putcode, inspire_record=None): """ @@ -96,7 +95,7 @@ def has_work_content_changed(self, inspire_record): return self._cached_hash_value != self._new_hash_value -class _OrcidHasher(object): +class _OrcidHasher: def __init__(self, inspire_record): self.inspire_record = inspire_record diff --git a/backend/inspirehep/orcid/cli.py b/backend/inspirehep/orcid/cli.py index d987444d6f..5c94bab5f6 100644 --- a/backend/inspirehep/orcid/cli.py +++ b/backend/inspirehep/orcid/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/orcid/config.py b/backend/inspirehep/orcid/config.py index ff029d5645..4b6b85c107 100644 --- a/backend/inspirehep/orcid/config.py +++ b/backend/inspirehep/orcid/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/orcid/converter.py b/backend/inspirehep/orcid/converter.py index 06839ded88..e1b9ed568e 100644 --- a/backend/inspirehep/orcid/converter.py +++ b/backend/inspirehep/orcid/converter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,6 @@ """Handle conversion from INSPIRE records to ORCID.""" - import logging from collections import namedtuple @@ -17,18 +15,17 @@ from inspire_utils.record import get_value from inspire_utils.urls import record_url_by_pattern +from inspirehep.orcid.builder import OrcidBuilder from inspirehep.records.api import ConferencesRecord from inspirehep.records.serializers.bibtex import literature_bibtex -from .builder import OrcidBuilder - LOGGER = logging.getLogger(__name__) ExternalIdentifier = namedtuple("ExternalIdentifier", ("type", "value")) -class OrcidConverter(object): +class OrcidConverter: """Coverter for the Orcid format.""" # Maps INSPIRE author roles to ORCID contributor roles @@ -69,7 +66,6 @@ def __init__(self, record, url_pattern, put_code=None, visibility=None): self._bibtex_citation = None self._external_identifiers = [] - def get_xml(self, do_add_bibtex_citation=False): """Create an ORCID XML representation of the record. diff --git a/backend/inspirehep/orcid/domain_models.py b/backend/inspirehep/orcid/domain_models.py index 8b4a98161e..ef6db53b1c 100644 --- a/backend/inspirehep/orcid/domain_models.py +++ b/backend/inspirehep/orcid/domain_models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2016-2018 CERN. @@ -15,19 +14,18 @@ from invenio_db import db from invenio_pidstore.errors import PIDDoesNotExistError +from inspirehep.orcid import exceptions, push_access_tokens, utils +from inspirehep.orcid.cache import OrcidCache +from inspirehep.orcid.converter import OrcidConverter +from inspirehep.orcid.putcode_getter import OrcidPutcodeGetter from inspirehep.records.api import LiteratureRecord from inspirehep.utils import distributed_lock -from . import exceptions, push_access_tokens, utils -from .cache import OrcidCache -from .converter import OrcidConverter -from .putcode_getter import OrcidPutcodeGetter - LOGGER = structlog.getLogger() ORCID_REGEX = r"\d{4}-\d{4}-\d{4}-\d{3}[0-9X]" -class OrcidPusher(object): +class OrcidPusher: def __init__( self, orcid, @@ -43,7 +41,7 @@ def __init__( self.record_db_version = record_db_version self.inspire_record = self._get_inspire_record() self.cache = OrcidCache(orcid, recid) - self.lock_name = "orcid:{}".format(self.orcid) + self.lock_name = f"orcid:{self.orcid}" self.client = OrcidClient(self.oauth_token, self.orcid) self.converter = None self.cached_author_putcodes = {} @@ -55,8 +53,8 @@ def _get_inspire_record(self): ) except PIDDoesNotExistError as exc: raise exceptions.RecordNotFoundException( - "recid={} not found for pid_type=lit".format(self.recid), from_exc=exc - ) + f"recid={self.recid} not found for pid_type=lit", from_exc=exc + ) from exc # If the record_db_version was given, then ensure we are about to push # the right record version. @@ -71,10 +69,8 @@ def _get_inspire_record(self): and inspire_record.model.version_id < self.record_db_version ): raise exceptions.StaleRecordDBVersionException( - "Requested push for db version={}, but actual record db" - " version={}".format( - self.record_db_version, inspire_record.model.version_id - ) + f"Requested push for db version={self.record_db_version}, but actual record db" + f" version={inspire_record.model.version_id}" ) return inspire_record @@ -102,7 +98,7 @@ def _is_record_deleted(self): return True return self.inspire_record.get("deleted", False) - def push(self): # noqa: C901 + def push(self): # noqa C901 putcode = None if not self._do_force_cache_miss: putcode = self.cache.read_work_putcode() @@ -128,7 +124,7 @@ def push(self): # noqa: C901 try: putcode = self._post_or_put_work(putcode) - except orcid_client_exceptions.WorkAlreadyExistsException: + except orcid_client_exceptions.WorkAlreadyExistsException as e: # We POSTed the record as new work, but it failed because # a work with the same identifier is already in ORCID. # This can mean two things: @@ -139,19 +135,19 @@ def push(self): # noqa: C901 # so we try to push once again works with clashing identifiers # to update them and resolve the potential conflict. if self.pushing_duplicated_identifier: - raise exceptions.DuplicatedExternalIdentifierPusherException + raise exceptions.DuplicatedExternalIdentifierPusherException from e putcode = self._cache_all_author_putcodes() if not putcode: try: self._push_work_with_clashing_identifier() putcode = self._post_or_put_work(putcode) - except orcid_client_exceptions.WorkAlreadyExistsException: + except orcid_client_exceptions.WorkAlreadyExistsException as e: # The PUT/POST failed despite pushing works with clashing identifiers # and we can't do anything about this. - raise exceptions.DuplicatedExternalIdentifierPusherException + raise exceptions.DuplicatedExternalIdentifierPusherException from e else: self._post_or_put_work(putcode) - except orcid_client_exceptions.DuplicatedExternalIdentifierException: + except orcid_client_exceptions.DuplicatedExternalIdentifierException as e: # We PUT a record changing its identifier, but there is another work # in ORCID with the same identifier. We need to find out the recid # of the clashing work in ORCID and push a fresh version of that @@ -160,7 +156,7 @@ def push(self): # noqa: C901 if not self.pushing_duplicated_identifier: self._push_work_with_clashing_identifier() # Raised exception will cause retry of celery task - raise exceptions.DuplicatedExternalIdentifierPusherException + raise exceptions.DuplicatedExternalIdentifierPusherException from e except orcid_client_exceptions.PutcodeNotFoundPutException: # We try to push the work with invalid putcode, so we delete # its putcode and push it without any putcode. @@ -179,19 +175,19 @@ def push(self): # noqa: C901 orcid_client_exceptions.TokenInvalidException, orcid_client_exceptions.TokenMismatchException, orcid_client_exceptions.TokenWithWrongPermissionException, - ): + ) as e: LOGGER.info( "Deleting Orcid push access", token=self.oauth_token, orcid=self.orcid ) push_access_tokens.delete_access_token(self.oauth_token, self.orcid) db.session.commit() - raise exceptions.TokenInvalidDeletedException + raise exceptions.TokenInvalidDeletedException from e except orcid_client_exceptions.MovedPermanentlyException as exc: old_orcid, new_orcid = re.findall(ORCID_REGEX, exc.args[0]) utils.update_moved_orcid(old_orcid, new_orcid) raise except orcid_client_exceptions.BaseOrcidClientJsonException as exc: - raise exceptions.InputDataInvalidException(from_exc=exc) + raise exceptions.InputDataInvalidException(from_exc=exc) from exc self.cache.write_work_putcode(putcode, self.inspire_record) return putcode @@ -247,10 +243,8 @@ def _cache_all_author_putcodes(self): # at this moment it helps isolate a potential issue. if putcode and not self.cache.read_work_putcode(): raise exceptions.PutcodeNotFoundInCacheAfterCachingAllPutcodes( - "No putcode={} found in cache for recid={} after having" - " cached all author putcodes for orcid={}".format( - self.putcode, self.recid, self.orcid - ) + f"No putcode={self.putcode} found in cache for recid={self.recid} after having" + f" cached all author putcodes for orcid={self.orcid}" ) return putcode @@ -271,7 +265,7 @@ def _delete_work(self, putcode=None): # Such putcode does not exists (anymore?) in orcid. pass except orcid_client_exceptions.BaseOrcidClientJsonException as exc: - raise exceptions.InputDataInvalidException(from_exc=exc) + raise exceptions.InputDataInvalidException(from_exc=exc) from exc self.cache.delete_work_putcode() @@ -286,8 +280,7 @@ def _push_work_with_clashing_identifier(self): putcodes_recids ) - for (recid, putcode) in updated_putcodes_recid.items(): - + for recid, putcode in updated_putcodes_recid.items(): if not putcode or not recid: continue if recid == self.recid: diff --git a/backend/inspirehep/orcid/exceptions.py b/backend/inspirehep/orcid/exceptions.py index 43aa126c55..244382ab3a 100644 --- a/backend/inspirehep/orcid/exceptions.py +++ b/backend/inspirehep/orcid/exceptions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2016-2018 CERN. @@ -12,15 +11,13 @@ def __init__(self, *args, **kwargs): # kwargs['from_exc'] used as a sort of exception chaining in Python 2. # No need in Python 3 with the statement: raise exc from cause self.from_exc = kwargs.get("from_exc") - super(BaseOrcidPusherException, self).__init__(*args) + super().__init__(*args) def __str__(self, *args, **kwargs): - output = super(BaseOrcidPusherException, self).__str__(*args, **kwargs) + output = super().__str__(*args, **kwargs) if not self.from_exc: return output - output += "\nThis exception was directly caused by the following exception:\n{}".format( - repr(self.from_exc) - ) + output += f"\nThis exception was directly caused by the following exception:\n{repr(self.from_exc)}" return output diff --git a/backend/inspirehep/orcid/ext.py b/backend/inspirehep/orcid/ext.py index 6280893a59..ef0222ec81 100644 --- a/backend/inspirehep/orcid/ext.py +++ b/backend/inspirehep/orcid/ext.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,11 +6,10 @@ """Search extension.""" - import inspire_service_orcid.conf -class InspireOrcid(object): +class InspireOrcid: def __init__(self, app=None): if app: self.init_app(app) diff --git a/backend/inspirehep/orcid/push_access_tokens.py b/backend/inspirehep/orcid/push_access_tokens.py index 8b3703d40a..672287b80c 100644 --- a/backend/inspirehep/orcid/push_access_tokens.py +++ b/backend/inspirehep/orcid/push_access_tokens.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2016-2018 CERN. @@ -66,7 +65,7 @@ def is_access_token_invalid(token_plain): return cache.does_invalid_token_exist() -class _OrcidInvalidTokensCache(object): +class _OrcidInvalidTokensCache: def __init__(self, token_plain): self.token_plain = token_plain @@ -83,8 +82,8 @@ def redis(self): def _key(self): prefix = "" if CACHE_PREFIX: - prefix = "{}:".format(CACHE_PREFIX) - return "{}orcidinvalidtoken:{}".format(prefix, self.token_plain) + prefix = f"{CACHE_PREFIX}:" + return f"{prefix}orcidinvalidtoken:{self.token_plain}" def write_invalid_token(self, orcid): data = {"orcid": orcid} diff --git a/backend/inspirehep/orcid/putcode_getter.py b/backend/inspirehep/orcid/putcode_getter.py index 862807d40e..3ab41689d0 100644 --- a/backend/inspirehep/orcid/putcode_getter.py +++ b/backend/inspirehep/orcid/putcode_getter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2016-2018 CERN. @@ -16,11 +15,10 @@ from inspire_service_orcid.client import OrcidClient from invenio_db import db +from inspirehep.orcid import exceptions, push_access_tokens from inspirehep.orcid.converter import ExternalIdentifier from inspirehep.pidstore.api.base import PidStoreBase -from . import exceptions, push_access_tokens - INSPIRE_WORK_URL_REGEX = re.compile( r"https?://(?:labs\.)?inspirehep\.net/(?:record|literature)/(\d+)", re.IGNORECASE ) @@ -29,7 +27,7 @@ LOGGER = structlog.getLogger() -class OrcidPutcodeGetter(object): +class OrcidPutcodeGetter: def __init__(self, orcid, oauth_token): self.orcid = orcid self.oauth_token = oauth_token @@ -77,7 +75,7 @@ def _get_all_works_summary(self): orcid_client_exceptions.TokenInvalidException, orcid_client_exceptions.TokenMismatchException, orcid_client_exceptions.TokenWithWrongPermissionException, - ): + ) as e: LOGGER.info( "OrcidPutcodeGetter: deleting Orcid push access", token=self.oauth_token, @@ -85,9 +83,9 @@ def _get_all_works_summary(self): ) push_access_tokens.delete_access_token(self.oauth_token, self.orcid) db.session.commit() - raise exceptions.TokenInvalidDeletedException + raise exceptions.TokenInvalidDeletedException from e except orcid_client_exceptions.BaseOrcidClientJsonException as exc: - raise exceptions.InputDataInvalidException(from_exc=exc) + raise exceptions.InputDataInvalidException(from_exc=exc) from exc return response def _get_putcodes_and_recids_iter(self, putcodes): @@ -115,7 +113,7 @@ def _get_urls_for_putcodes_iter(self, putcodes): try: response.raise_for_result() except orcid_client_exceptions.BaseOrcidClientJsonException as exc: - raise exceptions.InputDataInvalidException(from_exc=exc) + raise exceptions.InputDataInvalidException(from_exc=exc) from exc chained = itertools.chain(chained, response.get_putcodes_and_urls_iter()) return chained diff --git a/backend/inspirehep/orcid/tasks.py b/backend/inspirehep/orcid/tasks.py index c982c82fce..d7b4f897ca 100644 --- a/backend/inspirehep/orcid/tasks.py +++ b/backend/inspirehep/orcid/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,6 +5,8 @@ # the terms of the MIT License; see LICENSE file for more details. """Manage ORCID OAUTH token migration from INSPIRE legacy instance.""" + +import contextlib import re import structlog @@ -22,11 +23,10 @@ from simplejson import loads from sqlalchemy.orm.exc import FlushError +from inspirehep.orcid import domain_models, exceptions from inspirehep.orcid import exceptions as domain_exceptions from inspirehep.orcid.utils import get_literature_recids_for_orcid -from . import domain_models, exceptions - LOGGER = structlog.getLogger() USER_EMAIL_EMPTY_PATTERN = "{}@FAKEEMAILINSPIRE.FAKE" ORCID_REGEX = r"\d{4}-\d{4}-\d{4}-\d{3}[0-9X]" @@ -66,13 +66,11 @@ def _link_user_and_token(user, name, orcid, access_token): Returns: str: the ORCID associated with the token newly created/enabled. """ - try: + # Note: AlreadyLinkedError becomes FlushError when testing with isolated_app. + # If excpection: User already has their ORCID linked + with contextlib.suppress(AlreadyLinkedError, FlushError): # Link user and ORCID oauth_link_external_id(user, {"id": orcid, "method": "orcid"}) - # Note: AlreadyLinkedError becomes FlushError when testing with isolated_app. - except (AlreadyLinkedError, FlushError): - # User already has their ORCID linked - pass # Search for existing tokens. # Note: there can be only 1 RemoteToken per given RemoteAccount. @@ -143,7 +141,7 @@ def __init__(self, user, orcids): "A RemoteToken already exists for User={} and it is" " associated to a different ORCID: {}" ).format(user, " != ".join(orcids)) - super(RemoteTokenOrcidMismatch, self).__init__(msg) + super().__init__(msg) def _register_user(name, email, orcid, token): @@ -234,11 +232,9 @@ def orcid_push(self, orcid, rec_id, oauth_token, kwargs_to_pusher=None): # Enrich exception message. if isinstance(exc, RequestException): message = (exc.args[0:1] or ("",))[0] - try: - message += "\nResponse={}".format(exc.response.content) - except AttributeError: - pass - message += "\nRequest={} {}".format(exc.request.method, exc.request.url) + with contextlib.suppress(AttributeError): + message += f"\nResponse={exc.response.content}" + message += f"\nRequest={exc.request.method} {exc.request.url}" exc.args = (message,) + exc.args[1:] # If max_retries=3, then self.request.retries is: [0, 1, 2, 3] @@ -250,7 +246,7 @@ def orcid_push(self, orcid, rec_id, oauth_token, kwargs_to_pusher=None): recid=rec_id, orcid=orcid, ) - raise self.retry(max_retries=3, countdown=backoff, exc=exc) + raise self.retry(max_retries=3, countdown=backoff, exc=exc) from exc except ( exceptions.DuplicatedExternalIdentifierPusherException, domain_exceptions.RecordNotFoundException, @@ -259,7 +255,7 @@ def orcid_push(self, orcid, rec_id, oauth_token, kwargs_to_pusher=None): # If max_retries=4, then self.request.retries is: [0, 1, 2, 3, 4] # thus backoff power 5 is (secs): [5, 25, 125, 625] backoff = 5 ** (self.request.retries + 1) - raise self.retry(max_retries=4, countdown=backoff, exc=exc) + raise self.retry(max_retries=4, countdown=backoff, exc=exc) from exc except Exception: LOGGER.warning("Orcid_push task failed", recid=rec_id, orcid=orcid) raise diff --git a/backend/inspirehep/orcid/utils.py b/backend/inspirehep/orcid/utils.py index fe19ca7821..eac317e492 100644 --- a/backend/inspirehep/orcid/utils.py +++ b/backend/inspirehep/orcid/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2016-2018 CERN. @@ -7,11 +6,9 @@ # under the terms of the MIT License; see LICENSE file for more details. """ORCID utils.""" - from itertools import chain import structlog -from opensearch_dsl import Q from flask import current_app from inspire_dojson.utils import get_recid_from_ref from inspire_utils.record import get_value, get_values_for_schema @@ -21,6 +18,7 @@ from invenio_oauthclient.utils import oauth_link_external_id from invenio_pidstore.models import PersistentIdentifier from invenio_records.models import RecordMetadata +from opensearch_dsl import Q from sqlalchemy import cast, type_coerce from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm.exc import NoResultFound @@ -45,7 +43,7 @@ def _split_lists(sequence, chunk_size): List[List[Any]]: the split list """ return [ - sequence[i : i + chunk_size] # noqa:E203 + sequence[i : i + chunk_size] # noqaE203 for i in range(0, len(sequence), chunk_size) ] diff --git a/backend/inspirehep/pidstore/api/authors.py b/backend/inspirehep/pidstore/api/authors.py index a4139eabc9..ab0d427465 100644 --- a/backend/inspirehep/pidstore/api/authors.py +++ b/backend/inspirehep/pidstore/api/authors.py @@ -1,16 +1,13 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +from inspirehep.pidstore.api.base import PidStoreBase from inspirehep.pidstore.minters.bai import BAIMinter - -from ..minters.control_number import AuthorsMinter -from ..minters.orcid import OrcidMinter -from .base import PidStoreBase +from inspirehep.pidstore.minters.control_number import AuthorsMinter +from inspirehep.pidstore.minters.orcid import OrcidMinter class PidStoreAuthors(PidStoreBase): - minters = [AuthorsMinter, OrcidMinter, BAIMinter] diff --git a/backend/inspirehep/pidstore/api/base.py b/backend/inspirehep/pidstore/api/base.py index c596b42921..96f5a2a45c 100644 --- a/backend/inspirehep/pidstore/api/base.py +++ b/backend/inspirehep/pidstore/api/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -14,7 +13,7 @@ LOGGER = structlog.getLogger() -class PidStoreBase(object): +class PidStoreBase: minters = [] def __init__(self, object_uuid, data): diff --git a/backend/inspirehep/pidstore/api/conferences.py b/backend/inspirehep/pidstore/api/conferences.py index af9dab08b0..791cf7ecac 100644 --- a/backend/inspirehep/pidstore/api/conferences.py +++ b/backend/inspirehep/pidstore/api/conferences.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -12,5 +11,4 @@ class PidStoreConferences(PidStoreBase): - minters = [ConferencesMinter, CNUMMinter] diff --git a/backend/inspirehep/pidstore/api/data.py b/backend/inspirehep/pidstore/api/data.py index 5594dcf29f..e51ae77424 100644 --- a/backend/inspirehep/pidstore/api/data.py +++ b/backend/inspirehep/pidstore/api/data.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,10 +5,9 @@ # the terms of the MIT License; see LICENSE file for more details. -from ..minters.control_number import DataMinter -from .base import PidStoreBase +from inspirehep.pidstore.api.base import PidStoreBase +from inspirehep.pidstore.minters.control_number import DataMinter class PidStoreData(PidStoreBase): - minters = [DataMinter] diff --git a/backend/inspirehep/pidstore/api/experiments.py b/backend/inspirehep/pidstore/api/experiments.py index 9ae7cc566a..d7a9435a27 100644 --- a/backend/inspirehep/pidstore/api/experiments.py +++ b/backend/inspirehep/pidstore/api/experiments.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,10 +5,9 @@ # the terms of the MIT License; see LICENSE file for more details. -from ..minters.control_number import ExperimentsMinter -from .base import PidStoreBase +from inspirehep.pidstore.api.base import PidStoreBase +from inspirehep.pidstore.minters.control_number import ExperimentsMinter class PidStoreExperiments(PidStoreBase): - minters = [ExperimentsMinter] diff --git a/backend/inspirehep/pidstore/api/institutions.py b/backend/inspirehep/pidstore/api/institutions.py index f7e9138687..ea251c9511 100644 --- a/backend/inspirehep/pidstore/api/institutions.py +++ b/backend/inspirehep/pidstore/api/institutions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,10 +5,9 @@ # the terms of the MIT License; see LICENSE file for more details. -from ..minters.control_number import InstitutionsMinter -from .base import PidStoreBase +from inspirehep.pidstore.api.base import PidStoreBase +from inspirehep.pidstore.minters.control_number import InstitutionsMinter class PidStoreInstitutions(PidStoreBase): - minters = [InstitutionsMinter] diff --git a/backend/inspirehep/pidstore/api/jobs.py b/backend/inspirehep/pidstore/api/jobs.py index 106581a184..f795c2aa57 100644 --- a/backend/inspirehep/pidstore/api/jobs.py +++ b/backend/inspirehep/pidstore/api/jobs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,10 +5,9 @@ # the terms of the MIT License; see LICENSE file for more details. -from ..minters.control_number import JobsMinter -from .base import PidStoreBase +from inspirehep.pidstore.api.base import PidStoreBase +from inspirehep.pidstore.minters.control_number import JobsMinter class PidStoreJobs(PidStoreBase): - minters = [JobsMinter] diff --git a/backend/inspirehep/pidstore/api/journals.py b/backend/inspirehep/pidstore/api/journals.py index b05c7899e1..b5469cbb4b 100644 --- a/backend/inspirehep/pidstore/api/journals.py +++ b/backend/inspirehep/pidstore/api/journals.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,10 +5,9 @@ # the terms of the MIT License; see LICENSE file for more details. -from ..minters.control_number import JournalsMinter -from .base import PidStoreBase +from inspirehep.pidstore.api.base import PidStoreBase +from inspirehep.pidstore.minters.control_number import JournalsMinter class PidStoreJournals(PidStoreBase): - minters = [JournalsMinter] diff --git a/backend/inspirehep/pidstore/api/literature.py b/backend/inspirehep/pidstore/api/literature.py index cabae74b3e..a30fb99082 100644 --- a/backend/inspirehep/pidstore/api/literature.py +++ b/backend/inspirehep/pidstore/api/literature.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,5 +12,4 @@ class PidStoreLiterature(PidStoreBase): - minters = [LiteratureMinter, ArxivMinter, DoiMinter, TexKeyMinter] diff --git a/backend/inspirehep/pidstore/api/seminars.py b/backend/inspirehep/pidstore/api/seminars.py index 20c39b93e6..86e4006277 100644 --- a/backend/inspirehep/pidstore/api/seminars.py +++ b/backend/inspirehep/pidstore/api/seminars.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,10 +5,9 @@ # the terms of the MIT License; see LICENSE file for more details. -from ..minters.control_number import SeminarsMinter -from .base import PidStoreBase +from inspirehep.pidstore.api.base import PidStoreBase +from inspirehep.pidstore.minters.control_number import SeminarsMinter class PidStoreSeminars(PidStoreBase): - minters = [SeminarsMinter] diff --git a/backend/inspirehep/pidstore/cli.py b/backend/inspirehep/pidstore/cli.py index fcc8cf559d..7b16eaaee2 100644 --- a/backend/inspirehep/pidstore/cli.py +++ b/backend/inspirehep/pidstore/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -183,10 +182,9 @@ def fast_mint_bais(yes_i_know): author_uuid=author_uuid, control_number=author_record.get("control_number"), ) - if not yes_i_know: - if not click.confirm( - f"There are {length} Author records without BAI for which BAI will be created. Do you want to continue?" - ): - db.session.rollback() - return + if not yes_i_know and not click.confirm( + f"There are {length} Author records without BAI for which BAI will be created. Do you want to continue?" + ): + db.session.rollback() + return db.session.commit() diff --git a/backend/inspirehep/pidstore/converters.py b/backend/inspirehep/pidstore/converters.py index e0e079451d..bafe742d2a 100644 --- a/backend/inspirehep/pidstore/converters.py +++ b/backend/inspirehep/pidstore/converters.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -12,7 +11,7 @@ from invenio_records_rest.utils import PIDConverter, obj_or_import_string from werkzeug.routing import PathConverter -from .resolvers import InspireResolver +from inspirehep.pidstore.resolvers import InspireResolver LOGGER = structlog.getLogger() diff --git a/backend/inspirehep/pidstore/errors.py b/backend/inspirehep/pidstore/errors.py index 21781c9963..76a9c38c47 100644 --- a/backend/inspirehep/pidstore/errors.py +++ b/backend/inspirehep/pidstore/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/pidstore/minters/arxiv.py b/backend/inspirehep/pidstore/minters/arxiv.py index 218a011fd0..62d3c17d1b 100644 --- a/backend/inspirehep/pidstore/minters/arxiv.py +++ b/backend/inspirehep/pidstore/minters/arxiv.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,7 +5,7 @@ # the terms of the MIT License; see LICENSE file for more details. -from .base import Minter +from inspirehep.pidstore.minters.base import Minter class ArxivMinter(Minter): diff --git a/backend/inspirehep/pidstore/minters/bai.py b/backend/inspirehep/pidstore/minters/bai.py index ad8c921049..3bbd2bb115 100644 --- a/backend/inspirehep/pidstore/minters/bai.py +++ b/backend/inspirehep/pidstore/minters/bai.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -46,10 +45,13 @@ def get_pid_values(self): def create(self, pid_value=None, **kwargs): bai = super().create(pid_value, data=self.data, **kwargs) - if current_app.config.get("FEATURE_FLAG_ENABLE_BAI_PROVIDER", False): - if bai and pid_value is None: - self.data.setdefault("ids", []) - self.data["ids"].append( - {"schema": "INSPIRE BAI", "value": bai.pid.pid_value} - ) + if ( + current_app.config.get("FEATURE_FLAG_ENABLE_BAI_PROVIDER", False) + and bai + and pid_value is None + ): + self.data.setdefault("ids", []) + self.data["ids"].append( + {"schema": "INSPIRE BAI", "value": bai.pid.pid_value} + ) return bai diff --git a/backend/inspirehep/pidstore/minters/base.py b/backend/inspirehep/pidstore/minters/base.py index eeff66f88a..68fab46a51 100644 --- a/backend/inspirehep/pidstore/minters/base.py +++ b/backend/inspirehep/pidstore/minters/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,17 +10,14 @@ from invenio_pidstore.errors import PIDAlreadyExists, PIDDoesNotExistError from invenio_pidstore.models import PersistentIdentifier, PIDStatus -from inspirehep.pidstore.errors import MissingSchema +from inspirehep.pidstore.errors import MissingSchema, PIDAlreadyExistsError from inspirehep.pidstore.providers.external import InspireExternalIdProvider from inspirehep.pidstore.providers.recid import InspireRecordIdProvider -from ..errors import PIDAlreadyExistsError - LOGGER = structlog.getLogger() class Minter: - provider = InspireExternalIdProvider object_type = "rec" pid_type = None @@ -37,7 +33,7 @@ def validate(self): def get_pid_values(self): pid_values = get_value(self.data, self.pid_value_path, default=[]) - if not isinstance(pid_values, (tuple, list)): + if not isinstance(pid_values, tuple | list): pid_values = force_list(pid_values) return set(pid_values) @@ -64,7 +60,7 @@ def create(self, pid_value, **kwargs): pid_value=pid_value, object_type=self.object_type, object_uuid=self.object_uuid, - **kwargs + **kwargs, ) except PIDAlreadyExists as e: raise PIDAlreadyExistsError(e.pid_type, e.pid_value) from e @@ -120,7 +116,6 @@ def get_all_pidstore_pids(self): class ControlNumberMinter(Minter): - pid_value_path = "control_number" provider = InspireRecordIdProvider diff --git a/backend/inspirehep/pidstore/minters/cnum.py b/backend/inspirehep/pidstore/minters/cnum.py index 57d839fddd..89afa53ad4 100644 --- a/backend/inspirehep/pidstore/minters/cnum.py +++ b/backend/inspirehep/pidstore/minters/cnum.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -12,7 +11,6 @@ class CNUMMinter(Minter): - pid_value_path = "cnum" pid_type = "cnum" provider = InspireCNUMProvider @@ -76,7 +74,7 @@ def update(cls, object_uuid, data): if cnum_pid: if cnum_pid.pid_value != data["cnum"]: - raise CNUMChanged(f"CNUM identifier changed from the prev version") + raise CNUMChanged("CNUM identifier changed from the prev version") if cnum_pid.status == PIDStatus.DELETED: cnum_pid.status = PIDStatus.REGISTERED # un-delete pid diff --git a/backend/inspirehep/pidstore/minters/control_number.py b/backend/inspirehep/pidstore/minters/control_number.py index 2fb9f4c8aa..3a88a192bd 100644 --- a/backend/inspirehep/pidstore/minters/control_number.py +++ b/backend/inspirehep/pidstore/minters/control_number.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,7 +5,7 @@ # the terms of the MIT License; see LICENSE file for more details. -from .base import ControlNumberMinter +from inspirehep.pidstore.minters.base import ControlNumberMinter class LiteratureMinter(ControlNumberMinter): diff --git a/backend/inspirehep/pidstore/minters/doi.py b/backend/inspirehep/pidstore/minters/doi.py index 071c7bb096..183fa4e95b 100644 --- a/backend/inspirehep/pidstore/minters/doi.py +++ b/backend/inspirehep/pidstore/minters/doi.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,7 +5,7 @@ # the terms of the MIT License; see LICENSE file for more details. -from .base import Minter +from inspirehep.pidstore.minters.base import Minter class DoiMinter(Minter): diff --git a/backend/inspirehep/pidstore/minters/orcid.py b/backend/inspirehep/pidstore/minters/orcid.py index 1e242bc454..688e13a213 100644 --- a/backend/inspirehep/pidstore/minters/orcid.py +++ b/backend/inspirehep/pidstore/minters/orcid.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,7 @@ from inspire_utils.record import get_values_for_schema -from .base import Minter +from inspirehep.pidstore.minters.base import Minter class OrcidMinter(Minter): diff --git a/backend/inspirehep/pidstore/minters/texkey.py b/backend/inspirehep/pidstore/minters/texkey.py index 11d1948869..19e37501b1 100644 --- a/backend/inspirehep/pidstore/minters/texkey.py +++ b/backend/inspirehep/pidstore/minters/texkey.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/pidstore/models.py b/backend/inspirehep/pidstore/models.py index a4bceb4dbc..da2e90142a 100644 --- a/backend/inspirehep/pidstore/models.py +++ b/backend/inspirehep/pidstore/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -94,7 +93,7 @@ def _create_new_redirection(cls, old_pid, new_pid): old_pid.status = PIDStatus.REDIRECTED db.session.add(old_pid) except IntegrityError as e: - raise PIDInvalidAction(e) + raise PIDInvalidAction(e) from e except SQLAlchemyError: LOGGER.exception( "Failed to redirect record", old_pid=old_pid, new_pid=new_pid diff --git a/backend/inspirehep/pidstore/providers/bai.py b/backend/inspirehep/pidstore/providers/bai.py index e5f2d24adb..90d11a8b64 100644 --- a/backend/inspirehep/pidstore/providers/bai.py +++ b/backend/inspirehep/pidstore/providers/bai.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/pidstore/providers/base.py b/backend/inspirehep/pidstore/providers/base.py index 8631375fec..99a4004d1b 100644 --- a/backend/inspirehep/pidstore/providers/base.py +++ b/backend/inspirehep/pidstore/providers/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/pidstore/providers/cnum.py b/backend/inspirehep/pidstore/providers/cnum.py index ae63f9b8f1..ca978ae059 100644 --- a/backend/inspirehep/pidstore/providers/cnum.py +++ b/backend/inspirehep/pidstore/providers/cnum.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -73,10 +72,7 @@ def next(cls, data): cnum_value = cnum_entry.pid_value.split(".") if len(cnum_value) > 1: all_postfixes.append(int(cnum_value[-1])) - if all_postfixes: - postfix = sorted(all_postfixes)[-1] + 1 - else: - postfix = 1 + postfix = sorted(all_postfixes)[-1] + 1 if all_postfixes else 1 full_cnum = f"{cnum}.{postfix}" diff --git a/backend/inspirehep/pidstore/providers/external.py b/backend/inspirehep/pidstore/providers/external.py index ebc85aca98..d1664b831e 100644 --- a/backend/inspirehep/pidstore/providers/external.py +++ b/backend/inspirehep/pidstore/providers/external.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -14,7 +13,6 @@ class InspireExternalIdProvider(InspireBaseProvider): - pid_provider = "external" default_status = PIDStatus.REGISTERED diff --git a/backend/inspirehep/pidstore/providers/recid.py b/backend/inspirehep/pidstore/providers/recid.py index 84b9dfce00..59ec8930d0 100644 --- a/backend/inspirehep/pidstore/providers/recid.py +++ b/backend/inspirehep/pidstore/providers/recid.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/pidstore/providers/texkey.py b/backend/inspirehep/pidstore/providers/texkey.py index 0c307876d4..d716369ce6 100644 --- a/backend/inspirehep/pidstore/providers/texkey.py +++ b/backend/inspirehep/pidstore/providers/texkey.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -44,10 +43,7 @@ def sanitize(cls, name): @classmethod def build_texkey_first_part(cls, data): full_name = get_value(data, "authors[0].full_name") - if full_name: - last_name = full_name.split(",")[0] - else: - last_name = None + last_name = full_name.split(",")[0] if full_name else None if last_name and len(data["authors"]) < 10: return cls.sanitize(last_name) @@ -66,7 +62,6 @@ def build_texkey_second_part(cls, data): date = get_literature_earliest_date(data) if date: return date.split("-", 1)[0] - None @classmethod def get_texkey_with_random_part(cls, texkey): @@ -177,5 +172,4 @@ def query_pid(cls, pid_value, object_uuid=None, object_type="rec"): @classmethod def get(cls, pid_value, object_uuid=None): - return cls(cls.query_pid(pid_value, object_uuid).one()) diff --git a/backend/inspirehep/pidstore/resolvers.py b/backend/inspirehep/pidstore/resolvers.py index 9aed7f6375..69c483b938 100644 --- a/backend/inspirehep/pidstore/resolvers.py +++ b/backend/inspirehep/pidstore/resolvers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -40,9 +39,8 @@ def resolve(self, pid_value): obj = None raise PIDDeletedError(pid, obj) - if pid.is_redirected(): - if request.method != "PUT": - raise PIDRedirectedError(pid, InspireRedirect.get_redirect(pid)) + if pid.is_redirected() and request.method != "PUT": + raise PIDRedirectedError(pid, InspireRedirect.get_redirect(pid)) obj_id = pid.get_assigned_object(object_type=self.object_type) if not obj_id: diff --git a/backend/inspirehep/records/__init__.py b/backend/inspirehep/records/__init__.py index 11d29c15b5..58ca71f077 100644 --- a/backend/inspirehep/records/__init__.py +++ b/backend/inspirehep/records/__init__.py @@ -7,5 +7,4 @@ # flake8: noqa """Inspire""" - from .ext import InspireRecords diff --git a/backend/inspirehep/records/api/__init__.py b/backend/inspirehep/records/api/__init__.py index 0d26069043..0a7ff108ef 100644 --- a/backend/inspirehep/records/api/__init__.py +++ b/backend/inspirehep/records/api/__init__.py @@ -8,14 +8,13 @@ """INSPIRE module that adds more fun to the platform.""" - -from .authors import AuthorsRecord # noqa: F401 -from .base import InspireRecord # noqa: F401 -from .conferences import ConferencesRecord # noqa: F401 -from .data import DataRecord # noqa: F401 -from .experiments import ExperimentsRecord # noqa: F401 -from .institutions import InstitutionsRecord # noqa: F401 -from .jobs import JobsRecord # noqa: F401 -from .journals import JournalsRecord # noqa: F401 -from .literature import LiteratureRecord # noqa: F401 -from .seminars import SeminarsRecord # noqa: F401 +from .authors import AuthorsRecord # noqa F401 +from .base import InspireRecord # noqa F401 +from .conferences import ConferencesRecord # noqa F401 +from .data import DataRecord # noqa F401 +from .experiments import ExperimentsRecord # noqa F401 +from .institutions import InstitutionsRecord # noqa F401 +from .jobs import JobsRecord # noqa F401 +from .journals import JournalsRecord # noqa F401 +from .literature import LiteratureRecord # noqa F401 +from .seminars import SeminarsRecord # noqa F401 diff --git a/backend/inspirehep/records/api/authors.py b/backend/inspirehep/records/api/authors.py index 587662dd22..99b5304933 100644 --- a/backend/inspirehep/records/api/authors.py +++ b/backend/inspirehep/records/api/authors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,17 +12,16 @@ from sqlalchemy import cast, type_coerce from sqlalchemy.dialects.postgresql import JSONB +from inspirehep.pidstore.api import PidStoreAuthors from inspirehep.pidstore.api.base import PidStoreBase +from inspirehep.records.api.base import InspireRecord from inspirehep.records.api.mixins import StudentsAdvisorMixin from inspirehep.records.marshmallow.authors import AuthorsElasticSearchSchema from inspirehep.records.models import RecordCitations, RecordsAuthors +from inspirehep.records.utils import get_author_by_recid from inspirehep.search.api import AuthorsSearch from inspirehep.utils import chunker -from ...pidstore.api import PidStoreAuthors -from ..utils import get_author_by_recid -from .base import InspireRecord - LOGGER = structlog.getLogger() @@ -81,7 +79,7 @@ def create(cls, data, id_=None, *args, **kwargs): return record def assign_author_to_papers(self): - from .literature import LiteratureRecord + from inspirehep.records.api.literature import LiteratureRecord author_papers_ids = [ str(record_control_number) diff --git a/backend/inspirehep/records/api/base.py b/backend/inspirehep/records/api/base.py index fdd4871356..41c57d5dd3 100644 --- a/backend/inspirehep/records/api/base.py +++ b/backend/inspirehep/records/api/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -12,7 +11,6 @@ from itertools import chain import structlog -from opensearchpy import NotFoundError from flask import current_app from inspire_dojson.utils import strip_empty_values from inspire_schemas.utils import get_validation_errors @@ -25,6 +23,7 @@ from invenio_records.models import RecordMetadata from invenio_records.signals import after_record_revert, before_record_revert from jsonschema import ValidationError +from opensearchpy import NotFoundError from sqlalchemy import tuple_ from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm.attributes import flag_modified @@ -117,8 +116,8 @@ def get_record_by_pid_value( with_deleted = original_record or with_deleted try: return cls.get_record(record_uuid, with_deleted=with_deleted) - except NoResultFound: - raise PIDDoesNotExistError(pid_type, pid_value) + except NoResultFound as e: + raise PIDDoesNotExistError(pid_type, pid_value) from e @classmethod def get_subclasses(cls): @@ -136,13 +135,13 @@ def _get_record_version(cls, record_uuid, record_version): record = RecordMetadataVersion.query.filter_by( id=record_uuid, version_id=record_version ).one() - except NoResultFound: + except NoResultFound as e: LOGGER.warning( "Reading stale data", uuid=str(record_uuid), version=record_version, ) - raise StaleDataError() + raise StaleDataError() from e record_class = InspireRecord.get_class_for_record(record.json) if record_class != cls: record = record_class(record.json, model=record) @@ -244,7 +243,7 @@ def get_class_for_record(cls, data): def create(cls, data, id_=None, *args, **kwargs): record_class = cls.get_class_for_record(data) if record_class != cls: - return record_class.create(data, id_=id_, *args, **kwargs) + return record_class.create(data, *args, id_=id_, **kwargs) data = cls.strip_empty_values(data) deleted = data.get("deleted", False) diff --git a/backend/inspirehep/records/api/conferences.py b/backend/inspirehep/records/api/conferences.py index f380ef6aba..13755d6bc3 100644 --- a/backend/inspirehep/records/api/conferences.py +++ b/backend/inspirehep/records/api/conferences.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/api/data.py b/backend/inspirehep/records/api/data.py index bcdbb0e8f3..84289f208a 100644 --- a/backend/inspirehep/records/api/data.py +++ b/backend/inspirehep/records/api/data.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +from inspirehep.pidstore.api import PidStoreData +from inspirehep.records.api.base import InspireRecord from inspirehep.records.api.mixins import CitationMixin from inspirehep.records.marshmallow.data import DataElasticSearchSchema -from ...pidstore.api import PidStoreData -from .base import InspireRecord - class DataRecord(CitationMixin, InspireRecord): """Data Record.""" diff --git a/backend/inspirehep/records/api/experiments.py b/backend/inspirehep/records/api/experiments.py index 39b46e3240..096629133c 100644 --- a/backend/inspirehep/records/api/experiments.py +++ b/backend/inspirehep/records/api/experiments.py @@ -1,14 +1,12 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +from inspirehep.pidstore.api import PidStoreExperiments +from inspirehep.records.api.base import InspireRecord from inspirehep.records.marshmallow.experiments import ExperimentsElasticSearchSchema - -from ...pidstore.api import PidStoreExperiments -from ..models import ExperimentLiterature -from .base import InspireRecord +from inspirehep.records.models import ExperimentLiterature class ExperimentsRecord(InspireRecord): diff --git a/backend/inspirehep/records/api/institutions.py b/backend/inspirehep/records/api/institutions.py index 617edf3f6a..60377b9b50 100644 --- a/backend/inspirehep/records/api/institutions.py +++ b/backend/inspirehep/records/api/institutions.py @@ -1,14 +1,12 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +from inspirehep.pidstore.api import PidStoreInstitutions +from inspirehep.records.api.base import InspireRecord from inspirehep.records.marshmallow.institutions import InstitutionsElasticSearchSchema - -from ...pidstore.api import PidStoreInstitutions -from ..models import InstitutionLiterature -from .base import InspireRecord +from inspirehep.records.models import InstitutionLiterature class InstitutionsRecord(InspireRecord): diff --git a/backend/inspirehep/records/api/jobs.py b/backend/inspirehep/records/api/jobs.py index 2b35954087..9302e7446f 100644 --- a/backend/inspirehep/records/api/jobs.py +++ b/backend/inspirehep/records/api/jobs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/api/journals.py b/backend/inspirehep/records/api/journals.py index 3e7b889e92..3aa479809a 100644 --- a/backend/inspirehep/records/api/journals.py +++ b/backend/inspirehep/records/api/journals.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +from inspirehep.pidstore.api import PidStoreJournals +from inspirehep.records.api.base import InspireRecord from inspirehep.records.marshmallow.journals import JournalsElasticSearchSchema from inspirehep.records.models import JournalLiterature -from ...pidstore.api import PidStoreJournals -from .base import InspireRecord - class JournalsRecord(InspireRecord): """Journals Record.""" diff --git a/backend/inspirehep/records/api/literature.py b/backend/inspirehep/records/api/literature.py index 3e369cae5b..568c28725c 100644 --- a/backend/inspirehep/records/api/literature.py +++ b/backend/inspirehep/records/api/literature.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -32,6 +31,7 @@ from inspirehep.hal.api import push_to_hal from inspirehep.orcid.api import push_to_orcid from inspirehep.pidstore.api import PidStoreLiterature +from inspirehep.records.api.base import InspireRecord from inspirehep.records.api.mixins import ( CitationMixin, ConferencePaperAndProceedingsMixin, @@ -61,8 +61,6 @@ from inspirehep.search.api import LiteratureSearch from inspirehep.utils import chunker, hash_data -from .base import InspireRecord - LOGGER = structlog.getLogger() PLACEHOLDER = "" diff --git a/backend/inspirehep/records/api/mixins.py b/backend/inspirehep/records/api/mixins.py index 549a0b7a05..1262bedc82 100644 --- a/backend/inspirehep/records/api/mixins.py +++ b/backend/inspirehep/records/api/mixins.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -117,7 +116,7 @@ def _citation_query(self, exclude_self_citations=False): Returns: query: Query containing all citations for this record """ - query = RecordCitations.query.filter(RecordCitations.cited_id==self.id) + query = RecordCitations.query.filter(RecordCitations.cited_id == self.id) if exclude_self_citations and current_app.config.get( "FEATURE_FLAG_ENABLE_SELF_CITATIONS" ): @@ -370,7 +369,9 @@ def get_all_connected_records_uuids_of_modified_collaborations(self): if diff: citers = { citer[0] - for citer in RecordCitations.query.filter(RecordCitations.cited_id==self.id) + for citer in RecordCitations.query.filter( + RecordCitations.cited_id == self.id + ) .with_entities(RecordCitations.citer_id) .all() } @@ -462,7 +463,7 @@ def get_newest_linked_conferences_uuid(self): allowed_types = set( [option.value for option in list(ConferenceToLiteratureRelationshipType)] ) - type_changed = True if doc_type_diff.intersection(allowed_types) else False + type_changed = bool(doc_type_diff.intersection(allowed_types)) pids_previous = set( self._previous_version.get_linked_pids_from_field( diff --git a/backend/inspirehep/records/api/seminars.py b/backend/inspirehep/records/api/seminars.py index ba96f14f74..f52592fad3 100644 --- a/backend/inspirehep/records/api/seminars.py +++ b/backend/inspirehep/records/api/seminars.py @@ -1,12 +1,11 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -from ...pidstore.api import PidStoreSeminars -from ..marshmallow.seminars.es import SeminarsElasticSearchSchema -from .base import InspireRecord +from inspirehep.pidstore.api import PidStoreSeminars +from inspirehep.records.api.base import InspireRecord +from inspirehep.records.marshmallow.seminars.es import SeminarsElasticSearchSchema class SeminarsRecord(InspireRecord): diff --git a/backend/inspirehep/records/cli.py b/backend/inspirehep/records/cli.py index 6e354561a6..09052e6ca6 100644 --- a/backend/inspirehep/records/cli.py +++ b/backend/inspirehep/records/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -229,7 +228,7 @@ def close_expired_jobs(notify): only_jobs_collection = record_json["_collections"].contains(["Jobs"]) only_not_closed = not_(record_json["status"].astext == "closed") only_not_deleted = or_( - not_(record_json.has_key("deleted")), # noqa: W601 + not_(record_json.has_key("deleted")), # noqa W601 not_(record_json["deleted"] == cast(True, JSONB)), ) expired_jobs = RecordMetadata.query.filter( @@ -346,7 +345,7 @@ def remove_bai_from_literature_records( @click.option("--pid-type", required=True) @with_appcontext def update_relationships_for_records(pid_type): - if pid_type not in current_app.config["PID_TYPES_TO_ENDPOINTS"].keys(): + if pid_type not in current_app.config["PID_TYPES_TO_ENDPOINTS"]: click.echo("PID type not found") return records_ids_query = ( diff --git a/backend/inspirehep/records/config.py b/backend/inspirehep/records/config.py index dee45d6acb..ebdc58be1d 100644 --- a/backend/inspirehep/records/config.py +++ b/backend/inspirehep/records/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/errors.py b/backend/inspirehep/records/errors.py index 45033e49e1..7e61af4e7f 100644 --- a/backend/inspirehep/records/errors.py +++ b/backend/inspirehep/records/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/ext.py b/backend/inspirehep/records/ext.py index b0cba4ff6f..74bd36ae00 100644 --- a/backend/inspirehep/records/ext.py +++ b/backend/inspirehep/records/ext.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,7 +7,7 @@ """Flask extension for inspirehep.""" -class InspireRecords(object): +class InspireRecords: """inspirehep extension.""" def __init__(self, app=None): @@ -19,4 +18,4 @@ def __init__(self, app=None): def init_app(self, app): """Flask application initialization.""" app.extensions["inspirehep-records"] = self - from inspirehep.records import receivers # noqa: F401 + from inspirehep.records import receivers # noqa F401 diff --git a/backend/inspirehep/records/links.py b/backend/inspirehep/records/links.py index 8b29fe2cc3..3556f4fc93 100644 --- a/backend/inspirehep/records/links.py +++ b/backend/inspirehep/records/links.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,13 +12,13 @@ def inspire_detail_links_factory(pid, record=None, record_hit=None, *args, **kwargs): links = {} endpoint = find_record_endpoint(pid, record_hit) - endpoint_item = "invenio_records_rest.{0}_item".format(endpoint) + endpoint_item = f"invenio_records_rest.{endpoint}_item" self = url_for(endpoint_item, pid_value=pid.pid_value, _external=True) serializers_endpoints = current_app.config[endpoint.upper()].get( "record_serializers_aliases", {} ) - for serializer_endpoint in serializers_endpoints.keys(): + for serializer_endpoint in serializers_endpoints: links[serializer_endpoint] = f"{self}?format={serializer_endpoint}" additional_endpoints = current_app.config.get("ADDITIONAL_LINKS", {}).get( diff --git a/backend/inspirehep/records/marshmallow/authors/base.py b/backend/inspirehep/records/marshmallow/authors/base.py index 9b32f81962..b5267a0a97 100644 --- a/backend/inspirehep/records/marshmallow/authors/base.py +++ b/backend/inspirehep/records/marshmallow/authors/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -10,11 +9,10 @@ from invenio_records.api import RecordMetadata from marshmallow import Schema, fields +from inspirehep.records.marshmallow.base import RecordBaseSchema +from inspirehep.records.marshmallow.fields import NonHiddenRaw from inspirehep.records.models import StudentsAdvisors -from ..base import RecordBaseSchema -from ..fields import NonHiddenRaw - class AuthorsRawSchema(RecordBaseSchema): positions = NonHiddenRaw(dump_only=True) diff --git a/backend/inspirehep/records/marshmallow/authors/common/advisor.py b/backend/inspirehep/records/marshmallow/authors/common/advisor.py index 744960a304..0eb3f263b8 100644 --- a/backend/inspirehep/records/marshmallow/authors/common/advisor.py +++ b/backend/inspirehep/records/marshmallow/authors/common/advisor.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/authors/common/position.py b/backend/inspirehep/records/marshmallow/authors/common/position.py index 82dec8e0b2..29c3de6e83 100644 --- a/backend/inspirehep/records/marshmallow/authors/common/position.py +++ b/backend/inspirehep/records/marshmallow/authors/common/position.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -20,13 +19,13 @@ def get_display_date(self, data): start_date = data.get("start_date") end_date = data.get("end_date") - suffixed_start_date = "{}-".format(start_date) if start_date else "" + suffixed_start_date = f"{start_date}-" if start_date else "" if current: - return "{}present".format(suffixed_start_date) + return f"{suffixed_start_date}present" if end_date: - return "{}{}".format(suffixed_start_date, end_date) + return f"{suffixed_start_date}{end_date}" if start_date: return start_date diff --git a/backend/inspirehep/records/marshmallow/authors/es.py b/backend/inspirehep/records/marshmallow/authors/es.py index 92c16d9d97..401737e129 100644 --- a/backend/inspirehep/records/marshmallow/authors/es.py +++ b/backend/inspirehep/records/marshmallow/authors/es.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -10,12 +9,11 @@ from inspire_utils.helpers import force_list from marshmallow import fields -from ..base import ElasticSearchBaseSchema -from .base import AuthorsRawSchema +from inspirehep.records.marshmallow.authors.base import AuthorsRawSchema +from inspirehep.records.marshmallow.base import ElasticSearchBaseSchema class AuthorsElasticSearchSchema(ElasticSearchBaseSchema, AuthorsRawSchema): - author_suggest = fields.Method("get_author_suggest", dump_only=True) students = fields.Method("populate_students_field", dump_only=True) diff --git a/backend/inspirehep/records/marshmallow/authors/ui.py b/backend/inspirehep/records/marshmallow/authors/ui.py index 261e717a30..4359e094f8 100644 --- a/backend/inspirehep/records/marshmallow/authors/ui.py +++ b/backend/inspirehep/records/marshmallow/authors/ui.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,16 +7,15 @@ from marshmallow import fields from inspirehep.accounts.api import can_user_edit_author_record - -from ..fields import NonHiddenNested -from ..utils import ( +from inspirehep.records.marshmallow.authors.base import AuthorsPublicSchema +from inspirehep.records.marshmallow.authors.common import PositionSchemaV1 +from inspirehep.records.marshmallow.authors.common.advisor import AdvisorSchemaV1 +from inspirehep.records.marshmallow.fields import NonHiddenNested +from inspirehep.records.marshmallow.utils import ( get_acquisition_source_without_email, get_facet_author_name_for_author, get_first_value_for_schema, ) -from .base import AuthorsPublicSchema -from .common import PositionSchemaV1 -from .common.advisor import AdvisorSchemaV1 class AuthorsBaseSchema(AuthorsPublicSchema): @@ -61,7 +59,6 @@ def get_twitter(data): @staticmethod def get_all_urls_field(data): - urls = data.get("urls", []) external_author_profiles = [ { diff --git a/backend/inspirehep/records/marshmallow/base.py b/backend/inspirehep/records/marshmallow/base.py index 9e14bb7328..29a92cc0de 100644 --- a/backend/inspirehep/records/marshmallow/base.py +++ b/backend/inspirehep/records/marshmallow/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/common/accelerator_experiment.py b/backend/inspirehep/records/marshmallow/common/accelerator_experiment.py index 053fdcabd5..54b8b50669 100644 --- a/backend/inspirehep/records/marshmallow/common/accelerator_experiment.py +++ b/backend/inspirehep/records/marshmallow/common/accelerator_experiment.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,7 +12,6 @@ class AcceleratorExperimentSchemaV1(Schema): - name = fields.Method("get_name") record = fields.Raw(attribute="self", dump_only=True) @@ -40,9 +38,8 @@ def get_resolved_record_or_experiment(self, experiment_records_map, experiment): record_ref = experiment.get("record") experiment_record_id = get_recid_from_ref(record_ref) experiment_record = experiment_records_map.get(experiment_record_id) - if experiment_record: - if "legacy_name" not in experiment_record: - experiment_record["legacy_name"] = experiment.get("legacy_name") + if experiment_record and "legacy_name" not in experiment_record: + experiment_record["legacy_name"] = experiment.get("legacy_name") return experiment_record or experiment @@ -51,5 +48,5 @@ def get_name(self, item): accelerator = get_value(item, "accelerator.value") experiment = get_value(item, "experiment.value") if institution and accelerator and experiment: - return "{}-{}-{}".format(institution, accelerator, experiment) + return f"{institution}-{accelerator}-{experiment}" return item.get("legacy_name") diff --git a/backend/inspirehep/records/marshmallow/common/mixins.py b/backend/inspirehep/records/marshmallow/common/mixins.py index bae62788dd..03e9c06b80 100644 --- a/backend/inspirehep/records/marshmallow/common/mixins.py +++ b/backend/inspirehep/records/marshmallow/common/mixins.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/conferences/base.py b/backend/inspirehep/records/marshmallow/conferences/base.py index 229b6a703f..caf80c15f4 100644 --- a/backend/inspirehep/records/marshmallow/conferences/base.py +++ b/backend/inspirehep/records/marshmallow/conferences/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/conferences/common/proceeding_info_item.py b/backend/inspirehep/records/marshmallow/conferences/common/proceeding_info_item.py index 66388dafb6..86b499d676 100644 --- a/backend/inspirehep/records/marshmallow/conferences/common/proceeding_info_item.py +++ b/backend/inspirehep/records/marshmallow/conferences/common/proceeding_info_item.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/conferences/es.py b/backend/inspirehep/records/marshmallow/conferences/es.py index 80f0c0cf0a..bf34a0361e 100644 --- a/backend/inspirehep/records/marshmallow/conferences/es.py +++ b/backend/inspirehep/records/marshmallow/conferences/es.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/conferences/ui.py b/backend/inspirehep/records/marshmallow/conferences/ui.py index 324610c4bc..cd10c26ab2 100644 --- a/backend/inspirehep/records/marshmallow/conferences/ui.py +++ b/backend/inspirehep/records/marshmallow/conferences/ui.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/data/base.py b/backend/inspirehep/records/marshmallow/data/base.py index ee7131fc63..cc447c0eb1 100644 --- a/backend/inspirehep/records/marshmallow/data/base.py +++ b/backend/inspirehep/records/marshmallow/data/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/data/es.py b/backend/inspirehep/records/marshmallow/data/es.py index cc0d52f9d3..e6ba2e36c6 100644 --- a/backend/inspirehep/records/marshmallow/data/es.py +++ b/backend/inspirehep/records/marshmallow/data/es.py @@ -1,12 +1,11 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -from ..base import ElasticSearchBaseSchema -from .base import DataRawSchema +from inspirehep.records.marshmallow.base import ElasticSearchBaseSchema +from inspirehep.records.marshmallow.data.base import DataRawSchema class DataElasticSearchSchema(ElasticSearchBaseSchema, DataRawSchema): diff --git a/backend/inspirehep/records/marshmallow/experiments/base.py b/backend/inspirehep/records/marshmallow/experiments/base.py index f9016b4560..491893326c 100644 --- a/backend/inspirehep/records/marshmallow/experiments/base.py +++ b/backend/inspirehep/records/marshmallow/experiments/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/experiments/common/related_records.py b/backend/inspirehep/records/marshmallow/experiments/common/related_records.py index 5da3699a12..00adaaaf55 100644 --- a/backend/inspirehep/records/marshmallow/experiments/common/related_records.py +++ b/backend/inspirehep/records/marshmallow/experiments/common/related_records.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/experiments/es.py b/backend/inspirehep/records/marshmallow/experiments/es.py index 60c1543ae6..b4297f1767 100644 --- a/backend/inspirehep/records/marshmallow/experiments/es.py +++ b/backend/inspirehep/records/marshmallow/experiments/es.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -12,8 +11,8 @@ from inspire_utils.helpers import force_list from marshmallow import fields -from ..base import ElasticSearchBaseSchema -from .base import ExperimentsRawSchema +from inspirehep.records.marshmallow.base import ElasticSearchBaseSchema +from inspirehep.records.marshmallow.experiments.base import ExperimentsRawSchema class ExperimentsElasticSearchSchema(ElasticSearchBaseSchema, ExperimentsRawSchema): @@ -43,7 +42,6 @@ def normalize_subgroups(self, original_object): return self.build_normalized_names(original_object, paths) def get_facet_inspire_classification(self, data): - classifications = data.get("inspire_classification", []) cleaned_classifications = [ re.sub(" [Ee]xperiments", "", classification) diff --git a/backend/inspirehep/records/marshmallow/experiments/ui.py b/backend/inspirehep/records/marshmallow/experiments/ui.py index f26a91301e..1b345b9c28 100644 --- a/backend/inspirehep/records/marshmallow/experiments/ui.py +++ b/backend/inspirehep/records/marshmallow/experiments/ui.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/fields/list_with_limit.py b/backend/inspirehep/records/marshmallow/fields/list_with_limit.py index ee0332ac06..a712a95ffd 100644 --- a/backend/inspirehep/records/marshmallow/fields/list_with_limit.py +++ b/backend/inspirehep/records/marshmallow/fields/list_with_limit.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -14,5 +13,5 @@ def _serialize(self, value, attr, obj): limit = self.metadata.get("limit") value = [item for item in value if item] if limit: - return super(ListWithLimit, self)._serialize(value[:limit], attr, obj) - return super(ListWithLimit, self)._serialize(value, attr, obj) + return super()._serialize(value[:limit], attr, obj) + return super()._serialize(value, attr, obj) diff --git a/backend/inspirehep/records/marshmallow/fields/nested_without_empty_objects.py b/backend/inspirehep/records/marshmallow/fields/nested_without_empty_objects.py index 93df18bd27..cc17ccedee 100644 --- a/backend/inspirehep/records/marshmallow/fields/nested_without_empty_objects.py +++ b/backend/inspirehep/records/marshmallow/fields/nested_without_empty_objects.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -10,7 +9,7 @@ class NestedField(fields.Nested): def _serialize(self, nested_obj, attr, obj): - result = super(NestedField, self)._serialize(nested_obj, attr, obj) + result = super()._serialize(nested_obj, attr, obj) if result is None: return self.default if self.default is not missing else None return result diff --git a/backend/inspirehep/records/marshmallow/fields/non_hidden.py b/backend/inspirehep/records/marshmallow/fields/non_hidden.py index 8a15a3fd66..d0aedd3b08 100644 --- a/backend/inspirehep/records/marshmallow/fields/non_hidden.py +++ b/backend/inspirehep/records/marshmallow/fields/non_hidden.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/institutions/base.py b/backend/inspirehep/records/marshmallow/institutions/base.py index cee664945a..6319bd9756 100644 --- a/backend/inspirehep/records/marshmallow/institutions/base.py +++ b/backend/inspirehep/records/marshmallow/institutions/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/institutions/common/__init__.py b/backend/inspirehep/records/marshmallow/institutions/common/__init__.py index 71f4302bdb..ffa3e31145 100644 --- a/backend/inspirehep/records/marshmallow/institutions/common/__init__.py +++ b/backend/inspirehep/records/marshmallow/institutions/common/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/institutions/common/related_records.py b/backend/inspirehep/records/marshmallow/institutions/common/related_records.py index 09aa4ef421..8c64dbfa75 100644 --- a/backend/inspirehep/records/marshmallow/institutions/common/related_records.py +++ b/backend/inspirehep/records/marshmallow/institutions/common/related_records.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/institutions/es.py b/backend/inspirehep/records/marshmallow/institutions/es.py index ab750963da..7c153c410e 100644 --- a/backend/inspirehep/records/marshmallow/institutions/es.py +++ b/backend/inspirehep/records/marshmallow/institutions/es.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -10,8 +9,8 @@ from inspire_utils.helpers import force_list from marshmallow import fields -from ..base import ElasticSearchBaseSchema -from .base import InstitutionsRawSchema +from inspirehep.records.marshmallow.base import ElasticSearchBaseSchema +from inspirehep.records.marshmallow.institutions.base import InstitutionsRawSchema class InstitutionsElasticSearchSchema(ElasticSearchBaseSchema, InstitutionsRawSchema): diff --git a/backend/inspirehep/records/marshmallow/institutions/ui.py b/backend/inspirehep/records/marshmallow/institutions/ui.py index 4ae4069e2d..414c778439 100644 --- a/backend/inspirehep/records/marshmallow/institutions/ui.py +++ b/backend/inspirehep/records/marshmallow/institutions/ui.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/jobs/base.py b/backend/inspirehep/records/marshmallow/jobs/base.py index fca2dbe798..05a2d8f1ec 100644 --- a/backend/inspirehep/records/marshmallow/jobs/base.py +++ b/backend/inspirehep/records/marshmallow/jobs/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,9 +12,10 @@ AcceleratorExperimentSchemaV1, ContactDetailsItemWithoutEmail, ) - -from ..utils import get_acquisition_source_without_email -from .utils import get_reference_letters_without_email +from inspirehep.records.marshmallow.jobs.utils import ( + get_reference_letters_without_email, +) +from inspirehep.records.marshmallow.utils import get_acquisition_source_without_email class JobsRawSchema(RecordBaseSchema): diff --git a/backend/inspirehep/records/marshmallow/jobs/es.py b/backend/inspirehep/records/marshmallow/jobs/es.py index 9c4abb8058..611915dbf5 100644 --- a/backend/inspirehep/records/marshmallow/jobs/es.py +++ b/backend/inspirehep/records/marshmallow/jobs/es.py @@ -1,12 +1,11 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -from ..base import ElasticSearchBaseSchema -from .base import JobsRawSchema +from inspirehep.records.marshmallow.base import ElasticSearchBaseSchema +from inspirehep.records.marshmallow.jobs.base import JobsRawSchema class JobsElasticSearchSchema(ElasticSearchBaseSchema, JobsRawSchema): diff --git a/backend/inspirehep/records/marshmallow/jobs/ui.py b/backend/inspirehep/records/marshmallow/jobs/ui.py index 3b833c448f..e7b9fd01d4 100644 --- a/backend/inspirehep/records/marshmallow/jobs/ui.py +++ b/backend/inspirehep/records/marshmallow/jobs/ui.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -13,11 +12,12 @@ ) from inspirehep.records.marshmallow.common import ContactDetailsItemWithoutEmail from inspirehep.records.marshmallow.jobs.base import JobsPublicSchema +from inspirehep.records.marshmallow.jobs.utils import ( + get_reference_letters_without_email, +) +from inspirehep.records.marshmallow.utils import get_acquisition_source_without_email from inspirehep.submissions.utils import has_30_days_passed_after_deadline -from ..utils import get_acquisition_source_without_email -from .utils import get_reference_letters_without_email - class JobsBaseSchema(JobsPublicSchema): can_edit = fields.Method("is_job_editable", dump_only=True) @@ -48,9 +48,9 @@ def is_job_editable(data): return True deadline = get_value(data, "deadline_date") - if status == "closed" and not has_30_days_passed_after_deadline(deadline): - return True - return False + return bool( + status == "closed" and not has_30_days_passed_after_deadline(deadline) + ) class JobsDetailSchema(JobsBaseSchema): diff --git a/backend/inspirehep/records/marshmallow/journals/base.py b/backend/inspirehep/records/marshmallow/journals/base.py index dcca565ead..cb7ac45757 100644 --- a/backend/inspirehep/records/marshmallow/journals/base.py +++ b/backend/inspirehep/records/marshmallow/journals/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/journals/es.py b/backend/inspirehep/records/marshmallow/journals/es.py index 61e42aeec1..de2be65139 100644 --- a/backend/inspirehep/records/marshmallow/journals/es.py +++ b/backend/inspirehep/records/marshmallow/journals/es.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,8 +6,8 @@ from marshmallow import fields -from ..base import ElasticSearchBaseSchema -from .base import JournalsRawSchema +from inspirehep.records.marshmallow.base import ElasticSearchBaseSchema +from inspirehep.records.marshmallow.journals.base import JournalsRawSchema class JournalsElasticSearchSchema(ElasticSearchBaseSchema, JournalsRawSchema): diff --git a/backend/inspirehep/records/marshmallow/journals/ui.py b/backend/inspirehep/records/marshmallow/journals/ui.py index 3a6eccfba8..4db5396b9e 100644 --- a/backend/inspirehep/records/marshmallow/journals/ui.py +++ b/backend/inspirehep/records/marshmallow/journals/ui.py @@ -1,14 +1,14 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2022 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. +from inspire_utils.record import get_value from marshmallow import fields from inspirehep.records.marshmallow.journals.base import JournalsPublicListSchema -from inspire_utils.record import get_value + class JournalsBaseSchema(JournalsPublicListSchema): public_notes = fields.Raw() @@ -23,10 +23,10 @@ def get_journal_title(self, original_object): return journal_title + class JournalsListSchema(JournalsBaseSchema): pass - + class JournalsDetailSchema(JournalsListSchema): pass - diff --git a/backend/inspirehep/records/marshmallow/literature/authors.py b/backend/inspirehep/records/marshmallow/literature/authors.py index cd5013178c..f20e83a76f 100644 --- a/backend/inspirehep/records/marshmallow/literature/authors.py +++ b/backend/inspirehep/records/marshmallow/literature/authors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,9 +7,9 @@ from inspire_dojson.utils import strip_empty_values from marshmallow import Schema, fields, post_dump -from ..fields import NestedField -from .common import AuthorSchemaV1 -from .common.author import SupervisorSchema +from inspirehep.records.marshmallow.fields import NestedField +from inspirehep.records.marshmallow.literature.common import AuthorSchemaV1 +from inspirehep.records.marshmallow.literature.common.author import SupervisorSchema class LiteratureAuthorsSchema(Schema): diff --git a/backend/inspirehep/records/marshmallow/literature/base.py b/backend/inspirehep/records/marshmallow/literature/base.py index b512ba0368..77f125a09e 100644 --- a/backend/inspirehep/records/marshmallow/literature/base.py +++ b/backend/inspirehep/records/marshmallow/literature/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,10 +6,10 @@ from marshmallow import fields -from ..base import RecordBaseSchema -from ..fields import NonHiddenRaw -from ..utils import get_acquisition_source_without_email -from .utils import get_authors_without_emails +from inspirehep.records.marshmallow.base import RecordBaseSchema +from inspirehep.records.marshmallow.fields import NonHiddenRaw +from inspirehep.records.marshmallow.literature.utils import get_authors_without_emails +from inspirehep.records.marshmallow.utils import get_acquisition_source_without_email class LiteratureRawSchema(RecordBaseSchema): diff --git a/backend/inspirehep/records/marshmallow/literature/bibtex.py b/backend/inspirehep/records/marshmallow/literature/bibtex.py index a297b2fa2e..7fc78e9011 100644 --- a/backend/inspirehep/records/marshmallow/literature/bibtex.py +++ b/backend/inspirehep/records/marshmallow/literature/bibtex.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -19,13 +18,12 @@ from prometheus_client import Counter from six import text_type +from inspirehep.records.marshmallow.base import BaseSchema from inspirehep.records.marshmallow.literature.utils import ( get_parent_record, latex_encode, ) -from ..base import BaseSchema - bibtex_conversion_success = Counter( "bibtex_title_serialization_success", "Bibtex title serialization success" ) @@ -93,7 +91,6 @@ class BibTexCommonSchema(BaseSchema): @staticmethod def get_date(data, doc_type): - publication_year = BibTexCommonSchema.get_best_publication_info(data).get( "year" ) @@ -268,7 +265,6 @@ def get_title(self, data): part_witouth_mathml = part_witouth_mathml.replace("DUMMYAND", "&") bibtex_conversion_success_with_and_replacement.inc() except XMLSyntaxError: - part_witouth_mathml = part bibtex_conversion_errors.inc() latex_encoded_part = ( @@ -300,7 +296,7 @@ def get_type(self, data): doc_type = data.get("doc_type") degree_type = get_value(data, "thesis_info.degree_type", "other") if doc_type == "mastersthesis" and degree_type not in ("master", "diploma"): - return "{} thesis".format(degree_type.title()) + return f"{degree_type.title()} thesis" def get_report_number(self, data): report_number = ", ".join( diff --git a/backend/inspirehep/records/marshmallow/literature/common/abstract.py b/backend/inspirehep/records/marshmallow/literature/common/abstract.py index d8cde20160..c26baca05b 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/abstract.py +++ b/backend/inspirehep/records/marshmallow/literature/common/abstract.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/literature/common/author.py b/backend/inspirehep/records/marshmallow/literature/common/author.py index 7a79383a09..3df90a9aac 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/author.py +++ b/backend/inspirehep/records/marshmallow/literature/common/author.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -22,7 +21,6 @@ class FirstAuthorSchemaV1(Schema): - recid = fields.Method("get_recid", default=missing, attribute="record") emails = fields.Raw() full_name = fields.Raw() diff --git a/backend/inspirehep/records/marshmallow/literature/common/citation_item.py b/backend/inspirehep/records/marshmallow/literature/common/citation_item.py index 44ee38f05d..33b8ec6d6b 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/citation_item.py +++ b/backend/inspirehep/records/marshmallow/literature/common/citation_item.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,11 +7,17 @@ from inspire_dojson.utils import strip_empty_values from marshmallow import Schema, fields, post_dump -from ...fields import ListWithLimit, NestedField -from .author import AuthorSchemaV1 -from .collaboration import CollaborationSchemaV1 -from .collaboration_with_suffix import CollaborationWithSuffixSchemaV1 -from .publication_info_item import PublicationInfoItemSchemaV1 +from inspirehep.records.marshmallow.fields import ListWithLimit, NestedField +from inspirehep.records.marshmallow.literature.common.author import AuthorSchemaV1 +from inspirehep.records.marshmallow.literature.common.collaboration import ( + CollaborationSchemaV1, +) +from inspirehep.records.marshmallow.literature.common.collaboration_with_suffix import ( + CollaborationWithSuffixSchemaV1, +) +from inspirehep.records.marshmallow.literature.common.publication_info_item import ( + PublicationInfoItemSchemaV1, +) class CitationItemSchemaV1(Schema): diff --git a/backend/inspirehep/records/marshmallow/literature/common/collaboration.py b/backend/inspirehep/records/marshmallow/literature/common/collaboration.py index 36e0c9f16b..31b8d55168 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/collaboration.py +++ b/backend/inspirehep/records/marshmallow/literature/common/collaboration.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/literature/common/collaboration_with_suffix.py b/backend/inspirehep/records/marshmallow/literature/common/collaboration_with_suffix.py index bba427a81a..f8e6f68921 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/collaboration_with_suffix.py +++ b/backend/inspirehep/records/marshmallow/literature/common/collaboration_with_suffix.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -9,7 +8,9 @@ from marshmallow import pre_dump -from .collaboration import CollaborationSchemaV1 +from inspirehep.records.marshmallow.literature.common.collaboration import ( + CollaborationSchemaV1, +) class CollaborationWithSuffixSchemaV1(CollaborationSchemaV1): diff --git a/backend/inspirehep/records/marshmallow/literature/common/conference_info_item.py b/backend/inspirehep/records/marshmallow/literature/common/conference_info_item.py index 38eef4cf91..af802c694c 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/conference_info_item.py +++ b/backend/inspirehep/records/marshmallow/literature/common/conference_info_item.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/literature/common/doi.py b/backend/inspirehep/records/marshmallow/literature/common/doi.py index 9db747aa11..6ca51f9b64 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/doi.py +++ b/backend/inspirehep/records/marshmallow/literature/common/doi.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/literature/common/external_system_identifier.py b/backend/inspirehep/records/marshmallow/literature/common/external_system_identifier.py index 801705b7b3..9b60cc205d 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/external_system_identifier.py +++ b/backend/inspirehep/records/marshmallow/literature/common/external_system_identifier.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -55,7 +54,7 @@ def get_url_link(self, data): if kekscan_link is None: return missing return kekscan_link - return "{}{}".format(url_prefix, value) + return f"{url_prefix}{value}" @classmethod def get_link_for_kekscan_schema(cls, external_system_id_value): @@ -73,7 +72,7 @@ def get_link_for_kekscan_schema(cls, external_system_id_value): return None yymm = extid[:4] kekscan_url_prefix = cls.schema_to_url_link_prefix_map["kekscan"] - return "{}{}/{}/{}.pdf".format(kekscan_url_prefix, year, yymm, extid) + return f"{kekscan_url_prefix}{year}/{yymm}/{extid}.pdf" @post_dump(pass_many=True) def filter(self, data, many): diff --git a/backend/inspirehep/records/marshmallow/literature/common/isbn.py b/backend/inspirehep/records/marshmallow/literature/common/isbn.py index c44b771060..69b8388fb7 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/isbn.py +++ b/backend/inspirehep/records/marshmallow/literature/common/isbn.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/literature/common/publication_info_item.py b/backend/inspirehep/records/marshmallow/literature/common/publication_info_item.py index b9887a2119..d646a495da 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/publication_info_item.py +++ b/backend/inspirehep/records/marshmallow/literature/common/publication_info_item.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/literature/common/reference_item.py b/backend/inspirehep/records/marshmallow/literature/common/reference_item.py index a3c113c100..2de9258085 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/reference_item.py +++ b/backend/inspirehep/records/marshmallow/literature/common/reference_item.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,11 +10,16 @@ from marshmallow import Schema, fields, missing, post_dump, pre_dump from inspirehep.records.marshmallow.fields import ListWithLimit, NestedField - -from .author import AuthorSchemaV1 -from .collaboration import CollaborationSchemaV1 -from .collaboration_with_suffix import CollaborationWithSuffixSchemaV1 -from .publication_info_item import PublicationInfoItemSchemaV1 +from inspirehep.records.marshmallow.literature.common.author import AuthorSchemaV1 +from inspirehep.records.marshmallow.literature.common.collaboration import ( + CollaborationSchemaV1, +) +from inspirehep.records.marshmallow.literature.common.collaboration_with_suffix import ( + CollaborationWithSuffixSchemaV1, +) +from inspirehep.records.marshmallow.literature.common.publication_info_item import ( + PublicationInfoItemSchemaV1, +) class ReferenceItemSchemaV1(Schema): diff --git a/backend/inspirehep/records/marshmallow/literature/common/thesis_info.py b/backend/inspirehep/records/marshmallow/literature/common/thesis_info.py index 83dfec5108..90fe356f43 100644 --- a/backend/inspirehep/records/marshmallow/literature/common/thesis_info.py +++ b/backend/inspirehep/records/marshmallow/literature/common/thesis_info.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/literature/cv.py b/backend/inspirehep/records/marshmallow/literature/cv.py index 1cf11d7db2..cf23621be4 100644 --- a/backend/inspirehep/records/marshmallow/literature/cv.py +++ b/backend/inspirehep/records/marshmallow/literature/cv.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/literature/es.py b/backend/inspirehep/records/marshmallow/literature/es.py index ff0eeaca37..338fda08f5 100644 --- a/backend/inspirehep/records/marshmallow/literature/es.py +++ b/backend/inspirehep/records/marshmallow/literature/es.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -21,6 +20,8 @@ from inspirehep.oai.utils import is_cds_set, is_cern_arxiv_set from inspirehep.pidstore.api import PidStoreBase from inspirehep.records.api import InspireRecord +from inspirehep.records.marshmallow.base import ElasticSearchBaseSchema +from inspirehep.records.marshmallow.literature.base import LiteratureRawSchema from inspirehep.records.marshmallow.literature.common.abstract import AbstractSource from inspirehep.records.marshmallow.literature.common.author import ( AuthorsInfoSchemaForES, @@ -30,13 +31,13 @@ from inspirehep.records.marshmallow.literature.common.thesis_info import ( ThesisInfoSchemaForESV1, ) +from inspirehep.records.marshmallow.literature.ui import LiteratureDetailSchema +from inspirehep.records.marshmallow.utils import ( + get_display_name_for_author_name, + get_facet_author_name_for_author, +) from inspirehep.records.models import RecordCitations, RecordsAuthors -from ..base import ElasticSearchBaseSchema -from ..utils import get_display_name_for_author_name, get_facet_author_name_for_author -from .base import LiteratureRawSchema -from .ui import LiteratureDetailSchema - LOGGER = structlog.getLogger() diff --git a/backend/inspirehep/records/marshmallow/literature/latex.py b/backend/inspirehep/records/marshmallow/literature/latex.py index cd554b5701..005df4195a 100644 --- a/backend/inspirehep/records/marshmallow/literature/latex.py +++ b/backend/inspirehep/records/marshmallow/literature/latex.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -17,9 +16,9 @@ from marshmallow import fields, missing from prometheus_client import Counter -from ..base import BaseSchema -from .bibtex import BibTexCommonSchema -from .utils import latex_encode +from inspirehep.records.marshmallow.base import BaseSchema +from inspirehep.records.marshmallow.literature.bibtex import BibTexCommonSchema +from inspirehep.records.marshmallow.literature.utils import latex_encode latex_conversion_success = Counter( "latex_title_serialization_success", "Latex title serialization success" @@ -70,7 +69,6 @@ def cleanup_publication_info(pub_info): return publication_info def get_author_names(self, data): - authors = data.get("authors") if not authors: @@ -144,7 +142,6 @@ def get_collaborations(self, data): return [latex_encode(collab["value"]) for collab in data.get("collaborations")] def get_note(self, data): - erratums = [ self.cleanup_publication_info(publication) for publication in get_value(data, "publication_info", []) @@ -178,7 +175,6 @@ def get_book_publication_info(self, data): return missing def get_isbn(self, data): - isbns = get_value(data, "isbns.value") if not isbns: return missing diff --git a/backend/inspirehep/records/marshmallow/literature/references.py b/backend/inspirehep/records/marshmallow/literature/references.py index 5d14ec45ad..2b1faa9066 100644 --- a/backend/inspirehep/records/marshmallow/literature/references.py +++ b/backend/inspirehep/records/marshmallow/literature/references.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,8 +7,8 @@ from inspire_dojson.utils import strip_empty_values from marshmallow import Schema, post_dump -from ..fields import NestedField -from .common import ReferenceItemSchemaV1 +from inspirehep.records.marshmallow.fields import NestedField +from inspirehep.records.marshmallow.literature.common import ReferenceItemSchemaV1 class LiteratureReferencesSchema(Schema): diff --git a/backend/inspirehep/records/marshmallow/literature/ui.py b/backend/inspirehep/records/marshmallow/literature/ui.py index fb3128f6d4..fe10b53ac4 100644 --- a/backend/inspirehep/records/marshmallow/literature/ui.py +++ b/backend/inspirehep/records/marshmallow/literature/ui.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -18,21 +17,14 @@ ) from inspirehep.assign.utils import can_claim, is_assign_view_enabled from inspirehep.files.api import current_s3_instance +from inspirehep.records.marshmallow.base import EnvelopeSchema +from inspirehep.records.marshmallow.common import AcceleratorExperimentSchemaV1 from inspirehep.records.marshmallow.common.mixins import ( CanEditByCollectionPermissionMixin, ) -from inspirehep.records.marshmallow.literature.utils import ( - get_authors_without_emails, - get_pages, - get_parent_records, -) -from inspirehep.records.utils import get_literature_earliest_date - -from ..base import EnvelopeSchema -from ..common import AcceleratorExperimentSchemaV1 -from ..fields import ListWithLimit, NonHiddenNested -from .base import LiteraturePublicSchema -from .common import ( +from inspirehep.records.marshmallow.fields import ListWithLimit, NonHiddenNested +from inspirehep.records.marshmallow.literature.base import LiteraturePublicSchema +from inspirehep.records.marshmallow.literature.common import ( AuthorSchemaV1, CollaborationSchemaV1, CollaborationWithSuffixSchemaV1, @@ -43,7 +35,15 @@ PublicationInfoItemSchemaV1, ThesisInfoSchemaV1, ) -from .pdg_identifiers_latex import PDG_IDS_TO_LATEX_DESCRIPTION_MAPPING +from inspirehep.records.marshmallow.literature.pdg_identifiers_latex import ( + PDG_IDS_TO_LATEX_DESCRIPTION_MAPPING, +) +from inspirehep.records.marshmallow.literature.utils import ( + get_authors_without_emails, + get_pages, + get_parent_records, +) +from inspirehep.records.utils import get_literature_earliest_date DATASET_SCHEMA_TO_URL_PREFIX_MAP = { "hepdata": "https://www.hepdata.net/record/", @@ -163,7 +163,7 @@ def get_arxiv_fulltext_link(self, data): if arxiv_id: return { "description": description, - "value": "https://arxiv.org/pdf/%s" % arxiv_id, + "value": f"https://arxiv.org/pdf/{arxiv_id}", } return missing @@ -203,7 +203,7 @@ def get_linked_books(self, data): merged_list = [ {"page_start": start, "page_end": end, **record} for start, end, record in zip( - pages["page_start"], pages["page_end"], records + pages["page_start"], pages["page_end"], records, strict=False ) ] diff --git a/backend/inspirehep/records/marshmallow/literature/utils.py b/backend/inspirehep/records/marshmallow/literature/utils.py index 64570eb5ba..d5f8b6322d 100644 --- a/backend/inspirehep/records/marshmallow/literature/utils.py +++ b/backend/inspirehep/records/marshmallow/literature/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/seminars/base.py b/backend/inspirehep/records/marshmallow/seminars/base.py index d1c9d82ac0..490272818f 100644 --- a/backend/inspirehep/records/marshmallow/seminars/base.py +++ b/backend/inspirehep/records/marshmallow/seminars/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -15,7 +14,6 @@ class SeminarsRawSchema(RecordBaseSchema): - address = fields.Method("get_address") @staticmethod diff --git a/backend/inspirehep/records/marshmallow/seminars/common/literature_record.py b/backend/inspirehep/records/marshmallow/seminars/common/literature_record.py index 71782315d5..ae874bcc2f 100644 --- a/backend/inspirehep/records/marshmallow/seminars/common/literature_record.py +++ b/backend/inspirehep/records/marshmallow/seminars/common/literature_record.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/marshmallow/seminars/common/speaker.py b/backend/inspirehep/records/marshmallow/seminars/common/speaker.py index 331942da0b..fabce59c01 100644 --- a/backend/inspirehep/records/marshmallow/seminars/common/speaker.py +++ b/backend/inspirehep/records/marshmallow/seminars/common/speaker.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,7 +10,6 @@ class SpeakerSchemaV1(Schema): - affiliations = fields.Raw() curated_relation = fields.Raw() ids = fields.Raw() diff --git a/backend/inspirehep/records/marshmallow/seminars/es.py b/backend/inspirehep/records/marshmallow/seminars/es.py index deced323a4..8a9b9cecc7 100644 --- a/backend/inspirehep/records/marshmallow/seminars/es.py +++ b/backend/inspirehep/records/marshmallow/seminars/es.py @@ -1,12 +1,11 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. -from ..base import ElasticSearchBaseSchema -from .base import SeminarsRawSchema +from inspirehep.records.marshmallow.base import ElasticSearchBaseSchema +from inspirehep.records.marshmallow.seminars.base import SeminarsRawSchema class SeminarsElasticSearchSchema(ElasticSearchBaseSchema, SeminarsRawSchema): diff --git a/backend/inspirehep/records/marshmallow/seminars/ui.py b/backend/inspirehep/records/marshmallow/seminars/ui.py index 18bc5a86ad..04c0e06f4b 100644 --- a/backend/inspirehep/records/marshmallow/seminars/ui.py +++ b/backend/inspirehep/records/marshmallow/seminars/ui.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -13,8 +12,7 @@ LiteratureRecordSchemaV1, ) from inspirehep.records.marshmallow.seminars.common.speaker import SpeakerSchemaV1 - -from ..utils import get_acquisition_source_without_email +from inspirehep.records.marshmallow.utils import get_acquisition_source_without_email class SeminarsBaseSchema(SeminarsPublicSchema): diff --git a/backend/inspirehep/records/marshmallow/utils.py b/backend/inspirehep/records/marshmallow/utils.py index 107c500276..7885c95116 100644 --- a/backend/inspirehep/records/marshmallow/utils.py +++ b/backend/inspirehep/records/marshmallow/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -69,6 +68,6 @@ def get_last_name(name): def get_acquisition_source_without_email(data): acquisition_source = data.get("acquisition_source", {}) - if "email" in acquisition_source.keys(): + if "email" in acquisition_source: del acquisition_source["email"] return acquisition_source diff --git a/backend/inspirehep/records/models.py b/backend/inspirehep/records/models.py index 1b3eb7a78e..ece9bab702 100644 --- a/backend/inspirehep/records/models.py +++ b/backend/inspirehep/records/models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,6 +5,7 @@ # the terms of the MIT License; see LICENSE file for more details. """Additional models used by InspireRecords.""" + import enum from datetime import datetime @@ -249,7 +249,7 @@ class StudentsAdvisors(db.Model): ) -class Timestamp(object): +class Timestamp: """Timestamp model mix-in with fractional seconds support. SQLAlchemy-Utils timestamp model does not have support for fractional seconds. @@ -274,7 +274,6 @@ def timestamp_before_update(mapper, connection, target): class WorkflowsRecordSources(db.Model, Timestamp): - __tablename__ = "workflows_record_sources" __table_args__ = (db.PrimaryKeyConstraint("record_uuid", "source"),) diff --git a/backend/inspirehep/records/receivers.py b/backend/inspirehep/records/receivers.py index e14bdff6ea..a95593ac94 100644 --- a/backend/inspirehep/records/receivers.py +++ b/backend/inspirehep/records/receivers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -24,16 +23,19 @@ def index_after_commit(sender, changes): has been really committed to the DB. """ for model_instance, change in changes: - if isinstance(model_instance, RecordMetadata): - if change in ("insert", "update", "delete"): - LOGGER.debug( - "Record commited, indexing.", - change=change, - uuid=str(model_instance.id), - ) - force_delete = "delete" == change - InspireRecord(model_instance.json, model=model_instance).index( - force_delete=force_delete - ) - if "new_record" in model_instance.json: - redirect_references_to_merged_record.delay(str(model_instance.id)) + if isinstance(model_instance, RecordMetadata) and change in ( + "insert", + "update", + "delete", + ): + LOGGER.debug( + "Record commited, indexing.", + change=change, + uuid=str(model_instance.id), + ) + force_delete = change == "delete" + InspireRecord(model_instance.json, model=model_instance).index( + force_delete=force_delete + ) + if "new_record" in model_instance.json: + redirect_references_to_merged_record.delay(str(model_instance.id)) diff --git a/backend/inspirehep/records/serializers/bibtex.py b/backend/inspirehep/records/serializers/bibtex.py index 38c51336a9..972a564a3d 100644 --- a/backend/inspirehep/records/serializers/bibtex.py +++ b/backend/inspirehep/records/serializers/bibtex.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,10 +7,9 @@ from pybtex.database import BibliographyData, Entry, Person from pybtex.database.output.bibtex import Writer +from inspirehep.records.marshmallow.literature.bibtex import BibTexCommonSchema from inspirehep.serializers import record_responsify, search_responsify -from ..marshmallow.literature.bibtex import BibTexCommonSchema - LOGGER = structlog.getLogger() @@ -26,7 +24,7 @@ def _write_persons(self, stream, persons, role): stream, role, self._format_name(stream, persons[0]) + " and others" ) else: - super(BibtexWriter, self)._write_persons(stream, persons, role) + super()._write_persons(stream, persons, role) class BibTexSerializer: diff --git a/backend/inspirehep/records/serializers/cv.py b/backend/inspirehep/records/serializers/cv.py index a7f0d86881..8344c32484 100644 --- a/backend/inspirehep/records/serializers/cv.py +++ b/backend/inspirehep/records/serializers/cv.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -12,10 +11,9 @@ from invenio_records_rest.serializers.marshmallow import MarshmallowMixin from inspirehep.records.marshmallow.literature.cv import CVSchema +from inspirehep.records.serializers.jinja import jinja_cv_env from inspirehep.serializers import record_responsify, search_responsify -from .jinja import jinja_cv_env - class CVHTMLSerializer(MarshmallowMixin, PreprocessorMixin): wrapping_html = dedent( @@ -30,7 +28,7 @@ class CVHTMLSerializer(MarshmallowMixin, PreprocessorMixin): ) def __init__(self, **kwargs): - super(CVHTMLSerializer, self).__init__(**kwargs) + super().__init__(**kwargs) @staticmethod def cv_template(): diff --git a/backend/inspirehep/records/serializers/jinja.py b/backend/inspirehep/records/serializers/jinja.py index bad4758aca..7dd8f41602 100644 --- a/backend/inspirehep/records/serializers/jinja.py +++ b/backend/inspirehep/records/serializers/jinja.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/records/serializers/json/__init__.py b/backend/inspirehep/records/serializers/json/__init__.py index a8522c4331..f40d00e762 100644 --- a/backend/inspirehep/records/serializers/json/__init__.py +++ b/backend/inspirehep/records/serializers/json/__init__.py @@ -41,7 +41,7 @@ journals_json_response, journals_json_response_search, journals_json_detail_response, - journals_json_list_response + journals_json_list_response, ) from .literature import ( facets_json_response_search, diff --git a/backend/inspirehep/records/serializers/json/authors.py b/backend/inspirehep/records/serializers/json/authors.py index 1a48958a5c..02911f604f 100644 --- a/backend/inspirehep/records/serializers/json/authors.py +++ b/backend/inspirehep/records/serializers/json/authors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/serializers/json/conferences.py b/backend/inspirehep/records/serializers/json/conferences.py index 25737b2373..8eeca2536a 100644 --- a/backend/inspirehep/records/serializers/json/conferences.py +++ b/backend/inspirehep/records/serializers/json/conferences.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/serializers/json/experiments.py b/backend/inspirehep/records/serializers/json/experiments.py index a243f74ea4..0a88fb6f4f 100644 --- a/backend/inspirehep/records/serializers/json/experiments.py +++ b/backend/inspirehep/records/serializers/json/experiments.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/serializers/json/institutions.py b/backend/inspirehep/records/serializers/json/institutions.py index 384a350954..0611917bdc 100644 --- a/backend/inspirehep/records/serializers/json/institutions.py +++ b/backend/inspirehep/records/serializers/json/institutions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/serializers/json/jobs.py b/backend/inspirehep/records/serializers/json/jobs.py index ef29afc5f3..26b004cfa1 100644 --- a/backend/inspirehep/records/serializers/json/jobs.py +++ b/backend/inspirehep/records/serializers/json/jobs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/serializers/json/journals.py b/backend/inspirehep/records/serializers/json/journals.py index ee94591fa2..a335f4fbde 100644 --- a/backend/inspirehep/records/serializers/json/journals.py +++ b/backend/inspirehep/records/serializers/json/journals.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/serializers/json/literature.py b/backend/inspirehep/records/serializers/json/literature.py index 8f46f03ae9..42375972fd 100644 --- a/backend/inspirehep/records/serializers/json/literature.py +++ b/backend/inspirehep/records/serializers/json/literature.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/serializers/json/raw.py b/backend/inspirehep/records/serializers/json/raw.py index 8b3a365798..5d762dc6cf 100644 --- a/backend/inspirehep/records/serializers/json/raw.py +++ b/backend/inspirehep/records/serializers/json/raw.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/serializers/json/seminars.py b/backend/inspirehep/records/serializers/json/seminars.py index 5ceebfeeb3..6db11e10d5 100644 --- a/backend/inspirehep/records/serializers/json/seminars.py +++ b/backend/inspirehep/records/serializers/json/seminars.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/records/serializers/latex.py b/backend/inspirehep/records/serializers/latex.py index bd425e16b3..23c1ff32f0 100644 --- a/backend/inspirehep/records/serializers/latex.py +++ b/backend/inspirehep/records/serializers/latex.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,11 +6,10 @@ from invenio_records_rest.serializers.json import MarshmallowMixin, PreprocessorMixin +from inspirehep.records.marshmallow.literature.latex import LatexSchema +from inspirehep.records.serializers.jinja import jinja_latex_env from inspirehep.serializers import record_responsify, search_responsify -from ..marshmallow.literature.latex import LatexSchema -from .jinja import jinja_latex_env - class LatexSerializer(MarshmallowMixin, PreprocessorMixin): """Latex serializer for records.""" @@ -19,7 +17,7 @@ class LatexSerializer(MarshmallowMixin, PreprocessorMixin): def __init__(self, format, **kwargs): """Initialize record.""" self.format = format - super(LatexSerializer, self).__init__(**kwargs) + super().__init__(**kwargs) def serialize(self, pid, record, links_factory=None, **kwargs): """Serialize a single record and persistent identifier. diff --git a/backend/inspirehep/records/serializers/oai.py b/backend/inspirehep/records/serializers/oai.py index d764bdcdea..5724951e13 100644 --- a/backend/inspirehep/records/serializers/oai.py +++ b/backend/inspirehep/records/serializers/oai.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/records/tasks.py b/backend/inspirehep/records/tasks.py index 319bcfba0e..677abf12dd 100644 --- a/backend/inspirehep/records/tasks.py +++ b/backend/inspirehep/records/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/records/utils.py b/backend/inspirehep/records/utils.py index 7e79901799..6c03b5607b 100644 --- a/backend/inspirehep/records/utils.py +++ b/backend/inspirehep/records/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -83,14 +82,13 @@ def download_file_from_url(url, check_file_size=False): content_length = int(content_length) if check_file_size and content_length and content_length > file_size_limit: - raise FileSizeExceededError( f"Can't download file from url {download_url}. File size {content_length} is larger than the limit {file_size_limit}." ) except requests.exceptions.RequestException as exc: raise DownloadFileError( f"Cannot download file from url {download_url}. Reason: {exc}" - ) + ) from exc return request.content @@ -187,7 +185,9 @@ def _create_ticket_self_curation(record_control_number, record_revision_id, user def get_changed_reference(old_record_version, new_record_version): old_references = old_record_version.get("references", []) new_references = new_record_version.get("references", []) - for idx, (old_ref, new_ref) in enumerate(zip(old_references, new_references)): + for idx, (old_ref, new_ref) in enumerate( + zip(old_references, new_references, strict=False) + ): if old_ref != new_ref: return { "current_version": new_ref, diff --git a/backend/inspirehep/records/views.py b/backend/inspirehep/records/views.py index b709c13aac..1bcbf53412 100644 --- a/backend/inspirehep/records/views.py +++ b/backend/inspirehep/records/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -35,13 +34,15 @@ LiteratureReferencesSchema, ) from inspirehep.records.models import WorkflowsRecordSources -from inspirehep.records.utils import _create_ticket_self_curation +from inspirehep.records.utils import ( + _create_ticket_self_curation, + get_changed_reference, + get_ref_from_pid, +) +from inspirehep.search.api import LiteratureSearch from inspirehep.serializers import jsonify from inspirehep.submissions.serializers import literature_v1 -from ..search.api import LiteratureSearch -from .utils import get_changed_reference, get_ref_from_pid - LOGGER = structlog.getLogger() blueprint = Blueprint("inspirehep_records", __name__, url_prefix="") parser = FlaskParser() @@ -100,7 +101,6 @@ def get(self, pid, record): class WorkflowsRecordSourcesResource(MethodView): - view_name = "workflows_record_sources" decorators = [ login_required_with_roles([Roles.superuser.value, Roles.cataloger.value]) @@ -133,7 +133,12 @@ def get(self, args): if not results: return jsonify({"message": "Workflow source not found"}), 404 results_data = [ - {key: val for key, val in zip(required_fields_mapping.keys(), result)} + { + key: val + for key, val in zip( + required_fields_mapping.keys(), result, strict=False + ) + } for result in results ] return jsonify({"workflow_sources": results_data}), 200 @@ -271,7 +276,6 @@ def reference_self_curation(args): def literature_reference_difference_between_versions( pid_value, old_revision, new_revision ): - if new_revision <= old_revision: return ( jsonify({"message": "Old revision must be lower than new revision"}), diff --git a/backend/inspirehep/search/aggregations.py b/backend/inspirehep/search/aggregations.py index 5f798f20e1..f502a93ae3 100644 --- a/backend/inspirehep/search/aggregations.py +++ b/backend/inspirehep/search/aggregations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -730,7 +729,6 @@ def seminar_accessibility_aggregation( def experiment_inspire_classification_aggregation( order, title="Experiments", agg_type="tree" ): - return { "experiments": { "terms": {"field": "facet_inspire_classification", "size": 500}, diff --git a/backend/inspirehep/search/api.py b/backend/inspirehep/search/api.py index 9bcf421a85..851baa6497 100644 --- a/backend/inspirehep/search/api.py +++ b/backend/inspirehep/search/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,13 +6,13 @@ import structlog -from opensearchpy import RequestError -from opensearch_dsl.query import Match, Q from flask import current_app, request from inspire_schemas.utils import convert_old_publication_info_to_new from inspire_utils.record import get_value from invenio_search import current_search_client as es from invenio_search.api import DefaultFilter, RecordsSearch +from opensearch_dsl.query import Match, Q +from opensearchpy import RequestError from requests.exceptions import RequestException from inspirehep.accounts.api import ( @@ -25,17 +24,15 @@ match_reference_control_numbers_with_relaxed_journal_titles, ) from inspirehep.pidstore.api import PidStoreBase -from inspirehep.search.errors import MaximumSearchPageSizeExceeded +from inspirehep.search.errors import MalformatedQuery, MaximumSearchPageSizeExceeded from inspirehep.search.factories import inspire_query_factory from inspirehep.search.utils import RecursionLimit -from .errors import MalformatedQuery - IQ = inspire_query_factory() LOGGER = structlog.getLogger() -class SearchMixin(object): +class SearchMixin: """Mixin that adds helper functions to ElasticSearch DSL classes.""" @property @@ -62,8 +59,8 @@ def query_from_iq(self, query_string): default_field="_all", default_operator="AND", ) - except ValueError: - raise MalformatedQuery + except ValueError as e: + raise MalformatedQuery from e def get_source(self, uuid, **kwargs): """Get source from a given uuid. @@ -179,9 +176,9 @@ def normalize_journal_title(self, reference): journal_title = get_value( reference, "reference.publication_info.journal_title" ) - reference["reference"]["publication_info"][ - "journal_title" - ] = JournalsSearch().normalize_title(journal_title) + reference["reference"]["publication_info"]["journal_title"] = ( + JournalsSearch().normalize_title(journal_title) + ) except KeyError: pass return reference diff --git a/backend/inspirehep/search/config.py b/backend/inspirehep/search/config.py index acc8136e28..a7b58d60c8 100644 --- a/backend/inspirehep/search/config.py +++ b/backend/inspirehep/search/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/search/errors.py b/backend/inspirehep/search/errors.py index 4b30f77c1d..a689db23b0 100644 --- a/backend/inspirehep/search/errors.py +++ b/backend/inspirehep/search/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/search/facets.py b/backend/inspirehep/search/facets.py index e6822b73cf..ffdae2a9c3 100644 --- a/backend/inspirehep/search/facets.py +++ b/backend/inspirehep/search/facets.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -85,7 +84,6 @@ def must_match_all_or_missing_filter(field, missing_field_value): """Bool filter containing a list of must matches.""" def inner(values): - if missing_field_value in values: filters = ~Q("exists", field=field) diff --git a/backend/inspirehep/search/factories/__init__.py b/backend/inspirehep/search/factories/__init__.py index bcfebc6d04..5eb886d10d 100644 --- a/backend/inspirehep/search/factories/__init__.py +++ b/backend/inspirehep/search/factories/__init__.py @@ -8,5 +8,4 @@ """inspirehep.""" - from .query import inspire_query_factory diff --git a/backend/inspirehep/search/factories/facet.py b/backend/inspirehep/search/factories/facet.py index 4f21dd228e..569b257abe 100644 --- a/backend/inspirehep/search/factories/facet.py +++ b/backend/inspirehep/search/factories/facet.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,7 +7,7 @@ from invenio_records_rest.facets import _aggregations from werkzeug.datastructures import MultiDict -from ..utils import get_facet_configuration +from inspirehep.search.utils import get_facet_configuration def inspire_facets_factory(search, index): diff --git a/backend/inspirehep/search/factories/filter.py b/backend/inspirehep/search/factories/filter.py index 82635bf566..6dd1382f6f 100644 --- a/backend/inspirehep/search/factories/filter.py +++ b/backend/inspirehep/search/factories/filter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,7 +7,7 @@ from invenio_records_rest.facets import _post_filter, _query_filter from werkzeug.datastructures import MultiDict -from ..utils import get_facet_configuration +from inspirehep.search.utils import get_facet_configuration def inspire_filter_factory(search, index): diff --git a/backend/inspirehep/search/factories/query.py b/backend/inspirehep/search/factories/query.py index 9928b3130f..90c7df3d77 100644 --- a/backend/inspirehep/search/factories/query.py +++ b/backend/inspirehep/search/factories/query.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,8 +5,8 @@ # the terms of the MIT License; see LICENSE file for more details. import inspire_query_parser -from opensearch_dsl import Q from flask import current_app +from opensearch_dsl import Q from inspirehep.pidstore.api import PidStoreBase from inspirehep.search.errors import MalformatedQuery @@ -15,14 +14,13 @@ def replace_recid_in_citedby_query(query): - citedby_query_fields = set(["index", "id", "path"]) self_ref_field = "self.$ref.raw" def _replace_recid_in_citedby_query(obj): if isinstance(obj, dict): obj_keys = obj.keys() - for key, val in obj.items(): + for _key, val in obj.items(): if isinstance(val, list): _replace_recid_in_citedby_query(val) elif isinstance(val, dict): @@ -56,8 +54,8 @@ def inspire_query(query_string, search): with RecursionLimit(current_app.config.get("SEARCH_MAX_RECURSION_LIMIT", 5000)): try: query = Q(inspire_query_parser.parse_query(query_string)) - except ValueError: - raise MalformatedQuery + except ValueError as e: + raise MalformatedQuery from e if "citedby" in query_string: query = query.to_dict() replace_recid_in_citedby_query(query) diff --git a/backend/inspirehep/search/factories/search.py b/backend/inspirehep/search/factories/search.py index 3cb64bc1d2..dc56fd8b9d 100644 --- a/backend/inspirehep/search/factories/search.py +++ b/backend/inspirehep/search/factories/search.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -10,9 +9,9 @@ from invenio_records_rest.errors import InvalidQueryRESTError from invenio_records_rest.sorter import default_sorter_factory -from ..errors import FieldsParamForbidden -from .facet import inspire_facets_factory -from .filter import inspire_filter_factory +from inspirehep.search.errors import FieldsParamForbidden +from inspirehep.search.factories.facet import inspire_facets_factory +from inspirehep.search.factories.filter import inspire_filter_factory LOGGER = structlog.getLogger() @@ -26,9 +25,7 @@ def can_request_fields(): ): return False requested_format = request.values.get("format", "json", type=str) - if requested_format != "json": - return False - return True + return requested_format == "json" def get_search_with_source(search): diff --git a/backend/inspirehep/search/mappings/os-v2/records/seminars.json b/backend/inspirehep/search/mappings/os-v2/records/seminars.json index a66da36a54..a00cf4847b 100644 --- a/backend/inspirehep/search/mappings/os-v2/records/seminars.json +++ b/backend/inspirehep/search/mappings/os-v2/records/seminars.json @@ -386,4 +386,4 @@ } } } -} \ No newline at end of file +} diff --git a/backend/inspirehep/search/mappings/v7/records/seminars.json b/backend/inspirehep/search/mappings/v7/records/seminars.json index a66da36a54..a00cf4847b 100644 --- a/backend/inspirehep/search/mappings/v7/records/seminars.json +++ b/backend/inspirehep/search/mappings/v7/records/seminars.json @@ -386,4 +386,4 @@ } } } -} \ No newline at end of file +} diff --git a/backend/inspirehep/search/serializers.py b/backend/inspirehep/search/serializers.py index d952c08cf8..be6e04af2e 100644 --- a/backend/inspirehep/search/serializers.py +++ b/backend/inspirehep/search/serializers.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. import orjson +from flask import current_app +from freezegun.api import FakeDatetime from opensearchpy import SerializationError from opensearchpy.compat import string_types from opensearchpy.serializer import JSONSerializer -from flask import current_app -from freezegun.api import FakeDatetime from inspirehep.search.utils import RecursionLimit @@ -20,7 +19,7 @@ def orjson_default(self, data): # this is only way to somehow serialize date during tests as Freezegun is replacing datetime # with itself and it's not possible to easily change type back to datetime return data.isoformat() - raise TypeError("Unable to serialize %r (type: %s)" % (data, type(data))) + raise TypeError(f"Unable to serialize {data!r} (type: {type(data)})") def dumps(self, data): if isinstance(data, string_types): @@ -36,10 +35,10 @@ def dumps(self, data): # As currently we can only turn off recursion limit but we don't want that as # then we should handle overflow error in rust return super().dumps(data) - raise SerializationError(data, ex) + raise SerializationError(data, ex) from ex def loads(self, s): try: return orjson.loads(s) except (ValueError, TypeError) as ex: - raise SerializationError(s, ex) + raise SerializationError(s, ex) from ex diff --git a/backend/inspirehep/search/utils.py b/backend/inspirehep/search/utils.py index 8eb9ace0f7..e9f3180ef2 100644 --- a/backend/inspirehep/search/utils.py +++ b/backend/inspirehep/search/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/search/views.py b/backend/inspirehep/search/views.py index a04b2b2f6b..fb0c3b4474 100644 --- a/backend/inspirehep/search/views.py +++ b/backend/inspirehep/search/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/serializers.py b/backend/inspirehep/serializers.py index b2674ac45e..aae6ac330e 100644 --- a/backend/inspirehep/serializers.py +++ b/backend/inspirehep/serializers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -9,7 +8,6 @@ import orjson import pytz import structlog -from opensearch_dsl import Q from flask import current_app, request from inspire_utils.record import get_value from invenio_records_rest.serializers.json import ( @@ -17,6 +15,7 @@ ) from invenio_records_rest.utils import set_headers_for_record_caching_and_concurrency from invenio_search.utils import build_alias_name +from opensearch_dsl import Q from inspirehep.accounts.api import is_user_logged_in from inspirehep.errors import DB_TASK_EXCEPTIONS, ES_TASK_EXCEPTIONS @@ -96,8 +95,8 @@ def serialize_search( ), links=links, ) - except (ValueError, *DB_TASK_EXCEPTIONS, *ES_TASK_EXCEPTIONS): - raise NonSerializableSearchResult + except (ValueError, *DB_TASK_EXCEPTIONS, *ES_TASK_EXCEPTIONS) as e: + raise NonSerializableSearchResult from e sort_options = self._get_sort_options() if sort_options: data["sort_options"] = sort_options @@ -188,7 +187,7 @@ def flatten_aggregations(aggregations): "key": bucket_key, "doc_count": agg_value["buckets"][bucket_key]["doc_count"], } - for bucket_key in agg_value["buckets"].keys() + for bucket_key in agg_value["buckets"] if agg_value["buckets"][bucket_key]["doc_count"] != 0 ] new_aggs[agg_key] = agg_value @@ -317,9 +316,8 @@ def dumps(obj, app=None, **kwargs): """ encoding = kwargs.pop("encoding", None) rv = orjson.dumps(obj, option=orjson.OPT_NON_STR_KEYS).decode("utf-8") - if encoding is not None: - if isinstance(rv, str): - return rv.encode(encoding) + if encoding is not None and isinstance(rv, str): + return rv.encode(encoding) return rv @@ -393,5 +391,5 @@ def add_link_header(response, links): """ if links is not None: response.headers.extend( - {"Link": ", ".join([f'<{r}>; rel="{l}"' for r, l in links.items()])} + {"Link": ", ".join([f'<{r}>; rel="{link}"' for r, link in links.items()])} ) diff --git a/backend/inspirehep/sitemap/cli.py b/backend/inspirehep/sitemap/cli.py index 7b0ec35bba..a9191a5737 100644 --- a/backend/inspirehep/sitemap/cli.py +++ b/backend/inspirehep/sitemap/cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -9,7 +8,7 @@ import structlog from flask.cli import with_appcontext -from .tasks import create_sitemap +from inspirehep.sitemap.tasks import create_sitemap LOGGER = structlog.getLogger() diff --git a/backend/inspirehep/sitemap/collections.py b/backend/inspirehep/sitemap/collections.py index fe01f5c8b0..515d9ff4af 100644 --- a/backend/inspirehep/sitemap/collections.py +++ b/backend/inspirehep/sitemap/collections.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/sitemap/config.py b/backend/inspirehep/sitemap/config.py index 212305df2e..6399a60a75 100644 --- a/backend/inspirehep/sitemap/config.py +++ b/backend/inspirehep/sitemap/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/sitemap/sitemap.py b/backend/inspirehep/sitemap/sitemap.py index 353073d7d1..dfe216ea53 100644 --- a/backend/inspirehep/sitemap/sitemap.py +++ b/backend/inspirehep/sitemap/sitemap.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -6,10 +5,9 @@ # the terms of the MIT License; see LICENSE file for more details. from inspirehep.pidstore.api import PidStoreBase +from inspirehep.sitemap.collections import get_indexable_record_searches from inspirehep.utils import get_inspirehep_url -from .collections import get_indexable_record_searches - # TODO: maybe move to PidStoreBase ? def get_endpoint_from_schema(schema): @@ -33,5 +31,4 @@ def generate_sitemap_items_from_search(record_search): def generate_sitemap_items(): for record_search in get_indexable_record_searches(): - for sitemap_item in generate_sitemap_items_from_search(record_search): - yield sitemap_item + yield from generate_sitemap_items_from_search(record_search) diff --git a/backend/inspirehep/sitemap/tasks.py b/backend/inspirehep/sitemap/tasks.py index 9ff65da1c1..0828797562 100644 --- a/backend/inspirehep/sitemap/tasks.py +++ b/backend/inspirehep/sitemap/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -10,11 +9,13 @@ from flask import current_app, render_template from inspirehep.errors import ES_TASK_EXCEPTIONS +from inspirehep.sitemap.sitemap import generate_sitemap_items +from inspirehep.sitemap.utils import ( + get_sitemap_page_absolute_url, + write_sitemap_page_content, +) from inspirehep.utils import chunker -from .sitemap import generate_sitemap_items -from .utils import get_sitemap_page_absolute_url, write_sitemap_page_content - LOGGER = structlog.getLogger() diff --git a/backend/inspirehep/sitemap/templates/sitemap/index.xml b/backend/inspirehep/sitemap/templates/sitemap/index.xml index 79313cfefa..160320836c 100644 --- a/backend/inspirehep/sitemap/templates/sitemap/index.xml +++ b/backend/inspirehep/sitemap/templates/sitemap/index.xml @@ -8,4 +8,4 @@ {%- endif %} {%- endfor %} - \ No newline at end of file + diff --git a/backend/inspirehep/sitemap/templates/sitemap/page.xml b/backend/inspirehep/sitemap/templates/sitemap/page.xml index 47b3deeedc..759cd8b093 100644 --- a/backend/inspirehep/sitemap/templates/sitemap/page.xml +++ b/backend/inspirehep/sitemap/templates/sitemap/page.xml @@ -14,4 +14,4 @@ {%- endif %} {%- endfor %} - \ No newline at end of file + diff --git a/backend/inspirehep/sitemap/utils.py b/backend/inspirehep/sitemap/utils.py index d6c8fe1a28..bbbc1ca977 100644 --- a/backend/inspirehep/sitemap/utils.py +++ b/backend/inspirehep/sitemap/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # diff --git a/backend/inspirehep/sitemap/views.py b/backend/inspirehep/sitemap/views.py index fa8127b921..b2d707f9f1 100644 --- a/backend/inspirehep/sitemap/views.py +++ b/backend/inspirehep/sitemap/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/snow/api.py b/backend/inspirehep/snow/api.py index ba8ecb4607..8e717b18ea 100644 --- a/backend/inspirehep/snow/api.py +++ b/backend/inspirehep/snow/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # @@ -16,11 +15,14 @@ from inspire_utils.record import get_value from invenio_cache import current_cache +from inspirehep.snow.errors import ( + CreateTicketException, + EditTicketException, + SnowAuthenticationError, +) +from inspirehep.snow.utils import get_response_result, strip_lines from inspirehep.utils import DistributedLockError, distributed_lock -from .errors import CreateTicketException, EditTicketException, SnowAuthenticationError -from .utils import get_response_result, strip_lines - LOGGER = structlog.getLogger() @@ -80,11 +82,11 @@ def write_token_to_cache(self, token): try: with distributed_lock("snow-token-lock", blocking=False): current_cache.set("snow-token-value", token) - except DistributedLockError: + except DistributedLockError as e: LOGGER.error( "Can't acquire snow token from cache, other process acquired the lock." ) - raise SnowAuthenticationError + raise SnowAuthenticationError from e def get_token(self): login_payload = { @@ -100,8 +102,8 @@ def get_token(self): response = requests.post(self.auth_url, data=login_payload, headers=headers) response.raise_for_status() return response.json()["access_token"] - except (requests.exceptions.RequestException, KeyError): - raise SnowAuthenticationError + except (requests.exceptions.RequestException, KeyError) as e: + raise SnowAuthenticationError from e @relogin_if_needed def create_ticket(self, endpoint, payload): @@ -402,9 +404,9 @@ def get_tickets_by_recid(self, recid, exclude_resolved=True): third_party_search_query_string = f"u_third_party_ticket_id={recid}" tickets_search_parameters = dict(u_third_party="INSPIRE") if exclude_resolved: - tickets_search_parameters[ - "u_current_task_state" - ] = f"{self.ticket_status_mapping['waiting']}^ORu_current_task_state={self.ticket_status_mapping['assigned']}^ORu_current_task_state={self.ticket_status_mapping['in progress']}" + tickets_search_parameters["u_current_task_state"] = ( + f"{self.ticket_status_mapping['waiting']}^ORu_current_task_state={self.ticket_status_mapping['assigned']}^ORu_current_task_state={self.ticket_status_mapping['in progress']}" + ) try: third_party_ticket = self.search( self.third_party_ticket_endpoint, third_party_search_query_string @@ -497,8 +499,8 @@ def create_inspire_ticket( ticket_id, str(recid), assignee=assignee_id ) return ticket_id - except requests.exceptions.RequestException: - raise CreateTicketException() + except requests.exceptions.RequestException as e: + raise CreateTicketException() from e def create_inspire_ticket_with_template( self, @@ -539,10 +541,7 @@ def resolve_ticket(self, ticket_id, user_email=None, message=None): user_email (str): Email of the user as which action should be performed. message (str): message to be added when resolving the ticket. """ - if user_email: - snow_user_id = self._get_user_by_email(user_email) - else: - snow_user_id = None + snow_user_id = self._get_user_by_email(user_email) if user_email else None try: unassign_payload = {"assigned_to": ""} @@ -557,11 +556,11 @@ def resolve_ticket(self, ticket_id, user_email=None, message=None): "comments": message or "Ticket has been closed", } self.edit_ticket(ticket_id, payload) - except requests.exceptions.RequestException: + except requests.exceptions.RequestException as e: # Raise exception only if ticket isn't already resolved ticket = self.get_ticket(ticket_id) if ticket["u_current_task_state"] != self.ticket_status_mapping["resolved"]: - raise EditTicketException() + raise EditTicketException() from e def resolve_ticket_with_template( self, ticket_id, user_email=None, template_path=None, template_context=None diff --git a/backend/inspirehep/snow/config.py b/backend/inspirehep/snow/config.py index 7afb89eda9..b092935706 100644 --- a/backend/inspirehep/snow/config.py +++ b/backend/inspirehep/snow/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # diff --git a/backend/inspirehep/snow/errors.py b/backend/inspirehep/snow/errors.py index d845ca1674..d594078f1d 100644 --- a/backend/inspirehep/snow/errors.py +++ b/backend/inspirehep/snow/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # diff --git a/backend/inspirehep/snow/utils.py b/backend/inspirehep/snow/utils.py index ab2a76883a..af153cdef5 100644 --- a/backend/inspirehep/snow/utils.py +++ b/backend/inspirehep/snow/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # diff --git a/backend/inspirehep/submissions/errors.py b/backend/inspirehep/submissions/errors.py index ce257f5d99..2ac47fb2ff 100644 --- a/backend/inspirehep/submissions/errors.py +++ b/backend/inspirehep/submissions/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -23,7 +22,7 @@ class LoaderDataError(RESTException): def __init__( self, description="Something went wrong while processing your data, please try again later.", - **kwargs + **kwargs, ): super().__init__(**kwargs) self.description = description diff --git a/backend/inspirehep/submissions/loaders.py b/backend/inspirehep/submissions/loaders.py index bc0c9c1084..8a15322df4 100644 --- a/backend/inspirehep/submissions/loaders.py +++ b/backend/inspirehep/submissions/loaders.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,16 +10,15 @@ from invenio_records_rest.loaders.marshmallow import MarshmallowErrors from inspirehep.submissions.errors import LoaderDataError, RESTDataError - -from .marshmallow import ( +from inspirehep.submissions.marshmallow import ( Author, Conference, Experiment, Institution, Job, + Journal, Literature, Seminar, - Journal, ) diff --git a/backend/inspirehep/submissions/marshmallow/author.py b/backend/inspirehep/submissions/marshmallow/author.py index 75a6393fc7..f6de69f369 100644 --- a/backend/inspirehep/submissions/marshmallow/author.py +++ b/backend/inspirehep/submissions/marshmallow/author.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/submissions/marshmallow/conferences.py b/backend/inspirehep/submissions/marshmallow/conferences.py index d817829f17..0cb725edef 100644 --- a/backend/inspirehep/submissions/marshmallow/conferences.py +++ b/backend/inspirehep/submissions/marshmallow/conferences.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/submissions/marshmallow/experiments.py b/backend/inspirehep/submissions/marshmallow/experiments.py index f3a570736f..80964b8b94 100644 --- a/backend/inspirehep/submissions/marshmallow/experiments.py +++ b/backend/inspirehep/submissions/marshmallow/experiments.py @@ -11,9 +11,7 @@ class Experiment(Schema): @post_load def build_experiment(self, data): - if get_value(data, "legacy_name") and get_value(data, "project_type"): - return { "_collections": ["Experiments"], "$schema": url_for( diff --git a/backend/inspirehep/submissions/marshmallow/institutions.py b/backend/inspirehep/submissions/marshmallow/institutions.py index 4ec6d3aaaf..d2c12958f1 100644 --- a/backend/inspirehep/submissions/marshmallow/institutions.py +++ b/backend/inspirehep/submissions/marshmallow/institutions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -20,9 +19,7 @@ class Institution(Schema): @post_load def build_institution(self, data): - if get_value(data, "legacy_ICN") and get_value(data, "ICN"): - return { "_collections": ["Institutions"], "$schema": url_for( diff --git a/backend/inspirehep/submissions/marshmallow/job.py b/backend/inspirehep/submissions/marshmallow/job.py index ff97b413d4..937550cb06 100644 --- a/backend/inspirehep/submissions/marshmallow/job.py +++ b/backend/inspirehep/submissions/marshmallow/job.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/submissions/marshmallow/journals.py b/backend/inspirehep/submissions/marshmallow/journals.py index 09204c4a8e..404904135c 100644 --- a/backend/inspirehep/submissions/marshmallow/journals.py +++ b/backend/inspirehep/submissions/marshmallow/journals.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2022 CERN. # diff --git a/backend/inspirehep/submissions/marshmallow/literature.py b/backend/inspirehep/submissions/marshmallow/literature.py index 3d9b8466c3..56b360cfa0 100644 --- a/backend/inspirehep/submissions/marshmallow/literature.py +++ b/backend/inspirehep/submissions/marshmallow/literature.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,6 +5,7 @@ # the terms of the MIT License; see LICENSE file for more details. """JSON Schemas.""" + from inspire_dojson.utils import get_recid_from_ref from inspire_schemas.builders.literature import LiteratureBuilder from inspire_schemas.utils import split_page_artid diff --git a/backend/inspirehep/submissions/marshmallow/seminars.py b/backend/inspirehep/submissions/marshmallow/seminars.py index fd4f8afe0f..be1c3634c7 100644 --- a/backend/inspirehep/submissions/marshmallow/seminars.py +++ b/backend/inspirehep/submissions/marshmallow/seminars.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -190,10 +189,10 @@ def build_seminar(self, data) -> dict: for literature_record_pid in data.get("literature_records", []): try: LiteratureRecord.get_record_by_pid_value(literature_record_pid) - except PIDDoesNotExistError: + except PIDDoesNotExistError as e: raise InvalidDataError( f"{literature_record_pid} is not a valid literature record." - ) + ) from e record = { "$ref": f"{get_inspirehep_url()}/api/literature/{literature_record_pid}" } diff --git a/backend/inspirehep/submissions/serializers.py b/backend/inspirehep/submissions/serializers.py index d0fbb71bd0..f1a6591473 100644 --- a/backend/inspirehep/submissions/serializers.py +++ b/backend/inspirehep/submissions/serializers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -9,17 +8,16 @@ from invenio_records_rest.serializers.json import JSONSerializer -from inspirehep.submissions.marshmallow.job import Job - -from .marshmallow import ( +from inspirehep.submissions.marshmallow import ( Author, Conference, Experiment, Institution, + Journal, Literature, Seminar, - Journal, ) +from inspirehep.submissions.marshmallow.job import Job literature_v1 = JSONSerializer(Literature) author_v1 = JSONSerializer(Author) diff --git a/backend/inspirehep/submissions/tasks.py b/backend/inspirehep/submissions/tasks.py index bec5410885..fae17becb7 100644 --- a/backend/inspirehep/submissions/tasks.py +++ b/backend/inspirehep/submissions/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/submissions/utils.py b/backend/inspirehep/submissions/utils.py index ff87ef9bc5..f55316590a 100644 --- a/backend/inspirehep/submissions/utils.py +++ b/backend/inspirehep/submissions/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/submissions/views.py b/backend/inspirehep/submissions/views.py index 8949f72c85..ce1194e5e0 100644 --- a/backend/inspirehep/submissions/views.py +++ b/backend/inspirehep/submissions/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -40,23 +39,21 @@ SeminarsRecord, ) from inspirehep.serializers import jsonify -from inspirehep.submissions.errors import RESTDataError +from inspirehep.submissions.errors import RESTDataError, WorkflowStartError +from inspirehep.submissions.loaders import author_v1 as author_loader_v1 +from inspirehep.submissions.loaders import conference_v1 as conference_loader_v1 +from inspirehep.submissions.loaders import experiment_v1 as experiment_loader_v1 +from inspirehep.submissions.loaders import institution_v1 as institution_loader_v1 +from inspirehep.submissions.loaders import job_v1 as job_loader_v1 +from inspirehep.submissions.loaders import journal_v1 as journal_loader_v1 +from inspirehep.submissions.loaders import literature_v1 as literature_loader_v1 +from inspirehep.submissions.loaders import seminar_v1 as seminar_loader_v1 +from inspirehep.submissions.serializers import author_v1, job_v1 +from inspirehep.submissions.serializers import seminar_v1 as seminar_serializer_v1 +from inspirehep.submissions.tasks import async_create_ticket_with_template +from inspirehep.submissions.utils import has_30_days_passed_after_deadline from inspirehep.utils import get_inspirehep_url -from .errors import WorkflowStartError -from .loaders import author_v1 as author_loader_v1 -from .loaders import conference_v1 as conference_loader_v1 -from .loaders import experiment_v1 as experiment_loader_v1 -from .loaders import institution_v1 as institution_loader_v1 -from .loaders import job_v1 as job_loader_v1 -from .loaders import journal_v1 as journal_loader_v1 -from .loaders import literature_v1 as literature_loader_v1 -from .loaders import seminar_v1 as seminar_loader_v1 -from .serializers import author_v1, job_v1 -from .serializers import seminar_v1 as seminar_serializer_v1 -from .tasks import async_create_ticket_with_template -from .utils import has_30_days_passed_after_deadline - blueprint = Blueprint("inspirehep_submissions", __name__, url_prefix="/submissions") LOGGER = structlog.getLogger() @@ -103,7 +100,7 @@ def send_post_request_to_workflows( ) try: response.raise_for_status() - except RequestException: + except RequestException as e: LOGGER.exception( "Error during workflow creation.", response=response.text, @@ -112,7 +109,7 @@ def send_post_request_to_workflows( endpoint=endpoint, bearer_keyword=bearer_keyword, ) - raise WorkflowStartError + raise WorkflowStartError from e LOGGER.info("Workflow creation successful", response=response.text) return response.json() @@ -312,7 +309,6 @@ def create_ticket(self, record, rt_template): class ExperimentSubmissionsResource(BaseSubmissionsResource): - decorators = [login_required_with_roles([Roles.cataloger.value])] def load_data_from_request(self): @@ -441,7 +437,6 @@ def start_workflow_for_submission(self, control_number=None): class JobSubmissionsResource(BaseSubmissionsResource): - data_loader_from_request = job_loader_v1 user_allowed_status_changes = { @@ -569,7 +564,7 @@ def get_valid_record_data_from_builder(self, builder): builder.validate_record() except ValidationError as e: LOGGER.exception("Cannot process job submission") - raise RESTDataError(e.args[0]) + raise RESTDataError(e.args[0]) from e except SchemaError as e: LOGGER.exception("Schema is broken") abort(500, str(e)) @@ -612,7 +607,6 @@ def create_ticket(self, record, rt_template): class InstitutionSubmissionsResource(BaseSubmissionsResource): - decorators = [login_required_with_roles([Roles.cataloger.value])] def load_data_from_request(self): @@ -629,7 +623,6 @@ def post(self): class JournalSubmissionsResource(BaseSubmissionsResource): - decorators = [login_required_with_roles([Roles.cataloger.value])] def load_data_from_request(self): diff --git a/backend/inspirehep/tools/errors.py b/backend/inspirehep/tools/errors.py index 220097bdfa..18fc83342d 100644 --- a/backend/inspirehep/tools/errors.py +++ b/backend/inspirehep/tools/errors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/inspirehep/tools/utils.py b/backend/inspirehep/tools/utils.py index 37fd763f33..3ff3fb88b6 100644 --- a/backend/inspirehep/tools/utils.py +++ b/backend/inspirehep/tools/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -55,9 +54,7 @@ def find_references(references, requested_format): keyword = "external_system_identifiers.value" elif re.search(r".*\:\d{4}\w\w\w?", ref): keyword = "texkey" - elif re.search(r".*\/\d{7}", ref): - keyword = "eprint" - elif re.search(r"\d{4}\.\d{4,5}", ref): + elif re.search(r".*\/\d{7}", ref) or re.search(r"\d{4}\.\d{4,5}", ref): keyword = "eprint" elif re.search(r"\w\.\w+\.\w", ref): keyword = "j" diff --git a/backend/inspirehep/tools/views.py b/backend/inspirehep/tools/views.py index d11de7c1a5..129887c2c3 100644 --- a/backend/inspirehep/tools/views.py +++ b/backend/inspirehep/tools/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -12,6 +11,11 @@ from inspirehep.files.api import current_s3_instance from inspirehep.serializers import jsonify +from inspirehep.tools.errors import ( + FileFormatNotSupportedError, + FileTooBigError, + NoReferencesFoundError, +) from inspirehep.tools.utils import ( find_references, get_filename, @@ -20,8 +24,6 @@ ) from inspirehep.utils import hash_data -from .errors import FileFormatNotSupportedError, FileTooBigError, NoReferencesFoundError - blueprint = Blueprint("inspirehep_tools", __name__, url_prefix="") ALLOWED_EXTENSIONS = {".tex"} diff --git a/backend/inspirehep/utils.py b/backend/inspirehep/utils.py index 428273dd7b..37eb1fdd72 100644 --- a/backend/inspirehep/utils.py +++ b/backend/inspirehep/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -22,9 +21,9 @@ def include_table_check(object, name, type_, *args, **kwargs): - if type_ == "table" and name in current_app.config.get("ALEMBIC_SKIP_TABLES"): - return False - return True + return not ( + type_ == "table" and name in current_app.config.get("ALEMBIC_SKIP_TABLES") + ) def get_inspirehep_url(): @@ -68,7 +67,7 @@ def chunker(iterable, max_chunk_size, min_num_chunks=0): def flatten_list(input_list): - if isinstance(input_list, (list, tuple)): + if isinstance(input_list, list | tuple): return [ element for innerList in input_list for element in flatten_list(innerList) ] @@ -153,7 +152,7 @@ def next_batch(iterator, batch_size): batch = [] try: - for idx in range(batch_size): + for _idx in range(batch_size): batch.append(next(iterator)) except StopIteration: pass diff --git a/backend/inspirehep/version.py b/backend/inspirehep/version.py index 36a9d990c3..d03c5c3e7b 100644 --- a/backend/inspirehep/version.py +++ b/backend/inspirehep/version.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,5 +10,4 @@ and parsed by ``setup.py``. """ - __version__ = "1.0.0" diff --git a/backend/inspirehep/views.py b/backend/inspirehep/views.py index 73bf532ec4..c3be97331f 100644 --- a/backend/inspirehep/views.py +++ b/backend/inspirehep/views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/inspirehep/wsgi.py b/backend/inspirehep/wsgi.py index 73578c98de..2dcd40a249 100644 --- a/backend/inspirehep/wsgi.py +++ b/backend/inspirehep/wsgi.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 46505ae5c1..9ca1063130 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -40,4 +39,4 @@ def es(appctx): """Setup all registered Elasticsearch indices.""" from invenio_search import current_search_client - yield current_search_client + return current_search_client diff --git a/backend/tests/helpers/cleanups.py b/backend/tests/helpers/cleanups.py index fe8e14d75c..1b726d55b9 100644 --- a/backend/tests/helpers/cleanups.py +++ b/backend/tests/helpers/cleanups.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,16 +5,15 @@ # the terms of the MIT License; see LICENSE file for more details. from itertools import chain -from opensearchpy import ConflictError, NotFoundError, RequestError -from opensearchpy.client.ingest import IngestClient from flask import current_app from helpers.utils import get_index_alias +from inspirehep.indexer.cli import _put_files_pipeline from invenio_search.errors import IndexAlreadyExistsError +from opensearchpy import ConflictError, NotFoundError, RequestError +from opensearchpy.client.ingest import IngestClient from pytest_invenio.fixtures import _search_create_indexes from sqlalchemy_utils import create_database, database_exists -from inspirehep.indexer.cli import _put_files_pipeline - def es_cleanup(es): """Removes all data from es indexes @@ -35,9 +33,7 @@ def es_cleanup(es): x["aliases"].keys() for x in es.indices.get_alias("*").values() ) existing_mappings = set(chain.from_iterable(existing_mappings_nested)) - required_mappings = { - get_index_alias(index) for index in current_search.mappings.keys() - } + required_mappings = {get_index_alias(index) for index in current_search.mappings} missing_mappings = required_mappings.difference(existing_mappings) try: if len(missing_mappings): diff --git a/backend/tests/helpers/compare.py b/backend/tests/helpers/compare.py index a1a15a1e05..4586ad12d7 100644 --- a/backend/tests/helpers/compare.py +++ b/backend/tests/helpers/compare.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/tests/helpers/factories/db/base.py b/backend/tests/helpers/factories/db/base.py index 6a39afd9ad..6863b89d8a 100644 --- a/backend/tests/helpers/factories/db/base.py +++ b/backend/tests/helpers/factories/db/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,7 +12,7 @@ @six.add_metaclass(abc.ABCMeta) -class TestBaseModel(object): +class TestBaseModel: model_class = None @classmethod diff --git a/backend/tests/helpers/factories/db/invenio_accounts.py b/backend/tests/helpers/factories/db/invenio_accounts.py index 5fd1e0d418..07f258cf36 100644 --- a/backend/tests/helpers/factories/db/invenio_accounts.py +++ b/backend/tests/helpers/factories/db/invenio_accounts.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,10 +5,9 @@ # the terms of the MIT License; see LICENSE file for more details. +from helpers.factories.db.base import TestBaseModel from invenio_accounts.models import User -from .base import TestBaseModel - class TestUser(TestBaseModel): """ @@ -29,5 +27,5 @@ class TestUser(TestBaseModel): @classmethod def create_from_kwargs(cls, **kwargs): instance = cls() - instance.user = super(TestUser, cls).create_from_kwargs(kwargs) + instance.user = super().create_from_kwargs(kwargs) return instance diff --git a/backend/tests/helpers/factories/db/invenio_oauthclient.py b/backend/tests/helpers/factories/db/invenio_oauthclient.py index 59e2db643c..2846766057 100644 --- a/backend/tests/helpers/factories/db/invenio_oauthclient.py +++ b/backend/tests/helpers/factories/db/invenio_oauthclient.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,14 +5,12 @@ # the terms of the MIT License; see LICENSE file for more details. +from helpers.factories.db.base import TestBaseModel +from helpers.factories.db.invenio_accounts import TestUser +from inspirehep.accounts.fixtures import generate_random_string from invenio_db import db from invenio_oauthclient.models import RemoteAccount, RemoteToken, UserIdentity -from inspirehep.accounts.fixtures import generate_random_string - -from .base import TestBaseModel -from .invenio_accounts import TestUser - class TestUserIdentity(TestBaseModel): """ @@ -48,9 +45,7 @@ def create_from_kwargs(cls, **kwargs): db.session.flush() updated_kwargs["id_user"] = instance.user.id - instance.user_identity = super(TestUserIdentity, cls).create_from_kwargs( - updated_kwargs - ) + instance.user_identity = super().create_from_kwargs(updated_kwargs) return instance @classmethod @@ -93,9 +88,7 @@ def create_from_kwargs(cls, **kwargs): db.session.flush() updated_kwargs["user_id"] = instance.user.id - instance.remote_account = super(TestRemoteAccount, cls).create_from_kwargs( - updated_kwargs - ) + instance.remote_account = super().create_from_kwargs(updated_kwargs) return instance @classmethod @@ -136,9 +129,7 @@ def create_from_kwargs(cls, **kwargs): db.session.flush() updated_kwargs["remote_account"] = instance.remote_account - instance.remote_token = super(TestRemoteToken, cls).create_from_kwargs( - updated_kwargs - ) + instance.remote_token = super().create_from_kwargs(updated_kwargs) return instance @classmethod diff --git a/backend/tests/helpers/factories/db/invenio_pidstore.py b/backend/tests/helpers/factories/db/invenio_pidstore.py index 9bfc39d70d..c1b3fad8bb 100644 --- a/backend/tests/helpers/factories/db/invenio_pidstore.py +++ b/backend/tests/helpers/factories/db/invenio_pidstore.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -10,10 +9,9 @@ import random import uuid +from helpers.factories.db.base import TestBaseModel from invenio_pidstore.models import PersistentIdentifier, PIDStatus -from .base import TestBaseModel - class TestPersistentIdentifier(TestBaseModel): """Create a PersistentIdentifier instance. @@ -44,8 +42,6 @@ def create_from_kwargs(cls, **kwargs): if not kwargs.pop("status", None): updated_kwargs["status"] = PIDStatus.REGISTERED - instance.persistent_identifier = super( - TestPersistentIdentifier, cls - ).create_from_kwargs(updated_kwargs) + instance.persistent_identifier = super().create_from_kwargs(updated_kwargs) return instance diff --git a/backend/tests/helpers/factories/db/invenio_records.py b/backend/tests/helpers/factories/db/invenio_records.py index b72433cf14..4f2d2b52c3 100644 --- a/backend/tests/helpers/factories/db/invenio_records.py +++ b/backend/tests/helpers/factories/db/invenio_records.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -13,14 +12,12 @@ import orjson import pkg_resources -from invenio_records.models import RecordMetadata -from invenio_search import current_search_client as es - +from helpers.factories.db.base import TestBaseModel +from helpers.factories.db.invenio_pidstore import TestPersistentIdentifier from inspirehep.accounts.fixtures import generate_random_string from inspirehep.records.api import InspireRecord - -from .base import TestBaseModel -from .invenio_pidstore import TestPersistentIdentifier +from invenio_records.models import RecordMetadata +from invenio_search import current_search_client as es USED_RECIDS = {} MAX_ES_INT = 2147483647 @@ -96,9 +93,7 @@ def create_from_kwargs( updated_kwargs["json"] = json_ - instance.record_metadata = super(TestRecordMetadata, cls).create_from_kwargs( - updated_kwargs - ) + instance.record_metadata = super().create_from_kwargs(updated_kwargs) if index_name: instance.es_index_result = es.index( @@ -110,7 +105,7 @@ def create_from_kwargs( TestPersistentIdentifier.create_from_kwargs( object_uuid=instance.record_metadata.id, pid_value=instance.record_metadata.json.get("control_number"), - **kwargs + **kwargs, ).persistent_identifier ) diff --git a/backend/tests/helpers/factories/models/base.py b/backend/tests/helpers/factories/models/base.py index 1c20610a5e..2fee0105d6 100644 --- a/backend/tests/helpers/factories/models/base.py +++ b/backend/tests/helpers/factories/models/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,6 @@ """INSPIRE module that adds more fun to the platform.""" - import factory diff --git a/backend/tests/helpers/factories/models/pidstore.py b/backend/tests/helpers/factories/models/pidstore.py index 956864c612..09d78ea685 100644 --- a/backend/tests/helpers/factories/models/pidstore.py +++ b/backend/tests/helpers/factories/models/pidstore.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,6 @@ """INSPIRE module that adds more fun to the platform.""" - from helpers.factories.models.base import BaseFactory from invenio_pidstore.models import PersistentIdentifier, PIDStatus diff --git a/backend/tests/helpers/factories/models/records.py b/backend/tests/helpers/factories/models/records.py index 4c4cfc0c20..2ab85b84c3 100644 --- a/backend/tests/helpers/factories/models/records.py +++ b/backend/tests/helpers/factories/models/records.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,6 @@ """INSPIRE module that adds more fun to the platform.""" - import factory from helpers.factories.models.base import BaseFactory from helpers.providers.faker import faker diff --git a/backend/tests/helpers/factories/models/user_access_token.py b/backend/tests/helpers/factories/models/user_access_token.py index c9591eb6c7..1d486b0d1c 100644 --- a/backend/tests/helpers/factories/models/user_access_token.py +++ b/backend/tests/helpers/factories/models/user_access_token.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -31,13 +30,13 @@ def _create( allow_push=None, token=None, *args, - **kwargs + **kwargs, ): ds = current_app.extensions["invenio-accounts"].datastore role = ds.find_or_create_role(role) user = ds.create_user( id=fake.random_number(digits=8, fix_len=True), - email=fake.email() if not email else email, + email=email if email else fake.email(), password=hash_password(fake.password()), active=True, roles=[role], @@ -66,10 +65,7 @@ class Meta: @classmethod def _create(cls, model_class, *args, **kwargs): - if "role" in kwargs: - user = UserFactory(role=kwargs["role"]) - else: - user = UserFactory() + user = UserFactory(role=kwargs["role"]) if "role" in kwargs else UserFactory() user = User.query.filter(User.email == user.email).one_or_none() token = Token.create_personal( fake.first_name() + " " + fake.last_name(), diff --git a/backend/tests/helpers/providers/faker.py b/backend/tests/helpers/providers/faker.py index fa39aec4f4..9dae3469ef 100644 --- a/backend/tests/helpers/providers/faker.py +++ b/backend/tests/helpers/providers/faker.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,6 @@ """INSPIRE module that adds more fun to the platform.""" - from faker import Factory from helpers.providers.record_provider import RecordProvider diff --git a/backend/tests/helpers/providers/record_provider.py b/backend/tests/helpers/providers/record_provider.py index 41d619d37d..72a8f36429 100644 --- a/backend/tests/helpers/providers/record_provider.py +++ b/backend/tests/helpers/providers/record_provider.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -30,17 +29,11 @@ def control_number(): @staticmethod def doi(): - return "10.{}/{}".format( - fake.random_number(digits=4, fix_len=True), - fake.random_number(digits=8, fix_len=True), - ) + return f"10.{fake.random_number(digits=4, fix_len=True)}/{fake.random_number(digits=8, fix_len=True)}" @staticmethod def arxiv(): - return "20{}.{}".format( - fake.random_number(digits=2, fix_len=True), - fake.random_number(digits=5, fix_len=True), - ) + return f"20{fake.random_number(digits=2, fix_len=True)}.{fake.random_number(digits=5, fix_len=True)}" @staticmethod def orcid(): @@ -196,7 +189,7 @@ def generate_special_pids(cls, key, pid_generator, count): if not isinstance(count, int) or count <= 0: return {} data = {key: []} - for i in range(count): + for _i in range(count): data[key].append({"value": pid_generator()}) return data @@ -209,19 +202,25 @@ def add_arxiv_eprints(cls, arxiv_eprints): def add_dois(cls, dois): return cls.generate_special_pids("dois", cls.doi, dois) - # flake8: noqa: C901 + # flake8: noqa C901 def record( self, record_type, data=None, with_control_number=False, - literature_citations=[], # TODO: call `literature_references` - data_citations=[], + literature_citations=None, # TODO: call `literature_references` + data_citations=None, skip_validation=False, - other_pids=[], + other_pids=None, arxiv_eprints=None, dois=None, ): + if other_pids is None: + other_pids = [] + if data_citations is None: + data_citations = [] + if literature_citations is None: + literature_citations = [] if record_type == "lit": record = self.hep_record() elif record_type == "aut": diff --git a/backend/tests/helpers/utils.py b/backend/tests/helpers/utils.py index 9c30e3bf86..7887c4e9f9 100644 --- a/backend/tests/helpers/utils.py +++ b/backend/tests/helpers/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -16,15 +15,14 @@ from helpers.factories.models.records import RecordMetadataFactory from helpers.factories.models.user_access_token import AccessTokenFactory, UserFactory from helpers.providers.faker import faker +from inspirehep.files import current_s3_instance +from inspirehep.records.api import InspireRecord, LiteratureRecord +from inspirehep.utils import get_inspirehep_url from invenio_db import db from invenio_search import current_search from invenio_search.utils import build_alias_name from tenacity import retry -from inspirehep.files import current_s3_instance -from inspirehep.records.api import InspireRecord, LiteratureRecord -from inspirehep.utils import get_inspirehep_url - SENSITIVE_RESPONSE_KEYS = ["user.name", "user.email"] ENABLED_USER_DATA = ["marcjanna.jedrych@cern.ch", "Marcjanna Jedrych"] @@ -94,7 +92,9 @@ def create_record(record_type, data=None, **kwargs): return record -def create_s3_file(bucket, key, data, metadata={}, **kwargs): +def create_s3_file(bucket, key, data, metadata=None, **kwargs): + if metadata is None: + metadata = {} current_s3_instance.client.put_object( Bucket=bucket, Key=key, Body=data, Metadata=metadata, **kwargs ) @@ -133,7 +133,7 @@ def logout(client): del sess["user_id"] -def orcid_app_cli_runner(): +def orcid_app_cli_runner(inspire_app): """Click CLI runner inside the Flask application for orcid tests.""" runner = CliRunner() obj = ScriptInfo(create_app=lambda: inspire_app) @@ -144,11 +144,13 @@ def orcid_app_cli_runner(): def generate_records( count=10, record_type=LiteratureRecord, - data={}, + data=None, skip_validation=False, with_control_number=True, ): - for i in range(count): + if data is None: + data = {} + for _i in range(count): record_data = faker.record( record_type.pid_type, data=data, diff --git a/backend/tests/integration-async/accounts/test_views.py b/backend/tests/integration-async/accounts/test_views.py index 1722198b60..84666d2e1e 100644 --- a/backend/tests/integration-async/accounts/test_views.py +++ b/backend/tests/integration-async/accounts/test_views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # diff --git a/backend/tests/integration-async/assign/test_tasks.py b/backend/tests/integration-async/assign/test_tasks.py index 7538afbc3a..076c7906c9 100644 --- a/backend/tests/integration-async/assign/test_tasks.py +++ b/backend/tests/integration-async/assign/test_tasks.py @@ -1,11 +1,10 @@ from helpers.utils import create_record, retry_test -from invenio_db import db -from invenio_search import current_search -from tenacity import stop_after_delay, wait_fixed - from inspirehep.assign.tasks import assign_papers from inspirehep.records.api import AuthorsRecord from inspirehep.search.api import LiteratureSearch +from invenio_db import db +from invenio_search import current_search +from tenacity import stop_after_delay, wait_fixed def test_assign_from_an_author_to_another(inspire_app, clean_celery_session): diff --git a/backend/tests/integration-async/assign/test_views.py b/backend/tests/integration-async/assign/test_views.py index 82fe267dfd..151434e51e 100644 --- a/backend/tests/integration-async/assign/test_views.py +++ b/backend/tests/integration-async/assign/test_views.py @@ -1,15 +1,14 @@ import orjson from helpers.providers.faker import faker from helpers.utils import create_user, retry_test -from invenio_accounts.testutils import login_user_via_session -from invenio_db import db -from invenio_search import current_search -from tenacity import stop_after_delay, wait_fixed - from inspirehep.accounts.roles import Roles from inspirehep.records.api import AuthorsRecord, LiteratureRecord from inspirehep.records.utils import get_author_by_recid from inspirehep.search.api import LiteratureSearch +from invenio_accounts.testutils import login_user_via_session +from invenio_db import db +from invenio_search import current_search +from tenacity import stop_after_delay, wait_fixed def test_assign_regression(inspire_app, datadir, override_config, clean_celery_session): diff --git a/backend/tests/integration-async/assign/test_views/2171912.json b/backend/tests/integration-async/assign/test_views/2171912.json index 9bf9a64828..2992228c96 100644 --- a/backend/tests/integration-async/assign/test_views/2171912.json +++ b/backend/tests/integration-async/assign/test_views/2171912.json @@ -35088,4 +35088,4 @@ "filename":"8bd36c1faeaddc281f8923665ad3f080" } ] - } \ No newline at end of file + } diff --git a/backend/tests/integration-async/cds/conftest.py b/backend/tests/integration-async/cds/conftest.py index 46abbd58a1..ab1e2ff092 100644 --- a/backend/tests/integration-async/cds/conftest.py +++ b/backend/tests/integration-async/cds/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -12,7 +11,7 @@ import requests_mock -@pytest.fixture(scope="function") +@pytest.fixture() def inspire_app_with_config_for_cds_sync(inspire_app, override_config): with override_config( FEATURE_FLAG_ENABLE_CDS_SYNC=True, CDS_SERVER_API="http://localhost:9876/api/" @@ -20,7 +19,7 @@ def inspire_app_with_config_for_cds_sync(inspire_app, override_config): yield inspire_app -@pytest.fixture(scope="function") +@pytest.fixture() def inspire_app_for_cds_sync(inspire_app_with_config_for_cds_sync): with requests_mock.Mocker() as mocker: mocker.get( diff --git a/backend/tests/integration-async/cds/test_api.py b/backend/tests/integration-async/cds/test_api.py index c71a90f8c0..ba1bf72442 100644 --- a/backend/tests/integration-async/cds/test_api.py +++ b/backend/tests/integration-async/cds/test_api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -7,11 +6,10 @@ import pytest import requests from helpers.providers.faker import faker -from invenio_db import db - from inspirehep.cds.api import sync_identifiers from inspirehep.cds.models import CDSRun, CDSRunStatus from inspirehep.records.api import LiteratureRecord +from invenio_db import db def test_cds_sync_one_record_happy_flow(inspire_app_for_cds_sync): diff --git a/backend/tests/integration-async/cds/test_cli.py b/backend/tests/integration-async/cds/test_cli.py index 63995b0ca9..deb88e1d73 100644 --- a/backend/tests/integration-async/cds/test_cli.py +++ b/backend/tests/integration-async/cds/test_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -8,10 +7,9 @@ import uuid from helpers.providers.faker import faker -from invenio_db import db - from inspirehep.cds.models import CDSRun, CDSRunStatus from inspirehep.records.api import LiteratureRecord +from invenio_db import db def test_cds_sync(inspire_app_for_cds_sync, cli): diff --git a/backend/tests/integration-async/cds/test_models.py b/backend/tests/integration-async/cds/test_models.py index 3d5cc18ca0..0678b77d1a 100644 --- a/backend/tests/integration-async/cds/test_models.py +++ b/backend/tests/integration-async/cds/test_models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -7,9 +6,8 @@ import datetime import uuid -from invenio_db import db - from inspirehep.cds.models import CDSRun, CDSRunStatus +from invenio_db import db def test_create_new_run(inspire_app): diff --git a/backend/tests/integration-async/conftest.py b/backend/tests/integration-async/conftest.py index b14994069b..8396ac7967 100755 --- a/backend/tests/integration-async/conftest.py +++ b/backend/tests/integration-async/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -14,14 +13,13 @@ from click.testing import CliRunner from flask.cli import ScriptInfo from helpers.cleanups import db_cleanup, es_cleanup -from invenio_search import current_search_client as es -from moto import mock_s3 -from redis import StrictRedis - from inspirehep.celery import CeleryTask from inspirehep.cli import cli as inspire_cli from inspirehep.factory import create_app as inspire_create_app from inspirehep.files.api.s3 import S3 +from invenio_search import current_search_client as es +from moto import mock_s3 +from redis import StrictRedis LOGGER = structlog.getLogger() @@ -36,9 +34,9 @@ def app(): app_config["CELERY_TASK_EAGER_PROPAGATES"] = False app_config["TESTING"] = True app_config["SEARCH_INDEX_PREFIX"] = "test-integration-async-" - app_config[ - "SQLALCHEMY_DATABASE_URI" - ] = "postgresql+psycopg2://inspirehep:inspirehep@localhost/test-inspirehep-async" + app_config["SQLALCHEMY_DATABASE_URI"] = ( + "postgresql+psycopg2://inspirehep:inspirehep@localhost/test-inspirehep-async" + ) app_config["FEATURE_FLAG_ENABLE_REDIRECTION_OF_PIDS"] = True app.wsgi_app.mounts["/api"].config.update(app_config) # We cannot have `api` app with the same SERVER_NAME @@ -64,8 +62,8 @@ def celery_worker_parameters(): } -@pytest.fixture(scope="function", autouse=True) -def clear_environment(app): +@pytest.fixture(autouse=True) +def _clear_environment(app): from invenio_db import db as db_ with app.app_context(): @@ -86,10 +84,10 @@ def celery_session_app(app, celery_session_app): celery_session_app.Task = CeleryTask celery_session_app.flask_app = app - yield celery_session_app + return celery_session_app -@pytest.fixture(scope="function") +@pytest.fixture() def clean_celery_session(celery_session_app, celery_session_worker): celery_session_app.control.purge() yield celery_session_worker @@ -101,16 +99,16 @@ def celery_config(): return {} -@pytest.fixture(scope="function") +@pytest.fixture() def cli(inspire_app): """Click CLI runner inside the Flask application.""" runner = CliRunner() obj = ScriptInfo(create_app=lambda: inspire_app) runner.invoke = partial(runner.invoke, inspire_cli, obj=obj) - yield runner + return runner -@pytest.fixture(scope="function") +@pytest.fixture() def redis(inspire_app): redis_url = inspire_app.config.get("CACHE_REDIS_URL") redis = StrictRedis.from_url(redis_url, decode_responses=True) @@ -120,7 +118,7 @@ def redis(inspire_app): redis.close() -@pytest.fixture(scope="function") +@pytest.fixture() def override_config(inspire_app): @contextmanager def _override_config(**kwargs): @@ -136,26 +134,27 @@ def _override_config(**kwargs): ): ... """ - with mock.patch.dict(inspire_app.config, kwargs), mock.patch.dict( - inspire_app.wsgi_app.mounts["/api"].config, kwargs + with ( + mock.patch.dict(inspire_app.config, kwargs), + mock.patch.dict(inspire_app.wsgi_app.mounts["/api"].config, kwargs), ): yield return _override_config -@pytest.fixture(scope="function") -def inspire_app(app, cache, clear_environment): - yield app +@pytest.fixture() +def inspire_app(app, cache): + return app -@pytest.fixture(scope="function") +@pytest.fixture() def enable_self_citations(inspire_app, override_config): with override_config(FEATURE_FLAG_ENABLE_SELF_CITATIONS=True): yield inspire_app -@pytest.fixture(scope="function") +@pytest.fixture() def enable_disambiguation(inspire_app, override_config): with override_config( FEATURE_FLAG_ENABLE_AUTHOR_DISAMBIGUATION=True, @@ -165,7 +164,7 @@ def enable_disambiguation(inspire_app, override_config): yield inspire_app -@pytest.fixture(scope="function") +@pytest.fixture() def enable_files(inspire_app, override_config): with override_config(FEATURE_FLAG_ENABLE_FILES=True): yield inspire_app diff --git a/backend/tests/integration-async/curation/test_curation_cli.py b/backend/tests/integration-async/curation/test_curation_cli.py index 4f5f5c19a0..8fd09250b1 100644 --- a/backend/tests/integration-async/curation/test_curation_cli.py +++ b/backend/tests/integration-async/curation/test_curation_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2022 CERN. # @@ -11,11 +10,10 @@ from helpers.providers.faker import faker from helpers.utils import retry_test from inspire_utils.record import get_values_for_schema -from invenio_db import db -from tenacity import stop_after_delay, wait_fixed - from inspirehep.records.api.literature import LiteratureRecord from inspirehep.search.api import LiteratureSearch +from invenio_db import db +from tenacity import stop_after_delay, wait_fixed @mock.patch( @@ -163,7 +161,6 @@ def assert_keywords_are_updated(): def test_update_pdg_keywords_in_record_without_any_pdg_keywords( inspire_app, clean_celery_session, cli ): - rec_without_pdg_keywords_on_pdg_list = LiteratureRecord.create( faker.record( "lit", diff --git a/backend/tests/integration-async/disambiguation/test_disambiguation_cli.py b/backend/tests/integration-async/disambiguation/test_disambiguation_cli.py index 0882fe41f6..94b7470ad4 100644 --- a/backend/tests/integration-async/disambiguation/test_disambiguation_cli.py +++ b/backend/tests/integration-async/disambiguation/test_disambiguation_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2022 CERN. # @@ -6,11 +5,10 @@ # the terms of the MIT License; see LICENSE file for more details. from helpers.utils import create_record, retry_test +from inspirehep.search.api import LiteratureSearch from invenio_db import db from tenacity import stop_after_delay, wait_fixed -from inspirehep.search.api import LiteratureSearch - def test_disambiguate_all(inspire_app, cli, clean_celery_session, override_config): with override_config( diff --git a/backend/tests/integration-async/disambiguation/test_disambiguation_tasks.py b/backend/tests/integration-async/disambiguation/test_disambiguation_tasks.py index 3545e73f81..db05b87825 100644 --- a/backend/tests/integration-async/disambiguation/test_disambiguation_tasks.py +++ b/backend/tests/integration-async/disambiguation/test_disambiguation_tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,17 +10,16 @@ from helpers.providers.faker import faker from helpers.utils import create_user, retry_test from inspire_utils.record import get_values_for_schema -from invenio_accounts.testutils import login_user_via_session -from invenio_db import db -from redis import StrictRedis -from tenacity import stop_after_delay, wait_fixed - from inspirehep.disambiguation.tasks import disambiguate_authors from inspirehep.editor.editor_soft_lock import EditorSoftLock from inspirehep.records.api import AuthorsRecord, InspireRecord from inspirehep.records.api.literature import LiteratureRecord from inspirehep.records.receivers import index_after_commit from inspirehep.search.api import AuthorsSearch, InspireSearch +from invenio_accounts.testutils import login_user_via_session +from invenio_db import db +from redis import StrictRedis +from tenacity import stop_after_delay, wait_fixed def test_disambiguation_runs_after_record_creation( @@ -103,7 +101,8 @@ def test_disambiguate_many_authors_runs_after_record_creation( def assert_authors_records_exist_in_es(): author_record_1_from_es = InspireSearch.get_record_data_from_es(author_record_1) author_record_2_from_es = InspireSearch.get_record_data_from_es(author_record_2) - assert author_record_1_from_es and author_record_2_from_es + assert author_record_1_from_es + assert author_record_2_from_es assert_authors_records_exist_in_es() @@ -433,7 +432,8 @@ def assert_lit_records_exist_in_es(): lit_record_2_from_es = InspireSearch.get_record_data_from_es( literature_record_2 ) - assert lit_record_1_from_es and lit_record_2_from_es + assert lit_record_1_from_es + assert lit_record_2_from_es assert_lit_records_exist_in_es() @@ -513,7 +513,8 @@ def assert_lit_records_exist_in_es(): lit_record_2_from_es = InspireSearch.get_record_data_from_es( literature_record_2 ) - assert lit_record_1_from_es and lit_record_2_from_es + assert lit_record_1_from_es + assert lit_record_2_from_es assert_lit_records_exist_in_es() @@ -603,7 +604,8 @@ def assert_lit_records_exist_in_es(): lit_record_2_from_es = InspireSearch.get_record_data_from_es( literature_record_2 ) - assert lit_record_1_from_es and lit_record_2_from_es + assert lit_record_1_from_es + assert lit_record_2_from_es assert_lit_records_exist_in_es() @@ -799,6 +801,7 @@ def assert_first_disambiguation_no_match(): lit_record.update(dict(lit_record)) db.session.commit() + # TODO I'm pretty sure this test is not testing what it should @retry_test(stop=stop_after_delay(30), wait=wait_fixed(2)) def assert_disambiguation_on_record_update(): literature_record_from_es = InspireSearch.get_record_data_from_es( @@ -898,23 +901,6 @@ def assert_first_disambiguation_no_match(): assert_first_disambiguation_no_match() - db.session.expire_all() - lit_record = InspireRecord.get_record(literature_record_3.id) - lit_record["authors"][0]["affiliations"] = [{"value": "CERN"}] - lit_record.update(dict(lit_record)) - db.session.commit() - - @retry_test(stop=stop_after_delay(90), wait=wait_fixed(5)) - def assert_disambiguation_on_record_update(): - literature_record_from_es = InspireSearch.get_record_data_from_es( - literature_record_3 - ) - literature_record_from_es["authors"][0]["record"] != lit_record["authors"][0][ - "record" - ] - - assert_disambiguation_on_record_update() - def test_disambiguation_on_record_update_unambiguous_match( inspire_app, clean_celery_session, enable_disambiguation @@ -993,8 +979,8 @@ def test_disambiguation_handle_deleted_records( try: db.session.commit() - except Exception: - assert False + except Exception as e: + raise AssertionError() from e def test_disambiguation_races_assign( @@ -1048,7 +1034,6 @@ def assert_disambiguation_on_record_update(): literature_record_from_es = InspireSearch.get_record_data_from_es( lit_record ) - assert ( literature_record_from_es["authors"][0]["record"] != author_record["self"] @@ -1303,7 +1288,7 @@ def assert_disambiguation_task(): in literature_record_from_es_authors[0]["record"]["$ref"] ) - assert "Davis Gross, Brian" == literature_record_from_es_authors[0]["full_name"] + assert literature_record_from_es_authors[0]["full_name"] == "Davis Gross, Brian" assert_disambiguation_task() @@ -1311,7 +1296,6 @@ def assert_disambiguation_task(): def test_author_disambiguation_manually_when_empty_authors( inspire_app, clean_celery_session, enable_disambiguation ): - data = faker.record("lit", with_control_number=True) record = LiteratureRecord.create(data) record_version = record.model.version_id @@ -1334,7 +1318,6 @@ def assert_disambiguation_task(): def test_editor_lock_is_created_when_disambiguation_runs( mocked_remove_lock, inspire_app, clean_celery_session ): - redis_url = current_app.config.get("CACHE_REDIS_URL") redis = StrictRedis.from_url(redis_url) author_data = faker.record("aut", with_control_number=True) @@ -1470,7 +1453,8 @@ def assert_lit_records_exist_in_es(): lit_record_2_from_es = InspireSearch.get_record_data_from_es( literature_record_2 ) - assert lit_record_1_from_es and lit_record_2_from_es + assert lit_record_1_from_es + assert lit_record_2_from_es assert_lit_records_exist_in_es() diff --git a/backend/tests/integration-async/editor/test_editor_api.py b/backend/tests/integration-async/editor/test_editor_api.py index cbf519cbd4..5dcb49768f 100644 --- a/backend/tests/integration-async/editor/test_editor_api.py +++ b/backend/tests/integration-async/editor/test_editor_api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -10,15 +9,14 @@ import pytest from celery.app.annotations import MapAnnotation, resolve_all from helpers.utils import create_record_async, create_user, logout, retry_test +from inspirehep.accounts.roles import Roles +from inspirehep.records.api import LiteratureRecord from invenio_accounts.testutils import login_user_via_session from invenio_db import db from tenacity import stop_after_delay, wait_fixed -from inspirehep.accounts.roles import Roles -from inspirehep.records.api import LiteratureRecord - -@pytest.fixture(scope="function") +@pytest.fixture() def record_with_two_revisions(inspire_app, clean_celery_session): record_data = { "$schema": "http://localhost:5000/schemas/records/hep.json", @@ -38,7 +36,7 @@ def record_with_two_revisions(inspire_app, clean_celery_session): return record["control_number"] -@pytest.fixture(scope="function") +@pytest.fixture() def hidden_record_with_two_revisions(inspire_app, clean_celery_session): record_data = { "$schema": "http://localhost:5000/schemas/records/hep.json", @@ -354,7 +352,6 @@ def test_editor_locks_are_passed_in_payload_when_another_user_editing( from inspirehep.disambiguation.tasks import disambiguate_authors with override_config(FEATURE_FLAG_ENABLE_HAL_PUSH=True): - celery_task_annotation = MapAnnotation({"countdown": 1.5}) user = create_user(role=Roles.cataloger.value) diff --git a/backend/tests/integration-async/hal/test_hal_tasks.py b/backend/tests/integration-async/hal/test_hal_tasks.py index 10d3f49571..4c7a23e360 100644 --- a/backend/tests/integration-async/hal/test_hal_tasks.py +++ b/backend/tests/integration-async/hal/test_hal_tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -7,11 +6,10 @@ import pytest from helpers.providers.faker import faker -from invenio_db import db -from sqlalchemy.orm.exc import StaleDataError - from inspirehep.hal.tasks import hal_push from inspirehep.records.api import LiteratureRecord +from invenio_db import db +from sqlalchemy.orm.exc import StaleDataError def test_hal_push_retry_on_stale_data_error(inspire_app): diff --git a/backend/tests/integration-async/matcher/test_cli.py b/backend/tests/integration-async/matcher/test_cli.py index 01ec52da0d..5e29f2ae03 100644 --- a/backend/tests/integration-async/matcher/test_cli.py +++ b/backend/tests/integration-async/matcher/test_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -7,11 +6,10 @@ from helpers.utils import create_record_async, es_search, retry_test from inspire_utils.record import get_value +from inspirehep.records.api import LiteratureRecord from invenio_search import current_search from tenacity import stop_after_delay, wait_fixed -from inspirehep.records.api import LiteratureRecord - def test_match_references(inspire_app, cli, clean_celery_session): cited_data = { diff --git a/backend/tests/integration-async/oai/test_views.py b/backend/tests/integration-async/oai/test_views.py index 1a9b59ae84..0727578416 100644 --- a/backend/tests/integration-async/oai/test_views.py +++ b/backend/tests/integration-async/oai/test_views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -11,13 +10,12 @@ from helpers.utils import es_search, retry_test from inspire_dojson.api import record2marcxml from inspire_utils.record import get_value +from inspirehep.records.api import LiteratureRecord from invenio_db import db from invenio_oaiserver.models import OAISet from invenio_search import current_search from tenacity import stop_after_delay, wait_fixed -from inspirehep.records.api import LiteratureRecord - RECORD_INDENT = " " diff --git a/backend/tests/integration-async/pidstore/minters/test_minter_cnum.py b/backend/tests/integration-async/pidstore/minters/test_minter_cnum.py index 638aea5ce1..90e4c115f1 100644 --- a/backend/tests/integration-async/pidstore/minters/test_minter_cnum.py +++ b/backend/tests/integration-async/pidstore/minters/test_minter_cnum.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,11 +6,10 @@ import pytest from helpers.providers.faker import faker -from invenio_db import db -from invenio_pidstore.models import PersistentIdentifier, PIDStatus - from inspirehep.pidstore.errors import CNUMChanged from inspirehep.records.api import ConferencesRecord +from invenio_db import db +from invenio_pidstore.models import PersistentIdentifier, PIDStatus def test_minter_update_conference_record_with_different_cnum_raises_error( diff --git a/backend/tests/integration-async/pidstore/minters/test_minter_texkey.py b/backend/tests/integration-async/pidstore/minters/test_minter_texkey.py index 4b61c95133..c9be204942 100644 --- a/backend/tests/integration-async/pidstore/minters/test_minter_texkey.py +++ b/backend/tests/integration-async/pidstore/minters/test_minter_texkey.py @@ -1,7 +1,6 @@ from helpers.providers.faker import faker -from invenio_db import db - from inspirehep.records.api import LiteratureRecord +from invenio_db import db def test_minter_saves_texkey_on_hidden_collection(inspire_app, override_config): diff --git a/backend/tests/integration-async/pidstore/providers/test_bai_provider.py b/backend/tests/integration-async/pidstore/providers/test_bai_provider.py index 38ca944a50..b9f3fc82c9 100644 --- a/backend/tests/integration-async/pidstore/providers/test_bai_provider.py +++ b/backend/tests/integration-async/pidstore/providers/test_bai_provider.py @@ -1,10 +1,9 @@ import mock from helpers.providers.faker import faker +from inspirehep.records.api import AuthorsRecord from invenio_db import db from invenio_pidstore.models import PersistentIdentifier -from inspirehep.records.api import AuthorsRecord - def test_session_still_works_after_integrity_error_on_bai_provider( inspire_app, override_config diff --git a/backend/tests/integration-async/pidstore/test_cli.py b/backend/tests/integration-async/pidstore/test_cli.py index 7943488504..0d95cb40d2 100644 --- a/backend/tests/integration-async/pidstore/test_cli.py +++ b/backend/tests/integration-async/pidstore/test_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -20,22 +19,24 @@ def test_fast_bai_minter_cli(inspire_app, cli, override_config): ) assert ( - 2 - == PersistentIdentifier.query.filter_by(pid_type="aut", status="R").count() + PersistentIdentifier.query.filter_by(pid_type="aut", status="R").count() + == 2 ) assert ( - 1 - == PersistentIdentifier.query.filter_by(pid_type="bai", status="R").count() + PersistentIdentifier.query.filter_by(pid_type="bai", status="R").count() + == 1 ) - with override_config(FEATURE_FLAG_ENABLE_BAI_CREATION=True, FEATURE_FLAG_ENABLE_BAI_PROVIDER=True): + with override_config( + FEATURE_FLAG_ENABLE_BAI_CREATION=True, FEATURE_FLAG_ENABLE_BAI_PROVIDER=True + ): cli.invoke(["inspire-pidstore", "fast-mint-new-bais", "--yes-i-know"]) assert ( - 2 - == PersistentIdentifier.query.filter_by(pid_type="aut", status="R").count() + PersistentIdentifier.query.filter_by(pid_type="aut", status="R").count() + == 2 ) assert ( - 2 - == PersistentIdentifier.query.filter_by(pid_type="bai", status="R").count() + PersistentIdentifier.query.filter_by(pid_type="bai", status="R").count() + == 2 ) diff --git a/backend/tests/integration-async/pidstore/test_models.py b/backend/tests/integration-async/pidstore/test_models.py index 0f165f132d..046567bd9f 100644 --- a/backend/tests/integration-async/pidstore/test_models.py +++ b/backend/tests/integration-async/pidstore/test_models.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,10 +6,9 @@ import uuid import pytest -from invenio_pidstore.models import PersistentIdentifier, PIDStatus - from inspirehep.pidstore.errors import PidRedirectionMissing from inspirehep.pidstore.models import InspireRedirect +from invenio_pidstore.models import PersistentIdentifier, PIDStatus def test_redirection_invalidates_correctly_pid_objects(inspire_app): diff --git a/backend/tests/integration-async/records/api/__init__.py b/backend/tests/integration-async/records/api/__init__.py index 71f4302bdb..ffa3e31145 100644 --- a/backend/tests/integration-async/records/api/__init__.py +++ b/backend/tests/integration-async/records/api/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/tests/integration-async/records/api/test_authors.py b/backend/tests/integration-async/records/api/test_authors.py index b5dadae4e2..3daef9a19e 100644 --- a/backend/tests/integration-async/records/api/test_authors.py +++ b/backend/tests/integration-async/records/api/test_authors.py @@ -1,7 +1,6 @@ from helpers.providers.faker import faker -from invenio_db import db - from inspirehep.records.api import AuthorsRecord +from invenio_db import db def test_get_linked_advisors_when_name_changes(inspire_app): diff --git a/backend/tests/integration-async/records/api/test_base.py b/backend/tests/integration-async/records/api/test_base.py index 6270ac5e5d..a84629906d 100644 --- a/backend/tests/integration-async/records/api/test_base.py +++ b/backend/tests/integration-async/records/api/test_base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,11 +7,10 @@ import pytest from helpers.providers.faker import faker +from inspirehep.records.api import InspireRecord, LiteratureRecord from invenio_db import db from jsonschema import ValidationError -from inspirehep.records.api import InspireRecord, LiteratureRecord - def test_record_versioning(inspire_app, clean_celery_session): data = { @@ -219,7 +217,7 @@ def test_more_than_one_update_works_correctly(inspire_app): # Revision 1 will be missing as there were 2 updates in one commit, # so revision number increased but revision itself was not created in DB rec.revert(1) - assert dict(rec) == expectet_rev_1 + assert dict(rec) != expectet_rev_1 rec.revert(0) assert dict(rec) == expectet_rev_0 diff --git a/backend/tests/integration-async/records/api/test_literature.py b/backend/tests/integration-async/records/api/test_literature.py index 7e18edbded..7086560435 100644 --- a/backend/tests/integration-async/records/api/test_literature.py +++ b/backend/tests/integration-async/records/api/test_literature.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,12 +7,11 @@ from helpers.providers.faker import faker from helpers.utils import retry_test -from invenio_db import db -from tenacity import stop_after_delay, wait_fixed - from inspirehep.records.api import ConferencesRecord, LiteratureRecord from inspirehep.records.api.base import InspireRecord from inspirehep.records.models import RecordsAuthors +from invenio_db import db +from tenacity import stop_after_delay, wait_fixed def test_authors_uuids_added_after_create_and_update(inspire_app, clean_celery_session): diff --git a/backend/tests/integration-async/records/indexer/__init__.py b/backend/tests/integration-async/records/indexer/__init__.py index 71f4302bdb..ffa3e31145 100644 --- a/backend/tests/integration-async/records/indexer/__init__.py +++ b/backend/tests/integration-async/records/indexer/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/tests/integration-async/records/indexer/test_authors.py b/backend/tests/integration-async/records/indexer/test_authors.py index b29b5c3451..478515001d 100755 --- a/backend/tests/integration-async/records/indexer/test_authors.py +++ b/backend/tests/integration-async/records/indexer/test_authors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -9,13 +8,12 @@ from helpers.providers.faker import faker from helpers.utils import es_search, retry_test from inspire_utils.record import get_value +from inspirehep.records.api import AuthorsRecord, LiteratureRecord +from inspirehep.search.api import AuthorsSearch, LiteratureSearch from invenio_db import db from invenio_search import current_search from tenacity import stop_after_delay, wait_fixed -from inspirehep.records.api import AuthorsRecord, LiteratureRecord -from inspirehep.search.api import AuthorsSearch, LiteratureSearch - def test_aut_record_appear_in_es_when_created(inspire_app, clean_celery_session): data = faker.record("aut") diff --git a/backend/tests/integration-async/records/indexer/test_conferences.py b/backend/tests/integration-async/records/indexer/test_conferences.py index 34b193be34..1f205d0a2d 100644 --- a/backend/tests/integration-async/records/indexer/test_conferences.py +++ b/backend/tests/integration-async/records/indexer/test_conferences.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -10,15 +9,14 @@ from helpers.providers.faker import faker from helpers.utils import es_search, retry_test from inspire_utils.record import get_value -from invenio_db import db -from invenio_search import current_search -from tenacity import stop_after_delay, wait_fixed - from inspirehep.records.api import ConferencesRecord, LiteratureRecord from inspirehep.records.marshmallow.conferences.common.proceeding_info_item import ( ProceedingInfoItemSchemaV1, ) from inspirehep.search.api import ConferencesSearch +from invenio_db import db +from invenio_search import current_search +from tenacity import stop_after_delay, wait_fixed def test_conference_record_updates_in_es_when_lit_rec_refers_to_it( diff --git a/backend/tests/integration-async/records/indexer/test_data.py b/backend/tests/integration-async/records/indexer/test_data.py index bc8ef19c0d..c81797277a 100644 --- a/backend/tests/integration-async/records/indexer/test_data.py +++ b/backend/tests/integration-async/records/indexer/test_data.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -7,13 +6,12 @@ from helpers.providers.faker import faker from helpers.utils import retry_test +from inspirehep.records.api import DataRecord +from inspirehep.search.api import DataSearch from invenio_db import db from invenio_search import current_search from tenacity import stop_after_delay, wait_fixed -from inspirehep.records.api import DataRecord -from inspirehep.search.api import DataSearch - def test_indexer_deletes_record_from_es(inspire_app): @retry_test(stop=stop_after_delay(30), wait=wait_fixed(2)) diff --git a/backend/tests/integration-async/records/indexer/test_experiments.py b/backend/tests/integration-async/records/indexer/test_experiments.py index a39e901657..b6a2885e4c 100644 --- a/backend/tests/integration-async/records/indexer/test_experiments.py +++ b/backend/tests/integration-async/records/indexer/test_experiments.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020-2021 CERN. # @@ -7,13 +6,12 @@ from helpers.providers.faker import faker from helpers.utils import retry_test -from invenio_db import db -from invenio_search import current_search -from tenacity import stop_after_delay, wait_fixed - from inspirehep.records.api import LiteratureRecord from inspirehep.records.api.experiments import ExperimentsRecord from inspirehep.search.api import ExperimentsSearch +from invenio_db import db +from invenio_search import current_search +from tenacity import stop_after_delay, wait_fixed def test_experiment_record_updates_in_es_when_lit_rec_refers_to_it( diff --git a/backend/tests/integration-async/records/indexer/test_fulltext.py b/backend/tests/integration-async/records/indexer/test_fulltext.py index e8d14054d5..0576acea1a 100644 --- a/backend/tests/integration-async/records/indexer/test_fulltext.py +++ b/backend/tests/integration-async/records/indexer/test_fulltext.py @@ -5,15 +5,14 @@ from flask_sqlalchemy import models_committed from helpers.providers.faker import faker from helpers.utils import create_s3_bucket, create_s3_file, retry_test -from invenio_db import db -from invenio_search import current_search -from tenacity import stop_after_delay, wait_fixed - from inspirehep.files.api import current_s3_instance from inspirehep.indexer.tasks import batch_index from inspirehep.records.api import LiteratureRecord from inspirehep.records.receivers import index_after_commit from inspirehep.search.api import LiteratureSearch +from invenio_db import db +from invenio_search import current_search +from tenacity import stop_after_delay, wait_fixed KEY = "b50c2ea2d26571e0c5a3411e320586289fd715c2" @@ -39,7 +38,6 @@ def assert_hits(): "filename": "arXiv:nucl-th_9310030.pdf", "url": "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", "text": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", - "text": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", } ], ) @@ -162,7 +160,7 @@ def assert_update_in_es(): record_lit_es = ( LiteratureSearch().get_record(str(record.id)).execute().hits.hits[0] ) - assert "new_doc.pdf" == record_lit_es._source["documents"][0]["key"] + assert record_lit_es._source["documents"][0]["key"] == "new_doc.pdf" assert ( record_first_attachment != record_lit_es._source["documents"][0]["attachment"] @@ -297,7 +295,8 @@ def assert_records_in_es(): lit_record_from_es_2 = LiteratureSearch.get_record_data_from_es( lit_record_2 ) - assert lit_record_from_es and lit_record_from_es_2 + assert lit_record_from_es + assert lit_record_from_es_2 assert_records_in_es() diff --git a/backend/tests/integration-async/records/indexer/test_indexer_cli.py b/backend/tests/integration-async/records/indexer/test_indexer_cli.py index 915f08ebd6..9f0ef0d0a5 100755 --- a/backend/tests/integration-async/records/indexer/test_indexer_cli.py +++ b/backend/tests/integration-async/records/indexer/test_indexer_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,9 +6,6 @@ import pytest from helpers.utils import generate_records -from invenio_records.models import RecordMetadata -from mock import patch - from inspirehep.indexer.cli import get_query_records_to_index from inspirehep.records.api import ( AuthorsRecord, @@ -21,6 +17,8 @@ JournalsRecord, LiteratureRecord, ) +from invenio_records.models import RecordMetadata +from mock import patch def check_n_records_reindex_for_pidtype( @@ -56,9 +54,11 @@ def test_reindex_record_lit_fails_with_invalid_record( inspire_app, clean_celery_session, cli ): broken_field = {"_desy_bookkeeping": {"date": '"2013-01-14_final'}} - with patch("inspirehep.indexer.base.InspireRecordIndexer"): - with patch("inspirehep.records.api.base.get_validation_errors"): - generate_records(count=1, data=broken_field, skip_validation=True) + with ( + patch("inspirehep.indexer.base.InspireRecordIndexer"), + patch("inspirehep.records.api.base.get_validation_errors"), + ): + generate_records(count=1, data=broken_field, skip_validation=True) check_n_records_reindex_for_pidtype(inspire_app, cli, "lit", n_fail=1) @@ -68,9 +68,11 @@ def test_reindex_record_lit_fails_with_invalid_field_content( ): invalid_field = {"keywords": ["17B05"]} - with patch("inspirehep.indexer.base.InspireRecordIndexer"): - with patch("inspirehep.records.api.base.get_validation_errors"): - generate_records(count=1, data=invalid_field, skip_validation=True) + with ( + patch("inspirehep.indexer.base.InspireRecordIndexer"), + patch("inspirehep.records.api.base.get_validation_errors"), + ): + generate_records(count=1, data=invalid_field, skip_validation=True) check_n_records_reindex_for_pidtype(inspire_app, cli, "lit", n_fail=1) @@ -81,9 +83,11 @@ def test_reindex_records_lit_one_fails_and_two_ok( invalid_field = {"keywords": ["17B05"]} generate_records(count=2) - with patch("inspirehep.indexer.base.InspireRecordIndexer"): - with patch("inspirehep.records.api.base.get_validation_errors"): - generate_records(count=1, data=invalid_field, skip_validation=True) + with ( + patch("inspirehep.indexer.base.InspireRecordIndexer"), + patch("inspirehep.records.api.base.get_validation_errors"), + ): + generate_records(count=1, data=invalid_field, skip_validation=True) check_n_records_reindex_for_pidtype(inspire_app, cli, "lit", n_success=2, n_fail=1) diff --git a/backend/tests/integration-async/records/indexer/test_institutions.py b/backend/tests/integration-async/records/indexer/test_institutions.py index 0e888d841b..cba052d5ed 100644 --- a/backend/tests/integration-async/records/indexer/test_institutions.py +++ b/backend/tests/integration-async/records/indexer/test_institutions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -7,13 +6,12 @@ import orjson from helpers.providers.faker import faker from helpers.utils import retry_test -from invenio_db import db -from invenio_search import current_search -from tenacity import stop_after_delay, wait_fixed - from inspirehep.records.api import LiteratureRecord from inspirehep.records.api.institutions import InstitutionsRecord from inspirehep.search.api import InstitutionsSearch +from invenio_db import db +from invenio_search import current_search +from tenacity import stop_after_delay, wait_fixed def test_institutions_record_updates_in_es_when_lit_rec_refers_to_it( diff --git a/backend/tests/integration-async/records/indexer/test_jobs.py b/backend/tests/integration-async/records/indexer/test_jobs.py index f7d850d009..9a0730a1c1 100644 --- a/backend/tests/integration-async/records/indexer/test_jobs.py +++ b/backend/tests/integration-async/records/indexer/test_jobs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -6,13 +5,12 @@ # the terms of the MIT License; see LICENSE file for more details. from helpers.providers.faker import faker from helpers.utils import retry_test +from inspirehep.records.api import JobsRecord +from inspirehep.search.api import JobsSearch from invenio_db import db from invenio_search import current_search from tenacity import stop_after_delay, wait_fixed -from inspirehep.records.api import JobsRecord -from inspirehep.search.api import JobsSearch - def test_indexer_deletes_record_from_es(inspire_app): @retry_test(stop=stop_after_delay(30), wait=wait_fixed(2)) diff --git a/backend/tests/integration-async/records/indexer/test_journals.py b/backend/tests/integration-async/records/indexer/test_journals.py index a8c0d5b005..c3ee03f0a8 100644 --- a/backend/tests/integration-async/records/indexer/test_journals.py +++ b/backend/tests/integration-async/records/indexer/test_journals.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -6,13 +5,12 @@ # the terms of the MIT License; see LICENSE file for more details. import orjson from helpers.utils import retry_test +from inspirehep.records.api import JournalsRecord +from inspirehep.search.api import JournalsSearch from invenio_db import db from invenio_search import current_search from tenacity import stop_after_delay, wait_fixed -from inspirehep.records.api import JournalsRecord -from inspirehep.search.api import JournalsSearch - def test_indexer_deletes_record_from_es(inspire_app, datadir): @retry_test(stop=stop_after_delay(30), wait=wait_fixed(2)) diff --git a/backend/tests/integration-async/records/indexer/test_literature.py b/backend/tests/integration-async/records/indexer/test_literature.py index a65f7e5036..8cdbf79c61 100755 --- a/backend/tests/integration-async/records/indexer/test_literature.py +++ b/backend/tests/integration-async/records/indexer/test_literature.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,15 +10,14 @@ from helpers.factories.models.user_access_token import AccessTokenFactory from helpers.providers.faker import faker from helpers.utils import retry_test -from invenio_db import db -from invenio_search import current_search -from sqlalchemy.orm.exc import StaleDataError -from tenacity import stop_after_delay, wait_fixed - from inspirehep.indexer.tasks import index_record from inspirehep.records.api import AuthorsRecord, InspireRecord, LiteratureRecord from inspirehep.records.receivers import index_after_commit from inspirehep.search.api import AuthorsSearch, LiteratureSearch +from invenio_db import db +from invenio_search import current_search +from sqlalchemy.orm.exc import StaleDataError +from tenacity import stop_after_delay, wait_fixed def assert_citation_count(cited_record, expected_count): @@ -260,7 +258,7 @@ def assert_record(): def test_many_records_in_one_commit(inspire_app, clean_celery_session): record_recids = set() - for x in range(4): + for _x in range(4): data = faker.record("lit") record = LiteratureRecord.create(data) record_recids.add(record["control_number"]) @@ -362,7 +360,7 @@ def assert_record(): def assert_record(): current_search.flush_and_refresh("records-hep") record_from_es = LiteratureSearch().get_record_data_from_es(citer) - assert ["Jean.L.Picard.1"] == record_from_es["referenced_authors_bais"] + assert record_from_es["referenced_authors_bais"] == ["Jean.L.Picard.1"] assert_record() @@ -375,7 +373,7 @@ def assert_record(): current_search.flush_and_refresh("records-hep") record = LiteratureRecord.get_record_by_pid_value(citer_control_number) record_from_es = LiteratureSearch().get_record_data_from_es(record) - assert ["J.Picard.2"] == record_from_es["referenced_authors_bais"] + assert record_from_es["referenced_authors_bais"] == ["J.Picard.2"] assert_record() diff --git a/backend/tests/integration-async/records/indexer/test_seminars.py b/backend/tests/integration-async/records/indexer/test_seminars.py index 3378fa3fa3..e9d3f20b6f 100644 --- a/backend/tests/integration-async/records/indexer/test_seminars.py +++ b/backend/tests/integration-async/records/indexer/test_seminars.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -6,13 +5,12 @@ # the terms of the MIT License; see LICENSE file for more details. import orjson from helpers.utils import retry_test +from inspirehep.records.api import SeminarsRecord +from inspirehep.search.api import SeminarsSearch from invenio_db import db from invenio_search import current_search from tenacity import stop_after_delay, wait_fixed -from inspirehep.records.api import SeminarsRecord -from inspirehep.search.api import SeminarsSearch - def test_indexer_deletes_record_from_es(inspire_app, datadir): @retry_test(stop=stop_after_delay(30), wait=wait_fixed(2)) diff --git a/backend/tests/integration-async/records/test_fulltext.py b/backend/tests/integration-async/records/test_fulltext.py index c3a4955dda..47d9264844 100644 --- a/backend/tests/integration-async/records/test_fulltext.py +++ b/backend/tests/integration-async/records/test_fulltext.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -12,13 +11,12 @@ from flask_sqlalchemy import models_committed from helpers.providers.faker import faker from helpers.utils import retry_test -from invenio_db import db -from invenio_search import current_search -from tenacity import stop_after_delay, wait_fixed - from inspirehep.records.api import LiteratureRecord from inspirehep.records.receivers import index_after_commit from inspirehep.search.api import LiteratureSearch +from invenio_db import db +from invenio_search import current_search +from tenacity import stop_after_delay, wait_fixed @mock.patch( @@ -31,7 +29,6 @@ "filename": "arXiv:nucl-th_9310030.pdf", "url": "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", "text": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", - "text": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=", } ], ) diff --git a/backend/tests/integration-async/records/test_record_views.py b/backend/tests/integration-async/records/test_record_views.py index a91ec9f3df..ade75ab9ff 100644 --- a/backend/tests/integration-async/records/test_record_views.py +++ b/backend/tests/integration-async/records/test_record_views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # @@ -9,14 +8,13 @@ import orjson from helpers.providers.faker import faker from helpers.utils import create_user, retry_test +from inspirehep.records.api import InspireRecord +from inspirehep.search.api import InspireSearch from invenio_accounts.testutils import login_user_via_session from invenio_db import db from sqlalchemy.orm.exc import StaleDataError from tenacity import stop_after_delay, wait_fixed -from inspirehep.records.api import InspireRecord -from inspirehep.search.api import InspireSearch - def test_reference_diff(inspire_app, clean_celery_session): user = create_user(role="cataloger") @@ -362,17 +360,19 @@ def assert_record_updated(): assert_record_updated() new_record_revision = record.revision_id - with inspire_app.test_client() as client: - with mock.patch( + with ( + inspire_app.test_client() as client, + mock.patch( "inspirehep.records.views.InspireRecord.get_record", side_effect=StaleDataError, - ): - login_user_via_session(client, email=user.email) - response = client.get( - f"/api/literature/{record['control_number']}/diff/{old_record_revision}..{new_record_revision}", - ) - assert response.status_code == 400 - assert response.json["message"] == "Record in given revision was not found" + ), + ): + login_user_via_session(client, email=user.email) + response = client.get( + f"/api/literature/{record['control_number']}/diff/{old_record_revision}..{new_record_revision}", + ) + assert response.status_code == 400 + assert response.json["message"] == "Record in given revision was not found" def test_reference_diff_when_user_not_authenticated(inspire_app, clean_celery_session): diff --git a/backend/tests/integration-async/records/test_records_cli.py b/backend/tests/integration-async/records/test_records_cli.py index 5836bd2e20..2efc9fe010 100644 --- a/backend/tests/integration-async/records/test_records_cli.py +++ b/backend/tests/integration-async/records/test_records_cli.py @@ -1,12 +1,11 @@ from helpers.providers.faker import faker from helpers.utils import retry_test -from invenio_db import db -from invenio_search import current_search -from tenacity import stop_after_delay, wait_fixed - from inspirehep.records.api import InspireRecord from inspirehep.records.models import JournalLiterature from inspirehep.search.api import JournalsSearch, LiteratureSearch +from invenio_db import db +from invenio_search import current_search +from tenacity import stop_after_delay, wait_fixed def test_populate_journal_literature_table(inspire_app, cli, clean_celery_session): @@ -27,7 +26,8 @@ def assert_records_added(): record_jou_es = ( JournalsSearch().get_record(str(journal.id)).execute().hits.hits[0] ) - assert record_lit_es and record_jou_es + assert record_lit_es + assert record_jou_es assert_records_added() diff --git a/backend/tests/integration-async/records/test_tasks.py b/backend/tests/integration-async/records/test_tasks.py index 3c432e5fa7..c930c8e959 100644 --- a/backend/tests/integration-async/records/test_tasks.py +++ b/backend/tests/integration-async/records/test_tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -9,11 +8,10 @@ from helpers.providers.faker import faker from helpers.utils import retry_test from inspire_utils.record import get_value -from invenio_db import db -from tenacity import stop_after_delay, wait_fixed - from inspirehep.records.api import InspireRecord from inspirehep.search.api import InspireSearch, LiteratureSearch +from invenio_db import db +from tenacity import stop_after_delay, wait_fixed def test_recalculate_references_after_literature_record_merge( @@ -571,7 +569,8 @@ def test_recalculate_references_after_journal_record_merge( def assert_all_records_in_es(): literature_record_from_es = InspireSearch.get_record_data_from_es(literature) journal_record_from_es = InspireSearch.get_record_data_from_es(journal) - assert literature_record_from_es and journal_record_from_es + assert literature_record_from_es + assert journal_record_from_es assert_all_records_in_es() @@ -618,7 +617,8 @@ def test_recalculate_references_after_conference_record_merge( def assert_all_records_in_es(): literature_record_from_es = InspireSearch.get_record_data_from_es(literature) conference_record_from_es = InspireSearch.get_record_data_from_es(conference) - assert literature_record_from_es and conference_record_from_es + assert literature_record_from_es + assert conference_record_from_es assert_all_records_in_es() @@ -654,7 +654,7 @@ def test_recalculate_references_recalculates_more_than_10_references( literature_data.update( {"publication_info": [{"journal_record": {"$ref": journal_record_reference}}]} ) - for i in range(11): + for _i in range(11): InspireRecord.create(literature_data) db.session.commit() @@ -670,7 +670,8 @@ def assert_all_records_in_es(): ) journal_record_from_es = InspireSearch.get_record_data_from_es(journal) - assert len(literature_records_from_es) == 11 and journal_record_from_es + assert len(literature_records_from_es) == 11 + assert journal_record_from_es assert_all_records_in_es() diff --git a/backend/tests/integration/__init__.py b/backend/tests/integration/__init__.py index d764bdcdea..5724951e13 100644 --- a/backend/tests/integration/__init__.py +++ b/backend/tests/integration/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/tests/integration/accounts/__init__.py b/backend/tests/integration/accounts/__init__.py index d764bdcdea..5724951e13 100644 --- a/backend/tests/integration/accounts/__init__.py +++ b/backend/tests/integration/accounts/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/tests/integration/accounts/test_account_handlers.py b/backend/tests/integration/accounts/test_account_handlers.py index 7a0d6779eb..e117924589 100644 --- a/backend/tests/integration/accounts/test_account_handlers.py +++ b/backend/tests/integration/accounts/test_account_handlers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -7,7 +6,6 @@ import mock from helpers.utils import create_record, create_user - from inspirehep.accounts.handlers import get_current_user_data diff --git a/backend/tests/integration/accounts/test_cli.py b/backend/tests/integration/accounts/test_cli.py index 85f54498b4..0dd2d91f0a 100644 --- a/backend/tests/integration/accounts/test_cli.py +++ b/backend/tests/integration/accounts/test_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # diff --git a/backend/tests/integration/accounts/test_decorators.py b/backend/tests/integration/accounts/test_decorators.py index a1b3cb30aa..360cae0573 100644 --- a/backend/tests/integration/accounts/test_decorators.py +++ b/backend/tests/integration/accounts/test_decorators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,11 +7,10 @@ """INSPIRE module that adds more fun to the platform.""" import pytest -from mock import Mock, patch -from werkzeug.exceptions import Forbidden, Unauthorized - from inspirehep.accounts.decorators import login_required_with_roles from inspirehep.accounts.roles import Roles +from mock import Mock, patch +from werkzeug.exceptions import Forbidden, Unauthorized class MockUserWithRoleA: @@ -51,9 +49,12 @@ def test_login_required_with_roles_without_roles(mock_is_authenticated, inspire_ def test_login_required_with_roles_unauthenticated(mock_is_authenticated, inspire_app): func = Mock() decorated_func = login_required_with_roles()(func) - with pytest.raises(Unauthorized): + with pytest.raises( + Unauthorized, + match="401 Unauthorized: The server could not verify that you are authorized to access the URL requested.", + ): decorated_func() - assert func.called + assert not func.called @patch( @@ -64,9 +65,12 @@ def test_login_required_with_roles_unauthenticated(mock_is_authenticated, inspir def test_login_required_with_roles_unauthorized(mock_is_authenticated, inspire_app): func = Mock() decorated_func = login_required_with_roles(["role_a"])(func) - with pytest.raises(Forbidden): + with pytest.raises( + Forbidden, + match="403 Forbidden: You don't have the permission to access the requested resource.", + ): decorated_func() - assert func.called + assert not func.called @patch( diff --git a/backend/tests/integration/accounts/test_views.py b/backend/tests/integration/accounts/test_views.py index 083877b47a..cf54d49ba0 100644 --- a/backend/tests/integration/accounts/test_views.py +++ b/backend/tests/integration/accounts/test_views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/tests/integration/alembic/conftest.py b/backend/tests/integration/alembic/conftest.py index a48452fb66..f2941a6a7d 100644 --- a/backend/tests/integration/alembic/conftest.py +++ b/backend/tests/integration/alembic/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/tests/integration/alembic/test_alembic.py b/backend/tests/integration/alembic/test_alembic.py index b5ba9b4f72..52f5a42482 100644 --- a/backend/tests/integration/alembic/test_alembic.py +++ b/backend/tests/integration/alembic/test_alembic.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -405,7 +404,7 @@ def _check_column_in_table(table, column): def get_primary_keys(tablename): insp = reflection.Inspector.from_engine(db.engine) - primary_keys = insp.get_pk_constraint(tablename)['constrained_columns'] + primary_keys = insp.get_pk_constraint(tablename)["constrained_columns"] return primary_keys diff --git a/backend/tests/integration/assign/test_tasks.py b/backend/tests/integration/assign/test_tasks.py index 564326fbd6..c291e45e42 100644 --- a/backend/tests/integration/assign/test_tasks.py +++ b/backend/tests/integration/assign/test_tasks.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # # inspirehep is free software; you can redistribute it and/or modify it under # the terms of the MIT License; see LICENSE file for more details. from helpers.utils import create_record - from inspirehep.assign.tasks import ( assign_conference, assign_paper_to_conference, diff --git a/backend/tests/integration/assign/test_views.py b/backend/tests/integration/assign/test_views.py index 0862bb81af..807e398780 100644 --- a/backend/tests/integration/assign/test_views.py +++ b/backend/tests/integration/assign/test_views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -8,11 +7,10 @@ import mock import orjson from helpers.utils import create_record, create_user -from invenio_accounts.testutils import login_user_via_session -from sqlalchemy.exc import ResourceClosedError - from inspirehep.accounts.roles import Roles from inspirehep.records.api import AuthorsRecord, LiteratureRecord +from invenio_accounts.testutils import login_user_via_session +from sqlalchemy.exc import ResourceClosedError def test_assign_without_login(inspire_app): @@ -469,8 +467,8 @@ def test_assign_doesnt_raise_resource_closed_error(inspire_app, override_config) ), content_type="application/json", ) - except ResourceClosedError: - assert False + except ResourceClosedError as e: + raise AssertionError() from e @mock.patch("inspirehep.assign.tasks.async_create_ticket_with_template") diff --git a/backend/tests/integration/cds/test_api.py b/backend/tests/integration/cds/test_api.py index a00f6c418f..00e51efa71 100644 --- a/backend/tests/integration/cds/test_api.py +++ b/backend/tests/integration/cds/test_api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -6,7 +5,6 @@ # the terms of the MIT License; see LICENSE file for more details. import pytest from helpers.utils import create_record - from inspirehep.cds.api import ( get_record_for_pid_or_none, get_record_for_provided_ids, @@ -15,7 +13,7 @@ @pytest.mark.parametrize( - "data, pid_value, pid_type", + ("data", "pid_value", "pid_type"), [ ( {}, @@ -46,7 +44,7 @@ def test_get_record_for_pid_or_none_control_number_record_exists( @pytest.mark.parametrize( - "data, pid_value, pid_type", + ("data", "pid_value", "pid_type"), [ ( {}, @@ -110,7 +108,7 @@ def test_query_missing_report_number(inspire_app): @pytest.mark.parametrize( - "data, arxivs, dois, report_numbers", + ("data", "arxivs", "dois", "report_numbers"), [ ( {}, diff --git a/backend/tests/integration/config/test_views_suggesters.py b/backend/tests/integration/config/test_views_suggesters.py index ab7abcede0..b5ec2c5366 100644 --- a/backend/tests/integration/config/test_views_suggesters.py +++ b/backend/tests/integration/config/test_views_suggesters.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -90,17 +89,17 @@ def test_author_suggesters(inspire_app): assert result_rec_id == expected_rec_id -@pytest.mark.xfail +@pytest.mark.xfail() def test_jobs_suggester(): raise NotImplementedError("Missing serializer") -@pytest.mark.xfail +@pytest.mark.xfail() def test_journals_suggesters(): raise NotImplementedError("Missing serializer") -@pytest.mark.xfail +@pytest.mark.xfail() def test_experiments_suggesters(): raise NotImplementedError("Missing serializer") @@ -192,11 +191,11 @@ def test_seminars_series_name_suggester_ignores_duplicates(inspire_app): assert suggestion_count == 1 -@pytest.mark.xfail +@pytest.mark.xfail() def test_data_suggesters(): raise NotImplementedError("Missing serializer") -@pytest.mark.xfail +@pytest.mark.xfail() def test_institutions_suggesters(): raise NotImplementedError("Missing serializer") diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py index 9cd3f04920..028dc36d97 100644 --- a/backend/tests/integration/conftest.py +++ b/backend/tests/integration/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,6 +5,7 @@ # the terms of the MIT License; see LICENSE file for more details. """INSPIRE module that adds more fun to the platform.""" + from contextlib import contextmanager from functools import partial @@ -19,23 +19,22 @@ from helpers.factories.models.base import BaseFactory from helpers.factories.models.pidstore import PersistentIdentifierFactory from helpers.factories.models.records import RecordMetadataFactory -from invenio_cache import current_cache -from moto import mock_s3 -from redis import StrictRedis - from inspirehep.cli import cli as inspire_cli from inspirehep.factory import create_app as inspire_create_app from inspirehep.files.api.s3 import S3 +from invenio_cache import current_cache +from moto import mock_s3 +from redis import StrictRedis @pytest.fixture(scope="module") -def instance_path(): +def _instance_path(): """Override pytest-invenio fixture creating a temp dir.""" return @pytest.fixture(scope="module") -def app_config(instance_path, app_config): +def app_config(app_config): # add extra global config if you would like to customize the config # for a specific test you can change create fixture per-directory # using ``conftest.py`` or per-file. @@ -43,39 +42,39 @@ def app_config(instance_path, app_config): app_config["JSONSCHEMAS_HOST"] = "localhost:5000" app_config["SERVER_NAME"] = "localhost:5000" app_config["SEARCH_INDEX_PREFIX"] = "test-integration-" - app_config[ - "SQLALCHEMY_DATABASE_URI" - ] = "postgresql+psycopg2://inspirehep:inspirehep@localhost/test-inspirehep" + app_config["SQLALCHEMY_DATABASE_URI"] = ( + "postgresql+psycopg2://inspirehep:inspirehep@localhost/test-inspirehep" + ) app_config["FEATURE_FLAG_ENABLE_REDIRECTION_OF_PIDS"] = True app_config["FILES_MAX_UPLOAD_THREADS"] = 1 return app_config -@pytest.fixture(scope="function") +@pytest.fixture() def enable_files(inspire_app, override_config): with override_config(FEATURE_FLAG_ENABLE_FILES=True): yield inspire_app -@pytest.fixture(scope="function") +@pytest.fixture() def disable_files(inspire_app, override_config): with override_config(FEATURE_FLAG_ENABLE_FILES=False): yield inspire_app -@pytest.fixture(scope="function") +@pytest.fixture() def enable_self_citations(inspire_app, override_config): with override_config(FEATURE_FLAG_ENABLE_SELF_CITATIONS=True): yield inspire_app -@pytest.fixture(scope="function") +@pytest.fixture() def enable_hal_push(inspire_app, override_config): with override_config(FEATURE_FLAG_ENABLE_HAL_PUSH=True): yield inspire_app -@pytest.fixture(scope="function") +@pytest.fixture() def disable_hal_push(inspire_app, override_config): with override_config(FEATURE_FLAG_ENABLE_HAL_PUSH=False): yield inspire_app @@ -96,7 +95,7 @@ def database(appctx): db_.session.remove() -@pytest.fixture(scope="function") +@pytest.fixture() def db_(database): """Creates a new database session for a test. Scope: function @@ -139,27 +138,27 @@ def restart_savepoint(sess, trans): database.session = old_session -@pytest.fixture(scope="function") +@pytest.fixture() def db(db_): - yield db_ + return db_ -@pytest.fixture(scope="function") +@pytest.fixture() def es_clear(es): es_cleanup(es) - yield es + return es -@pytest.fixture(scope="function") +@pytest.fixture() def cli(inspire_app): """Click CLI runner inside the Flask application.""" runner = CliRunner() obj = ScriptInfo(create_app=lambda: inspire_app) runner.invoke = partial(runner.invoke, inspire_cli, obj=obj) - yield runner + return runner -@pytest.fixture(scope="function") +@pytest.fixture() def redis(inspire_app): redis_url = inspire_app.config.get("CACHE_REDIS_URL") redis = StrictRedis.from_url(redis_url, decode_responses=True) @@ -169,14 +168,14 @@ def redis(inspire_app): redis.close() -@pytest.fixture(scope="function") +@pytest.fixture() def inspire_app(base_app, db, es_clear, vcr_config): # Make sure the API app has the same config base_app.wsgi_app.mounts["/api"].config.update(base_app.config) - yield base_app + return base_app -@pytest.fixture(scope="function") +@pytest.fixture() def override_config(inspire_app): @contextmanager def _override_config(**kwargs): @@ -192,8 +191,9 @@ def _override_config(**kwargs): ): ... """ - with mock.patch.dict(inspire_app.config, kwargs), mock.patch.dict( - inspire_app.wsgi_app.mounts["/api"].config, kwargs + with ( + mock.patch.dict(inspire_app.config, kwargs), + mock.patch.dict(inspire_app.wsgi_app.mounts["/api"].config, kwargs), ): yield @@ -219,8 +219,8 @@ class MockedInspireS3: inspire_app.extensions["inspirehep-s3"] = real_inspirehep_s3 -@pytest.fixture(scope="function") -def mocked_inspire_snow(mocker): +@pytest.fixture() +def _mocked_inspire_snow(mocker): # If SNOW_AUTH_URL (and SNOW_CLIENT_ID, SNOW_CLIENT_SECRET) is set, we dont need to mock the token if not current_app.config.get("SNOW_AUTH_URL"): mocker.patch( @@ -234,8 +234,8 @@ def mocked_inspire_snow(mocker): mocker.patch("inspirehep.snow.api.InspireSnow.get_token", return_value="abcd") -@pytest.fixture(scope="function") -def teardown_cache(): +@pytest.fixture() +def _teardown_cache(): yield current_cache.delete("snow_users") current_cache.delete("snow_functional_categories") diff --git a/backend/tests/integration/curation/test_api.py b/backend/tests/integration/curation/test_api.py index f1f0d2c435..7c6d4f3a57 100644 --- a/backend/tests/integration/curation/test_api.py +++ b/backend/tests/integration/curation/test_api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2023 CERN. # @@ -11,7 +10,6 @@ import orjson import pytest from helpers.utils import create_record - from inspirehep.curation.api import ( assign_institution_reference_to_affiliations, normalize_affiliations, @@ -26,16 +24,16 @@ def create_records_from_datadir(datadir, record_type, path_in_datadir): create_record(record_type, data=data) -@pytest.fixture(scope="function") -def insert_experiments_into_db(inspire_app, datadir): +@pytest.fixture() +def _insert_experiments_into_db(inspire_app, datadir): experiments_path = os.path.join(datadir, "experiments") for record_filename in os.listdir(experiments_path): data = orjson.loads((datadir / "experiments" / record_filename).read_text()) create_record("exp", data=data) -@pytest.fixture(scope="function") -def insert_ambiguous_experiments_into_db(inspire_app, datadir): +@pytest.fixture() +def _insert_ambiguous_experiments_into_db(inspire_app, datadir): ambiguous_experiments_path = os.path.join(datadir, "ambiguous_experiments") for record_filename in os.listdir(ambiguous_experiments_path): data = orjson.loads( @@ -44,17 +42,18 @@ def insert_ambiguous_experiments_into_db(inspire_app, datadir): create_record("exp", data=data) -@pytest.fixture(scope="function") -def insert_literature_in_db(inspire_app, datadir): +@pytest.fixture() +def _insert_literature_in_db(inspire_app, datadir): create_records_from_datadir(datadir, "lit", "literature") -@pytest.fixture(scope="function") -def insert_institutions_in_db(inspire_app, datadir): +@pytest.fixture() +def _insert_institutions_in_db(inspire_app, datadir): create_records_from_datadir(datadir, "ins", "institutions") -def test_normalize_collaborations(inspire_app, insert_experiments_into_db): +@pytest.mark.usefixtures("_insert_experiments_into_db") +def test_normalize_collaborations(inspire_app): record = { "_collections": ["Literature"], "titles": ["A title"], @@ -91,8 +90,9 @@ def test_normalize_collaborations(inspire_app, insert_experiments_into_db): assert result["accelerator_experiments"] == expected_accelerator_experiments +@pytest.mark.usefixtures("_insert_experiments_into_db") def test_normalize_collaborations_with_different_name_variants( - inspire_app, insert_experiments_into_db + inspire_app, ): record = { "_collections": ["Literature"], @@ -133,8 +133,9 @@ def test_normalize_collaborations_with_different_name_variants( assert result["accelerator_experiments"] == expected_accelerator_experiments +@pytest.mark.usefixtures("_insert_ambiguous_experiments_into_db") def test_normalize_collaborations_doesnt_link_experiment_when_ambiguous_collaboration_names( - inspire_app, insert_ambiguous_experiments_into_db, caplog + inspire_app, caplog ): record = { "_collections": ["Literature"], @@ -156,8 +157,9 @@ def test_normalize_collaborations_doesnt_link_experiment_when_ambiguous_collabor assert "'Ambiguous match for collaboration'" in caplog.text +@pytest.mark.usefixtures("_insert_ambiguous_experiments_into_db") def test_normalize_collaborations_doesnt_link_experiment_when_ambiguous_subgroup( - inspire_app, insert_ambiguous_experiments_into_db, caplog + inspire_app, caplog ): record = { "_collections": ["Literature"], @@ -178,9 +180,8 @@ def test_normalize_collaborations_doesnt_link_experiment_when_ambiguous_subgroup assert "Ambiguous match for collaboration" in caplog.text -def test_normalize_affiliations_happy_flow( - inspire_app, insert_literature_in_db, insert_institutions_in_db, caplog -): +@pytest.mark.usefixtures("_insert_literature_in_db", "_insert_institutions_in_db") +def test_normalize_affiliations_happy_flow(inspire_app, caplog): record = { "_collections": ["Literature"], "titles": ["A title"], @@ -216,9 +217,8 @@ def test_normalize_affiliations_happy_flow( assert "Found matching affiliation" in caplog.text -def test_normalize_affiliations_when_authors_has_two_happy_flow( - inspire_app, insert_literature_in_db -): +@pytest.mark.usefixtures("_insert_literature_in_db") +def test_normalize_affiliations_when_authors_has_two_happy_flow(inspire_app): record = { "_collections": ["Literature"], "titles": ["A title"], @@ -249,8 +249,9 @@ def test_normalize_affiliations_when_authors_has_two_happy_flow( } in data["normalized_affiliations"][0] +@pytest.mark.usefixtures("_insert_literature_in_db", "_insert_institutions_in_db") def test_normalize_affiliations_when_lit_affiliation_missing_institution_ref( - inspire_app, insert_literature_in_db, insert_institutions_in_db + inspire_app, ): record = { "_collections": ["Literature"], @@ -280,8 +281,9 @@ def test_normalize_affiliations_when_lit_affiliation_missing_institution_ref( "inspirehep.curation.utils.find_unambiguous_affiliation", return_value={"value": "CERN"}, ) +@pytest.mark.usefixtures("_insert_literature_in_db") def test_normalize_affiliations_run_query_only_once_when_authors_have_same_raw_aff( - mock_assign_matched_affiliation_to_author, inspire_app, insert_literature_in_db + mock_assign_matched_affiliation_to_author, inspire_app ): record = { "_collections": ["Literature"], @@ -308,9 +310,8 @@ def test_normalize_affiliations_run_query_only_once_when_authors_have_same_raw_a assert mock_assign_matched_affiliation_to_author.called_once() -def test_normalize_affiliations_handle_not_found_affiliations( - inspire_app, insert_literature_in_db -): +@pytest.mark.usefixtures("_insert_literature_in_db") +def test_normalize_affiliations_handle_not_found_affiliations(inspire_app): record = { "_collections": ["Literature"], "titles": ["A title"], @@ -329,9 +330,9 @@ def test_normalize_affiliations_handle_not_found_affiliations( assert "Non existing aff" in data["ambiguous_affiliations"] +@pytest.mark.usefixtures("_insert_literature_in_db") def test_normalize_affiliations_doesnt_return_nested_affiliations_if_using_memoized( inspire_app, - insert_literature_in_db, ): record = { "_collections": ["Literature"], @@ -369,9 +370,9 @@ def test_normalize_affiliations_doesnt_return_nested_affiliations_if_using_memoi ] +@pytest.mark.usefixtures("_insert_literature_in_db") def test_normalize_affiliations_doesnt_add_duplicated_affiliations( inspire_app, - insert_literature_in_db, ): record = { "_collections": ["Literature"], @@ -397,8 +398,9 @@ def test_normalize_affiliations_doesnt_add_duplicated_affiliations( } +@pytest.mark.usefixtures("_insert_literature_in_db") def test_aff_normalization_if_no_match_from_highlighting_and_no_other_matches( - inspire_app, insert_literature_in_db + inspire_app, ): record = { "_collections": ["Literature"], @@ -421,9 +423,9 @@ def test_aff_normalization_if_no_match_from_highlighting_and_no_other_matches( assert not data["normalized_affiliations"][0] +@pytest.mark.usefixtures("_insert_literature_in_db") def test_normalize_affiliations_assign_only_matching_affiliation_when_multiple_raw_affs_in_matched_author( inspire_app, - insert_literature_in_db, ): record = { "_collections": ["Literature"], @@ -451,9 +453,9 @@ def test_normalize_affiliations_assign_only_matching_affiliation_when_multiple_r ] +@pytest.mark.usefixtures("_insert_literature_in_db") def test_normalize_affiliations_doesnt_add_not_valid_stuff_to_affiliation( inspire_app, - insert_literature_in_db, ): record = { "_collections": ["Literature"], @@ -487,9 +489,9 @@ def test_normalize_affiliations_doesnt_add_not_valid_stuff_to_affiliation( ] +@pytest.mark.usefixtures("_insert_literature_in_db") def test_normalize_affiliations_assign_all_affiliations_if_one_raw_aff_in_matched_lit_author( inspire_app, - insert_literature_in_db, ): record = { "_collections": ["Literature"], @@ -526,9 +528,8 @@ def test_normalize_affiliations_assign_all_affiliations_if_one_raw_aff_in_matche ] -def test_assign_institution_reference_to_affiliations( - inspire_app, insert_institutions_in_db -): +@pytest.mark.usefixtures("_insert_institutions_in_db") +def test_assign_institution_reference_to_affiliations(inspire_app): affiliations = [{"value": "CERN"}, {"value": "Warsaw U."}] assign_institution_reference_to_affiliations(affiliations, {}) assert ["record" in affiliation for affiliation in affiliations] diff --git a/backend/tests/integration/curation/test_api/literature/literature_1459277.json b/backend/tests/integration/curation/test_api/literature/literature_1459277.json index da3e4a788d..1bc7f0d6dd 100644 --- a/backend/tests/integration/curation/test_api/literature/literature_1459277.json +++ b/backend/tests/integration/curation/test_api/literature/literature_1459277.json @@ -1958,4 +1958,4 @@ "schema": "ADS" } ] -} \ No newline at end of file +} diff --git a/backend/tests/integration/curation/test_api/literature/literature_1800446.json b/backend/tests/integration/curation/test_api/literature/literature_1800446.json index d8fadff67d..16f59c0e57 100644 --- a/backend/tests/integration/curation/test_api/literature/literature_1800446.json +++ b/backend/tests/integration/curation/test_api/literature/literature_1800446.json @@ -1928,4 +1928,4 @@ } ], "legacy_creation_date": "2020-06-11" -} \ No newline at end of file +} diff --git a/backend/tests/integration/curation/test_api/literature/literature_1836272.json b/backend/tests/integration/curation/test_api/literature/literature_1836272.json index 506d0d306d..d0be9f650a 100644 --- a/backend/tests/integration/curation/test_api/literature/literature_1836272.json +++ b/backend/tests/integration/curation/test_api/literature/literature_1836272.json @@ -1046,4 +1046,4 @@ } ], "legacy_creation_date": "2020-12-14" -} \ No newline at end of file +} diff --git a/backend/tests/integration/curation/test_api/literature/literature_1862822.json b/backend/tests/integration/curation/test_api/literature/literature_1862822.json index b438384a4d..fb7973cbf0 100644 --- a/backend/tests/integration/curation/test_api/literature/literature_1862822.json +++ b/backend/tests/integration/curation/test_api/literature/literature_1862822.json @@ -1752,4 +1752,4 @@ } ], "legacy_creation_date": "2021-05-12" -} \ No newline at end of file +} diff --git a/backend/tests/integration/curation/test_api/literature/literature_1863053.json b/backend/tests/integration/curation/test_api/literature/literature_1863053.json index 0b9328adb1..2f939adbd5 100644 --- a/backend/tests/integration/curation/test_api/literature/literature_1863053.json +++ b/backend/tests/integration/curation/test_api/literature/literature_1863053.json @@ -3396,4 +3396,4 @@ "schema": "CDS" } ] -} \ No newline at end of file +} diff --git a/backend/tests/integration/curation/test_curation_views.py b/backend/tests/integration/curation/test_curation_views.py index aaa7105842..8207cbea04 100644 --- a/backend/tests/integration/curation/test_curation_views.py +++ b/backend/tests/integration/curation/test_curation_views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,10 +6,9 @@ import orjson from helpers.utils import create_record, create_user -from invenio_accounts.testutils import login_user_via_session - from inspirehep.accounts.roles import Roles from inspirehep.records.api import LiteratureRecord +from invenio_accounts.testutils import login_user_via_session def test_add_keywords_replace_old_keywords_with_new(inspire_app): diff --git a/backend/tests/integration/curation/test_search_check_do_base.py b/backend/tests/integration/curation/test_search_check_do_base.py index a4dbe747d9..c20973ee31 100644 --- a/backend/tests/integration/curation/test_search_check_do_base.py +++ b/backend/tests/integration/curation/test_search_check_do_base.py @@ -6,15 +6,13 @@ import os from helpers.utils import create_record -from inspirehep.records.api import LiteratureRecord -from mock import patch - from inspirehep.curation.search_check_do import SearchCheckDo -from inspirehep.records.api import InspireRecord +from inspirehep.records.api import InspireRecord, LiteratureRecord +from mock import patch def test_search_check_do(inspire_app): - not_matching_query_data= {"titles": [{"title": "Some random title"}]} + not_matching_query_data = {"titles": [{"title": "Some random title"}]} not_passing_check_data = {"titles": [{"title": "A title not to modify"}]} to_modify_data = {"titles": [{"title": "A title to modify"}]} @@ -41,9 +39,13 @@ def do(record, logger, state): ModifyTitleWithState() - not_matching_query = LiteratureRecord.get_record_by_pid_value(not_matching_query['control_number']) - not_passing_check = LiteratureRecord.get_record_by_pid_value(not_passing_check['control_number']) - to_modify = LiteratureRecord.get_record_by_pid_value(to_modify['control_number']) + not_matching_query = LiteratureRecord.get_record_by_pid_value( + not_matching_query["control_number"] + ) + not_passing_check = LiteratureRecord.get_record_by_pid_value( + not_passing_check["control_number"] + ) + to_modify = LiteratureRecord.get_record_by_pid_value(to_modify["control_number"]) assert not_matching_query.get_value("titles.title") == ["Some random title"] assert not_passing_check.get_value("titles.title") == ["A title not to modify"] diff --git a/backend/tests/integration/curation/test_utils.py b/backend/tests/integration/curation/test_utils.py index 75c09c51dc..9052023c8e 100644 --- a/backend/tests/integration/curation/test_utils.py +++ b/backend/tests/integration/curation/test_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,7 +5,6 @@ # the terms of the MIT License; see LICENSE file for more details. from helpers.utils import create_record - from inspirehep.curation.utils import ( assign_institution, set_refereed_and_fix_document_type, diff --git a/backend/tests/integration/disambiguation/__init__.py b/backend/tests/integration/disambiguation/__init__.py index 71f4302bdb..ffa3e31145 100644 --- a/backend/tests/integration/disambiguation/__init__.py +++ b/backend/tests/integration/disambiguation/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/tests/integration/disambiguation/test_cli.py b/backend/tests/integration/disambiguation/test_cli.py index af36b5b5f6..f9b99b2789 100644 --- a/backend/tests/integration/disambiguation/test_cli.py +++ b/backend/tests/integration/disambiguation/test_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2021 CERN. # @@ -6,7 +5,6 @@ # the terms of the MIT License; see LICENSE file for more details. import mock from helpers.utils import create_record - from inspirehep.records.api import InspireRecord diff --git a/backend/tests/integration/disambiguation/test_utils.py b/backend/tests/integration/disambiguation/test_utils.py index 50f809cc1c..27b5a89661 100644 --- a/backend/tests/integration/disambiguation/test_utils.py +++ b/backend/tests/integration/disambiguation/test_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -6,7 +5,6 @@ # the terms of the MIT License; see LICENSE file for more detailss. from freezegun import freeze_time - from inspirehep.disambiguation.utils import create_new_stub_author diff --git a/backend/tests/integration/editor/conftest.py b/backend/tests/integration/editor/conftest.py index e0a1d45d61..c33b8d5e67 100644 --- a/backend/tests/integration/editor/conftest.py +++ b/backend/tests/integration/editor/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of INSPIRE. # Copyright (C) 2018 CERN. @@ -24,8 +23,8 @@ from redis import StrictRedis -@pytest.fixture(scope="function", autouse=True) -def clear_redis_editor_locks(inspire_app): +@pytest.fixture(autouse=True) +def _clear_redis_editor_locks(inspire_app): with inspire_app.app_context(): redis_url = current_app.config.get("CACHE_REDIS_URL") redis = StrictRedis.from_url(redis_url) diff --git a/backend/tests/integration/editor/test_editor_views.py b/backend/tests/integration/editor/test_editor_views.py index 0c33d79498..4156f8be20 100644 --- a/backend/tests/integration/editor/test_editor_views.py +++ b/backend/tests/integration/editor/test_editor_views.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # @@ -23,15 +22,14 @@ ) from inspire_schemas.api import load_schema, validate from inspire_utils.record import get_value +from inspirehep.accounts.roles import Roles +from inspirehep.files import current_s3_instance +from inspirehep.snow.api import InspireSnow from invenio_accounts.testutils import login_user_via_session from mock import patch from redis import StrictRedis from werkzeug.datastructures import FileStorage -from inspirehep.accounts.roles import Roles -from inspirehep.files import current_s3_instance -from inspirehep.snow.api import InspireSnow - def test_get_record_and_schema(inspire_app): cataloger = create_user(role=Roles.cataloger.value) @@ -93,9 +91,8 @@ def test_get_record_and_schema_requires_cataloger_logged_in(inspire_app): before_record_request=filter_out_authentication, before_record_response=filter_out_user_data_and_cookie_headers(), ) -def test_create_snow_ticket( - mocked_inspire_snow, inspire_app, override_config, teardown_cache -): +@pytest.mark.usefixtures("_mocked_inspire_snow", "_teardown_cache") +def test_create_snow_ticket(inspire_app, override_config): user = create_user(role=Roles.cataloger.value, email="marcjanna.jedrych@cern.ch") with inspire_app.test_client() as client: @@ -126,9 +123,8 @@ def test_create_snow_ticket( before_record_request=filter_out_authentication, before_record_response=filter_out_user_data_and_cookie_headers(), ) -def test_create_snow_ticket_only_needs_queue_and_recid( - mocked_inspire_snow, inspire_app, override_config, teardown_cache -): +@pytest.mark.usefixtures("_mocked_inspire_snow", "_teardown_cache") +def test_create_snow_ticket_only_needs_queue_and_recid(inspire_app, override_config): user = create_user(role=Roles.cataloger.value, email="marcjanna.jedrych@cern.ch") with inspire_app.test_client() as client: @@ -150,12 +146,11 @@ def test_create_snow_ticket_only_needs_queue_and_recid( before_record_response=filter_out_user_data_and_cookie_headers(), ) @patch("inspirehep.snow.api.requests.put") +@pytest.mark.usefixtures("_mocked_inspire_snow", "_teardown_cache") def test_create_snow_ticket_returns_500_on_error( mocked_update_ticket_with_inspire_recid, - mocked_inspire_snow, inspire_app, override_config, - teardown_cache, ): with override_config(SNOW_URL="https://non-existing-url.com"): user = create_user(role=Roles.cataloger.value) @@ -192,9 +187,8 @@ def test_create_snow_ticket_returns_500_on_error( before_record_request=filter_out_authentication, before_record_response=filter_out_user_data_and_cookie_headers(), ) -def test_resolve_snow_ticket( - mocked_inspire_snow, inspire_app, override_config, teardown_cache -): +@pytest.mark.usefixtures("_mocked_inspire_snow", "_teardown_cache") +def test_resolve_snow_ticket(inspire_app, override_config): user = create_user(role=Roles.cataloger.value, email="marcjanna.jedrych@cern.ch") with inspire_app.test_client() as client: @@ -211,8 +205,9 @@ def test_resolve_snow_ticket( assert expected == result +@pytest.mark.usefixtures("_mocked_inspire_snow") def test_resolve_snow_ticket_returns_403_on_authentication_error( - mocked_inspire_snow, inspire_app, override_config + inspire_app, override_config ): user = create_user() @@ -228,9 +223,8 @@ def test_resolve_snow_ticket_returns_403_on_authentication_error( before_record_request=filter_out_authentication, before_record_response=filter_out_user_data_and_cookie_headers(), ) -def test_get_snow_tickets_for_record( - mocked_inspire_snow, inspire_app, override_config, teardown_cache -): +@pytest.mark.usefixtures("_mocked_inspire_snow", "_teardown_cache") +def test_get_snow_tickets_for_record(inspire_app, override_config): user = create_user(role=Roles.cataloger.value, email="marcjanna.jedrych@cern.ch") with inspire_app.test_client() as client: @@ -269,9 +263,8 @@ def test_get_tickets_for_record_returns_403_on_authentication_error(inspire_app) before_record_request=filter_out_authentication, before_record_response=filter_out_user_data_and_cookie_headers(), ) -def test_get_snow_users( - mocked_inspire_snow, inspire_app, override_config, teardown_cache -): +@pytest.mark.usefixtures("_mocked_inspire_snow", "_teardown_cache") +def test_get_snow_users(inspire_app, override_config): user = create_user(role=Roles.cataloger.value, email="marcjanna.jedrych@cern.ch") with inspire_app.test_client() as client: @@ -283,8 +276,9 @@ def test_get_snow_users( assert next((user.email == snow_user["email"] for snow_user in response.json), None) +@pytest.mark.usefixtures("_mocked_inspire_snow") def test_get_snow_users_returns_403_on_authentication_error( - mocked_inspire_snow, inspire_app, override_config + inspire_app, override_config ): user = create_user() with inspire_app.test_client() as client: @@ -294,8 +288,9 @@ def test_get_snow_users_returns_403_on_authentication_error( assert response.status_code == 403 +@pytest.mark.usefixtures("_mocked_inspire_snow") def test_get_snow_functional_categories_returns_403_on_authentication_error( - mocked_inspire_snow, inspire_app, override_config + inspire_app, override_config ): user = create_user() @@ -311,9 +306,8 @@ def test_get_snow_functional_categories_returns_403_on_authentication_error( before_record_request=filter_out_authentication, before_record_response=filter_out_user_data_and_cookie_headers(), ) -def test_get_snow_functional_categories( - mocked_inspire_snow, inspire_app, override_config, teardown_cache -): +@pytest.mark.usefixtures("_mocked_inspire_snow", "_teardown_cache") +def test_get_snow_functional_categories(inspire_app, override_config): user = create_user(role=Roles.cataloger.value, email="marcjanna.jedrych@cern.ch") with inspire_app.test_client() as client: @@ -464,11 +458,12 @@ def test_file_upload(inspire_app, s3, datadir, override_config): current_s3_instance.client.create_bucket(Bucket="inspire-editor") user = create_user(role=Roles.cataloger.value) record = create_record("lit") - with override_config( - EDITOR_UPLOAD_ALLOWED_EXTENSIONS=".pdf" - ), inspire_app.test_client() as client: + with ( + override_config(EDITOR_UPLOAD_ALLOWED_EXTENSIONS=".pdf"), + inspire_app.test_client() as client, + open(f"{datadir}/test.pdf", "rb") as file_pdf, + ): login_user_via_session(client, email=user.email) - file_pdf = open(f"{datadir}/test.pdf", "rb") bytes_file = FileStorage(file_pdf) data = {"file": bytes_file} response = client.post( @@ -497,11 +492,12 @@ def test_file_upload_with_wrong_mimetype(inspire_app, s3, datadir, override_conf current_s3_instance.client.create_bucket(Bucket="inspire-editor") user = create_user(role=Roles.cataloger.value) record = create_record("lit") - with override_config( - EDITOR_UPLOAD_ALLOWED_EXTENSIONS=".pdf" - ), inspire_app.test_client() as client: + with ( + override_config(EDITOR_UPLOAD_ALLOWED_EXTENSIONS=".pdf"), + inspire_app.test_client() as client, + open(f"{datadir}/test.txt", "rb") as file_txt, + ): login_user_via_session(client, email=user.email) - file_txt = open(f"{datadir}/test.txt", "rb") bytes_file = FileStorage(file_txt) data = {"file": bytes_file} response = client.post( @@ -515,8 +511,10 @@ def test_file_upload_with_wrong_mimetype(inspire_app, s3, datadir, override_conf def test_file_upload_without_permissions(inspire_app, s3, datadir): current_s3_instance.client.create_bucket(Bucket="inspire-editor") record = create_record("lit") - with inspire_app.test_client() as client: - file_pdf = open(f"{datadir}/test.pdf", "rb") + with ( + inspire_app.test_client() as client, + open(f"{datadir}/test.pdf", "rb") as file_pdf, + ): bytes_file = FileStorage(file_pdf) data = {"file": bytes_file} response = client.post( @@ -541,18 +539,19 @@ def test_file_upload_with_read_write_access(inspire_app, s3, datadir, override_c "If-Match": '"0"', } - with override_config( - EDITOR_UPLOAD_ALLOWED_EXTENSIONS=".pdf" - ), inspire_app.test_client() as client: + with ( + override_config(EDITOR_UPLOAD_ALLOWED_EXTENSIONS=".pdf"), + inspire_app.test_client() as client, + ): login_user_via_session(client, email=user.email) - file_pdf = open(f"{datadir}/test.pdf", "rb") - bytes_file = FileStorage(file_pdf) - data = {"file": bytes_file} - response = client.post( - "/editor/literature/{}/upload".format(record["control_number"]), - data=data, - headers=headers_readwrite, - ) + with open(f"{datadir}/test.pdf", "rb") as file_pdf: + bytes_file = FileStorage(file_pdf) + data = {"file": bytes_file} + response = client.post( + "/editor/literature/{}/upload".format(record["control_number"]), + data=data, + headers=headers_readwrite, + ) expected_status_code = 200 assert expected_status_code == response.status_code @@ -573,18 +572,19 @@ def test_file_upload_with_read_access(inspire_app, s3, datadir, override_config) "If-Match": '"0"', } - with override_config( - EDITOR_UPLOAD_ALLOWED_EXTENSIONS=".pdf" - ), inspire_app.test_client() as client: + with ( + override_config(EDITOR_UPLOAD_ALLOWED_EXTENSIONS=".pdf"), + inspire_app.test_client() as client, + ): login_user_via_session(client, email=user.email) - file_pdf = open(f"{datadir}/test.pdf", "rb") - bytes_file = FileStorage(file_pdf) - data = {"file": bytes_file} - response = client.post( - "/editor/literature/{}/upload".format(record["control_number"]), - data=data, - headers=headers_readwrite, - ) + with open(f"{datadir}/test.pdf", "rb") as file_pdf: + bytes_file = FileStorage(file_pdf) + data = {"file": bytes_file} + response = client.post( + "/editor/literature/{}/upload".format(record["control_number"]), + data=data, + headers=headers_readwrite, + ) expected_status_code = 403 assert expected_status_code == response.status_code diff --git a/backend/tests/integration/files/__init__.py b/backend/tests/integration/files/__init__.py index 71f4302bdb..ffa3e31145 100644 --- a/backend/tests/integration/files/__init__.py +++ b/backend/tests/integration/files/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # diff --git a/backend/tests/integration/files/test_api_s3.py b/backend/tests/integration/files/test_api_s3.py index 610f8d51c7..b43303f911 100644 --- a/backend/tests/integration/files/test_api_s3.py +++ b/backend/tests/integration/files/test_api_s3.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -11,9 +10,8 @@ import pytest from botocore.exceptions import ClientError from helpers.utils import create_s3_bucket, create_s3_file -from mock import patch - from inspirehep.files.api import current_s3_instance +from mock import patch KEY = "b50c2ea2d26571e0c5a3411e320586289fd715c2" diff --git a/backend/tests/integration/files/test_cli.py b/backend/tests/integration/files/test_cli.py index 8c563ddec0..d4009da445 100644 --- a/backend/tests/integration/files/test_cli.py +++ b/backend/tests/integration/files/test_cli.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. # diff --git a/backend/tests/integration/files/test_ext.py b/backend/tests/integration/files/test_ext.py index 590b33f438..13772112ec 100644 --- a/backend/tests/integration/files/test_ext.py +++ b/backend/tests/integration/files/test_ext.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Copyright (C) 2019 CERN. # @@ -7,7 +6,6 @@ from concurrent.futures import ThreadPoolExecutor from flask import Flask, current_app - from inspirehep.files import InspireS3 from inspirehep.files.proxies import current_s3_instance diff --git a/backend/tests/integration/fixtures/1663923.xml b/backend/tests/integration/fixtures/1663923.xml index b668ba4cdb..5e5d1fc9a0 100644 --- a/backend/tests/integration/fixtures/1663923.xml +++ b/backend/tests/integration/fixtures/1663923.xml @@ -119,4 +119,4 @@ - \ No newline at end of file + diff --git a/backend/tests/integration/fixtures/1663924.xml b/backend/tests/integration/fixtures/1663924.xml index 9487c16f29..00a2e0eae1 100644 --- a/backend/tests/integration/fixtures/1663924.xml +++ b/backend/tests/integration/fixtures/1663924.xml @@ -187,4 +187,4 @@ - \ No newline at end of file + diff --git a/backend/tests/integration/hal/conftest.py b/backend/tests/integration/hal/conftest.py index c545adc4f6..4fda1dd25f 100644 --- a/backend/tests/integration/hal/conftest.py +++ b/backend/tests/integration/hal/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # This file is part of INSPIRE. # Copyright (C) 2018 CERN. @@ -27,7 +26,7 @@ import pytest -@pytest.fixture +@pytest.fixture() def get_fixture(): def _get_fixture(filename): return pkg_resources.resource_string(__name__, os.path.join("data", filename)) diff --git a/backend/tests/integration/hal/data/_get_hal_id_map.json b/backend/tests/integration/hal/data/_get_hal_id_map.json index 39a544bbd3..2d0fdd21b5 100644 --- a/backend/tests/integration/hal/data/_get_hal_id_map.json +++ b/backend/tests/integration/hal/data/_get_hal_id_map.json @@ -546,4 +546,4 @@ "date":"2018-08-28" } ] -} \ No newline at end of file +} diff --git a/backend/tests/integration/hal/data/_get_hal_id_map_institution.json b/backend/tests/integration/hal/data/_get_hal_id_map_institution.json index eb2969d646..b71e5ef22f 100644 --- a/backend/tests/integration/hal/data/_get_hal_id_map_institution.json +++ b/backend/tests/integration/hal/data/_get_hal_id_map_institution.json @@ -98,4 +98,4 @@ "value": "http://irfu.cea.fr/" } ] -} \ No newline at end of file +} diff --git a/backend/tests/integration/hal/data/aofr.xsd b/backend/tests/integration/hal/data/aofr.xsd index d75727d218..41cd8b9bca 100644 --- a/backend/tests/integration/hal/data/aofr.xsd +++ b/backend/tests/integration/hal/data/aofr.xsd @@ -68,7 +68,7 @@ - + @@ -135,7 +135,7 @@ - + @@ -241,10 +241,10 @@ - - - - + + + + @@ -303,7 +303,7 @@ - + @@ -378,7 +378,7 @@ - + @@ -400,14 +400,14 @@ - + - + @@ -419,7 +419,7 @@