diff --git a/backend/poetry.lock b/backend/poetry.lock index 8e79e1f9f..f596bcb49 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "alembic" @@ -2020,12 +2020,12 @@ tests = ["flake8 (>=3.5.0,<4.0)", "mock (>=2.0.0,<3.0)", "pytest (>=3.2.2,<4.0)" [[package]] name = "inspire-schemas" -version = "61.6.4" +version = "61.6.8" description = "Inspire JSON schemas and utilities to use them." optional = false python-versions = "*" files = [ - {file = "inspire_schemas-61.6.4-py2.py3-none-any.whl", hash = "sha256:03551e9655791b52cb04117169a26e2c007f007f2a5acd3a6042568bd1df6a00"}, + {file = "inspire_schemas-61.6.8-py2.py3-none-any.whl", hash = "sha256:97fc76694b04f8a7ad3be66e8d1834bfe91784ce7f3653a991a606c26c765215"}, ] [package.dependencies] @@ -6568,4 +6568,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.12" -content-hash = "dbf6e6127522a6980a521f7cfb4bad2deb03904ba7b8a91bbd8590bde1ecfb00" +content-hash = "52ba06113b83054ae0a57a55cf3845fa5cba96bf168ca7bcadeefa15fef0446a" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8f08ea2fb..93c6223e6 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -21,7 +21,7 @@ Flask = ">=1.0.2" msgpack = "==0.6.2" python = ">=3.11,<3.12" -inspire-schemas = ">=61.6.4" +inspire-schemas = ">=61.6.8" inspire-utils = ">=3.0.63" inspire-service-orcid = {git = "https://github.com/inspirehep/inspire-service-orcid.git", rev = "1a0e762e58bc9cac65f7665b9831993c017fd8bb"} inspire-json-merger = "^11.0.37" diff --git a/workflows/tests/cassettes/TestDataHarvest.test_load_record_post.yaml b/workflows/tests/cassettes/TestDataHarvest.test_load_record_post.yaml index 62584ffb7..6cce111e3 100644 --- a/workflows/tests/cassettes/TestDataHarvest.test_load_record_post.yaml +++ b/workflows/tests/cassettes/TestDataHarvest.test_load_record_post.yaml @@ -9,7 +9,7 @@ interactions: Connection: - keep-alive method: GET - uri: http://host.docker.internal:8080/api/doi/10.1234567/test + uri: http://host.docker.internal:8080/api/doi/10.1234568/test response: body: string: '{"status": 404, "message": "PID does not exist."}' @@ -21,7 +21,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 11 Dec 2024 16:16:05 GMT + - Fri, 10 Jan 2025 10:28:29 GMT Server: - nginx/1.19.1 status: @@ -37,7 +37,7 @@ interactions: Connection: - keep-alive method: GET - uri: http://host.docker.internal:8080/api/doi/10.1234567/test + uri: http://host.docker.internal:8080/api/doi/10.1234568/test response: body: string: '{"status": 404, "message": "PID does not exist."}' @@ -49,7 +49,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 11 Dec 2024 16:16:06 GMT + - Fri, 10 Jan 2025 10:28:30 GMT Server: - nginx/1.19.1 status: @@ -65,7 +65,7 @@ interactions: Connection: - keep-alive method: GET - uri: http://host.docker.internal:8080/api/doi/10.1234567/test + uri: http://host.docker.internal:8080/api/doi/10.1234568/test response: body: string: '{"status": 404, "message": "PID does not exist."}' @@ -77,7 +77,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 11 Dec 2024 16:16:08 GMT + - Fri, 10 Jan 2025 10:28:32 GMT Server: - nginx/1.19.1 status: @@ -93,7 +93,7 @@ interactions: Connection: - keep-alive method: GET - uri: http://host.docker.internal:8080/api/doi/10.1234567/test + uri: http://host.docker.internal:8080/api/doi/10.1234568/test response: body: string: '{"status": 404, "message": "PID does not exist."}' @@ -105,7 +105,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 11 Dec 2024 16:16:12 GMT + - Fri, 10 Jan 2025 10:28:36 GMT Server: - nginx/1.19.1 status: @@ -121,7 +121,7 @@ interactions: Connection: - keep-alive method: GET - uri: http://host.docker.internal:8080/api/doi/10.1234567/test + uri: http://host.docker.internal:8080/api/doi/10.1234568/test response: body: string: '{"status": 404, "message": "PID does not exist."}' @@ -133,7 +133,7 @@ interactions: Content-Type: - application/json Date: - - Wed, 11 Dec 2024 16:16:20 GMT + - Fri, 10 Jan 2025 10:28:44 GMT Server: - nginx/1.19.1 status: @@ -141,7 +141,9 @@ interactions: message: NOT FOUND - request: body: '{"_collections": ["Data"], "$schema": "https://inspirehep.net/schemas/records/data.json", - "dois": [{"value": "10.1234567/test", "material": "data"}]}' + "dois": [{"value": "10.1234568/test", "material": "data"}], "acquisition_source": + {"source": "inspirehep", "submission_number": "2829504", "datetime": "2025-01-09T16:14:51.647320+00:00", + "method": "inspirehep"}}' headers: Accept: - application/vnd+inspire.record.raw+json @@ -150,37 +152,37 @@ interactions: Connection: - keep-alive Content-Length: - - '149' + - '301' Content-Type: - application/json method: POST uri: http://host.docker.internal:8080/api/data response: body: - string: '{"metadata":{"_collections":["Data"],"$schema":"https://inspirehep.net/schemas/records/data.json","dois":[{"value":"10.1234567/test","material":"data"}],"control_number":2610637,"self":{"$ref":"http://localhost:8000/api/data/2610637"}},"revision_id":0,"updated":"2024-12-11T16:16:20.351311+00:00","uuid":"b765f128-72f8-4120-ac43-9242ad2c40a6","created":"2024-12-11T16:16:20.351308+00:00","id":"{2610637}","links":{"json":"http://localhost:8000/api/data/%7B2610637%7D?format=json"}}' + string: '{"links":{"json":"http://localhost:8000/api/data/%7B2727285%7D?format=json"},"revision_id":0,"uuid":"d0e25a25-c167-43c4-a5ed-0746758572df","created":"2025-01-10T10:28:44.881337+00:00","updated":"2025-01-10T10:28:44.881390+00:00","metadata":{"_collections":["Data"],"$schema":"https://inspirehep.net/schemas/records/data.json","dois":[{"value":"10.1234568/test","material":"data"}],"acquisition_source":{"source":"inspirehep","submission_number":"2829504","datetime":"2025-01-09T16:14:51.647320+00:00","method":"inspirehep"},"control_number":2727285,"self":{"$ref":"http://localhost:8000/api/data/2727285"}},"id":"{2727285}"}' headers: Cache-Control: - no-cache Connection: - keep-alive Content-Length: - - '481' + - '624' Content-Type: - application/vnd+inspire.record.raw+json Date: - - Wed, 11 Dec 2024 16:16:20 GMT + - Fri, 10 Jan 2025 10:28:44 GMT ETag: - W/"0" Last-Modified: - - Wed, 11 Dec 2024 16:16:20 GMT + - Fri, 10 Jan 2025 10:28:44 GMT Link: - - ; rel="http://localhost:8000/api/data/%7B2610637%7D?format=json" + - ; rel="http://localhost:8000/api/data/%7B2727285%7D?format=json" Server: - nginx/1.19.1 Vary: - Accept,Accept-Encoding,Accept-Language location: - - http://localhost:8000/api/data/%7B2610637%7D + - http://localhost:8000/api/data/%7B2727285%7D status: code: 201 message: CREATED diff --git a/workflows/tests/cassettes/TestDataHarvest.test_load_record_put.yaml b/workflows/tests/cassettes/TestDataHarvest.test_load_record_put.yaml index c48bd28e4..56faaa7d8 100644 --- a/workflows/tests/cassettes/TestDataHarvest.test_load_record_put.yaml +++ b/workflows/tests/cassettes/TestDataHarvest.test_load_record_put.yaml @@ -12,7 +12,10 @@ interactions: uri: http://host.docker.internal:8080/api/doi/10.8756/tTM response: body: - string: '{"metadata":{"dois":[{"value":"10.8756/tTM","material":"data"}],"self":{"$ref":"http://localhost:8000/api/data/777"},"urls":[{"value":"http://HoXZ","description":"tempor + string: '{"links":{"bibtex":"http://localhost:8000/api/doi/10.8756/ttm?format=bibtex","latex-eu":"http://localhost:8000/api/doi/10.8756/ttm?format=latex-eu","latex-us":"http://localhost:8000/api/doi/10.8756/ttm?format=latex-us","json":"http://localhost:8000/api/doi/10.8756/ttm?format=json","cv":"http://localhost:8000/api/doi/10.8756/ttm?format=cv"},"revision_id":0,"uuid":"639d8a22-e982-4fdb-b60d-8fce2e78fd34","created":"2025-01-10T08:59:12.166145+00:00","updated":"2025-01-10T08:59:12.166147+00:00","metadata":{"dois":[{"value":"10.8756/tTM","source":"in + ad et","material":"data"},{"value":"10.5/.Aww=bT@","source":"mollit deserunt + eu","material":"version"},{"value":"10.0.9747720/#}O=W:$","source":"adipisicing + et","material":"part"}],"self":{"$ref":"http://localhost:8000/api/data/777"},"urls":[{"value":"http://HoXZ","description":"tempor laboris nostrud amet dolor"},{"value":"http://CBBgCi","description":"irure"},{"value":"https://aE,","description":"reprehenderit lab"},{"value":"http://Erus/87","description":"magna"},{"value":"https://lHCOlUYMP","description":"laborum eiusmod nulla ad ea"}],"titles":[{"title":"Test title","source":"pariatur @@ -40,22 +43,22 @@ interactions: eiusmod anim aute in","record":{"$ref":"http://lWwb6D/api/experiments/547678"}}],"control_number":777,"legacy_version":"ea","deleted_records":[{"$ref":"http://J3vLBvgzhC/api/data/546456"},{"$ref":"http://txGvpx7fpfb/api/data/34534567"}],"acquisition_source":{"email":"xnSAdJWSg358dN@gQEsvHyr.ga","orcid":"7331-1113-6863-4887","method":"oai","source":"dolore mollit","datetime":"5096-09-22T20:00:59.888Z","internal_uid":-19966101,"submission_number":"n"},"accelerator_experiments":[{"record":{"$ref":"http://Wba/api/experiments/12234"},"experiment":"exercitation officia sint","accelerator":"ipsum ess","institution":"veniam","legacy_name":"reprehenderit - incididunt eiusmod exercitation","curated_relation":true}]},"revision_id":7,"updated":"2024-12-11T14:56:27.449027+00:00","uuid":"e5430afb-8761-4eaf-9188-e9d3a7bd4207","created":"2024-12-11T14:02:11.066149+00:00","id":"10.8756/ttm","links":{"bibtex":"http://localhost:8000/api/doi/10.8756/ttm?format=bibtex","latex-eu":"http://localhost:8000/api/doi/10.8756/ttm?format=latex-eu","latex-us":"http://localhost:8000/api/doi/10.8756/ttm?format=latex-us","json":"http://localhost:8000/api/doi/10.8756/ttm?format=json","cv":"http://localhost:8000/api/doi/10.8756/ttm?format=cv"}}' + incididunt eiusmod exercitation","curated_relation":true}]},"id":"10.8756/ttm"}' headers: Cache-Control: - no-cache Connection: - keep-alive Content-Length: - - '5272' + - '5446' Content-Type: - application/vnd+inspire.record.raw+json Date: - - Wed, 11 Dec 2024 16:15:43 GMT + - Fri, 10 Jan 2025 10:29:41 GMT ETag: - - W/"7" + - W/"0" Last-Modified: - - Wed, 11 Dec 2024 14:56:27 GMT + - Fri, 10 Jan 2025 08:59:12 GMT Link: - ; rel="http://localhost:8000/api/doi/10.8756/ttm?format=bibtex", ; rel="http://localhost:8000/api/doi/10.8756/ttm?format=latex-eu", ; @@ -123,9 +126,8 @@ interactions: "ut eiusmod anim aute in", "record": {"$ref": "http://lWwb6D/api/experiments/547678"}}], "control_number": 777, "legacy_version": "ea", "deleted_records": [{"$ref": "http://J3vLBvgzhC/api/data/546456"}, {"$ref": "http://txGvpx7fpfb/api/data/34534567"}], - "acquisition_source": {"email": "xnSAdJWSg358dN@gQEsvHyr.ga", "orcid": "7331-1113-6863-4887", - "method": "oai", "source": "dolore mollit", "datetime": "5096-09-22T20:00:59.888Z", - "internal_uid": -19966101, "submission_number": "n"}, "accelerator_experiments": + "acquisition_source": {"source": "inspirehep", "submission_number": "2829504", + "datetime": "2025-01-09T16:14:51.647320+00:00", "method": "inspirehep"}, "accelerator_experiments": [{"record": {"$ref": "http://Wba/api/experiments/12234"}, "experiment": "exercitation officia sint", "accelerator": "ipsum ess", "institution": "veniam", "legacy_name": "reprehenderit incididunt eiusmod exercitation", "curated_relation": true}]}' @@ -137,16 +139,16 @@ interactions: Connection: - keep-alive Content-Length: - - '5059' + - '4979' Content-Type: - application/json If-Match: - - '"7"' + - '"0"' method: PUT uri: http://host.docker.internal:8080/api/data/777 response: body: - string: '{"metadata":{"dois":[{"value":"10.8756/tTM","material":"data"}],"self":{"$ref":"http://localhost:8000/api/data/777"},"urls":[{"value":"http://HoXZ","description":"tempor + string: '{"links":{"json":"http://localhost:8000/api/data/777?format=json"},"revision_id":1,"uuid":"639d8a22-e982-4fdb-b60d-8fce2e78fd34","created":"2025-01-10T08:59:12.166145+00:00","updated":"2025-01-10T10:29:41.568306+00:00","metadata":{"dois":[{"value":"10.8756/tTM","material":"data"}],"self":{"$ref":"http://localhost:8000/api/data/777"},"urls":[{"value":"http://HoXZ","description":"tempor laboris nostrud amet dolor"},{"value":"http://CBBgCi","description":"irure"},{"value":"https://aE,","description":"reprehenderit lab"},{"value":"http://Erus/87","description":"magna"},{"value":"https://lHCOlUYMP","description":"laborum eiusmod nulla ad ea"}],"titles":[{"title":"Test title","source":"pariatur @@ -171,25 +173,24 @@ interactions: id sunt","source":"ut"},{"value":"mollit","source":"dolor eu et pariatur"}],"literature":[{"doi":{"value":"10.912/W/K[Bz","source":"amet Ut"},"record":{"$ref":"http://XVxZ15mkm/api/literature/44707"}},{"record":{"$ref":"http://l/api/literature/873915"}}],"_collections":["Data"],"creation_date":"2005-05-03","collaborations":[{"value":"ex occaecat sunt eu","record":{"$ref":"http://XcyD6G/api/experiments/1234456"}},{"value":"ut - eiusmod anim aute in","record":{"$ref":"http://lWwb6D/api/experiments/547678"}}],"control_number":777,"legacy_version":"ea","deleted_records":[{"$ref":"http://J3vLBvgzhC/api/data/546456"},{"$ref":"http://txGvpx7fpfb/api/data/34534567"}],"acquisition_source":{"email":"xnSAdJWSg358dN@gQEsvHyr.ga","orcid":"7331-1113-6863-4887","method":"oai","source":"dolore - mollit","datetime":"5096-09-22T20:00:59.888Z","internal_uid":-19966101,"submission_number":"n"},"accelerator_experiments":[{"record":{"$ref":"http://Wba/api/experiments/12234"},"experiment":"exercitation + eiusmod anim aute in","record":{"$ref":"http://lWwb6D/api/experiments/547678"}}],"control_number":777,"legacy_version":"ea","deleted_records":[{"$ref":"http://J3vLBvgzhC/api/data/546456"},{"$ref":"http://txGvpx7fpfb/api/data/34534567"}],"acquisition_source":{"source":"inspirehep","submission_number":"2829504","datetime":"2025-01-09T16:14:51.647320+00:00","method":"inspirehep"},"accelerator_experiments":[{"record":{"$ref":"http://Wba/api/experiments/12234"},"experiment":"exercitation officia sint","accelerator":"ipsum ess","institution":"veniam","legacy_name":"reprehenderit - incididunt eiusmod exercitation","curated_relation":true}]},"revision_id":8,"updated":"2024-12-11T16:15:43.855471+00:00","uuid":"e5430afb-8761-4eaf-9188-e9d3a7bd4207","created":"2024-12-11T14:02:11.066149+00:00","id":"777","links":{"json":"http://localhost:8000/api/data/777?format=json"}}' + incididunt eiusmod exercitation","curated_relation":true}]},"id":"777"}' headers: Cache-Control: - no-cache Connection: - keep-alive Content-Length: - - '4989' + - '4915' Content-Type: - application/vnd+inspire.record.raw+json Date: - - Wed, 11 Dec 2024 16:15:43 GMT + - Fri, 10 Jan 2025 10:29:41 GMT ETag: - - W/"8" + - W/"1" Last-Modified: - - Wed, 11 Dec 2024 16:15:43 GMT + - Fri, 10 Jan 2025 10:29:41 GMT Link: - ; rel="http://localhost:8000/api/data/777?format=json" Server: diff --git a/workflows/tests/conftest.py b/workflows/tests/conftest.py index dd3565978..1c52b90c2 100644 --- a/workflows/tests/conftest.py +++ b/workflows/tests/conftest.py @@ -8,4 +8,5 @@ def vcr_config(): "decode_compressed_response": True, "filter_headers": ("Authorization", "User-Agent"), "record_mode": "once", + "match_on": ["method", "scheme", "host", "port", "path", "query", "body"], } diff --git a/workflows/tests/test_data_tasks.py b/workflows/tests/test_data_tasks.py index 15a342835..e41d29fe9 100644 --- a/workflows/tests/test_data_tasks.py +++ b/workflows/tests/test_data_tasks.py @@ -99,6 +99,12 @@ def test_load_record_put(self): "_collections": ["Data"], "$schema": "https://inspirehep.net/schemas/records/data.json", "dois": [{"value": "10.8756/tTM", "material": "data"}], + "acquisition_source": { + "source": "inspirehep", + "submission_number": "2829504", + "datetime": "2025-01-09T16:14:51.647320+00:00", + "method": "inspirehep", + }, } task = self.dag.get_task("process_record.load_record") task.op_args = (record,) @@ -126,9 +132,16 @@ def test_load_record_post(self): record = { "_collections": ["Data"], "$schema": "https://inspirehep.net/schemas/records/data.json", - "dois": [{"value": "10.1234567/test", "material": "data"}], + "dois": [{"value": "10.1234568/test", "material": "data"}], + "acquisition_source": { + "source": "inspirehep", + "submission_number": "2829504", + "datetime": "2025-01-09T16:14:51.647320+00:00", + "method": "inspirehep", + }, } task = self.dag.get_task("process_record.load_record") task.op_args = (record,) json_response = task.execute(context=self.context) + # import pdb; pdb.set_trace() assert json_response