From 423f85d62f3045bbc85e56a4af6fa595ef817056 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Tue, 15 Oct 2024 16:47:06 +0200 Subject: [PATCH] try to fix tests --- openatlas/models/imports.py | 28 +++++----------------------- openatlas/static/example.csv | 2 +- openatlas/views/imports.py | 11 ++--------- sphinx/source/admin/import.rst | 6 +++--- tests/invalid_2.csv | 2 +- tests/test_export_import.py | 1 - 6 files changed, 12 insertions(+), 38 deletions(-) diff --git a/openatlas/models/imports.py b/openatlas/models/imports.py index bdcbbbc14..457d57402 100644 --- a/openatlas/models/imports.py +++ b/openatlas/models/imports.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from collections import defaultdict from typing import Any, Optional @@ -161,16 +162,14 @@ def link_types(entity: Entity, row: dict[str, Any], class_: str) -> None: def link_references(entity: Entity, row: dict[str, Any], class_: str) -> None: if data := row.get('references'): - if '"' in str(data): - data = clean_reference_pages(str(data)) - for references in str(data).split(): + for references in clean_reference_pages(str(data)): reference = references.split(';') if len(reference) <= 2 and reference[0].isdigit(): try: ref_entity = ApiEntity.get_by_id(int(reference[0])) except EntityDoesNotExistError: continue - page = reference[1].replace('|', ' ') or None + page = reference[1] or None ref_entity.link('P67', entity, page) match_types = get_match_types() systems = list(set(i for i in row if i.startswith('reference_system_'))) @@ -218,22 +217,5 @@ def insert_gis(entity: Entity, row: dict[str, Any], project: Project) -> None: Gis.insert_wkt(entity, location, project, wkt_) -def clean_reference_pages(value: str) -> str: - new_string = "" - inside_quotes = False - current_part = "" - - for char in value: - if char == '"': - if inside_quotes: - modified_part = current_part.replace(' ', '|') - new_string += modified_part - inside_quotes = False - current_part = "" - else: - inside_quotes = True - elif inside_quotes: - current_part += char - else: - new_string += char - return new_string +def clean_reference_pages(value: str) -> list[str]: + return re.findall(r'\d+;.*?(?=\d+;|$)', value) diff --git a/openatlas/static/example.csv b/openatlas/static/example.csv index 874751eb0..91685d22c 100644 --- a/openatlas/static/example.csv +++ b/openatlas/static/example.csv @@ -1,4 +1,4 @@ id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,references,reference_system_wikidata,reference_system_geonames,administrative_unit,historical_place -place_1,Vienna,Wien;City of Vienna,Capital of Austria,1500-01-01,1500-12-31,It was a rainy day.,2045-01-01,2049-12-31,We'll see about that.,"POLYGON((16.1203 48.30671, 16.606275 48.30671, 16.606275 48.3154, 16.1203 48.3154, 16.1203 48.30671))",80 184895,128787;-13.56,117293;"IV, 23-45" 23235;"45 34",Q152419;close_match,2761369;exact_match,87,221630 +place_1,Vienna,Wien;City of Vienna,Capital of Austria,1500-01-01,1500-12-31,It was a rainy day.,2045-01-01,2049-12-31,We'll see about that.,"POLYGON((16.1203 48.30671, 16.606275 48.30671, 16.606275 48.3154, 16.1203 48.3154, 16.1203 48.30671))",80 184895,128787;-13.56,"117293;IV, 23-45 23235;45 34",Q152419;close_match,2761369;exact_match,87,221630 place_2,London,,,,,,,,,"POINT (-0.1290 51.5053)",,,,,,, place_3,Rom,,,,,,,,,"LINESTRING (12.458533781141528 41.922205268362234, 12.53062334955289 41.917606998887024, 12.52169797441624 41.888476931243254)",,,,,,, diff --git a/openatlas/views/imports.py b/openatlas/views/imports.py index 0ef2d8c48..08c06a2f6 100644 --- a/openatlas/views/imports.py +++ b/openatlas/views/imports.py @@ -50,7 +50,6 @@ _('invalid value type values') _('invalid coordinates') _('invalid OpenAtlas class') -_('invalid references') _('invalid reference id') _('empty names') _('empty ids') @@ -490,14 +489,8 @@ def check_cell_value( value = ' '.join(value_types) case 'references' if value: references = [] - if '"' in str(value): - value = clean_reference_pages(str(value)) - for reference in str(value).split(): + for reference in clean_reference_pages(str(value)): values = str(reference).split(';') - if len(values) > 2: - references.append(error_span(reference)) - checks.set_warning('invalid_references', id_) - continue if not values[0].isdigit(): values[0] = error_span(values[0]) checks.set_warning('invalid_reference_id', id_) @@ -507,7 +500,7 @@ def check_cell_value( except EntityDoesNotExistError: values[0] = error_span(values[0]) checks.set_warning('invalid_reference_id', id_) - references.append((';'.join(values)).replace('|', ' ')) + references.append(';'.join(values)) value = ' '.join(references) case 'wkt' if value: try: diff --git a/sphinx/source/admin/import.rst b/sphinx/source/admin/import.rst index 0265ac428..7570864e1 100644 --- a/sphinx/source/admin/import.rst +++ b/sphinx/source/admin/import.rst @@ -122,9 +122,9 @@ References It is possible to link existing :doc:`/entity/reference` to imported entities. * :doc:`/entity/reference` ID and pages are separated with a semicolon (**;**), e.g. 1234;56-78 -* To link :doc:`/entity/reference` with multiple page numbers, wrap the numbers in quotation marks, e.g. 1234;"IV, 56-78" -* To link :doc:`/entity/reference` without page number, just add the ID without semicolon (**;**) -* You can enter multiple :doc:`/entity/reference` separated with a space, e.g. 1234;56-78 5678 +* To link :doc:`/entity/reference` with multiple page numbers, wrap the whole cell in quotation marks, e.g. "1234;IV, 56-78" +* To link :doc:`/entity/reference` without page number, just add the ID semicolon (**;**) without additional information +* You can enter multiple :doc:`/entity/reference` separated with a space, e.g. 1234;56-78 5678; * The ID of a :doc:`/entity/reference` can be looked up at the detail view of the entity .. _WKT import: diff --git a/tests/invalid_2.csv b/tests/invalid_2.csv index d42077ae8..6fd6f4561 100644 --- a/tests/invalid_2.csv +++ b/tests/invalid_2.csv @@ -1,4 +1,4 @@ id,name,alias,description,begin_from,begin_to,begin_comment,end_from,end_to,end_comment,wkt,type_ids,value_types,references,reference_system_wikidata,reference_system_geon,administrative_unit,historical_place,not_existing_column -place_1,Vienna,Wien,Capital of Austria,not_a_date,NaT,It was a rainy day.,,2049-12-31,"We'll see about that.","MULTILINESTRING ((10 10, 20 20, 10 40),(40 40, 30 30, 40 20, 30 10))",666,666;12B34,666;213;41 12b 666;IV,123;away,juhhu;4545,777,888, +place_1,Vienna,Wien,Capital of Austria,not_a_date,NaT,It was a rainy day.,,2049-12-31,"We'll see about that.","MULTILINESTRING ((10 10, 20 20, 10 40),(40 40, 30 30, 40 20, 30 10))",666,666;12B34,666;213;41 12b 666b;IV,123;away,juhhu;4545,777,888, place_1,,,,,,,,,,"MULTILINESTRING ((BLA 10, 20 20, 10 40),(40 40, 30 30, 40 20, 30 10))",,,,,, place_1,Vienna,,,,,,,,,,,,,, diff --git a/tests/test_export_import.py b/tests/test_export_import.py index 2d6b8a95d..4bd79ebbf 100644 --- a/tests/test_export_import.py +++ b/tests/test_export_import.py @@ -114,7 +114,6 @@ def test_export(self) -> None: assert b'invalid value type ids' in rv.data assert b'invalid value type values' in rv.data assert b'invalid reference system' in rv.data - assert b'invalid references' in rv.data assert b'invalid reference id' in rv.data assert b'empty names' in rv.data assert b'double IDs in import' in rv.data