Merge branch 'develop' into feature_docker_k8s

config/api_config.py

@@ -0,0 +1,43 @@
+API_CONTEXT = {
+    'LPF': 'https://raw.githubusercontent.com/LinkedPasts/linked-places/'
+           'master/linkedplaces-context-v1.1.jsonld',
+    'LOUD': 'https://linked.art/ns/v1/linked-art.json'}
+
+CORS_ALLOWANCE = '*'  # Cross-Origin source (CORS)
+ALLOWED_IPS = ['127.0.0.1']
+API_PROXY = ''
+
+RDF_FORMATS = {
+    'pretty-xml': 'application/rdf+xml',
+    'n3': 'text/rdf+n3',
+    'turtle': 'application/x-turtle',
+    'nt': 'text/plain',
+    'xml': 'application/xml'}
+JSON_FORMATS = {
+    'lp': 'application/ld+json',
+    'loud': 'application/ld+json',
+    'geojson': 'application/json',
+    'geojson-v2': 'application/json'}
+API_FORMATS = RDF_FORMATS | JSON_FORMATS
+
+LOGICAL_OPERATOR: list[str] = ['and', 'or']
+STR_CATEGORIES: list[str] = [
+    "entityName", "entityDescription", "entityAliases", "entityCidocClass",
+    "entitySystemClass", "typeName", "typeNameWithSubs",
+    "beginFrom", "beginTo", "endFrom", "endTo"]
+INT_CATEGORIES: list[str] = [
+    "entityID", "typeID", "typeIDWithSubs", "relationToID"]
+SET_CATEGORIES: list[str] = ["valueTypeID"]
+VALID_CATEGORIES: list[str] = [
+    *STR_CATEGORIES,
+    *INT_CATEGORIES,
+    *SET_CATEGORIES]
+COMPARE_OPERATORS: list[str] = [
+    'equal', 'notEqual', 'greaterThan', 'lesserThan', 'greaterThanEqual',
+    'lesserThanEqual', 'like']
+
+# Used to connect to ACDH-CH ARCHE systems
+ARCHE = {'id': None, 'url': None}
+
+# Used to connect to password protected Vocabs systems
+VOCABS_PASS = ''

config/default.py

@@ -3,16 +3,9 @@
 
 from config.database_versions import DATABASE_VERSIONS
 
-VERSION = '7.16.1'
+VERSION = '7.17.0'
 DATABASE_VERSION = DATABASE_VERSIONS[0]
 DEMO_MODE = False  # If activated some options are disabled, login is prefilled
-
-LANGUAGES = {
-    'ca': 'Català',
-    'de': 'Deutsch',
-    'en': 'English',
-    'es': 'Español',
-    'fr': 'Français'}
 DEBUG = False
 
 DATABASE_NAME = 'openatlas'
@@ -23,6 +16,13 @@
 MAIL_PASSWORD = 'CHANGE ME'
 SECRET_KEY = 'CHANGE ME'  # Used for cookies
 
+LANGUAGES = {
+    'ca': 'Català',
+    'de': 'Deutsch',
+    'en': 'English',
+    'es': 'Español',
+    'fr': 'Français'}
+
 # Files with these extensions are can be displayed in the browser
 DISPLAY_FILE_EXTENSIONS = \
     ['.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', '.svg']
@@ -31,13 +31,13 @@
 # To override them (in instance/production.py) either use them like here
 # or use absolute paths like e.g. pathlib.Path('/some/location/somewhere')
 FILES_PATH = Path(__file__).parent.parent / 'files'
-EXPORT_DIR = Path(FILES_PATH) / 'export'
-UPLOAD_DIR = Path(FILES_PATH) / 'uploads'
-TMP_DIR = Path('/tmp')  # used e.g. for processing imports and export files
+EXPORT_PATH = Path(FILES_PATH) / 'export'
+UPLOAD_PATH = Path(FILES_PATH) / 'uploads'
+TMP_PATH = Path('/tmp')  # used e.g. for processing imports and export files
 
 # Image processing
-PROCESSED_IMAGE_DIR = Path(FILES_PATH) / 'processed_images'
-RESIZED_IMAGES = Path(PROCESSED_IMAGE_DIR) / 'resized'
+PROCESSED_IMAGE_PATH = Path(FILES_PATH) / 'processed_images'
+RESIZED_IMAGES = Path(PROCESSED_IMAGE_PATH) / 'resized'
 IMAGE_SIZE = {
     'thumbnail': '200',
     'table': '100'}
@@ -46,84 +46,24 @@
 PROCESSED_EXT = '.jpeg'
 
 # For system checks
-WRITABLE_DIRS = [
-    UPLOAD_DIR,
-    EXPORT_DIR,
+WRITABLE_PATHS = [
+    UPLOAD_PATH,
+    EXPORT_PATH,
     RESIZED_IMAGES]
 
 # Security
 SESSION_COOKIE_SECURE = False  # Should be True in production.py if using HTTPS
 REMEMBER_COOKIE_SECURE = True
 SESSION_COOKIE_SAMESITE = 'Lax'
 
-# API
-API_SCHEMA = \
-    'https://raw.githubusercontent.com/LinkedPasts/linked-places' \
-    '/master/linkedplaces-context-v1.1.jsonld'
-CORS_ALLOWANCE = '*'  # Cross-Origin source (CORS)
-ALLOWED_IPS = ['127.0.0.1']
-RDF_FORMATS = {
-    'pretty-xml': 'application/rdf+xml',
-    'n3': 'text/rdf+n3',
-    'turtle': 'application/x-turtle',
-    'nt': 'text/plain',
-    'xml': 'application/xml'}
-JSON_FORMATS = {
-    'lp': 'application/ld+json',
-    'loud': 'application/ld+json',
-    'geojson': 'application/json',
-    'geojson-v2': 'application/json'}
-API_FORMATS = RDF_FORMATS | JSON_FORMATS
-
-API_PROXY = ''
-
-LOGICAL_OPERATOR: list[str] = ['and', 'or']
-STR_CATEGORIES: list[str] = [
-    "entityName", "entityDescription", "entityAliases", "entityCidocClass",
-    "entitySystemClass", "typeName", "typeNameWithSubs",
-    "beginFrom", "beginTo", "endFrom", "endTo"]
-INT_CATEGORIES: list[str] = [
-    "entityID", "typeID", "typeIDWithSubs", "relationToID"]
-SET_CATEGORIES: list[str] = ["valueTypeID"]
-VALID_CATEGORIES: list[str] = [
-    *STR_CATEGORIES,
-    *INT_CATEGORIES,
-    *SET_CATEGORIES]
-COMPARE_OPERATORS: list[str] = [
-    'equal', 'notEqual', 'greaterThan', 'lesserThan', 'greaterThanEqual',
-    'lesserThanEqual', 'like']
-
-# Used to connect to ACDH-CH ARCHE systems
-ARCHE = {'id': None, 'url': None}
-
-# Used to connect to password protected Vocabs systems
-VOCABS_PASS = ''
-
 # Table options
 TABLE_ROWS = {10: '10', 25: '25', 50: '50', 100: '100'}
 
 # Minimum required characters for table filters
 MIN_CHARS_JSTREE_SEARCH = 1
 
-LOG_LEVELS = {
-    0: 'emergency',
-    1: 'alert',
-    2: 'critical',
-    3: 'error',
-    4: 'warn',
-    5: 'notice',
-    6: 'info',
-    7: 'debug'}
-
 CSS = {
     'string_field': 'form-control form-control-sm',
     'button': {
         'primary': 'btn btn-outline-primary btn-sm',
         'secondary': 'btn btn-outline-secondary btn-sm'}}
-
-# Property types work differently, e.g. no move functionality
-PROPERTY_TYPES = [
-    'Actor relation',
-    'Actor function',
-    'External reference match',
-    'Involvement']

install/data_test.sql

@@ -22,3 +22,7 @@ INSERT INTO web.i18n (name, language, text) VALUES ('citation_example', 'en', 'c
 
 -- Log entry for none existing entity
 INSERT INTO web.user_log (user_id, entity_id, action) VALUES (2, 6666, 'insert');
+
+-- Needed to test external reference systems for hierarchies
+INSERT INTO web.reference_system_openatlas_class (reference_system_id, openatlas_class_name) VALUES
+  ((SELECT entity_id FROM web.reference_system WHERE name='Wikidata'), 'type');

install/upgrade/database_upgrade.py

@@ -21,10 +21,11 @@
 
 sys.path.insert(0, str(Path(__file__).parent.parent.parent))
 
+# pylint: disable=wrong-import-position
 from config.database_versions import DATABASE_VERSIONS
 from config.default import (
     DATABASE_PASS, VERSION, DATABASE_VERSION, DATABASE_NAME, DATABASE_USER,
-    DATABASE_HOST, DATABASE_PORT, EXPORT_DIR)
+    DATABASE_HOST, DATABASE_PORT, EXPORT_PATH)
 from instance import production
 from openatlas.database.connect import open_connection
 from openatlas.database.settings import Settings
@@ -106,7 +107,7 @@ def check_database_version_supported() -> None:
 
 
 def backup_database() -> None:
-    path = EXPORT_DIR
+    path = EXPORT_PATH
     if not os.access(path, os.W_OK):
         finish(
             f'Directory for database backup not writeable ({path}). Aborting!')

instance/example_testing.py

@@ -15,4 +15,4 @@
 
 # For Windows user
 # from pathlib import Path
-# TMP_DIR = Path('C:\\Path\\to\\tmp')
+# TMP_PATH = Path('C:\\Path\\to\\tmp')

openatlas/__init__.py

@@ -1,6 +1,5 @@
 import locale
-from pathlib import Path
-from typing import Any, Optional
+from typing import Optional
 
 from flask import Flask, Response, g, request, session
 from flask_babel import Babel
@@ -14,6 +13,7 @@
 app: Flask = Flask(__name__, instance_relative_config=True)
 csrf = CSRFProtect(app)  # Make sure all forms are CSRF protected
 app.config.from_object('config.default')
+app.config.from_object('config.api_config')
 app.config.from_pyfile('production.py')
 app.config['WTF_CSRF_TIME_LIMIT'] = None  # Set CSRF token valid for session
 
@@ -29,6 +29,7 @@
     file, hierarchy, index, imports, link, login, model, note, overlay,
     profile, search, sql, type as type_, user, vocabs)
 
+
 @babel.localeselector
 def get_locale() -> str:
     if 'language' in session:
@@ -49,22 +50,25 @@ def before_request() -> None:
 
     if request.path.startswith('/static'):
         return  # Avoid files overhead if not using Apache with static alias
+
     g.logger = Logger()
     g.db = open_connection(app.config)
     g.db.autocommit = True
     g.cursor = g.db.cursor(cursor_factory=extras.DictCursor)
     g.settings = Settings.get_settings()
     session['language'] = get_locale()
-    g.cidoc_classes = CidocClass.get_all()
-    g.properties = CidocProperty.get_all()
+    g.cidoc_classes = CidocClass.get_all(session['language'])
+    g.properties = CidocProperty.get_all(session['language'])
     g.classes = OpenatlasClass.get_all()
     g.types = Type.get_all()
     g.reference_systems = ReferenceSystem.get_all()
     g.view_class_mapping = view_class_mapping
     g.class_view_mapping = OpenatlasClass.get_class_view_mapping()
     g.table_headers = OpenatlasClass.get_table_headers()
-    g.file_stats = get_file_stats()
-
+    g.files = {}
+    for file_ in app.config['UPLOAD_PATH'].iterdir():
+        if file_.stem.isdigit():
+            g.files[int(file_.stem)] = file_
     # Set max file upload in MB
     app.config['MAX_CONTENT_LENGTH'] = \
         g.settings['file_upload_max_size'] * 1024 * 1024
@@ -90,14 +94,3 @@ def apply_caching(response: Response) -> Response:
 @app.teardown_request
 def teardown_request(_exception: Optional[Exception]) -> None:
     close_connection()
-
-
-def get_file_stats(
-        path: Path = app.config['UPLOAD_DIR']) -> dict[int, dict[str, Any]]:
-    stats: dict[int, dict[str, Any]] = {}
-    for file_ in filter(lambda x: x.stem.isdigit(), path.iterdir()):
-        stats[int(file_.stem)] = {
-            'ext': file_.suffix,
-            'size': convert_size(file_.stat().st_size),
-            'date': file_.stat().st_ctime}
-    return stats

openatlas/api/endpoints/display_image.py

@@ -1,14 +1,17 @@
 from pathlib import Path as Pathlib_path
+from typing import Any
 
-from flask import Response, send_file
-from flask_restful import Resource
+from flask import Response, send_file, url_for
+from flask_restful import Resource, marshal
 
 from openatlas import app
 from openatlas.api.resources.error import DisplayFileNotFoundError, \
     NoLicenseError
-from openatlas.api.resources.parser import image
+from openatlas.api.resources.parser import image, files
+from openatlas.api.resources.templates import licensed_file_template
 from openatlas.api.resources.util import get_license_name
-from openatlas.api.resources.model_mapper import get_entity_by_id
+from openatlas.api.resources.model_mapper import get_entity_by_id, \
+    get_entities_by_system_classes, get_entities_by_ids
 from openatlas.display.util import get_file_path
 
 
@@ -26,3 +29,30 @@ def get(filename: str) -> Response:
         if not filepath:
             raise DisplayFileNotFoundError
         return send_file(filepath, as_attachment=bool(parser['download']))
+
+
+class LicensedFileOverview(Resource):
+    @staticmethod
+    def get() -> tuple[Any, int]:
+        parser = files.parse_args()
+        if parser['file_id']:
+            entities = get_entities_by_ids(parser['file_id'])
+        else:
+            entities = get_entities_by_system_classes(['file'])
+        files_dict = {}
+        for entity in entities:
+            if license_ := get_license_name(entity):
+                if path := get_file_path(entity):
+                    files_dict[path.stem] = {
+                        'extension': path.suffix,
+                        'display': url_for(
+                            'api.display',
+                            filename=path.stem,
+                            _external=True),
+                        'thumbnail': url_for(
+                            'api.display',
+                            image_size='thumbnail',
+                            filename=path.stem,
+                            _external=True),
+                        'license': license_}
+        return marshal(files_dict, licensed_file_template(entities)), 200

openatlas/api/endpoints/type.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 from typing import Any, Union
 
-from flask import Response, g, url_for
+from flask import Response, g, url_for, jsonify
 from flask_restful import Resource, marshal
 
 from openatlas.api.resources.parser import default, entity_
@@ -79,9 +79,12 @@ def get_type_overview() -> dict[str, dict[Entity, str]]:
 class GetTypeTree(Resource):
     @staticmethod
     def get() -> Union[tuple[Resource, int], Response]:
+        parser = entity_.parse_args()
         type_tree = {'typeTree': GetTypeTree.get_type_tree()}
-        if entity_.parse_args()['download']:
+        if parser['download']:
             return download(type_tree, type_tree_template(), 'type_tree')
+        if parser['count'] == 'true':
+            return jsonify(len(type_tree['typeTree']))
         return marshal(type_tree, type_tree_template()), 200
 
     @staticmethod

openatlas/api/formats/linked_places.py

@@ -20,7 +20,7 @@ def get_linked_places_entity(
     links_inverse = entity_dict['links_inverse']
     return {
         'type': 'FeatureCollection',
-        '@context': app.config['API_SCHEMA'],
+        '@context': app.config['API_CONTEXT']['LPF'],
         'features': [replace_empty_list_values_in_dict_with_none({
             '@id': url_for('view', id_=entity.id, _external=True),
             'type': 'Feature',

openatlas/api/formats/loud.py

@@ -3,6 +3,7 @@
 
 from flask import url_for
 
+from openatlas import app
 from openatlas.display.util import get_file_path
 from openatlas.models.gis import Gis
 from openatlas.api.resources.util import remove_spaces_dashes, date_to_str, \
@@ -93,8 +94,8 @@ def get_domain_links() -> dict[str, Any]:
                 "_label": label,
                 "type": "DigitalObject"}]})
 
-    return {'@context': "https://linked.art/ns/v1/linked-art.json"} | \
-        base_entity_dict() | properties_set  # type: ignore
+    return {'@context': app.config['API_CONTEXT']['LOUD']} | \
+        base_entity_dict() | properties_set
 
 
 def get_loud_timespan(entity: Entity) -> dict[str, Any]: