Merge branch 'develop' into feature_docker_k8s

.github/workflows/starter.yaml

@@ -135,9 +135,14 @@ jobs:
           WTF_CSRF_METHODS: list[str] = []
           ARCHE = {
               'id': 0,
-              'collection_ids': [0],
-              'base_url': 'https://arche-curation.acdh-dev.oeaw.ac.at/',
-              'thumbnail_url': 'https://arche-thumbnails.acdh.oeaw.ac.at/'}
+              'url': 'https://arche-curation.acdh-dev.oeaw.ac.at/'}
+          IIIF = {
+              'enabled': True,
+              'path': '/var/www/iipsrv/',
+              'url': 'http://localhost:8080/iiif/',
+              'version': 2,
+              'conversion': True,
+              'compression': 'jpeg'}
           EOF
           sudo chown 33:33 testing.py
           docker cp -a testing.py openatlas-openatlas-1:/var/www/openatlas/instance/

config/default.py

@@ -23,33 +23,30 @@
     'es': 'Español',
     'fr': 'Français'}
 
-# Files with these extensions are can be displayed in the browser
-DISPLAY_FILE_EXTENSIONS = \
-    ['.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', '.svg']
-
 # Paths are implemented operating system independent using pathlib.
 # To override them (in instance/production.py) either use them like here
 # or use absolute paths like e.g. pathlib.Path('/some/location/somewhere')
 FILES_PATH = Path(__file__).parent.parent / 'files'
 EXPORT_PATH = Path(FILES_PATH) / 'export'
 UPLOAD_PATH = Path(FILES_PATH) / 'uploads'
-TMP_PATH = Path('/tmp')  # used e.g. for processing imports and export files
+TMP_PATH = Path('/tmp')  # For processing files e.g. at import and export
 
 # Image processing
+DISPLAY_FILE_EXT = ['.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', '.svg']
+PROCESSABLE_EXT = ['.tiff', '.tif']
+PROCESSED_EXT = '.jpeg'
 PROCESSED_IMAGE_PATH = Path(FILES_PATH) / 'processed_images'
 RESIZED_IMAGES = Path(PROCESSED_IMAGE_PATH) / 'resized'
 IMAGE_SIZE = {
     'thumbnail': '200',
     'table': '100'}
-NONE_DISPLAY_EXT = ['.tiff', '.tif']
-ALLOWED_IMAGE_EXT = DISPLAY_FILE_EXTENSIONS + NONE_DISPLAY_EXT
-PROCESSED_EXT = '.jpeg'
-
-# For system checks
-WRITABLE_PATHS = [
-    UPLOAD_PATH,
-    EXPORT_PATH,
-    RESIZED_IMAGES]
+IIIF = {
+    'enabled': False,
+    'path': '',
+    'url': '',
+    'version': 2,
+    'conversion': True,
+    'compression': 'deflate'}  # 'deflate' or 'jpeg'
 
 # Security
 SESSION_COOKIE_SECURE = False  # Should be True in production.py if using HTTPS

install/Dockerfile

@@ -8,10 +8,16 @@ RUN --mount=type=cache,target=/var/cache/apt \
     apt install -y --no-install-recommends python3-pandas python3-jinja2 python3-flask-cors python3-flask-restful p7zip-full &&\
     apt install -y --no-install-recommends python3-wand python3-rdflib python3-requests python3-dicttoxml python3-rdflib-jsonld python3-flasgger &&\
     apt install -y --no-install-recommends apache2 libapache2-mod-wsgi-py3 python3-coverage python3-nose exiftran &&\
+    apt install -y --no-install-recommends iipimage-server libvips-tools &&\
     apt install -y --no-install-recommends gettext npm python3-pip git postgresql-client-13 &&\
     apt install -y --no-install-recommends dos2unix locales locales-all w3c-sgml-lib &&\
     mkdir -p /var/www/openatlas /var/www/.cache /var/www/.local /var/www/.npm &&\
     chown -R www-data:www-data /var/www/.cache /var/www/.local /var/www/.npm /var/log/apache2 /var/run/apache2
+RUN cp -rp /usr/lib/iipimage-server/ /var/www/iipsrv/ &&\
+    chown -R www-data /var/www/iipsrv/ &&\
+    chmod 777 -R /var/www/iipsrv/
+RUN rm /etc/apache2/mods-available/iipsrv.conf
+COPY /install/iipsrv.conf /etc/apache2/mods-available/iipsrv.conf
 COPY --chown=www-data:www-data / /var/www/openatlas/
 RUN cd /var/www/openatlas && cp install/entrypoint.sh /entrypoint.sh &&\
     cp install/example_apache.conf /etc/apache2/sites-available/000-default.conf &&\
@@ -26,6 +32,8 @@ RUN cd /var/www/openatlas &&\
     cd openatlas/static &&\
     pip3 install -e ./ &&\
     ~/.local/bin/calmjs npm --install openatlas
+RUN a2enmod iipsrv &&\
+    service apache2 restart
 
 EXPOSE 8080
 ENV APACHE_CONFDIR /etc/apache2

install/iipsrv.conf

@@ -0,0 +1,34 @@
+# Create a directory for the iipsrv binary
+ScriptAlias /iiif "/var/www/iipsrv/iipsrv.fcgi"
+
+# Set the options on that directory
+<Location "iipsrv/">
+   AllowOverride None
+   Options None
+   <IfModule mod_version.c>
+     <IfVersion < 2.4>
+       Order allow,deny
+       Allow from all
+     </IfVersion>
+     <IfVersion >= 2.4>
+       Require all granted
+     </IfVersion>
+   </IfModule>
+   # Set the module handler
+   AddHandler fcgid-script .fcgi
+</Location>
+
+# Set our environment variables for the IIP server
+FcgidInitialEnv VERBOSITY "6"
+FcgidInitialEnv LOGFILE "/var/log/iipsrv.log"
+FcgidInitialEnv MAX_IMAGE_CACHE_SIZE "10"
+FcgidInitialEnv JPEG_QUALITY "90"
+FcgidInitialEnv MAX_CVT "5000"
+FcgidInitialEnv MEMCACHED_SERVERS "localhost"
+FcgidInitialEnv CORS "*"
+FcgidInitialEnv URI_MAP "iiif=>IIIF"
+
+# Define the idle timeout as unlimited and the number of
+# processes we want
+FcgidIdleTimeout 0
+FcgidMaxProcessesPerClass 1

install/upgrade/upgrade.md

@@ -19,6 +19,26 @@ then run the database upgrade script, then restart Apache:
     sudo python3 install/upgrade/database_upgrade.py
     sudo service apache2 restart
 
+### 7.16.x to 7.17.0
+
+#### Configuration
+
+The configuration was refactored, especially path parameters. You should
+compare the instance/production.py with config/default.py in case you made
+adaptions to these. The same goes for instance/tests.py in case you are also
+using tests.
+
+#### IIIF
+For the IIIF implementation new NPM packages are needed:
+
+    $ cd openatlas/static
+    $ rm package.json
+    $ pip3 install -e ./
+    $ ~/.local/bin/calmjs npm --install openatlas
+
+If you want to use IIIF, please read the
+[instructions](https://redmine.openatlas.eu/projects/uni/wiki/IIIF). 
+
 ### 7.16.0 to 7.16.1
 A code base update (e.g. with git pull) and a webserver restart is sufficient.
 

openatlas/__init__.py

@@ -13,7 +13,7 @@
 app: Flask = Flask(__name__, instance_relative_config=True)
 csrf = CSRFProtect(app)  # Make sure all forms are CSRF protected
 app.config.from_object('config.default')
-app.config.from_object('config.api_config')
+app.config.from_object('config.api')
 app.config.from_pyfile('production.py')
 app.config['WTF_CSRF_TIME_LIMIT'] = None  # Set CSRF token valid for session
 
@@ -69,10 +69,16 @@ def before_request() -> None:
     for file_ in app.config['UPLOAD_PATH'].iterdir():
         if file_.stem.isdigit():
             g.files[int(file_.stem)] = file_
-    # Set max file upload in MB
     app.config['MAX_CONTENT_LENGTH'] = \
-        g.settings['file_upload_max_size'] * 1024 * 1024
-
+        g.settings['file_upload_max_size'] * 1024 * 1024  # Max upload in MB
+    g.display_file_ext = app.config['DISPLAY_FILE_EXT']
+    if g.settings['image_processing']:
+        g.display_file_ext += app.config['PROCESSABLE_EXT']
+    g.writable_paths = [
+        app.config['EXPORT_PATH'],
+        app.config['RESIZED_IMAGES'],
+        app.config['UPLOAD_PATH'],
+        app.config['TMP_PATH']]
     if request.path.startswith('/api/'):
         ip = request.environ.get('HTTP_X_REAL_IP', request.remote_addr)
         if not current_user.is_authenticated \

openatlas/api/endpoints/iiif.py

@@ -0,0 +1,148 @@
+from typing import Any
+
+import requests
+from flask import jsonify, Response, url_for, g
+from flask_restful import Resource
+
+from openatlas import app
+from openatlas.api.resources.model_mapper import get_entity_by_id
+from openatlas.api.resources.util import get_license_name
+from openatlas.models.entity import Entity
+
+
+class IIIFSequenceV2(Resource):
+    @staticmethod
+    def get(id_: int) -> Response:
+        return jsonify(
+            {"@context": "https://iiif.io/api/presentation/2/context.json"} |
+            IIIFSequenceV2.build_sequence(get_metadata(get_entity_by_id(id_))))
+
+    @staticmethod
+    def build_sequence(metadata: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "@id": url_for(
+                'api.iiif_sequence',
+                id_=metadata['entity'].id,
+                version=2,
+                _external=True),
+            "@type": "sc:Sequence",
+            "label": [{
+                "@value": "Normal Sequence",
+                "@language": "en"}],
+            "canvases": [
+                IIIFCanvasV2.build_canvas(metadata)]}
+
+
+class IIIFCanvasV2(Resource):
+    @staticmethod
+    def get(id_: int) -> Response:
+        return jsonify(
+            {"@context": "https://iiif.io/api/presentation/2/context.json"} |
+            IIIFCanvasV2.build_canvas(get_metadata(get_entity_by_id(id_))))
+
+    @staticmethod
+    def build_canvas(metadata: dict[str, Any]) -> dict[str, Any]:
+        entity = metadata['entity']
+        return {
+            "@id": url_for(
+                'api.iiif_canvas', id_=entity.id, version=2, _external=True),
+            "@type": "sc:Canvas",
+            "label": entity.name,
+            "height": metadata['img_api']['height'],
+            "width": metadata['img_api']['width'],
+            "description": {
+                "@value": entity.description,
+                "@language": "en"},
+            "images": [IIIFImageV2.build_image(metadata)],
+            "related": "",
+            "thumbnail": {
+                "@id": f'{metadata["img_url"]}/full/!200,200/0/default.jpg',
+                "@type": "dctypes:Image",
+                "format": "image/jpeg",
+                "height": 200,
+                "width": 200,
+                "service": {
+                    "@context": "https://iiif.io/api/image/2/context.json",
+                    "@id": metadata['img_url'],
+                    "profile": metadata['img_api']['profile']}}}
+
+
+class IIIFImageV2(Resource):
+    @staticmethod
+    def get(id_: int) -> Response:
+        return jsonify(
+            IIIFImageV2.build_image(get_metadata(get_entity_by_id(id_))))
+
+    @staticmethod
+    def build_image(metadata: dict[str, Any]) -> dict[str, Any]:
+        id_ = metadata['entity'].id
+        return {
+            "@context": "https://iiif.io/api/presentation/2/context.json",
+            "@id":
+                url_for('api.iiif_image', id_=id_, version=2, _external=True),
+            "@type": "oa:Annotation",
+            "motivation": "sc:painting",
+            "resource": {
+                "@id": metadata['img_url'],
+                "@type": "dctypes:Image",
+                "format": "image/jpeg",
+                "service": {
+                    "@context": "https://iiif.io/api/image/2/context.json",
+                    "@id": metadata['img_url'],
+                    "profile": metadata['img_api']['profile']},
+                "height": metadata['img_api']['height'],
+                "width": metadata['img_api']['width']},
+            "on":
+                url_for('api.iiif_canvas', id_=id_, version=2, _external=True)}
+
+
+class IIIFManifest(Resource):
+    @staticmethod
+    def get(version: int, id_: int) -> Response:
+        operation = getattr(IIIFManifest, f'get_manifest_version_{version}')
+        return jsonify(operation(id_))
+
+    @staticmethod
+    def get_manifest_version_2(id_: int) -> dict[str, Any]:
+        entity = get_entity_by_id(id_)
+        return {
+            "@context": "https://iiif.io/api/presentation/2/context.json",
+            "@id":
+                url_for(
+                    'api.iiif_manifest',
+                    id_=id_,
+                    version=2,
+                    _external=True),
+            "@type": "sc:Manifest",
+            "label": entity.name,
+            "metadata": [],
+            "description": [{
+                "@value": entity.description,
+                "@language": "en"}],
+            "license": get_license_name(entity),
+            "attribution": "By OpenAtlas",
+            "logo": get_logo(),
+            "sequences": [
+                IIIFSequenceV2.build_sequence(get_metadata(entity))],
+            "structures": []}
+
+
+def get_metadata(entity: Entity) -> dict[str, Any]:
+    ext = '.tiff' if app.config['IIIF']['conversion'] \
+        else entity.get_file_ext()
+    image_url = f"{app.config['IIIF']['url']}{entity.id}{ext}"
+    req = requests.get(f"{image_url}/info.json")
+    image_api = req.json()
+    return {'entity': entity, 'img_url': image_url, 'img_api': image_api}
+
+
+def get_logo() -> dict[str, Any]:
+    return {
+        "@id": url_for(
+            'api.display',
+            filename=g.settings['logo_file_id'],
+            _external=True),
+        "service": {
+            "@context": "https://iiif.io/api/image/2/context.json",
+            "@id": url_for('overview', _external=True),
+            "profile": "https://iiif.io/api/image/2/level2.json"}}

openatlas/api/import_scripts/util.py

@@ -50,6 +50,8 @@ def vocabs_requests(
 def request_arche_metadata(id_: int) -> dict[str, Any]:
     req = requests.get(
         f"{app.config['ARCHE']['url']}/api/{id_}/metadata",
-        headers={'Accept': 'application/ld+json'},
+        headers={
+            'Accept': 'application/ld+json',
+            'X-METADATA-READ-MODE': '1_1_0_0'},
         timeout=60)
     return req.json()

openatlas/api/import_scripts/vocabs.py

@@ -27,8 +27,8 @@ def fetch_top_groups(
     duplicates = []
     if ref := get_vocabs_reference_system(details):
         for group in form_data['choices']:
-            if (not Type.check_hierarchy_exists(group[1]) and
-                    group[0] in form_data['top_concepts']):
+            if not Type.check_hierarchy_exists(group[1]) and \
+                    group[0] in form_data['top_concepts']:
                 hierarchy = Entity.insert(
                     'type',
                     group[1],
@@ -47,8 +47,8 @@ def fetch_top_groups(
                     ref,
                     hierarchy)
                 count.append(group[0])
-            if (Type.check_hierarchy_exists(group[1])
-                    and group[0] in form_data['top_concepts']):
+            if Type.check_hierarchy_exists(group[1]) \
+                    and group[0] in form_data['top_concepts']:
                 duplicates.append(group[1])
     return count, duplicates