diff --git a/.github/workflows/starter.yaml b/.github/workflows/starter.yaml index a2d68b17d..2d2b4f81c 100644 --- a/.github/workflows/starter.yaml +++ b/.github/workflows/starter.yaml @@ -135,9 +135,14 @@ jobs: WTF_CSRF_METHODS: list[str] = [] ARCHE = { 'id': 0, - 'collection_ids': [0], - 'base_url': 'https://arche-curation.acdh-dev.oeaw.ac.at/', - 'thumbnail_url': 'https://arche-thumbnails.acdh.oeaw.ac.at/'} + 'url': 'https://arche-curation.acdh-dev.oeaw.ac.at/'} + IIIF = { + 'enabled': True, + 'path': '/var/www/iipsrv/', + 'url': 'http://localhost:8080/iiif/', + 'version': 2, + 'conversion': True, + 'compression': 'jpeg'} EOF sudo chown 33:33 testing.py docker cp -a testing.py openatlas-openatlas-1:/var/www/openatlas/instance/ diff --git a/config/api_config.py b/config/api.py similarity index 100% rename from config/api_config.py rename to config/api.py diff --git a/config/default.py b/config/default.py index aab3fed06..7f2aeb328 100644 --- a/config/default.py +++ b/config/default.py @@ -23,33 +23,30 @@ 'es': 'Español', 'fr': 'Français'} -# Files with these extensions are can be displayed in the browser -DISPLAY_FILE_EXTENSIONS = \ - ['.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', '.svg'] - # Paths are implemented operating system independent using pathlib. # To override them (in instance/production.py) either use them like here # or use absolute paths like e.g. pathlib.Path('/some/location/somewhere') FILES_PATH = Path(__file__).parent.parent / 'files' EXPORT_PATH = Path(FILES_PATH) / 'export' UPLOAD_PATH = Path(FILES_PATH) / 'uploads' -TMP_PATH = Path('/tmp') # used e.g. for processing imports and export files +TMP_PATH = Path('/tmp') # For processing files e.g. at import and export # Image processing +DISPLAY_FILE_EXT = ['.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', '.svg'] +PROCESSABLE_EXT = ['.tiff', '.tif'] +PROCESSED_EXT = '.jpeg' PROCESSED_IMAGE_PATH = Path(FILES_PATH) / 'processed_images' RESIZED_IMAGES = Path(PROCESSED_IMAGE_PATH) / 'resized' IMAGE_SIZE = { 'thumbnail': '200', 'table': '100'} -NONE_DISPLAY_EXT = ['.tiff', '.tif'] -ALLOWED_IMAGE_EXT = DISPLAY_FILE_EXTENSIONS + NONE_DISPLAY_EXT -PROCESSED_EXT = '.jpeg' - -# For system checks -WRITABLE_PATHS = [ - UPLOAD_PATH, - EXPORT_PATH, - RESIZED_IMAGES] +IIIF = { + 'enabled': False, + 'path': '', + 'url': '', + 'version': 2, + 'conversion': True, + 'compression': 'deflate'} # 'deflate' or 'jpeg' # Security SESSION_COOKIE_SECURE = False # Should be True in production.py if using HTTPS diff --git a/install/Dockerfile b/install/Dockerfile index e124f1f73..17856673a 100644 --- a/install/Dockerfile +++ b/install/Dockerfile @@ -8,10 +8,16 @@ RUN --mount=type=cache,target=/var/cache/apt \ apt install -y --no-install-recommends python3-pandas python3-jinja2 python3-flask-cors python3-flask-restful p7zip-full &&\ apt install -y --no-install-recommends python3-wand python3-rdflib python3-requests python3-dicttoxml python3-rdflib-jsonld python3-flasgger &&\ apt install -y --no-install-recommends apache2 libapache2-mod-wsgi-py3 python3-coverage python3-nose exiftran &&\ + apt install -y --no-install-recommends iipimage-server libvips-tools &&\ apt install -y --no-install-recommends gettext npm python3-pip git postgresql-client-13 &&\ apt install -y --no-install-recommends dos2unix locales locales-all w3c-sgml-lib &&\ mkdir -p /var/www/openatlas /var/www/.cache /var/www/.local /var/www/.npm &&\ chown -R www-data:www-data /var/www/.cache /var/www/.local /var/www/.npm /var/log/apache2 /var/run/apache2 +RUN cp -rp /usr/lib/iipimage-server/ /var/www/iipsrv/ &&\ + chown -R www-data /var/www/iipsrv/ &&\ + chmod 777 -R /var/www/iipsrv/ +RUN rm /etc/apache2/mods-available/iipsrv.conf +COPY /install/iipsrv.conf /etc/apache2/mods-available/iipsrv.conf COPY --chown=www-data:www-data / /var/www/openatlas/ RUN cd /var/www/openatlas && cp install/entrypoint.sh /entrypoint.sh &&\ cp install/example_apache.conf /etc/apache2/sites-available/000-default.conf &&\ @@ -26,6 +32,8 @@ RUN cd /var/www/openatlas &&\ cd openatlas/static &&\ pip3 install -e ./ &&\ ~/.local/bin/calmjs npm --install openatlas +RUN a2enmod iipsrv &&\ + service apache2 restart EXPOSE 8080 ENV APACHE_CONFDIR /etc/apache2 diff --git a/install/iipsrv.conf b/install/iipsrv.conf new file mode 100644 index 000000000..caa64ee86 --- /dev/null +++ b/install/iipsrv.conf @@ -0,0 +1,34 @@ +# Create a directory for the iipsrv binary +ScriptAlias /iiif "/var/www/iipsrv/iipsrv.fcgi" + +# Set the options on that directory + + AllowOverride None + Options None + + + Order allow,deny + Allow from all + + = 2.4> + Require all granted + + + # Set the module handler + AddHandler fcgid-script .fcgi + + +# Set our environment variables for the IIP server +FcgidInitialEnv VERBOSITY "6" +FcgidInitialEnv LOGFILE "/var/log/iipsrv.log" +FcgidInitialEnv MAX_IMAGE_CACHE_SIZE "10" +FcgidInitialEnv JPEG_QUALITY "90" +FcgidInitialEnv MAX_CVT "5000" +FcgidInitialEnv MEMCACHED_SERVERS "localhost" +FcgidInitialEnv CORS "*" +FcgidInitialEnv URI_MAP "iiif=>IIIF" + +# Define the idle timeout as unlimited and the number of +# processes we want +FcgidIdleTimeout 0 +FcgidMaxProcessesPerClass 1 diff --git a/install/upgrade/upgrade.md b/install/upgrade/upgrade.md index eeb706b7e..e4b87156f 100644 --- a/install/upgrade/upgrade.md +++ b/install/upgrade/upgrade.md @@ -19,6 +19,26 @@ then run the database upgrade script, then restart Apache: sudo python3 install/upgrade/database_upgrade.py sudo service apache2 restart +### 7.16.x to 7.17.0 + +#### Configuration + +The configuration was refactored, especially path parameters. You should +compare the instance/production.py with config/default.py in case you made +adaptions to these. The same goes for instance/tests.py in case you are also +using tests. + +#### IIIF +For the IIIF implementation new NPM packages are needed: + + $ cd openatlas/static + $ rm package.json + $ pip3 install -e ./ + $ ~/.local/bin/calmjs npm --install openatlas + +If you want to use IIIF, please read the +[instructions](https://redmine.openatlas.eu/projects/uni/wiki/IIIF). + ### 7.16.0 to 7.16.1 A code base update (e.g. with git pull) and a webserver restart is sufficient. diff --git a/openatlas/__init__.py b/openatlas/__init__.py index a100dce16..80a995c1e 100644 --- a/openatlas/__init__.py +++ b/openatlas/__init__.py @@ -13,7 +13,7 @@ app: Flask = Flask(__name__, instance_relative_config=True) csrf = CSRFProtect(app) # Make sure all forms are CSRF protected app.config.from_object('config.default') -app.config.from_object('config.api_config') +app.config.from_object('config.api') app.config.from_pyfile('production.py') app.config['WTF_CSRF_TIME_LIMIT'] = None # Set CSRF token valid for session @@ -69,10 +69,16 @@ def before_request() -> None: for file_ in app.config['UPLOAD_PATH'].iterdir(): if file_.stem.isdigit(): g.files[int(file_.stem)] = file_ - # Set max file upload in MB app.config['MAX_CONTENT_LENGTH'] = \ - g.settings['file_upload_max_size'] * 1024 * 1024 - + g.settings['file_upload_max_size'] * 1024 * 1024 # Max upload in MB + g.display_file_ext = app.config['DISPLAY_FILE_EXT'] + if g.settings['image_processing']: + g.display_file_ext += app.config['PROCESSABLE_EXT'] + g.writable_paths = [ + app.config['EXPORT_PATH'], + app.config['RESIZED_IMAGES'], + app.config['UPLOAD_PATH'], + app.config['TMP_PATH']] if request.path.startswith('/api/'): ip = request.environ.get('HTTP_X_REAL_IP', request.remote_addr) if not current_user.is_authenticated \ diff --git a/openatlas/api/endpoints/iiif.py b/openatlas/api/endpoints/iiif.py new file mode 100644 index 000000000..7c2bfcdd4 --- /dev/null +++ b/openatlas/api/endpoints/iiif.py @@ -0,0 +1,148 @@ +from typing import Any + +import requests +from flask import jsonify, Response, url_for, g +from flask_restful import Resource + +from openatlas import app +from openatlas.api.resources.model_mapper import get_entity_by_id +from openatlas.api.resources.util import get_license_name +from openatlas.models.entity import Entity + + +class IIIFSequenceV2(Resource): + @staticmethod + def get(id_: int) -> Response: + return jsonify( + {"@context": "https://iiif.io/api/presentation/2/context.json"} | + IIIFSequenceV2.build_sequence(get_metadata(get_entity_by_id(id_)))) + + @staticmethod + def build_sequence(metadata: dict[str, Any]) -> dict[str, Any]: + return { + "@id": url_for( + 'api.iiif_sequence', + id_=metadata['entity'].id, + version=2, + _external=True), + "@type": "sc:Sequence", + "label": [{ + "@value": "Normal Sequence", + "@language": "en"}], + "canvases": [ + IIIFCanvasV2.build_canvas(metadata)]} + + +class IIIFCanvasV2(Resource): + @staticmethod + def get(id_: int) -> Response: + return jsonify( + {"@context": "https://iiif.io/api/presentation/2/context.json"} | + IIIFCanvasV2.build_canvas(get_metadata(get_entity_by_id(id_)))) + + @staticmethod + def build_canvas(metadata: dict[str, Any]) -> dict[str, Any]: + entity = metadata['entity'] + return { + "@id": url_for( + 'api.iiif_canvas', id_=entity.id, version=2, _external=True), + "@type": "sc:Canvas", + "label": entity.name, + "height": metadata['img_api']['height'], + "width": metadata['img_api']['width'], + "description": { + "@value": entity.description, + "@language": "en"}, + "images": [IIIFImageV2.build_image(metadata)], + "related": "", + "thumbnail": { + "@id": f'{metadata["img_url"]}/full/!200,200/0/default.jpg', + "@type": "dctypes:Image", + "format": "image/jpeg", + "height": 200, + "width": 200, + "service": { + "@context": "https://iiif.io/api/image/2/context.json", + "@id": metadata['img_url'], + "profile": metadata['img_api']['profile']}}} + + +class IIIFImageV2(Resource): + @staticmethod + def get(id_: int) -> Response: + return jsonify( + IIIFImageV2.build_image(get_metadata(get_entity_by_id(id_)))) + + @staticmethod + def build_image(metadata: dict[str, Any]) -> dict[str, Any]: + id_ = metadata['entity'].id + return { + "@context": "https://iiif.io/api/presentation/2/context.json", + "@id": + url_for('api.iiif_image', id_=id_, version=2, _external=True), + "@type": "oa:Annotation", + "motivation": "sc:painting", + "resource": { + "@id": metadata['img_url'], + "@type": "dctypes:Image", + "format": "image/jpeg", + "service": { + "@context": "https://iiif.io/api/image/2/context.json", + "@id": metadata['img_url'], + "profile": metadata['img_api']['profile']}, + "height": metadata['img_api']['height'], + "width": metadata['img_api']['width']}, + "on": + url_for('api.iiif_canvas', id_=id_, version=2, _external=True)} + + +class IIIFManifest(Resource): + @staticmethod + def get(version: int, id_: int) -> Response: + operation = getattr(IIIFManifest, f'get_manifest_version_{version}') + return jsonify(operation(id_)) + + @staticmethod + def get_manifest_version_2(id_: int) -> dict[str, Any]: + entity = get_entity_by_id(id_) + return { + "@context": "https://iiif.io/api/presentation/2/context.json", + "@id": + url_for( + 'api.iiif_manifest', + id_=id_, + version=2, + _external=True), + "@type": "sc:Manifest", + "label": entity.name, + "metadata": [], + "description": [{ + "@value": entity.description, + "@language": "en"}], + "license": get_license_name(entity), + "attribution": "By OpenAtlas", + "logo": get_logo(), + "sequences": [ + IIIFSequenceV2.build_sequence(get_metadata(entity))], + "structures": []} + + +def get_metadata(entity: Entity) -> dict[str, Any]: + ext = '.tiff' if app.config['IIIF']['conversion'] \ + else entity.get_file_ext() + image_url = f"{app.config['IIIF']['url']}{entity.id}{ext}" + req = requests.get(f"{image_url}/info.json") + image_api = req.json() + return {'entity': entity, 'img_url': image_url, 'img_api': image_api} + + +def get_logo() -> dict[str, Any]: + return { + "@id": url_for( + 'api.display', + filename=g.settings['logo_file_id'], + _external=True), + "service": { + "@context": "https://iiif.io/api/image/2/context.json", + "@id": url_for('overview', _external=True), + "profile": "https://iiif.io/api/image/2/level2.json"}} diff --git a/openatlas/api/import_scripts/util.py b/openatlas/api/import_scripts/util.py index b41eeefc2..278927498 100644 --- a/openatlas/api/import_scripts/util.py +++ b/openatlas/api/import_scripts/util.py @@ -50,6 +50,8 @@ def vocabs_requests( def request_arche_metadata(id_: int) -> dict[str, Any]: req = requests.get( f"{app.config['ARCHE']['url']}/api/{id_}/metadata", - headers={'Accept': 'application/ld+json'}, + headers={ + 'Accept': 'application/ld+json', + 'X-METADATA-READ-MODE': '1_1_0_0'}, timeout=60) return req.json() diff --git a/openatlas/api/import_scripts/vocabs.py b/openatlas/api/import_scripts/vocabs.py index 48b1efc47..3f1ebd950 100644 --- a/openatlas/api/import_scripts/vocabs.py +++ b/openatlas/api/import_scripts/vocabs.py @@ -27,8 +27,8 @@ def fetch_top_groups( duplicates = [] if ref := get_vocabs_reference_system(details): for group in form_data['choices']: - if (not Type.check_hierarchy_exists(group[1]) and - group[0] in form_data['top_concepts']): + if not Type.check_hierarchy_exists(group[1]) and \ + group[0] in form_data['top_concepts']: hierarchy = Entity.insert( 'type', group[1], @@ -47,8 +47,8 @@ def fetch_top_groups( ref, hierarchy) count.append(group[0]) - if (Type.check_hierarchy_exists(group[1]) - and group[0] in form_data['top_concepts']): + if Type.check_hierarchy_exists(group[1]) \ + and group[0] in form_data['top_concepts']: duplicates.append(group[1]) return count, duplicates diff --git a/openatlas/api/routes.py b/openatlas/api/routes.py index cf399de22..f9afe15b6 100644 --- a/openatlas/api/routes.py +++ b/openatlas/api/routes.py @@ -1,16 +1,18 @@ from flask_restful import Api +from openatlas.api.endpoints.iiif import \ + (IIIFManifest, IIIFImageV2, IIIFCanvasV2, IIIFSequenceV2) from openatlas.api.endpoints.content import ClassMapping, \ GetContent, SystemClassCount from openatlas.api.endpoints.special import GetGeometricEntities, \ ExportDatabase, GetSubunits -from openatlas.api.endpoints.display_image import DisplayImage, LicensedFileOverview +from openatlas.api.endpoints.display_image import \ + (DisplayImage, LicensedFileOverview) from openatlas.api.endpoints.entities import GetByCidocClass, \ GetBySystemClass, GetByViewClass, GetEntitiesLinkedToEntity, GetEntity, \ GetLatest, GetQuery, GetTypeEntities, GetTypeEntitiesAll -from openatlas.api.endpoints.type import GetTypeByViewClass, \ - GetTypeOverview, \ - GetTypeTree +from openatlas.api.endpoints.type import \ + (GetTypeByViewClass, GetTypeOverview, GetTypeTree) def add_routes_v03(api: Api) -> None: @@ -97,3 +99,20 @@ def add_routes_v03(api: Api) -> None: LicensedFileOverview, '/licensed_file_overview/', endpoint='licensed_file_overview') + + api.add_resource( + IIIFManifest, + '/iiif_manifest//', + endpoint='iiif_manifest') + api.add_resource( + IIIFImageV2, + '/iiif_image/.json', + endpoint='iiif_image') + api.add_resource( + IIIFCanvasV2, + '/iiif_canvas/.json', + endpoint='iiif_canvas') + api.add_resource( + IIIFSequenceV2, + '/iiif_sequence/.json', + endpoint='iiif_sequence') diff --git a/openatlas/display/base_display.py b/openatlas/display/base_display.py index 69f48829d..5d026696a 100644 --- a/openatlas/display/base_display.py +++ b/openatlas/display/base_display.py @@ -13,7 +13,7 @@ bookmark_toggle, button, description, edit_link, format_date, format_entity_date, get_appearance, get_base_table_data, get_system_data, is_authorized, link, manual, profile_image_table_link, remove_link, - get_chart_data) + get_chart_data, check_iiif_file_exist) from openatlas.models.entity import Entity from openatlas.models.gis import Gis from openatlas.models.link import Link @@ -78,15 +78,15 @@ def get_type_data(self) -> dict[str, Any]: return {key: data[key] for key in sorted(data.keys())} def add_file_tab_thumbnails(self) -> None: - if 'file' in self.tabs \ - and current_user.settings['table_show_icons'] \ - and g.settings['image_processing']: - self.tabs['file'].table.header.insert(1, _('icon')) - for row in self.tabs['file'].table.rows: - row.insert(1, file_preview( - int(row[0] - .replace(' None: self.add_data() @@ -202,11 +202,9 @@ def add_reference_tables_data(self) -> None: domain = link_.domain data = get_base_table_data(domain) if domain.class_.view == 'file': - extension = data[3] - data.append( - profile_image_table_link(entity, domain, extension)) - if not entity.image_id \ - and extension in app.config['DISPLAY_FILE_EXTENSIONS']: + ext = data[3] + data.append(profile_image_table_link(entity, domain, ext)) + if not entity.image_id and ext in g.display_file_ext: entity.image_id = domain.id elif domain.class_.view != 'source': data.append(link_.description) @@ -382,17 +380,15 @@ def add_tabs(self) -> None: domain = link_.domain data = get_base_table_data(domain) if domain.class_.view == 'file': - extension = data[3] - data.append( - profile_image_table_link(entity, domain, extension)) - if not entity.image_id \ - and extension in app.config['DISPLAY_FILE_EXTENSIONS']: + ext = data[3] + data.append(profile_image_table_link(entity, domain, ext)) + if not entity.image_id and ext in g.display_file_ext: entity.image_id = domain.id if entity.class_.view == 'place' \ and is_authorized('editor') \ and current_user.settings['module_map_overlay']: content = '' - if extension in app.config['DISPLAY_FILE_EXTENSIONS']: + if ext in app.config['DISPLAY_FILE_EXT']: overlays = Overlay.get_by_object(entity) if domain.id in overlays and (html_link := edit_link( url_for( diff --git a/openatlas/display/display.py b/openatlas/display/display.py index 89d12c486..e5ce54fef 100644 --- a/openatlas/display/display.py +++ b/openatlas/display/display.py @@ -5,6 +5,7 @@ from flask import g, url_for from flask_babel import lazy_gettext as _ +from openatlas import app from openatlas.display.base_display import ( ActorDisplay, BaseDisplay, EventsDisplay, PlaceBaseDisplay, ReferenceBaseDisplay, TypeBaseDisplay) @@ -12,7 +13,8 @@ from openatlas.display.table import Table from openatlas.display.util import ( button, description, edit_link, format_entity_date, get_base_table_data, - get_file_path, is_authorized, link, remove_link, uc_first) + get_file_path, is_authorized, link, remove_link, uc_first, + check_iiif_activation, check_iiif_file_exist) from openatlas.models.entity import Entity from openatlas.views.tools import carbon_result, sex_result @@ -66,13 +68,24 @@ class FileDisplay(BaseDisplay): def add_data(self) -> None: super().add_data() self.data[_('size')] = self.entity.get_file_size() - self.data[_('extension')] = self.entity.get_file_extension() + self.data[_('extension')] = self.entity.get_file_ext() def add_button_others(self) -> None: if path := get_file_path(self.entity.id): self.buttons.append(button( _('download'), url_for('download_file', filename=path.name))) + if check_iiif_activation() \ + and self.entity.get_file_ext() in g.display_file_ext: + if check_iiif_file_exist(self.entity.id) \ + or not app.config['IIIF']['conversion']: + self.buttons.append(button( + _('view in IIIF'), + url_for('view_iiif', id_=self.entity.id))) + else: + self.buttons.append(button( + _('enable IIIF view'), + url_for('make_iiif_available', id_=self.entity.id))) return self.buttons.append( '' + uc_first(_("missing file")) + '') diff --git a/openatlas/display/image_processing.py b/openatlas/display/image_processing.py index 6021960b5..748951037 100644 --- a/openatlas/display/image_processing.py +++ b/openatlas/display/image_processing.py @@ -8,13 +8,11 @@ def resize_image(filename: str) -> None: file_format = '.' + filename.split('.', 1)[1].lower() - if file_format in app.config['ALLOWED_IMAGE_EXT']: - loop_resize_image(filename.rsplit('.', 1)[0].lower(), file_format) - - -def loop_resize_image(name: str, file_format: str) -> None: - for size in app.config['IMAGE_SIZE'].values(): - safe_resize_image(name, file_format, size) + if file_format in g.display_file_ext: + for size in app.config['IMAGE_SIZE'].values(): + safe_resize_image( + filename.rsplit('.', 1)[0].lower(), + file_format, size) def safe_resize_image(name: str, file_format: str, size: str) -> bool: @@ -32,23 +30,23 @@ def safe_resize_image(name: str, file_format: str, size: str) -> bool: def image_resizing(name: str, format_: str, size: str) -> bool: - conf = app.config - filename = Path(conf['UPLOAD_PATH']) / f"{name}{format_}[0]" + filename = Path(app.config['UPLOAD_PATH']) / f"{name}{format_}[0]" with Image(filename=filename) as src: - ext = conf['PROCESSED_EXT'] \ - if format_ in conf['NONE_DISPLAY_EXT'] else format_ - with src.convert(ext.replace('.', '')) as img: + if format_ in app.config['PROCESSABLE_EXT']: + format_ = app.config['PROCESSED_EXT'] # pragma: no cover + with src.convert(format_.replace('.', '')) as img: img.transform(resize=f"{size}x{size}>") img.compression_quality = 75 img.save( - filename=Path(conf['RESIZED_IMAGES']) / size / f"{name}{ext}") + filename=Path( + app.config['RESIZED_IMAGES']) / size / f"{name}{format_}") return True def check_processed_image(filename: str) -> bool: file_format = '.' + filename.split('.', 1)[1].lower() try: - if file_format in app.config['ALLOWED_IMAGE_EXT']: + if file_format in g.display_file_ext: return loop_through_processed_folders( filename.rsplit('.', 1)[0].lower(), file_format) @@ -62,8 +60,9 @@ def check_processed_image(filename: str) -> bool: def loop_through_processed_folders(name: str, file_format: str) -> bool: - ext = app.config['PROCESSED_EXT'] \ - if file_format in app.config['NONE_DISPLAY_EXT'] else file_format + ext = file_format + if file_format in app.config['PROCESSABLE_EXT']: + ext = app.config['PROCESSED_EXT'] # pragma: no cover for size in app.config['IMAGE_SIZE'].values(): path = Path(app.config['RESIZED_IMAGES']) / size / f"{name}{ext}" if not path.is_file() \ @@ -97,7 +96,6 @@ def delete_orphaned_resized_images() -> None: def create_resized_images() -> None: from openatlas.models.entity import Entity - for entity in Entity.get_by_class('file'): - if entity.id in g.files: - if entity.get_file_extension() in app.config['ALLOWED_IMAGE_EXT']: - resize_image(f"{entity.id}{entity.get_file_extension()}") + for e in Entity.get_by_class('file'): + if e.id in g.files and e.get_file_ext() in g.display_file_ext: + resize_image(f"{e.id}{e.get_file_ext()}") diff --git a/openatlas/display/tab.py b/openatlas/display/tab.py index 246430f55..6dfb4cd9e 100644 --- a/openatlas/display/tab.py +++ b/openatlas/display/tab.py @@ -118,12 +118,12 @@ def set_buttons(self, name: str, entity: Optional[Entity] = None) -> None: g.classes[item].label, url_for('insert', class_=item, origin_id=id_))) elif name == 'artifact': - if (entity and entity.class_.name in + if entity and entity.class_.name in \ ['place', 'artifact', 'human_remains', 'feature', - 'stratigraphic_unit']): + 'stratigraphic_unit']: self.buttons.append( button(_('add subunit'), url_for('add_subunit', super_id=id_))) diff --git a/openatlas/display/util.py b/openatlas/display/util.py index 97ce708d7..9a1a7b89c 100644 --- a/openatlas/display/util.py +++ b/openatlas/display/util.py @@ -4,6 +4,7 @@ import os import re import smtplib +import subprocess from datetime import datetime, timedelta from email.header import Header from email.mime.text import MIMEText @@ -65,8 +66,8 @@ def ext_references(links: list[Link]) -> str: f'{system.resolver_url}{link_.description}', external=True) if system.resolver_url else link_.description html += \ - f' ({ g.types[link_.type.id].name } ' + _('at') + \ - f' { link(link_.domain) })
' + f' ({g.types[link_.type.id].name} ' + _('at') + \ + f' {link(link_.domain)})
' return html @@ -127,17 +128,12 @@ def format_entity_date( return html + (f" ({comment})" if comment else '') -def profile_image_table_link( - entity: Entity, - file: Entity, - extension: str) -> str: +def profile_image_table_link(entity: Entity, file: Entity, ext: str) -> str: if file.id == entity.image_id: return link( _('unset'), url_for('file_remove_profile_image', entity_id=entity.id)) - if extension in app.config['DISPLAY_FILE_EXTENSIONS'] or ( - g.settings['image_processing'] - and extension in app.config['ALLOWED_IMAGE_EXT']): + if ext in g.display_file_ext: return link( _('set'), url_for('set_profile_image', id_=file.id, origin_id=entity.id)) @@ -231,40 +227,45 @@ def display_menu(entity: Optional[Entity], origin: Optional[Entity]) -> str: def profile_image(entity: Entity) -> str: if not entity.image_id: return '' - path = get_file_path(entity.image_id) - if not path: + if not (path := get_file_path(entity.image_id)): return '' # pragma: no cover - resized = None - size = app.config['IMAGE_SIZE']['thumbnail'] - if g.settings['image_processing'] and check_processed_image(path.name): - if path_ := get_file_path(entity.image_id, size): - resized = url_for('display_file', filename=path_.name, size=size) - url = url_for('display_file', filename=path.name) - src = resized or url - style = f'max-width:{g.settings["profile_image_width"]}px;' - ext = app.config["DISPLAY_FILE_EXTENSIONS"] - if resized: - style = f'max-width:{app.config["IMAGE_SIZE"]["thumbnail"]}px;' - ext = app.config["ALLOWED_IMAGE_EXT"] + + src = url_for('display_file', filename=path.name) + url = src + width = g.settings["profile_image_width"] + if app.config['IIIF']['enabled'] and check_iiif_file_exist(entity.id): + url = url_for('view_iiif', id_=entity.id) + iiif_ext = '.tiff' if app.config['IIIF']['conversion'] \ + else g.files[entity.id].suffix + src = \ + f"{app.config['IIIF']['url']}{entity.id}{iiif_ext}" \ + f"/full/!{width},{width}/0/default.jpg" + elif g.settings['image_processing'] and check_processed_image(path.name): + if path_ := get_file_path( + entity.image_id, + app.config['IMAGE_SIZE']['thumbnail']): + src = url_for( + 'display_file', + size=app.config['IMAGE_SIZE']['thumbnail'], + filename=path_.name) + external = False if entity.class_.view == 'file': - html = \ - '

' + _('no preview available') + '

' - if path.suffix.lower() in ext: - html = link( - f'image', - url, - external=True) + external = True + if path.suffix.lower() not in g.display_file_ext: + return '

' + _('no preview available') + '

' else: - html = link( - f'image', - url_for('view', id_=entity.image_id)) - return f'{html}' + url = url_for('view', id_=entity.image_id) + html = link( + f'{entity.name}', + url, + external=external) + return html @app.template_filter() def get_js_messages(lang: str) -> str: js_message_file = Path('static') / 'vendor' / 'jquery_validation_plugin' \ - / f'messages_{lang}.js' + / f'messages_{lang}.js' if not (Path(app.root_path) / js_message_file).is_file(): return '' return f'' @@ -313,7 +314,7 @@ def get_backup_file_data() -> dict[str, Any]: latest_file = None latest_file_date = None for file in [ - f for f in path.iterdir() + f for f in path.iterdir() if (path / f).is_file() and f.name != '.gitignore']: file_date = datetime.utcfromtimestamp((path / file).stat().st_ctime) if not latest_file_date or file_date > latest_file_date: @@ -339,7 +340,7 @@ def get_base_table_data(entity: Entity, show_links: bool = True) -> list[Any]: data.append(entity.standard_type.name if entity.standard_type else '') if entity.class_.name == 'file': data.append(entity.get_file_size()) - data.append(entity.get_file_extension()) + data.append(entity.get_file_ext()) if entity.class_.view in ['actor', 'artifact', 'event', 'place']: data.append(entity.first) data.append(entity.last) @@ -426,11 +427,11 @@ def system_warnings(_context: str, _unneeded_string: str) -> str: warnings.append( f"Database version {app.config['DATABASE_VERSION']} is needed but " f"current version is {g.settings['database_version']}") - for path in app.config['WRITABLE_PATHS']: - if not os.access(path, os.W_OK): - warnings.append( - '

' + _('directory not writable') + - f" {str(path).replace(app.root_path, '')}

") + if app.config['IIIF']['enabled']: + if path := app.config['IIIF']['path']: + check_write_access(path, warnings) + for path in g.writable_paths: + check_write_access(path, warnings) if is_authorized('admin'): from openatlas.models.user import User user = User.get_by_username('OpenAtlas') @@ -448,6 +449,14 @@ def system_warnings(_context: str, _unneeded_string: str) -> str: f'{"
".join(warnings)}' if warnings else '' +def check_write_access(path: Path, warnings: list[str]) -> list[str]: + if not os.access(path, os.W_OK): + warnings.append( + '

' + _('directory not writable') + + f" {str(path).replace(app.root_path, '')}

") + return warnings + + @app.template_filter() def tooltip(text: str) -> str: if not text: @@ -466,7 +475,7 @@ def get_file_path( return None ext = g.files[id_].suffix if size: - if ext in app.config['NONE_DISPLAY_EXT']: + if ext in app.config['PROCESSABLE_EXT']: ext = app.config['PROCESSED_EXT'] # pragma: no cover path = app.config['RESIZED_IMAGES'] / size / f"{id_}{ext}" return path if os.path.exists(path) else None @@ -639,8 +648,9 @@ def manual(site: str) -> str: return '' return \ '
' + f'href="/static/manual/{site}.html" class="manual" ' \ + f'target="_blank" rel="noopener noreferrer">' \ + f'' @app.template_filter() @@ -740,3 +750,37 @@ def get_entities_linked_to_type_recursive( for sub_id in g.types[id_].subs: get_entities_linked_to_type_recursive(sub_id, data) return data + + +def check_iiif_activation() -> bool: + iiif = app.config['IIIF'] + return bool(iiif['enabled'] and os.access(Path(iiif['path']), os.W_OK)) + + +def check_iiif_file_exist(id_: int) -> bool: + if app.config['IIIF']['conversion']: + return get_iiif_file_path(id_).is_file() + return bool(get_file_path(id_)) + + +def get_iiif_file_path(id_: int) -> Path: + ext = '.tiff' if app.config['IIIF']['conversion'] \ + else g.files[id_].suffix + return Path(app.config['IIIF']['path']) / f'{id_}{ext}' + + +def convert_image_to_iiif(id_: int) -> None: + compression = app.config['IIIF']['compression'] \ + if app.config['IIIF']['compression'] in ['deflate', 'jpeg'] \ + else 'deflate' + vips = "vips" if os.name == 'posix' else "vips.exe" + command = \ + (f"{vips} tiffsave {get_file_path(id_)} {get_iiif_file_path(id_)} " + f"--tile --pyramid --compression {compression} " + f"--tile-width 128 --tile-height 128") + try: + process = subprocess.Popen(command, shell=True) + process.wait() + flash(_('IIIF converted'), 'info') + except Exception as e: # pragma: no cover + flash(f"{_('failed to convert image')}: {e}", 'error') diff --git a/openatlas/models/entity.py b/openatlas/models/entity.py index 286358b9b..443735154 100644 --- a/openatlas/models/entity.py +++ b/openatlas/models/entity.py @@ -335,7 +335,7 @@ def get_file_size(self) -> str: return convert_size(g.files[self.id].stat().st_size) \ if self.id in g.files else 'N/A' - def get_file_extension(self) -> str: + def get_file_ext(self) -> str: return g.files[self.id].suffix if self.id in g.files else 'N/A' @staticmethod @@ -379,9 +379,8 @@ def get_by_view( def get_display_files() -> list[Entity]: entities = [] for row in Db.get_by_class('file', types=True): - ext = g.files[row['id']].suffix \ - if row['id'] in g.files else 'N/A' - if ext in app.config['DISPLAY_FILE_EXTENSIONS']: + ext = g.files[row['id']].suffix if row['id'] in g.files else 'N/A' + if ext in app.config['DISPLAY_FILE_EXT']: entities.append(Entity(row)) return entities diff --git a/openatlas/static/setup.py b/openatlas/static/setup.py index 608106bff..58e2f2ab1 100644 --- a/openatlas/static/setup.py +++ b/openatlas/static/setup.py @@ -26,6 +26,7 @@ "leaflet.fullscreen": "2.2.0", "leaflet.markercluster": "^1.5.3", "leaflet": "^1.7.1", + "mirador": "^3.3.0", "save-svg-as-png": "^1.4.17", "tinymce": "^5.10.3", } diff --git a/openatlas/templates/iiif.html b/openatlas/templates/iiif.html new file mode 100644 index 000000000..0d8a46be5 --- /dev/null +++ b/openatlas/templates/iiif.html @@ -0,0 +1,31 @@ +
+
+ + +
\ No newline at end of file diff --git a/openatlas/views/admin.py b/openatlas/views/admin.py index dc6d9ac75..8571b33b4 100644 --- a/openatlas/views/admin.py +++ b/openatlas/views/admin.py @@ -25,7 +25,7 @@ from openatlas.display.util import ( button, convert_size, display_form, display_info, format_date, get_file_path, is_authorized, link, manual, required_group, sanitize, - send_mail, uc_first) + send_mail, uc_first, convert_image_to_iiif) from openatlas.forms.field import SubmitField from openatlas.forms.setting import ( ApiForm, ContentForm, FilesForm, GeneralForm, LogForm, MailForm, MapForm, @@ -616,7 +616,7 @@ def admin_logo(id_: Optional[int] = None) -> Union[str, Response]: entity.name, link(entity.standard_type), entity.get_file_size(), - entity.get_file_extension(), + entity.get_file_ext(), entity.description, date]) return render_template( diff --git a/openatlas/views/entity.py b/openatlas/views/entity.py index 518346168..778a96304 100644 --- a/openatlas/views/entity.py +++ b/openatlas/views/entity.py @@ -15,7 +15,7 @@ from openatlas.display.image_processing import resize_image from openatlas.display.util import ( button, get_base_table_data, get_file_path, is_authorized, link, - required_group) + required_group, get_iiif_file_path, check_iiif_file_exist) from openatlas.forms.base_manager import BaseManager from openatlas.forms.form import get_manager from openatlas.forms.util import was_modified @@ -123,11 +123,11 @@ def update(id_: int, copy: Optional[str] = None) -> Union[str, Response]: if entity.class_.view in ['artifact', 'place']: manager.entity.image_id = manager.entity.get_profile_image_id() if not manager.entity.image_id: - for l in manager.entity.get_links('P67', inverse=True): - if l.domain.class_.view == 'file' \ - and get_base_table_data(l.domain)[3] \ - in app.config['DISPLAY_FILE_EXTENSIONS']: - manager.entity.image_id = l.domain.id + for link_ in manager.entity.get_links('P67', inverse=True): + if link_.domain.class_.view == 'file' \ + and get_base_table_data(link_.domain)[3] \ + in g.display_file_ext: + manager.entity.image_id = link_.domain.id break return render_template( 'entity/update.html', @@ -211,7 +211,7 @@ def insert_files(manager: BaseManager) -> Union[str, Response]: ext = secure_filename(file.filename).rsplit('.', 1)[1].lower() path = app.config['UPLOAD_PATH'] / name file.save(str(path)) - if f'.{ext}' in app.config['DISPLAY_FILE_EXTENSIONS']: + if f'.{ext}' in g.display_file_ext: call(f'exiftran -ai {path}', shell=True) # Fix rotation filenames.append(name) if g.settings['image_processing']: @@ -331,3 +331,6 @@ def delete_files(id_: int) -> None: path.unlink() for resized_path in app.config['RESIZED_IMAGES'].glob(f'**/{id_}.*'): resized_path.unlink() + if app.config['IIIF']['enabled'] and check_iiif_file_exist(id_): + if path := get_iiif_file_path(id_): + path.unlink() diff --git a/openatlas/views/entity_index.py b/openatlas/views/entity_index.py index 6ca2cf84f..d24df5b7b 100644 --- a/openatlas/views/entity_index.py +++ b/openatlas/views/entity_index.py @@ -10,7 +10,7 @@ from openatlas.display.table import Table from openatlas.display.util import ( button, format_date, get_base_table_data, get_file_path, is_authorized, - link, manual, required_group) + link, manual, required_group, check_iiif_file_exist) from openatlas.models.entity import Entity from openatlas.models.gis import Gis @@ -42,7 +42,7 @@ def get_table(view: str) -> Table: if view == 'file': table.order = [[0, 'desc']] table.header = ['date'] + table.header - if g.settings['image_processing'] \ + if (g.settings['image_processing'] or app.config['IIIF']['enabled']) \ and current_user.settings['table_show_icons']: table.header.insert(1, _('icon')) for entity in Entity.get_by_class('file', types=True): @@ -51,9 +51,10 @@ def get_table(view: str) -> Table: link(entity), link(entity.standard_type), entity.get_file_size(), - entity.get_file_extension(), + entity.get_file_ext(), entity.description] - if g.settings['image_processing'] \ + if (g.settings['image_processing'] + or app.config['IIIF']['enabled']) \ and current_user.settings['table_show_icons']: data.insert(1, file_preview(entity.id)) table.rows.append(data) @@ -77,15 +78,22 @@ def get_table(view: str) -> Table: def file_preview(entity_id: int) -> str: size = app.config['IMAGE_SIZE']['table'] - parameter = f"loading='lazy' alt='image' width='{size}'" + param = f"loading='lazy' alt='image' max-width='100px' max-height='100px'" + if app.config['IIIF']['enabled'] and check_iiif_file_exist(entity_id): + ext = '.tiff' if app.config['IIIF']['conversion'] \ + else g.files[entity_id].suffix + url = (f"{app.config['IIIF']['url']}{entity_id}{ext}" + f"/full/!100,100/0/default.jpg") + return f"" \ + if ext in g.display_file_ext else '' if icon_path := get_file_path( entity_id, app.config['IMAGE_SIZE']['table']): url = url_for('display_file', filename=icon_path.name, size=size) - return f"" + return f"" path = get_file_path(entity_id) if path and check_processed_image(path.name): if icon := get_file_path(entity_id, app.config['IMAGE_SIZE']['table']): url = url_for('display_file', filename=icon.name, size=size) - return f"" + return f"" return '' diff --git a/openatlas/views/file.py b/openatlas/views/file.py index eae768718..1bd295d49 100644 --- a/openatlas/views/file.py +++ b/openatlas/views/file.py @@ -6,7 +6,7 @@ from werkzeug.wrappers import Response from openatlas import app -from openatlas.display.util import required_group +from openatlas.display.util import required_group, convert_image_to_iiif from openatlas.forms.form import get_table_form from openatlas.models.entity import Entity @@ -66,3 +66,23 @@ def file_add(id_: int, view: str) -> Union[str, Response]: [_(entity.class_.view), url_for('index', view=entity.class_.view)], entity, f"{_('link')} {_(view)}"]) + + +@app.route('/file/convert_iiif/', methods=['GET']) +@required_group('contributor') +def make_iiif_available(id_: int) -> Response: + convert_image_to_iiif(id_) + return redirect(url_for('view', id_=id_)) + + +@app.route('/view_iiif/', methods=['GET']) +@required_group('contributor') +def view_iiif(id_: int) -> str: + return render_template( + 'iiif.html', + manifest_url= + url_for( + 'api.iiif_manifest', + id_=id_, + version=app.config['IIIF']['version'], + _external=True)) diff --git a/sphinx/source/entity/file.rst b/sphinx/source/entity/file.rst index befcc6aa1..abf1ab0ba 100644 --- a/sphinx/source/entity/file.rst +++ b/sphinx/source/entity/file.rst @@ -52,7 +52,7 @@ extensions and can be changed in the configuration file .. code-block:: python - DISPLAY_FILE_EXTENSIONS = ['.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', '.svg'] + DISPLAY_FILE_EXT = ['.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', '.svg'] Logo ---- diff --git a/tests/test_file.py b/tests/test_file.py index c949381c4..815d3cfcc 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Any from flask import url_for @@ -20,7 +21,7 @@ def test_file(self) -> None: / 'static' / 'images' / 'layout' / 'logo.png' with open(logo, 'rb') as img_1, open(logo, 'rb') as img_2: - rv = self.app.post( + rv: Any = self.app.post( url_for('insert', class_='file', origin_id=place.id), data={'name': 'OpenAtlas logo', 'file': [img_1, img_2]}, follow_redirects=True) @@ -93,7 +94,7 @@ def test_file(self) -> None: assert b'File keeper' in rv.data rv = self.app.get(url_for('update', id_=place.id)) - assert b'alt="image"' in rv.data + assert b'File keeper' in rv.data rv = self.app.post( url_for('entity_add_file', id_=get_hierarchy('Sex').subs[0]), @@ -101,6 +102,54 @@ def test_file(self) -> None: follow_redirects=True) assert b'Updated file' in rv.data + rv = self.app.get(url_for('view', id_=file_id)) + assert b'Logo' in rv.data + + rv = self.app.get(url_for('view', id_=file_id)) + assert b'enable IIIF view' in rv.data + + rv = self.app.get( + url_for('make_iiif_available', id_=file_id), + follow_redirects=True) + assert b'IIIF converted' in rv.data + + rv = self.app.get(url_for('view', id_=file_id)) + assert b'view in IIIF' in rv.data + + rv = self.app.get( + url_for( + 'api.iiif_manifest', + id_=file_id, + version=app.config['IIIF']['version'])) + rv = rv.get_json() + assert bool(rv['label'] == 'Updated file') + + rv = self.app.get(url_for('api.iiif_sequence', id_=file_id)) + rv = rv.get_json() + assert bool(str(file_id) in rv['@id']) + rv = self.app.get(url_for('api.iiif_image', id_=file_id)) + rv = rv.get_json() + assert bool(str(file_id) in rv['@id']) + rv = self.app.get(url_for('api.iiif_canvas', id_=file_id)) + rv = rv.get_json() + assert bool(str(file_id) in rv['@id']) + + rv = self.app.get(url_for('view_iiif', id_=file_id)) + assert b'Mirador' in rv.data + + rv = self.app.get(url_for('view', id_=place.id)) + assert b'/full/!100,100/0/default.jpg' in rv.data + + app.config['IIIF']['conversion'] = False + rv = self.app.get(url_for('view', id_=place.id)) + assert b'/full/!100,100/0/default.jpg' in rv.data + + app.config['IIIF']['activate'] = False + rv = self.app.get(url_for('view', id_=place.id)) + assert b'Logo' in rv.data + + app.config['IIIF']['activate'] = True + app.config['IIIF']['conversion'] = True for file in files: rv = self.app.get( url_for('delete', id_=file.id), diff --git a/tests/test_index.py b/tests/test_index.py index 863fdc007..bd2db64ea 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1,6 +1,6 @@ from pathlib import Path -from flask import url_for +from flask import url_for, g from openatlas import app from tests.base import TestBaseCase @@ -27,8 +27,9 @@ def test_index(self) -> None: self.app.get(url_for('set_locale', language='en')) - app.config['WRITABLE_PATHS'].append(Path(app.root_path) / 'error') + g.writable_paths.append(Path(app.root_path) / 'error') app.config['DATABASE_VERSION'] = 'error' + app.config['EXPORT_PATH'] = Path('error') rv = self.app.get(url_for('view', id_=666), follow_redirects=True) assert b'teapot' in rv.data assert b'OpenAtlas with default password is still' in rv.data