Merge branch 'develop' into feature_docker_k8s_merge

# Conflicts:
#	.github/workflows/starter.yaml
#	install/Dockerfile

.pylintrc

@@ -3,7 +3,6 @@ disable=C0111, broad-except, cyclic-import, duplicate-code
 good-names=f,i,j,k,ex,Run,_,e,js,bc,id,rv,ip,to  # default is i,j,k,ex,Run,_
 ignore=openatlas.wsgi
 min-public-methods=0
---disable=W cidoc_rtfs_parser
 
 [FORMAT]
 max-line-length=79

README.md

@@ -26,7 +26,9 @@ the [DPP](https://dpp.oeaw.ac.at/) project.
 
 # Installation
 
-Please refer to [install.md](install.md) for requirements and installation.
+Please refer to [install.md](install.md) for requirements and installation and
+to [upgrade.md](install/upgrade/upgrade.md) when upgrading to newer
+versions.
 
 # Contributing
 

config/api.py

@@ -1,8 +1,9 @@
 API_VERSIONS = ['0.3', '0.4']
 
 API_CONTEXT = {
-    'LPF': 'https://raw.githubusercontent.com/LinkedPasts/linked-places/'
-           'master/linkedplaces-context-v1.1.jsonld',
+    'LPF':
+        'https://raw.githubusercontent.com/LinkedPasts/linked-places/'
+        'master/linkedplaces-context-v1.1.jsonld',
     'LOUD': 'https://linked.art/ns/v1/linked-art.json'}
 
 CORS_ALLOWANCE = '*'  # Cross-Origin source (CORS)
@@ -28,7 +29,10 @@
     'entitySystemClass', 'typeName', 'typeNameWithSubs',
     'beginFrom', 'beginTo', 'endFrom', 'endTo']
 INT_CATEGORIES: list[str] = [
-    'entityID', 'typeID', 'typeIDWithSubs', 'relationToID']
+    'entityID',
+    'typeID',
+    'typeIDWithSubs',
+    'relationToID']
 SET_CATEGORIES: list[str] = ['valueTypeID']
 VALID_CATEGORIES: list[str] = [
     *STR_CATEGORIES,

config/database_versions.py

@@ -1,5 +1,6 @@
-# Used for automatic database upgrades and database version checks
+# Used for automatic database upgrades and version checks
 DATABASE_VERSIONS = [
+    '8.0.0',
     '7.17.0',
     '7.16.0',
     '7.15.0',

config/default.py

@@ -3,7 +3,7 @@
 
 from config.database_versions import DATABASE_VERSIONS
 
-VERSION = '7.18.0'
+VERSION = '8.0.0'
 DATABASE_VERSION = DATABASE_VERSIONS[0]
 DEMO_MODE = False  # If activated some options are disabled, login is prefilled
 DEBUG = False

docker-compose.yaml

@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   postgres:
-    image: "postgis/postgis:13-3.3"
+    image: "postgis/postgis:13-3.4"
     environment:
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}

install.md

@@ -2,17 +2,17 @@
 Some knowledge about package installation, web server and database
 configuration will be needed.
 
-This software was developed and tested on Linux/Debian 11.5
-(codename "bullseye") and the easiest way to install would be on a
-[Debian](https://www.debian.org/) 11.5 system following these instructions.
+This software was developed and tested on Linux/Debian 12.2
+(codename "bookwom") and the easiest way to install would be on a
+[Debian](https://www.debian.org/) 12.2 system following these instructions.
 
 Another (experimental) way to install it would be via
 [Docker](https://www.docker.com/).
 For more information take a look at the end of this document.
 
-It also may work on other Linux distributions or even on non Linux systems, and
-we provided a [requirements.txt](requirements.txt). But it is experimental too
-and would need substantially more knowledge about server administration.
+It may also work on other Linux distributions or even non Linux systems with
+using the [requirements.txt](requirements.txt), but substantially more
+knowledge about server administration would be needed.
 
 Feel free to also consult our own
 [documentation](https://redmine.openatlas.eu/projects/uni/wiki/Debian_server_installation)
@@ -24,21 +24,21 @@ that we are using to set up Debian servers for OpenAtlas installations.
 * [Docker](#Docker) (alternative installation method)
 
 ## Requirements
-### Python 3.9 and Flask 1.1.2
+### Python 3.11 and Flask 2.2.2
     sudo apt install python3 python3-bcrypt python3-dateutil python3-psycopg2 python3-fuzzywuzzy python3-flask
     sudo apt install python3-flask-babel python3-flask-login python3-flaskext.wtf python3-markdown python3-numpy
     sudo apt install python3-pandas python3-jinja2 python3-flask-cors python3-flask-restful p7zip-full
     sudo apt install python3-wand python3-rdflib python3-dicttoxml python3-rdflib-jsonld python3-flasgger
-    sudo apt install python3-requests exiftran
+    sudo apt install python3-requests exiftran python3-email-validator
 
 ### Apache 2.4
     sudo apt install apache2 libapache2-mod-wsgi-py3
 
-### PostgreSQL 13 and PostGIS 3
-    sudo apt install postgresql postgresql-13-postgis-3 postgresql-13-postgis-3-scripts
+### PostgreSQL 15 and PostGIS 3
+    sudo apt install postgresql postgresql-15-postgis-3 postgresql-15-postgis-3-scripts
 
 ### gettext, pip, npm
-    sudo apt install gettext npm python3-pip
+    sudo apt install gettext npm 
 
 ## Installation
 ### Files
@@ -50,10 +50,8 @@ adapt them accordingly as regular user:
 ### Frontend libraries
 Execute this lines as regular user too:
 
-    pip3 install calmjs
     cd openatlas/static
-    pip3 install -e ./
-    ~/.local/bin/calmjs npm --install openatlas
+    npm install
 
 ### Database
 Executed statements below as **postgres** user.
@@ -135,6 +133,7 @@ high-quality, attributed digital objects online at scale. Be aware that:
 ### Installation
 
     sudo apt install iipimage-server libvips-tools
+    sudo a2enmod fcgid
     sudo service apache2 restart
 
 You can test http://your.server/iipsrv/iipsrv.fcgi to see if it runs.

install/3_data_web.sql

@@ -15,14 +15,16 @@ INSERT INTO web.user (username, password, active, email, group_id) VALUES (
   (SELECT id FROM web.group WHERE name = 'admin'));
 
 INSERT INTO web.settings (name, value) VALUES
-  ('database_version', '7.17.0'),
+  ('database_version', '8.0.0'),
   ('api_public', ''),
   ('default_language', 'en'),
   ('table_rows', '25'),
   ('failed_login_forget_minutes', '1'),
   ('failed_login_tries', '3'),
   ('file_upload_max_size', '10'),
   ('file_upload_allowed_extension', 'gif jpeg jpg pdf png txt zip'),
+  ('frontend_website_url', ''),
+  ('frontend_resolver_url', ''),
   ('geonames_username', 'openatlas'),
   ('iiif', ''),
   ('iiif_path', ''),

install/Dockerfile

@@ -1,16 +1,16 @@
-FROM debian:11
+FROM debian:12
 
 RUN --mount=type=cache,target=/var/cache/apt \
     export DEBIAN_FRONTEND=noninteractive &&\
     apt update && apt dist-upgrade -y --no-install-recommends &&\
     apt install -y --no-install-recommends python3 python3-bcrypt python3-dateutil python3-psycopg2 python3-fuzzywuzzy python3-flask &&\
     apt install -y --no-install-recommends python3-flask-babel python3-flask-login python3-flaskext.wtf python3-markdown python3-numpy &&\
     apt install -y --no-install-recommends python3-pandas python3-jinja2 python3-flask-cors python3-flask-restful p7zip-full &&\
-    apt install -y --no-install-recommends python3-wand python3-rdflib python3-requests python3-dicttoxml python3-rdflib-jsonld python3-flasgger &&\
+    apt install -y --no-install-recommends python3-wand python3-rdflib python3-requests python3-dicttoxml python3-flasgger &&\
     apt install -y --no-install-recommends apache2 libapache2-mod-wsgi-py3 python3-coverage python3-nose exiftran &&\
-    apt install -y --no-install-recommends iipimage-server libvips-tools &&\
-    apt install -y --no-install-recommends gettext npm python3-pip git postgresql-client-13 &&\
-    apt install -y --no-install-recommends dos2unix locales locales-all w3c-sgml-lib vim nano &&\
+    apt install -y --no-install-recommends iipimage-server libvips-tools python3-email-validator &&\
+    apt install -y --no-install-recommends gettext npm python3-pip git postgresql-client-15 &&\
+    apt install -y --no-install-recommends dos2unix locales locales-all vim w3c-sgml-lib vim nano &&\
     mkdir -p /var/www/openatlas /var/www/.cache /var/www/.local /var/www/.npm &&\
     chown -R www-data:www-data /var/www/.cache /var/www/.local /var/www/.npm /var/log/apache2 /var/run/apache2
 RUN cp -rp /usr/lib/iipimage-server/ /var/www/iipsrv/ &&\
@@ -29,11 +29,10 @@ RUN cd /var/www/openatlas && cp install/entrypoint.sh /entrypoint.sh &&\
     sed -i 's~</VirtualHost>~ErrorLogFormat "{ \"time\":\"%{%Y-%m-%d}tT%{%T}t.%{msec_frac}tZ\", \"function\" : \"[%-m:%l]\" , \"process\" : \"[pid %P:tid %T]\" , \"message\" : \"%M\" ,\ \"referer\"\ : \"%{Referer}i\" },"\nLogFormat "{ \"time\":\"%{%Y-%m-%d}tT%{%T}t.%{msec_frac}tZ\", \"process\":\"%D\", \"filename\":\"%f\", \"remoteIP\":\"%a\", \"host\":\"%V\", \"request\":\"%U\", \"query\":\"%q\", \"method\":\"%m\", \"status\":\"%>s\", \"userAgent\":\"%{User-agent}i\", \"referer\":\"%{Referer}i\" }," combined\nCustomLog /dev/stdout combined\n</VirtualHost>~' /etc/apache2/sites-available/000-default.conf
 USER www-data
 RUN cd /var/www/openatlas &&\
-    pip3 install calmjs &&\
     cd openatlas/static &&\
-    pip3 install -e ./ &&\
-    ~/.local/bin/calmjs npm --install openatlas
+    npm install
 RUN a2enmod iipsrv &&\
+    a2enmod fcgid &&\
     service apache2 restart
 
 EXPOSE 8080

install/crm/cidoc_rtfs_parser.py

@@ -30,7 +30,7 @@
 #
 
 import time
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 import psycopg2.extras
 from rdflib import URIRef
@@ -50,7 +50,7 @@
 DATABASE_PASS = 'CHANGE ME'
 
 
-def connect() -> psycopg2.connect:
+def connect() -> Any:
     return psycopg2.connect(
         database=DATABASE_NAME,
         user=DATABASE_USER,
@@ -89,11 +89,16 @@ def import_cidoc() -> None:
         except Exception:
             print(f'Not able to parse subject: {subject}')
             continue
-        item = Item(code, name.replace('_', ' '), graph.comment(subject))
+        item = Item(
+            code,
+            name.replace('_', ' '),
+            graph.comment(subject))  # type: ignore
 
         # Translations
         for language in ['de', 'en', 'fr', 'ru', 'el', 'pt', 'zh']:
-            translation = graph.preferredLabel(subject, lang=language)
+            translation = graph.preferredLabel(
+                subject,
+                lang=language)  # type: ignore
             if translation and translation[0][1]:
                 item.label[language] = translation[0][1]
 

install/data_test.sql

@@ -15,6 +15,7 @@ VALUES
   ((SELECT id FROM web.user WHERE username = 'Alice'), 'entity_show_import', 'True'),
   ((SELECT id FROM web.user WHERE username = 'Alice'), 'newsletter', 'True'),
   ((SELECT id FROM web.user WHERE username = 'Alice'), 'module_time', 'True'),
+  ((SELECT id FROM web.user WHERE username = 'Alice'), 'table_rows', '25'),
   ((SELECT id FROM web.user WHERE username = 'Alice'), 'table_show_icons', 'True');
 
 -- Citation example
@@ -33,3 +34,4 @@ UPDATE web.settings SET value = '/var/www/iipsrv' WHERE name = 'iiif_path';
 UPDATE web.settings SET value = 'http://localhost/iiif/' WHERE name = 'iiif_url';
 UPDATE web.settings SET value = '2' WHERE name = 'iiif_version';
 UPDATE web.settings SET value = 'deflate' WHERE name = 'iiif_conversion';
+UPDATE web.settings SET value = 'https://frontend-demo.openatlas.eu/entity/' WHERE name = 'frontend_resolver_url';

install/data_test_api.sql

@@ -1,8 +1,3 @@
-
--- Content example for API
-INSERT INTO web.i18n (name, language, text) VALUES ('intro_for_frontend', 'en', 'This is English');
-INSERT INTO web.i18n (name, language, text) VALUES ('intro_for_frontend', 'de', 'Das ist Deutsch');
-
 INSERT INTO model.entity (
   cidoc_class_code, openatlas_class_name, name, description, begin_from, begin_to, begin_comment,
   end_from, end_to, end_comment, created, modified

install/upgrade/8.0.0.sql

@@ -0,0 +1,21 @@
+-- Upgrade 7.17.x to 8.0.0
+-- Be sure to backup the database and read the upgrade notes before executing.
+
+BEGIN;
+
+-- Raise database version
+UPDATE web.settings SET value = '8.0.0' WHERE name = 'database_version';
+
+-- #2054: Remove content for presentation sites
+DELETE FROM web.i18n WHERE name IN (
+    'contact_for_frontend',
+    'intro_for_frontend',
+    'legal_notice_for_frontend',
+    'site_name_for_frontend');
+
+-- #2096: Add presentation site link in backend
+INSERT INTO web.settings (name, value) VALUES
+    ('frontend_website_url', ''),
+    ('frontend_resolver_url', '');
+
+END;

install/upgrade/upgrade.md

@@ -1,5 +1,5 @@
 ## INFO
-Beginning from 6.6.0 you can use the database update script which takes care
+From 6.6.0 on you can use the database update script which takes care
 of database changes even over multiple versions. A backup is made before
 changes happen. You still should read the upgrade notes about important
 information.
@@ -18,8 +18,28 @@ then run the database upgrade script, then restart Apache:
     sudo python3 install/upgrade/database_upgrade.py
     sudo service apache2 restart
 
-### 7.17.x to 7.17.4
-A code base update (e.g. with git pull) and a webserver restart is sufficient.
+### 7.17.x to 8.0.0
+This is a major upgrade which utilizes newer versions of underlying software.
+Please consult the install.md about installation. In case you are using a
+Debian system, feel free to use our own upgrade notes from the issue
+description: https://redmine.openatlas.eu/issues/2038, which is already
+tested.
+
+8.0.0.sql is needed but will be taken care of by the database upgrade script.
+
+#### Breaking changes
+**Removal of frontend content management**
+
+Because the new presentation site functionality doesn't require content
+management in the backend anymore, this functionality was removed.
+In case older frontend versions are still in use they would have to be
+switched to the new one (OpenAtlas Discovery) or manually adapted.
+
+**New stable API version 0.4**
+
+The current stable API version is **0.4**. Support for API versions **0.3**
+is dropped with this release including the frontend content queries mentioned
+above.
 
 ### 7.16.x to 7.17.0
 7.17.0.sql is needed but will be taken care of by the database upgrade script.

openatlas/__init__.py

@@ -1,5 +1,5 @@
 import locale
-from typing import Optional
+from typing import Any, Optional
 
 from flask import Flask, Response, g, request, session
 from flask_babel import Babel
@@ -109,5 +109,5 @@ def apply_caching(response: Response) -> Response:
 
 
 @app.teardown_request
-def teardown_request(_exception: Optional[Exception]) -> None:
+def teardown_request(_exception: Optional[Any]) -> None:
     close_connection()

openatlas/api/api.py

@@ -4,8 +4,9 @@
 from flask_restful import Api
 
 from openatlas import app
-from openatlas.api.routes import entity_routes, deprecated_routes, \
-    display_routes, type_routes, admin_routes, special_routes
+from openatlas.api.routes import (
+    entity_routes,  display_routes, type_routes, admin_routes, special_routes)
+
 
 app.config['SWAGGER'] = {
     'openapi': '3.0.2',
@@ -19,13 +20,7 @@
             "name": "Apache 2.0",
             "url": "https://www.apache.org/licenses/LICENSE-2.0.html"},
         "route": '/openapi.json',
-        "rule_filter": lambda rule: rule.endpoint.startswith('api_04')}, {
-        "endpoint": '03',
-        "license": {
-            "name": "Apache 2.0",
-            "url": "https://www.apache.org/licenses/LICENSE-2.0.html"},
-        "route": '/openapi.json',
-        "rule_filter": lambda rule: rule.endpoint.startswith('api_03')}],
+        "rule_filter": lambda rule: rule.endpoint.startswith('api_04')}],
     "specs_route": "/swagger/"}
 
 app.config['PROPAGATE_EXCEPTIONS'] = True
@@ -40,19 +35,8 @@
 type_routes(api)
 admin_routes(api)
 special_routes(api)
-deprecated_routes(api)
 app.register_blueprint(api_bp)
 
-api_bp_03 = Blueprint('api_03', __name__, url_prefix='/api/0.3')
-api_03 = Api(api_bp_03)
-entity_routes(api_03)
-display_routes(api_03)
-type_routes(api_03)
-admin_routes(api_03)
-special_routes(api_03)
-deprecated_routes(api_03)
-app.register_blueprint(api_bp_03)
-
 api_bp_04 = Blueprint('api_04', __name__, url_prefix='/api/0.4')
 api_04 = Api(api_bp_04)
 entity_routes(api_04)