Merge branch 'develop' into feature_docker_k8s

# Conflicts: # .github/workflows/starter.yaml # install.md
acdh-oeaw · Nov 14, 2023 · 9767bf1 · 9767bf1
2 parents e556ea0 + 56c911e
commit 9767bf1
Show file tree

Hide file tree

Showing 159 changed files with 5,521 additions and 2,448 deletions.
diff --git a/.github/workflows/starter.yaml b/.github/workflows/starter.yaml
@@ -123,11 +123,11 @@ jobs:
           set -x
           cat <<EOF > testing.py
           SERVER_NAME='local.host'
-          DATABASE_NAME='openatlas'
-          DATABASE_USER='openatlas'
+          DATABASE_NAME='${{env.POSTGRES_DB}}'
+          DATABASE_USER='${{env.POSTGRES_USER}}'
           DATABASE_HOST='postgres'
           DATABASE_PORT=5432
-          DATABASE_PASS='verysecret'
+          DATABASE_PASS='${{env.POSTGRES_PASSWORD}}'
           MAIL_PASSWORD='asdQWEtzu123'
           SECRET_KEY='$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c 32;echo;)'  # Used for cookies
           DEBUG = True
@@ -146,8 +146,8 @@ jobs:
           EOF
           sudo chown 33:33 testing.py
           docker cp -a testing.py openatlas-openatlas-1:/var/www/openatlas/instance/
-          docker exec -i openatlas-openatlas-1 /bin/bash -c "cd /var/www/openatlas/instance/ && ls -la && cat testing.py"
-
+          docker exec -i openatlas-openatlas-1 /bin/bash -c "cd /var/www/openatlas/install/ && sed -i 's@http://localhost/iiif/@http://localhost:8080/iiif/@g' data_test.sql"
+          docker exec -i openatlas-openatlas-1 /bin/bash -c "cd /var/www/openatlas/instance/ && ls -la && cat testing.py && cat production.py"
       - name: Run tests
         run: |
           docker exec -i openatlas-openatlas-1 /bin/bash -c "cd /var/www/openatlas/tests && nosetests3 -c .noserc --verbosity=2 --nologcapture --nocapture && echo passed"

diff --git a/config/api.py b/config/api.py
@@ -1,3 +1,5 @@
+API_VERSIONS = ['0.3', '0.4']
+
 API_CONTEXT = {
     'LPF': 'https://raw.githubusercontent.com/LinkedPasts/linked-places/'
            'master/linkedplaces-context-v1.1.jsonld',
@@ -22,12 +24,12 @@
 
 LOGICAL_OPERATOR: list[str] = ['and', 'or']
 STR_CATEGORIES: list[str] = [
-    "entityName", "entityDescription", "entityAliases", "entityCidocClass",
-    "entitySystemClass", "typeName", "typeNameWithSubs",
-    "beginFrom", "beginTo", "endFrom", "endTo"]
+    'entityName', 'entityDescription', 'entityAliases', 'entityCidocClass',
+    'entitySystemClass', 'typeName', 'typeNameWithSubs',
+    'beginFrom', 'beginTo', 'endFrom', 'endTo']
 INT_CATEGORIES: list[str] = [
-    "entityID", "typeID", "typeIDWithSubs", "relationToID"]
-SET_CATEGORIES: list[str] = ["valueTypeID"]
+    'entityID', 'typeID', 'typeIDWithSubs', 'relationToID']
+SET_CATEGORIES: list[str] = ['valueTypeID']
 VALID_CATEGORIES: list[str] = [
     *STR_CATEGORIES,
     *INT_CATEGORIES,

diff --git a/config/database_versions.py b/config/database_versions.py
@@ -1,5 +1,6 @@
 # Used for automatic database upgrades and database version checks
 DATABASE_VERSIONS = [
+    '7.17.0',
     '7.16.0',
     '7.15.0',
     '7.14.0',

diff --git a/config/default.py b/config/default.py
@@ -3,7 +3,7 @@
 
 from config.database_versions import DATABASE_VERSIONS
 
-VERSION = '7.17.0'
+VERSION = '7.18.0'
 DATABASE_VERSION = DATABASE_VERSIONS[0]
 DEMO_MODE = False  # If activated some options are disabled, login is prefilled
 DEBUG = False
@@ -40,13 +40,6 @@
 IMAGE_SIZE = {
     'thumbnail': '200',
     'table': '100'}
-IIIF = {
-    'enabled': False,
-    'path': '',
-    'url': '',
-    'version': 2,
-    'conversion': True,
-    'compression': 'deflate'}  # 'deflate' or 'jpeg'
 
 # Security
 SESSION_COOKIE_SECURE = False  # Should be True in production.py if using HTTPS
@@ -64,3 +57,6 @@
     'button': {
         'primary': 'btn btn-outline-primary btn-sm',
         'secondary': 'btn btn-outline-secondary btn-sm'}}
+
+# Tests
+LOAD_WINDOWS_TEST_SQL = False
diff --git a/install.md b/install.md
@@ -18,57 +18,62 @@ Feel free to also consult our own
 [documentation](https://redmine.openatlas.eu/projects/uni/wiki/Debian_server_installation)
 that we are using to set up Debian servers for OpenAtlas installations.
 
+* [Requirements](#Requirements)
+* [IIIF](#IIIF) (optional)
+* [Tests](#Tests) (optional)
+* [Docker](#Docker) (alternative installation method)
+
 ## Requirements
 ### Python 3.9 and Flask 1.1.2
-    # apt install python3 python3-bcrypt python3-dateutil python3-psycopg2 python3-fuzzywuzzy python3-flask
-    # apt install python3-flask-babel python3-flask-login python3-flaskext.wtf python3-markdown python3-numpy
-    # apt install python3-pandas python3-jinja2 python3-flask-cors python3-flask-restful p7zip-full
-    # apt install python3-wand python3-rdflib python3-dicttoxml python3-rdflib-jsonld python3-flasgger
-    # apt install python3-requests exiftran
+    sudo apt install python3 python3-bcrypt python3-dateutil python3-psycopg2 python3-fuzzywuzzy python3-flask
+    sudo apt install python3-flask-babel python3-flask-login python3-flaskext.wtf python3-markdown python3-numpy
+    sudo apt install python3-pandas python3-jinja2 python3-flask-cors python3-flask-restful p7zip-full
+    sudo apt install python3-wand python3-rdflib python3-dicttoxml python3-rdflib-jsonld python3-flasgger
+    sudo apt install python3-requests exiftran
 
 ### Apache 2.4
-    # apt install apache2 libapache2-mod-wsgi-py3
+    sudo apt install apache2 libapache2-mod-wsgi-py3
 
 ### PostgreSQL 13 and PostGIS 3
-    # apt install postgresql postgresql-13-postgis-3 postgresql-13-postgis-3-scripts
+    sudo apt install postgresql postgresql-13-postgis-3 postgresql-13-postgis-3-scripts
 
 ### gettext, pip, npm
-    # apt install gettext npm python3-pip
+    sudo apt install gettext npm python3-pip
 
 ## Installation
 ### Files
 Copy the files to /var/www/your_site_name or clone OpenAtlas from GitHub and
 adapt them accordingly as regular user:
 
-    $ git clone https://github.com/craws/OpenAtlas.git
+    git clone https://github.com/craws/OpenAtlas.git
 
 ### Frontend libraries
 Execute this lines as regular user too:
 
-    $ pip3 install calmjs
-    $ cd openatlas/static
-    $ pip3 install -e ./
-    $ ~/.local/bin/calmjs npm --install openatlas
+    pip3 install calmjs
+    cd openatlas/static
+    pip3 install -e ./
+    ~/.local/bin/calmjs npm --install openatlas
 
 ### Database
 Executed statements below as **postgres** user.
 
 Create an openatlas database user
 
-    $ createuser openatlas -P
+    createuser openatlas -P
 
 Create an openatlas database, make openatlas the owner of it
 
-    $ createdb openatlas -O openatlas
+    createdb openatlas -O openatlas
 
-Add postgis and unaccent extension to the database
+Add the [PostGIS](https://postgis.net/) and unaccent extension to the database
 
-    $ psql openatlas -c "CREATE EXTENSION postgis; CREATE EXTENSION unaccent;"
+    psql openatlas -c "CREATE EXTENSION postgis; CREATE EXTENSION unaccent;"
 
 Import the SQL files:
 
-    $ cd install
-    $ cat 1_structure.sql 2_data_model.sql 3_data_web.sql 4_data_type.sql | psql -d openatlas -f -
+    cd install
+    cat 1_structure.sql 2_data_model.sql 3_data_web.sql 4_data_type.sql | psql -d openatlas -f -
 
 A user with username **OpenAtlas** is created with the password
 **change_me_PLEASE!**
@@ -79,29 +84,29 @@ admins until this account is changed.
 ### Configuration
 Copy instance/example_production.py to instance/production.py
 
-    $ cp instance/example_production.py instance/production.py
+    cp instance/example_production.py instance/production.py
 
 Add/change values as appropriate. See config.py which settings are available.
 
 ### Apache
 As root copy and adapt install/example_apache.conf for a new vhost, activate
 the site:
 
-    # a2ensite your_sitename
+    sudo a2ensite your_sitename
 
 Test Apache configuration and restart
 
-    # apache2ctl configtest
-    # service apache2 restart
+    sudo apache2ctl configtest
+    sudo service apache2 restart
 
 Make the **files** directory writable for the Apache user, e.g.:
 
-    # chown -R www-data files
+    sudo chown -R www-data files
 
 ### Finishing
 Login with username "OpenAtlas" and password "change_me_PLEASE!" and change the
-password in profile. You may want to check the admin area to set up default site
-settings, email and similar.
+password in profile. You may want to check the admin area to set up default
+site settings, email and similar.
 
 ### Upgrade
 If you later like to upgrade the application be sure to read and follow the
@@ -118,38 +123,70 @@ instance/production.py:
 
     SESSION_COOKIE_SECURE = True
 
-### Tests (optional)
+## IIIF
+
+[IIIF](https://iiif.io/) is a set of open standards for delivering
+high-quality, attributed digital objects online at scale. Be aware that:
+
+* IIIF is **optional** for an OpenAtlas installation
+* Although already working and in use we still consider it **experimental**
+* Enabling IIIF can expose files to the public (without login)
+
+### Installation
+
+    sudo apt install iipimage-server libvips-tools
+    sudo service apache2 restart
+
+You can test http://your.server/iipsrv/iipsrv.fcgi to see if it runs.
+
+    sudo mkdir /var/www/iipsrv
+    sudo cp -p /usr/lib/iipimage-server/iipsrv.fcgi /var/www/iipsrv/
+    sudo chown -R www-data /var/www/iipsrv
+
+### Configuration
+
+Edit the configuration to your needs, see example at
+[install/iipsrv.conf](install/iipsrv.conf) and restart Apache:
+
+    sudo vim /etc/apache2/mods-available/iipsrv.conf
+    sudo service apache2 restart
+
+Further configuration can be done at the IIIF tab in the admin area of the web
+application.
+
+## Tests
 Install required packages:
 
-    # apt install python3-coverage python3-nose
+    sudo apt install python3-coverage python3-nose
 
 As postgres:
 
-    $ createdb openatlas_test -O openatlas
+    createdb openatlas_test -O openatlas
+    psql openatlas_test -c "CREATE EXTENSION postgis; CREATE EXTENSION unaccent;"
 
 Copy instance/example_testing.py to instance/testing.py and adapt as needed:
 
-    $ cp instance/example_testing.py instance/testing.py
+    cp instance/example_testing.py instance/testing.py
 
 Run tests
 
-    $ nosetest3
+    nosetest3
 
 Run tests with coverage
 
-    $ nosetests3 -c tests/.noserc
+    nosetests3 -c tests/.noserc
 
-## Docker 
+## Docker
 Be aware, the [Docker](https://www.docker.com/) installation is experimental
 and is **not** recommended for usage on a productive system.
 
 To run OpenAtlas as a Docker container clone the repository
 
-    $ git clone https://github.com/craws/OpenAtlas.git
+    git clone https://github.com/craws/OpenAtlas.git
 
 Open an CLI in the directory where you cloned OpenAtlas and run
-    
-    $ docker compose up --detach
+
+    docker compose up --detach
 
 After the containers are build an OpenAtlas instance is available under
 **localhost:8080**.
@@ -164,4 +201,4 @@ To restore a database SQL dump uncomment following command in
 database is installed (e.g. delete ./data/db/), as the dump will not be
 executed.
 
-    $ - ./files/export/dump.sql:/docker-entrypoint-initdb.d/dump.sql
+    - ./files/export/dump.sql:/docker-entrypoint-initdb.d/dump.sql
diff --git a/install/3_data_web.sql b/install/3_data_web.sql
@@ -15,7 +15,7 @@ INSERT INTO web.user (username, password, active, email, group_id) VALUES (
   (SELECT id FROM web.group WHERE name = 'admin'));
 
 INSERT INTO web.settings (name, value) VALUES
-  ('database_version', '7.16.0'),
+  ('database_version', '7.17.0'),
   ('api_public', ''),
   ('default_language', 'en'),
   ('table_rows', '25'),
@@ -24,6 +24,11 @@ INSERT INTO web.settings (name, value) VALUES
   ('file_upload_max_size', '10'),
   ('file_upload_allowed_extension', 'gif jpeg jpg pdf png txt zip'),
   ('geonames_username', 'openatlas'),
+  ('iiif', ''),
+  ('iiif_path', ''),
+  ('iiif_url', ''),
+  ('iiif_version', '2'),
+  ('iiif_conversion', ''),
   ('image_processing', 'True'),
   ('log_level', '6'),
   ('logo_file_id', ''),

diff --git a/install/data_test.sql b/install/data_test.sql
@@ -26,3 +26,10 @@ INSERT INTO web.user_log (user_id, entity_id, action) VALUES (2, 6666, 'insert')
 -- Needed to test external reference systems for hierarchies
 INSERT INTO web.reference_system_openatlas_class (reference_system_id, openatlas_class_name) VALUES
   ((SELECT entity_id FROM web.reference_system WHERE name='Wikidata'), 'type');
+
+-- IIIF activation
+UPDATE web.settings SET value = 'True' WHERE name = 'iiif';
+UPDATE web.settings SET value = '/var/www/iipsrv' WHERE name = 'iiif_path';
+UPDATE web.settings SET value = 'http://localhost/iiif/' WHERE name = 'iiif_url';
+UPDATE web.settings SET value = '2' WHERE name = 'iiif_version';
+UPDATE web.settings SET value = 'deflate' WHERE name = 'iiif_conversion';
diff --git a/install/data_test_windows.sql b/install/data_test_windows.sql
@@ -0,0 +1,3 @@
+-- IIIF activation
+UPDATE web.settings SET value = 'c:\iiif' WHERE name = 'iiif_path';
+UPDATE web.settings SET value = 'http://localhost:8182/iiif/2/' WHERE name = 'iiif_url';
diff --git a/install/iipsrv.conf b/install/iipsrv.conf
@@ -19,7 +19,7 @@ ScriptAlias /iiif "/var/www/iipsrv/iipsrv.fcgi"
 </Location>
 
 # Set our environment variables for the IIP server
-FcgidInitialEnv VERBOSITY "6"
+FcgidInitialEnv VERBOSITY "1"
 FcgidInitialEnv LOGFILE "/var/log/iipsrv.log"
 FcgidInitialEnv MAX_IMAGE_CACHE_SIZE "10"
 FcgidInitialEnv JPEG_QUALITY "90"

diff --git a/install/upgrade/7.17.0.sql b/install/upgrade/7.17.0.sql
@@ -0,0 +1,17 @@
+-- Upgrade 7.16.x to 7.17.0
+-- Be sure to backup the database and read the upgrade notes before executing.
+
+BEGIN;
+
+-- Raise database version
+UPDATE web.settings SET value = '7.17.0' WHERE name = 'database_version';
+
+-- #2094: IIIF configuration and documentation
+INSERT INTO web.settings (name, value) VALUES
+    ('iiif', ''),
+    ('iiif_path', ''),
+    ('iiif_url', ''),
+    ('iiif_version', '2'),
+    ('iiif_conversion', '');
+
+END;