diff --git a/.ansible-lint b/.ansible-lint index 27fb9d4..8d34551 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,3 +1,6 @@ +exclude_paths: + - ./.github + - ./docker skip_list: - no-changed-when # Commands should not change things if nothing needs doing - no-handler # Tasks that run when changed should likely be handlers diff --git a/README.md b/README.md index 7956aff..0ac22d6 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ sudo adduser vagrant vboxsf Next, restart the server: ```bash -sudo restart +sudo reboot ``` After rebooting ssh into epos-msl again and the content of the share should be visible within /media/sf_epos! @@ -84,6 +84,21 @@ sudo ln -s /media/sf_epos /var/www/msl_api Check to see if the login page is accessible by navigating to https://epos-msl.ckan.test/webservice/login. A reboot might be needed. +#### CKAN extensions + +The same method can be used to work with local folders containing the msl CKAN extensions by creating shared folders +within the Virtualbox management program as described above. For example after creating a shared folder containing +the 'msl_ckan_core' repo checkout can be used within the epos-msl server by first removing the currently present version +on the server with: +```bash +sudo rm -r /usr/lib/ckan/msl_ckan_core +``` + +Followed by creating a symlink to the shared mount: +```bash +sudo ln -s /media/sf_ckanext-msl_ckan /usr/lib/ckan/msl_ckan_core +``` + ### Seeding test admin panel account(s) The msl_api project contains a specific seeder for adding test admin accounts. Contents can be adjusted to add or @@ -165,6 +180,16 @@ sudo -u www-data /usr/bin/php8.0 artisan db:seed sudo -u www-data /usr/bin/php8.0 artisan storage:link ``` +## Connecting to the mysql database from local machine + +To connect from the host machine to the mysql database used by msl_api use the following command: + +``` +vagrant ssh epos-msl -- -L 3306:127.0.0.1:3306 -N epos-msl +``` + +While running, you can connect to the database using the default credentials. + ## Configuration The main configuration settings are: @@ -179,6 +204,10 @@ The main configuration settings are: |msl_api_asset_url | asset URL for the MSL API web service, e.g. https://epos-catalog.mydomain.nl/webservice | |ckan_api_token | the MSL API uses this value to authenticate to the CKAN API. this should currently be the API key (not API token!) of the ckanadmin account. The current way to use this field is: deploy the catalog using a dummy value for this parameter, log in on CKAN using the ckanadmin account, generate an API key, replace the dummy value in the host\_vars file with the real API key, and run the playbook a second time. |msl_api_app_key | the MSL API application key. The current way to configure this is to deploy the application, generate the app key by running `sudo -u www-data /usr/bin/php8.0 artisan key:generate && sudo -u www-data /usr/bin/php8.0 artisan config:cache` in /var/www/msl\_api. Finally copy the generated key in /var/www/msl\_api/.env to the host\_vars file. +| ckan_install_spatial_plugin | whether to install the ckanext-spatial plugin (default: false) | +| ckan_spatial_plugin_repo | Github repository to use for the ckanext-spatial plugin | +| ckan_spatial_plugin_version | Branch or tag to use for the ckanext-spatial plugin | +| ckan_plugins_editable_mode | Whether to install CKAN plugins in editable mode. This is convenient for development and testing purposes. Enabled on development environment; default value is `false`. | ## CKAN catalog diff --git a/docker/.env b/docker/.env new file mode 100644 index 0000000..3d4ee5e --- /dev/null +++ b/docker/.env @@ -0,0 +1,48 @@ +# Variables in this file will be substituted into docker-compose.yml +# Save a copy of this file as .env and insert your own values. +# Verify correct substitution with "docker-compose config" +# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: +# docker-compose down +# docker rmi -f docker_ckan docker_db +# docker rmi $(docker images -f dangling=true -q) +# docker-compose build +# docker-compose up -d +# docker-compose restart ckan # give the db service time to initialize the db cluster on first run + +# Image: ckan +CKAN_SITE_ID=default + +EPOS_MSL_HOST=epos-msl.ckan +# +# On AWS, your CKAN_SITE_URL is the output of: +# curl -s http://169.254.169.254/latest/meta-data/public-hostname +# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com +# On OSX with Docker for Mac, your CKAN_SITE_URL is +# CKAN_SITE_URL=http://docker.for.mac.localhost:5000 +# When running locally, CKAN_SITE_URL must contain the port +CKAN_SITE_URL=http://localhost:5000 +# +# CKAN_PORT must be available on the host: sudo netstat -na +# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan +CKAN_PORT=5000 +# +# Email settings +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost +# +# Image: db +POSTGRES_PASSWORD=ckan +# +# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432 +# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build +POSTGRES_PORT=5432 +# +# The datastore database will be created in the db container as docs +# Readwrite user/pass will be ckan:POSTGRES_PASSWORD +# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD +DATASTORE_READONLY_PASSWORD=datastore + +MYSQL_ROOT_PASSWORD=testtest diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..221698e --- /dev/null +++ b/docker/README.md @@ -0,0 +1,32 @@ +# EPOS-MSL Docker development setup + +This Docker setup currently has an experimental status, and is in development. + +If you use Windows, ensure that core.autocrlf is set to false in your git client before you clone the EPOS-MSL +repository: _git config --global core.autocrlf false_ Otherwise the Docker images may not work due to line +ending changes. + +## Building the images + +The images of the EPOS-MSL are not yet available in a registry, so you'll have to build them locally first. + +``` +./build-local-images.sh +``` + +## Using the Docker setup + +First add an entry to your `/etc/hosts` file (or equivalent) so that queries for the development setup +interface resolve to your loopback interface. For example: + +``` +127.0.0.1 epos-msl.ckan +``` + +Start the Docker Compose setup: +``` +docker compose up +``` + +Wait until CKAN has started. Then navigate to [https://epos-msl.ckan](https://epos-msl.ckan) in your browser. The +development VM runs with self-signed certificates, so you'll need to accept the security warning. diff --git a/docker/build-local-images.sh b/docker/build-local-images.sh new file mode 100755 index 0000000..0d72b48 --- /dev/null +++ b/docker/build-local-images.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -e + +cd images + +for image in ckan nginx solr msl-api +do cd "$image" + echo "Building image $image ..." + ./build.sh + cd .. +done + +echo "Building images completed." diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..cb89915 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,102 @@ +# This is a locally modified version of the upstream CKAN docker compose file. +# It contains image names for pushing locally built images to a central registry at ECR + +# docker-compose build && docker-compose up -d +# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times. +version: "3" + +volumes: + ckan_config: + ckan_home: + ckan_storage: + ckan_coveragedata: + pg_data: + solr_data: + nginx_config: + nginx_certificates: + mslapi_signal: + mslapi_storage: + +services: + ckan: + container_name: ckan + image: epos-msl-ckan:latest + links: + - db + - solr + - redis + depends_on: + - db + - solr + - redis + ports: + - "0.0.0.0:${CKAN_PORT}:5000" + environment: + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan_default + - CKAN_SOLR_URL=http://solr:8983/solr/ckan + - CKAN_REDIS_URL=redis://redis:6379/1 + - CKAN_SITE_URL=${CKAN_SITE_URL} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + volumes: + - ckan_config:/etc/ckan + - ckan_home:/usr/lib/ckan + - ckan_storage:/var/lib/ckan + - ckan_coveragedata:/coverage + + nginx: + container_name: nginx + image: epos-msl-nginx:latest + environment: + - EPOS_MSL_HOST=${EPOS_MSL_HOST} + ports: + - "18443:443" + volumes: + - nginx_config:/etc/nginx/conf.d + - nginx_certificates:/etc/certificates + + db: + container_name: db + image: postgres:12.20 + environment: + - POSTGRES_DB=ckan_default + - POSTGRES_USER=ckan + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - PGDATA=/var/lib/postgresql/data/db + volumes: + - pg_data:/var/lib/postgresql/data + + mslapi_db: + container_name: mslapi_db + image: mysql:8.0.39-debian + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + + mslapi_webserver: + container_name: mslapi_web + image: epos-msl-api:latest + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MSLAPI_ROLE=WEBSERVER + volumes: + - mslapi_signal:/signal + - mslapi_storage:/storage + + mslapi_worker: + container_name: mslapi_worker + image: epos-msl-api:latest + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MSLAPI_ROLE=QUEUE_WORKER + volumes: + - mslapi_signal:/signal + - mslapi_storage:/storage + + solr: + container_name: solr + image: epos-msl-solr:latest + volumes: + - solr_data:/opt/solr/server/solr/ckan/data + + redis: + container_name: redis + image: redis:6.2 diff --git a/docker/images/ckan/Dockerfile b/docker/images/ckan/Dockerfile new file mode 100644 index 0000000..de6d49e --- /dev/null +++ b/docker/images/ckan/Dockerfile @@ -0,0 +1,106 @@ +# This is a locally modified version of the upstream CKAN Dockerfile + +# See CKAN docs on installation from Docker Compose on usage +FROM ubuntu:focal-20210119 +MAINTAINER Yoda team + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Setting the locale +ENV LC_ALL=en_US.UTF-8 +RUN adduser ckan +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Install required system packages +RUN apt-get -q -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \ + && apt-get -q -y install \ + python3.8 \ + python3-dev \ + python3-pip \ + python3-venv \ + python3-wheel \ + libpq-dev \ + python3-pastescript \ + python3-virtualenv \ + libxml2-dev \ + libxslt-dev \ + libgeos-dev \ + libssl-dev \ + libffi-dev \ + postgresql-client \ + build-essential \ + git-core \ + vim \ + wget \ + curl \ + nmap \ + sqlite3 \ + pwgen \ + uuid-runtime \ + && apt-get -q clean \ + && rm -rf /var/lib/apt/lists/* + +# Define environment variables +ENV CKAN_HOME /usr/lib/ckan +ENV CKAN_VENV $CKAN_HOME/default +ENV CKAN_CONFIG /etc/ckan +ENV CKAN_STORAGE_PATH=/ckanstorage +ENV CKAN_VERSION=2.9.11 +ENV CKAN_SCHEMING_VERSION=release-2.1.0 +ENV CKAN_MSL_CORE_VERSION=1.4.0 +ENV CKAN_MSL_UTIL_VERSION=1.0.0 + +# Create storage path +RUN mkdir -p $CKAN_STORAGE_PATH/webassets $CKAN_STORAGE_PATH/storage + +# Build-time variables specified by docker-compose.yml / .env +ARG CKAN_SITE_URL + +# Setup virtual environment for CKAN +RUN mkdir -p $CKAN_VENV $CKAN_CONFIG/default && \ + python3 -m venv $CKAN_VENV && \ + ln -s $CKAN_VENV/bin/pip3 /usr/local/bin/ckan-pip3 &&\ + ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan +ADD ckan.ini /etc/ckan/default/ckan.ini +ADD who.ini /etc/ckan/default/who.ini +ADD wsgi.py /etc/ckan/default/wsgi.py +ADD ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini + +# Virtual environment binaries/scripts to be used first +ENV PATH=${CKAN_VENV}/bin:${PATH} +ENV EPOS_MSL_FQDN=epos-msl.local +ENV CKAN_ADMIN_PASSWORD="testtest" + +# Install CKAN and plugins +RUN ckan-pip3 install -U pip && \ + ckan-pip3 install setuptools==44.1.0 && \ + ckan-pip3 install --upgrade pip && \ + ckan-pip3 install wheel && \ + ckan-pip3 install -e "git+https://github.com/ckan/ckan@ckan-${CKAN_VERSION}#egg=ckan[requirements]" && \ + ckan-pip3 install uwsgi && \ + ckan-pip3 install -e "git+https://github.com/ckan/ckanext-scheming@${CKAN_SCHEMING_VERSION}#egg=ckanext-scheming" && \ + ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_core@${CKAN_MSL_CORE_VERSION}#egg=ckanext-msl_ckan" && \ + ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_util@${CKAN_MSL_UTIL_VERSION}#egg=ckanext-msl_ckan_util" && \ + ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \ + cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \ + chmod +x /ckan-entrypoint.sh && \ + chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH $COVERAGE_DIR && \ + rm /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \ + ln -sf /usr/lib/ckan/default/src/ckanext-msl-ckan/ckanext/msl_ckan/config/solr/schema.xml /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \ + perl -n -i.bak -e 'print unless /defaultSearchField/ or /solrQueryParser/' /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml + +ADD ./ckan-entrypoint.sh /ckan-entrypoint.sh +ENTRYPOINT ["/ckan-entrypoint.sh"] +RUN chmod +x /ckan-entrypoint.sh + +USER ckan +EXPOSE 8080 + +CMD ["/ckan-entrypoint.sh"] diff --git a/docker/images/ckan/build.sh b/docker/images/ckan/build.sh new file mode 100755 index 0000000..9712778 --- /dev/null +++ b/docker/images/ckan/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-ckan:latest . diff --git a/docker/images/ckan/ckan-entrypoint.sh b/docker/images/ckan/ckan-entrypoint.sh new file mode 100644 index 0000000..53dfa4e --- /dev/null +++ b/docker/images/ckan/ckan-entrypoint.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -u + +check_port() { + local host="$1" + local port="$2" + + echo "Checking service availability at ${host}:${port}..." + + while true; do + # Use nmap to scan the specified port on the given host + nmap -p "${port}" "${host}" | grep -q "open" + + # If the port is open, exit the loop + if [ $? -eq 0 ]; then + echo "Server is available at ${host}:${port}" + break + else + echo "Server at ${host}:${port} is not available. Retrying in 1 seconds..." + sleep 1 + fi + done +} + +## Check DB up +check_port db 5432 + +## Check Solr up +check_port solr 8983 + +## Check Redis up +check_port redis 6379 + +## Initialize CKAN config, database and admin account +CKAN_CONFIG_FILE=/etc/ckan/default/ckan.ini +CKAN_INIT_STATUS_FILE=/etc/ckan/default/.ckan_initialized + +if test -f "$CKAN_INIT_STATUS_FILE" +then echo "Configuration and database already initialized." +else echo "Initializing configuration ..." + export BEAKER_SESSION_SECRET=$(openssl rand -base64 32) + export APP_INSTANCE_UUID=$(uuidgen --name "$EPOS_MSL_FQDN" --namespace "@url" --sha1) + export CKAN_DATABASE_PASSWORD=$(pwgen -n 16 -N 1) + export CKAN_MSL_VOCABULARIES_ENDPOINT="https://${EPOS_MSL_FQDN}/webservice/api/vocabularies" + perl -pi.bak -e '$beaker_session_secret=$ENV{BEAKER_SESSION_SECRET}; s/BEAKER_SESSION_SECRET/$beaker_session_secret/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$app_instance_uuid=$ENV{APP_INSTANCE_UUID}; s/APP_INSTANCE_UUID/$app_instance_uuid/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$ckan_database_password=$ENV{CKAN_DATABASE_PASSWORD}; s/CKAN_DATABASE_PASSWORD/$ckan_database_password/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$epos_msl_fqdn=$ENV{EPOS_MSL_FQDN}; s/EPOS_MSL_FQDN/$epos_msl_fqdn/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$ckan_msl_vocabularies_endpoint=$ENV{CKAN_MSL_VOCABULARIES_ENDPOINT}; s/CKAN_MSL_VOCABULARIES_ENDPOINT/$ckan_msl_vocabularies_endpoint/ge' "$CKAN_CONFIG_FILE" + echo "Initializing database ..." + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" db init + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" user add ckanadmin password="$CKAN_ADMIN_PASSWORD" email=ckanadmin@localhost name=ckanadmin + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" sysadmin add ckanadmin + touch "$CKAN_INIT_STATUS_FILE" + echo "Configuration and database initialization finished." +fi + +## Start CKAN +echo "Starting CKAN ..." +while true +do /usr/lib/ckan/default/bin/uwsgi -i /etc/ckan/default/ckan-uwsgi.ini + sleep 1 + echo "CKAN process terminated; trying to restart ..." +done diff --git a/docker/images/ckan/ckan-uwsgi.ini b/docker/images/ckan/ckan-uwsgi.ini new file mode 100644 index 0000000..d30aa91 --- /dev/null +++ b/docker/images/ckan/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] + +http = 0.0.0.0:8080 +uid = www-data +gid = www-data +wsgi-file = /etc/ckan/default/wsgi.py +virtualenv = /usr/lib/ckan/default +module = wsgi:application +master = true +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application +strict = true diff --git a/docker/images/ckan/ckan.ini b/docker/images/ckan/ckan.ini new file mode 100644 index 0000000..c1f591b --- /dev/null +++ b/docker/images/ckan/ckan.ini @@ -0,0 +1,274 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. +debug = false + +[app:main] +use = egg:ckan + +## Development settings +ckan.devserver.host = localhost +ckan.devserver.port = 5000 + +## Session settings +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `ckan generate config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = BEAKER_SESSION_SECRET + +# `ckan generate config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = APP_INSTANCE_UUID + +# Default number of search facets returned in a query +search.facets.limit = 10000 + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +who.timeout = 3600 + +## Database Settings +sqlalchemy.url = postgresql://ckan:CKAN_DATABASE_PASSWORD@db/ckan_default + +#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default +#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + + +## Site Settings + +ckan.site_url = https://EPOS_MSL_FQDN +#ckan.use_pylons_response_cleanup_middleware = true + +# Default timeout for Requests +#ckan.requests.timeout = 10 + + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = false +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = false +ckan.auth.public_activity_stream_detail = false +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false + +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret = string:HvlXPoquK7PcVZ1eMo7Lwf_Rs +api_token.jwt.decode.secret = string:HvlXPoquK7PcVZ1eMo7Lwf_Rs +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 + +## Search Settings + +ckan.site_id = default + +solr_url = http://solr:8983/solr/ckan + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +ckan.redis.url = redis://redis:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +#ckan.plugins = stats text_view image_view recline_view +ckan.plugins = stats text_view image_view recline_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields + +scheming.dataset_schemas = ckanext.msl_ckan:schemas/datasets/data_publication.yml ckanext.msl_ckan:schemas/datasets/labs.json +scheming.organization_schemas = ckanext.msl_ckan:schemas/organizations/organization.json + +mslfacets.dataset_config = ckanext.msl_ckan:config/facets.json + +mslindexfields.field_config = ckanext.msl_ckan:config/msl_index_fields.json + +mslvocabularies.endpoint_root = CKAN_MSL_VOCABULARIES_ENDPOINT + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +ckan.storage_path = /ckanstorage +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 +# Default timeout for fetching proxied items +#ckan.resource_proxy.timeout = 10 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +#smtp.reply_to = + +## Background Job Settings +ckan.jobs.timeout = 180 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s + +# msl_ckan_util settings +ckan.mslfacets.dataset_config = ckanext.msl_ckan_util:samples/facets.json +ckan.mslindexfields.fields_config = ckanext.msl_ckan_util:samples/msl_index_fields.json diff --git a/docker/images/ckan/who.ini b/docker/images/ckan/who.ini new file mode 100644 index 0000000..eb2a5a5 --- /dev/null +++ b/docker/images/ckan/who.ini @@ -0,0 +1,32 @@ +[plugin:auth_tkt] +use = ckan.lib.repoze_plugins.auth_tkt:make_plugin +# If no secret key is defined here, beaker.session.secret will be used +#secret = somesecret + +[plugin:friendlyform] +use = ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin +login_form_url= /user/login +login_handler_path = /login_generic +logout_handler_path = /user/logout +rememberer_name = auth_tkt +post_login_url = /user/logged_in +post_logout_url = /user/logged_out +charset = utf-8 + +[general] +request_classifier = repoze.who.classifiers:default_request_classifier +challenge_decider = repoze.who.classifiers:default_challenge_decider + +[identifiers] +plugins = + friendlyform;browser + auth_tkt + +[authenticators] +plugins = + auth_tkt + ckan.lib.authenticator:UsernamePasswordAuthenticator + +[challengers] +plugins = + friendlyform;browser diff --git a/docker/images/ckan/wsgi.py b/docker/images/ckan/wsgi.py new file mode 100644 index 0000000..9880af9 --- /dev/null +++ b/docker/images/ckan/wsgi.py @@ -0,0 +1,12 @@ +# -- coding: utf-8 -- + +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/docker/images/msl-api/Dockerfile b/docker/images/msl-api/Dockerfile new file mode 100644 index 0000000..1028e7b --- /dev/null +++ b/docker/images/msl-api/Dockerfile @@ -0,0 +1,64 @@ +# This is an image for running the MSL-API application, that is part of the EPOS-MSL catalog + +# See CKAN docs on installation from Docker Compose on usage +FROM ubuntu:focal-20210119 +MAINTAINER Yoda team + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Setting the locale +ENV LC_ALL=en_US.UTF-8 +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Install required system packages +ENV COMPOSER_VERSION=2.5.4 +RUN apt-get -q -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \ + && apt-get -q -y install \ + nginx \ + acl \ + git \ + nginx \ + wget \ + nmap \ + sudo \ + software-properties-common \ + && apt-add-repository ppa:ondrej/php \ + && apt-get -q -y install \ + php8.0 \ + php8.0-bcmath \ + php8.0-common \ + php8.0-curl \ + php8.0-fpm \ + php8.0-gd \ + php8.0-mbstring \ + php8.0-mysql \ + php8.0-xml \ + php8.0-zip \ + mysql-client \ + && apt-get -q clean \ + && rm -rf /var/lib/apt/lists/* \ + && wget "https://getcomposer.org/download/$COMPOSER_VERSION/composer.phar" \ + -O /usr/local/bin/composer2 + +ADD msl-api.site /etc/nginx/conf.d/msl-api.conf + +ADD msl-api-entrypoint.sh /msl-api-entrypoint.sh +RUN chmod +x /msl-api-entrypoint.sh + +RUN cd /var/www && \ + git clone https://github.com/utrechtUniversity/msl_api && \ + chown -R www-data:www-data msl_api +USER www-data +ADD msl-api.env /var/www/msl_api/.env +RUN cd /var/www/msl_api && \ + /usr/bin/php8.0 /usr/local/bin/composer2 install + +USER root +CMD ["/msl-api-entrypoint.sh"] diff --git a/docker/images/msl-api/build.sh b/docker/images/msl-api/build.sh new file mode 100755 index 0000000..0485a35 --- /dev/null +++ b/docker/images/msl-api/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-api:latest . diff --git a/docker/images/msl-api/msl-api-entrypoint.sh b/docker/images/msl-api/msl-api-entrypoint.sh new file mode 100644 index 0000000..98151b8 --- /dev/null +++ b/docker/images/msl-api/msl-api-entrypoint.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +set -u + +check_port() { + local host="$1" + local port="$2" + + echo "Checking service availability at ${host}:${port}..." + + while true; do + # Use nmap to scan the specified port on the given host + nmap -p "${port}" "${host}" | grep -q "open" + + # If the port is open, exit the loop + if [ $? -eq 0 ]; then + echo "Server is available at ${host}:${port}" + break + else + echo "Server at ${host}:${port} is not available. Retrying in 1 seconds..." + sleep 1 + fi + done +} + +## Check DB up +check_port mslapi_db 3306 + +# The web server container initializes the database and application +# The queue worker container waits for the web server to finish initialization +SIGNALFILE="/signal/mslapi_initialized.sig" +if [ -f "$SIGNALFILE" ] +then echo "MSL-API already initialized. Proceeding with starting $MSLAPI_ROLE ..." +else + if [ "$MSLAPI_ROLE" == "WEBSERVER" ] + then # Initialize the MSL-API database + mysql -u root "-p$MYSQL_ROOT_PASSWORD" -h mslapi_db -e " +CREATE DATABASE mslapi; +CREATE USER 'msl'@'%' IDENTIFIED BY 'msl'; +GRANT ALL PRIVILEGES ON mslapi.* TO 'msl'@'%'; +FLUSH PRIVILEGES; +" + cd /var/www/msl_api + # Initialize the MSL-API application + set -x + sudo -u www-data /usr/bin/php8.0 artisan key:generate + sudo -u www-data /usr/bin/php8.0 artisan config:cache + sudo -u www-data /usr/bin/php8.0 artisan migrate --force + sudo -u www-data /usr/bin/php8.0 artisan db:seed --force + sudo -u www-data /usr/bin/php8.0 artisan storage:link + set +x + touch "$SIGNALFILE" + elif [ "$MSLAPI_ROLE" == "QUEUE_WORKER" ] + then while ! [ -f "$SIGNALFILE" ] + do echo "Waiting for web server to initialize MSL-API application ..." + sleep 2 + done + echo "MSL-API application initialized. Proceeding ..." + else echo "Error: unknown MSL API role: $MSLAPI_ROLE" + exit 1 + fi +fi + +## Run main process +if [ "$MSLAPI_ROLE" == "QUEUE_WORKER" ] +then while true + do sudo -u www-data /usr/bin/php8.0 /var/www/msl_api/artisan queue:work --rest=1 --tries=3 --timeout=300 + sleep 3 + echo "Restarting queue worker after exit..." + done +elif [ "$MSLAPI_ROLE" == "WEBSERVER" ] +then /usr/sbin/nginx -g 'daemon off;' +else echo "Error: unknown MSL API role: $MSLAPI_ROLE" +fi diff --git a/docker/images/msl-api/msl-api.env b/docker/images/msl-api/msl-api.env new file mode 100644 index 0000000..ae2e0c9 --- /dev/null +++ b/docker/images/msl-api/msl-api.env @@ -0,0 +1,55 @@ +APP_NAME=MSLAPI +APP_ENV=production +APP_DEBUG=false +APP_URL=https://epos-msl.ckan/webservice +ASSET_URL=https://epos-msl.ckan/webservice + +LOG_CHANNEL=stack +LOG_LEVEL=debug + +DB_CONNECTION=mysql +DB_HOST=mslapi_db +DB_PORT=3306 +DB_DATABASE=mslapi +DB_USERNAME=msl +DB_PASSWORD=msl + +BROADCAST_DRIVER=log +CACHE_DRIVER=file +FILESYSTEM_DRIVER=local +QUEUE_CONNECTION=database +SESSION_DRIVER=file +SESSION_LIFETIME=120 + +MEMCACHED_HOST=127.0.0.1 + +REDIS_HOST=127.0.0.1 +REDIS_PASSWORD=null +REDIS_PORT=6379 + +MAIL_MAILER=smtp +MAIL_HOST=mailhog +MAIL_PORT=1025 +MAIL_USERNAME=null +MAIL_PASSWORD=null +MAIL_ENCRYPTION=null +MAIL_FROM_ADDRESS=null +MAIL_FROM_NAME="${APP_NAME}" + +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_DEFAULT_REGION=us-east-1 +AWS_BUCKET= +AWS_USE_PATH_STYLE_ENDPOINT=false + +PUSHER_APP_ID= +PUSHER_APP_KEY= +PUSHER_APP_SECRET= +PUSHER_APP_CLUSTER=mt1 + +MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}" +MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}" + +CKAN_API_URL= https://ckan/api/3/ +CKAN_API_TOKEN= PUT_API_TOKEN_HERE +CKAN_ROOT_URL= https://epos-msl.ckan/ diff --git a/docker/images/msl-api/msl-api.site b/docker/images/msl-api/msl-api.site new file mode 100644 index 0000000..1bb1cdc --- /dev/null +++ b/docker/images/msl-api/msl-api.site @@ -0,0 +1,34 @@ +proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m; +proxy_temp_path /tmp/nginx_proxy 1 2; + +server { + listen 80 default_server; + listen [::]:80 default_server; + + location / { + # This server only serves MSL-API traffic + return 404; + } + + location /webservice/ { + alias /var/www/msl_api/public; + try_files /index.php =404; + autoindex on; + + fastcgi_split_path_info ^(.+\.php)(/.+)$; + fastcgi_pass unix:/run/php/php8.0-fpm.sock; + fastcgi_index index.php; + + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root/index.php; + } + + location /webservice/css/ { + alias /var/www/msl_api/public/css/; + } + + location /webservice/js/ { + alias /var/www/msl_api/public/js/; + } +} + diff --git a/docker/images/nginx/Dockerfile b/docker/images/nginx/Dockerfile new file mode 100644 index 0000000..13ccae5 --- /dev/null +++ b/docker/images/nginx/Dockerfile @@ -0,0 +1,11 @@ +# Image for Nginx reverse proxy configured for CKAN + +FROM nginx:1.27 +MAINTAINER Yoda team + +ADD ckan.site /etc/nginx/conf.d/ckan.conf +RUN rm /etc/nginx/conf.d/default.conf +ADD epos-msl.cnf /etc/certificates/epos-msl.cnf +ADD nginx-entrypoint.sh /nginx-entrypoint.sh +RUN chmod +x /nginx-entrypoint.sh +CMD ["/nginx-entrypoint.sh"] diff --git a/docker/images/nginx/build.sh b/docker/images/nginx/build.sh new file mode 100755 index 0000000..4c015cc --- /dev/null +++ b/docker/images/nginx/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-nginx:latest . diff --git a/docker/images/nginx/ckan.site b/docker/images/nginx/ckan.site new file mode 100644 index 0000000..1e9b5fe --- /dev/null +++ b/docker/images/nginx/ckan.site @@ -0,0 +1,68 @@ +proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m; +proxy_temp_path /tmp/nginx_proxy 1 2; + +server { + listen 80 default_server; + listen [::]:80 default_server; + + location / { + return 301 https://$host$request_uri; + } +} + +server { + + listen 443 ssl http2; + listen [::]:443 ssl http2; + + ssl_certificate /etc/certificates/epos-msl.pem; + ssl_certificate_key /etc/certificates/epos-msl.key; + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + + # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam + ssl_dhparam /etc/certificates/dhparams.pem; + + # intermediate configuration + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + + client_max_body_size 100M; + + location / { + proxy_pass http://ckan:8080/; + proxy_set_header Host $host; + proxy_cache cache; + proxy_cache_bypass $cookie_auth_tkt; + proxy_no_cache $cookie_auth_tkt; + proxy_cache_valid 30m; + proxy_cache_key $host$scheme$proxy_host$request_uri; + # In emergency comment out line to force caching + # proxy_ignore_headers X-Accel-Expires Expires Cache-Control; + } + + location /webservice/ { + proxy_pass http://mslapi_webserver:80/; + proxy_set_header Host $host; + proxy_cache cache; + proxy_cache_bypass $cookie_auth_tkt; + proxy_no_cache $cookie_auth_tkt; + proxy_cache_valid 30m; + proxy_cache_key $host$scheme$proxy_host$request_uri; + # In emergency comment out line to force caching + # proxy_ignore_headers X-Accel-Expires Expires Cache-Control; + } + +} diff --git a/docker/images/nginx/epos-msl.cnf b/docker/images/nginx/epos-msl.cnf new file mode 100644 index 0000000..6cd1532 --- /dev/null +++ b/docker/images/nginx/epos-msl.cnf @@ -0,0 +1,13 @@ +[ req ] +default_bits = 4096 +default_md = sha256 +prompt = no +encrypt_key = no +distinguished_name = dn + +[ dn ] +CN = "EPOS_MSL_HOST" +C = NL +O = Yoda +OU = ITS +L = Utrecht diff --git a/docker/images/nginx/nginx-entrypoint.sh b/docker/images/nginx/nginx-entrypoint.sh new file mode 100644 index 0000000..8043edd --- /dev/null +++ b/docker/images/nginx/nginx-entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +## Adapt URL in config +export EPOS_MSL_HOST="$EPOS_MSL_HOST" +perl -pi.bak -e '$epos_msl_host=$ENV{EPOS_MSL_HOST}; s/EPOS_MSL_HOST/$epos_msl_host/ge' /etc/certificates/epos-msl.cnf + +## Generate certificates if needed +cd /etc/certificates +if [ -f "epos-msl.pem" ] +then echo "Skipping certificate generation, because certificate files are already present." +else echo "Generating certificates for reverse proxy at https://$EPOS_MSL_HOST ..." + openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout epos-msl.key -out epos-msl.pem -config epos-msl.cnf + echo "Certificate generation complete." +fi + +if [ -f "dhparam.pem" ] +then echo "Skipping DHParam generation, because DHParam file is already present." +else echo "Generating DHParam configuration..." + openssl dhparam -dsaparam -out dhparams.pem 4096 + echo "DHParam generation complete." +fi + +## Run Nginx +nginx -g "daemon off;" diff --git a/docker/images/solr/Dockerfile b/docker/images/solr/Dockerfile new file mode 100644 index 0000000..e9028da --- /dev/null +++ b/docker/images/solr/Dockerfile @@ -0,0 +1,27 @@ +# Image for Solr configured for CKAN + +FROM solr:8.11 +MAINTAINER Yoda team + +ENV SOLR_CONFIG_DIR="/opt/solr/server/solr/configsets" + +# Create a CKAN configset based on the default one +USER root +RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan + +# Download the EPOS-MSL core plugin (for the schema) +RUN cd /usr/lib && \ + apt update && \ + apt install -y git && \ + git clone https://github.com/UtrechtUniversity/msl_ckan_core.git + +# Change the CKAN configset to use the EPOS-MSL schema, and enable Query Elevation +USER root +ADD solrconfig.xml /opt/solr-8.11.4/server/solr/configsets/ckan/conf/solrconfig.xml +RUN ln -s /usr/lib/msl_ckan_core/ckanext/msl_ckan/config/solr/schema.xml $SOLR_CONFIG_DIR/ckan/conf/schema.xml && \ + perl -n -i.bak -e 'print unless /defaultSearchField/ or /solrQueryParser/' $SOLR_CONFIG_DIR/ckan/conf/schema.xml && \ + cp $SOLR_CONFIG_DIR/ckan/conf/schema.xml $SOLR_CONFIG_DIR/ckan/conf/managed-schema && \ + cp /opt/solr/server/solr/configsets/sample_techproducts_configs/conf/elevate.xml $SOLR_CONFIG_DIR/ckan/conf/elevate.xml + +USER solr +CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] diff --git a/docker/images/solr/build.sh b/docker/images/solr/build.sh new file mode 100755 index 0000000..957e4cd --- /dev/null +++ b/docker/images/solr/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-solr:latest . diff --git a/docker/images/solr/solrconfig.xml b/docker/images/solr/solrconfig.xml new file mode 100644 index 0000000..43edf9b --- /dev/null +++ b/docker/images/solr/solrconfig.xml @@ -0,0 +1,1360 @@ + + + + + + + + 6.6.3 + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + AND + explicit + 10 + + + + + + + + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + _text_ + + + + + + + true + ignored_ + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + + + true + false + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + + 5 + + + + + + + + + + + + + + + diff --git a/environments/development/host_vars/epos-msl b/environments/development/host_vars/epos-msl index 0c8c336..63d8295 100644 --- a/environments/development/host_vars/epos-msl +++ b/environments/development/host_vars/epos-msl @@ -18,6 +18,8 @@ epos_msl_environment: development # EPOS-MSL environment: development, ckan_database_password: test # CKAN Postgresql database password ckan_admin_password: testtest # CKAN admin user ('ckanadmin') password ckan_harvest_password: testtest # CKAN harvest user ('harvest') password +ckan_install_spatial_plugin: true +ckan_plugins_editable_mode: true ########################## # MSL API configuration @@ -28,3 +30,9 @@ msl_api_app_url: http://epos-msl.ckan.test/webservice msl_api_asset_url: https://epos-msl.ckan.test/webservice msl_api_app_key: "" ckan_api_token: none + +######################### +# Solr configuration +######################### + +solr_install_native: false diff --git a/roles/ckan/defaults/main.yml b/roles/ckan/defaults/main.yml index 970495d..7789bd4 100644 --- a/roles/ckan/defaults/main.yml +++ b/roles/ckan/defaults/main.yml @@ -23,6 +23,7 @@ openssl_crt_signed: localhost.crt ckan_package_url: https://packaging.ckan.org/python-ckan_2.9-py3-focal_amd64.deb ckan_package_file: /tmp/ckan_package.deb ckan_package_upgrade: false +ckan_plugins_editable_mode: false ckan_ini_file: /etc/ckan/default/ckan.ini ckan_storage_path: /ckandata ckan_search_facets_limit: 10000 @@ -44,3 +45,7 @@ ckan_msl_core_plugin_version: 1.4.0 ckan_msl_core_plugin_branch: 1.4.0 ckan_msl_util_plugin_branch: 1.0.0 ckan_msl_vocabularies_endpoint: https://epos-msl.ckan.test/webservice/api/vocabularies + +ckan_install_spatial_plugin: false +ckan_spatial_plugin_repo: ckan/ckanext-spatial +ckan_spatial_plugin_version: v2.1.1 diff --git a/roles/ckan/tasks/main.yml b/roles/ckan/tasks/main.yml index 5d9c9f5..1327c0d 100644 --- a/roles/ckan/tasks/main.yml +++ b/roles/ckan/tasks/main.yml @@ -146,35 +146,21 @@ virtualenv: /usr/lib/ckan/default -- name: Install CKAN scheming plugin - ansible.builtin.pip: - name: "https://github.com/ckan/ckanext-scheming/archive/{{ ckanext_scheming_version }}.tar.gz" - virtualenv: /usr/lib/ckan/default - extra_args: "--upgrade" - notify: Restart ckan-uwsgi +- name: Install CKAN spatial plugin, if enabled + ansible.builtin.import_tasks: spatial_plugin.yml + when: ckan_install_spatial_plugin -- name: Install CKAN MSL Core plugin - ansible.builtin.pip: - name: "git+https://github.com/UtrechtUniversity/msl_ckan_core@{{ ckan_msl_core_plugin_branch }}" - virtualenv: /usr/lib/ckan/default - extra_args: "--upgrade" - notify: Restart ckan-uwsgi +- name: Install CKAN scheming plugin + ansible.builtin.import_tasks: scheming_plugin.yml -- name: Clone CKAN MSL Core plugin for copying images - ansible.builtin.git: - repo: "https://github.com/UtrechtUniversity/msl_ckan_core" - dest: /usr/lib/ckan/msl_ckan_core - version: "{{ ckan_msl_core_plugin_branch }}" +- name: Install MSL Core plugin + ansible.builtin.import_tasks: msl_core_plugin.yml -# Workaround for issue where pip doesn't install static images in MSL core plugin -- name: Synchronize MSL Core plugin images - ansible.posix.synchronize: - src: /usr/lib/ckan/msl_ckan_core/ckanext/msl_ckan/public/ - dest: /usr/lib/ckan/default/lib/python3.8/site-packages/ckanext/msl_ckan/public/ - delegate_to: "{{ inventory_hostname }}" +- name: Install MSL util plugin + ansible.builtin.import_tasks: msl_util_plugin.yml - name: Set Solr schema directory for native Solr package @@ -247,22 +233,6 @@ state: link -# Workaround for issue where pip doesn't install static images in MSL core plugin -- name: Synchronize MSL Core plugin images - ansible.posix.synchronize: - src: /usr/lib/ckan/msl_ckan_core/ckanext/msl_ckan/public/ - dest: /usr/lib/ckan/default/lib/python3.8/site-packages/ckanext/msl_ckan/public/ - delegate_to: "{{ inventory_hostname }}" - - -- name: Install CKAN MSL Util plugin - ansible.builtin.pip: - name: "git+https://github.com/UtrechtUniversity/msl_ckan_util@{{ ckan_msl_util_plugin_branch }}" - virtualenv: /usr/lib/ckan/default - extra_args: "--upgrade" - notify: Restart ckan-uwsgi - - - name: Check CKAN database initialized become: true become_user: postgres diff --git a/roles/ckan/tasks/msl_core_plugin.yml b/roles/ckan/tasks/msl_core_plugin.yml new file mode 100644 index 0000000..df4917b --- /dev/null +++ b/roles/ckan/tasks/msl_core_plugin.yml @@ -0,0 +1,37 @@ +--- +# copyright Utrecht University + +# It is always cloned for copying images, and depending on settings, also for an +# editable install of the plugin. +- name: Clone CKAN MSL Core plugin + ansible.builtin.git: + repo: "https://github.com/UtrechtUniversity/msl_ckan_core" + dest: /usr/lib/ckan/msl_ckan_core + version: "{{ ckan_msl_core_plugin_branch }}" + + +- name: Install CKAN MSL Core plugin (regular mode) + ansible.builtin.pip: + name: "git+https://github.com/UtrechtUniversity/msl_ckan_core@{{ ckan_msl_core_plugin_branch }}" + virtualenv: /usr/lib/ckan/default + extra_args: "--upgrade" + notify: Restart ckan-uwsgi + when: not ckan_plugins_editable_mode + + +- name: Install CKAN MSL Core plugin (editable mode) + ansible.builtin.pip: + name: /usr/lib/ckan/msl_ckan_core + virtualenv: /usr/lib/ckan/default + extra_args: "-e" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi + + +# Workaround for issue where pip doesn't install static images in MSL core plugin +- name: Synchronize MSL Core plugin images + ansible.posix.synchronize: + src: /usr/lib/ckan/msl_ckan_core/ckanext/msl_ckan/public/ + dest: /usr/lib/ckan/default/lib/python3.8/site-packages/ckanext/msl_ckan/public/ + delegate_to: "{{ inventory_hostname }}" + when: not ckan_plugins_editable_mode diff --git a/roles/ckan/tasks/msl_util_plugin.yml b/roles/ckan/tasks/msl_util_plugin.yml new file mode 100644 index 0000000..22e1217 --- /dev/null +++ b/roles/ckan/tasks/msl_util_plugin.yml @@ -0,0 +1,27 @@ +--- +# copyright Utrecht University + +- name: Clone CKAN MSL util plugin + ansible.builtin.git: + repo: "https://github.com/utrechtuniversity/msl_ckan_util" + dest: /usr/lib/ckan/msl_ckan_util + version: "{{ ckan_msl_util_plugin_branch }}" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi + + +- name: Install CKAN MSL Util plugin + ansible.builtin.pip: + name: "git+https://github.com/UtrechtUniversity/msl_ckan_util@{{ ckan_msl_util_plugin_branch }}" + virtualenv: /usr/lib/ckan/default + extra_args: "--upgrade" + notify: Restart ckan-uwsgi + + +- name: Install CKAN MSL Util plugin (editable mode) + ansible.builtin.pip: + name: /usr/lib/ckan/msl_ckan_util + virtualenv: /usr/lib/ckan/default + extra_args: "-e" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi diff --git a/roles/ckan/tasks/scheming_plugin.yml b/roles/ckan/tasks/scheming_plugin.yml new file mode 100644 index 0000000..6babedd --- /dev/null +++ b/roles/ckan/tasks/scheming_plugin.yml @@ -0,0 +1,27 @@ +--- +# copyright Utrecht University + +- name: Clone scheming CKAN plugin + ansible.builtin.git: + repo: "https://github.com/ckan/ckanext-scheming" + dest: /usr/lib/ckan/ckanext-scheming + version: "{{ ckanext_scheming_version }}" + notify: Restart ckan-uwsgi + + +- name: Install CKAN scheming plugin (regular mode) + ansible.builtin.pip: + name: "https://github.com/ckan/ckanext-scheming/archive/{{ ckanext_scheming_version }}.tar.gz" + virtualenv: /usr/lib/ckan/default + extra_args: "--upgrade" + when: not ckan_plugins_editable_mode + notify: Restart ckan-uwsgi + + +- name: Install CKAN scheming plugin (editable mode) + ansible.builtin.pip: + name: /usr/lib/ckan/ckanext-scheming + virtualenv: /usr/lib/ckan/default + extra_args: "-e" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi diff --git a/roles/ckan/tasks/spatial_plugin.yml b/roles/ckan/tasks/spatial_plugin.yml new file mode 100644 index 0000000..8392037 --- /dev/null +++ b/roles/ckan/tasks/spatial_plugin.yml @@ -0,0 +1,43 @@ +--- +# copyright Utrecht University +# +- name: Install spatial CKAN plugin package dependencies + ansible.builtin.apt: + pkg: + - python-dev + - libxml2-dev + - libxslt1-dev + - libgeos-c1v5 + state: present + + +- name: Clone spatial CKAN plugin + ansible.builtin.git: + repo: "https://github.com/{{ ckan_spatial_plugin_repo }}" + dest: /usr/lib/ckan/ckanext-spatial + version: "{{ ckan_spatial_plugin_version }}" + + +- name: Install spatial CKAN plugin Python dependencies + ansible.builtin.pip: + requirements: /usr/lib/ckan/ckanext-spatial/requirements.txt + virtualenv: /usr/lib/ckan/default + notify: Restart ckan-uwsgi + + +- name: Install spatial CKAN plugin (regular mode) + ansible.builtin.pip: + name: "git+https://github.com/{{ ckan_spatial_plugin_repo }}@{{ ckan_spatial_plugin_version }}" + virtualenv: /usr/lib/ckan/default + extra_args: "--upgrade" + when: not ckan_plugins_editable_mode + notify: Restart ckan-uwsgi + + +- name: Install spatial CKAN plugin (editable mode) + ansible.builtin.pip: + name: /usr/lib/ckan/ckanext-spatial + virtualenv: /usr/lib/ckan/default + extra_args: "-e" + when: ckan_plugins_editable_mode + notify: Restart ckan-uwsgi diff --git a/roles/ckan/templates/api.php.j2 b/roles/ckan/templates/api.php.j2 deleted file mode 100644 index 3b64585..0000000 --- a/roles/ckan/templates/api.php.j2 +++ /dev/null @@ -1,164 +0,0 @@ - 'author', - 'Keywords' => 'tags', - 'Title' => 'title', - 'Description' => 'notes', - 'Lab' => 'organization', - 'Dataset_contact' => 'extras_Datasetcontact', - 'References' => 'extras_References', - 'Is_Supplement_To' => 'extras_Issupplementto', - 'Cites' => 'extras_Cites', - 'Publisher' => 'extras_Publisher', - 'PublicationYear' => 'extras_Publicationdate' -); - -$solrQuery = []; -foreach ($querystring as $param => $value) { - if (array_key_exists($param, $fieldMapping)) { - $param = $fieldMapping[$param]; - - //str_replace(' ', '%20', $value) - - $solrQuery[] = sprintf( - '(%s:"%s")', - $param, - $value - ); - } -} - -$api_url = sprintf( - 'https://localhost/api/3/action/package_search?q=%s', - implode(' AND ', $solrQuery) -); - -$s = file_get_contents(str_replace(' ', '%20', $api_url)); - -$response = json_decode($s, JSON_OBJECT_AS_ARRAY); -$countDPs = 0; -$totalDPs = array(); - -if ($response['success'] == 1){ - if ($response['result']['count']>0) { - $results = $response['result']['results']; - - foreach ($results as $result) { - $tags = array(); - foreach ($result['tags'] as $tag) { - $tags[] = $tag['name']; - } - $subDomains = array(); - foreach ($result['groups'] as $subDomain) { - $subDomains = $subDomain['display_name']; - } - - //geoJSON - bounding box - $bbX1 = 0; - $bbY1 = 0; - $bbX2 = 0; - $bbY2 = 0; - - $dpData = array( - 'Title' => $result['title'], - 'Description' => $result['notes'], - 'Author' => $result['author'], - 'Keywords' => $tags, - 'Subdomain' => $subDomains, - 'Lab' => $result['organization']['title'], - - 'Publisher' => $result['maintainer'], // provided by - 'Publisher_email' => $result['maintainer_email'], // - 'Url' => $result['url'], - 'License' => $result['license_title'], - ); - - foreach ($result['extras'] as $extra) { //pairs are enumerated - if (substr($extra['key'],0,7)!='harvest') { - if (substr($extra['key'],0,6)=='geobox') { - // Fill the coordinates with floats - $val = floatval($extra['value']); - switch ($extra['key']) { - case 'geobox-eLong': - $bbX1 = $val; - break; - case 'geobox-nLat': - $bbY1 = $val; - break; - case 'geobox-wLong': - $bbX2 = $val; - break; - case 'geobox-sLat': - $bbY2 = $val; - } - } - else { - $dpData[$extra['key']] = $extra['value']; - } - } - } - - $geoJSON = array( - 'type' => 'Feature', - 'properties' => $dpData, - 'geometry' => array( - 'type' => 'Polygon', - 'coordinates' => array( - array( - array($bbX1,$bbY1), array($bbX2,$bbY1), array($bbX2,$bbY2), array($bbX1,$bbY2),array($bbX1,$bbY1) - ) - ) - ) - ); - - $totalDPs[] = $geoJSON; - $countDPs += 1; - } - } -} - -echo json_encode($totalDPs); -exit; diff --git a/roles/ckan/templates/ckan.ini.j2 b/roles/ckan/templates/ckan.ini.j2 index 942d416..875358b 100644 --- a/roles/ckan/templates/ckan.ini.j2 +++ b/roles/ckan/templates/ckan.ini.j2 @@ -129,7 +129,11 @@ solr_url = http://127.0.0.1:{{ solr_port }}/solr/ckan # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy #ckan.plugins = stats text_view image_view recline_view +{% if ckan_install_spatial_plugin %} +ckan.plugins = stats text_view image_view recline_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields spatial_metadata spatial_query +{% else %} ckan.plugins = stats text_view image_view recline_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields +{% endif %} scheming.dataset_schemas = ckanext.msl_ckan:schemas/datasets/data_publication.yml ckanext.msl_ckan:schemas/datasets/labs.json scheming.organization_schemas = ckanext.msl_ckan:schemas/organizations/organization.json diff --git a/roles/solr/defaults/main.yml b/roles/solr/defaults/main.yml index 8622c01..54904a7 100644 --- a/roles/solr/defaults/main.yml +++ b/roles/solr/defaults/main.yml @@ -3,7 +3,7 @@ # This installs an old version of Solr that comes with a native Ubuntu package, # rather than a version from Solr.org -solr_install_native: true +solr_install_native: false # CKAN 2.9 only supports Solr 7 and 8. This playbook uses the Solr 8 CKAN schema, # so it only works with Solr 8.x The settings below only take effect when solr_install_native