Add support for Solr 8

roles/ckan/tasks/main.yml

@@ -173,30 +173,56 @@
   delegate_to: "{{ inventory_hostname }}"
 
 
+- name: Set Solr schema directory for native Solr package
+  ansible.builtin.set_fact:
+    ckan_solr_schema_directory: /etc/solr/conf
+  when: solr_install_native
+
+
+- name: Set Solr schema directory for upstream Solr package
+  ansible.builtin.set_fact:
+    ckan_solr_schema_directory: /var/solr/data/ckan/conf
+  when: not solr_install_native
+
+
 - name: Check Solr schema
   ansible.builtin.stat:
-    path: /etc/solr/conf/schema.xml
+    path: "{{ ckan_solr_schema_directory }}/schema.xml"
   register: solrschema
+  when: not ansible_check_mode
 
 
 - name: Ensure default schema has been backed up
   ansible.builtin.command:
-    cmd: "mv /etc/solr/conf/schema.xml /etc/solr/conf/schema.xml.orig"
-    creates: /etc/solr/conf/schema.xml.orig
-  when: solrschema.stat.exists and solrschema.stat.isreg is defined and solrschema.stat.isreg
+    cmd: "mv {{ ckan_solr_schema_directory }}/schema.xml {{ ckan_solr_schema_directory }}/schema.xml.orig"
+    creates: "{{ ckan_solr_schema_directory }}/schema.xml.orig"
+  when: not ansible_check_mode and solrschema.stat.exists and solrschema.stat.isreg is defined and solrschema.stat.isreg
 
 
-- name: Use custom Solr scheme for EPOS-MSL
+- name: Use custom Solr schema for EPOS-MSL
   become_user: root
   become: true
   ansible.posix.synchronize:
     src: /usr/lib/ckan/msl_ckan_core/ckanext/msl_ckan/config/solr/schema.xml
-    dest: /etc/solr/conf/schema.xml
+    dest: "{{ ckan_solr_schema_directory }}/schema.xml"
   notify: Restart Solr
   delegate_to: "{{ inventory_hostname }}"
   when: not ansible_check_mode
 
 
+- name: Remove legacy settings no longer compatible with Solr 8
+  become_user: root
+  become: true
+  ansible.builtin.lineinfile:
+    path: "{{ ckan_solr_schema_directory }}/schema.xml"
+    regexp: "{{ item }}"
+    state: absent
+  with_items:
+    - "defaultSearchField"
+    - "solrQueryParser"
+  when: not solr_install_native
+
+
 - name: Check who.ini
   ansible.builtin.stat:
     path: /etc/ckan/default/who.ini

roles/ckan/templates/ckan.ini.j2

@@ -99,7 +99,12 @@ expire_api_token.default_lifetime = 3600
 ## Search Settings
 
 ckan.site_id = default
+
+{% if solr_install_native %}
 solr_url = http://127.0.0.1:{{ solr_port }}/solr
+{% else %}
+solr_url = http://127.0.0.1:{{ solr_port }}/solr/ckan
+{% endif %}
 
 
 ## Redis Settings

roles/solr/defaults/main.yml

@@ -1,4 +1,23 @@
 ---
 # copyright Utrecht University
 
+# This installs an old version of Solr that comes with a native Ubuntu package,
+# rather than a version from Solr.org
+solr_install_native: true
+
+# CKAN 2.9 only supports Solr 7 and 8. This playbook uses the Solr 8 CKAN schema,
+# so it only works with Solr 8.x The settings below only take effect when solr_install_native
+# has been disabled.
+#
+# After upgrading from Solr 3 to Solr 8, you will need to manually rebuild indexes:
+# $ sudo systemctl restart solr
+# $ sudo -iu ckan /usr/lib/ckan/default/bin/ckan -c /etc/ckan/default/ckan.ini search-index build -r
+solr_version: "8.11.3"
+solr_checksum: sha512:10f09b163bd9c31b2a8fdf889cf624114648e76881d69a4c096d473272c47b3cbe37ec9f9bd1980fd90967352c4052477065e165127eccb2c49a60c8d9860afc
+
+solr_config_dir: "/var/solr/data/ckan/conf"
+solr_schema_filename: "schema.xml"
+solr_elevate_example_dir: "/opt/solr-{{ solr_version }}/server/solr/configsets/sample_techproducts_configs/conf"
+
+# Other settings
 solr_port: 8983

roles/solr/files/solr.service

@@ -0,0 +1,18 @@
+# Adapted from mhitza version at https://gist.github.com/hammady/3d7b5964c7b0f90997865ebef40bf5e1
+
+[Unit]
+Description=Solr
+
+[Service]
+Type=forking
+User=solr
+Environment=SOLR_INCLUDE=/etc/default/solr.in.sh
+ExecStart=/opt/solr/bin/solr start
+ExecStop=/opt/solr/bin/solr stop
+Restart=on-failure
+LimitNOFILE=65000
+LimitNPROC=65000
+TimeoutSec=180s
+
+[Install]
+WantedBy=multi-user.target

roles/solr/tasks/install-native-solr.yml

@@ -0,0 +1,34 @@
+---
+# copyright Utrecht University
+
+- name: Ensure Solr is installed
+  ansible.builtin.package:
+    name: '{{ item }}'
+    state: present
+  with_items:
+    - solr-tomcat
+
+
+- name: Install Solr server configuration
+  ansible.builtin.template:
+    src: templates/server.xml.j2
+    dest: /etc/tomcat9/server.xml
+    owner: root
+    group: tomcat
+    mode: "0640"
+  register: serverxml
+
+
+# Restarting Solr immediately so that it runs on the right port. If
+# we would use the handler, CKAN wouldn't be able to find Solr.
+- name: Restart Solr if config if server configuration changed
+  ansible.builtin.service:
+    name: tomcat9
+    state: restarted
+  when: serverxml.changed
+
+
+- name: Ensure Solr is enabled on boot
+  ansible.builtin.service:
+    name: tomcat9
+    enabled: true

roles/solr/tasks/install-solr8.yml

@@ -0,0 +1,127 @@
+---
+# copyright Utrecht University
+
+- name: Populate service facts
+  ansible.builtin.service_facts:
+
+
+- name: Ensure native Solr is disabled and stopped, if present
+  ansible.builtin.service:
+    name: tomcat9
+    enabled: false
+    state: stopped
+  when: ansible_facts.services['tomcat9.service'] is defined
+
+
+- name: Ensure Solr dependencies are installed
+  ansible.builtin.package:
+    name: '{{ item }}'
+    state: present
+  with_items:
+    - default-jdk
+    - unzip
+
+
+- name: Check if Solr is installed
+  ansible.builtin.stat:
+    path: /opt/solr
+  register: solr_installed
+
+
+- name: "Download Solr archive"
+  ansible.builtin.get_url:
+    url: "https://www.apache.org/dyn/closer.lua/lucene/solr/{{ solr_version }}/solr-{{ solr_version }}.tgz?action=download"
+    dest: "~/solr-{{ solr_version }}.tgz"
+    checksum: "{{ solr_checksum }}"
+    force: false
+    owner: root
+    group: root
+    mode: "0644"
+    timeout: 300
+  when: solr_installed.stat.isdir is not defined
+  register: solr_downloaded
+
+
+- name: Expand Solr archive
+  ansible.builtin.unarchive:
+    src: "~/solr-{{ solr_version }}.tgz"
+    dest: ~/
+    copy: false
+  when: solr_downloaded.changed
+
+
+- name: Run Solr installation script
+  ansible.builtin.command:
+    cmd: "~/solr-{{ solr_version }}/bin/install_solr_service.sh ~/solr-{{ solr_version }}.tgz"
+    creates: /opt/solr/bin/solr
+  register: solr_installed
+
+
+- name: Remove the Solr installation files
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "~/solr-{{ solr_version }}.tgz"
+    - "~/solr-{{ solr_version }}/"
+
+
+- name: Create Solr CKAN configuration
+  become_user: solr
+  become: true
+  ansible.builtin.command:
+    cmd: /opt/solr/bin/solr create -c ckan
+    creates: /var/solr/data/ckan/core.properties
+
+
+- name: Ensure solrconfig.xml for CKAN is present
+  ansible.builtin.template:
+    src: "solrconfig.xml.j2"
+    dest: "/var/solr/data/ckan/conf/solrconfig.xml"
+    owner: solr
+    group: solr
+    mode: "0660"
+
+
+- name: Ensure Solr QueryElevationComponent is initialized
+  ansible.builtin.copy:
+    src: "{{ solr_elevate_example_dir }}/elevate.xml"
+    dest: "{{ solr_config_dir }}/elevate.xml"
+    owner: solr
+    group: solr
+    mode: '0644'
+    remote_src: true
+  register: solr_init_queryelevation
+
+
+# Solr might be started outside of systemd. Stop it with command to workaround
+# service limitations, so it can be restarted via systemd.
+- name: Ensure Solr is stopped
+  # noqa command-instead-of-module no-handler no-changed-when
+  ansible.builtin.command: service solr stop
+  when: solr_installed.changed
+
+
+- name: Install Solr systemd unit file
+  ansible.builtin.copy:
+    src: "solr.service"
+    dest: "/etc/systemd/system/solr.service"
+    owner: root
+    group: root
+    mode: "0644"
+
+
+- name: Ensure Solr daemon is started using systemd
+  ansible.builtin.systemd:
+    name: solr
+    state: started
+    daemon_reload: true
+    enabled: true
+
+
+- name: Ensure Solr config directory is present
+  ansible.builtin.file:
+    path: "{{ solr_config_dir }}"
+    state: directory
+    mode: "0755"
+    owner: root

roles/solr/tasks/main.yml

@@ -1,34 +1,11 @@
 ---
 # copyright Utrecht University
 
-- name: Ensure Solr is installed
-  ansible.builtin.package:
-    name: '{{ item }}'
-    state: present
-  with_items:
-    - solr-tomcat
+- name: Install old version of Solr using distribution-native package
+  ansible.builtin.import_tasks: install-native-solr.yml
+  when: solr_install_native
 
 
-- name: Install Solr server configuration
-  ansible.builtin.template:
-    src: templates/server.xml.j2
-    dest: /etc/tomcat9/server.xml
-    owner: root
-    group: tomcat
-    mode: "0640"
-  register: serverxml
-
-
-# Restarting Solr immediately so that it runs on the right port. If
-# we would use the handler, CKAN wouldn't be able to find Solr.
-- name: Restart Solr if config if server configuration changed
-  ansible.builtin.service:
-    name: tomcat9
-    state: restarted
-  when: serverxml.changed
-
-
-- name: Ensure Solr is enabled on boot
-  ansible.builtin.service:
-    name: tomcat9
-    enabled: true
+- name: Install Solr 8 from official package
+  ansible.builtin.import_tasks: install-solr8.yml
+  when: not solr_install_native