feat: add Elasticsearch support in tutor-discovery (#89)

- As Tutor and Open edX have shifted to Meilisearch, and course-discovery still depends on Elasticsearch, running the Elasticsearch container with tutor-discovery will facilitate smoother operation for the course-discovery service.
- It's related PR from tutor: overhangio/tutor#1141.

Co-authored-by: Muhammad Faraz  Maqsood <[email protected]>

changelog.d/20241025_181736_faraz.maqsood_enable_elastic_search_for_discovery.md

@@ -0,0 +1,2 @@
+- [Feature] Add Elasticsearch support in tutor-discovery. As Tutor and Open edX have shifted to Meilisearch, and course-discovery still depends on Elasticsearch, running the Elasticsearch container with tutor-discovery will facilitate smoother operation for the course-discovery service. (by @Faraz32123)
+  - It's related PR from tutor: https://github.com/overhangio/tutor/pull/1141.

tutordiscovery/patches/discovery-common-settings

@@ -0,0 +1 @@
+DISCOVERY_RUN_ELASTICSEARCH = {{ DISCOVERY_RUN_ELASTICSEARCH }}

tutordiscovery/patches/k8s-deployments

@@ -32,3 +32,54 @@ spec:
         - name: settings
           configMap:
             name: discovery-settings
+
+{% if DISCOVERY_RUN_ELASTICSEARCH %}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch
+  labels:
+    app.kubernetes.io/name: elasticsearch
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: elasticsearch
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: elasticsearch
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        fsGroupChangePolicy: "OnRootMismatch"
+      containers:
+        - name: elasticsearch
+          image: {{ DISCOVERY_DOCKER_IMAGE_ELASTICSEARCH }}
+          env:
+            - name: cluster.name
+              value: "openedx"
+            - name: bootstrap.memory_lock
+              value: "true"
+            - name: discovery.type
+              value: "single-node"
+            - name: ES_JAVA_OPTS
+              value: "-Xms{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }}"
+            - name: TAKE_FILE_OWNERSHIP
+              value: "1"
+          ports:
+            - containerPort: 9200
+          securityContext:
+            allowPrivilegeEscalation: false
+          volumeMounts:
+            - mountPath: /usr/share/elasticsearch/data
+              name: data
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: elasticsearch
+{% endif %}

tutordiscovery/patches/k8s-jobs

@@ -23,4 +23,3 @@ spec:
         - name: settings
           configMap:
             name: discovery-settings
-

tutordiscovery/patches/k8s-services

@@ -10,3 +10,19 @@ spec:
       protocol: TCP
   selector:
     app.kubernetes.io/name: discovery
+{% if DISCOVERY_RUN_ELASTICSEARCH %}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  labels:
+    app.kubernetes.io/name: elasticsearch
+spec:
+  type: ClusterIP
+  ports:
+    - port: 9200
+      protocol: TCP
+  selector:
+    app.kubernetes.io/name: elasticsearch
+{% endif %}

tutordiscovery/patches/k8s-volumes

@@ -0,0 +1,16 @@
+{% if DISCOVERY_RUN_ELASTICSEARCH %}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: elasticsearch
+  labels:
+    app.kubernetes.io/component: volume
+    app.kubernetes.io/name: elasticsearch
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+{% endif %}

tutordiscovery/patches/local-docker-compose-dev-services

@@ -15,3 +15,12 @@ discovery:
     default:
       aliases:
         - "{{ DISCOVERY_HOST }}"
+
+{% if DISCOVERY_RUN_ELASTICSEARCH and is_docker_rootless() %}
+  elasticsearch:
+    ulimits:
+      memlock:
+        # Fixes error setting rlimits for ready process in rootless docker
+        soft: 0  # zero means "unset" in the memlock context
+        hard: 0
+  {% endif %}

tutordiscovery/patches/local-docker-compose-permissions-command

@@ -0,0 +1 @@
+{% if DISCOVERY_RUN_ELASTICSEARCH %}setowner 1000 /mounts/elasticsearch{% endif %}

tutordiscovery/patches/local-docker-compose-permissions-volumes

@@ -0,0 +1 @@
+{% if DISCOVERY_RUN_ELASTICSEARCH %}- ../../data/elasticsearch:/mounts/elasticsearch{% endif %}

tutordiscovery/patches/local-docker-compose-services

@@ -6,7 +6,28 @@ discovery:
   volumes:
     - ../plugins/discovery/apps/settings/tutor:/openedx/discovery/course_discovery/settings/tutor:ro
     - ../../data/discovery/media:/openedx/discovery/course_discovery/media
+    {% if DISCOVERY_RUN_ELASTICSEARCH %}- ../../data/elasticsearch:/mounts/elasticsearch{% endif %}
   depends_on: 
     - lms
     {% if RUN_MYSQL %}- mysql{% endif %}
-    {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %}
+    {% if DISCOVERY_RUN_ELASTICSEARCH %}- elasticsearch{% endif %}
+
+{% if DISCOVERY_RUN_ELASTICSEARCH -%}
+  elasticsearch:
+    image: {{ DISCOVERY_DOCKER_IMAGE_ELASTICSEARCH }}
+    environment:
+      - cluster.name=openedx
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - "ES_JAVA_OPTS=-Xms{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }}"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    restart: unless-stopped
+    user: "1000:1000"
+    volumes:
+      - ../../data/elasticsearch:/usr/share/elasticsearch/data
+    depends_on:
+      - permissions
+{%- endif %}

tutordiscovery/plugin.py

@@ -36,6 +36,12 @@
         "EXTRA_PIP_REQUIREMENTS": [],
         "REPOSITORY": "https://github.com/openedx/course-discovery.git",
         "REPOSITORY_VERSION": "{{ OPENEDX_COMMON_VERSION }}",
+        "RUN_ELASTICSEARCH": True,
+        "DOCKER_IMAGE_ELASTICSEARCH": "docker.io/elasticsearch:7.17.13",
+        "ELASTICSEARCH_HOST": "elasticsearch",
+        "ELASTICSEARCH_PORT": 9200,
+        "ELASTICSEARCH_SCHEME": "http",
+        "ELASTICSEARCH_HEAP_SIZE": "1g",
     },
     "unique": {
         "MYSQL_PASSWORD": "{{ 8|random_string }}",

tutordiscovery/templates/discovery/apps/settings/partials/common.py

@@ -20,8 +20,14 @@
     }
 }
 
+DISCOVERY_DOCKER_IMAGE_ELASTICSEARCH = "{{ DISCOVERY_DOCKER_IMAGE_ELASTICSEARCH }}"
+DISCOVERY_ELASTICSEARCH_HOST = "{{ DISCOVERY_ELASTICSEARCH_HOST }}"
+DISCOVERY_ELASTICSEARCH_PORT = "{{ DISCOVERY_ELASTICSEARCH_PORT }}"
+DISCOVERY_ELASTICSEARCH_SCHEME = "{{ DISCOVERY_ELASTICSEARCH_SCHEME }}"
+DISCOVERY_ELASTICSEARCH_HEAP_SIZE = "{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }}"
+
 ELASTICSEARCH_DSL['default'].update({
-    'hosts': "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}/"
+    'hosts': "{{ DISCOVERY_ELASTICSEARCH_SCHEME }}://{{ DISCOVERY_ELASTICSEARCH_HOST }}:{{ DISCOVERY_ELASTICSEARCH_PORT }}/"
 })
 
 {% for name, index in DISCOVERY_INDEX_OVERRIDES.items() %}