From 403a5e297d7ed1489686d84ac174a9902faa0b0d Mon Sep 17 00:00:00 2001 From: Muhammad Faraz Maqsood Date: Fri, 1 Nov 2024 22:22:22 +0500 Subject: [PATCH] feat: add Elasticsearch support in tutor-discovery (#89) - As Tutor and Open edX have shifted to Meilisearch, and course-discovery still depends on Elasticsearch, running the Elasticsearch container with tutor-discovery will facilitate smoother operation for the course-discovery service. - It's related PR from tutor: overhangio/tutor#1141. Co-authored-by: Muhammad Faraz Maqsood --- ...ood_enable_elastic_search_for_discovery.md | 2 + .../patches/discovery-common-settings | 1 + tutordiscovery/patches/k8s-deployments | 51 +++++++++++++++++++ tutordiscovery/patches/k8s-jobs | 1 - tutordiscovery/patches/k8s-services | 16 ++++++ tutordiscovery/patches/k8s-volumes | 16 ++++++ .../patches/local-docker-compose-dev-services | 9 ++++ .../local-docker-compose-permissions-command | 1 + .../local-docker-compose-permissions-volumes | 1 + .../patches/local-docker-compose-services | 23 ++++++++- tutordiscovery/plugin.py | 6 +++ .../apps/settings/partials/common.py | 8 ++- 12 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20241025_181736_faraz.maqsood_enable_elastic_search_for_discovery.md create mode 100644 tutordiscovery/patches/discovery-common-settings create mode 100644 tutordiscovery/patches/k8s-volumes create mode 100644 tutordiscovery/patches/local-docker-compose-permissions-command create mode 100644 tutordiscovery/patches/local-docker-compose-permissions-volumes diff --git a/changelog.d/20241025_181736_faraz.maqsood_enable_elastic_search_for_discovery.md b/changelog.d/20241025_181736_faraz.maqsood_enable_elastic_search_for_discovery.md new file mode 100644 index 0000000..551ea27 --- /dev/null +++ b/changelog.d/20241025_181736_faraz.maqsood_enable_elastic_search_for_discovery.md @@ -0,0 +1,2 @@ +- [Feature] Add Elasticsearch support in tutor-discovery. As Tutor and Open edX have shifted to Meilisearch, and course-discovery still depends on Elasticsearch, running the Elasticsearch container with tutor-discovery will facilitate smoother operation for the course-discovery service. (by @Faraz32123) + - It's related PR from tutor: https://github.com/overhangio/tutor/pull/1141. diff --git a/tutordiscovery/patches/discovery-common-settings b/tutordiscovery/patches/discovery-common-settings new file mode 100644 index 0000000..8bc08f1 --- /dev/null +++ b/tutordiscovery/patches/discovery-common-settings @@ -0,0 +1 @@ +DISCOVERY_RUN_ELASTICSEARCH = {{ DISCOVERY_RUN_ELASTICSEARCH }} diff --git a/tutordiscovery/patches/k8s-deployments b/tutordiscovery/patches/k8s-deployments index 15a1b3f..06e959c 100644 --- a/tutordiscovery/patches/k8s-deployments +++ b/tutordiscovery/patches/k8s-deployments @@ -32,3 +32,54 @@ spec: - name: settings configMap: name: discovery-settings + +{% if DISCOVERY_RUN_ELASTICSEARCH %} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch + labels: + app.kubernetes.io/name: elasticsearch +spec: + selector: + matchLabels: + app.kubernetes.io/name: elasticsearch + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: elasticsearch + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" + containers: + - name: elasticsearch + image: {{ DISCOVERY_DOCKER_IMAGE_ELASTICSEARCH }} + env: + - name: cluster.name + value: "openedx" + - name: bootstrap.memory_lock + value: "true" + - name: discovery.type + value: "single-node" + - name: ES_JAVA_OPTS + value: "-Xms{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }}" + - name: TAKE_FILE_OWNERSHIP + value: "1" + ports: + - containerPort: 9200 + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /usr/share/elasticsearch/data + name: data + volumes: + - name: data + persistentVolumeClaim: + claimName: elasticsearch +{% endif %} diff --git a/tutordiscovery/patches/k8s-jobs b/tutordiscovery/patches/k8s-jobs index d45e8d7..b3409ab 100644 --- a/tutordiscovery/patches/k8s-jobs +++ b/tutordiscovery/patches/k8s-jobs @@ -23,4 +23,3 @@ spec: - name: settings configMap: name: discovery-settings - diff --git a/tutordiscovery/patches/k8s-services b/tutordiscovery/patches/k8s-services index feb9752..3592338 100644 --- a/tutordiscovery/patches/k8s-services +++ b/tutordiscovery/patches/k8s-services @@ -10,3 +10,19 @@ spec: protocol: TCP selector: app.kubernetes.io/name: discovery +{% if DISCOVERY_RUN_ELASTICSEARCH %} +--- +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + labels: + app.kubernetes.io/name: elasticsearch +spec: + type: ClusterIP + ports: + - port: 9200 + protocol: TCP + selector: + app.kubernetes.io/name: elasticsearch +{% endif %} diff --git a/tutordiscovery/patches/k8s-volumes b/tutordiscovery/patches/k8s-volumes new file mode 100644 index 0000000..4f3ba6b --- /dev/null +++ b/tutordiscovery/patches/k8s-volumes @@ -0,0 +1,16 @@ +{% if DISCOVERY_RUN_ELASTICSEARCH %} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: elasticsearch + labels: + app.kubernetes.io/component: volume + app.kubernetes.io/name: elasticsearch +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +{% endif %} diff --git a/tutordiscovery/patches/local-docker-compose-dev-services b/tutordiscovery/patches/local-docker-compose-dev-services index 95906db..43356e7 100644 --- a/tutordiscovery/patches/local-docker-compose-dev-services +++ b/tutordiscovery/patches/local-docker-compose-dev-services @@ -15,3 +15,12 @@ discovery: default: aliases: - "{{ DISCOVERY_HOST }}" + +{% if DISCOVERY_RUN_ELASTICSEARCH and is_docker_rootless() %} + elasticsearch: + ulimits: + memlock: + # Fixes error setting rlimits for ready process in rootless docker + soft: 0 # zero means "unset" in the memlock context + hard: 0 + {% endif %} diff --git a/tutordiscovery/patches/local-docker-compose-permissions-command b/tutordiscovery/patches/local-docker-compose-permissions-command new file mode 100644 index 0000000..9f470cd --- /dev/null +++ b/tutordiscovery/patches/local-docker-compose-permissions-command @@ -0,0 +1 @@ +{% if DISCOVERY_RUN_ELASTICSEARCH %}setowner 1000 /mounts/elasticsearch{% endif %} diff --git a/tutordiscovery/patches/local-docker-compose-permissions-volumes b/tutordiscovery/patches/local-docker-compose-permissions-volumes new file mode 100644 index 0000000..5bae503 --- /dev/null +++ b/tutordiscovery/patches/local-docker-compose-permissions-volumes @@ -0,0 +1 @@ +{% if DISCOVERY_RUN_ELASTICSEARCH %}- ../../data/elasticsearch:/mounts/elasticsearch{% endif %} diff --git a/tutordiscovery/patches/local-docker-compose-services b/tutordiscovery/patches/local-docker-compose-services index ddbf6ab..bd1bd57 100644 --- a/tutordiscovery/patches/local-docker-compose-services +++ b/tutordiscovery/patches/local-docker-compose-services @@ -6,7 +6,28 @@ discovery: volumes: - ../plugins/discovery/apps/settings/tutor:/openedx/discovery/course_discovery/settings/tutor:ro - ../../data/discovery/media:/openedx/discovery/course_discovery/media + {% if DISCOVERY_RUN_ELASTICSEARCH %}- ../../data/elasticsearch:/mounts/elasticsearch{% endif %} depends_on: - lms {% if RUN_MYSQL %}- mysql{% endif %} - {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} + {% if DISCOVERY_RUN_ELASTICSEARCH %}- elasticsearch{% endif %} + +{% if DISCOVERY_RUN_ELASTICSEARCH -%} + elasticsearch: + image: {{ DISCOVERY_DOCKER_IMAGE_ELASTICSEARCH }} + environment: + - cluster.name=openedx + - bootstrap.memory_lock=true + - discovery.type=single-node + - "ES_JAVA_OPTS=-Xms{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }}" + ulimits: + memlock: + soft: -1 + hard: -1 + restart: unless-stopped + user: "1000:1000" + volumes: + - ../../data/elasticsearch:/usr/share/elasticsearch/data + depends_on: + - permissions +{%- endif %} diff --git a/tutordiscovery/plugin.py b/tutordiscovery/plugin.py index 6633d5d..9546569 100644 --- a/tutordiscovery/plugin.py +++ b/tutordiscovery/plugin.py @@ -36,6 +36,12 @@ "EXTRA_PIP_REQUIREMENTS": [], "REPOSITORY": "https://github.com/openedx/course-discovery.git", "REPOSITORY_VERSION": "{{ OPENEDX_COMMON_VERSION }}", + "RUN_ELASTICSEARCH": True, + "DOCKER_IMAGE_ELASTICSEARCH": "docker.io/elasticsearch:7.17.13", + "ELASTICSEARCH_HOST": "elasticsearch", + "ELASTICSEARCH_PORT": 9200, + "ELASTICSEARCH_SCHEME": "http", + "ELASTICSEARCH_HEAP_SIZE": "1g", }, "unique": { "MYSQL_PASSWORD": "{{ 8|random_string }}", diff --git a/tutordiscovery/templates/discovery/apps/settings/partials/common.py b/tutordiscovery/templates/discovery/apps/settings/partials/common.py index 4cd4758..424639a 100644 --- a/tutordiscovery/templates/discovery/apps/settings/partials/common.py +++ b/tutordiscovery/templates/discovery/apps/settings/partials/common.py @@ -20,8 +20,14 @@ } } +DISCOVERY_DOCKER_IMAGE_ELASTICSEARCH = "{{ DISCOVERY_DOCKER_IMAGE_ELASTICSEARCH }}" +DISCOVERY_ELASTICSEARCH_HOST = "{{ DISCOVERY_ELASTICSEARCH_HOST }}" +DISCOVERY_ELASTICSEARCH_PORT = "{{ DISCOVERY_ELASTICSEARCH_PORT }}" +DISCOVERY_ELASTICSEARCH_SCHEME = "{{ DISCOVERY_ELASTICSEARCH_SCHEME }}" +DISCOVERY_ELASTICSEARCH_HEAP_SIZE = "{{ DISCOVERY_ELASTICSEARCH_HEAP_SIZE }}" + ELASTICSEARCH_DSL['default'].update({ - 'hosts': "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}/" + 'hosts': "{{ DISCOVERY_ELASTICSEARCH_SCHEME }}://{{ DISCOVERY_ELASTICSEARCH_HOST }}:{{ DISCOVERY_ELASTICSEARCH_PORT }}/" }) {% for name, index in DISCOVERY_INDEX_OVERRIDES.items() %}