diff --git a/changelog.d/20241017_125209_regis_meilisearch.md b/changelog.d/20241017_125209_regis_meilisearch.md new file mode 100644 index 0000000000..8144dda24b --- /dev/null +++ b/changelog.d/20241017_125209_regis_meilisearch.md @@ -0,0 +1 @@ +- 💥[Feature] Replace Elasticsearch by Meilisearch. Elasticsearch was both a source of complexity and high resource usage. With this change, we no longer run Elasticsearch to perform common search queries across Open edX. This includes: course discovery, courseware search and studio search. Instead, we index all these documents in a Meilisearch instance, which is much more lightweight in terms of memory consumption. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index 6b1a4c8277..424613768d 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -40,7 +40,7 @@ With an up-to-date environment, Tutor is ready to launch an Open edX platform an Individual service activation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- ``RUN_ELASTICSEARCH`` (default: ``true``) +- ``RUN_MEILISEARCH`` (default: ``true``) - ``RUN_MONGODB`` (default: ``true``) - ``RUN_MYSQL`` (default: ``true``) - ``RUN_REDIS`` (default: ``true``) @@ -71,9 +71,9 @@ This configuration parameter defines the name of the Docker image to run the dev This configuration parameter defines which Caddy Docker image to use. -- ``DOCKER_IMAGE_ELASTICSEARCH`` (default: ``"docker.io/elasticsearch:7.17.9"``) +- ``DOCKER_IMAGE_MEILISEARCH`` (default: ``"docker.io/getmeili/meilisearch:v1.8.4"``) -This configuration parameter defines which Elasticsearch Docker image to use. +This configuration parameter defines which Meilisearch Docker image to use. - ``DOCKER_IMAGE_MONGODB`` (default: ``"docker.io/mongo:7.0.7"``) @@ -228,13 +228,19 @@ By default, a running Open edX platform deployed with Tutor includes all necessa .. note:: When configuring an external MySQL database, please make sure it is using version 8.4. -Elasticsearch -************* +Meilisearch +*********** + +- ``MEILISEARCH_URL`` (default: ``"http://meilisearch:7700"``): internal URL used for backend-to-backend communication. +- ``MEILISEARCH_PUBLIC_URL`` (default: ``"{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://meilisearch.{{ LMS_HOST }}"``): external URL from which the frontend will access the Meilisearch instance. +- ``MEILISEARCH_INDEX_PREFIX`` (default: ``"tutor_"``) +- ``MEILISEARCH_MASTER_KEY`` (default: ``"{{ 24|random_string }}"``) +- ``MEILISEARCH_API_KEY_UID`` (default: ``"{{ 4|uuid }}"``): UID used to sign the API key. +- ``MEILISEARCH_API_KEY`` (default: ``"{{ MEILISEARCH_MASTER_KEY|uid_master_hash(MEILISEARCH_API_KEY_UID) }}"``) + +To reset the Meilisearch API key, make sure to unset both the API key and it's UID: -- ``ELASTICSEARCH_SCHEME`` (default: ``"http"``) -- ``ELASTICSEARCH_HOST`` (default: ``"elasticsearch"``) -- ``ELASTICSEARCH_PORT`` (default: ``9200``) -- ``ELASTICSEARCH_HEAP_SIZE`` (default: ``"1g"``) + tutor config save --unset MEILISEARCH_API_KEY_UID MEILISEARCH_API_KEY MongoDB ******* diff --git a/docs/tutorials/nightly.rst b/docs/tutorials/nightly.rst index 6ee60f3de0..e8bf59825c 100644 --- a/docs/tutorials/nightly.rst +++ b/docs/tutorials/nightly.rst @@ -58,7 +58,7 @@ When running Tutor Nightly, you usually do not want to override your existing Tu Making changes to Tutor Nightly ------------------------------- -In general pull requests should be open on the "master" branch of Tutor: the "master" branch is automatically merged on the "nightly" branch at every commit, such that changes made to Tutor releases find their way to Tutor Nightly as soon as they are merged. However, sometimes you want to make changes to Tutor Nightly exclusively, and not to the Tutor releases. This might be the case for instance when upgrading the running version of a third-party service (for instance: Elasticsearch, MySQL), or when the master branch requires specific changes. In that case, you should follow the instructions from the :ref:`contributing` section of the docs, with the following differences: +In general pull requests should be open on the "master" branch of Tutor: the "master" branch is automatically merged on the "nightly" branch at every commit, such that changes made to Tutor releases find their way to Tutor Nightly as soon as they are merged. However, sometimes you want to make changes to Tutor Nightly exclusively, and not to the Tutor releases. This might be the case for instance when upgrading the running version of a third-party service (for instance: Meilisearch, MySQL), or when the master branch requires specific changes. In that case, you should follow the instructions from the :ref:`contributing` section of the docs, with the following differences: - Open your pull request on top of the "nightly" branch instead of "master". - Add a description of your changes by creating a changelog entry with `make changelog-entry`, as in the master branch. diff --git a/docs/tutorials/scale.rst b/docs/tutorials/scale.rst index 4cecdb3ef2..ea54331f66 100644 --- a/docs/tutorials/scale.rst +++ b/docs/tutorials/scale.rst @@ -37,11 +37,11 @@ Offloading data storage Aside from web workers, the most resource-intensive services are in the data persistence layer. They are, by decreasing resource usage: -- `Elasticsearch `__: indexing of course contents and forum topics, mostly for search. Elasticsearch is never a source of truth in Open edX, and the data can thus be trashed and re-created safely. - `MySQL `__: structured, consistent data storage which is the default destination of all data. - `MongoDB `__: structured storage of course data. - `Redis `__: caching and asynchronous task management. - `MinIO `__: S3-like object storage for user-uploaded files, which is enabled by the `tutor-minio `__ plugin. It is possible to replace MinIO by direct filesystem storage (the default), but scaling will then become much more difficult down the road. +- `Meilisearch `__: indexing of course contents and forum topics, mostly for search. Meilisearch is never a source of truth in Open edX, and the data can thus be trashed and re-created safely. When attempting to scale a single-server deployment, we recommend starting by offloading some of these stateful data storage components, in the same order of priority. There are multiple benefits: diff --git a/tutor/commands/images.py b/tutor/commands/images.py index ed3174af63..d8478c580b 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -63,7 +63,7 @@ def _add_images_to_pull( """ vendor_images = [ ("caddy", "DOCKER_IMAGE_CADDY"), - ("elasticsearch", "DOCKER_IMAGE_ELASTICSEARCH"), + ("meilisearch", "DOCKER_IMAGE_MEILISEARCH"), ("mongodb", "DOCKER_IMAGE_MONGODB"), ("mysql", "DOCKER_IMAGE_MYSQL"), ("redis", "DOCKER_IMAGE_REDIS"), diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 7510a83b31..10893ccd19 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -42,6 +42,10 @@ def _add_core_init_tasks() -> None: hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh")) ) + with hooks.Contexts.app("meilisearch").enter(): + hooks.Filters.CLI_DO_INIT_TASKS.add_item( + ("lms", env.read_core_template_file("jobs", "init", "meilisearch.sh")) + ) with hooks.Contexts.app("lms").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( ( diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 4af21ac1d7..0ecde92dfe 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -390,7 +390,7 @@ def _start_base_deployments(_job_name: str, *_args: Any, **_kwargs: Any) -> None """ config = tutor_config.load(context.root) wait_for_deployment_ready(config, "caddy") - for name in ["elasticsearch", "mysql", "mongodb"]: + for name in ["meilisearch", "mysql", "mongodb"]: if tutor_config.is_service_activated(config, name): wait_for_deployment_ready(config, name) diff --git a/tutor/config.py b/tutor/config.py index 531056792c..5d29c17f40 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -235,7 +235,6 @@ def upgrade_obsolete(config: Config) -> None: for name in [ "ACTIVATE_LMS", "ACTIVATE_CMS", - "ACTIVATE_ELASTICSEARCH", "ACTIVATE_MONGODB", "ACTIVATE_MYSQL", "ACTIVATE_REDIS", diff --git a/tutor/env.py b/tutor/env.py index 4049e9295a..4d60594b9f 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -1,10 +1,10 @@ from __future__ import annotations +from copy import deepcopy import os import re import shutil import typing as t -from copy import deepcopy import importlib_resources import jinja2 @@ -56,6 +56,8 @@ def _prepare_environment() -> None: ("reverse_host", utils.reverse_host), ("rsa_import_key", utils.rsa_import_key), ("rsa_private_key", utils.rsa_private_key), + ("uuid", utils.uuid), + ("uid_master_hash", utils.uid_master_hash), ], ) # Template variables diff --git a/tutor/plugins/openedx.py b/tutor/plugins/openedx.py index 2984d60792..c3f3dfd02d 100644 --- a/tutor/plugins/openedx.py +++ b/tutor/plugins/openedx.py @@ -1,5 +1,6 @@ from __future__ import annotations + import os import re import typing as t @@ -28,6 +29,17 @@ def _edx_platform_public_hosts( return hosts +@hooks.Filters.APP_PUBLIC_HOSTS.add() +def _meilisearch_public_hosts( + hosts: list[str], context_name: t.Literal["local", "dev"] +) -> list[str]: + if context_name == "dev": + hosts.append("{{ MEILISEARCH_PUBLIC_URL.split('://')[1] }}:7700") + else: + hosts.append("{{ MEILISEARCH_PUBLIC_URL.split('://')[1] }}") + return hosts + + @hooks.Filters.IMAGES_BUILD_MOUNTS.add() def _mount_edx_platform_build( volumes: list[tuple[str, str]], path: str diff --git a/tutor/templates/apps/caddy/Caddyfile b/tutor/templates/apps/caddy/Caddyfile index 96a6d9891b..77eb152ec4 100644 --- a/tutor/templates/apps/caddy/Caddyfile +++ b/tutor/templates/apps/caddy/Caddyfile @@ -82,4 +82,10 @@ } } +{% if RUN_MEILISEARCH %} +{{ MEILISEARCH_PUBLIC_URL.split("://")[1] }}{$default_site_port} { + import proxy "meilisearch:7700" +} +{% endif %} + {{ patch("caddyfile") }} diff --git a/tutor/templates/apps/openedx/config/cms.env.yml b/tutor/templates/apps/openedx/config/cms.env.yml index 53e99b0c2d..8eb7b462d8 100644 --- a/tutor/templates/apps/openedx/config/cms.env.yml +++ b/tutor/templates/apps/openedx/config/cms.env.yml @@ -9,7 +9,6 @@ FEATURES: {{ patch("cms-env-features")|indent(2) }} CERTIFICATES_HTML_VIEW: true PREVIEW_LMS_BASE: "{{ PREVIEW_LMS_HOST }}" - ENABLE_COURSEWARE_INDEX: true ENABLE_CSMH_EXTENDED: false ENABLE_LEARNER_RECORDS: false ENABLE_LIBRARY_INDEX: true diff --git a/tutor/templates/apps/openedx/config/lms.env.yml b/tutor/templates/apps/openedx/config/lms.env.yml index 565820c81b..bc9927f23c 100644 --- a/tutor/templates/apps/openedx/config/lms.env.yml +++ b/tutor/templates/apps/openedx/config/lms.env.yml @@ -9,10 +9,7 @@ FEATURES: {{ patch("lms-env-features")|indent(2) }} CERTIFICATES_HTML_VIEW: true PREVIEW_LMS_BASE: "{{ PREVIEW_LMS_HOST }}" - ENABLE_COURSE_DISCOVERY: true - ENABLE_COURSEWARE_SEARCH: true ENABLE_CSMH_EXTENDED: false - ENABLE_DASHBOARD_SEARCH: true ENABLE_COMBINED_LOGIN_REGISTRATION: true ENABLE_GRADE_DOWNLOADS: true ENABLE_LEARNER_RECORDS: false diff --git a/tutor/templates/apps/openedx/settings/cms/development.py b/tutor/templates/apps/openedx/settings/cms/development.py index af7af18e12..ee4c3b3b5b 100644 --- a/tutor/templates/apps/openedx/settings/cms/development.py +++ b/tutor/templates/apps/openedx/settings/cms/development.py @@ -2,20 +2,22 @@ import os from cms.envs.devstack import * +{% include "apps/openedx/settings/partials/common_cms.py" %} + LMS_BASE = "{{ LMS_HOST }}:8000" LMS_ROOT_URL = "http://" + LMS_BASE CMS_BASE = "{{ CMS_HOST }}:8001" CMS_ROOT_URL = "http://" + CMS_BASE +MEILISEARCH_PUBLIC_URL = "{{ MEILISEARCH_PUBLIC_URL }}:7700" + # Authentication SOCIAL_AUTH_EDX_OAUTH2_KEY = "{{ CMS_OAUTH2_KEY_SSO_DEV }}" SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT = LMS_ROOT_URL FEATURES["PREVIEW_LMS_BASE"] = "{{ PREVIEW_LMS_HOST }}:8000" -{% include "apps/openedx/settings/partials/common_cms.py" %} - # Setup correct webpack configuration file for development WEBPACK_CONFIG_PATH = "webpack.dev.config.js" diff --git a/tutor/templates/apps/openedx/settings/lms/development.py b/tutor/templates/apps/openedx/settings/lms/development.py index ed0c277366..8e06352e1d 100644 --- a/tutor/templates/apps/openedx/settings/lms/development.py +++ b/tutor/templates/apps/openedx/settings/lms/development.py @@ -15,6 +15,8 @@ CMS_ROOT_URL = "http://{}".format(CMS_BASE) LOGIN_REDIRECT_WHITELIST.append(CMS_BASE) +MEILISEARCH_PUBLIC_URL = "{{ MEILISEARCH_PUBLIC_URL }}:7700" + # Session cookie SESSION_COOKIE_DOMAIN = "{{ LMS_HOST }}" SESSION_COOKIE_SECURE = False diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index 105ac5383c..f156b0e423 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -34,12 +34,14 @@ # Behave like memcache when it comes to connection errors DJANGO_REDIS_IGNORE_EXCEPTIONS = True -# Elasticsearch connection parameters -ELASTIC_SEARCH_CONFIG = [{ - {% if ELASTICSEARCH_SCHEME == "https" %}"use_ssl": True,{% endif %} - "host": "{{ ELASTICSEARCH_HOST }}", - "port": {{ ELASTICSEARCH_PORT }}, -}] +# Meilisearch connection parameters +MEILISEARCH_ENABLED = True +MEILISEARCH_URL = "{{ MEILISEARCH_URL }}" +MEILISEARCH_PUBLIC_URL = "{{ MEILISEARCH_PUBLIC_URL }}" +MEILISEARCH_INDEX_PREFIX = "{{ MEILISEARCH_INDEX_PREFIX }}" +MEILISEARCH_API_KEY = "{{ MEILISEARCH_API_KEY }}" +MEILISEARCH_MASTER_KEY = "{{ MEILISEARCH_MASTER_KEY }}" +SEARCH_ENGINE = "search.meilisearch.MeilisearchEngine" # Common cache config CACHES = { diff --git a/tutor/templates/apps/openedx/settings/partials/common_cms.py b/tutor/templates/apps/openedx/settings/partials/common_cms.py index c5dde04400..e1263100d5 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_cms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_cms.py @@ -20,6 +20,9 @@ FRONTEND_LOGIN_URL = LMS_ROOT_URL + '/login' FRONTEND_REGISTER_URL = LMS_ROOT_URL + '/register' +# Enable "reindex" button +FEATURES["ENABLE_COURSEWARE_INDEX"] = True + # Create folders if necessary for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]: if not os.path.exists(folder): diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index 1a06d613c2..befd6d6d2f 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -37,6 +37,11 @@ "LOCATION": "staticfiles_lms", } +# Enable search features +FEATURES["ENABLE_COURSE_DISCOVERY"] = True +FEATURES["ENABLE_COURSEWARE_SEARCH"] = True +FEATURES["ENABLE_DASHBOARD_SEARCH"] = True + # Create folders if necessary for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]: if not os.path.exists(folder): diff --git a/tutor/templates/apps/permissions/setowners.sh b/tutor/templates/apps/permissions/setowners.sh index d4044f9067..6ec6e969f9 100644 --- a/tutor/templates/apps/permissions/setowners.sh +++ b/tutor/templates/apps/permissions/setowners.sh @@ -1,6 +1,6 @@ #! /bin/sh setowner $OPENEDX_USER_ID /mounts/lms /mounts/cms /mounts/openedx -{% if RUN_ELASTICSEARCH %}setowner 1000 /mounts/elasticsearch{% endif %} +{% if RUN_MEILISEARCH %}setowner 1000 /mounts/meilisearch{% endif %} {% if RUN_MONGODB %}setowner 999 /mounts/mongodb{% endif %} {% if RUN_MYSQL %}setowner 999 /mounts/mysql{% endif %} {% if RUN_REDIS %}setowner 1000 /mounts/redis{% endif %} diff --git a/tutor/templates/config/base.yml b/tutor/templates/config/base.yml index b1d6a14afa..61e4459a8d 100644 --- a/tutor/templates/config/base.yml +++ b/tutor/templates/config/base.yml @@ -2,6 +2,9 @@ CMS_OAUTH2_SECRET: "{{ 24|random_string }}" ID: "{{ 24|random_string }}" JWT_RSA_PRIVATE_KEY: "{{ 2048|rsa_private_key }}" +MEILISEARCH_MASTER_KEY: "{{ 24|random_string }}" +MEILISEARCH_API_KEY_UID: "{{ 4|uuid }}" +MEILISEARCH_API_KEY: "{{ MEILISEARCH_MASTER_KEY|uid_master_hash(MEILISEARCH_API_KEY_UID) }}" MYSQL_ROOT_PASSWORD: "{{ 8|random_string }}" OPENEDX_MYSQL_PASSWORD: "{{ 8|random_string }}" OPENEDX_SECRET_KEY: "{{ 24|random_string }}" diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index ed22f0c442..e11167140c 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -16,8 +16,8 @@ DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}" # https://hub.docker.com/_/caddy/tags DOCKER_IMAGE_CADDY: "docker.io/caddy:2.7.4" -# https://hub.docker.com/_/elasticsearch/tags -DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.13" +# https://hub.docker.com/r/getmeili/meilisearch/tags +DOCKER_IMAGE_MEILISEARCH: "docker.io/getmeili/meilisearch:v1.8.4" # https://hub.docker.com/_/mongo/tags DOCKER_IMAGE_MONGODB: "docker.io/mongo:7.0.7" # https://hub.docker.com/_/mysql/tags @@ -29,10 +29,6 @@ DOCKER_IMAGE_REDIS: "docker.io/redis:7.2.4" DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.96-r1-0" EDX_PLATFORM_REPOSITORY: "https://github.com/openedx/edx-platform.git" EDX_PLATFORM_VERSION: "{{ OPENEDX_COMMON_VERSION }}" -ELASTICSEARCH_HOST: "elasticsearch" -ELASTICSEARCH_PORT: 9200 -ELASTICSEARCH_SCHEME: "http" -ELASTICSEARCH_HEAP_SIZE: 1g ENABLE_HTTPS: false ENABLE_WEB_PROXY: true JWT_COMMON_AUDIENCE: "openedx" @@ -42,6 +38,9 @@ K8S_NAMESPACE: "openedx" LANGUAGE_CODE: "en" LMS_HOST: "www.myopenedx.com" LOCAL_PROJECT_NAME: "{{ TUTOR_APP }}_local" +MEILISEARCH_URL: "http://meilisearch:7700" +MEILISEARCH_PUBLIC_URL: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://meilisearch.{{ LMS_HOST }}" +MEILISEARCH_INDEX_PREFIX: "tutor_" MONGODB_AUTH_MECHANISM: "" MONGODB_AUTH_SOURCE: "admin" MONGODB_HOST: "mongodb" @@ -73,7 +72,7 @@ REDIS_HOST: "redis" REDIS_PORT: 6379 REDIS_USERNAME: "" REDIS_PASSWORD: "" -RUN_ELASTICSEARCH: true +RUN_MEILISEARCH: true RUN_MONGODB: true RUN_MYSQL: true RUN_REDIS: true diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index acbaa56925..5ef5387d39 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -32,19 +32,20 @@ services: ports: - "8001:8000" + {% if RUN_MEILISEARCH -%} + meilisearch: + ports: + - "127.0.0.1:7700:7700" + networks: + default: + aliases: + - "{{ MEILISEARCH_PUBLIC_URL.split('://')[1] }}" + {%- endif %} + # Additional service for watching theme changes watchthemes: <<: *openedx-service command: npm run watch-sass restart: unless-stopped - {% if RUN_ELASTICSEARCH and is_docker_rootless() %} - elasticsearch: - ulimits: - memlock: - # Fixes error setting rlimits for ready process in rootless docker - soft: 0 # zero means "unset" in the memlock context - hard: 0 - {% endif %} - {{ patch("local-docker-compose-dev-services")|indent(2) }} diff --git a/tutor/templates/jobs/init/cms.sh b/tutor/templates/jobs/init/cms.sh index 1d420a48ab..2b30ad8011 100644 --- a/tutor/templates/jobs/init/cms.sh +++ b/tutor/templates/jobs/init/cms.sh @@ -16,3 +16,7 @@ fi # Create waffle switches to enable some features, if they have not been explicitly defined before # Copy-paste of units in Studio (highly requested new feature, but defaults to off in Quince) (./manage.py cms waffle_flag --list | grep contentstore.enable_copy_paste_units) || ./manage.py lms waffle_flag --create contentstore.enable_copy_paste_units --everyone + +# Re-index studio and courseware content +./manage.py cms reindex_studio --experimental +./manage.py cms reindex_course --active diff --git a/tutor/templates/jobs/init/lms.sh b/tutor/templates/jobs/init/lms.sh index 88c94625c5..93d179c127 100644 --- a/tutor/templates/jobs/init/lms.sh +++ b/tutor/templates/jobs/init/lms.sh @@ -10,6 +10,9 @@ echo "Loading settings $DJANGO_SETTINGS_MODULE" ./manage.py lms migrate +# Create meilisearch indexes +./manage.py lms shell -c "import search.meilisearch; search.meilisearch.create_indexes()" + # Create oauth2 apps for CMS SSO # https://github.com/openedx/edx-platform/blob/master/docs/guides/studio_oauth.rst ./manage.py lms manage_user cms cms@openedx --unusable-password diff --git a/tutor/templates/jobs/init/meilisearch.sh b/tutor/templates/jobs/init/meilisearch.sh new file mode 100644 index 0000000000..2adb79c4e8 --- /dev/null +++ b/tutor/templates/jobs/init/meilisearch.sh @@ -0,0 +1,18 @@ +# Get or create Meilisearch API key +python -c " +import meilisearch +client = meilisearch.Client('{{ MEILISEARCH_URL }}', '{{ MEILISEARCH_MASTER_KEY }}') +try: + client.get_key('{{ MEILISEARCH_API_KEY_UID }}') + print('Key already exists') +except meilisearch.errors.MeilisearchApiError: + print('Key does not exist: creating...') + client.create_key({ + 'name': 'Open edX backend API key', + 'uid': '{{ MEILISEARCH_API_KEY_UID }}', + 'actions': ['*'], + 'indexes': ['{{ MEILISEARCH_INDEX_PREFIX }}*'], + 'expiresAt': None, + 'description': 'Use it for backend API calls -- Created by Tutor', + }) +" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index bf50b48eb7..cb4af9fffc 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -275,24 +275,24 @@ spec: - name: config configMap: name: openedx-config -{% if RUN_ELASTICSEARCH %} +{% if RUN_MEILISEARCH %} --- apiVersion: apps/v1 kind: Deployment metadata: - name: elasticsearch + name: meilisearch labels: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch spec: selector: matchLabels: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch strategy: type: Recreate template: metadata: labels: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch spec: securityContext: runAsUser: 1000 @@ -300,30 +300,22 @@ spec: fsGroup: 1000 fsGroupChangePolicy: "OnRootMismatch" containers: - - name: elasticsearch - image: {{ DOCKER_IMAGE_ELASTICSEARCH }} + - name: meilisearch + image: {{ DOCKER_IMAGE_MEILISEARCH }} env: - - name: cluster.name - value: "openedx" - - name: bootstrap.memory_lock - value: "true" - - name: discovery.type - value: "single-node" - - name: ES_JAVA_OPTS - value: "-Xms{{ ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ ELASTICSEARCH_HEAP_SIZE }}" - - name: TAKE_FILE_OWNERSHIP - value: "1" + - name: MEILI_MASTER_KEY + value: "{{ MEILISEARCH_MASTER_KEY }}" ports: - - containerPort: 9200 + - containerPort: 7700 securityContext: allowPrivilegeEscalation: false volumeMounts: - - mountPath: /usr/share/elasticsearch/data + - mountPath: /meili_data name: data volumes: - name: data persistentVolumeClaim: - claimName: elasticsearch + claimName: meilisearch {% endif %} {% if RUN_MONGODB %} --- diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index c34d2255d8..6cfb554350 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -67,21 +67,21 @@ spec: protocol: TCP selector: app.kubernetes.io/name: lms -{% if RUN_ELASTICSEARCH %} +{% if RUN_MEILISEARCH %} --- apiVersion: v1 kind: Service metadata: - name: elasticsearch + name: meilisearch labels: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch spec: type: ClusterIP ports: - - port: 9200 + - port: 7700 protocol: TCP selector: - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch {% endif %} {% if RUN_MONGODB %} --- diff --git a/tutor/templates/k8s/volumes.yml b/tutor/templates/k8s/volumes.yml index ffb4b66486..aa31555ba0 100644 --- a/tutor/templates/k8s/volumes.yml +++ b/tutor/templates/k8s/volumes.yml @@ -14,21 +14,21 @@ spec: requests: storage: 1Gi {% endif %} -{% if RUN_ELASTICSEARCH %} +{% if RUN_MEILISEARCH %} --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: elasticsearch + name: meilisearch labels: app.kubernetes.io/component: volume - app.kubernetes.io/name: elasticsearch + app.kubernetes.io/name: meilisearch spec: accessModes: - ReadWriteOnce resources: requests: - storage: 2Gi + storage: 5Gi {% endif %} {% if RUN_MONGODB %} --- @@ -78,4 +78,4 @@ spec: requests: storage: 1Gi {% endif %} -{{ patch("k8s-volumes") }} \ No newline at end of file +{{ patch("k8s-volumes") }} diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index 6155cd688a..1e463a02b1 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -43,6 +43,6 @@ services: {%- for mount in iter_mounts(MOUNTS, "openedx", "cms-job") %} - {{ mount }} {%- endfor %} - depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB), ("elasticsearch", RUN_ELASTICSEARCH), ("redis", RUN_REDIS)]|list_if }} + depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB), ("meilisearch", RUN_MEILISEARCH), ("redis", RUN_REDIS)]|list_if }} {{ patch("local-docker-compose-jobs-services")|indent(4) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index da33b653ff..3c5a44d9b9 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -18,7 +18,7 @@ services: - ../../data/openedx-media-private:/mounts/openedx-private {% if RUN_MONGODB %}- ../../data/mongodb:/mounts/mongodb{% endif %} {% if RUN_MYSQL %}- ../../data/mysql:/mounts/mysql{% endif %} - {% if RUN_ELASTICSEARCH %}- ../../data/elasticsearch:/mounts/elasticsearch{% endif %} + {% if RUN_MEILISEARCH %}- ../../data/meilisearch:/mounts/meilisearch{% endif %} {% if RUN_REDIS %}- ../../data/redis:/mounts/redis{% endif %} {{ patch("local-docker-compose-permissions-volumes")|indent(6) }} @@ -54,22 +54,15 @@ services: MYSQL_ROOT_PASSWORD: "{{ MYSQL_ROOT_PASSWORD }}" {%- endif %} - {% if RUN_ELASTICSEARCH -%} - elasticsearch: - image: {{ DOCKER_IMAGE_ELASTICSEARCH }} + {% if RUN_MEILISEARCH -%} + meilisearch: + image: {{ DOCKER_IMAGE_MEILISEARCH }} environment: - - cluster.name=openedx - - bootstrap.memory_lock=true - - discovery.type=single-node - - "ES_JAVA_OPTS=-Xms{{ ELASTICSEARCH_HEAP_SIZE }} -Xmx{{ ELASTICSEARCH_HEAP_SIZE }}" - ulimits: - memlock: - soft: -1 - hard: -1 + MEILI_MASTER_KEY: "{{ MEILISEARCH_MASTER_KEY }}" + volumes: + - ../../data/meilisearch:/meili_data restart: unless-stopped user: "1000:1000" - volumes: - - ../../data/elasticsearch:/usr/share/elasticsearch/data depends_on: - permissions {%- endif %} @@ -120,7 +113,7 @@ services: depends_on: - permissions {% if RUN_MYSQL %}- mysql{% endif %} - {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} + {% if RUN_MEILISEARCH %}- meilisearch{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} @@ -148,7 +141,7 @@ services: - permissions - lms {% if RUN_MYSQL %}- mysql{% endif %} - {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} + {% if RUN_MEILISEARCH %}- meilisearch{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} diff --git a/tutor/utils.py b/tutor/utils.py index cdb082f38a..344f7bcfc0 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -1,4 +1,6 @@ import base64 +import hashlib +import hmac import json import os import random @@ -13,6 +15,7 @@ from typing import List, Tuple from urllib.error import URLError from urllib.request import urlopen +import uuid as uuid_module import click from Crypto.Protocol.KDF import bcrypt, bcrypt_check @@ -110,6 +113,24 @@ def rsa_private_key(bits: int = 2048) -> str: return key.export_key().decode() +def uuid(size: int) -> str: + """ + Return a random uuid string with a given size. + """ + fn = getattr(uuid_module, f"uuid{size}") + return str(fn()) + + +def uid_master_hash(master_key: str, uid: str) -> str: + """ + Hash a key UID and master key to generate an API key + + This is used specifically for meilisearch. + Source: https://www.meilisearch.com/docs/reference/api/keys#key + """ + return hmac.new(master_key.encode(), uid.encode(), hashlib.sha256).hexdigest() + + def rsa_import_key(key: str) -> RsaKey: """ Import PEM-formatted RSA key and return the corresponding object.