Change reservation unit search from Elastic to Postgres FTS

.env.example

@@ -2,7 +2,6 @@
 
 DATABASE_URL=postgis://tvp:[email protected]:5432/tvp
 REDIS_URL=redis://127.0.0.1:6379/0
-ELASTICSEARCH_URL=http://localhost:9200
 
 QUERY_LOGGING_ENABLED=false
 #QUERY_LOGGING_SKIP_ROUTES=/admin,/static

.github/workflows/test.yml

@@ -49,14 +49,6 @@ jobs:
         ports:
           - 6379:6379
 
-      elastic:
-        image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
-        env:
-          xpack.security.enabled: false
-          discovery.type: single-node
-        ports:
-          - 9200:9200
-
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v4

Makefile

@@ -79,9 +79,6 @@ generate:
 hooks:
 	@pre-commit install
 
-indices:
-	@python manage.py rebuild_search_index reservation_units
-
 lint:
 	@pre-commit run --all-files
 
@@ -95,7 +92,7 @@ run:
 	@docker compose up --detach --build
 
 services:
-	@docker compose up --detach --build db elastic redis
+	@docker compose up --detach --build db redis
 
 stop:
 	@docker compose stop

README.md

@@ -39,7 +39,6 @@ The preferred contact method is through Helsinki City Slack.
 - [PostgreSQL] (with the [PostGIS] extension) for database needs
 - [Redis] for in-memory caching
 - [Celery] for scheduling and background task handling
-- [Elasticsearch] you know, for search
 - [Poetry] for dependency management
 - [Docker] for containerization
 - [Django] as the web framework
@@ -87,13 +86,7 @@ To generate test data, follow the steps below.
 make bash
 ```
 
-2. (Re)create elastic search index.
-
-```shell
-make indices
-```
-
-3. Generate test data.
+2. Generate test data.
 
 ```shell
 make generate
@@ -112,7 +105,6 @@ Before you start, you'll need the following dependencies:
 - [Make] ([Windows][Make (Windows)], [Mac][Make (Mac)])
 - [PostgreSQL] (with the [PostGIS] extension) (version 13 or newer)
 - [Redis] (version 7 or newer)
-- [Elasticsearch] (version 8.8)
 - [GDAL] (version compatible with Django, check their documentation for more info)
     - Ubuntu: `sudo apt-get install gdal-bin`
     - Mac: `brew install gdal`
@@ -126,7 +118,7 @@ Before you start, you'll need the following dependencies:
 > change over time, so please refer to the official documentation for each dependency
 > on how to set them up correctly.
 
-> You can skip the dependencies for Postgres, Redis, and Elasticsearch by running
+> You can skip the dependencies for Postgres and Redis by running
 > them using Docker. To do this, install [Docker] and run `make services`.
 
 Now, follow the steps below.
@@ -166,19 +158,13 @@ poetry run make hooks
 poetry run make migrate
 ```
 
-6. (Re)create elastic search indices
-
-```shell
-poetry run make indices
-```
-
-7. Generate test data
+6. Generate test data
 
 ```shell
 poetry run make generate
 ```
 
-8. Start the server
+7. Start the server
 
 ```shell
 poetry run make dev
@@ -193,7 +179,6 @@ Tests are run with `pytest`.
 Some flags that can save time when running tests:
 
 - To skip slow-running tests: `pytest --skip-slow`
-- To skip tests requiring Elasticsearch: `pytest --skip-elastic`
 - To retain test database between runs: `pytest --reuse-db`
 - To skip migration-checks at the start of tests: `pytest --no-migrations`
 - To run tests in parallel: `pytest -n 8 --dist=loadscope` (=8 cores, use `-n auto` to use all available cores)
@@ -308,7 +293,6 @@ In settings there are four configurations:
 [django-helusers]: https://github.com/City-of-Helsinki/django-helusers
 [Django]: https://www.djangoproject.com/
 [Docker]: https://www.docker.com/
-[Elasticsearch]: https://www.elastic.co/guide/en/elasticsearch/reference/current/elasticsearch-intro.html
 [GDAL]: https://gdal.org/index.html
 [gettext]: https://www.gnu.org/software/gettext/
 [Graphene]: https://github.com/graphql-python/graphene-django

config/settings.py

@@ -1,4 +1,6 @@
 # ruff: noqa: N802
+from __future__ import annotations
+
 import os
 import zoneinfo
 from pathlib import Path
@@ -71,7 +73,6 @@ class Common(Environment):
         "django_extensions",
         "django_filters",
         "easy_thumbnails",
-        "elasticsearch_django",
         "graphene_django",
         "import_export",
         "mptt",
@@ -414,7 +415,7 @@ def OIDC_API_TOKEN_AUTH(cls):
     CELERY_PROCESSED_FOLDER = values.StringValue(default="/broker/processed/")
 
     @classproperty
-    def CELERY_BROKER_URL(cls):
+    def CELERY_BROKER_URL(cls) -> str:
         return "filesystem://"
 
     @classproperty
@@ -443,44 +444,6 @@ def CACHES(cls):
             }
         }
 
-    # --- Elasticsearch settings -------------------------------------------------------------------------------------
-
-    ELASTICSEARCH_URL = values.StringValue()
-    DISABLE_ELASTICSEARCH_INDEXES = values.BooleanValue(default=False)
-
-    @classproperty
-    def SEARCH_SETTINGS(cls):
-        return {
-            "connections": {
-                "default": cls.ELASTICSEARCH_URL,
-            },
-            "indexes": (
-                {}
-                if cls.DISABLE_ELASTICSEARCH_INDEXES
-                else {
-                    "reservation_units": {
-                        "models": [
-                            "tilavarauspalvelu.ReservationUnit",
-                        ]
-                    }
-                }
-            ),
-            "settings": {
-                # batch size for ES bulk api operations
-                "chunk_size": 500,
-                # default page size for search results
-                "page_size": 10000,
-                # set to True to connect post_save/delete signals
-                "auto_sync": True,
-                # List of models which will never auto_sync even if auto_sync is True
-                "never_auto_sync": [],
-                # if true, then indexes must have mapping files
-                "strict_validation": False,
-                # Path to the elasticsearch mapping files
-                "mappings_dir": str(Common.BASE_DIR / "config" / "elasticsearch"),
-            },
-        }
-
     # --- Admin data views ------------------------------------------------------------------------------------------
 
     ADMIN_DATA_VIEWS = {
@@ -520,6 +483,7 @@ def SEARCH_SETTINGS(cls):
     TPREK_UNIT_URL = values.URLValue()
     GRAPHQL_CODEGEN_ENABLED = False
     UPDATE_RESERVATION_UNIT_HIERARCHY = True
+    UPDATE_SEARCH_VECTORS = True
     UPDATE_AFFECTING_TIME_SPANS = True
     SAVE_RESERVATION_STATISTICS = True
     REBUILD_SPACE_HIERARCHY = True
@@ -562,7 +526,6 @@ class EmptyDefaults:
 
     DATABASES = {}
     REDIS_URL = ""
-    ELASTICSEARCH_URL = ""
 
     STATIC_ROOT = Common.BASE_DIR / "staticroot"
     MEDIA_ROOT = Common.BASE_DIR / "media"
@@ -683,22 +646,13 @@ class Local(Common, overrides_from=LocalMixin):
 
     REDIS_URL = values.StringValue(default="redis://127.0.0.1:6379/0")
 
-    # --- Elasticsearch settings -------------------------------------------------------------------------------------
-
-    ELASTICSEARCH_URL = values.StringValue(default="http://localhost:9200")
-
-    @classproperty
-    def SEARCH_SETTINGS(cls):
-        search_settings = super().SEARCH_SETTINGS
-        search_settings["settings"]["mappings_dir"] = str(Common.BASE_DIR / "config" / "elasticsearch")
-        return search_settings
-
     # --- Misc settings-----------------------------------------------------------------------------------------------
 
     SENTRY_LOGGER_ALWAYS_RE_RAISE = True
     GRAPHQL_CODEGEN_ENABLED = values.BooleanValue(default=False)
     ICAL_HASH_SECRET = values.StringValue(default="")  # nosec # NOSONAR
     UPDATE_RESERVATION_UNIT_HIERARCHY = values.BooleanValue(default=True)
+    UPDATE_SEARCH_VECTORS = values.BooleanValue(default=True)
     UPDATE_AFFECTING_TIME_SPANS = values.BooleanValue(default=True)
     SAVE_RESERVATION_STATISTICS = values.BooleanValue(default=True)
     REBUILD_SPACE_HIERARCHY = values.BooleanValue(default=True)
@@ -740,7 +694,6 @@ class Docker(Common, overrides_from=DockerMixin):
     SOCIAL_AUTH_TUNNISTAMO_INACTIVE_USER_URL = "http://localhost:3000/deactivated-account"
 
     REDIS_URL = values.StringValue(default="redis://redis:6379/0")
-    ELASTICSEARCH_URL = values.StringValue(default="http://elastic:9200")
 
     @classproperty
     def CELERY_BROKER_URL(cls):
@@ -864,17 +817,6 @@ class AutomatedTests(EmptyDefaults, Common, dotenv_path=None, overrides_from=Aut
 
     REDIS_URL = values.StringValue(default="redis://localhost:6379/0")
 
-    # --- Elasticsearch settings -------------------------------------------------------------------------------------
-
-    ELASTICSEARCH_URL = values.StringValue(default="http://localhost:9200")
-
-    @classproperty
-    def SEARCH_SETTINGS(cls):
-        search_settings = super().SEARCH_SETTINGS
-        search_settings["settings"]["mappings_dir"] = str(Common.BASE_DIR / "config" / "elasticsearch")
-        search_settings["settings"]["auto_sync"] = False
-        return search_settings
-
     # --- Misc settings ----------------------------------------------------------------------------------------------
 
     CACHES = {
@@ -890,7 +832,11 @@ def SEARCH_SETTINGS(cls):
     # since they slow them down a lot in CI. Refresh should be called manually when needed.
     UPDATE_RESERVATION_UNIT_HIERARCHY = False
     UPDATE_AFFECTING_TIME_SPANS = False
+    # Turn off search vector updates from signals during tests.
+    UPDATE_SEARCH_VECTORS = False
+    # Turn off reservation unit thumbnail updates from signals during tests.
     UPDATE_RESERVATION_UNIT_THUMBNAILS = False
+    # Turn off image downloads during tests.
     DOWNLOAD_IMAGES_FOR_TEST_DATA = False
     # Turn off statistics saving during tests for performance reasons
     SAVE_RESERVATION_STATISTICS = False
@@ -959,7 +905,7 @@ def CACHES(cls):
     # --- Celery settings --------------------------------------------------------------------------------------------
 
     @classproperty
-    def CELERY_BROKER_URL(cls):
+    def CELERY_BROKER_URL(cls) -> str:
         return f"sentinel://:{cls.REDIS_PASSWORD}@{cls.REDIS_SENTINEL_SERVICE}"
 
     @classproperty

docker-compose.yml

@@ -17,21 +17,6 @@ services:
     volumes:
       - tvp-postgres:/var/lib/postgresql/data
 
-  elastic:
-    container_name: tvp-elastic
-    image: "docker.elastic.co/elasticsearch/elasticsearch:8.8.0"
-    environment:
-      xpack.security.enabled: false
-      discovery.type: single-node
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    ports:
-      - "127.0.0.1:9200:9200"
-    volumes:
-      - tvp-elastic:/usr/share/elasticsearch/data
-
   redis:
     container_name: tvp-redis
     image: "redis:7-alpine"
@@ -86,8 +71,6 @@ services:
 volumes:
   tvp-postgres:
   tvp-media:
-  tvp-elastic:
-    driver: local
 
 networks:
   default: