Search service with local storage (#2701)

* search service with local storage

* Unmount all references to volumeDuckDBIndex

* fix style

* Remove duckdb mount

* Add clean directory call randomly

* Fix missing env vars

* validate disk space

* profile only when running cleaning

* fix space validation

* remove extra profile step

* fix style and apply code review suggestion

* Apply code review suggestion

chart/env/prod.yaml

@@ -227,16 +227,6 @@ backfillRetryableErrors:
       cpu: 2
       memory: "8Gi"
 
-cleanDuckdbIndexCache:
-  nodeSelector:
-    role-datasets-server: "true"
-  expiredTimeIntervalSeconds: 259_200 # 3 days
-
-cleanDuckdbIndexDownloads:
-  nodeSelector:
-    role-datasets-server: "true"
-  expiredTimeIntervalSeconds: 259_200 # 3 days
-
 postMessages:
   nodeSelector:
     role-datasets-server: "true"
@@ -387,7 +377,7 @@ search:
   uvicornNumWorkers: "2"
   nodeSelector:
     role-datasets-server-search: "true"
-  replicas: 7
+  replicas: 4
   service:
     type: NodePort
   ingress:

chart/templates/_common/_helpers.tpl

@@ -109,16 +109,6 @@ app.kubernetes.io/component: "{{ include "name" . }}-backfill"
 app.kubernetes.io/component: "{{ include "name" . }}-backfill-retryable-errors"
 {{- end -}}
 
-{{- define "labels.cleanDuckdbIndexCache" -}}
-{{ include "hf.labels.commons" . }}
-app.kubernetes.io/component: "{{ include "name" . }}-clean-duckdb-cache"
-{{- end -}}
-
-{{- define "labels.cleanDuckdbIndexDownloads" -}}
-{{ include "hf.labels.commons" . }}
-app.kubernetes.io/component: "{{ include "name" . }}-clean-duckdb-downloads"
-{{- end -}}
-
 {{- define "labels.postMessages" -}}
 {{ include "hf.labels.commons" . }}
 app.kubernetes.io/component: "{{ include "name" . }}-post-messages"

chart/templates/services/admin/_container.tpl

@@ -15,8 +15,6 @@
   {{ include "envAssets" . | nindent 2 }}
   {{ include "envCachedAssets" . | nindent 2 }}
   {{ include "envParquetMetadata" . | nindent 2 }}
-  - name: DUCKDB_INDEX_CACHE_DIRECTORY
-    value: {{ .Values.duckDBIndex.cacheDirectory | quote }}
   # service
   - name: ADMIN_HF_ORGANIZATION
     value: {{ .Values.admin.hfOrganization | quote }}
@@ -41,7 +39,6 @@
   - name: ADMIN_UVICORN_PORT
     value: {{ .Values.admin.uvicornPort | quote }}
   volumeMounts:
-  {{ include "volumeMountDuckDBIndexRO" . | nindent 2 }}
   {{ include "volumeMountParquetMetadataRO" . | nindent 2 }}
   securityContext:
     allowPrivilegeEscalation: false

chart/templates/services/admin/deployment.yaml

@@ -25,12 +25,10 @@ spec:
       {{- include "dnsConfig" . | nindent 6 }}
       {{- include "image.imagePullSecrets" . | nindent 6 }}
       initContainers: 
-        {{ include "initContainerDuckDBIndex" . | nindent 8 }}
         {{ include "initContainerParquetMetadata" . | nindent 8 }}
       containers: {{ include "containerAdmin" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.admin.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.admin.tolerations | nindent 8 }}
       volumes:
-        {{ include "volumeDuckDBIndex" . | nindent 8 }}
         {{ include "volumeParquetMetadata" . | nindent 8 }}
       securityContext: {{ include "securityContext" . | nindent 8 }}

chart/templates/services/search/_container.tpl

@@ -35,12 +35,14 @@
     value: {{ .Values.duckDBIndex.targetRevision | quote }}
   - name: DUCKDB_INDEX_CACHE_DIRECTORY
     value: {{ .Values.duckDBIndex.cacheDirectory | quote }}
+  - name: DUCKDB_INDEX_CACHE_CLEAN_CACHE_PROBA
+    value: {{ .Values.search.cleanCacheProba | quote }}
+  - name: DUCKDB_INDEX_CACHE_EXPIRED_TIME_INTERVAL_SECONDS
+    value: {{ .Values.search.expiredTimeIntervalSeconds | quote }}
   - name: DUCKDB_INDEX_EXTENSIONS_DIRECTORY
     value: "/tmp/duckdb-extensions"
   - name: HF_HUB_ENABLE_HF_TRANSFER
     value: "1"
-  volumeMounts:
-  {{ include "volumeMountDuckDBIndexRW" . | nindent 2 }}
   securityContext:
     allowPrivilegeEscalation: false
   readinessProbe:

chart/templates/services/search/deployment.yaml

@@ -24,11 +24,7 @@ spec:
     spec:
       {{- include "dnsConfig" . | nindent 6 }}
       {{- include "image.imagePullSecrets" . | nindent 6 }}
-      initContainers:
-        {{ include "initContainerDuckDBIndex" . | nindent 8 }}
       containers: {{ include "containerSearch" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.search.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.search.tolerations | nindent 8 }}
-      volumes: 
-        {{ include "volumeDuckDBIndex" . | nindent 8 }}
       securityContext: {{ include "securityContext" . | nindent 8 }}

chart/templates/storage-admin/_container.tpl

@@ -6,12 +6,7 @@
   image: {{ include "services.storageAdmin.image" . }}
   imagePullPolicy: {{ .Values.images.pullPolicy }}
   volumeMounts:
-  {{ include "volumeMountDuckDBIndexRW" . | nindent 2 }}
   {{ include "volumeMountParquetMetadataRW" . | nindent 2 }}
-  - mountPath: /volumes/duckdb-index
-    mountPropagation: None
-    name: volume-duckdb-index
-    readOnly: false
   - mountPath: /volumes/parquet-metadata
     mountPropagation: None
     name: volume-parquet-metadata

chart/templates/storage-admin/deployment.yaml

@@ -18,11 +18,9 @@ spec:
       labels: {{ include "labels.storageAdmin" . | nindent 8 }}
     spec:
       initContainers:
-        {{ include "initContainerDuckDBIndex" . | nindent 8 }}
         {{ include "initContainerParquetMetadata" . | nindent 8 }}
       containers: {{ include "containerStorageAdmin" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.storageAdmin.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.storageAdmin.tolerations | nindent 8 }}
       volumes:
-        {{ include "volumeDuckDBIndex" . | nindent 8 }}
         {{ include "volumeParquetMetadata" . | nindent 8 }}

chart/values.yaml

@@ -144,8 +144,6 @@ uid: 1000
 gid: 3000
 
 persistence:
-  duckDBIndex:
-    existingClaim: ""
   parquetMetadata:
     existingClaim: ""
 
@@ -259,8 +257,8 @@ parquetMetadata:
   storageDirectory: "/storage/parquet-metadata"
 
 duckDBIndex:
-  # Directory on the shared storage (used to cache the duckdb files for /filter and /search)
-  cacheDirectory: "/storage/duckdb-index"
+  # Directory on the local storage (used to cache the duckdb files for /filter and /search)
+  cacheDirectory: "/tmp/duckdb-index"
   # Directory on the worker (used temporarily to prepare the duckdb indexes before sending to the Hub)
   workerDirectory: "/tmp/duckdb-index"
   # the git commit message when the duckdb index file is uploaded to the Hub. Defaults to `Update duckdb index files`.
@@ -337,42 +335,6 @@ backfillRetryableErrors:
       cpu: 0
   tolerations: []
 
-cleanDuckdbIndexCache:
-  enabled: true
-  log:
-    level: "info"
-  action: "clean-directory"
-  schedule: "0 */3 * * *"
-  # every 3 hours
-  nodeSelector: {}
-  resources:
-    requests:
-      cpu: 0
-    limits:
-      cpu: 0
-  tolerations: []
-  # the time interval at which cached downloaded data files will be considered as expired and will be deleted
-  expiredTimeIntervalSeconds: 600
-  subfolderPattern: "cache/*"
-
-cleanDuckdbIndexDownloads:
-  enabled: true
-  log:
-    level: "info"
-  action: "clean-directory"
-  schedule: "0 */3 * * *"
-  # every 3 hours
-  nodeSelector: {}
-  resources:
-    requests:
-      cpu: 0
-    limits:
-      cpu: 0
-  tolerations: []
-  # the time interval at which cached downloaded data files will be considered as expired and will be deleted
-  expiredTimeIntervalSeconds: 600
-  subfolderPattern: "downloads/*"
-
 postMessages:
   enabled: true
   log:
@@ -572,6 +534,10 @@ search:
   uvicornNumWorkers: "1"
   # Application endpoint port
   uvicornPort: 8080
+  # Probability of cleaning the downloads folder at each request.
+  cleanCacheProba: 0.05
+  # Retention period for downloads.
+  expiredTimeIntervalSeconds: 43_200 # 12 hours
 
   nodeSelector: {}
   replicas: 1

e2e/tests/test_31_admin_metrics.py

@@ -58,12 +58,6 @@ def test_metrics() -> None:
         ), f"responses_in_cache_total - cache kind {cache_kind} found in {metrics}"
 
     # the disk usage metrics, on the other end, are computed at runtime, so we should see them
-    assert has_metric(
-        name="duckdb_disk_usage",
-        labels={"type": "total", "pid": "[0-9]*"},
-        metric_names=metric_names,
-    ), "duckdb_disk_usage"
-
     assert has_metric(
         name="parquet_metadata_disk_usage",
         labels={"type": "total", "pid": "[0-9]*"},