Revert delete assets (#2066)

* Revert "fix reverse proxy (#2063)"

This reverts commit f3bf823.

* Revert "remove assets and cached assets (#2040)"

This reverts commit 7d1213c.

chart/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 2.0.0
+version: 1.21.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

chart/env/prod.yaml

@@ -112,12 +112,16 @@ secrets:
       secretName: "datasets-server-prod-secrets"
 
 persistence:
+  cachedAssets:
+    existingClaim: "datasets-server-cached-assets-pvc"
   descriptiveStatistics:
     existingClaim: "datasets-server-statistics-pvc"
   duckDBIndex:
     existingClaim: "datasets-server-duckdb-pvc"
   hfDatasetsCache:
     existingClaim: "datasets-server-cache-pvc"
+  nfs:
+    existingClaim: "nfs-datasets-server-pvc"
   parquetMetadata:
     existingClaim: "datasets-server-parquet-pvc"
 

chart/env/staging.yaml

@@ -107,12 +107,16 @@ secrets:
       secretName: "datasets-server-staging-secrets"
 
 persistence:
+  cachedAssets:
+    existingClaim: "datasets-server-cached-assets-pvc"
   descriptiveStatistics:
     existingClaim: "datasets-server-statistics-pvc"
   duckDBIndex:
     existingClaim: "datasets-server-duckdb-pvc"
   hfDatasetsCache:
     existingClaim: "datasets-server-cache-pvc"
+  nfs:
+    existingClaim: "nfs-datasets-server-pvc"
   parquetMetadata:
     existingClaim: "datasets-server-parquet-pvc"
 

chart/nginx-templates/default.conf.template

@@ -29,6 +29,14 @@ server {
     return 307 https://raw.githubusercontent.com/huggingface/datasets-server/main/${OPENAPI_FILE};
   }
 
+  location /assets/ {
+    alias ${ASSETS_DIRECTORY}/;
+  }
+
+  location /cached-assets/ {
+    alias ${CACHED_ASSETS_DIRECTORY}/;
+  }
+
   location /admin/ {
     # note the trailing slash, to remove the /admin/ prefix
     proxy_pass ${URL_ADMIN}/;

chart/templates/_common/_helpers.tpl

@@ -201,6 +201,15 @@ The assets base URL
 {{- printf "%s%s/assets" (include "datasetsServer.ingress.scheme" .) (include "datasetsServer.ingress.hostname" .) }}
 {{- end }}
 
+{{/*
+The assets/ subpath in the NFS
+- in a subdirectory named as the chart (datasets-server/), and below it,
+- in a subdirectory named as the Release, so that Releases will not share the same dir
+*/}}
+{{- define "assets.subpath" -}}
+{{- printf "%s/%s/%s/" .Chart.Name .Release.Name "assets" }}
+{{- end }}
+
 {{/*
 The cached-assets base URL
 */}}
@@ -209,7 +218,16 @@ The cached-assets base URL
 {{- end }}
 
 {{/*
-The parquet-metadata/ subpath in the EFS
+The cached-assets/ subpath in the NFS
+- in a subdirectory named as the chart (datasets-server/), and below it,
+- in a subdirectory named as the Release, so that Releases will not share the same dir
+*/}}
+{{- define "cachedAssets.subpath" -}}
+{{- printf "%s/%s/%s/" .Chart.Name .Release.Name "cached-assets" }}
+{{- end }}
+
+{{/*
+The parquet-metadata/ subpath in the NFS
 - in a subdirectory named as the chart (datasets-server/), and below it,
 - in a subdirectory named as the Release, so that Releases will not share the same dir
 */}}
@@ -227,7 +245,7 @@ The duckdb-index/ subpath in EFS
 {{- end }}
 
 {{/*
-The stats-cache/ subpath in the EFS
+The stats-cache/ subpath in the NFS
 - in a subdirectory named as the chart (datasets-server/), and below it,
 - in a subdirectory named as the Release, so that Releases will not share the same dir
 */}}

chart/templates/_env/_envWorker.tpl

@@ -25,6 +25,8 @@
 - name: TMPDIR
   value: "/tmp"
   # ^ensure the temporary files are created in /tmp, which is writable
+- name: WORKER_STORAGE_PATHS
+  value: {{ .Values.assets.storageDirectory | quote }}
 # specific to the /first-rows job runner
 - name: FIRST_ROWS_MAX_BYTES
   value: {{ .Values.firstRows.maxBytes | quote }}

chart/templates/_initContainers/_initContainerAssets.tpl

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 The HuggingFace Authors.
+
+{{- define "initContainerAssets" -}}
+- name: prepare-assets
+  image: ubuntu:focal
+  imagePullPolicy: {{ .Values.images.pullPolicy }}
+  command: ["/bin/sh", "-c"]
+  args:
+  - chown {{ .Values.uid }}:{{ .Values.gid }} /mounted-path;
+  volumeMounts:
+  - mountPath: /mounted-path
+    mountPropagation: None
+    name: volume-nfs
+    subPath: "{{ include "assets.subpath" . }}"
+    readOnly: false
+  securityContext:
+    runAsNonRoot: false
+    runAsUser: 0
+    runAsGroup: 0
+{{- end -}}

chart/templates/_initContainers/_initContainerCachedAssets.tpl

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 The HuggingFace Authors.
+
+{{- define "initContainerCachedAssets" -}}
+- name: prepare-cached-assets
+  image: ubuntu:focal
+  imagePullPolicy: {{ .Values.images.pullPolicy }}
+  command: ["/bin/sh", "-c"]
+  args:
+  - chown {{ .Values.uid }}:{{ .Values.gid }} /mounted-path;
+  volumeMounts:
+  - mountPath: /mounted-path
+    mountPropagation: None
+    name: volume-cached-assets
+    subPath: "{{ include "cachedAssets.subpath" . }}"
+    readOnly: false
+  securityContext:
+    runAsNonRoot: false
+    runAsUser: 0
+    runAsGroup: 0
+{{- end -}}

chart/templates/_volumeMounts/_volumeMountAssets.tpl

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 The HuggingFace Authors.
+
+{{- define "volumeMountAssetsRO" -}}
+- mountPath: {{ .Values.assets.storageDirectory | quote }}
+  mountPropagation: None
+  name: volume-nfs
+  subPath: "{{ include "assets.subpath" . }}"
+  readOnly: true
+{{- end -}}
+
+{{- define "volumeMountAssetsRW" -}}
+- mountPath: {{ .Values.assets.storageDirectory | quote }}
+  mountPropagation: None
+  name: volume-nfs
+  subPath: "{{ include "assets.subpath" . }}"
+  readOnly: false
+{{- end -}}

chart/templates/_volumeMounts/_volumeMountCachedAssets.tpl

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 The HuggingFace Authors.
+
+{{- define "volumeMountCachedAssetsRO" -}}
+- mountPath: {{ .Values.cachedAssets.storageDirectory | quote }}
+  mountPropagation: None
+  name: volume-cached-assets
+  subPath: "{{ include "cachedAssets.subpath" . }}"
+  readOnly: true
+{{- end -}}
+
+{{- define "volumeMountCachedAssetsRW" -}}
+- mountPath: {{ .Values.cachedAssets.storageDirectory | quote }}
+  mountPropagation: None
+  name: volume-cached-assets
+  subPath: "{{ include "cachedAssets.subpath" . }}"
+  readOnly: false
+{{- end -}}

chart/templates/_volumes/_volumeCachedAssets.tpl

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 The HuggingFace Authors.
+
+{{- define "volumeCachedAssets" -}}
+- name: volume-cached-assets
+  persistentVolumeClaim:
+    claimName: {{ .Values.persistence.cachedAssets.existingClaim | default (include "name" .) }}
+{{- end -}}

chart/templates/_volumes/_volumeNfs.tpl

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 The HuggingFace Authors.
+
+{{- define "volumeNfs" -}}
+- name: volume-nfs
+  persistentVolumeClaim:
+    claimName: {{ .Values.persistence.nfs.existingClaim | default (include "name" .) }}
+{{- end -}}

chart/templates/pvc.yaml

@@ -0,0 +1,17 @@
+{{- if (not .Values.persistence.nfs.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  labels: {{ include "hf.labels.commons" . | nindent 4 }}
+  name: {{ include "name" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: {{ .Values.persistence.nfs.size }}
+  {{ if ne "" .Values.persistence.nfs.storageClass }}
+  storageClassName: {{ .Values.persistence.nfs.storageClass }}
+  {{ end }}
+{{- end }}

chart/templates/reverse-proxy/_container.tpl

@@ -6,6 +6,10 @@
   image: {{ include "reverseproxy.image" . }}
   imagePullPolicy: {{ .Values.images.pullPolicy }}
   env:
+  - name: ASSETS_DIRECTORY
+    value: {{ .Values.assets.storageDirectory | quote }}
+  - name: CACHED_ASSETS_DIRECTORY
+    value: {{ .Values.cachedAssets.storageDirectory | quote }}
   - name: OPENAPI_FILE
     value: {{ .Values.reverseProxy.openapiFile | quote }}
   - name: HOST
@@ -22,6 +26,9 @@
     value: {{ include "search.url" . | quote }}
   - name: URL_SSE_API
     value: {{ include "sseApi.url" . | quote }}
+  volumeMounts:
+  {{ include "volumeMountAssetsRO" . | nindent 2 }}
+  {{ include "volumeMountCachedAssetsRO" . | nindent 2 }}
   - name: nginx-templates
     mountPath: /etc/nginx/templates
     mountPropagation: None

chart/templates/reverse-proxy/deployment.yaml

@@ -27,10 +27,14 @@ spec:
     spec:
       {{- include "image.imagePullSecrets" . | nindent 6 }}
       initContainers: 
+        {{ include "initContainerAssets" . | nindent 8 }}
+        {{ include "initContainerCachedAssets" . | nindent 8 }}
       containers: {{ include "containerReverseProxy" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.reverseProxy.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.reverseProxy.tolerations | nindent 8 }}
       volumes:
+      {{ include "volumeCachedAssets" . | nindent 6 }}
+      {{ include "volumeNfs" . | nindent 6 }}
       - name: nginx-templates
         configMap:
           name: "{{ include "name" . }}-reverse-proxy"

chart/templates/services/admin/_container.tpl

@@ -45,6 +45,8 @@
   - name: ADMIN_UVICORN_PORT
     value: {{ .Values.admin.uvicornPort | quote }}
   volumeMounts:
+  {{ include "volumeMountAssetsRW" . | nindent 2 }}
+  {{ include "volumeMountCachedAssetsRW" . | nindent 2 }}
   {{ include "volumeMountDescriptiveStatisticsRO" . | nindent 2 }}
   {{ include "volumeMountDuckDBIndexRO" . | nindent 2 }}
   {{ include "volumeMountHfDatasetsCacheRO" . | nindent 2 }}

chart/templates/services/admin/deployment.yaml

@@ -25,6 +25,8 @@ spec:
       {{- include "dnsConfig" . | nindent 6 }}
       {{- include "image.imagePullSecrets" . | nindent 6 }}
       initContainers: 
+        {{ include "initContainerAssets" . | nindent 8 }}
+        {{ include "initContainerCachedAssets" . | nindent 8 }}
         {{ include "initContainerDescriptiveStatistics" . | nindent 8 }}
         {{ include "initContainerDuckDBIndex" . | nindent 8 }}
         {{ include "initContainerHfDatasetsCache" . | nindent 8 }}
@@ -36,5 +38,7 @@ spec:
         {{ include "volumeDescriptiveStatistics" . | nindent 8 }}
         {{ include "volumeDuckDBIndex" . | nindent 8 }}
         {{ include "volumeHfDatasetsCache" . | nindent 8 }}
+        {{ include "volumeNfs" . | nindent 8 }}
         {{ include "volumeParquetMetadata" . | nindent 8 }}
+        {{ include "volumeCachedAssets" . | nindent 8 }}
       securityContext: {{ include "securityContext" . | nindent 8 }}

chart/templates/services/rows/_container.tpl

@@ -33,6 +33,7 @@
   - name: ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY
     value: {{ .Values.rowsIndex.maxArrowDataInMemory | quote }}
   volumeMounts:
+  {{ include "volumeMountCachedAssetsRW" . | nindent 2 }}
   {{ include "volumeMountParquetMetadataRO" . | nindent 2 }}
   securityContext:
     allowPrivilegeEscalation: false

chart/templates/services/rows/deployment.yaml

@@ -25,10 +25,12 @@ spec:
       {{- include "dnsConfig" . | nindent 6 }}
       {{- include "image.imagePullSecrets" . | nindent 6 }}
       initContainers:
+        {{ include "initContainerCachedAssets" . | nindent 8 }}
         {{ include "initContainerParquetMetadata" . | nindent 8 }}
       containers: {{ include "containerRows" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.rows.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.rows.tolerations | nindent 8 }}
       volumes: 
+        {{ include "volumeCachedAssets" . | nindent 8 }}
         {{ include "volumeParquetMetadata" . | nindent 8 }}
       securityContext: {{ include "securityContext" . | nindent 8 }}

chart/templates/services/search/_container.tpl

@@ -34,6 +34,7 @@
   - name: DUCKDB_INDEX_CACHE_DIRECTORY
     value: {{ .Values.duckDBIndex.cacheDirectory | quote }}
   volumeMounts:
+  {{ include "volumeMountCachedAssetsRW" . | nindent 2 }}
   {{ include "volumeMountDuckDBIndexRW" . | nindent 2 }}
   securityContext:
     allowPrivilegeEscalation: false

chart/templates/services/search/deployment.yaml

@@ -25,10 +25,12 @@ spec:
       {{- include "dnsConfig" . | nindent 6 }}
       {{- include "image.imagePullSecrets" . | nindent 6 }}
       initContainers:
+        {{ include "initContainerCachedAssets" . | nindent 8 }}
         {{ include "initContainerDuckDBIndex" . | nindent 8 }}
       containers: {{ include "containerSearch" . | nindent 8 }}
       nodeSelector: {{ toYaml .Values.search.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.search.tolerations | nindent 8 }}
       volumes: 
+        {{ include "volumeCachedAssets" . | nindent 8 }}
         {{ include "volumeDuckDBIndex" . | nindent 8 }}
       securityContext: {{ include "securityContext" . | nindent 8 }}

chart/templates/storage-admin/_container.tpl

@@ -6,10 +6,16 @@
   image: {{ include "services.storageAdmin.image" . }}
   imagePullPolicy: {{ .Values.images.pullPolicy }}
   volumeMounts:
+  {{ include "volumeMountAssetsRW" . | nindent 2 }}
+  {{ include "volumeMountCachedAssetsRW" . | nindent 2 }}
   {{ include "volumeMountDescriptiveStatisticsRW" . | nindent 2 }}
   {{ include "volumeMountDuckDBIndexRW" . | nindent 2 }}
   {{ include "volumeMountHfDatasetsCacheRW" . | nindent 2 }}
   {{ include "volumeMountParquetMetadataRW" . | nindent 2 }}
+  - mountPath: /volumes/cached-assets
+    mountPropagation: None
+    name: volume-cached-assets
+    readOnly: false
   - mountPath: /volumes/descriptive-statistics
     mountPropagation: None
     name: volume-descriptive-statistics
@@ -22,6 +28,10 @@
     mountPropagation: None
     name: volume-hf-datasets-cache
     readOnly: false
+  - mountPath: /volumes/nfs
+    mountPropagation: None
+    name: volume-nfs
+    readOnly: false
   - mountPath: /volumes/parquet-metadata
     mountPropagation: None
     name: volume-parquet-metadata

chart/templates/storage-admin/deployment.yaml

@@ -18,6 +18,8 @@ spec:
       labels: {{ include "labels.storageAdmin" . | nindent 8 }}
     spec:
       initContainers:
+        {{ include "initContainerAssets" . | nindent 8 }}
+        {{ include "initContainerCachedAssets" . | nindent 8 }}
         {{ include "initContainerDescriptiveStatistics" . | nindent 8 }}
         {{ include "initContainerDuckDBIndex" . | nindent 8 }}
         {{ include "initContainerHfDatasetsCache" . | nindent 8 }}
@@ -26,7 +28,9 @@ spec:
       nodeSelector: {{ toYaml .Values.storageAdmin.nodeSelector | nindent 8 }}
       tolerations: {{ toYaml .Values.storageAdmin.tolerations | nindent 8 }}
       volumes:
+        {{ include "volumeCachedAssets" . | nindent 8 }}
         {{ include "volumeDescriptiveStatistics" . | nindent 8 }}
         {{ include "volumeDuckDBIndex" . | nindent 8 }}
         {{ include "volumeHfDatasetsCache" . | nindent 8 }}
+        {{ include "volumeNfs" . | nindent 8 }}
         {{ include "volumeParquetMetadata" . | nindent 8 }}

chart/templates/worker/_container.tpl

@@ -29,6 +29,7 @@
   - name: ROWS_INDEX_MAX_ARROW_DATA_IN_MEMORY
     value: {{ .Values.rowsIndex.maxArrowDataInMemory | quote }}
   volumeMounts:
+  {{ include "volumeMountAssetsRW" . | nindent 2 }}
   {{ include "volumeMountDescriptiveStatisticsRW" . | nindent 2 }}
   {{ include "volumeMountDuckDBIndexRW" . | nindent 2 }}
   {{ include "volumeMountHfDatasetsCacheRW" . | nindent 2 }}