From 8ce0972a4a0009d4f7bd7603a9750d7c90f905d4 Mon Sep 17 00:00:00 2001 From: Odd Eirik Igland Date: Wed, 27 Nov 2024 13:06:13 +0100 Subject: [PATCH 1/4] adds gcp proxy --- charts/lakefs/Chart.yaml | 2 +- charts/lakefs/README.md | 2 ++ .../lakefs/templates/_gcp_proxy_container.tpl | 18 ++++++++++++++++++ charts/lakefs/templates/deployment.yaml | 1 + charts/lakefs/values.yaml | 3 +++ 5 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 charts/lakefs/templates/_gcp_proxy_container.tpl diff --git a/charts/lakefs/Chart.yaml b/charts/lakefs/Chart.yaml index 4836b91..394cbab 100644 --- a/charts/lakefs/Chart.yaml +++ b/charts/lakefs/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: lakefs description: A Helm chart for running LakeFS on Kubernetes type: application -version: 1.3.20 +version: 1.3.21 appVersion: 1.43.0 home: https://lakefs.io diff --git a/charts/lakefs/README.md b/charts/lakefs/README.md index 9baf08e..e6ef36c 100644 --- a/charts/lakefs/README.md +++ b/charts/lakefs/README.md @@ -130,5 +130,7 @@ helm upgrade -f my-values.yaml my-lakefs lakefs/lakefs --set kv_upgrade=true | `s3Fallback.enabled` | If set to true, an [S3Proxy](https://github.com/gaul/s3proxy) container will be started. Requests to lakeFS S3 gateway with a non-existing repository will be forwarded to this container. | | | `s3Fallback.aws_access_key` | An AWS access key to be used by the S3Proxy for authentication | | | `s3Fallback.aws_secret_key` | An AWS secret key to be used by the S3Proxy for authentication | | +| `gcpFallback.enabled` | If set to true, an [GCP Proxy](https://github.com/GoogleCloudPlatform/cloud-sql-proxy) container will be started. | | +| `gcpFallback.instances` | The instances to connect to. | | | `committedLocalCacheVolume` | A volume definition to be mounted by lakeFS and used for caching committed metadata. See [here](https://kubernetes.io/docs/concepts/storage/volumes/#volume-types) for a list of supported volume types. The default values.yaml file shows an example of how to use this parameter. | | | `serviceAccount.name` | Name of the service account to use for the lakeFS pods. If not set, use the `default` service account. | | diff --git a/charts/lakefs/templates/_gcp_proxy_container.tpl b/charts/lakefs/templates/_gcp_proxy_container.tpl new file mode 100644 index 0000000..f3d5a4c --- /dev/null +++ b/charts/lakefs/templates/_gcp_proxy_container.tpl @@ -0,0 +1,18 @@ +{{- define "lakefs.gcpProxyContainer" }} +{{- if .Values.lakefsConfig }} +{{ $config := .Values.lakefsConfig | fromYaml }} +{{- end }} +{{- if .Values.gcpFallback.enabled }} +- name: gcp-proxy + image: eu.gcr.io/cloudsql-docker/gce-proxy:1.33.4 + imagePullPolicy: IfNotPresent + command: + - /cloud_sql_proxy + - -term_timeout=10s + env: +{{- if .Values.gcpFallback.instances }} + - name: INSTANCES + value: {{ .Values.gcpFallback.instances }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/lakefs/templates/deployment.yaml b/charts/lakefs/templates/deployment.yaml index a6711b9..cb41932 100644 --- a/charts/lakefs/templates/deployment.yaml +++ b/charts/lakefs/templates/deployment.yaml @@ -91,6 +91,7 @@ spec: {{- end }} {{- include "lakefs.env" . | nindent 10 }} {{- include "lakefs.s3proxyContainer" . | nindent 8}} + {{- include "lakefs.gcpProxyContainer" . | nindent 8}} volumes: {{- include "lakefs.volumes" . | nindent 8 }} {{- with .Values.nodeSelector }} diff --git a/charts/lakefs/values.yaml b/charts/lakefs/values.yaml index 195ce35..957210e 100644 --- a/charts/lakefs/values.yaml +++ b/charts/lakefs/values.yaml @@ -59,6 +59,9 @@ topologySpreadConstraints: [] s3Fallback: enabled: false +gcpFallback: + enabled: false + lakefsConfig: | database: type: local From 647626f015f86ede4f3c1a96f6b9d9c9fee595f5 Mon Sep 17 00:00:00 2001 From: Odd Eirik Igland Date: Wed, 27 Nov 2024 13:56:14 +0100 Subject: [PATCH 2/4] my values example --- charts/lakefs/README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/charts/lakefs/README.md b/charts/lakefs/README.md index e6ef36c..e11a8a9 100644 --- a/charts/lakefs/README.md +++ b/charts/lakefs/README.md @@ -48,6 +48,26 @@ lakefsConfig: | domain_name: s3.lakefs.example.com ``` +Example `my-values.yaml` using PostgreSQL with Cloud SQL Auth Proxy in GCP: + +```yaml +secrets: + databaseConnectionString: postgres://:@localhost:5432/ + authEncryptSecretKey: +lakefsConfig: | + database: + type: postgres + blockstore: + type: gs + gs: + credentials_json: '' +serviceAccount: + name: +gcpFallback: + enabled: true + instances: +``` + Example `my-values.yaml` using DynamoDB: ```yaml secrets: From 46c60ee565c20731151d6b30a9504e059e490cc5 Mon Sep 17 00:00:00 2001 From: Odd Eirik Igland Date: Mon, 2 Dec 2024 10:15:48 +0100 Subject: [PATCH 3/4] change to instance --- charts/lakefs/README.md | 8 ++++---- charts/lakefs/templates/_gcp_proxy_container.tpl | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/lakefs/README.md b/charts/lakefs/README.md index e11a8a9..956a782 100644 --- a/charts/lakefs/README.md +++ b/charts/lakefs/README.md @@ -65,7 +65,7 @@ serviceAccount: name: gcpFallback: enabled: true - instances: + instance: ``` Example `my-values.yaml` using DynamoDB: @@ -150,7 +150,7 @@ helm upgrade -f my-values.yaml my-lakefs lakefs/lakefs --set kv_upgrade=true | `s3Fallback.enabled` | If set to true, an [S3Proxy](https://github.com/gaul/s3proxy) container will be started. Requests to lakeFS S3 gateway with a non-existing repository will be forwarded to this container. | | | `s3Fallback.aws_access_key` | An AWS access key to be used by the S3Proxy for authentication | | | `s3Fallback.aws_secret_key` | An AWS secret key to be used by the S3Proxy for authentication | | -| `gcpFallback.enabled` | If set to true, an [GCP Proxy](https://github.com/GoogleCloudPlatform/cloud-sql-proxy) container will be started. | | -| `gcpFallback.instances` | The instances to connect to. | | +| `gcpFallback.enabled` | If set to true, an [GCP Proxy](https://github.com/GoogleCloudPlatform/cloud-sql-proxy) container will be started. | | +| `gcpFallback.instance` | The instance to connect to. See the example above for the format. | | | `committedLocalCacheVolume` | A volume definition to be mounted by lakeFS and used for caching committed metadata. See [here](https://kubernetes.io/docs/concepts/storage/volumes/#volume-types) for a list of supported volume types. The default values.yaml file shows an example of how to use this parameter. | | -| `serviceAccount.name` | Name of the service account to use for the lakeFS pods. If not set, use the `default` service account. | | +| `serviceAccount.name` | Name of the service account to use for the lakeFS pods. If not set, use the `default` service account. | | diff --git a/charts/lakefs/templates/_gcp_proxy_container.tpl b/charts/lakefs/templates/_gcp_proxy_container.tpl index f3d5a4c..56ac84d 100644 --- a/charts/lakefs/templates/_gcp_proxy_container.tpl +++ b/charts/lakefs/templates/_gcp_proxy_container.tpl @@ -10,9 +10,9 @@ - /cloud_sql_proxy - -term_timeout=10s env: -{{- if .Values.gcpFallback.instances }} +{{- if .Values.gcpFallback.instance }} - name: INSTANCES - value: {{ .Values.gcpFallback.instances }} + value: {{ .Values.gcpFallback.instance }} {{- end }} {{- end }} {{- end }} From 312c2f19797625a904685ad44f0543b961362c81 Mon Sep 17 00:00:00 2001 From: Odd Eirik Igland Date: Mon, 2 Dec 2024 10:49:16 +0100 Subject: [PATCH 4/4] bump version --- charts/lakefs/Chart.yaml | 2 +- charts/lakefs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/lakefs/Chart.yaml b/charts/lakefs/Chart.yaml index 394cbab..b90a576 100644 --- a/charts/lakefs/Chart.yaml +++ b/charts/lakefs/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: lakefs description: A Helm chart for running LakeFS on Kubernetes type: application -version: 1.3.21 +version: 1.3.22 appVersion: 1.43.0 home: https://lakefs.io diff --git a/charts/lakefs/README.md b/charts/lakefs/README.md index 956a782..d8b879e 100644 --- a/charts/lakefs/README.md +++ b/charts/lakefs/README.md @@ -65,7 +65,7 @@ serviceAccount: name: gcpFallback: enabled: true - instance: + instance: ::=tcp:5432 ``` Example `my-values.yaml` using DynamoDB: