feat(traefik): enhance file upload handling and timeouts

- Add buffering middleware for large file uploads (100MB limit) - Configure dial and response timeouts (120s) - Add network connectivity for MinIO integration - Update service configuration for improved reliability This resolves context timeout issues and enables handling of large file uploads.
jembi · Dec 5, 2024 · f9d1c6f · f9d1c6f
1 parent 9a5a967
commit f9d1c6f
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 8 deletions.
diff --git a/database-postgres/package-metadata.json b/database-postgres/package-metadata.json
@@ -9,7 +9,7 @@
     "REPMGR_PRIMARY_HOST": "postgres-1",
     "REPMGR_PARTNER_NODES": "postgres-1",
     "REPMGR_PASSWORD": "instant101",
-    "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14",
+    "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14@sha256:bdf1e4903710c4e0b465664b886d4556897e1b18c07d7c513a4fc1ceba929e02",
     "POSTGRES_1_PLACEMENT": "node-1",
     "POSTGRES_2_PLACEMENT": "node-2",
     "POSTGRES_3_PLACEMENT": "node-3",

diff --git a/datalake/docker-compose.yml b/datalake/docker-compose.yml
@@ -24,16 +24,31 @@ services:
       labels:
         - traefik.enable=true
         - traefik.docker.network=reverse-proxy-traefik_public
-        - traefik.http.routers.minio.rule=${DOMAIN_NAME_HOST_TRAEFIK} && PathPrefix(`/minio`)
-        - traefik.http.services.minio.loadbalancer.server.port=9001
-        - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio
-        - traefik.http.routers.minio.middlewares=minio-stripprefix
+        - traefik.http.routers.minio-console.rule=Host(`${DOMAIN_NAME_HOST_TRAEFIK}`) && PathPrefix(`/minio-console`)
+        - traefik.http.routers.minio-console.priority=100
+        - traefik.http.routers.minio-console.service=minio-console-service
+        - traefik.http.services.minio-console-service.loadbalancer.server.port=9001
+        - traefik.http.middlewares.minio-console-stripprefix.stripprefix.prefixes=/minio-console/
+        - traefik.http.routers.minio-console.middlewares=minio-console-stripprefix
+        - traefik.http.routers.minio-console.tls=${TLS}
+
+        - traefik.http.routers.minio-api.rule=Host(`${DOMAIN_NAME_HOST_TRAEFIK}`) && PathPrefix(`/minio-api`)
+        - traefik.http.routers.minio-api.priority=100
+        - traefik.http.routers.minio-api.service=minio-api-service
+        - traefik.http.services.minio-api-service.loadbalancer.server.port=9090
+        - traefik.http.middlewares.minio-api-stripprefix.stripprefix.prefixes=/minio-api/
+        - traefik.http.routers.minio-api.middlewares=minio-api-stripprefix
+        - traefik.http.routers.minio-api.tls=${TLS}
     networks:
       public:
+      traefik:
 networks:
   public:
     name: minio_public
     external: true
+  traefik:
+    name: reverse-proxy-traefik_public
+    external: true
 
 volumes:
   minio-01-data1:

diff --git a/documentation/packages/reverse-proxy-traefik/README.md b/documentation/packages/reverse-proxy-traefik/README.md
@@ -92,14 +92,30 @@ Set the following environment variables in the package-metadata.json in the "./d
 Set the following environment variables in the package-metadata.json in the "monitoring" directory
 
 ```bash
-
 "environmentVariables":
 {
 # Other Configurations
 ...
-    "MINIO_BROWSER_REDIRECT_URL": "https://domain/minio/"
+    "MINIO_BROWSER_REDIRECT_URL": "https://domain/minio-console/"
 }
+```
+
+### MinIO Configuration
+
+The MinIO server is configured to run with the following port settings:
+
+- **API Port**: 9090
+- **Console Port**: 9001
+
+Ensure that your Traefik configuration reflects these ports to properly route traffic to the MinIO services. The API can be accessed at `https://<domain>/minio` and the Console at `https://<domain>/minio-console`.
+
+Update your Traefik labels in the `docker-compose.yml` to match these settings:
 
+```yaml
+# API Configuration
+- traefik.http.services.minio.loadbalancer.server.port=9090
+# Console Configuration
+- traefik.http.services.minio-console.loadbalancer.server.port=9001
 ```
 
 ### Enabling Grafana

diff --git a/monitoring/package-metadata.json b/monitoring/package-metadata.json
@@ -29,7 +29,6 @@
     "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only",
     "MO_RETENTION_TIME": "15d",
     "GF_SERVER_SERVE_FROM_SUB_PATH": "false",
-    "MINIO_BROWSER_REDIRECT_URL": "",
     "DOCKER_SOCK_FOLDER": "/var/run/docker.sock",
     "DOCKER_LIB_FOLDER": "/var/lib/docker/"
   }

diff --git a/reverse-proxy-traefik/docker-compose.yml b/reverse-proxy-traefik/docker-compose.yml
@@ -49,6 +49,10 @@ services:
         - traefik.http.middlewares.to-https.redirectscheme.permanent=${REDIRECT_TO_HTTPS}
         - traefik.http.middlewares.auth.basicauth.users=${USERNAME}:${PASSWORD}
 
+        - "traefik.http.middlewares.bigfiles.buffering.maxRequestBodyBytes=100000000"
+        - "traefik.http.service.traefik.loadbalancer.server.forwardingTimeouts.dialTimeout=120s"
+        - "traefik.http.service.traefik.loadbalancer.server.forwardingTimeouts.responseHeaderTimeout=120s"
+
       placement:
         max_replicas_per_node: 1
         constraints: