From f9d1c6f727ad10a2ca027c1a496200e6f052757a Mon Sep 17 00:00:00 2001 From: drono Date: Thu, 5 Dec 2024 13:46:50 +0300 Subject: [PATCH] feat(traefik): enhance file upload handling and timeouts - Add buffering middleware for large file uploads (100MB limit) - Configure dial and response timeouts (120s) - Add network connectivity for MinIO integration - Update service configuration for improved reliability This resolves context timeout issues and enables handling of large file uploads. --- database-postgres/package-metadata.json | 2 +- datalake/docker-compose.yml | 23 +++++++++++++++---- .../packages/reverse-proxy-traefik/README.md | 20 ++++++++++++++-- monitoring/package-metadata.json | 1 - reverse-proxy-traefik/docker-compose.yml | 4 ++++ 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/database-postgres/package-metadata.json b/database-postgres/package-metadata.json index 10beda95..f31c9f0c 100644 --- a/database-postgres/package-metadata.json +++ b/database-postgres/package-metadata.json @@ -9,7 +9,7 @@ "REPMGR_PRIMARY_HOST": "postgres-1", "REPMGR_PARTNER_NODES": "postgres-1", "REPMGR_PASSWORD": "instant101", - "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14", + "POSTGRES_IMAGE": "bitnami/postgresql-repmgr:14@sha256:bdf1e4903710c4e0b465664b886d4556897e1b18c07d7c513a4fc1ceba929e02", "POSTGRES_1_PLACEMENT": "node-1", "POSTGRES_2_PLACEMENT": "node-2", "POSTGRES_3_PLACEMENT": "node-3", diff --git a/datalake/docker-compose.yml b/datalake/docker-compose.yml index 0c1f2cb2..b2f759c3 100644 --- a/datalake/docker-compose.yml +++ b/datalake/docker-compose.yml @@ -24,16 +24,31 @@ services: labels: - traefik.enable=true - traefik.docker.network=reverse-proxy-traefik_public - - traefik.http.routers.minio.rule=${DOMAIN_NAME_HOST_TRAEFIK} && PathPrefix(`/minio`) - - traefik.http.services.minio.loadbalancer.server.port=9001 - - traefik.http.middlewares.minio-stripprefix.stripprefix.prefixes=/minio - - traefik.http.routers.minio.middlewares=minio-stripprefix + - traefik.http.routers.minio-console.rule=Host(`${DOMAIN_NAME_HOST_TRAEFIK}`) && PathPrefix(`/minio-console`) + - traefik.http.routers.minio-console.priority=100 + - traefik.http.routers.minio-console.service=minio-console-service + - traefik.http.services.minio-console-service.loadbalancer.server.port=9001 + - traefik.http.middlewares.minio-console-stripprefix.stripprefix.prefixes=/minio-console/ + - traefik.http.routers.minio-console.middlewares=minio-console-stripprefix + - traefik.http.routers.minio-console.tls=${TLS} + + - traefik.http.routers.minio-api.rule=Host(`${DOMAIN_NAME_HOST_TRAEFIK}`) && PathPrefix(`/minio-api`) + - traefik.http.routers.minio-api.priority=100 + - traefik.http.routers.minio-api.service=minio-api-service + - traefik.http.services.minio-api-service.loadbalancer.server.port=9090 + - traefik.http.middlewares.minio-api-stripprefix.stripprefix.prefixes=/minio-api/ + - traefik.http.routers.minio-api.middlewares=minio-api-stripprefix + - traefik.http.routers.minio-api.tls=${TLS} networks: public: + traefik: networks: public: name: minio_public external: true + traefik: + name: reverse-proxy-traefik_public + external: true volumes: minio-01-data1: diff --git a/documentation/packages/reverse-proxy-traefik/README.md b/documentation/packages/reverse-proxy-traefik/README.md index 872651f0..08c04c1d 100644 --- a/documentation/packages/reverse-proxy-traefik/README.md +++ b/documentation/packages/reverse-proxy-traefik/README.md @@ -92,14 +92,30 @@ Set the following environment variables in the package-metadata.json in the "./d Set the following environment variables in the package-metadata.json in the "monitoring" directory ```bash - "environmentVariables": { # Other Configurations ... - "MINIO_BROWSER_REDIRECT_URL": "https://domain/minio/" + "MINIO_BROWSER_REDIRECT_URL": "https://domain/minio-console/" } +``` + +### MinIO Configuration + +The MinIO server is configured to run with the following port settings: + +- **API Port**: 9090 +- **Console Port**: 9001 + +Ensure that your Traefik configuration reflects these ports to properly route traffic to the MinIO services. The API can be accessed at `https:///minio` and the Console at `https:///minio-console`. + +Update your Traefik labels in the `docker-compose.yml` to match these settings: +```yaml +# API Configuration +- traefik.http.services.minio.loadbalancer.server.port=9090 +# Console Configuration +- traefik.http.services.minio-console.loadbalancer.server.port=9001 ``` ### Enabling Grafana diff --git a/monitoring/package-metadata.json b/monitoring/package-metadata.json index 31691759..52d582f9 100644 --- a/monitoring/package-metadata.json +++ b/monitoring/package-metadata.json @@ -29,7 +29,6 @@ "MO_SECURITY_ADMIN_PASSWORD": "dev_password_only", "MO_RETENTION_TIME": "15d", "GF_SERVER_SERVE_FROM_SUB_PATH": "false", - "MINIO_BROWSER_REDIRECT_URL": "", "DOCKER_SOCK_FOLDER": "/var/run/docker.sock", "DOCKER_LIB_FOLDER": "/var/lib/docker/" } diff --git a/reverse-proxy-traefik/docker-compose.yml b/reverse-proxy-traefik/docker-compose.yml index ebd83ea3..09918f96 100644 --- a/reverse-proxy-traefik/docker-compose.yml +++ b/reverse-proxy-traefik/docker-compose.yml @@ -49,6 +49,10 @@ services: - traefik.http.middlewares.to-https.redirectscheme.permanent=${REDIRECT_TO_HTTPS} - traefik.http.middlewares.auth.basicauth.users=${USERNAME}:${PASSWORD} + - "traefik.http.middlewares.bigfiles.buffering.maxRequestBodyBytes=100000000" + - "traefik.http.service.traefik.loadbalancer.server.forwardingTimeouts.dialTimeout=120s" + - "traefik.http.service.traefik.loadbalancer.server.forwardingTimeouts.responseHeaderTimeout=120s" + placement: max_replicas_per_node: 1 constraints: