Skip to content

Commit

Permalink
alllow for database connection, grpc worker thread pool, and k8s clie…
Browse files Browse the repository at this point in the history 
…nt level tuning of the watcher and api servers

- add open and idle database connection pool settings to the config.env file
- but don't change golang's database/sql defaults
- add the ablity to increase GRPC worker thread poll count
- add the ability set set K8s client QPS and burst settings to the api sever via config.env and to the watcher via command line arguments

rh-pre-commit.version: 2.2.0
rh-pre-commit.check-secrets: ENABLED
  • Loading branch information
@stuartwdouglas @tekton-robot
stuartwdouglas authored and tekton-robot committed Apr 16, 2024
1 parent 9309cee commit 714dfd2
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 34 deletions.
67 changes: 36 additions & 31 deletions cmd/api/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,42 @@

## Variables

| Environment Variable | Description | Example |
|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------|
| DB_USER | Postgres Database user | user |
| DB_PASSWORD | Postgres Database Password | hunter2 |
| DB_HOST | Postgres Database host | /cloudsql/my-project:us-east1:tekton-results |
| DB_NAME | Postgres Database name | tekton_results |
| DB_SSLMODE | Database SSL mode | verify-full |
| DB_SSLROOTCERT | Path to CA cert used to validate Database cert | /etc/tls/db/ca.crt |
| DB_ENABLE_AUTO_MIGRATION | Auto-migrate the database on startup (create/update schemas). For further details, refer to <https://gorm.io/docs/migration.html> | true (default) |
| PROFILING | Enable profiling server | false (default) |
| PROFILING_PORT | Profiling Server Port | 6060 (default) |
| SERVER_PORT | gRPC and REST Server Port | 8080 (default) |
| PROMETHEUS_PORT | Prometheus Port | 9090 (default) |
| PROMETHEUS_HISTOGRAM | Enable Prometheus histogram metrics to measure latency distributions of RPCs | false (default) |
| TLS_PATH | Path to TLS files | /etc/tls |
| AUTH_DISABLE | Disable RBAC check for resources | false (default) |
| AUTH_IMPERSONATE | Enable RBAC impersonation | true (default) |
| LOG_LEVEL | Log level for api server | info (default) |
| LOGS_API | Enable logs storage service | false (default) |
| LOGS_TYPE | Determine Logs storage backend type | File (default) |
| LOGS_BUFFER_SIZE | Buffer for streaming logs | 32768 (default) |
| LOGS_PATH | Logs storage path | logs (default) |
| S3_BUCKET_NAME | S3 Bucket name | <S3 Bucket Name> |
| S3_ENDPOINT | S3 Endpoint | https://s3.ap-south-1.amazonaws.com |
| S3_HOSTNAME_IMMUTABLE | S3 Hostname immutable | false (default) |
| S3_REGION | S3 Region | ap-south-1 |
| S3_ACCESS_KEY_ID | S3 Access Key ID | <S3 Acces Key> |
| S3_SECRET_ACCESS_KEY | S3 Secret Access Key | <S3 Access Secret> |
| S3_MULTI_PART_SIZE | S3 Multi part size | 5242880 (default) |
| GCS_BUCKET_NAME | GCS Bucket Name | <GCS Bucket Name> |
| STORAGE_EMULATOR_HOST | GCS Storage Emulator Server | http://localhost:9004 |
| Environment Variable | Description | Example |
|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------|
| DB_USER | Postgres Database user | user |
| DB_PASSWORD | Postgres Database Password | hunter2 |
| DB_HOST | Postgres Database host | /cloudsql/my-project:us-east1:tekton-results |
| DB_NAME | Postgres Database name | tekton_results |
| DB_SSLMODE | Database SSL mode | verify-full |
| DB_SSLROOTCERT | Path to CA cert used to validate Database cert | /etc/tls/db/ca.crt |
| DB_ENABLE_AUTO_MIGRATION | Auto-migrate the database on startup (create/update schemas). For further details, refer to <https://gorm.io/docs/migration.html> | true (default) |
| PROFILING | Enable profiling server | false (default) |
| PROFILING_PORT | Profiling Server Port | 6060 (default) |
| DB_MAX_IDLE_CONNECTIONS | The number of idle database connections to keep open | 2 (default for golang, but specific database drivers may have settings for this too) |
| DB_MAX_OPEN_CONNECTIONS | The maximum number of database connections, for best performance it should equal DB_MAX_IDLE_CONNECTIONS | unlimited (default for golang, but specific database drivers may have settings for this too) |
| GRPC_WORKER_POOL | The maximum number of goroutines pre-allocated for process GRPC requests. The GRPC server will also dynamically create threads. | 2 (default) |
| K8S_QPS | The QPS setting for the kubernetes client created. | 5 (default) |
| K8S_BURST | The burst setting for the kubernetes client created. | 10 (default) |
| SERVER_PORT | gRPC and REST Server Port | 8080 (default) |
| PROMETHEUS_PORT | Prometheus Port | 9090 (default) |
| PROMETHEUS_HISTOGRAM | Enable Prometheus histogram metrics to measure latency distributions of RPCs | false (default) |
| TLS_PATH | Path to TLS files | /etc/tls |
| AUTH_DISABLE | Disable RBAC check for resources | false (default) |
| AUTH_IMPERSONATE | Enable RBAC impersonation | true (default) |
| LOG_LEVEL | Log level for api server | info (default) |
| LOGS_API | Enable logs storage service | false (default) |
| LOGS_TYPE | Determine Logs storage backend type | File (default) |
| LOGS_BUFFER_SIZE | Buffer for streaming logs | 32768 (default) |
| LOGS_PATH | Logs storage path | logs (default) |
| S3_BUCKET_NAME | S3 Bucket name | <S3 Bucket Name> |
| S3_ENDPOINT | S3 Endpoint | https://s3.ap-south-1.amazonaws.com |
| S3_HOSTNAME_IMMUTABLE | S3 Hostname immutable | false (default) |
| S3_REGION | S3 Region | ap-south-1 |
| S3_ACCESS_KEY_ID | S3 Access Key ID | <S3 Acces Key> |
| S3_SECRET_ACCESS_KEY | S3 Secret Access Key | <S3 Access Secret> |
| S3_MULTI_PART_SIZE | S3 Multi part size | 5242880 (default) |
| GCS_BUCKET_NAME | GCS Bucket Name | <GCS Bucket Name> |
| STORAGE_EMULATOR_HOST | GCS Storage Emulator Server | http://localhost:9004 |

These values can also be set in the config file located in the `config/env/config` directory.

Expand Down
45 changes: 43 additions & 2 deletions cmd/api/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package main
import (
"context"
"crypto/tls"
"database/sql"
"fmt"
"net/http"
"path"
Expand Down Expand Up @@ -119,6 +120,33 @@ func main() {
log.Fatalf("Failed to connect to database: %v", err)
}

var sqlDB *sql.DB

// Set DB connection limits
maxIdle := serverConfig.DB_MAX_IDLE_CONNECTIONS
maxOpen := serverConfig.DB_MAX_OPEN_CONNECTIONS
if maxOpen > 0 {
sqlDB, err = db.DB()
if err != nil {
log.Fatalf("Error getting database configuration for updating max open connections: %s", err.Error())
}
sqlDB.SetMaxOpenConns(maxOpen)
}
if maxIdle > 0 {
sqlDB, err = db.DB()
if err != nil {
log.Fatalf("Error getting database configuration for updating max open connections: %s", err.Error())
}
sqlDB.SetMaxIdleConns(maxIdle)
}

// Set grpc worker pool
grpcWorkers := serverConfig.GRPC_WORKER_POOL
var streamWorkers grpc.ServerOption
if grpcWorkers > 2 {
streamWorkers = grpc.NumStreamWorkers((uint32)(grpcWorkers))
}

// Create the authorization authCheck
var authCheck auth.Checker
var serverMuxOptions []runtime.ServeMuxOption
Expand All @@ -132,6 +160,15 @@ func main() {
if err != nil {
log.Fatal("Error getting kubernetes client config:", err)
}
// Override k8s client qps/burts settings
qps := serverConfig.K8S_QPS
burst := serverConfig.K8S_BURST
if qps > 0 {
k8sConfig.QPS = (float32)(qps)
}
if burst > 0 {
k8sConfig.Burst = burst
}
k8s, err := kubernetes.NewForConfig(k8sConfig)
if err != nil {
log.Fatal("Error creating kubernetes clientset:", err)
Expand Down Expand Up @@ -163,7 +200,7 @@ func main() {
// Customize logger, so it can be passed to the gRPC interceptors
grpcLogger := log.Desugar().With(zap.Bool("grpc.auth_disabled", serverConfig.AUTH_DISABLE))

gs := grpc.NewServer(
svrOpts := []grpc.ServerOption{
grpc.Creds(creds),
grpc_middleware.WithUnaryServerChain(
// The grpc_ctxtags context updater should be before everything else
Expand All @@ -181,7 +218,11 @@ func main() {
prometheus.StreamServerInterceptor,
recovery.StreamServerInterceptor(recovery.WithRecoveryHandler(recoveryHandler)),
),
)
}
if streamWorkers != nil {
svrOpts = append(svrOpts, streamWorkers)
}
gs := grpc.NewServer(svrOpts...)
v1alpha2pb.RegisterResultsServer(gs, v1a2)
if serverConfig.LOGS_API {
v1alpha2pb.RegisterLogsServer(gs, v1a2)
Expand Down
Loading

0 comments on commit 714dfd2

Please sign in to comment.