Make Kafka REST Proxy & Kafka Connect hosts default and improve Schem…

…a Registry config (#354)
bakdata · Oct 5, 2023 · b800dcc · b800dcc
1 parent fed04f6
commit b800dcc
Show file tree

Hide file tree

Showing 53 changed files with 620 additions and 380 deletions.
diff --git a/config.yaml b/config.yaml
@@ -1,5 +1,2 @@
 environment: development
-brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
-kafka_connect_host: "http://localhost:8083"
-kafka_rest_host: "http://localhost:8082"
-schema_registry_url: "http://localhost:8081"
+kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
diff --git a/docs/docs/developer/auto-generation.md b/docs/docs/developer/auto-generation.md
@@ -10,7 +10,7 @@ Auto generation happens mostly with [`pre-commit`](https://pre-commit.com/) hook
 
 - `cli_env_vars.env` -- All CLI environment variables in a `dotenv` file.
 - `cli_env_vars.md` -- All CLI environment variables in a table.
-- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in [`PipelineConfig`](https://github.com/bakdata/kpops/blob/main/kpops/cli/pipeline_config.py) whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it.
+- `config_env_vars.env` -- Almost all pipeline config environment variables in a `dotenv` file. The script checks for each field in [`PipelineConfig`](https://github.com/bakdata/kpops/blob/main/kpops/cli/kpops_config.py) whether it has an `env` attribute defined. The script is currently unable to visit the classes of fields like `topic_name_config`, hence any environment variables defined there would remain unknown to it.
 - `config_env_vars.env` -- Almost all pipeline config environment variables in a table.
 - `variable_substitution.yaml` -- A copy of `./tests/pipeline/resources/component-type-substitution/pipeline.yaml` used as an example of substitution.
 

diff --git a/docs/docs/resources/pipeline-components/kafka-app.yaml b/docs/docs/resources/pipeline-components/kafka-app.yaml
@@ -49,7 +49,7 @@
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app

diff --git a/docs/docs/resources/pipeline-components/pipeline.yaml b/docs/docs/resources/pipeline-components/pipeline.yaml
@@ -49,7 +49,7 @@
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
@@ -275,7 +275,7 @@
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:
@@ -346,7 +346,7 @@
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1

diff --git a/docs/docs/resources/pipeline-components/producer-app.yaml b/docs/docs/resources/pipeline-components/producer-app.yaml
@@ -32,7 +32,7 @@
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:

diff --git a/docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml b/docs/docs/resources/pipeline-components/sections/app-kafka-app.yaml
@@ -2,7 +2,7 @@
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
diff --git a/docs/docs/resources/pipeline-components/sections/app-producer-app.yaml b/docs/docs/resources/pipeline-components/sections/app-producer-app.yaml
@@ -2,7 +2,7 @@
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:

diff --git a/docs/docs/resources/pipeline-components/sections/app-streams-app.yaml b/docs/docs/resources/pipeline-components/sections/app-streams-app.yaml
@@ -4,7 +4,7 @@
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1

diff --git a/docs/docs/resources/pipeline-components/streams-app.yaml b/docs/docs/resources/pipeline-components/streams-app.yaml
@@ -51,7 +51,7 @@
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1

diff --git a/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-kafka-app.yaml
@@ -7,7 +7,7 @@ kafka-app:
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app

diff --git a/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-producer-app.yaml
@@ -10,7 +10,7 @@ producer-app:
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:

diff --git a/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml b/docs/docs/resources/pipeline-defaults/defaults-streams-app.yaml
@@ -9,7 +9,7 @@ streams-app:
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1

diff --git a/docs/docs/resources/pipeline-defaults/defaults.yaml b/docs/docs/resources/pipeline-defaults/defaults.yaml
@@ -7,7 +7,7 @@ kafka-app:
   # add the key-value pairs they need.
   app: # required
     streams: # required
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
     nameOverride: override-with-this-name # kafka-app-specific
     imageTag: "1.0.0" # Example values that are shared between streams-app and producer-app
@@ -170,7 +170,7 @@ producer-app:
   # https://github.com/bakdata/streams-bootstrap/tree/master/charts/producer-app
   app: # required
     streams: # required, producer-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       outputTopic: output_topic
       extraOutputTopics:
@@ -188,7 +188,7 @@ streams-app:
   app: # required
     # Streams Bootstrap streams section
     streams: # required, streams-app-specific
-      brokers: ${brokers} # required
+      brokers: ${kafka_brokers} # required
       schemaRegistryUrl: ${schema_registry_url}
       inputTopics:
         - topic1

diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env
@@ -9,18 +9,18 @@
 # Suffix your environment files with this value (e.g.
 # defaults_development.yaml for environment=development).
 KPOPS_ENVIRONMENT # No default value, required
-# brokers
+# kafka_brokers
 # The comma separated Kafka brokers address.
 KPOPS_KAFKA_BROKERS # No default value, required
-# schema_registry_url
+# url
 # Address of the Schema Registry.
-KPOPS_SCHEMA_REGISTRY_URL # No default value, not required
-# kafka_rest_host
+KPOPS_SCHEMA_REGISTRY_URL=http://localhost:8081
+# url
 # Address of the Kafka REST Proxy.
-KPOPS_REST_PROXY_HOST # No default value, not required
-# kafka_connect_host
+KPOPS_KAFKA_REST_URL=http://localhost:8082
+# url
 # Address of Kafka Connect.
-KPOPS_CONNECT_HOST # No default value, not required
+KPOPS_KAFKA_CONNECT_URL=http://localhost:8083
 # timeout
 # The timeout in seconds that specifies when actions like deletion or
 # deploy timeout.

diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md
@@ -1,11 +1,11 @@
 These variables are a lower priority alternative to the settings in `config.yaml`. Variables marked as required can instead be set in the pipeline config.
 
-|          Name           |Default Value|Required|                                                                                Description                                                                                 |   Setting name    |
-|-------------------------|-------------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------|
-|KPOPS_ENVIRONMENT        |             |True    |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).|environment        |
-|KPOPS_KAFKA_BROKERS      |             |True    |The comma separated Kafka brokers address.                                                                                                                                  |brokers            |
-|KPOPS_SCHEMA_REGISTRY_URL|             |False   |Address of the Schema Registry.                                                                                                                                             |schema_registry_url|
-|KPOPS_REST_PROXY_HOST    |             |False   |Address of the Kafka REST Proxy.                                                                                                                                            |kafka_rest_host    |
-|KPOPS_CONNECT_HOST       |             |False   |Address of Kafka Connect.                                                                                                                                                   |kafka_connect_host |
-|KPOPS_TIMEOUT            |          300|False   |The timeout in seconds that specifies when actions like deletion or deploy timeout.                                                                                         |timeout            |
-|KPOPS_RETAIN_CLEAN_JOBS  |False        |False   |Whether to retain clean up jobs in the cluster or uninstall the, after completion.                                                                                          |retain_clean_jobs  |
+|          Name           |    Default Value    |Required|                                                                                Description                                                                                 |  Setting name   |
+|-------------------------|---------------------|--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|
+|KPOPS_ENVIRONMENT        |                     |True    |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).|environment      |
+|KPOPS_KAFKA_BROKERS      |                     |True    |The comma separated Kafka brokers address.                                                                                                                                  |kafka_brokers    |
+|KPOPS_SCHEMA_REGISTRY_URL|http://localhost:8081|False   |Address of the Schema Registry.                                                                                                                                             |url              |
+|KPOPS_KAFKA_REST_URL     |http://localhost:8082|False   |Address of the Kafka REST Proxy.                                                                                                                                            |url              |
+|KPOPS_KAFKA_CONNECT_URL  |http://localhost:8083|False   |Address of Kafka Connect.                                                                                                                                                   |url              |
+|KPOPS_TIMEOUT            |300                  |False   |The timeout in seconds that specifies when actions like deletion or deploy timeout.                                                                                         |timeout          |
+|KPOPS_RETAIN_CLEAN_JOBS  |False                |False   |Whether to retain clean up jobs in the cluster or uninstall the, after completion.                                                                                          |retain_clean_jobs|