From 5106b3185e3d32da74530474d66184bfa9fec1c5 Mon Sep 17 00:00:00 2001 From: morenol Date: Fri, 12 May 2023 20:42:06 +0000 Subject: [PATCH] Split readme and update publish workflow (#40) Fixes #29 Fixes #37 --- .github/workflows/publish-dev-kafka-sink.yaml | 6 +- .../workflows/publish-dev-kafka-source.yaml | 5 +- .github/workflows/publish-kafka-sink.yaml | 5 +- .github/workflows/publish-kafka-source.yaml | 5 +- README.md | 112 +----------------- crates/kafka-sink/README.md | 78 ++++++++++++ crates/kafka-source/README.md | 39 ++++++ 7 files changed, 132 insertions(+), 118 deletions(-) create mode 100644 crates/kafka-sink/README.md create mode 100644 crates/kafka-source/README.md diff --git a/.github/workflows/publish-dev-kafka-sink.yaml b/.github/workflows/publish-dev-kafka-sink.yaml index 811e9b5..ab41b21 100644 --- a/.github/workflows/publish-dev-kafka-sink.yaml +++ b/.github/workflows/publish-dev-kafka-sink.yaml @@ -17,8 +17,10 @@ jobs: with: package-name: kafka-sink branch: ${{ github.event.inputs.branch }} - cloud-url: ${{ vars.DEV_CLOUD_URL }} - fail-fast: false + cloud-url: ${{ vars.DEV_CLOUD_URL }} + fail-fast: false + readme: "./crates/kafka-sink/README.md" + secrets: CLOUD_USER_EMAIL: ${{ secrets.DEV_HUB_USER_EMAIL }} CLOUD_USER_PASSWORD: ${{ secrets.DEV_HUB_USER_PASSWORD }} diff --git a/.github/workflows/publish-dev-kafka-source.yaml b/.github/workflows/publish-dev-kafka-source.yaml index 98ee02d..a54af93 100644 --- a/.github/workflows/publish-dev-kafka-source.yaml +++ b/.github/workflows/publish-dev-kafka-source.yaml @@ -17,8 +17,9 @@ jobs: with: package-name: kafka-source branch: ${{ github.event.inputs.branch }} - cloud-url: ${{ vars.DEV_CLOUD_URL }} - fail-fast: false + readme: "./crates/kafka-source/README.md" + cloud-url: ${{ vars.DEV_CLOUD_URL }} + fail-fast: false secrets: CLOUD_USER_EMAIL: ${{ secrets.DEV_HUB_USER_EMAIL }} CLOUD_USER_PASSWORD: ${{ secrets.DEV_HUB_USER_PASSWORD }} diff --git a/.github/workflows/publish-kafka-sink.yaml b/.github/workflows/publish-kafka-sink.yaml index e7f05cd..e3b3e22 100644 --- a/.github/workflows/publish-kafka-sink.yaml +++ b/.github/workflows/publish-kafka-sink.yaml @@ -4,7 +4,7 @@ permissions: read-all on: push: - tags: + tags: - v* workflow_dispatch: inputs: @@ -19,7 +19,8 @@ jobs: with: package-name: kafka-sink branch: ${{ github.event.inputs.branch }} - fail-fast: false + readme: "./crates/kafka-sink/README.md" + fail-fast: false secrets: CLOUD_USER_EMAIL: ${{ secrets.HUB_USER_EMAIL }} CLOUD_USER_PASSWORD: ${{ secrets.HUB_USER_PASSWORD }} diff --git a/.github/workflows/publish-kafka-source.yaml b/.github/workflows/publish-kafka-source.yaml index eb339cb..ee6fba3 100644 --- a/.github/workflows/publish-kafka-source.yaml +++ b/.github/workflows/publish-kafka-source.yaml @@ -4,7 +4,7 @@ permissions: read-all on: push: - tags: + tags: - v* workflow_dispatch: inputs: @@ -19,7 +19,8 @@ jobs: with: package-name: kafka-source branch: ${{ github.event.inputs.branch }} - fail-fast: false + readme: "./crates/kafka-source/README.md" + fail-fast: false secrets: CLOUD_USER_EMAIL: ${{ secrets.HUB_USER_EMAIL }} CLOUD_USER_PASSWORD: ${{ secrets.HUB_USER_PASSWORD }} diff --git a/README.md b/README.md index 0a9c4b3..17804c3 100644 --- a/README.md +++ b/README.md @@ -2,117 +2,9 @@ Official Infinyon Kafka connectors ## Sink Connector -This is a connector for taking data from a Fluvio topic and sending to a Kafka topic. - -See [docs](https://www.fluvio.io/connectors/outbound/kafka/) here. - -### Configuration - -| Opt | default | type | description | -| :--- | :--- | :--- | :---- | -| url | - | String | The url for the kafka connector | -| topic | - | String | The kafka topic | -| partition | 0 | Integer | The kafka partition | -| create-topic | false | Boolean | Create or not a topic before start | -| options | - | Mapping | The kafka client options | -| security | - | Mapping | Optional. The kafka security config | - -#### Security configuration -| Option | default | type | description | -| :--- | :--- | :--- | :---- | -| security_protocol | ssl | String | The kafka security protocol | -| ssl_key | - | Mapping | The SSL key file to use | -| ssl_cert | - | Mapping | The SSL cert file to use | -| ssl_ca | - | Mapping | The SSL ca file to use | - -Parameters `ssl_key`, `ssl_cert` and `ssl_ca` can be defined via `file` - path to the file, or `pem` - content as string value. - -Example without security: -```yaml -apiVersion: 0.1.0 -meta: - version: 0.1.1 - name: my-kafka-connector - type: kafka-sink - topic: kafka-topic - create-topic: true -kafka: - url: "localhost:9092" - topic: fluvio-topic - create-topic: true -``` - -Example with security enabled: -```yaml -apiVersion: 0.1.0 -meta: - version: 0.1.1 - name: my-kafka-connector - type: kafka-sink - topic: kafka-topic - create-topic: true - secrets: - - name: KAFKA_BROKER_URL - - name: SSL_CERT_PEM -kafka: - url: ${{ secrets.KAFKA_BROKER_URL }} - topic: fluvio-topic - create-topic: true - security: - ssl_key: - file: /path/to/file - ssl_cert: - pem: "${{ secrets.SSL_CERT_PEM }}" - ssl_ca: - file: /path/to/file - security_protocol: ssl -``` - -### Usage -To try out Kafka Sink connector locally, you can use Fluvio CDK tool: -```bash -fluvio install cdk - -cdk deploy -p kafka-sink start --config crates/kafka-sink/config-example.yaml -``` +Fluvio Kafka Sink Connector is a connector for taking data from a Fluvio topic and sending to a Kafka topic. See [reference](./crates/kafka-sink/) here. ## Source Connector -This is a connector for taking data from a Kafka topic and sending to a Fluvio topic. - -See [docs](https://www.fluvio.io/connectors/inbound/kafka/) here. - -### Configuration - -| Opt | default | type | description | -| :--- | :--- | :--- | :---- | -| url | - | String | The url for the kafka connector | -| topic | - | String | The kafka topic | -| partition | 0 | Integer | The kafka partition | -| group | fluvio-kafka-source | String | The kafka consumer group | - -Example: -```yaml -apiVersion: 0.1.0 -meta: - version: 0.2.0 - name: my-kafka-connector - type: kafka-source - topic: kafka-topic - create-topic: true -kafka: - url: "localhost:9092" - topic: fluvio-topic -``` - -### Usage -To try out Kafka Source connector locally, you can use Fluvio CDK tool: -```bash -fluvio install cdk - -cdk deploy -p kafka-source start --config crates/kafka-source/config-example.yaml -``` - -## Transformations -Fluvio Kafka Connectors support [Transformations](https://www.fluvio.io/docs/concepts/transformations-chain/). +Fluvio Kafka Source Connector is a connector for taking data from a Kafka topic and sending to a Fluvio topic. See [reference](./crates/kafka-source/) here. \ No newline at end of file diff --git a/crates/kafka-sink/README.md b/crates/kafka-sink/README.md new file mode 100644 index 0000000..416bbe4 --- /dev/null +++ b/crates/kafka-sink/README.md @@ -0,0 +1,78 @@ +# Fluvio Kafka Outbound Connector + +This is a connector for taking data from a Fluvio topic and sending to a Kafka topic. + +See [docs](https://www.fluvio.io/connectors/outbound/kafka/) here. + +### Configuration + +| Opt | default | type | description | +| :--- | :--- | :--- | :---- | +| url | - | String | The url for the kafka connector | +| topic | - | String | The kafka topic | +| partition | 0 | Integer | The kafka partition | +| create-topic | false | Boolean | Create or not a topic before start | +| options | - | Mapping | The kafka client options | +| security | - | Mapping | Optional. The kafka security config | + +#### Security configuration +| Option | default | type | description | +| :--- | :--- | :--- | :---- | +| security_protocol | ssl | String | The kafka security protocol | +| ssl_key | - | Mapping | The SSL key file to use | +| ssl_cert | - | Mapping | The SSL cert file to use | +| ssl_ca | - | Mapping | The SSL ca file to use | + +Parameters `ssl_key`, `ssl_cert` and `ssl_ca` can be defined via `file` - path to the file, or `pem` - content as string value. + +Example without security: +```yaml +apiVersion: 0.1.0 +meta: + version: 0.2.1 + name: my-kafka-connector + type: kafka-sink + topic: kafka-topic + create-topic: true +kafka: + url: "localhost:9092" + topic: fluvio-topic + create-topic: true +``` + +Example with security enabled: +```yaml +apiVersion: 0.1.0 +meta: + version: 0.2.1 + name: my-kafka-connector + type: kafka-sink + topic: kafka-topic + create-topic: true + secrets: + - name: KAFKA_BROKER_URL + - name: SSL_CERT_PEM +kafka: + url: ${{ secrets.KAFKA_BROKER_URL }} + topic: fluvio-topic + create-topic: true + security: + ssl_key: + file: /path/to/file + ssl_cert: + pem: "${{ secrets.SSL_CERT_PEM }}" + ssl_ca: + file: /path/to/file + security_protocol: ssl +``` + +### Usage +To try out Kafka Sink connector locally, you can use Fluvio CDK tool: +```bash +fluvio install cdk + +cdk deploy -p kafka-sink start --config crates/kafka-sink/config-example.yaml +``` + +## Transformations +Fluvio Kafka Connectors support [Transformations](https://www.fluvio.io/docs/concepts/transformations-chain/). diff --git a/crates/kafka-source/README.md b/crates/kafka-source/README.md new file mode 100644 index 0000000..cfe41a7 --- /dev/null +++ b/crates/kafka-source/README.md @@ -0,0 +1,39 @@ +# Fluvio Kafka Inbound Connector + +This is a connector for taking data from a Kafka topic and sending to a Fluvio topic. + +See [docs](https://www.fluvio.io/connectors/inbound/kafka/) here. + +### Configuration + +| Opt | default | type | description | +| :--- | :--- | :--- | :---- | +| url | - | String | The url for the kafka connector | +| topic | - | String | The kafka topic | +| partition | 0 | Integer | The kafka partition | +| group | fluvio-kafka-source | String | The kafka consumer group | + +Example: +```yaml +apiVersion: 0.1.0 +meta: + version: 0.2.1 + name: my-kafka-connector + type: kafka-source + topic: kafka-topic + create-topic: true +kafka: + url: "localhost:9092" + topic: fluvio-topic +``` + +### Usage +To try out Kafka Source connector locally, you can use Fluvio CDK tool: +```bash +fluvio install cdk + +cdk deploy -p kafka-source start --config crates/kafka-source/config-example.yaml +``` + +## Transformations +Fluvio Kafka Connectors support [Transformations](https://www.fluvio.io/docs/concepts/transformations-chain/).