From 159e62f1f1262856ae6b1bc7f01a066676e9efd6 Mon Sep 17 00:00:00 2001 From: Thiago Pinto Date: Wed, 27 Jun 2018 14:46:49 -0300 Subject: [PATCH] Update readme --- README.md | 40 +++++++++++++++++++++++++++++++++++---- clients/kafka_consumer.py | 2 +- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 411ccad..8632798 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,41 @@ -[![Build Status](https://travis-ci.org/quintoandar/python-kafka-idempotence-client.svg?branch=master)](https://travis-ci.org/quintoandar/python-kafka-idempotence-client) +[![Build Status](https://travis-ci.org/quintoandar/python-kafka.svg?branch=master)](https://travis-ci.org/quintoandar/python-kafka) -# Idempotence Client for Kafka +# Python Kafka -Checks messages to avoid reprocessing events. +QuintoAndar's kafka-python lib wrapper with additional fuctionalities. -## Usage +## KafkaConsumer + +A simple wrapper for kafka-python lib. + +### Configuration + +| Name | Description | +| ----------------- | -------------------------------------------- | +| group_id | The consumer group id | +| bootstrap_servers | The bootstrap servers | +| topic | The topic to consume from | +| processor | The function that processes the event | +| idempotenceClient | (optional) A client that checks for repeated events. Must implement the same interface as [IdempotenceClient](/clients/idempotence_client.py) | +| deserializer | (optional) The deserializer | + +## IdempotenceClient for Kafka + +A redis implementation to check messages and avoid reprocessing events. + +### Usage `markConsumedMessage(topic, message)` creates a hash from the message and concatenates it with the topic and the group id. The message is reigstered as consumed with a default ttl of 2 weeks. + +`isUnique(message)` verifies if the message has already been processed. + +### Configuration + +| Name | Description | +| ------------- | ------------------------------- | +| host | The redis host | +| port | The redis port | +| groupId | The consumer group id | +| db | (optional) The redis db option | +| expire | (optional) The redis ttl | +| key_extractor | (optional) A funtion that extracts a key that uniquely identifies the event | \ No newline at end of file diff --git a/clients/kafka_consumer.py b/clients/kafka_consumer.py index 810d264..a73d814 100644 --- a/clients/kafka_consumer.py +++ b/clients/kafka_consumer.py @@ -8,7 +8,7 @@ class KafkaConsumerClient: def __init__(self, group_id, bootstrap_servers, topic, - processor, idempotenceClient=None, deserializer=None): + processor, deserializer=None, idempotenceClient=None): self.log = logging.getLogger(__name__) self.topic = topic self.processor = processor