diff --git a/docs/introduction.rst b/docs/introduction.rst new file mode 100644 index 0000000..a17ed57 --- /dev/null +++ b/docs/introduction.rst @@ -0,0 +1,54 @@ +================== +Spooldir Connector +================== + +This Kafka Connect connector provides the capability to watch a directory for files and read the data as new files are +written to the input directory. Each of the records in the input file will be converted based on the user supplied schema. + +The CSVRecordProcessor supports reading CSV or TSV files. It can convert a CSV on the fly to the strongly typed Kafka +Connect data types. It currently has support for all of the schema types and logical types that are supported in Kafka 0.10.x. +If you couple this with the Avro converter and Schema Registry by Confluent, you will be able to process csv files to +strongly typed Avro data in real time. + + + +.. toctree:: + :maxdepth: 1 + :caption: Source Connectors: + :hidden: + :glob: + + sources/* + + +.. toctree:: + :maxdepth: 1 + :caption: Sink Connectors: + :hidden: + :glob: + + sinks/* + + +.. toctree:: + :maxdepth: 1 + :caption: Transformations: + :hidden: + :glob: + + transformations/* + + +.. toctree:: + :maxdepth: 0 + :caption: Schemas: + :hidden: + + schemas + + + + + + + diff --git a/pom.xml b/pom.xml index 53ff56b..2bba650 100644 --- a/pom.xml +++ b/pom.xml @@ -19,14 +19,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - 3.3.0 - - com.github.jcustenborder.kafka.connect kafka-connect-parent - 0.10.2.0-cp1 + 0.10.2.1-cp2 kafka-connect-spooldir @@ -79,4 +75,4 @@ 3.8 - \ No newline at end of file + diff --git a/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnector.java b/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnector.java index 75e2738..bcc0969 100644 --- a/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnector.java +++ b/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirCsvSourceConnector.java @@ -21,8 +21,8 @@ import java.util.Map; -@Description("The `SpoolDirCsvSourceConnector` will monitor the directory specified in `input.path` for files and read them as a CSV " + - "converting each of the records to the strongly typed equavalent specified in `key.schema` and `value.schema`.") +@Description("The SpoolDirCsvSourceConnector will monitor the directory specified in `input.path` for files and read them as a CSV " + + "converting each of the records to the strongly typed equivalent specified in `key.schema` and `value.schema`.") public class SpoolDirCsvSourceConnector extends SpoolDirSourceConnector { @Override protected SpoolDirCsvSourceConnectorConfig config(Map settings) { diff --git a/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceConnector.java b/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceConnector.java index 8f03753..01acc9b 100644 --- a/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceConnector.java +++ b/src/main/java/com/github/jcustenborder/kafka/connect/spooldir/SpoolDirJsonSourceConnector.java @@ -21,7 +21,7 @@ import java.util.Map; -@Description("This connector is used to [stream](https://en.wikipedia.org/wiki/JSON_Streaming) JSON files from a directory " + +@Description("This connector is used to `stream ` JSON files from a directory " + "while converting the data based on the schema supplied in the configuration.") public class SpoolDirJsonSourceConnector extends SpoolDirSourceConnector {