Added e2e tests to ensure kafka, postgres, mysql samples run with the latest dozer

connectors/kafka/README.md

@@ -54,7 +54,7 @@ The producer will start generating sample data and publish it to Redpanda.
 2. Run the following command to start Dozer:
 
 ```bash
-dozer -c dozer-config.yaml
+dozer run -c dozer-config.yaml
 ```
 
 Dozer will ingest the data from Redpanda and perform the specified operations based on the configuration.

connectors/kafka/docker-compose.yml

@@ -1,5 +1,4 @@
 version: "3.7"
-name: redpanda-quickstart
 networks:
   redpanda_network:
     driver: bridge

connectors/kafka/producer.py

@@ -13,7 +13,7 @@
 producer = KafkaProducer(bootstrap_servers=kafka_bootstrap_servers)
 
 # Generate mock transaction data and send it to the Kafka topic
-for index in range(10000000):
+for index in range(10):
     # Generate mock transaction data using the Faker library
     transaction_data = {
         'id': index,

connectors/postgres/README.md

@@ -16,13 +16,13 @@ docker-compose up
 
 Running Dozer
 ```
-dozer
+dozer run
 ```
 
 That's all to it. You have APIs instantly available over REST and gRPC.
 
 ```
- dozer
+ dozer run
 
 ____   ___ __________ ____
 |  _ \ / _ \__  / ____|  _ \

connectors/postgres/dozer-config.yaml

@@ -7,7 +7,7 @@ connections:
       password: postgres
       host: localhost
       port: 5433
-      database: film
+      database: pagila
 
 sources:
   - name: actors

package.json

@@ -20,7 +20,7 @@
   },
   "devDependencies": {
     "@types/google-protobuf": "^3.15.7",
-    "@types/mocha": "^10.0.1",
+    "@types/mocha": "^10.0.2",
     "@types/node": "^20.6.5",
     "@typescript-eslint/eslint-plugin": "^6.7.2",
     "@typescript-eslint/parser": "^6.7.2",
@@ -50,4 +50,4 @@
     ],
     "recursive": "test/**/*.js"
   }
-}
+}

test/connectors/initKafka.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+BASEDIR=$(dirname "$0")
+cd ${BASEDIR}/../../connectors/kafka
+
+#start RedPanda
+docker-compose up -d  
+
+#Register schema
+curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" --data '{"schema": "{\"type\":\"record\",\"name\":\"transaction\",\"namespace\":\"dozer.samples\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"}, {\"name\":\"customer_id\",\"type\":\"int\"},{\"name\":\"amount\",\"type\":\"float\"},{\"name\":\"location\",\"type\":\"string\"},{\"name\":\"provider\",\"type\":\"string\"}]}"}' http://localhost:18081/subjects/transactions-value/versions
+
+
+curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" --data '{"schema": "{\"type\":\"record\",\"name\":\"transactions\",\"namespace\":\"dozer.samples\",\"fields\":[{\"name\":\"id\",\"type\":\"int\"}]}"}' http://localhost:18081/subjects/transactions-key/versions
+
+#Run producer script
+pip install kafka-python
+pip install Faker
+python producer.py
+
+
+

test/connectors/initMySQL.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+BASEDIR=$(dirname "$0")
+cd ${BASEDIR}/../../
+
+docker-compose -f ./connectors/mysql/docker-compose.yml up -d

test/connectors/initPostgres.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+BASEDIR=$(dirname "$0")
+cd ${BASEDIR}/../../connectors/postgres
+
+sh download.sh
+
+docker-compose up -d

test/connectors/mysql.ts

@@ -0,0 +1,30 @@
+import { execSync } from 'child_process';
+import path from 'path';
+import {
+  assertEndpointsWithRetry, initDozer,
+} from '../helper';
+
+const TEST_PATH = './connectors/mysql';
+
+describe('Connectors: MySQL', async () => {
+  beforeEach(async () => {
+    process.chdir('../../'); // go to root
+    console.log(`Starting directory: ${process.cwd()}`);
+
+    // Download init.sql and setup docker image
+    execSync(`${__dirname}/initMySQL.sh`, { stdio: 'inherit' });
+
+    // navigate to test path
+    const baseDir = path.join(__dirname, '../../');
+    const fullPath = path.join(baseDir, TEST_PATH);
+    process.chdir(fullPath);
+    execSync('rm -rf .dozer && rm -f dozer.lock', { stdio: 'inherit' });
+  });
+
+  it('should run and return API endpoints', async () => {
+    const dozer = await initDozer();
+    await assertEndpointsWithRetry();
+    dozer.kill(9);
+    console.log('Killed dozer mysql');
+  });
+});

test/connectors/postgres.ts

@@ -0,0 +1,26 @@
+import { execSync } from 'child_process';
+import {
+  assertEndpointsWithRetry, initDozer,
+} from '../helper';
+
+const TEST_PATH = './connectors/postgres';
+
+describe('Connectors: Postgres', async () => {
+  beforeEach(async () => {
+    process.chdir('../../'); // go to root
+    console.log(`Starting directory: ${process.cwd()}`);
+
+    // Download init.sql and setup docker image
+    execSync(`${__dirname}/initPostgres.sh`, { stdio: 'inherit' });
+
+    process.chdir(TEST_PATH);
+    execSync('rm -rf .dozer && rm -f dozer.lock', { stdio: 'inherit' });
+  });
+
+  it('should run and return API endpoints', async () => {
+    const dozer = await initDozer();
+    await assertEndpointsWithRetry();
+    dozer.kill(9);
+    console.log('Killed dozer postgres');
+  });
+});

test/connectors/runkafka.ts

@@ -0,0 +1,27 @@
+import { execSync } from 'child_process';
+import {
+  assertEndpointsWithRetry, initDozer,
+} from '../helper';
+
+const TEST_PATH = './connectors/kafka';
+
+describe('Connectors: Kafka', async () => {
+  beforeEach(async () => {
+    process.chdir('../../'); // go to root
+    console.log(`Starting directory: ${process.cwd()}`);
+
+    // Download init.sql and setup docker image
+    execSync(`${__dirname}/initKafka.sh`, { stdio: 'inherit' });
+
+    process.chdir(TEST_PATH);
+    execSync('rm -rf .dozer && rm -f dozer.lock', { stdio: 'inherit' });
+  });
+
+  it('should run and return API endpoints', async () => {
+    const dozer = await initDozer();
+    console.log('Dozer started');
+    await assertEndpointsWithRetry();
+    dozer.kill(9);
+    console.log('Killed dozer Kafka');
+  });
+});