Mock Schema Registry (#2037)

* minor style fixes * correct host name * minor change * minor correction * minor update * minor change * minor change * added additional mock directories for dependabot scan * Automated commit: Unpin versions of updated images * update * added back ssl changes for kafka producer * lint * minor change * minor update --------- Co-authored-by: VRO Machine User <[email protected]> Co-authored-by: Teja <[email protected]>
department-of-veterans-affairs · Oct 30, 2023 · 5a7232b · 5a7232b
1 parent f6c797a
commit 5a7232b
Show file tree

Hide file tree

Showing 13 changed files with 994 additions and 2 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -56,3 +56,17 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 10
+
+  - package-ecosystem: "docker"
+    directory: "/mock/mock-bie-kafka"
+    target-branch: "develop"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 10
+
+  - package-ecosystem: "docker"
+    directory: "/mock/mock-bie-schema-registry"
+    target-branch: "develop"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 10
diff --git a/.github/workflows/bie-kafka-end2end-test.yml b/.github/workflows/bie-kafka-end2end-test.yml
@@ -45,7 +45,7 @@ jobs:
 
           ./gradlew :dockerComposeUp
 
-          ./gradlew -p mocks :mock-bie-kafka:docker
+          ./gradlew -p mocks docker
           ./gradlew -p mocks :dockerComposeUp
 
           ./gradlew :domain-xample:dockerComposeUp

diff --git a/app/docker-compose.yml b/app/docker-compose.yml
@@ -121,6 +121,7 @@ services:
     environment:
       <<: [*rabbitmq-placeholder-vars, *common-vars]
       BIE_KAFKA_PLACEHOLDERS_BROKERS: mock-bie-kafka:9092
+      BIE_KAFKA_PLACEHOLDERS_SCHEMA_REGISTRY: mock-bie-schema-registry:8081
       BIE_KAFKA_KEYSTORE_INBASE64: ${BIE_KAFKA_KEYSTORE_INBASE64}
       BIE_KAFKA_KEYSTORE_PASSWORD: ${BIE_KAFKA_KEYSTORE_PASSWORD}
       BIE_KAFKA_TRUSTSTORE_INBASE64: ${BIE_KAFKA_TRUSTSTORE_INBASE64}

diff --git a/mocks/docker-compose.yml b/mocks/docker-compose.yml
@@ -96,7 +96,7 @@ services:
       # Set up SSL -- see "Security" section of https://hub.docker.com/r/bitnami/kafka/
       KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:SSL
       KAFKA_CFG_SSL_CLIENT_AUTH: required
-      # PEM https://medium.com/analytics-vidhya/how-to-use-pem-certificates-with-apache-kafka-f3b444a00816
+    # PEM https://medium.com/analytics-vidhya/how-to-use-pem-certificates-with-apache-kafka-f3b444a00816
       KAFKA_TLS_TYPE: JKS
       KAFKA_CERTIFICATE_PASSWORD: mockKafka
 

diff --git a/mocks/mock-bie-schema-registry/Dockerfile b/mocks/mock-bie-schema-registry/Dockerfile
@@ -0,0 +1,13 @@
+# Use the official Confluent Schema Registry image as the base
+FROM confluentinc/cp-schema-registry:7.5.1
+
+USER root
+
+# Install jq using yum (as root)
+RUN yum install -y jq-1.6 && yum clean all
+
+# Create a new user 'tron' and create a home directory for the user
+RUN useradd -m tron
+
+# Switch to the new user 'tron'
+USER tron
diff --git a/mocks/mock-bie-schema-registry/build.gradle b/mocks/mock-bie-schema-registry/build.gradle
@@ -0,0 +1,6 @@
+plugins {
+  id 'shared.docker.container-conventions'
+}
+
+docker {
+}
diff --git a/mocks/mock-bie-schema-registry/docker-entry.sh b/mocks/mock-bie-schema-registry/docker-entry.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Start Schema Registry in the background
+/etc/confluent/docker/run &
+
+# Define the Schema Registry URL
+SCHEMA_REGISTRY_URL="http://localhost:8081"
+
+# Wait for Schema Registry to be available
+until curl --silent --output /dev/null --fail "$SCHEMA_REGISTRY_URL"; do
+  echo "Waiting for Schema Registry to be available..."
+  sleep 5
+done
+
+
+# Navigate to /avro/events folder
+cd /avro/events
+
+# Loop through all .avsc files in the folder
+for file in *.avsc; do
+  # Extract the filename without extension to be used as subject name
+  subject="${file%.avsc}"
+
+  # Read the file content as schema data
+  schema=$(cat "$file")
+
+  # Register the schema with the Schema Registry
+  echo "Registering schema for subject: $subject"
+  curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \
+    --data "$(jq -n --arg schema "$schema" '{"schema": $schema}')" \
+    "$SCHEMA_REGISTRY_URL/subjects/$subject/versions"
+done
+
+
+# Wait for Schema Registry process to complete
+wait $!
diff --git a/...registry/resources/avro/events/TST_CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02.avsc b/...registry/resources/avro/events/TST_CONTENTION_BIE_CONTENTION_ASSOCIATED_TO_CLAIM_V02.avsc
@@ -0,0 +1,188 @@
+{
+  "name":"ContentionAssociatedToClaim",
+  "type":"record",
+  "namespace":"gov.va.bip.bie.claim.contention",
+  "fields":[
+    {
+      "name": "ActionName",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ActionResultName",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ActorApplicationId",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ActorStation",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "ActorUserId",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "AutomationIndicator",
+      "type": "boolean"
+    },
+    {
+      "name":"BenefitClaimTypeCode",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name":"ClaimId",
+      "type":"long"
+    },
+    {
+      "name":"ContentionClassificationName",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name":"ContentionId",
+      "type":"long"
+    },
+    {
+      "name":"ContentionStatusTypeCode",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name":"ContentionTypeCode",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name":"CurrentLifecycleStatus",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name":"DateAdded",
+      "type": [
+        "null",
+        {
+          "type": "long",
+          "connect.version": 1,
+          "connect.name": "org.apache.kafka.connect.data.Timestamp",
+          "logicalType": "timestamp-millis"
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "Details",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name":"DiagnosticTypeCode",
+      "type": [
+        "null",
+        "string"
+      ],
+      "default": null
+    },
+    {
+      "name": "EventTime",
+      "type":
+      {
+        "type": "long",
+        "connect.version": 1,
+        "connect.name": "org.apache.kafka.connect.data.Timestamp",
+        "logicalType": "timestamp-millis"
+      }
+    },
+    {
+      "name":"VeteranParticipantId",
+      "type": [
+        "null",
+        "long"
+      ],
+      "default": null
+    },
+    {
+      "name": "BieTs",
+      "doc": "The timestamp that BIE processed the event.",
+      "type": [
+        "null",
+        {
+          "type": "long",
+          "connect.version": 1,
+          "connect.name": "org.apache.kafka.connect.data.Timestamp",
+          "logicalType": "timestamp-millis"
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "SourceTs",
+      "doc": "The timestamp of the source event. This is the time of the database operation.",
+      "type": [
+        "null",
+        {
+          "type": "long",
+          "connect.version": 1,
+          "connect.name": "org.apache.kafka.connect.data.Timestamp",
+          "logicalType": "timestamp-millis"
+        }
+      ],
+      "default": null
+    },
+    {
+      "name": "ConnectorTs",
+      "doc": "The timestamp that the connector processed the event.",
+      "type": [
+        "null",
+        {
+          "type": "long",
+          "connect.version": 1,
+          "connect.name": "org.apache.kafka.connect.data.Timestamp",
+          "logicalType": "timestamp-millis"
+        }
+      ],
+      "default": null
+    }
+  ]
+}