Add Schema Registry to the image

Dockerfile

@@ -4,27 +4,47 @@ FROM centos
 
 ENV SCALA_VERSION 2.12
 ENV KAFKA_VERSION 2.3.0
+ENV SCHEMA_VERSION 5.3.2
 ENV KAFKA_HOME /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION"
+ENV SCHEMA_REGISTRY_HOME /opt/schema-registry-"$SCHEMA_VERSION"
 
 # Install Kafka, Zookeeper and other needed things
 RUN yum update -y && \
-    yum install -y epel-release wget nc net-tools openssl krb5-workstation krb5-libs java which && \
+    yum install -y epel-release wget nc net-tools openssl krb5-workstation krb5-libs java maven which && \
     yum install -y python3-pip && \
     pip3 install supervisor && \
     wget -q \
         http://apache.mirrors.spacedump.net/kafka/"$KAFKA_VERSION"/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz \
         -O /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \
     tar xfz /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -C /opt && \
     rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \
+    wget -q https://github.com/confluentinc/common/archive/v"$SCHEMA_VERSION".tar.gz \
+        -O /tmp/cp-common.tgz && \
+    tar xfz /tmp/cp-common.tgz -C /opt && \
+    wget -q https://github.com/confluentinc/rest-utils/archive/v"$SCHEMA_VERSION".tar.gz \
+        -O /tmp/restutils.tgz && \
+    tar xfz /tmp/restutils.tgz -C /opt && \
+    wget -q https://github.com/confluentinc/schema-registry/archive/v"$SCHEMA_VERSION".tar.gz \
+        -O /tmp/schema_registry.tgz && \
+    tar xfz /tmp/schema_registry.tgz -C /opt
+
+RUN cd /opt/common-"$SCHEMA_VERSION" && \
+    mvn install && \
+    cd /opt/rest-utils-"$SCHEMA_VERSION" && \
+    mvn install && \
+    cd /opt/schema-registry-"$SCHEMA_VERSION" && \
+    mvn install -DskipTests=true && \
     yum clean all
 
 ADD scripts/start-all.sh /usr/bin/start-all.sh
 ADD scripts/start-kafka.sh /usr/bin/start-kafka.sh
 ADD scripts/start-zookeeper.sh /usr/bin/start-zookeeper.sh
+ADD scripts/start-schemaregistry.sh /usr/bin/start-schemaregistry.sh
 ADD scripts/create-kafka-topics.sh /usr/bin/create-kafka-topics.sh
 ADD scripts/configureKerberosClient.sh /usr/bin/configureKerberosClient.sh
 
 ADD config/log4j.properties config/zookeeper.jaas.tmpl config/kafka.jaas.tmpl "$KAFKA_HOME"/config/
+ADD config/schema-registry.properties "$SCHEMA_REGISTRY_HOME"/config/
 
 RUN mkdir -p /tmp/zookeeper && \
     mkdir -p /tmp/kafka-logs && \
@@ -36,17 +56,19 @@ RUN mkdir -p /tmp/zookeeper && \
     chmod -R 777 /var/log/zookeeper/ && \
     chmod -R 777 /var/run/ && \
     chmod -R 777 "$KAFKA_HOME"/logs && \
-    chmod -R 777 "$KAFKA_HOME"/config && \
+    chmod -R 777 "$KAFKA_HOME"/logs && \
+    chmod -R 777 "$SCHEMA_REGISTRY_HOME"/config && \
+    chmod -R 777 "$SCHEMA_REGISTRY_HOME"/ && \
     chmod -R 777  /tmp/zookeeper && \
     chmod -R 777  /tmp/kafka-logs && \
     chmod -R 777 /var/private/ssl
 
 # Supervisor config
-ADD supervisor/initialize.ini supervisor/kafka.ini supervisor/zookeeper.ini /etc/supervisord.d/
+ADD supervisor/initialize.ini supervisor/kafka.ini supervisor/zookeeper.ini supervisor/schemaregistry.ini /etc/supervisord.d/
 
 RUN echo_supervisord_conf | sed -e 's:;\[include\]:\[include\]:g' | sed -e 's:;files = relative/directory/\*.ini:files = /etc/supervisord.d/\*.ini:g' > /etc/supervisord.conf
 
-# 2181 is zookeeper, 9092-9099 is kafka (for different listeners like SSL, INTERNAL, PLAINTEXT etc.)
-EXPOSE 2181 9092 9093 9094 9095 9096 9097 9098 9099
+# 2181 is zookeeper, 8081 is Schema Registry, 9092-9099 is kafka (for different listeners like SSL, INTERNAL, PLAINTEXT etc.)
+EXPOSE 2181 8081 9092 9093 9094 9095 9096 9097 9098 9099
 
 CMD ["supervisord", "-n"]

config/schema-registry.properties

@@ -0,0 +1,27 @@
+# The address the socket server listens on.
+#   FORMAT:
+#     listeners = listener_name://host_name:port
+#   EXAMPLE:
+#     listeners = PLAINTEXT://your.host.name:9092
+listeners=http://0.0.0.0:8081
+
+# Zookeeper connection string for the Zookeeper cluster used by your Kafka cluster
+# (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+kafkastore.connection.url=localhost:2181
+
+# Alternatively, Schema Registry can now operate without Zookeeper, handling all coordination via
+# Kafka brokers. Use this setting to specify the bootstrap servers for your Kafka cluster and it
+# will be used both for selecting the master schema registry instance and for storing the data for
+# registered schemas.
+# (Note that you cannot mix the two modes; use this mode only on new deployments or by shutting down
+# all instances, switching to the new configuration, and then starting the schema registry
+# instances again.)
+#kafkastore.bootstrap.servers=PLAINTEXT://localhost:9092
+
+# The name of the topic to store schemas in
+kafkastore.topic=_schemas
+
+# If true, API requests that fail will include extra debugging information, including stack traces
+debug=false

scripts/start-all.sh

@@ -10,7 +10,7 @@ start(){
 
   # Create a lock file to prevent multiple instantiations.
   touch $LOCKFILE
-
+  
   if [ ! -z "$ENABLE_KERBEROS" ]; then
     # Start Kerberos keytab creation
     echo "Starting keytab creation..."
@@ -57,6 +57,28 @@ start(){
     echo "Kafka has been started!"
   fi
 
+  until echo exit | nc --send-only $(hostname -s) 9092;
+  do 
+    echo "Waiting for Kafka Broker to be really available"
+    sleep 10
+  done
+
+  # Start Schema registry on master node.
+  echo "Starting schema registry..."
+  supervisorctl start schemaregistry
+  # Wait for Kafka to start.
+  while [ "$(supervisorctl status schemaregistry | tr -s ' ' | cut -f2 -d' ')" == "STARTING" ] 
+  do
+    sleep 10
+  done
+  schema_status=$(supervisorctl status schemaregistry | tr -s ' ' | cut -f2 -d' ')
+  if [ "$schema_status" != "RUNNING" ]
+  then
+    echo "SchemaRegistry has failed to start with code $schema_status."
+  else
+    echo "SchemaRegistry has been started!"
+  fi  
+
   # Start Topic creation
   echo "Starting topic creation..."
   # Wait for kafka broker to be available
@@ -66,11 +88,6 @@ start(){
     sleep 10
   done
 
-  until echo exit | nc --send-only $(hostname -s) 9092;
-  do 
-    echo "Waiting for Kafka Broker to be really available"
-    sleep 10
-  done
 
   /usr/bin/create-kafka-topics.sh >> $KAFKA_HOME/logs/create-kafka-topics.log
   returnedValue=$?
@@ -92,6 +109,17 @@ start(){
 }
 
 stop(){
+
+  # Stop Schema Registry.
+  supervisorctl stop schemaregistry
+  schema_status=$(supervisorctl status schemaregistry | tr -s ' ' | cut -f2 -d' ')
+  if [ "$schema_status" == "STOPPED" ]
+  then
+    echo "SchemaRegistry has been stopped!"
+  else
+    echo "SchemaRegistry has failed to stop with returned code $schema_status"
+  fi
+
   # Stop Kafka.
   supervisorctl stop kafka
   kafka_status=$(supervisorctl status kafka | tr -s ' ' | cut -f2 -d' ')

scripts/start-schemaregistry.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# Optional ENV variables:
+# * SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka broker address
+
+echo "Make sure new config items are put at end of config file even if no newline is present as final character in the config"
+echo >> $SCHEMA_REGISTRY_HOME/config/schema-registry.properties    
+
+
+if [ ! -z "$SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS" ]; then
+    echo "Schema Registry Bootstrap Server: $SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS"
+    sed -r -i "s/#?(kafkastore.connection.url)=(.*)/\1=$SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS/g" $SCHEMA_REGISTRY_HOME/config/schema-registry.properties    
+fi
+
+$SCHEMA_REGISTRY_HOME/bin/schema-registry-start $SCHEMA_REGISTRY_HOME/config/schema-registry.properties    
+

supervisor/schemaregistry.ini

@@ -0,0 +1,4 @@
+[program:schemaregistry]
+command=/usr/bin/start-schemaregistry.sh
+autostart=false
+autorestart=false