Image upgrades and fixes

.gitignore

@@ -1,2 +1,4 @@
 secrets/*
 !secrets/.gitkeep
+.idea
+.vscode

README.md

@@ -10,7 +10,7 @@ Also included:
 0. Ensure Docker and Docker Compose are installed
 1. Build the Docker containers.
    ```sh
-     docker-compose build.
+     docker compose build
    ```
 
 ### Starting up
@@ -60,4 +60,4 @@ See below to learn more about running `kafka-tools` for admin tasks.
   # Example, delete a Topic
   ./auto/kafka-tools.sh kafka-topics --bootstrap-server=broker.local:19092 --command-config /etc/kafka/config/command.properties --delete --topic UserEmail
 -
-```
+```

auto/clean-up.sh

@@ -1,11 +1,11 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euf -o pipefail
 
 cd "$(dirname "$0")/.." || exit
 
 ./auto/down.sh
 
-echo "💣  Deleting volumes for a clean slate."
+echo "💣  Deleting volumes for a clean state."
 
 docker volume rm zk-data > /dev/null
 docker volume rm zk-txn-logs > /dev/null

auto/create-certs.sh

@@ -1,43 +1,72 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euf -o pipefail
 
 cd "$(dirname "$0")/../secrets/" || exit
 
+function usage {
+    printf "Usage:\n"
+    printf "$0 [--prompt|-p]\n"
+    exit 1
+}
+
+function argparse {
+  while [ $# -gt 0 ]; do
+    case "$1" in
+      --prompt|-p)
+        # optional: activate prompt for certificate trust with keytool (default: no prompt)
+        export NO_PROMPT=""
+        shift
+        ;;
+      *)
+        printf "ERROR: Parameters invalid\n"
+        usage
+    esac
+  done
+}
+
+#
+# init
+export NO_PROMPT="-noprompt"
+argparse $*
+
 echo "🔖  Generating some fake certificates and other secrets."
-echo "⚠️  Remember to type in \"yes\" for all prompts."
+[[ -z "$NO_PROMPT" ]] && echo "⚠️  Remember to type in \"yes\" for all prompts."
 sleep 2
 
 TLD="local"
 PASSWORD="awesomekafka"
+COUNTRY_CODE="AU"
+
+CA_NAME="fake-ca-1"
 
 # Generate CA key
-openssl req -new -x509 -keyout fake-ca-1.key \
-	-out fake-ca-1.crt -days 9999 \
-	-subj "/CN=ca1.${TLD}/OU=CIA/O=REA/L=Melbourne/S=VIC/C=AU" \
+openssl req -new -x509 -keyout ${CA_NAME}.key \
+	-out ${CA_NAME}.crt -days 9999 \
+	-subj "/CN=ca1.${TLD}/OU=CIA/O=REA/L=Melbourne/ST=VIC/C=${COUNTRY_CODE}" \
 	-passin pass:$PASSWORD -passout pass:$PASSWORD
 
 for i in broker control-center metrics schema-registry kafka-tools rest-proxy; do
 	echo ${i}
 	# Create keystores
 	keytool -genkey -noprompt \
 		-alias ${i} \
-		-dname "CN=${i}.${TLD}, OU=CIA, O=REA, L=Melbourne, S=VIC, C=AU" \
+		-dname "CN=${i}.${TLD}, OU=CIA, O=REA, L=Melbourne, ST=VIC, C=${COUNTRY_CODE}" \
 		-keystore kafka.${i}.keystore.jks \
 		-keyalg RSA \
 		-storepass $PASSWORD \
 		-keypass $PASSWORD
 
 	# Create CSR, sign the key and import back into keystore
-	keytool -keystore kafka.$i.keystore.jks -alias $i -certreq -file $i.csr -storepass $PASSWORD -keypass $PASSWORD
+	keytool ${NO_PROMPT} -keystore kafka.$i.keystore.jks -alias $i -certreq -file $i.csr -storepass $PASSWORD -keypass $PASSWORD
 
-	openssl x509 -req -CA fake-ca-1.crt -CAkey fake-ca-1.key -in $i.csr -out $i-ca1-signed.crt -days 9999 -CAcreateserial -passin pass:$PASSWORD
+	openssl x509 -req -CA ${CA_NAME}.crt -CAkey ${CA_NAME}.key -in $i.csr -out $i-ca1-signed.crt -days 9999 -CAcreateserial -passin pass:$PASSWORD
 
-	keytool -keystore kafka.$i.keystore.jks -alias CARoot -import -file fake-ca-1.crt -storepass $PASSWORD -keypass $PASSWORD
+	keytool ${NO_PROMPT} -keystore kafka.$i.keystore.jks -alias CARoot -import -file ${CA_NAME}.crt -storepass $PASSWORD -keypass $PASSWORD
 
-	keytool -keystore kafka.$i.keystore.jks -alias $i -import -file $i-ca1-signed.crt -storepass $PASSWORD -keypass $PASSWORD
+	keytool ${NO_PROMPT} -keystore kafka.$i.keystore.jks -alias $i -import -file $i-ca1-signed.crt -storepass $PASSWORD -keypass $PASSWORD
 
 	# Create truststore and import the CA cert.
-	keytool -keystore kafka.$i.truststore.jks -alias CARoot -import -file fake-ca-1.crt -storepass $PASSWORD -keypass $PASSWORD
+	keytool ${NO_PROMPT} -keystore kafka.$i.truststore.jks -alias CARoot -import -file ${CA_NAME}.crt -storepass $PASSWORD -keypass $PASSWORD
 
 	echo $PASSWORD >${i}_sslkey_creds
 	echo $PASSWORD >${i}_keystore_creds

auto/down.sh

@@ -1,12 +1,12 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euf -o pipefail
 
 cd "$(dirname "$0"..)" || exit
 
 echo "🧹  Stopping containers and cleaning up."
 echo ""
 
-docker-compose down
+docker compose down
 
 echo ""
 echo "✨  All done."

auto/kafka-tools.sh

@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euf -o pipefail
 
 cd "$(dirname "$0"..)" || exit
 
-docker-compose run --rm --name=kafka-tools kafka-tools "$@"
+docker compose run --rm --name=kafka-tools kafka-tools "$@"

auto/logs.sh

@@ -1,9 +1,9 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euf -o pipefail
 
 cd "$(dirname "$0"..)" || exit
 
 echo "🌲  Here are some logs"
 sleep 2
 
-docker-compose logs --follow
+docker compose logs --follow

auto/up.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euf -o pipefail
 
 cd "$(dirname "$0"..)" || exit
@@ -10,7 +10,7 @@ docker volume create --name zk-txn-logs > /dev/null
 docker volume create --name kafka-data > /dev/null
 
 # Don't need kafka-tools to start up
-docker-compose up --detach --scale kafka-tools=0
+docker compose up --detach --scale kafka-tools=0
 
 echo ""
 echo "🐳  Kicked off the containers. Should be up in one minute (literally)."

config/command.properties

@@ -3,6 +3,6 @@ retry.backoff.ms=500
 security.protocol=SSL
 ssl.truststore.location=/etc/kafka/secrets/kafka.control-center.truststore.jks
 ssl.keystore.location=/etc/kafka/secrets/kafka.control-center.keystore.jks
-ssl.truststore.password=kafka
-ssl.keystore.password=kafka
-ssl.key.password=kafka
+ssl.truststore.password=awesomekafka
+ssl.keystore.password=awesomekafka
+ssl.key.password=awesomekafka

docker-compose.yaml

@@ -2,7 +2,7 @@
 version: "3"
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:5.3.1
+    image: confluentinc/cp-zookeeper:7.5.1
     container_name: zookeeper
     networks:
        kafka:
@@ -19,7 +19,7 @@ services:
       - zk-txn-logs:/var/lib/zookeeper/log
 
   broker:
-    image: confluentinc/cp-enterprise-kafka:5.3.1
+    image: confluentinc/cp-enterprise-kafka:7.5.1
     container_name: broker
     networks:
        kafka:
@@ -62,86 +62,6 @@ services:
       - kafka-data:/var/lib/kafka/data
       - ./secrets:/etc/kafka/secrets
 
-  schema-registry:
-    image: confluentinc/cp-schema-registry:5.3.1
-    depends_on:
-      - zookeeper
-      - broker
-    container_name: schema-registry
-    networks:
-       kafka:
-          aliases:
-            - schema-registry.local
-    ports:
-      - 8081:8081
-    environment:
-      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker.local:19092
-      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SSL
-      SCHEMA_REGISTRY_HOST_NAME: schema-registry.local
-      SCHEMA_REGISTRY_LISTENERS: "https://schema-registry.local:8081"
-      SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "https"
-      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
-      SCHEMA_REGISTRY_LOG4J_LOGLEVEL: ERROR
-      SCHEMA_REGISTRY_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.schema-registry.truststore.jks
-      SCHEMA_REGISTRY_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.schema-registry.keystore.jks
-      SCHEMA_REGISTRY_SSL_TRUSTSTORE_PASSWORD: awesomekafka
-      SCHEMA_REGISTRY_SSL_KEYSTORE_PASSWORD: awesomekafka
-      SCHEMA_REGISTRY_SSL_KEY_PASSWORD: awesomekafka
-      SCHEMA_REGISTRY_KAFKASTORE_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS"
-      SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.schema-registry.truststore.jks
-      SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.schema-registry.keystore.jks
-      SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_PASSWORD: awesomekafka
-      SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_PASSWORD: awesomekafka
-      SCHEMA_REGISTRY_KAFKASTORE_SSL_KEY_PASSWORD: awesomekafka
-      SCHEMA_REGISTRY_SSL_CLIENT_AUTH: "true"
-    volumes:
-      - ./secrets:/etc/kafka/secrets
-
-  control-center:
-    image: confluentinc/cp-enterprise-control-center:5.3.1
-    container_name: control-center
-    networks:
-       kafka:
-          aliases:
-            - control-center.local
-    depends_on:
-      - zookeeper
-      - broker
-      - schema-registry
-    ports:
-      - "9021:9021"
-    environment:
-      CONTROL_CENTER_LOG4J_ROOT_LOGLEVEL: INFO
-      CONTROL_CENTER_LOG4J_LOGLEVEL: INFO
-      CONTROL_CENTER_BOOTSTRAP_SERVERS: broker.local:19092
-      CONTROL_CENTER_ZOOKEEPER_CONNECT: zookeeper.local:22181
-      CONTROL_CENTER_SCHEMA_REGISTRY_URL: "https://schema-registry.local:8081"
-      CONTROL_CENTER_STREAMS_SECURITY_PROTOCOL: SSL
-      CONTROL_CENTER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
-      CONTROL_CENTER_STREAMS_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.control-center.truststore.jks
-      CONTROL_CENTER_STREAMS_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.control-center.keystore.jks
-      CONTROL_CENTER_STREAMS_SSL_TRUSTSTORE_PASSWORD: awesomekafka
-      CONTROL_CENTER_STREAMS_SSL_KEYSTORE_PASSWORD: awesomekafka
-      CONTROL_CENTER_STREAMS_SSL_KEY_PASSWORD: awesomekafka
-      CONTROL_CENTER_STREAMS_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS"
-      CONTROL_CENTER_REPLICATION_FACTOR: 1
-      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
-      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
-      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
-      CONTROL_CENTER_REST_LISTENERS: "http://0.0.0.0:9021"
-      CONTROL_CENTER_REST_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.control-center.truststore.jks
-      CONTROL_CENTER_REST_SSL_TRUSTSTORE_PASSWORD: awesomekafka
-      CONTROL_CENTER_REST_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.control-center.keystore.jks
-      CONTROL_CENTER_REST_SSL_KEYSTORE_PASSWORD: awesomekafka
-      CONTROL_CENTER_REST_SSL_KEY_PASSWORD: awesomekafka
-      CONTROL_CENTER_OPTS: -Djavax.net.ssl.trustStore=/etc/kafka/secrets/kafka.control-center.truststore.jks
-                  -Djavax.net.ssl.trustStorePassword=kafka
-                  -Djavax.net.ssl.keyStore=/etc/kafka/secrets/kafka.control-center.keystore.jks
-                  -Djavax.net.ssl.keyStorePassword=kafka
-      PORT: 9021
-    volumes:
-      - ./secrets:/etc/kafka/secrets
-
   kafka-tools:
     build:
       context: ./kafka-tools
@@ -162,45 +82,17 @@ services:
       KAFKA_SSL_TRUSTSTORE_PASSWORD: awesomekafka
       KAFKA_SSL_KEYSTORE_PASSWORD: awesomekafka
       KAFKA_SSL_KEY_PASSWORD: awesomekafka
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper.local:22181
+      KAFKA_ADVERTISED_LISTENERS: SSL://broker.local:19092
+      KAFKA_SSL_KEYSTORE_FILENAME: kafka.kafka-tools.keystore.jks
+      KAFKA_SSL_KEYSTORE_CREDENTIALS: kafka-tools_keystore_creds
+      KAFKA_SSL_KEY_CREDENTIALS: kafka-tools_sslkey_creds
+      KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.kafka-tools.truststore.jks
+      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: kafka-tools_truststore_creds
     volumes:
       - ./secrets:/etc/kafka/secrets
       - ./config:/etc/kafka/config
-
-  rest-proxy:
-    image: confluentinc/cp-kafka-rest:5.3.1
-    depends_on:
-      - zookeeper
-      - broker
-    ports:
-      - 8082:8082
-    hostname: rest-proxy
-    container_name: rest-proxy
-    networks:
-       kafka:
-          aliases:
-            - rest-proxy.local
-    environment:
-      KAFKA_REST_LOG4J_ROOT_LOGLEVEL: ERROR
-      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " "
-      KAFKA_REST_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.rest-proxy.truststore.jks
-      KAFKA_REST_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.rest-proxy.keystore.jks
-      KAFKA_REST_SSL_TRUSTSTORE_PASSWORD: awesomekafka
-      KAFKA_REST_SSL_KEYSTORE_PASSWORD: awesomekafka
-      KAFKA_REST_SSL_KEY_PASSWORD: awesomekafka
-      KAFKA_REST_LOG4J_LOGLEVEL: ERROR
-      KAFKA_REST_HOST_NAME: rest-proxy
-      KAFKA_REST_BOOTSTRAP_SERVERS: broker.local:19092
-      KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper.local:22181
-      KAFKA_REST_CLIENT_SECURITY_PROTOCOL: SSL
-      KAFKA_REST_CLIENT_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.rest-proxy.truststore.jks
-      KAFKA_REST_CLIENT_SSL_TRUSTSTORE_PASSWORD: awesomekafka
-      KAFKA_REST_CLIENT_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.rest-proxy.keystore.jks
-      KAFKA_REST_CLIENT_SSL_KEYSTORE_PASSWORD: awesomekafka
-      KAFKA_REST_CLIENT_SSL_KEY_PASSWORD: awesomekafka
-      KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
-      KAFKA_HEAP_OPTS: "-Xmx1G -Xms1G"
-    volumes:
-      - ./secrets:/etc/kafka/secrets
+      - kafka-data:/var/lib/kafka/data
 
 volumes:
   zk-data:

kafka-tools/Dockerfile

@@ -1,10 +1,4 @@
-FROM  confluentinc/cp-enterprise-kafka:5.3.1
+FROM  confluentinc/cp-enterprise-kafka:7.5.1
 
-# Set up a user to run Kafka
-RUN groupadd kafka && \
-  useradd -d /kafka -g kafka -s /bin/false kafka && \
-  chown -R kafka:kafka /usr/bin/
-
-USER kafka
 ENV PATH /usr/bin:$PATH
 WORKDIR /usr/bin