diff --git a/.gitignore b/.gitignore index 10eda8d..e539312 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ secrets/* !secrets/.gitkeep +.idea +.vscode \ No newline at end of file diff --git a/README.md b/README.md index 1f5a511..15d4d2b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Also included: 0. Ensure Docker and Docker Compose are installed 1. Build the Docker containers. ```sh - docker-compose build. + docker compose build ``` ### Starting up @@ -60,4 +60,4 @@ See below to learn more about running `kafka-tools` for admin tasks. # Example, delete a Topic ./auto/kafka-tools.sh kafka-topics --bootstrap-server=broker.local:19092 --command-config /etc/kafka/config/command.properties --delete --topic UserEmail - -``` +``` \ No newline at end of file diff --git a/auto/clean-up.sh b/auto/clean-up.sh index 5a044ef..cd9390e 100755 --- a/auto/clean-up.sh +++ b/auto/clean-up.sh @@ -1,11 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash set -euf -o pipefail cd "$(dirname "$0")/.." || exit ./auto/down.sh -echo "๐Ÿ’ฃ Deleting volumes for a clean slate." +echo "๐Ÿ’ฃ Deleting volumes for a clean state." docker volume rm zk-data > /dev/null docker volume rm zk-txn-logs > /dev/null diff --git a/auto/create-certs.sh b/auto/create-certs.sh index aff0c2b..594d320 100755 --- a/auto/create-certs.sh +++ b/auto/create-certs.sh @@ -1,19 +1,48 @@ -#!/bin/bash +#!/usr/bin/env bash set -euf -o pipefail cd "$(dirname "$0")/../secrets/" || exit +function usage { + printf "Usage:\n" + printf "$0 [--prompt|-p]\n" + exit 1 +} + +function argparse { + while [ $# -gt 0 ]; do + case "$1" in + --prompt|-p) + # optional: activate prompt for certificate trust with keytool (default: no prompt) + export NO_PROMPT="" + shift + ;; + *) + printf "ERROR: Parameters invalid\n" + usage + esac + done +} + +# +# init +export NO_PROMPT="-noprompt" +argparse $* + echo "๐Ÿ”– Generating some fake certificates and other secrets." -echo "โš ๏ธ Remember to type in \"yes\" for all prompts." +[[ -z "$NO_PROMPT" ]] && echo "โš ๏ธ Remember to type in \"yes\" for all prompts." sleep 2 TLD="local" PASSWORD="awesomekafka" +COUNTRY_CODE="AU" + +CA_NAME="fake-ca-1" # Generate CA key -openssl req -new -x509 -keyout fake-ca-1.key \ - -out fake-ca-1.crt -days 9999 \ - -subj "/CN=ca1.${TLD}/OU=CIA/O=REA/L=Melbourne/S=VIC/C=AU" \ +openssl req -new -x509 -keyout ${CA_NAME}.key \ + -out ${CA_NAME}.crt -days 9999 \ + -subj "/CN=ca1.${TLD}/OU=CIA/O=REA/L=Melbourne/ST=VIC/C=${COUNTRY_CODE}" \ -passin pass:$PASSWORD -passout pass:$PASSWORD for i in broker control-center metrics schema-registry kafka-tools rest-proxy; do @@ -21,23 +50,23 @@ for i in broker control-center metrics schema-registry kafka-tools rest-proxy; d # Create keystores keytool -genkey -noprompt \ -alias ${i} \ - -dname "CN=${i}.${TLD}, OU=CIA, O=REA, L=Melbourne, S=VIC, C=AU" \ + -dname "CN=${i}.${TLD}, OU=CIA, O=REA, L=Melbourne, ST=VIC, C=${COUNTRY_CODE}" \ -keystore kafka.${i}.keystore.jks \ -keyalg RSA \ -storepass $PASSWORD \ -keypass $PASSWORD # Create CSR, sign the key and import back into keystore - keytool -keystore kafka.$i.keystore.jks -alias $i -certreq -file $i.csr -storepass $PASSWORD -keypass $PASSWORD + keytool ${NO_PROMPT} -keystore kafka.$i.keystore.jks -alias $i -certreq -file $i.csr -storepass $PASSWORD -keypass $PASSWORD - openssl x509 -req -CA fake-ca-1.crt -CAkey fake-ca-1.key -in $i.csr -out $i-ca1-signed.crt -days 9999 -CAcreateserial -passin pass:$PASSWORD + openssl x509 -req -CA ${CA_NAME}.crt -CAkey ${CA_NAME}.key -in $i.csr -out $i-ca1-signed.crt -days 9999 -CAcreateserial -passin pass:$PASSWORD - keytool -keystore kafka.$i.keystore.jks -alias CARoot -import -file fake-ca-1.crt -storepass $PASSWORD -keypass $PASSWORD + keytool ${NO_PROMPT} -keystore kafka.$i.keystore.jks -alias CARoot -import -file ${CA_NAME}.crt -storepass $PASSWORD -keypass $PASSWORD - keytool -keystore kafka.$i.keystore.jks -alias $i -import -file $i-ca1-signed.crt -storepass $PASSWORD -keypass $PASSWORD + keytool ${NO_PROMPT} -keystore kafka.$i.keystore.jks -alias $i -import -file $i-ca1-signed.crt -storepass $PASSWORD -keypass $PASSWORD # Create truststore and import the CA cert. - keytool -keystore kafka.$i.truststore.jks -alias CARoot -import -file fake-ca-1.crt -storepass $PASSWORD -keypass $PASSWORD + keytool ${NO_PROMPT} -keystore kafka.$i.truststore.jks -alias CARoot -import -file ${CA_NAME}.crt -storepass $PASSWORD -keypass $PASSWORD echo $PASSWORD >${i}_sslkey_creds echo $PASSWORD >${i}_keystore_creds diff --git a/auto/down.sh b/auto/down.sh index d5ffe2f..dfca26d 100755 --- a/auto/down.sh +++ b/auto/down.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euf -o pipefail cd "$(dirname "$0"..)" || exit @@ -6,7 +6,7 @@ cd "$(dirname "$0"..)" || exit echo "๐Ÿงน Stopping containers and cleaning up." echo "" -docker-compose down +docker compose down echo "" echo "โœจ All done." diff --git a/auto/kafka-tools.sh b/auto/kafka-tools.sh index da4b2d2..9036cf9 100755 --- a/auto/kafka-tools.sh +++ b/auto/kafka-tools.sh @@ -1,6 +1,6 @@ -#!/bin/bash +#!/usr/bin/env bash set -euf -o pipefail cd "$(dirname "$0"..)" || exit -docker-compose run --rm --name=kafka-tools kafka-tools "$@" +docker compose run --rm --name=kafka-tools kafka-tools "$@" diff --git a/auto/logs.sh b/auto/logs.sh index aa0ba33..7fb78fc 100755 --- a/auto/logs.sh +++ b/auto/logs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euf -o pipefail cd "$(dirname "$0"..)" || exit @@ -6,4 +6,4 @@ cd "$(dirname "$0"..)" || exit echo "๐ŸŒฒ Here are some logs" sleep 2 -docker-compose logs --follow +docker compose logs --follow diff --git a/auto/up.sh b/auto/up.sh index cbeec66..514c667 100755 --- a/auto/up.sh +++ b/auto/up.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euf -o pipefail cd "$(dirname "$0"..)" || exit @@ -10,7 +10,7 @@ docker volume create --name zk-txn-logs > /dev/null docker volume create --name kafka-data > /dev/null # Don't need kafka-tools to start up -docker-compose up --detach --scale kafka-tools=0 +docker compose up --detach --scale kafka-tools=0 echo "" echo "๐Ÿณ Kicked off the containers. Should be up in one minute (literally)." diff --git a/config/command.properties b/config/command.properties index 0f61392..caa2027 100644 --- a/config/command.properties +++ b/config/command.properties @@ -3,6 +3,6 @@ retry.backoff.ms=500 security.protocol=SSL ssl.truststore.location=/etc/kafka/secrets/kafka.control-center.truststore.jks ssl.keystore.location=/etc/kafka/secrets/kafka.control-center.keystore.jks -ssl.truststore.password=kafka -ssl.keystore.password=kafka -ssl.key.password=kafka +ssl.truststore.password=awesomekafka +ssl.keystore.password=awesomekafka +ssl.key.password=awesomekafka diff --git a/docker-compose.yaml b/docker-compose.yaml index 6f6fc12..7235734 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3" services: zookeeper: - image: confluentinc/cp-zookeeper:5.3.1 + image: confluentinc/cp-zookeeper:7.5.1 container_name: zookeeper networks: kafka: @@ -19,7 +19,7 @@ services: - zk-txn-logs:/var/lib/zookeeper/log broker: - image: confluentinc/cp-enterprise-kafka:5.3.1 + image: confluentinc/cp-enterprise-kafka:7.5.1 container_name: broker networks: kafka: @@ -62,86 +62,6 @@ services: - kafka-data:/var/lib/kafka/data - ./secrets:/etc/kafka/secrets - schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 - depends_on: - - zookeeper - - broker - container_name: schema-registry - networks: - kafka: - aliases: - - schema-registry.local - ports: - - 8081:8081 - environment: - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker.local:19092 - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SSL - SCHEMA_REGISTRY_HOST_NAME: schema-registry.local - SCHEMA_REGISTRY_LISTENERS: "https://schema-registry.local:8081" - SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "https" - SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO - SCHEMA_REGISTRY_LOG4J_LOGLEVEL: ERROR - SCHEMA_REGISTRY_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.schema-registry.truststore.jks - SCHEMA_REGISTRY_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.schema-registry.keystore.jks - SCHEMA_REGISTRY_SSL_TRUSTSTORE_PASSWORD: awesomekafka - SCHEMA_REGISTRY_SSL_KEYSTORE_PASSWORD: awesomekafka - SCHEMA_REGISTRY_SSL_KEY_PASSWORD: awesomekafka - SCHEMA_REGISTRY_KAFKASTORE_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS" - SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.schema-registry.truststore.jks - SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.schema-registry.keystore.jks - SCHEMA_REGISTRY_KAFKASTORE_SSL_TRUSTSTORE_PASSWORD: awesomekafka - SCHEMA_REGISTRY_KAFKASTORE_SSL_KEYSTORE_PASSWORD: awesomekafka - SCHEMA_REGISTRY_KAFKASTORE_SSL_KEY_PASSWORD: awesomekafka - SCHEMA_REGISTRY_SSL_CLIENT_AUTH: "true" - volumes: - - ./secrets:/etc/kafka/secrets - - control-center: - image: confluentinc/cp-enterprise-control-center:5.3.1 - container_name: control-center - networks: - kafka: - aliases: - - control-center.local - depends_on: - - zookeeper - - broker - - schema-registry - ports: - - "9021:9021" - environment: - CONTROL_CENTER_LOG4J_ROOT_LOGLEVEL: INFO - CONTROL_CENTER_LOG4J_LOGLEVEL: INFO - CONTROL_CENTER_BOOTSTRAP_SERVERS: broker.local:19092 - CONTROL_CENTER_ZOOKEEPER_CONNECT: zookeeper.local:22181 - CONTROL_CENTER_SCHEMA_REGISTRY_URL: "https://schema-registry.local:8081" - CONTROL_CENTER_STREAMS_SECURITY_PROTOCOL: SSL - CONTROL_CENTER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "" - CONTROL_CENTER_STREAMS_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.control-center.truststore.jks - CONTROL_CENTER_STREAMS_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.control-center.keystore.jks - CONTROL_CENTER_STREAMS_SSL_TRUSTSTORE_PASSWORD: awesomekafka - CONTROL_CENTER_STREAMS_SSL_KEYSTORE_PASSWORD: awesomekafka - CONTROL_CENTER_STREAMS_SSL_KEY_PASSWORD: awesomekafka - CONTROL_CENTER_STREAMS_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS" - CONTROL_CENTER_REPLICATION_FACTOR: 1 - CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 - CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 - CONFLUENT_METRICS_TOPIC_REPLICATION: 1 - CONTROL_CENTER_REST_LISTENERS: "http://0.0.0.0:9021" - CONTROL_CENTER_REST_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.control-center.truststore.jks - CONTROL_CENTER_REST_SSL_TRUSTSTORE_PASSWORD: awesomekafka - CONTROL_CENTER_REST_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.control-center.keystore.jks - CONTROL_CENTER_REST_SSL_KEYSTORE_PASSWORD: awesomekafka - CONTROL_CENTER_REST_SSL_KEY_PASSWORD: awesomekafka - CONTROL_CENTER_OPTS: -Djavax.net.ssl.trustStore=/etc/kafka/secrets/kafka.control-center.truststore.jks - -Djavax.net.ssl.trustStorePassword=kafka - -Djavax.net.ssl.keyStore=/etc/kafka/secrets/kafka.control-center.keystore.jks - -Djavax.net.ssl.keyStorePassword=kafka - PORT: 9021 - volumes: - - ./secrets:/etc/kafka/secrets - kafka-tools: build: context: ./kafka-tools @@ -162,45 +82,17 @@ services: KAFKA_SSL_TRUSTSTORE_PASSWORD: awesomekafka KAFKA_SSL_KEYSTORE_PASSWORD: awesomekafka KAFKA_SSL_KEY_PASSWORD: awesomekafka + KAFKA_ZOOKEEPER_CONNECT: zookeeper.local:22181 + KAFKA_ADVERTISED_LISTENERS: SSL://broker.local:19092 + KAFKA_SSL_KEYSTORE_FILENAME: kafka.kafka-tools.keystore.jks + KAFKA_SSL_KEYSTORE_CREDENTIALS: kafka-tools_keystore_creds + KAFKA_SSL_KEY_CREDENTIALS: kafka-tools_sslkey_creds + KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.kafka-tools.truststore.jks + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: kafka-tools_truststore_creds volumes: - ./secrets:/etc/kafka/secrets - ./config:/etc/kafka/config - - rest-proxy: - image: confluentinc/cp-kafka-rest:5.3.1 - depends_on: - - zookeeper - - broker - ports: - - 8082:8082 - hostname: rest-proxy - container_name: rest-proxy - networks: - kafka: - aliases: - - rest-proxy.local - environment: - KAFKA_REST_LOG4J_ROOT_LOGLEVEL: ERROR - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: " " - KAFKA_REST_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.rest-proxy.truststore.jks - KAFKA_REST_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.rest-proxy.keystore.jks - KAFKA_REST_SSL_TRUSTSTORE_PASSWORD: awesomekafka - KAFKA_REST_SSL_KEYSTORE_PASSWORD: awesomekafka - KAFKA_REST_SSL_KEY_PASSWORD: awesomekafka - KAFKA_REST_LOG4J_LOGLEVEL: ERROR - KAFKA_REST_HOST_NAME: rest-proxy - KAFKA_REST_BOOTSTRAP_SERVERS: broker.local:19092 - KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper.local:22181 - KAFKA_REST_CLIENT_SECURITY_PROTOCOL: SSL - KAFKA_REST_CLIENT_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.rest-proxy.truststore.jks - KAFKA_REST_CLIENT_SSL_TRUSTSTORE_PASSWORD: awesomekafka - KAFKA_REST_CLIENT_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.rest-proxy.keystore.jks - KAFKA_REST_CLIENT_SSL_KEYSTORE_PASSWORD: awesomekafka - KAFKA_REST_CLIENT_SSL_KEY_PASSWORD: awesomekafka - KAFKA_REST_LISTENERS: "http://0.0.0.0:8082" - KAFKA_HEAP_OPTS: "-Xmx1G -Xms1G" - volumes: - - ./secrets:/etc/kafka/secrets + - kafka-data:/var/lib/kafka/data volumes: zk-data: diff --git a/kafka-tools/Dockerfile b/kafka-tools/Dockerfile index a4dd3df..55d6e57 100644 --- a/kafka-tools/Dockerfile +++ b/kafka-tools/Dockerfile @@ -1,10 +1,4 @@ -FROM confluentinc/cp-enterprise-kafka:5.3.1 +FROM confluentinc/cp-enterprise-kafka:7.5.1 -# Set up a user to run Kafka -RUN groupadd kafka && \ - useradd -d /kafka -g kafka -s /bin/false kafka && \ - chown -R kafka:kafka /usr/bin/ - -USER kafka ENV PATH /usr/bin:$PATH WORKDIR /usr/bin