Use an external database for conduktor (#134)

* add conduktor in all compose & use an external database

* add healthcheck

* compose cli v1 => v2

* remove depends on

* only target full.yml

* fix test

* Update conduktor.yml

Co-authored-by: Quentin <qboileau@conduktor.io>

* increase timeout

* fix ci

---------

Co-authored-by: Quentin <qboileau@conduktor.io>

Author: rokh-conduktor

.github/workflows/main.yml

@@ -51,5 +51,5 @@ jobs:
       - name: run test multiple-schema-registry 7
         run: ./test.sh zk-multiple-kafka-multiple-schema-registry.yml 7
 
-      - name: run test full-stack 7
-        run: ./test.sh full-stack.yml 7
+      - name: run test full-stack 8
+        run: ./test.sh full-stack.yml 8

README.md

@@ -9,7 +9,7 @@ Just connect against `localhost:9092`. If you are on Mac or Windows and want to
 
 # kafka-stack-docker-compose
 
-This replicates as well as possible real deployment configurations, where you have your zookeeper servers and kafka servers actually all distinct from each other. This solves all the networking hurdles that comes with Docker and docker-compose, and is compatible cross platform.
+This replicates as well as possible real deployment configurations, where you have your zookeeper servers and kafka servers actually all distinct from each other. This solves all the networking hurdles that comes with Docker and docker compose, and is compatible cross platform.
 
 **UPDATE**: No /etc/hosts file changes are necessary anymore. Explanations at: https://rmoff.net/2018/08/02/kafka-listeners-explained/
 
@@ -65,10 +65,10 @@ password: `admin`
 
  Run with:
  ```
- docker-compose -f full-stack.yml up
- docker-compose -f full-stack.yml down
+ docker compose -f full-stack.yml up
+ docker compose -f full-stack.yml down
  ```
-** Note: if you find that you can not connect to [localhost:8080](http://localhost:8080/) please run `docker-compose -f full-stack.yml build` to rebuild the port mappings.
+** Note: if you find that you can not connect to [localhost:8080](http://localhost:8080/) please run `docker compose -f full-stack.yml build` to rebuild the port mappings.
 
 ## Single Zookeeper / Single Kafka
 
@@ -80,8 +80,8 @@ This configuration fits most development requirements.
 
 Run with:
 ```
-docker-compose -f zk-single-kafka-single.yml up
-docker-compose -f zk-single-kafka-single.yml down
+docker compose -f zk-single-kafka-single.yml up
+docker compose -f zk-single-kafka-single.yml down
 ```
 
 ## Single Zookeeper / Multiple Kafka
@@ -94,8 +94,8 @@ If you want to have three brokers and experiment with kafka replication / fault-
 
 Run with:
 ```
-docker-compose -f zk-single-kafka-multiple.yml up
-docker-compose -f zk-single-kafka-multiple.yml down
+docker compose -f zk-single-kafka-multiple.yml up
+docker compose -f zk-single-kafka-multiple.yml down
 ```
 
 ## Multiple Zookeeper / Single Kafka
@@ -108,8 +108,8 @@ If you want to have three zookeeper nodes and experiment with zookeeper fault-to
 
 Run with:
 ```
-docker-compose -f zk-multiple-kafka-single.yml up
-docker-compose -f zk-multiple-kafka-single.yml down
+docker compose -f zk-multiple-kafka-single.yml up
+docker compose -f zk-multiple-kafka-single.yml down
 ```
 
 
@@ -122,8 +122,8 @@ If you want to have three zookeeper nodes and three kafka brokers to experiment
 
 Run with:
 ```
-docker-compose -f zk-multiple-kafka-multiple.yml up
-docker-compose -f zk-multiple-kafka-multiple.yml down
+docker compose -f zk-multiple-kafka-multiple.yml up
+docker compose -f zk-multiple-kafka-multiple.yml down
 ```
 
 # FAQ
@@ -132,11 +132,11 @@ docker-compose -f zk-multiple-kafka-multiple.yml down
 
 **Q: Kafka's log is too verbose, how can I reduce it?**
 
-A: Add the following line to your docker-compose environment variables: `KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"`. Full logging control can be accessed here: https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template
+A: Add the following line to your docker compose environment variables: `KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"`. Full logging control can be accessed here: https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template
 
 **Q: How do I delete data to start fresh?**
 
-A: Your data is persisted from within the docker compose folder, so if you want for example to reset the data in the full-stack docker compose, do a `docker-compose -f full-stack.yml down`.
+A: Your data is persisted from within the docker compose folder, so if you want for example to reset the data in the full-stack docker compose, do a `docker compose -f full-stack.yml down`.
 
 **Q: Can I change the zookeeper ports?**
 

conduktor.yml

@@ -0,0 +1,37 @@
+version: '3.8'
+
+services:  
+  
+  postgresql:
+    image: postgres:14
+    hostname: postgresql
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: "conduktor-platform"
+      POSTGRES_USER: "conduktor"
+      POSTGRES_PASSWORD: "some_password"
+      POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
+
+  conduktor-platform:
+    image: conduktor/conduktor-platform:latest
+    ports:
+      - "8080:8080"
+    volumes:
+      - conduktor_data:/var/conduktor
+    environment:
+      CDK_ORGANIZATION_NAME: "demo"
+      CDK_ADMIN_EMAIL: "admin@admin.io"
+      CDK_ADMIN_PASSWORD: "admin"
+      CDK_DATABASE_URL: "postgresql://conduktor:some_password@postgresql:5432/conduktor-platform"
+      CDK_CLUSTERS_0_ID: "default"
+      CDK_CLUSTERS_0_NAME: "My Local Kafka Cluster"
+      CDK_CLUSTERS_0_COLOR: "#0013E7"
+      CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "PLAINTEXT://kafka1:19092"
+      CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://kafka-schema-registry:8081"
+      CDK_CLUSTERS_0_KAFKACONNECTS_0_URL: "http://kafka-connect:8083"
+      CDK_CLUSTERS_0_KAFKACONNECTS_0_NAME: "full stack kafka connect"
+
+volumes:
+  pg_data: {}
+  conduktor_data: {}

full-stack.yml

@@ -126,28 +126,17 @@ services:
       - zoo1
       - kafka1
 
+  postgresql:
+    extends:
+      service: postgresql
+      file: conduktor.yml 
+            
   conduktor-platform:
-    image: conduktor/conduktor-platform:latest
-    ports:
-      - "8080:8080"
-    volumes:
-      - conduktor_data:/var/conduktor
-    depends_on:
-      - kafka1
-      - kafka-schema-registry
-      - kafka-connect
-    environment:
-      CDK_ORGANIZATION_NAME: "demo"
-      CDK_ADMIN_EMAIL: "admin@admin.io"
-      CDK_ADMIN_PASSWORD: "admin"
-      CDK_CLUSTERS_0_ID: "default"
-      CDK_CLUSTERS_0_NAME: "My Local Kafka Cluster"
-      CDK_CLUSTERS_0_COLOR: "#0013E7"
-      CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "PLAINTEXT://kafka1:19092"
-      CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://kafka-schema-registry:8081"
-      CDK_CLUSTERS_0_KAFKACONNECTS_0_URL: "http://kafka-connect:8083"
-      CDK_CLUSTERS_0_KAFKACONNECTS_0_NAME: "full stack kafka connect"
+    extends:
+      service: conduktor-platform
+      file: conduktor.yml
 
 volumes:
+  pg_data: {}
   conduktor_data: {}
 

test.sh

@@ -14,20 +14,20 @@ f () {
     # ${BASH_LINENO[0]} contains the line number in the script of that command
     # exit the script or return to try again, etc.
     # creating stack...
-    docker-compose -f $file down
+    docker compose -f $file down
     exit $errcode  # or use some other value or do return instead
 }
 trap f ERR
 
 all_great(){
     # for testing
     echo "Verifying Process"
-    running=`docker-compose -f $1 ps | grep Up | wc -l`
+    running=`docker compose -f $1 ps | grep Up | wc -l`
     if [ "$running" != "$2" ]; then
         # for logging
-        docker-compose -f $1 ps
+        docker compose -f $1 ps
         # debug
-        docker-compose -f $1 logs
+        docker compose -f $1 logs
         exit 1
     fi
 }
@@ -51,14 +51,15 @@ kafka_tests(){
 }
 
 # creating stack...
-docker-compose -f $file up -d
-sleep 10
+docker compose -f $file down -v
+docker compose -f $file up -d
+sleep 30
 # logging
-docker-compose -f $file ps
+docker compose -f $file ps
 # tests
 all_great $1 $2
 kafka_tests $1
 all_great $1 $2
 # teardown
-docker-compose -f $file down
+docker compose -f $file down -v
 echo "Success!"

zk-multiple-kafka-multiple-schema-registry.yml

@@ -115,3 +115,4 @@ services:
       SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka1:19092,PLAINTEXT://kafka2:19093,PLAINTEXT://kafka3:19094'
       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
+

zk-single-kafka-multiple.yml

@@ -69,4 +69,4 @@ services:
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
     depends_on:
-      - zoo1
+      - zoo1

zk-single-kafka-single.yml

@@ -36,3 +36,6 @@ services:
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
     depends_on:
       - zoo1
+
+
+