feat(kafka): add Kafka integration for event streaming

- Introduced Kafka support in the Evolution API, allowing for real-time event streaming and processing.
- Updated environment configuration to include Kafka-related variables.
- Added KafkaController and KafkaRouter for managing Kafka events.
- Enhanced event management to support Kafka alongside existing integrations.
- Updated database schemas and migrations for Kafka integration in both MySQL and PostgreSQL.
- Documented Kafka integration in the README file.

Author: DavidsonGomes

.env.example

@@ -190,6 +190,60 @@ PUSHER_EVENTS_CALL=true
 PUSHER_EVENTS_TYPEBOT_START=false
 PUSHER_EVENTS_TYPEBOT_CHANGE_STATUS=false
 
+# Kafka - Environment variables
+KAFKA_ENABLED=false
+KAFKA_CLIENT_ID=evolution-api
+KAFKA_BROKERS=localhost:9092
+KAFKA_CONNECTION_TIMEOUT=3000
+KAFKA_REQUEST_TIMEOUT=30000
+# Global events - By enabling this variable, events from all instances are sent to global Kafka topics.
+KAFKA_GLOBAL_ENABLED=false
+KAFKA_CONSUMER_GROUP_ID=evolution-api-consumers
+KAFKA_TOPIC_PREFIX=evolution
+KAFKA_NUM_PARTITIONS=1
+KAFKA_REPLICATION_FACTOR=1
+KAFKA_AUTO_CREATE_TOPICS=false
+# Choose the events you want to send to Kafka
+KAFKA_EVENTS_APPLICATION_STARTUP=false
+KAFKA_EVENTS_INSTANCE_CREATE=false
+KAFKA_EVENTS_INSTANCE_DELETE=false
+KAFKA_EVENTS_QRCODE_UPDATED=false
+KAFKA_EVENTS_MESSAGES_SET=false
+KAFKA_EVENTS_MESSAGES_UPSERT=false
+KAFKA_EVENTS_MESSAGES_EDITED=false
+KAFKA_EVENTS_MESSAGES_UPDATE=false
+KAFKA_EVENTS_MESSAGES_DELETE=false
+KAFKA_EVENTS_SEND_MESSAGE=false
+KAFKA_EVENTS_SEND_MESSAGE_UPDATE=false
+KAFKA_EVENTS_CONTACTS_SET=false
+KAFKA_EVENTS_CONTACTS_UPSERT=false
+KAFKA_EVENTS_CONTACTS_UPDATE=false
+KAFKA_EVENTS_PRESENCE_UPDATE=false
+KAFKA_EVENTS_CHATS_SET=false
+KAFKA_EVENTS_CHATS_UPSERT=false
+KAFKA_EVENTS_CHATS_UPDATE=false
+KAFKA_EVENTS_CHATS_DELETE=false
+KAFKA_EVENTS_GROUPS_UPSERT=false
+KAFKA_EVENTS_GROUPS_UPDATE=false
+KAFKA_EVENTS_GROUP_PARTICIPANTS_UPDATE=false
+KAFKA_EVENTS_CONNECTION_UPDATE=false
+KAFKA_EVENTS_LABELS_EDIT=false
+KAFKA_EVENTS_LABELS_ASSOCIATION=false
+KAFKA_EVENTS_CALL=false
+KAFKA_EVENTS_TYPEBOT_START=false
+KAFKA_EVENTS_TYPEBOT_CHANGE_STATUS=false
+# SASL Authentication (optional)
+KAFKA_SASL_ENABLED=false
+KAFKA_SASL_MECHANISM=plain
+KAFKA_SASL_USERNAME=
+KAFKA_SASL_PASSWORD=
+# SSL Configuration (optional)
+KAFKA_SSL_ENABLED=false
+KAFKA_SSL_REJECT_UNAUTHORIZED=true
+KAFKA_SSL_CA=
+KAFKA_SSL_KEY=
+KAFKA_SSL_CERT=
+
 # WhatsApp Business API - Environment variables
 # Token used to validate the webhook on the Facebook APP
 WA_BUSINESS_TOKEN_WEBHOOK=evolution

README.md

@@ -55,6 +55,9 @@ Evolution API supports various integrations to enhance its functionality. Below
 - [RabbitMQ](https://www.rabbitmq.com/):
   - Receive events from the Evolution API via RabbitMQ.
 
+- [Apache Kafka](https://kafka.apache.org/):
+  - Receive events from the Evolution API via Apache Kafka for real-time event streaming and processing.
+
 - [Amazon SQS](https://aws.amazon.com/pt/sqs/):
   - Receive events from the Evolution API via Amazon SQS.
 

package-lock.json

@@ -95,6 +95,7 @@
     "json-schema": "^0.4.0",
     "jsonschema": "^1.4.1",
     "jsonwebtoken": "^9.0.2",
+    "kafkajs": "^2.2.4",
     "link-preview-js": "^3.0.13",
     "long": "^5.2.3",
     "mediainfo.js": "^0.3.4",

package.json

@@ -88,6 +88,7 @@ model Instance {
   Rabbitmq                Rabbitmq?
   Nats                    Nats?
   Sqs                     Sqs?
+  Kafka                   Kafka?
   Websocket               Websocket?
   Typebot                 Typebot[]
   Session                 Session?
@@ -105,8 +106,11 @@ model Instance {
   EvolutionBotSetting     EvolutionBotSetting?
   Flowise                 Flowise[]
   FlowiseSetting          FlowiseSetting?
-  Pusher                  Pusher?
   N8n                     N8n[]
+  N8nSetting              N8nSetting?
+  Evoai                   Evoai[]
+  EvoaiSetting            EvoaiSetting?
+  Pusher                  Pusher?
 }
 
 model Session {
@@ -309,6 +313,16 @@ model Sqs {
   instanceId String    @unique
 }
 
+model Kafka {
+  id         String    @id @default(cuid())
+  enabled    Boolean   @default(false)
+  events     Json      @db.Json
+  createdAt  DateTime? @default(dbgenerated("CURRENT_TIMESTAMP")) @db.Timestamp
+  updatedAt  DateTime  @updatedAt @db.Timestamp
+  Instance   Instance  @relation(fields: [instanceId], references: [id], onDelete: Cascade)
+  instanceId String    @unique
+}
+
 model Websocket {
   id         String    @id @default(cuid())
   enabled    Boolean   @default(false)
@@ -647,7 +661,7 @@ model IsOnWhatsapp {
 
 model N8n {
   id              String           @id @default(cuid())
-  enabled         Boolean          @default(true) @db.TinyInt(1)
+  enabled         Boolean          @default(true) @db.TinyInt()
   description     String?          @db.VarChar(255)
   webhookUrl      String?          @db.VarChar(255)
   basicAuthUser   String?          @db.VarChar(255)
@@ -666,7 +680,7 @@ model N8n {
   triggerType     TriggerType?
   triggerOperator TriggerOperator?
   triggerValue    String?
-  createdAt       DateTime?        @default(now()) @db.Timestamp
+  createdAt       DateTime?        @default(dbgenerated("CURRENT_TIMESTAMP")) @db.Timestamp
   updatedAt       DateTime         @updatedAt @db.Timestamp
   Instance        Instance         @relation(fields: [instanceId], references: [id], onDelete: Cascade)
   instanceId      String
@@ -686,7 +700,7 @@ model N8nSetting {
   ignoreJids      Json?
   splitMessages   Boolean?  @default(false)
   timePerChar     Int?      @default(50) @db.Int
-  createdAt       DateTime? @default(now()) @db.Timestamp
+  createdAt       DateTime? @default(dbgenerated("CURRENT_TIMESTAMP")) @db.Timestamp
   updatedAt       DateTime  @updatedAt @db.Timestamp
   Fallback        N8n?      @relation(fields: [n8nIdFallback], references: [id])
   n8nIdFallback   String?   @db.VarChar(100)
@@ -696,7 +710,7 @@ model N8nSetting {
 
 model Evoai {
   id              String           @id @default(cuid())
-  enabled         Boolean          @default(true) @db.TinyInt(1)
+  enabled         Boolean          @default(true) @db.TinyInt()
   description     String?          @db.VarChar(255)
   agentUrl        String?          @db.VarChar(255)
   apiKey          String?          @db.VarChar(255)
@@ -714,7 +728,7 @@ model Evoai {
   triggerType     TriggerType?
   triggerOperator TriggerOperator?
   triggerValue    String?
-  createdAt       DateTime?        @default(now()) @db.Timestamp
+  createdAt       DateTime?        @default(dbgenerated("CURRENT_TIMESTAMP")) @db.Timestamp
   updatedAt       DateTime         @updatedAt @db.Timestamp
   Instance        Instance         @relation(fields: [instanceId], references: [id], onDelete: Cascade)
   instanceId      String
@@ -734,7 +748,7 @@ model EvoaiSetting {
   ignoreJids      Json?
   splitMessages   Boolean?  @default(false)
   timePerChar     Int?      @default(50) @db.Int
-  createdAt       DateTime? @default(now()) @db.Timestamp
+  createdAt       DateTime? @default(dbgenerated("CURRENT_TIMESTAMP")) @db.Timestamp
   updatedAt       DateTime  @updatedAt @db.Timestamp
   Fallback        Evoai?    @relation(fields: [evoaiIdFallback], references: [id])
   evoaiIdFallback String?   @db.VarChar(100)

prisma/mysql-migrations/20250918183910_add_kafka_integration/migration.sql

@@ -0,0 +1,17 @@
+-- CreateTable
+CREATE TABLE "public"."Kafka" (
+    "id" TEXT NOT NULL,
+    "enabled" BOOLEAN NOT NULL DEFAULT false,
+    "events" JSONB NOT NULL,
+    "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP NOT NULL,
+    "instanceId" TEXT NOT NULL,
+
+    CONSTRAINT "Kafka_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Kafka_instanceId_key" ON "public"."Kafka"("instanceId");
+
+-- AddForeignKey
+ALTER TABLE "public"."Kafka" ADD CONSTRAINT "Kafka_instanceId_fkey" FOREIGN KEY ("instanceId") REFERENCES "public"."Instance"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/mysql-schema.prisma

@@ -88,6 +88,7 @@ model Instance {
   Rabbitmq                Rabbitmq?
   Nats                    Nats?
   Sqs                     Sqs?
+  Kafka                   Kafka?
   Websocket               Websocket?
   Typebot                 Typebot[]
   Session                 Session?
@@ -312,6 +313,16 @@ model Sqs {
   instanceId String    @unique
 }
 
+model Kafka {
+  id         String    @id @default(cuid())
+  enabled    Boolean   @default(false) @db.Boolean
+  events     Json      @db.JsonB
+  createdAt  DateTime? @default(now()) @db.Timestamp
+  updatedAt  DateTime  @updatedAt @db.Timestamp
+  Instance   Instance  @relation(fields: [instanceId], references: [id], onDelete: Cascade)
+  instanceId String    @unique
+}
+
 model Websocket {
   id         String    @id @default(cuid())
   enabled    Boolean   @default(false) @db.Boolean

prisma/postgresql-migrations/20250918182355_add_kafka_integration/migration.sql

@@ -89,6 +89,7 @@ model Instance {
   Rabbitmq                Rabbitmq?
   Nats                    Nats?
   Sqs                     Sqs?
+  Kafka                   Kafka?
   Websocket               Websocket?
   Typebot                 Typebot[]
   Session                 Session?
@@ -313,6 +314,16 @@ model Sqs {
   instanceId String    @unique
 }
 
+model Kafka {
+  id         String    @id @default(cuid())
+  enabled    Boolean   @default(false) @db.Boolean
+  events     Json      @db.JsonB
+  createdAt  DateTime? @default(now()) @db.Timestamp
+  updatedAt  DateTime  @updatedAt @db.Timestamp
+  Instance   Instance  @relation(fields: [instanceId], references: [id], onDelete: Cascade)
+  instanceId String    @unique
+}
+
 model Websocket {
   id         String    @id @default(cuid())
   enabled    Boolean   @default(false) @db.Boolean

prisma/postgresql-schema.prisma

@@ -40,6 +40,11 @@ export class EventDto {
     useTLS?: boolean;
     events?: string[];
   };
+
+  kafka?: {
+    enabled?: boolean;
+    events?: string[];
+  };
 }
 
 export function EventInstanceMixin<TBase extends Constructor>(Base: TBase) {
@@ -82,5 +87,10 @@ export function EventInstanceMixin<TBase extends Constructor>(Base: TBase) {
       useTLS?: boolean;
       events?: string[];
     };
+
+    kafka?: {
+      enabled?: boolean;
+      events?: string[];
+    };
   };
 }