Merge pull request #1750 from kube-logging/fluentbit-max-worker-conn

pepov · web-flow · commit c4df1e5c811c · 2024-05-29T05:59:36.000+02:00
fluentbit: net.max_worker_connections
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_fluentbitagents.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_fluentbitagents.yaml
@@ -1653,6 +1653,8 @@ spec:
                   keepaliveMaxRecycle:
                     format: int32
                     type: integer
+                  maxWorkerConnections:
+                    type: integer
                   sourceAddress:
                     type: string
                 type: object
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml
@@ -2670,6 +2670,8 @@ spec:
                       keepaliveMaxRecycle:
                         format: int32
                         type: integer
+                      maxWorkerConnections:
+                        type: integer
                       sourceAddress:
                         type: string
                     type: object
@@ -11206,6 +11208,8 @@ spec:
                             keepaliveMaxRecycle:
                               format: int32
                               type: integer
+                            maxWorkerConnections:
+                              type: integer
                             sourceAddress:
                               type: string
                           type: object
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_nodeagents.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_nodeagents.yaml
@@ -4066,6 +4066,8 @@ spec:
                       keepaliveMaxRecycle:
                         format: int32
                         type: integer
+                      maxWorkerConnections:
+                        type: integer
                       sourceAddress:
                         type: string
                     type: object
diff --git a/config/crd/bases/logging.banzaicloud.io_fluentbitagents.yaml b/config/crd/bases/logging.banzaicloud.io_fluentbitagents.yaml
@@ -1653,6 +1653,8 @@ spec:
                   keepaliveMaxRecycle:
                     format: int32
                     type: integer
+                  maxWorkerConnections:
+                    type: integer
                   sourceAddress:
                     type: string
                 type: object
diff --git a/config/crd/bases/logging.banzaicloud.io_loggings.yaml b/config/crd/bases/logging.banzaicloud.io_loggings.yaml
@@ -2670,6 +2670,8 @@ spec:
                       keepaliveMaxRecycle:
                         format: int32
                         type: integer
+                      maxWorkerConnections:
+                        type: integer
                       sourceAddress:
                         type: string
                     type: object
@@ -11206,6 +11208,8 @@ spec:
                             keepaliveMaxRecycle:
                               format: int32
                               type: integer
+                            maxWorkerConnections:
+                              type: integer
                             sourceAddress:
                               type: string
                           type: object
diff --git a/config/crd/bases/logging.banzaicloud.io_nodeagents.yaml b/config/crd/bases/logging.banzaicloud.io_nodeagents.yaml
@@ -4066,6 +4066,8 @@ spec:
                       keepaliveMaxRecycle:
                         format: int32
                         type: integer
+                      maxWorkerConnections:
+                        type: integer
                       sourceAddress:
                         type: string
                     type: object
diff --git a/config/samples/syslog-ng-simple.yaml b/config/samples/syslog-ng-simple.yaml
@@ -5,6 +5,10 @@ metadata:
 spec:
   controlNamespace: default
   fluentbit: {}
+#    network:
+#      maxWorkerConnections: 2
+#    syslogng_output:
+#      Workers: 5
   syslogNG:
     globalOptions:
       log_level: trace
diff --git a/docs/configuration/crds/v1beta1/fluentbit_types.md b/docs/configuration/crds/v1beta1/fluentbit_types.md
@@ -313,6 +313,12 @@ How many times a TCP keepalive connection can be used before being recycled
 
 Default: 0, disabled
 
+### maxWorkerConnections (int, optional) {#fluentbitnetwork-maxworkerconnections}
+
+Set maximum number of TCP connections that can be established per worker.
+
+Default: 0, unlimited
+
 ### sourceAddress (string, optional) {#fluentbitnetwork-sourceaddress}
 
 Specify network address (interface) to use for connection and data traffic.
diff --git a/pkg/resources/fluentbit/config.go b/pkg/resources/fluentbit/config.go
@@ -185,6 +185,9 @@ var fluentbitNetworkTemplate = `
     {{- if .Network.SourceAddress }}
     net.source_address {{ .Network.SourceAddress }}
     {{- end }}
+    {{- if .Network.MaxWorkerConnections }}
+    net.max_worker_connections {{ .Network.MaxWorkerConnections }}
+    {{- end }}
     {{- end }}
 `
 
diff --git a/pkg/resources/fluentbit/configsecret.go b/pkg/resources/fluentbit/configsecret.go
@@ -125,6 +125,7 @@ type FluentbitNetwork struct {
 	KeepaliveMaxRecycleSet    bool
 	KeepaliveMaxRecycle       uint32
 	SourceAddress             string
+	MaxWorkerConnections      int
 }
 
 // https://docs.fluentbit.io/manual/pipeline/outputs/tcp-and-tls
@@ -187,6 +188,10 @@ func newFluentbitNetwork(network v1beta1.FluentbitNetwork) (result FluentbitNetw
 	if network.SourceAddress != "" {
 		result.SourceAddress = network.SourceAddress
 	}
+
+	if network.MaxWorkerConnections != 0 {
+		result.MaxWorkerConnections = network.MaxWorkerConnections
+	}
 	return
 }
 
diff --git a/pkg/sdk/logging/api/v1beta1/fluentbit_types.go b/pkg/sdk/logging/api/v1beta1/fluentbit_types.go
@@ -175,6 +175,8 @@ type FluentbitNetwork struct {
 	KeepaliveMaxRecycle *uint32 `json:"keepaliveMaxRecycle,omitempty"`
 	// Specify network address (interface) to use for connection and data traffic. (default: disabled)
 	SourceAddress string `json:"sourceAddress,omitempty"`
+	// Set maximum number of TCP connections that can be established per worker. (default: 0, unlimited)
+	MaxWorkerConnections int `json:"maxWorkerConnections,omitempty"`
 }
 
 // GetPrometheusPortFromAnnotation gets the port value from annotation

Original file line number	Diff line number	Diff line change
@@ -185,6 +185,9 @@ var fluentbitNetworkTemplate = `
`185`	`185`	`{{- if .Network.SourceAddress }}`
`186`	`186`	`net.source_address {{ .Network.SourceAddress }}`
`187`	`187`	`{{- end }}`
	`188`	`+ {{- if .Network.MaxWorkerConnections }}`
	`189`	`+ net.max_worker_connections {{ .Network.MaxWorkerConnections }}`
	`190`	`+ {{- end }}`
`188`	`191`	`{{- end }}`
`189`	`192`	`
`190`	`193`
Original file line number	Diff line number	Diff line change
`@@ -125,6 +125,7 @@ type FluentbitNetwork struct {`
`125`	`125`	`KeepaliveMaxRecycleSet bool`
`126`	`126`	`KeepaliveMaxRecycle uint32`
`127`	`127`	`SourceAddress string`
	`128`	`+ MaxWorkerConnections int`
`128`	`129`	`}`
`129`	`130`
`130`	`131`	`// https://docs.fluentbit.io/manual/pipeline/outputs/tcp-and-tls`
`@@ -187,6 +188,10 @@ func newFluentbitNetwork(network v1beta1.FluentbitNetwork) (result FluentbitNetw`
`187`	`188`	`if network.SourceAddress != "" {`
`188`	`189`	`result.SourceAddress = network.SourceAddress`
`189`	`190`	`}`
	`191`	`+`
	`192`	`+ if network.MaxWorkerConnections != 0 {`
	`193`	`+ result.MaxWorkerConnections = network.MaxWorkerConnections`
	`194`	`+ }`
`190`	`195`	`return`
`191`	`196`	`}`
`192`	`197`