diff --git a/apis/fluentd/v1alpha1/plugins/output/opensearch.go b/apis/fluentd/v1alpha1/plugins/output/opensearch.go index ae07a6336..69b5ea996 100644 --- a/apis/fluentd/v1alpha1/plugins/output/opensearch.go +++ b/apis/fluentd/v1alpha1/plugins/output/opensearch.go @@ -28,4 +28,95 @@ type Opensearch struct { Password *plugins.Secret `json:"password,omitempty"` // Optional, Force certificate validation SslVerify *bool `json:"sslVerify,omitempty"` + // Optional, Absolute path to CA certificate file + CAFile *string `json:"caFile,omitempty"` + // Optional, Absolute path to client Certificate file + ClientCert *string `json:"clientCert,omitempty"` + // Optional, Absolute path to client private Key file + ClientKey *string `json:"clientKey,omitempty"` + // Optional, password for ClientKey file + ClientKeyPassword *plugins.Secret `json:"clientKeyPassword,omitempty"` + // Optional, You can specify SSL/TLS version (default: TLSv1_2) + SslVersion *string `json:"sslVersion,omitempty"` + // Optional, Minimum SSL/TLS version + SslMinVersion *string `json:"sslMinVersion,omitempty"` + // Optional, Maximum SSL/TLS version + SslMaxVersion *string `json:"sslMaxVersion,omitempty"` + // Optional, Enable logging of 400 reason without enabling debug log level (default: false) + LogOs400Reason *bool `json:"logOs400Reason,omitempty"` + // Optional, HTTP request timeout in seconds (default: 5s) + // +kubebuilder:validation:Pattern:="^\\d+(s|m|h|d)$" + RequestTimeout *string `json:"requestTimeout,omitempty"` + // Optional, Indicates that the plugin should reset connection on any error (reconnect on next send) (default: false) + ReconnectOnError *bool `json:"reconnectOnError,omitempty"` + // Optional, Automatically reload connection after 10000 documents (default: true) + ReloadConnections *bool `json:"reloadConnections,omitempty"` + // Optional, When ReloadConnections true, this is the integer number of operations after which the plugin will reload the connections (default: 10000) + ReloadAfter *uint32 `json:"reloadAfter,omitempty"` + // Optional, Indicates that the opensearch-transport will try to reload the nodes addresses if there is a failure while making the request (default: false) + ReloadOnFailure *bool `json:"reloadOnFailure,omitempty"` + // Optional, You can specify times of retry obtaining OpenSearch version (default: 15) + MaxRetryGetOsVersion *uint32 `json:"maxRetryGetOsVersion,omitempty"` + // Optional, Indicates whether to fail when max_retry_get_os_version is exceeded (default: true) + FailOnDetectingOsVersionRetryExceed *bool `json:"failOnDetectingOsVersionRetryExceed,omitempty"` + // Optional, Default OpenSearch version (default: 1) + DefaultOpensearchVersion *uint32 `json:"defaultOpensearchVersion,omitempty"` + // Optional, Validate OpenSearch version at startup (default: true) + VerifyOsVersionAtStartup *bool `json:"verifyOsVersionAtStartup,omitempty"` + // Optional, Always update the template, even if it already exists (default: false) + TemplateOverwrite *bool `json:"templateOverwrite,omitempty"` + // Optional, You can specify times of retry putting template (default: 10) + MaxRetryPuttingTemplate *uint32 `json:"maxRetryPuttingTemplate,omitempty"` + // Optional, Indicates whether to fail when max_retry_putting_template is exceeded (default: true) + FailOnPuttingTemplateRetryExceed *bool `json:"failOnPuttingTemplateRetryExceed,omitempty"` + // Optional, Provide a different sniffer class name + SnifferClassName *string `json:"snifferClassName,omitempty"` + // Optional, Provide a selector class name + SelectorClassName *string `json:"selectorClassName,omitempty"` + // Optional, You can specify HTTP backend (default: excon). Options: excon, typhoeus + HttpBackend *string `json:"httpBackend,omitempty"` + // Optional, With http_backend_excon_nonblock false, plugin uses excon with nonblock=false (default: true) + HttpBackendExconNonblock *bool `json:"httpBackendExconNonblock,omitempty"` + // Optional, You can specify the compression level (default: no_compression). Options: no_compression, best_compression, best_speed, default_compression + CompressionLevel *string `json:"compressionLevel,omitempty"` + // Optional, With default behavior, plugin uses Yajl as JSON encoder/decoder. Set to true to use Oj (default: false) + PreferOjSerializer *bool `json:"preferOjSerializer,omitempty"` + // Optional, Suppress '[types removal]' warnings on OpenSearch 2.x (default: true for OS2+) + SuppressTypeName *bool `json:"suppressTypeName,omitempty"` + // Optional, With content_type application/x-ndjson, plugin adds application/x-ndjson as Content-Type (default: application/json) + ContentType *string `json:"contentType,omitempty"` + // Optional, Include tag key in record (default: false) + IncludeTagKey *bool `json:"includeTagKey,omitempty"` + // Optional, Tag key name when include_tag_key is true (default: tag) + TagKey *string `json:"tagKey,omitempty"` + // Optional, Record accessor syntax to specify the field to use as _id in OpenSearch + IdKey *string `json:"idKey,omitempty"` + // Optional, Remove specified keys from the event record + RemoveKeys *string `json:"removeKeys,omitempty"` + // Optional, Remove keys when record is being updated + RemoveKeysOnUpdate *string `json:"removeKeysOnUpdate,omitempty"` + // Optional, The write operation (default: index). Options: index, create, update, upsert + WriteOperation *string `json:"writeOperation,omitempty"` + // Optional, When write_operation is not index, setting this true will cause plugin to emit_error_event of records which do not include _id field (default: false) + EmitErrorForMissingId *bool `json:"emitErrorForMissingId,omitempty"` + // Optional, Custom headers in Hash format + CustomHeaders *string `json:"customHeaders,omitempty"` + // Optional, Pipeline name + Pipeline *string `json:"pipeline,omitempty"` + // Optional, UTC index (default: false for local time) + UtcIndex *bool `json:"utcIndex,omitempty"` + // Optional, Suppress doc_wrap (default: false) + SuppressDocWrap *bool `json:"suppressDocWrap,omitempty"` + // Optional, List of exception classes to ignore + IgnoreExceptions *string `json:"ignoreExceptions,omitempty"` + // Optional, Backup chunk when ignore exception occurs (default: true) + ExceptionBackup *bool `json:"exceptionBackup,omitempty"` + // Optional, Configure bulk_message request splitting threshold size (default: -1 unlimited) + BulkMessageRequestThreshold *int32 `json:"bulkMessageRequestThreshold,omitempty"` + // Optional, Specify the application name for the rollover index to be created (default: default) + ApplicationName *string `json:"applicationName,omitempty"` + // Optional, Specify the index date pattern for creating a rollover index (default: now/d) + IndexDatePattern *string `json:"indexDatePattern,omitempty"` + // Optional, Use legacy template or not (default: false for composable templates) + UseLegacyTemplate *bool `json:"useLegacyTemplate,omitempty"` } diff --git a/apis/fluentd/v1alpha1/plugins/output/types.go b/apis/fluentd/v1alpha1/plugins/output/types.go index a00e4dae6..e96ba78f8 100644 --- a/apis/fluentd/v1alpha1/plugins/output/types.go +++ b/apis/fluentd/v1alpha1/plugins/output/types.go @@ -458,80 +458,133 @@ func (o *Output) elasticsearchDataStreamPlugin(parent *params.PluginStore, loade } func (o *Output) opensearchPlugin(parent *params.PluginStore, loader plugins.SecretLoader) (*params.PluginStore, error) { - if o.Opensearch.Host != nil { - parent.InsertPairs("host", fmt.Sprint(*o.Opensearch.Host)) + if err := o.opensearchBasicConnection(parent, loader); err != nil { + return nil, err } - - if o.Opensearch.Port != nil { - parent.InsertPairs("port", fmt.Sprint(*o.Opensearch.Port)) + o.opensearchIndexConfig(parent) + if err := o.opensearchSSLConfig(parent, loader); err != nil { + return nil, err } + o.opensearchConnectionManagement(parent) + o.opensearchVersionDetection(parent) + o.opensearchTemplateManagement(parent) + o.opensearchPerformanceTuning(parent) + o.opensearchRecordHandling(parent) + o.opensearchAdvancedOptions(parent) - if o.Opensearch.Hosts != nil { - parent.InsertPairs("hosts", fmt.Sprint(*o.Opensearch.Hosts)) - } + return parent, nil +} +func (o *Output) opensearchBasicConnection(parent *params.PluginStore, loader plugins.SecretLoader) error { + params.InsertPairs(parent, "host", o.Opensearch.Host) + params.InsertPairs(parent, "port", o.Opensearch.Port) + params.InsertPairs(parent, "hosts", o.Opensearch.Hosts) if o.Opensearch.User != nil { user, err := loader.LoadSecret(*o.Opensearch.User) if err != nil { - return nil, err + return err } parent.InsertPairs("user", user) } - if o.Opensearch.Password != nil { pwd, err := loader.LoadSecret(*o.Opensearch.Password) if err != nil { - return nil, err + return err } parent.InsertPairs("password", pwd) } + params.InsertPairs(parent, "scheme", o.Opensearch.Scheme) + params.InsertPairs(parent, "path", o.Opensearch.Path) + return nil +} - if o.Opensearch.Scheme != nil { - parent.InsertPairs("scheme", fmt.Sprint(*o.Opensearch.Scheme)) - } +func (o *Output) opensearchIndexConfig(parent *params.PluginStore) { + params.InsertPairs(parent, "index_name", o.Opensearch.IndexName) + params.InsertPairs(parent, "logstash_format", o.Opensearch.LogstashFormat) + params.InsertPairs(parent, "logstash_prefix", o.Opensearch.LogstashPrefix) + params.InsertPairs(parent, "index_date_pattern", o.Opensearch.IndexDatePattern) + params.InsertPairs(parent, "utc_index", o.Opensearch.UtcIndex) +} - if o.Opensearch.Path != nil { - parent.InsertPairs("path", fmt.Sprint(*o.Opensearch.Path)) +func (o *Output) opensearchSSLConfig(parent *params.PluginStore, loader plugins.SecretLoader) error { + params.InsertPairs(parent, "ssl_verify", o.Opensearch.SslVerify) + params.InsertPairs(parent, "ca_file", o.Opensearch.CAFile) + params.InsertPairs(parent, "client_cert", o.Opensearch.ClientCert) + params.InsertPairs(parent, "client_key", o.Opensearch.ClientKey) + if o.Opensearch.ClientKeyPassword != nil { + pwd, err := loader.LoadSecret(*o.Opensearch.ClientKeyPassword) + if err != nil { + return err + } + parent.InsertPairs("client_key_pass", pwd) } + params.InsertPairs(parent, "ssl_version", o.Opensearch.SslVersion) + params.InsertPairs(parent, "ssl_min_version", o.Opensearch.SslMinVersion) + params.InsertPairs(parent, "ssl_max_version", o.Opensearch.SslMaxVersion) + return nil +} - if o.Opensearch.IndexName != nil { - parent.InsertPairs("index_name", fmt.Sprint(*o.Opensearch.IndexName)) - } +func (o *Output) opensearchConnectionManagement(parent *params.PluginStore) { + params.InsertPairs(parent, "log_os_400_reason", o.Opensearch.LogOs400Reason) + params.InsertPairs(parent, "request_timeout", o.Opensearch.RequestTimeout) + params.InsertPairs(parent, "reconnect_on_error", o.Opensearch.ReconnectOnError) + params.InsertPairs(parent, "reload_connections", o.Opensearch.ReloadConnections) + params.InsertPairs(parent, "reload_after", o.Opensearch.ReloadAfter) + params.InsertPairs(parent, "reload_on_failure", o.Opensearch.ReloadOnFailure) +} - if o.Opensearch.LogstashFormat != nil { - parent.InsertPairs("logstash_format", fmt.Sprint(*o.Opensearch.LogstashFormat)) - } +func (o *Output) opensearchVersionDetection(parent *params.PluginStore) { + params.InsertPairs(parent, "max_retry_get_os_version", o.Opensearch.MaxRetryGetOsVersion) + params.InsertPairs(parent, "fail_on_detecting_os_version_retry_exceed", o.Opensearch.FailOnDetectingOsVersionRetryExceed) + params.InsertPairs(parent, "default_opensearch_version", o.Opensearch.DefaultOpensearchVersion) + params.InsertPairs(parent, "verify_os_version_at_startup", o.Opensearch.VerifyOsVersionAtStartup) +} - if o.Opensearch.LogstashPrefix != nil { - parent.InsertPairs("logstash_prefix", fmt.Sprint(*o.Opensearch.LogstashPrefix)) - } +func (o *Output) opensearchTemplateManagement(parent *params.PluginStore) { + params.InsertPairs(parent, "template_overwrite", o.Opensearch.TemplateOverwrite) + params.InsertPairs(parent, "max_retry_putting_template", o.Opensearch.MaxRetryPuttingTemplate) + params.InsertPairs(parent, "fail_on_putting_template_retry_exceed", o.Opensearch.FailOnPuttingTemplateRetryExceed) + params.InsertPairs(parent, "use_legacy_template", o.Opensearch.UseLegacyTemplate) +} - if o.Opensearch.SslVerify != nil { - parent.InsertPairs("ssl_verify", fmt.Sprint(*o.Opensearch.SslVerify)) - } +func (o *Output) opensearchPerformanceTuning(parent *params.PluginStore) { + params.InsertPairs(parent, "sniffer_class_name", o.Opensearch.SnifferClassName) + params.InsertPairs(parent, "selector_class_name", o.Opensearch.SelectorClassName) + params.InsertPairs(parent, "http_backend", o.Opensearch.HttpBackend) + params.InsertPairs(parent, "http_backend_excon_nonblock", o.Opensearch.HttpBackendExconNonblock) + params.InsertPairs(parent, "compression_level", o.Opensearch.CompressionLevel) + params.InsertPairs(parent, "prefer_oj_serializer", o.Opensearch.PreferOjSerializer) + params.InsertPairs(parent, "bulk_message_request_threshold", o.Opensearch.BulkMessageRequestThreshold) +} - return parent, nil +func (o *Output) opensearchRecordHandling(parent *params.PluginStore) { + params.InsertPairs(parent, "suppress_type_name", o.Opensearch.SuppressTypeName) + params.InsertPairs(parent, "content_type", o.Opensearch.ContentType) + params.InsertPairs(parent, "include_tag_key", o.Opensearch.IncludeTagKey) + params.InsertPairs(parent, "tag_key", o.Opensearch.TagKey) + params.InsertPairs(parent, "id_key", o.Opensearch.IdKey) + params.InsertPairs(parent, "remove_keys", o.Opensearch.RemoveKeys) + params.InsertPairs(parent, "remove_keys_on_update", o.Opensearch.RemoveKeysOnUpdate) + params.InsertPairs(parent, "write_operation", o.Opensearch.WriteOperation) + params.InsertPairs(parent, "emit_error_for_missing_id", o.Opensearch.EmitErrorForMissingId) + params.InsertPairs(parent, "suppress_doc_wrap", o.Opensearch.SuppressDocWrap) +} + +func (o *Output) opensearchAdvancedOptions(parent *params.PluginStore) { + params.InsertPairs(parent, "custom_headers", o.Opensearch.CustomHeaders) + params.InsertPairs(parent, "pipeline", o.Opensearch.Pipeline) + params.InsertPairs(parent, "ignore_exceptions", o.Opensearch.IgnoreExceptions) + params.InsertPairs(parent, "exception_backup", o.Opensearch.ExceptionBackup) + params.InsertPairs(parent, "application_name", o.Opensearch.ApplicationName) } func (o *Output) kafka2Plugin(parent *params.PluginStore, _ plugins.SecretLoader) *params.PluginStore { - if o.Kafka.Brokers != nil { - parent.InsertPairs("brokers", fmt.Sprint(*o.Kafka.Brokers)) - } - if o.Kafka.TopicKey != nil { - parent.InsertPairs("topic_key", fmt.Sprint(*o.Kafka.TopicKey)) - } - if o.Kafka.DefaultTopic != nil { - parent.InsertPairs("default_topic", fmt.Sprint(*o.Kafka.DefaultTopic)) - } - if o.Kafka.UseEventTime != nil { - parent.InsertPairs("use_event_time", fmt.Sprint(*o.Kafka.UseEventTime)) - } - if o.Kafka.RequiredAcks != nil { - parent.InsertPairs("required_acks", fmt.Sprint(*o.Kafka.RequiredAcks)) - } - if o.Kafka.CompressionCodec != nil { - parent.InsertPairs("compression_codec", fmt.Sprint(*o.Kafka.CompressionCodec)) - } + params.InsertPairs(parent, "brokers", o.Kafka.Brokers) + params.InsertPairs(parent, "topic_key", o.Kafka.TopicKey) + params.InsertPairs(parent, "default_topic", o.Kafka.DefaultTopic) + params.InsertPairs(parent, "use_event_time", o.Kafka.UseEventTime) + params.InsertPairs(parent, "required_acks", o.Kafka.RequiredAcks) + params.InsertPairs(parent, "compression_codec", o.Kafka.CompressionCodec) return parent } diff --git a/apis/fluentd/v1alpha1/plugins/params/model.go b/apis/fluentd/v1alpha1/plugins/params/model.go index 69a39bce2..6ac01f237 100644 --- a/apis/fluentd/v1alpha1/plugins/params/model.go +++ b/apis/fluentd/v1alpha1/plugins/params/model.go @@ -36,7 +36,7 @@ func (ps *PluginStore) InsertPairs(key, value string) { } type ValueType interface { - *string | *bool | *int | *int16 | *uint16 | *uint32 + *string | *bool | *int | *int16 | *int32 | *uint16 | *uint32 } func InsertPairs[T ValueType](ps *PluginStore, key string, value T) { @@ -52,6 +52,8 @@ func InsertPairs[T ValueType](ps *PluginStore, key string, value T) { ps.InsertPairs(key, strconv.FormatInt(int64(*v), 10)) case *int16: ps.InsertPairs(key, strconv.FormatInt(int64(*v), 10)) + case *int32: + ps.InsertPairs(key, strconv.FormatInt(int64(*v), 10)) case *uint16: ps.InsertPairs(key, strconv.FormatUint(uint64(*v), 10)) case *uint32: diff --git a/charts/fluentd-crds/templates/fluentd.fluent.io_clusteroutputs.yaml b/charts/fluentd-crds/templates/fluentd.fluent.io_clusteroutputs.yaml index 47c95755c..b92f7206b 100644 --- a/charts/fluentd-crds/templates/fluentd.fluent.io_clusteroutputs.yaml +++ b/charts/fluentd-crds/templates/fluentd.fluent.io_clusteroutputs.yaml @@ -2192,6 +2192,95 @@ spec: opensearch: description: out_opensearch plugin properties: + applicationName: + description: 'Optional, Specify the application name for + the rollover index to be created (default: default)' + type: string + bulkMessageRequestThreshold: + description: 'Optional, Configure bulk_message request splitting + threshold size (default: -1 unlimited)' + format: int32 + type: integer + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compressionLevel: + description: 'Optional, You can specify the compression + level (default: no_compression). Options: no_compression, + best_compression, best_speed, default_compression' + type: string + contentType: + description: 'Optional, With content_type application/x-ndjson, + plugin adds application/x-ndjson as Content-Type (default: + application/json)' + type: string + customHeaders: + description: Optional, Custom headers in Hash format + type: string + defaultOpensearchVersion: + description: 'Optional, Default OpenSearch version (default: + 1)' + format: int32 + type: integer + emitErrorForMissingId: + description: 'Optional, When write_operation is not index, + setting this true will cause plugin to emit_error_event + of records which do not include _id field (default: false)' + type: boolean + exceptionBackup: + description: 'Optional, Backup chunk when ignore exception + occurs (default: true)' + type: boolean + failOnDetectingOsVersionRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_get_os_version + is exceeded (default: true)' + type: boolean + failOnPuttingTemplateRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_putting_template + is exceeded (default: true)' + type: boolean host: description: 'The hostname of your Opensearch node (default: localhost).' @@ -2200,10 +2289,38 @@ spec: description: Hosts defines a list of hosts if you want to connect to more than one Openearch nodes type: string + httpBackend: + description: 'Optional, You can specify HTTP backend (default: + excon). Options: excon, typhoeus' + type: string + httpBackendExconNonblock: + description: 'Optional, With http_backend_excon_nonblock + false, plugin uses excon with nonblock=false (default: + true)' + type: boolean + idKey: + description: Optional, Record accessor syntax to specify + the field to use as _id in OpenSearch + type: string + ignoreExceptions: + description: Optional, List of exception classes to ignore + type: string + includeTagKey: + description: 'Optional, Include tag key in record (default: + false)' + type: boolean + indexDatePattern: + description: 'Optional, Specify the index date pattern for + creating a rollover index (default: now/d)' + type: string indexName: description: IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. type: string + logOs400Reason: + description: 'Optional, Enable logging of 400 reason without + enabling debug log level (default: false)' + type: boolean logstashFormat: description: 'If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option @@ -2214,6 +2331,16 @@ spec: index name to write events when logstash_format is true (default: logstash).' type: string + maxRetryGetOsVersion: + description: 'Optional, You can specify times of retry obtaining + OpenSearch version (default: 15)' + format: int32 + type: integer + maxRetryPuttingTemplate: + description: 'Optional, You can specify times of retry putting + template (default: 10)' + format: int32 + type: integer password: description: Optional, The login credentials to connect to Opensearch @@ -2253,6 +2380,9 @@ spec: description: 'Path defines the REST API endpoint of Opensearch to post write requests (default: nil).' type: string + pipeline: + description: Optional, Pipeline name + type: string port: description: 'The port number of your Opensearch node (default: 9200).' @@ -2260,13 +2390,87 @@ spec: maximum: 65535 minimum: 1 type: integer + preferOjSerializer: + description: 'Optional, With default behavior, plugin uses + Yajl as JSON encoder/decoder. Set to true to use Oj (default: + false)' + type: boolean + reconnectOnError: + description: 'Optional, Indicates that the plugin should + reset connection on any error (reconnect on next send) + (default: false)' + type: boolean + reloadAfter: + description: 'Optional, When ReloadConnections true, this + is the integer number of operations after which the plugin + will reload the connections (default: 10000)' + format: int32 + type: integer + reloadConnections: + description: 'Optional, Automatically reload connection + after 10000 documents (default: true)' + type: boolean + reloadOnFailure: + description: 'Optional, Indicates that the opensearch-transport + will try to reload the nodes addresses if there is a failure + while making the request (default: false)' + type: boolean + removeKeys: + description: Optional, Remove specified keys from the event + record + type: string + removeKeysOnUpdate: + description: Optional, Remove keys when record is being + updated + type: string + requestTimeout: + description: 'Optional, HTTP request timeout in seconds + (default: 5s)' + pattern: ^\d+(s|m|h|d)$ + type: string scheme: description: 'Specify https if your Opensearch endpoint supports SSL (default: http).' type: string + selectorClassName: + description: Optional, Provide a selector class name + type: string + snifferClassName: + description: Optional, Provide a different sniffer class + name + type: string + sslMaxVersion: + description: Optional, Maximum SSL/TLS version + type: string + sslMinVersion: + description: Optional, Minimum SSL/TLS version + type: string sslVerify: description: Optional, Force certificate validation type: boolean + sslVersion: + description: 'Optional, You can specify SSL/TLS version + (default: TLSv1_2)' + type: string + suppressDocWrap: + description: 'Optional, Suppress doc_wrap (default: false)' + type: boolean + suppressTypeName: + description: 'Optional, Suppress ''[types removal]'' warnings + on OpenSearch 2.x (default: true for OS2+)' + type: boolean + tagKey: + description: 'Optional, Tag key name when include_tag_key + is true (default: tag)' + type: string + templateOverwrite: + description: 'Optional, Always update the template, even + if it already exists (default: false)' + type: boolean + useLegacyTemplate: + description: 'Optional, Use legacy template or not (default: + false for composable templates)' + type: boolean user: description: Optional, The login credentials to connect to Opensearch @@ -2302,6 +2506,18 @@ spec: x-kubernetes-map-type: atomic type: object type: object + utcIndex: + description: 'Optional, UTC index (default: false for local + time)' + type: boolean + verifyOsVersionAtStartup: + description: 'Optional, Validate OpenSearch version at startup + (default: true)' + type: boolean + writeOperation: + description: 'Optional, The write operation (default: index). + Options: index, create, update, upsert' + type: string type: object s3: description: out_s3 plugin diff --git a/charts/fluentd-crds/templates/fluentd.fluent.io_outputs.yaml b/charts/fluentd-crds/templates/fluentd.fluent.io_outputs.yaml index cbae4a559..760742120 100644 --- a/charts/fluentd-crds/templates/fluentd.fluent.io_outputs.yaml +++ b/charts/fluentd-crds/templates/fluentd.fluent.io_outputs.yaml @@ -2192,6 +2192,95 @@ spec: opensearch: description: out_opensearch plugin properties: + applicationName: + description: 'Optional, Specify the application name for + the rollover index to be created (default: default)' + type: string + bulkMessageRequestThreshold: + description: 'Optional, Configure bulk_message request splitting + threshold size (default: -1 unlimited)' + format: int32 + type: integer + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compressionLevel: + description: 'Optional, You can specify the compression + level (default: no_compression). Options: no_compression, + best_compression, best_speed, default_compression' + type: string + contentType: + description: 'Optional, With content_type application/x-ndjson, + plugin adds application/x-ndjson as Content-Type (default: + application/json)' + type: string + customHeaders: + description: Optional, Custom headers in Hash format + type: string + defaultOpensearchVersion: + description: 'Optional, Default OpenSearch version (default: + 1)' + format: int32 + type: integer + emitErrorForMissingId: + description: 'Optional, When write_operation is not index, + setting this true will cause plugin to emit_error_event + of records which do not include _id field (default: false)' + type: boolean + exceptionBackup: + description: 'Optional, Backup chunk when ignore exception + occurs (default: true)' + type: boolean + failOnDetectingOsVersionRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_get_os_version + is exceeded (default: true)' + type: boolean + failOnPuttingTemplateRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_putting_template + is exceeded (default: true)' + type: boolean host: description: 'The hostname of your Opensearch node (default: localhost).' @@ -2200,10 +2289,38 @@ spec: description: Hosts defines a list of hosts if you want to connect to more than one Openearch nodes type: string + httpBackend: + description: 'Optional, You can specify HTTP backend (default: + excon). Options: excon, typhoeus' + type: string + httpBackendExconNonblock: + description: 'Optional, With http_backend_excon_nonblock + false, plugin uses excon with nonblock=false (default: + true)' + type: boolean + idKey: + description: Optional, Record accessor syntax to specify + the field to use as _id in OpenSearch + type: string + ignoreExceptions: + description: Optional, List of exception classes to ignore + type: string + includeTagKey: + description: 'Optional, Include tag key in record (default: + false)' + type: boolean + indexDatePattern: + description: 'Optional, Specify the index date pattern for + creating a rollover index (default: now/d)' + type: string indexName: description: IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. type: string + logOs400Reason: + description: 'Optional, Enable logging of 400 reason without + enabling debug log level (default: false)' + type: boolean logstashFormat: description: 'If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option @@ -2214,6 +2331,16 @@ spec: index name to write events when logstash_format is true (default: logstash).' type: string + maxRetryGetOsVersion: + description: 'Optional, You can specify times of retry obtaining + OpenSearch version (default: 15)' + format: int32 + type: integer + maxRetryPuttingTemplate: + description: 'Optional, You can specify times of retry putting + template (default: 10)' + format: int32 + type: integer password: description: Optional, The login credentials to connect to Opensearch @@ -2253,6 +2380,9 @@ spec: description: 'Path defines the REST API endpoint of Opensearch to post write requests (default: nil).' type: string + pipeline: + description: Optional, Pipeline name + type: string port: description: 'The port number of your Opensearch node (default: 9200).' @@ -2260,13 +2390,87 @@ spec: maximum: 65535 minimum: 1 type: integer + preferOjSerializer: + description: 'Optional, With default behavior, plugin uses + Yajl as JSON encoder/decoder. Set to true to use Oj (default: + false)' + type: boolean + reconnectOnError: + description: 'Optional, Indicates that the plugin should + reset connection on any error (reconnect on next send) + (default: false)' + type: boolean + reloadAfter: + description: 'Optional, When ReloadConnections true, this + is the integer number of operations after which the plugin + will reload the connections (default: 10000)' + format: int32 + type: integer + reloadConnections: + description: 'Optional, Automatically reload connection + after 10000 documents (default: true)' + type: boolean + reloadOnFailure: + description: 'Optional, Indicates that the opensearch-transport + will try to reload the nodes addresses if there is a failure + while making the request (default: false)' + type: boolean + removeKeys: + description: Optional, Remove specified keys from the event + record + type: string + removeKeysOnUpdate: + description: Optional, Remove keys when record is being + updated + type: string + requestTimeout: + description: 'Optional, HTTP request timeout in seconds + (default: 5s)' + pattern: ^\d+(s|m|h|d)$ + type: string scheme: description: 'Specify https if your Opensearch endpoint supports SSL (default: http).' type: string + selectorClassName: + description: Optional, Provide a selector class name + type: string + snifferClassName: + description: Optional, Provide a different sniffer class + name + type: string + sslMaxVersion: + description: Optional, Maximum SSL/TLS version + type: string + sslMinVersion: + description: Optional, Minimum SSL/TLS version + type: string sslVerify: description: Optional, Force certificate validation type: boolean + sslVersion: + description: 'Optional, You can specify SSL/TLS version + (default: TLSv1_2)' + type: string + suppressDocWrap: + description: 'Optional, Suppress doc_wrap (default: false)' + type: boolean + suppressTypeName: + description: 'Optional, Suppress ''[types removal]'' warnings + on OpenSearch 2.x (default: true for OS2+)' + type: boolean + tagKey: + description: 'Optional, Tag key name when include_tag_key + is true (default: tag)' + type: string + templateOverwrite: + description: 'Optional, Always update the template, even + if it already exists (default: false)' + type: boolean + useLegacyTemplate: + description: 'Optional, Use legacy template or not (default: + false for composable templates)' + type: boolean user: description: Optional, The login credentials to connect to Opensearch @@ -2302,6 +2506,18 @@ spec: x-kubernetes-map-type: atomic type: object type: object + utcIndex: + description: 'Optional, UTC index (default: false for local + time)' + type: boolean + verifyOsVersionAtStartup: + description: 'Optional, Validate OpenSearch version at startup + (default: true)' + type: boolean + writeOperation: + description: 'Optional, The write operation (default: index). + Options: index, create, update, upsert' + type: string type: object s3: description: out_s3 plugin diff --git a/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml index 2a0447000..5d0e2ecb2 100644 --- a/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml @@ -2189,6 +2189,95 @@ spec: opensearch: description: out_opensearch plugin properties: + applicationName: + description: 'Optional, Specify the application name for + the rollover index to be created (default: default)' + type: string + bulkMessageRequestThreshold: + description: 'Optional, Configure bulk_message request splitting + threshold size (default: -1 unlimited)' + format: int32 + type: integer + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compressionLevel: + description: 'Optional, You can specify the compression + level (default: no_compression). Options: no_compression, + best_compression, best_speed, default_compression' + type: string + contentType: + description: 'Optional, With content_type application/x-ndjson, + plugin adds application/x-ndjson as Content-Type (default: + application/json)' + type: string + customHeaders: + description: Optional, Custom headers in Hash format + type: string + defaultOpensearchVersion: + description: 'Optional, Default OpenSearch version (default: + 1)' + format: int32 + type: integer + emitErrorForMissingId: + description: 'Optional, When write_operation is not index, + setting this true will cause plugin to emit_error_event + of records which do not include _id field (default: false)' + type: boolean + exceptionBackup: + description: 'Optional, Backup chunk when ignore exception + occurs (default: true)' + type: boolean + failOnDetectingOsVersionRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_get_os_version + is exceeded (default: true)' + type: boolean + failOnPuttingTemplateRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_putting_template + is exceeded (default: true)' + type: boolean host: description: 'The hostname of your Opensearch node (default: localhost).' @@ -2197,10 +2286,38 @@ spec: description: Hosts defines a list of hosts if you want to connect to more than one Openearch nodes type: string + httpBackend: + description: 'Optional, You can specify HTTP backend (default: + excon). Options: excon, typhoeus' + type: string + httpBackendExconNonblock: + description: 'Optional, With http_backend_excon_nonblock + false, plugin uses excon with nonblock=false (default: + true)' + type: boolean + idKey: + description: Optional, Record accessor syntax to specify + the field to use as _id in OpenSearch + type: string + ignoreExceptions: + description: Optional, List of exception classes to ignore + type: string + includeTagKey: + description: 'Optional, Include tag key in record (default: + false)' + type: boolean + indexDatePattern: + description: 'Optional, Specify the index date pattern for + creating a rollover index (default: now/d)' + type: string indexName: description: IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. type: string + logOs400Reason: + description: 'Optional, Enable logging of 400 reason without + enabling debug log level (default: false)' + type: boolean logstashFormat: description: 'If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option @@ -2211,6 +2328,16 @@ spec: index name to write events when logstash_format is true (default: logstash).' type: string + maxRetryGetOsVersion: + description: 'Optional, You can specify times of retry obtaining + OpenSearch version (default: 15)' + format: int32 + type: integer + maxRetryPuttingTemplate: + description: 'Optional, You can specify times of retry putting + template (default: 10)' + format: int32 + type: integer password: description: Optional, The login credentials to connect to Opensearch @@ -2250,6 +2377,9 @@ spec: description: 'Path defines the REST API endpoint of Opensearch to post write requests (default: nil).' type: string + pipeline: + description: Optional, Pipeline name + type: string port: description: 'The port number of your Opensearch node (default: 9200).' @@ -2257,13 +2387,87 @@ spec: maximum: 65535 minimum: 1 type: integer + preferOjSerializer: + description: 'Optional, With default behavior, plugin uses + Yajl as JSON encoder/decoder. Set to true to use Oj (default: + false)' + type: boolean + reconnectOnError: + description: 'Optional, Indicates that the plugin should + reset connection on any error (reconnect on next send) + (default: false)' + type: boolean + reloadAfter: + description: 'Optional, When ReloadConnections true, this + is the integer number of operations after which the plugin + will reload the connections (default: 10000)' + format: int32 + type: integer + reloadConnections: + description: 'Optional, Automatically reload connection + after 10000 documents (default: true)' + type: boolean + reloadOnFailure: + description: 'Optional, Indicates that the opensearch-transport + will try to reload the nodes addresses if there is a failure + while making the request (default: false)' + type: boolean + removeKeys: + description: Optional, Remove specified keys from the event + record + type: string + removeKeysOnUpdate: + description: Optional, Remove keys when record is being + updated + type: string + requestTimeout: + description: 'Optional, HTTP request timeout in seconds + (default: 5s)' + pattern: ^\d+(s|m|h|d)$ + type: string scheme: description: 'Specify https if your Opensearch endpoint supports SSL (default: http).' type: string + selectorClassName: + description: Optional, Provide a selector class name + type: string + snifferClassName: + description: Optional, Provide a different sniffer class + name + type: string + sslMaxVersion: + description: Optional, Maximum SSL/TLS version + type: string + sslMinVersion: + description: Optional, Minimum SSL/TLS version + type: string sslVerify: description: Optional, Force certificate validation type: boolean + sslVersion: + description: 'Optional, You can specify SSL/TLS version + (default: TLSv1_2)' + type: string + suppressDocWrap: + description: 'Optional, Suppress doc_wrap (default: false)' + type: boolean + suppressTypeName: + description: 'Optional, Suppress ''[types removal]'' warnings + on OpenSearch 2.x (default: true for OS2+)' + type: boolean + tagKey: + description: 'Optional, Tag key name when include_tag_key + is true (default: tag)' + type: string + templateOverwrite: + description: 'Optional, Always update the template, even + if it already exists (default: false)' + type: boolean + useLegacyTemplate: + description: 'Optional, Use legacy template or not (default: + false for composable templates)' + type: boolean user: description: Optional, The login credentials to connect to Opensearch @@ -2299,6 +2503,18 @@ spec: x-kubernetes-map-type: atomic type: object type: object + utcIndex: + description: 'Optional, UTC index (default: false for local + time)' + type: boolean + verifyOsVersionAtStartup: + description: 'Optional, Validate OpenSearch version at startup + (default: true)' + type: boolean + writeOperation: + description: 'Optional, The write operation (default: index). + Options: index, create, update, upsert' + type: string type: object s3: description: out_s3 plugin diff --git a/config/crd/bases/fluentd.fluent.io_outputs.yaml b/config/crd/bases/fluentd.fluent.io_outputs.yaml index f7c023b33..73b4ac5b7 100644 --- a/config/crd/bases/fluentd.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentd.fluent.io_outputs.yaml @@ -2189,6 +2189,95 @@ spec: opensearch: description: out_opensearch plugin properties: + applicationName: + description: 'Optional, Specify the application name for + the rollover index to be created (default: default)' + type: string + bulkMessageRequestThreshold: + description: 'Optional, Configure bulk_message request splitting + threshold size (default: -1 unlimited)' + format: int32 + type: integer + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compressionLevel: + description: 'Optional, You can specify the compression + level (default: no_compression). Options: no_compression, + best_compression, best_speed, default_compression' + type: string + contentType: + description: 'Optional, With content_type application/x-ndjson, + plugin adds application/x-ndjson as Content-Type (default: + application/json)' + type: string + customHeaders: + description: Optional, Custom headers in Hash format + type: string + defaultOpensearchVersion: + description: 'Optional, Default OpenSearch version (default: + 1)' + format: int32 + type: integer + emitErrorForMissingId: + description: 'Optional, When write_operation is not index, + setting this true will cause plugin to emit_error_event + of records which do not include _id field (default: false)' + type: boolean + exceptionBackup: + description: 'Optional, Backup chunk when ignore exception + occurs (default: true)' + type: boolean + failOnDetectingOsVersionRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_get_os_version + is exceeded (default: true)' + type: boolean + failOnPuttingTemplateRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_putting_template + is exceeded (default: true)' + type: boolean host: description: 'The hostname of your Opensearch node (default: localhost).' @@ -2197,10 +2286,38 @@ spec: description: Hosts defines a list of hosts if you want to connect to more than one Openearch nodes type: string + httpBackend: + description: 'Optional, You can specify HTTP backend (default: + excon). Options: excon, typhoeus' + type: string + httpBackendExconNonblock: + description: 'Optional, With http_backend_excon_nonblock + false, plugin uses excon with nonblock=false (default: + true)' + type: boolean + idKey: + description: Optional, Record accessor syntax to specify + the field to use as _id in OpenSearch + type: string + ignoreExceptions: + description: Optional, List of exception classes to ignore + type: string + includeTagKey: + description: 'Optional, Include tag key in record (default: + false)' + type: boolean + indexDatePattern: + description: 'Optional, Specify the index date pattern for + creating a rollover index (default: now/d)' + type: string indexName: description: IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. type: string + logOs400Reason: + description: 'Optional, Enable logging of 400 reason without + enabling debug log level (default: false)' + type: boolean logstashFormat: description: 'If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option @@ -2211,6 +2328,16 @@ spec: index name to write events when logstash_format is true (default: logstash).' type: string + maxRetryGetOsVersion: + description: 'Optional, You can specify times of retry obtaining + OpenSearch version (default: 15)' + format: int32 + type: integer + maxRetryPuttingTemplate: + description: 'Optional, You can specify times of retry putting + template (default: 10)' + format: int32 + type: integer password: description: Optional, The login credentials to connect to Opensearch @@ -2250,6 +2377,9 @@ spec: description: 'Path defines the REST API endpoint of Opensearch to post write requests (default: nil).' type: string + pipeline: + description: Optional, Pipeline name + type: string port: description: 'The port number of your Opensearch node (default: 9200).' @@ -2257,13 +2387,87 @@ spec: maximum: 65535 minimum: 1 type: integer + preferOjSerializer: + description: 'Optional, With default behavior, plugin uses + Yajl as JSON encoder/decoder. Set to true to use Oj (default: + false)' + type: boolean + reconnectOnError: + description: 'Optional, Indicates that the plugin should + reset connection on any error (reconnect on next send) + (default: false)' + type: boolean + reloadAfter: + description: 'Optional, When ReloadConnections true, this + is the integer number of operations after which the plugin + will reload the connections (default: 10000)' + format: int32 + type: integer + reloadConnections: + description: 'Optional, Automatically reload connection + after 10000 documents (default: true)' + type: boolean + reloadOnFailure: + description: 'Optional, Indicates that the opensearch-transport + will try to reload the nodes addresses if there is a failure + while making the request (default: false)' + type: boolean + removeKeys: + description: Optional, Remove specified keys from the event + record + type: string + removeKeysOnUpdate: + description: Optional, Remove keys when record is being + updated + type: string + requestTimeout: + description: 'Optional, HTTP request timeout in seconds + (default: 5s)' + pattern: ^\d+(s|m|h|d)$ + type: string scheme: description: 'Specify https if your Opensearch endpoint supports SSL (default: http).' type: string + selectorClassName: + description: Optional, Provide a selector class name + type: string + snifferClassName: + description: Optional, Provide a different sniffer class + name + type: string + sslMaxVersion: + description: Optional, Maximum SSL/TLS version + type: string + sslMinVersion: + description: Optional, Minimum SSL/TLS version + type: string sslVerify: description: Optional, Force certificate validation type: boolean + sslVersion: + description: 'Optional, You can specify SSL/TLS version + (default: TLSv1_2)' + type: string + suppressDocWrap: + description: 'Optional, Suppress doc_wrap (default: false)' + type: boolean + suppressTypeName: + description: 'Optional, Suppress ''[types removal]'' warnings + on OpenSearch 2.x (default: true for OS2+)' + type: boolean + tagKey: + description: 'Optional, Tag key name when include_tag_key + is true (default: tag)' + type: string + templateOverwrite: + description: 'Optional, Always update the template, even + if it already exists (default: false)' + type: boolean + useLegacyTemplate: + description: 'Optional, Use legacy template or not (default: + false for composable templates)' + type: boolean user: description: Optional, The login credentials to connect to Opensearch @@ -2299,6 +2503,18 @@ spec: x-kubernetes-map-type: atomic type: object type: object + utcIndex: + description: 'Optional, UTC index (default: false for local + time)' + type: boolean + verifyOsVersionAtStartup: + description: 'Optional, Validate OpenSearch version at startup + (default: true)' + type: boolean + writeOperation: + description: 'Optional, The write operation (default: index). + Options: index, create, update, upsert' + type: string type: object s3: description: out_s3 plugin diff --git a/docs/plugins/fluentd/output/opensearch.md b/docs/plugins/fluentd/output/opensearch.md index a90e09a6a..ece03660a 100644 --- a/docs/plugins/fluentd/output/opensearch.md +++ b/docs/plugins/fluentd/output/opensearch.md @@ -2,6 +2,9 @@ Opensearch defines the parameters for out_opensearch plugin +## Configuration Parameters + +### Basic Connection | Field | Description | Scheme | | ----- | ----------- | ------ | @@ -10,9 +13,206 @@ Opensearch defines the parameters for out_opensearch plugin | hosts | Hosts defines a list of hosts if you want to connect to more than one Openearch nodes | *string | | scheme | Specify https if your Opensearch endpoint supports SSL (default: http). | *string | | path | Path defines the REST API endpoint of Opensearch to post write requests (default: nil). | *string | + +### Authentication + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| user | Optional, The login credentials to connect to Opensearch | *[plugins.Secret](../secret.md) | +| password | Optional, The login credentials to connect to Opensearch | *[plugins.Secret](../secret.md) | + +### Index Configuration + +| Field | Description | Scheme | +| ----- | ----------- | ------ | | indexName | IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. | *string | | logstashFormat | If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option supersedes the index_name option. | *bool | | logstashPrefix | LogstashPrefix defines the logstash prefix index name to write events when logstash_format is true (default: logstash). | *string | -| user | Optional, The login credentials to connect to Opensearch | *[plugins.Secret](../secret.md) | -| password | Optional, The login credentials to connect to Opensearch | *[plugins.Secret](../secret.md) | + +### SSL/TLS Configuration + +| Field | Description | Scheme | +| ----- | ----------- | ------ | | sslVerify | Optional, Force certificate validation | *bool | +| caFile | Optional, Absolute path to CA certificate file | *string | +| clientCert | Optional, Absolute path to client Certificate file | *string | +| clientKey | Optional, Absolute path to client private Key file | *string | +| clientKeyPassword | Optional, password for ClientKey file | *[plugins.Secret](../secret.md) | +| sslVersion | Optional, You can specify SSL/TLS version (default: TLSv1_2) | *string | +| sslMinVersion | Optional, Minimum SSL/TLS version | *string | +| sslMaxVersion | Optional, Maximum SSL/TLS version | *string | + +### Debugging & Error Handling + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| logOs400Reason | Optional, Enable logging of 400 reason without enabling debug log level (default: false). **Critical for troubleshooting 400 errors!** | *bool | +| reconnectOnError | Optional, Indicates that the plugin should reset connection on any error (reconnect on next send) (default: false) | *bool | +| ignoreExceptions | Optional, List of exception classes to ignore | *string | +| exceptionBackup | Optional, Backup chunk when ignore exception occurs (default: true) | *bool | + +### Connection Management + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| requestTimeout | Optional, HTTP request timeout in seconds (default: 5s) | *string | +| reloadConnections | Optional, Automatically reload connection after 10000 documents (default: true) | *bool | +| reloadAfter | Optional, When ReloadConnections true, this is the integer number of operations after which the plugin will reload the connections (default: 10000) | *uint32 | +| reloadOnFailure | Optional, Indicates that the opensearch-transport will try to reload the nodes addresses if there is a failure while making the request (default: false) | *bool | + +### Performance Tuning + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| compressionLevel | Optional, You can specify the compression level (default: no_compression). Options: no_compression, best_compression, best_speed, default_compression | *string | +| httpBackend | Optional, You can specify HTTP backend (default: excon). Options: excon, typhoeus | *string | +| httpBackendExconNonblock | Optional, With http_backend_excon_nonblock false, plugin uses excon with nonblock=false (default: true) | *bool | +| preferOjSerializer | Optional, With default behavior, plugin uses Yajl as JSON encoder/decoder. Set to true to use Oj (default: false) | *bool | +| bulkMessageRequestThreshold | Optional, Configure bulk_message request splitting threshold size (default: -1 unlimited) | *int32 | + +### Record Handling + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| includeTagKey | Optional, Include tag key in record (default: false) | *bool | +| tagKey | Optional, Tag key name when include_tag_key is true (default: tag) | *string | +| idKey | Optional, Record accessor syntax to specify the field to use as _id in OpenSearch | *string | +| removeKeys | Optional, Remove specified keys from the event record | *string | +| removeKeysOnUpdate | Optional, Remove keys when record is being updated | *string | +| writeOperation | Optional, The write operation (default: index). Options: index, create, update, upsert | *string | +| emitErrorForMissingId | Optional, When write_operation is not index, setting this true will cause plugin to emit_error_event of records which do not include _id field (default: false) | *bool | + +### Template & Version Management + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| templateOverwrite | Optional, Always update the template, even if it already exists (default: false) | *bool | +| maxRetryPuttingTemplate | Optional, You can specify times of retry putting template (default: 10) | *uint32 | +| failOnPuttingTemplateRetryExceed | Optional, Indicates whether to fail when max_retry_putting_template is exceeded (default: true) | *bool | +| verifyOsVersionAtStartup | Optional, Validate OpenSearch version at startup (default: true) | *bool | +| maxRetryGetOsVersion | Optional, You can specify times of retry obtaining OpenSearch version (default: 15) | *uint32 | +| failOnDetectingOsVersionRetryExceed | Optional, Indicates whether to fail when max_retry_get_os_version is exceeded (default: true) | *bool | +| defaultOpensearchVersion | Optional, Default OpenSearch version (default: 1) | *uint32 | +| applicationName | Optional, Specify the application name for the rollover index to be created (default: default) | *string | +| indexDatePattern | Optional, Specify the index date pattern for creating a rollover index (default: now/d) | *string | +| useLegacyTemplate | Optional, Use legacy template or not (default: false for composable templates) | *bool | + +### Advanced Options + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| suppressTypeName | Optional, Suppress '[types removal]' warnings on OpenSearch 2.x (default: true for OS2+) | *bool | +| contentType | Optional, With content_type application/x-ndjson, plugin adds application/x-ndjson as Content-Type (default: application/json) | *string | +| customHeaders | Optional, Custom headers in Hash format | *string | +| pipeline | Optional, Pipeline name | *string | +| utcIndex | Optional, UTC index (default: false for local time) | *bool | +| suppressDocWrap | Optional, Suppress doc_wrap (default: false) | *bool | +| snifferClassName | Optional, Provide a different sniffer class name | *string | +| selectorClassName | Optional, Provide a selector class name | *string | + +## Examples + +### Basic Configuration + +```yaml +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterOutput +metadata: + name: opensearch-basic +spec: + outputs: + - opensearch: + host: opensearch.logging.svc.cluster.local + port: 9200 + logstashFormat: true + logstashPrefix: app-logs +``` + +### Production Configuration + +```yaml +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterOutput +metadata: + name: opensearch-production +spec: + outputs: + - opensearch: + # Connection + hosts: "opensearch-node1:9200,opensearch-node2:9200,opensearch-node3:9200" + scheme: https + + # Authentication + user: + valueFrom: + secretKeyRef: + name: opensearch-creds + key: username + password: + valueFrom: + secretKeyRef: + name: opensearch-creds + key: password + + # SSL/TLS with mutual authentication + sslVerify: true + caFile: /etc/ssl/opensearch/ca.crt + clientCert: /etc/ssl/opensearch/client.crt + clientKey: /etc/ssl/opensearch/client.key + sslMinVersion: TLSv1_2 + sslMaxVersion: TLSv1_3 + + # Index configuration + logstashFormat: true + logstashPrefix: kubernetes-logs + + # CRITICAL: Enable 400 error debugging + logOs400Reason: true + + # Reliability + reconnectOnError: true + requestTimeout: 30s + reloadConnections: true + reloadOnFailure: true + + # Performance + compressionLevel: best_speed + bulkMessageRequestThreshold: 20971520 # 20MB + + # Document handling + includeTagKey: true + idKey: kubernetes.pod_id + writeOperation: upsert +``` + +### Development/Debug Configuration + +```yaml +apiVersion: fluentd.fluent.io/v1alpha1 +kind: ClusterOutput +metadata: + name: opensearch-debug +spec: + outputs: + - opensearch: + host: opensearch-dev.local + port: 9200 + scheme: http + + # Enable detailed error logging + logOs400Reason: true + + # Relaxed SSL for development + sslVerify: false + + # Simple index + indexName: dev-logs + + # Debug helpers + includeTagKey: true + suppressDocWrap: false + + # Don't fail on errors during development + failOnPuttingTemplateRetryExceed: false + failOnDetectingOsVersionRetryExceed: false +``` diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 1d4f2e8f7..97007466c 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -10720,6 +10720,95 @@ spec: opensearch: description: out_opensearch plugin properties: + applicationName: + description: 'Optional, Specify the application name for + the rollover index to be created (default: default)' + type: string + bulkMessageRequestThreshold: + description: 'Optional, Configure bulk_message request splitting + threshold size (default: -1 unlimited)' + format: int32 + type: integer + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compressionLevel: + description: 'Optional, You can specify the compression + level (default: no_compression). Options: no_compression, + best_compression, best_speed, default_compression' + type: string + contentType: + description: 'Optional, With content_type application/x-ndjson, + plugin adds application/x-ndjson as Content-Type (default: + application/json)' + type: string + customHeaders: + description: Optional, Custom headers in Hash format + type: string + defaultOpensearchVersion: + description: 'Optional, Default OpenSearch version (default: + 1)' + format: int32 + type: integer + emitErrorForMissingId: + description: 'Optional, When write_operation is not index, + setting this true will cause plugin to emit_error_event + of records which do not include _id field (default: false)' + type: boolean + exceptionBackup: + description: 'Optional, Backup chunk when ignore exception + occurs (default: true)' + type: boolean + failOnDetectingOsVersionRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_get_os_version + is exceeded (default: true)' + type: boolean + failOnPuttingTemplateRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_putting_template + is exceeded (default: true)' + type: boolean host: description: 'The hostname of your Opensearch node (default: localhost).' @@ -10728,10 +10817,38 @@ spec: description: Hosts defines a list of hosts if you want to connect to more than one Openearch nodes type: string + httpBackend: + description: 'Optional, You can specify HTTP backend (default: + excon). Options: excon, typhoeus' + type: string + httpBackendExconNonblock: + description: 'Optional, With http_backend_excon_nonblock + false, plugin uses excon with nonblock=false (default: + true)' + type: boolean + idKey: + description: Optional, Record accessor syntax to specify + the field to use as _id in OpenSearch + type: string + ignoreExceptions: + description: Optional, List of exception classes to ignore + type: string + includeTagKey: + description: 'Optional, Include tag key in record (default: + false)' + type: boolean + indexDatePattern: + description: 'Optional, Specify the index date pattern for + creating a rollover index (default: now/d)' + type: string indexName: description: IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. type: string + logOs400Reason: + description: 'Optional, Enable logging of 400 reason without + enabling debug log level (default: false)' + type: boolean logstashFormat: description: 'If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option @@ -10742,6 +10859,16 @@ spec: index name to write events when logstash_format is true (default: logstash).' type: string + maxRetryGetOsVersion: + description: 'Optional, You can specify times of retry obtaining + OpenSearch version (default: 15)' + format: int32 + type: integer + maxRetryPuttingTemplate: + description: 'Optional, You can specify times of retry putting + template (default: 10)' + format: int32 + type: integer password: description: Optional, The login credentials to connect to Opensearch @@ -10781,6 +10908,9 @@ spec: description: 'Path defines the REST API endpoint of Opensearch to post write requests (default: nil).' type: string + pipeline: + description: Optional, Pipeline name + type: string port: description: 'The port number of your Opensearch node (default: 9200).' @@ -10788,13 +10918,87 @@ spec: maximum: 65535 minimum: 1 type: integer + preferOjSerializer: + description: 'Optional, With default behavior, plugin uses + Yajl as JSON encoder/decoder. Set to true to use Oj (default: + false)' + type: boolean + reconnectOnError: + description: 'Optional, Indicates that the plugin should + reset connection on any error (reconnect on next send) + (default: false)' + type: boolean + reloadAfter: + description: 'Optional, When ReloadConnections true, this + is the integer number of operations after which the plugin + will reload the connections (default: 10000)' + format: int32 + type: integer + reloadConnections: + description: 'Optional, Automatically reload connection + after 10000 documents (default: true)' + type: boolean + reloadOnFailure: + description: 'Optional, Indicates that the opensearch-transport + will try to reload the nodes addresses if there is a failure + while making the request (default: false)' + type: boolean + removeKeys: + description: Optional, Remove specified keys from the event + record + type: string + removeKeysOnUpdate: + description: Optional, Remove keys when record is being + updated + type: string + requestTimeout: + description: 'Optional, HTTP request timeout in seconds + (default: 5s)' + pattern: ^\d+(s|m|h|d)$ + type: string scheme: description: 'Specify https if your Opensearch endpoint supports SSL (default: http).' type: string + selectorClassName: + description: Optional, Provide a selector class name + type: string + snifferClassName: + description: Optional, Provide a different sniffer class + name + type: string + sslMaxVersion: + description: Optional, Maximum SSL/TLS version + type: string + sslMinVersion: + description: Optional, Minimum SSL/TLS version + type: string sslVerify: description: Optional, Force certificate validation type: boolean + sslVersion: + description: 'Optional, You can specify SSL/TLS version + (default: TLSv1_2)' + type: string + suppressDocWrap: + description: 'Optional, Suppress doc_wrap (default: false)' + type: boolean + suppressTypeName: + description: 'Optional, Suppress ''[types removal]'' warnings + on OpenSearch 2.x (default: true for OS2+)' + type: boolean + tagKey: + description: 'Optional, Tag key name when include_tag_key + is true (default: tag)' + type: string + templateOverwrite: + description: 'Optional, Always update the template, even + if it already exists (default: false)' + type: boolean + useLegacyTemplate: + description: 'Optional, Use legacy template or not (default: + false for composable templates)' + type: boolean user: description: Optional, The login credentials to connect to Opensearch @@ -10830,6 +11034,18 @@ spec: x-kubernetes-map-type: atomic type: object type: object + utcIndex: + description: 'Optional, UTC index (default: false for local + time)' + type: boolean + verifyOsVersionAtStartup: + description: 'Optional, Validate OpenSearch version at startup + (default: true)' + type: boolean + writeOperation: + description: 'Optional, The write operation (default: index). + Options: index, create, update, upsert' + type: string type: object s3: description: out_s3 plugin @@ -40527,6 +40743,95 @@ spec: opensearch: description: out_opensearch plugin properties: + applicationName: + description: 'Optional, Specify the application name for + the rollover index to be created (default: default)' + type: string + bulkMessageRequestThreshold: + description: 'Optional, Configure bulk_message request splitting + threshold size (default: -1 unlimited)' + format: int32 + type: integer + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compressionLevel: + description: 'Optional, You can specify the compression + level (default: no_compression). Options: no_compression, + best_compression, best_speed, default_compression' + type: string + contentType: + description: 'Optional, With content_type application/x-ndjson, + plugin adds application/x-ndjson as Content-Type (default: + application/json)' + type: string + customHeaders: + description: Optional, Custom headers in Hash format + type: string + defaultOpensearchVersion: + description: 'Optional, Default OpenSearch version (default: + 1)' + format: int32 + type: integer + emitErrorForMissingId: + description: 'Optional, When write_operation is not index, + setting this true will cause plugin to emit_error_event + of records which do not include _id field (default: false)' + type: boolean + exceptionBackup: + description: 'Optional, Backup chunk when ignore exception + occurs (default: true)' + type: boolean + failOnDetectingOsVersionRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_get_os_version + is exceeded (default: true)' + type: boolean + failOnPuttingTemplateRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_putting_template + is exceeded (default: true)' + type: boolean host: description: 'The hostname of your Opensearch node (default: localhost).' @@ -40535,10 +40840,38 @@ spec: description: Hosts defines a list of hosts if you want to connect to more than one Openearch nodes type: string + httpBackend: + description: 'Optional, You can specify HTTP backend (default: + excon). Options: excon, typhoeus' + type: string + httpBackendExconNonblock: + description: 'Optional, With http_backend_excon_nonblock + false, plugin uses excon with nonblock=false (default: + true)' + type: boolean + idKey: + description: Optional, Record accessor syntax to specify + the field to use as _id in OpenSearch + type: string + ignoreExceptions: + description: Optional, List of exception classes to ignore + type: string + includeTagKey: + description: 'Optional, Include tag key in record (default: + false)' + type: boolean + indexDatePattern: + description: 'Optional, Specify the index date pattern for + creating a rollover index (default: now/d)' + type: string indexName: description: IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. type: string + logOs400Reason: + description: 'Optional, Enable logging of 400 reason without + enabling debug log level (default: false)' + type: boolean logstashFormat: description: 'If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option @@ -40549,6 +40882,16 @@ spec: index name to write events when logstash_format is true (default: logstash).' type: string + maxRetryGetOsVersion: + description: 'Optional, You can specify times of retry obtaining + OpenSearch version (default: 15)' + format: int32 + type: integer + maxRetryPuttingTemplate: + description: 'Optional, You can specify times of retry putting + template (default: 10)' + format: int32 + type: integer password: description: Optional, The login credentials to connect to Opensearch @@ -40588,6 +40931,9 @@ spec: description: 'Path defines the REST API endpoint of Opensearch to post write requests (default: nil).' type: string + pipeline: + description: Optional, Pipeline name + type: string port: description: 'The port number of your Opensearch node (default: 9200).' @@ -40595,13 +40941,87 @@ spec: maximum: 65535 minimum: 1 type: integer + preferOjSerializer: + description: 'Optional, With default behavior, plugin uses + Yajl as JSON encoder/decoder. Set to true to use Oj (default: + false)' + type: boolean + reconnectOnError: + description: 'Optional, Indicates that the plugin should + reset connection on any error (reconnect on next send) + (default: false)' + type: boolean + reloadAfter: + description: 'Optional, When ReloadConnections true, this + is the integer number of operations after which the plugin + will reload the connections (default: 10000)' + format: int32 + type: integer + reloadConnections: + description: 'Optional, Automatically reload connection + after 10000 documents (default: true)' + type: boolean + reloadOnFailure: + description: 'Optional, Indicates that the opensearch-transport + will try to reload the nodes addresses if there is a failure + while making the request (default: false)' + type: boolean + removeKeys: + description: Optional, Remove specified keys from the event + record + type: string + removeKeysOnUpdate: + description: Optional, Remove keys when record is being + updated + type: string + requestTimeout: + description: 'Optional, HTTP request timeout in seconds + (default: 5s)' + pattern: ^\d+(s|m|h|d)$ + type: string scheme: description: 'Specify https if your Opensearch endpoint supports SSL (default: http).' type: string + selectorClassName: + description: Optional, Provide a selector class name + type: string + snifferClassName: + description: Optional, Provide a different sniffer class + name + type: string + sslMaxVersion: + description: Optional, Maximum SSL/TLS version + type: string + sslMinVersion: + description: Optional, Minimum SSL/TLS version + type: string sslVerify: description: Optional, Force certificate validation type: boolean + sslVersion: + description: 'Optional, You can specify SSL/TLS version + (default: TLSv1_2)' + type: string + suppressDocWrap: + description: 'Optional, Suppress doc_wrap (default: false)' + type: boolean + suppressTypeName: + description: 'Optional, Suppress ''[types removal]'' warnings + on OpenSearch 2.x (default: true for OS2+)' + type: boolean + tagKey: + description: 'Optional, Tag key name when include_tag_key + is true (default: tag)' + type: string + templateOverwrite: + description: 'Optional, Always update the template, even + if it already exists (default: false)' + type: boolean + useLegacyTemplate: + description: 'Optional, Use legacy template or not (default: + false for composable templates)' + type: boolean user: description: Optional, The login credentials to connect to Opensearch @@ -40637,6 +41057,18 @@ spec: x-kubernetes-map-type: atomic type: object type: object + utcIndex: + description: 'Optional, UTC index (default: false for local + time)' + type: boolean + verifyOsVersionAtStartup: + description: 'Optional, Validate OpenSearch version at startup + (default: true)' + type: boolean + writeOperation: + description: 'Optional, The write operation (default: index). + Options: index, create, update, upsert' + type: string type: object s3: description: out_s3 plugin diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 291d4bb56..b8e8cdc95 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -10720,6 +10720,95 @@ spec: opensearch: description: out_opensearch plugin properties: + applicationName: + description: 'Optional, Specify the application name for + the rollover index to be created (default: default)' + type: string + bulkMessageRequestThreshold: + description: 'Optional, Configure bulk_message request splitting + threshold size (default: -1 unlimited)' + format: int32 + type: integer + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compressionLevel: + description: 'Optional, You can specify the compression + level (default: no_compression). Options: no_compression, + best_compression, best_speed, default_compression' + type: string + contentType: + description: 'Optional, With content_type application/x-ndjson, + plugin adds application/x-ndjson as Content-Type (default: + application/json)' + type: string + customHeaders: + description: Optional, Custom headers in Hash format + type: string + defaultOpensearchVersion: + description: 'Optional, Default OpenSearch version (default: + 1)' + format: int32 + type: integer + emitErrorForMissingId: + description: 'Optional, When write_operation is not index, + setting this true will cause plugin to emit_error_event + of records which do not include _id field (default: false)' + type: boolean + exceptionBackup: + description: 'Optional, Backup chunk when ignore exception + occurs (default: true)' + type: boolean + failOnDetectingOsVersionRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_get_os_version + is exceeded (default: true)' + type: boolean + failOnPuttingTemplateRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_putting_template + is exceeded (default: true)' + type: boolean host: description: 'The hostname of your Opensearch node (default: localhost).' @@ -10728,10 +10817,38 @@ spec: description: Hosts defines a list of hosts if you want to connect to more than one Openearch nodes type: string + httpBackend: + description: 'Optional, You can specify HTTP backend (default: + excon). Options: excon, typhoeus' + type: string + httpBackendExconNonblock: + description: 'Optional, With http_backend_excon_nonblock + false, plugin uses excon with nonblock=false (default: + true)' + type: boolean + idKey: + description: Optional, Record accessor syntax to specify + the field to use as _id in OpenSearch + type: string + ignoreExceptions: + description: Optional, List of exception classes to ignore + type: string + includeTagKey: + description: 'Optional, Include tag key in record (default: + false)' + type: boolean + indexDatePattern: + description: 'Optional, Specify the index date pattern for + creating a rollover index (default: now/d)' + type: string indexName: description: IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. type: string + logOs400Reason: + description: 'Optional, Enable logging of 400 reason without + enabling debug log level (default: false)' + type: boolean logstashFormat: description: 'If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option @@ -10742,6 +10859,16 @@ spec: index name to write events when logstash_format is true (default: logstash).' type: string + maxRetryGetOsVersion: + description: 'Optional, You can specify times of retry obtaining + OpenSearch version (default: 15)' + format: int32 + type: integer + maxRetryPuttingTemplate: + description: 'Optional, You can specify times of retry putting + template (default: 10)' + format: int32 + type: integer password: description: Optional, The login credentials to connect to Opensearch @@ -10781,6 +10908,9 @@ spec: description: 'Path defines the REST API endpoint of Opensearch to post write requests (default: nil).' type: string + pipeline: + description: Optional, Pipeline name + type: string port: description: 'The port number of your Opensearch node (default: 9200).' @@ -10788,13 +10918,87 @@ spec: maximum: 65535 minimum: 1 type: integer + preferOjSerializer: + description: 'Optional, With default behavior, plugin uses + Yajl as JSON encoder/decoder. Set to true to use Oj (default: + false)' + type: boolean + reconnectOnError: + description: 'Optional, Indicates that the plugin should + reset connection on any error (reconnect on next send) + (default: false)' + type: boolean + reloadAfter: + description: 'Optional, When ReloadConnections true, this + is the integer number of operations after which the plugin + will reload the connections (default: 10000)' + format: int32 + type: integer + reloadConnections: + description: 'Optional, Automatically reload connection + after 10000 documents (default: true)' + type: boolean + reloadOnFailure: + description: 'Optional, Indicates that the opensearch-transport + will try to reload the nodes addresses if there is a failure + while making the request (default: false)' + type: boolean + removeKeys: + description: Optional, Remove specified keys from the event + record + type: string + removeKeysOnUpdate: + description: Optional, Remove keys when record is being + updated + type: string + requestTimeout: + description: 'Optional, HTTP request timeout in seconds + (default: 5s)' + pattern: ^\d+(s|m|h|d)$ + type: string scheme: description: 'Specify https if your Opensearch endpoint supports SSL (default: http).' type: string + selectorClassName: + description: Optional, Provide a selector class name + type: string + snifferClassName: + description: Optional, Provide a different sniffer class + name + type: string + sslMaxVersion: + description: Optional, Maximum SSL/TLS version + type: string + sslMinVersion: + description: Optional, Minimum SSL/TLS version + type: string sslVerify: description: Optional, Force certificate validation type: boolean + sslVersion: + description: 'Optional, You can specify SSL/TLS version + (default: TLSv1_2)' + type: string + suppressDocWrap: + description: 'Optional, Suppress doc_wrap (default: false)' + type: boolean + suppressTypeName: + description: 'Optional, Suppress ''[types removal]'' warnings + on OpenSearch 2.x (default: true for OS2+)' + type: boolean + tagKey: + description: 'Optional, Tag key name when include_tag_key + is true (default: tag)' + type: string + templateOverwrite: + description: 'Optional, Always update the template, even + if it already exists (default: false)' + type: boolean + useLegacyTemplate: + description: 'Optional, Use legacy template or not (default: + false for composable templates)' + type: boolean user: description: Optional, The login credentials to connect to Opensearch @@ -10830,6 +11034,18 @@ spec: x-kubernetes-map-type: atomic type: object type: object + utcIndex: + description: 'Optional, UTC index (default: false for local + time)' + type: boolean + verifyOsVersionAtStartup: + description: 'Optional, Validate OpenSearch version at startup + (default: true)' + type: boolean + writeOperation: + description: 'Optional, The write operation (default: index). + Options: index, create, update, upsert' + type: string type: object s3: description: out_s3 plugin @@ -40527,6 +40743,95 @@ spec: opensearch: description: out_opensearch plugin properties: + applicationName: + description: 'Optional, Specify the application name for + the rollover index to be created (default: default)' + type: string + bulkMessageRequestThreshold: + description: 'Optional, Configure bulk_message request splitting + threshold size (default: -1 unlimited)' + format: int32 + type: integer + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + compressionLevel: + description: 'Optional, You can specify the compression + level (default: no_compression). Options: no_compression, + best_compression, best_speed, default_compression' + type: string + contentType: + description: 'Optional, With content_type application/x-ndjson, + plugin adds application/x-ndjson as Content-Type (default: + application/json)' + type: string + customHeaders: + description: Optional, Custom headers in Hash format + type: string + defaultOpensearchVersion: + description: 'Optional, Default OpenSearch version (default: + 1)' + format: int32 + type: integer + emitErrorForMissingId: + description: 'Optional, When write_operation is not index, + setting this true will cause plugin to emit_error_event + of records which do not include _id field (default: false)' + type: boolean + exceptionBackup: + description: 'Optional, Backup chunk when ignore exception + occurs (default: true)' + type: boolean + failOnDetectingOsVersionRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_get_os_version + is exceeded (default: true)' + type: boolean + failOnPuttingTemplateRetryExceed: + description: 'Optional, Indicates whether to fail when max_retry_putting_template + is exceeded (default: true)' + type: boolean host: description: 'The hostname of your Opensearch node (default: localhost).' @@ -40535,10 +40840,38 @@ spec: description: Hosts defines a list of hosts if you want to connect to more than one Openearch nodes type: string + httpBackend: + description: 'Optional, You can specify HTTP backend (default: + excon). Options: excon, typhoeus' + type: string + httpBackendExconNonblock: + description: 'Optional, With http_backend_excon_nonblock + false, plugin uses excon with nonblock=false (default: + true)' + type: boolean + idKey: + description: Optional, Record accessor syntax to specify + the field to use as _id in OpenSearch + type: string + ignoreExceptions: + description: Optional, List of exception classes to ignore + type: string + includeTagKey: + description: 'Optional, Include tag key in record (default: + false)' + type: boolean + indexDatePattern: + description: 'Optional, Specify the index date pattern for + creating a rollover index (default: now/d)' + type: string indexName: description: IndexName defines the placeholder syntax of Fluentd plugin API. See https://docs.fluentd.org/configuration/buffer-section. type: string + logOs400Reason: + description: 'Optional, Enable logging of 400 reason without + enabling debug log level (default: false)' + type: boolean logstashFormat: description: 'If true, Fluentd uses the conventional index name format logstash-%Y.%m.%d (default: false). This option @@ -40549,6 +40882,16 @@ spec: index name to write events when logstash_format is true (default: logstash).' type: string + maxRetryGetOsVersion: + description: 'Optional, You can specify times of retry obtaining + OpenSearch version (default: 15)' + format: int32 + type: integer + maxRetryPuttingTemplate: + description: 'Optional, You can specify times of retry putting + template (default: 10)' + format: int32 + type: integer password: description: Optional, The login credentials to connect to Opensearch @@ -40588,6 +40931,9 @@ spec: description: 'Path defines the REST API endpoint of Opensearch to post write requests (default: nil).' type: string + pipeline: + description: Optional, Pipeline name + type: string port: description: 'The port number of your Opensearch node (default: 9200).' @@ -40595,13 +40941,87 @@ spec: maximum: 65535 minimum: 1 type: integer + preferOjSerializer: + description: 'Optional, With default behavior, plugin uses + Yajl as JSON encoder/decoder. Set to true to use Oj (default: + false)' + type: boolean + reconnectOnError: + description: 'Optional, Indicates that the plugin should + reset connection on any error (reconnect on next send) + (default: false)' + type: boolean + reloadAfter: + description: 'Optional, When ReloadConnections true, this + is the integer number of operations after which the plugin + will reload the connections (default: 10000)' + format: int32 + type: integer + reloadConnections: + description: 'Optional, Automatically reload connection + after 10000 documents (default: true)' + type: boolean + reloadOnFailure: + description: 'Optional, Indicates that the opensearch-transport + will try to reload the nodes addresses if there is a failure + while making the request (default: false)' + type: boolean + removeKeys: + description: Optional, Remove specified keys from the event + record + type: string + removeKeysOnUpdate: + description: Optional, Remove keys when record is being + updated + type: string + requestTimeout: + description: 'Optional, HTTP request timeout in seconds + (default: 5s)' + pattern: ^\d+(s|m|h|d)$ + type: string scheme: description: 'Specify https if your Opensearch endpoint supports SSL (default: http).' type: string + selectorClassName: + description: Optional, Provide a selector class name + type: string + snifferClassName: + description: Optional, Provide a different sniffer class + name + type: string + sslMaxVersion: + description: Optional, Maximum SSL/TLS version + type: string + sslMinVersion: + description: Optional, Minimum SSL/TLS version + type: string sslVerify: description: Optional, Force certificate validation type: boolean + sslVersion: + description: 'Optional, You can specify SSL/TLS version + (default: TLSv1_2)' + type: string + suppressDocWrap: + description: 'Optional, Suppress doc_wrap (default: false)' + type: boolean + suppressTypeName: + description: 'Optional, Suppress ''[types removal]'' warnings + on OpenSearch 2.x (default: true for OS2+)' + type: boolean + tagKey: + description: 'Optional, Tag key name when include_tag_key + is true (default: tag)' + type: string + templateOverwrite: + description: 'Optional, Always update the template, even + if it already exists (default: false)' + type: boolean + useLegacyTemplate: + description: 'Optional, Use legacy template or not (default: + false for composable templates)' + type: boolean user: description: Optional, The login credentials to connect to Opensearch @@ -40637,6 +41057,18 @@ spec: x-kubernetes-map-type: atomic type: object type: object + utcIndex: + description: 'Optional, UTC index (default: false for local + time)' + type: boolean + verifyOsVersionAtStartup: + description: 'Optional, Validate OpenSearch version at startup + (default: true)' + type: boolean + writeOperation: + description: 'Optional, The write operation (default: index). + Options: index, create, update, upsert' + type: string type: object s3: description: out_s3 plugin