From 35a102418d4567a41af2507690c6649faf8494e5 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Tue, 23 Dec 2025 16:11:17 +0100 Subject: [PATCH 1/7] TRACING-5814 | RHOSDT 3.9 Document google managed prometheus Signed-off-by: Pavol Loffay --- ...ing-data-to-google-managed-prometheus.adoc | 55 +++++++++++++++++++ .../otel/otel-forwarding-telemetry-data.adoc | 2 + 2 files changed, 57 insertions(+) create mode 100644 modules/otel-forwarding-data-to-google-managed-prometheus.adoc diff --git a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc new file mode 100644 index 000000000000..6d1e2fba2023 --- /dev/null +++ b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc @@ -0,0 +1,55 @@ +// Module included in the following assemblies: +// +// * observability/otel/otel-forwarding-telemetry-data.adoc + +:_mod-docs-content-type: CONCEPT +[id="otel-forwarding-data-to-google-managed-prometheus_{context}"] += Forwarding telemetry data to Google Managed Prometheus + +[role="_abstract"] +To forward metrics to the Google Managed Prometheus, use the OpenTelemetry Collector with the OTLP exporter, metricstarttime processor and googleclientauth extension. + +.OpenTelemetry Collector custom resource with a configured File Storage Extension that persists an OTLP sending queue +[source,yaml] +---- +# ... + config: + extensions: + file_storage/all_settings: + directory: /var/lib/otelcol/mydir # <1> + timeout: 1s # <2> + compaction: + on_start: true # <3> + directory: /tmp/ # <4> + max_transaction_size: 65_536 # <5> + fsync: false # <6> + processors: + transform/collision: + metric_statements: + - context: datapoint + statements: + - set(attributes["exported_location"], attributes["location"]) + - delete_key(attributes, "location") + - set(attributes["exported_cluster"], attributes["cluster"]) + - delete_key(attributes, "cluster") + - set(attributes["exported_namespace"], attributes["namespace"]) + - delete_key(attributes, "namespace") + - set(attributes["exported_job"], attributes["job"]) + - delete_key(attributes, "job") + - set(attributes["exported_instance"], attributes["instance"]) + - delete_key(attributes, "instance") + - set(attributes["exported_project_id"], attributes["project_id"]) + - delete_key(attributes, "project_id") + + exporters: + otlp: + sending_queue: + storage: file_storage/all_settings # <7> + + service: + extensions: [file_storage/all_settings] # <8> + pipelines: + traces: + receivers: [otlp] + exporters: [otlp] +# ... diff --git a/observability/otel/otel-forwarding-telemetry-data.adoc b/observability/otel/otel-forwarding-telemetry-data.adoc index bf86771546c4..9bdb253582f5 100644 --- a/observability/otel/otel-forwarding-telemetry-data.adoc +++ b/observability/otel/otel-forwarding-telemetry-data.adoc @@ -26,6 +26,8 @@ include::modules/otel-forwarding-data-to-google-cloud.adoc[leveloffset=+1] .Additional resources * xref:../../observability/otel/otel-collector/otel-collector-exporters.adoc#otel-collector-exporters[Exporters] +include::modules/otel-forwarding-data-to-google-managed-prometheus.adoc[leveloffset=+1] + [role="_additional-resources"] [id="additional-resources_{context}"] == Additional resources From 4f37f2339074d00429de25c8b48f8bbdd12720fe Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Tue, 23 Dec 2025 16:38:32 +0100 Subject: [PATCH 2/7] TRACING-5814 | RHOSDT 3.9 Document google managed prometheus Signed-off-by: Pavol Loffay --- ...ing-data-to-google-managed-prometheus.adoc | 75 +++++++++++-------- .../otel/otel-forwarding-telemetry-data.adoc | 5 ++ 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc index 6d1e2fba2023..9f2a8981abe7 100644 --- a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc +++ b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc @@ -13,43 +13,54 @@ To forward metrics to the Google Managed Prometheus, use the OpenTelemetry Colle [source,yaml] ---- # ... + mode: sidecar config: extensions: - file_storage/all_settings: - directory: /var/lib/otelcol/mydir # <1> - timeout: 1s # <2> - compaction: - on_start: true # <3> - directory: /tmp/ # <4> - max_transaction_size: 65_536 # <5> - fsync: false # <6> - processors: - transform/collision: - metric_statements: - - context: datapoint - statements: - - set(attributes["exported_location"], attributes["location"]) - - delete_key(attributes, "location") - - set(attributes["exported_cluster"], attributes["cluster"]) - - delete_key(attributes, "cluster") - - set(attributes["exported_namespace"], attributes["namespace"]) - - delete_key(attributes, "namespace") - - set(attributes["exported_job"], attributes["job"]) - - delete_key(attributes, "job") - - set(attributes["exported_instance"], attributes["instance"]) - - delete_key(attributes, "instance") - - set(attributes["exported_project_id"], attributes["project_id"]) - - delete_key(attributes, "project_id") + googleclientauth: {} exporters: - otlp: - sending_queue: - storage: file_storage/all_settings # <7> + otlphttp: + encoding: json + endpoint: https://telemetry.googleapis.com + auth: + authenticator: googleclientauth + + processors: + metricstarttime: + strategy: subtract_initial_point # <2> + + resource/gcp_project_id: + attributes: + - action: insert + value: project_id # <1> + key: gcp.project_id + + k8sattributes: {} + + transform/collision: + metric_statements: + - context: datapoint + statements: + - set(attributes["exported_location"], attributes["location"]) + - delete_key(attributes, "location") + - set(attributes["exported_cluster"], attributes["cluster"]) + - delete_key(attributes, "cluster") + - set(attributes["exported_namespace"], attributes["namespace"]) + - delete_key(attributes, "namespace") + - set(attributes["exported_job"], attributes["job"]) + - delete_key(attributes, "job") + - set(attributes["exported_instance"], attributes["instance"]) + - delete_key(attributes, "instance") + - set(attributes["exported_project_id"], attributes["project_id"]) + - delete_key(attributes, "project_id") service: - extensions: [file_storage/all_settings] # <8> + extensions: [googleclientauth] pipelines: - traces: - receivers: [otlp] - exporters: [otlp] + metrics: + processors: [k8sattributes, resource/gcp_project_id, transform/collision, metricstarttime] + exporters: [otlphttp] # ... +---- +<1> The `subtract_initial_point` strategy is stateful, requiring the Collector to run as a sidecar to maintain per-pod state. Alternative strategies available; choose the one that best fits your use case. +<2> Replace with your GCP project ID. \ No newline at end of file diff --git a/observability/otel/otel-forwarding-telemetry-data.adoc b/observability/otel/otel-forwarding-telemetry-data.adoc index 9bdb253582f5..7da16294b53a 100644 --- a/observability/otel/otel-forwarding-telemetry-data.adoc +++ b/observability/otel/otel-forwarding-telemetry-data.adoc @@ -28,6 +28,11 @@ include::modules/otel-forwarding-data-to-google-cloud.adoc[leveloffset=+1] include::modules/otel-forwarding-data-to-google-managed-prometheus.adoc[leveloffset=+1] +[role="_additional-resources"] +.Additional resources +* xref:../../observability/otel/otel-collector/otel-collector-processors.adoc#otel-collector-processors[Processors] +* xref:../../observability/otel/otel-collector/otel-collector-extensions.adoc#otel-collector-extensions[Extensions] + [role="_additional-resources"] [id="additional-resources_{context}"] == Additional resources From a08a279faedc02a7cff26884ff3ab9a1b0915ddd Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Tue, 23 Dec 2025 17:19:58 +0100 Subject: [PATCH 3/7] Google link Signed-off-by: Pavol Loffay --- observability/otel/otel-forwarding-telemetry-data.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/observability/otel/otel-forwarding-telemetry-data.adoc b/observability/otel/otel-forwarding-telemetry-data.adoc index 7da16294b53a..e4cb6a372768 100644 --- a/observability/otel/otel-forwarding-telemetry-data.adoc +++ b/observability/otel/otel-forwarding-telemetry-data.adoc @@ -32,6 +32,7 @@ include::modules/otel-forwarding-data-to-google-managed-prometheus.adoc[leveloff .Additional resources * xref:../../observability/otel/otel-collector/otel-collector-processors.adoc#otel-collector-processors[Processors] * xref:../../observability/otel/otel-collector/otel-collector-extensions.adoc#otel-collector-extensions[Extensions] +* https://docs.cloud.google.com/stackdriver/docs/managed-prometheus [role="_additional-resources"] [id="additional-resources_{context}"] From da06f79843991d996aa29f6cd9e88e4fe7e2761d Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Wed, 7 Jan 2026 17:03:38 +0100 Subject: [PATCH 4/7] Add google auth extension Signed-off-by: Pavol Loffay --- .../otel-extensions-googleauth-extension.adoc | 38 +++++++++++++++++++ ...ing-data-to-google-managed-prometheus.adoc | 6 +-- .../otel-collector-extensions.adoc | 2 + 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 modules/otel-extensions-googleauth-extension.adoc diff --git a/modules/otel-extensions-googleauth-extension.adoc b/modules/otel-extensions-googleauth-extension.adoc new file mode 100644 index 000000000000..7b66e976b394 --- /dev/null +++ b/modules/otel-extensions-googleauth-extension.adoc @@ -0,0 +1,38 @@ +// Module included in the following assemblies: +// +// * observability/otel/otel-collector/otel-collector-extensions.adoc + +:_mod-docs-content-type: REFERENCE +[id="otel-extensions-googleauth-extension_{context}"] += Google Client Authentication Extension + +[role="_abstract"] + +The Google Client Authentication extension provides Google OAuth2 Client Credentials and Metadata for gRPC and http based exporters. + +.OpenTelemetry Collector custom resource with the configured Google Client Auth Extension +[source,yaml] +---- +# ... + config: + extensions: + googleclientauth: + project: "my-project" # <1> + + exporters: + otlphttp: + encoding: json + endpoint: https://telemetry.googleapis.com + auth: + authenticator: googleclientauth + + service: + extensions: [googleclientauth] + pipelines: + traces: + receivers: [otlp] + exporters: [debug] +# ... +---- + +<1> The Google Cloud Project telemetry is sent to if the `gcp.project.id` resource attribute is not set. If unspecified, this is determined using application default credentials. diff --git a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc index 9f2a8981abe7..f910a07e050f 100644 --- a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc +++ b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc @@ -21,18 +21,18 @@ To forward metrics to the Google Managed Prometheus, use the OpenTelemetry Colle exporters: otlphttp: encoding: json - endpoint: https://telemetry.googleapis.com + endpoint: https://telemetry.googleapis.com auth: authenticator: googleclientauth processors: metricstarttime: - strategy: subtract_initial_point # <2> + strategy: subtract_initial_point # <1> resource/gcp_project_id: attributes: - action: insert - value: project_id # <1> + value: project_id # <2> key: gcp.project_id k8sattributes: {} diff --git a/observability/otel/otel-collector/otel-collector-extensions.adoc b/observability/otel/otel-collector/otel-collector-extensions.adoc index 948da06b6610..f632e2dc9cc0 100644 --- a/observability/otel/otel-collector/otel-collector-extensions.adoc +++ b/observability/otel/otel-collector/otel-collector-extensions.adoc @@ -23,6 +23,8 @@ include::modules/otel-extensions-jaegerremotesampling-extension.adoc[leveloffset include::modules/otel-extensions-pprof-extension.adoc[leveloffset=+1] +include::modules/otel-extensions-googleauth-extension.adoc[leveloffset=+1] + include::modules/otel-extensions-healthcheck-extension.adoc[leveloffset=+1] include::modules/otel-extensions-zpages-extension.adoc[leveloffset=+1] From c20634e82d4400b26e461ea7a45ad8a504883af8 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 8 Jan 2026 13:08:52 +0100 Subject: [PATCH 5/7] Use WIF Signed-off-by: Pavol Loffay --- ...ing-data-to-google-managed-prometheus.adoc | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc index f910a07e050f..85199e21a63c 100644 --- a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc +++ b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc @@ -14,6 +14,27 @@ To forward metrics to the Google Managed Prometheus, use the OpenTelemetry Colle ---- # ... mode: sidecar + env: + - name: GOOGLE_APPLICATION_CREDENTIALS # <1> + value: "/etc/workload-identity/credential-configuration.json" + volumes: + - name: workload-identity-credential-configuration + configMap: + name: gcp-wif-credentials # <2> + - name: service-account-token-volume + projected: + sources: + - serviceAccountToken: + audience: openshift + expirationSeconds: 3600 + path: token + volumeMounts: + - name: workload-identity-credential-configuration + mountPath: "/etc/workload-identity" + readOnly: true + - name: service-account-token-volume + mountPath: "/var/run/secrets/otel/serviceaccount" <3> + readOnly: true config: extensions: googleclientauth: {} @@ -27,12 +48,12 @@ To forward metrics to the Google Managed Prometheus, use the OpenTelemetry Colle processors: metricstarttime: - strategy: subtract_initial_point # <1> + strategy: subtract_initial_point # <4> resource/gcp_project_id: attributes: - action: insert - value: project_id # <2> + value: project_id # <5> key: gcp.project_id k8sattributes: {} @@ -62,5 +83,8 @@ To forward metrics to the Google Managed Prometheus, use the OpenTelemetry Colle exporters: [otlphttp] # ... ---- -<1> The `subtract_initial_point` strategy is stateful, requiring the Collector to run as a sidecar to maintain per-pod state. Alternative strategies available; choose the one that best fits your use case. -<2> Replace with your GCP project ID. \ No newline at end of file +<1> The environment variable `GOOGLE_APPLICATION_CREDENTIALS` can be configured to use the traditional secret or use the workload identity federation (WIF). This specific example uses WIF. +<2> Config map contains the Google WIF configuration file `credential-configuration.json`. +<3> The path to the service account token used by WIF. In this case `/var/run/secrets/otel/serviceaccount/token`. +<4> The `subtract_initial_point` strategy is stateful, requiring the Collector to run as a sidecar to maintain per-pod state. Alternative strategies available; choose the one that best fits your use case. +<5> Replace with your GCP project ID. \ No newline at end of file From 5c73d11a5bea2db136403965a6c121ea3ba61a26 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 8 Jan 2026 13:17:39 +0100 Subject: [PATCH 6/7] Use WIF Signed-off-by: Pavol Loffay --- .../otel-forwarding-data-to-google-managed-prometheus.adoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc index 85199e21a63c..de0e7f1650a0 100644 --- a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc +++ b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc @@ -7,9 +7,10 @@ = Forwarding telemetry data to Google Managed Prometheus [role="_abstract"] -To forward metrics to the Google Managed Prometheus, use the OpenTelemetry Collector with the OTLP exporter, metricstarttime processor and googleclientauth extension. +To forward metrics to the Google Managed Prometheus, use the OpenTelemetry Collector with the OTLP exporter, Metric Start Time processor and Google Client Auth extension. +The OTLP exporter should be configured with Google Client Auth extension, which can use standard secret authentication or Workload Identity Federation (WIF). -.OpenTelemetry Collector custom resource with a configured File Storage Extension that persists an OTLP sending queue +.OpenTelemetry Collector custom resource with configured OTLP exporter and Google WIF authentication. [source,yaml] ---- # ... From 725d3af64abea7e7a65c23c0bdf5421107eeb226 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 8 Jan 2026 13:18:11 +0100 Subject: [PATCH 7/7] Use WIF Signed-off-by: Pavol Loffay --- .../otel-forwarding-data-to-google-managed-prometheus.adoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc index de0e7f1650a0..297cc25c48a9 100644 --- a/modules/otel-forwarding-data-to-google-managed-prometheus.adoc +++ b/modules/otel-forwarding-data-to-google-managed-prometheus.adoc @@ -88,4 +88,7 @@ The OTLP exporter should be configured with Google Client Auth extension, which <2> Config map contains the Google WIF configuration file `credential-configuration.json`. <3> The path to the service account token used by WIF. In this case `/var/run/secrets/otel/serviceaccount/token`. <4> The `subtract_initial_point` strategy is stateful, requiring the Collector to run as a sidecar to maintain per-pod state. Alternative strategies available; choose the one that best fits your use case. -<5> Replace with your GCP project ID. \ No newline at end of file +<5> Replace with your GCP project ID. + + +Other resources: https://docs.cloud.google.com/iam/docs/workload-identity-federation-with-kubernetes \ No newline at end of file