feat!: Add dedicated -metrics service (#748)

* feat!: Add dedicated -metrics service

* changelog

* improve doc comment

* linter

* linter

* Add container port

* fix port problem

* fix tests und role service name

* fix linter

* Update rust/operator-binary/src/controller.rs

Co-authored-by: Malte Sander <contact@maltesander.com>

* changelog

* changelog

* changelog

* Update CHANGELOG.md

Co-authored-by: Malte Sander <contact@maltesander.com>

* fix tests

* ruff ruff

---------

Co-authored-by: Malte Sander <malte.sander.it@gmail.com>
Co-authored-by: Malte Sander <contact@maltesander.com>

Author: 2 people

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+
+- Add a dedicated per-rolegroup `-metrics` Service, which can be used to get Prometheus metrics ([#748]).
+- Expose more Prometheus metrics, such as successful or failed bundle loads and information about the OPA environment ([#748]).
+
+### Changed
+
+- BREAKING: The per-rolegroup services now only serves the HTTP port and has a `-headless` suffix to better indicate their
+  purpose and to be consistent with other operators ([#748]).
+- BREAKING: The per-role server service is now prefixed with `-server` to be consistent with other operators ([#748]).
+
+[#748]: https://github.com/stackabletech/opa-operator/pull/748
+
 ## [25.7.0] - 2025-07-23
 
 ## [25.7.0-rc1] - 2025-07-18

rust/operator-binary/src/controller.rs

@@ -50,7 +50,7 @@ use stackable_operator::{
         core::{DeserializeGuard, error_boundary},
         runtime::{controller::Action, reflector::ObjectRef},
     },
-    kvp::{Label, LabelError, Labels, ObjectLabels},
+    kvp::{LabelError, Labels, ObjectLabels},
     logging::controller::ReconcilerError,
     memory::{BinaryMultiple, MemoryQuantity},
     product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config},
@@ -93,6 +93,7 @@ pub const BUNDLES_ACTIVE_DIR: &str = "/bundles/active";
 pub const BUNDLES_INCOMING_DIR: &str = "/bundles/incoming";
 pub const BUNDLES_TMP_DIR: &str = "/bundles/tmp";
 pub const BUNDLE_BUILDER_PORT: i32 = 3030;
+pub const OPA_STACKABLE_SERVICE_NAME: &str = "stackable";
 
 const CONFIG_VOLUME_NAME: &str = "config";
 const CONFIG_DIR: &str = "/stackable/config";
@@ -189,6 +190,12 @@ pub enum Error {
         rolegroup: RoleGroupRef<v1alpha1::OpaCluster>,
     },
 
+    #[snafu(display("failed to apply metrics Service for [{rolegroup}]"))]
+    ApplyRoleGroupMetricsService {
+        source: stackable_operator::cluster_resources::Error,
+        rolegroup: RoleGroupRef<v1alpha1::OpaCluster>,
+    },
+
     #[snafu(display("failed to build ConfigMap for [{rolegroup}]"))]
     BuildRoleGroupConfig {
         source: stackable_operator::builder::configmap::Error,
@@ -346,19 +353,20 @@ pub struct OpaClusterConfigFile {
     bundles: OpaClusterBundle,
     #[serde(skip_serializing_if = "Option::is_none")]
     decision_logs: Option<OpaClusterConfigDecisionLog>,
+    status: Option<OpaClusterConfigStatus>,
 }
 
 impl OpaClusterConfigFile {
     pub fn new(decision_logging: Option<OpaClusterConfigDecisionLog>) -> Self {
         Self {
             services: vec![OpaClusterConfigService {
-                name: String::from("stackable"),
-                url: String::from("http://localhost:3030/opa/v1"),
+                name: OPA_STACKABLE_SERVICE_NAME.to_owned(),
+                url: "http://localhost:3030/opa/v1".to_owned(),
             }],
             bundles: OpaClusterBundle {
                 stackable: OpaClusterBundleConfig {
-                    service: String::from("stackable"),
-                    resource: String::from("opa/bundle.tar.gz"),
+                    service: OPA_STACKABLE_SERVICE_NAME.to_owned(),
+                    resource: "opa/bundle.tar.gz".to_owned(),
                     persist: true,
                     polling: OpaClusterBundleConfigPolling {
                         min_delay_seconds: 10,
@@ -367,6 +375,12 @@ impl OpaClusterConfigFile {
                 },
             },
             decision_logs: decision_logging,
+            // Enable more Prometheus metrics, such as bundle loads
+            // See https://www.openpolicyagent.org/docs/monitoring#status-metrics
+            status: Some(OpaClusterConfigStatus {
+                service: OPA_STACKABLE_SERVICE_NAME.to_owned(),
+                prometheus: true,
+            }),
         }
     }
 }
@@ -401,6 +415,12 @@ pub struct OpaClusterConfigDecisionLog {
     console: bool,
 }
 
+#[derive(Serialize, Deserialize)]
+struct OpaClusterConfigStatus {
+    service: String,
+    prometheus: bool,
+}
+
 pub async fn reconcile_opa(
     opa: Arc<DeserializeGuard<v1alpha1::OpaCluster>>,
     ctx: Arc<Ctx>,
@@ -498,7 +518,10 @@ pub async fn reconcile_opa(
             &rolegroup,
             &merged_config,
         )?;
-        let rg_service = build_rolegroup_service(opa, &resolved_product_image, &rolegroup)?;
+        let rg_service =
+            build_rolegroup_headless_service(opa, &resolved_product_image, &rolegroup)?;
+        let rg_metrics_service =
+            build_rolegroup_metrics_service(opa, &resolved_product_image, &rolegroup)?;
         let rg_daemonset = build_server_rolegroup_daemonset(
             opa,
             &resolved_product_image,
@@ -524,6 +547,12 @@ pub async fn reconcile_opa(
             .with_context(|_| ApplyRoleGroupServiceSnafu {
                 rolegroup: rolegroup.clone(),
             })?;
+        cluster_resources
+            .add(client, rg_metrics_service)
+            .await
+            .with_context(|_| ApplyRoleGroupServiceSnafu {
+                rolegroup: rolegroup.clone(),
+            })?;
         ds_cond_builder.add(
             cluster_resources
                 .add(client, rg_daemonset.clone())
@@ -647,17 +676,14 @@ pub fn build_server_role_service(
 /// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup
 ///
 /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing.
-fn build_rolegroup_service(
+fn build_rolegroup_headless_service(
     opa: &v1alpha1::OpaCluster,
     resolved_product_image: &ResolvedProductImage,
     rolegroup: &RoleGroupRef<v1alpha1::OpaCluster>,
 ) -> Result<Service> {
-    let prometheus_label =
-        Label::try_from(("prometheus.io/scrape", "true")).context(BuildLabelSnafu)?;
-
     let metadata = ObjectMetaBuilder::new()
         .name_and_namespace(opa)
-        .name(rolegroup.object_name())
+        .name(rolegroup.rolegroup_headless_service_name())
         .ownerreference_from_resource(opa, None, Some(true))
         .context(ObjectMissingMetadataForOwnerRefSnafu)?
         .with_recommended_labels(build_recommended_labels(
@@ -667,19 +693,20 @@ fn build_rolegroup_service(
             &rolegroup.role_group,
         ))
         .context(ObjectMetaSnafu)?
-        .with_label(prometheus_label)
         .build();
 
-    let service_selector_labels =
-        Labels::role_group_selector(opa, APP_NAME, &rolegroup.role, &rolegroup.role_group)
-            .context(BuildLabelSnafu)?;
-
     let service_spec = ServiceSpec {
-        // Internal communication does not need to be exposed
+        // Currently we don't offer listener-exposition of OPA mostly due to security concerns.
+        // OPA is currently public within the Kubernetes (without authentication).
+        // Opening it up to outside of Kubernetes might worsen things.
+        // We are open to implement listener-integration, but this needs to be thought through before
+        // implementing it.
+        // Note: We have kind of similar situations for HMS and Zookeeper, as the authentication
+        // options there are non-existent (mTLS still opens plain port) or suck (Kerberos).
         type_: Some("ClusterIP".to_string()),
         cluster_ip: Some("None".to_string()),
-        ports: Some(service_ports(opa.spec.cluster_config.tls.is_some())),
-        selector: Some(service_selector_labels.into()),
+        ports: Some(data_service_ports_with_tls(opa.spec.cluster_config.tls.is_some())),
+        selector: Some(role_group_selector_labels(opa, rolegroup)?.into()),
         publish_not_ready_addresses: Some(true),
         ..ServiceSpec::default()
     };
@@ -691,6 +718,55 @@ fn build_rolegroup_service(
     })
 }
 
+/// The rolegroup metrics [`Service`] is a service that exposes metrics and has the
+/// prometheus.io/scrape label.
+fn build_rolegroup_metrics_service(
+    opa: &v1alpha1::OpaCluster,
+    resolved_product_image: &ResolvedProductImage,
+    rolegroup: &RoleGroupRef<v1alpha1::OpaCluster>,
+) -> Result<Service> {
+    let labels = Labels::try_from([("prometheus.io/scrape", "true")])
+        .expect("static Prometheus labels must be valid");
+
+    let metadata = ObjectMetaBuilder::new()
+        .name_and_namespace(opa)
+        .name(rolegroup.rolegroup_metrics_service_name())
+        .ownerreference_from_resource(opa, None, Some(true))
+        .context(ObjectMissingMetadataForOwnerRefSnafu)?
+        .with_recommended_labels(build_recommended_labels(
+            opa,
+            &resolved_product_image.app_version_label,
+            &rolegroup.role,
+            &rolegroup.role_group,
+        ))
+        .context(ObjectMetaSnafu)?
+        .with_labels(labels)
+        .build();
+
+    let service_spec = ServiceSpec {
+        type_: Some("ClusterIP".to_string()),
+        cluster_ip: Some("None".to_string()),
+        ports: Some(vec![metrics_service_port_with_tls(opa.spec.cluster_config.tls.is_some())]),
+        selector: Some(role_group_selector_labels(opa, rolegroup)?.into()),
+        ..ServiceSpec::default()
+    };
+
+    Ok(Service {
+        metadata,
+        spec: Some(service_spec),
+        status: None,
+    })
+}
+
+/// Returns the [`Labels`] that can be used to select all Pods that are part of the roleGroup.
+fn role_group_selector_labels(
+    opa: &v1alpha1::OpaCluster,
+    rolegroup: &RoleGroupRef<v1alpha1::OpaCluster>,
+) -> Result<Labels> {
+    Labels::role_group_selector(opa, APP_NAME, &rolegroup.role, &rolegroup.role_group)
+        .context(BuildLabelSnafu)
+}
+
 /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator
 fn build_server_rolegroup_config_map(
     opa: &v1alpha1::OpaCluster,
@@ -923,6 +999,11 @@ fn build_server_rolegroup_daemonset(
         );
 
     // Add appropriate container port based on TLS configuration
+    // If we also add a container port "metrics" pointing to the same port number, we get a
+    //
+    // .spec.template.spec.containers[name="opa"].ports: duplicate entries for key [containerPort=8081,protocol="TCP"]
+    //
+    // So we don't do that
     if opa_tls_config.is_some() {
         cb_opa.add_container_port(APP_TLS_PORT_NAME, APP_TLS_PORT.into());
         cb_opa
@@ -1455,36 +1536,35 @@ fn build_prepare_start_command(
     prepare_container_args
 }
 
-fn service_ports(tls_enabled: bool) -> Vec<ServicePort> {
-    let (port_name, port, target_port) = if tls_enabled {
-        (
-            APP_TLS_PORT_NAME,
-            APP_TLS_PORT,
-            IntOrString::String(APP_TLS_PORT_NAME.to_string()),
-        )
+fn data_service_ports_with_tls(tls_enabled: bool) -> Vec<ServicePort> {
+    let (port_name, port) = if tls_enabled {
+        (APP_TLS_PORT_NAME, APP_TLS_PORT)
     } else {
-        (
-            APP_PORT_NAME,
-            APP_PORT,
-            IntOrString::String(APP_PORT_NAME.to_string()),
-        )
+        (APP_PORT_NAME, APP_PORT)
     };
 
-    vec![
-        ServicePort {
-            name: Some(port_name.to_string()),
-            port: port.into(),
-            protocol: Some("TCP".to_string()),
-            ..ServicePort::default()
-        },
-        ServicePort {
-            name: Some(METRICS_PORT_NAME.to_string()),
-            port: 9504, // Arbitrary port number, this is never actually used anywhere
-            protocol: Some("TCP".to_string()),
-            target_port: Some(target_port),
-            ..ServicePort::default()
-        },
-    ]
+    vec![ServicePort {
+        name: Some(port_name.to_string()),
+        port: port.into(),
+        protocol: Some("TCP".to_string()),
+        ..ServicePort::default()
+    }]
+}
+
+fn metrics_service_port_with_tls(tls_enabled: bool) -> ServicePort {
+    let port = if tls_enabled {
+        APP_TLS_PORT
+    } else {
+        APP_PORT
+    };
+
+    ServicePort {
+        name: Some(METRICS_PORT_NAME.to_string()),
+        // The metrics are served on the same port as the HTTP traffic
+        port: port.into(),
+        protocol: Some("TCP".to_string()),
+        ..ServicePort::default()
+    }
 }
 
 /// Creates recommended `ObjectLabels` to be used in deployed resources

rust/operator-binary/src/crd/mod.rs

@@ -17,7 +17,7 @@ use stackable_operator::{
         merge::Merge,
     },
     k8s_openapi::apimachinery::pkg::api::resource::Quantity,
-    kube::CustomResource,
+    kube::{CustomResource, ResourceExt},
     product_config_utils::Configuration,
     product_logging::{self, spec::Logging},
     role_utils::{
@@ -338,7 +338,11 @@ impl v1alpha1::OpaCluster {
 
     /// The name of the role-level load-balanced Kubernetes `Service`
     pub fn server_role_service_name(&self) -> Option<String> {
-        self.metadata.name.clone()
+        Some(format!(
+            "{cluster_name}-{role}",
+            cluster_name = self.name_any(),
+            role = v1alpha1::OpaRole::Server
+        ))
     }
 
     /// The fully-qualified domain name of the role-level load-balanced Kubernetes `Service`

tests/templates/kuttl/aas-user-info/30-assert.yaml

@@ -4,4 +4,4 @@ kind: TestAssert
 metadata:
   name: test-regorule
 commands:
-  - script: kubectl exec -n $NAMESPACE test-regorule-0 -- python /tmp/test-regorule.py -u 'http://test-opa-server-default:8081/v1/data/test'
+  - script: kubectl exec -n $NAMESPACE test-regorule-0 -- python /tmp/test-regorule.py -u 'http://test-opa-server:8081/v1/data/test'

tests/templates/kuttl/ad-user-info/30-assert.yaml

@@ -4,4 +4,4 @@ kind: TestAssert
 metadata:
   name: test-regorule
 commands:
-  - script: kubectl exec -n $NAMESPACE test-regorule-0 -- python /tmp/test-regorule.py -u 'http://test-opa-server-default:8081/v1/data/test'
+  - script: kubectl exec -n $NAMESPACE test-regorule-0 -- python /tmp/test-regorule.py -u 'http://test-opa-server:8081/v1/data/test'

tests/templates/kuttl/keycloak-user-info/30-assert.yaml

@@ -4,4 +4,4 @@ kind: TestAssert
 metadata:
   name: test-regorule
 commands:
-  - script: kubectl exec -n $NAMESPACE test-regorule-0 -- python /tmp/test-regorule.py -u 'http://test-opa-server-default:8081/v1/data/test'
+  - script: kubectl exec -n $NAMESPACE test-regorule-0 -- python /tmp/test-regorule.py -u 'http://test-opa-server:8081/v1/data/test'