Add larger request duration histogram bucket to better identify slow requests (#5815)

Author: rdettai

quickwit/quickwit-search/src/metrics.rs

@@ -38,6 +38,11 @@ pub struct SearchMetrics {
     pub searcher_local_kv_store_size_bytes: IntGauge,
 }
 
+/// From 0.008s to 131.072s
+fn duration_buckets() -> Vec<f64> {
+    exponential_buckets(0.008, 2.0, 15).unwrap()
+}
+
 impl Default for SearchMetrics {
     fn default() -> Self {
         let targeted_splits_buckets: Vec<f64> = [
@@ -85,7 +90,7 @@ impl Default for SearchMetrics {
                 "search",
                 &[("kind", "server")],
                 ["status"],
-                exponential_buckets(0.001, 2.0, 15).unwrap(),
+                duration_buckets(),
             ),
             root_search_targeted_splits: new_histogram_vec(
                 "root_search_targeted_splits",
@@ -108,7 +113,7 @@ impl Default for SearchMetrics {
                 "search",
                 &[("kind", "server")],
                 ["status"],
-                exponential_buckets(0.001, 2.0, 15).unwrap(),
+                duration_buckets(),
             ),
             leaf_search_targeted_splits: new_histogram_vec(
                 "leaf_search_targeted_splits",
@@ -129,7 +134,7 @@ impl Default for SearchMetrics {
                 "Number of seconds required to run a leaf search over a single split. The timer \
                  starts after the semaphore is obtained.",
                 "search",
-                exponential_buckets(0.001, 2.0, 15).unwrap(),
+                duration_buckets(),
             ),
             leaf_search_single_split_tasks_ongoing: leaf_search_single_split_tasks
                 .with_label_values(["ongoing"]),

quickwit/quickwit-serve/src/metrics.rs

@@ -48,7 +48,8 @@ impl Default for ServeMetrics {
                 "",
                 &[],
                 ["method", "status_code"],
-                quickwit_common::metrics::exponential_buckets(0.02, 2.0, 8).unwrap(),
+                // last bucket is 163.84s
+                quickwit_common::metrics::exponential_buckets(0.02, 2.0, 14).unwrap(),
             ),
             ongoing_requests: new_gauge_vec(
                 "ongoing_requests",