Skip to content

Commit efad408

Browse files
nikhilsinhaparseablenikhilsinhaparseable
authored
Merge branch 'main' into improve-metrics
2 parents edc749a + 7d5600b commit efad408

File tree

4 files changed

+52
-27
lines changed

4 files changed

+52
-27
lines changed

‎resources/ingest_demo_data.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ create_target() {
377377
echo "Creating webhook target..." >&2
378378

379379
response=$(curl -s -H "Content-Type: application/json" -H "$AUTH_HEADER" -X POST "$P_URL/api/v1/targets" -d @- << EOF
380-
{"type":"webhook","endpoint":"https://webhook.site/8e1f26bd-2f5b-47a2-9d0b-3b3dabb30710","name":"Test Webhook","auth":{"username":"","password":""},"skipTlsCheck":false,"notificationConfig":{"interval":1,"times":1}}
380+
{"type":"webhook","endpoint":"https://webhook.site/8e1f26bd-2f5b-47a2-9d0b-3b3dabb30710","name":"Test Webhook","auth":{"username":"","password":""},"skipTlsCheck":false}
381381
EOF
382382
)
383383

@@ -415,7 +415,7 @@ create_alerts() {
415415
echo "Creating alerts with target ID: $target_id"
416416

417417
# Alert 1: Error Count (severity_number = 18)
418-
alert1_json="{\"severity\":\"high\",\"title\":\"error count\",\"alertType\":\"threshold\",\"query\": \"select count(severity_number) as count_severity_number from demodata where severity_number=18\",\"thresholdConfig\":{\"operator\":\">\",\"value\":1000},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5h\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"]}"
418+
alert1_json="{\"severity\":\"high\",\"title\":\"error count\",\"alertType\":\"threshold\",\"query\": \"select count(severity_number) as count_severity_number from demodata where severity_number=18\",\"thresholdConfig\":{\"operator\":\">\",\"value\":1000},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5 hours\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"],\"tags\":[],\"notificationConfig\":{\"interval\":1}}"
419419

420420
response1=$(curl_with_retry "$P_URL/api/v1/alerts" "POST" "$alert1_json" "application/json" 3)
421421
if [[ $? -eq 0 ]]; then
@@ -426,7 +426,7 @@ create_alerts() {
426426
fi
427427

428428
# Alert 2: 400 Errors
429-
alert2_json="{\"severity\":\"critical\",\"title\":\"400 Errors\",\"alertType\":\"threshold\",\"query\": \"select count(body) as count_body from demodata where body like '%400%'\",\"thresholdConfig\":{\"operator\":\">\",\"value\":10},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5h\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"]}"
429+
alert2_json="{\"severity\":\"critical\",\"title\":\"400 Errors\",\"alertType\":\"threshold\",\"query\": \"select count(body) as count_body from demodata where body like '%400%'\",\"thresholdConfig\":{\"operator\":\">\",\"value\":10},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5 hours\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"],\"tags\":[],\"notificationConfig\":{\"interval\":1}}"
430430

431431
response2=$(curl_with_retry "$P_URL/api/v1/alerts" "POST" "$alert2_json" "application/json" 3)
432432
if [[ $? -eq 0 ]]; then
@@ -437,7 +437,7 @@ create_alerts() {
437437
fi
438438

439439
# Alert 3: Trace ID null
440-
alert3_json="{\"severity\":\"high\",\"title\":\"Trace ID null\",\"alertType\":\"threshold\",\"query\": \"select count(trace_id) as count_trace_id from demodata where trace_id is null\",\"thresholdConfig\":{\"operator\":\">\",\"value\":0},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5h\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"]}"
440+
alert3_json="{\"severity\":\"high\",\"title\":\"Trace ID null\",\"alertType\":\"threshold\",\"query\": \"select count(trace_id) as count_trace_id from demodata where trace_id is null\",\"thresholdConfig\":{\"operator\":\">\",\"value\":0},\"evalConfig\":{\"rollingWindow\":{\"evalStart\":\"5 hours\",\"evalEnd\":\"now\",\"evalFrequency\":1}},\"targets\":[\"$target_id\"],\"tags\":[],\"notificationConfig\":{\"interval\":1}}"
441441
response3=$(curl_with_retry "$P_URL/api/v1/alerts" "POST" "$alert3_json" "application/json" 3)
442442
if [[ $? -eq 0 ]]; then
443443
echo "Alert 3 (Trace ID null) created successfully"

‎src/handlers/http/query.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ async fn handle_count_query(
171171
stream: table_name.to_string(),
172172
start_time: query_request.start_time.clone(),
173173
end_time: query_request.end_time.clone(),
174-
num_bins: 1,
174+
num_bins: Some(1),
175175
conditions: None,
176176
};
177177
let count_records = counts_req.get_bin_density().await?;

‎src/prism/logstream/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ impl PrismDatasetRequest {
351351
stream: stream.to_owned(),
352352
start_time: "1h".to_owned(),
353353
end_time: "now".to_owned(),
354-
num_bins: 10,
354+
num_bins: Some(10),
355355
conditions: None,
356356
};
357357

‎src/query/mod.rs‎

Lines changed: 46 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -298,8 +298,10 @@ impl Query {
298298
/// Record of counts for a given time bin.
299299
#[derive(Debug, Serialize, Clone, Deserialize)]
300300
pub struct CountsRecord {
301+
#[serde(alias = "_bin_start_time_")]
301302
/// Start time of the bin
302303
pub start_time: String,
304+
#[serde(alias = "_bin_end_time_")]
303305
/// End time of the bin
304306
pub end_time: String,
305307
/// Number of logs in the bin
@@ -330,8 +332,8 @@ pub struct CountsRequest {
330332
pub start_time: String,
331333
/// Excluded end time for counts query
332334
pub end_time: String,
333-
/// Number of bins to divide the time range into
334-
pub num_bins: u64,
335+
/// optional number of bins to divide the time range into
336+
pub num_bins: Option<u64>,
335337
/// Conditions
336338
pub conditions: Option<CountConditions>,
337339
}
@@ -400,9 +402,28 @@ impl CountsRequest {
400402
.signed_duration_since(time_range.start)
401403
.num_minutes() as u64;
402404

405+
let num_bins = if let Some(num_bins) = self.num_bins {
406+
num_bins
407+
} else {
408+
// create number of bins based on total minutes
409+
if total_minutes <= 60 * 5 {
410+
// till 5 hours, 1 bin = 1 min
411+
total_minutes
412+
} else if total_minutes <= 60 * 24 {
413+
// till 1 day, 1 bin = 5 min
414+
total_minutes.div_ceil(5)
415+
} else if total_minutes <= 60 * 24 * 10 {
416+
// till 10 days, 1 bin = 1 hour
417+
total_minutes.div_ceil(60)
418+
} else {
419+
// > 10 days, 1 bin = 1 day
420+
total_minutes.div_ceil(1440)
421+
}
422+
};
423+
403424
// divide minutes by num bins to get minutes per bin
404-
let quotient = total_minutes / self.num_bins;
405-
let remainder = total_minutes % self.num_bins;
425+
let quotient = total_minutes / num_bins;
426+
let remainder = total_minutes % num_bins;
406427
let have_remainder = remainder > 0;
407428

408429
// now create multiple bounds [startTime, endTime)
@@ -412,9 +433,9 @@ impl CountsRequest {
412433
let mut start = time_range.start;
413434

414435
let loop_end = if have_remainder {
415-
self.num_bins
436+
num_bins
416437
} else {
417-
self.num_bins - 1
438+
num_bins - 1
418439
};
419440

420441
// Create bins for all but the last date
@@ -449,36 +470,40 @@ impl CountsRequest {
449470

450471
let dur = time_range.end.signed_duration_since(time_range.start);
451472

452-
let date_bin = if dur.num_minutes() <= 60 * 10 {
453-
// date_bin 1 minute
473+
let table_name = &self.stream;
474+
let start_time_col_name = "_bin_start_time_";
475+
let end_time_col_name = "_bin_end_time_";
476+
let date_bin = if dur.num_minutes() <= 60 * 5 {
477+
// less than 5 hour = 1 min bin
478+
format!(
479+
"CAST(DATE_BIN('1m', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as {start_time_col_name}, DATE_BIN('1m', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1m' as {end_time_col_name}"
480+
)
481+
} else if dur.num_minutes() <= 60 * 24 {
482+
// 1 day = 5 min bin
454483
format!(
455-
"CAST(DATE_BIN('1 minute', \"{}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as start_time, DATE_BIN('1 minute', \"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1 minute' as end_time",
456-
self.stream
484+
"CAST(DATE_BIN('5m', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as {start_time_col_name}, DATE_BIN('5m', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '5m' as {end_time_col_name}"
457485
)
458-
} else if dur.num_minutes() > 60 * 10 && dur.num_minutes() < 60 * 240 {
459-
// date_bin 1 hour
486+
} else if dur.num_minutes() < 60 * 24 * 10 {
487+
// 10 days = 1 hour bin
460488
format!(
461-
"CAST(DATE_BIN('1 hour', \"{}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as start_time, DATE_BIN('1 hour', \"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1 hour' as end_time",
462-
self.stream
489+
"CAST(DATE_BIN('1h', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as {start_time_col_name}, DATE_BIN('1h', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1h' as {end_time_col_name}"
463490
)
464491
} else {
465-
// date_bin 1 day
492+
// 1 day
466493
format!(
467-
"CAST(DATE_BIN('1 day', \"{}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as start_time, DATE_BIN('1 day', \"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1 day' as end_time",
468-
self.stream
494+
"CAST(DATE_BIN('1d', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') AS TEXT) as {start_time_col_name}, DATE_BIN('1d', \"{table_name}\".\"{time_column}\", TIMESTAMP '1970-01-01 00:00:00+00') + INTERVAL '1d' as {end_time_col_name}"
469495
)
470496
};
471497

472498
let query = if let Some(conditions) = &count_conditions.conditions {
473499
let f = get_filter_string(conditions).map_err(QueryError::CustomError)?;
474500
format!(
475-
"SELECT {date_bin}, COUNT(*) as count FROM \"{}\" WHERE {} GROUP BY end_time,start_time ORDER BY end_time",
476-
self.stream, f
501+
"SELECT {date_bin}, COUNT(*) as count FROM \"{table_name}\" WHERE {} GROUP BY {end_time_col_name},{start_time_col_name} ORDER BY {end_time_col_name}",
502+
f
477503
)
478504
} else {
479505
format!(
480-
"SELECT {date_bin}, COUNT(*) as count FROM \"{}\" GROUP BY end_time,start_time ORDER BY end_time",
481-
self.stream
506+
"SELECT {date_bin}, COUNT(*) as count FROM \"{table_name}\" GROUP BY {end_time_col_name},{start_time_col_name} ORDER BY {end_time_col_name}",
482507
)
483508
};
484509
Ok(query)

0 commit comments

Comments
 (0)