use super::funejson::Value; use super::monitor_burn_rate::burn_rate; use super::monitor_burn_rate_alert_data::SRE_WORKBOOK; // bad/total >= threshold/1000 x (10000 - target)/10000, cross-multiplied so a // burn of 14.3995x never fires a 14.4x rule by rounding up. fn at_or_above(w: &WindowCount, threshold_milli: i64, target_basis_points: i64) -> bool { if w.total_events == 0 { return false; } w.bad_events as i128 * 10_000_000 >= threshold_milli as i128 * w.total_events as i128 * (10000 - target_basis_points) as i128 } /// Multiwindow, multi-burn-rate SLO alerting, as the Google SRE workbook /// recommends: a rule fires only when both its long window (enough budget /// spent to matter) and its short window (still happening now) burn at or /// above its threshold. With rules None it uses the workbook's Table 5-8. /// /// min_events guards small samples: a rule whose long window saw fewer /// events stays quiet, since 2 bad checks out of 11 is a 36x burn of a /// 99.5% budget but proves little. The short window is not guarded: it only /// confirms the burn is still going on. /// /// # Panics /// Panics on a bad target, window, count or rule, or a rule whose window has no counts. pub fn burn_rate_alert( target_basis_points: i64, windows: &[WindowCount], rules: Option<&[BurnRule]>, min_events: Option, ) -> BurnAlert { if !(1..=9999).contains(&target_basis_points) { panic!( "targetBasisPoints must be a whole number from 1 to 9999 (10000 leaves no error budget), received {}", target_basis_points ); } if let Some(m) = min_events { if m < 0 { panic!("minEvents must be null or a whole number of at least 0, received {}", m); } } let minimum = min_events.unwrap_or(0); let mut counts: Vec<(&WindowCount, i64)> = Vec::new(); for w in windows { if w.window_seconds < 1 { panic!("windowSeconds must be at least 1, received {}", w.window_seconds); } if counts.iter().any(|(c, _)| c.window_seconds == w.window_seconds) { panic!("duplicate counts for a {}-second window", w.window_seconds); } // burn_rate checks the counts, so every window is checked, used or not. counts.push((w, burn_rate(target_basis_points, w.total_events, w.bad_events))); } let workbook: Vec; let chosen: &[BurnRule] = match rules { Some(r) => r, None => { workbook = SRE_WORKBOOK .iter() .map(|r| BurnRule { severity: r.severity.to_string(), long_window_seconds: r.long_window_seconds, short_window_seconds: r.short_window_seconds, burn_rate_milli: r.burn_rate_milli, }) .collect(); &workbook } }; if chosen.is_empty() { panic!("rules must not be empty; pass null for the SRE workbook rules"); } let find = |seconds: i64| -> (&WindowCount, i64) { match counts.iter().find(|(c, _)| c.window_seconds == seconds) { Some((c, burn)) => (*c, *burn), None => panic!("no counts for a {}-second window", seconds), } }; let mut results: Vec = Vec::new(); let mut severity: Option = None; for rule in chosen { if rule.severity.is_empty() { panic!("severity must not be empty"); } if rule.short_window_seconds < 1 { panic!("shortWindowSeconds must be at least 1, received {}", rule.short_window_seconds); } if rule.short_window_seconds > rule.long_window_seconds { panic!( "shortWindowSeconds must not exceed longWindowSeconds: {} > {}", rule.short_window_seconds, rule.long_window_seconds ); } if rule.burn_rate_milli < 1 { panic!("burnRateMilli must be at least 1, received {}", rule.burn_rate_milli); } let (long_count, long_burn) = find(rule.long_window_seconds); let (short_count, short_burn) = find(rule.short_window_seconds); let enough_events = long_count.total_events >= minimum; let firing = enough_events && at_or_above(long_count, rule.burn_rate_milli, target_basis_points) && at_or_above(short_count, rule.burn_rate_milli, target_basis_points); if firing && severity.is_none() { severity = Some(rule.severity.clone()); } results.push(BurnRuleResult { severity: rule.severity.clone(), long_window_seconds: rule.long_window_seconds, short_window_seconds: rule.short_window_seconds, threshold_milli: rule.burn_rate_milli, long_burn_milli: long_burn, short_burn_milli: short_burn, enough_events, firing, }); } BurnAlert { firing: severity.is_some(), severity, rules: results } } pub fn window_count_from_value(v: &Value) -> WindowCount { WindowCount { window_seconds: v.get("windowSeconds").as_i64(), total_events: v.get("totalEvents").as_i64(), bad_events: v.get("badEvents").as_i64(), } } pub fn burn_rule_from_value(v: &Value) -> BurnRule { BurnRule { severity: v.get("severity").as_str().to_string(), long_window_seconds: v.get("longWindowSeconds").as_i64(), short_window_seconds: v.get("shortWindowSeconds").as_i64(), burn_rate_milli: v.get("burnRateMilli").as_i64(), } } pub fn burn_rule_result_to_value(r: &BurnRuleResult) -> Value { Value::obj(vec![ ("severity", Value::str(&r.severity)), ("longWindowSeconds", Value::Int(r.long_window_seconds)), ("shortWindowSeconds", Value::Int(r.short_window_seconds)), ("thresholdMilli", Value::Int(r.threshold_milli)), ("longBurnMilli", Value::Int(r.long_burn_milli)), ("shortBurnMilli", Value::Int(r.short_burn_milli)), ("enoughEvents", Value::Bool(r.enough_events)), ("firing", Value::Bool(r.firing)), ]) } pub fn burn_alert_to_value(a: &BurnAlert) -> Value { Value::obj(vec![ ("firing", Value::Bool(a.firing)), ("severity", match &a.severity { Some(s) => Value::str(s), None => Value::Null }), ("rules", Value::Arr(a.rules.iter().map(burn_rule_result_to_value).collect())), ]) } pub fn fune_vector(args: &[Value]) -> Value { let windows: Vec = args[1].as_arr().iter().map(window_count_from_value).collect(); let rules: Option> = if args[2].is_null() { None } else { Some(args[2].as_arr().iter().map(burn_rule_from_value).collect()) }; // minEvents: null, or a whole number; refuse a fractional one with the text TypeScript and Python use. let min_events: Option = match &args[3] { Value::Null => None, Value::Int(i) => Some(*i), other => panic!("minEvents must be null or a whole number of at least 0, received {}", other.as_f64()), }; burn_alert_to_value(&burn_rate_alert(args[0].as_i64(), &windows, rules.as_deref(), min_events)) }