cloudflare/pint
Publicmirrored from https://github.com/cloudflare/pintAvailable
internal/checks/alerts_count.go
112lines · modecode
| 1 | package checks |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "fmt" |
| 6 | "sort" |
| 7 | "time" |
| 8 | |
| 9 | "github.com/cloudflare/pint/internal/parser" |
| 10 | "github.com/cloudflare/pint/internal/promapi" |
| 11 | ) |
| 12 | |
| 13 | const ( |
| 14 | AlertsCheckName = "alerts/count" |
| 15 | ) |
| 16 | |
| 17 | func NewAlertsCheck(prom *promapi.Prometheus, lookBack, step, resolve time.Duration) AlertsCheck { |
| 18 | return AlertsCheck{ |
| 19 | prom: prom, |
| 20 | lookBack: lookBack, |
| 21 | step: step, |
| 22 | resolve: resolve, |
| 23 | } |
| 24 | } |
| 25 | |
| 26 | type AlertsCheck struct { |
| 27 | prom *promapi.Prometheus |
| 28 | lookBack time.Duration |
| 29 | step time.Duration |
| 30 | resolve time.Duration |
| 31 | } |
| 32 | |
| 33 | func (c AlertsCheck) String() string { |
| 34 | return fmt.Sprintf("%s(%s)", AlertsCheckName, c.prom.Name()) |
| 35 | } |
| 36 | |
| 37 | func (c AlertsCheck) Reporter() string { |
| 38 | return AlertsCheckName |
| 39 | } |
| 40 | |
| 41 | func (c AlertsCheck) Check(ctx context.Context, rule parser.Rule) (problems []Problem) { |
| 42 | if rule.AlertingRule == nil { |
| 43 | return |
| 44 | } |
| 45 | |
| 46 | if rule.AlertingRule.Expr.SyntaxError != nil { |
| 47 | return |
| 48 | } |
| 49 | |
| 50 | end := time.Now() |
| 51 | start := end.Add(-1 * c.lookBack) |
| 52 | |
| 53 | qr, err := c.prom.RangeQuery(ctx, rule.AlertingRule.Expr.Value.Value, start, end, c.step) |
| 54 | if err != nil { |
| 55 | problems = append(problems, Problem{ |
| 56 | Fragment: rule.AlertingRule.Expr.Value.Value, |
| 57 | Lines: rule.AlertingRule.Expr.Lines(), |
| 58 | Reporter: c.Reporter(), |
| 59 | Text: fmt.Sprintf("query using %s failed with: %s", c.prom.Name(), err), |
| 60 | Severity: Bug, |
| 61 | }) |
| 62 | return |
| 63 | } |
| 64 | |
| 65 | var forDur time.Duration |
| 66 | if rule.AlertingRule.For != nil { |
| 67 | forDur, _ = time.ParseDuration(rule.AlertingRule.For.Value.Value) |
| 68 | } |
| 69 | |
| 70 | var alerts int |
| 71 | for _, sample := range qr.Samples { |
| 72 | var isAlerting, isNew bool |
| 73 | var firstTime, lastTime time.Time |
| 74 | for _, value := range sample.Values { |
| 75 | isNew = value.Timestamp.Time().After(lastTime.Add(c.step)) |
| 76 | if isNew { |
| 77 | if rule.AlertingRule.For != nil { |
| 78 | isAlerting = false |
| 79 | } else { |
| 80 | isAlerting = true |
| 81 | alerts++ |
| 82 | } |
| 83 | firstTime = value.Timestamp.Time() |
| 84 | } else { |
| 85 | if !isAlerting && rule.AlertingRule.For != nil { |
| 86 | if !value.Timestamp.Time().Before(firstTime.Add(forDur)) { |
| 87 | isAlerting = true |
| 88 | alerts++ |
| 89 | } |
| 90 | } |
| 91 | } |
| 92 | lastTime = value.Timestamp.Time() |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | lines := []int{} |
| 97 | lines = append(lines, rule.AlertingRule.Expr.Lines()...) |
| 98 | if rule.AlertingRule.For != nil { |
| 99 | lines = append(lines, rule.AlertingRule.For.Lines()...) |
| 100 | } |
| 101 | sort.Ints(lines) |
| 102 | |
| 103 | delta := qr.End.Sub(qr.Start) |
| 104 | problems = append(problems, Problem{ |
| 105 | Fragment: rule.AlertingRule.Expr.Value.Value, |
| 106 | Lines: lines, |
| 107 | Reporter: c.Reporter(), |
| 108 | Text: fmt.Sprintf("query using %s would trigger %d alert(s) in the last %s", c.prom.Name(), alerts, promapi.HumanizeDuration(delta)), |
| 109 | Severity: Information, |
| 110 | }) |
| 111 | return |
| 112 | } |
| 113 | |