package alert import ( "fmt" "log" "os" "strings" "time" "nasmon/internal/config" "nasmon/internal/store" ) // Manager reconciles evaluated events with the alerts table and notifies. type Manager struct { Store *store.Store Config *config.Store Send func(cfg config.SMTP, subject, body string) error lastErr string // last mail error logged, to avoid repeating it every cycle nextMailTry time.Time // backoff after a delivery failure } const mailRetryAfter = 10 * time.Minute // Outcome describes what one reconciliation did. type Outcome struct { Opened []store.Alert Resolved []store.Alert Reminded []store.Alert Mailed bool MailErr error } func escalated(old, cur string) bool { return old == LevelWarn && cur == LevelCrit } func frozen(key string, prefixes []string) bool { for _, p := range prefixes { if strings.HasPrefix(key, p) { return true } } return false } // Reconcile opens/resolves alerts based on the active events and sends one // email describing everything that changed in this cycle. func (m *Manager) Reconcile(in Input, events []Event) (Outcome, error) { var out Outcome cfg := m.Config.Get() now := in.Now open, err := m.Store.OpenAlerts() if err != nil { return out, err } openByKey := map[string]store.Alert{} for _, a := range open { openByKey[a.Key] = a } active := map[string]Event{} for _, e := range events { active[e.Key] = e } // alerts re-raised in this cycle (acknowledged, but the problem changed) reraised := map[int64]bool{} for _, e := range events { if a, ok := openByKey[e.Key]; ok { changed := a.Detail != e.Detail || a.Level != e.Level if changed { _ = m.Store.UpdateAlertDetail(a.ID, e.Level, e.Detail) } // an acknowledged alert stays quiet until the problem gets worse: // escalated level or different details (grown counters, new errors) if a.Acked() && (a.Detail != e.Detail || escalated(a.Level, e.Level)) { _ = m.Store.ReraiseAlert(a.ID) a.AckedAt, a.LastNotifiedAt, a.Level, a.Detail = nil, nil, e.Level, e.Detail openByKey[e.Key] = a reraised[a.ID] = true log.Printf("alert re-raised: %s — %s", a.Key, a.Title) out.Opened = append(out.Opened, a) } continue } a, err := m.Store.OpenAlert(e.Key, e.Level, e.Title, e.Detail, now) if err != nil { return out, err } log.Printf("alert opened: %s — %s", a.Key, a.Title) out.Opened = append(out.Opened, a) } remind := time.Duration(cfg.RemindHours) * time.Hour for _, a := range open { if _, still := active[a.Key]; still { if reraised[a.ID] { continue } switch { case a.Acked(): // acknowledged: no reminders, no retries case a.LastNotifiedAt == nil: // never delivered (mail failed earlier): retry as new out.Opened = append(out.Opened, a) case remind > 0 && now.Sub(*a.LastNotifiedAt) >= remind: out.Reminded = append(out.Reminded, a) } continue } if frozen(a.Key, in.Frozen) { continue } if err := m.Store.ResolveAlert(a.ID, now); err != nil { return out, err } log.Printf("alert resolved: %s — %s", a.Key, a.Title) out.Resolved = append(out.Resolved, a) } if !cfg.AlertsEnabled || m.Send == nil { return out, nil } if len(out.Opened) == 0 && len(out.Reminded) == 0 && (len(out.Resolved) == 0 || !cfg.NotifyResolved) { return out, nil } if now.Before(m.nextMailTry) { return out, nil } subject, body := compose(cfg, out) if err := m.Send(cfg.SMTP, subject, body); err != nil { out.MailErr = err m.nextMailTry = now.Add(mailRetryAfter) if err.Error() != m.lastErr { log.Printf("alert mail failed (retry in %s): %v", mailRetryAfter, err) m.lastErr = err.Error() } return out, nil } m.lastErr, m.nextMailTry = "", time.Time{} out.Mailed = true var ids []int64 for _, a := range append(append([]store.Alert{}, out.Opened...), out.Reminded...) { ids = append(ids, a.ID) } _ = m.Store.MarkNotified(ids, now) return out, nil } func compose(cfg config.Settings, o Outcome) (subject, body string) { host, _ := os.Hostname() var parts []string if n := len(o.Opened); n > 0 { parts = append(parts, fmt.Sprintf("%d new", n)) } if n := len(o.Reminded); n > 0 { parts = append(parts, fmt.Sprintf("%d still open", n)) } if n := len(o.Resolved); n > 0 && cfg.NotifyResolved { parts = append(parts, fmt.Sprintf("%d resolved", n)) } subject = strings.TrimSpace(cfg.SMTP.SubjectPrefix + " " + strings.Join(parts, ", ") + " on " + host) if len(o.Opened) == 1 && len(o.Reminded) == 0 && len(o.Resolved) == 0 { subject = strings.TrimSpace(cfg.SMTP.SubjectPrefix + " " + o.Opened[0].Title) } var b strings.Builder fmt.Fprintf(&b, "nasmon on %s, %s\n\n", host, time.Now().Format("2006-01-02 15:04:05 MST")) section := func(title string, list []store.Alert) { if len(list) == 0 { return } fmt.Fprintf(&b, "== %s ==\n\n", title) for _, a := range list { fmt.Fprintf(&b, "[%s] %s\n", strings.ToUpper(a.Level), a.Title) for _, l := range strings.Split(a.Detail, "\n") { fmt.Fprintf(&b, " %s\n", l) } fmt.Fprintf(&b, " since %s\n\n", a.OpenedAt.Format("2006-01-02 15:04")) } } section("NEW", o.Opened) section("STILL OPEN", o.Reminded) if cfg.NotifyResolved { section("RESOLVED", o.Resolved) } return subject, b.String() }