package alert import ( "path/filepath" "strings" "testing" "time" "nasmon/internal/collect" "nasmon/internal/config" "nasmon/internal/store" ) func keys(ev []Event) []string { out := make([]string, len(ev)) for i, e := range ev { out[i] = e.Key } return out } func has(ev []Event, key string) bool { for _, e := range ev { if e.Key == key { return true } } return false } func TestEvaluateRules(t *testing.T) { now := time.Date(2025, 9, 13, 12, 0, 0, 0, time.Local) old := now.Add(-60 * 24 * time.Hour) th := config.Defaults().Thresholds in := Input{ Now: now, Th: th, PoolsOK: true, RootOK: true, Smarts: []collect.Smart{ {Device: "/dev/sda", Serial: "A", Model: "HDD", Supported: true, Healthy: true, HasTemp: true, TempC: 55, Attrs: []collect.SmartAttr{{ID: 5, Raw: 12}, {ID: 199, Raw: 5}}}, {Device: "/dev/sdb", Serial: "B", Model: "HDD", Supported: true, Healthy: false, HasTemp: true, TempC: 30, LastTest: &collect.SelfTest{Type: "Short offline", Status: "Completed: read failure", Passed: false}}, {Device: "/dev/nvme0n1", Serial: "N", Model: "NVMe", Supported: true, Healthy: true, NVMe: &collect.NVMeHealth{PercentageUsed: 95, AvailableSpare: 100}}, {Device: "/dev/sdc", Serial: "C", Model: "HDD", Supported: true, Healthy: true, HasTemp: true, TempC: 35}, }, Prev: map[string]*collect.Smart{"A": {Serial: "A", Attrs: []collect.SmartAttr{{ID: 199, Raw: 3}}}}, Disks: []collect.Disk{{Serial: "A"}, {Serial: "B"}, {Serial: "N"}, {Serial: "C"}}, Seen: []store.SeenDisk{{Serial: "A"}, {Serial: "GONE", Model: "Old", Device: "/dev/sdz"}}, Pools: []collect.Pool{ {Name: "good", Health: "ONLINE", CapPct: 40, Size: 100, Free: 60, LastScrub: &now, Errors: "No known data errors"}, {Name: "bad", Health: "DEGRADED", CapPct: 90, Size: 100, Free: 10, LastScrub: &old, Vdevs: []collect.Vdev{{Name: "sda", Cksum: 2}}, Errors: "No known data errors"}, {Name: "never", Health: "ONLINE", CapPct: 10, Size: 100, Free: 90}, }, CPUTemp: 85, HasCPU: true, RootFS: collect.FSUsage{Total: 100, Free: 5}, } ev := Evaluate(in) want := []string{ "smart:A:attr", "smart:A:crc", "smart:A:temp", "smart:B:health", "smart:B:selftest", "smart:N:nvme", "disk:GONE:missing", "zfs:bad:health", "zfs:bad:errors", "zfs:bad:capacity", "zfs:bad:scrub", "zfs:never:scrub", "sys:cpu:temp", "sys:rootfs:space", } for _, k := range want { if !has(ev, k) { t.Errorf("missing event %s; got %v", k, keys(ev)) } } if len(ev) != len(want) { t.Errorf("got %d events, want %d: %v", len(ev), len(want), keys(ev)) } for _, e := range ev { if strings.HasPrefix(e.Key, "smart:C:") || strings.HasPrefix(e.Key, "zfs:good:") { t.Errorf("unexpected event %s", e.Key) } if e.Key == "smart:A:attr" && e.Level != LevelCrit { t.Errorf("reallocated sectors must be crit, got %s", e.Level) } } // timeouts / reported-uncorrect alone are only a warning ev = Evaluate(Input{Now: now, Th: th, Smarts: []collect.Smart{{Serial: "S", Supported: true, Healthy: true, Attrs: []collect.SmartAttr{{ID: 187, Raw: 6}, {ID: 188, Raw: 3}}}}}) if len(ev) != 1 || ev[0].Key != "smart:S:attr" || ev[0].Level != LevelWarn { t.Errorf("187/188 only: %+v", ev) } } func TestEvaluateDisabledThresholds(t *testing.T) { th := config.Thresholds{} // all zero = disabled in := Input{Now: time.Now(), Th: th, PoolsOK: true, RootOK: true, HasCPU: true, CPUTemp: 99, Smarts: []collect.Smart{{Serial: "A", Supported: true, Healthy: true, HasTemp: true, TempC: 99}}, Pools: []collect.Pool{{Name: "p", Health: "ONLINE", CapPct: 99, Errors: "No known data errors"}}, RootFS: collect.FSUsage{Total: 100, Free: 1}} if ev := Evaluate(in); len(ev) != 0 { t.Errorf("thresholds disabled but got %v", keys(ev)) } } type mailSink struct { subjects []string bodies []string fail bool } func (m *mailSink) send(_ config.SMTP, subject, body string) error { if m.fail { return errTest } m.subjects = append(m.subjects, subject) m.bodies = append(m.bodies, body) return nil } type testErr struct{} func (testErr) Error() string { return "boom" } var errTest = testErr{} func newManager(t *testing.T) (*Manager, *mailSink) { t.Helper() dir := t.TempDir() st, err := store.Open(filepath.Join(dir, "t.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { st.Close() }) cfg, err := config.Load(filepath.Join(dir, "config.json")) if err != nil { t.Fatal(err) } c := cfg.Get() c.SMTP.Host, c.SMTP.To = "localhost", []string{"a@b"} c.RemindHours = 2 if err := cfg.Save(c); err != nil { t.Fatal(err) } sink := &mailSink{} return &Manager{Store: st, Config: cfg, Send: sink.send}, sink } func TestReconcileLifecycle(t *testing.T) { m, sink := newManager(t) t0 := time.Date(2025, 9, 13, 10, 0, 0, 0, time.UTC) ev := []Event{{Key: "zfs:tank:health", Level: LevelCrit, Title: "Pool tank is DEGRADED", Detail: "d"}} out, err := m.Reconcile(Input{Now: t0}, ev) if err != nil { t.Fatal(err) } if len(out.Opened) != 1 || !out.Mailed || len(sink.subjects) != 1 { t.Fatalf("open: %+v mails=%v", out, sink.subjects) } if !strings.Contains(sink.subjects[0], "Pool tank is DEGRADED") { t.Errorf("subject = %q", sink.subjects[0]) } // same event again shortly after: nothing new, no mail out, _ = m.Reconcile(Input{Now: t0.Add(time.Hour)}, ev) if len(out.Opened) != 0 || out.Mailed || len(sink.subjects) != 1 { t.Fatalf("no-change cycle mailed: %+v", out) } // after RemindHours: reminder mail out, _ = m.Reconcile(Input{Now: t0.Add(3 * time.Hour)}, ev) if len(out.Reminded) != 1 || !out.Mailed || len(sink.subjects) != 2 { t.Fatalf("reminder: %+v mails=%v", out, sink.subjects) } if !strings.Contains(sink.bodies[1], "STILL OPEN") { t.Errorf("reminder body: %q", sink.bodies[1]) } // frozen prefix: alert must survive a cycle without data out, _ = m.Reconcile(Input{Now: t0.Add(4 * time.Hour), Frozen: []string{"zfs:"}}, nil) if len(out.Resolved) != 0 { t.Fatalf("frozen alert was resolved") } // condition gone: resolved + mail out, _ = m.Reconcile(Input{Now: t0.Add(5 * time.Hour)}, nil) if len(out.Resolved) != 1 || !out.Mailed || !strings.Contains(sink.bodies[len(sink.bodies)-1], "RESOLVED") { t.Fatalf("resolve: %+v", out) } openNow, _ := m.Store.OpenAlerts() if len(openNow) != 0 { t.Errorf("still open: %+v", openNow) } } func TestReconcileMailFailureKeepsUnnotified(t *testing.T) { m, sink := newManager(t) sink.fail = true t0 := time.Now() ev := []Event{{Key: "smart:A:health", Level: LevelCrit, Title: "x", Detail: "y"}} out, _ := m.Reconcile(Input{Now: t0}, ev) if out.Mailed || out.MailErr == nil { t.Fatalf("expected mail error: %+v", out) } open, _ := m.Store.OpenAlerts() if len(open) != 1 || open[0].LastNotifiedAt != nil { t.Fatalf("alert should be open and un-notified: %+v", open) } // within the backoff window nothing is attempted sink.fail = false out, _ = m.Reconcile(Input{Now: t0.Add(time.Minute)}, ev) if out.Mailed || out.MailErr != nil { t.Fatalf("mail attempted inside backoff: %+v", out) } // after the backoff the un-notified alert is retried as new out, _ = m.Reconcile(Input{Now: t0.Add(mailRetryAfter + time.Minute)}, ev) if !out.Mailed || len(out.Opened) != 1 { t.Fatalf("un-notified alert not retried: %+v", out) } } func TestReconcileAcknowledged(t *testing.T) { m, sink := newManager(t) t0 := time.Date(2025, 9, 13, 10, 0, 0, 0, time.UTC) ev := []Event{{Key: "smart:A:attr", Level: LevelWarn, Title: "SMART attributes degraded", Detail: "187 Reported_Uncorrect raw=6"}} out, _ := m.Reconcile(Input{Now: t0}, ev) if len(out.Opened) != 1 || len(sink.subjects) != 1 { t.Fatalf("open: %+v", out) } id := out.Opened[0].ID if err := m.Store.AckAlert(id, t0.Add(time.Minute)); err != nil { t.Fatal(err) } // acknowledged + unchanged: no reminder even after RemindHours out, _ = m.Reconcile(Input{Now: t0.Add(5 * time.Hour)}, ev) if out.Mailed || len(out.Reminded) != 0 { t.Fatalf("acked alert reminded: %+v", out) } if act, _ := m.Store.ActiveAlerts(); len(act) != 0 { t.Fatalf("acked alert still active: %+v", act) } // counters grew: re-raised as new, mailed, visible again ev[0].Detail = "187 Reported_Uncorrect raw=9" out, _ = m.Reconcile(Input{Now: t0.Add(6 * time.Hour)}, ev) if len(out.Opened) != 1 || !out.Mailed || len(sink.subjects) != 2 { t.Fatalf("re-raise: %+v mails=%v", out, sink.subjects) } act, _ := m.Store.ActiveAlerts() if len(act) != 1 || act[0].Detail != "187 Reported_Uncorrect raw=9" || act[0].Acked() { t.Fatalf("re-raised alert: %+v", act) } // acknowledged again, then escalated warn -> crit: re-raised _ = m.Store.AckAlert(id, t0.Add(7*time.Hour)) ev[0].Level = LevelCrit out, _ = m.Reconcile(Input{Now: t0.Add(8 * time.Hour)}, ev) if len(out.Opened) != 1 || len(sink.subjects) != 3 { t.Fatalf("escalation not re-raised: %+v", out) } // acknowledged, then the problem disappears: resolved normally _ = m.Store.AckAlert(id, t0.Add(9*time.Hour)) out, _ = m.Reconcile(Input{Now: t0.Add(10 * time.Hour)}, nil) if len(out.Resolved) != 1 || len(sink.subjects) != 4 { t.Fatalf("resolve of acked: %+v", out) } if open, _ := m.Store.OpenAlerts(); len(open) != 0 { t.Fatalf("still open: %+v", open) } } func TestStoreAlertLog(t *testing.T) { m, _ := newManager(t) st := m.Store t0 := time.Now().Add(-30 * 24 * time.Hour) old, _ := st.OpenAlert("k:old", LevelWarn, "old", "d", t0) _ = st.ResolveAlert(old.ID, t0.Add(time.Hour)) fresh, _ := st.OpenAlert("k:fresh", LevelWarn, "fresh", "d", time.Now().Add(-2*time.Hour)) _ = st.ResolveAlert(fresh.ID, time.Now().Add(-time.Hour)) cur, _ := st.OpenAlert("k:cur", LevelCrit, "cur", "d", time.Now()) recent, _ := st.RecentAlerts(100, time.Now().Add(-7*24*time.Hour)) if len(recent) != 2 { t.Fatalf("recent = %+v", recent) } if n, _ := st.CountResolvedBefore(time.Now().Add(-7 * 24 * time.Hour)); n != 1 { t.Errorf("hidden count = %d", n) } all, _ := st.RecentAlerts(100, time.Time{}) if len(all) != 3 { t.Fatalf("all = %+v", all) } // open alerts cannot be deleted, resolved ones can _ = st.DeleteAlert(cur.ID) _ = st.DeleteAlert(old.ID) all, _ = st.RecentAlerts(100, time.Time{}) if len(all) != 2 { t.Fatalf("after delete = %+v", all) } _ = st.PruneAlerts(time.Now().Add(-time.Minute)) all, _ = st.RecentAlerts(100, time.Time{}) if len(all) != 1 || all[0].ID != cur.ID { t.Fatalf("after prune = %+v", all) } }