Initial import: nasmon health monitor, split out of home_nas
Go service (nasmon/) plus nfpm packaging (packaging/), carried over from the home_nas monorepo. Adds rpm packaging alongside the existing deb build (Fedora/RHEL/Rocky dependency names via nfpm overrides, per-format systemd unit path, dpkg+rpm-compatible maintainer scripts) and a Gitea Actions workflow that builds both formats on every push. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,305 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"nasmon/internal/collect"
|
||||
"nasmon/internal/config"
|
||||
"nasmon/internal/store"
|
||||
)
|
||||
|
||||
func keys(ev []Event) []string {
|
||||
out := make([]string, len(ev))
|
||||
for i, e := range ev {
|
||||
out[i] = e.Key
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func has(ev []Event, key string) bool {
|
||||
for _, e := range ev {
|
||||
if e.Key == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestEvaluateRules(t *testing.T) {
|
||||
now := time.Date(2025, 9, 13, 12, 0, 0, 0, time.Local)
|
||||
old := now.Add(-60 * 24 * time.Hour)
|
||||
th := config.Defaults().Thresholds
|
||||
in := Input{
|
||||
Now: now, Th: th, PoolsOK: true, RootOK: true,
|
||||
Smarts: []collect.Smart{
|
||||
{Device: "/dev/sda", Serial: "A", Model: "HDD", Supported: true, Healthy: true, HasTemp: true, TempC: 55,
|
||||
Attrs: []collect.SmartAttr{{ID: 5, Raw: 12}, {ID: 199, Raw: 5}}},
|
||||
{Device: "/dev/sdb", Serial: "B", Model: "HDD", Supported: true, Healthy: false, HasTemp: true, TempC: 30,
|
||||
LastTest: &collect.SelfTest{Type: "Short offline", Status: "Completed: read failure", Passed: false}},
|
||||
{Device: "/dev/nvme0n1", Serial: "N", Model: "NVMe", Supported: true, Healthy: true,
|
||||
NVMe: &collect.NVMeHealth{PercentageUsed: 95, AvailableSpare: 100}},
|
||||
{Device: "/dev/sdc", Serial: "C", Model: "HDD", Supported: true, Healthy: true, HasTemp: true, TempC: 35},
|
||||
},
|
||||
Prev: map[string]*collect.Smart{"A": {Serial: "A", Attrs: []collect.SmartAttr{{ID: 199, Raw: 3}}}},
|
||||
Disks: []collect.Disk{{Serial: "A"}, {Serial: "B"}, {Serial: "N"}, {Serial: "C"}},
|
||||
Seen: []store.SeenDisk{{Serial: "A"}, {Serial: "GONE", Model: "Old", Device: "/dev/sdz"}},
|
||||
Pools: []collect.Pool{
|
||||
{Name: "good", Health: "ONLINE", CapPct: 40, Size: 100, Free: 60, LastScrub: &now, Errors: "No known data errors"},
|
||||
{Name: "bad", Health: "DEGRADED", CapPct: 90, Size: 100, Free: 10, LastScrub: &old,
|
||||
Vdevs: []collect.Vdev{{Name: "sda", Cksum: 2}}, Errors: "No known data errors"},
|
||||
{Name: "never", Health: "ONLINE", CapPct: 10, Size: 100, Free: 90},
|
||||
},
|
||||
CPUTemp: 85, HasCPU: true,
|
||||
RootFS: collect.FSUsage{Total: 100, Free: 5},
|
||||
}
|
||||
ev := Evaluate(in)
|
||||
want := []string{
|
||||
"smart:A:attr", "smart:A:crc", "smart:A:temp",
|
||||
"smart:B:health", "smart:B:selftest",
|
||||
"smart:N:nvme",
|
||||
"disk:GONE:missing",
|
||||
"zfs:bad:health", "zfs:bad:errors", "zfs:bad:capacity", "zfs:bad:scrub",
|
||||
"zfs:never:scrub",
|
||||
"sys:cpu:temp", "sys:rootfs:space",
|
||||
}
|
||||
for _, k := range want {
|
||||
if !has(ev, k) {
|
||||
t.Errorf("missing event %s; got %v", k, keys(ev))
|
||||
}
|
||||
}
|
||||
if len(ev) != len(want) {
|
||||
t.Errorf("got %d events, want %d: %v", len(ev), len(want), keys(ev))
|
||||
}
|
||||
for _, e := range ev {
|
||||
if strings.HasPrefix(e.Key, "smart:C:") || strings.HasPrefix(e.Key, "zfs:good:") {
|
||||
t.Errorf("unexpected event %s", e.Key)
|
||||
}
|
||||
if e.Key == "smart:A:attr" && e.Level != LevelCrit {
|
||||
t.Errorf("reallocated sectors must be crit, got %s", e.Level)
|
||||
}
|
||||
}
|
||||
// timeouts / reported-uncorrect alone are only a warning
|
||||
ev = Evaluate(Input{Now: now, Th: th, Smarts: []collect.Smart{{Serial: "S", Supported: true, Healthy: true,
|
||||
Attrs: []collect.SmartAttr{{ID: 187, Raw: 6}, {ID: 188, Raw: 3}}}}})
|
||||
if len(ev) != 1 || ev[0].Key != "smart:S:attr" || ev[0].Level != LevelWarn {
|
||||
t.Errorf("187/188 only: %+v", ev)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateDisabledThresholds(t *testing.T) {
|
||||
th := config.Thresholds{} // all zero = disabled
|
||||
in := Input{Now: time.Now(), Th: th, PoolsOK: true, RootOK: true, HasCPU: true, CPUTemp: 99,
|
||||
Smarts: []collect.Smart{{Serial: "A", Supported: true, Healthy: true, HasTemp: true, TempC: 99}},
|
||||
Pools: []collect.Pool{{Name: "p", Health: "ONLINE", CapPct: 99, Errors: "No known data errors"}},
|
||||
RootFS: collect.FSUsage{Total: 100, Free: 1}}
|
||||
if ev := Evaluate(in); len(ev) != 0 {
|
||||
t.Errorf("thresholds disabled but got %v", keys(ev))
|
||||
}
|
||||
}
|
||||
|
||||
type mailSink struct {
|
||||
subjects []string
|
||||
bodies []string
|
||||
fail bool
|
||||
}
|
||||
|
||||
func (m *mailSink) send(_ config.SMTP, subject, body string) error {
|
||||
if m.fail {
|
||||
return errTest
|
||||
}
|
||||
m.subjects = append(m.subjects, subject)
|
||||
m.bodies = append(m.bodies, body)
|
||||
return nil
|
||||
}
|
||||
|
||||
type testErr struct{}
|
||||
|
||||
func (testErr) Error() string { return "boom" }
|
||||
|
||||
var errTest = testErr{}
|
||||
|
||||
func newManager(t *testing.T) (*Manager, *mailSink) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
st, err := store.Open(filepath.Join(dir, "t.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
cfg, err := config.Load(filepath.Join(dir, "config.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := cfg.Get()
|
||||
c.SMTP.Host, c.SMTP.To = "localhost", []string{"a@b"}
|
||||
c.RemindHours = 2
|
||||
if err := cfg.Save(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &mailSink{}
|
||||
return &Manager{Store: st, Config: cfg, Send: sink.send}, sink
|
||||
}
|
||||
|
||||
func TestReconcileLifecycle(t *testing.T) {
|
||||
m, sink := newManager(t)
|
||||
t0 := time.Date(2025, 9, 13, 10, 0, 0, 0, time.UTC)
|
||||
ev := []Event{{Key: "zfs:tank:health", Level: LevelCrit, Title: "Pool tank is DEGRADED", Detail: "d"}}
|
||||
|
||||
out, err := m.Reconcile(Input{Now: t0}, ev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(out.Opened) != 1 || !out.Mailed || len(sink.subjects) != 1 {
|
||||
t.Fatalf("open: %+v mails=%v", out, sink.subjects)
|
||||
}
|
||||
if !strings.Contains(sink.subjects[0], "Pool tank is DEGRADED") {
|
||||
t.Errorf("subject = %q", sink.subjects[0])
|
||||
}
|
||||
|
||||
// same event again shortly after: nothing new, no mail
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(time.Hour)}, ev)
|
||||
if len(out.Opened) != 0 || out.Mailed || len(sink.subjects) != 1 {
|
||||
t.Fatalf("no-change cycle mailed: %+v", out)
|
||||
}
|
||||
|
||||
// after RemindHours: reminder mail
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(3 * time.Hour)}, ev)
|
||||
if len(out.Reminded) != 1 || !out.Mailed || len(sink.subjects) != 2 {
|
||||
t.Fatalf("reminder: %+v mails=%v", out, sink.subjects)
|
||||
}
|
||||
if !strings.Contains(sink.bodies[1], "STILL OPEN") {
|
||||
t.Errorf("reminder body: %q", sink.bodies[1])
|
||||
}
|
||||
|
||||
// frozen prefix: alert must survive a cycle without data
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(4 * time.Hour), Frozen: []string{"zfs:"}}, nil)
|
||||
if len(out.Resolved) != 0 {
|
||||
t.Fatalf("frozen alert was resolved")
|
||||
}
|
||||
|
||||
// condition gone: resolved + mail
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(5 * time.Hour)}, nil)
|
||||
if len(out.Resolved) != 1 || !out.Mailed || !strings.Contains(sink.bodies[len(sink.bodies)-1], "RESOLVED") {
|
||||
t.Fatalf("resolve: %+v", out)
|
||||
}
|
||||
openNow, _ := m.Store.OpenAlerts()
|
||||
if len(openNow) != 0 {
|
||||
t.Errorf("still open: %+v", openNow)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileMailFailureKeepsUnnotified(t *testing.T) {
|
||||
m, sink := newManager(t)
|
||||
sink.fail = true
|
||||
t0 := time.Now()
|
||||
ev := []Event{{Key: "smart:A:health", Level: LevelCrit, Title: "x", Detail: "y"}}
|
||||
out, _ := m.Reconcile(Input{Now: t0}, ev)
|
||||
if out.Mailed || out.MailErr == nil {
|
||||
t.Fatalf("expected mail error: %+v", out)
|
||||
}
|
||||
open, _ := m.Store.OpenAlerts()
|
||||
if len(open) != 1 || open[0].LastNotifiedAt != nil {
|
||||
t.Fatalf("alert should be open and un-notified: %+v", open)
|
||||
}
|
||||
// within the backoff window nothing is attempted
|
||||
sink.fail = false
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(time.Minute)}, ev)
|
||||
if out.Mailed || out.MailErr != nil {
|
||||
t.Fatalf("mail attempted inside backoff: %+v", out)
|
||||
}
|
||||
// after the backoff the un-notified alert is retried as new
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(mailRetryAfter + time.Minute)}, ev)
|
||||
if !out.Mailed || len(out.Opened) != 1 {
|
||||
t.Fatalf("un-notified alert not retried: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileAcknowledged(t *testing.T) {
|
||||
m, sink := newManager(t)
|
||||
t0 := time.Date(2025, 9, 13, 10, 0, 0, 0, time.UTC)
|
||||
ev := []Event{{Key: "smart:A:attr", Level: LevelWarn, Title: "SMART attributes degraded", Detail: "187 Reported_Uncorrect raw=6"}}
|
||||
out, _ := m.Reconcile(Input{Now: t0}, ev)
|
||||
if len(out.Opened) != 1 || len(sink.subjects) != 1 {
|
||||
t.Fatalf("open: %+v", out)
|
||||
}
|
||||
id := out.Opened[0].ID
|
||||
if err := m.Store.AckAlert(id, t0.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// acknowledged + unchanged: no reminder even after RemindHours
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(5 * time.Hour)}, ev)
|
||||
if out.Mailed || len(out.Reminded) != 0 {
|
||||
t.Fatalf("acked alert reminded: %+v", out)
|
||||
}
|
||||
if act, _ := m.Store.ActiveAlerts(); len(act) != 0 {
|
||||
t.Fatalf("acked alert still active: %+v", act)
|
||||
}
|
||||
|
||||
// counters grew: re-raised as new, mailed, visible again
|
||||
ev[0].Detail = "187 Reported_Uncorrect raw=9"
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(6 * time.Hour)}, ev)
|
||||
if len(out.Opened) != 1 || !out.Mailed || len(sink.subjects) != 2 {
|
||||
t.Fatalf("re-raise: %+v mails=%v", out, sink.subjects)
|
||||
}
|
||||
act, _ := m.Store.ActiveAlerts()
|
||||
if len(act) != 1 || act[0].Detail != "187 Reported_Uncorrect raw=9" || act[0].Acked() {
|
||||
t.Fatalf("re-raised alert: %+v", act)
|
||||
}
|
||||
|
||||
// acknowledged again, then escalated warn -> crit: re-raised
|
||||
_ = m.Store.AckAlert(id, t0.Add(7*time.Hour))
|
||||
ev[0].Level = LevelCrit
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(8 * time.Hour)}, ev)
|
||||
if len(out.Opened) != 1 || len(sink.subjects) != 3 {
|
||||
t.Fatalf("escalation not re-raised: %+v", out)
|
||||
}
|
||||
|
||||
// acknowledged, then the problem disappears: resolved normally
|
||||
_ = m.Store.AckAlert(id, t0.Add(9*time.Hour))
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(10 * time.Hour)}, nil)
|
||||
if len(out.Resolved) != 1 || len(sink.subjects) != 4 {
|
||||
t.Fatalf("resolve of acked: %+v", out)
|
||||
}
|
||||
if open, _ := m.Store.OpenAlerts(); len(open) != 0 {
|
||||
t.Fatalf("still open: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreAlertLog(t *testing.T) {
|
||||
m, _ := newManager(t)
|
||||
st := m.Store
|
||||
t0 := time.Now().Add(-30 * 24 * time.Hour)
|
||||
old, _ := st.OpenAlert("k:old", LevelWarn, "old", "d", t0)
|
||||
_ = st.ResolveAlert(old.ID, t0.Add(time.Hour))
|
||||
fresh, _ := st.OpenAlert("k:fresh", LevelWarn, "fresh", "d", time.Now().Add(-2*time.Hour))
|
||||
_ = st.ResolveAlert(fresh.ID, time.Now().Add(-time.Hour))
|
||||
cur, _ := st.OpenAlert("k:cur", LevelCrit, "cur", "d", time.Now())
|
||||
|
||||
recent, _ := st.RecentAlerts(100, time.Now().Add(-7*24*time.Hour))
|
||||
if len(recent) != 2 {
|
||||
t.Fatalf("recent = %+v", recent)
|
||||
}
|
||||
if n, _ := st.CountResolvedBefore(time.Now().Add(-7 * 24 * time.Hour)); n != 1 {
|
||||
t.Errorf("hidden count = %d", n)
|
||||
}
|
||||
all, _ := st.RecentAlerts(100, time.Time{})
|
||||
if len(all) != 3 {
|
||||
t.Fatalf("all = %+v", all)
|
||||
}
|
||||
// open alerts cannot be deleted, resolved ones can
|
||||
_ = st.DeleteAlert(cur.ID)
|
||||
_ = st.DeleteAlert(old.ID)
|
||||
all, _ = st.RecentAlerts(100, time.Time{})
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("after delete = %+v", all)
|
||||
}
|
||||
_ = st.PruneAlerts(time.Now().Add(-time.Minute))
|
||||
all, _ = st.RecentAlerts(100, time.Time{})
|
||||
if len(all) != 1 || all[0].ID != cur.ID {
|
||||
t.Fatalf("after prune = %+v", all)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user