Initial import: nasmon health monitor, split out of home_nas
Go service (nasmon/) plus nfpm packaging (packaging/), carried over from the home_nas monorepo. Adds rpm packaging alongside the existing deb build (Fedora/RHEL/Rocky dependency names via nfpm overrides, per-format systemd unit path, dpkg+rpm-compatible maintainer scripts) and a Gitea Actions workflow that builds both formats on every push. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,305 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"nasmon/internal/collect"
|
||||
"nasmon/internal/config"
|
||||
"nasmon/internal/store"
|
||||
)
|
||||
|
||||
func keys(ev []Event) []string {
|
||||
out := make([]string, len(ev))
|
||||
for i, e := range ev {
|
||||
out[i] = e.Key
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func has(ev []Event, key string) bool {
|
||||
for _, e := range ev {
|
||||
if e.Key == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestEvaluateRules(t *testing.T) {
|
||||
now := time.Date(2025, 9, 13, 12, 0, 0, 0, time.Local)
|
||||
old := now.Add(-60 * 24 * time.Hour)
|
||||
th := config.Defaults().Thresholds
|
||||
in := Input{
|
||||
Now: now, Th: th, PoolsOK: true, RootOK: true,
|
||||
Smarts: []collect.Smart{
|
||||
{Device: "/dev/sda", Serial: "A", Model: "HDD", Supported: true, Healthy: true, HasTemp: true, TempC: 55,
|
||||
Attrs: []collect.SmartAttr{{ID: 5, Raw: 12}, {ID: 199, Raw: 5}}},
|
||||
{Device: "/dev/sdb", Serial: "B", Model: "HDD", Supported: true, Healthy: false, HasTemp: true, TempC: 30,
|
||||
LastTest: &collect.SelfTest{Type: "Short offline", Status: "Completed: read failure", Passed: false}},
|
||||
{Device: "/dev/nvme0n1", Serial: "N", Model: "NVMe", Supported: true, Healthy: true,
|
||||
NVMe: &collect.NVMeHealth{PercentageUsed: 95, AvailableSpare: 100}},
|
||||
{Device: "/dev/sdc", Serial: "C", Model: "HDD", Supported: true, Healthy: true, HasTemp: true, TempC: 35},
|
||||
},
|
||||
Prev: map[string]*collect.Smart{"A": {Serial: "A", Attrs: []collect.SmartAttr{{ID: 199, Raw: 3}}}},
|
||||
Disks: []collect.Disk{{Serial: "A"}, {Serial: "B"}, {Serial: "N"}, {Serial: "C"}},
|
||||
Seen: []store.SeenDisk{{Serial: "A"}, {Serial: "GONE", Model: "Old", Device: "/dev/sdz"}},
|
||||
Pools: []collect.Pool{
|
||||
{Name: "good", Health: "ONLINE", CapPct: 40, Size: 100, Free: 60, LastScrub: &now, Errors: "No known data errors"},
|
||||
{Name: "bad", Health: "DEGRADED", CapPct: 90, Size: 100, Free: 10, LastScrub: &old,
|
||||
Vdevs: []collect.Vdev{{Name: "sda", Cksum: 2}}, Errors: "No known data errors"},
|
||||
{Name: "never", Health: "ONLINE", CapPct: 10, Size: 100, Free: 90},
|
||||
},
|
||||
CPUTemp: 85, HasCPU: true,
|
||||
RootFS: collect.FSUsage{Total: 100, Free: 5},
|
||||
}
|
||||
ev := Evaluate(in)
|
||||
want := []string{
|
||||
"smart:A:attr", "smart:A:crc", "smart:A:temp",
|
||||
"smart:B:health", "smart:B:selftest",
|
||||
"smart:N:nvme",
|
||||
"disk:GONE:missing",
|
||||
"zfs:bad:health", "zfs:bad:errors", "zfs:bad:capacity", "zfs:bad:scrub",
|
||||
"zfs:never:scrub",
|
||||
"sys:cpu:temp", "sys:rootfs:space",
|
||||
}
|
||||
for _, k := range want {
|
||||
if !has(ev, k) {
|
||||
t.Errorf("missing event %s; got %v", k, keys(ev))
|
||||
}
|
||||
}
|
||||
if len(ev) != len(want) {
|
||||
t.Errorf("got %d events, want %d: %v", len(ev), len(want), keys(ev))
|
||||
}
|
||||
for _, e := range ev {
|
||||
if strings.HasPrefix(e.Key, "smart:C:") || strings.HasPrefix(e.Key, "zfs:good:") {
|
||||
t.Errorf("unexpected event %s", e.Key)
|
||||
}
|
||||
if e.Key == "smart:A:attr" && e.Level != LevelCrit {
|
||||
t.Errorf("reallocated sectors must be crit, got %s", e.Level)
|
||||
}
|
||||
}
|
||||
// timeouts / reported-uncorrect alone are only a warning
|
||||
ev = Evaluate(Input{Now: now, Th: th, Smarts: []collect.Smart{{Serial: "S", Supported: true, Healthy: true,
|
||||
Attrs: []collect.SmartAttr{{ID: 187, Raw: 6}, {ID: 188, Raw: 3}}}}})
|
||||
if len(ev) != 1 || ev[0].Key != "smart:S:attr" || ev[0].Level != LevelWarn {
|
||||
t.Errorf("187/188 only: %+v", ev)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateDisabledThresholds(t *testing.T) {
|
||||
th := config.Thresholds{} // all zero = disabled
|
||||
in := Input{Now: time.Now(), Th: th, PoolsOK: true, RootOK: true, HasCPU: true, CPUTemp: 99,
|
||||
Smarts: []collect.Smart{{Serial: "A", Supported: true, Healthy: true, HasTemp: true, TempC: 99}},
|
||||
Pools: []collect.Pool{{Name: "p", Health: "ONLINE", CapPct: 99, Errors: "No known data errors"}},
|
||||
RootFS: collect.FSUsage{Total: 100, Free: 1}}
|
||||
if ev := Evaluate(in); len(ev) != 0 {
|
||||
t.Errorf("thresholds disabled but got %v", keys(ev))
|
||||
}
|
||||
}
|
||||
|
||||
type mailSink struct {
|
||||
subjects []string
|
||||
bodies []string
|
||||
fail bool
|
||||
}
|
||||
|
||||
func (m *mailSink) send(_ config.SMTP, subject, body string) error {
|
||||
if m.fail {
|
||||
return errTest
|
||||
}
|
||||
m.subjects = append(m.subjects, subject)
|
||||
m.bodies = append(m.bodies, body)
|
||||
return nil
|
||||
}
|
||||
|
||||
type testErr struct{}
|
||||
|
||||
func (testErr) Error() string { return "boom" }
|
||||
|
||||
var errTest = testErr{}
|
||||
|
||||
func newManager(t *testing.T) (*Manager, *mailSink) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
st, err := store.Open(filepath.Join(dir, "t.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
cfg, err := config.Load(filepath.Join(dir, "config.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := cfg.Get()
|
||||
c.SMTP.Host, c.SMTP.To = "localhost", []string{"a@b"}
|
||||
c.RemindHours = 2
|
||||
if err := cfg.Save(c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &mailSink{}
|
||||
return &Manager{Store: st, Config: cfg, Send: sink.send}, sink
|
||||
}
|
||||
|
||||
func TestReconcileLifecycle(t *testing.T) {
|
||||
m, sink := newManager(t)
|
||||
t0 := time.Date(2025, 9, 13, 10, 0, 0, 0, time.UTC)
|
||||
ev := []Event{{Key: "zfs:tank:health", Level: LevelCrit, Title: "Pool tank is DEGRADED", Detail: "d"}}
|
||||
|
||||
out, err := m.Reconcile(Input{Now: t0}, ev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(out.Opened) != 1 || !out.Mailed || len(sink.subjects) != 1 {
|
||||
t.Fatalf("open: %+v mails=%v", out, sink.subjects)
|
||||
}
|
||||
if !strings.Contains(sink.subjects[0], "Pool tank is DEGRADED") {
|
||||
t.Errorf("subject = %q", sink.subjects[0])
|
||||
}
|
||||
|
||||
// same event again shortly after: nothing new, no mail
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(time.Hour)}, ev)
|
||||
if len(out.Opened) != 0 || out.Mailed || len(sink.subjects) != 1 {
|
||||
t.Fatalf("no-change cycle mailed: %+v", out)
|
||||
}
|
||||
|
||||
// after RemindHours: reminder mail
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(3 * time.Hour)}, ev)
|
||||
if len(out.Reminded) != 1 || !out.Mailed || len(sink.subjects) != 2 {
|
||||
t.Fatalf("reminder: %+v mails=%v", out, sink.subjects)
|
||||
}
|
||||
if !strings.Contains(sink.bodies[1], "STILL OPEN") {
|
||||
t.Errorf("reminder body: %q", sink.bodies[1])
|
||||
}
|
||||
|
||||
// frozen prefix: alert must survive a cycle without data
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(4 * time.Hour), Frozen: []string{"zfs:"}}, nil)
|
||||
if len(out.Resolved) != 0 {
|
||||
t.Fatalf("frozen alert was resolved")
|
||||
}
|
||||
|
||||
// condition gone: resolved + mail
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(5 * time.Hour)}, nil)
|
||||
if len(out.Resolved) != 1 || !out.Mailed || !strings.Contains(sink.bodies[len(sink.bodies)-1], "RESOLVED") {
|
||||
t.Fatalf("resolve: %+v", out)
|
||||
}
|
||||
openNow, _ := m.Store.OpenAlerts()
|
||||
if len(openNow) != 0 {
|
||||
t.Errorf("still open: %+v", openNow)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileMailFailureKeepsUnnotified(t *testing.T) {
|
||||
m, sink := newManager(t)
|
||||
sink.fail = true
|
||||
t0 := time.Now()
|
||||
ev := []Event{{Key: "smart:A:health", Level: LevelCrit, Title: "x", Detail: "y"}}
|
||||
out, _ := m.Reconcile(Input{Now: t0}, ev)
|
||||
if out.Mailed || out.MailErr == nil {
|
||||
t.Fatalf("expected mail error: %+v", out)
|
||||
}
|
||||
open, _ := m.Store.OpenAlerts()
|
||||
if len(open) != 1 || open[0].LastNotifiedAt != nil {
|
||||
t.Fatalf("alert should be open and un-notified: %+v", open)
|
||||
}
|
||||
// within the backoff window nothing is attempted
|
||||
sink.fail = false
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(time.Minute)}, ev)
|
||||
if out.Mailed || out.MailErr != nil {
|
||||
t.Fatalf("mail attempted inside backoff: %+v", out)
|
||||
}
|
||||
// after the backoff the un-notified alert is retried as new
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(mailRetryAfter + time.Minute)}, ev)
|
||||
if !out.Mailed || len(out.Opened) != 1 {
|
||||
t.Fatalf("un-notified alert not retried: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileAcknowledged(t *testing.T) {
|
||||
m, sink := newManager(t)
|
||||
t0 := time.Date(2025, 9, 13, 10, 0, 0, 0, time.UTC)
|
||||
ev := []Event{{Key: "smart:A:attr", Level: LevelWarn, Title: "SMART attributes degraded", Detail: "187 Reported_Uncorrect raw=6"}}
|
||||
out, _ := m.Reconcile(Input{Now: t0}, ev)
|
||||
if len(out.Opened) != 1 || len(sink.subjects) != 1 {
|
||||
t.Fatalf("open: %+v", out)
|
||||
}
|
||||
id := out.Opened[0].ID
|
||||
if err := m.Store.AckAlert(id, t0.Add(time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// acknowledged + unchanged: no reminder even after RemindHours
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(5 * time.Hour)}, ev)
|
||||
if out.Mailed || len(out.Reminded) != 0 {
|
||||
t.Fatalf("acked alert reminded: %+v", out)
|
||||
}
|
||||
if act, _ := m.Store.ActiveAlerts(); len(act) != 0 {
|
||||
t.Fatalf("acked alert still active: %+v", act)
|
||||
}
|
||||
|
||||
// counters grew: re-raised as new, mailed, visible again
|
||||
ev[0].Detail = "187 Reported_Uncorrect raw=9"
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(6 * time.Hour)}, ev)
|
||||
if len(out.Opened) != 1 || !out.Mailed || len(sink.subjects) != 2 {
|
||||
t.Fatalf("re-raise: %+v mails=%v", out, sink.subjects)
|
||||
}
|
||||
act, _ := m.Store.ActiveAlerts()
|
||||
if len(act) != 1 || act[0].Detail != "187 Reported_Uncorrect raw=9" || act[0].Acked() {
|
||||
t.Fatalf("re-raised alert: %+v", act)
|
||||
}
|
||||
|
||||
// acknowledged again, then escalated warn -> crit: re-raised
|
||||
_ = m.Store.AckAlert(id, t0.Add(7*time.Hour))
|
||||
ev[0].Level = LevelCrit
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(8 * time.Hour)}, ev)
|
||||
if len(out.Opened) != 1 || len(sink.subjects) != 3 {
|
||||
t.Fatalf("escalation not re-raised: %+v", out)
|
||||
}
|
||||
|
||||
// acknowledged, then the problem disappears: resolved normally
|
||||
_ = m.Store.AckAlert(id, t0.Add(9*time.Hour))
|
||||
out, _ = m.Reconcile(Input{Now: t0.Add(10 * time.Hour)}, nil)
|
||||
if len(out.Resolved) != 1 || len(sink.subjects) != 4 {
|
||||
t.Fatalf("resolve of acked: %+v", out)
|
||||
}
|
||||
if open, _ := m.Store.OpenAlerts(); len(open) != 0 {
|
||||
t.Fatalf("still open: %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreAlertLog(t *testing.T) {
|
||||
m, _ := newManager(t)
|
||||
st := m.Store
|
||||
t0 := time.Now().Add(-30 * 24 * time.Hour)
|
||||
old, _ := st.OpenAlert("k:old", LevelWarn, "old", "d", t0)
|
||||
_ = st.ResolveAlert(old.ID, t0.Add(time.Hour))
|
||||
fresh, _ := st.OpenAlert("k:fresh", LevelWarn, "fresh", "d", time.Now().Add(-2*time.Hour))
|
||||
_ = st.ResolveAlert(fresh.ID, time.Now().Add(-time.Hour))
|
||||
cur, _ := st.OpenAlert("k:cur", LevelCrit, "cur", "d", time.Now())
|
||||
|
||||
recent, _ := st.RecentAlerts(100, time.Now().Add(-7*24*time.Hour))
|
||||
if len(recent) != 2 {
|
||||
t.Fatalf("recent = %+v", recent)
|
||||
}
|
||||
if n, _ := st.CountResolvedBefore(time.Now().Add(-7 * 24 * time.Hour)); n != 1 {
|
||||
t.Errorf("hidden count = %d", n)
|
||||
}
|
||||
all, _ := st.RecentAlerts(100, time.Time{})
|
||||
if len(all) != 3 {
|
||||
t.Fatalf("all = %+v", all)
|
||||
}
|
||||
// open alerts cannot be deleted, resolved ones can
|
||||
_ = st.DeleteAlert(cur.ID)
|
||||
_ = st.DeleteAlert(old.ID)
|
||||
all, _ = st.RecentAlerts(100, time.Time{})
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("after delete = %+v", all)
|
||||
}
|
||||
_ = st.PruneAlerts(time.Now().Add(-time.Minute))
|
||||
all, _ = st.RecentAlerts(100, time.Time{})
|
||||
if len(all) != 1 || all[0].ID != cur.ID {
|
||||
t.Fatalf("after prune = %+v", all)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net"
|
||||
"net/smtp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"nasmon/internal/config"
|
||||
)
|
||||
|
||||
// SendMail delivers a plain-text message using the configured SMTP account.
|
||||
func SendMail(cfg config.SMTP, subject, body string) error {
|
||||
if cfg.Host == "" {
|
||||
return errors.New("SMTP host is not configured")
|
||||
}
|
||||
if len(cfg.To) == 0 {
|
||||
return errors.New("no recipients configured")
|
||||
}
|
||||
from := cfg.From
|
||||
if from == "" {
|
||||
from = cfg.User
|
||||
}
|
||||
if from == "" {
|
||||
return errors.New("sender address (from) is not configured")
|
||||
}
|
||||
addr := net.JoinHostPort(cfg.Host, fmt.Sprint(cfg.Port))
|
||||
dialer := net.Dialer{Timeout: 20 * time.Second}
|
||||
|
||||
var c *smtp.Client
|
||||
switch cfg.Security {
|
||||
case "tls":
|
||||
conn, err := tls.DialWithDialer(&dialer, "tcp", addr, &tls.Config{ServerName: cfg.Host})
|
||||
if err != nil {
|
||||
return fmt.Errorf("tls connect: %w", err)
|
||||
}
|
||||
c, err = smtp.NewClient(conn, cfg.Host)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return err
|
||||
}
|
||||
default:
|
||||
conn, err := dialer.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect: %w", err)
|
||||
}
|
||||
conn.SetDeadline(time.Now().Add(60 * time.Second))
|
||||
c, err = smtp.NewClient(conn, cfg.Host)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return err
|
||||
}
|
||||
if cfg.Security == "starttls" {
|
||||
if ok, _ := c.Extension("STARTTLS"); !ok {
|
||||
c.Close()
|
||||
return errors.New("server does not support STARTTLS")
|
||||
}
|
||||
if err := c.StartTLS(&tls.Config{ServerName: cfg.Host}); err != nil {
|
||||
c.Close()
|
||||
return fmt.Errorf("starttls: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
if cfg.User != "" {
|
||||
ok, mechs := c.Extension("AUTH")
|
||||
var auth smtp.Auth
|
||||
switch {
|
||||
case ok && strings.Contains(mechs, "PLAIN"):
|
||||
auth = smtp.PlainAuth("", cfg.User, cfg.Pass, cfg.Host)
|
||||
case ok && strings.Contains(mechs, "LOGIN"):
|
||||
auth = loginAuth{cfg.User, cfg.Pass}
|
||||
case ok && strings.Contains(mechs, "CRAM-MD5"):
|
||||
auth = smtp.CRAMMD5Auth(cfg.User, cfg.Pass)
|
||||
default:
|
||||
auth = smtp.PlainAuth("", cfg.User, cfg.Pass, cfg.Host)
|
||||
}
|
||||
if err := c.Auth(auth); err != nil {
|
||||
return fmt.Errorf("auth: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.Mail(from); err != nil {
|
||||
return fmt.Errorf("MAIL FROM: %w", err)
|
||||
}
|
||||
for _, to := range cfg.To {
|
||||
if err := c.Rcpt(to); err != nil {
|
||||
return fmt.Errorf("RCPT TO %s: %w", to, err)
|
||||
}
|
||||
}
|
||||
w, err := c.Data()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := fmt.Sprintf("From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\nMIME-Version: 1.0\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Transfer-Encoding: 8bit\r\n\r\n%s",
|
||||
from, strings.Join(cfg.To, ", "), mime.QEncoding.Encode("utf-8", subject),
|
||||
time.Now().Format(time.RFC1123Z), strings.ReplaceAll(body, "\n", "\r\n"))
|
||||
if _, err := w.Write([]byte(msg)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return c.Quit()
|
||||
}
|
||||
|
||||
// loginAuth implements the legacy AUTH LOGIN mechanism (Yandex, Mail.ru, Exchange).
|
||||
type loginAuth struct{ user, pass string }
|
||||
|
||||
func (a loginAuth) Start(*smtp.ServerInfo) (string, []byte, error) { return "LOGIN", nil, nil }
|
||||
|
||||
func (a loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
|
||||
if !more {
|
||||
return nil, nil
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(string(fromServer))) {
|
||||
case "username:":
|
||||
return []byte(a.user), nil
|
||||
case "password:":
|
||||
return []byte(a.pass), nil
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected LOGIN challenge %q", fromServer)
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
// Package alert evaluates health rules, tracks open alerts and sends mail.
|
||||
package alert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"nasmon/internal/collect"
|
||||
"nasmon/internal/config"
|
||||
"nasmon/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
LevelWarn = "warn"
|
||||
LevelCrit = "crit"
|
||||
)
|
||||
|
||||
// Event is one active problem. Key is stable across evaluations so the same
|
||||
// condition maps to the same open alert: "<area>:<object>:<rule>".
|
||||
type Event struct {
|
||||
Key string
|
||||
Level string
|
||||
Title string
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Input is everything the rules look at in one evaluation.
|
||||
type Input struct {
|
||||
Now time.Time
|
||||
Th config.Thresholds
|
||||
Smarts []collect.Smart // disks with usable SMART data (standby ones excluded)
|
||||
Prev map[string]*collect.Smart // previous snapshot by serial, for growth rules
|
||||
Disks []collect.Disk
|
||||
Seen []store.SeenDisk
|
||||
Pools []collect.Pool
|
||||
PoolsOK bool
|
||||
CPUTemp float64
|
||||
HasCPU bool
|
||||
RootFS collect.FSUsage
|
||||
RootOK bool
|
||||
// Frozen lists key prefixes whose open alerts must be left untouched
|
||||
// because no fresh data was available for them in this cycle.
|
||||
Frozen []string
|
||||
}
|
||||
|
||||
// criticalATA are attributes whose non-zero raw value is a warning sign.
|
||||
var criticalATA = map[int]string{
|
||||
5: "Reallocated_Sector_Ct",
|
||||
187: "Reported_Uncorrect",
|
||||
188: "Command_Timeout",
|
||||
197: "Current_Pending_Sector",
|
||||
198: "Offline_Uncorrectable",
|
||||
}
|
||||
|
||||
func diskLabel(s collect.Smart) string {
|
||||
return fmt.Sprintf("%s (%s, %s)", s.Device, s.Model, s.Serial)
|
||||
}
|
||||
|
||||
// Evaluate returns the list of currently active events.
|
||||
func Evaluate(in Input) []Event {
|
||||
var ev []Event
|
||||
add := func(key, level, title, detail string) {
|
||||
ev = append(ev, Event{Key: key, Level: level, Title: title, Detail: detail})
|
||||
}
|
||||
|
||||
for _, s := range in.Smarts {
|
||||
if s.Error != "" && !s.Supported {
|
||||
continue
|
||||
}
|
||||
pfx := "smart:" + s.Serial + ":"
|
||||
lbl := diskLabel(s)
|
||||
if s.Supported && !s.Healthy {
|
||||
add(pfx+"health", LevelCrit, "SMART health FAILED: "+lbl, "smartctl reports overall-health self-assessment: FAILED. Back up the data and replace the disk.")
|
||||
}
|
||||
var bad []string
|
||||
level := LevelWarn
|
||||
for _, a := range s.Attrs {
|
||||
if _, ok := criticalATA[a.ID]; ok && a.Raw > 0 {
|
||||
bad = append(bad, fmt.Sprintf("%d %s raw=%d", a.ID, a.Name, a.Raw))
|
||||
if a.ID != 187 && a.ID != 188 { // uncorrect/timeouts are a hint, bad sectors are damage
|
||||
level = LevelCrit
|
||||
}
|
||||
}
|
||||
if a.WhenFailed != "" {
|
||||
bad = append(bad, fmt.Sprintf("%d %s when_failed=%s (value %d, thresh %d)", a.ID, a.Name, a.WhenFailed, a.Value, a.Thresh))
|
||||
level = LevelCrit
|
||||
}
|
||||
}
|
||||
if len(bad) > 0 {
|
||||
add(pfx+"attr", level, "SMART attributes degraded: "+lbl, strings.Join(bad, "\n"))
|
||||
}
|
||||
if cur, ok := s.Attr(199); ok {
|
||||
if prev := in.Prev[s.Serial]; prev != nil {
|
||||
if p, ok := prev.Attr(199); ok && cur.Raw > p.Raw {
|
||||
add(pfx+"crc", LevelWarn, "UDMA CRC errors increasing: "+lbl,
|
||||
fmt.Sprintf("199 UDMA_CRC_Error_Count %d -> %d. Usually a cable/connector problem.", p.Raw, cur.Raw))
|
||||
}
|
||||
}
|
||||
}
|
||||
if n := s.NVMe; n != nil {
|
||||
var msgs []string
|
||||
if n.CriticalWarning != 0 {
|
||||
msgs = append(msgs, fmt.Sprintf("critical_warning=0x%02x", n.CriticalWarning))
|
||||
}
|
||||
if n.MediaErrors > 0 {
|
||||
msgs = append(msgs, fmt.Sprintf("media_errors=%d", n.MediaErrors))
|
||||
}
|
||||
if n.PercentageUsed >= in.Th.NVMeUsedPct {
|
||||
msgs = append(msgs, fmt.Sprintf("percentage_used=%d%% (>= %d%%)", n.PercentageUsed, in.Th.NVMeUsedPct))
|
||||
}
|
||||
if n.AvailableSpare < in.Th.NVMeSparePct {
|
||||
msgs = append(msgs, fmt.Sprintf("available_spare=%d%% (< %d%%)", n.AvailableSpare, in.Th.NVMeSparePct))
|
||||
}
|
||||
if len(msgs) > 0 {
|
||||
add(pfx+"nvme", LevelCrit, "NVMe health: "+lbl, strings.Join(msgs, "\n"))
|
||||
}
|
||||
}
|
||||
if s.HasTemp && in.Th.DiskTempC > 0 && s.TempC >= in.Th.DiskTempC {
|
||||
add(pfx+"temp", LevelWarn, fmt.Sprintf("Disk temperature %d°C: %s", s.TempC, lbl),
|
||||
fmt.Sprintf("Threshold is %d°C.", in.Th.DiskTempC))
|
||||
}
|
||||
if t := s.LastTest; t != nil && !t.InProgress && !t.Passed {
|
||||
add(pfx+"selftest", LevelCrit, "SMART self-test failed: "+lbl,
|
||||
fmt.Sprintf("%s: %s (at %d power-on hours)", t.Type, t.Status, t.Hours))
|
||||
}
|
||||
}
|
||||
|
||||
// disks that were seen before but are gone now
|
||||
present := map[string]bool{}
|
||||
for _, d := range in.Disks {
|
||||
present[d.Serial] = true
|
||||
}
|
||||
for _, d := range in.Seen {
|
||||
if !present[d.Serial] {
|
||||
add("disk:"+d.Serial+":missing", LevelCrit, fmt.Sprintf("Disk disappeared: %s (%s)", d.Model, d.Serial),
|
||||
fmt.Sprintf("Last seen as %s at %s. If it was removed on purpose, forget it on the disk page.", d.Device, d.LastSeen.Format(time.RFC3339)))
|
||||
}
|
||||
}
|
||||
|
||||
if in.PoolsOK {
|
||||
for _, p := range in.Pools {
|
||||
pfx := "zfs:" + p.Name + ":"
|
||||
if p.Health != "ONLINE" {
|
||||
detail := "State: " + p.Health
|
||||
if p.Status != "" {
|
||||
detail += "\nStatus: " + p.Status
|
||||
}
|
||||
if p.Action != "" {
|
||||
detail += "\nAction: " + p.Action
|
||||
}
|
||||
add(pfx+"health", LevelCrit, fmt.Sprintf("Pool %s is %s", p.Name, p.Health), detail)
|
||||
}
|
||||
var errs []string
|
||||
for _, v := range p.Vdevs {
|
||||
if v.Read > 0 || v.Write > 0 || v.Cksum > 0 {
|
||||
errs = append(errs, fmt.Sprintf("%s: read=%d write=%d cksum=%d", v.Name, v.Read, v.Write, v.Cksum))
|
||||
}
|
||||
}
|
||||
if p.Errors != "" && !strings.HasPrefix(p.Errors, "No known data errors") {
|
||||
errs = append(errs, "errors: "+p.Errors)
|
||||
}
|
||||
if p.ScanErrs > 0 {
|
||||
errs = append(errs, fmt.Sprintf("last scan finished with %d errors", p.ScanErrs))
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
add(pfx+"errors", LevelCrit, "Pool "+p.Name+" has errors", strings.Join(errs, "\n"))
|
||||
}
|
||||
if in.Th.PoolCapacityPct > 0 && p.CapPct >= in.Th.PoolCapacityPct {
|
||||
add(pfx+"capacity", LevelWarn, fmt.Sprintf("Pool %s is %d%% full", p.Name, p.CapPct),
|
||||
fmt.Sprintf("Pool size %s. Threshold %d%%.", humanBytes(p.Size), in.Th.PoolCapacityPct))
|
||||
}
|
||||
if in.Th.ScrubMaxAgeDays > 0 && !p.Scrubbing {
|
||||
maxAge := time.Duration(in.Th.ScrubMaxAgeDays) * 24 * time.Hour
|
||||
if p.LastScrub == nil {
|
||||
add(pfx+"scrub", LevelWarn, "Pool "+p.Name+" was never scrubbed", "Run: zpool scrub "+p.Name)
|
||||
} else if age := in.Now.Sub(*p.LastScrub); age > maxAge {
|
||||
add(pfx+"scrub", LevelWarn, fmt.Sprintf("Pool %s: last scrub %d days ago", p.Name, int(age.Hours()/24)),
|
||||
fmt.Sprintf("Last scrub %s. Threshold %d days.", p.LastScrub.Format("2006-01-02"), in.Th.ScrubMaxAgeDays))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if in.HasCPU && in.Th.CPUTempC > 0 && in.CPUTemp >= float64(in.Th.CPUTempC) {
|
||||
add("sys:cpu:temp", LevelWarn, fmt.Sprintf("CPU temperature %.0f°C", in.CPUTemp), fmt.Sprintf("Threshold %d°C.", in.Th.CPUTempC))
|
||||
}
|
||||
if in.RootOK && in.RootFS.Total > 0 && in.Th.RootFSFreePct > 0 {
|
||||
pct := int(in.RootFS.Free * 100 / in.RootFS.Total)
|
||||
if pct < in.Th.RootFSFreePct {
|
||||
add("sys:rootfs:space", LevelWarn, fmt.Sprintf("Root filesystem %d%% free", pct),
|
||||
fmt.Sprintf("Root filesystem size %s. Threshold %d%%.", humanBytes(in.RootFS.Total), in.Th.RootFSFreePct))
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(ev, func(i, j int) bool { return ev[i].Key < ev[j].Key })
|
||||
return ev
|
||||
}
|
||||
|
||||
func humanBytes(b uint64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := uint64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"nasmon/internal/config"
|
||||
"nasmon/internal/store"
|
||||
)
|
||||
|
||||
// Manager reconciles evaluated events with the alerts table and notifies.
|
||||
type Manager struct {
|
||||
Store *store.Store
|
||||
Config *config.Store
|
||||
Send func(cfg config.SMTP, subject, body string) error
|
||||
|
||||
lastErr string // last mail error logged, to avoid repeating it every cycle
|
||||
nextMailTry time.Time // backoff after a delivery failure
|
||||
}
|
||||
|
||||
const mailRetryAfter = 10 * time.Minute
|
||||
|
||||
// Outcome describes what one reconciliation did.
|
||||
type Outcome struct {
|
||||
Opened []store.Alert
|
||||
Resolved []store.Alert
|
||||
Reminded []store.Alert
|
||||
Mailed bool
|
||||
MailErr error
|
||||
}
|
||||
|
||||
func escalated(old, cur string) bool { return old == LevelWarn && cur == LevelCrit }
|
||||
|
||||
func frozen(key string, prefixes []string) bool {
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(key, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Reconcile opens/resolves alerts based on the active events and sends one
|
||||
// email describing everything that changed in this cycle.
|
||||
func (m *Manager) Reconcile(in Input, events []Event) (Outcome, error) {
|
||||
var out Outcome
|
||||
cfg := m.Config.Get()
|
||||
now := in.Now
|
||||
|
||||
open, err := m.Store.OpenAlerts()
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
openByKey := map[string]store.Alert{}
|
||||
for _, a := range open {
|
||||
openByKey[a.Key] = a
|
||||
}
|
||||
active := map[string]Event{}
|
||||
for _, e := range events {
|
||||
active[e.Key] = e
|
||||
}
|
||||
|
||||
// alerts re-raised in this cycle (acknowledged, but the problem changed)
|
||||
reraised := map[int64]bool{}
|
||||
for _, e := range events {
|
||||
if a, ok := openByKey[e.Key]; ok {
|
||||
changed := a.Detail != e.Detail || a.Level != e.Level
|
||||
if changed {
|
||||
_ = m.Store.UpdateAlertDetail(a.ID, e.Level, e.Detail)
|
||||
}
|
||||
// an acknowledged alert stays quiet until the problem gets worse:
|
||||
// escalated level or different details (grown counters, new errors)
|
||||
if a.Acked() && (a.Detail != e.Detail || escalated(a.Level, e.Level)) {
|
||||
_ = m.Store.ReraiseAlert(a.ID)
|
||||
a.AckedAt, a.LastNotifiedAt, a.Level, a.Detail = nil, nil, e.Level, e.Detail
|
||||
openByKey[e.Key] = a
|
||||
reraised[a.ID] = true
|
||||
log.Printf("alert re-raised: %s — %s", a.Key, a.Title)
|
||||
out.Opened = append(out.Opened, a)
|
||||
}
|
||||
continue
|
||||
}
|
||||
a, err := m.Store.OpenAlert(e.Key, e.Level, e.Title, e.Detail, now)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
log.Printf("alert opened: %s — %s", a.Key, a.Title)
|
||||
out.Opened = append(out.Opened, a)
|
||||
}
|
||||
|
||||
remind := time.Duration(cfg.RemindHours) * time.Hour
|
||||
for _, a := range open {
|
||||
if _, still := active[a.Key]; still {
|
||||
if reraised[a.ID] {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case a.Acked():
|
||||
// acknowledged: no reminders, no retries
|
||||
case a.LastNotifiedAt == nil:
|
||||
// never delivered (mail failed earlier): retry as new
|
||||
out.Opened = append(out.Opened, a)
|
||||
case remind > 0 && now.Sub(*a.LastNotifiedAt) >= remind:
|
||||
out.Reminded = append(out.Reminded, a)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if frozen(a.Key, in.Frozen) {
|
||||
continue
|
||||
}
|
||||
if err := m.Store.ResolveAlert(a.ID, now); err != nil {
|
||||
return out, err
|
||||
}
|
||||
log.Printf("alert resolved: %s — %s", a.Key, a.Title)
|
||||
out.Resolved = append(out.Resolved, a)
|
||||
}
|
||||
|
||||
if !cfg.AlertsEnabled || m.Send == nil {
|
||||
return out, nil
|
||||
}
|
||||
if len(out.Opened) == 0 && len(out.Reminded) == 0 && (len(out.Resolved) == 0 || !cfg.NotifyResolved) {
|
||||
return out, nil
|
||||
}
|
||||
if now.Before(m.nextMailTry) {
|
||||
return out, nil
|
||||
}
|
||||
subject, body := compose(cfg, out)
|
||||
if err := m.Send(cfg.SMTP, subject, body); err != nil {
|
||||
out.MailErr = err
|
||||
m.nextMailTry = now.Add(mailRetryAfter)
|
||||
if err.Error() != m.lastErr {
|
||||
log.Printf("alert mail failed (retry in %s): %v", mailRetryAfter, err)
|
||||
m.lastErr = err.Error()
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
m.lastErr, m.nextMailTry = "", time.Time{}
|
||||
out.Mailed = true
|
||||
var ids []int64
|
||||
for _, a := range append(append([]store.Alert{}, out.Opened...), out.Reminded...) {
|
||||
ids = append(ids, a.ID)
|
||||
}
|
||||
_ = m.Store.MarkNotified(ids, now)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func compose(cfg config.Settings, o Outcome) (subject, body string) {
|
||||
host, _ := os.Hostname()
|
||||
var parts []string
|
||||
if n := len(o.Opened); n > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d new", n))
|
||||
}
|
||||
if n := len(o.Reminded); n > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d still open", n))
|
||||
}
|
||||
if n := len(o.Resolved); n > 0 && cfg.NotifyResolved {
|
||||
parts = append(parts, fmt.Sprintf("%d resolved", n))
|
||||
}
|
||||
subject = strings.TrimSpace(cfg.SMTP.SubjectPrefix + " " + strings.Join(parts, ", ") + " on " + host)
|
||||
if len(o.Opened) == 1 && len(o.Reminded) == 0 && len(o.Resolved) == 0 {
|
||||
subject = strings.TrimSpace(cfg.SMTP.SubjectPrefix + " " + o.Opened[0].Title)
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "nasmon on %s, %s\n\n", host, time.Now().Format("2006-01-02 15:04:05 MST"))
|
||||
section := func(title string, list []store.Alert) {
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(&b, "== %s ==\n\n", title)
|
||||
for _, a := range list {
|
||||
fmt.Fprintf(&b, "[%s] %s\n", strings.ToUpper(a.Level), a.Title)
|
||||
for _, l := range strings.Split(a.Detail, "\n") {
|
||||
fmt.Fprintf(&b, " %s\n", l)
|
||||
}
|
||||
fmt.Fprintf(&b, " since %s\n\n", a.OpenedAt.Format("2006-01-02 15:04"))
|
||||
}
|
||||
}
|
||||
section("NEW", o.Opened)
|
||||
section("STILL OPEN", o.Reminded)
|
||||
if cfg.NotifyResolved {
|
||||
section("RESOLVED", o.Resolved)
|
||||
}
|
||||
return subject, b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user