Initial import: nasmon health monitor, split out of home_nas

Go service (nasmon/) plus nfpm packaging (packaging/), carried over from the home_nas monorepo. Adds rpm packaging alongside the existing deb build (Fedora/RHEL/Rocky dependency names via nfpm overrides, per-format systemd unit path, dpkg+rpm-compatible maintainer scripts) and a Gitea Actions workflow that builds both formats on every push. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-19 10:24:34 +03:00
commit 09159a5f31
61 changed files with 7882 additions and 0 deletions
@@ -0,0 +1,212 @@
+// Package alert evaluates health rules, tracks open alerts and sends mail.
+package alert
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"nasmon/internal/collect"
+	"nasmon/internal/config"
+	"nasmon/internal/store"
+)
+
+const (
+	LevelWarn = "warn"
+	LevelCrit = "crit"
+)
+
+// Event is one active problem. Key is stable across evaluations so the same
+// condition maps to the same open alert: "<area>:<object>:<rule>".
+type Event struct {
+	Key    string
+	Level  string
+	Title  string
+	Detail string
+}
+
+// Input is everything the rules look at in one evaluation.
+type Input struct {
+	Now     time.Time
+	Th      config.Thresholds
+	Smarts  []collect.Smart           // disks with usable SMART data (standby ones excluded)
+	Prev    map[string]*collect.Smart // previous snapshot by serial, for growth rules
+	Disks   []collect.Disk
+	Seen    []store.SeenDisk
+	Pools   []collect.Pool
+	PoolsOK bool
+	CPUTemp float64
+	HasCPU  bool
+	RootFS  collect.FSUsage
+	RootOK  bool
+	// Frozen lists key prefixes whose open alerts must be left untouched
+	// because no fresh data was available for them in this cycle.
+	Frozen []string
+}
+
+// criticalATA are attributes whose non-zero raw value is a warning sign.
+var criticalATA = map[int]string{
+	5:   "Reallocated_Sector_Ct",
+	187: "Reported_Uncorrect",
+	188: "Command_Timeout",
+	197: "Current_Pending_Sector",
+	198: "Offline_Uncorrectable",
+}
+
+func diskLabel(s collect.Smart) string {
+	return fmt.Sprintf("%s (%s, %s)", s.Device, s.Model, s.Serial)
+}
+
+// Evaluate returns the list of currently active events.
+func Evaluate(in Input) []Event {
+	var ev []Event
+	add := func(key, level, title, detail string) {
+		ev = append(ev, Event{Key: key, Level: level, Title: title, Detail: detail})
+	}
+
+	for _, s := range in.Smarts {
+		if s.Error != "" && !s.Supported {
+			continue
+		}
+		pfx := "smart:" + s.Serial + ":"
+		lbl := diskLabel(s)
+		if s.Supported && !s.Healthy {
+			add(pfx+"health", LevelCrit, "SMART health FAILED: "+lbl, "smartctl reports overall-health self-assessment: FAILED. Back up the data and replace the disk.")
+		}
+		var bad []string
+		level := LevelWarn
+		for _, a := range s.Attrs {
+			if _, ok := criticalATA[a.ID]; ok && a.Raw > 0 {
+				bad = append(bad, fmt.Sprintf("%d %s raw=%d", a.ID, a.Name, a.Raw))
+				if a.ID != 187 && a.ID != 188 { // uncorrect/timeouts are a hint, bad sectors are damage
+					level = LevelCrit
+				}
+			}
+			if a.WhenFailed != "" {
+				bad = append(bad, fmt.Sprintf("%d %s when_failed=%s (value %d, thresh %d)", a.ID, a.Name, a.WhenFailed, a.Value, a.Thresh))
+				level = LevelCrit
+			}
+		}
+		if len(bad) > 0 {
+			add(pfx+"attr", level, "SMART attributes degraded: "+lbl, strings.Join(bad, "\n"))
+		}
+		if cur, ok := s.Attr(199); ok {
+			if prev := in.Prev[s.Serial]; prev != nil {
+				if p, ok := prev.Attr(199); ok && cur.Raw > p.Raw {
+					add(pfx+"crc", LevelWarn, "UDMA CRC errors increasing: "+lbl,
+						fmt.Sprintf("199 UDMA_CRC_Error_Count %d -> %d. Usually a cable/connector problem.", p.Raw, cur.Raw))
+				}
+			}
+		}
+		if n := s.NVMe; n != nil {
+			var msgs []string
+			if n.CriticalWarning != 0 {
+				msgs = append(msgs, fmt.Sprintf("critical_warning=0x%02x", n.CriticalWarning))
+			}
+			if n.MediaErrors > 0 {
+				msgs = append(msgs, fmt.Sprintf("media_errors=%d", n.MediaErrors))
+			}
+			if n.PercentageUsed >= in.Th.NVMeUsedPct {
+				msgs = append(msgs, fmt.Sprintf("percentage_used=%d%% (>= %d%%)", n.PercentageUsed, in.Th.NVMeUsedPct))
+			}
+			if n.AvailableSpare < in.Th.NVMeSparePct {
+				msgs = append(msgs, fmt.Sprintf("available_spare=%d%% (< %d%%)", n.AvailableSpare, in.Th.NVMeSparePct))
+			}
+			if len(msgs) > 0 {
+				add(pfx+"nvme", LevelCrit, "NVMe health: "+lbl, strings.Join(msgs, "\n"))
+			}
+		}
+		if s.HasTemp && in.Th.DiskTempC > 0 && s.TempC >= in.Th.DiskTempC {
+			add(pfx+"temp", LevelWarn, fmt.Sprintf("Disk temperature %d°C: %s", s.TempC, lbl),
+				fmt.Sprintf("Threshold is %d°C.", in.Th.DiskTempC))
+		}
+		if t := s.LastTest; t != nil && !t.InProgress && !t.Passed {
+			add(pfx+"selftest", LevelCrit, "SMART self-test failed: "+lbl,
+				fmt.Sprintf("%s: %s (at %d power-on hours)", t.Type, t.Status, t.Hours))
+		}
+	}
+
+	// disks that were seen before but are gone now
+	present := map[string]bool{}
+	for _, d := range in.Disks {
+		present[d.Serial] = true
+	}
+	for _, d := range in.Seen {
+		if !present[d.Serial] {
+			add("disk:"+d.Serial+":missing", LevelCrit, fmt.Sprintf("Disk disappeared: %s (%s)", d.Model, d.Serial),
+				fmt.Sprintf("Last seen as %s at %s. If it was removed on purpose, forget it on the disk page.", d.Device, d.LastSeen.Format(time.RFC3339)))
+		}
+	}
+
+	if in.PoolsOK {
+		for _, p := range in.Pools {
+			pfx := "zfs:" + p.Name + ":"
+			if p.Health != "ONLINE" {
+				detail := "State: " + p.Health
+				if p.Status != "" {
+					detail += "\nStatus: " + p.Status
+				}
+				if p.Action != "" {
+					detail += "\nAction: " + p.Action
+				}
+				add(pfx+"health", LevelCrit, fmt.Sprintf("Pool %s is %s", p.Name, p.Health), detail)
+			}
+			var errs []string
+			for _, v := range p.Vdevs {
+				if v.Read > 0 || v.Write > 0 || v.Cksum > 0 {
+					errs = append(errs, fmt.Sprintf("%s: read=%d write=%d cksum=%d", v.Name, v.Read, v.Write, v.Cksum))
+				}
+			}
+			if p.Errors != "" && !strings.HasPrefix(p.Errors, "No known data errors") {
+				errs = append(errs, "errors: "+p.Errors)
+			}
+			if p.ScanErrs > 0 {
+				errs = append(errs, fmt.Sprintf("last scan finished with %d errors", p.ScanErrs))
+			}
+			if len(errs) > 0 {
+				add(pfx+"errors", LevelCrit, "Pool "+p.Name+" has errors", strings.Join(errs, "\n"))
+			}
+			if in.Th.PoolCapacityPct > 0 && p.CapPct >= in.Th.PoolCapacityPct {
+				add(pfx+"capacity", LevelWarn, fmt.Sprintf("Pool %s is %d%% full", p.Name, p.CapPct),
+					fmt.Sprintf("Pool size %s. Threshold %d%%.", humanBytes(p.Size), in.Th.PoolCapacityPct))
+			}
+			if in.Th.ScrubMaxAgeDays > 0 && !p.Scrubbing {
+				maxAge := time.Duration(in.Th.ScrubMaxAgeDays) * 24 * time.Hour
+				if p.LastScrub == nil {
+					add(pfx+"scrub", LevelWarn, "Pool "+p.Name+" was never scrubbed", "Run: zpool scrub "+p.Name)
+				} else if age := in.Now.Sub(*p.LastScrub); age > maxAge {
+					add(pfx+"scrub", LevelWarn, fmt.Sprintf("Pool %s: last scrub %d days ago", p.Name, int(age.Hours()/24)),
+						fmt.Sprintf("Last scrub %s. Threshold %d days.", p.LastScrub.Format("2006-01-02"), in.Th.ScrubMaxAgeDays))
+				}
+			}
+		}
+	}
+
+	if in.HasCPU && in.Th.CPUTempC > 0 && in.CPUTemp >= float64(in.Th.CPUTempC) {
+		add("sys:cpu:temp", LevelWarn, fmt.Sprintf("CPU temperature %.0f°C", in.CPUTemp), fmt.Sprintf("Threshold %d°C.", in.Th.CPUTempC))
+	}
+	if in.RootOK && in.RootFS.Total > 0 && in.Th.RootFSFreePct > 0 {
+		pct := int(in.RootFS.Free * 100 / in.RootFS.Total)
+		if pct < in.Th.RootFSFreePct {
+			add("sys:rootfs:space", LevelWarn, fmt.Sprintf("Root filesystem %d%% free", pct),
+				fmt.Sprintf("Root filesystem size %s. Threshold %d%%.", humanBytes(in.RootFS.Total), in.Th.RootFSFreePct))
+		}
+	}
+
+	sort.Slice(ev, func(i, j int) bool { return ev[i].Key < ev[j].Key })
+	return ev
+}
+
+func humanBytes(b uint64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := uint64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
+}