nasmon/internal/alert/rules.go

// Package alert evaluates health rules, tracks open alerts and sends mail.
package alert

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"nasmon/internal/collect"
	"nasmon/internal/config"
	"nasmon/internal/store"
)

const (
	LevelWarn = "warn"
	LevelCrit = "crit"
)

// Event is one active problem. Key is stable across evaluations so the same
// condition maps to the same open alert: "<area>:<object>:<rule>".
type Event struct {
	Key    string
	Level  string
	Title  string
	Detail string
}

// Input is everything the rules look at in one evaluation.
type Input struct {
	Now     time.Time
	Th      config.Thresholds
	Smarts  []collect.Smart           // disks with usable SMART data (standby ones excluded)
	Prev    map[string]*collect.Smart // previous snapshot by serial, for growth rules
	Disks   []collect.Disk
	Seen    []store.SeenDisk
	Pools   []collect.Pool
	PoolsOK bool
	CPUTemp float64
	HasCPU  bool
	RootFS  collect.FSUsage
	RootOK  bool
	// Frozen lists key prefixes whose open alerts must be left untouched
	// because no fresh data was available for them in this cycle.
	Frozen []string
}

// criticalATA are attributes whose non-zero raw value is a warning sign.
var criticalATA = map[int]string{
	5:   "Reallocated_Sector_Ct",
	187: "Reported_Uncorrect",
	188: "Command_Timeout",
	197: "Current_Pending_Sector",
	198: "Offline_Uncorrectable",
}

func diskLabel(s collect.Smart) string {
	return fmt.Sprintf("%s (%s, %s)", s.Device, s.Model, s.Serial)
}

// Evaluate returns the list of currently active events.
func Evaluate(in Input) []Event {
	var ev []Event
	add := func(key, level, title, detail string) {
		ev = append(ev, Event{Key: key, Level: level, Title: title, Detail: detail})
	}

	for _, s := range in.Smarts {
		if s.Error != "" && !s.Supported {
			continue
		}
		pfx := "smart:" + s.Serial + ":"
		lbl := diskLabel(s)
		if s.Supported && !s.Healthy {
			add(pfx+"health", LevelCrit, "SMART health FAILED: "+lbl, "smartctl reports overall-health self-assessment: FAILED. Back up the data and replace the disk.")
		}
		var bad []string
		level := LevelWarn
		for _, a := range s.Attrs {
			if _, ok := criticalATA[a.ID]; ok && a.Raw > 0 {
				bad = append(bad, fmt.Sprintf("%d %s raw=%d", a.ID, a.Name, a.Raw))
				if a.ID != 187 && a.ID != 188 { // uncorrect/timeouts are a hint, bad sectors are damage
					level = LevelCrit
				}
			}
			if a.WhenFailed != "" {
				bad = append(bad, fmt.Sprintf("%d %s when_failed=%s (value %d, thresh %d)", a.ID, a.Name, a.WhenFailed, a.Value, a.Thresh))
				level = LevelCrit
			}
		}
		if len(bad) > 0 {
			add(pfx+"attr", level, "SMART attributes degraded: "+lbl, strings.Join(bad, "\n"))
		}
		if cur, ok := s.Attr(199); ok {
			if prev := in.Prev[s.Serial]; prev != nil {
				if p, ok := prev.Attr(199); ok && cur.Raw > p.Raw {
					add(pfx+"crc", LevelWarn, "UDMA CRC errors increasing: "+lbl,
						fmt.Sprintf("199 UDMA_CRC_Error_Count %d -> %d. Usually a cable/connector problem.", p.Raw, cur.Raw))
				}
			}
		}
		if n := s.NVMe; n != nil {
			var msgs []string
			if n.CriticalWarning != 0 {
				msgs = append(msgs, fmt.Sprintf("critical_warning=0x%02x", n.CriticalWarning))
			}
			if n.MediaErrors > 0 {
				msgs = append(msgs, fmt.Sprintf("media_errors=%d", n.MediaErrors))
			}
			if n.PercentageUsed >= in.Th.NVMeUsedPct {
				msgs = append(msgs, fmt.Sprintf("percentage_used=%d%% (>= %d%%)", n.PercentageUsed, in.Th.NVMeUsedPct))
			}
			if n.AvailableSpare < in.Th.NVMeSparePct {
				msgs = append(msgs, fmt.Sprintf("available_spare=%d%% (< %d%%)", n.AvailableSpare, in.Th.NVMeSparePct))
			}
			if len(msgs) > 0 {
				add(pfx+"nvme", LevelCrit, "NVMe health: "+lbl, strings.Join(msgs, "\n"))
			}
		}
		if s.HasTemp && in.Th.DiskTempC > 0 && s.TempC >= in.Th.DiskTempC {
			add(pfx+"temp", LevelWarn, fmt.Sprintf("Disk temperature %d°C: %s", s.TempC, lbl),
				fmt.Sprintf("Threshold is %d°C.", in.Th.DiskTempC))
		}
		if t := s.LastTest; t != nil && !t.InProgress && !t.Passed {
			add(pfx+"selftest", LevelCrit, "SMART self-test failed: "+lbl,
				fmt.Sprintf("%s: %s (at %d power-on hours)", t.Type, t.Status, t.Hours))
		}
	}

	// disks that were seen before but are gone now
	present := map[string]bool{}
	for _, d := range in.Disks {
		present[d.Serial] = true
	}
	for _, d := range in.Seen {
		if !present[d.Serial] {
			add("disk:"+d.Serial+":missing", LevelCrit, fmt.Sprintf("Disk disappeared: %s (%s)", d.Model, d.Serial),
				fmt.Sprintf("Last seen as %s at %s. If it was removed on purpose, forget it on the disk page.", d.Device, d.LastSeen.Format(time.RFC3339)))
		}
	}

	if in.PoolsOK {
		for _, p := range in.Pools {
			pfx := "zfs:" + p.Name + ":"
			if p.Health != "ONLINE" {
				detail := "State: " + p.Health
				if p.Status != "" {
					detail += "\nStatus: " + p.Status
				}
				if p.Action != "" {
					detail += "\nAction: " + p.Action
				}
				add(pfx+"health", LevelCrit, fmt.Sprintf("Pool %s is %s", p.Name, p.Health), detail)
			}
			var errs []string
			for _, v := range p.Vdevs {
				if v.Read > 0 || v.Write > 0 || v.Cksum > 0 {
					errs = append(errs, fmt.Sprintf("%s: read=%d write=%d cksum=%d", v.Name, v.Read, v.Write, v.Cksum))
				}
			}
			if p.Errors != "" && !strings.HasPrefix(p.Errors, "No known data errors") {
				errs = append(errs, "errors: "+p.Errors)
			}
			if p.ScanErrs > 0 {
				errs = append(errs, fmt.Sprintf("last scan finished with %d errors", p.ScanErrs))
			}
			if len(errs) > 0 {
				add(pfx+"errors", LevelCrit, "Pool "+p.Name+" has errors", strings.Join(errs, "\n"))
			}
			if in.Th.PoolCapacityPct > 0 && p.CapPct >= in.Th.PoolCapacityPct {
				add(pfx+"capacity", LevelWarn, fmt.Sprintf("Pool %s is %d%% full", p.Name, p.CapPct),
					fmt.Sprintf("Pool size %s. Threshold %d%%.", humanBytes(p.Size), in.Th.PoolCapacityPct))
			}
			if in.Th.ScrubMaxAgeDays > 0 && !p.Scrubbing {
				maxAge := time.Duration(in.Th.ScrubMaxAgeDays) * 24 * time.Hour
				if p.LastScrub == nil {
					add(pfx+"scrub", LevelWarn, "Pool "+p.Name+" was never scrubbed", "Run: zpool scrub "+p.Name)
				} else if age := in.Now.Sub(*p.LastScrub); age > maxAge {
					add(pfx+"scrub", LevelWarn, fmt.Sprintf("Pool %s: last scrub %d days ago", p.Name, int(age.Hours()/24)),
						fmt.Sprintf("Last scrub %s. Threshold %d days.", p.LastScrub.Format("2006-01-02"), in.Th.ScrubMaxAgeDays))
				}
			}
		}
	}

	if in.HasCPU && in.Th.CPUTempC > 0 && in.CPUTemp >= float64(in.Th.CPUTempC) {
		add("sys:cpu:temp", LevelWarn, fmt.Sprintf("CPU temperature %.0f°C", in.CPUTemp), fmt.Sprintf("Threshold %d°C.", in.Th.CPUTempC))
	}
	if in.RootOK && in.RootFS.Total > 0 && in.Th.RootFSFreePct > 0 {
		pct := int(in.RootFS.Free * 100 / in.RootFS.Total)
		if pct < in.Th.RootFSFreePct {
			add("sys:rootfs:space", LevelWarn, fmt.Sprintf("Root filesystem %d%% free", pct),
				fmt.Sprintf("Root filesystem size %s. Threshold %d%%.", humanBytes(in.RootFS.Total), in.Th.RootFSFreePct))
		}
	}

	sort.Slice(ev, func(i, j int) bool { return ev[i].Key < ev[j].Key })
	return ev
}

func humanBytes(b uint64) string {
	const unit = 1024
	if b < unit {
		return fmt.Sprintf("%d B", b)
	}
	div, exp := uint64(unit), 0
	for n := b / unit; n >= unit; n /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}