// Package alert evaluates health rules, tracks open alerts and sends mail.
package alert
import (
"fmt"
"sort"
"strings"
"time"
"nasmon/internal/collect"
"nasmon/internal/config"
"nasmon/internal/store"
)
const (
LevelWarn = "warn"
LevelCrit = "crit"
)
// Event is one active problem. Key is stable across evaluations so the same
// condition maps to the same open alert: " ::".
type Event struct {
Key string
Level string
Title string
Detail string
}
// Input is everything the rules look at in one evaluation.
type Input struct {
Now time.Time
Th config.Thresholds
Smarts []collect.Smart // disks with usable SMART data (standby ones excluded)
Prev map[string]*collect.Smart // previous snapshot by serial, for growth rules
Disks []collect.Disk
Seen []store.SeenDisk
Pools []collect.Pool
PoolsOK bool
CPUTemp float64
HasCPU bool
RootFS collect.FSUsage
RootOK bool
// RepoISO is the polled RepoISO health (nil when the integration is off).
RepoISO *collect.RepoISO
// Frozen lists key prefixes whose open alerts must be left untouched
// because no fresh data was available for them in this cycle.
Frozen []string
}
// criticalATA are attributes whose non-zero raw value is a warning sign.
var criticalATA = map[int]string{
5: "Reallocated_Sector_Ct",
187: "Reported_Uncorrect",
188: "Command_Timeout",
197: "Current_Pending_Sector",
198: "Offline_Uncorrectable",
}
func diskLabel(s collect.Smart) string {
return fmt.Sprintf("%s (%s, %s)", s.Device, s.Model, s.Serial)
}
// Evaluate returns the list of currently active events.
func Evaluate(in Input) []Event {
var ev []Event
add := func(key, level, title, detail string) {
ev = append(ev, Event{Key: key, Level: level, Title: title, Detail: detail})
}
for _, s := range in.Smarts {
if s.Error != "" && !s.Supported {
continue
}
pfx := "smart:" + s.Serial + ":"
lbl := diskLabel(s)
if s.Supported && !s.Healthy {
add(pfx+"health", LevelCrit, "SMART health FAILED: "+lbl, "smartctl reports overall-health self-assessment: FAILED. Back up the data and replace the disk.")
}
var bad []string
level := LevelWarn
for _, a := range s.Attrs {
if _, ok := criticalATA[a.ID]; ok && a.Raw > 0 {
bad = append(bad, fmt.Sprintf("%d %s raw=%d", a.ID, a.Name, a.Raw))
if a.ID != 187 && a.ID != 188 { // uncorrect/timeouts are a hint, bad sectors are damage
level = LevelCrit
}
}
if a.WhenFailed != "" {
bad = append(bad, fmt.Sprintf("%d %s when_failed=%s (value %d, thresh %d)", a.ID, a.Name, a.WhenFailed, a.Value, a.Thresh))
level = LevelCrit
}
}
if len(bad) > 0 {
add(pfx+"attr", level, "SMART attributes degraded: "+lbl, strings.Join(bad, "\n"))
}
if cur, ok := s.Attr(199); ok {
if prev := in.Prev[s.Serial]; prev != nil {
if p, ok := prev.Attr(199); ok && cur.Raw > p.Raw {
add(pfx+"crc", LevelWarn, "UDMA CRC errors increasing: "+lbl,
fmt.Sprintf("199 UDMA_CRC_Error_Count %d -> %d. Usually a cable/connector problem.", p.Raw, cur.Raw))
}
}
}
if n := s.NVMe; n != nil {
var msgs []string
if n.CriticalWarning != 0 {
msgs = append(msgs, fmt.Sprintf("critical_warning=0x%02x", n.CriticalWarning))
}
if n.MediaErrors > 0 {
msgs = append(msgs, fmt.Sprintf("media_errors=%d", n.MediaErrors))
}
if n.PercentageUsed >= in.Th.NVMeUsedPct {
msgs = append(msgs, fmt.Sprintf("percentage_used=%d%% (>= %d%%)", n.PercentageUsed, in.Th.NVMeUsedPct))
}
if n.AvailableSpare < in.Th.NVMeSparePct {
msgs = append(msgs, fmt.Sprintf("available_spare=%d%% (< %d%%)", n.AvailableSpare, in.Th.NVMeSparePct))
}
if len(msgs) > 0 {
add(pfx+"nvme", LevelCrit, "NVMe health: "+lbl, strings.Join(msgs, "\n"))
}
}
if s.HasTemp && in.Th.DiskTempC > 0 && s.TempC >= in.Th.DiskTempC {
add(pfx+"temp", LevelWarn, fmt.Sprintf("Disk temperature %d°C: %s", s.TempC, lbl),
fmt.Sprintf("Threshold is %d°C.", in.Th.DiskTempC))
}
if t := s.LastTest; t != nil && !t.InProgress && !t.Passed {
add(pfx+"selftest", LevelCrit, "SMART self-test failed: "+lbl,
fmt.Sprintf("%s: %s (at %d power-on hours)", t.Type, t.Status, t.Hours))
}
}
// disks that were seen before but are gone now
present := map[string]bool{}
for _, d := range in.Disks {
present[d.Serial] = true
}
for _, d := range in.Seen {
if !present[d.Serial] {
add("disk:"+d.Serial+":missing", LevelCrit, fmt.Sprintf("Disk disappeared: %s (%s)", d.Model, d.Serial),
fmt.Sprintf("Last seen as %s at %s. If it was removed on purpose, forget it on the disk page.", d.Device, d.LastSeen.Format(time.RFC3339)))
}
}
if in.PoolsOK {
for _, p := range in.Pools {
pfx := "zfs:" + p.Name + ":"
if p.Health != "ONLINE" {
detail := "State: " + p.Health
if p.Status != "" {
detail += "\nStatus: " + p.Status
}
if p.Action != "" {
detail += "\nAction: " + p.Action
}
add(pfx+"health", LevelCrit, fmt.Sprintf("Pool %s is %s", p.Name, p.Health), detail)
}
var errs []string
for _, v := range p.Vdevs {
if v.Read > 0 || v.Write > 0 || v.Cksum > 0 {
errs = append(errs, fmt.Sprintf("%s: read=%d write=%d cksum=%d", v.Name, v.Read, v.Write, v.Cksum))
}
}
if p.Errors != "" && !strings.HasPrefix(p.Errors, "No known data errors") {
errs = append(errs, "errors: "+p.Errors)
}
if p.ScanErrs > 0 {
errs = append(errs, fmt.Sprintf("last scan finished with %d errors", p.ScanErrs))
}
if len(errs) > 0 {
add(pfx+"errors", LevelCrit, "Pool "+p.Name+" has errors", strings.Join(errs, "\n"))
}
if in.Th.PoolCapacityPct > 0 && p.CapPct >= in.Th.PoolCapacityPct {
add(pfx+"capacity", LevelWarn, fmt.Sprintf("Pool %s is %d%% full", p.Name, p.CapPct),
fmt.Sprintf("Pool size %s. Threshold %d%%.", humanBytes(p.Size), in.Th.PoolCapacityPct))
}
if in.Th.ScrubMaxAgeDays > 0 && !p.Scrubbing {
maxAge := time.Duration(in.Th.ScrubMaxAgeDays) * 24 * time.Hour
if p.LastScrub == nil {
add(pfx+"scrub", LevelWarn, "Pool "+p.Name+" was never scrubbed", "Run: zpool scrub "+p.Name)
} else if age := in.Now.Sub(*p.LastScrub); age > maxAge {
add(pfx+"scrub", LevelWarn, fmt.Sprintf("Pool %s: last scrub %d days ago", p.Name, int(age.Hours()/24)),
fmt.Sprintf("Last scrub %s. Threshold %d days.", p.LastScrub.Format("2006-01-02"), in.Th.ScrubMaxAgeDays))
}
}
}
}
if in.HasCPU && in.Th.CPUTempC > 0 && in.CPUTemp >= float64(in.Th.CPUTempC) {
add("sys:cpu:temp", LevelWarn, fmt.Sprintf("CPU temperature %.0f°C", in.CPUTemp), fmt.Sprintf("Threshold %d°C.", in.Th.CPUTempC))
}
if in.RootOK && in.RootFS.Total > 0 && in.Th.RootFSFreePct > 0 {
pct := int(in.RootFS.Free * 100 / in.RootFS.Total)
if pct < in.Th.RootFSFreePct {
add("sys:rootfs:space", LevelWarn, fmt.Sprintf("Root filesystem %d%% free", pct),
fmt.Sprintf("Root filesystem size %s. Threshold %d%%.", humanBytes(in.RootFS.Total), in.Th.RootFSFreePct))
}
}
if r := in.RepoISO; r != nil {
switch {
case !r.Reachable:
add("repoiso:api:down", LevelCrit, "RepoISO is not responding",
fmt.Sprintf("%s/api/integration/status: %s. Check: systemctl status repoiso-api nginx", r.URL, r.Error))
case !r.WorkerAlive:
add("repoiso:worker:stale", LevelWarn, "RepoISO worker is not running",
fmt.Sprintf("Last worker heartbeat: %s. Downloads, mounts and Proxmox exports are not processed. Check: systemctl status repoiso-worker", orNever(r.WorkerHeartbeat)))
}
if r.Reachable && r.JobsFailed24h > 0 {
add("repoiso:jobs:failed", LevelWarn, fmt.Sprintf("RepoISO: %d failed job(s) in 24 h", r.JobsFailed24h),
"See the Jobs page of RepoISO for the errors.")
}
if r.Reachable && r.PublicationsError > 0 {
add("repoiso:publications:error", LevelWarn, fmt.Sprintf("RepoISO: %d publication(s) not mounted", r.PublicationsError),
"See the Publications page of RepoISO; remount or unpublish the image.")
}
}
sort.Slice(ev, func(i, j int) bool { return ev[i].Key < ev[j].Key })
return ev
}
func orNever(s string) string {
if s == "" {
return "never"
}
return s
}
func humanBytes(b uint64) string {
const unit = 1024
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := uint64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
}