// Package alert evaluates health rules, tracks open alerts and sends mail. package alert import ( "fmt" "sort" "strings" "time" "nasmon/internal/collect" "nasmon/internal/config" "nasmon/internal/store" ) const ( LevelWarn = "warn" LevelCrit = "crit" ) // Event is one active problem. Key is stable across evaluations so the same // condition maps to the same open alert: "::". type Event struct { Key string Level string Title string Detail string } // Input is everything the rules look at in one evaluation. type Input struct { Now time.Time Th config.Thresholds Smarts []collect.Smart // disks with usable SMART data (standby ones excluded) Prev map[string]*collect.Smart // previous snapshot by serial, for growth rules Disks []collect.Disk Seen []store.SeenDisk Pools []collect.Pool PoolsOK bool CPUTemp float64 HasCPU bool RootFS collect.FSUsage RootOK bool // RepoISO is the polled RepoISO health (nil when the integration is off). RepoISO *collect.RepoISO // Frozen lists key prefixes whose open alerts must be left untouched // because no fresh data was available for them in this cycle. Frozen []string } // criticalATA are attributes whose non-zero raw value is a warning sign. var criticalATA = map[int]string{ 5: "Reallocated_Sector_Ct", 187: "Reported_Uncorrect", 188: "Command_Timeout", 197: "Current_Pending_Sector", 198: "Offline_Uncorrectable", } func diskLabel(s collect.Smart) string { return fmt.Sprintf("%s (%s, %s)", s.Device, s.Model, s.Serial) } // Evaluate returns the list of currently active events. func Evaluate(in Input) []Event { var ev []Event add := func(key, level, title, detail string) { ev = append(ev, Event{Key: key, Level: level, Title: title, Detail: detail}) } for _, s := range in.Smarts { if s.Error != "" && !s.Supported { continue } pfx := "smart:" + s.Serial + ":" lbl := diskLabel(s) if s.Supported && !s.Healthy { add(pfx+"health", LevelCrit, "SMART health FAILED: "+lbl, "smartctl reports overall-health self-assessment: FAILED. Back up the data and replace the disk.") } var bad []string level := LevelWarn for _, a := range s.Attrs { if _, ok := criticalATA[a.ID]; ok && a.Raw > 0 { bad = append(bad, fmt.Sprintf("%d %s raw=%d", a.ID, a.Name, a.Raw)) if a.ID != 187 && a.ID != 188 { // uncorrect/timeouts are a hint, bad sectors are damage level = LevelCrit } } if a.WhenFailed != "" { bad = append(bad, fmt.Sprintf("%d %s when_failed=%s (value %d, thresh %d)", a.ID, a.Name, a.WhenFailed, a.Value, a.Thresh)) level = LevelCrit } } if len(bad) > 0 { add(pfx+"attr", level, "SMART attributes degraded: "+lbl, strings.Join(bad, "\n")) } if cur, ok := s.Attr(199); ok { if prev := in.Prev[s.Serial]; prev != nil { if p, ok := prev.Attr(199); ok && cur.Raw > p.Raw { add(pfx+"crc", LevelWarn, "UDMA CRC errors increasing: "+lbl, fmt.Sprintf("199 UDMA_CRC_Error_Count %d -> %d. Usually a cable/connector problem.", p.Raw, cur.Raw)) } } } if n := s.NVMe; n != nil { var msgs []string if n.CriticalWarning != 0 { msgs = append(msgs, fmt.Sprintf("critical_warning=0x%02x", n.CriticalWarning)) } if n.MediaErrors > 0 { msgs = append(msgs, fmt.Sprintf("media_errors=%d", n.MediaErrors)) } if n.PercentageUsed >= in.Th.NVMeUsedPct { msgs = append(msgs, fmt.Sprintf("percentage_used=%d%% (>= %d%%)", n.PercentageUsed, in.Th.NVMeUsedPct)) } if n.AvailableSpare < in.Th.NVMeSparePct { msgs = append(msgs, fmt.Sprintf("available_spare=%d%% (< %d%%)", n.AvailableSpare, in.Th.NVMeSparePct)) } if len(msgs) > 0 { add(pfx+"nvme", LevelCrit, "NVMe health: "+lbl, strings.Join(msgs, "\n")) } } if s.HasTemp && in.Th.DiskTempC > 0 && s.TempC >= in.Th.DiskTempC { add(pfx+"temp", LevelWarn, fmt.Sprintf("Disk temperature %d°C: %s", s.TempC, lbl), fmt.Sprintf("Threshold is %d°C.", in.Th.DiskTempC)) } if t := s.LastTest; t != nil && !t.InProgress && !t.Passed { add(pfx+"selftest", LevelCrit, "SMART self-test failed: "+lbl, fmt.Sprintf("%s: %s (at %d power-on hours)", t.Type, t.Status, t.Hours)) } } // disks that were seen before but are gone now present := map[string]bool{} for _, d := range in.Disks { present[d.Serial] = true } for _, d := range in.Seen { if !present[d.Serial] { add("disk:"+d.Serial+":missing", LevelCrit, fmt.Sprintf("Disk disappeared: %s (%s)", d.Model, d.Serial), fmt.Sprintf("Last seen as %s at %s. If it was removed on purpose, forget it on the disk page.", d.Device, d.LastSeen.Format(time.RFC3339))) } } if in.PoolsOK { for _, p := range in.Pools { pfx := "zfs:" + p.Name + ":" if p.Health != "ONLINE" { detail := "State: " + p.Health if p.Status != "" { detail += "\nStatus: " + p.Status } if p.Action != "" { detail += "\nAction: " + p.Action } add(pfx+"health", LevelCrit, fmt.Sprintf("Pool %s is %s", p.Name, p.Health), detail) } var errs []string for _, v := range p.Vdevs { if v.Read > 0 || v.Write > 0 || v.Cksum > 0 { errs = append(errs, fmt.Sprintf("%s: read=%d write=%d cksum=%d", v.Name, v.Read, v.Write, v.Cksum)) } } if p.Errors != "" && !strings.HasPrefix(p.Errors, "No known data errors") { errs = append(errs, "errors: "+p.Errors) } if p.ScanErrs > 0 { errs = append(errs, fmt.Sprintf("last scan finished with %d errors", p.ScanErrs)) } if len(errs) > 0 { add(pfx+"errors", LevelCrit, "Pool "+p.Name+" has errors", strings.Join(errs, "\n")) } if in.Th.PoolCapacityPct > 0 && p.CapPct >= in.Th.PoolCapacityPct { add(pfx+"capacity", LevelWarn, fmt.Sprintf("Pool %s is %d%% full", p.Name, p.CapPct), fmt.Sprintf("Pool size %s. Threshold %d%%.", humanBytes(p.Size), in.Th.PoolCapacityPct)) } if in.Th.ScrubMaxAgeDays > 0 && !p.Scrubbing { maxAge := time.Duration(in.Th.ScrubMaxAgeDays) * 24 * time.Hour if p.LastScrub == nil { add(pfx+"scrub", LevelWarn, "Pool "+p.Name+" was never scrubbed", "Run: zpool scrub "+p.Name) } else if age := in.Now.Sub(*p.LastScrub); age > maxAge { add(pfx+"scrub", LevelWarn, fmt.Sprintf("Pool %s: last scrub %d days ago", p.Name, int(age.Hours()/24)), fmt.Sprintf("Last scrub %s. Threshold %d days.", p.LastScrub.Format("2006-01-02"), in.Th.ScrubMaxAgeDays)) } } } } if in.HasCPU && in.Th.CPUTempC > 0 && in.CPUTemp >= float64(in.Th.CPUTempC) { add("sys:cpu:temp", LevelWarn, fmt.Sprintf("CPU temperature %.0f°C", in.CPUTemp), fmt.Sprintf("Threshold %d°C.", in.Th.CPUTempC)) } if in.RootOK && in.RootFS.Total > 0 && in.Th.RootFSFreePct > 0 { pct := int(in.RootFS.Free * 100 / in.RootFS.Total) if pct < in.Th.RootFSFreePct { add("sys:rootfs:space", LevelWarn, fmt.Sprintf("Root filesystem %d%% free", pct), fmt.Sprintf("Root filesystem size %s. Threshold %d%%.", humanBytes(in.RootFS.Total), in.Th.RootFSFreePct)) } } if r := in.RepoISO; r != nil { switch { case !r.Reachable: add("repoiso:api:down", LevelCrit, "RepoISO is not responding", fmt.Sprintf("%s/api/integration/status: %s. Check: systemctl status repoiso-api nginx", r.URL, r.Error)) case !r.WorkerAlive: add("repoiso:worker:stale", LevelWarn, "RepoISO worker is not running", fmt.Sprintf("Last worker heartbeat: %s. Downloads, mounts and Proxmox exports are not processed. Check: systemctl status repoiso-worker", orNever(r.WorkerHeartbeat))) } if r.Reachable && r.JobsFailed24h > 0 { add("repoiso:jobs:failed", LevelWarn, fmt.Sprintf("RepoISO: %d failed job(s) in 24 h", r.JobsFailed24h), "See the Jobs page of RepoISO for the errors.") } if r.Reachable && r.PublicationsError > 0 { add("repoiso:publications:error", LevelWarn, fmt.Sprintf("RepoISO: %d publication(s) not mounted", r.PublicationsError), "See the Publications page of RepoISO; remount or unpublish the image.") } } sort.Slice(ev, func(i, j int) bool { return ev[i].Key < ev[j].Key }) return ev } func orNever(s string) string { if s == "" { return "never" } return s } func humanBytes(b uint64) string { const unit = 1024 if b < unit { return fmt.Sprintf("%d B", b) } div, exp := uint64(unit), 0 for n := b / unit; n >= unit; n /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp]) }