Initial import: nasmon health monitor, split out of home_nas
Go service (nasmon/) plus nfpm packaging (packaging/), carried over from the home_nas monorepo. Adds rpm packaging alongside the existing deb build (Fedora/RHEL/Rocky dependency names via nfpm overrides, per-format systemd unit path, dpkg+rpm-compatible maintainer scripts) and a Gitea Actions workflow that builds both formats on every push. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,188 @@
|
||||
package alert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"nasmon/internal/config"
|
||||
"nasmon/internal/store"
|
||||
)
|
||||
|
||||
// Manager reconciles evaluated events with the alerts table and notifies.
|
||||
type Manager struct {
|
||||
Store *store.Store
|
||||
Config *config.Store
|
||||
Send func(cfg config.SMTP, subject, body string) error
|
||||
|
||||
lastErr string // last mail error logged, to avoid repeating it every cycle
|
||||
nextMailTry time.Time // backoff after a delivery failure
|
||||
}
|
||||
|
||||
const mailRetryAfter = 10 * time.Minute
|
||||
|
||||
// Outcome describes what one reconciliation did.
|
||||
type Outcome struct {
|
||||
Opened []store.Alert
|
||||
Resolved []store.Alert
|
||||
Reminded []store.Alert
|
||||
Mailed bool
|
||||
MailErr error
|
||||
}
|
||||
|
||||
func escalated(old, cur string) bool { return old == LevelWarn && cur == LevelCrit }
|
||||
|
||||
func frozen(key string, prefixes []string) bool {
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(key, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Reconcile opens/resolves alerts based on the active events and sends one
|
||||
// email describing everything that changed in this cycle.
|
||||
func (m *Manager) Reconcile(in Input, events []Event) (Outcome, error) {
|
||||
var out Outcome
|
||||
cfg := m.Config.Get()
|
||||
now := in.Now
|
||||
|
||||
open, err := m.Store.OpenAlerts()
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
openByKey := map[string]store.Alert{}
|
||||
for _, a := range open {
|
||||
openByKey[a.Key] = a
|
||||
}
|
||||
active := map[string]Event{}
|
||||
for _, e := range events {
|
||||
active[e.Key] = e
|
||||
}
|
||||
|
||||
// alerts re-raised in this cycle (acknowledged, but the problem changed)
|
||||
reraised := map[int64]bool{}
|
||||
for _, e := range events {
|
||||
if a, ok := openByKey[e.Key]; ok {
|
||||
changed := a.Detail != e.Detail || a.Level != e.Level
|
||||
if changed {
|
||||
_ = m.Store.UpdateAlertDetail(a.ID, e.Level, e.Detail)
|
||||
}
|
||||
// an acknowledged alert stays quiet until the problem gets worse:
|
||||
// escalated level or different details (grown counters, new errors)
|
||||
if a.Acked() && (a.Detail != e.Detail || escalated(a.Level, e.Level)) {
|
||||
_ = m.Store.ReraiseAlert(a.ID)
|
||||
a.AckedAt, a.LastNotifiedAt, a.Level, a.Detail = nil, nil, e.Level, e.Detail
|
||||
openByKey[e.Key] = a
|
||||
reraised[a.ID] = true
|
||||
log.Printf("alert re-raised: %s — %s", a.Key, a.Title)
|
||||
out.Opened = append(out.Opened, a)
|
||||
}
|
||||
continue
|
||||
}
|
||||
a, err := m.Store.OpenAlert(e.Key, e.Level, e.Title, e.Detail, now)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
log.Printf("alert opened: %s — %s", a.Key, a.Title)
|
||||
out.Opened = append(out.Opened, a)
|
||||
}
|
||||
|
||||
remind := time.Duration(cfg.RemindHours) * time.Hour
|
||||
for _, a := range open {
|
||||
if _, still := active[a.Key]; still {
|
||||
if reraised[a.ID] {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case a.Acked():
|
||||
// acknowledged: no reminders, no retries
|
||||
case a.LastNotifiedAt == nil:
|
||||
// never delivered (mail failed earlier): retry as new
|
||||
out.Opened = append(out.Opened, a)
|
||||
case remind > 0 && now.Sub(*a.LastNotifiedAt) >= remind:
|
||||
out.Reminded = append(out.Reminded, a)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if frozen(a.Key, in.Frozen) {
|
||||
continue
|
||||
}
|
||||
if err := m.Store.ResolveAlert(a.ID, now); err != nil {
|
||||
return out, err
|
||||
}
|
||||
log.Printf("alert resolved: %s — %s", a.Key, a.Title)
|
||||
out.Resolved = append(out.Resolved, a)
|
||||
}
|
||||
|
||||
if !cfg.AlertsEnabled || m.Send == nil {
|
||||
return out, nil
|
||||
}
|
||||
if len(out.Opened) == 0 && len(out.Reminded) == 0 && (len(out.Resolved) == 0 || !cfg.NotifyResolved) {
|
||||
return out, nil
|
||||
}
|
||||
if now.Before(m.nextMailTry) {
|
||||
return out, nil
|
||||
}
|
||||
subject, body := compose(cfg, out)
|
||||
if err := m.Send(cfg.SMTP, subject, body); err != nil {
|
||||
out.MailErr = err
|
||||
m.nextMailTry = now.Add(mailRetryAfter)
|
||||
if err.Error() != m.lastErr {
|
||||
log.Printf("alert mail failed (retry in %s): %v", mailRetryAfter, err)
|
||||
m.lastErr = err.Error()
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
m.lastErr, m.nextMailTry = "", time.Time{}
|
||||
out.Mailed = true
|
||||
var ids []int64
|
||||
for _, a := range append(append([]store.Alert{}, out.Opened...), out.Reminded...) {
|
||||
ids = append(ids, a.ID)
|
||||
}
|
||||
_ = m.Store.MarkNotified(ids, now)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func compose(cfg config.Settings, o Outcome) (subject, body string) {
|
||||
host, _ := os.Hostname()
|
||||
var parts []string
|
||||
if n := len(o.Opened); n > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d new", n))
|
||||
}
|
||||
if n := len(o.Reminded); n > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d still open", n))
|
||||
}
|
||||
if n := len(o.Resolved); n > 0 && cfg.NotifyResolved {
|
||||
parts = append(parts, fmt.Sprintf("%d resolved", n))
|
||||
}
|
||||
subject = strings.TrimSpace(cfg.SMTP.SubjectPrefix + " " + strings.Join(parts, ", ") + " on " + host)
|
||||
if len(o.Opened) == 1 && len(o.Reminded) == 0 && len(o.Resolved) == 0 {
|
||||
subject = strings.TrimSpace(cfg.SMTP.SubjectPrefix + " " + o.Opened[0].Title)
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "nasmon on %s, %s\n\n", host, time.Now().Format("2006-01-02 15:04:05 MST"))
|
||||
section := func(title string, list []store.Alert) {
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(&b, "== %s ==\n\n", title)
|
||||
for _, a := range list {
|
||||
fmt.Fprintf(&b, "[%s] %s\n", strings.ToUpper(a.Level), a.Title)
|
||||
for _, l := range strings.Split(a.Detail, "\n") {
|
||||
fmt.Fprintf(&b, " %s\n", l)
|
||||
}
|
||||
fmt.Fprintf(&b, " since %s\n\n", a.OpenedAt.Format("2006-01-02 15:04"))
|
||||
}
|
||||
}
|
||||
section("NEW", o.Opened)
|
||||
section("STILL OPEN", o.Reminded)
|
||||
if cfg.NotifyResolved {
|
||||
section("RESOLVED", o.Resolved)
|
||||
}
|
||||
return subject, b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user