nasmon/internal/alert/state.go

package alert

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"

	"nasmon/internal/config"
	"nasmon/internal/store"
)

// Manager reconciles evaluated events with the alerts table and notifies.
type Manager struct {
	Store  *store.Store
	Config *config.Store
	Send   func(cfg config.SMTP, subject, body string) error

	lastErr     string    // last mail error logged, to avoid repeating it every cycle
	nextMailTry time.Time // backoff after a delivery failure
}

const mailRetryAfter = 10 * time.Minute

// Outcome describes what one reconciliation did.
type Outcome struct {
	Opened   []store.Alert
	Resolved []store.Alert
	Reminded []store.Alert
	Mailed   bool
	MailErr  error
}

func escalated(old, cur string) bool { return old == LevelWarn && cur == LevelCrit }

func frozen(key string, prefixes []string) bool {
	for _, p := range prefixes {
		if strings.HasPrefix(key, p) {
			return true
		}
	}
	return false
}

// Reconcile opens/resolves alerts based on the active events and sends one
// email describing everything that changed in this cycle.
func (m *Manager) Reconcile(in Input, events []Event) (Outcome, error) {
	var out Outcome
	cfg := m.Config.Get()
	now := in.Now

	open, err := m.Store.OpenAlerts()
	if err != nil {
		return out, err
	}
	openByKey := map[string]store.Alert{}
	for _, a := range open {
		openByKey[a.Key] = a
	}
	active := map[string]Event{}
	for _, e := range events {
		active[e.Key] = e
	}

	// alerts re-raised in this cycle (acknowledged, but the problem changed)
	reraised := map[int64]bool{}
	for _, e := range events {
		if a, ok := openByKey[e.Key]; ok {
			changed := a.Detail != e.Detail || a.Level != e.Level
			if changed {
				_ = m.Store.UpdateAlertDetail(a.ID, e.Level, e.Detail)
			}
			// an acknowledged alert stays quiet until the problem gets worse:
			// escalated level or different details (grown counters, new errors)
			if a.Acked() && (a.Detail != e.Detail || escalated(a.Level, e.Level)) {
				_ = m.Store.ReraiseAlert(a.ID)
				a.AckedAt, a.LastNotifiedAt, a.Level, a.Detail = nil, nil, e.Level, e.Detail
				openByKey[e.Key] = a
				reraised[a.ID] = true
				log.Printf("alert re-raised: %s — %s", a.Key, a.Title)
				out.Opened = append(out.Opened, a)
			}
			continue
		}
		a, err := m.Store.OpenAlert(e.Key, e.Level, e.Title, e.Detail, now)
		if err != nil {
			return out, err
		}
		log.Printf("alert opened: %s — %s", a.Key, a.Title)
		out.Opened = append(out.Opened, a)
	}

	remind := time.Duration(cfg.RemindHours) * time.Hour
	for _, a := range open {
		if _, still := active[a.Key]; still {
			if reraised[a.ID] {
				continue
			}
			switch {
			case a.Acked():
				// acknowledged: no reminders, no retries
			case a.LastNotifiedAt == nil:
				// never delivered (mail failed earlier): retry as new
				out.Opened = append(out.Opened, a)
			case remind > 0 && now.Sub(*a.LastNotifiedAt) >= remind:
				out.Reminded = append(out.Reminded, a)
			}
			continue
		}
		if frozen(a.Key, in.Frozen) {
			continue
		}
		if err := m.Store.ResolveAlert(a.ID, now); err != nil {
			return out, err
		}
		log.Printf("alert resolved: %s — %s", a.Key, a.Title)
		out.Resolved = append(out.Resolved, a)
	}

	if !cfg.AlertsEnabled || m.Send == nil {
		return out, nil
	}
	if len(out.Opened) == 0 && len(out.Reminded) == 0 && (len(out.Resolved) == 0 || !cfg.NotifyResolved) {
		return out, nil
	}
	if now.Before(m.nextMailTry) {
		return out, nil
	}
	subject, body := compose(cfg, out)
	if err := m.Send(cfg.SMTP, subject, body); err != nil {
		out.MailErr = err
		m.nextMailTry = now.Add(mailRetryAfter)
		if err.Error() != m.lastErr {
			log.Printf("alert mail failed (retry in %s): %v", mailRetryAfter, err)
			m.lastErr = err.Error()
		}
		return out, nil
	}
	m.lastErr, m.nextMailTry = "", time.Time{}
	out.Mailed = true
	var ids []int64
	for _, a := range append(append([]store.Alert{}, out.Opened...), out.Reminded...) {
		ids = append(ids, a.ID)
	}
	_ = m.Store.MarkNotified(ids, now)
	return out, nil
}

func compose(cfg config.Settings, o Outcome) (subject, body string) {
	host, _ := os.Hostname()
	var parts []string
	if n := len(o.Opened); n > 0 {
		parts = append(parts, fmt.Sprintf("%d new", n))
	}
	if n := len(o.Reminded); n > 0 {
		parts = append(parts, fmt.Sprintf("%d still open", n))
	}
	if n := len(o.Resolved); n > 0 && cfg.NotifyResolved {
		parts = append(parts, fmt.Sprintf("%d resolved", n))
	}
	subject = strings.TrimSpace(cfg.SMTP.SubjectPrefix + " " + strings.Join(parts, ", ") + " on " + host)
	if len(o.Opened) == 1 && len(o.Reminded) == 0 && len(o.Resolved) == 0 {
		subject = strings.TrimSpace(cfg.SMTP.SubjectPrefix + " " + o.Opened[0].Title)
	}

	var b strings.Builder
	fmt.Fprintf(&b, "nasmon on %s, %s\n\n", host, time.Now().Format("2006-01-02 15:04:05 MST"))
	section := func(title string, list []store.Alert) {
		if len(list) == 0 {
			return
		}
		fmt.Fprintf(&b, "== %s ==\n\n", title)
		for _, a := range list {
			fmt.Fprintf(&b, "[%s] %s\n", strings.ToUpper(a.Level), a.Title)
			for _, l := range strings.Split(a.Detail, "\n") {
				fmt.Fprintf(&b, "    %s\n", l)
			}
			fmt.Fprintf(&b, "    since %s\n\n", a.OpenedAt.Format("2006-01-02 15:04"))
		}
	}
	section("NEW", o.Opened)
	section("STILL OPEN", o.Reminded)
	if cfg.NotifyResolved {
		section("RESOLVED", o.Resolved)
	}
	return subject, b.String()
}