165 lines
5.6 KiB
Go
165 lines
5.6 KiB
Go
// alert.go Day 40(A-3.12):通道告警服务。
|
|
// 复用 Day 27 余额预警的 SystemAlertRule(target_type=channel)+notify webhook 基建:
|
|
// - metric=link_down : 应建链而全断(running 且 healthy==0,或启用中 cmpp 通道未建链)→ 触发;
|
|
// - metric=fail_rate : 60s 窗口提交失败占比 > threshold(0~1,语义固定 gt)→ 触发;
|
|
//
|
|
// 触发后走 NotifyType/NotifyTo webhook(企微/钉钉),CooldownMin 冷却去重(LastAlertAt 锚点)。
|
|
package channel
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"gorm.io/gorm"
|
|
|
|
"go-sms/internal/model/entity"
|
|
"go-sms/internal/pkg/notify"
|
|
"go-sms/internal/service/channel/pool"
|
|
)
|
|
|
|
// 支持的通道告警指标。
|
|
const (
|
|
AlertMetricLinkDown = "link_down"
|
|
AlertMetricFailRate = "fail_rate"
|
|
)
|
|
|
|
// AlertScanResult 扫描结果(测试/验收视图)。
|
|
type AlertScanResult struct {
|
|
Rules int // 启用中的通道规则数
|
|
Triggered int // 触发通知数
|
|
}
|
|
|
|
// ChannelAlertService 通道告警(nil 安全:未注入时 Evaluate/ScanOnce 为 no-op)。
|
|
type ChannelAlertService struct {
|
|
db *gorm.DB
|
|
mon *Monitor
|
|
links func(id int64) *pool.LinkStatus
|
|
newNotifier func(typ, url string) notify.Notifier
|
|
onLogged func(channelID int64, kind string, level int8, msgID, detail string, latencyMs int)
|
|
}
|
|
|
|
// NewChannelAlertService 构造;links 为链路状态读取函数(解依赖环:linkManager ↔ alert)。
|
|
// newNotifier 为 nil 时用 notify.NewWebhook(测试注入 fake)。
|
|
func NewChannelAlertService(db *gorm.DB, mon *Monitor, links func(id int64) *pool.LinkStatus,
|
|
newNotifier func(typ, url string) notify.Notifier) *ChannelAlertService {
|
|
if newNotifier == nil {
|
|
newNotifier = func(typ, url string) notify.Notifier { return notify.NewWebhook(typ, url) }
|
|
}
|
|
return &ChannelAlertService{db: db, mon: mon, links: links, newNotifier: newNotifier}
|
|
}
|
|
|
|
// EvaluateChannel 单通道即时评估(断连事件触发;冷却由规则 CooldownMin 保证)。
|
|
func (s *ChannelAlertService) EvaluateChannel(ctx context.Context, channelID int64) {
|
|
if s == nil || s.db == nil {
|
|
return
|
|
}
|
|
s.evaluate(ctx, &entity.SystemAlertRule{TargetType: "channel", TargetID: channelID}, false)
|
|
}
|
|
|
|
// ScanOnce 全量扫描启用中的通道规则(cron 30s;含单通道)。
|
|
func (s *ChannelAlertService) ScanOnce(ctx context.Context) (AlertScanResult, error) {
|
|
if s == nil || s.db == nil {
|
|
return AlertScanResult{}, nil
|
|
}
|
|
var rules []entity.SystemAlertRule
|
|
if err := s.db.WithContext(ctx).Where(
|
|
"status = 1 AND target_type = ?", "channel",
|
|
).Find(&rules).Error; err != nil {
|
|
return AlertScanResult{}, err
|
|
}
|
|
res := AlertScanResult{Rules: len(rules)}
|
|
for i := range rules {
|
|
if s.evaluate(ctx, &rules[i], true) {
|
|
res.Triggered++
|
|
}
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// evaluate 单规则评估;返回是否触发。skipMissing=true 时通道不存在/停用则跳过(cron 路径),
|
|
// 事件路径(false)仍评估链路断告警(断连后通道可能被停用但事件有效)。
|
|
func (s *ChannelAlertService) evaluate(ctx context.Context, rule *entity.SystemAlertRule, cronPath bool) bool {
|
|
var ch entity.ChannelChannel
|
|
err := s.db.WithContext(ctx).Where("id = ?", rule.TargetID).Take(&ch).Error
|
|
if err != nil {
|
|
return false
|
|
}
|
|
if cronPath && ch.Status != 1 {
|
|
return false // 停用通道不告(cron 路径)
|
|
}
|
|
|
|
var hit bool
|
|
var cur float64
|
|
switch rule.Metric {
|
|
case AlertMetricLinkDown:
|
|
st := s.linkStatus(ch.ID)
|
|
// 应有链路而全断:启用中 cmpp 通道(未建链或池健康数 0)。
|
|
if ch.Protocol == "cmpp" && ch.Status == 1 {
|
|
if st == nil || !st.Running || st.Healthy == 0 {
|
|
hit, cur = true, 0
|
|
}
|
|
}
|
|
case AlertMetricFailRate:
|
|
submit, fail, _, _, _ := s.mon.Stats(ch.ID)
|
|
if submit > 0 {
|
|
cur = float64(fail) / float64(submit)
|
|
hit = cur > rule.Threshold
|
|
}
|
|
default:
|
|
return false // 未知指标静默跳过(规则录入侧约束)
|
|
}
|
|
if !hit {
|
|
return false
|
|
}
|
|
// 冷却期内不重复通知。
|
|
now := time.Now()
|
|
if rule.LastAlertAt != nil && now.Sub(*rule.LastAlertAt) < time.Duration(rule.CooldownMin)*time.Minute {
|
|
return false
|
|
}
|
|
n := s.newNotifier(rule.NotifyType, rule.NotifyTo)
|
|
title := "【go-sms 通道告警】"
|
|
content := s.alertContent(&ch, rule, cur)
|
|
if err := n.Send(ctx, title, content); err != nil {
|
|
return false // 通知失败不写 LastAlertAt,下轮重试
|
|
}
|
|
_ = s.db.WithContext(ctx).Model(&entity.SystemAlertRule{}).
|
|
Where("id = ?", rule.ID).Update("last_alert_at", now).Error
|
|
// 告警留痕进通道日志(逐条)。
|
|
if s.onLogged != nil {
|
|
s.onLogged(ch.ID, pool.LogKindAlert, 1, "", content, 0)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// SetLogHook 注入告警留痕回调(Service 装配时指向 ChannelLogger,避免依赖环)。
|
|
func (s *ChannelAlertService) SetLogHook(fn func(channelID int64, kind string, level int8, msgID, detail string, latencyMs int)) {
|
|
s.onLogged = fn
|
|
}
|
|
|
|
// alertContent 组装通知文案。
|
|
func (s *ChannelAlertService) alertContent(ch *entity.ChannelChannel, rule *entity.SystemAlertRule, cur float64) string {
|
|
switch rule.Metric {
|
|
case AlertMetricLinkDown:
|
|
st := s.linkStatus(ch.ID)
|
|
running, healthy := false, 0
|
|
if st != nil {
|
|
running, healthy = st.Running, st.Healthy
|
|
}
|
|
return fmt.Sprintf("通道 %s(id=%d,%s)链路异常:running=%v healthy=%d,请检查网关/网络。",
|
|
ch.Name, ch.ID, ch.Protocol, running, healthy)
|
|
case AlertMetricFailRate:
|
|
return fmt.Sprintf("通道 %s(id=%d)60s 窗口失败率 %.1f%% 超阈值 %.1f%%,请关注上游状态。",
|
|
ch.Name, ch.ID, cur*100, rule.Threshold*100)
|
|
}
|
|
return "未知指标"
|
|
}
|
|
|
|
// linkStatus 链路状态读取(links 函数可为 nil→未建链)。
|
|
func (s *ChannelAlertService) linkStatus(id int64) *pool.LinkStatus {
|
|
if s.links == nil {
|
|
return nil
|
|
}
|
|
return s.links(id)
|
|
}
|