174 lines
5.3 KiB
Go
174 lines
5.3 KiB
Go
// alert_test.go Day 40(A-3.12):通道告警测试——link_down/fail_rate 触发、冷却去重、停用跳过。
|
|
package channel
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"testing"
|
|
|
|
"go-sms/internal/model/entity"
|
|
"go-sms/internal/pkg/notify"
|
|
)
|
|
|
|
// fakeNotifier 收集通知(实现 notify.Notifier,测试注入)。
|
|
type fakeNotifier struct {
|
|
mu sync.Mutex
|
|
sends []string
|
|
failAll bool
|
|
}
|
|
|
|
func (f *fakeNotifier) Send(_ context.Context, title, content string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.failAll {
|
|
return errFakeSend
|
|
}
|
|
f.sends = append(f.sends, title+"/"+content)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeNotifier) count() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return len(f.sends)
|
|
}
|
|
|
|
var errFakeSend = &sendErr{}
|
|
|
|
type sendErr struct{}
|
|
|
|
func (*sendErr) Error() string { return "fake send failed" }
|
|
|
|
// fakeFactory 构造注入 NewChannelAlertService 的 notifier 工厂。
|
|
func fakeFactory(fake *fakeNotifier) func(string, string) notify.Notifier {
|
|
return func(_, _ string) notify.Notifier { return fake }
|
|
}
|
|
|
|
// TestChannelAlert_LinkDownAndCooldown 验收:启用中 cmpp 通道未建链 → link_down 触发一次;
|
|
// 冷却期内重复扫描/事件路径均不重复通知。
|
|
func TestChannelAlert_LinkDownAndCooldown(t *testing.T) {
|
|
s := newSvc35(t)
|
|
ctx := context.Background()
|
|
ch, err := s.Create(ctx, &CreateReq{
|
|
Name: "down", Code: "al-down", Protocol: "cmpp",
|
|
Host: "127.0.0.1", Port: 1, Username: "u", Password: "pw"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rule := &entity.SystemAlertRule{
|
|
TargetType: "channel", TargetID: ch.ID, Metric: AlertMetricLinkDown,
|
|
Op: "lt", Threshold: 1, NotifyType: "wechat", NotifyTo: "http://fake",
|
|
CooldownMin: 30, Status: 1,
|
|
}
|
|
if err := s.db.Create(rule).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fake := &fakeNotifier{}
|
|
svc := NewChannelAlertService(s.db, s.mon, s.links.Status, fakeFactory(fake))
|
|
|
|
res, err := svc.ScanOnce(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.Rules != 1 || res.Triggered != 1 {
|
|
t.Fatalf("首扫: %+v 期望 rules=1 triggered=1", res)
|
|
}
|
|
if fake.count() != 1 {
|
|
t.Fatalf("通知数=%d 期望 1", fake.count())
|
|
}
|
|
// 冷却:LastAlertAt 刚写 → CooldownMin 内不再通知(cron + 事件路径)。
|
|
res2, _ := svc.ScanOnce(ctx)
|
|
if res2.Triggered != 0 || fake.count() != 1 {
|
|
t.Fatalf("冷却失效: %+v notified=%d", res2, fake.count())
|
|
}
|
|
svc.EvaluateChannel(ctx, ch.ID)
|
|
if fake.count() != 1 {
|
|
t.Fatalf("事件路径冷却失效: notified=%d", fake.count())
|
|
}
|
|
}
|
|
|
|
// TestChannelAlert_NotifyRetry 通知失败不写 LastAlertAt(下轮重试)。
|
|
func TestChannelAlert_NotifyRetry(t *testing.T) {
|
|
s := newSvc35(t)
|
|
ctx := context.Background()
|
|
ch, _ := s.Create(ctx, &CreateReq{
|
|
Name: "retry", Code: "al-retry", Protocol: "cmpp",
|
|
Host: "127.0.0.1", Port: 1, Username: "u", Password: "pw"})
|
|
if err := s.db.Create(&entity.SystemAlertRule{
|
|
TargetType: "channel", TargetID: ch.ID, Metric: AlertMetricLinkDown,
|
|
NotifyType: "wechat", NotifyTo: "http://fake", CooldownMin: 30, Status: 1,
|
|
}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fake := &fakeNotifier{failAll: true}
|
|
svc := NewChannelAlertService(s.db, s.mon, s.links.Status, fakeFactory(fake))
|
|
if res, _ := svc.ScanOnce(ctx); res.Triggered != 0 {
|
|
t.Fatalf("失败通知不应计入触发: %+v", res)
|
|
}
|
|
var rule entity.SystemAlertRule
|
|
s.db.First(&rule)
|
|
if rule.LastAlertAt != nil {
|
|
t.Fatal("通知失败不应写 LastAlertAt")
|
|
}
|
|
// 恢复通知后重试成功。
|
|
fake.mu.Lock()
|
|
fake.failAll = false
|
|
fake.mu.Unlock()
|
|
if res, _ := svc.ScanOnce(ctx); res.Triggered != 1 {
|
|
t.Fatalf("重试未触发: %+v", res)
|
|
}
|
|
}
|
|
|
|
// TestChannelAlert_FailRate 验收:60s 窗口失败率超阈值触发。
|
|
func TestChannelAlert_FailRate(t *testing.T) {
|
|
s := newSvc35(t)
|
|
ctx := context.Background()
|
|
ch, err := s.Create(ctx, &CreateReq{Name: "fr", Code: "al-fr", Protocol: "http", Host: "http://up"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// 2 成 1 败(33%)> 阈值 30%。
|
|
s.mon.ReportSubmit(ch.ID, true, 5)
|
|
s.mon.ReportSubmit(ch.ID, true, 5)
|
|
s.mon.ReportSubmit(ch.ID, false, 0)
|
|
if err := s.db.Create(&entity.SystemAlertRule{
|
|
TargetType: "channel", TargetID: ch.ID, Metric: AlertMetricFailRate,
|
|
Op: "gt", Threshold: 0.3, NotifyType: "wechat", NotifyTo: "http://fake",
|
|
CooldownMin: 30, Status: 1,
|
|
}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fake := &fakeNotifier{}
|
|
svc := NewChannelAlertService(s.db, s.mon, s.links.Status, fakeFactory(fake))
|
|
res, err := svc.ScanOnce(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if res.Triggered != 1 {
|
|
t.Fatalf("fail_rate 未触发: %+v", res)
|
|
}
|
|
}
|
|
|
|
// TestChannelAlert_DisabledChannelSkipped 停用通道 cron 路径跳过。
|
|
func TestChannelAlert_DisabledChannelSkipped(t *testing.T) {
|
|
s := newSvc35(t)
|
|
ctx := context.Background()
|
|
ch, _ := s.Create(ctx, &CreateReq{
|
|
Name: "off", Code: "al-off", Protocol: "cmpp",
|
|
Host: "127.0.0.1", Port: 1, Username: "u", Password: "pw"})
|
|
if err := s.SetStatus(ctx, ch.ID, 0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.db.Create(&entity.SystemAlertRule{
|
|
TargetType: "channel", TargetID: ch.ID, Metric: AlertMetricLinkDown,
|
|
NotifyType: "wechat", NotifyTo: "http://fake", CooldownMin: 30, Status: 1,
|
|
}).Error; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
fake := &fakeNotifier{}
|
|
svc := NewChannelAlertService(s.db, s.mon, s.links.Status, fakeFactory(fake))
|
|
if res, _ := svc.ScanOnce(ctx); res.Triggered != 0 {
|
|
t.Fatalf("停用通道不应告: %+v", res)
|
|
}
|
|
}
|