Files
go-sms-V2/deploy/monitoring/alert_rules.yml
T

80 lines
3.6 KiB
YAML

# alert_rules.yml Day 53(runbook §6.2):告警规则(指标现实核对版)。
# 已可用指标(internal/pkg/metrics/prometheus.go):http_requests_total{method,route,status}/
# http_request_duration_seconds_bucket{method,route}/http_requests_in_flight。
# 挂账注记:
# - ~~Kafka lag(P1)~~(Day55 已清偿:gateway 侧 LagReader 采样导出 go_sms_kafka_lag{topic},
# 规则见 gateway-p1 组;背压三级水位 L1 口径一致 lag>10 万);
# - 通道速率异常(P0 资损):Monitor 内存指标未导出 Prometheus——挂账 Day56 并入;
# - BalanceReconcileDiff(P0·R1 前置接入位):Day54 对账任务导出
# go_sms_balance_reconcile_diff{account} 后本规则即生效(abs>0 for 1m)。
groups:
- name: gateway-p0
rules:
- alert: HTTPError5xxBurst
expr: sum by (route) (rate(http_requests_total{status=~"5.."}[5m])) > 1
for: 2m
labels:
severity: P0
annotations:
summary: "5xx 突增 {{ $labels.route }}"
description: "5 分钟 5xx 速率 {{ $value }}/s 超 1(runbook §0:全站不可用/资损级)"
- alert: BalanceReconcileDiff
expr: abs(go_sms_balance_reconcile_diff) > 0
for: 1m
labels:
severity: P0
annotations:
summary: "R1 余额对账差额 account={{ $labels.account }}"
description: "Redis 预扣余额与 MySQL 流水重算差 {{ $value }} 条(流水为唯一事实源,runbook §3 数据修复专项)"
- name: gateway-p1
rules:
- alert: HTTPP99High
expr: histogram_quantile(0.99, sum by (route, le) (rate(http_request_duration_seconds_bucket[5m]))) > 0.5
for: 5m
labels:
severity: P1
annotations:
summary: "P99>500ms {{ $labels.route }}"
description: "接口 P99 {{ $value }}s(runbook §6.2 P1 口径)"
- alert: KafkaLagHigh
expr: go_sms_kafka_lag > 100000
for: 2m
labels:
severity: P1
annotations:
summary: "submit 消费组 lag>10 万 topic={{ $labels.topic }}"
description: "Kafka 积压 {{ $value }} 条(背压 L1 告警口径;持续不收敛查 consumer 通道侧限速/暂停,runbook §2)"
# Day 69 百万级长短信压测观测组(gateway: 入向聚合健康度/consumer: 回执聚合健康度)。
- name: longsms-1m
rules:
- alert: LongSmsExpireFailBurst
expr: increase(go_sms_longsms_expire_fail_total[1h]) > 10
for: 5m
labels:
severity: P1
annotations:
summary: "长短信挂起组 72h 到期失败突增"
description: "1h 到期失败 {{ $value }} 组——客户端拆片后大面积丢片(如 ISC:004 32% 超时率),建议推进客户端背靠背发片整改"
- alert: LongSmsOverflowDegraded
expr: increase(go_sms_longsms_degrade_total{kind=~"overflow|ref_conflict"}[5m]) > 0
for: 1m
labels:
severity: P1
annotations:
summary: "长短信聚合溢出/协议异常退化"
description: "并发未到齐组逼近 4096 上限或同组 total 不一致(5 分钟 {{ $value }} 片)——压测速率×片间隔需下调"
- alert: ReportPendingNoFinalize
expr: rate(go_sms_report_pending_total[5m]) > 1 and rate(go_sms_report_recheck_total{kind="finalized"}[5m]) == 0
for: 10m
labels:
severity: P1
annotations:
summary: "回执聚合持续挂起且复查零终态"
description: "pending 事件持续产生而 recheck 补刀为 0——回执段丢失(通道未回执),将挂至 48h 补偿,核对上游通道回执率"