80 lines
3.6 KiB
YAML
80 lines
3.6 KiB
YAML
# alert_rules.yml Day 53(runbook §6.2):告警规则(指标现实核对版)。
|
|
# 已可用指标(internal/pkg/metrics/prometheus.go):http_requests_total{method,route,status}/
|
|
# http_request_duration_seconds_bucket{method,route}/http_requests_in_flight。
|
|
# 挂账注记:
|
|
# - ~~Kafka lag(P1)~~(Day55 已清偿:gateway 侧 LagReader 采样导出 go_sms_kafka_lag{topic},
|
|
# 规则见 gateway-p1 组;背压三级水位 L1 口径一致 lag>10 万);
|
|
# - 通道速率异常(P0 资损):Monitor 内存指标未导出 Prometheus——挂账 Day56 并入;
|
|
# - BalanceReconcileDiff(P0·R1 前置接入位):Day54 对账任务导出
|
|
# go_sms_balance_reconcile_diff{account} 后本规则即生效(abs>0 for 1m)。
|
|
groups:
|
|
- name: gateway-p0
|
|
rules:
|
|
- alert: HTTPError5xxBurst
|
|
expr: sum by (route) (rate(http_requests_total{status=~"5.."}[5m])) > 1
|
|
for: 2m
|
|
labels:
|
|
severity: P0
|
|
annotations:
|
|
summary: "5xx 突增 {{ $labels.route }}"
|
|
description: "5 分钟 5xx 速率 {{ $value }}/s 超 1(runbook §0:全站不可用/资损级)"
|
|
|
|
- alert: BalanceReconcileDiff
|
|
expr: abs(go_sms_balance_reconcile_diff) > 0
|
|
for: 1m
|
|
labels:
|
|
severity: P0
|
|
annotations:
|
|
summary: "R1 余额对账差额 account={{ $labels.account }}"
|
|
description: "Redis 预扣余额与 MySQL 流水重算差 {{ $value }} 条(流水为唯一事实源,runbook §3 数据修复专项)"
|
|
|
|
- name: gateway-p1
|
|
rules:
|
|
- alert: HTTPP99High
|
|
expr: histogram_quantile(0.99, sum by (route, le) (rate(http_request_duration_seconds_bucket[5m]))) > 0.5
|
|
for: 5m
|
|
labels:
|
|
severity: P1
|
|
annotations:
|
|
summary: "P99>500ms {{ $labels.route }}"
|
|
description: "接口 P99 {{ $value }}s(runbook §6.2 P1 口径)"
|
|
|
|
- alert: KafkaLagHigh
|
|
expr: go_sms_kafka_lag > 100000
|
|
for: 2m
|
|
labels:
|
|
severity: P1
|
|
annotations:
|
|
summary: "submit 消费组 lag>10 万 topic={{ $labels.topic }}"
|
|
description: "Kafka 积压 {{ $value }} 条(背压 L1 告警口径;持续不收敛查 consumer 通道侧限速/暂停,runbook §2)"
|
|
|
|
# Day 69 百万级长短信压测观测组(gateway: 入向聚合健康度/consumer: 回执聚合健康度)。
|
|
- name: longsms-1m
|
|
rules:
|
|
- alert: LongSmsExpireFailBurst
|
|
expr: increase(go_sms_longsms_expire_fail_total[1h]) > 10
|
|
for: 5m
|
|
labels:
|
|
severity: P1
|
|
annotations:
|
|
summary: "长短信挂起组 72h 到期失败突增"
|
|
description: "1h 到期失败 {{ $value }} 组——客户端拆片后大面积丢片(如 ISC:004 32% 超时率),建议推进客户端背靠背发片整改"
|
|
|
|
- alert: LongSmsOverflowDegraded
|
|
expr: increase(go_sms_longsms_degrade_total{kind=~"overflow|ref_conflict"}[5m]) > 0
|
|
for: 1m
|
|
labels:
|
|
severity: P1
|
|
annotations:
|
|
summary: "长短信聚合溢出/协议异常退化"
|
|
description: "并发未到齐组逼近 4096 上限或同组 total 不一致(5 分钟 {{ $value }} 片)——压测速率×片间隔需下调"
|
|
|
|
- alert: ReportPendingNoFinalize
|
|
expr: rate(go_sms_report_pending_total[5m]) > 1 and rate(go_sms_report_recheck_total{kind="finalized"}[5m]) == 0
|
|
for: 10m
|
|
labels:
|
|
severity: P1
|
|
annotations:
|
|
summary: "回执聚合持续挂起且复查零终态"
|
|
description: "pending 事件持续产生而 recheck 补刀为 0——回执段丢失(通道未回执),将挂至 48h 补偿,核对上游通道回执率"
|