Files
go-sms-V2/internal/pkg/metrics/prometheus.go
T
Tory bb60d60529
ci / test (push) Canceled after 0s
ci / web-test (push) Canceled after 0s
ci / build (push) Canceled after 0s
ci / deploy-dev (push) Canceled after 0s
S4-D1/D2: C1 回调推送引擎+C2 上行自动拉黑——验收项「死信可查/拒收即拉黑」✅: ①C1=callback_task 队列表(迁移 v21,状态机 pending→sent/dead+attempt/next_retry_at/last_code)+PushWorker(2s 扫/批 100/并发 8/2^n 秒指数退避封顶 10min/10 次耗尽死信/UPDATE CAS 防竞态)+双挂点(report 终态首锁[Consumer+PairService,与 CMPP Deliver 同点同语义]+uplink 消费尾)+URL 门控 account_config.report_url/uplink_url(60s TTL 缓存,未配置零任务)+GET /ops/callback(列表+状态计数面板)/POST /ops/callback/:id/resend(casbin ops callback:read/write v21 所种)+前端回调管理页(ops-callback 路由系统管理组+api/callback.ts+状态面板/筛选/载荷展开/死信重派;matrix+guard 33 菜单显式更)+metrics callback_push_total{type,result}/blacklist_total+openapi v1.4.0;②C2=AutoBlacklister 等值词表(trim+lower 防误伤,默认 R/TD/退订/拒收;SystemConfig uplink.blacklist.keywords 可配 60s 刷新)→policy_blacklist(source=2/账号级/uk 幂等)+Redis Set 即时生效,拉黑不影响上行推送(需求原文);③真机全链六步实证(dev 中间件):回执 DELIVRD→detail 终态→HTTP 回调 JSON 送达(200/sent)+上行 R→拉黑(source=2+Redis=1)+推送送达+坏 URL→attempt10→dead→resend ISC:000→修复重投 sent+status_counts 面板;一次性探针 cmd/cbprobe 用后即删+冒烟数据全清;④单测 +13(callback 6[入队门控/载荷/状态机全链/并发批]+uplink 5[词表/配置覆盖/幂等缓存/消费链]+report 2[首锁一次/nil]);⑤门禁=build/vet/-race 40 包/lint 0/eslint0/vue-tsc0/vitest 148/build 全绿;迁移测试 v15..v21 两遍幂等+dev 实跑 v21;gofmt 存量 8 文件顺手清零(S2 遗留);⑥口径诚实标注:载荷无签名挂账/拦截终态 HTTP 未挂(CMPP 侧已有)/PushWorker 单实例 at-least-once;验收单 s4-d1d2-callback.md+CHANGELOG+memory-bank 回填
2026-11-16 11:19:55 +08:00

234 lines
9.5 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package metrics Prometheus 指标中心(Day 13):
// 独立 Registry(不污染全局)+ HTTP 服务指标三件套 + 业务指标预留。
// 暴露方式:app 装配 metrics.Handler() 至 GET /metrics。
package metrics
import (
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// durationBuckets 耗时直方图分桶(对齐 P99<50ms 目标精化低区间)。
var durationBuckets = []float64{
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
}
// HTTP 指标标签维度。
const (
LabelMethod = "method"
LabelRoute = "route"
LabelStatus = "status"
LabelTopic = "topic"
LabelKind = "kind"
)
var (
// HTTPRequestsTotal HTTP 请求总数(method/route/status)。
HTTPRequestsTotal *prometheus.CounterVec
// HTTPRequestDuration HTTP 请求耗时分布(秒,method/route)。
HTTPRequestDuration *prometheus.HistogramVec
// HTTPRequestsInFlight 当前在途请求数。
HTTPRequestsInFlight prometheus.Gauge
// SMSSubmitTotal 短信提交总数(阶段 4 发送链路埋点:result)。
SMSSubmitTotal *prometheus.CounterVec
// SMSReportTotal 状态回执处理总数(阶段 5:status)。
SMSReportTotal *prometheus.CounterVec
// KafkaProducerMessages Kafka 生产结果(topic/kind=ok|retry|dlq)。
KafkaProducerMessages *prometheus.CounterVec
// ---- 资金域(Day54 R1,前缀 go_sms_balance_ 与 alert_rules.yml/评审纪要三方锁定) ----
// BalanceReconcileDiff 对账差额(Redis 余额 vs 流水重算;覆写前采样,{account})。
BalanceReconcileDiff *prometheus.GaugeVec
// BalanceMode 当前预扣模式(1=redis Lua/0=mysql 行锁;flag 切换即置)。
BalanceMode prometheus.Gauge
// BalanceFallbackTotal Redis 基础设施错回退 MySQL 次数(kill Redis 验收观测点)。
BalanceFallbackTotal prometheus.Counter
// BalanceFlowlogPending 异步流水队列水位(批写链路健康度)。
BalanceFlowlogPending prometheus.Gauge
// ---- 消费域(Day55,挂账清偿) ----
// KafkaLag submit 消费组 lag 合计({topic};背压 L1 告警源,Prometheus 抓取)。
KafkaLag *prometheus.GaugeVec
// ---- 长短信聚合域(Day 69,百万级压测观测;Day 71 挂起制语义更新) ----
// LongSmsMergedTotal 入向长短信聚合成功数(组到齐合并单次受理)。
LongSmsMergedTotal prometheus.Counter
// LongSmsDegradeTotal 退化直发片数({reason}: overflow=并发组超上限新组拒绝/
// ref_conflict=同组 total 不一致协议异常片直发。Day 71 起 timeout 语义废除,
// 挂起组到期走 LongSmsExpireFailTotal)。
LongSmsDegradeTotal *prometheus.CounterVec
// LongSmsExpireFailTotal Day 71:挂起组 72h 到期失败终态数(不齐不发,
// 建 LP task+失败 detail+UNDELIV 回执给下家)。
LongSmsExpireFailTotal prometheus.Counter
// LongSmsDupFragTotal 同 idx 重复片幂等覆盖数(客户端重发观测)。
LongSmsDupFragTotal prometheus.Counter
// ReportRequeueTotal unpaired 回执短重投次数(秒回竞态窗口观测)。
ReportRequeueTotal prometheus.Counter
// ---- 回执聚合域(Day 69 二轮,百万级压测观测) ----
// ReportPendingTotal 多段聚合未到齐挂起事件数。
ReportPendingTotal prometheus.Counter
// ReportRecheckTotal pending 后新事务复查结果({result}: finalized=补刀
// 终态/still_pending=真未到齐;双盲竞态健康度=finalized 占比)。
ReportRecheckTotal *prometheus.CounterVec
// SettleAggregateRows 结算域消耗日汇总落账行数(scope: account=billing_
// consume_log / channel=stats_channel_daily;幂等 UPSERT 重刷亦计入,
// 单调累计供观测 job 活性,S5 基线期再评估是否细化)。
SettleAggregateRows *prometheus.CounterVec
// SettleAggregateErrors 消耗日汇总执行失败次数(失败 Warn 不中断,
// 幂等下轮自愈;持续增长=真故障信号)。
SettleAggregateErrors prometheus.Counter
// ---- 回调推送域(S4·C1) ----
// CallbackPushTotal 推送结果计数({type}: report/uplink × {result}:
// sent=2xx 送达/retry=失败待重试/dead=死信)。
CallbackPushTotal *prometheus.CounterVec
// CallbackBlacklistTotal 上行自动拉黑新增数(S4·C2;幂等冲突不计)。
CallbackBlacklistTotal prometheus.Counter
)
var registry = prometheus.NewRegistry()
func init() {
HTTPRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "http_requests_total",
Help: "HTTP 请求总数",
}, []string{LabelMethod, LabelRoute, LabelStatus})
HTTPRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "http_request_duration_seconds",
Help: "HTTP 请求耗时分布(秒)",
Buckets: durationBuckets,
}, []string{LabelMethod, LabelRoute})
HTTPRequestsInFlight = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "http_requests_in_flight",
Help: "当前在途请求数",
})
SMSSubmitTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "sms_submit_total",
Help: "短信提交总数(阶段 4 发送链路)",
}, []string{"result"})
SMSReportTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "sms_report_total",
Help: "状态回执处理总数(阶段 5)",
}, []string{"status"})
KafkaProducerMessages = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "kafka_producer_messages_total",
Help: "Kafka 生产消息结果(ok/retry/dlq)",
}, []string{LabelTopic, LabelKind})
BalanceReconcileDiff = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "go_sms_balance_reconcile_diff",
Help: "R1 对账差额:Redis 预扣余额-流水重算值(覆写前采样;0=一致)",
}, []string{"account"})
BalanceMode = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "go_sms_balance_mode",
Help: "预扣模式(1=redis Lua/0=mysql 行锁;GOSMS_BALANCE_MODE 切换)",
})
BalanceFallbackTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "go_sms_balance_fallback_total",
Help: "Redis 预扣基础设施错单次回退 MySQL 行锁次数",
})
BalanceFlowlogPending = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "go_sms_balance_flowlog_pending",
Help: "Redis 模式异步流水队列待写条数",
})
KafkaLag = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "go_sms_kafka_lag",
Help: "Kafka 消费组 lag 合计(高水位-已提交;背压 L1 告警源,Day53 挂账 Day55 清偿)",
}, []string{LabelTopic})
LongSmsMergedTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "go_sms_longsms_merged_total",
Help: "入向长短信聚合成功数(UDH 组到齐合并单次受理,Day69)",
})
LongSmsDegradeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "go_sms_longsms_degrade_total",
Help: "长短信退化直发片数(reason: overflow=并发组超上限/ref_conflict=协议异常;Day71 起 timeout 废除)",
}, []string{LabelKind})
LongSmsExpireFailTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "go_sms_longsms_expire_fail_total",
Help: "长短信挂起组 72h 到期失败终态数(不齐不发,LP task+UNDELIV 回执,Day71)",
})
LongSmsDupFragTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "go_sms_longsms_dup_frag_total",
Help: "长短信同 idx 重复片幂等覆盖数(客户端重发观测)",
})
ReportRequeueTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "go_sms_report_requeue_total",
Help: "unpaired 回执短重投次数(seg 落库竞态窗口,Day69 三轮)",
})
ReportPendingTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "go_sms_report_pending_total",
Help: "多段回执聚合未到齐挂起事件数(Day69 二轮)",
})
ReportRecheckTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "go_sms_report_recheck_total",
Help: "pending 后新事务复查结果(result: finalized=补刀终态/still_pending=真未到齐)",
}, []string{LabelKind})
SettleAggregateRows = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "go_sms_settle_aggregate_rows_total",
Help: "结算域消耗日汇总落账行数(scope: account/channel;UPSERT 重刷亦计入,S2)",
}, []string{LabelKind})
SettleAggregateErrors = prometheus.NewCounter(prometheus.CounterOpts{
Name: "go_sms_settle_aggregate_errors_total",
Help: "消耗日汇总执行失败次数(幂等下轮自愈,持续增长=真故障,S2)",
})
CallbackPushTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "go_sms_callback_push_total",
Help: "回调推送结果(type: report/uplink × result: sent/retry/dead,S4·C1)",
}, []string{"type", "result"})
CallbackBlacklistTotal = prometheus.NewCounter(prometheus.CounterOpts{
Name: "go_sms_callback_blacklist_total",
Help: "上行自动拉黑新增数(幂等冲突不计,S4·C2)",
})
registry.MustRegister(
HTTPRequestsTotal, HTTPRequestDuration, HTTPRequestsInFlight,
SMSSubmitTotal, SMSReportTotal, KafkaProducerMessages,
BalanceReconcileDiff, BalanceMode, BalanceFallbackTotal, BalanceFlowlogPending,
KafkaLag,
LongSmsMergedTotal, LongSmsDegradeTotal, LongSmsDupFragTotal,
LongSmsExpireFailTotal,
ReportPendingTotal, ReportRecheckTotal, ReportRequeueTotal,
SettleAggregateRows, SettleAggregateErrors,
CallbackPushTotal, CallbackBlacklistTotal,
)
// Go 运行时与进程指标
registry.MustRegister(
collectors.NewGoCollector(),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
)
}
// Registry 返回指标注册中心(测试或自托管采集用)。
func Registry() *prometheus.Registry { return registry }
// Handler /metrics 输出 handler(app 装配:gin.WrapH(metrics.Handler()))。
func Handler() http.Handler {
return promhttp.HandlerFor(registry, promhttp.HandlerOpts{EnableOpenMetrics: true})
}