216 lines
8.7 KiB
Go
216 lines
8.7 KiB
Go
// Package metrics Prometheus 指标中心(Day 13):
|
|
// 独立 Registry(不污染全局)+ HTTP 服务指标三件套 + 业务指标预留。
|
|
// 暴露方式:app 装配 metrics.Handler() 至 GET /metrics。
|
|
package metrics
|
|
|
|
import (
|
|
"net/http"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/collectors"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
)
|
|
|
|
// durationBuckets 耗时直方图分桶(对齐 P99<50ms 目标精化低区间)。
|
|
var durationBuckets = []float64{
|
|
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
|
|
}
|
|
|
|
// HTTP 指标标签维度。
|
|
const (
|
|
LabelMethod = "method"
|
|
LabelRoute = "route"
|
|
LabelStatus = "status"
|
|
LabelTopic = "topic"
|
|
LabelKind = "kind"
|
|
)
|
|
|
|
var (
|
|
// HTTPRequestsTotal HTTP 请求总数(method/route/status)。
|
|
HTTPRequestsTotal *prometheus.CounterVec
|
|
// HTTPRequestDuration HTTP 请求耗时分布(秒,method/route)。
|
|
HTTPRequestDuration *prometheus.HistogramVec
|
|
// HTTPRequestsInFlight 当前在途请求数。
|
|
HTTPRequestsInFlight prometheus.Gauge
|
|
|
|
// SMSSubmitTotal 短信提交总数(阶段 4 发送链路埋点:result)。
|
|
SMSSubmitTotal *prometheus.CounterVec
|
|
// SMSReportTotal 状态回执处理总数(阶段 5:status)。
|
|
SMSReportTotal *prometheus.CounterVec
|
|
// KafkaProducerMessages Kafka 生产结果(topic/kind=ok|retry|dlq)。
|
|
KafkaProducerMessages *prometheus.CounterVec
|
|
|
|
// ---- 资金域(Day54 R1,前缀 go_sms_balance_ 与 alert_rules.yml/评审纪要三方锁定) ----
|
|
// BalanceReconcileDiff 对账差额(Redis 余额 vs 流水重算;覆写前采样,{account})。
|
|
BalanceReconcileDiff *prometheus.GaugeVec
|
|
// BalanceMode 当前预扣模式(1=redis Lua/0=mysql 行锁;flag 切换即置)。
|
|
BalanceMode prometheus.Gauge
|
|
// BalanceFallbackTotal Redis 基础设施错回退 MySQL 次数(kill Redis 验收观测点)。
|
|
BalanceFallbackTotal prometheus.Counter
|
|
// BalanceFlowlogPending 异步流水队列水位(批写链路健康度)。
|
|
BalanceFlowlogPending prometheus.Gauge
|
|
|
|
// ---- 消费域(Day55,挂账清偿) ----
|
|
// KafkaLag submit 消费组 lag 合计({topic};背压 L1 告警源,Prometheus 抓取)。
|
|
KafkaLag *prometheus.GaugeVec
|
|
|
|
// ---- 长短信聚合域(Day 69,百万级压测观测;Day 71 挂起制语义更新) ----
|
|
// LongSmsMergedTotal 入向长短信聚合成功数(组到齐合并单次受理)。
|
|
LongSmsMergedTotal prometheus.Counter
|
|
// LongSmsDegradeTotal 退化直发片数({reason}: overflow=并发组超上限新组拒绝/
|
|
// ref_conflict=同组 total 不一致协议异常片直发。Day 71 起 timeout 语义废除,
|
|
// 挂起组到期走 LongSmsExpireFailTotal)。
|
|
LongSmsDegradeTotal *prometheus.CounterVec
|
|
// LongSmsExpireFailTotal Day 71:挂起组 72h 到期失败终态数(不齐不发,
|
|
// 建 LP task+失败 detail+UNDELIV 回执给下家)。
|
|
LongSmsExpireFailTotal prometheus.Counter
|
|
// LongSmsDupFragTotal 同 idx 重复片幂等覆盖数(客户端重发观测)。
|
|
LongSmsDupFragTotal prometheus.Counter
|
|
// ReportRequeueTotal unpaired 回执短重投次数(秒回竞态窗口观测)。
|
|
ReportRequeueTotal prometheus.Counter
|
|
|
|
// ---- 回执聚合域(Day 69 二轮,百万级压测观测) ----
|
|
// ReportPendingTotal 多段聚合未到齐挂起事件数。
|
|
ReportPendingTotal prometheus.Counter
|
|
// ReportRecheckTotal pending 后新事务复查结果({result}: finalized=补刀
|
|
// 终态/still_pending=真未到齐;双盲竞态健康度=finalized 占比)。
|
|
ReportRecheckTotal *prometheus.CounterVec
|
|
|
|
// SettleAggregateRows 结算域消耗日汇总落账行数(scope: account=billing_
|
|
// consume_log / channel=stats_channel_daily;幂等 UPSERT 重刷亦计入,
|
|
// 单调累计供观测 job 活性,S5 基线期再评估是否细化)。
|
|
SettleAggregateRows *prometheus.CounterVec
|
|
// SettleAggregateErrors 消耗日汇总执行失败次数(失败 Warn 不中断,
|
|
// 幂等下轮自愈;持续增长=真故障信号)。
|
|
SettleAggregateErrors prometheus.Counter
|
|
)
|
|
|
|
var registry = prometheus.NewRegistry()
|
|
|
|
func init() {
|
|
HTTPRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "http_requests_total",
|
|
Help: "HTTP 请求总数",
|
|
}, []string{LabelMethod, LabelRoute, LabelStatus})
|
|
|
|
HTTPRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "http_request_duration_seconds",
|
|
Help: "HTTP 请求耗时分布(秒)",
|
|
Buckets: durationBuckets,
|
|
}, []string{LabelMethod, LabelRoute})
|
|
|
|
HTTPRequestsInFlight = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "http_requests_in_flight",
|
|
Help: "当前在途请求数",
|
|
})
|
|
|
|
SMSSubmitTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "sms_submit_total",
|
|
Help: "短信提交总数(阶段 4 发送链路)",
|
|
}, []string{"result"})
|
|
|
|
SMSReportTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "sms_report_total",
|
|
Help: "状态回执处理总数(阶段 5)",
|
|
}, []string{"status"})
|
|
|
|
KafkaProducerMessages = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "kafka_producer_messages_total",
|
|
Help: "Kafka 生产消息结果(ok/retry/dlq)",
|
|
}, []string{LabelTopic, LabelKind})
|
|
|
|
BalanceReconcileDiff = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "go_sms_balance_reconcile_diff",
|
|
Help: "R1 对账差额:Redis 预扣余额-流水重算值(覆写前采样;0=一致)",
|
|
}, []string{"account"})
|
|
|
|
BalanceMode = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "go_sms_balance_mode",
|
|
Help: "预扣模式(1=redis Lua/0=mysql 行锁;GOSMS_BALANCE_MODE 切换)",
|
|
})
|
|
|
|
BalanceFallbackTotal = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "go_sms_balance_fallback_total",
|
|
Help: "Redis 预扣基础设施错单次回退 MySQL 行锁次数",
|
|
})
|
|
|
|
BalanceFlowlogPending = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "go_sms_balance_flowlog_pending",
|
|
Help: "Redis 模式异步流水队列待写条数",
|
|
})
|
|
|
|
KafkaLag = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "go_sms_kafka_lag",
|
|
Help: "Kafka 消费组 lag 合计(高水位-已提交;背压 L1 告警源,Day53 挂账 Day55 清偿)",
|
|
}, []string{LabelTopic})
|
|
|
|
LongSmsMergedTotal = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "go_sms_longsms_merged_total",
|
|
Help: "入向长短信聚合成功数(UDH 组到齐合并单次受理,Day69)",
|
|
})
|
|
|
|
LongSmsDegradeTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "go_sms_longsms_degrade_total",
|
|
Help: "长短信退化直发片数(reason: overflow=并发组超上限/ref_conflict=协议异常;Day71 起 timeout 废除)",
|
|
}, []string{LabelKind})
|
|
|
|
LongSmsExpireFailTotal = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "go_sms_longsms_expire_fail_total",
|
|
Help: "长短信挂起组 72h 到期失败终态数(不齐不发,LP task+UNDELIV 回执,Day71)",
|
|
})
|
|
|
|
LongSmsDupFragTotal = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "go_sms_longsms_dup_frag_total",
|
|
Help: "长短信同 idx 重复片幂等覆盖数(客户端重发观测)",
|
|
})
|
|
|
|
ReportRequeueTotal = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "go_sms_report_requeue_total",
|
|
Help: "unpaired 回执短重投次数(seg 落库竞态窗口,Day69 三轮)",
|
|
})
|
|
|
|
ReportPendingTotal = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "go_sms_report_pending_total",
|
|
Help: "多段回执聚合未到齐挂起事件数(Day69 二轮)",
|
|
})
|
|
|
|
ReportRecheckTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "go_sms_report_recheck_total",
|
|
Help: "pending 后新事务复查结果(result: finalized=补刀终态/still_pending=真未到齐)",
|
|
}, []string{LabelKind})
|
|
|
|
SettleAggregateRows = prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "go_sms_settle_aggregate_rows_total",
|
|
Help: "结算域消耗日汇总落账行数(scope: account/channel;UPSERT 重刷亦计入,S2)",
|
|
}, []string{LabelKind})
|
|
|
|
SettleAggregateErrors = prometheus.NewCounter(prometheus.CounterOpts{
|
|
Name: "go_sms_settle_aggregate_errors_total",
|
|
Help: "消耗日汇总执行失败次数(幂等下轮自愈,持续增长=真故障,S2)",
|
|
})
|
|
|
|
registry.MustRegister(
|
|
HTTPRequestsTotal, HTTPRequestDuration, HTTPRequestsInFlight,
|
|
SMSSubmitTotal, SMSReportTotal, KafkaProducerMessages,
|
|
BalanceReconcileDiff, BalanceMode, BalanceFallbackTotal, BalanceFlowlogPending,
|
|
KafkaLag,
|
|
LongSmsMergedTotal, LongSmsDegradeTotal, LongSmsDupFragTotal,
|
|
LongSmsExpireFailTotal,
|
|
ReportPendingTotal, ReportRecheckTotal, ReportRequeueTotal,
|
|
SettleAggregateRows, SettleAggregateErrors,
|
|
)
|
|
// Go 运行时与进程指标
|
|
registry.MustRegister(
|
|
collectors.NewGoCollector(),
|
|
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
|
)
|
|
}
|
|
|
|
// Registry 返回指标注册中心(测试或自托管采集用)。
|
|
func Registry() *prometheus.Registry { return registry }
|
|
|
|
// Handler /metrics 输出 handler(app 装配:gin.WrapH(metrics.Handler()))。
|
|
func Handler() http.Handler {
|
|
return promhttp.HandlerFor(registry, promhttp.HandlerOpts{EnableOpenMetrics: true})
|
|
}
|