360 lines
14 KiB
Go
360 lines
14 KiB
Go
// cmd/consumer Day 48:submit 消费者——sms-submit 事件驱动立即任务下发。
|
|
// 组件:消费组(重试+死信)|ReclaimScanner 补偿认领(事件丢失/审核通过兜底)|
|
|
// ChannelDispatcher(router 选路→聚合/直发)|ChannelPacer(令牌桶+在途窗口+降速)|
|
|
// DetailWriter(终态批量落库)|Backpressure(三级水位,L3 暂停消费)。
|
|
// 与 gateway 同库同 Redis 同 Kafka;HTTP 路由不注册(纯 worker 进程)。
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"go-sms/internal/app"
|
|
"go-sms/internal/config"
|
|
"go-sms/internal/model/entity"
|
|
"go-sms/internal/pkg/metrics"
|
|
"go-sms/internal/pkg/queue"
|
|
"go-sms/internal/service/audit"
|
|
"go-sms/internal/service/balance"
|
|
"go-sms/internal/service/callback"
|
|
"go-sms/internal/service/channel"
|
|
"go-sms/internal/service/interceptor"
|
|
"go-sms/internal/service/report"
|
|
"go-sms/internal/service/router"
|
|
"go-sms/internal/service/send"
|
|
"go-sms/internal/service/uplink"
|
|
"go-sms/internal/util"
|
|
|
|
"github.com/IBM/sarama"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
func main() {
|
|
a, err := app.New(app.Options{ServiceName: "go-sms-consumer"})
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
db, rc := a.DB(), a.Redis()
|
|
cfg := a.Config()
|
|
if db == nil {
|
|
a.Lg().Error("consumer_db_required")
|
|
os.Exit(1)
|
|
}
|
|
if len(cfg.Kafka.Brokers) == 0 {
|
|
a.Lg().Error("consumer_kafka_required")
|
|
os.Exit(1)
|
|
}
|
|
submitTopic := cfg.Kafka.Topics["submit"]
|
|
if submitTopic == "" {
|
|
submitTopic = queue.TopicSubmit
|
|
}
|
|
ccfg := cfg.Kafka.Consumer
|
|
groupID := ccfg.GroupID
|
|
if groupID == "" {
|
|
groupID = "go-sms-consumer"
|
|
}
|
|
// Day58:起步 offset 可配(默认 oldest=Day48 存量消费语义;dev 止血用
|
|
// env GOSMS_KAFKA_CONSUMER_OFFSET_MODE=latest 跳过积压——仅对无已提交
|
|
// offset 的消费组生效)+消费 pacing(防 oldest 冷启动存量回放打爆 dev MySQL)。
|
|
offsetMode := ccfg.OffsetMode
|
|
if offsetMode == "" {
|
|
offsetMode = queue.OffsetModeOldest
|
|
}
|
|
paceEvery, paceSleep := ccfg.PaceEvery, ccfg.PaceSleep
|
|
|
|
// 通道链路(常驻 CMPP;下发出口)。
|
|
chSvc := channel.NewService(db)
|
|
// Day 49:上行域——本进程长连接同样接收 Deliver MO,注入上行出口
|
|
// (码号分配→message_uplink→投 sms-up;消费组见下方 uplinkConsumer)。
|
|
upAsg := uplink.NewAssigner(db)
|
|
var upProd uplink.Producer
|
|
if p := a.Producer(); p != nil {
|
|
upProd = p
|
|
}
|
|
chSvc.SetUplinkSink(uplink.NewService(db, upAsg, upProd))
|
|
upCtx, stopUpAsg := context.WithCancel(context.Background())
|
|
go upAsg.Start(upCtx, uplink.DefaultReloadEvery)
|
|
a.OnShutdown(stopUpAsg)
|
|
// Day 56:回执域——本进程长连接接收状态报告,注入回执出口
|
|
// (投 sms-report;消费组见下方 reportConsumer:归一化+段级配对)。
|
|
if p := a.Producer(); p != nil {
|
|
chSvc.SetReportSink(report.NewSink(p))
|
|
}
|
|
chSvc.BootstrapLinks(context.Background())
|
|
a.OnShutdown(chSvc.ShutdownLinks)
|
|
a.OnShutdown(chSvc.Close)
|
|
|
|
// pacer 限速(通道 speed 装载)+聚合器+分发器+批量 writer。
|
|
pacer := send.NewChannelPacer(ccfg.Enable)
|
|
var sumSpeed int64 = 0
|
|
var chans []entity.ChannelChannel
|
|
if err := db.Where("status = ?", 1).Find(&chans).Error; err == nil {
|
|
for _, ch := range chans {
|
|
if ch.Speed <= 0 {
|
|
sumSpeed += send.DefaultChannelSpeed
|
|
} else {
|
|
sumSpeed += int64(ch.Speed)
|
|
}
|
|
pacer.SetSpeed(ch.ID, float64(ch.Speed))
|
|
}
|
|
}
|
|
writer := send.NewDetailWriter(db, send.DefaultWriterBatch, send.DefaultWriterWorker)
|
|
a.OnShutdown(writer.Close)
|
|
aggregator := send.NewSubmitAggregator(chSvc, send.DefaultAggrWindow, send.DefaultAggrMaxBatch).WithDB(db) // Day56 seg 配对登记(装配遗漏修复)
|
|
aggCtx, stopAgg := context.WithCancel(context.Background())
|
|
go aggregator.Run(aggCtx)
|
|
a.OnShutdown(func() { stopAgg(); aggregator.FlushAll() })
|
|
dp := send.NewChannelDispatcher(db, router.NewRouter(db), chSvc, aggregator).
|
|
WithPacer(pacer).WithDetailWriter(writer)
|
|
|
|
// 事件消费者 + 补偿扫描。
|
|
sc := send.NewSubmitConsumer(db, dp, a.Lg())
|
|
// P0-1(Day 63)Deferred 补全依赖——拦截链+审核分流(与 gateway 同构);
|
|
// 资金 Store 待 balStore 装配后注入(下方 WithConsumerBalance)。
|
|
var ichain *interceptor.Chain
|
|
if rc != nil {
|
|
ichain = interceptor.NewDefaultChain(db, rc,
|
|
interceptor.NewDBRecorder(db),
|
|
interceptor.Options{FreqStore: interceptor.NewRedisFreqStore(rc)})
|
|
} else {
|
|
ichain = interceptor.NewDefaultChain(db, nil,
|
|
interceptor.NewDBRecorder(db), interceptor.Options{})
|
|
}
|
|
sc = sc.WithConsumerChain(ichain).WithConsumerRouter(audit.NewReviewService(db))
|
|
// Day62c:拦截终态 CMPP 下发桥——受理期拦截明细无上游回执,不桥则客户收不到
|
|
// 状态(真机报障);复用 Day61 同一 ProducerPushSink 投 cmpp-deliver。
|
|
if p := a.Producer(); p != nil {
|
|
sc = sc.WithBlockedDeliverSink(blockedDeliverAdapter{sink: &report.ProducerPushSink{P: p}})
|
|
}
|
|
reclaimEvery := ccfg.ReclaimEvery
|
|
if reclaimEvery <= 0 {
|
|
reclaimEvery = 5 * time.Second
|
|
}
|
|
rs := send.NewReclaimScanner(db, dp, a.Lg())
|
|
if ccfg.ReclaimBatch > 0 {
|
|
rs = rs.WithBatch(ccfg.ReclaimBatch)
|
|
}
|
|
rsCtx, stopRS := context.WithCancel(context.Background())
|
|
go rs.Run(rsCtx, reclaimEvery)
|
|
a.OnShutdown(stopRS)
|
|
|
|
// 消费组(失败重试+死信转投)。
|
|
cg, err := queue.NewConsumerGroup(queue.ConsumerGroupConfig{
|
|
Consumer: queue.ConsumerConfig{
|
|
Brokers: cfg.Kafka.Brokers,
|
|
GroupID: groupID,
|
|
Topics: []string{submitTopic},
|
|
Logger: a.Lg(),
|
|
OffsetMode: offsetMode, // Day58 起步 offset 可配(默认 oldest=Day48 历史积压认领幂等,不双发)
|
|
PaceEvery: paceEvery,
|
|
PaceSleep: paceSleep,
|
|
// Day65:每分区 worker 池——Deferred 补全为 DB 慢事务链(单条 0.3~0.7s),
|
|
// 串行 16 分区并发封顶 ~60/s;workers=4 → 64 并发。offset 按前缀有序 Mark,
|
|
// at-least-once 不变。回执/上行组保持串行(乱序风险零容忍优先)。
|
|
WorkersPerPartition: ccfg.Workers,
|
|
Handler: func(ctx context.Context, msg *sarama.ConsumerMessage) error {
|
|
return sc.HandlePayload(ctx, msg.Value)
|
|
},
|
|
},
|
|
Producer: queue.ProducerConfig{Brokers: cfg.Kafka.Brokers, Logger: a.Lg()},
|
|
})
|
|
if err != nil {
|
|
a.Lg().Error("consumer_group_init_failed", zap.Error(err))
|
|
os.Exit(1)
|
|
}
|
|
a.OnShutdown(func() { _ = cg.Close() })
|
|
|
|
// 背压三级水位(L1 告警/L2 入口收紧键/L3 暂停消费;采样 lag=submit topic)。
|
|
bpEvery := ccfg.BPSampleEvery
|
|
if bpEvery <= 0 {
|
|
bpEvery = 10 * time.Second
|
|
}
|
|
lagR, lagErr := queue.NewLagReader(cfg.Kafka.Brokers, groupID, a.Lg())
|
|
bp := send.NewBackpressure(send.NewRedisBPStore(rc), a.Lg(),
|
|
cg.Pause, cg.Resume, func() int64 { return sumSpeed })
|
|
if lagErr != nil {
|
|
a.Lg().Warn("lag_reader_disabled_backpressure_degraded", zap.Error(lagErr))
|
|
} else {
|
|
a.OnShutdown(func() { _ = lagR.Close() })
|
|
bpCtx, stopBP := context.WithCancel(context.Background())
|
|
go bp.Run(bpCtx, bpEvery, func(ctx context.Context) (int64, error) {
|
|
return lagR.Total(ctx, []string{submitTopic})
|
|
})
|
|
a.OnShutdown(stopBP)
|
|
}
|
|
|
|
a.Lg().Info("submit_consumer_ready",
|
|
zap.String("topic", submitTopic), zap.String("group", groupID),
|
|
zap.Int("channels", len(chans)), zap.Int64("sum_speed", sumSpeed),
|
|
zap.Bool("pacer", pacer.Open()),
|
|
zap.String("reclaim", reclaimEvery.String()), zap.Bool("lag_reader", lagErr == nil))
|
|
|
|
// S4·C1/C2:回调推送引擎(HTTP 回执终态+上行推送,指数退避+死信可查)
|
|
// + 上行自动拉黑(词表 SystemConfig 可配,拒收即拉黑)。
|
|
cbSvc := callback.NewService(db)
|
|
upBl := uplink.NewAutoBlacklister(db, rc)
|
|
blCtx, stopBl := context.WithCancel(context.Background())
|
|
go upBl.Start(blCtx)
|
|
a.OnShutdown(stopBl)
|
|
cbW := callback.NewPushWorker(db)
|
|
cbWCtx, stopCbW := context.WithCancel(context.Background())
|
|
go cbW.Run(cbWCtx, callback.DefaultScanEvery)
|
|
a.OnShutdown(stopCbW)
|
|
a.Lg().Info("callback_engine_ready",
|
|
zap.Bool("blacklist", true), zap.Duration("scan_every", callback.DefaultScanEvery))
|
|
|
|
// Day 49:上行消费组(sms-up;15 天窗口匹配下行明细回填 match_task_id;
|
|
// S4:上行推送+自动拉黑挂消费尾部,不阻断匹配主链)。独立 group 前缀防 offset 互扰。
|
|
upTopic := cfg.Kafka.Topics["up"]
|
|
if upTopic == "" {
|
|
upTopic = queue.TopicUp
|
|
}
|
|
uc := uplink.NewConsumer(db).WithCallback(cbSvc).WithBlacklist(upBl)
|
|
ucg, err := queue.NewConsumerGroup(queue.ConsumerGroupConfig{
|
|
Consumer: queue.ConsumerConfig{
|
|
Brokers: cfg.Kafka.Brokers,
|
|
GroupID: groupID + "-uplink",
|
|
Topics: []string{upTopic},
|
|
Logger: a.Lg(),
|
|
OffsetMode: offsetMode, // Day58 起步 offset 可配(默认 oldest=15 天窗口回放语义)
|
|
PaceEvery: paceEvery,
|
|
PaceSleep: paceSleep,
|
|
Handler: func(ctx context.Context, msg *sarama.ConsumerMessage) error {
|
|
return uc.HandlePayload(ctx, msg.Value)
|
|
},
|
|
},
|
|
Producer: queue.ProducerConfig{Brokers: cfg.Kafka.Brokers, Logger: a.Lg()},
|
|
})
|
|
if err != nil {
|
|
a.Lg().Error("uplink_consumer_group_init_failed", zap.Error(err))
|
|
os.Exit(1)
|
|
}
|
|
a.OnShutdown(func() { _ = ucg.Close() })
|
|
a.Lg().Info("uplink_consumer_ready",
|
|
zap.String("topic", upTopic), zap.String("group", groupID+"-uplink"))
|
|
|
|
// Day 56:回执消费组(sms-report;归一化三态+段级配对→detail 终态+
|
|
// message_report 落库)。独立 group 前缀防 offset 互扰。
|
|
repTopic := cfg.Kafka.Topics["report"]
|
|
if repTopic == "" {
|
|
repTopic = queue.TopicReport
|
|
}
|
|
// Day 57:计费确认——balance Store(与 gateway 同构;退款走同一 Redis/
|
|
// MySQL 双级)。Reconciler 仅 gateway 独占(幂等覆写,双进程双跑无害但
|
|
// 浪费);ModeWatcher 两进程各自热切保持模式一致。雪花 workerID=2
|
|
// (gateway=1,跨进程不冲突)。
|
|
balStore := balance.Store(balance.NewMySQLStore(db))
|
|
if balance.NormalizeMode(cfg.Balance.Mode) == balance.ModeRedis && rc != nil {
|
|
sf2, sfErr := util.NewSnowflake(2)
|
|
if sfErr != nil {
|
|
a.Lg().Warn("consumer_balance_snowflake_fallback_mysql", zap.Error(sfErr))
|
|
} else {
|
|
rs := balance.NewRedisStore(rc, db, sf2, a.Lg(), balance.RedisOptions{})
|
|
fb := balance.NewFallbackStore(rs, balance.NewMySQLStore(db), a.Lg())
|
|
balStore = fb
|
|
balCtx, stopBal := context.WithCancel(context.Background())
|
|
go balance.RunModeWatcher(balCtx, fb, func() string { return a.Config().Balance.Mode })
|
|
a.OnShutdown(func() { stopBal(); rs.Close() })
|
|
}
|
|
}
|
|
bill := report.NewBilling(db, balStore)
|
|
// P0-1(Day 63):Deferred 拦截/审核拒退费与计费确认共用同一 Store。
|
|
sc = sc.WithConsumerBalance(balStore)
|
|
// 48h 无回执补偿扫描→UNKNOWN(4) 结案+冻结释放(启动即一轮,10m/轮)。
|
|
comp := report.NewCompensator(db, bill, report.CompensatorOptions{})
|
|
compCtx, stopComp := context.WithCancel(context.Background())
|
|
go comp.Run(compCtx, 10*time.Minute)
|
|
a.OnShutdown(stopComp)
|
|
|
|
repC := report.NewConsumer(db).WithBilling(bill).WithCallback(cbSvc) // S4·C1:回执终态 HTTP 回调入队
|
|
// Day 61(P0.5):CMPP 回执协议侧下发桥——终态锁定后投 cmpp-deliver
|
|
// (gateway 进程消费→PushDeliver);producer 缺省时不注入(HTTP 来源无此诉求)。
|
|
if p := a.Producer(); p != nil {
|
|
repC = repC.WithPushSink(&report.ProducerPushSink{P: p})
|
|
// Day 69 三轮:unpaired 短重投出口——秒回模拟器回执跑赢发送侧 seg
|
|
// 落库的固有竞态,回投 sms-report 尾部经队列延迟后重配(≤3 次)。
|
|
repC = repC.WithRequeue(func(ev *report.Event) error {
|
|
payload, err := json.Marshal(ev)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return a.Producer().Send(repTopic, []byte(ev.MsgID), payload)
|
|
})
|
|
}
|
|
rcg, err := queue.NewConsumerGroup(queue.ConsumerGroupConfig{
|
|
Consumer: queue.ConsumerConfig{
|
|
Brokers: cfg.Kafka.Brokers,
|
|
GroupID: groupID + "-report",
|
|
Topics: []string{repTopic},
|
|
Logger: a.Lg(),
|
|
Oldest: true, // 历史积压回执补处理(seg 幂等,detail 终态锁定防翻转);恒 oldest 不读 offset_mode(回执不丢)
|
|
PaceEvery: paceEvery,
|
|
PaceSleep: paceSleep,
|
|
Handler: func(ctx context.Context, msg *sarama.ConsumerMessage) error {
|
|
return repC.HandlePayload(ctx, msg.Value)
|
|
},
|
|
},
|
|
Producer: queue.ProducerConfig{Brokers: cfg.Kafka.Brokers, Logger: a.Lg()},
|
|
})
|
|
if err != nil {
|
|
a.Lg().Error("report_consumer_group_init_failed", zap.Error(err))
|
|
os.Exit(1)
|
|
}
|
|
a.OnShutdown(func() { _ = rcg.Close() })
|
|
a.Lg().Info("report_consumer_ready",
|
|
zap.String("topic", repTopic), zap.String("group", groupID+"-report"))
|
|
|
|
// 优雅退出(SIGINT/SIGTERM→app.Shutdown 钩子链:writer 冲刷/链路拆除/消费组退出)。
|
|
go func() {
|
|
if err := cg.Run(context.Background()); err != nil {
|
|
a.Lg().Error("consumer_group_exit", zap.Error(err))
|
|
}
|
|
}()
|
|
go func() {
|
|
if err := ucg.Run(context.Background()); err != nil {
|
|
a.Lg().Error("uplink_consumer_group_exit", zap.Error(err))
|
|
}
|
|
}()
|
|
go func() {
|
|
if err := rcg.Run(context.Background()); err != nil {
|
|
a.Lg().Error("report_consumer_group_exit", zap.Error(err))
|
|
}
|
|
}()
|
|
// Day 69 百万级压测观测:consumer 侧独立 /metrics(:8082,Prometheus
|
|
// 抓取,deploy/monitoring/prometheus.yml 同步)。失败仅日志不阻断 worker。
|
|
go func() {
|
|
mux := http.NewServeMux()
|
|
mux.Handle("/metrics", metrics.Handler())
|
|
if err := http.ListenAndServe(":8082", mux); err != nil {
|
|
a.Lg().Error("metrics_http_failed", zap.Error(err))
|
|
}
|
|
}()
|
|
quit := make(chan os.Signal, 1)
|
|
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
|
<-quit
|
|
a.Lg().Info("submit_consumer_stopping")
|
|
a.Shutdown(10 * time.Second)
|
|
}
|
|
|
|
// 编译期确认 config 引用(避免未来重命名漂移)。
|
|
var _ = config.Config{}
|
|
|
|
// blockedDeliverAdapter Day62c:send 域拦截终态出口 → report.DeliverEvent 桥
|
|
// (stat 映射 ISC:004→SIGNREJ;与 Day61 回执事件同 topic 同载荷形状,
|
|
// gateway 侧 PushDeliver 链路零改动)。
|
|
type blockedDeliverAdapter struct{ sink *report.ProducerPushSink }
|
|
|
|
func (b blockedDeliverAdapter) PushBlocked(ctx context.Context,
|
|
accountID, detailID int64, mobile, errCode, submitAt string) {
|
|
ev := &report.DeliverEvent{
|
|
AccountID: accountID, DetailID: detailID, Mobile: mobile,
|
|
Stat: report.StatForBlocked(errCode), SubmitAt: submitAt, DoneAt: submitAt,
|
|
}
|
|
_ = b.sink.PushDeliver(ctx, ev) // 失败已 Warn:终态不回滚,下发尽力而为(Day61 同口径)
|
|
}
|