Files
go-sms-V2/internal/service/callback/worker.go
T
Tory bb60d60529
ci / test (push) Canceled after 0s
ci / web-test (push) Canceled after 0s
ci / build (push) Canceled after 0s
ci / deploy-dev (push) Canceled after 0s
S4-D1/D2: C1 回调推送引擎+C2 上行自动拉黑——验收项「死信可查/拒收即拉黑」✅: ①C1=callback_task 队列表(迁移 v21,状态机 pending→sent/dead+attempt/next_retry_at/last_code)+PushWorker(2s 扫/批 100/并发 8/2^n 秒指数退避封顶 10min/10 次耗尽死信/UPDATE CAS 防竞态)+双挂点(report 终态首锁[Consumer+PairService,与 CMPP Deliver 同点同语义]+uplink 消费尾)+URL 门控 account_config.report_url/uplink_url(60s TTL 缓存,未配置零任务)+GET /ops/callback(列表+状态计数面板)/POST /ops/callback/:id/resend(casbin ops callback:read/write v21 所种)+前端回调管理页(ops-callback 路由系统管理组+api/callback.ts+状态面板/筛选/载荷展开/死信重派;matrix+guard 33 菜单显式更)+metrics callback_push_total{type,result}/blacklist_total+openapi v1.4.0;②C2=AutoBlacklister 等值词表(trim+lower 防误伤,默认 R/TD/退订/拒收;SystemConfig uplink.blacklist.keywords 可配 60s 刷新)→policy_blacklist(source=2/账号级/uk 幂等)+Redis Set 即时生效,拉黑不影响上行推送(需求原文);③真机全链六步实证(dev 中间件):回执 DELIVRD→detail 终态→HTTP 回调 JSON 送达(200/sent)+上行 R→拉黑(source=2+Redis=1)+推送送达+坏 URL→attempt10→dead→resend ISC:000→修复重投 sent+status_counts 面板;一次性探针 cmd/cbprobe 用后即删+冒烟数据全清;④单测 +13(callback 6[入队门控/载荷/状态机全链/并发批]+uplink 5[词表/配置覆盖/幂等缓存/消费链]+report 2[首锁一次/nil]);⑤门禁=build/vet/-race 40 包/lint 0/eslint0/vue-tsc0/vitest 148/build 全绿;迁移测试 v15..v21 两遍幂等+dev 实跑 v21;gofmt 存量 8 文件顺手清零(S2 遗留);⑥口径诚实标注:载荷无签名挂账/拦截终态 HTTP 未挂(CMPP 侧已有)/PushWorker 单实例 at-least-once;验收单 s4-d1d2-callback.md+CHANGELOG+memory-bank 回填
2026-11-16 11:19:55 +08:00

176 lines
5.3 KiB
Go

// worker.go S4·C1:回调推送 Worker——周期扫描 callback_task(status=0 且到期)
// →HTTP POST→2xx 置 sent/失败指数退避/耗尽置 dead。单实例部署(与 consumer
// 同进程);多实例时批次 SELECT 无锁会重复推送(at-least-once,UPDATE CAS
// 保证状态机不乱,重复送达客户侧幂等兜底——与 CMPP Deliver 同语义)。
package callback
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"time"
"go-sms/internal/model/entity"
"go-sms/internal/pkg/logger"
"go-sms/internal/pkg/metrics"
"go.uber.org/zap"
"gorm.io/gorm"
)
// 推送参数默认值(HDD→SSD 后保守起步;S5 基线期按压测重标)。
const (
DefaultScanEvery = 2 * time.Second // 扫描周期
DefaultBatch = 100 // 单轮上限
DefaultConcurrency = 8 // 并发推送信号量
DefaultTimeout = 5 * time.Second // 单次 HTTP 超时
DefaultMaxAttempt = 10 // 重试上限(超过=死信)
DefaultMaxBackoff = 10 * time.Minute // 退避封顶
)
// PushWorker 回调推送器。
type PushWorker struct {
db *gorm.DB
client *http.Client
batch int
concurrency int
maxAttempt int
maxBackoff time.Duration
}
// NewPushWorker 构造(零值参数取默认)。
func NewPushWorker(db *gorm.DB) *PushWorker {
return &PushWorker{
db: db,
client: &http.Client{
Timeout: DefaultTimeout,
},
batch: DefaultBatch,
concurrency: DefaultConcurrency,
maxAttempt: DefaultMaxAttempt,
maxBackoff: DefaultMaxBackoff,
}
}
// Run 常驻循环(caller 起 goroutine;ctx 取消即退出,每轮独立 tick)。
func (w *PushWorker) Run(ctx context.Context, every time.Duration) {
if every <= 0 {
every = DefaultScanEvery
}
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
w.Tick(ctx)
}
}
}
// Tick 单轮扫描推送(导出供测试/手动触发);返回本轮处理条数。
func (w *PushWorker) Tick(ctx context.Context) int {
var rows []entity.CallbackTask
if err := w.db.WithContext(ctx).
Where("status = ? AND next_retry_at <= ?", entity.CallbackPending, time.Now()).
Order("id ASC").Limit(w.batch).Find(&rows).Error; err != nil {
logger.L().Warn("callback_scan_failed", zap.Error(err))
return 0
}
sem := make(chan struct{}, w.concurrency)
done := make(chan struct{}, len(rows))
for i := range rows {
sem <- struct{}{}
go func(task entity.CallbackTask) {
defer func() { <-sem; done <- struct{}{} }()
w.pushOne(ctx, task)
}(rows[i])
}
for range rows {
<-done
}
return len(rows)
}
// pushOne 单条推送+状态机推进(CAS WHERE status=0:与 Resend/并发实例至多一次生效)。
func (w *PushWorker) pushOne(ctx context.Context, task entity.CallbackTask) {
typ := "report"
if task.Type == entity.CallbackTypeUplink {
typ = "uplink"
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, task.URL, bytes.NewReader([]byte(task.Payload)))
if err != nil {
w.markRetry(ctx, &task, 0, err.Error(), typ)
return
}
req.Header.Set("Content-Type", "application/json")
resp, err := w.client.Do(req)
if err != nil {
w.markRetry(ctx, &task, 0, truncate(err.Error(), 255), typ)
return
}
defer func() { _ = resp.Body.Close() }()
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10))
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
res := w.db.WithContext(ctx).Model(&entity.CallbackTask{}).
Where("id = ? AND status = ?", task.ID, entity.CallbackPending).
Updates(map[string]any{
"status": entity.CallbackSent, "last_code": resp.StatusCode,
"last_err": "", "sent_at": time.Now(),
})
if res.Error != nil {
logger.L().Warn("callback_mark_sent_failed", zap.Int64("id", task.ID), zap.Error(res.Error))
}
if res.RowsAffected > 0 {
metrics.CallbackPushTotal.WithLabelValues(typ, "sent").Inc()
}
return
}
w.markRetry(ctx, &task, resp.StatusCode, fmt.Sprintf("http %d", resp.StatusCode), typ)
}
// markRetry 失败推进:attempt+1 → 耗尽置 dead,否则退避重排。
func (w *PushWorker) markRetry(ctx context.Context, task *entity.CallbackTask, httpCode int, msg, typ string) {
attempt := task.Attempt + 1
updates := map[string]any{
"attempt": attempt, "last_code": httpCode, "last_err": truncate(msg, 255),
}
if attempt >= w.maxAttempt {
updates["status"] = entity.CallbackDead
updates["next_retry_at"] = time.Now() // MySQL 不收 zero time;死信不扫靠 status
} else {
backoff := time.Duration(1<<uint(attempt)) * time.Second // 2^attempt s
if backoff > w.maxBackoff {
backoff = w.maxBackoff
}
updates["next_retry_at"] = time.Now().Add(backoff)
}
res := w.db.WithContext(ctx).Model(&entity.CallbackTask{}).
Where("id = ? AND status = ?", task.ID, entity.CallbackPending).
Updates(updates)
if res.Error != nil {
logger.L().Warn("callback_mark_retry_failed", zap.Int64("id", task.ID), zap.Error(res.Error))
return
}
if res.RowsAffected > 0 {
if attempt >= w.maxAttempt {
metrics.CallbackPushTotal.WithLabelValues(typ, "dead").Inc()
logger.L().Warn("callback_task_dead",
zap.Int64("id", task.ID), zap.Int64("account_id", task.AccountID),
zap.Int("attempt", attempt), zap.String("url", task.URL))
} else {
metrics.CallbackPushTotal.WithLabelValues(typ, "retry").Inc()
}
}
}
func truncate(s string, n int) string {
if r := []rune(s); len(r) > n {
return string(r[:n])
}
return s
}