176 lines
5.3 KiB
Go
176 lines
5.3 KiB
Go
// worker.go S4·C1:回调推送 Worker——周期扫描 callback_task(status=0 且到期)
|
|
// →HTTP POST→2xx 置 sent/失败指数退避/耗尽置 dead。单实例部署(与 consumer
|
|
// 同进程);多实例时批次 SELECT 无锁会重复推送(at-least-once,UPDATE CAS
|
|
// 保证状态机不乱,重复送达客户侧幂等兜底——与 CMPP Deliver 同语义)。
|
|
package callback
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
|
|
"go-sms/internal/model/entity"
|
|
"go-sms/internal/pkg/logger"
|
|
"go-sms/internal/pkg/metrics"
|
|
|
|
"go.uber.org/zap"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// 推送参数默认值(HDD→SSD 后保守起步;S5 基线期按压测重标)。
|
|
const (
|
|
DefaultScanEvery = 2 * time.Second // 扫描周期
|
|
DefaultBatch = 100 // 单轮上限
|
|
DefaultConcurrency = 8 // 并发推送信号量
|
|
DefaultTimeout = 5 * time.Second // 单次 HTTP 超时
|
|
DefaultMaxAttempt = 10 // 重试上限(超过=死信)
|
|
DefaultMaxBackoff = 10 * time.Minute // 退避封顶
|
|
)
|
|
|
|
// PushWorker 回调推送器。
|
|
type PushWorker struct {
|
|
db *gorm.DB
|
|
client *http.Client
|
|
batch int
|
|
concurrency int
|
|
maxAttempt int
|
|
maxBackoff time.Duration
|
|
}
|
|
|
|
// NewPushWorker 构造(零值参数取默认)。
|
|
func NewPushWorker(db *gorm.DB) *PushWorker {
|
|
return &PushWorker{
|
|
db: db,
|
|
client: &http.Client{
|
|
Timeout: DefaultTimeout,
|
|
},
|
|
batch: DefaultBatch,
|
|
concurrency: DefaultConcurrency,
|
|
maxAttempt: DefaultMaxAttempt,
|
|
maxBackoff: DefaultMaxBackoff,
|
|
}
|
|
}
|
|
|
|
// Run 常驻循环(caller 起 goroutine;ctx 取消即退出,每轮独立 tick)。
|
|
func (w *PushWorker) Run(ctx context.Context, every time.Duration) {
|
|
if every <= 0 {
|
|
every = DefaultScanEvery
|
|
}
|
|
t := time.NewTicker(every)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
w.Tick(ctx)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tick 单轮扫描推送(导出供测试/手动触发);返回本轮处理条数。
|
|
func (w *PushWorker) Tick(ctx context.Context) int {
|
|
var rows []entity.CallbackTask
|
|
if err := w.db.WithContext(ctx).
|
|
Where("status = ? AND next_retry_at <= ?", entity.CallbackPending, time.Now()).
|
|
Order("id ASC").Limit(w.batch).Find(&rows).Error; err != nil {
|
|
logger.L().Warn("callback_scan_failed", zap.Error(err))
|
|
return 0
|
|
}
|
|
sem := make(chan struct{}, w.concurrency)
|
|
done := make(chan struct{}, len(rows))
|
|
for i := range rows {
|
|
sem <- struct{}{}
|
|
go func(task entity.CallbackTask) {
|
|
defer func() { <-sem; done <- struct{}{} }()
|
|
w.pushOne(ctx, task)
|
|
}(rows[i])
|
|
}
|
|
for range rows {
|
|
<-done
|
|
}
|
|
return len(rows)
|
|
}
|
|
|
|
// pushOne 单条推送+状态机推进(CAS WHERE status=0:与 Resend/并发实例至多一次生效)。
|
|
func (w *PushWorker) pushOne(ctx context.Context, task entity.CallbackTask) {
|
|
typ := "report"
|
|
if task.Type == entity.CallbackTypeUplink {
|
|
typ = "uplink"
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, task.URL, bytes.NewReader([]byte(task.Payload)))
|
|
if err != nil {
|
|
w.markRetry(ctx, &task, 0, err.Error(), typ)
|
|
return
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
resp, err := w.client.Do(req)
|
|
if err != nil {
|
|
w.markRetry(ctx, &task, 0, truncate(err.Error(), 255), typ)
|
|
return
|
|
}
|
|
defer func() { _ = resp.Body.Close() }()
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10))
|
|
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
|
res := w.db.WithContext(ctx).Model(&entity.CallbackTask{}).
|
|
Where("id = ? AND status = ?", task.ID, entity.CallbackPending).
|
|
Updates(map[string]any{
|
|
"status": entity.CallbackSent, "last_code": resp.StatusCode,
|
|
"last_err": "", "sent_at": time.Now(),
|
|
})
|
|
if res.Error != nil {
|
|
logger.L().Warn("callback_mark_sent_failed", zap.Int64("id", task.ID), zap.Error(res.Error))
|
|
}
|
|
if res.RowsAffected > 0 {
|
|
metrics.CallbackPushTotal.WithLabelValues(typ, "sent").Inc()
|
|
}
|
|
return
|
|
}
|
|
w.markRetry(ctx, &task, resp.StatusCode, fmt.Sprintf("http %d", resp.StatusCode), typ)
|
|
}
|
|
|
|
// markRetry 失败推进:attempt+1 → 耗尽置 dead,否则退避重排。
|
|
func (w *PushWorker) markRetry(ctx context.Context, task *entity.CallbackTask, httpCode int, msg, typ string) {
|
|
attempt := task.Attempt + 1
|
|
updates := map[string]any{
|
|
"attempt": attempt, "last_code": httpCode, "last_err": truncate(msg, 255),
|
|
}
|
|
if attempt >= w.maxAttempt {
|
|
updates["status"] = entity.CallbackDead
|
|
updates["next_retry_at"] = time.Now() // MySQL 不收 zero time;死信不扫靠 status
|
|
} else {
|
|
backoff := time.Duration(1<<uint(attempt)) * time.Second // 2^attempt s
|
|
if backoff > w.maxBackoff {
|
|
backoff = w.maxBackoff
|
|
}
|
|
updates["next_retry_at"] = time.Now().Add(backoff)
|
|
}
|
|
res := w.db.WithContext(ctx).Model(&entity.CallbackTask{}).
|
|
Where("id = ? AND status = ?", task.ID, entity.CallbackPending).
|
|
Updates(updates)
|
|
if res.Error != nil {
|
|
logger.L().Warn("callback_mark_retry_failed", zap.Int64("id", task.ID), zap.Error(res.Error))
|
|
return
|
|
}
|
|
if res.RowsAffected > 0 {
|
|
if attempt >= w.maxAttempt {
|
|
metrics.CallbackPushTotal.WithLabelValues(typ, "dead").Inc()
|
|
logger.L().Warn("callback_task_dead",
|
|
zap.Int64("id", task.ID), zap.Int64("account_id", task.AccountID),
|
|
zap.Int("attempt", attempt), zap.String("url", task.URL))
|
|
} else {
|
|
metrics.CallbackPushTotal.WithLabelValues(typ, "retry").Inc()
|
|
}
|
|
}
|
|
}
|
|
|
|
func truncate(s string, n int) string {
|
|
if r := []rune(s); len(r) > n {
|
|
return string(r[:n])
|
|
}
|
|
return s
|
|
}
|