204 lines
6.9 KiB
Go
204 lines
6.9 KiB
Go
// job.go S4·D3:归档 job——热窗外明细逐日迁出 MySQL→ClickHouse,校验通过才
|
|
// DROP MySQL 分区(数据零丢失方向:先 CH 后 MySQL;CH 侧重写=DROP PARTITION
|
|
// +INSERT 当日全量,幂等可重入)。周期 1h(启动先 EnsureSchema+Partitions)。
|
|
package archive
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"go-sms/internal/model/entity"
|
|
"go-sms/internal/pkg/logger"
|
|
"go-sms/internal/pkg/metrics"
|
|
"go-sms/internal/repository/clickhouse"
|
|
|
|
"go.uber.org/zap"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// 默认参数(SSD 基线保守起步)。
|
|
const (
|
|
DefaultArchiveEvery = time.Hour // 归档周期
|
|
archiveBatch = 2000 // MySQL 分批读取行数
|
|
archiveMaxDays = 62 // 单轮最多处理天数(两月保护,防首轮长事务风暴)
|
|
)
|
|
|
|
// Service 归档服务(db=MySQL;ch 可 nil=禁用归档链路,仅分区维护可用)。
|
|
type Service struct {
|
|
db *gorm.DB
|
|
ch clickhouse.Conn
|
|
}
|
|
|
|
// NewService 构造。
|
|
func NewService(db *gorm.DB, ch clickhouse.Conn) *Service { return &Service{db: db, ch: ch} }
|
|
|
|
// Run 常驻循环(caller 起 goroutine;每轮 Ensure→归档;失败 Warn 下轮自愈)。
|
|
func (s *Service) Run(ctx context.Context, every time.Duration) {
|
|
if every <= 0 {
|
|
every = DefaultArchiveEvery
|
|
}
|
|
t := time.NewTicker(every)
|
|
defer t.Stop()
|
|
s.EnsureOnce(ctx) // 幂等前置:CH 建表+分区维护
|
|
s.ArchiveOnce(ctx) // 启动先归档一轮(重启即清退热窗外积压)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
s.ArchiveOnce(ctx)
|
|
}
|
|
}
|
|
}
|
|
|
|
// EnsureOnce 幂等前置(CH 建表+分区维护)。
|
|
func (s *Service) EnsureOnce(ctx context.Context) {
|
|
if err := s.EnsureCHSchema(ctx); err != nil {
|
|
logger.L().Warn("archive_ch_schema_failed", zap.Error(err))
|
|
}
|
|
if _, err := s.EnsurePartitions(ctx); err != nil {
|
|
logger.L().Warn("archive_partition_ensure_failed", zap.Error(err))
|
|
}
|
|
}
|
|
|
|
// ArchiveOnce 单轮归档:找热窗外 distinct 日期→逐日迁出。
|
|
func (s *Service) ArchiveOnce(ctx context.Context) int {
|
|
if s.ch == nil {
|
|
return 0 // CH 未装配:链路禁用(分区维护仍由 EnsureOnce 承担)
|
|
}
|
|
floor := HotFloor(time.Now())
|
|
// DISTINCT 时刻级 created_at(MySQL/glebarez 双方言直 Scan time.Time;
|
|
// DATE() 在 sqlite 返回 TEXT 难以统一转换——D3 真机+单测双教训),
|
|
// Go 侧归一化当日零点去重。
|
|
var times []time.Time
|
|
if err := s.db.WithContext(ctx).Raw(
|
|
"SELECT DISTINCT created_at FROM message_detail "+
|
|
"WHERE created_at < ? ORDER BY created_at LIMIT ?", floor, archiveMaxDays*4).
|
|
Scan(×).Error; err != nil {
|
|
logger.L().Warn("archive_scan_days_failed", zap.Error(err))
|
|
metrics.ArchiveErrors.Inc()
|
|
return 0
|
|
}
|
|
seen := map[string]bool{}
|
|
var dayTimes []time.Time
|
|
for _, t := range times {
|
|
if t.IsZero() {
|
|
continue
|
|
}
|
|
day := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())
|
|
key := day.Format("2006-01-02")
|
|
if !seen[key] {
|
|
seen[key] = true
|
|
dayTimes = append(dayTimes, day)
|
|
}
|
|
}
|
|
done := 0
|
|
for _, day := range dayTimes {
|
|
logger.L().Info("archive_day_begin", zap.String("day", day.Format("2006-01-02")))
|
|
if s.archiveDay(ctx, day) {
|
|
done++
|
|
}
|
|
}
|
|
if len(dayTimes) > 0 {
|
|
logger.L().Info("archive_round_done",
|
|
zap.Int("pending_days", len(dayTimes)), zap.Int("archived_days", done))
|
|
}
|
|
return done
|
|
}
|
|
|
|
// archiveDay 单日归档(MySQL count → CH 重写 → CH count 校验 → DROP 分区/兜底 DELETE)。
|
|
func (s *Service) archiveDay(ctx context.Context, day time.Time) bool {
|
|
lo := day
|
|
hi := day.AddDate(0, 0, 1)
|
|
var mysqlCount int64
|
|
if err := s.db.WithContext(ctx).Table("message_detail").
|
|
Where("created_at >= ? AND created_at < ?", lo, hi).Count(&mysqlCount).Error; err != nil {
|
|
logger.L().Warn("archive_mysql_count_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err))
|
|
metrics.ArchiveErrors.Inc()
|
|
return false
|
|
}
|
|
if mysqlCount == 0 {
|
|
// 无数据但分区存在(历史残留):直接清分区。
|
|
if dropped, err := s.DropPartition(ctx, day); err == nil && dropped {
|
|
logger.L().Info("archive_empty_partition_dropped", zap.String("day", day.Format("2006-01-02")))
|
|
}
|
|
return false
|
|
}
|
|
logger.L().Info("archive_day_counted", zap.String("day", day.Format("2006-01-02")), zap.Int64("rows", mysqlCount))
|
|
// CH 侧重写(幂等):先 DROP 当日分区,再分批 INSERT 当日全量。
|
|
if err := s.dropCHPartition(ctx, day); err != nil {
|
|
logger.L().Warn("archive_ch_drop_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err))
|
|
metrics.ArchiveErrors.Inc()
|
|
return false
|
|
}
|
|
var lastID int64
|
|
batches := 0
|
|
for {
|
|
var rows []entity.MessageDetail
|
|
if err := s.db.WithContext(ctx).
|
|
Where("created_at >= ? AND created_at < ? AND id > ?", lo, hi, lastID).
|
|
Order("id ASC").Limit(archiveBatch).Find(&rows).Error; err != nil {
|
|
logger.L().Warn("archive_mysql_read_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err))
|
|
metrics.ArchiveErrors.Inc()
|
|
return false
|
|
}
|
|
if len(rows) == 0 {
|
|
break
|
|
}
|
|
if err := s.ch.BatchInsert(ctx, ArchiveTable, archiveColumns, detailToRows(rows)); err != nil {
|
|
logger.L().Warn("archive_ch_insert_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err))
|
|
metrics.ArchiveErrors.Inc()
|
|
return false
|
|
}
|
|
lastID = rows[len(rows)-1].ID
|
|
batches++
|
|
if len(rows) < archiveBatch {
|
|
break
|
|
}
|
|
}
|
|
// 校验:CH count == MySQL count 才清 MySQL(不等=保留重试,零丢失方向)。
|
|
chCount, err := s.chCountDay(ctx, day)
|
|
if err != nil || chCount != mysqlCount {
|
|
logger.L().Warn("archive_verify_mismatch_keep_mysql",
|
|
zap.String("day", day.Format("2006-01-02")),
|
|
zap.Int64("mysql", mysqlCount), zap.Int64("ch", chCount), zap.Error(err))
|
|
metrics.ArchiveErrors.Inc()
|
|
return false
|
|
}
|
|
dropped, err := s.DropPartition(ctx, day)
|
|
if err != nil {
|
|
logger.L().Warn("archive_mysql_drop_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err))
|
|
metrics.ArchiveErrors.Inc()
|
|
return false
|
|
}
|
|
if !dropped {
|
|
// 非分区残留(历史数据无对应分区):DELETE 兜底(限当日窗)。
|
|
if err := s.db.WithContext(ctx).
|
|
Where("created_at >= ? AND created_at < ?", lo, hi).
|
|
Delete(&entity.MessageDetail{}).Error; err != nil {
|
|
logger.L().Warn("archive_mysql_delete_fallback_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err))
|
|
metrics.ArchiveErrors.Inc()
|
|
return false
|
|
}
|
|
}
|
|
metrics.ArchiveRows.Add(float64(mysqlCount))
|
|
logger.L().Info("archive_day_done",
|
|
zap.String("day", day.Format("2006-01-02")), zap.Int64("rows", mysqlCount),
|
|
zap.Int("batches", batches), zap.Bool("partition_dropped", dropped))
|
|
return true
|
|
}
|
|
|
|
// detailToRows 实体→CH 行(列序=archiveColumns)。
|
|
func detailToRows(rows []entity.MessageDetail) [][]any {
|
|
out := make([][]any, 0, len(rows))
|
|
for i := range rows {
|
|
d := &rows[i]
|
|
out = append(out, []any{
|
|
d.ID, d.CreatedAt, d.TaskID, d.AccountID, d.ChannelID, d.Mobile,
|
|
d.Content, d.Province, d.Carrier, d.Status, d.ErrorCode, d.ExtCode,
|
|
d.CallData, d.Fee, d.SendAt, d.ReportAt, d.Business, d.Source,
|
|
})
|
|
}
|
|
return out
|
|
}
|