Files
go-sms-V2/internal/service/archive/partition.go
T
Tory 01c8874831
ci / test (push) Canceled after 0s
ci / web-test (push) Canceled after 0s
ci / build (push) Canceled after 0s
ci / deploy-dev (push) Canceled after 0s
S4-D3: 明细按天分区热5天+ClickHouse冷热切换——S4泳道收官(冷查16ms<1s验收✅): ①分区=迁移v22 PARTITION BY RANGE(id)(雪花时间位当天零点边界,主键即分区键零唯一键改造;热5天+未来3天缓冲+pmax兜底;MySQL守卫sqlite跳过)+archive/partition.go EnsurePartitions幂等滚动(REORGANIZE pmax拆分;过期分区归档校验后才DROP)+真机9分区存量无损;②归档=archive域(CH表toDate分区/ArchiveJob 1h:CH当日分区重写幂等→分批2000迁出→count校验相等才DROP MySQL分区或DELETE兜底=零丢失方向;consumer装配启动即首轮)+metrics archive_rows/errors;③冷查=GET /record/history(CH查询/task_no MySQL转译回填/D26契约直连)真机16ms+mobile筛选✓,导出挂账D29下载中心;④settle-check热窗感知=月中对账DetailPartial=true明细↔汇总退化两方基线(日汇总↔结算单,reconcile头注预声明落地)+settle.Service时钟注入nowFn+测试相对时间/period动态;⑤真机5连坑全修存档:MySQL8.0 information_schema非分区表也返回NULL行(判据加PARTITION_NAME IS NOT NULL)/parseTime下DATE() Scan []string带RFC3339尾巴(DISTINCT时刻Scan []time.Time+Go去重双方言)/CH24.3无DROP PARTITION IF EXISTS(先查system.parts)/CH count()返回UInt64(*int64不支持+count错误不可吞)/CH绑定time.Time UTC歧义(预格式化墙钟串)——techContext存档;⑥门禁=build/vet/lint 0/-race 41包全绿/迁移v16..v22两遍幂等(v15断言随窗口右移移除);archive单测5件+settle热窗外基线用例;验收单s4-d3-partition-archive.md+CHANGELOG+memory-bank(S4收官快照/S5下一步/挂账清单)
2026-11-16 12:11:31 +08:00

150 lines
5.4 KiB
Go

// Package archive S4·D3:明细热 5 天(MySQL 按天 RANGE(id) 分区)+ 冷数据
// ClickHouse 归档。分区键=雪花 id(时间位单调,RANGE(id) 边界=当天零点雪花
// 值,无需改主键);分区集=热 5 天(今日-4..今日)+ 未来 3 天缓冲 + pmax 兜底
// (Manager 挂掉数据仍可写,恢复后 REORGANIZE 拆分——运维口径)。过期分区
// DROP 只发生在归档 job 校验通过后(先 CH 后 MySQL,数据零丢失方向)。
package archive
import (
"context"
"fmt"
"strings"
"time"
"go-sms/internal/pkg/logger"
"go-sms/internal/util"
"go.uber.org/zap"
"gorm.io/gorm"
)
// HotDays 热窗天数(含今日;需求 2.5「近 5 天热数据」)。
const HotDays = 5
// BufferDays 未来预建分区缓冲天数(写入永不停摆的余量)。
const BufferDays = 3
// partPrefix MySQL 分区名前缀(pYYYYMMDD)。
const partPrefix = "p"
// DayFloorSnowflake 某日 00:00(本地时区)对应的雪花 ID 边界值:
// id 高 41 位=相对 Epoch 毫秒 → 边界 = (dayStartMilli - Epoch) << 22。
func DayFloorSnowflake(day time.Time) int64 {
dayStart := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, day.Location())
return (dayStart.UnixMilli() - util.Epoch) << (util.WorkerIDBits + util.SequenceBits)
}
// PartName 分区名(本地时区日期)。
func PartName(day time.Time) string { return partPrefix + day.Format("20060102") }
// hotRange 当前应存在的按天分区日期集(今日-4 .. 今日+3)。
func hotRange(now time.Time) []time.Time {
today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
out := make([]time.Time, 0, HotDays-1+BufferDays+1)
for i := -(HotDays - 1); i <= BufferDays; i++ {
out = append(out, today.AddDate(0, 0, i))
}
return out
}
// existingParts 查询 message_detail 现有分区(名称→边界描述)。
// 空 map+isPartitioned=false 表示表未分区。
func existingParts(ctx context.Context, db *gorm.DB) (map[string]string, bool, error) {
var rows []struct {
Name string
Desc string
}
err := db.WithContext(ctx).Raw(
"SELECT PARTITION_NAME AS name, PARTITION_DESCRIPTION AS desc_ FROM information_schema.PARTITIONS " +
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'message_detail' AND PARTITION_NAME IS NOT NULL").
Scan(&rows).Error
if err != nil {
return nil, false, err
}
out := map[string]string{}
for _, r := range rows {
if r.Name == "" {
continue // NULL 行(非分区表在 MySQL 8.0 也占一行;方言护栏双保险)
}
out[r.Name] = r.Desc
}
return out, len(out) > 0, nil // 存在非空分区名=已分区
}
// partDefs 按天分区子句列表(每天一个,VALUES LESS THAN 次日边界)。
func partDefs(days []time.Time) []string {
out := make([]string, 0, len(days)+1)
for _, d := range days {
out = append(out, fmt.Sprintf("PARTITION %s VALUES LESS THAN (%d)",
PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1))))
}
return out
}
// EnsurePartitions 幂等维护(MySQL only;sqlite 单测跳过):未分区→一次性
// RANGE(id) 建(热集+pmax);已分区→补缺失的按天分区(过期分区不在此删,
// 归档 job 校验后 DROP)。每轮返回是否发生了 DDL(观测用)。
func (s *Service) EnsurePartitions(ctx context.Context) (bool, error) {
if s.db.Name() != "mysql" {
return false, nil // sqlite 内存库无分区(物理优化,MySQL 语义等价)
}
existing, partitioned, err := existingParts(ctx, s.db)
if err != nil {
return false, err
}
days := hotRange(time.Now())
if !partitioned {
ddl := "ALTER TABLE message_detail PARTITION BY RANGE (id) (" +
strings.Join(partDefs(days), ", ") + ", PARTITION pmax VALUES LESS THAN MAXVALUE)"
if err := s.db.WithContext(ctx).Exec(ddl).Error; err != nil {
return true, fmt.Errorf("archive: 首次分区失败: %w", err)
}
logger.L().Info("archive_partition_initialized", zap.Int("parts", len(days)+1))
return true, nil
}
changed := false
for _, d := range days {
if _, ok := existing[PartName(d)]; ok {
continue
}
ddl := fmt.Sprintf("ALTER TABLE message_detail ADD PARTITION (%s VALUES LESS THAN (%d))",
PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1)))
// pmax 已存在时不能 ADD(区间必须单调追加)——pmax 场景走 REORGANIZE 拆分。
if _, hasMax := existing["pmax"]; hasMax {
ddl = fmt.Sprintf("ALTER TABLE message_detail REORGANIZE PARTITION pmax INTO (%s VALUES LESS THAN (%d), PARTITION pmax VALUES LESS THAN MAXVALUE)",
PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1)))
}
if err := s.db.WithContext(ctx).Exec(ddl).Error; err != nil {
return changed, fmt.Errorf("archive: 补分区 %s 失败: %w", PartName(d), err)
}
existing[PartName(d)] = ""
changed = true
}
return changed, nil
}
// DropPartition 删除指定日分区(归档校验后调用;分区不存在返回 false 不报错)。
func (s *Service) DropPartition(ctx context.Context, day time.Time) (bool, error) {
if s.db.Name() != "mysql" {
return false, nil
}
existing, _, err := existingParts(ctx, s.db)
if err != nil {
return false, err
}
if _, ok := existing[PartName(day)]; !ok {
return false, nil
}
if err := s.db.WithContext(ctx).
Exec("ALTER TABLE message_detail DROP PARTITION " + PartName(day)).Error; err != nil {
return false, err
}
return true, nil
}
// HotFloor 热窗下限(今日-4 零点;此前明细属归档对象)。
func HotFloor(now time.Time) time.Time {
today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
return today.AddDate(0, 0, -(HotDays - 1))
}