150 lines
5.4 KiB
Go
150 lines
5.4 KiB
Go
// Package archive S4·D3:明细热 5 天(MySQL 按天 RANGE(id) 分区)+ 冷数据
|
|
// ClickHouse 归档。分区键=雪花 id(时间位单调,RANGE(id) 边界=当天零点雪花
|
|
// 值,无需改主键);分区集=热 5 天(今日-4..今日)+ 未来 3 天缓冲 + pmax 兜底
|
|
// (Manager 挂掉数据仍可写,恢复后 REORGANIZE 拆分——运维口径)。过期分区
|
|
// DROP 只发生在归档 job 校验通过后(先 CH 后 MySQL,数据零丢失方向)。
|
|
package archive
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"go-sms/internal/pkg/logger"
|
|
"go-sms/internal/util"
|
|
|
|
"go.uber.org/zap"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// HotDays 热窗天数(含今日;需求 2.5「近 5 天热数据」)。
|
|
const HotDays = 5
|
|
|
|
// BufferDays 未来预建分区缓冲天数(写入永不停摆的余量)。
|
|
const BufferDays = 3
|
|
|
|
// partPrefix MySQL 分区名前缀(pYYYYMMDD)。
|
|
const partPrefix = "p"
|
|
|
|
// DayFloorSnowflake 某日 00:00(本地时区)对应的雪花 ID 边界值:
|
|
// id 高 41 位=相对 Epoch 毫秒 → 边界 = (dayStartMilli - Epoch) << 22。
|
|
func DayFloorSnowflake(day time.Time) int64 {
|
|
dayStart := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, day.Location())
|
|
return (dayStart.UnixMilli() - util.Epoch) << (util.WorkerIDBits + util.SequenceBits)
|
|
}
|
|
|
|
// PartName 分区名(本地时区日期)。
|
|
func PartName(day time.Time) string { return partPrefix + day.Format("20060102") }
|
|
|
|
// hotRange 当前应存在的按天分区日期集(今日-4 .. 今日+3)。
|
|
func hotRange(now time.Time) []time.Time {
|
|
today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
|
|
out := make([]time.Time, 0, HotDays-1+BufferDays+1)
|
|
for i := -(HotDays - 1); i <= BufferDays; i++ {
|
|
out = append(out, today.AddDate(0, 0, i))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// existingParts 查询 message_detail 现有分区(名称→边界描述)。
|
|
// 空 map+isPartitioned=false 表示表未分区。
|
|
func existingParts(ctx context.Context, db *gorm.DB) (map[string]string, bool, error) {
|
|
var rows []struct {
|
|
Name string
|
|
Desc string
|
|
}
|
|
err := db.WithContext(ctx).Raw(
|
|
"SELECT PARTITION_NAME AS name, PARTITION_DESCRIPTION AS desc_ FROM information_schema.PARTITIONS " +
|
|
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'message_detail' AND PARTITION_NAME IS NOT NULL").
|
|
Scan(&rows).Error
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
out := map[string]string{}
|
|
for _, r := range rows {
|
|
if r.Name == "" {
|
|
continue // NULL 行(非分区表在 MySQL 8.0 也占一行;方言护栏双保险)
|
|
}
|
|
out[r.Name] = r.Desc
|
|
}
|
|
return out, len(out) > 0, nil // 存在非空分区名=已分区
|
|
}
|
|
|
|
// partDefs 按天分区子句列表(每天一个,VALUES LESS THAN 次日边界)。
|
|
func partDefs(days []time.Time) []string {
|
|
out := make([]string, 0, len(days)+1)
|
|
for _, d := range days {
|
|
out = append(out, fmt.Sprintf("PARTITION %s VALUES LESS THAN (%d)",
|
|
PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1))))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// EnsurePartitions 幂等维护(MySQL only;sqlite 单测跳过):未分区→一次性
|
|
// RANGE(id) 建(热集+pmax);已分区→补缺失的按天分区(过期分区不在此删,
|
|
// 归档 job 校验后 DROP)。每轮返回是否发生了 DDL(观测用)。
|
|
func (s *Service) EnsurePartitions(ctx context.Context) (bool, error) {
|
|
if s.db.Name() != "mysql" {
|
|
return false, nil // sqlite 内存库无分区(物理优化,MySQL 语义等价)
|
|
}
|
|
existing, partitioned, err := existingParts(ctx, s.db)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
days := hotRange(time.Now())
|
|
if !partitioned {
|
|
ddl := "ALTER TABLE message_detail PARTITION BY RANGE (id) (" +
|
|
strings.Join(partDefs(days), ", ") + ", PARTITION pmax VALUES LESS THAN MAXVALUE)"
|
|
if err := s.db.WithContext(ctx).Exec(ddl).Error; err != nil {
|
|
return true, fmt.Errorf("archive: 首次分区失败: %w", err)
|
|
}
|
|
logger.L().Info("archive_partition_initialized", zap.Int("parts", len(days)+1))
|
|
return true, nil
|
|
}
|
|
changed := false
|
|
for _, d := range days {
|
|
if _, ok := existing[PartName(d)]; ok {
|
|
continue
|
|
}
|
|
ddl := fmt.Sprintf("ALTER TABLE message_detail ADD PARTITION (%s VALUES LESS THAN (%d))",
|
|
PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1)))
|
|
// pmax 已存在时不能 ADD(区间必须单调追加)——pmax 场景走 REORGANIZE 拆分。
|
|
if _, hasMax := existing["pmax"]; hasMax {
|
|
ddl = fmt.Sprintf("ALTER TABLE message_detail REORGANIZE PARTITION pmax INTO (%s VALUES LESS THAN (%d), PARTITION pmax VALUES LESS THAN MAXVALUE)",
|
|
PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1)))
|
|
}
|
|
if err := s.db.WithContext(ctx).Exec(ddl).Error; err != nil {
|
|
return changed, fmt.Errorf("archive: 补分区 %s 失败: %w", PartName(d), err)
|
|
}
|
|
existing[PartName(d)] = ""
|
|
changed = true
|
|
}
|
|
return changed, nil
|
|
}
|
|
|
|
// DropPartition 删除指定日分区(归档校验后调用;分区不存在返回 false 不报错)。
|
|
func (s *Service) DropPartition(ctx context.Context, day time.Time) (bool, error) {
|
|
if s.db.Name() != "mysql" {
|
|
return false, nil
|
|
}
|
|
existing, _, err := existingParts(ctx, s.db)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if _, ok := existing[PartName(day)]; !ok {
|
|
return false, nil
|
|
}
|
|
if err := s.db.WithContext(ctx).
|
|
Exec("ALTER TABLE message_detail DROP PARTITION " + PartName(day)).Error; err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// HotFloor 热窗下限(今日-4 零点;此前明细属归档对象)。
|
|
func HotFloor(now time.Time) time.Time {
|
|
today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
|
|
return today.AddDate(0, 0, -(HotDays - 1))
|
|
}
|