Files
go-sms-V2/internal/service/archive/schema.go
T
Tory 01c8874831
ci / test (push) Canceled after 0s
ci / web-test (push) Canceled after 0s
ci / build (push) Canceled after 0s
ci / deploy-dev (push) Canceled after 0s
S4-D3: 明细按天分区热5天+ClickHouse冷热切换——S4泳道收官(冷查16ms<1s验收✅): ①分区=迁移v22 PARTITION BY RANGE(id)(雪花时间位当天零点边界,主键即分区键零唯一键改造;热5天+未来3天缓冲+pmax兜底;MySQL守卫sqlite跳过)+archive/partition.go EnsurePartitions幂等滚动(REORGANIZE pmax拆分;过期分区归档校验后才DROP)+真机9分区存量无损;②归档=archive域(CH表toDate分区/ArchiveJob 1h:CH当日分区重写幂等→分批2000迁出→count校验相等才DROP MySQL分区或DELETE兜底=零丢失方向;consumer装配启动即首轮)+metrics archive_rows/errors;③冷查=GET /record/history(CH查询/task_no MySQL转译回填/D26契约直连)真机16ms+mobile筛选✓,导出挂账D29下载中心;④settle-check热窗感知=月中对账DetailPartial=true明细↔汇总退化两方基线(日汇总↔结算单,reconcile头注预声明落地)+settle.Service时钟注入nowFn+测试相对时间/period动态;⑤真机5连坑全修存档:MySQL8.0 information_schema非分区表也返回NULL行(判据加PARTITION_NAME IS NOT NULL)/parseTime下DATE() Scan []string带RFC3339尾巴(DISTINCT时刻Scan []time.Time+Go去重双方言)/CH24.3无DROP PARTITION IF EXISTS(先查system.parts)/CH count()返回UInt64(*int64不支持+count错误不可吞)/CH绑定time.Time UTC歧义(预格式化墙钟串)——techContext存档;⑥门禁=build/vet/lint 0/-race 41包全绿/迁移v16..v22两遍幂等(v15断言随窗口右移移除);archive单测5件+settle热窗外基线用例;验收单s4-d3-partition-archive.md+CHANGELOG+memory-bank(S4收官快照/S5下一步/挂账清单)
2026-11-16 12:11:31 +08:00

243 lines
7.1 KiB
Go

// schema.go S4·D3:ClickHouse 归档表 DDL(幂等)+ 冷查行扫描。
package archive
import (
"context"
"fmt"
"time"
"go-sms/internal/repository/clickhouse"
)
// ArchiveTable CH 归档表名(字段与 message_detail 对齐;fee Decimal→Float64,
// 可空时间列 Nullable;按天分区 toDate(created_at),ORDER BY id)。
const ArchiveTable = "sms_detail_archive"
// archiveColumns 归档列(与 DDL/INSERT/SELECT 三方对齐,顺序稳定)。
var archiveColumns = []string{
"id", "created_at", "task_id", "account_id", "channel_id", "mobile",
"content", "province", "carrier", "status", "error_code", "ext_code",
"call_data", "fee", "send_at", "report_at", "business", "source",
}
// ArchiveTableDDL 建表语句(幂等;PARTITION BY toDate(created_at) 的分区名
// =YYYYMMDD,与 ArchiveDayPartition 一致,归档重写=先 DROP PARTITION 再 INSERT)。
const ArchiveTableDDL = `
CREATE TABLE IF NOT EXISTS sms_detail_archive (
id Int64,
created_at DateTime64(3),
task_id Int64,
account_id Int64,
channel_id Int64,
mobile String,
content String,
province LowCardinality(String),
carrier LowCardinality(String),
status Int8,
error_code LowCardinality(String),
ext_code LowCardinality(String),
call_data String,
fee Float64,
send_at Nullable(DateTime64(3)),
report_at Nullable(DateTime64(3)),
business LowCardinality(String),
source Int8
) ENGINE = MergeTree
PARTITION BY toDate(created_at)
ORDER BY id
SETTINGS index_granularity = 8192`
// EnsureCHSchema 幂等建归档表。
func (s *Service) EnsureCHSchema(ctx context.Context) error {
if s.ch == nil {
return nil // CH 未装配:归档链路禁用(单测/最小部署)
}
return s.ch.Exec(ctx, ArchiveTableDDL)
}
// ArchiveDayPartition 某日在 CH 的分区名(toDate 表达式分区名=YYYYMMDD)。
func ArchiveDayPartition(day time.Time) string { return day.Format("20060102") }
// dropCHPartition 删除 CH 当日分区(CH 24.3 无 DROP PARTITION IF EXISTS——
// 先查 system.parts 存在才 DROP;不存在静默通过)。
func (s *Service) dropCHPartition(ctx context.Context, day time.Time) error {
rows, err := s.ch.Query(ctx,
"SELECT count() FROM system.parts WHERE database = currentDatabase() "+
"AND table = ? AND partition = ?", ArchiveTable, ArchiveDayPartition(day))
if err != nil {
return err
}
n := uint64(0) // CH count() 返回 UInt64(Scan *int64 不支持,真机教训)
if rows.Next() {
if err := rows.Scan(&n); err != nil {
_ = rows.Close()
return err
}
}
_ = rows.Close()
if n == 0 {
return nil
}
return s.ch.Exec(ctx,
fmt.Sprintf("ALTER TABLE `%s` DROP PARTITION '%s'", ArchiveTable, ArchiveDayPartition(day)))
}
// chCountDay CH 当日行数(归档校验锚)。
func (s *Service) chCountDay(ctx context.Context, day time.Time) (int64, error) {
rows, err := s.ch.Query(ctx,
fmt.Sprintf("SELECT count() FROM `%s` WHERE toDate(created_at) = ?", ArchiveTable),
ArchiveDayPartition(day))
if err != nil {
return 0, err
}
defer func() { _ = rows.Close() }()
if !rows.Next() {
return 0, fmt.Errorf("archive: CH count 无行返回")
}
var n uint64 // CH count() 返回 UInt64
if err := rows.Scan(&n); err != nil {
return 0, err
}
return int64(n), nil
}
// ArchiveRow 冷查明细行(CH 扫描 DTO;时间字段 CH 驱动返回 time.Time)。
type ArchiveRow struct {
ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
TaskID int64 `json:"task_id"`
AccountID int64 `json:"account_id"`
ChannelID int64 `json:"channel_id"`
Mobile string `json:"mobile"`
Content string `json:"content"`
Province string `json:"province"`
Carrier string `json:"carrier"`
Status int8 `json:"status"`
ErrorCode string `json:"error_code"`
ExtCode string `json:"ext_code"`
CallData string `json:"call_data"`
Fee float64 `json:"fee"`
SendAt *time.Time `json:"send_at"`
ReportAt *time.Time `json:"report_at"`
Business string `json:"business"`
Source int8 `json:"source"`
}
// QueryHistory 冷查(CH;分页+账号/手机号/时间窗筛选;验收=冷查<1s)。
// 返回行列表与总数(总数另查 count();行窗 id 排序稳定)。
func (s *Service) QueryHistory(ctx context.Context, q HistoryQuery) ([]ArchiveRow, int64, error) {
if s.ch == nil {
return nil, 0, fmt.Errorf("archive: ClickHouse 未配置")
}
page, size := q.Page, q.PageSize
if page < 1 {
page = 1
}
if size < 1 || size > 1000 {
size = 50
}
where, args := historyWhere(q)
var total int64
crows, err := s.ch.Query(ctx,
fmt.Sprintf("SELECT count() FROM `%s`%s", ArchiveTable, where), args...)
if err != nil {
return nil, 0, err
}
{ // CH count() 返回 UInt64(*int64 Scan 不支持——错误不可吞,D3 真机教训)
var n uint64
if crows.Next() {
if err := crows.Scan(&n); err != nil {
_ = crows.Close()
return nil, 0, err
}
}
_ = crows.Close()
total = int64(n)
}
if total == 0 {
return []ArchiveRow{}, 0, nil
}
lrows, err := s.ch.Query(ctx,
fmt.Sprintf("SELECT %s FROM `%s`%s ORDER BY id DESC LIMIT %d OFFSET %d",
joinColumns(), ArchiveTable, where, size, (page-1)*size), args...)
if err != nil {
return nil, 0, err
}
defer func() { _ = lrows.Close() }()
out := make([]ArchiveRow, 0, size)
for lrows.Next() {
var r ArchiveRow
if err := lrows.Scan(&r.ID, &r.CreatedAt, &r.TaskID, &r.AccountID, &r.ChannelID,
&r.Mobile, &r.Content, &r.Province, &r.Carrier, &r.Status, &r.ErrorCode,
&r.ExtCode, &r.CallData, &r.Fee, &r.SendAt, &r.ReportAt, &r.Business, &r.Source); err != nil {
return nil, 0, err
}
out = append(out, r)
}
return out, total, nil
}
// HistoryQuery 冷查入参(全部可选;时间窗起闭止开)。
type HistoryQuery struct {
AccountID int64
ChannelID int64
TaskID int64
Mobile string
Status *int8
Start *time.Time
End *time.Time
Page int
PageSize int
}
func historyWhere(q HistoryQuery) (string, []any) {
conds := []string{"1=1"}
var args []any
if q.AccountID > 0 {
conds = append(conds, "account_id = ?")
args = append(args, q.AccountID)
}
if q.ChannelID > 0 {
conds = append(conds, "channel_id = ?")
args = append(args, q.ChannelID)
}
if q.TaskID > 0 {
conds = append(conds, "task_id = ?")
args = append(args, q.TaskID)
}
if q.Mobile != "" {
conds = append(conds, "mobile = ?")
args = append(args, q.Mobile)
}
if q.Status != nil {
conds = append(conds, "status = ?")
args = append(args, *q.Status)
}
if q.Start != nil {
conds = append(conds, "created_at >= ?")
// CH 绑定 time.Time 存在 UTC 化歧义;归档写入=墙钟语义,查询同口径
// 预格式化字符串(D3 真机教训)。
args = append(args, q.Start.Format("2006-01-02 15:04:05"))
}
if q.End != nil {
conds = append(conds, "created_at < ?")
args = append(args, q.End.Format("2006-01-02 15:04:05"))
}
return " WHERE " + joinStrings(conds, " AND "), args
}
func joinStrings(ss []string, sep string) string {
out := ""
for i, s := range ss {
if i > 0 {
out += sep
}
out += s
}
return out
}
func joinColumns() string { return joinStrings(archiveColumns, ", ") }
var _ clickhouse.Conn = (clickhouse.Conn)(nil) // 引用守卫(接口扩展与本包解耦)