From 01c88748319b35b237d9cc4f6bd63dfe801b9f77 Mon Sep 17 00:00:00 2001 From: Tory Date: Mon, 16 Nov 2026 12:11:23 +0800 Subject: [PATCH] =?UTF-8?q?S4-D3:=20=E6=98=8E=E7=BB=86=E6=8C=89=E5=A4=A9?= =?UTF-8?q?=E5=88=86=E5=8C=BA=E7=83=AD5=E5=A4=A9+ClickHouse=E5=86=B7?= =?UTF-8?q?=E7=83=AD=E5=88=87=E6=8D=A2=E2=80=94=E2=80=94S4=E6=B3=B3?= =?UTF-8?q?=E9=81=93=E6=94=B6=E5=AE=98(=E5=86=B7=E6=9F=A516ms<1s=E9=AA=8C?= =?UTF-8?q?=E6=94=B6=E2=9C=85):=20=E2=91=A0=E5=88=86=E5=8C=BA=3D=E8=BF=81?= =?UTF-8?q?=E7=A7=BBv22=20PARTITION=20BY=20RANGE(id)(=E9=9B=AA=E8=8A=B1?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=BD=8D=E5=BD=93=E5=A4=A9=E9=9B=B6=E7=82=B9?= =?UTF-8?q?=E8=BE=B9=E7=95=8C,=E4=B8=BB=E9=94=AE=E5=8D=B3=E5=88=86?= =?UTF-8?q?=E5=8C=BA=E9=94=AE=E9=9B=B6=E5=94=AF=E4=B8=80=E9=94=AE=E6=94=B9?= =?UTF-8?q?=E9=80=A0;=E7=83=AD5=E5=A4=A9+=E6=9C=AA=E6=9D=A53=E5=A4=A9?= =?UTF-8?q?=E7=BC=93=E5=86=B2+pmax=E5=85=9C=E5=BA=95;MySQL=E5=AE=88?= =?UTF-8?q?=E5=8D=ABsqlite=E8=B7=B3=E8=BF=87)+archive/partition.go=20Ensur?= =?UTF-8?q?ePartitions=E5=B9=82=E7=AD=89=E6=BB=9A=E5=8A=A8(REORGANIZE=20pm?= =?UTF-8?q?ax=E6=8B=86=E5=88=86;=E8=BF=87=E6=9C=9F=E5=88=86=E5=8C=BA?= =?UTF-8?q?=E5=BD=92=E6=A1=A3=E6=A0=A1=E9=AA=8C=E5=90=8E=E6=89=8DDROP)+?= =?UTF-8?q?=E7=9C=9F=E6=9C=BA9=E5=88=86=E5=8C=BA=E5=AD=98=E9=87=8F?= =?UTF-8?q?=E6=97=A0=E6=8D=9F;=E2=91=A1=E5=BD=92=E6=A1=A3=3Darchive?= =?UTF-8?q?=E5=9F=9F(CH=E8=A1=A8toDate=E5=88=86=E5=8C=BA/ArchiveJob=201h:C?= =?UTF-8?q?H=E5=BD=93=E6=97=A5=E5=88=86=E5=8C=BA=E9=87=8D=E5=86=99?= =?UTF-8?q?=E5=B9=82=E7=AD=89=E2=86=92=E5=88=86=E6=89=B92000=E8=BF=81?= =?UTF-8?q?=E5=87=BA=E2=86=92count=E6=A0=A1=E9=AA=8C=E7=9B=B8=E7=AD=89?= =?UTF-8?q?=E6=89=8DDROP=20MySQL=E5=88=86=E5=8C=BA=E6=88=96DELETE=E5=85=9C?= =?UTF-8?q?=E5=BA=95=3D=E9=9B=B6=E4=B8=A2=E5=A4=B1=E6=96=B9=E5=90=91;consu?= =?UTF-8?q?mer=E8=A3=85=E9=85=8D=E5=90=AF=E5=8A=A8=E5=8D=B3=E9=A6=96?= =?UTF-8?q?=E8=BD=AE)+metrics=20archive=5Frows/errors;=E2=91=A2=E5=86=B7?= =?UTF-8?q?=E6=9F=A5=3DGET=20/record/history(CH=E6=9F=A5=E8=AF=A2/task=5Fn?= =?UTF-8?q?o=20MySQL=E8=BD=AC=E8=AF=91=E5=9B=9E=E5=A1=AB/D26=E5=A5=91?= =?UTF-8?q?=E7=BA=A6=E7=9B=B4=E8=BF=9E)=E7=9C=9F=E6=9C=BA16ms+mobile?= =?UTF-8?q?=E7=AD=9B=E9=80=89=E2=9C=93,=E5=AF=BC=E5=87=BA=E6=8C=82?= =?UTF-8?q?=E8=B4=A6D29=E4=B8=8B=E8=BD=BD=E4=B8=AD=E5=BF=83;=E2=91=A3settl?= =?UTF-8?q?e-check=E7=83=AD=E7=AA=97=E6=84=9F=E7=9F=A5=3D=E6=9C=88?= =?UTF-8?q?=E4=B8=AD=E5=AF=B9=E8=B4=A6DetailPartial=3Dtrue=E6=98=8E?= =?UTF-8?q?=E7=BB=86=E2=86=94=E6=B1=87=E6=80=BB=E9=80=80=E5=8C=96=E4=B8=A4?= =?UTF-8?q?=E6=96=B9=E5=9F=BA=E7=BA=BF(=E6=97=A5=E6=B1=87=E6=80=BB?= =?UTF-8?q?=E2=86=94=E7=BB=93=E7=AE=97=E5=8D=95,reconcile=E5=A4=B4?= =?UTF-8?q?=E6=B3=A8=E9=A2=84=E5=A3=B0=E6=98=8E=E8=90=BD=E5=9C=B0)+settle.?= =?UTF-8?q?Service=E6=97=B6=E9=92=9F=E6=B3=A8=E5=85=A5nowFn+=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=9B=B8=E5=AF=B9=E6=97=B6=E9=97=B4/period=E5=8A=A8?= =?UTF-8?q?=E6=80=81;=E2=91=A4=E7=9C=9F=E6=9C=BA5=E8=BF=9E=E5=9D=91?= =?UTF-8?q?=E5=85=A8=E4=BF=AE=E5=AD=98=E6=A1=A3:MySQL8.0=20information=5Fs?= =?UTF-8?q?chema=E9=9D=9E=E5=88=86=E5=8C=BA=E8=A1=A8=E4=B9=9F=E8=BF=94?= =?UTF-8?q?=E5=9B=9ENULL=E8=A1=8C(=E5=88=A4=E6=8D=AE=E5=8A=A0PARTITION=5FN?= =?UTF-8?q?AME=20IS=20NOT=20NULL)/parseTime=E4=B8=8BDATE()=20Scan=20[]stri?= =?UTF-8?q?ng=E5=B8=A6RFC3339=E5=B0=BE=E5=B7=B4(DISTINCT=E6=97=B6=E5=88=BB?= =?UTF-8?q?Scan=20[]time.Time+Go=E5=8E=BB=E9=87=8D=E5=8F=8C=E6=96=B9?= =?UTF-8?q?=E8=A8=80)/CH24.3=E6=97=A0DROP=20PARTITION=20IF=20EXISTS(?= =?UTF-8?q?=E5=85=88=E6=9F=A5system.parts)/CH=20count()=E8=BF=94=E5=9B=9EU?= =?UTF-8?q?Int64(*int64=E4=B8=8D=E6=94=AF=E6=8C=81+count=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E4=B8=8D=E5=8F=AF=E5=90=9E)/CH=E7=BB=91=E5=AE=9Atime.Time=20UT?= =?UTF-8?q?C=E6=AD=A7=E4=B9=89(=E9=A2=84=E6=A0=BC=E5=BC=8F=E5=8C=96?= =?UTF-8?q?=E5=A2=99=E9=92=9F=E4=B8=B2)=E2=80=94=E2=80=94techContext?= =?UTF-8?q?=E5=AD=98=E6=A1=A3;=E2=91=A5=E9=97=A8=E7=A6=81=3Dbuild/vet/lint?= =?UTF-8?q?=200/-race=2041=E5=8C=85=E5=85=A8=E7=BB=BF/=E8=BF=81=E7=A7=BBv1?= =?UTF-8?q?6..v22=E4=B8=A4=E9=81=8D=E5=B9=82=E7=AD=89(v15=E6=96=AD?= =?UTF-8?q?=E8=A8=80=E9=9A=8F=E7=AA=97=E5=8F=A3=E5=8F=B3=E7=A7=BB=E7=A7=BB?= =?UTF-8?q?=E9=99=A4);archive=E5=8D=95=E6=B5=8B5=E4=BB=B6+settle=E7=83=AD?= =?UTF-8?q?=E7=AA=97=E5=A4=96=E5=9F=BA=E7=BA=BF=E7=94=A8=E4=BE=8B;?= =?UTF-8?q?=E9=AA=8C=E6=94=B6=E5=8D=95s4-d3-partition-archive.md+CHANGELOG?= =?UTF-8?q?+memory-bank(S4=E6=94=B6=E5=AE=98=E5=BF=AB=E7=85=A7/S5=E4=B8=8B?= =?UTF-8?q?=E4=B8=80=E6=AD=A5/=E6=8C=82=E8=B4=A6=E6=B8=85=E5=8D=95)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 9 + cmd/consumer/main.go | 20 ++ cmd/gateway/main.go | 20 ++ docs/acceptance/s4-d3-partition-archive.md | 55 +++++ internal/api/http/v1/account.go | 6 +- internal/api/http/v1/record_handler.go | 108 ++++++++- internal/pkg/metrics/prometheus.go | 17 ++ internal/repository/clickhouse/clickhouse.go | 28 +++ internal/repository/mysql/base_test.go | 25 +- internal/repository/mysql/migrations.go | 42 ++++ internal/service/archive/archive_test.go | 180 ++++++++++++++ internal/service/archive/job.go | 203 ++++++++++++++++ internal/service/archive/partition.go | 149 ++++++++++++ internal/service/archive/schema.go | 242 +++++++++++++++++++ internal/service/settle/consume.go | 8 +- internal/service/settle/reconcile.go | 57 ++++- internal/service/settle/reconcile_test.go | 62 ++++- memory-bank/activeContext.md | 18 +- memory-bank/progress.md | 4 +- memory-bank/techContext.md | 2 + 20 files changed, 1204 insertions(+), 51 deletions(-) create mode 100644 docs/acceptance/s4-d3-partition-archive.md create mode 100644 internal/service/archive/archive_test.go create mode 100644 internal/service/archive/job.go create mode 100644 internal/service/archive/partition.go create mode 100644 internal/service/archive/schema.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f2f402e..c95841fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ > 本仓 CHANGELOG 自重建起记(POC 明细见归档仓 /data/poc-archive/repos/go-sms.git 的 CHANGELOG.md,已冻结)。 +## 2026-11-16(下午)· S4-D3:明细按天分区+CH 冷热切换——S4 泳道收官(三项验收全过) + +- **分区**:迁移 v22 `PARTITION BY RANGE (id)`(雪花时间位当天零点边界,主键即分区键零唯一键改造;热 5 天+未来 3 天+pmax 兜底;MySQL 守卫 sqlite 跳过)+EnsurePartitions 幂等滚动(archive/partition.go);真机 9 分区落地存量无损。 +- **归档**:archive 域(schema CH 表 toDate 分区/ArchiveJob 1h:CH 当日分区重写幂等→分批 2000 迁出→**count 校验相等才 DROP MySQL 分区/DELETE 兜底**=零丢失方向;consumer 装配启动即首轮)+metrics archive_rows/errors。 +- **冷查**:GET /record/history(CH;task_no MySQL 转译回填;D26 契约直连)——**真机 16ms(<1s 验收线)**,mobile 筛选✓;history 导出挂账 D29 下载中心。 +- **settle-check 热窗感知**:月中对账 DetailPartial=true→明细↔汇总退化两方基线(日汇总↔结算单;reconcile 头注预声明落地);settle.Service 时钟注入(nowFn)。 +- **真机 5 连坑全修**(教训存档验收单):①MySQL 8.0 information_schema 对非分区表也返回 NULL 行(判据加 PARTITION_NAME IS NOT NULL)②parseTime 下 DATE() Scan []string 带 RFC3339 尾巴(改 DISTINCT 时刻 Scan []time.Time+Go 去重,双方言兼容)③CH 24.3 无 DROP PARTITION IF EXISTS(先查 system.parts)④CH count() 返回 UInt64(*int64 Scan 不支持;count 错误曾被吞致 total 恒 0)⑤CH 绑定 time.Time UTC 歧义(时间窗参数预格式化墙钟串)。 +- **门禁**:build/vet/lint 0/-race **41 包全绿**;迁移测试 v16..v22 两遍幂等;archive 单测 5 件+settle 热窗外基线用例;settle 测试改相对时间+period 动态(v22 前固定日期在热窗外误触发新逻辑);gofmt 全仓零;验收单 s4-d3-partition-archive.md。 + ## 2026-11-16 · S4-D1/D2:C1 回调推送引擎 + C2 上行自动拉黑(附 Issue#1 二轮入库) - **Issue#1 第二轮对齐入库(前会话遗留)**:ChannelList 20列→8列双语义聚合+ChannelEdit 分组深度对齐+chrome-session.mjs 工具;门禁复验四件全绿。 diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index faf8dc98..ccb598d5 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -19,6 +19,8 @@ import ( "go-sms/internal/model/entity" "go-sms/internal/pkg/metrics" "go-sms/internal/pkg/queue" + chrepo "go-sms/internal/repository/clickhouse" + "go-sms/internal/service/archive" "go-sms/internal/service/audit" "go-sms/internal/service/balance" "go-sms/internal/service/callback" @@ -208,6 +210,24 @@ func main() { a.Lg().Info("callback_engine_ready", zap.Bool("blacklist", true), zap.Duration("scan_every", callback.DefaultScanEvery)) + // S4·D3:明细冷热归档 job(CH 不可达=分区维护仍跑、迁出禁用自愈)。 + var chConn chrepo.Conn + if c, chErr := chrepo.Open(chrepo.Config{ + Addrs: cfg.ClickHouse.Addrs, Database: cfg.ClickHouse.Database, + Username: cfg.ClickHouse.Username, Password: cfg.ClickHouse.Password, + DialTimeout: cfg.ClickHouse.DialTimeout, + }); chErr != nil { + a.Lg().Warn("clickhouse_open_failed_archive_degraded", zap.Error(chErr)) + } else { + chConn = c + a.OnShutdown(func() { _ = chConn.Close() }) + } + arcSvc := archive.NewService(db, chConn) + arcCtx, stopArc := context.WithCancel(context.Background()) + go arcSvc.Run(arcCtx, archive.DefaultArchiveEvery) + a.OnShutdown(stopArc) + a.Lg().Info("archive_job_ready", zap.Bool("clickhouse", chConn != nil)) + // Day 49:上行消费组(sms-up;15 天窗口匹配下行明细回填 match_task_id; // S4:上行推送+自动拉黑挂消费尾部,不阻断匹配主链)。独立 group 前缀防 offset 互扰。 upTopic := cfg.Kafka.Topics["up"] diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 53215e01..ac919575 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -18,10 +18,12 @@ import ( "go-sms/internal/pkg/metrics" "go-sms/internal/pkg/queue" "go-sms/internal/pkg/storage" + chrepo "go-sms/internal/repository/clickhouse" cmppproto "go-sms/internal/protocol/cmpp" smppproto "go-sms/internal/protocol/smpp" "go-sms/internal/service/access" "go-sms/internal/service/account" + "go-sms/internal/service/archive" "go-sms/internal/service/audit" "go-sms/internal/service/auth" "go-sms/internal/service/balance" @@ -212,6 +214,22 @@ func main() { if p := a.Producer(); p != nil { pairSvc = pairSvc.WithPushSink(&report.ProducerPushSink{P: p}) } + // S4·D3:明细冷热归档——gateway 侧仅挂 /record/history 冷查 + // (归档 job 在 consumer 进程);CH 不可达=端点不注册(404 优于半可用)。 + var arcSvc *archive.Service + if chConn, chErr := chrepo.Open(chrepo.Config{ + Addrs: cfg.ClickHouse.Addrs, Database: cfg.ClickHouse.Database, + Username: cfg.ClickHouse.Username, Password: cfg.ClickHouse.Password, + DialTimeout: cfg.ClickHouse.DialTimeout, + }); chErr != nil { + a.Lg().Warn("clickhouse_open_failed_history_disabled", zap.Error(chErr)) + } else { + arcSvc = archive.NewService(db, chConn) + if err := arcSvc.EnsureCHSchema(context.Background()); err != nil { + a.Lg().Warn("archive_ch_schema_failed", zap.Error(err)) + } + a.OnShutdown(func() { _ = chConn.Close() }) + } v1.Register(rg, v1.Deps{ Account: svc, Warning: warnSvc, @@ -241,6 +259,8 @@ func main() { Settle: settle.NewService(db), // S4·C1:/ops/callback 死信查询与重派(管理面;投递在 consumer 进程) Callback: cbSvc, + // S4·D3:/record/history 冷查(CH;nil=不注册) + Archive: arcSvc, }) // Day68:运营总览汇总刷新(回补 30 天+60s ticker 增量 UPSERT stats_daily; // 幂等多实例无害,详见 stats_handler.go StartStatsRefresher)。 diff --git a/docs/acceptance/s4-d3-partition-archive.md b/docs/acceptance/s4-d3-partition-archive.md new file mode 100644 index 00000000..fa03568e --- /dev/null +++ b/docs/acceptance/s4-d3-partition-archive.md @@ -0,0 +1,55 @@ +# S4 验收单 D3:明细按天分区热 5 天 + ClickHouse 冷热切换(2026-11-16) + +> 泳道:S4 第 3 日;验收项「冷查 <1s」✅(16ms,索引冷缓存口径)。S4 三项验收全过,泳道收官。 + +## 一、交付范围 + +### 明细按天 RANGE(id) 分区(热 5 天) + +| 组件 | 说明 | +|---|---| +| 迁移 v22 | `ALTER TABLE message_detail PARTITION BY RANGE (id)`——分区键=雪花 id(时间位单调,当天零点边界=`(dayStart-Epoch)<<22` 精确计算;**主键即分区键,唯一键零改造**);分区集=热 5 天+未来 3 天缓冲+pmax 兜底;MySQL only(sqlite 方言守卫) | +| `archive/partition.go` | DayFloorSnowflake/PartName/HotFloor 纯函数 + EnsurePartitions 幂等滚动(缺补 ADD/REORGANIZE pmax 拆分;**过期分区不在此删**——归档校验通过才 DROP)+ DropPartition | +| 真机分区状态 | p20261112..p20261119 + pmax 共 9 分区;存量 3 行无损迁入 | + +### CH 冷热归档(`internal/service/archive`) + +| 组件 | 说明 | +|---|---| +| CH 表 | sms_detail_archive(MergeTree,PARTITION BY toDate(created_at),ORDER BY id,18 列与 message_detail 对齐;EnsureCHSchema 幂等) | +| ArchiveJob(consumer 进程,1h) | 热窗外逐日:CH DROP 当日分区(重写幂等)→MySQL 分批 2000 读→CH BatchInsert→**count 校验相等才 DROP MySQL 分区/DELETE 兜底**(零丢失方向:不等保留重试) | +| 冷查 | GET /record/history(CH 查询;task_no 经 MySQL message_task 转译+批量回填;契约=web record.ts D26);metrics archive_rows/errors | + +### settle-check 热窗感知 + +对账窗起点早于热窗下限(月中对账常态)→ `DetailPartial=true`:明细↔汇总比对退化为**两方基线(日汇总↔结算单)**,明细侧不判差异(归档清退≠丢失,历史明细核对走 CH 冷查)——reconcile.go 头注预声明口径落地;settle.Service 时钟注入(nowFn)供测试。 + +## 二、真机全链(dev MySQL/CH,2026-11-16 11:44-11:58) + +1. **分区**:v22 迁移→9 分区落地,存量 3 行保留 ✓ +2. **归档**:造 6 天前(11-10)2 行+今天 1 行→consumer 重启即首轮归档→`archive_day_done rows=2`(DELETE 兜底分支)→MySQL 冷数据 0 行/热数据保留,CH 2 行(content/mobile 完整),metrics archive_rows_total=2 ✓ +3. **冷查**:`GET /record/history?start_time=2026-11-01&end_time=2026-11-12` → **16ms**(**验收线 <1s 达标**,两次测 7.5/16ms),total=2 行数据全对;mobile 精筛 ✓ +4. **settle-check**:月中 2026-11 → detail 侧=0(热窗感知生效)+BillMissing 正常列行;all_zero=false 仅因 S2 走查种子残留(account=2 账单 1955 vs 消耗 16050 不齐——用户走查在途数据,不清理) +5. 冒烟数据全清(MySQL 3 行+当日消耗行;CH 20261110 分区 DROP→count=0) + +## 三、门禁(七件套) + +- 后端:build ✓ / vet ✓ / golangci-lint **0** / **-race 全包绿**(后台终跑) / 迁移测试 v16..v22 两遍幂等(v22 sqlite 守卫跳过;v15 断言随窗口右移移除)。 +- 单测:archive 5 件(边界单调/归档全链含幂等/校验不匹配保留/冷查条件)+ settle +1(热窗外两方基线,时钟注入月 20 号稳定复现)。 +- 前端:无改动(契约 D26 已在;history 页零改动直连新端点)。 + +## 四、真机调试教训留痕(5 连坑,全部已修) + +1. **MySQL 8.0 information_schema.PARTITIONS 对非分区表也返回一行**(PARTITION_NAME=NULL)——已分区判据必须 `AND PARTITION_NAME IS NOT NULL`(迁移 v22+existingParts 双处)。 +2. **go-sql-driver parseTime=True 下 DATE() 结果 Scan 到 []string 带 RFC3339 尾巴**('2026-11-10T00:00:00+08:00')→ Parse 失败静默跳天;改为 DISTINCT 时刻级 created_at Scan []time.Time+Go 侧归一化去重(MySQL/glebarez 双方言兼容)。 +3. **CH 24.3 无 `DROP PARTITION IF EXISTS`**(语法错)→ 先查 system.parts 存在才 DROP。 +4. **CH count() 返回 UInt64,Scan *int64 不支持**——dropCHPartition/chCountDay 用 *uint64;QueryHistory 的 count Scan 错误曾被 `_=` 吞掉致 total 恒 0(错误不可吞)。 +5. **CH 绑定 time.Time 的 UTC 化歧义**——查询时间窗参数预格式化墙钟字符串(与归档写入同口径)。 + +## 五、口径与假设(诚实标注) + +1. **归档 job 单实例**(consumer 进程);多实例重复迁出由「CH 分区重写+count 校验」幂等吸收。 +2. **CH 无备份/TTL**(dev 口径);生产建议 CH 副本+冷分区 S3 挂载(S5 运维项)。 +3. **history 导出(/record/history/export 与下载中心)挂账 D29/P2**——D26 契约先查后导,异步导出走 MinIO/RustFS 下载中心(S5+ 择机)。 +4. **pmax 兜底分区语义**:EnsurePartitions 挂掉期间数据落入 pmax,恢复后 REORGANIZE 拆分(运维 runbook 项)。 +5. 明细写入侧零改动(雪花 id 天然落在当日分区);watchdog/明细页查询语义不变(分区对 SQL 透明)。 diff --git a/internal/api/http/v1/account.go b/internal/api/http/v1/account.go index 2c6afa69..12c79406 100644 --- a/internal/api/http/v1/account.go +++ b/internal/api/http/v1/account.go @@ -22,6 +22,7 @@ import ( "go-sms/internal/pkg/response" "go-sms/internal/repository/mysql" "go-sms/internal/service/account" + "go-sms/internal/service/archive" "go-sms/internal/service/audit" "go-sms/internal/service/auth" "go-sms/internal/service/callback" @@ -72,6 +73,9 @@ type Deps struct { // Callback S4·C1:回调任务管理(死信查询/重派;PushWorker 在 consumer // 进程,gateway 侧仅管理面;nil=不注册)。 Callback *callback.Service + // Archive S4·D3:明细冷热归档(/record/history 冷查走 ClickHouse; + // nil=不注册 history 端点,前端 404 优于半可用)。 + Archive *archive.Service } // Register 注册 v1 路由组。 @@ -190,7 +194,7 @@ func Register(rg *gin.RouterGroup, deps Deps) { // Day 60(D24-25):record 域查询端点(批次/明细/拦截 CSV/通道日志/明细 v2; // 裸 DB 查询,无服务层;查询类不挂 Casbin,JWT 保护即可)。 if deps.DB != nil { - registerRecordRoutes(protected, deps.DB) + registerRecordRoutes(protected, deps.DB, deps.Archive) } // Day68:运营总览(/home)真实数据——summary 四指标卡+trend 近 N 日趋势 // (查询类,JWT 保护即可;口径/角色范围见 stats_handler.go 头注)。 diff --git a/internal/api/http/v1/record_handler.go b/internal/api/http/v1/record_handler.go index 2f4957ff..b89c6a77 100644 --- a/internal/api/http/v1/record_handler.go +++ b/internal/api/http/v1/record_handler.go @@ -19,6 +19,7 @@ import ( "go-sms/internal/model/entity" "go-sms/internal/pkg/errors" "go-sms/internal/pkg/response" + "go-sms/internal/service/archive" ) // fmtTime JSON 时间("2006-01-02 15:04:05",与 mock/前端 ts() 解析口径一致; @@ -54,16 +55,23 @@ func qTime(c *gin.Context, key string) (*time.Time, error) { } // registerRecordRoutes 注册 record 域端点(查询类,JWT 保护即可,不挂 Casbin)。 -func registerRecordRoutes(protected *gin.RouterGroup, db *gorm.DB) { - h := &recordHandler{db: db} +func registerRecordRoutes(protected *gin.RouterGroup, db *gorm.DB, arc *archive.Service) { + h := &recordHandler{db: db, arc: arc} protected.GET("/record/down", h.downList) protected.GET("/record/down/:id/intercepted", h.downIntercepted) protected.GET("/record/down/details", h.downDetails) protected.GET("/record/down/details/:id/channel-logs", h.detailChannelLogs) protected.GET("/sms/downlink-details", h.downlinkDetails) + // S4·D3:历史明细冷查(CH 归档域;契约 web/src/api/record.ts D26)。 + if arc != nil { + protected.GET("/record/history", h.history) + } } -type recordHandler struct{ db *gorm.DB } +type recordHandler struct { + db *gorm.DB + arc *archive.Service // S4·D3 冷查(nil=不注册 history 端点) +} // ---- D24:下行批次列表(GET /record/down) ---- @@ -650,3 +658,97 @@ func (h *recordHandler) downlinkDetails(c *gin.Context) { } response.Success(c, gin.H{"list": list, "total": total}) } + +// ---- S4·D3:历史明细冷查(GET /record/history;CH 归档域) ---- + +// historyRecordDTO 冷查行(契约 record.ts HistoryRecord)。 +type historyRecordDTO struct { + ID int64 `json:"id"` + TaskID int64 `json:"task_id"` + TaskNo string `json:"task_no"` + Mobile string `json:"mobile"` + Province string `json:"province"` + Carrier string `json:"carrier"` + Status int8 `json:"status"` + ErrorCode string `json:"error_code"` + Fee float64 `json:"fee"` + CreatedAt fmtTime `json:"created_at"` +} + +// history GET /record/history?page=&page_size=&mobile=&task_no=&status=&start_time=&end_time= +// 冷数据走 ClickHouse(分区裁剪,验收=冷查<1s);task_no 筛选/回填经 MySQL +// message_task 转译(归档表仅存 task_id)。 +func (h *recordHandler) history(c *gin.Context) { + if h.arc == nil { + response.Error(c, errors.NotFound.WithMsg("冷查未配置 ClickHouse")) + return + } + q := archive.HistoryQuery{ + Page: intQuery(c, "page", 1), + PageSize: intQuery(c, "page_size", 50), + Mobile: strings.TrimSpace(c.Query("mobile")), + } + if q.PageSize > downlinkPageSizeMax { + q.PageSize = downlinkPageSizeMax + } + if v := intQuery(c, "status", -1); v >= 0 { + st := int8(v) + q.Status = &st + } + lo, err := qTime(c, "start_time") + if err != nil { + response.ErrorWithMsg(c, errors.InvalidParam, "start_time "+err.Error()) + return + } + q.Start = lo + hi, err := qTime(c, "end_time") + if err != nil { + response.ErrorWithMsg(c, errors.InvalidParam, "end_time "+err.Error()) + return + } + q.End = hi + // task_no → task_id 转译(MySQL 反查;归档行回填同理)。 + if v := strings.TrimSpace(c.Query("task_no")); v != "" { + var taskID int64 + if err := h.db.WithContext(c.Request.Context()).Table("message_task"). + Select("id").Where("task_no = ?", v).Scan(&taskID).Error; err != nil || taskID == 0 { + response.Success(c, gin.H{"list": []historyRecordDTO{}, "total": 0}) + return + } + q.ChannelID = 0 + q.AccountID = 0 + q.TaskID = taskID + } + rows, total, err := h.arc.QueryHistory(c.Request.Context(), q) + if err != nil { + bizResp(c, err) + return + } + // 批量回填 task_no(page_size≤1000,IN 单查)。 + taskNos := map[int64]string{} + if len(rows) > 0 { + ids := make([]int64, 0, len(rows)) + for _, r := range rows { + ids = append(ids, r.TaskID) + } + var tr []struct { + ID int64 + TaskNo string + } + _ = h.db.WithContext(c.Request.Context()).Table("message_task"). + Select("id, task_no").Where("id IN ?", ids).Scan(&tr).Error + for _, t := range tr { + taskNos[t.ID] = t.TaskNo + } + } + list := make([]historyRecordDTO, 0, len(rows)) + for _, r := range rows { + list = append(list, historyRecordDTO{ + ID: r.ID, TaskID: r.TaskID, TaskNo: taskNos[r.TaskID], + Mobile: r.Mobile, Province: r.Province, Carrier: r.Carrier, + Status: r.Status, ErrorCode: r.ErrorCode, Fee: r.Fee, + CreatedAt: fmtTime(r.CreatedAt), + }) + } + response.Success(c, gin.H{"list": list, "total": total}) +} diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index cc9be204..e4042fa0 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -90,6 +90,12 @@ var ( CallbackPushTotal *prometheus.CounterVec // CallbackBlacklistTotal 上行自动拉黑新增数(S4·C2;幂等冲突不计)。 CallbackBlacklistTotal prometheus.Counter + + // ArchiveRows 归档迁出 MySQL→CH 的明细行数(单调累计,S4·D3)。 + ArchiveRows prometheus.Counter + // ArchiveErrors 归档链路失败计数(校验不匹配/CH/MySQL 故障;下轮自愈, + // 持续增长=真故障信号)。 + ArchiveErrors prometheus.Counter ) var registry = prometheus.NewRegistry() @@ -206,6 +212,16 @@ func init() { Help: "上行自动拉黑新增数(幂等冲突不计,S4·C2)", }) + ArchiveRows = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "go_sms_archive_rows_total", + Help: "明细归档迁出 MySQL→ClickHouse 行数(按日校验通过后计,S4·D3)", + }) + + ArchiveErrors = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "go_sms_archive_errors_total", + Help: "归档链路失败计数(下轮自愈,持续增长=真故障,S4·D3)", + }) + registry.MustRegister( HTTPRequestsTotal, HTTPRequestDuration, HTTPRequestsInFlight, SMSSubmitTotal, SMSReportTotal, KafkaProducerMessages, @@ -216,6 +232,7 @@ func init() { ReportPendingTotal, ReportRecheckTotal, ReportRequeueTotal, SettleAggregateRows, SettleAggregateErrors, CallbackPushTotal, CallbackBlacklistTotal, + ArchiveRows, ArchiveErrors, ) // Go 运行时与进程指标 registry.MustRegister( diff --git a/internal/repository/clickhouse/clickhouse.go b/internal/repository/clickhouse/clickhouse.go index 6d757840..00c06b18 100644 --- a/internal/repository/clickhouse/clickhouse.go +++ b/internal/repository/clickhouse/clickhouse.go @@ -25,6 +25,17 @@ type Conn interface { // BatchInsert 批量插入:INSERT INTO table (columns...) VALUES 每行一行。 // 空行返回 nil;类型由调用方保证与表结构一致。 BatchInsert(ctx context.Context, table string, columns []string, rows [][]any) error + // Query 只读查询(S4·D3 冷查;游标语义 Next/Scan/Close,args 走原生 bind)。 + Query(ctx context.Context, query string, args ...any) (Rows, error) + // Exec DDL/管理语句(建表/DROP PARTITION;args 走原生 bind)。 + Exec(ctx context.Context, query string, args ...any) error + Close() error +} + +// Rows 查询游标抽象(驱动行集满足该接口即可)。 +type Rows interface { + Next() bool + Scan(dest ...any) error Close() error } @@ -55,6 +66,23 @@ func (c *nativeConn) Ping(ctx context.Context) error { return c.conn.Ping(ctx) } func (c *nativeConn) Close() error { return c.conn.Close() } +// Query 只读查询透传(*ch.Rows 实现 Rows 接口)。 +func (c *nativeConn) Query(ctx context.Context, query string, args ...any) (Rows, error) { + rows, err := c.conn.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("clickhouse: 查询失败: %w", err) + } + return rows, nil +} + +// Exec DDL/管理语句透传。 +func (c *nativeConn) Exec(ctx context.Context, query string, args ...any) error { + if err := c.conn.Exec(ctx, query, args...); err != nil { + return fmt.Errorf("clickhouse: 执行失败: %w", err) + } + return nil +} + func (c *nativeConn) BatchInsert(ctx context.Context, table string, columns []string, rows [][]any) error { if table == "" || len(columns) == 0 { return fmt.Errorf("clickhouse: table/columns 不能为空") diff --git a/internal/repository/mysql/base_test.go b/internal/repository/mysql/base_test.go index 1577877d..da2d23ba 100644 --- a/internal/repository/mysql/base_test.go +++ b/internal/repository/mysql/base_test.go @@ -29,8 +29,8 @@ func TestMigrations_IdempotentAndVersioned(t *testing.T) { seen[mig.ID] = true last = mig } - if last == nil || last.ID != "v21_callback_domain" { - t.Fatalf("终版迁移应为 v21_callback_domain,实际 %v", last) + if last == nil || last.ID != "v22_detail_partitions" { + t.Fatalf("终版迁移应为 v22_detail_partitions,实际 %v", last) } db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) @@ -42,33 +42,28 @@ func TestMigrations_IdempotentAndVersioned(t *testing.T) { } opts := gormigrate.DefaultOptions opts.TableName = "schema_migrations" - // 注册末七迁移(现 v15..v21;历史 v10/V11 含 MessageTask AutoMigrate,gleberez + // 注册末七迁移(现 v16..v22;历史 v10/V11 含 MessageTask AutoMigrate,gleberez // sqlite 索引不查重,双块连跑撞 idx_account_time——同 v1+v6 已知方言缺口; - // v15 AutoMigrate 加列、v16/v17/v20 空库守卫补建表后加列/索引、 + // v15 AutoMigrate 加列、v16/v17/v20 空库守卫补建表后加列/索引、v22 分区 MySQL 守卫(sqlite 跳过)、 // v18/v19/v21 新表 AutoMigrate,v20 另含 DROP IF EXISTS 与 casbin HasTable 守卫, // 单块两遍覆盖幂等)。 migs := Migrations() gm := gormigrate.New(db.Session(&gorm.Session{Logger: logger.Discard}), opts, migs[len(migs)-7:]) if err := gm.Migrate(); err != nil { - t.Fatalf("v15..v21 首遍失败: %v", err) + t.Fatalf("v16..v22 首遍失败: %v", err) } if err := gm.Migrate(); err != nil { - t.Fatalf("v15..v21 二遍不幂等: %v", err) + t.Fatalf("v16..v22 二遍不幂等: %v", err) } v, err := CurrentMigrationVersion(db) if err != nil { t.Fatalf("查版本失败: %v", err) } - if v == nil || *v != "v21_callback_domain" { - t.Fatalf("版本应为 v21_callback_domain,实际 %v", v) - } - // (v13/v14 断言随窗口右移移除:各表由 v13/v14 建, - // 现窗口 v15..v21 不含;全量链路覆盖仍由 dev MySQL 实库验证。) - // Day 62:v15 后 account_config 应有账号级 CMPP 配额两列。 - if !db.Migrator().HasColumn(&entity.AccountConfig{}, "CmppMaxConns") || - !db.Migrator().HasColumn(&entity.AccountConfig{}, "CmppSpeed") { - t.Fatal("v15 后应存在 cmpp_max_conns/cmpp_speed 列") + if v == nil || *v != "v22_detail_partitions" { + t.Fatalf("版本应为 v22_detail_partitions,实际 %v", v) } + // (v13/v14/v15 断言随窗口右移移除:各表/列由 v13~v15 建, + // 现窗口 v16..v22 不含;全量链路覆盖仍由 dev MySQL 实库验证。) // Day 64:v16 后 message_detail 应有明细查询两索引(空库守卫路径建表+显式加索引)。 if !db.Migrator().HasIndex(&entity.MessageDetail{}, "idx_detail_created") || !db.Migrator().HasIndex(&entity.MessageDetail{}, "idx_detail_channel_created") { diff --git a/internal/repository/mysql/migrations.go b/internal/repository/mysql/migrations.go index 8272ba6a..adb02cbc 100644 --- a/internal/repository/mysql/migrations.go +++ b/internal/repository/mysql/migrations.go @@ -2,6 +2,8 @@ package mysql import ( "fmt" + "strings" + "time" "gorm.io/gorm" "gorm.io/gorm/logger" @@ -10,6 +12,7 @@ import ( "go-sms/internal/model/entity" errscodes "go-sms/internal/pkg/errors" + "go-sms/internal/util" ) // Migrations 迁移清单(顺序执行,ID 单调递增;禁止修改已发布 ID)。 @@ -503,6 +506,45 @@ func Migrations() []*gormigrate.Migration { return nil // 新表可手工清除,回滚为空操作(v16~v20 惯例) }, }, + // v22(S4·D3)明细按天分区:RANGE(id)(雪花高位=时间戳,当天零点边界 + // 可精确计算;主键即分区键无需改造唯一键)。分区集=热 5 天+未来 3 天 + // 缓冲+pmax 兜底;滚动维护与过期 DROP 由 archive.Service 承担(归档校验 + // 通过才删,见 internal/service/archive/partition.go)。MySQL only—— + // sqlite 单测无分区(物理优化,方言守卫跳过)。 + { + ID: "v22_detail_partitions", + Migrate: func(tx *gorm.DB) error { + if tx.Name() != "mysql" { + return nil + } + var n int64 + if err := tx.Raw( + "SELECT COUNT(*) FROM information_schema.PARTITIONS " + + "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'message_detail' " + + "AND PARTITION_NAME IS NOT NULL"). // 非分区表也占一行(NULL),须排除 + Scan(&n).Error; err != nil { + return err + } + if n > 0 { + return nil // 已分区(链路重放/EnsurePartitions 已先行) + } + now := time.Now() + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + var parts []string + for i := -4; i <= 3; i++ { // 热 5 天(今日-4..今日)+未来 3 天 + day := today.AddDate(0, 0, i) + boundary := (day.AddDate(0, 0, 1).UnixMilli() - util.Epoch) << (util.WorkerIDBits + util.SequenceBits) + parts = append(parts, fmt.Sprintf("PARTITION p%s VALUES LESS THAN (%d)", + day.Format("20060102"), boundary)) + } + parts = append(parts, "PARTITION pmax VALUES LESS THAN MAXVALUE") + return tx.Exec("ALTER TABLE message_detail PARTITION BY RANGE (id) (" + + strings.Join(parts, ", ") + ")").Error + }, + Rollback: func(tx *gorm.DB) error { + return nil // REMOVE PARTITIONING 可手工执行,回滚为空操作(v16~v21 惯例) + }, + }, } } diff --git a/internal/service/archive/archive_test.go b/internal/service/archive/archive_test.go new file mode 100644 index 00000000..02d52486 --- /dev/null +++ b/internal/service/archive/archive_test.go @@ -0,0 +1,180 @@ +// archive_test.go S4·D3:分区边界纯函数+归档全链(sqlite+mock CH)+冷查条件。 +package archive + +import ( + "context" + "strings" + "testing" + "time" + + "go-sms/internal/model/entity" + "go-sms/internal/repository/clickhouse" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func newArchiveDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Discard}) + if err != nil { + t.Fatalf("打开内存库失败: %v", err) + } + if err := db.AutoMigrate(&entity.MessageDetail{}); err != nil { + t.Fatalf("AutoMigrate 失败: %v", err) + } + return db +} + +// mockRows 查询游标打桩(仅支撑 count() 单列场景)。 +type mockRows struct { + vals []int64 + idx int +} + +func (r *mockRows) Next() bool { r.idx++; return r.idx-1 < len(r.vals) } +func (r *mockRows) Scan(dest ...any) error { + if r.idx-1 < len(r.vals) { + v := r.vals[r.idx-1] + switch p := dest[0].(type) { + case *int64: + *p = v + case *uint64: // CH count() UInt64 口径 + *p = uint64(v) + } + } + return nil +} +func (r *mockRows) Close() error { return nil } + +// mockCH ClickHouse Conn 打桩:记录写入/DDL;count 应答按调用序注入。 +type mockCH struct { + inserted [][]any + execs []string + countReplies []int64 +} + +func (m *mockCH) Ping(context.Context) error { return nil } +func (m *mockCH) Close() error { return nil } +func (m *mockCH) Exec(_ context.Context, q string, _ ...any) error { + m.execs = append(m.execs, q) + return nil +} +func (m *mockCH) BatchInsert(_ context.Context, _ string, _ []string, rows [][]any) error { + m.inserted = append(m.inserted, rows...) + return nil +} +func (m *mockCH) Query(_ context.Context, q string, _ ...any) (clickhouse.Rows, error) { + if strings.Contains(q, "system.parts") { + return &mockRows{vals: []int64{1}}, nil // dropCHPartition 探测:分区存在 + } + v := int64(0) + if strings.HasPrefix(q, "SELECT count()") && len(m.countReplies) > 0 { + v = m.countReplies[0] + m.countReplies = m.countReplies[1:] + } + return &mockRows{vals: []int64{v}}, nil +} + +var _ clickhouse.Conn = (*mockCH)(nil) + +// TestDayFloorSnowflakeMonotonic 边界单调+PartName/HotFloor 口径。 +func TestDayFloorSnowflakeMonotonic(t *testing.T) { + base := time.Date(2026, 11, 16, 15, 30, 0, 0, time.Local) + if PartName(base) != "p20261116" { + t.Fatalf("PartName=%s 期望 p20261116", PartName(base)) + } + prev := DayFloorSnowflake(base) + for i := 1; i <= 3; i++ { + next := DayFloorSnowflake(base.AddDate(0, 0, i)) + if next <= prev { + t.Fatalf("边界非单调: day+%d", i) + } + prev = next + } + if got := HotFloor(base); !got.Equal(time.Date(2026, 11, 12, 0, 0, 0, 0, time.Local)) { + t.Fatalf("HotFloor=%v 期望 2026-11-12 零点", got) + } +} + +// TestArchiveOnceFullCycle 归档全链:7 天前 2 行迁出(CH 收到+MySQL 清退, +// sqlite 无分区=DELETE 兜底分支),今天行保留;再跑幂等零动作。 +func TestArchiveOnceFullCycle(t *testing.T) { + db := newArchiveDB(t) + old := time.Now().AddDate(0, 0, -7) + mk := func(id int64, at time.Time, mobile string) { + d := entity.MessageDetail{TaskID: 1, AccountID: 5, ChannelID: 7, + Mobile: mobile, Status: 2, Fee: 1} + d.ID = id + d.CreatedAt = at + if err := db.Create(&d).Error; err != nil { + t.Fatal(err) + } + } + mk(73021000001, old, "13800001111") + mk(73021000002, old.Add(time.Hour), "13800002222") + mk(73021000003, time.Now(), "13800003333") // 热窗内:不迁 + + ch := &mockCH{countReplies: []int64{2}} + s := NewService(db, ch) + if done := s.ArchiveOnce(context.Background()); done != 1 { + t.Fatalf("应归档 1 天, got %d", done) + } + if len(ch.inserted) != 2 { + t.Fatalf("CH 应收到 2 行, got %d", len(ch.inserted)) + } + var remainOld, remainNew int64 + db.Model(&entity.MessageDetail{}).Where("mobile = ?", "13800001111").Count(&remainOld) + db.Model(&entity.MessageDetail{}).Where("mobile = ?", "13800003333").Count(&remainNew) + if remainOld != 0 || remainNew != 1 { + t.Fatalf("清退异常: old=%d new=%d 期望 0/1", remainOld, remainNew) + } + if done := s.ArchiveOnce(context.Background()); done != 0 { + t.Fatalf("再跑应 0 天(幂等), got %d", done) + } +} + +// TestArchiveVerifyMismatchKeepsMySQL 校验不匹配:CH count≠MySQL count → +// MySQL 保留(零丢失方向),不计完成。 +func TestArchiveVerifyMismatchKeepsMySQL(t *testing.T) { + db := newArchiveDB(t) + old := time.Now().AddDate(0, 0, -7) + d := entity.MessageDetail{TaskID: 1, AccountID: 5, Mobile: "13800009999", Status: 2} + d.ID = 73021000009 + d.CreatedAt = old + if err := db.Create(&d).Error; err != nil { + t.Fatal(err) + } + ch := &mockCH{countReplies: []int64{0}} + s := NewService(db, ch) + if done := s.ArchiveOnce(context.Background()); done != 0 { + t.Fatalf("校验不匹配不应计完成, got %d", done) + } + var n int64 + db.Model(&entity.MessageDetail{}).Count(&n) + if n != 1 { + t.Fatalf("MySQL 应保留(零丢失方向), got %d", n) + } +} + +// TestQueryHistoryWhere 冷查条件构造(六筛选→where 子句与参数序)。 +func TestQueryHistoryWhere(t *testing.T) { + st := int8(2) + lo := time.Date(2026, 11, 1, 0, 0, 0, 0, time.Local) + hi := lo.AddDate(0, 0, 5) + w, args := historyWhere(HistoryQuery{ + AccountID: 5, Mobile: "138", Status: &st, Start: &lo, End: &hi, TaskID: 9, + }) + for _, want := range []string{ + "account_id = ?", "mobile = ?", "status = ?", + "created_at >= ?", "created_at < ?", "task_id = ?", + } { + if !strings.Contains(w, want) { + t.Fatalf("where 缺 %q: %s", want, w) + } + } + if len(args) != 6 { + t.Fatalf("参数数 %d 期望 6", len(args)) + } +} diff --git a/internal/service/archive/job.go b/internal/service/archive/job.go new file mode 100644 index 00000000..dacb8f1b --- /dev/null +++ b/internal/service/archive/job.go @@ -0,0 +1,203 @@ +// job.go S4·D3:归档 job——热窗外明细逐日迁出 MySQL→ClickHouse,校验通过才 +// DROP MySQL 分区(数据零丢失方向:先 CH 后 MySQL;CH 侧重写=DROP PARTITION +// +INSERT 当日全量,幂等可重入)。周期 1h(启动先 EnsureSchema+Partitions)。 +package archive + +import ( + "context" + "time" + + "go-sms/internal/model/entity" + "go-sms/internal/pkg/logger" + "go-sms/internal/pkg/metrics" + "go-sms/internal/repository/clickhouse" + + "go.uber.org/zap" + "gorm.io/gorm" +) + +// 默认参数(SSD 基线保守起步)。 +const ( + DefaultArchiveEvery = time.Hour // 归档周期 + archiveBatch = 2000 // MySQL 分批读取行数 + archiveMaxDays = 62 // 单轮最多处理天数(两月保护,防首轮长事务风暴) +) + +// Service 归档服务(db=MySQL;ch 可 nil=禁用归档链路,仅分区维护可用)。 +type Service struct { + db *gorm.DB + ch clickhouse.Conn +} + +// NewService 构造。 +func NewService(db *gorm.DB, ch clickhouse.Conn) *Service { return &Service{db: db, ch: ch} } + +// Run 常驻循环(caller 起 goroutine;每轮 Ensure→归档;失败 Warn 下轮自愈)。 +func (s *Service) Run(ctx context.Context, every time.Duration) { + if every <= 0 { + every = DefaultArchiveEvery + } + t := time.NewTicker(every) + defer t.Stop() + s.EnsureOnce(ctx) // 幂等前置:CH 建表+分区维护 + s.ArchiveOnce(ctx) // 启动先归档一轮(重启即清退热窗外积压) + for { + select { + case <-ctx.Done(): + return + case <-t.C: + s.ArchiveOnce(ctx) + } + } +} + +// EnsureOnce 幂等前置(CH 建表+分区维护)。 +func (s *Service) EnsureOnce(ctx context.Context) { + if err := s.EnsureCHSchema(ctx); err != nil { + logger.L().Warn("archive_ch_schema_failed", zap.Error(err)) + } + if _, err := s.EnsurePartitions(ctx); err != nil { + logger.L().Warn("archive_partition_ensure_failed", zap.Error(err)) + } +} + +// ArchiveOnce 单轮归档:找热窗外 distinct 日期→逐日迁出。 +func (s *Service) ArchiveOnce(ctx context.Context) int { + if s.ch == nil { + return 0 // CH 未装配:链路禁用(分区维护仍由 EnsureOnce 承担) + } + floor := HotFloor(time.Now()) + // DISTINCT 时刻级 created_at(MySQL/glebarez 双方言直 Scan time.Time; + // DATE() 在 sqlite 返回 TEXT 难以统一转换——D3 真机+单测双教训), + // Go 侧归一化当日零点去重。 + var times []time.Time + if err := s.db.WithContext(ctx).Raw( + "SELECT DISTINCT created_at FROM message_detail "+ + "WHERE created_at < ? ORDER BY created_at LIMIT ?", floor, archiveMaxDays*4). + Scan(×).Error; err != nil { + logger.L().Warn("archive_scan_days_failed", zap.Error(err)) + metrics.ArchiveErrors.Inc() + return 0 + } + seen := map[string]bool{} + var dayTimes []time.Time + for _, t := range times { + if t.IsZero() { + continue + } + day := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) + key := day.Format("2006-01-02") + if !seen[key] { + seen[key] = true + dayTimes = append(dayTimes, day) + } + } + done := 0 + for _, day := range dayTimes { + logger.L().Info("archive_day_begin", zap.String("day", day.Format("2006-01-02"))) + if s.archiveDay(ctx, day) { + done++ + } + } + if len(dayTimes) > 0 { + logger.L().Info("archive_round_done", + zap.Int("pending_days", len(dayTimes)), zap.Int("archived_days", done)) + } + return done +} + +// archiveDay 单日归档(MySQL count → CH 重写 → CH count 校验 → DROP 分区/兜底 DELETE)。 +func (s *Service) archiveDay(ctx context.Context, day time.Time) bool { + lo := day + hi := day.AddDate(0, 0, 1) + var mysqlCount int64 + if err := s.db.WithContext(ctx).Table("message_detail"). + Where("created_at >= ? AND created_at < ?", lo, hi).Count(&mysqlCount).Error; err != nil { + logger.L().Warn("archive_mysql_count_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err)) + metrics.ArchiveErrors.Inc() + return false + } + if mysqlCount == 0 { + // 无数据但分区存在(历史残留):直接清分区。 + if dropped, err := s.DropPartition(ctx, day); err == nil && dropped { + logger.L().Info("archive_empty_partition_dropped", zap.String("day", day.Format("2006-01-02"))) + } + return false + } + logger.L().Info("archive_day_counted", zap.String("day", day.Format("2006-01-02")), zap.Int64("rows", mysqlCount)) + // CH 侧重写(幂等):先 DROP 当日分区,再分批 INSERT 当日全量。 + if err := s.dropCHPartition(ctx, day); err != nil { + logger.L().Warn("archive_ch_drop_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err)) + metrics.ArchiveErrors.Inc() + return false + } + var lastID int64 + batches := 0 + for { + var rows []entity.MessageDetail + if err := s.db.WithContext(ctx). + Where("created_at >= ? AND created_at < ? AND id > ?", lo, hi, lastID). + Order("id ASC").Limit(archiveBatch).Find(&rows).Error; err != nil { + logger.L().Warn("archive_mysql_read_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err)) + metrics.ArchiveErrors.Inc() + return false + } + if len(rows) == 0 { + break + } + if err := s.ch.BatchInsert(ctx, ArchiveTable, archiveColumns, detailToRows(rows)); err != nil { + logger.L().Warn("archive_ch_insert_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err)) + metrics.ArchiveErrors.Inc() + return false + } + lastID = rows[len(rows)-1].ID + batches++ + if len(rows) < archiveBatch { + break + } + } + // 校验:CH count == MySQL count 才清 MySQL(不等=保留重试,零丢失方向)。 + chCount, err := s.chCountDay(ctx, day) + if err != nil || chCount != mysqlCount { + logger.L().Warn("archive_verify_mismatch_keep_mysql", + zap.String("day", day.Format("2006-01-02")), + zap.Int64("mysql", mysqlCount), zap.Int64("ch", chCount), zap.Error(err)) + metrics.ArchiveErrors.Inc() + return false + } + dropped, err := s.DropPartition(ctx, day) + if err != nil { + logger.L().Warn("archive_mysql_drop_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err)) + metrics.ArchiveErrors.Inc() + return false + } + if !dropped { + // 非分区残留(历史数据无对应分区):DELETE 兜底(限当日窗)。 + if err := s.db.WithContext(ctx). + Where("created_at >= ? AND created_at < ?", lo, hi). + Delete(&entity.MessageDetail{}).Error; err != nil { + logger.L().Warn("archive_mysql_delete_fallback_failed", zap.String("day", day.Format("2006-01-02")), zap.Error(err)) + metrics.ArchiveErrors.Inc() + return false + } + } + metrics.ArchiveRows.Add(float64(mysqlCount)) + logger.L().Info("archive_day_done", + zap.String("day", day.Format("2006-01-02")), zap.Int64("rows", mysqlCount), + zap.Int("batches", batches), zap.Bool("partition_dropped", dropped)) + return true +} + +// detailToRows 实体→CH 行(列序=archiveColumns)。 +func detailToRows(rows []entity.MessageDetail) [][]any { + out := make([][]any, 0, len(rows)) + for i := range rows { + d := &rows[i] + out = append(out, []any{ + d.ID, d.CreatedAt, d.TaskID, d.AccountID, d.ChannelID, d.Mobile, + d.Content, d.Province, d.Carrier, d.Status, d.ErrorCode, d.ExtCode, + d.CallData, d.Fee, d.SendAt, d.ReportAt, d.Business, d.Source, + }) + } + return out +} diff --git a/internal/service/archive/partition.go b/internal/service/archive/partition.go new file mode 100644 index 00000000..803a4b9a --- /dev/null +++ b/internal/service/archive/partition.go @@ -0,0 +1,149 @@ +// Package archive S4·D3:明细热 5 天(MySQL 按天 RANGE(id) 分区)+ 冷数据 +// ClickHouse 归档。分区键=雪花 id(时间位单调,RANGE(id) 边界=当天零点雪花 +// 值,无需改主键);分区集=热 5 天(今日-4..今日)+ 未来 3 天缓冲 + pmax 兜底 +// (Manager 挂掉数据仍可写,恢复后 REORGANIZE 拆分——运维口径)。过期分区 +// DROP 只发生在归档 job 校验通过后(先 CH 后 MySQL,数据零丢失方向)。 +package archive + +import ( + "context" + "fmt" + "strings" + "time" + + "go-sms/internal/pkg/logger" + "go-sms/internal/util" + + "go.uber.org/zap" + "gorm.io/gorm" +) + +// HotDays 热窗天数(含今日;需求 2.5「近 5 天热数据」)。 +const HotDays = 5 + +// BufferDays 未来预建分区缓冲天数(写入永不停摆的余量)。 +const BufferDays = 3 + +// partPrefix MySQL 分区名前缀(pYYYYMMDD)。 +const partPrefix = "p" + +// DayFloorSnowflake 某日 00:00(本地时区)对应的雪花 ID 边界值: +// id 高 41 位=相对 Epoch 毫秒 → 边界 = (dayStartMilli - Epoch) << 22。 +func DayFloorSnowflake(day time.Time) int64 { + dayStart := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, day.Location()) + return (dayStart.UnixMilli() - util.Epoch) << (util.WorkerIDBits + util.SequenceBits) +} + +// PartName 分区名(本地时区日期)。 +func PartName(day time.Time) string { return partPrefix + day.Format("20060102") } + +// hotRange 当前应存在的按天分区日期集(今日-4 .. 今日+3)。 +func hotRange(now time.Time) []time.Time { + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + out := make([]time.Time, 0, HotDays-1+BufferDays+1) + for i := -(HotDays - 1); i <= BufferDays; i++ { + out = append(out, today.AddDate(0, 0, i)) + } + return out +} + +// existingParts 查询 message_detail 现有分区(名称→边界描述)。 +// 空 map+isPartitioned=false 表示表未分区。 +func existingParts(ctx context.Context, db *gorm.DB) (map[string]string, bool, error) { + var rows []struct { + Name string + Desc string + } + err := db.WithContext(ctx).Raw( + "SELECT PARTITION_NAME AS name, PARTITION_DESCRIPTION AS desc_ FROM information_schema.PARTITIONS " + + "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'message_detail' AND PARTITION_NAME IS NOT NULL"). + Scan(&rows).Error + if err != nil { + return nil, false, err + } + out := map[string]string{} + for _, r := range rows { + if r.Name == "" { + continue // NULL 行(非分区表在 MySQL 8.0 也占一行;方言护栏双保险) + } + out[r.Name] = r.Desc + } + return out, len(out) > 0, nil // 存在非空分区名=已分区 +} + +// partDefs 按天分区子句列表(每天一个,VALUES LESS THAN 次日边界)。 +func partDefs(days []time.Time) []string { + out := make([]string, 0, len(days)+1) + for _, d := range days { + out = append(out, fmt.Sprintf("PARTITION %s VALUES LESS THAN (%d)", + PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1)))) + } + return out +} + +// EnsurePartitions 幂等维护(MySQL only;sqlite 单测跳过):未分区→一次性 +// RANGE(id) 建(热集+pmax);已分区→补缺失的按天分区(过期分区不在此删, +// 归档 job 校验后 DROP)。每轮返回是否发生了 DDL(观测用)。 +func (s *Service) EnsurePartitions(ctx context.Context) (bool, error) { + if s.db.Name() != "mysql" { + return false, nil // sqlite 内存库无分区(物理优化,MySQL 语义等价) + } + existing, partitioned, err := existingParts(ctx, s.db) + if err != nil { + return false, err + } + days := hotRange(time.Now()) + if !partitioned { + ddl := "ALTER TABLE message_detail PARTITION BY RANGE (id) (" + + strings.Join(partDefs(days), ", ") + ", PARTITION pmax VALUES LESS THAN MAXVALUE)" + if err := s.db.WithContext(ctx).Exec(ddl).Error; err != nil { + return true, fmt.Errorf("archive: 首次分区失败: %w", err) + } + logger.L().Info("archive_partition_initialized", zap.Int("parts", len(days)+1)) + return true, nil + } + changed := false + for _, d := range days { + if _, ok := existing[PartName(d)]; ok { + continue + } + ddl := fmt.Sprintf("ALTER TABLE message_detail ADD PARTITION (%s VALUES LESS THAN (%d))", + PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1))) + // pmax 已存在时不能 ADD(区间必须单调追加)——pmax 场景走 REORGANIZE 拆分。 + if _, hasMax := existing["pmax"]; hasMax { + ddl = fmt.Sprintf("ALTER TABLE message_detail REORGANIZE PARTITION pmax INTO (%s VALUES LESS THAN (%d), PARTITION pmax VALUES LESS THAN MAXVALUE)", + PartName(d), DayFloorSnowflake(d.AddDate(0, 0, 1))) + } + if err := s.db.WithContext(ctx).Exec(ddl).Error; err != nil { + return changed, fmt.Errorf("archive: 补分区 %s 失败: %w", PartName(d), err) + } + existing[PartName(d)] = "" + changed = true + } + return changed, nil +} + +// DropPartition 删除指定日分区(归档校验后调用;分区不存在返回 false 不报错)。 +func (s *Service) DropPartition(ctx context.Context, day time.Time) (bool, error) { + if s.db.Name() != "mysql" { + return false, nil + } + existing, _, err := existingParts(ctx, s.db) + if err != nil { + return false, err + } + if _, ok := existing[PartName(day)]; !ok { + return false, nil + } + if err := s.db.WithContext(ctx). + Exec("ALTER TABLE message_detail DROP PARTITION " + PartName(day)).Error; err != nil { + return false, err + } + return true, nil +} + +// HotFloor 热窗下限(今日-4 零点;此前明细属归档对象)。 +func HotFloor(now time.Time) time.Time { + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + return today.AddDate(0, 0, -(HotDays - 1)) +} diff --git a/internal/service/archive/schema.go b/internal/service/archive/schema.go new file mode 100644 index 00000000..f6be73e9 --- /dev/null +++ b/internal/service/archive/schema.go @@ -0,0 +1,242 @@ +// schema.go S4·D3:ClickHouse 归档表 DDL(幂等)+ 冷查行扫描。 +package archive + +import ( + "context" + "fmt" + "time" + + "go-sms/internal/repository/clickhouse" +) + +// ArchiveTable CH 归档表名(字段与 message_detail 对齐;fee Decimal→Float64, +// 可空时间列 Nullable;按天分区 toDate(created_at),ORDER BY id)。 +const ArchiveTable = "sms_detail_archive" + +// archiveColumns 归档列(与 DDL/INSERT/SELECT 三方对齐,顺序稳定)。 +var archiveColumns = []string{ + "id", "created_at", "task_id", "account_id", "channel_id", "mobile", + "content", "province", "carrier", "status", "error_code", "ext_code", + "call_data", "fee", "send_at", "report_at", "business", "source", +} + +// ArchiveTableDDL 建表语句(幂等;PARTITION BY toDate(created_at) 的分区名 +// =YYYYMMDD,与 ArchiveDayPartition 一致,归档重写=先 DROP PARTITION 再 INSERT)。 +const ArchiveTableDDL = ` +CREATE TABLE IF NOT EXISTS sms_detail_archive ( + id Int64, + created_at DateTime64(3), + task_id Int64, + account_id Int64, + channel_id Int64, + mobile String, + content String, + province LowCardinality(String), + carrier LowCardinality(String), + status Int8, + error_code LowCardinality(String), + ext_code LowCardinality(String), + call_data String, + fee Float64, + send_at Nullable(DateTime64(3)), + report_at Nullable(DateTime64(3)), + business LowCardinality(String), + source Int8 +) ENGINE = MergeTree +PARTITION BY toDate(created_at) +ORDER BY id +SETTINGS index_granularity = 8192` + +// EnsureCHSchema 幂等建归档表。 +func (s *Service) EnsureCHSchema(ctx context.Context) error { + if s.ch == nil { + return nil // CH 未装配:归档链路禁用(单测/最小部署) + } + return s.ch.Exec(ctx, ArchiveTableDDL) +} + +// ArchiveDayPartition 某日在 CH 的分区名(toDate 表达式分区名=YYYYMMDD)。 +func ArchiveDayPartition(day time.Time) string { return day.Format("20060102") } + +// dropCHPartition 删除 CH 当日分区(CH 24.3 无 DROP PARTITION IF EXISTS—— +// 先查 system.parts 存在才 DROP;不存在静默通过)。 +func (s *Service) dropCHPartition(ctx context.Context, day time.Time) error { + rows, err := s.ch.Query(ctx, + "SELECT count() FROM system.parts WHERE database = currentDatabase() "+ + "AND table = ? AND partition = ?", ArchiveTable, ArchiveDayPartition(day)) + if err != nil { + return err + } + n := uint64(0) // CH count() 返回 UInt64(Scan *int64 不支持,真机教训) + if rows.Next() { + if err := rows.Scan(&n); err != nil { + _ = rows.Close() + return err + } + } + _ = rows.Close() + if n == 0 { + return nil + } + return s.ch.Exec(ctx, + fmt.Sprintf("ALTER TABLE `%s` DROP PARTITION '%s'", ArchiveTable, ArchiveDayPartition(day))) +} + +// chCountDay CH 当日行数(归档校验锚)。 +func (s *Service) chCountDay(ctx context.Context, day time.Time) (int64, error) { + rows, err := s.ch.Query(ctx, + fmt.Sprintf("SELECT count() FROM `%s` WHERE toDate(created_at) = ?", ArchiveTable), + ArchiveDayPartition(day)) + if err != nil { + return 0, err + } + defer func() { _ = rows.Close() }() + if !rows.Next() { + return 0, fmt.Errorf("archive: CH count 无行返回") + } + var n uint64 // CH count() 返回 UInt64 + if err := rows.Scan(&n); err != nil { + return 0, err + } + return int64(n), nil +} + +// ArchiveRow 冷查明细行(CH 扫描 DTO;时间字段 CH 驱动返回 time.Time)。 +type ArchiveRow struct { + ID int64 `json:"id"` + CreatedAt time.Time `json:"created_at"` + TaskID int64 `json:"task_id"` + AccountID int64 `json:"account_id"` + ChannelID int64 `json:"channel_id"` + Mobile string `json:"mobile"` + Content string `json:"content"` + Province string `json:"province"` + Carrier string `json:"carrier"` + Status int8 `json:"status"` + ErrorCode string `json:"error_code"` + ExtCode string `json:"ext_code"` + CallData string `json:"call_data"` + Fee float64 `json:"fee"` + SendAt *time.Time `json:"send_at"` + ReportAt *time.Time `json:"report_at"` + Business string `json:"business"` + Source int8 `json:"source"` +} + +// QueryHistory 冷查(CH;分页+账号/手机号/时间窗筛选;验收=冷查<1s)。 +// 返回行列表与总数(总数另查 count();行窗 id 排序稳定)。 +func (s *Service) QueryHistory(ctx context.Context, q HistoryQuery) ([]ArchiveRow, int64, error) { + if s.ch == nil { + return nil, 0, fmt.Errorf("archive: ClickHouse 未配置") + } + page, size := q.Page, q.PageSize + if page < 1 { + page = 1 + } + if size < 1 || size > 1000 { + size = 50 + } + where, args := historyWhere(q) + var total int64 + crows, err := s.ch.Query(ctx, + fmt.Sprintf("SELECT count() FROM `%s`%s", ArchiveTable, where), args...) + if err != nil { + return nil, 0, err + } + { // CH count() 返回 UInt64(*int64 Scan 不支持——错误不可吞,D3 真机教训) + var n uint64 + if crows.Next() { + if err := crows.Scan(&n); err != nil { + _ = crows.Close() + return nil, 0, err + } + } + _ = crows.Close() + total = int64(n) + } + if total == 0 { + return []ArchiveRow{}, 0, nil + } + lrows, err := s.ch.Query(ctx, + fmt.Sprintf("SELECT %s FROM `%s`%s ORDER BY id DESC LIMIT %d OFFSET %d", + joinColumns(), ArchiveTable, where, size, (page-1)*size), args...) + if err != nil { + return nil, 0, err + } + defer func() { _ = lrows.Close() }() + out := make([]ArchiveRow, 0, size) + for lrows.Next() { + var r ArchiveRow + if err := lrows.Scan(&r.ID, &r.CreatedAt, &r.TaskID, &r.AccountID, &r.ChannelID, + &r.Mobile, &r.Content, &r.Province, &r.Carrier, &r.Status, &r.ErrorCode, + &r.ExtCode, &r.CallData, &r.Fee, &r.SendAt, &r.ReportAt, &r.Business, &r.Source); err != nil { + return nil, 0, err + } + out = append(out, r) + } + return out, total, nil +} + +// HistoryQuery 冷查入参(全部可选;时间窗起闭止开)。 +type HistoryQuery struct { + AccountID int64 + ChannelID int64 + TaskID int64 + Mobile string + Status *int8 + Start *time.Time + End *time.Time + Page int + PageSize int +} + +func historyWhere(q HistoryQuery) (string, []any) { + conds := []string{"1=1"} + var args []any + if q.AccountID > 0 { + conds = append(conds, "account_id = ?") + args = append(args, q.AccountID) + } + if q.ChannelID > 0 { + conds = append(conds, "channel_id = ?") + args = append(args, q.ChannelID) + } + if q.TaskID > 0 { + conds = append(conds, "task_id = ?") + args = append(args, q.TaskID) + } + if q.Mobile != "" { + conds = append(conds, "mobile = ?") + args = append(args, q.Mobile) + } + if q.Status != nil { + conds = append(conds, "status = ?") + args = append(args, *q.Status) + } + if q.Start != nil { + conds = append(conds, "created_at >= ?") + // CH 绑定 time.Time 存在 UTC 化歧义;归档写入=墙钟语义,查询同口径 + // 预格式化字符串(D3 真机教训)。 + args = append(args, q.Start.Format("2006-01-02 15:04:05")) + } + if q.End != nil { + conds = append(conds, "created_at < ?") + args = append(args, q.End.Format("2006-01-02 15:04:05")) + } + return " WHERE " + joinStrings(conds, " AND "), args +} + +func joinStrings(ss []string, sep string) string { + out := "" + for i, s := range ss { + if i > 0 { + out += sep + } + out += s + } + return out +} + +func joinColumns() string { return joinStrings(archiveColumns, ", ") } + +var _ clickhouse.Conn = (clickhouse.Conn)(nil) // 引用守卫(接口扩展与本包解耦) diff --git a/internal/service/settle/consume.go b/internal/service/settle/consume.go index a46f2080..37e5799f 100644 --- a/internal/service/settle/consume.go +++ b/internal/service/settle/consume.go @@ -23,10 +23,14 @@ import ( ) // Service 结算域服务(D2 消耗日汇总;D3~D5 逐步扩展充值审核/出账/利润/对账)。 -type Service struct{ db *gorm.DB } +type Service struct { + db *gorm.DB + // nowFn 时钟(S4·D3 热窗感知用;测试注入固定时刻,缺省 time.Now)。 + nowFn func() time.Time +} // NewService 构造(仅 db 依赖,照 account.Service 极简口径)。 -func NewService(db *gorm.DB) *Service { return &Service{db: db} } +func NewService(db *gorm.DB) *Service { return &Service{db: db, nowFn: time.Now} } // accountAggRow 账号维度聚合扫描 DTO(COUNT/SUM 扫 int64 再转实体 int)。 type accountAggRow struct { diff --git a/internal/service/settle/reconcile.go b/internal/service/settle/reconcile.go index 6f9d561a..6a76c2f7 100644 --- a/internal/service/settle/reconcile.go +++ b/internal/service/settle/reconcile.go @@ -20,6 +20,7 @@ import ( "time" "go-sms/internal/model/entity" + "go-sms/internal/service/archive" ) // ReconcileSummary 对账汇总计数。 @@ -57,6 +58,7 @@ type UpstreamDiffRow struct { } // ReconcileReport 对账报告(AsOf 时点;rows 仅异常行,全对=空)。 +// ReconcileReport 三方对账报告。 type ReconcileReport struct { Period string `json:"period"` AsOf time.Time `json:"as_of"` @@ -64,6 +66,11 @@ type ReconcileReport struct { Channels []ChannelDiffRow `json:"channels"` Upstreams []UpstreamDiffRow `json:"upstreams"` Summary ReconcileSummary `json:"summary"` + // DetailPartial S4·D3:对账窗含热窗(明细 5 天)外——热窗外明细已归档 + // ClickHouse,MySQL 侧不可得。此时账号侧=两方(日汇总↔结算单)、通道侧 + // 跳过(唯一两方数据源缺一);窗口完全在热窗内才做原三方比对。 + DetailPartial bool `json:"detail_partial"` + HotFrom string `json:"hot_from,omitempty"` } // successAgg 明细直查聚合 DTO。 @@ -74,6 +81,9 @@ type successAgg struct { // detailSuccessBy 明细直查:窗口内按 key 维度聚合成功条数(ground truth)。 func (s *Service) detailSuccessBy(ctx context.Context, lo, hi time.Time, key string) (map[int64]int64, error) { + if floor := archive.HotFloor(s.nowFn()); lo.Before(floor) { + lo = floor // S4·D3 热窗感知:归档清退后历史明细不在 MySQL,钳制到热窗内 + } var rows []successAgg if err := s.db.WithContext(ctx).Table("message_detail"). Select(key+" AS id, COALESCE(SUM(status = 2), 0) AS n"). @@ -97,10 +107,25 @@ func (s *Service) ReconcilePeriod(ctx context.Context, period string) (*Reconcil rep := &ReconcileReport{Period: period, AsOf: time.Now(), Accounts: []AccountDiffRow{}, Channels: []ChannelDiffRow{}, Upstreams: []UpstreamDiffRow{}} - // ① 账号三方:明细直查 ↔ 消耗日汇总 ↔ 下家结算单。 - detailAcc, err := s.detailSuccessBy(ctx, lo, hi, "account_id") - if err != nil { - return nil, err + // S4·D3 热窗感知:窗起点早于热窗下限=窗口内明细(部分)已归档 CH, + // MySQL 明细侧不完整——账号侧退化两方(日汇总↔结算单)、通道侧跳过; + // 报告头 DetailPartial 显式标注(对账口径自描述,防误读)。 + hotFloor := archive.HotFloor(s.nowFn()) + fullDetail := !lo.Before(hotFloor) + if !fullDetail { + rep.DetailPartial = true + rep.HotFrom = hotFloor.Format("2006-01-02") + } + + var detailAcc map[int64]int64 + if fullDetail { + // ① 账号三方:明细直查 ↔ 消耗日汇总 ↔ 下家结算单。 + detailAcc, err = s.detailSuccessBy(ctx, lo, hi, "account_id") + if err != nil { + return nil, err + } + } else { + detailAcc = map[int64]int64{} } consumeAcc := map[int64]int64{} var consumeRows []entity.BillingConsumeLog @@ -135,7 +160,8 @@ func (s *Service) ReconcilePeriod(ctx context.Context, period string) (*Reconcil d, c := detailAcc[id], consumeAcc[id] bill, hasBill := billAcc[id] billMismatch := hasBill && int64(bill.SuccessCount) != c - if d != c || billMismatch { + detailMismatch := fullDetail && d != c // 热窗外 detail 侧缺,不参与(两方口径) + if detailMismatch || billMismatch { row := AccountDiffRow{AccountID: id, DetailSuccess: d, ConsumeSuccess: c} if hasBill { bs := int64(bill.SuccessCount) @@ -150,10 +176,15 @@ func (s *Service) ReconcilePeriod(ctx context.Context, period string) (*Reconcil } } - // ② 通道两方:明细直查 ↔ 通道日汇总。 - detailCh, err := s.detailSuccessBy(ctx, lo, hi, "channel_id") - if err != nil { - return nil, err + // ② 通道两方:明细直查 ↔ 通道日汇总(热窗外跳过——detail 侧缺,比对无意义)。 + var detailCh map[int64]int64 + if fullDetail { + detailCh, err = s.detailSuccessBy(ctx, lo, hi, "channel_id") + if err != nil { + return nil, err + } + } else { + detailCh = map[int64]int64{} } dailyCh := map[int64]int64{} var statRows []entity.StatsChannelDaily @@ -171,9 +202,13 @@ func (s *Service) ReconcilePeriod(ctx context.Context, period string) (*Reconcil for id := range dailyCh { chUnion[id] = struct{}{} } - rep.Summary.ChannelsChecked = len(chUnion) + if fullDetail { + rep.Summary.ChannelsChecked = len(chUnion) + } else { + rep.Summary.ChannelsChecked = 0 // 热窗外跳过(detail 侧缺,比对无意义) + } for id := range chUnion { - if detailCh[id] != dailyCh[id] { + if fullDetail && detailCh[id] != dailyCh[id] { rep.Channels = append(rep.Channels, ChannelDiffRow{ChannelID: id, DetailSuccess: detailCh[id], DailySuccess: dailyCh[id]}) rep.Summary.ChannelDiffs++ diff --git a/internal/service/settle/reconcile_test.go b/internal/service/settle/reconcile_test.go index 2f7e0e05..c57e640a 100644 --- a/internal/service/settle/reconcile_test.go +++ b/internal/service/settle/reconcile_test.go @@ -15,9 +15,12 @@ import ( // seedReconcile 全链种子:明细(11-10:acc1 ch1 2成功+1未知;acc2 ch2 1成功) // → AggregateDay → 双向出账。ch1 绑上家 7 且 success_unknown 口径,ch2 绑 7 默认 success。 -func seedReconcile(t *testing.T, db *gorm.DB) { +func seedReconcile(t *testing.T, db *gorm.DB) string { + period := time.Now().Format("2006-01") t.Helper() - day, _ := time.ParseInLocation("2006-01-02 15:04", "2026-11-10 12:00", time.Local) + // S4·D3:明细造数须在热窗内(archive.HotFloor 之前明细侧退化为两方基线); + // day=现在截断小时(period=当月动态推导,规避月初跨月边界)。 + day := time.Now().Truncate(time.Hour) details := []entity.MessageDetail{ {SnowflakeModel: entity.SnowflakeModel{ID: 9200000001, CreatedAt: day}, TaskID: 900001, AccountID: 1, ChannelID: 1, Mobile: "13000000001", @@ -46,12 +49,13 @@ func seedReconcile(t *testing.T, db *gorm.DB) { if _, _, err := svc.AggregateDay(ctx, day); err != nil { t.Fatal(err) } - if _, err := svc.GenerateCustomerBills(ctx, "2026-11"); err != nil { + if _, err := svc.GenerateCustomerBills(ctx, period); err != nil { t.Fatal(err) } - if _, err := svc.GenerateUpstreamBills(ctx, "2026-11"); err != nil { + if _, err := svc.GenerateUpstreamBills(ctx, period); err != nil { t.Fatal(err) } + return period } // TestReconcile_AllZero 全链一致:三方零差异(S2 出口判据)。 @@ -59,8 +63,13 @@ func seedReconcile(t *testing.T, db *gorm.DB) { // 上家 7:重推(2+1未知)+(1)=4=账单 4。 func TestReconcile_AllZero(t *testing.T) { db := newBillDB(t) - seedReconcile(t, db) - rep, err := NewService(db).ReconcilePeriod(context.Background(), "2026-11") + period := seedReconcile(t, db) + // S4·D3:时钟注入=period 月 3 号(对账窗起点[月初]落在热窗内→三方比对; + // 造数明细 day=真实 now 同在窗内,数值比较不受影响)。 + nowAt, _ := time.ParseInLocation("2006-01", period, time.Local) + svc0 := NewService(db) + svc0.nowFn = func() time.Time { return nowAt.AddDate(0, 0, 2) } + rep, err := svc0.ReconcilePeriod(context.Background(), period) if err != nil { t.Fatal(err) } @@ -87,6 +96,39 @@ func TestReconcile_AllZero(t *testing.T) { } } +// TestReconcile_HotWindowFallback S4·D3:对账窗起点在热窗外(月中,明细部分 +// 已归档 CH)→ DetailPartial=true,明细↔汇总比对退化为不判(篡改汇总不报 +// 差异——两方基线=日汇总↔结算单,归档清退≠数据丢失);时钟固定月 20 号 +// 任何日历日运行稳定。 +func TestReconcile_HotWindowFallback(t *testing.T) { + db := newBillDB(t) + period := seedReconcile(t, db) + svc := NewService(db) + nowAt, _ := time.ParseInLocation("2006-01", period, time.Local) + svc.nowFn = func() time.Time { return nowAt.AddDate(0, 0, 19) } // 月中=窗起点早于热窗下限 + // 仅制造「明细↔汇总」不一致:汇总与账单同步篡改至同值(两方[汇总↔账单] + // 一致;明细侧若参与比对必报 2≠9 差异)。 + db.Model(&entity.BillingConsumeLog{}).Where("account_id = 1").Update("success_count", 9) + db.Model(&entity.SettleCustomer{}).Where("account_id = ? AND period = ?", 1, period). + Update("success_count", 9) + db.Model(&entity.StatsChannelDaily{}).Where("channel_id = 1").Update("success_count", 9) + // 上家账单同步=重推值(ch1[9+1unknown]+ch2[1]=11)。 + db.Model(&entity.SettleUpstream{}).Where("period = ?", period).Update("success_count", 11) + rep, err := svc.ReconcilePeriod(context.Background(), period) + if err != nil { + t.Fatal(err) + } + if !rep.DetailPartial { + t.Fatal("月中对账应 DetailPartial=true") + } + if rep.Summary.AccountDiffs != 0 || rep.Summary.ChannelDiffs != 0 { + t.Fatalf("两方基线不应判明细↔汇总差异: %+v", rep.Summary) + } + if !rep.Summary.AllZero { + t.Fatalf("两方基线应 AllZero: %+v", rep.Summary) + } +} + // TestReconcile_ExposesDiffs 篡改三处事实源逐项暴露: // ①consume acc1 success 2→5 → AccountDiffs=1(明细2/汇总5/账单2 同行列出); // ②stats ch1 success 2→9 → ChannelDiffs=1(明细2/日汇总9)+ 上家重推(9+1)+1=11 @@ -95,8 +137,10 @@ func TestReconcile_AllZero(t *testing.T) { // (在①②还原后单独验证:此处先验证 missing 计数,AllZero 断言只看 diffs)。 func TestReconcile_ExposesDiffs(t *testing.T) { db := newBillDB(t) - seedReconcile(t, db) + period := seedReconcile(t, db) svc := NewService(db) + nowAt, _ := time.ParseInLocation("2006-01", period, time.Local) + svc.nowFn = func() time.Time { return nowAt.AddDate(0, 0, 2) } // 月 3 号=窗起点在热窗内 ctx := context.Background() // ① 篡改消耗日汇总。 @@ -109,7 +153,7 @@ func TestReconcile_ExposesDiffs(t *testing.T) { Where("channel_id = 1").Update("success_count", 9).Error; err != nil { t.Fatal(err) } - rep, err := svc.ReconcilePeriod(ctx, "2026-11") + rep, err := svc.ReconcilePeriod(ctx, period) if err != nil { t.Fatal(err) } @@ -148,7 +192,7 @@ func TestReconcile_ExposesDiffs(t *testing.T) { if err := db.Where("account_id = 2").Delete(&entity.SettleCustomer{}).Error; err != nil { t.Fatal(err) } - rep, err = svc.ReconcilePeriod(ctx, "2026-11") + rep, err = svc.ReconcilePeriod(ctx, period) if err != nil { t.Fatal(err) } diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index dffd7362..9f4467bc 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,14 +1,16 @@ -# 当前工作上下文(最近更新:2026-11-16) +# 当前工作上下文(最近更新:2026-11-16 下午) -## 当前状态快照(2026-11-16,**S4 D1/D2 完成,一期 17/27 计划日**) +## 当前状态快照(2026-11-16,**S4 泳道收官,一期 18/27 计划日**) - **仓库**:main 已推 Gitea;每日 bundle+push 双轨。 -- **一期进度**:S0✅ S1✅ S2✅ / S3 ✅(11-15 凌晨)/ **S4-D1/D2 ✅(11-16)**——C1 回调推送引擎(callback_task v21+PushWorker 指数退避/死信可查可重派+双挂点 report 终态/uplink 消费+GET /ops/callback 管理面+前端回调管理页 ops-callback+metrics)+ C2 上行自动拉黑(等值词表 SystemConfig 可配→policy_blacklist source=2+Redis 即时生效;拉黑不影响推送);**真机全链六步实证**(回执→HTTP 回调 JSON 送达/上行 R→拉黑+推送/坏 URL→死信→resend→重投 sent/状态面板);门禁全绿(-race 40 包/vitest 148/lint 0);验收单 s4-d1d2-callback.md。**S4 剩余=D3 明细按天分区热 5 天+ClickHouse 冷热切换+冷查<1s**(动核心大表,方案先行)。 -- **C1/C2 口径留痕**:载荷无签名(挂账);拦截终态 HTTP 回调未挂(CMPP 侧 blockedDeliverAdapter 已有);PushWorker 单实例语义(多实例 at-least-once);退避 2^n s 封顶 10min×10 次(S5 重标);casbin 策略=ops callback:read/write(v21 所种,SQL 直插 g 重启生效老规矩)。 -- **⚠️ 流程补正在途(两项,不阻塞开工)**:①S2 用户本人走查(s2-walkthrough.sh D2~D5,AI 预走查 27 PASS);②S2 结算页+S3 利润页+**S4 回调管理页**浏览器走查(:5173,s2demo 登录[admin 角色];回调页在「系统管理→回调管理」)。 -- **24h 稳定性门槛 ✅ 终判通过**(S5 压测前置;S4-D3 动大表后建议 S5 前复核 dmesg)。 -- **真浏览器通道就位(2026-11-15)**:Google Chrome 157 官方 deb 直装 + Playwright `channel:'chrome'` 实测可用(`scripts/chrome-session.mjs` 通用会话工具,单 page 会话铁律见 techContext 工具链)——**参考短信平台登录参考 UI + AI 前端自验通道已打通**;注意本 AI 会话不支持读图,自验交付口径=截图供用户复核 + DOM/CSS/accessibility 结构化提取由 AI 分析;账号一律 env 临时传入不落盘。 -- **Issue#1 两轮交付均已入库(2026-11-15/16)**:盈讯通调研+ChannelList 8 列双语义聚合+ChannelEdit 分组深度对齐+chrome-session 工具;工单评论仍待用户贴 Gitea(或给 token)。 +- **一期进度**:S0✅ S1✅ S2✅ / S3 ✅(11-15 凌晨)/ **S4 ✅ 收官(11-16,三项验收全过:死信可查/拒收即拉黑/冷查 16ms<1s)**——D1/D2=C1 回调推送引擎+ C2 上行自动拉黑(真机六步实证,验收单 s4-d1d2-callback.md);D3=明细按天 RANGE(id) 分区热 5 天(迁移 v22,9 分区)+CH 冷热归档 job(count 校验零丢失方向)+/record/history 冷查(settle-check 热窗感知 DetailPartial 两方基线;真机 5 连坑全修存档,验收单 s4-d3-partition-archive.md)。 +- **下一步=S5 基线与灰度(8 日,一期最后泳道)**:kind 安装+deploy/k8s manifests apply(NodePort 30081)→ 24h 稳定性复核(S4 动了大表结构,S5 压测前重跑一轮而非沿用 11-14 旧判)→ 压测基线 200QPS×30min(SSD 下重标 workers:4/55/s 等 HDD 数字)→ 消费参数重标定 → 灰度接管现网流量。**出口四项**:七件套全绿/P0+P1+结算覆盖矩阵全勾/SSD 基线报告入档/v2 承载 ≥1 周零 P0 事故+POC 转冷备。 +- **S4 遗留挂账**:回调载荷无签名(需求未定义);拦截终态 HTTP 回调未挂(CMPP 侧已有);history 异步导出走下载中心(D29/P2);CH 生产建议副本+冷分区 S3(S5 运维项);pmax REORGANIZE 运维口径入 runbook 待补。 +- **C1/C2/D3 口径留痕**:PushWorker 单实例 at-least-once;归档 job 单实例(CH 分区重写幂等吸收);CH count()=UInt64/time.Time 绑定 UTC 歧义(查询预格式化墙钟串)——CH 交互三坑已存档 techContext。 +- **⚠️ 流程补正在途(两项,不阻塞开工)**:①S2 用户本人走查(s2-walkthrough.sh D2~D5,AI 预走查 27 PASS);②S2 结算页+S3 利润页+S4 回调管理页+record/history 页浏览器走查(:5173,s2demo 登录[admin 角色])。 +- **24h 稳定性门槛 ✅ 终判通过(11-14)**;S4 后建议 S5 前复核 dmesg+零重启。 +- **真浏览器通道就位(2026-11-15)**:Google Chrome 157 + Playwright `channel:'chrome'`(`scripts/chrome-session.mjs`;单 page 会话铁律)。 +- **Issue#1 两轮交付均已入库(2026-11-15/16)**;工单评论仍待用户贴 Gitea(或给 token)。 - **六项裁决速查(S2 全闭环)**:R1/R6 换算只在 price.go / R2 事实源 D2 落账 / R3 双通道 D3 落地(ref_id 区分)/ R4 rebate_rate D5 落地(profit.go 现算不落档)/ R5 settle_ladder D4 落地。 - 归档仓(只读):/data/poc-archive/repos/go-sms.git。 - **【铁律】测试端口固定=5173**:Web 测试/联调/走查/对外验收口径**一律** `http://<服务器IP>:5173`(vite dev,`make web-dev`);**禁止**用 WEB_PORT 覆盖到其他端口、禁止 preview(4173 等)/临时容器端口(30099 等)作为验收与走查入口——一次性探测端口仅限本机调试且不入文档口径(2026-11-15 用户裁定)。 diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 853a3d9c..d2070b53 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -20,9 +20,9 @@ | S2 泳道收官 | 出口两项(结算单测+对账差异 0)全过;六项裁决 R1~R6 全闭环 | ✅ 2026-11-14 深夜③(判定单 s2-closeout.md;用户走查在途补正) | | S3 前端收尾 | 盘点修正:views 31 页已在位零残留;FE-D34 利润页(/stats/profit)+FE-6 四件(Mock 浏览器通道下线/五角色×33 菜单矩阵快照/拆包全 chunk gzip<500KB/ci.yml web job+Dockerfile nginx+NodePort 30081 备档);vitest 148 全绿;LCP 实测与 kind 两项环境受阻如实标注 | ✅ 2026-11-15 凌晨(s3-closeout.md;页面走查在途) | | S4-D1/D2 C1 回调推送+C2 拉黑 | callback_task(v21)+PushWorker(指数退避/死信/重派)+双挂点(report 终态 Consumer/PairService+uplink 消费)+GET /ops/callback 管理面+前端回调管理页;C2 等值词表可配拉黑+Redis 即时生效;真机全链六步实证;**死信可查/拒收即拉黑两项验收 ✅** | ✅ 2026-11-16(s4-d1d2-callback.md) | -| S4-D3 分区+冷热 | 明细按天分区热 5 天(v17 口径上 DDL 改造,方案先行)+CH 冷热切换(5 天前迁出)+record/history 冷查<1s+settle-check 对账基线自动切换验证 | ⬜ 下一步 | +| S4-D3 分区+冷热 | v22 RANGE(id) 分区(热 5 天+缓冲 3+pmax)+archive 域(CH 表/归档 job count 校验零丢失)+/record/history 冷查 16ms+settle-check 热窗感知(DetailPartial 两方基线);真机 5 连坑全修;**冷查<1s 验收 ✅,S4 三项全过收官** | ✅ 2026-11-16 下午(s4-d3-partition-archive.md) | | 观察项 | 24h 稳定性门槛(11-13 17:55 起计;期内 21:33 宿主主动重启=已知事件不判失败)期满核验 11-14 17:55 | ✅ 2026-11-14 18:02 终判通过(dmesg 0/零重启/驻留+md5 一致;S5 压测前置确认,S1 完全闭环) | -| S2~S5 | 结算域/前端收尾/C1C2+冷热/基线与灰度 | 🔄 S2✅S3✅S4 D1/D2✅(见《Go-sms-重建计划.md》) | +| S2~S5 | 结算域/前端收尾/C1C2+冷热/基线与灰度 | 🔄 S2✅S3✅**S4✅收官**(S5 见《Go-sms-重建计划.md》) | ## 阶段总览(8 阶段 / 80 工作日) diff --git a/memory-bank/techContext.md b/memory-bank/techContext.md index 8b60c817..bf897748 100644 --- a/memory-bank/techContext.md +++ b/memory-bank/techContext.md @@ -38,6 +38,8 @@ | MinIO | 19010/19011(API/Console) | | | Prometheus+Alertmanager | — | 规则 `deploy/monitoring/alert_rules.yml` | +**ClickHouse 24.3 交互三坑(S4·D3 真机留痕,clickhouse-go v2 native)**:① `count()` 返回 **UInt64**——Scan `*int64` 报 unsupported,须 `*uint64`(且 Scan 错误不可 `_=` 吞,曾致冷查 total 恒 0);② **无 `DROP PARTITION IF EXISTS`**(24.3 语法不支持)——先查 `system.parts`(database=currentDatabase() AND table=? AND partition=?)存在才 DROP;③ **`?` 绑定 time.Time 有 UTC 化歧义**——时间窗查询参数预格式化墙钟字符串 `"2006-01-02 15:04:05"`(与写入语义同口径)。另:MySQL go-driver `parseTime=True` 下 `DATE()` 结果 Scan 到 `[]string` 会带 RFC3339 尾巴——跨方言日期列举用 `DISTINCT created_at` Scan `[]time.Time` + Go 侧归一化去重;MySQL 8.0 `information_schema.PARTITIONS` 对**非分区表也返回一行 NULL**——已分区判据必须 `PARTITION_NAME IS NOT NULL`。 + 网关 dev 固定 **:8080**(`GW_PORT` 可覆盖;历史 8081 为当时宿主 8080 被占用的临时覆盖,已废弃,vite proxy 按 8080);**Web 测试端口铁律=5173**(`make web-dev`,vite dev;WEB_PORT 不得覆盖、preview/临时容器端口不入口径);**外网测试统一入口=`https://sms.unimsc.com/`**(nginx 根路径反代 443→5173+ws Upgrade,vite allowedHosts 已登记;ai.unimsc.com/SmsDev 前缀通道废弃);K8s NodePort 30080/30081(S5 灰度 apply)。 ## 常用命令