Files
go-sms-V2/internal/service/settle/reconcile.go
T
Tory 01c8874831
ci / test (push) Canceled after 0s
ci / web-test (push) Canceled after 0s
ci / build (push) Canceled after 0s
ci / deploy-dev (push) Canceled after 0s
S4-D3: 明细按天分区热5天+ClickHouse冷热切换——S4泳道收官(冷查16ms<1s验收✅): ①分区=迁移v22 PARTITION BY RANGE(id)(雪花时间位当天零点边界,主键即分区键零唯一键改造;热5天+未来3天缓冲+pmax兜底;MySQL守卫sqlite跳过)+archive/partition.go EnsurePartitions幂等滚动(REORGANIZE pmax拆分;过期分区归档校验后才DROP)+真机9分区存量无损;②归档=archive域(CH表toDate分区/ArchiveJob 1h:CH当日分区重写幂等→分批2000迁出→count校验相等才DROP MySQL分区或DELETE兜底=零丢失方向;consumer装配启动即首轮)+metrics archive_rows/errors;③冷查=GET /record/history(CH查询/task_no MySQL转译回填/D26契约直连)真机16ms+mobile筛选✓,导出挂账D29下载中心;④settle-check热窗感知=月中对账DetailPartial=true明细↔汇总退化两方基线(日汇总↔结算单,reconcile头注预声明落地)+settle.Service时钟注入nowFn+测试相对时间/period动态;⑤真机5连坑全修存档:MySQL8.0 information_schema非分区表也返回NULL行(判据加PARTITION_NAME IS NOT NULL)/parseTime下DATE() Scan []string带RFC3339尾巴(DISTINCT时刻Scan []time.Time+Go去重双方言)/CH24.3无DROP PARTITION IF EXISTS(先查system.parts)/CH count()返回UInt64(*int64不支持+count错误不可吞)/CH绑定time.Time UTC歧义(预格式化墙钟串)——techContext存档;⑥门禁=build/vet/lint 0/-race 41包全绿/迁移v16..v22两遍幂等(v15断言随窗口右移移除);archive单测5件+settle热窗外基线用例;验收单s4-d3-partition-archive.md+CHANGELOG+memory-bank(S4收官快照/S5下一步/挂账清单)
2026-11-16 12:11:31 +08:00

272 lines
9.6 KiB
Go

// reconcile.go S2-D5:三方对账——明细直查 vs 日汇总(R2 事实源)vs 结算单,
// diff=0 为 S2 泳道出口判据(《重建计划》S2 行)。
//
// 比对口径(全部为**条数**,status=2 成功侧):
// - 账号三方:message_detail 直查聚合 ↔ billing_consume_log 月和 ↔
// settle_customer.success_count(有单才比;未生成=流程状态,BillMissing
// 单列不判失败);
// - 通道两方:message_detail 直查聚合 ↔ stats_channel_daily 月和;
// - 上家:settle_upstream 快照 ↔ 日汇总×**当前**通道归并/billing_mode 重推
// (Q4/Q6 快照语义:生成后换绑/改模式/重刷会如实暴露差异——这正是对账目的);
// - 金额不比:结算单金额是出账时点价/阶梯快照(Q6),事后调价重算必生噪音;
// 金额核对走账单导出 XLSX 人工比对(D4 已验)。
//
// 注意:明细热窗仅 5 天(S4 分区规划)——历史月的"明细直查"侧在数据清退后
// 不再完整,届时对账基线=日汇总 vs 结算单两方;当前 dev 明细全量在库。
package settle
import (
"context"
"time"
"go-sms/internal/model/entity"
"go-sms/internal/service/archive"
)
// ReconcileSummary 对账汇总计数。
type ReconcileSummary struct {
AccountsChecked int `json:"accounts_checked"` // 账号并集(明细∪汇总∪账单)
AccountDiffs int `json:"account_diffs"` // 账号侧差异行数
BillMissing int `json:"bill_missing"` // 有消耗无账单(流程状态,不判失败)
ChannelsChecked int `json:"channels_checked"` // 通道并集(明细∪汇总)
ChannelDiffs int `json:"channel_diffs"` // 通道侧差异行数
UpstreamsChecked int `json:"upstreams_checked"` // 上家并集(重推∪账单)
UpstreamDiffs int `json:"upstream_diffs"` // 上家侧差异行数(缺账单不计入)
AllZero bool `json:"all_zero"` // 三方差异全零(S2 出口判据)
}
// AccountDiffRow 账号对账行(仅差异/缺单账号入列)。
type AccountDiffRow struct {
AccountID int64 `json:"account_id"`
DetailSuccess int64 `json:"detail_success"` // 明细直查
ConsumeSuccess int64 `json:"consume_success"` // 日汇总
BillSuccess *int64 `json:"bill_success"` // 结算单(nil=未生成)
}
// ChannelDiffRow 通道对账行(仅差异通道入列)。
type ChannelDiffRow struct {
ChannelID int64 `json:"channel_id"`
DetailSuccess int64 `json:"detail_success"`
DailySuccess int64 `json:"daily_success"`
}
// UpstreamDiffRow 上家对账行(差异/缺单入列;重推=当前归并×当前口径)。
type UpstreamDiffRow struct {
UpstreamID int64 `json:"upstream_id"`
DailyCount int64 `json:"daily_count"` // 日汇总重推
BillSuccess *int64 `json:"bill_success"` // 账单快照(nil=未生成)
}
// ReconcileReport 对账报告(AsOf 时点;rows 仅异常行,全对=空)。
// ReconcileReport 三方对账报告。
type ReconcileReport struct {
Period string `json:"period"`
AsOf time.Time `json:"as_of"`
Accounts []AccountDiffRow `json:"accounts"`
Channels []ChannelDiffRow `json:"channels"`
Upstreams []UpstreamDiffRow `json:"upstreams"`
Summary ReconcileSummary `json:"summary"`
// DetailPartial S4·D3:对账窗含热窗(明细 5 天)外——热窗外明细已归档
// ClickHouse,MySQL 侧不可得。此时账号侧=两方(日汇总↔结算单)、通道侧
// 跳过(唯一两方数据源缺一);窗口完全在热窗内才做原三方比对。
DetailPartial bool `json:"detail_partial"`
HotFrom string `json:"hot_from,omitempty"`
}
// successAgg 明细直查聚合 DTO。
type successAgg struct {
ID int64
N int64
}
// detailSuccessBy 明细直查:窗口内按 key 维度聚合成功条数(ground truth)。
func (s *Service) detailSuccessBy(ctx context.Context, lo, hi time.Time, key string) (map[int64]int64, error) {
if floor := archive.HotFloor(s.nowFn()); lo.Before(floor) {
lo = floor // S4·D3 热窗感知:归档清退后历史明细不在 MySQL,钳制到热窗内
}
var rows []successAgg
if err := s.db.WithContext(ctx).Table("message_detail").
Select(key+" AS id, COALESCE(SUM(status = 2), 0) AS n").
Where("created_at >= ? AND created_at < ?", lo, hi).
Group(key).Scan(&rows).Error; err != nil {
return nil, err
}
out := make(map[int64]int64, len(rows))
for _, r := range rows {
out[r.ID] = r.N
}
return out, nil
}
// ReconcilePeriod 三方对账(period=YYYY-MM)。
func (s *Service) ReconcilePeriod(ctx context.Context, period string) (*ReconcileReport, error) {
lo, hi, err := PeriodBounds(period)
if err != nil {
return nil, err
}
rep := &ReconcileReport{Period: period, AsOf: time.Now(),
Accounts: []AccountDiffRow{}, Channels: []ChannelDiffRow{}, Upstreams: []UpstreamDiffRow{}}
// S4·D3 热窗感知:窗起点早于热窗下限=窗口内明细(部分)已归档 CH,
// MySQL 明细侧不完整——账号侧退化两方(日汇总↔结算单)、通道侧跳过;
// 报告头 DetailPartial 显式标注(对账口径自描述,防误读)。
hotFloor := archive.HotFloor(s.nowFn())
fullDetail := !lo.Before(hotFloor)
if !fullDetail {
rep.DetailPartial = true
rep.HotFrom = hotFloor.Format("2006-01-02")
}
var detailAcc map[int64]int64
if fullDetail {
// ① 账号三方:明细直查 ↔ 消耗日汇总 ↔ 下家结算单。
detailAcc, err = s.detailSuccessBy(ctx, lo, hi, "account_id")
if err != nil {
return nil, err
}
} else {
detailAcc = map[int64]int64{}
}
consumeAcc := map[int64]int64{}
var consumeRows []entity.BillingConsumeLog
if err := s.db.WithContext(ctx).
Where("stat_date >= ? AND stat_date < ?", lo, hi).Find(&consumeRows).Error; err != nil {
return nil, err
}
for _, r := range consumeRows {
consumeAcc[r.AccountID] += int64(r.SuccessCount)
}
var bills []entity.SettleCustomer
if err := s.db.WithContext(ctx).
Where("period = ?", period).Find(&bills).Error; err != nil {
return nil, err
}
billAcc := map[int64]entity.SettleCustomer{}
for _, b := range bills {
billAcc[b.AccountID] = b
}
accUnion := map[int64]struct{}{}
for id := range detailAcc {
accUnion[id] = struct{}{}
}
for id := range consumeAcc {
accUnion[id] = struct{}{}
}
for id := range billAcc {
accUnion[id] = struct{}{}
}
rep.Summary.AccountsChecked = len(accUnion)
for id := range accUnion {
d, c := detailAcc[id], consumeAcc[id]
bill, hasBill := billAcc[id]
billMismatch := hasBill && int64(bill.SuccessCount) != c
detailMismatch := fullDetail && d != c // 热窗外 detail 侧缺,不参与(两方口径)
if detailMismatch || billMismatch {
row := AccountDiffRow{AccountID: id, DetailSuccess: d, ConsumeSuccess: c}
if hasBill {
bs := int64(bill.SuccessCount)
row.BillSuccess = &bs
}
rep.Accounts = append(rep.Accounts, row)
rep.Summary.AccountDiffs++
} else if !hasBill {
rep.Accounts = append(rep.Accounts,
AccountDiffRow{AccountID: id, DetailSuccess: d, ConsumeSuccess: c})
rep.Summary.BillMissing++
}
}
// ② 通道两方:明细直查 ↔ 通道日汇总(热窗外跳过——detail 侧缺,比对无意义)。
var detailCh map[int64]int64
if fullDetail {
detailCh, err = s.detailSuccessBy(ctx, lo, hi, "channel_id")
if err != nil {
return nil, err
}
} else {
detailCh = map[int64]int64{}
}
dailyCh := map[int64]int64{}
var statRows []entity.StatsChannelDaily
if err := s.db.WithContext(ctx).
Where("stat_date >= ? AND stat_date < ?", lo, hi).Find(&statRows).Error; err != nil {
return nil, err
}
for _, r := range statRows {
dailyCh[r.ChannelID] += int64(r.SuccessCount)
}
chUnion := map[int64]struct{}{}
for id := range detailCh {
chUnion[id] = struct{}{}
}
for id := range dailyCh {
chUnion[id] = struct{}{}
}
if fullDetail {
rep.Summary.ChannelsChecked = len(chUnion)
} else {
rep.Summary.ChannelsChecked = 0 // 热窗外跳过(detail 侧缺,比对无意义)
}
for id := range chUnion {
if fullDetail && detailCh[id] != dailyCh[id] {
rep.Channels = append(rep.Channels,
ChannelDiffRow{ChannelID: id, DetailSuccess: detailCh[id], DailySuccess: dailyCh[id]})
rep.Summary.ChannelDiffs++
}
}
// ③ 上家:日汇总×当前归并/口径重推 ↔ 账单快照。
dailyUp := map[int64]int64{}
var channels []entity.ChannelChannel
if err := s.db.WithContext(ctx).Find(&channels).Error; err != nil {
return nil, err
}
modeOf := map[int64]string{}
upstreamOf := map[int64]int64{}
for _, ch := range channels {
modeOf[ch.ID] = ch.BillingMode
upstreamOf[ch.ID] = ch.UpstreamID
}
for _, r := range statRows {
uid := upstreamOf[r.ChannelID]
if uid == 0 {
continue
}
dailyUp[uid] += int64(billableCount(modeOf[r.ChannelID], r))
}
var upBills []entity.SettleUpstream
if err := s.db.WithContext(ctx).
Where("period = ?", period).Find(&upBills).Error; err != nil {
return nil, err
}
billUp := map[int64]entity.SettleUpstream{}
for _, b := range upBills {
billUp[b.UpstreamID] = b
}
upUnion := map[int64]struct{}{}
for id := range dailyUp {
upUnion[id] = struct{}{}
}
for id := range billUp {
upUnion[id] = struct{}{}
}
rep.Summary.UpstreamsChecked = len(upUnion)
for id := range upUnion {
bill, hasBill := billUp[id]
if hasBill && int64(bill.SuccessCount) != dailyUp[id] {
bs := int64(bill.SuccessCount)
rep.Upstreams = append(rep.Upstreams,
UpstreamDiffRow{UpstreamID: id, DailyCount: dailyUp[id], BillSuccess: &bs})
rep.Summary.UpstreamDiffs++
} else if !hasBill {
// 有日汇总无账单:与账号侧 BillMissing 同语义,列行不计数。
rep.Upstreams = append(rep.Upstreams,
UpstreamDiffRow{UpstreamID: id, DailyCount: dailyUp[id]})
}
}
rep.Summary.AllZero = rep.Summary.AccountDiffs == 0 &&
rep.Summary.ChannelDiffs == 0 && rep.Summary.UpstreamDiffs == 0
return rep, nil
}