180 lines
5.7 KiB
Go
180 lines
5.7 KiB
Go
package queue
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"math/rand"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/IBM/sarama"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// parallelSession 嵌入接口仅覆写 consumeParallel 依赖的三个方法(未用方法 nil panic 即测试失败)。
|
|
type parallelSession struct {
|
|
sarama.ConsumerGroupSession
|
|
ctx context.Context
|
|
mu sync.Mutex
|
|
marked []int64
|
|
commits int
|
|
}
|
|
|
|
func (s *parallelSession) MarkMessage(msg *sarama.ConsumerMessage, _ string) {
|
|
s.mu.Lock()
|
|
s.marked = append(s.marked, msg.Offset)
|
|
s.mu.Unlock()
|
|
}
|
|
func (s *parallelSession) Commit() {
|
|
s.mu.Lock()
|
|
s.commits++
|
|
s.mu.Unlock()
|
|
}
|
|
func (s *parallelSession) Context() context.Context { return s.ctx }
|
|
func (s *parallelSession) markedSnapshot() []int64 {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
return append([]int64(nil), s.marked...)
|
|
}
|
|
|
|
// parallelClaim 单分区消息通道。
|
|
type parallelClaim struct{ ch chan *sarama.ConsumerMessage }
|
|
|
|
func (c *parallelClaim) Topic() string { return "t-parallel" }
|
|
func (c *parallelClaim) Partition() int32 { return 0 }
|
|
func (c *parallelClaim) InitialOffset() int64 { return 0 }
|
|
func (c *parallelClaim) HighWaterMarkOffset() int64 { return 0 }
|
|
func (c *parallelClaim) Messages() <-chan *sarama.ConsumerMessage { return c.ch }
|
|
|
|
// runParallel 驱动 consumeParallel:投消息→等 handler 全部完成且 Mark 达期望→
|
|
// 关闭消息通道触发退出(通道关闭=会话结束路径,与生产语义一致)。
|
|
func runParallel(t *testing.T, workers, n int, h Handler, wantMarked int) *parallelSession {
|
|
t.Helper()
|
|
msgs := make([]*sarama.ConsumerMessage, n)
|
|
for i := range msgs {
|
|
msgs[i] = &sarama.ConsumerMessage{Topic: "t-parallel", Partition: 0, Offset: int64(i), Value: []byte("v")}
|
|
}
|
|
sess := ¶llelSession{ctx: context.Background()}
|
|
claim := ¶llelClaim{ch: make(chan *sarama.ConsumerMessage, n)}
|
|
gh := &groupHandler{
|
|
cfg: ConsumerConfig{Handler: h, WorkersPerPartition: workers, CommitEveryN: 5},
|
|
log: zap.NewNop(),
|
|
}
|
|
done := make(chan struct{})
|
|
go func() { _ = gh.consumeParallel(sess, claim); close(done) }()
|
|
for _, m := range msgs {
|
|
claim.ch <- m
|
|
}
|
|
// 等 Mark 追平期望(处理完成与 Mark 之间有主循环异步窗口,轮询避免竞态)。
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if len(sess.markedSnapshot()) >= wantMarked {
|
|
break
|
|
}
|
|
time.Sleep(2 * time.Millisecond)
|
|
}
|
|
close(claim.ch)
|
|
select {
|
|
case <-done:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("consumeParallel 未在超时内退出")
|
|
}
|
|
return sess
|
|
}
|
|
|
|
// TestConsumeParallel_OrderedMark Day65 核心不变量:多 worker 乱序处理,
|
|
// MarkMessage 严格按 offset 升序(仅连续完成前缀),全部消息各处理一次。
|
|
func TestConsumeParallel_OrderedMark(t *testing.T) {
|
|
rng := rand.New(rand.NewSource(64))
|
|
delays := make([]time.Duration, 30)
|
|
for i := range delays {
|
|
delays[i] = time.Duration(rng.Intn(8)) * time.Millisecond // 预生成(*Rand 非并发安全)
|
|
}
|
|
var mu sync.Mutex
|
|
seen := map[int64]int{}
|
|
h := func(_ context.Context, m *sarama.ConsumerMessage) error {
|
|
time.Sleep(delays[m.Offset]) // 乱序完成
|
|
mu.Lock()
|
|
seen[m.Offset]++
|
|
mu.Unlock()
|
|
return nil
|
|
}
|
|
sess := runParallel(t, 4, 30, h, 30)
|
|
|
|
got := sess.markedSnapshot()
|
|
if len(got) != 30 {
|
|
t.Fatalf("Mark 数 = %d, 期望 30(got=%v)", len(got), got)
|
|
}
|
|
for i, off := range got {
|
|
if off != int64(i) {
|
|
t.Fatalf("Mark 序列须严格升序连续:第 %d 个 = offset %d(序列 %v)", i, off, got)
|
|
}
|
|
}
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
if len(seen) != 30 {
|
|
t.Fatalf("处理消息数 = %d, 期望 30", len(seen))
|
|
}
|
|
for off, c := range seen {
|
|
if c != 1 {
|
|
t.Fatalf("offset %d 处理 %d 次, 期望 1(至少一次语义,不应重复)", off, c)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestConsumeParallel_FailureNotMarkedButAdvances 失败消息不 Mark 但窗口越过
|
|
// (与串行版「失败 continue、后继 Mark 越过」等价),其余按序 Mark。
|
|
func TestConsumeParallel_FailureNotMarkedButAdvances(t *testing.T) {
|
|
h := func(_ context.Context, m *sarama.ConsumerMessage) error {
|
|
if m.Offset == 3 || m.Offset == 7 {
|
|
return errors.New("biz fail")
|
|
}
|
|
return nil
|
|
}
|
|
sess := runParallel(t, 3, 10, h, 8)
|
|
|
|
got := sess.markedSnapshot()
|
|
want := []int64{0, 1, 2, 4, 5, 6, 8, 9}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("Mark 数 = %d, 期望 %d(got=%v)", len(got), len(want), got)
|
|
}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Fatalf("Mark 序列 = %v, 期望 %v(失败 offset 3/7 跳过且不阻塞)", got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestConsumeParallel_AllFail 全部失败:零 Mark、正常退出不卡死(毒丸不阻塞分区)。
|
|
func TestConsumeParallel_AllFail(t *testing.T) {
|
|
h := func(_ context.Context, _ *sarama.ConsumerMessage) error { return errors.New("boom") }
|
|
sess := runParallel(t, 2, 6, h, 0)
|
|
if got := sess.markedSnapshot(); len(got) != 0 {
|
|
t.Fatalf("全败应零 Mark, 实际 %v", got)
|
|
}
|
|
}
|
|
|
|
// TestConsumeParallel_NoDeadlockUnderLoad Day65 死锁回归:大量消息+快速 handler,
|
|
// done 结果积压期间主循环仍须持续消费(修复前:投 tasks 嵌套 select 独占等待,
|
|
// done 满后与 worker 互锁,生产实证进程全静默)。超时即死锁。
|
|
func TestConsumeParallel_NoDeadlockUnderLoad(t *testing.T) {
|
|
const n = 500
|
|
h := func(_ context.Context, m *sarama.ConsumerMessage) error {
|
|
if m.Offset%97 == 0 {
|
|
return errors.New("flaky")
|
|
}
|
|
return nil
|
|
}
|
|
sess := runParallel(t, 4, n, h, n-6) // 500 条中 6 条(offset 0,97,194,291,388,486?)失败
|
|
got := sess.markedSnapshot()
|
|
if len(got) == 0 {
|
|
t.Fatal("零 Mark:消费未推进(疑似死锁)")
|
|
}
|
|
for i := 1; i < len(got); i++ {
|
|
if got[i] <= got[i-1] {
|
|
t.Fatalf("Mark 序列须严格升序: %v", got[:i+1])
|
|
}
|
|
}
|
|
}
|