perf(cmpp): instrument inbound flow and unbatch submit worker

2026-08-20 11:27:35 +08:00
parent c4f36fc50d
commit b9a71fe0b9
23 changed files with 760 additions and 138 deletions
@@ -2,6 +2,7 @@ package submitworker

 import (
 	"context"
+	"sync/atomic"
 	"testing"
 	"time"

@@ -121,11 +122,21 @@ func TestMinIdleDefaultsToThirtySeconds(t *testing.T) {
 	}
 }

-func TestProcessMessagesDoesNotLetOneChannelBlockAnother(t *testing.T) {
+func TestConcurrencyUsesBoundedDefaultAndMaximum(t *testing.T) {
+	if got := (&Worker{}).ConfiguredConcurrency(); got != defaultConcurrency {
+		t.Fatalf("default concurrency = %d, want %d", got, defaultConcurrency)
+	}
+	if got := (&Worker{Concurrency: 2048}).ConfiguredConcurrency(); got != 1024 {
+		t.Fatalf("capped concurrency = %d, want 1024", got)
+	}
+}
+
+func TestMessageWorkPoolContinuouslyRefillsWithoutWaitingForSlowSibling(t *testing.T) {
 	mr := miniredis.RunT(t)
 	client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
 	startedA := make(chan struct{})
 	startedB := make(chan struct{})
+	startedC := make(chan struct{})
 	releaseA := make(chan struct{})
 	worker := &Worker{
 		Redis: client,
@@ -136,19 +147,17 @@ func TestProcessMessagesDoesNotLetOneChannelBlockAnother(t *testing.T) {
 				<-releaseA
 			case "channel-b":
 				close(startedB)
+			case "channel-c":
+				close(startedC)
 			}
 			return queue.SubmitResult{SubmitStatus: "accepted"}, nil
 		},
 	}
-	messages := []redis.XMessage{
-		{ID: "1-0", Values: submitCommandValues("message-a", "channel-a")},
-		{ID: "2-0", Values: submitCommandValues("message-b", "channel-b")},
+	pool := newMessageWorkPool(context.Background(), worker, 2)
+	if !pool.dispatch(redis.XMessage{ID: "1-0", Values: submitCommandValues("message-a", "channel-a")}) ||
+		!pool.dispatch(redis.XMessage{ID: "2-0", Values: submitCommandValues("message-b", "channel-b")}) {
+		t.Fatal("initial messages were not dispatched")
 	}
-	done := make(chan struct{})
-	go func() {
-		_ = worker.processMessages(context.Background(), messages)
-		close(done)
-	}()
 	select {
 	case <-startedA:
 	case <-time.After(time.Second):
@@ -159,11 +168,125 @@ func TestProcessMessagesDoesNotLetOneChannelBlockAnother(t *testing.T) {
 	case <-time.After(200 * time.Millisecond):
 		t.Fatal("channel-b was blocked by channel-a")
 	}
-	close(releaseA)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if _, err := pool.waitForCapacity(ctx); err != nil {
+		t.Fatalf("wait for refill capacity: %v", err)
+	}
+	if !pool.dispatch(redis.XMessage{ID: "3-0", Values: submitCommandValues("message-c", "channel-c")}) {
+		t.Fatal("refill message was not dispatched")
+	}
 	select {
-	case <-done:
-	case <-time.After(time.Second):
-		t.Fatal("message batch did not complete")
+	case <-startedC:
+	case <-time.After(200 * time.Millisecond):
+		t.Fatal("pool waited for the slow sibling instead of refilling its free slot")
+	}
+	close(releaseA)
+	pool.wait()
+}
+
+func TestMessageWorkPoolAcknowledgesFastMessageBeforeSlowSiblingCompletes(t *testing.T) {
+	mr := miniredis.RunT(t)
+	client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+	worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway"}
+	ctx := context.Background()
+	if err := worker.ensureGroup(ctx); err != nil {
+		t.Fatalf("ensureGroup: %v", err)
+	}
+	for _, entry := range []struct{ id, messageID, channelID string }{
+		{"1-0", "message-slow", "channel-slow"},
+		{"2-0", "message-fast", "channel-fast"},
+	} {
+		if err := client.XAdd(ctx, &redis.XAddArgs{Stream: worker.stream(), ID: entry.id, Values: submitCommandValues(entry.messageID, entry.channelID)}).Err(); err != nil {
+			t.Fatalf("xadd %s: %v", entry.id, err)
+		}
+	}
+	streams, err := client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: worker.group(), Consumer: worker.consumer(), Streams: []string{worker.stream(), ">"}, Count: 2}).Result()
+	if err != nil || len(streams) != 1 || len(streams[0].Messages) != 2 {
+		t.Fatalf("xreadgroup: streams=%+v err=%v", streams, err)
+	}
+	slowStarted := make(chan struct{})
+	fastReturned := make(chan struct{})
+	releaseSlow := make(chan struct{})
+	worker.Submit = func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
+		if command.ChannelID == "channel-slow" {
+			close(slowStarted)
+			<-releaseSlow
+		} else {
+			close(fastReturned)
+		}
+		return queue.SubmitResult{SubmitStatus: "accepted"}, nil
+	}
+	pool := newMessageWorkPool(ctx, worker, 2)
+	for _, message := range streams[0].Messages {
+		if !pool.dispatch(message) {
+			t.Fatalf("message %s was not dispatched", message.ID)
+		}
+	}
+	<-slowStarted
+	<-fastReturned
+	deadline := time.Now().Add(time.Second)
+	for {
+		pending, pendingErr := client.XPending(ctx, worker.stream(), worker.group()).Result()
+		if pendingErr != nil {
+			t.Fatalf("xpending: %v", pendingErr)
+		}
+		if pending.Count == 1 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("pending count = %d, want 1 while slow sibling is still running", pending.Count)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	close(releaseSlow)
+	pool.wait()
+}
+
+func TestPendingRecoveryDoesNotDuplicateAnActiveOrAlreadyAcknowledgedMessage(t *testing.T) {
+	mr := miniredis.RunT(t)
+	client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+	worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", MinIdle: time.Millisecond}
+	ctx := context.Background()
+	if err := worker.ensureGroup(ctx); err != nil {
+		t.Fatalf("ensureGroup: %v", err)
+	}
+	if err := client.XAdd(ctx, &redis.XAddArgs{Stream: worker.stream(), ID: "3-0", Values: submitCommandValues("message-active", "channel-active")}).Err(); err != nil {
+		t.Fatalf("xadd: %v", err)
+	}
+	streams, err := client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: worker.group(), Consumer: worker.consumer(), Streams: []string{worker.stream(), ">"}, Count: 1}).Result()
+	if err != nil || len(streams) != 1 || len(streams[0].Messages) != 1 {
+		t.Fatalf("xreadgroup: streams=%+v err=%v", streams, err)
+	}
+	message := streams[0].Messages[0]
+	started := make(chan struct{})
+	release := make(chan struct{})
+	var submits atomic.Int32
+	worker.Submit = func(_ context.Context, _ queue.SubmitCommand) (queue.SubmitResult, error) {
+		submits.Add(1)
+		close(started)
+		<-release
+		return queue.SubmitResult{SubmitStatus: "accepted"}, nil
+	}
+	pool := newMessageWorkPool(ctx, worker, 2)
+	if !pool.dispatch(message) {
+		t.Fatal("active message was not dispatched")
+	}
+	<-started
+	time.Sleep(5 * time.Millisecond)
+	if err := worker.recoverPending(ctx, pool); err != nil {
+		t.Fatalf("recoverPending: %v", err)
+	}
+	if got := submits.Load(); got != 1 {
+		t.Fatalf("active message submit count = %d, want 1", got)
+	}
+	close(release)
+	pool.wait()
+	if err := pool.dispatchRecovered(ctx, message); err != nil {
+		t.Fatalf("dispatch acknowledged recovery: %v", err)
+	}
+	if got := submits.Load(); got != 1 {
+		t.Fatalf("acknowledged message submit count = %d, want 1", got)
 	}
 }