perf: expand gateway capacity and prevent receipt replay

gateway/internal/resultoutbox/outbox_test.go

@@ -2,6 +2,7 @@ package resultoutbox
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"sync/atomic"
@@ -32,6 +33,99 @@ func TestPublishSubmitSegmentIsIdempotent(t *testing.T) {
 	}
 }
 
+func TestBatchCallbackSendsMultipleEventsInOneRequest(t *testing.T) {
+	mr := miniredis.RunT(t)
+	client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+	var requests atomic.Int32
+	var eventCount atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+		requests.Add(1)
+		var batch callbackBatch
+		if err := json.NewDecoder(request.Body).Decode(&batch); err != nil {
+			t.Errorf("decode batch: %v", err)
+			return
+		}
+		eventCount.Store(int32(len(batch.Events)))
+		result := callbackBatchResponse{BatchID: batch.BatchID}
+		for _, event := range batch.Events {
+			result.Results = append(result.Results, callbackEventResult{EventID: event.EventID, Accepted: true})
+		}
+		_ = json.NewEncoder(response).Encode(result)
+	}))
+	defer server.Close()
+	outbox := New(client)
+	outbox.APIBaseURL = server.URL
+	outbox.BatchEnabled = true
+	outbox.BatchSize = 50
+	outbox.BatchWait = 10 * time.Millisecond
+	outbox.GatewayInstanceID = "gateway-test"
+	command := testCommand()
+	for index := 1; index <= 2; index++ {
+		if err := outbox.PublishSubmitSegment(context.Background(), command, queue.SubmitSegmentResult{SegmentTotal: 2, SegmentIndex: index, SequenceID: uint32(index), GatewayMessageID: "88", SubmitStatus: "accepted"}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- outbox.Run(ctx) }()
+	deadline := time.Now().Add(3 * time.Second)
+	for client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
+		if time.Now().After(deadline) {
+			t.Fatal("batch did not drain")
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	cancel()
+	<-done
+	if requests.Load() != 1 || eventCount.Load() != 2 {
+		t.Fatalf("requests/events=%d/%d, want 1/2", requests.Load(), eventCount.Load())
+	}
+}
+
+func TestBatchCallbackReplaysWholeRequestAfterHTTPFailure(t *testing.T) {
+	mr := miniredis.RunT(t)
+	client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
+	var calls atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+		var batch callbackBatch
+		_ = json.NewDecoder(request.Body).Decode(&batch)
+		if calls.Add(1) == 1 {
+			http.Error(response, "busy", http.StatusServiceUnavailable)
+			return
+		}
+		result := callbackBatchResponse{BatchID: batch.BatchID}
+		for _, event := range batch.Events {
+			result.Results = append(result.Results, callbackEventResult{EventID: event.EventID, Accepted: true})
+		}
+		_ = json.NewEncoder(response).Encode(result)
+	}))
+	defer server.Close()
+	outbox := New(client)
+	outbox.APIBaseURL = server.URL
+	outbox.BatchEnabled = true
+	outbox.BatchWait = 5 * time.Millisecond
+	outbox.MinIdle = 5 * time.Millisecond
+	outbox.GatewayInstanceID = "g"
+	if err := outbox.PublishSubmitSegment(context.Background(), testCommand(), queue.SubmitSegmentResult{SegmentTotal: 1, SegmentIndex: 1, SequenceID: 1, GatewayMessageID: "1", SubmitStatus: "accepted"}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- outbox.Run(ctx) }()
+	deadline := time.Now().Add(3 * time.Second)
+	for client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
+		if time.Now().After(deadline) {
+			t.Fatal("replayed batch did not drain")
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	cancel()
+	<-done
+	if calls.Load() < 2 {
+		t.Fatalf("calls=%d want replay", calls.Load())
+	}
+}
+
 func TestPublishAggregateAndCommandAckAreAtomicAndIdempotent(t *testing.T) {
 	mr := miniredis.RunT(t)
 	client := redis.NewClient(&redis.Options{Addr: mr.Addr()})