hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: expand gateway capacity and prevent receipt replay

Browse Source
This commit is contained in:
hectorzhao
2026-08-25 16:08:30 +08:00
parent 761c123b65
commit 9292352be1
48 changed files with 2001 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/internal/resultoutbox/batch.go
+170
View File
@@ -0,0 +1,170 @@
package resultoutbox
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
type callbackBatch struct {
BatchID string `json:"batchId"`
GatewayInstanceID string `json:"gatewayInstanceId"`
CreatedAt time.Time `json:"createdAt"`
Events []callbackBatchEvent `json:"events"`
}
type callbackBatchEvent struct {
EventID string `json:"eventId"`
Type string `json:"type"`
Payload json.RawMessage `json:"payload"`
}
type callbackBatchResponse struct {
BatchID string `json:"batchId"`
Results []callbackEventResult `json:"results"`
}
type callbackEventResult struct {
EventID string `json:"eventId"`
Accepted bool `json:"accepted"`
Retryable bool `json:"retryable,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
}
func (o *Outbox) runBatches(ctx context.Context) error {
for ctx.Err() == nil {
messages, _, err := o.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: o.stream(), Group: o.group(), Consumer: o.consumer(), MinIdle: o.minIdle(), Start: "0-0", Count: int64(o.batchSize())}).Result()
if err != nil && !errors.Is(err, redis.Nil) {
return err
}
if len(messages) == 0 {
streams, readErr := o.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{Group: o.group(), Consumer: o.consumer(), Streams: []string{o.stream(), ">"}, Count: int64(o.batchSize()), Block: o.batchWait()}).Result()
if errors.Is(readErr, redis.Nil) {
continue
}
if readErr != nil {
return readErr
}
for _, stream := range streams {
messages = append(messages, stream.Messages...)
}
}
if len(messages) == 0 {
continue
}
if err := o.processBatch(ctx, messages); err != nil {
o.batchRetries.Add(int64(len(messages)))
sleep(ctx, 100*time.Millisecond)
}
}
return ctx.Err()
}
func (o *Outbox) processBatch(ctx context.Context, messages []redis.XMessage) error {
batch := callbackBatch{BatchID: fmt.Sprintf("CB-%d", time.Now().UnixNano()), GatewayInstanceID: o.GatewayInstanceID, CreatedAt: time.Now().UTC()}
byEvent := make(map[string]redis.XMessage, len(messages))
for _, message := range messages {
event, err := EventFromStreamValues(message.Values)
if err != nil {
_ = o.deadLetter(ctx, message, "INVALID_ENVELOPE", err.Error())
continue
}
batch.Events = append(batch.Events, callbackBatchEvent{EventID: event.EventID, Type: event.EventType, Payload: event.Payload})
byEvent[event.EventID] = message
}
if len(batch.Events) == 0 {
return nil
}
body, err := json.Marshal(batch)
if err != nil {
return err
}
if len(body) > 1024*1024 {
return fmt.Errorf("callback batch exceeds 1MB")
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(o.APIBaseURL, "/")+"/gateway/events/batch", bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: o.httpTimeout()}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
data, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return fmt.Errorf("batch callback returned %d: %s", resp.StatusCode, strings.TrimSpace(string(data)))
}
var result callbackBatchResponse
if err := json.NewDecoder(io.LimitReader(resp.Body, 1024*1024)).Decode(&result); err != nil {
return err
}
if result.BatchID != batch.BatchID {
return fmt.Errorf("callback batchId mismatch")
}
o.batchRequests.Add(1)
o.batchEvents.Add(int64(len(batch.Events)))
seen := make(map[string]struct{}, len(result.Results))
for _, item := range result.Results {
message, ok := byEvent[item.EventID]
if !ok {
continue
}
seen[item.EventID] = struct{}{}
if item.Accepted {
if err := o.ackDelete(ctx, message.ID); err != nil {
return err
}
continue
}
if !item.Retryable {
if err := o.deadLetter(ctx, message, item.ErrorCode, "non-retryable callback result"); err != nil {
return err
}
}
}
for eventID := range byEvent {
if _, ok := seen[eventID]; !ok {
return fmt.Errorf("batch response omitted event %s", eventID)
}
}
return nil
}
func (o *Outbox) ackDelete(ctx context.Context, id string) error {
return acknowledgeAndDeleteScript.Run(ctx, o.Redis, []string{o.stream()}, o.group(), id).Err()
}
func (o *Outbox) deadLetter(ctx context.Context, message redis.XMessage, code, detail string) error {
stream := o.DeadLetterStream
if stream == "" {
stream = o.stream() + ".dead"
}
pipe := o.Redis.TxPipeline()
pipe.XAdd(ctx, &redis.XAddArgs{Stream: stream, Values: map[string]any{"sourceId": message.ID, "errorCode": code, "detail": detail, "data": fmt.Sprint(message.Values["data"])}})
pipe.XAck(ctx, o.stream(), o.group(), message.ID)
pipe.XDel(ctx, o.stream(), message.ID)
if _, err := pipe.Exec(ctx); err != nil {
return err
}
o.deadLetters.Add(1)
return nil
}
func (o *Outbox) batchSize() int {
if o.BatchSize < 1 {
return 50
}
return min(o.BatchSize, 100)
}
func (o *Outbox) batchWait() time.Duration {
if o.BatchWait <= 0 {
return 10 * time.Millisecond
}
return o.BatchWait
}
gateway/internal/resultoutbox/outbox.go
+36 -12
View File
@@ -53,16 +53,25 @@ type Event struct {
}
type Outbox struct {
Redis *redis.Client
Stream string
Group string
Consumer string
DedupeTTL time.Duration
APIBaseURL string
HTTPTimeout time.Duration
Concurrency int
MinIdle time.Duration
inFlight atomic.Int64
Redis *redis.Client
Stream string
Group string
Consumer string
DedupeTTL time.Duration
APIBaseURL string
HTTPTimeout time.Duration
Concurrency int
MinIdle time.Duration
BatchEnabled bool
BatchSize int
BatchWait time.Duration
GatewayInstanceID string
DeadLetterStream string
inFlight atomic.Int64
batchRequests atomic.Int64
batchEvents atomic.Int64
batchRetries atomic.Int64
deadLetters atomic.Int64
}
func New(client *redis.Client) *Outbox {
@@ -145,6 +154,18 @@ func (o *Outbox) PublishSubmitResult(ctx context.Context, command queue.SubmitCo
return o.publish(ctx, event)
}
func (o *Outbox) PublishReceipt(ctx context.Context, event queue.ReceiptEvent) error {
return o.publishRaw(ctx, Event{SchemaVersion: queue.SchemaVersion, EventID: fmt.Sprintf("receipt:%s:%s:%d", event.GatewayMessageID, event.RawStatus, event.SequenceID), EventType: "receipt_intake", Path: "/gateway/events/receipt/intake", TraceID: event.TraceID, MessageID: event.MessageID, ChannelID: event.ChannelID, Payload: mustMarshal(event), CreatedAt: time.Now().UTC()})
}
func (o *Outbox) PublishUplink(ctx context.Context, event queue.UplinkEvent) error {
return o.publishRaw(ctx, Event{SchemaVersion: queue.SchemaVersion, EventID: fmt.Sprintf("uplink:%s:%d:%d", event.ChannelID, event.SequenceID, event.ReceivedAt.UnixNano()), EventType: "uplink", Path: "/gateway/events/uplink", TraceID: event.TraceID, MessageID: event.MessageID, ChannelID: event.ChannelID, Payload: mustMarshal(event), CreatedAt: time.Now().UTC()})
}
func mustMarshal(value any) json.RawMessage { data, _ := json.Marshal(value); return data }
func (o *Outbox) publishRaw(ctx context.Context, event Event) error { return o.publish(ctx, event) }
func (o *Outbox) publish(ctx context.Context, event Event) error {
if o.Redis == nil {
return fmt.Errorf("result Outbox Redis client is required")
@@ -211,10 +232,10 @@ func EventFromStreamValues(values map[string]interface{}) (Event, error) {
if err := json.Unmarshal([]byte(data), &event); err != nil {
return Event{}, err
}
if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" || event.SubmitID == "" {
if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" {
return Event{}, fmt.Errorf("invalid result Outbox envelope")
}
if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" {
if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" && event.Path != "/gateway/events/receipt/intake" && event.Path != "/gateway/events/uplink" && event.Path != "/gateway/events/dead-letter" {
return Event{}, fmt.Errorf("unsupported result Outbox path %q", event.Path)
}
if len(event.Payload) == 0 {
@@ -258,3 +279,6 @@ func (o *Outbox) dedupeKey(eventID string) string {
func (o *Outbox) StreamName() string { return o.stream() }
func (o *Outbox) GroupName() string { return o.group() }
func (o *Outbox) InFlight() int64 { return o.inFlight.Load() }
func (o *Outbox) BatchCounts() (int64, int64, int64, int64) {
return o.batchRequests.Load(), o.batchEvents.Load(), o.batchRetries.Load(), o.deadLetters.Load()
}
gateway/internal/resultoutbox/outbox_test.go
+94
View File
@@ -2,6 +2,7 @@ package resultoutbox
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"sync/atomic"
@@ -32,6 +33,99 @@ func TestPublishSubmitSegmentIsIdempotent(t *testing.T) {
}
}
func TestBatchCallbackSendsMultipleEventsInOneRequest(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
var requests atomic.Int32
var eventCount atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
requests.Add(1)
var batch callbackBatch
if err := json.NewDecoder(request.Body).Decode(&batch); err != nil {
t.Errorf("decode batch: %v", err)
return
}
eventCount.Store(int32(len(batch.Events)))
result := callbackBatchResponse{BatchID: batch.BatchID}
for _, event := range batch.Events {
result.Results = append(result.Results, callbackEventResult{EventID: event.EventID, Accepted: true})
}
_ = json.NewEncoder(response).Encode(result)
}))
defer server.Close()
outbox := New(client)
outbox.APIBaseURL = server.URL
outbox.BatchEnabled = true
outbox.BatchSize = 50
outbox.BatchWait = 10 * time.Millisecond
outbox.GatewayInstanceID = "gateway-test"
command := testCommand()
for index := 1; index <= 2; index++ {
if err := outbox.PublishSubmitSegment(context.Background(), command, queue.SubmitSegmentResult{SegmentTotal: 2, SegmentIndex: index, SequenceID: uint32(index), GatewayMessageID: "88", SubmitStatus: "accepted"}); err != nil {
t.Fatal(err)
}
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- outbox.Run(ctx) }()
deadline := time.Now().Add(3 * time.Second)
for client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
if time.Now().After(deadline) {
t.Fatal("batch did not drain")
}
time.Sleep(10 * time.Millisecond)
}
cancel()
<-done
if requests.Load() != 1 || eventCount.Load() != 2 {
t.Fatalf("requests/events=%d/%d, want 1/2", requests.Load(), eventCount.Load())
}
}
func TestBatchCallbackReplaysWholeRequestAfterHTTPFailure(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
var batch callbackBatch
_ = json.NewDecoder(request.Body).Decode(&batch)
if calls.Add(1) == 1 {
http.Error(response, "busy", http.StatusServiceUnavailable)
return
}
result := callbackBatchResponse{BatchID: batch.BatchID}
for _, event := range batch.Events {
result.Results = append(result.Results, callbackEventResult{EventID: event.EventID, Accepted: true})
}
_ = json.NewEncoder(response).Encode(result)
}))
defer server.Close()
outbox := New(client)
outbox.APIBaseURL = server.URL
outbox.BatchEnabled = true
outbox.BatchWait = 5 * time.Millisecond
outbox.MinIdle = 5 * time.Millisecond
outbox.GatewayInstanceID = "g"
if err := outbox.PublishSubmitSegment(context.Background(), testCommand(), queue.SubmitSegmentResult{SegmentTotal: 1, SegmentIndex: 1, SequenceID: 1, GatewayMessageID: "1", SubmitStatus: "accepted"}); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- outbox.Run(ctx) }()
deadline := time.Now().Add(3 * time.Second)
for client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
if time.Now().After(deadline) {
t.Fatal("replayed batch did not drain")
}
time.Sleep(10 * time.Millisecond)
}
cancel()
<-done
if calls.Load() < 2 {
t.Fatalf("calls=%d want replay", calls.Load())
}
}
func TestPublishAggregateAndCommandAckAreAtomicAndIdempotent(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
gateway/internal/resultoutbox/worker.go
+3
View File
@@ -46,6 +46,9 @@ func (o *Outbox) Run(ctx context.Context) error {
if err := o.ensureGroup(ctx); err != nil {
return err
}
if o.BatchEnabled {
return o.runBatches(ctx)
}
pool := newCallbackPool(ctx, o, o.concurrency())
defer pool.wait()
for {