perf(cmpp): decouple supplier result callbacks

2026-08-20 14:19:42 +08:00
parent 485af688d2
commit 67b760a599
27 changed files with 1000 additions and 66 deletions
@@ -0,0 +1,275 @@
+package resultoutbox
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"cmpp-platform/gateway/internal/metrics"
+
+	"github.com/redis/go-redis/v9"
+)
+
+const (
+	// Eight callbacks matched the test VM's API/PostgreSQL capacity through
+	// 40 rps. A larger default caused callback bursts to contend with inbound
+	// persistence; operators can still raise it after measuring both queues.
+	defaultConcurrency = 8
+	// Keep the reclaim threshold above the callback timeout. Otherwise another
+	// Gateway replica could reclaim a still-running callback and execute the same
+	// business transition concurrently before the API stores its idempotency ID.
+	defaultMinIdle     = 30 * time.Second
+	defaultBlock       = 2 * time.Second
+	defaultHTTPTimeout = 10 * time.Second
+)
+
+var acknowledgeAndDeleteScript = redis.NewScript(`
+redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
+redis.call('XDEL', KEYS[1], ARGV[2])
+return 1
+`)
+
+func (o *Outbox) Run(ctx context.Context) error {
+	if o.Redis == nil {
+		return fmt.Errorf("result Outbox Redis client is required")
+	}
+	if strings.TrimSpace(o.APIBaseURL) == "" {
+		return fmt.Errorf("result Outbox API base URL is required")
+	}
+	if err := o.ensureGroup(ctx); err != nil {
+		return err
+	}
+	pool := newCallbackPool(ctx, o, o.concurrency())
+	defer pool.wait()
+	for {
+		if err := o.recoverPending(ctx, pool); err != nil && ctx.Err() == nil {
+			log.Printf("gateway result Outbox pending recovery failed: %v", err)
+			sleep(ctx, time.Second)
+			continue
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if err := o.consumeOnce(ctx, pool); err != nil && ctx.Err() == nil {
+			log.Printf("gateway result Outbox consume failed: %v", err)
+			sleep(ctx, time.Second)
+			continue
+		}
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+	}
+}
+
+func (o *Outbox) ensureGroup(ctx context.Context) error {
+	err := o.Redis.XGroupCreateMkStream(ctx, o.stream(), o.group(), "0").Err()
+	if err == nil || strings.Contains(err.Error(), "BUSYGROUP") {
+		return nil
+	}
+	return err
+}
+
+func (o *Outbox) consumeOnce(ctx context.Context, pool *callbackPool) error {
+	available, err := pool.waitForCapacity(ctx)
+	if err != nil {
+		return err
+	}
+	streams, err := o.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{
+		Group: o.group(), Consumer: o.consumer(), Streams: []string{o.stream(), ">"},
+		Count: int64(available), Block: defaultBlock,
+	}).Result()
+	if errors.Is(err, redis.Nil) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	for _, stream := range streams {
+		for _, message := range stream.Messages {
+			if !pool.dispatch(message) {
+				return fmt.Errorf("gateway result Outbox capacity accounting mismatch")
+			}
+		}
+	}
+	return nil
+}
+
+func (o *Outbox) recoverPending(ctx context.Context, pool *callbackPool) error {
+	if pool.available() == 0 {
+		return nil
+	}
+	messages, _, err := o.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{
+		Stream: o.stream(), Group: o.group(), Consumer: o.consumer(), MinIdle: o.minIdle(),
+		Start: "0-0", Count: int64(pool.available()),
+	}).Result()
+	if errors.Is(err, redis.Nil) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	for _, message := range messages {
+		if !pool.dispatch(message) {
+			return fmt.Errorf("gateway result Outbox recovery capacity accounting mismatch")
+		}
+	}
+	return nil
+}
+
+func (o *Outbox) processMessage(ctx context.Context, message redis.XMessage) error {
+	event, err := EventFromStreamValues(message.Values)
+	if err != nil {
+		// Malformed internal events cannot be delivered. Keep them pending for operator
+		// evidence instead of ACKing and silently losing a supplier result.
+		return err
+	}
+	startedAt := time.Now()
+	err = o.post(ctx, event)
+	metrics.ObserveSubmitStage("api_callback", err == nil, time.Since(startedAt))
+	if err != nil {
+		return err
+	}
+	// Result events have a bounded dedupe key, so successful callbacks can be
+	// ACKed and deleted atomically instead of turning the Outbox into an archive.
+	return acknowledgeAndDeleteScript.Run(
+		ctx,
+		o.Redis,
+		[]string{o.stream()},
+		o.group(),
+		message.ID,
+	).Err()
+}
+
+func (o *Outbox) post(ctx context.Context, event Event) error {
+	client := &http.Client{Timeout: o.httpTimeout()}
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		strings.TrimRight(o.APIBaseURL, "/")+event.Path,
+		bytes.NewReader(event.Payload),
+	)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-CMPP-Result-Event-ID", event.EventID)
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
+		return fmt.Errorf("result callback returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	return nil
+}
+
+type callbackPool struct {
+	ctx       context.Context
+	outbox    *Outbox
+	slots     chan struct{}
+	completed chan struct{}
+	group     sync.WaitGroup
+	mu        sync.Mutex
+	active    map[string]struct{}
+}
+
+func newCallbackPool(ctx context.Context, outbox *Outbox, concurrency int) *callbackPool {
+	return &callbackPool{
+		ctx: ctx, outbox: outbox, slots: make(chan struct{}, concurrency), completed: make(chan struct{}, concurrency), active: make(map[string]struct{}),
+	}
+}
+
+func (p *callbackPool) available() int { return cap(p.slots) - len(p.slots) }
+
+func (p *callbackPool) waitForCapacity(ctx context.Context) (int, error) {
+	for p.available() == 0 {
+		select {
+		case <-ctx.Done():
+			return 0, ctx.Err()
+		case <-p.completed:
+		}
+	}
+	return p.available(), nil
+}
+
+func (p *callbackPool) dispatch(message redis.XMessage) bool {
+	p.mu.Lock()
+	if _, exists := p.active[message.ID]; exists {
+		p.mu.Unlock()
+		return true
+	}
+	select {
+	case p.slots <- struct{}{}:
+		p.active[message.ID] = struct{}{}
+		p.outbox.inFlight.Add(1)
+		p.group.Add(1)
+		p.mu.Unlock()
+	case <-p.ctx.Done():
+		p.mu.Unlock()
+		return false
+	default:
+		p.mu.Unlock()
+		return false
+	}
+	go func() {
+		defer func() {
+			p.mu.Lock()
+			delete(p.active, message.ID)
+			p.mu.Unlock()
+			<-p.slots
+			p.outbox.inFlight.Add(-1)
+			select {
+			case p.completed <- struct{}{}:
+			default:
+			}
+			p.group.Done()
+		}()
+		if err := p.outbox.processMessage(p.ctx, message); err != nil {
+			log.Printf("gateway result Outbox event %s failed: %v", message.ID, err)
+		}
+	}()
+	return true
+}
+
+func (p *callbackPool) wait() { p.group.Wait() }
+
+func (o *Outbox) concurrency() int {
+	if o.Concurrency > 0 {
+		return min(o.Concurrency, 1024)
+	}
+	return defaultConcurrency
+}
+
+func (o *Outbox) minIdle() time.Duration {
+	if o.MinIdle > 0 {
+		return o.MinIdle
+	}
+	return defaultMinIdle
+}
+
+func (o *Outbox) httpTimeout() time.Duration {
+	if o.HTTPTimeout > 0 {
+		return o.HTTPTimeout
+	}
+	return defaultHTTPTimeout
+}
+
+func (o *Outbox) ConfiguredConcurrency() int { return o.concurrency() }
+
+func sleep(ctx context.Context, duration time.Duration) {
+	timer := time.NewTimer(duration)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+	case <-timer.C:
+	}
+}