lislgosms/gateway/internal/resultoutbox/worker.go

package resultoutbox

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"

	"cmpp-platform/gateway/internal/metrics"

	"github.com/redis/go-redis/v9"
)

const (
	// Eight callbacks matched the test VM's API/PostgreSQL capacity through
	// 40 rps. A larger default caused callback bursts to contend with inbound
	// persistence; operators can still raise it after measuring both queues.
	defaultConcurrency = 8
	// Keep the reclaim threshold above the callback timeout. Otherwise another
	// Gateway replica could reclaim a still-running callback and execute the same
	// business transition concurrently before the API stores its idempotency ID.
	defaultMinIdle     = 30 * time.Second
	defaultBlock       = 2 * time.Second
	defaultHTTPTimeout = 10 * time.Second
)

var acknowledgeAndDeleteScript = redis.NewScript(`
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
redis.call('XDEL', KEYS[1], ARGV[2])
return 1
`)

func (o *Outbox) Run(ctx context.Context) error {
	if o.Redis == nil {
		return fmt.Errorf("result Outbox Redis client is required")
	}
	if strings.TrimSpace(o.APIBaseURL) == "" {
		return fmt.Errorf("result Outbox API base URL is required")
	}
	if o.BatchEnabled {
		return retryUntilCanceled(ctx, time.Second, func() error {
			if err := o.ensureGroup(ctx); err != nil {
				log.Printf("gateway result Outbox group initialization failed: %v", err)
				return err
			}
			if err := o.runBatches(ctx); err != nil && ctx.Err() == nil {
				// A transient Redis read error must not permanently stop the only
				// consumer for submit results, receipts, and uplink events.
				log.Printf("gateway result Outbox batch consume failed: %v", err)
				return err
			}
			return ctx.Err()
		})
	}
	if err := o.ensureGroup(ctx); err != nil {
		return err
	}
	pool := newCallbackPool(ctx, o, o.concurrency())
	defer pool.wait()
	for {
		if err := o.recoverPending(ctx, pool); err != nil && ctx.Err() == nil {
			log.Printf("gateway result Outbox pending recovery failed: %v", err)
			sleep(ctx, time.Second)
			continue
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
		if err := o.consumeOnce(ctx, pool); err != nil && ctx.Err() == nil {
			log.Printf("gateway result Outbox consume failed: %v", err)
			sleep(ctx, time.Second)
			continue
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
	}
}

func (o *Outbox) ensureGroup(ctx context.Context) error {
	err := o.Redis.XGroupCreateMkStream(ctx, o.stream(), o.group(), "0").Err()
	if err == nil || strings.Contains(err.Error(), "BUSYGROUP") {
		return nil
	}
	return err
}

func (o *Outbox) consumeOnce(ctx context.Context, pool *callbackPool) error {
	available, err := pool.waitForCapacity(ctx)
	if err != nil {
		return err
	}
	streams, err := o.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{
		Group: o.group(), Consumer: o.consumer(), Streams: []string{o.stream(), ">"},
		Count: int64(available), Block: defaultBlock,
	}).Result()
	if errors.Is(err, redis.Nil) {
		return nil
	}
	if err != nil {
		return err
	}
	for _, stream := range streams {
		for _, message := range stream.Messages {
			if !pool.dispatch(message) {
				return fmt.Errorf("gateway result Outbox capacity accounting mismatch")
			}
		}
	}
	return nil
}

func (o *Outbox) recoverPending(ctx context.Context, pool *callbackPool) error {
	if pool.available() == 0 {
		return nil
	}
	messages, _, err := o.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{
		Stream: o.stream(), Group: o.group(), Consumer: o.consumer(), MinIdle: o.minIdle(),
		Start: "0-0", Count: int64(pool.available()),
	}).Result()
	if errors.Is(err, redis.Nil) {
		return nil
	}
	if err != nil {
		return err
	}
	for _, message := range messages {
		if !pool.dispatch(message) {
			return fmt.Errorf("gateway result Outbox recovery capacity accounting mismatch")
		}
	}
	return nil
}

func (o *Outbox) processMessage(ctx context.Context, message redis.XMessage) error {
	event, err := EventFromStreamValues(message.Values)
	if err != nil {
		// Malformed internal events cannot be delivered. Keep them pending for operator
		// evidence instead of ACKing and silently losing a supplier result.
		return err
	}
	startedAt := time.Now()
	err = o.post(ctx, event)
	metrics.ObserveSubmitStage("api_callback", err == nil, time.Since(startedAt))
	if err != nil {
		return err
	}
	// Result events have a bounded dedupe key, so successful callbacks can be
	// ACKed and deleted atomically instead of turning the Outbox into an archive.
	return acknowledgeAndDeleteScript.Run(
		ctx,
		o.Redis,
		[]string{o.stream()},
		o.group(),
		message.ID,
	).Err()
}

func (o *Outbox) post(ctx context.Context, event Event) error {
	client := &http.Client{Timeout: o.httpTimeout()}
	req, err := http.NewRequestWithContext(
		ctx,
		http.MethodPost,
		strings.TrimRight(o.APIBaseURL, "/")+event.Path,
		bytes.NewReader(event.Payload),
	)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-CMPP-Result-Event-ID", event.EventID)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
		body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
		return fmt.Errorf("result callback returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
	}
	return nil
}

type callbackPool struct {
	ctx       context.Context
	outbox    *Outbox
	slots     chan struct{}
	completed chan struct{}
	group     sync.WaitGroup
	mu        sync.Mutex
	active    map[string]struct{}
}

func newCallbackPool(ctx context.Context, outbox *Outbox, concurrency int) *callbackPool {
	return &callbackPool{
		ctx: ctx, outbox: outbox, slots: make(chan struct{}, concurrency), completed: make(chan struct{}, concurrency), active: make(map[string]struct{}),
	}
}

func (p *callbackPool) available() int { return cap(p.slots) - len(p.slots) }

func (p *callbackPool) waitForCapacity(ctx context.Context) (int, error) {
	for p.available() == 0 {
		select {
		case <-ctx.Done():
			return 0, ctx.Err()
		case <-p.completed:
		}
	}
	return p.available(), nil
}

func (p *callbackPool) dispatch(message redis.XMessage) bool {
	p.mu.Lock()
	if _, exists := p.active[message.ID]; exists {
		p.mu.Unlock()
		return true
	}
	select {
	case p.slots <- struct{}{}:
		p.active[message.ID] = struct{}{}
		p.outbox.inFlight.Add(1)
		p.group.Add(1)
		p.mu.Unlock()
	case <-p.ctx.Done():
		p.mu.Unlock()
		return false
	default:
		p.mu.Unlock()
		return false
	}
	go func() {
		defer func() {
			p.mu.Lock()
			delete(p.active, message.ID)
			p.mu.Unlock()
			<-p.slots
			p.outbox.inFlight.Add(-1)
			select {
			case p.completed <- struct{}{}:
			default:
			}
			p.group.Done()
		}()
		if err := p.outbox.processMessage(p.ctx, message); err != nil {
			log.Printf("gateway result Outbox event %s failed: %v", message.ID, err)
		}
	}()
	return true
}

func (p *callbackPool) wait() { p.group.Wait() }

func (o *Outbox) concurrency() int {
	if o.Concurrency > 0 {
		return min(o.Concurrency, 1024)
	}
	return defaultConcurrency
}

func (o *Outbox) minIdle() time.Duration {
	if o.MinIdle > 0 {
		return o.MinIdle
	}
	return defaultMinIdle
}

func (o *Outbox) httpTimeout() time.Duration {
	if o.HTTPTimeout > 0 {
		return o.HTTPTimeout
	}
	return defaultHTTPTimeout
}

func (o *Outbox) ConfiguredConcurrency() int { return o.concurrency() }

func sleep(ctx context.Context, duration time.Duration) {
	timer := time.NewTimer(duration)
	defer timer.Stop()
	select {
	case <-ctx.Done():
	case <-timer.C:
	}
}

func retryUntilCanceled(ctx context.Context, delay time.Duration, operation func() error) error {
	for ctx.Err() == nil {
		if err := operation(); err != nil && ctx.Err() == nil {
			sleep(ctx, delay)
		}
	}
	return ctx.Err()
}