feat: complete cmpp gateway delivery recovery workflows

2026-07-08 16:30:06 +08:00
parent cc628d0214
commit 8144f08652
60 changed files with 8901 additions and 94 deletions
@@ -0,0 +1,101 @@
+package inbound
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+)
+
+func TestRedisRecoveryStoreBackoffAndStatuses(t *testing.T) {
+	mr := miniredis.RunT(t)
+	store, err := NewRedisRecoveryStore("redis://" + mr.Addr())
+	if err != nil {
+		t.Fatalf("new recovery store: %v", err)
+	}
+	store.Prefix = "test:recovery"
+
+	decision, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
+	if err != nil {
+		t.Fatalf("start recovery: %v", err)
+	}
+	if !decision.Allowed {
+		t.Fatalf("expected recovery allowed, got %+v", decision)
+	}
+	if decision.Status.LockOwner != "gateway-a" || decision.Status.LockToken == "" || decision.Status.LockExpiresAt.IsZero() {
+		t.Fatalf("missing recovery lock metadata: %+v", decision.Status)
+	}
+	status := decision.Status
+	status.State = "waiting_connection"
+	if err := store.CompleteAccountRecovery(context.Background(), status); err != nil {
+		t.Fatalf("complete recovery: %v", err)
+	}
+
+	statuses, err := store.ListRecoveryStatuses(context.Background())
+	if err != nil {
+		t.Fatalf("list recovery statuses: %v", err)
+	}
+	if len(statuses) != 1 || statuses[0].State != "waiting_connection" || statuses[0].AttemptCount != 1 {
+		t.Fatalf("unexpected statuses: %+v", statuses)
+	}
+	if statuses[0].FailureCategory != "client_disconnected" {
+		t.Fatalf("failure category = %q, want client_disconnected", statuses[0].FailureCategory)
+	}
+	if statuses[0].NextRetryAt.IsZero() {
+		t.Fatalf("expected next retry at after waiting connection: %+v", statuses[0])
+	}
+
+	decision, err = store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
+	if err != nil {
+		t.Fatalf("start recovery second time: %v", err)
+	}
+	if decision.Allowed || decision.SkipReason != "backoff" {
+		t.Fatalf("expected backoff skip, got %+v", decision)
+	}
+	if decision.Status.FailureCategory != "backoff" {
+		t.Fatalf("skip failure category = %q, want backoff", decision.Status.FailureCategory)
+	}
+}
+
+func TestRedisRecoveryStoreDoesNotReleaseLockOwnedByAnotherGateway(t *testing.T) {
+	mr := miniredis.RunT(t)
+	store, err := NewRedisRecoveryStore("redis://" + mr.Addr())
+	if err != nil {
+		t.Fatalf("new recovery store: %v", err)
+	}
+	store.Prefix = "test:recovery:takeover"
+
+	first, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
+	if err != nil {
+		t.Fatalf("start first recovery: %v", err)
+	}
+	if !first.Allowed {
+		t.Fatalf("expected first recovery allowed, got %+v", first)
+	}
+
+	mr.FastForward(defaultRecoveryLockTTL + time.Second)
+	second, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-b")
+	if err != nil {
+		t.Fatalf("start second recovery: %v", err)
+	}
+	if !second.Allowed {
+		t.Fatalf("expected second recovery to take over expired lock, got %+v", second)
+	}
+
+	stale := first.Status
+	stale.State = "success"
+	err = store.CompleteAccountRecovery(context.Background(), stale)
+	if !errors.Is(err, ErrRecoveryLockLost) {
+		t.Fatalf("stale completion error = %v, want ErrRecoveryLockLost", err)
+	}
+
+	lockValue, err := store.Client.Get(context.Background(), store.lockKey("100001")).Result()
+	if err != nil {
+		t.Fatalf("load current lock: %v", err)
+	}
+	if lockValue != second.Status.LockToken {
+		t.Fatalf("current lock was changed by stale completion: got %q want %q", lockValue, second.Status.LockToken)
+	}
+}