102 lines
3.3 KiB
Go
102 lines
3.3 KiB
Go
package inbound
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
)
|
|
|
|
func TestRedisRecoveryStoreBackoffAndStatuses(t *testing.T) {
|
|
mr := miniredis.RunT(t)
|
|
store, err := NewRedisRecoveryStore("redis://" + mr.Addr())
|
|
if err != nil {
|
|
t.Fatalf("new recovery store: %v", err)
|
|
}
|
|
store.Prefix = "test:recovery"
|
|
|
|
decision, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
|
|
if err != nil {
|
|
t.Fatalf("start recovery: %v", err)
|
|
}
|
|
if !decision.Allowed {
|
|
t.Fatalf("expected recovery allowed, got %+v", decision)
|
|
}
|
|
if decision.Status.LockOwner != "gateway-a" || decision.Status.LockToken == "" || decision.Status.LockExpiresAt.IsZero() {
|
|
t.Fatalf("missing recovery lock metadata: %+v", decision.Status)
|
|
}
|
|
status := decision.Status
|
|
status.State = "waiting_connection"
|
|
if err := store.CompleteAccountRecovery(context.Background(), status); err != nil {
|
|
t.Fatalf("complete recovery: %v", err)
|
|
}
|
|
|
|
statuses, err := store.ListRecoveryStatuses(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("list recovery statuses: %v", err)
|
|
}
|
|
if len(statuses) != 1 || statuses[0].State != "waiting_connection" || statuses[0].AttemptCount != 1 {
|
|
t.Fatalf("unexpected statuses: %+v", statuses)
|
|
}
|
|
if statuses[0].FailureCategory != "client_disconnected" {
|
|
t.Fatalf("failure category = %q, want client_disconnected", statuses[0].FailureCategory)
|
|
}
|
|
if statuses[0].NextRetryAt.IsZero() {
|
|
t.Fatalf("expected next retry at after waiting connection: %+v", statuses[0])
|
|
}
|
|
|
|
decision, err = store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
|
|
if err != nil {
|
|
t.Fatalf("start recovery second time: %v", err)
|
|
}
|
|
if decision.Allowed || decision.SkipReason != "backoff" {
|
|
t.Fatalf("expected backoff skip, got %+v", decision)
|
|
}
|
|
if decision.Status.FailureCategory != "backoff" {
|
|
t.Fatalf("skip failure category = %q, want backoff", decision.Status.FailureCategory)
|
|
}
|
|
}
|
|
|
|
func TestRedisRecoveryStoreDoesNotReleaseLockOwnedByAnotherGateway(t *testing.T) {
|
|
mr := miniredis.RunT(t)
|
|
store, err := NewRedisRecoveryStore("redis://" + mr.Addr())
|
|
if err != nil {
|
|
t.Fatalf("new recovery store: %v", err)
|
|
}
|
|
store.Prefix = "test:recovery:takeover"
|
|
|
|
first, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
|
|
if err != nil {
|
|
t.Fatalf("start first recovery: %v", err)
|
|
}
|
|
if !first.Allowed {
|
|
t.Fatalf("expected first recovery allowed, got %+v", first)
|
|
}
|
|
|
|
mr.FastForward(defaultRecoveryLockTTL + time.Second)
|
|
second, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-b")
|
|
if err != nil {
|
|
t.Fatalf("start second recovery: %v", err)
|
|
}
|
|
if !second.Allowed {
|
|
t.Fatalf("expected second recovery to take over expired lock, got %+v", second)
|
|
}
|
|
|
|
stale := first.Status
|
|
stale.State = "success"
|
|
err = store.CompleteAccountRecovery(context.Background(), stale)
|
|
if !errors.Is(err, ErrRecoveryLockLost) {
|
|
t.Fatalf("stale completion error = %v, want ErrRecoveryLockLost", err)
|
|
}
|
|
|
|
lockValue, err := store.Client.Get(context.Background(), store.lockKey("100001")).Result()
|
|
if err != nil {
|
|
t.Fatalf("load current lock: %v", err)
|
|
}
|
|
if lockValue != second.Status.LockToken {
|
|
t.Fatalf("current lock was changed by stale completion: got %q want %q", lockValue, second.Status.LockToken)
|
|
}
|
|
}
|