feat: improve channel resilience and operations

gateway/internal/upstream/connection_loss_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"testing"
+	"time"
 )
 
 func TestHandleConnectionLossNotifiesPendingSubmitters(t *testing.T) {
@@ -18,6 +19,8 @@ func TestHandleConnectionLossNotifiesPendingSubmitters(t *testing.T) {
 				reported = state
 				return nil
 			},
+			reconnectSignal: make(chan struct{}, 1),
+			stopCh:          make(chan struct{}),
 		},
 	}
 	conn.pool.conns = []*connection{conn}
@@ -45,6 +48,60 @@ func TestHandleConnectionLossNotifiesPendingSubmitters(t *testing.T) {
 	if reported.Status != "disconnected" || reported.CurrentConnections != 0 {
 		t.Fatalf("unexpected reported state: %+v", reported)
 	}
+	if conn.pool.reconnectCount != 1 || conn.pool.nextReconnectAt.IsZero() {
+		t.Fatalf("expected connection loss to schedule reconnect, got count=%d next=%v", conn.pool.reconnectCount, conn.pool.nextReconnectAt)
+	}
+	conn.pool.close()
+}
+
+func TestHeartbeatTimeoutClosesConnectionAndSchedulesReconnect(t *testing.T) {
+	pool := &connectionPool{
+		channelID:       "channel-1",
+		connectionID:    "channel-1:primary",
+		config:          normalizeUpstreamConfig(queueUpstreamConfigForTest()),
+		reconnectSignal: make(chan struct{}, 1),
+		stopCh:          make(chan struct{}),
+	}
+	conn := &connection{
+		pool:             pool,
+		pending:          make(map[uint32]chan submitPartResponse),
+		heartbeatPending: map[uint32]time.Time{1: time.Now(), 2: time.Now(), 3: time.Now()},
+	}
+	pool.conns = []*connection{conn}
+
+	if conn.sendHeartbeat() {
+		t.Fatal("expected heartbeat timeout to stop heartbeat loop")
+	}
+	if !conn.closed {
+		t.Fatal("expected heartbeat timeout to close the connection")
+	}
+	if pool.reconnectCount != 1 || pool.lastErrorCategory != "heartbeat_timeout" {
+		t.Fatalf("unexpected reconnect state: count=%d category=%s", pool.reconnectCount, pool.lastErrorCategory)
+	}
+	pool.close()
+}
+
+func TestReconnectDelayUsesCappedBackoffAndSlowAuthenticationRetry(t *testing.T) {
+	if got := reconnectDelay(1, "network"); got < 4*time.Second || got > 6*time.Second {
+		t.Fatalf("first reconnect delay = %v", got)
+	}
+	if got := reconnectDelay(6, "network"); got < 4*time.Minute || got > 6*time.Minute {
+		t.Fatalf("capped reconnect delay = %v", got)
+	}
+	if got := reconnectDelay(1, "authentication"); got != 5*time.Minute {
+		t.Fatalf("authentication reconnect delay = %v", got)
+	}
+}
+
+func TestHeartbeatResponseClearsOnlyMatchingRequest(t *testing.T) {
+	conn := &connection{heartbeatPending: map[uint32]time.Time{7: time.Now(), 8: time.Now()}}
+	conn.handleHeartbeatResponse(7)
+	if _, exists := conn.heartbeatPending[7]; exists {
+		t.Fatal("expected matching heartbeat request to be cleared")
+	}
+	if _, exists := conn.heartbeatPending[8]; !exists {
+		t.Fatal("expected unrelated heartbeat request to remain pending")
+	}
 }
 
 func TestTemporaryReadTimeoutDetection(t *testing.T) {