feat: add phone frequency controls and modularize codebase

2026-07-31 22:25:23 +08:00
parent 0af671b4ed
commit ca4f591a13
216 changed files with 41579 additions and 23694 deletions
@@ -0,0 +1,171 @@
+package upstream
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strings"
+	"time"
+)
+
+// Authentication failures intentionally use the slow retry class while
+// transient network failures use capped backoff; manual disconnect closes stopCh.
+
+func (p *connectionPool) startSupervisor() {
+	p.supervisorOnce.Do(func() {
+		go p.superviseReconnects()
+	})
+}
+
+func (p *connectionPool) stopped() bool {
+	select {
+	case <-p.stopCh:
+		return true
+	default:
+		return false
+	}
+}
+
+func (p *connectionPool) signalReconnect() {
+	select {
+	case <-p.stopCh:
+		return
+	default:
+	}
+	select {
+	case p.reconnectSignal <- struct{}{}:
+	default:
+	}
+}
+
+func (p *connectionPool) scheduleReconnect(stateErr error) {
+	select {
+	case <-p.stopCh:
+		return
+	default:
+	}
+	p.mu.Lock()
+	now := time.Now().UTC()
+	if !p.nextReconnectAt.IsZero() && p.nextReconnectAt.After(now) {
+		p.mu.Unlock()
+		return
+	}
+	p.reconnectCount++
+	p.lastReconnectAttemptAt = now
+	p.lastErrorCategory = connectionErrorCategory(stateErr)
+	delay := reconnectDelay(p.reconnectCount, p.lastErrorCategory)
+	p.nextReconnectAt = p.lastReconnectAttemptAt.Add(delay)
+	p.mu.Unlock()
+	p.signalReconnect()
+}
+
+func (p *connectionPool) resetReconnectState() {
+	p.mu.Lock()
+	p.reconnectCount = 0
+	p.lastReconnectAttemptAt = time.Time{}
+	p.nextReconnectAt = time.Time{}
+	p.lastErrorCategory = ""
+	p.mu.Unlock()
+	p.signalReconnect()
+}
+
+func (p *connectionPool) superviseReconnects() {
+	for {
+		select {
+		case <-p.stopCh:
+			return
+		case <-p.reconnectSignal:
+		}
+		for {
+			p.mu.Lock()
+			next := p.nextReconnectAt
+			p.mu.Unlock()
+			if next.IsZero() {
+				break
+			}
+			timer := time.NewTimer(time.Until(next))
+			select {
+			case <-p.stopCh:
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return
+			case <-p.reconnectSignal:
+				if !timer.Stop() {
+					<-timer.C
+				}
+				continue
+			case <-timer.C:
+			}
+			_ = p.reportState(context.Background(), "reconnecting", nil)
+			p.mu.Lock()
+			p.lastReconnectAttemptAt = time.Now().UTC()
+			p.mu.Unlock()
+			if err := p.ensureConnected(); err != nil {
+				select {
+				case <-p.stopCh:
+					return
+				default:
+				}
+				p.scheduleReconnect(err)
+				_ = p.reportState(context.Background(), "failed", err)
+				continue
+			}
+			select {
+			case <-p.stopCh:
+				return
+			default:
+			}
+			p.resetReconnectState()
+			_ = p.reportState(context.Background(), "connected", nil)
+			break
+		}
+	}
+}
+
+func connectionErrorCategory(err error) string {
+	if err == nil {
+		return ""
+	}
+	message := strings.ToLower(err.Error())
+	switch {
+	case strings.Contains(message, "auth"), strings.Contains(message, "password"), strings.Contains(message, "credential"):
+		return "authentication"
+	case strings.Contains(message, "heartbeat"):
+		return "heartbeat_timeout"
+	case strings.Contains(message, "timeout"):
+		return "timeout"
+	default:
+		return "network"
+	}
+}
+
+func reconnectDelay(attempt int, category string) time.Duration {
+	if category == "authentication" {
+		return defaultAuthReconnectDelay
+	}
+	if attempt < 1 {
+		attempt = 1
+	}
+	delays := []time.Duration{
+		defaultReconnectInitialDelay,
+		15 * time.Second,
+		30 * time.Second,
+		time.Minute,
+		2 * time.Minute,
+		defaultReconnectMaximumDelay,
+	}
+	delay := delays[min(attempt-1, len(delays)-1)]
+	// Deterministic ±10% jitter prevents a large set of channels from retrying together.
+	offsetPercent := (attempt*37)%21 - 10
+	delay += time.Duration(int64(delay) * int64(offsetPercent) / 100)
+	if delay < time.Second {
+		return time.Second
+	}
+	return delay
+}
+
+func isTemporaryReadTimeout(err error) bool {
+	var netErr net.Error
+	return errors.As(err, &netErr) && netErr.Timeout()
+}