feat: improve channel resilience and operations

This commit is contained in:
hectorzhao
2026-07-24 08:17:33 +08:00
parent 2f781ebb8a
commit afd3c96070
43 changed files with 1969 additions and 299 deletions
+106 -30
View File
@@ -14,6 +14,7 @@ import (
)
type ConnectFunc func(context.Context, ConnectChannelCommand) (ConnectionStateCallback, error)
type DisconnectFunc func(context.Context, DisconnectChannelCommand) (ConnectionStateCallback, error)
type SubmitFunc func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error)
type ConnectChannelCommand struct {
@@ -28,34 +29,51 @@ type ConnectChannelCommand struct {
}
type ChannelConfig struct {
Code string `json:"code"`
Name string `json:"name"`
GatewayHost string `json:"gatewayHost"`
GatewayPort int `json:"gatewayPort"`
Account string `json:"account"`
PasswordCipher string `json:"passwordCipher"`
SrcID string `json:"srcId"`
CMPPVersion string `json:"cmppVersion"`
RateLimitPerSecond int `json:"rateLimitPerSecond"`
Code string `json:"code"`
Name string `json:"name"`
GatewayHost string `json:"gatewayHost"`
GatewayPort int `json:"gatewayPort"`
Account string `json:"account"`
PasswordCipher string `json:"passwordCipher"`
SrcID string `json:"srcId"`
CMPPVersion string `json:"cmppVersion"`
RateLimitPerSecond int `json:"rateLimitPerSecond"`
WindowSize int `json:"windowSize,omitempty"`
HeartbeatIntervalSeconds int `json:"heartbeatIntervalSeconds,omitempty"`
HeartbeatMissThreshold int `json:"heartbeatMissThreshold,omitempty"`
}
type DisconnectChannelCommand struct {
SchemaVersion string `json:"schemaVersion"`
MessageType string `json:"messageType"`
TraceID string `json:"traceId"`
ChannelID string `json:"channelId"`
ConnectionID string `json:"connectionId"`
CreatedAt string `json:"createdAt"`
Reason string `json:"reason"`
}
type ConnectionStateCallback struct {
ChannelID string `json:"channelId"`
ConnectionID string `json:"connectionId"`
Status string `json:"status"`
DesiredConnections int `json:"desiredConnections"`
CurrentConnections int `json:"currentConnections"`
LastConnectedAt string `json:"lastConnectedAt,omitempty"`
LastDisconnectedAt string `json:"lastDisconnectedAt,omitempty"`
LastHeartbeatAt string `json:"lastHeartbeatAt,omitempty"`
ReconnectCount int `json:"reconnectCount,omitempty"`
LastError string `json:"lastError,omitempty"`
ChannelID string `json:"channelId"`
ConnectionID string `json:"connectionId"`
Status string `json:"status"`
DesiredConnections int `json:"desiredConnections"`
CurrentConnections int `json:"currentConnections"`
LastConnectedAt string `json:"lastConnectedAt,omitempty"`
LastDisconnectedAt string `json:"lastDisconnectedAt,omitempty"`
LastHeartbeatAt string `json:"lastHeartbeatAt,omitempty"`
ReconnectCount int `json:"reconnectCount,omitempty"`
LastReconnectAttemptAt string `json:"lastReconnectAttemptAt,omitempty"`
NextReconnectAt string `json:"nextReconnectAt,omitempty"`
LastErrorCategory string `json:"lastErrorCategory,omitempty"`
LastError string `json:"lastError,omitempty"`
}
type Server struct {
APIBaseURL string
HTTPClient *http.Client
Connect ConnectFunc
Disconnect DisconnectFunc
Submit SubmitFunc
Upstream *upstream.Manager
Limiter ratelimit.Limiter
@@ -78,10 +96,14 @@ func Register(mux *http.ServeMux, server Server) {
if server.Connect == nil {
server.Connect = server.connectChannel
}
if server.Disconnect == nil {
server.Disconnect = server.disconnectChannel
}
if server.Submit == nil {
server.Submit = server.Upstream.Submit
}
mux.HandleFunc("/connections/connect", server.handleConnectChannel)
mux.HandleFunc("/connections/disconnect", server.handleDisconnectChannel)
mux.HandleFunc("/upstream/submit", server.handleUpstreamSubmit)
mux.HandleFunc("/downstream/receipt", server.handleDownstreamReceipt)
mux.HandleFunc("/downstream/uplink", server.handleDownstreamUplink)
@@ -122,6 +144,29 @@ func (s Server) handleConnectChannel(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(status)
}
func (s Server) handleDisconnectChannel(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
var command DisconnectChannelCommand
if err := json.NewDecoder(r.Body).Decode(&command); err != nil {
http.Error(w, fmt.Sprintf("invalid disconnect command: %v", err), http.StatusBadRequest)
return
}
if command.MessageType != string(queue.MessageTypeDisconnectChannel) || command.ChannelID == "" || command.ConnectionID == "" {
http.Error(w, "messageType DisconnectChannel, channelId and connectionId are required", http.StatusBadRequest)
return
}
status, err := s.Disconnect(r.Context(), command)
if err != nil {
http.Error(w, fmt.Sprintf("failed to disconnect upstream pool: %v", err), http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(status)
}
func (s Server) handleUpstreamSubmit(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
@@ -265,29 +310,60 @@ func (s Server) connectChannel(ctx context.Context, command ConnectChannelComman
Reason: command.Reason,
DesiredConnections: command.DesiredConnections,
Channel: queue.ConnectChannelConfig{
Code: command.Channel.Code,
Name: command.Channel.Name,
GatewayHost: command.Channel.GatewayHost,
GatewayPort: command.Channel.GatewayPort,
Account: command.Channel.Account,
PasswordCipher: command.Channel.PasswordCipher,
SrcID: command.Channel.SrcID,
CMPPVersion: command.Channel.CMPPVersion,
RateLimitPerSecond: command.Channel.RateLimitPerSecond,
Code: command.Channel.Code,
Name: command.Channel.Name,
GatewayHost: command.Channel.GatewayHost,
GatewayPort: command.Channel.GatewayPort,
Account: command.Channel.Account,
PasswordCipher: command.Channel.PasswordCipher,
SrcID: command.Channel.SrcID,
CMPPVersion: command.Channel.CMPPVersion,
RateLimitPerSecond: command.Channel.RateLimitPerSecond,
WindowSize: command.Channel.WindowSize,
HeartbeatIntervalSeconds: command.Channel.HeartbeatIntervalSeconds,
HeartbeatMissThreshold: command.Channel.HeartbeatMissThreshold,
},
})
if err != nil {
return ConnectionStateCallback{}, err
}
return ConnectionStateCallback{
ChannelID: state.ChannelID,
ConnectionID: state.ConnectionID,
Status: state.Status,
DesiredConnections: state.DesiredConnections,
CurrentConnections: state.CurrentConnections,
LastConnectedAt: state.LastConnectedAt,
LastDisconnectedAt: state.LastDisconnectedAt,
LastHeartbeatAt: state.LastHeartbeatAt,
ReconnectCount: state.ReconnectCount,
LastReconnectAttemptAt: state.LastReconnectAttemptAt,
NextReconnectAt: state.NextReconnectAt,
LastErrorCategory: state.LastErrorCategory,
LastError: state.LastError,
}, nil
}
func (s Server) disconnectChannel(ctx context.Context, command DisconnectChannelCommand) (ConnectionStateCallback, error) {
state, err := s.Upstream.DisconnectChannel(ctx, queue.DisconnectChannelCommand{
SchemaVersion: command.SchemaVersion,
MessageType: queue.MessageTypeDisconnectChannel,
TraceID: command.TraceID,
ChannelID: command.ChannelID,
ConnectionID: command.ConnectionID,
CreatedAt: time.Now().UTC(),
Reason: command.Reason,
})
if err != nil {
return ConnectionStateCallback{}, err
}
return ConnectionStateCallback{
ChannelID: state.ChannelID,
ConnectionID: state.ConnectionID,
Status: state.Status,
DesiredConnections: state.DesiredConnections,
CurrentConnections: state.CurrentConnections,
LastConnectedAt: state.LastConnectedAt,
LastDisconnectedAt: state.LastDisconnectedAt,
LastHeartbeatAt: state.LastHeartbeatAt,
ReconnectCount: state.ReconnectCount,
LastError: state.LastError,
}, nil
+33
View File
@@ -140,6 +140,39 @@ func TestConnectChannelRejectsInvalidCommand(t *testing.T) {
}
}
func TestDisconnectChannelStopsSupplierPool(t *testing.T) {
var received DisconnectChannelCommand
handler := handlerWithServer(Server{
Disconnect: func(_ context.Context, command DisconnectChannelCommand) (ConnectionStateCallback, error) {
received = command
return ConnectionStateCallback{
ChannelID: command.ChannelID,
ConnectionID: command.ConnectionID,
Status: "disconnected",
CurrentConnections: 0,
}, nil
},
})
resp := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/connections/disconnect", strings.NewReader(`{
"schemaVersion":"v1",
"messageType":"DisconnectChannel",
"traceId":"trace-disconnect",
"channelId":"channel-1",
"connectionId":"channel-1:primary",
"reason":"channel_disabled"
}`))
handler.ServeHTTP(resp, req)
if resp.Code != http.StatusOK {
t.Fatalf("unexpected response status: %d body=%s", resp.Code, resp.Body.String())
}
if received.ChannelID != "channel-1" || received.Reason != "channel_disabled" {
t.Fatalf("unexpected disconnect command: %+v", received)
}
}
func TestRecoveryCandidatesEndpointReturnsView(t *testing.T) {
handler := handlerWithServer(Server{
RecoveryCandidates: func(context.Context) ([]inbound.DownstreamPresence, error) {