hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(cmpp): process inbound submits within client window

Browse Source
This commit is contained in:
hectorzhao
2026-08-20 11:45:16 +08:00
parent b9a71fe0b9
commit 0757a699ff
20 changed files with 389 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/internal/inbound/authentication.go
+4
View File
@@ -9,6 +9,7 @@ import (
"net"
"strings"
"sync"
"sync/atomic"
"time"
)
@@ -28,6 +29,7 @@ type authResponse struct {
Account string `json:"account"`
EnterpriseCode string `json:"enterpriseCode"`
MaxConnections int `json:"maxConnections"`
WindowSize int `json:"windowSize"`
}
func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger *log.Logger) (bool, error) {
@@ -59,6 +61,8 @@ func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger
applicationID: strings.TrimSpace(auth.ApplicationID),
enterpriseCode: strings.TrimSpace(auth.EnterpriseCode),
protocol: cmppVersionName(req.Version),
windowSize: boundedSubmitWindow(auth.WindowSize, s.MaxSubmitConcurrency),
submitInFlight: &atomic.Int64{},
srcID: strings.TrimSpace(auth.Account),
remoteIP: remoteIP(packet.Conn.Conn.RemoteAddr()),
connectedAt: now,
gateway/internal/inbound/server.go
+12 -1
View File
@@ -20,6 +20,7 @@ type Server struct {
PresenceStore PresenceStore
RecoveryStore RecoveryStore
GatewayInstanceID string
MaxSubmitConcurrency int
}
func (s Server) ListenAndServe() error {
@@ -30,9 +31,19 @@ func (s Server) ListenAndServe() error {
s.logRecoveryCandidates(log.Default())
go s.recoverPendingCandidates(log.Default())
go s.runPendingFlusher(log.Default())
return cmpp.ListenAndServeWithClose(addr, cmpp.V30, 30*time.Second, 3, s.LogWriter, s.handleConnectionClosed,
return cmpp.ListenAndServeWithCloseAndSubmitWindow(addr, cmpp.V30, 30*time.Second, 3, s.LogWriter, s.handleConnectionClosed, submitWindowByConn,
cmpp.HandlerFunc(s.handleLogin),
cmpp.HandlerFunc(s.handleSubmit),
cmpp.HandlerFunc(s.handleActivity),
)
}
func boundedSubmitWindow(applicationWindow int, gatewayMaximum int) int {
if applicationWindow < 1 {
applicationWindow = 1
}
if gatewayMaximum < 1 {
gatewayMaximum = 64
}
return min(applicationWindow, min(gatewayMaximum, 1024))
}
gateway/internal/inbound/server_test.go
+111
View File
@@ -5,6 +5,7 @@ import (
"context"
"encoding/binary"
"encoding/json"
"fmt"
"log"
"net"
"net/http"
@@ -237,6 +238,116 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
}
}
func TestInboundServerProcessesSubmitWithinAuthenticatedConnectionWindow(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
account := "100020"
password := "window-secret"
firstStarted := make(chan struct{})
secondStarted := make(chan struct{})
releaseFirst := make(chan struct{})
var releaseOnce sync.Once
defer releaseOnce.Do(func() { close(releaseFirst) })
var calls atomic.Int32
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/gateway/events/inbound/authenticate":
_ = json.NewEncoder(w).Encode(authResponse{
PasswordCipher: password, Account: account, EnterpriseCode: account, WindowSize: 2,
})
case "/api/gateway/events/inbound/submit":
call := calls.Add(1)
if call == 1 {
close(firstStarted)
<-releaseFirst
} else {
close(secondStarted)
}
_ = json.NewEncoder(w).Encode(submitResponse{Accepted: true, MessageID: fmt.Sprintf("MSG-WINDOW-%d", call)})
case "/api/gateway/events/inbound/connection", "/api/gateway/events/protocol-log":
w.WriteHeader(http.StatusOK)
case "/api/gateway/events/downstream/pending":
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
default:
w.WriteHeader(http.StatusOK)
}
}))
defer api.Close()
addr := reserveTCPAddr(t)
go func() {
_ = (Server{Addr: addr, APIBaseURL: api.URL + "/api", MaxSubmitConcurrency: 2}).ListenAndServe()
}()
time.Sleep(300 * time.Millisecond)
client := cmpp.NewClient(cmpp.V30)
defer client.Disconnect()
if err := client.Connect(addr, account, password, 2*time.Second); err != nil {
t.Fatalf("connect inbound cmpp: %v", err)
}
packet := func(phone string) *cmpp.Cmpp3SubmitReqPkt {
return &cmpp.Cmpp3SubmitReqPkt{
PkTotal: 1, PkNumber: 1, RegisteredDelivery: 1, MsgLevel: 1,
ServiceId: "cmpp", FeeUserType: 2, FeeTerminalId: phone, MsgFmt: 0,
MsgSrc: account, FeeType: "02", FeeCode: "0", SrcId: "10690000",
DestUsrTl: 1, DestTerminalId: []string{phone}, MsgLength: 6, MsgContent: "window",
}
}
firstSequence, err := client.SendReqPkt(packet("13800000001"))
if err != nil {
t.Fatalf("send first submit: %v", err)
}
<-firstStarted
secondSequence, err := client.SendReqPkt(packet("13800000002"))
if err != nil {
t.Fatalf("send second submit: %v", err)
}
select {
case <-secondStarted:
case <-time.After(time.Second):
t.Fatal("second Submit did not enter the authenticated connection window")
}
response := recvSubmitRsp(t, client)
if response.SeqId != secondSequence {
t.Fatalf("first completed response sequence=%d, want second sequence=%d (first=%d)", response.SeqId, secondSequence, firstSequence)
}
releaseOnce.Do(func() { close(releaseFirst) })
response = recvSubmitRsp(t, client)
if response.SeqId != firstSequence {
t.Fatalf("released response sequence=%d, want first sequence=%d", response.SeqId, firstSequence)
}
}
func TestBoundedSubmitWindowAndAggregateSlotSnapshot(t *testing.T) {
for _, test := range []struct {
application int
maximum int
want int
}{
{application: 0, maximum: 0, want: 1},
{application: 32, maximum: 64, want: 32},
{application: 32, maximum: 16, want: 16},
{application: 2048, maximum: 2048, want: 1024},
} {
if got := boundedSubmitWindow(test.application, test.maximum); got != test.want {
t.Fatalf("boundedSubmitWindow(%d, %d)=%d, want %d", test.application, test.maximum, got, test.want)
}
}
resetDownstreamRegistry()
defer resetDownstreamRegistry()
firstCounter := &atomic.Int64{}
secondCounter := &atomic.Int64{}
firstCounter.Store(3)
secondCounter.Store(1)
downstreamRegistry.byConn[&cmpp.Conn{}] = &downstreamSession{windowSize: 32, submitInFlight: firstCounter}
downstreamRegistry.byConn[&cmpp.Conn{}] = &downstreamSession{windowSize: 16, submitInFlight: secondCounter}
configured, inFlight := SubmitSlotSnapshot()
if configured != 48 || inFlight != 4 {
t.Fatalf("slot snapshot configured=%d in_flight=%d, want 48/4", configured, inFlight)
}
}
func TestDecodeInboundLongMessageStripsConcatUDHBeforeUCS2Decode(t *testing.T) {
payload, err := cmpputils.Utf8ToUcs2("【深圳市合正物业服务有限公司】第一片正文")
if err != nil {
gateway/internal/inbound/sessions.go
+36
View File
@@ -6,6 +6,7 @@ import (
"log"
"strings"
"sync"
"sync/atomic"
"time"
)
@@ -30,6 +31,8 @@ type downstreamSession struct {
applicationID string
enterpriseCode string
protocol string
windowSize int
submitInFlight *atomic.Int64
srcID string
phoneNumber string
gatewayMsgID uint64
@@ -123,6 +126,39 @@ func findSessionByConn(conn *cmpp.Conn) *downstreamSession {
return downstreamRegistry.byConn[conn]
}
func submitWindowByConn(conn *cmpp.Conn) int {
session := findSessionByConn(conn)
if session == nil || session.windowSize < 1 {
return 1
}
return session.windowSize
}
func beginInboundSubmit(session *downstreamSession) func() {
if session == nil || session.submitInFlight == nil {
return func() {}
}
session.submitInFlight.Add(1)
return func() { session.submitInFlight.Add(-1) }
}
func SubmitSlotSnapshot() (int, int64) {
downstreamRegistry.RLock()
defer downstreamRegistry.RUnlock()
configured := 0
var inFlight int64
for _, session := range downstreamRegistry.byConn {
if session == nil {
continue
}
configured += max(1, session.windowSize)
if session.submitInFlight != nil {
inFlight += session.submitInFlight.Load()
}
}
return configured, inFlight
}
func rememberDownstream(session downstreamSession) {
if session.messageID == "" || session.conn == nil {
return
gateway/internal/inbound/submit.go
+2
View File
@@ -69,6 +69,8 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
response.AfterSend = observeInboundSubmitResponse(handlerStartedAt, responseReadyAt, false, s.submitResponseProtocolLogger("", req.protocol, req.sequenceID, "", "", 0, 9))
return false, nil
}
releaseInboundSlot := beginInboundSubmit(session)
defer releaseInboundSlot()
account := session.account
enterpriseCode := strings.TrimRight(req.msgSrc, "\x00")
if session.enterpriseCode != "" && enterpriseCode != session.enterpriseCode {
gateway/internal/metrics/metrics.go
+13 -10
View File
@@ -37,16 +37,18 @@ var inboundStageHistograms [4][2]durationHistogram
var submitStageHistograms [5][2]durationHistogram
type Snapshot struct {
UpstreamDesired int
UpstreamConnected int
DownstreamConnected int
SubmitWorkerUp bool
SubmitWorkerConcurrency int
SubmitWorkerInFlight int64
QueueAvailable bool
QueuePending int64
QueueLag int64
QueueOldestAgeSeconds float64
UpstreamDesired int
UpstreamConnected int
DownstreamConnected int
SubmitWorkerUp bool
SubmitWorkerConcurrency int
SubmitWorkerInFlight int64
InboundSubmitConcurrency int
InboundSubmitInFlight int64
QueueAvailable bool
QueuePending int64
QueueLag int64
QueueOldestAgeSeconds float64
}
type SnapshotFunc func(context.Context) Snapshot
@@ -116,6 +118,7 @@ func Handler(load SnapshotFunc) http.Handler {
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight)
fmt.Fprintf(response, "# HELP cmpp_gateway_inbound_submit_slots Configured and active authenticated client Submit slots.\n# TYPE cmpp_gateway_inbound_submit_slots gauge\ncmpp_gateway_inbound_submit_slots{state=\"configured\"} %d\ncmpp_gateway_inbound_submit_slots{state=\"in_flight\"} %d\n", snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight)
if snapshot.QueueAvailable {
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag)
gateway/internal/metrics/metrics_test.go
+3 -1
View File
@@ -16,7 +16,7 @@ func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
response := httptest.NewRecorder()
Handler(func(context.Context) Snapshot {
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, SubmitWorkerConcurrency: 64, SubmitWorkerInFlight: 7, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12}
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, SubmitWorkerConcurrency: 64, SubmitWorkerInFlight: 7, InboundSubmitConcurrency: 96, InboundSubmitInFlight: 9, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12}
}).ServeHTTP(response, request)
body := response.Body.String()
@@ -28,6 +28,8 @@ func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
`cmpp_gateway_submit_stage_duration_seconds_count{stage="rate_limit_wait",result="success"} 1`,
`cmpp_gateway_submit_worker_slots{state="configured"} 64`,
`cmpp_gateway_submit_worker_slots{state="in_flight"} 7`,
`cmpp_gateway_inbound_submit_slots{state="configured"} 96`,
`cmpp_gateway_inbound_submit_slots{state="in_flight"} 9`,
} {
if !strings.Contains(body, expected) {
t.Fatalf("metrics response is missing %q: %s", expected, body)