hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden upstream and downstream receipt delivery

Browse Source
This commit is contained in:
hectorzhao
2026-07-25 23:32:13 +08:00
parent 5018167696
commit 54617c927e
18 changed files with 1797 additions and 779 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/internal/inbound/server.go
+92 -15
View File
@@ -209,6 +209,11 @@ var downstreamAckRegistry = struct {
items map[string]*downstreamAckTracker
}{items: make(map[string]*downstreamAckTracker)}
var downstreamSubmitBarrier = struct {
sync.RWMutex
byConn map[*cmpp.Conn]int
}{byConn: make(map[*cmpp.Conn]int)}
func (s Server) ListenAndServe() error {
addr := s.Addr
if addr == "" {
@@ -336,6 +341,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
}
contentHash := fmt.Sprintf("%x", md5.Sum([]byte(content)))
startedAt := time.Now()
releaseSubmitBarrier := beginDownstreamSubmitBarrier(packet.Conn)
result, err := s.submit(remote, submitRequest{
Account: account,
PhoneNumber: phone,
@@ -361,7 +367,11 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
clientProtocol, req.protocol, account, remote, req.sequenceID, phone, responseResult, time.Since(startedAt).Milliseconds(), len([]rune(content)), contentHash, reason,
)
setInboundSubmitResponse(response.Packer, 0, responseResult)
response.AfterSend = s.submitResponseProtocolLogger(account, clientProtocol, req.sequenceID, phone, result.MessageID, 0, responseResult)
protocolLogger := s.submitResponseProtocolLogger(account, clientProtocol, req.sequenceID, phone, result.MessageID, 0, responseResult)
response.AfterSend = func(sendErr error) {
releaseSubmitBarrier()
protocolLogger(sendErr)
}
return false, nil
}
gatewayMsgID := messageIDFrom(result.MessageID, req.sequenceID)
@@ -401,6 +411,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
go current.report(current, "submit", "")
}
response.AfterSend = func(sendErr error) {
releaseSubmitBarrier()
s.emitProtocolLog(protocolLogEvent{
Protocol: "cmpp",
Direction: "platform_to_client",
@@ -538,6 +549,9 @@ func (s Server) reportConnectionOrDisconnect(session *downstreamSession, status
}
func (s Server) handleConnectionClosed(conn *cmpp.Conn) {
downstreamSubmitBarrier.Lock()
delete(downstreamSubmitBarrier.byConn, conn)
downstreamSubmitBarrier.Unlock()
session := findSessionByConn(conn)
if session == nil {
return
@@ -705,10 +719,10 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
if err != nil {
result.FailedCount++
result.LastError = err.Error()
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]string{
"id": delivery.ID,
"errorMessage": err.Error(),
"failureType": "send_failed",
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
"id": delivery.ID, "errorMessage": err.Error(), "failureType": "send_failed",
"connectionId": sendResult.ConnectionID, "sequenceId": sendResult.SequenceID,
"messageId": sendResult.MessageID, "sentAt": sendResult.SentAt,
}, nil)
continue
}
@@ -716,6 +730,10 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
result.DeliveredCount++
continue
}
if sendResult.ReasonCode == "SUBMIT_RESPONSE_PENDING" {
result.WaitingCount++
continue
}
errorMessage := defaultString(sendResult.ErrorMessage, "gateway did not complete downstream delivery")
failureType := "unrecoverable"
if sendResult.Retryable {
@@ -725,10 +743,10 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
result.FailedCount++
}
result.LastError = errorMessage
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]string{
"id": delivery.ID,
"errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode),
"failureType": failureType,
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
"id": delivery.ID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode),
"failureType": failureType, "connectionId": sendResult.ConnectionID,
"sequenceId": sendResult.SequenceID, "messageId": sendResult.MessageID, "sentAt": sendResult.SentAt,
}, nil)
}
return result, nil
@@ -1153,6 +1171,13 @@ func pushReceiptWithResult(event DownstreamReceipt, allowRecovery bool) (Downstr
ErrorMessage: "下游客户端当前未连接，等待自动重试",
}, nil
}
if downstreamSubmitResponsePending(session.conn) {
return DownstreamSendResult{
Retryable: true,
ReasonCode: "SUBMIT_RESPONSE_PENDING",
ErrorMessage: "客户 SubmitResp 尚未完成写出，回执已保留并等待响应后投递",
}, nil
}
stat := strings.TrimSpace(event.RawStatus)
if stat == "" {
stat = cmppReceiptStatus(event.ReceiptStatus)
@@ -1179,6 +1204,36 @@ func pushReceiptWithResult(event DownstreamReceipt, allowRecovery bool) (Downstr
return sendDownstream(session, deliver, event.DeliveryID)
}
func beginDownstreamSubmitBarrier(conn *cmpp.Conn) func() {
if conn == nil {
return func() {}
}
downstreamSubmitBarrier.Lock()
downstreamSubmitBarrier.byConn[conn]++
downstreamSubmitBarrier.Unlock()
var once sync.Once
return func() {
once.Do(func() {
downstreamSubmitBarrier.Lock()
if downstreamSubmitBarrier.byConn[conn] <= 1 {
delete(downstreamSubmitBarrier.byConn, conn)
} else {
downstreamSubmitBarrier.byConn[conn]--
}
downstreamSubmitBarrier.Unlock()
})
}
}
func downstreamSubmitResponsePending(conn *cmpp.Conn) bool {
if conn == nil {
return false
}
downstreamSubmitBarrier.RLock()
defer downstreamSubmitBarrier.RUnlock()
return downstreamSubmitBarrier.byConn[conn] > 0
}
func findReceiptSession(messageID string, account string) *downstreamSession {
downstreamRegistry.RLock()
defer downstreamRegistry.RUnlock()
@@ -1288,6 +1343,13 @@ func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID
sequenceID := <-session.conn.SeqId
sentAt := time.Now().UTC()
ackDeadlineAt := sentAt.Add(downstreamAckTimeout())
result := DownstreamSendResult{
ConnectionID: session.connectionID,
SequenceID: strconv.FormatUint(uint64(sequenceID), 10),
MessageID: strconv.FormatUint(messageID, 10),
SentAt: formatRFC3339Nano(sentAt),
AckDeadlineAt: formatRFC3339Nano(ackDeadlineAt),
}
tracker := registerDownstreamAck(session, deliveryID, sequenceID, messageID, ackDeadlineAt)
if err := session.conn.SendPkt(deliver, sequenceID); err != nil {
removeDownstreamAck(tracker)
@@ -1296,14 +1358,13 @@ func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID
go session.report(session, "disconnected", err.Error())
}
forgetDownstream(session)
return DownstreamSendResult{}, err
result.Retryable = true
result.ReasonCode = "SEND_FAILED"
result.ErrorMessage = err.Error()
return result, nil
}
session.recordDownstreamProtocol(deliver, deliveryID, sequenceID, messageID, "success", "", nil)
result := DownstreamSendResult{
Sent: true, ConnectionID: session.connectionID,
SequenceID: strconv.FormatUint(uint64(sequenceID), 10), MessageID: strconv.FormatUint(messageID, 10),
SentAt: formatRFC3339Nano(sentAt), AckDeadlineAt: formatRFC3339Nano(ackDeadlineAt),
}
result.Sent = true
if deliveryID != "" && session.deliveryReport != nil {
go session.deliveryReport(downstreamDeliveryLifecycleEvent{
Kind: "sent", DeliveryID: deliveryID, ConnectionID: session.connectionID,
@@ -1433,6 +1494,22 @@ func handleDownstreamAcknowledgement(conn *cmpp.Conn, sequenceID uint32, message
if logger != nil {
logger.Printf("cmpp inbound event=deliver_ack_unmatched seq=%d message_id=%d result=%d", sequenceID, messageID, result)
}
if session := findSessionByConn(conn); session != nil && session.protocolLog != nil {
session.protocolLog(protocolLogEvent{
Protocol: "cmpp",
Direction: "client_to_platform",
EventType: "deliver_resp",
Status: "failed",
TenantID: session.tenantID,
ApplicationID: session.applicationID,
Account: session.account,
MessageID: session.messageID,
GatewayMessageID: strconv.FormatUint(messageID, 10),
Phone: session.phoneNumber,
ResultCode: strconv.FormatUint(uint64(result), 10),
Detail: map[string]any{"sequenceId": sequenceID, "unmatched": true},
})
}
return
}
if tracker.messageID != messageID {
gateway/internal/inbound/server_test.go
+38
View File
@@ -549,6 +549,41 @@ func TestRecoverableReceiptWaitsForClientConnection(t *testing.T) {
}
}
func TestReceiptWaitsUntilCurrentSubmitResponseHasBeenWritten(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
conn := &cmpp.Conn{}
session := &downstreamSession{
messageID: "MSG-LONG-1",
account: "100001",
conn: conn,
}
downstreamRegistry.Lock()
downstreamRegistry.byMessageID[session.messageID] = session
downstreamRegistry.byAccount[session.account] = session
downstreamRegistry.byConn[conn] = session
downstreamRegistry.Unlock()
release := beginDownstreamSubmitBarrier(conn)
result, err := PushReceiptWithResult(DownstreamReceipt{
DeliveryID: "delivery-long-failed",
Account: session.account,
MessageID: session.messageID,
ReceiptStatus: "undelivered",
})
if err != nil {
t.Fatalf("push guarded receipt: %v", err)
}
if result.Sent || !result.Retryable || result.ReasonCode != "SUBMIT_RESPONSE_PENDING" {
t.Fatalf("unexpected guarded result: %+v", result)
}
release()
if downstreamSubmitResponsePending(conn) {
t.Fatal("submit response barrier remained active after release")
}
}
func TestInboundServerNegotiatesCMPP2AndUsesAuthenticatedAccountForSubmit(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
@@ -1058,6 +1093,9 @@ func resetDownstreamRegistry() {
}
downstreamAckRegistry.items = make(map[string]*downstreamAckTracker)
downstreamAckRegistry.Unlock()
downstreamSubmitBarrier.Lock()
downstreamSubmitBarrier.byConn = make(map[*cmpp.Conn]int)
downstreamSubmitBarrier.Unlock()
}
func reserveTCPAddr(t *testing.T) string {