hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: expand gateway capacity and prevent receipt replay

Browse Source
This commit is contained in:
hectorzhao
2026-08-25 16:08:30 +08:00
parent 761c123b65
commit 9292352be1
48 changed files with 2001 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/internal/inbound/acknowledgement.go
+6 -5
View File
@@ -18,6 +18,7 @@ const defaultDownstreamAckTimeout = 30 * time.Second
type downstreamAckTracker struct {
deliveryID string
claimID string
connectionID string
sequenceID uint32
messageID uint64
@@ -69,29 +70,29 @@ func downstreamAckKey(conn *cmpp.Conn, sequenceID uint32) string {
return fmt.Sprintf("%p:%d", conn, sequenceID)
}
func registerDownstreamAck(session *downstreamSession, deliveryID string, sequenceID uint32, messageID uint64, deadline time.Time) *downstreamAckTracker {
func registerDownstreamAck(session *downstreamSession, deliveryID string, claimID string, sequenceID uint32, messageID uint64, deadline time.Time) *downstreamAckTracker {
if session == nil || session.conn == nil || strings.TrimSpace(deliveryID) == "" {
return nil
}
tracker := &downstreamAckTracker{
deliveryID: deliveryID, connectionID: session.connectionID,
deliveryID: deliveryID, claimID: claimID, connectionID: session.connectionID,
sequenceID: sequenceID, messageID: messageID, session: session,
}
key := downstreamAckKey(session.conn, sequenceID)
downstreamAckRegistry.Lock()
downstreamAckRegistry.items[key] = tracker
downstreamAckRegistry.Unlock()
tracker.timer = time.AfterFunc(time.Until(deadline), func() {
timedOut := takeDownstreamAck(session.conn, sequenceID)
if timedOut == nil || timedOut.session == nil || timedOut.session.deliveryReport == nil {
return
}
timedOut.session.deliveryReport(downstreamDeliveryLifecycleEvent{
Kind: "failed", DeliveryID: timedOut.deliveryID, ConnectionID: timedOut.connectionID,
Kind: "failed", DeliveryID: timedOut.deliveryID, ClaimID: timedOut.claimID, ConnectionID: timedOut.connectionID,
SequenceID: timedOut.sequenceID, MessageID: timedOut.messageID, ObservedAt: time.Now().UTC(),
FailureType: "ack_timeout", ErrorMessage: "CMPP_DELIVER_RESP timeout",
})
})
downstreamAckRegistry.Unlock()
return tracker
}
@@ -149,7 +150,7 @@ func handleDownstreamAcknowledgement(conn *cmpp.Conn, sequenceID uint32, message
}
if tracker.session != nil && tracker.session.deliveryReport != nil {
go tracker.session.deliveryReport(downstreamDeliveryLifecycleEvent{
Kind: "acknowledged", DeliveryID: tracker.deliveryID, ConnectionID: tracker.connectionID,
Kind: "acknowledged", DeliveryID: tracker.deliveryID, ClaimID: tracker.claimID, ConnectionID: tracker.connectionID,
SequenceID: sequenceID, MessageID: messageID, Result: result, ObservedAt: time.Now().UTC(),
})
}
gateway/internal/inbound/delivery.go
+9 -6
View File
@@ -16,6 +16,7 @@ import (
type DownstreamReceipt struct {
DeliveryID string `json:"deliveryId,omitempty"`
ClaimID string `json:"claimId,omitempty"`
Account string `json:"account,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
MessageID string `json:"messageId"`
@@ -31,6 +32,7 @@ type DownstreamReceipt struct {
type DownstreamUplink struct {
DeliveryID string `json:"deliveryId,omitempty"`
ClaimID string `json:"claimId,omitempty"`
Account string `json:"account,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
MessageID string `json:"messageId,omitempty"`
@@ -55,6 +57,7 @@ type DownstreamSendResult struct {
type downstreamDeliveryLifecycleEvent struct {
Kind string
DeliveryID string
ClaimID string
ConnectionID string
SequenceID uint32
MessageID uint64
@@ -70,7 +73,7 @@ func (s Server) reportDownstreamDelivery(event downstreamDeliveryLifecycleEvent)
return
}
payload := map[string]any{
"id": event.DeliveryID, "connectionId": event.ConnectionID,
"id": event.DeliveryID, "claimId": event.ClaimID, "connectionId": event.ConnectionID,
"sequenceId": strconv.FormatUint(uint64(event.SequenceID), 10),
"messageId": strconv.FormatUint(event.MessageID, 10),
}
@@ -160,7 +163,7 @@ func pushReceiptWithResult(event DownstreamReceipt, allowRecovery bool) (Downstr
return DownstreamSendResult{}, err
}
deliver := downstreamDeliverPacket(session, receiptMessageID, session.srcID, defaultString(event.PhoneNumber, session.phoneNumber), 0, 1, string(receiptBytes))
return sendDownstream(session, deliver, event.DeliveryID)
return sendDownstream(session, deliver, event.DeliveryID, event.ClaimID)
}
func downstreamReceiptMessageID(event DownstreamReceipt, session *downstreamSession) uint64 {
@@ -228,7 +231,7 @@ func PushUplinkWithResult(event DownstreamUplink) (DownstreamSendResult, error)
0,
content,
)
return sendDownstream(session, deliver, event.DeliveryID)
return sendDownstream(session, deliver, event.DeliveryID, event.ClaimID)
}
func errorMessageWithCode(message string, code string) string {
@@ -272,7 +275,7 @@ func findSession(messageID string, account string) *downstreamSession {
return nil
}
func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID string) (DownstreamSendResult, error) {
func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID string, claimID string) (DownstreamSendResult, error) {
session.mu.Lock()
defer session.mu.Unlock()
messageID := downstreamDeliverMessageID(deliver)
@@ -289,7 +292,7 @@ func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID
SentAt: formatRFC3339Nano(sentAt),
AckDeadlineAt: formatRFC3339Nano(ackDeadlineAt),
}
tracker := registerDownstreamAck(session, deliveryID, sequenceID, messageID, ackDeadlineAt)
tracker := registerDownstreamAck(session, deliveryID, claimID, sequenceID, messageID, ackDeadlineAt)
if err := session.conn.SendPkt(deliver, sequenceID); err != nil {
removeDownstreamAck(tracker)
session.recordDownstreamProtocol(deliver, deliveryID, sequenceID, messageID, "failed", "SEND_FAILED", err)
@@ -306,7 +309,7 @@ func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID
result.Sent = true
if deliveryID != "" && session.deliveryReport != nil {
go session.deliveryReport(downstreamDeliveryLifecycleEvent{
Kind: "sent", DeliveryID: deliveryID, ConnectionID: session.connectionID,
Kind: "sent", DeliveryID: deliveryID, ClaimID: claimID, ConnectionID: session.connectionID,
SequenceID: sequenceID, MessageID: messageID, ObservedAt: sentAt, AckDeadlineAt: ackDeadlineAt,
})
}
gateway/internal/inbound/pending_recovery.go
+93 -3
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"log"
"strings"
"sync"
"time"
)
@@ -15,6 +16,8 @@ import (
type pendingDeliveryRequest struct {
Account string `json:"account"`
Limit int `json:"limit,omitempty"`
ClaimID string `json:"claimId"`
LeaseMS int `json:"leaseMs"`
}
type pendingDelivery struct {
@@ -22,6 +25,7 @@ type pendingDelivery struct {
DeliveryType string `json:"deliveryType"`
Payload json.RawMessage `json:"payload"`
CreatedAt time.Time `json:"createdAt"`
ClaimID string `json:"claimId"`
}
type pendingFlushResult struct {
@@ -33,25 +37,96 @@ type pendingFlushResult struct {
LastError string
}
type pendingFlushCall struct {
done chan struct{}
result pendingFlushResult
err error
}
var pendingFlushSingleflight = struct {
sync.Mutex
byAccount map[string]*pendingFlushCall
}{byAccount: make(map[string]*pendingFlushCall)}
type pendingFlushSchedule struct {
first time.Time
timer *time.Timer
}
var pendingFlushDebouncer = struct {
sync.Mutex
byAccount map[string]*pendingFlushSchedule
}{byAccount: make(map[string]*pendingFlushSchedule)}
func (s Server) schedulePendingFlush(account string, logger *log.Logger) {
account = strings.TrimSpace(account)
if account == "" {
return
}
pendingFlushDebouncer.Lock()
if scheduled := pendingFlushDebouncer.byAccount[account]; scheduled != nil {
if time.Since(scheduled.first) < 250*time.Millisecond {
scheduled.timer.Reset(25 * time.Millisecond)
}
pendingFlushDebouncer.Unlock()
return
}
scheduled := &pendingFlushSchedule{first: time.Now()}
scheduled.timer = time.AfterFunc(25*time.Millisecond, func() {
pendingFlushDebouncer.Lock()
if pendingFlushDebouncer.byAccount[account] == scheduled {
delete(pendingFlushDebouncer.byAccount, account)
}
pendingFlushDebouncer.Unlock()
if _, err := s.flushPending(account, logger); err != nil {
logger.Printf("cmpp inbound event=scheduled_pending_flush_failed account=%s error=%q", account, err.Error())
}
})
pendingFlushDebouncer.byAccount[account] = scheduled
pendingFlushDebouncer.Unlock()
}
func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushResult, error) {
account = strings.TrimSpace(account)
result := pendingFlushResult{Account: account}
if account == "" {
return result, nil
}
pendingFlushSingleflight.Lock()
if active := pendingFlushSingleflight.byAccount[account]; active != nil {
pendingFlushSingleflight.Unlock()
<-active.done
return active.result, active.err
}
call := &pendingFlushCall{done: make(chan struct{})}
pendingFlushSingleflight.byAccount[account] = call
pendingFlushSingleflight.Unlock()
call.result, call.err = s.flushPendingClaimed(account, logger)
pendingFlushSingleflight.Lock()
delete(pendingFlushSingleflight.byAccount, account)
close(call.done)
pendingFlushSingleflight.Unlock()
return call.result, call.err
}
func (s Server) flushPendingClaimed(account string, logger *log.Logger) (pendingFlushResult, error) {
result := pendingFlushResult{Account: account}
claimID := fmt.Sprintf("%s:%s:%d", s.gatewayInstanceID(), account, time.Now().UnixNano())
var deliveries []pendingDelivery
if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100}, &deliveries); err != nil {
if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100, ClaimID: claimID, LeaseMS: 30000}, &deliveries); err != nil {
logger.Printf("cmpp inbound pending delivery fetch failed account=%s err=%v", account, err)
result.LastError = err.Error()
return result, err
}
result.Deliveries = len(deliveries)
for _, delivery := range deliveries {
delivery.ClaimID = defaultString(delivery.ClaimID, claimID)
sendResult, err := s.pushPendingDelivery(account, delivery)
if err != nil {
result.FailedCount++
result.LastError = err.Error()
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
"id": delivery.ID, "errorMessage": err.Error(), "failureType": "send_failed",
"id": delivery.ID, "claimId": delivery.ClaimID, "errorMessage": err.Error(), "failureType": "send_failed",
"connectionId": sendResult.ConnectionID, "sequenceId": sendResult.SequenceID,
"messageId": sendResult.MessageID, "sentAt": sendResult.SentAt,
}, nil)
@@ -63,6 +138,7 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
}
if sendResult.ReasonCode == "SUBMIT_RESPONSE_PENDING" {
result.WaitingCount++
_ = s.releasePendingClaim(delivery, sendResult)
continue
}
errorMessage := defaultString(sendResult.ErrorMessage, "gateway did not complete downstream delivery")
@@ -74,8 +150,12 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
result.FailedCount++
}
result.LastError = errorMessage
if sendResult.Retryable {
_ = s.releasePendingClaim(delivery, sendResult)
continue
}
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
"id": delivery.ID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode),
"id": delivery.ID, "claimId": delivery.ClaimID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode),
"failureType": failureType, "connectionId": sendResult.ConnectionID,
"sequenceId": sendResult.SequenceID, "messageId": sendResult.MessageID, "sentAt": sendResult.SentAt,
}, nil)
@@ -83,6 +163,14 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
return result, nil
}
func (s Server) releasePendingClaim(delivery pendingDelivery, sendResult DownstreamSendResult) error {
return s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
"id": delivery.ID, "claimId": delivery.ClaimID,
"errorMessage": errorMessageWithCode(defaultString(sendResult.ErrorMessage, "Gateway released downstream delivery claim"), sendResult.ReasonCode),
"failureType": "claim_released",
}, nil)
}
func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (DownstreamSendResult, error) {
switch delivery.DeliveryType {
case "receipt":
@@ -91,6 +179,7 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D
return DownstreamSendResult{}, err
}
event.DeliveryID = delivery.ID
event.ClaimID = delivery.ClaimID
event.Account = defaultString(event.Account, account)
allowRecovery := !delivery.CreatedAt.IsZero() && time.Since(delivery.CreatedAt) >= 5*time.Second
return pushReceiptWithResult(event, allowRecovery)
@@ -100,6 +189,7 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D
return DownstreamSendResult{}, err
}
event.DeliveryID = delivery.ID
event.ClaimID = delivery.ClaimID
event.Account = defaultString(event.Account, account)
return PushUplinkWithResult(event)
default:
gateway/internal/inbound/protocol_log.go
+11 -14
View File
@@ -1,6 +1,7 @@
package inbound
import (
"cmpp-platform/gateway/internal/protocollog"
"context"
"fmt"
cmpp "github.com/bigwhite/gocmpp"
@@ -8,20 +9,7 @@ import (
"strconv"
)
type protocolLogEvent struct {
Protocol string `json:"protocol"`
Direction string `json:"direction"`
EventType string `json:"eventType"`
Status string `json:"status"`
TenantID string `json:"tenantId,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
Account string `json:"account,omitempty"`
MessageID string `json:"messageId,omitempty"`
GatewayMessageID string `json:"gatewayMessageId,omitempty"`
Phone string `json:"phone,omitempty"`
ResultCode string `json:"resultCode,omitempty"`
Detail map[string]any `json:"detail,omitempty"`
}
type protocolLogEvent = protocollog.Event
func (s Server) submitResponseProtocolLogger(
account string,
@@ -71,6 +59,15 @@ func protocolSubmitResponseDetail(sequenceID uint32, sendErr error) map[string]a
}
func (s Server) emitProtocolLog(event protocolLogEvent) {
event.GatewayInstanceID = s.GatewayInstanceID
if s.ProtocolLogPublisher != nil {
go func() {
if err := s.ProtocolLogPublisher.Publish(context.Background(), event); err != nil {
log.Printf("cmpp inbound protocol log Redis publish failed account=%s message_id=%s error=%q", event.Account, event.MessageID, err)
}
}()
return
}
go func() {
ctx, cancel := context.WithTimeout(context.Background(), defaultHTTPTimeout)
defer cancel()
gateway/internal/inbound/server.go
+5
View File
@@ -1,6 +1,8 @@
package inbound
import (
"cmpp-platform/gateway/internal/protocollog"
"context"
cmpp "github.com/bigwhite/gocmpp"
"io"
"log"
@@ -22,6 +24,9 @@ type Server struct {
RecoveryStore RecoveryStore
GatewayInstanceID string
MaxSubmitConcurrency int
ProtocolLogPublisher interface {
Publish(context.Context, protocollog.Event) error
}
}
func (s Server) ListenAndServe() error {
gateway/internal/inbound/server_test.go
+39 -2
View File
@@ -930,6 +930,42 @@ func TestFlushOnlineAccountsFetchesPendingDeliveries(t *testing.T) {
}
}
func TestFlushPendingCoalescesConcurrentRequestsPerAccount(t *testing.T) {
var calls atomic.Int32
release := make(chan struct{})
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/gateway/events/downstream/pending" {
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
calls.Add(1)
<-release
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
}))
defer api.Close()
server := Server{APIBaseURL: api.URL + "/api", GatewayInstanceID: "gateway-a"}
results := make(chan error, 8)
for range 8 {
go func() {
_, err := server.flushPending("100001", log.Default())
results <- err
}()
}
deadline := time.Now().Add(time.Second)
for calls.Load() == 0 && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
close(release)
for range 8 {
if err := <-results; err != nil {
t.Fatalf("flush failed: %v", err)
}
}
if got := calls.Load(); got != 1 {
t.Fatalf("pending fetch calls = %d, want 1", got)
}
}
func TestRecoverPendingCandidatesFetchesPresenceAccounts(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
@@ -1085,7 +1121,7 @@ func TestDownstreamDeliveryRequiresAcknowledgement(t *testing.T) {
messageID: "MSG-LONG-1", phoneNumber: "18821203795",
protocolLog: func(event protocolLogEvent) { protocolEvents <- event },
}
registerDownstreamAck(session, "delivery-1", 37, 9016479179509871733, time.Now().Add(time.Second))
registerDownstreamAck(session, "delivery-1", "", 37, 9016479179509871733, time.Now().Add(time.Second))
handleDownstreamAcknowledgement(conn, 37, 9016479179509871733, 0, log.Default())
select {
@@ -1165,6 +1201,7 @@ func TestSendDownstreamRejectsZeroMessageID(t *testing.T) {
&downstreamSession{mu: &sync.Mutex{}},
&cmpp.Cmpp2DeliverReqPkt{MsgId: 0},
"delivery-zero",
"",
)
if err == nil || !strings.Contains(err.Error(), "Msg_Id=0") {
t.Fatalf("expected zero Msg_Id rejection, got %v", err)
@@ -1180,7 +1217,7 @@ func TestDownstreamDeliveryReportsAckTimeout(t *testing.T) {
conn: &cmpp.Conn{}, connectionID: "conn-1",
deliveryReport: func(event downstreamDeliveryLifecycleEvent) { events <- event },
}
registerDownstreamAck(session, "delivery-timeout", 38, 9017467844344255865, time.Now().Add(20*time.Millisecond))
registerDownstreamAck(session, "delivery-timeout", "", 38, 9017467844344255865, time.Now().Add(20*time.Millisecond))
select {
case event := <-events:
gateway/internal/inbound/submit.go
+1 -5
View File
@@ -212,11 +212,7 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
if sendErr != nil {
return
}
go func() {
if _, err := s.flushPending(account, logger); err != nil {
logger.Printf("cmpp inbound event=post_submit_pending_flush_failed account=%s message_id=%s error=%q", account, result.MessageID, err.Error())
}
}()
s.schedulePendingFlush(account, logger)
})
logger.Printf(
"cmpp inbound event=submit_accepted protocol=%s packet_type=%s account=%s remote=%s seq=%d phone=%s dest_count=%d accepted_count=%d result=0 message_id=%s gateway_message_id=%d duration_ms=%d content_chars=%d content_hash=%s",