fix: correlate downstream receipts with submit responses

gateway/internal/inbound/server.go

@@ -6,6 +6,7 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -75,6 +76,7 @@ type DownstreamReceipt struct {
 	ReceiptStatus    string `json:"receiptStatus"`
 	RawStatus        string `json:"rawStatus,omitempty"`
 	ErrorCode        string `json:"errorCode,omitempty"`
+	SubmitSequenceID uint32 `json:"submitSequenceId,omitempty"`
 	DeliveredAt      string `json:"deliveredAt,omitempty"`
 }
 
@@ -220,7 +222,11 @@ func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger
 	}
 	rememberAccount(session)
 	go s.reportConnection(&session, "connected", "")
-	go s.flushPending(defaultString(auth.Account, account), logger)
+	response.AfterSend = func(sendErr error) {
+		if sendErr == nil {
+			go s.flushPending(defaultString(auth.Account, account), logger)
+		}
+	}
 	logger.Printf(
 		"cmpp inbound event=login_accepted protocol=%s requested_version=0x%02x response_version=0x%02x account=%s remote=%s",
 		cmppVersionName(req.Version), uint8(req.Version), uint8(req.Version), account, packet.Conn.Conn.RemoteAddr(),
@@ -319,11 +325,16 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
 	if current := findSessionByConn(packet.Conn); current != nil && current.report != nil {
 		go current.report(current, "submit", "")
 	}
-	go func() {
-		if _, err := s.flushPending(account, logger); err != nil {
-			logger.Printf("cmpp inbound event=post_submit_pending_flush_failed account=%s message_id=%s error=%q", account, result.MessageID, err.Error())
+	response.AfterSend = func(sendErr error) {
+		if sendErr != nil {
+			return
 		}
-	}()
+		go func() {
+			if _, err := s.flushPending(account, logger); err != nil {
+				logger.Printf("cmpp inbound event=post_submit_pending_flush_failed account=%s message_id=%s error=%q", account, result.MessageID, err.Error())
+			}
+		}()
+	}
 	logger.Printf(
 		"cmpp inbound event=submit_accepted protocol=%s packet_type=%s account=%s remote=%s seq=%d phone=%s result=0 message_id=%s gateway_message_id=%d duration_ms=%d content_chars=%d content_hash=%s",
 		clientProtocol, req.protocol, account, remote, req.sequenceID, phone, result.MessageID, gatewayMsgID, time.Since(startedAt).Milliseconds(), len([]rune(content)), contentHash,
@@ -493,6 +504,7 @@ type pendingDelivery struct {
 	ID           string          `json:"id"`
 	DeliveryType string          `json:"deliveryType"`
 	Payload      json.RawMessage `json:"payload"`
+	CreatedAt    time.Time       `json:"createdAt"`
 }
 
 type pendingFlushResult struct {
@@ -546,7 +558,8 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D
 		}
 		event.DeliveryID = delivery.ID
 		event.Account = defaultString(event.Account, account)
-		return PushReceiptWithResult(event)
+		allowRecovery := !delivery.CreatedAt.IsZero() && time.Since(delivery.CreatedAt) >= 5*time.Second
+		return pushReceiptWithResult(event, allowRecovery)
 	case "uplink":
 		var event DownstreamUplink
 		if err := json.Unmarshal(delivery.Payload, &event); err != nil {
@@ -848,7 +861,14 @@ func PushReceipt(event DownstreamReceipt) (bool, error) {
 }
 
 func PushReceiptWithResult(event DownstreamReceipt) (DownstreamSendResult, error) {
-	session := findSession(event.MessageID, event.Account)
+	return pushReceiptWithResult(event, false)
+}
+
+func pushReceiptWithResult(event DownstreamReceipt, allowRecovery bool) (DownstreamSendResult, error) {
+	session := findReceiptSession(event.MessageID, event.Account)
+	if session == nil && allowRecovery {
+		session = recoverReceiptSession(event)
+	}
 	if session == nil {
 		return DownstreamSendResult{}, nil
 	}
@@ -878,6 +898,34 @@ func PushReceiptWithResult(event DownstreamReceipt) (DownstreamSendResult, error
 	return sendDownstream(session, deliver, event.DeliveryID)
 }
 
+func findReceiptSession(messageID string, account string) *downstreamSession {
+	downstreamRegistry.RLock()
+	defer downstreamRegistry.RUnlock()
+	if messageID != "" {
+		return downstreamRegistry.byMessageID[messageID]
+	}
+	if account != "" {
+		return downstreamRegistry.byAccount[account]
+	}
+	return nil
+}
+
+func recoverReceiptSession(event DownstreamReceipt) *downstreamSession {
+	if event.MessageID == "" || event.SubmitSequenceID == 0 || event.Account == "" {
+		return nil
+	}
+	downstreamRegistry.RLock()
+	accountSession := downstreamRegistry.byAccount[event.Account]
+	downstreamRegistry.RUnlock()
+	if accountSession == nil || accountSession.conn == nil {
+		return nil
+	}
+	recovered := *accountSession
+	recovered.messageID = event.MessageID
+	recovered.gatewayMsgID = messageIDFrom(event.MessageID, event.SubmitSequenceID)
+	return &recovered
+}
+
 func PushUplink(event DownstreamUplink) (bool, error) {
 	result, err := PushUplinkWithResult(event)
 	return result.Sent, err
@@ -936,8 +984,11 @@ func findSession(messageID string, account string) *downstreamSession {
 func sendDownstream(session *downstreamSession, deliver cmpp.Packer, deliveryID string) (DownstreamSendResult, error) {
 	session.mu.Lock()
 	defer session.mu.Unlock()
-	sequenceID := <-session.conn.SeqId
 	messageID := downstreamDeliverMessageID(deliver)
+	if messageID == 0 {
+		return DownstreamSendResult{}, errors.New("refusing downstream CMPP_DELIVER with Msg_Id=0")
+	}
+	sequenceID := <-session.conn.SeqId
 	sentAt := time.Now().UTC()
 	ackDeadlineAt := sentAt.Add(downstreamAckTimeout())
 	tracker := registerDownstreamAck(session, deliveryID, sequenceID, messageID, ackDeadlineAt)