fix: harden upstream and downstream receipt delivery

2026-07-25 23:32:13 +08:00
parent 5018167696
commit 54617c927e
18 changed files with 1797 additions and 779 deletions
@@ -73,7 +73,26 @@ func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.Su
 		return result, err
 	}

-	result, err := pool.submit(ctx, cmd)
+	result, err := pool.submit(ctx, cmd, func(segment queue.SubmitSegmentResult) {
+		payload := struct {
+			queue.Envelope
+			SubmitID string `json:"submitId,omitempty"`
+			queue.SubmitSegmentResult
+		}{
+			Envelope:            cmd.Envelope,
+			SubmitID:            cmd.SubmitID,
+			SubmitSegmentResult: segment,
+		}
+		callbackCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		postErr := m.post(callbackCtx, "/gateway/events/submit-segment-result", payload)
+		cancel()
+		if postErr != nil {
+			log.Printf(
+				"protocol_event protocol=cmpp direction=gateway_to_api event=submit_segment_result status=forward_failed channel_id=%s message_id=%s segment=%d/%d error=%q",
+				cmd.ChannelID, cmd.MessageID, segment.SegmentIndex, segment.SegmentTotal, postErr,
+			)
+		}
+	})
 	if err != nil {
 		if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
 			return result, postErr
@@ -404,7 +423,11 @@ func (p *connectionPool) superviseReconnects() {
 	}
 }

-func (p *connectionPool) submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
+func (p *connectionPool) submit(
+	ctx context.Context,
+	cmd queue.SubmitCommand,
+	onSegment func(queue.SubmitSegmentResult),
+) (queue.SubmitResult, error) {
 	parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content)
 	if err != nil {
 		result := submitResult(cmd, 0, "", "rejected", "ENCODE_FAILED", err.Error())
@@ -423,7 +446,11 @@ func (p *connectionPool) submit(ctx context.Context, cmd queue.SubmitCommand) (q
 		}
 		seq, gatewayMessageID, result, err := conn.submitPart(ctx, cmd, part)
 		release()
-		segments = append(segments, submitSegmentResult(part, seq, gatewayMessageID, result))
+		segment := submitSegmentResult(part, seq, gatewayMessageID, result)
+		segments = append(segments, segment)
+		if onSegment != nil {
+			onSegment(segment)
+		}
 		if firstSequence == 0 {
 			firstSequence = seq
 		}
@@ -878,13 +905,35 @@ func (c *connection) readLoop() {
 				ch <- submitPartResponse{seqID: p.SeqId, msgID: p.MsgId, result: p.Result}
 			}
 		case *cmpp.Cmpp2DeliverReqPkt:
-			responseErr := c.sendResponse(client, &cmpp.Cmpp2DeliverRspPkt{MsgId: p.MsgId, Result: 0}, p.SeqId)
-			c.emitDeliverResponse(deliverPacketFromCMPP2(p), responseErr)
-			c.handleDeliver(deliverPacketFromCMPP2(p))
+			deliver := deliverPacketFromCMPP2(p)
+			if deliver.registerDelivery == 1 {
+				if err := c.handleDeliver(deliver); err != nil {
+					log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_receipt status=intake_failed channel_id=%s sequence_id=%d error=%q", c.channelID, deliver.seqID, err)
+					c.handleConnectionLoss(fmt.Errorf("persist upstream receipt before DELIVER_RESP: %w", err))
+					return
+				}
+				responseErr := c.sendResponse(client, &cmpp.Cmpp2DeliverRspPkt{MsgId: p.MsgId, Result: 0}, p.SeqId)
+				c.emitDeliverResponse(deliver, responseErr)
+			} else {
+				responseErr := c.sendResponse(client, &cmpp.Cmpp2DeliverRspPkt{MsgId: p.MsgId, Result: 0}, p.SeqId)
+				c.emitDeliverResponse(deliver, responseErr)
+				_ = c.handleDeliver(deliver)
+			}
 		case *cmpp.Cmpp3DeliverReqPkt:
-			responseErr := c.sendResponse(client, &cmpp.Cmpp3DeliverRspPkt{MsgId: p.MsgId, Result: 0}, p.SeqId)
-			c.emitDeliverResponse(deliverPacketFromCMPP3(p), responseErr)
-			c.handleDeliver(deliverPacketFromCMPP3(p))
+			deliver := deliverPacketFromCMPP3(p)
+			if deliver.registerDelivery == 1 {
+				if err := c.handleDeliver(deliver); err != nil {
+					log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_receipt status=intake_failed channel_id=%s sequence_id=%d error=%q", c.channelID, deliver.seqID, err)
+					c.handleConnectionLoss(fmt.Errorf("persist upstream receipt before DELIVER_RESP: %w", err))
+					return
+				}
+				responseErr := c.sendResponse(client, &cmpp.Cmpp3DeliverRspPkt{MsgId: p.MsgId, Result: 0}, p.SeqId)
+				c.emitDeliverResponse(deliver, responseErr)
+			} else {
+				responseErr := c.sendResponse(client, &cmpp.Cmpp3DeliverRspPkt{MsgId: p.MsgId, Result: 0}, p.SeqId)
+				c.emitDeliverResponse(deliver, responseErr)
+				_ = c.handleDeliver(deliver)
+			}
 		case *cmpp.CmppActiveTestReqPkt:
 			_ = c.sendResponse(client, &cmpp.CmppActiveTestRspPkt{}, p.SeqId)
 			_ = c.pool.reportState(context.Background(), "heartbeat", nil)
@@ -1069,12 +1118,12 @@ func deliverPacketFromCMPP3(pkt *cmpp.Cmpp3DeliverReqPkt) deliverPacket {
 	}
 }

-func (c *connection) handleDeliver(pkt deliverPacket) {
+func (c *connection) handleDeliver(pkt deliverPacket) error {
 	if pkt.registerDelivery == 1 {
 		var receipt cmpp.CmppReceiptPkt
 		if err := receipt.Unpack([]byte(pkt.msgContent)); err != nil {
 			log.Printf("protocol_event protocol=cmpp direction=channel_to_platform event=deliver_receipt status=parse_failed channel_id=%s sequence_id=%d packet_msg_id=%d error=%q", c.channelID, pkt.seqID, pkt.msgID, err)
-			return
+			return err
 		}
 		log.Printf("protocol_event protocol=cmpp direction=channel_to_platform event=deliver_receipt status=received channel_id=%s sequence_id=%d gateway_message_id=%d raw_status=%s", c.channelID, pkt.seqID, receipt.MsgId, strings.TrimSpace(receipt.Stat))
 		cmd, ok := c.commandFor(receipt.MsgId)
@@ -1104,22 +1153,24 @@ func (c *connection) handleDeliver(pkt deliverPacket) {
 			ReceiptStatus:    receiptStatus(receipt.Stat),
 			RawStatus:        strings.TrimSpace(receipt.Stat),
 			DeliveredAt:      time.Now().UTC(),
+			ConnectionID:     c.identity(),
 		}
-		if err := postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt", event); err != nil {
+		if err := postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt/intake", event); err != nil {
 			log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_receipt status=forward_failed channel_id=%s sequence_id=%d gateway_message_id=%d error=%q", c.channelID, pkt.seqID, receipt.MsgId, err)
+			return err
 		} else {
 			log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_receipt status=forwarded channel_id=%s sequence_id=%d gateway_message_id=%d", c.channelID, pkt.seqID, receipt.MsgId)
 		}
-		return
+		return nil
 	}

 	content, complete, err := c.decodeUplinkContent(pkt)
 	if err != nil {
 		log.Printf("protocol_event protocol=cmpp direction=channel_to_platform event=deliver_uplink status=decode_failed channel_id=%s sequence_id=%d packet_msg_id=%d error=%q", c.channelID, pkt.seqID, pkt.msgID, err)
-		return
+		return err
 	}
 	if !complete {
-		return
+		return nil
 	}
 	cmd, _ := c.commandFor(pkt.msgID)
 	event := queue.UplinkEvent{
@@ -1142,6 +1193,14 @@ func (c *connection) handleDeliver(pkt deliverPacket) {
 	} else {
 		log.Printf("protocol_event protocol=cmpp direction=gateway_to_api event=deliver_uplink status=forwarded channel_id=%s sequence_id=%d packet_msg_id=%d", c.channelID, pkt.seqID, pkt.msgID)
 	}
+	return nil
+}
+
+func (c *connection) identity() string {
+	if c.pool != nil && strings.TrimSpace(c.pool.connectionID) != "" {
+		return fmt.Sprintf("%s-%d", c.pool.connectionID, c.index)
+	}
+	return fmt.Sprintf("%s-%d", c.channelID, c.index)
 }

 func (c *connection) decodeUplinkContent(pkt deliverPacket) (string, bool, error) {