feat: complete cmpp gateway delivery recovery workflows

This commit is contained in:
hectorzhao
2026-07-08 16:30:06 +08:00
parent cc628d0214
commit 8144f08652
60 changed files with 8901 additions and 94 deletions
+697
View File
@@ -0,0 +1,697 @@
package upstream
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
"cmpp-platform/gateway/internal/queue"
cmpp "github.com/bigwhite/gocmpp"
cmpputils "github.com/bigwhite/gocmpp/utils"
)
const (
defaultConnectTimeout = 5 * time.Second
defaultSubmitTimeout = 10 * time.Second
defaultHTTPTimeout = 10 * time.Second
defaultWindowSize = 16
)
type Manager struct {
APIBaseURL string
HTTPClient *http.Client
mu sync.Mutex
conns map[string]*connectionPool
}
func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
if err := validateSubmitCommand(cmd); err != nil {
result := submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error())
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err
}
pool, err := m.connectionFor(cmd)
if err != nil {
result := submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error())
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err
}
result, err := pool.submit(ctx, cmd)
if err != nil {
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err
}
if err := m.post(ctx, "/gateway/events/submit-result", result); err != nil {
return result, err
}
return result, nil
}
func (m *Manager) connectionFor(cmd queue.SubmitCommand) (*connectionPool, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.HTTPClient == nil {
m.HTTPClient = &http.Client{Timeout: defaultHTTPTimeout}
}
if m.conns == nil {
m.conns = make(map[string]*connectionPool)
}
pool := m.conns[cmd.ChannelID]
if pool == nil || !pool.matches(cmd.Upstream) {
if pool != nil {
pool.close()
}
pool = &connectionPool{
channelID: cmd.ChannelID,
config: normalizeUpstreamConfig(cmd.Upstream),
apiBaseURL: m.APIBaseURL,
httpClient: m.HTTPClient,
}
m.conns[cmd.ChannelID] = pool
}
if err := pool.ensureConnected(); err != nil {
delete(m.conns, cmd.ChannelID)
return nil, err
}
return pool, nil
}
func (m *Manager) post(ctx context.Context, path string, payload any) error {
client := m.HTTPClient
if client == nil {
client = &http.Client{Timeout: defaultHTTPTimeout}
}
return postJSON(ctx, client, m.APIBaseURL, path, payload)
}
type connectionPool struct {
channelID string
config queue.UpstreamConfig
apiBaseURL string
httpClient *http.Client
mu sync.Mutex
conns []*connection
next int
}
func (p *connectionPool) matches(config queue.UpstreamConfig) bool {
return p.config == normalizeUpstreamConfig(config)
}
func (p *connectionPool) ensureConnected() error {
p.mu.Lock()
defer p.mu.Unlock()
desired := p.config.DesiredConnections
if desired <= 0 {
desired = 1
}
for len(p.conns) < desired {
index := len(p.conns)
conn := &connection{
channelID: p.channelID,
config: p.config,
index: index,
apiBaseURL: p.apiBaseURL,
httpClient: p.httpClient,
window: make(chan struct{}, p.config.WindowSize),
pending: make(map[uint32]chan submitPartResponse),
tracker: make(map[uint64]queue.SubmitCommand),
longUplink: make(map[string]*longUplinkAssembly),
}
if err := conn.ensureConnected(); err != nil {
conn.close()
p.closeLocked()
return err
}
p.conns = append(p.conns, conn)
}
return nil
}
func (p *connectionPool) submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content)
if err != nil {
result := submitResult(cmd, 0, "", "rejected", "ENCODE_FAILED", err.Error())
return result, err
}
var firstSequence uint32
var firstGatewayMessageID string
segments := make([]queue.SubmitSegmentResult, 0, len(parts))
for _, part := range parts {
conn, release, err := p.acquireConnection(ctx)
if err != nil {
result := submitResult(cmd, 0, "", "timeout", "WINDOW_TIMEOUT", err.Error())
result.Segments = segments
return result, err
}
seq, gatewayMessageID, result, err := conn.submitPart(ctx, cmd, part)
release()
segments = append(segments, submitSegmentResult(part, seq, gatewayMessageID, result))
if firstSequence == 0 {
firstSequence = seq
}
if firstGatewayMessageID == "" {
firstGatewayMessageID = gatewayMessageID
}
if err != nil {
result.Segments = segments
return result, err
}
if result.SubmitStatus != "accepted" {
result.Segments = segments
return result, nil
}
}
result := submitResult(cmd, firstSequence, firstGatewayMessageID, "accepted", "", "")
result.Segments = segments
return result, nil
}
func (p *connectionPool) acquireConnection(ctx context.Context) (*connection, func(), error) {
waitCtx, cancel := context.WithTimeout(ctx, defaultSubmitTimeout)
defer cancel()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
if conn, release := p.tryAcquireConnection(); conn != nil {
if err := conn.ensureConnected(); err != nil {
release()
select {
case <-waitCtx.Done():
return nil, nil, waitCtx.Err()
case <-ticker.C:
continue
}
}
return conn, release, nil
}
select {
case <-waitCtx.Done():
return nil, nil, waitCtx.Err()
case <-ticker.C:
}
}
}
func (p *connectionPool) tryAcquireConnection() (*connection, func()) {
p.mu.Lock()
defer p.mu.Unlock()
if len(p.conns) == 0 {
return nil, nil
}
for i := 0; i < len(p.conns); i++ {
index := (p.next + i) % len(p.conns)
conn := p.conns[index]
if conn.tryAcquireWindow() {
p.next = (index + 1) % len(p.conns)
return conn, conn.releaseWindow
}
}
return nil, nil
}
func (p *connectionPool) close() {
p.mu.Lock()
defer p.mu.Unlock()
p.closeLocked()
}
func (p *connectionPool) closeLocked() {
for _, conn := range p.conns {
conn.close()
}
p.conns = nil
}
type connection struct {
channelID string
config queue.UpstreamConfig
index int
apiBaseURL string
httpClient *http.Client
mu sync.Mutex
sendMu sync.Mutex
client *cmpp.Client
window chan struct{}
pending map[uint32]chan submitPartResponse
tracker map[uint64]queue.SubmitCommand
longUplink map[string]*longUplinkAssembly
readOnce sync.Once
closed bool
}
type submitPartResponse struct {
rsp *cmpp.Cmpp3SubmitRspPkt
err error
}
func (c *connection) matches(config queue.UpstreamConfig) bool {
return c.config == normalizeUpstreamConfig(config)
}
func (c *connection) ensureConnected() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.client != nil && !c.closed {
return nil
}
client := cmpp.NewClient(protocolVersion(c.config.CMPPVersion))
addr := fmt.Sprintf("%s:%d", c.config.GatewayHost, c.config.GatewayPort)
if err := client.Connect(addr, c.config.Account, c.config.PasswordCipher, defaultConnectTimeout); err != nil {
client.Disconnect()
return err
}
c.client = client
c.closed = false
go c.readLoop()
return nil
}
func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) {
rspCh := make(chan submitPartResponse, 1)
pkt := &cmpp.Cmpp3SubmitReqPkt{
PkTotal: part.PkTotal,
PkNumber: part.PkNumber,
TpUdhi: part.TpUdhi,
RegisteredDelivery: uint8(cmd.CMPP.RegisteredDelivery),
MsgLevel: 1,
ServiceId: cmd.CMPP.ServiceID,
FeeUserType: uint8(defaultInt(cmd.CMPP.FeeUserType, 2)),
FeeTerminalId: cmd.PhoneNumber,
MsgFmt: uint8(cmd.CMPP.MsgFmt),
MsgSrc: c.config.Account,
FeeType: defaultString(cmd.CMPP.FeeType, "02"),
FeeCode: defaultString(cmd.CMPP.FeeCode, "0"),
SrcId: cmd.CMPP.SrcID,
DestUsrTl: 1,
DestTerminalId: []string{cmd.PhoneNumber},
MsgLength: uint8(len(part.MsgContent)),
MsgContent: part.MsgContent,
}
c.sendMu.Lock()
seq, err := c.client.SendReqPkt(pkt)
c.sendMu.Unlock()
if err != nil {
c.close()
result := submitResult(cmd, 0, "", "timeout", "SEND_FAILED", err.Error())
return 0, "", result, err
}
c.mu.Lock()
c.pending[seq] = rspCh
c.mu.Unlock()
defer func() {
c.mu.Lock()
delete(c.pending, seq)
c.mu.Unlock()
}()
waitCtx, cancel := context.WithTimeout(ctx, defaultSubmitTimeout)
defer cancel()
select {
case <-waitCtx.Done():
result := submitResult(cmd, seq, "", "timeout", "SUBMIT_TIMEOUT", waitCtx.Err().Error())
return seq, "", result, waitCtx.Err()
case rsp := <-rspCh:
if rsp.err != nil {
result := submitResult(cmd, seq, "", "timeout", "CONNECTION_LOST", rsp.err.Error())
return seq, "", result, rsp.err
}
if rsp.rsp == nil {
err := fmt.Errorf("submit response is empty")
result := submitResult(cmd, seq, "", "timeout", "EMPTY_SUBMIT_RESPONSE", err.Error())
return seq, "", result, err
}
gatewayMessageID := fmt.Sprint(rsp.rsp.MsgId)
status := "accepted"
errorCode := ""
errorMessage := ""
if rsp.rsp.Result != 0 {
status = "rejected"
errorCode = fmt.Sprint(rsp.rsp.Result)
errorMessage = fmt.Sprintf("upstream submit rejected with result %d", rsp.rsp.Result)
}
if rsp.rsp.Result == 0 {
c.mu.Lock()
c.tracker[rsp.rsp.MsgId] = cmd
c.mu.Unlock()
}
return seq, gatewayMessageID, submitResult(cmd, seq, gatewayMessageID, status, errorCode, errorMessage), nil
}
}
func (c *connection) tryAcquireWindow() bool {
if c.window == nil {
c.window = make(chan struct{}, defaultWindowSize)
}
select {
case c.window <- struct{}{}:
return true
default:
return false
}
}
func (c *connection) releaseWindow() {
if c.window == nil {
return
}
select {
case <-c.window:
default:
}
}
func (c *connection) readLoop() {
for {
pkt, err := c.client.RecvAndUnpackPkt(time.Second)
if err != nil {
c.mu.Lock()
closed := c.closed
c.mu.Unlock()
if closed {
return
}
if isTemporaryReadTimeout(err) {
continue
}
c.handleConnectionLoss(err)
continue
}
switch p := pkt.(type) {
case *cmpp.Cmpp3SubmitRspPkt:
c.mu.Lock()
ch := c.pending[p.SeqId]
c.mu.Unlock()
if ch != nil {
ch <- submitPartResponse{rsp: p}
}
case *cmpp.Cmpp3DeliverReqPkt:
c.handleDeliver(p)
case *cmpp.CmppActiveTestReqPkt:
_ = c.client.SendRspPkt(&cmpp.CmppActiveTestRspPkt{}, p.SeqId)
}
}
}
func (c *connection) handleDeliver(pkt *cmpp.Cmpp3DeliverReqPkt) {
_ = c.client.SendRspPkt(&cmpp.Cmpp3DeliverRspPkt{MsgId: pkt.MsgId, Result: 0}, pkt.SeqId)
if pkt.RegisterDelivery == 1 {
var receipt cmpp.CmppReceiptPkt
if err := receipt.Unpack([]byte(pkt.MsgContent)); err != nil {
return
}
cmd, ok := c.commandFor(receipt.MsgId)
if !ok {
cmd, ok = c.commandFor(pkt.MsgId)
}
traceID := fmt.Sprintf("receipt-%d", receipt.MsgId)
messageID := fmt.Sprintf("receipt-%d", receipt.MsgId)
channelID := c.channelID
if ok {
traceID = cmd.TraceID
messageID = cmd.MessageID
channelID = cmd.ChannelID
}
event := queue.ReceiptEvent{
Envelope: queue.Envelope{
SchemaVersion: queue.SchemaVersion,
MessageType: queue.MessageTypeReceiptEvent,
TraceID: traceID,
MessageID: messageID,
ChannelID: channelID,
CreatedAt: time.Now().UTC(),
},
SequenceID: pkt.SeqId,
GatewayMessageID: fmt.Sprint(receipt.MsgId),
PhoneNumber: strings.TrimSpace(receipt.DestTerminalId),
ReceiptStatus: receiptStatus(receipt.Stat),
RawStatus: strings.TrimSpace(receipt.Stat),
DeliveredAt: time.Now().UTC(),
}
_ = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt", event)
return
}
content, complete, err := c.decodeUplinkContent(pkt)
if err != nil {
return
}
if !complete {
return
}
cmd, _ := c.commandFor(pkt.MsgId)
event := queue.UplinkEvent{
Envelope: queue.Envelope{
SchemaVersion: queue.SchemaVersion,
MessageType: queue.MessageTypeUplinkEvent,
TraceID: cmd.TraceID,
MessageID: cmd.MessageID,
ChannelID: c.channelID,
CreatedAt: time.Now().UTC(),
},
SequenceID: pkt.SeqId,
PhoneNumber: strings.TrimSpace(pkt.SrcTerminalId),
DestID: strings.TrimSpace(pkt.DestId),
Content: content,
ReceivedAt: time.Now().UTC(),
}
_ = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/uplink", event)
}
func (c *connection) decodeUplinkContent(pkt *cmpp.Cmpp3DeliverReqPkt) (string, bool, error) {
if pkt.TpUdhi != 1 {
content, err := decodeContent(pkt.MsgFmt, pkt.MsgContent)
return content, true, err
}
ref, total, number, payload, ok := parseConcatSegment(pkt.MsgContent)
if !ok {
content, err := decodeContent(pkt.MsgFmt, pkt.MsgContent)
return content, true, err
}
key := fmt.Sprintf("%s:%s:%s:%d:%d", c.channelID, strings.TrimSpace(pkt.SrcTerminalId), strings.TrimSpace(pkt.DestId), ref, total)
c.mu.Lock()
if c.longUplink == nil {
c.longUplink = make(map[string]*longUplinkAssembly)
}
pruneLongUplinkAssemblies(c.longUplink, time.Now(), 10*time.Minute)
content, complete, err := assembleLongUplink(c.longUplink, key, pkt.MsgFmt, total, number, payload)
c.mu.Unlock()
return content, complete, err
}
func (c *connection) commandFor(gatewayMsgID uint64) (queue.SubmitCommand, bool) {
c.mu.Lock()
defer c.mu.Unlock()
cmd, ok := c.tracker[gatewayMsgID]
return cmd, ok
}
func (c *connection) close() {
c.handleConnectionLoss(fmt.Errorf("connection closed"))
}
func (c *connection) handleConnectionLoss(err error) {
c.mu.Lock()
if c.closed && c.client == nil {
c.mu.Unlock()
return
}
c.closed = true
pending := c.pending
c.pending = make(map[uint32]chan submitPartResponse)
if c.client != nil {
c.client.Disconnect()
c.client = nil
}
c.mu.Unlock()
for _, ch := range pending {
select {
case ch <- submitPartResponse{err: err}:
default:
}
}
}
func submitResult(cmd queue.SubmitCommand, sequenceID uint32, gatewayMessageID string, status string, code string, message string) queue.SubmitResult {
if gatewayMessageID == "" {
gatewayMessageID = fmt.Sprintf("GW-%s-%d", cmd.SubmitID, time.Now().UnixNano())
}
return queue.SubmitResult{
Envelope: queue.Envelope{
SchemaVersion: queue.SchemaVersion,
MessageType: queue.MessageTypeSubmitResult,
TraceID: cmd.TraceID,
MessageID: cmd.MessageID,
ChannelID: cmd.ChannelID,
CreatedAt: time.Now().UTC(),
},
SequenceID: sequenceID,
GatewayMessageID: gatewayMessageID,
SubmitStatus: status,
ErrorCode: code,
ErrorMessage: message,
SubmittedAt: time.Now().UTC(),
}
}
func submitSegmentResult(part submitPart, sequenceID uint32, gatewayMessageID string, result queue.SubmitResult) queue.SubmitSegmentResult {
if gatewayMessageID == "" {
gatewayMessageID = result.GatewayMessageID
}
return queue.SubmitSegmentResult{
SegmentTotal: int(part.PkTotal),
SegmentIndex: int(part.PkNumber),
SequenceID: sequenceID,
GatewayMessageID: gatewayMessageID,
SubmitStatus: result.SubmitStatus,
ErrorCode: result.ErrorCode,
ErrorMessage: result.ErrorMessage,
SubmittedAt: result.SubmittedAt,
}
}
func validateSubmitCommand(cmd queue.SubmitCommand) error {
if cmd.MessageType != queue.MessageTypeSubmitCommand {
return fmt.Errorf("unsupported messageType %q", cmd.MessageType)
}
if cmd.MessageID == "" || cmd.ChannelID == "" || cmd.SubmitID == "" {
return fmt.Errorf("messageId, channelId and submitId are required")
}
if cmd.Upstream.GatewayHost == "" || cmd.Upstream.GatewayPort <= 0 {
return fmt.Errorf("upstream gatewayHost and gatewayPort are required")
}
if cmd.Upstream.Account == "" || cmd.Upstream.PasswordCipher == "" {
return fmt.Errorf("upstream account and passwordCipher are required")
}
if len(cmd.PhoneNumber) == 0 || len(cmd.Content) == 0 {
return fmt.Errorf("phoneNumber and content are required")
}
return nil
}
func normalizeUpstreamConfig(config queue.UpstreamConfig) queue.UpstreamConfig {
if config.DesiredConnections <= 0 {
config.DesiredConnections = 1
}
if config.WindowSize <= 0 {
config.WindowSize = defaultWindowSize
}
return config
}
func isTemporaryReadTimeout(err error) bool {
var netErr net.Error
return errors.As(err, &netErr) && netErr.Timeout()
}
func postJSON(ctx context.Context, client *http.Client, apiBaseURL string, path string, payload any) error {
if client == nil {
client = &http.Client{Timeout: defaultHTTPTimeout}
}
body, err := json.Marshal(payload)
if err != nil {
return err
}
base := strings.TrimRight(apiBaseURL, "/")
if base == "" {
base = "http://127.0.0.1:3000/api"
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+path, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("api returned %s", resp.Status)
}
return nil
}
func encodeContent(format int, content string) (string, error) {
switch format {
case 8:
return cmpputils.Utf8ToUcs2(content)
case 15:
return cmpputils.Utf8ToGB18030(content)
default:
return content, nil
}
}
func decodeContent(format uint8, content string) (string, error) {
switch format {
case 8:
return cmpputils.Ucs2ToUtf8(content)
case 15:
return cmpputils.GB18030ToUtf8(content)
default:
return content, nil
}
}
func protocolVersion(version string) cmpp.Type {
if strings.HasPrefix(version, "2") {
return cmpp.V20
}
return cmpp.V30
}
func receiptStatus(stat string) string {
switch strings.ToUpper(strings.TrimSpace(stat)) {
case "DELIVRD":
return "delivered"
case "":
return "unknown"
default:
return "undelivered"
}
}
func defaultString(value string, fallback string) string {
if value == "" {
return fallback
}
return value
}
func defaultInt(value int, fallback int) int {
if value == 0 {
return fallback
}
return value
}