hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete cmpp gateway delivery recovery workflows

Browse Source
This commit is contained in:
hectorzhao
2026-07-08 16:30:06 +08:00
parent cc628d0214
commit 8144f08652
60 changed files with 8901 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/internal/control/server.go
+150 -3
View File
@@ -9,6 +9,10 @@ import (
"strings"
"time"
"cmpp-platform/gateway/internal/inbound"
"cmpp-platform/gateway/internal/queue"
"cmpp-platform/gateway/internal/upstream"
cmpp "github.com/bigwhite/gocmpp"
)
@@ -53,9 +57,17 @@ type ConnectionStateCallback struct {
}
type Server struct {
APIBaseURL string
HTTPClient *http.Client
Dial DialFunc
APIBaseURL string
HTTPClient *http.Client
Dial DialFunc
Upstream *upstream.Manager
RecoveryCandidates func(context.Context) ([]inbound.DownstreamPresence, error)
RecoveryStatuses func(context.Context) ([]inbound.DownstreamRecoveryStatus, error)
}
type DownstreamRecoveryOverview struct {
Candidates []inbound.DownstreamPresence `json:"candidates"`
Statuses []inbound.DownstreamRecoveryStatus `json:"statuses"`
}
func Register(mux *http.ServeMux, server Server) {
@@ -65,7 +77,16 @@ func Register(mux *http.ServeMux, server Server) {
if server.Dial == nil {
server.Dial = DialCMPP
}
if server.Upstream == nil {
server.Upstream = &upstream.Manager{APIBaseURL: server.APIBaseURL, HTTPClient: server.HTTPClient}
}
mux.HandleFunc("/connections/connect", server.handleConnectChannel)
mux.HandleFunc("/upstream/submit", server.handleUpstreamSubmit)
mux.HandleFunc("/downstream/receipt", server.handleDownstreamReceipt)
mux.HandleFunc("/downstream/uplink", server.handleDownstreamUplink)
mux.HandleFunc("/downstream/recovery-candidates", server.handleDownstreamRecoveryCandidates)
mux.HandleFunc("/downstream/recovery-statuses", server.handleDownstreamRecoveryStatuses)
mux.HandleFunc("/downstream/recovery-overview", server.handleDownstreamRecoveryOverview)
}
func (s Server) handleConnectChannel(w http.ResponseWriter, r *http.Request) {
@@ -111,6 +132,132 @@ func (s Server) handleConnectChannel(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(status)
}
func (s Server) handleUpstreamSubmit(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
var command queue.SubmitCommand
if err := json.NewDecoder(r.Body).Decode(&command); err != nil {
http.Error(w, fmt.Sprintf("invalid submit command: %v", err), http.StatusBadRequest)
return
}
result, err := s.Upstream.Submit(r.Context(), command)
if err != nil {
w.WriteHeader(http.StatusBadGateway)
_ = json.NewEncoder(w).Encode(result)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(result)
}
func (s Server) handleDownstreamReceipt(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
var event inbound.DownstreamReceipt
if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
http.Error(w, fmt.Sprintf("invalid downstream receipt: %v", err), http.StatusBadRequest)
return
}
delivered, err := inbound.PushReceipt(event)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"delivered": delivered})
}
func (s Server) handleDownstreamUplink(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
var event inbound.DownstreamUplink
if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
http.Error(w, fmt.Sprintf("invalid downstream uplink: %v", err), http.StatusBadRequest)
return
}
delivered, err := inbound.PushUplink(event)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"delivered": delivered})
}
func (s Server) handleDownstreamRecoveryCandidates(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if s.RecoveryCandidates == nil {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode([]inbound.DownstreamPresence{})
return
}
candidates, err := s.RecoveryCandidates(r.Context())
if err != nil {
http.Error(w, fmt.Sprintf("failed to load recovery candidates: %v", err), http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(candidates)
}
func (s Server) handleDownstreamRecoveryStatuses(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
if s.RecoveryStatuses == nil {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode([]inbound.DownstreamRecoveryStatus{})
return
}
statuses, err := s.RecoveryStatuses(r.Context())
if err != nil {
http.Error(w, fmt.Sprintf("failed to load recovery statuses: %v", err), http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(statuses)
}
func (s Server) handleDownstreamRecoveryOverview(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
candidates := []inbound.DownstreamPresence{}
if s.RecoveryCandidates != nil {
result, err := s.RecoveryCandidates(r.Context())
if err != nil {
http.Error(w, fmt.Sprintf("failed to load recovery candidates: %v", err), http.StatusBadGateway)
return
}
candidates = result
}
statuses := []inbound.DownstreamRecoveryStatus{}
if s.RecoveryStatuses != nil {
result, err := s.RecoveryStatuses(r.Context())
if err != nil {
http.Error(w, fmt.Sprintf("failed to load recovery statuses: %v", err), http.StatusBadGateway)
return
}
statuses = result
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(DownstreamRecoveryOverview{
Candidates: candidates,
Statuses: statuses,
})
}
func DialCMPP(ctx context.Context, command ConnectChannelCommand) error {
ctx, cancel := context.WithTimeout(ctx, defaultConnectTimeout)
defer cancel()
gateway/internal/control/server_test.go
+101 -1
View File
@@ -7,6 +7,9 @@ import (
"net/http/httptest"
"strings"
"testing"
"time"
"cmpp-platform/gateway/internal/inbound"
)
func TestConnectChannelCallbacksConnectedState(t *testing.T) {
@@ -84,6 +87,99 @@ func TestConnectChannelRejectsInvalidCommand(t *testing.T) {
}
}
func TestRecoveryCandidatesEndpointReturnsView(t *testing.T) {
handler := handlerWithServer(Server{
RecoveryCandidates: func(context.Context) ([]inbound.DownstreamPresence, error) {
return []inbound.DownstreamPresence{{
Account: "100001",
GatewayInstanceID: "gateway-a",
State: "connected",
UpdatedAt: time.Now().UTC(),
}}, nil
},
})
resp := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/downstream/recovery-candidates", nil)
handler.ServeHTTP(resp, req)
if resp.Code != http.StatusOK {
t.Fatalf("unexpected response status: %d body=%s", resp.Code, resp.Body.String())
}
var payload []inbound.DownstreamPresence
if err := json.Unmarshal(resp.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(payload) != 1 || payload[0].Account != "100001" {
t.Fatalf("unexpected payload: %+v", payload)
}
}
func TestRecoveryStatusesEndpointReturnsView(t *testing.T) {
handler := handlerWithServer(Server{
RecoveryStatuses: func(context.Context) ([]inbound.DownstreamRecoveryStatus, error) {
return []inbound.DownstreamRecoveryStatus{{
Account: "100001",
GatewayInstanceID: "gateway-a",
State: "waiting_connection",
AttemptCount: 2,
}}, nil
},
})
resp := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/downstream/recovery-statuses", nil)
handler.ServeHTTP(resp, req)
if resp.Code != http.StatusOK {
t.Fatalf("unexpected response status: %d body=%s", resp.Code, resp.Body.String())
}
var payload []inbound.DownstreamRecoveryStatus
if err := json.Unmarshal(resp.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(payload) != 1 || payload[0].State != "waiting_connection" || payload[0].AttemptCount != 2 {
t.Fatalf("unexpected payload: %+v", payload)
}
}
func TestRecoveryOverviewEndpointReturnsCombinedView(t *testing.T) {
handler := handlerWithServer(Server{
RecoveryCandidates: func(context.Context) ([]inbound.DownstreamPresence, error) {
return []inbound.DownstreamPresence{{
Account: "100001",
GatewayInstanceID: "gateway-a",
State: "connected",
}}, nil
},
RecoveryStatuses: func(context.Context) ([]inbound.DownstreamRecoveryStatus, error) {
return []inbound.DownstreamRecoveryStatus{{
Account: "100001",
GatewayInstanceID: "gateway-a",
State: "success",
}}, nil
},
})
resp := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/downstream/recovery-overview", nil)
handler.ServeHTTP(resp, req)
if resp.Code != http.StatusOK {
t.Fatalf("unexpected response status: %d body=%s", resp.Code, resp.Body.String())
}
var payload DownstreamRecoveryOverview
if err := json.Unmarshal(resp.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode response: %v", err)
}
if len(payload.Candidates) != 1 || payload.Candidates[0].Account != "100001" {
t.Fatalf("unexpected candidate payload: %+v", payload.Candidates)
}
if len(payload.Statuses) != 1 || payload.Statuses[0].State != "success" {
t.Fatalf("unexpected status payload: %+v", payload.Statuses)
}
}
type testDialError struct{}
func (testDialError) Error() string {
@@ -93,8 +189,12 @@ func (testDialError) Error() string {
var errTestDial testDialError
func handlerWithDial(apiBaseURL string, dial DialFunc) http.Handler {
return handlerWithServer(Server{APIBaseURL: apiBaseURL, Dial: dial})
}
func handlerWithServer(server Server) http.Handler {
mux := http.NewServeMux()
Register(mux, Server{APIBaseURL: apiBaseURL, Dial: dial})
Register(mux, server)
return mux
}
gateway/internal/inbound/presence.go
+182
View File
@@ -0,0 +1,182 @@
package inbound
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
const defaultPresenceTTL = 3 * time.Minute
type PresenceStore interface {
TouchAccount(ctx context.Context, snapshot DownstreamPresence) error
RemoveAccount(ctx context.Context, account string) error
ListAccounts(ctx context.Context) ([]DownstreamPresence, error)
}
type DownstreamPresence struct {
Account string `json:"account"`
SrcID string `json:"srcId,omitempty"`
RemoteIP string `json:"remoteIp,omitempty"`
GatewayInstanceID string `json:"gatewayInstanceId,omitempty"`
State string `json:"state"`
ConnectedAt time.Time `json:"connectedAt"`
UpdatedAt time.Time `json:"updatedAt"`
LastSubmitAt time.Time `json:"lastSubmitAt,omitempty"`
LastDeliverAt time.Time `json:"lastDeliverAt,omitempty"`
}
type RedisPresenceStore struct {
Client *redis.Client
TTL time.Duration
Prefix string
}
func NewRedisPresenceStore(redisURL string) (*RedisPresenceStore, error) {
if redisURL == "" {
redisURL = "redis://127.0.0.1:6379"
}
options, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}
return &RedisPresenceStore{
Client: redis.NewClient(options),
TTL: defaultPresenceTTL,
Prefix: "gateway:downstream:presence",
}, nil
}
func (s *RedisPresenceStore) TouchAccount(ctx context.Context, snapshot DownstreamPresence) error {
if s == nil || s.Client == nil || snapshot.Account == "" {
return nil
}
if snapshot.UpdatedAt.IsZero() {
snapshot.UpdatedAt = time.Now().UTC()
}
if snapshot.ConnectedAt.IsZero() {
snapshot.ConnectedAt = snapshot.UpdatedAt
}
payload, err := json.Marshal(snapshot)
if err != nil {
return err
}
ttl := s.ttl()
pipe := s.Client.TxPipeline()
pipe.Set(ctx, s.accountKey(snapshot.Account), payload, ttl)
pipe.SAdd(ctx, s.accountsKey(), snapshot.Account)
pipe.Expire(ctx, s.accountsKey(), ttl*4)
_, err = pipe.Exec(ctx)
return err
}
func (s *RedisPresenceStore) RemoveAccount(ctx context.Context, account string) error {
if s == nil || s.Client == nil || account == "" {
return nil
}
pipe := s.Client.TxPipeline()
pipe.Del(ctx, s.accountKey(account))
pipe.SRem(ctx, s.accountsKey(), account)
_, err := pipe.Exec(ctx)
return err
}
func (s *RedisPresenceStore) ListAccounts(ctx context.Context) ([]DownstreamPresence, error) {
if s == nil || s.Client == nil {
return nil, nil
}
accounts, err := s.Client.SMembers(ctx, s.accountsKey()).Result()
if err != nil {
return nil, err
}
result := make([]DownstreamPresence, 0, len(accounts))
for _, account := range accounts {
payload, getErr := s.Client.Get(ctx, s.accountKey(account)).Bytes()
if getErr == redis.Nil {
_ = s.Client.SRem(ctx, s.accountsKey(), account).Err()
continue
}
if getErr != nil {
return nil, getErr
}
var snapshot DownstreamPresence
if err := json.Unmarshal(payload, &snapshot); err != nil {
return nil, fmt.Errorf("decode presence %s: %w", account, err)
}
result = append(result, snapshot)
}
return result, nil
}
func (s *RedisPresenceStore) ttl() time.Duration {
if s != nil && s.TTL > 0 {
return s.TTL
}
return defaultPresenceTTL
}
func (s *RedisPresenceStore) accountKey(account string) string {
return fmt.Sprintf("%s:account:%s", s.prefix(), account)
}
func (s *RedisPresenceStore) accountsKey() string {
return fmt.Sprintf("%s:accounts", s.prefix())
}
func (s *RedisPresenceStore) prefix() string {
if s != nil && s.Prefix != "" {
return s.Prefix
}
return "gateway:downstream:presence"
}
func ListRecoveryCandidates(ctx context.Context, store PresenceStore) ([]DownstreamPresence, error) {
candidateMap := map[string]DownstreamPresence{}
if store != nil {
snapshots, err := store.ListAccounts(ctx)
if err != nil {
return nil, err
}
for _, snapshot := range snapshots {
account := strings.TrimSpace(snapshot.Account)
if account == "" {
continue
}
candidateMap[account] = snapshot
}
}
for _, account := range onlineAccounts() {
account = strings.TrimSpace(account)
if account == "" {
continue
}
current := candidateMap[account]
current.Account = account
if strings.TrimSpace(current.State) == "" {
current.State = "connected"
}
if current.UpdatedAt.IsZero() {
current.UpdatedAt = time.Now().UTC()
}
candidateMap[account] = current
}
result := make([]DownstreamPresence, 0, len(candidateMap))
for _, snapshot := range candidateMap {
result = append(result, snapshot)
}
sort.Slice(result, func(i, j int) bool {
if result[i].UpdatedAt.Equal(result[j].UpdatedAt) {
return result[i].Account < result[j].Account
}
return result[i].UpdatedAt.After(result[j].UpdatedAt)
})
return result, nil
}
gateway/internal/inbound/presence_test.go
+95
View File
@@ -0,0 +1,95 @@
package inbound
import (
"context"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
)
func TestRedisPresenceStoreTouchListAndRemove(t *testing.T) {
mr := miniredis.RunT(t)
store, err := NewRedisPresenceStore("redis://" + mr.Addr())
if err != nil {
t.Fatalf("new presence store: %v", err)
}
store.TTL = time.Minute
store.Prefix = "test:presence"
now := time.Now().UTC().Truncate(time.Second)
err = store.TouchAccount(context.Background(), DownstreamPresence{
Account: "100001",
SrcID: "10690000",
RemoteIP: "127.0.0.1",
GatewayInstanceID: "gateway-a",
State: "connected",
ConnectedAt: now,
UpdatedAt: now,
LastSubmitAt: now,
})
if err != nil {
t.Fatalf("touch account: %v", err)
}
accounts, err := store.ListAccounts(context.Background())
if err != nil {
t.Fatalf("list accounts: %v", err)
}
if len(accounts) != 1 {
t.Fatalf("accounts len = %d, want 1", len(accounts))
}
if accounts[0].Account != "100001" || accounts[0].GatewayInstanceID != "gateway-a" || accounts[0].State != "connected" {
t.Fatalf("unexpected presence snapshot: %+v", accounts[0])
}
if err := store.RemoveAccount(context.Background(), "100001"); err != nil {
t.Fatalf("remove account: %v", err)
}
accounts, err = store.ListAccounts(context.Background())
if err != nil {
t.Fatalf("list accounts after remove: %v", err)
}
if len(accounts) != 0 {
t.Fatalf("accounts len after remove = %d, want 0", len(accounts))
}
}
func TestListRecoveryCandidatesMergesPresenceAndInMemory(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
store := &memoryPresenceStore{
snapshots: map[string]DownstreamPresence{
"100001": {
Account: "100001",
GatewayInstanceID: "gateway-a",
State: "connected",
UpdatedAt: time.Now().UTC().Add(-time.Minute),
},
},
}
downstreamRegistry.Lock()
downstreamRegistry.byAccount["100002"] = &downstreamSession{account: "100002"}
downstreamRegistry.Unlock()
candidates, err := ListRecoveryCandidates(context.Background(), store)
if err != nil {
t.Fatalf("list recovery candidates: %v", err)
}
if len(candidates) != 2 {
t.Fatalf("candidates len = %d, want 2", len(candidates))
}
accounts := map[string]DownstreamPresence{}
for _, item := range candidates {
accounts[item.Account] = item
}
if _, ok := accounts["100001"]; !ok {
t.Fatal("expected redis presence candidate 100001")
}
if snapshot, ok := accounts["100002"]; !ok || snapshot.State != "connected" {
t.Fatalf("expected in-memory candidate 100002 connected, got %+v", snapshot)
}
}
gateway/internal/inbound/recovery.go
+323
View File
@@ -0,0 +1,323 @@
package inbound
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/redis/go-redis/v9"
)
const (
defaultRecoveryLockTTL = 30 * time.Second
defaultRecoveryBackoffBase = 30 * time.Second
defaultRecoveryBackoffMax = 10 * time.Minute
)
var ErrRecoveryLockLost = errors.New("recovery lock lost")
type RecoveryStore interface {
StartAccountRecovery(ctx context.Context, account string, instanceID string) (RecoveryStartDecision, error)
CompleteAccountRecovery(ctx context.Context, status DownstreamRecoveryStatus) error
GetAccountRecoveryStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error)
ListRecoveryStatuses(ctx context.Context) ([]DownstreamRecoveryStatus, error)
}
type RecoveryStartDecision struct {
Allowed bool
SkipReason string
Status DownstreamRecoveryStatus
}
type DownstreamRecoveryStatus struct {
Account string `json:"account"`
GatewayInstanceID string `json:"gatewayInstanceId,omitempty"`
State string `json:"state"`
LockOwner string `json:"lockOwner,omitempty"`
LockToken string `json:"lockToken,omitempty"`
LockAcquiredAt time.Time `json:"lockAcquiredAt,omitempty"`
LockExpiresAt time.Time `json:"lockExpiresAt,omitempty"`
LastAttemptAt time.Time `json:"lastAttemptAt,omitempty"`
LastSuccessAt time.Time `json:"lastSuccessAt,omitempty"`
LastFailureAt time.Time `json:"lastFailureAt,omitempty"`
NextRetryAt time.Time `json:"nextRetryAt,omitempty"`
AttemptCount int `json:"attemptCount"`
FailureCategory string `json:"failureCategory,omitempty"`
LastError string `json:"lastError,omitempty"`
LastSkipReason string `json:"lastSkipReason,omitempty"`
}
type RedisRecoveryStore struct {
Client *redis.Client
Prefix string
}
func NewRedisRecoveryStore(redisURL string) (*RedisRecoveryStore, error) {
if redisURL == "" {
redisURL = "redis://127.0.0.1:6379"
}
options, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}
return &RedisRecoveryStore{
Client: redis.NewClient(options),
Prefix: "gateway:downstream:recovery",
}, nil
}
func (s *RedisRecoveryStore) StartAccountRecovery(ctx context.Context, account string, instanceID string) (RecoveryStartDecision, error) {
if s == nil || s.Client == nil || account == "" {
return RecoveryStartDecision{Allowed: true}, nil
}
status, err := s.getStatus(ctx, account)
if err != nil {
return RecoveryStartDecision{}, err
}
now := time.Now().UTC()
if !status.NextRetryAt.IsZero() && status.NextRetryAt.After(now) {
status.LastSkipReason = "backoff"
status.FailureCategory = "backoff"
return RecoveryStartDecision{Allowed: false, SkipReason: "backoff", Status: status}, nil
}
lockToken := newRecoveryLockToken(instanceID)
acquired, err := s.Client.SetNX(ctx, s.lockKey(account), lockToken, defaultRecoveryLockTTL).Result()
if err != nil {
return RecoveryStartDecision{}, err
}
if !acquired {
status.LockToken, _ = s.Client.Get(ctx, s.lockKey(account)).Result()
status.LockOwner = recoveryLockOwner(status.LockToken)
status.LockExpiresAt = lockExpiresAt(ctx, s.Client, s.lockKey(account), now)
status.LastSkipReason = "locked"
status.FailureCategory = "lock_contended"
return RecoveryStartDecision{Allowed: false, SkipReason: "locked", Status: status}, nil
}
status.Account = account
status.GatewayInstanceID = instanceID
status.State = "running"
status.LockOwner = instanceID
status.LockToken = lockToken
status.LockAcquiredAt = now
status.LockExpiresAt = now.Add(defaultRecoveryLockTTL)
status.LastAttemptAt = now
status.LastSkipReason = ""
status.FailureCategory = ""
if err := s.saveStatus(ctx, status); err != nil {
_ = s.deleteLockIfOwned(ctx, account, lockToken)
return RecoveryStartDecision{}, err
}
return RecoveryStartDecision{Allowed: true, Status: status}, nil
}
func (s *RedisRecoveryStore) CompleteAccountRecovery(ctx context.Context, status DownstreamRecoveryStatus) error {
if s == nil || s.Client == nil || status.Account == "" {
return nil
}
now := time.Now().UTC()
if status.LastAttemptAt.IsZero() {
status.LastAttemptAt = now
}
if status.LockOwner == "" {
status.LockOwner = status.GatewayInstanceID
}
switch status.State {
case "success":
status.LastSuccessAt = now
status.AttemptCount = 0
status.NextRetryAt = time.Time{}
status.LastError = ""
status.FailureCategory = ""
case "waiting_connection", "failed", "partial":
status.AttemptCount++
if status.State == "failed" {
status.LastFailureAt = now
}
if status.FailureCategory == "" {
status.FailureCategory = classifyRecoveryFailure(status)
}
status.NextRetryAt = now.Add(recoveryBackoffDelay(status.AttemptCount))
default:
status.State = "unknown"
}
payload, err := json.Marshal(status)
if err != nil {
return err
}
result, err := s.Client.Eval(ctx, completeRecoveryScript, []string{s.lockKey(status.Account), s.statusKey(status.Account), s.accountsKey()}, status.LockToken, payload, int64((24*time.Hour)/time.Millisecond), status.Account).Int()
if err != nil {
return err
}
if result == 0 {
return ErrRecoveryLockLost
}
return nil
}
func classifyRecoveryFailure(status DownstreamRecoveryStatus) string {
switch status.State {
case "waiting_connection":
return "client_disconnected"
case "partial":
return "partial_delivery_failed"
case "failed":
if status.LastSkipReason == "backoff" {
return "backoff"
}
if status.LastSkipReason == "locked" {
return "lock_contended"
}
if status.LastError != "" {
return "flush_failed"
}
}
return "unknown"
}
func (s *RedisRecoveryStore) ListRecoveryStatuses(ctx context.Context) ([]DownstreamRecoveryStatus, error) {
if s == nil || s.Client == nil {
return nil, nil
}
accounts, err := s.Client.SMembers(ctx, s.accountsKey()).Result()
if err != nil {
return nil, err
}
result := make([]DownstreamRecoveryStatus, 0, len(accounts))
for _, account := range accounts {
status, getErr := s.getStatus(ctx, account)
if getErr == redis.Nil {
_ = s.Client.SRem(ctx, s.accountsKey(), account).Err()
continue
}
if getErr != nil {
return nil, getErr
}
result = append(result, status)
}
return result, nil
}
func (s *RedisRecoveryStore) GetAccountRecoveryStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error) {
if s == nil || s.Client == nil || account == "" {
return DownstreamRecoveryStatus{Account: account}, nil
}
return s.getStatus(ctx, account)
}
func (s *RedisRecoveryStore) getStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error) {
payload, err := s.Client.Get(ctx, s.statusKey(account)).Bytes()
if err != nil {
if err == redis.Nil {
return DownstreamRecoveryStatus{Account: account}, nil
}
return DownstreamRecoveryStatus{}, err
}
var status DownstreamRecoveryStatus
if err := json.Unmarshal(payload, &status); err != nil {
return DownstreamRecoveryStatus{}, fmt.Errorf("decode recovery status %s: %w", account, err)
}
if status.Account == "" {
status.Account = account
}
return status, nil
}
func (s *RedisRecoveryStore) saveStatus(ctx context.Context, status DownstreamRecoveryStatus) error {
payload, err := json.Marshal(status)
if err != nil {
return err
}
pipe := s.Client.TxPipeline()
pipe.Set(ctx, s.statusKey(status.Account), payload, 24*time.Hour)
pipe.SAdd(ctx, s.accountsKey(), status.Account)
pipe.Expire(ctx, s.accountsKey(), 24*time.Hour)
_, err = pipe.Exec(ctx)
return err
}
func (s *RedisRecoveryStore) deleteLockIfOwned(ctx context.Context, account string, token string) error {
if token == "" {
return nil
}
_, err := s.Client.Eval(ctx, deleteLockIfOwnedScript, []string{s.lockKey(account)}, token).Result()
return err
}
func (s *RedisRecoveryStore) statusKey(account string) string {
return fmt.Sprintf("%s:status:%s", s.prefix(), account)
}
func (s *RedisRecoveryStore) lockKey(account string) string {
return fmt.Sprintf("%s:lock:%s", s.prefix(), account)
}
func (s *RedisRecoveryStore) accountsKey() string {
return fmt.Sprintf("%s:accounts", s.prefix())
}
func (s *RedisRecoveryStore) prefix() string {
if s != nil && s.Prefix != "" {
return s.Prefix
}
return "gateway:downstream:recovery"
}
func recoveryBackoffDelay(attemptCount int) time.Duration {
if attemptCount <= 0 {
return defaultRecoveryBackoffBase
}
delay := defaultRecoveryBackoffBase
for step := 1; step < attemptCount; step++ {
delay *= 2
if delay >= defaultRecoveryBackoffMax {
return defaultRecoveryBackoffMax
}
}
if delay > defaultRecoveryBackoffMax {
return defaultRecoveryBackoffMax
}
return delay
}
func newRecoveryLockToken(instanceID string) string {
return fmt.Sprintf("%s:%d", instanceID, time.Now().UTC().UnixNano())
}
func recoveryLockOwner(token string) string {
for index, char := range token {
if char == ':' {
return token[:index]
}
}
return token
}
func lockExpiresAt(ctx context.Context, client *redis.Client, key string, now time.Time) time.Time {
ttl, err := client.TTL(ctx, key).Result()
if err != nil || ttl <= 0 {
return time.Time{}
}
return now.Add(ttl)
}
const completeRecoveryScript = `
local current = redis.call("GET", KEYS[1])
if current ~= ARGV[1] then
return 0
end
redis.call("SET", KEYS[2], ARGV[2], "PX", ARGV[3])
redis.call("SADD", KEYS[3], ARGV[4])
redis.call("PEXPIRE", KEYS[3], ARGV[3])
redis.call("DEL", KEYS[1])
return 1
`
const deleteLockIfOwnedScript = `
local current = redis.call("GET", KEYS[1])
if current == ARGV[1] then
return redis.call("DEL", KEYS[1])
end
return 0
`
gateway/internal/inbound/recovery_test.go
+101
View File
@@ -0,0 +1,101 @@
package inbound
import (
"context"
"errors"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
)
func TestRedisRecoveryStoreBackoffAndStatuses(t *testing.T) {
mr := miniredis.RunT(t)
store, err := NewRedisRecoveryStore("redis://" + mr.Addr())
if err != nil {
t.Fatalf("new recovery store: %v", err)
}
store.Prefix = "test:recovery"
decision, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
if err != nil {
t.Fatalf("start recovery: %v", err)
}
if !decision.Allowed {
t.Fatalf("expected recovery allowed, got %+v", decision)
}
if decision.Status.LockOwner != "gateway-a" || decision.Status.LockToken == "" || decision.Status.LockExpiresAt.IsZero() {
t.Fatalf("missing recovery lock metadata: %+v", decision.Status)
}
status := decision.Status
status.State = "waiting_connection"
if err := store.CompleteAccountRecovery(context.Background(), status); err != nil {
t.Fatalf("complete recovery: %v", err)
}
statuses, err := store.ListRecoveryStatuses(context.Background())
if err != nil {
t.Fatalf("list recovery statuses: %v", err)
}
if len(statuses) != 1 || statuses[0].State != "waiting_connection" || statuses[0].AttemptCount != 1 {
t.Fatalf("unexpected statuses: %+v", statuses)
}
if statuses[0].FailureCategory != "client_disconnected" {
t.Fatalf("failure category = %q, want client_disconnected", statuses[0].FailureCategory)
}
if statuses[0].NextRetryAt.IsZero() {
t.Fatalf("expected next retry at after waiting connection: %+v", statuses[0])
}
decision, err = store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
if err != nil {
t.Fatalf("start recovery second time: %v", err)
}
if decision.Allowed || decision.SkipReason != "backoff" {
t.Fatalf("expected backoff skip, got %+v", decision)
}
if decision.Status.FailureCategory != "backoff" {
t.Fatalf("skip failure category = %q, want backoff", decision.Status.FailureCategory)
}
}
func TestRedisRecoveryStoreDoesNotReleaseLockOwnedByAnotherGateway(t *testing.T) {
mr := miniredis.RunT(t)
store, err := NewRedisRecoveryStore("redis://" + mr.Addr())
if err != nil {
t.Fatalf("new recovery store: %v", err)
}
store.Prefix = "test:recovery:takeover"
first, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
if err != nil {
t.Fatalf("start first recovery: %v", err)
}
if !first.Allowed {
t.Fatalf("expected first recovery allowed, got %+v", first)
}
mr.FastForward(defaultRecoveryLockTTL + time.Second)
second, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-b")
if err != nil {
t.Fatalf("start second recovery: %v", err)
}
if !second.Allowed {
t.Fatalf("expected second recovery to take over expired lock, got %+v", second)
}
stale := first.Status
stale.State = "success"
err = store.CompleteAccountRecovery(context.Background(), stale)
if !errors.Is(err, ErrRecoveryLockLost) {
t.Fatalf("stale completion error = %v, want ErrRecoveryLockLost", err)
}
lockValue, err := store.Client.Get(context.Background(), store.lockKey("100001")).Result()
if err != nil {
t.Fatalf("load current lock: %v", err)
}
if lockValue != second.Status.LockToken {
t.Fatalf("current lock was changed by stale completion: got %q want %q", lockValue, second.Status.LockToken)
}
}
gateway/internal/inbound/server.go
+534 -3
View File
@@ -11,6 +11,7 @@ import (
"net"
"net/http"
"strings"
"sync"
"time"
cmpp "github.com/bigwhite/gocmpp"
@@ -20,9 +21,13 @@ import (
const defaultHTTPTimeout = 10 * time.Second
type Server struct {
Addr string
APIBaseURL string
HTTPClient *http.Client
Addr string
APIBaseURL string
HTTPClient *http.Client
PendingFlushInterval time.Duration
PresenceStore PresenceStore
RecoveryStore RecoveryStore
GatewayInstanceID string
}
type authRequest struct {
@@ -49,6 +54,56 @@ type submitResponse struct {
type authResponse struct {
PasswordCipher string `json:"passwordCipher"`
ApplicationID string `json:"applicationId"`
TenantID string `json:"tenantId"`
Account string `json:"account"`
}
type DownstreamReceipt struct {
DeliveryID string `json:"deliveryId,omitempty"`
Account string `json:"account,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
MessageID string `json:"messageId"`
GatewayMessageID string `json:"gatewayMessageId,omitempty"`
PhoneNumber string `json:"phoneNumber,omitempty"`
ReceiptStatus string `json:"receiptStatus"`
RawStatus string `json:"rawStatus,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
DeliveredAt string `json:"deliveredAt,omitempty"`
}
type DownstreamUplink struct {
DeliveryID string `json:"deliveryId,omitempty"`
Account string `json:"account,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
MessageID string `json:"messageId,omitempty"`
PhoneNumber string `json:"phoneNumber"`
DestID string `json:"destId"`
Content string `json:"content"`
ReceivedAt string `json:"receivedAt,omitempty"`
}
type downstreamSession struct {
messageID string
account string
srcID string
phoneNumber string
gatewayMsgID uint64
remoteIP string
connectedAt time.Time
conn *cmpp.Conn
mu *sync.Mutex
presence PresenceStore
instanceID string
}
var downstreamRegistry = struct {
sync.RWMutex
byMessageID map[string]*downstreamSession
byAccount map[string]*downstreamSession
}{
byMessageID: make(map[string]*downstreamSession),
byAccount: make(map[string]*downstreamSession),
}
func (s Server) ListenAndServe() error {
@@ -56,6 +111,9 @@ func (s Server) ListenAndServe() error {
if addr == "" {
addr = ":17890"
}
s.logRecoveryCandidates(log.Default())
go s.recoverPendingCandidates(log.Default())
go s.runPendingFlusher(log.Default())
return cmpp.ListenAndServe(addr, cmpp.V30, 30*time.Second, 3, nil,
cmpp.HandlerFunc(s.handleLogin),
cmpp.HandlerFunc(s.handleSubmit),
@@ -83,6 +141,18 @@ func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger
authSource := []byte(req.AuthSrc)
authISMG := md5.Sum(bytes.Join([][]byte{{byte(resp.Status)}, authSource, []byte(auth.PasswordCipher)}, nil))
resp.AuthIsmg = string(authISMG[:])
session := downstreamSession{
account: strings.TrimSpace(defaultString(auth.Account, account)),
srcID: strings.TrimSpace(auth.Account),
remoteIP: remoteIP(packet.Conn.Conn.RemoteAddr()),
connectedAt: time.Now().UTC(),
conn: packet.Conn,
mu: &sync.Mutex{},
presence: s.PresenceStore,
instanceID: s.gatewayInstanceID(),
}
rememberAccount(session)
go s.flushPending(defaultString(auth.Account, account), logger)
logger.Printf("cmpp inbound account=%s login ok remote=%s", account, packet.Conn.Conn.RemoteAddr())
return false, nil
}
@@ -120,6 +190,19 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
}
resp.MsgId = messageIDFrom(result.MessageID, req.SeqId)
resp.Result = 0
rememberDownstream(downstreamSession{
messageID: result.MessageID,
account: account,
srcID: strings.TrimSpace(req.SrcId),
phoneNumber: phone,
gatewayMsgID: resp.MsgId,
remoteIP: remoteIP(packet.Conn.Conn.RemoteAddr()),
connectedAt: time.Now().UTC(),
conn: packet.Conn,
mu: &sync.Mutex{},
presence: s.PresenceStore,
instanceID: s.gatewayInstanceID(),
})
return false, nil
}
@@ -142,6 +225,82 @@ func (s Server) submit(remote net.Addr, payload submitRequest) (submitResponse,
return result, err
}
type pendingDeliveryRequest struct {
Account string `json:"account"`
Limit int `json:"limit,omitempty"`
}
type pendingDelivery struct {
ID string `json:"id"`
DeliveryType string `json:"deliveryType"`
Payload json.RawMessage `json:"payload"`
}
type pendingFlushResult struct {
Account string
Deliveries int
DeliveredCount int
FailedCount int
WaitingCount int
LastError string
}
func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushResult, error) {
result := pendingFlushResult{Account: account}
if account == "" {
return result, nil
}
var deliveries []pendingDelivery
if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100}, &deliveries); err != nil {
logger.Printf("cmpp inbound pending delivery fetch failed account=%s err=%v", account, err)
result.LastError = err.Error()
return result, err
}
result.Deliveries = len(deliveries)
for _, delivery := range deliveries {
delivered, err := s.pushPendingDelivery(account, delivery)
if err != nil {
result.FailedCount++
result.LastError = err.Error()
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]string{
"id": delivery.ID,
"errorMessage": err.Error(),
}, nil)
continue
}
if delivered {
result.DeliveredCount++
_ = s.post(context.Background(), "/gateway/events/downstream/delivered", map[string]string{"id": delivery.ID}, nil)
continue
}
result.WaitingCount++
}
return result, nil
}
func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (bool, error) {
switch delivery.DeliveryType {
case "receipt":
var event DownstreamReceipt
if err := json.Unmarshal(delivery.Payload, &event); err != nil {
return false, err
}
event.DeliveryID = delivery.ID
event.Account = defaultString(event.Account, account)
return PushReceipt(event)
case "uplink":
var event DownstreamUplink
if err := json.Unmarshal(delivery.Payload, &event); err != nil {
return false, err
}
event.DeliveryID = delivery.ID
event.Account = defaultString(event.Account, account)
return PushUplink(event)
default:
return false, fmt.Errorf("unsupported downstream delivery type %q", delivery.DeliveryType)
}
}
func (s Server) post(ctx context.Context, path string, payload any, result any) error {
client := s.HTTPClient
if client == nil {
@@ -210,3 +369,375 @@ func messageIDFrom(value string, seq uint32) uint64 {
}
return result
}
func rememberDownstream(session downstreamSession) {
if session.messageID == "" || session.conn == nil {
return
}
session.touchPresence("connected", true, false)
downstreamRegistry.Lock()
downstreamRegistry.byMessageID[session.messageID] = &session
if session.account != "" {
downstreamRegistry.byAccount[session.account] = &session
}
downstreamRegistry.Unlock()
}
func rememberAccount(session downstreamSession) {
if session.account == "" || session.conn == nil {
return
}
session.touchPresence("connected", false, false)
downstreamRegistry.Lock()
downstreamRegistry.byAccount[session.account] = &session
downstreamRegistry.Unlock()
}
func forgetDownstream(session *downstreamSession) {
if session == nil {
return
}
downstreamRegistry.Lock()
if session.messageID != "" {
if current := downstreamRegistry.byMessageID[session.messageID]; current == session {
delete(downstreamRegistry.byMessageID, session.messageID)
}
}
if session.account != "" {
if current := downstreamRegistry.byAccount[session.account]; current == session {
delete(downstreamRegistry.byAccount, session.account)
}
}
downstreamRegistry.Unlock()
_ = session.removePresence()
}
func (s Server) runPendingFlusher(logger *log.Logger) {
ticker := time.NewTicker(s.pendingFlushInterval())
defer ticker.Stop()
for range ticker.C {
s.flushOnlineAccounts(logger)
s.recoverPendingCandidates(logger)
}
}
func (s Server) flushOnlineAccounts(logger *log.Logger) {
for _, account := range onlineAccounts() {
_, _ = s.flushPending(account, logger)
}
}
func (s Server) recoverPendingCandidates(logger *log.Logger) {
candidates, err := ListRecoveryCandidates(context.Background(), s.PresenceStore)
if err != nil {
logger.Printf("cmpp inbound recovery candidate refresh failed err=%v", err)
return
}
for _, candidate := range candidates {
account := strings.TrimSpace(candidate.Account)
if account == "" {
continue
}
activeRecovery := DownstreamRecoveryStatus{
Account: account,
GatewayInstanceID: s.gatewayInstanceID(),
}
if s.RecoveryStore != nil {
decision, recoveryErr := s.RecoveryStore.StartAccountRecovery(context.Background(), account, s.gatewayInstanceID())
if recoveryErr != nil {
logger.Printf("cmpp inbound recovery start failed account=%s err=%v", account, recoveryErr)
continue
}
if !decision.Allowed {
logger.Printf("cmpp inbound recovery skipped account=%s reason=%s", account, decision.SkipReason)
decision.Status.Account = account
decision.Status.GatewayInstanceID = s.gatewayInstanceID()
decision.Status.State = defaultString(decision.Status.State, "failed")
decision.Status.LastSkipReason = defaultString(decision.Status.LastSkipReason, decision.SkipReason)
if decision.Status.FailureCategory == "" {
decision.Status.FailureCategory = recoveryFailureCategory(decision.Status.State, "", decision.Status.LastSkipReason)
}
s.syncRecoveryStatus(logger, account, decision.Status)
continue
}
activeRecovery = decision.Status
activeRecovery.Account = account
activeRecovery.GatewayInstanceID = s.gatewayInstanceID()
}
result, flushErr := s.flushPending(account, logger)
if s.RecoveryStore != nil {
status := DownstreamRecoveryStatus{
Account: account,
GatewayInstanceID: s.gatewayInstanceID(),
LockToken: activeRecovery.LockToken,
LockOwner: defaultString(activeRecovery.LockOwner, s.gatewayInstanceID()),
LockAcquiredAt: activeRecovery.LockAcquiredAt,
LockExpiresAt: activeRecovery.LockExpiresAt,
LastAttemptAt: activeRecovery.LastAttemptAt,
}
switch {
case flushErr != nil:
status.State = "failed"
status.LastError = flushErr.Error()
status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason)
case result.WaitingCount > 0 && result.DeliveredCount == 0 && result.FailedCount == 0:
status.State = "waiting_connection"
status.LastError = "downstream client is not connected"
status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason)
case result.FailedCount > 0 && result.DeliveredCount > 0:
status.State = "partial"
status.LastError = result.LastError
status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason)
default:
status.State = "success"
status.LastError = ""
status.FailureCategory = ""
}
if err := s.RecoveryStore.CompleteAccountRecovery(context.Background(), status); err != nil {
logger.Printf("cmpp inbound recovery completion failed account=%s err=%v", account, err)
if err == ErrRecoveryLockLost {
status.State = "failed"
status.LastSkipReason = "lock_lost"
status.FailureCategory = "lock_lost"
s.syncRecoveryStatus(logger, account, status)
}
} else if persisted, err := s.RecoveryStore.GetAccountRecoveryStatus(context.Background(), account); err != nil {
logger.Printf("cmpp inbound recovery status fetch failed account=%s err=%v", account, err)
} else {
s.syncRecoveryStatus(logger, account, persisted)
}
}
}
}
func (s Server) syncRecoveryStatus(logger *log.Logger, account string, status DownstreamRecoveryStatus) {
if err := s.post(context.Background(), "/gateway/events/downstream/recovery-status", map[string]any{
"account": status.Account,
"gatewayInstanceId": status.GatewayInstanceID,
"state": status.State,
"lockOwner": status.LockOwner,
"lockExpiresAt": formatRFC3339Nano(status.LockExpiresAt),
"lastAttemptAt": formatRFC3339Nano(status.LastAttemptAt),
"lastSuccessAt": formatRFC3339Nano(status.LastSuccessAt),
"lastFailureAt": formatRFC3339Nano(status.LastFailureAt),
"nextRetryAt": formatRFC3339Nano(status.NextRetryAt),
"attemptCount": status.AttemptCount,
"failureCategory": status.FailureCategory,
"lastError": status.LastError,
"lastSkipReason": status.LastSkipReason,
}, nil); err != nil {
logger.Printf("cmpp inbound recovery status sync failed account=%s err=%v", account, err)
}
}
func recoveryFailureCategory(state string, lastError string, lastSkipReason string) string {
if state == "success" || state == "running" {
return ""
}
if lastSkipReason == "backoff" {
return "backoff"
}
if lastSkipReason == "locked" {
return "lock_contended"
}
if lastSkipReason == "lock_lost" {
return "lock_lost"
}
if state == "waiting_connection" {
return "client_disconnected"
}
if state == "partial" {
return "partial_delivery_failed"
}
if state == "failed" && strings.TrimSpace(lastError) != "" {
return "flush_failed"
}
return "unknown"
}
func onlineAccounts() []string {
downstreamRegistry.RLock()
defer downstreamRegistry.RUnlock()
accounts := make([]string, 0, len(downstreamRegistry.byAccount))
for account := range downstreamRegistry.byAccount {
if strings.TrimSpace(account) != "" {
accounts = append(accounts, account)
}
}
return accounts
}
func PushReceipt(event DownstreamReceipt) (bool, error) {
session := findSession(event.MessageID, event.Account)
if session == nil {
return false, nil
}
stat := strings.TrimSpace(event.RawStatus)
if stat == "" {
stat = cmppReceiptStatus(event.ReceiptStatus)
}
when := time.Now()
if event.DeliveredAt != "" {
if parsed, err := time.Parse(time.RFC3339Nano, event.DeliveredAt); err == nil {
when = parsed
}
}
receipt := &cmpp.CmppReceiptPkt{
MsgId: session.gatewayMsgID,
Stat: stat,
SubmitTime: when.Format("0601021504"),
DoneTime: when.Format("0601021504"),
DestTerminalId: defaultString(event.PhoneNumber, session.phoneNumber),
SmscSequence: uint32(time.Now().UnixNano() & 0xffffffff),
}
receiptBytes, err := receipt.Pack()
if err != nil {
return false, err
}
deliver := &cmpp.Cmpp3DeliverReqPkt{
MsgId: session.gatewayMsgID,
DestId: session.srcID,
ServiceId: "cmpp",
MsgFmt: 0,
SrcTerminalId: defaultString(event.PhoneNumber, session.phoneNumber),
RegisterDelivery: 1,
MsgLength: uint8(cmpp.CmppReceiptPktLen),
MsgContent: string(receiptBytes),
}
return sendDownstream(session, deliver)
}
func PushUplink(event DownstreamUplink) (bool, error) {
session := findSession(event.MessageID, event.Account)
if session == nil {
return false, nil
}
content, err := cmpputils.Utf8ToUcs2(event.Content)
if err != nil {
return false, err
}
deliver := &cmpp.Cmpp3DeliverReqPkt{
MsgId: messageIDFrom(defaultString(event.MessageID, event.Account), uint32(time.Now().UnixNano())),
DestId: defaultString(event.DestID, session.srcID),
ServiceId: "cmpp",
MsgFmt: 8,
SrcTerminalId: event.PhoneNumber,
RegisterDelivery: 0,
MsgLength: uint8(len(content)),
MsgContent: content,
}
return sendDownstream(session, deliver)
}
func findSession(messageID string, account string) *downstreamSession {
downstreamRegistry.RLock()
defer downstreamRegistry.RUnlock()
if messageID != "" {
if session := downstreamRegistry.byMessageID[messageID]; session != nil {
return session
}
}
if account != "" {
return downstreamRegistry.byAccount[account]
}
return nil
}
func sendDownstream(session *downstreamSession, deliver *cmpp.Cmpp3DeliverReqPkt) (bool, error) {
session.mu.Lock()
defer session.mu.Unlock()
if err := session.conn.SendPkt(deliver, <-session.conn.SeqId); err != nil {
forgetDownstream(session)
return false, err
}
session.touchPresence("connected", false, true)
return true, nil
}
func (s Server) pendingFlushInterval() time.Duration {
if s.PendingFlushInterval > 0 {
return s.PendingFlushInterval
}
return time.Minute
}
func (s Server) logRecoveryCandidates(logger *log.Logger) {
candidates, err := ListRecoveryCandidates(context.Background(), s.PresenceStore)
if err != nil {
logger.Printf("cmpp inbound recovery candidates load failed err=%v", err)
return
}
if len(candidates) == 0 {
logger.Printf("cmpp inbound recovery candidates loaded count=0")
return
}
accounts := make([]string, 0, len(candidates))
for _, item := range candidates {
if strings.TrimSpace(item.Account) != "" {
accounts = append(accounts, item.Account)
}
}
logger.Printf("cmpp inbound recovery candidates loaded count=%d accounts=%s", len(candidates), strings.Join(accounts, ","))
}
func cmppReceiptStatus(status string) string {
switch strings.ToLower(strings.TrimSpace(status)) {
case "delivered":
return "DELIVRD"
case "unknown":
return "UNKNOWN"
default:
return "UNDELIV"
}
}
func defaultString(value string, fallback string) string {
if value == "" {
return fallback
}
return value
}
func formatRFC3339Nano(value time.Time) string {
if value.IsZero() {
return ""
}
return value.UTC().Format(time.RFC3339Nano)
}
func (s Server) gatewayInstanceID() string {
if strings.TrimSpace(s.GatewayInstanceID) != "" {
return strings.TrimSpace(s.GatewayInstanceID)
}
return "gateway-1"
}
func (session downstreamSession) touchPresence(state string, includeSubmit bool, includeDeliver bool) {
if session.presence == nil || strings.TrimSpace(session.account) == "" {
return
}
now := time.Now().UTC()
snapshot := DownstreamPresence{
Account: strings.TrimSpace(session.account),
SrcID: strings.TrimSpace(session.srcID),
RemoteIP: strings.TrimSpace(session.remoteIP),
GatewayInstanceID: strings.TrimSpace(session.instanceID),
State: defaultString(strings.TrimSpace(state), "connected"),
ConnectedAt: session.connectedAt,
UpdatedAt: now,
}
if includeSubmit {
snapshot.LastSubmitAt = now
}
if includeDeliver {
snapshot.LastDeliverAt = now
}
_ = session.presence.TouchAccount(context.Background(), snapshot)
}
func (session downstreamSession) removePresence() error {
if session.presence == nil || strings.TrimSpace(session.account) == "" {
return nil
}
return session.presence.RemoveAccount(context.Background(), strings.TrimSpace(session.account))
}
gateway/internal/inbound/server_test.go
+259
View File
@@ -1,10 +1,13 @@
package inbound
import (
"context"
"encoding/json"
"log"
"net"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
@@ -12,6 +15,63 @@ import (
cmpputils "github.com/bigwhite/gocmpp/utils"
)
type memoryPresenceStore struct {
snapshots map[string]DownstreamPresence
removed []string
}
type memoryRecoveryStore struct {
decisions map[string]RecoveryStartDecision
completed []DownstreamRecoveryStatus
}
func (m *memoryPresenceStore) TouchAccount(_ context.Context, snapshot DownstreamPresence) error {
if m.snapshots == nil {
m.snapshots = map[string]DownstreamPresence{}
}
m.snapshots[snapshot.Account] = snapshot
return nil
}
func (m *memoryPresenceStore) RemoveAccount(_ context.Context, account string) error {
delete(m.snapshots, account)
m.removed = append(m.removed, account)
return nil
}
func (m *memoryPresenceStore) ListAccounts(_ context.Context) ([]DownstreamPresence, error) {
result := make([]DownstreamPresence, 0, len(m.snapshots))
for _, snapshot := range m.snapshots {
result = append(result, snapshot)
}
return result, nil
}
func (m *memoryRecoveryStore) StartAccountRecovery(_ context.Context, account string, _ string) (RecoveryStartDecision, error) {
if decision, ok := m.decisions[account]; ok {
return decision, nil
}
return RecoveryStartDecision{Allowed: true}, nil
}
func (m *memoryRecoveryStore) CompleteAccountRecovery(_ context.Context, status DownstreamRecoveryStatus) error {
m.completed = append(m.completed, status)
return nil
}
func (m *memoryRecoveryStore) ListRecoveryStatuses(_ context.Context) ([]DownstreamRecoveryStatus, error) {
return append([]DownstreamRecoveryStatus(nil), m.completed...), nil
}
func (m *memoryRecoveryStore) GetAccountRecoveryStatus(_ context.Context, account string) (DownstreamRecoveryStatus, error) {
for _, item := range m.completed {
if item.Account == account {
return item, nil
}
}
return DownstreamRecoveryStatus{Account: account}, nil
}
func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
account := "100001"
password := "secret-hash"
@@ -29,6 +89,8 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
t.Fatalf("decode submit: %v", err)
}
_ = json.NewEncoder(w).Encode(submitResponse{Accepted: true, MessageID: "MSG-1"})
case "/api/gateway/events/downstream/pending":
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
default:
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
@@ -76,6 +138,26 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
if rsp.Result != 0 || rsp.MsgId == 0 {
t.Fatalf("unexpected submit response: %+v", rsp)
}
delivered, err := PushReceipt(DownstreamReceipt{
MessageID: "MSG-1",
PhoneNumber: "13500002696",
ReceiptStatus: "delivered",
DeliveredAt: time.Now().UTC().Format(time.RFC3339Nano),
})
if err != nil || !delivered {
t.Fatalf("push receipt delivered=%v err=%v", delivered, err)
}
deliver := recvDeliver(t, client)
if deliver.RegisterDelivery != 1 {
t.Fatalf("expected receipt deliver, got %+v", deliver)
}
var receipt cmpp.CmppReceiptPkt
if err := receipt.Unpack([]byte(deliver.MsgContent)); err != nil {
t.Fatalf("unpack pushed receipt: %v", err)
}
if receipt.Stat != "DELIVRD" || receipt.DestTerminalId != "13500002696" {
t.Fatalf("unexpected pushed receipt: %+v", receipt)
}
if gotAuth.Account != account || gotAuth.AuthSource == "" || gotAuth.RemoteIP == "" {
t.Fatalf("unexpected auth payload: %+v", gotAuth)
}
@@ -84,6 +166,183 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
}
}
func TestFlushOnlineAccountsFetchesPendingDeliveries(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
account := "100001"
calls := 0
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/gateway/events/downstream/pending":
calls++
var payload pendingDeliveryRequest
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("decode pending request: %v", err)
}
if payload.Account != account {
t.Fatalf("unexpected account: %+v", payload)
}
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
default:
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
}))
defer api.Close()
downstreamRegistry.Lock()
downstreamRegistry.byAccount[account] = &downstreamSession{account: account}
downstreamRegistry.Unlock()
server := Server{APIBaseURL: api.URL + "/api"}
server.flushOnlineAccounts(log.Default())
if calls != 1 {
t.Fatalf("pending fetch calls = %d, want 1", calls)
}
}
func TestRecoverPendingCandidatesFetchesPresenceAccounts(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
account := "100009"
calls := 0
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/gateway/events/downstream/pending":
calls++
var payload pendingDeliveryRequest
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("decode pending request: %v", err)
}
if payload.Account != account {
t.Fatalf("unexpected account: %+v", payload)
}
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
default:
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
}))
defer api.Close()
store := &memoryPresenceStore{
snapshots: map[string]DownstreamPresence{
account: {
Account: account,
GatewayInstanceID: "gateway-a",
State: "connected",
UpdatedAt: time.Now().UTC(),
},
},
}
server := Server{APIBaseURL: api.URL + "/api", PresenceStore: store}
server.recoverPendingCandidates(log.Default())
if calls != 1 {
t.Fatalf("recovery pending fetch calls = %d, want 1", calls)
}
}
func TestRecoverPendingCandidatesWritesWaitingConnectionStatus(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
account := "100010"
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/gateway/events/downstream/pending":
_ = json.NewEncoder(w).Encode([]pendingDelivery{{
ID: "delivery-1",
DeliveryType: "receipt",
Payload: json.RawMessage(`{"messageId":"MSG-404","phoneNumber":"13800000001","receiptStatus":"delivered"}`),
}})
case "/api/gateway/events/downstream/recovery-status":
w.WriteHeader(http.StatusOK)
default:
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
}))
defer api.Close()
recovery := &memoryRecoveryStore{}
store := &memoryPresenceStore{
snapshots: map[string]DownstreamPresence{
account: {
Account: account,
GatewayInstanceID: "gateway-a",
State: "connected",
UpdatedAt: time.Now().UTC(),
},
},
}
server := Server{APIBaseURL: api.URL + "/api", PresenceStore: store, RecoveryStore: recovery}
server.recoverPendingCandidates(log.Default())
if len(recovery.completed) != 1 {
t.Fatalf("completed recovery statuses = %d, want 1", len(recovery.completed))
}
if recovery.completed[0].State != "waiting_connection" {
t.Fatalf("unexpected recovery status: %+v", recovery.completed[0])
}
}
func TestRememberAndForgetAccountUpdatesPresenceStore(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
store := &memoryPresenceStore{}
session := downstreamSession{
account: "100001",
srcID: "10690000",
remoteIP: "127.0.0.1",
connectedAt: time.Now().UTC(),
conn: &cmpp.Conn{},
mu: &sync.Mutex{},
presence: store,
instanceID: "gateway-a",
}
rememberAccount(session)
snapshot, ok := store.snapshots["100001"]
if !ok {
t.Fatal("expected presence snapshot to be stored")
}
if snapshot.Account != "100001" || snapshot.GatewayInstanceID != "gateway-a" || snapshot.State != "connected" {
t.Fatalf("unexpected snapshot: %+v", snapshot)
}
forgetDownstream(downstreamRegistry.byAccount["100001"])
if len(store.removed) != 1 || store.removed[0] != "100001" {
t.Fatalf("unexpected removed accounts: %+v", store.removed)
}
}
func recvDeliver(t *testing.T, client *cmpp.Client) *cmpp.Cmpp3DeliverReqPkt {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
packet, err := client.RecvAndUnpackPkt(200 * time.Millisecond)
if err != nil {
continue
}
if deliver, ok := packet.(*cmpp.Cmpp3DeliverReqPkt); ok {
return deliver
}
}
t.Fatal("timed out waiting deliver request")
return nil
}
func resetDownstreamRegistry() {
downstreamRegistry.Lock()
defer downstreamRegistry.Unlock()
downstreamRegistry.byAccount = make(map[string]*downstreamSession)
downstreamRegistry.byMessageID = make(map[string]*downstreamSession)
}
func reserveTCPAddr(t *testing.T) string {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
gateway/internal/queue/messages.go
+37 -13
View File
@@ -25,19 +25,20 @@ type Envelope struct {
type SubmitCommand struct {
Envelope
TenantID string `json:"tenantId"`
ApplicationID string `json:"applicationId"`
TaskID string `json:"taskId,omitempty"`
SubmitID string `json:"submitId"`
PhoneNumber string `json:"phoneNumber"`
Content string `json:"content"`
Signature string `json:"signature"`
TemplateID string `json:"templateId"`
BillingUnits int `json:"billingUnits"`
QueuePriority string `json:"queuePriority"`
Route Route `json:"route"`
CMPP CMPP `json:"cmpp"`
Retry Retry `json:"retry"`
TenantID string `json:"tenantId"`
ApplicationID string `json:"applicationId"`
TaskID string `json:"taskId,omitempty"`
SubmitID string `json:"submitId"`
PhoneNumber string `json:"phoneNumber"`
Content string `json:"content"`
Signature string `json:"signature"`
TemplateID string `json:"templateId"`
BillingUnits int `json:"billingUnits"`
QueuePriority string `json:"queuePriority"`
Route Route `json:"route"`
CMPP CMPP `json:"cmpp"`
Upstream UpstreamConfig `json:"upstream"`
Retry Retry `json:"retry"`
}
type Route struct {
@@ -57,6 +58,16 @@ type CMPP struct {
FeeType string `json:"feeType,omitempty"`
}
type UpstreamConfig struct {
GatewayHost string `json:"gatewayHost"`
GatewayPort int `json:"gatewayPort"`
Account string `json:"account"`
PasswordCipher string `json:"passwordCipher"`
CMPPVersion string `json:"cmppVersion"`
DesiredConnections int `json:"desiredConnections,omitempty"`
WindowSize int `json:"windowSize,omitempty"`
}
type Retry struct {
Attempt int `json:"attempt"`
MaxAttempts int `json:"maxAttempts"`
@@ -70,12 +81,25 @@ type SubmitResult struct {
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
SubmittedAt time.Time `json:"submittedAt"`
Segments []SubmitSegmentResult `json:"segments,omitempty"`
}
type SubmitSegmentResult struct {
SegmentTotal int `json:"segmentTotal"`
SegmentIndex int `json:"segmentIndex"`
SequenceID uint32 `json:"sequenceId"`
GatewayMessageID string `json:"gatewayMessageId"`
SubmitStatus string `json:"submitStatus"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
SubmittedAt time.Time `json:"submittedAt"`
}
type ReceiptEvent struct {
Envelope
SequenceID uint32 `json:"sequenceId"`
GatewayMessageID string `json:"gatewayMessageId"`
PhoneNumber string `json:"phoneNumber,omitempty"`
ReceiptStatus string `json:"receiptStatus"`
RawStatus string `json:"rawStatus"`
ErrorCode string `json:"errorCode,omitempty"`
gateway/internal/queue/messages_test.go
+15
View File
@@ -34,6 +34,15 @@ func TestSubmitCommandUnmarshalsQueuePriority(t *testing.T) {
"registeredDelivery": 1,
"msgFmt": 8
},
"upstream": {
"gatewayHost": "127.0.0.1",
"gatewayPort": 17890,
"account": "account-a",
"passwordCipher": "secret",
"cmppVersion": "3.0",
"desiredConnections": 2,
"windowSize": 16
},
"retry": {
"attempt": 0,
"maxAttempts": 3
@@ -47,4 +56,10 @@ func TestSubmitCommandUnmarshalsQueuePriority(t *testing.T) {
if command.QueuePriority != "priority" {
t.Fatalf("QueuePriority = %q, want priority", command.QueuePriority)
}
if command.Upstream.GatewayHost != "127.0.0.1" || command.Upstream.Account != "account-a" {
t.Fatalf("unexpected upstream config: %+v", command.Upstream)
}
if command.Upstream.DesiredConnections != 2 || command.Upstream.WindowSize != 16 {
t.Fatalf("unexpected upstream window config: %+v", command.Upstream)
}
}
gateway/internal/spike/simulator.go
+7
View File
@@ -134,6 +134,13 @@ func NewSubmitCommand(index int) queue.SubmitCommand {
FeeCode: "0",
FeeType: "01",
},
Upstream: queue.UpstreamConfig{
GatewayHost: "127.0.0.1",
GatewayPort: 17890,
Account: "cmpp-account-spike",
PasswordCipher: "secret-spike",
CMPPVersion: "3.0",
},
Retry: queue.Retry{Attempt: 0, MaxAttempts: 3},
}
}
gateway/internal/submitworker/worker.go
+431
View File
@@ -0,0 +1,431 @@
package submitworker
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strings"
"time"
"cmpp-platform/gateway/internal/queue"
"cmpp-platform/gateway/internal/upstream"
"github.com/redis/go-redis/v9"
)
const (
defaultStream = "gateway.submit.commands"
defaultGroup = "cmpp-gateway"
defaultConsumer = "gateway-1"
defaultMinIdle = 30 * time.Second
defaultMaxFails = 3
)
type Worker struct {
Redis *redis.Client
Upstream *upstream.Manager
Submit func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error)
ReportDeadLetter func(context.Context, DeadLetterEvent) error
Stream string
Group string
Consumer string
Block time.Duration
Count int64
MinIdle time.Duration
MaxFailures int
APIBaseURL string
HTTPClient *http.Client
Logger *log.Logger
}
type DeadLetterEvent struct {
StreamMessageID string `json:"streamMessageId"`
TraceID string `json:"traceId,omitempty"`
MessageID string `json:"messageId,omitempty"`
ChannelID string `json:"channelId,omitempty"`
TenantID string `json:"tenantId,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
SubmitID string `json:"submitId,omitempty"`
FailureCode string `json:"failureCode"`
FailureMessage string `json:"failureMessage"`
Attempts int `json:"attempts"`
MaxAttempts int `json:"maxAttempts"`
CommandPayload map[string]interface{} `json:"commandPayload,omitempty"`
RawPayload string `json:"rawPayload,omitempty"`
DeadLetteredAt time.Time `json:"deadLetteredAt"`
}
func New(redisURL string, manager *upstream.Manager) (*Worker, error) {
client, err := redisClient(redisURL)
if err != nil {
return nil, err
}
return &Worker{Redis: client, Upstream: manager}, nil
}
func (w *Worker) Run(ctx context.Context) error {
if w.Redis == nil {
return fmt.Errorf("redis client is required")
}
if w.Upstream == nil {
return fmt.Errorf("upstream manager is required")
}
for {
if err := w.ensureGroup(ctx); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
w.logf("gateway submit worker ensure group failed: %v", err)
sleep(ctx, 3*time.Second)
continue
}
if err := w.recoverPending(ctx); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
w.logf("gateway submit worker pending recovery failed: %v", err)
sleep(ctx, time.Second)
continue
}
if err := w.consumeOnce(ctx); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
w.logf("gateway submit worker consume failed: %v", err)
sleep(ctx, time.Second)
}
}
}
func (w *Worker) ensureGroup(ctx context.Context) error {
err := w.Redis.XGroupCreateMkStream(ctx, w.stream(), w.group(), "0").Err()
if err == nil || strings.Contains(err.Error(), "BUSYGROUP") {
return nil
}
return err
}
func (w *Worker) consumeOnce(ctx context.Context) error {
streams, err := w.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: w.group(),
Consumer: w.consumer(),
Streams: []string{w.stream(), ">"},
Count: w.count(),
Block: w.block(),
}).Result()
if errors.Is(err, redis.Nil) {
return nil
}
if err != nil {
return err
}
for _, stream := range streams {
if err := w.processMessages(ctx, stream.Messages); err != nil {
return err
}
}
return nil
}
func (w *Worker) recoverPending(ctx context.Context) error {
start := "0-0"
for {
messages, next, err := w.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{
Stream: w.stream(),
Group: w.group(),
Consumer: w.consumer(),
MinIdle: w.minIdle(),
Start: start,
Count: w.count(),
}).Result()
if errors.Is(err, redis.Nil) {
return nil
}
if err != nil {
return err
}
if len(messages) == 0 {
return nil
}
w.logf("gateway submit worker reclaimed %d pending message(s)", len(messages))
if err := w.processMessages(ctx, messages); err != nil {
return err
}
start = next
if next == "0-0" {
return nil
}
}
}
func (w *Worker) processMessages(ctx context.Context, messages []redis.XMessage) error {
for _, message := range messages {
if err := w.processMessage(ctx, message); err != nil {
w.logf("gateway submit worker message %s failed: %v", message.ID, err)
continue
}
}
return nil
}
func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) error {
command, err := CommandFromStreamValues(message.Values)
if err != nil {
return w.deadLetterMalformedMessage(ctx, message, err)
}
if err := w.handleCommand(ctx, command); err != nil {
attempts, attemptsErr := w.incrementFailureAttempt(ctx, message.ID)
if attemptsErr != nil {
w.logf("gateway submit worker increment failure %s failed: %v", message.ID, attemptsErr)
}
if attempts >= w.maxFailures() {
if reportErr := w.deadLetterCommand(ctx, message, command, attempts, err); reportErr != nil {
return reportErr
}
return w.ackAndClearFailure(ctx, message.ID)
}
return err
}
return w.ackAndClearFailure(ctx, message.ID)
}
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
submit := w.Submit
if submit == nil {
if w.Upstream == nil {
return fmt.Errorf("upstream manager is required")
}
submit = w.Upstream.Submit
}
result, err := submit(ctx, command)
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
return err
}
return nil
}
func CommandFromStreamValues(values map[string]interface{}) (queue.SubmitCommand, error) {
raw, ok := values["data"]
if !ok {
return queue.SubmitCommand{}, fmt.Errorf("stream data field is required")
}
var data string
switch value := raw.(type) {
case string:
data = value
case []byte:
data = string(value)
default:
data = fmt.Sprint(value)
}
var command queue.SubmitCommand
if err := json.Unmarshal([]byte(data), &command); err != nil {
return queue.SubmitCommand{}, err
}
if command.MessageType != queue.MessageTypeSubmitCommand {
return queue.SubmitCommand{}, fmt.Errorf("unsupported messageType %q", command.MessageType)
}
return command, nil
}
func (w *Worker) deadLetterMalformedMessage(ctx context.Context, message redis.XMessage, cause error) error {
event := DeadLetterEvent{
StreamMessageID: message.ID,
FailureCode: "INVALID_COMMAND_PAYLOAD",
FailureMessage: cause.Error(),
Attempts: 1,
MaxAttempts: 1,
RawPayload: extractRawPayload(message.Values),
DeadLetteredAt: time.Now().UTC(),
}
if err := w.reportDeadLetter(ctx, event); err != nil {
return err
}
return w.ackAndClearFailure(ctx, message.ID)
}
func (w *Worker) deadLetterCommand(ctx context.Context, message redis.XMessage, command queue.SubmitCommand, attempts int, cause error) error {
payload, payloadErr := commandPayload(command)
if payloadErr != nil {
w.logf("gateway submit worker marshal dead-letter command %s failed: %v", message.ID, payloadErr)
}
return w.reportDeadLetter(ctx, DeadLetterEvent{
StreamMessageID: message.ID,
TraceID: command.TraceID,
MessageID: command.MessageID,
ChannelID: command.ChannelID,
TenantID: command.TenantID,
ApplicationID: command.ApplicationID,
SubmitID: command.SubmitID,
FailureCode: "SUBMIT_PROCESSING_FAILED",
FailureMessage: cause.Error(),
Attempts: attempts,
MaxAttempts: w.maxFailures(),
CommandPayload: payload,
RawPayload: extractRawPayload(message.Values),
DeadLetteredAt: time.Now().UTC(),
})
}
func (w *Worker) reportDeadLetter(ctx context.Context, event DeadLetterEvent) error {
if w.ReportDeadLetter != nil {
return w.ReportDeadLetter(ctx, event)
}
if w.APIBaseURL == "" {
return fmt.Errorf("gateway submit dead-letter reporter is not configured")
}
client := w.HTTPClient
if client == nil {
client = &http.Client{Timeout: 10 * time.Second}
}
body, err := json.Marshal(event)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
strings.TrimRight(w.APIBaseURL, "/")+"/gateway/events/dead-letter",
bytes.NewReader(body),
)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("dead-letter endpoint returned %d", resp.StatusCode)
}
return nil
}
func redisClient(redisURL string) (*redis.Client, error) {
if redisURL == "" {
redisURL = "redis://127.0.0.1:6379"
}
options, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}
return redis.NewClient(options), nil
}
func (w *Worker) incrementFailureAttempt(ctx context.Context, messageID string) (int, error) {
value, err := w.Redis.HIncrBy(ctx, w.failureAttemptsKey(), messageID, 1).Result()
return int(value), err
}
func (w *Worker) ackAndClearFailure(ctx context.Context, messageID string) error {
if err := w.Redis.XAck(ctx, w.stream(), w.group(), messageID).Err(); err != nil {
return err
}
if err := w.Redis.HDel(ctx, w.failureAttemptsKey(), messageID).Err(); err != nil {
w.logf("gateway submit worker clear failure %s failed: %v", messageID, err)
}
return nil
}
func (w *Worker) stream() string {
if w.Stream != "" {
return w.Stream
}
return defaultStream
}
func (w *Worker) group() string {
if w.Group != "" {
return w.Group
}
return defaultGroup
}
func (w *Worker) consumer() string {
if w.Consumer != "" {
return w.Consumer
}
return defaultConsumer
}
func (w *Worker) block() time.Duration {
if w.Block > 0 {
return w.Block
}
return 5 * time.Second
}
func (w *Worker) count() int64 {
if w.Count > 0 {
return w.Count
}
return 10
}
func (w *Worker) minIdle() time.Duration {
if w.MinIdle > 0 {
return w.MinIdle
}
return defaultMinIdle
}
func (w *Worker) maxFailures() int {
if w.MaxFailures > 0 {
return w.MaxFailures
}
return defaultMaxFails
}
func (w *Worker) failureAttemptsKey() string {
return w.stream() + ":failure-attempts"
}
func (w *Worker) logf(format string, args ...interface{}) {
if w.Logger != nil {
w.Logger.Printf(format, args...)
return
}
log.Printf(format, args...)
}
func sleep(ctx context.Context, duration time.Duration) {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
func extractRawPayload(values map[string]interface{}) string {
raw, ok := values["data"]
if !ok {
return ""
}
switch value := raw.(type) {
case string:
return value
case []byte:
return string(value)
default:
return fmt.Sprint(value)
}
}
func commandPayload(command queue.SubmitCommand) (map[string]interface{}, error) {
data, err := json.Marshal(command)
if err != nil {
return nil, err
}
var payload map[string]interface{}
if err := json.Unmarshal(data, &payload); err != nil {
return nil, err
}
return payload, nil
}
gateway/internal/submitworker/worker_test.go
+180
View File
@@ -0,0 +1,180 @@
package submitworker
import (
"context"
"testing"
"cmpp-platform/gateway/internal/queue"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
)
func TestCommandFromStreamValuesParsesSubmitCommand(t *testing.T) {
command, err := CommandFromStreamValues(map[string]interface{}{
"messageType": "SubmitCommand",
"data": `{
"schemaVersion": "v1",
"messageType": "SubmitCommand",
"traceId": "trace-worker-0001",
"messageId": "msg-worker-0001",
"channelId": "channel-1",
"createdAt": "2026-07-07T10:00:00Z",
"tenantId": "tenant-1",
"applicationId": "app-1",
"submitId": "submit-1",
"phoneNumber": "13800138000",
"content": "hello",
"signature": "测试",
"templateId": "tpl-1",
"billingUnits": 1,
"queuePriority": "normal",
"route": {
"channelCode": "CMPP-A",
"cmppAccountCode": "account-a",
"priority": 0,
"rateLimitPerSecond": 100
},
"cmpp": {
"serviceId": "SMS",
"srcId": "10690000",
"registeredDelivery": 1,
"msgFmt": 8
},
"upstream": {
"gatewayHost": "127.0.0.1",
"gatewayPort": 17890,
"account": "account-a",
"passwordCipher": "secret",
"cmppVersion": "3.0"
},
"retry": {
"attempt": 0,
"maxAttempts": 1
}
}`,
})
if err != nil {
t.Fatalf("parse stream command: %v", err)
}
if command.MessageID != "msg-worker-0001" || command.Upstream.Account != "account-a" {
t.Fatalf("unexpected command: %+v", command)
}
}
func TestCommandFromStreamValuesRejectsMissingData(t *testing.T) {
_, err := CommandFromStreamValues(map[string]interface{}{"messageType": "SubmitCommand"})
if err == nil {
t.Fatal("expected missing data error")
}
}
func TestHandleMessageUsesInjectedSubmit(t *testing.T) {
var got queue.SubmitCommand
worker := &Worker{
Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
got = command
return queue.SubmitResult{SubmitStatus: "accepted"}, nil
},
}
if err := worker.handleCommand(context.Background(), queue.SubmitCommand{
Envelope: queue.Envelope{MessageID: "msg-worker-0002"},
SubmitID: "submit-2",
PhoneNumber: "13800138000",
Content: "hello",
Upstream: queue.UpstreamConfig{GatewayHost: "127.0.0.1", GatewayPort: 17890, Account: "account-a", PasswordCipher: "secret", CMPPVersion: "3.0"},
CMPP: queue.CMPP{ServiceID: "SMS", SrcID: "10690000", RegisteredDelivery: 1, MsgFmt: 8},
Route: queue.Route{ChannelCode: "CMPP-A"},
Retry: queue.Retry{Attempt: 0, MaxAttempts: 1},
ApplicationID: "app-1",
TenantID: "tenant-1",
}); err != nil {
t.Fatalf("handleCommand returned error: %v", err)
}
if got.MessageID != "msg-worker-0002" || got.SubmitID != "submit-2" {
t.Fatalf("unexpected command: %+v", got)
}
}
func TestMinIdleDefaultsToThirtySeconds(t *testing.T) {
worker := &Worker{}
if got := worker.minIdle(); got != defaultMinIdle {
t.Fatalf("minIdle = %v, want %v", got, defaultMinIdle)
}
}
func TestProcessMessageDeadLettersAfterMaxFailures(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
reported := []DeadLetterEvent{}
worker := &Worker{
Redis: client,
Stream: "gateway.submit.commands",
Group: "cmpp-gateway",
MaxFailures: 2,
Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
return queue.SubmitResult{}, context.DeadlineExceeded
},
ReportDeadLetter: func(_ context.Context, event DeadLetterEvent) error {
reported = append(reported, event)
return nil
},
}
ctx := context.Background()
if err := worker.ensureGroup(ctx); err != nil {
t.Fatalf("ensureGroup: %v", err)
}
message := redis.XMessage{
ID: "1710000000000-0",
Values: map[string]interface{}{
"messageType": "SubmitCommand",
"data": `{
"schemaVersion": "v1",
"messageType": "SubmitCommand",
"traceId": "trace-worker-0003",
"messageId": "msg-worker-0003",
"channelId": "channel-1",
"createdAt": "2026-07-08T10:00:00Z",
"tenantId": "tenant-1",
"applicationId": "app-1",
"submitId": "submit-3",
"phoneNumber": "13800138000",
"content": "hello",
"signature": "测试",
"templateId": "tpl-1",
"billingUnits": 1,
"queuePriority": "normal",
"route": { "channelCode": "CMPP-A", "cmppAccountCode": "account-a", "priority": 0 },
"cmpp": { "serviceId": "SMS", "srcId": "10690000", "registeredDelivery": 1, "msgFmt": 8 },
"upstream": { "gatewayHost": "127.0.0.1", "gatewayPort": 17890, "account": "account-a", "passwordCipher": "secret", "cmppVersion": "3.0" },
"retry": { "attempt": 0, "maxAttempts": 1 }
}`,
},
}
if err := client.XAdd(ctx, &redis.XAddArgs{
Stream: worker.stream(),
ID: message.ID,
Values: message.Values,
}).Err(); err != nil {
t.Fatalf("xadd: %v", err)
}
if err := worker.processMessage(ctx, message); err == nil {
t.Fatal("expected first failure")
}
if len(reported) != 0 {
t.Fatalf("unexpected dead letters on first failure: %+v", reported)
}
if err := worker.processMessage(ctx, message); err != nil {
t.Fatalf("second failure should dead-letter and ack, got %v", err)
}
if len(reported) != 1 {
t.Fatalf("dead letters = %d, want 1", len(reported))
}
if reported[0].FailureCode != "SUBMIT_PROCESSING_FAILED" || reported[0].Attempts != 2 {
t.Fatalf("unexpected dead letter: %+v", reported[0])
}
if client.HGet(ctx, worker.failureAttemptsKey(), message.ID).Err() != redis.Nil {
t.Fatalf("failure attempt key was not cleared")
}
}
gateway/internal/upstream/connection_loss_test.go
+50
View File
@@ -0,0 +1,50 @@
package upstream
import (
"errors"
"testing"
)
func TestHandleConnectionLossNotifiesPendingSubmitters(t *testing.T) {
conn := &connection{
pending: make(map[uint32]chan submitPartResponse),
}
waiter := make(chan submitPartResponse, 1)
conn.pending[7] = waiter
loss := errors.New("socket closed")
conn.handleConnectionLoss(loss)
select {
case result := <-waiter:
if !errors.Is(result.err, loss) {
t.Fatalf("pending waiter err = %v, want %v", result.err, loss)
}
default:
t.Fatal("expected pending waiter to be notified")
}
if !conn.closed {
t.Fatal("expected connection to be marked closed")
}
if len(conn.pending) != 0 {
t.Fatalf("expected pending map to be reset, got %d entries", len(conn.pending))
}
}
func TestTemporaryReadTimeoutDetection(t *testing.T) {
if !isTemporaryReadTimeout(fakeNetError{timeout: true}) {
t.Fatal("expected timeout error to be treated as temporary")
}
if isTemporaryReadTimeout(errors.New("eof")) {
t.Fatal("did not expect non-timeout error to be treated as temporary")
}
}
type fakeNetError struct {
timeout bool
}
func (f fakeNetError) Error() string { return "network error" }
func (f fakeNetError) Timeout() bool { return f.timeout }
func (f fakeNetError) Temporary() bool { return f.timeout }
gateway/internal/upstream/long_message.go
+153
View File
@@ -0,0 +1,153 @@
package upstream
import (
"fmt"
"strings"
"time"
)
const (
maxSingleMessageBytes = 140
maxMultipartPayloadBytes = 134
concatUDHLength = 6
maxMultipartSegments = 255
)
type submitPart struct {
PkTotal uint8
PkNumber uint8
TpUdhi uint8
MsgContent string
}
type longUplinkAssembly struct {
msgFmt uint8
total uint8
parts map[uint8]string
updatedAt time.Time
}
func splitSubmitContent(format int, content string) ([]submitPart, error) {
encoded, err := encodeContent(format, content)
if err != nil {
return nil, err
}
if len(encoded) <= maxSingleMessageBytes {
return []submitPart{{
PkTotal: 1,
PkNumber: 1,
TpUdhi: 0,
MsgContent: encoded,
}}, nil
}
chunks, err := splitEncodedContent(format, content, maxMultipartPayloadBytes)
if err != nil {
return nil, err
}
if len(chunks) > maxMultipartSegments {
return nil, fmt.Errorf("message requires %d segments, maximum is %d", len(chunks), maxMultipartSegments)
}
ref := uint8(time.Now().UnixNano())
parts := make([]submitPart, 0, len(chunks))
for i, chunk := range chunks {
total := uint8(len(chunks))
number := uint8(i + 1)
udh := []byte{0x05, 0x00, 0x03, ref, total, number}
content := append(udh, []byte(chunk)...)
parts = append(parts, submitPart{
PkTotal: total,
PkNumber: number,
TpUdhi: 1,
MsgContent: string(content),
})
}
return parts, nil
}
func splitEncodedContent(format int, content string, limit int) ([]string, error) {
var chunks []string
var current strings.Builder
currentLen := 0
for _, r := range content {
encoded, err := encodeContent(format, string(r))
if err != nil {
return nil, err
}
if len(encoded) > limit {
return nil, fmt.Errorf("single character exceeds segment payload limit")
}
if currentLen > 0 && currentLen+len(encoded) > limit {
chunks = append(chunks, current.String())
current.Reset()
currentLen = 0
}
current.WriteString(encoded)
currentLen += len(encoded)
}
if currentLen > 0 {
chunks = append(chunks, current.String())
}
if len(chunks) == 0 {
chunks = append(chunks, "")
}
return chunks, nil
}
func parseConcatSegment(content string) (ref uint8, total uint8, number uint8, payload string, ok bool) {
raw := []byte(content)
if len(raw) < concatUDHLength {
return 0, 0, 0, "", false
}
if raw[0] != 0x05 || raw[1] != 0x00 || raw[2] != 0x03 {
return 0, 0, 0, "", false
}
ref = raw[3]
total = raw[4]
number = raw[5]
if total == 0 || number == 0 || number > total {
return 0, 0, 0, "", false
}
return ref, total, number, string(raw[concatUDHLength:]), true
}
func assembleLongUplink(assemblies map[string]*longUplinkAssembly, key string, msgFmt uint8, total uint8, number uint8, payload string) (string, bool, error) {
assembly := assemblies[key]
if assembly == nil || assembly.total != total || assembly.msgFmt != msgFmt {
assembly = &longUplinkAssembly{
msgFmt: msgFmt,
total: total,
parts: make(map[uint8]string, int(total)),
}
assemblies[key] = assembly
}
assembly.parts[number] = payload
assembly.updatedAt = time.Now()
if len(assembly.parts) < int(total) {
return "", false, nil
}
var raw strings.Builder
for i := uint8(1); i <= total; i++ {
part, ok := assembly.parts[i]
if !ok {
return "", false, nil
}
raw.WriteString(part)
}
delete(assemblies, key)
content, err := decodeContent(msgFmt, raw.String())
if err != nil {
return "", false, err
}
return content, true, nil
}
func pruneLongUplinkAssemblies(assemblies map[string]*longUplinkAssembly, now time.Time, ttl time.Duration) {
for key, assembly := range assemblies {
if now.Sub(assembly.updatedAt) > ttl {
delete(assemblies, key)
}
}
}
gateway/internal/upstream/long_message_test.go
+95
View File
@@ -0,0 +1,95 @@
package upstream
import (
"strings"
"testing"
)
func TestSplitSubmitContentUCS2LongMessage(t *testing.T) {
content := strings.Repeat("测试", 40)
parts, err := splitSubmitContent(8, content)
if err != nil {
t.Fatalf("splitSubmitContent returned error: %v", err)
}
if len(parts) < 2 {
t.Fatalf("expected multipart content, got %d part", len(parts))
}
total := uint8(len(parts))
ref := []byte(parts[0].MsgContent)[3]
for i, part := range parts {
if part.PkTotal != total {
t.Fatalf("part %d PkTotal = %d, want %d", i, part.PkTotal, total)
}
if part.PkNumber != uint8(i+1) {
t.Fatalf("part %d PkNumber = %d, want %d", i, part.PkNumber, i+1)
}
if part.TpUdhi != 1 {
t.Fatalf("part %d TpUdhi = %d, want 1", i, part.TpUdhi)
}
raw := []byte(part.MsgContent)
if len(raw) > maxSingleMessageBytes {
t.Fatalf("part %d length = %d, want <= %d", i, len(raw), maxSingleMessageBytes)
}
if raw[0] != 0x05 || raw[1] != 0x00 || raw[2] != 0x03 {
t.Fatalf("part %d missing standard concat UDH: %v", i, raw[:concatUDHLength])
}
if raw[3] != ref || raw[4] != total || raw[5] != uint8(i+1) {
t.Fatalf("part %d UDH = %v, ref=%d total=%d number=%d", i, raw[:concatUDHLength], ref, total, i+1)
}
if len(raw[concatUDHLength:])%2 != 0 {
t.Fatalf("part %d UCS2 payload length must be even, got %d", i, len(raw[concatUDHLength:]))
}
}
}
func TestSplitSubmitContentSingleShortMessage(t *testing.T) {
parts, err := splitSubmitContent(15, "hello")
if err != nil {
t.Fatalf("splitSubmitContent returned error: %v", err)
}
if len(parts) != 1 {
t.Fatalf("expected one part, got %d", len(parts))
}
if parts[0].PkTotal != 1 || parts[0].PkNumber != 1 || parts[0].TpUdhi != 0 {
t.Fatalf("unexpected single part metadata: %+v", parts[0])
}
}
func TestAssembleLongUplinkOutOfOrder(t *testing.T) {
content := strings.Repeat("上行", 40)
parts, err := splitSubmitContent(8, content)
if err != nil {
t.Fatalf("splitSubmitContent returned error: %v", err)
}
if len(parts) < 2 {
t.Fatalf("expected multipart content, got %d part", len(parts))
}
assemblies := map[string]*longUplinkAssembly{}
key := "channel:phone:dest:ref"
for i := len(parts) - 1; i >= 0; i-- {
_, total, number, payload, ok := parseConcatSegment(parts[i].MsgContent)
if !ok {
t.Fatalf("part %d did not parse as concat segment", i)
}
assembled, complete, err := assembleLongUplink(assemblies, key, 8, total, number, payload)
if err != nil {
t.Fatalf("assembleLongUplink returned error: %v", err)
}
if i > 0 && complete {
t.Fatalf("assembly completed before all parts arrived")
}
if i == 0 {
if !complete {
t.Fatalf("assembly did not complete after all parts arrived")
}
if assembled != content {
t.Fatalf("assembled content mismatch: got %q want %q", assembled, content)
}
}
}
if len(assemblies) != 0 {
t.Fatalf("expected completed assembly to be removed, got %d", len(assemblies))
}
}
gateway/internal/upstream/manager.go
+697
View File
@@ -0,0 +1,697 @@
package upstream
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"strings"
"sync"
"time"
"cmpp-platform/gateway/internal/queue"
cmpp "github.com/bigwhite/gocmpp"
cmpputils "github.com/bigwhite/gocmpp/utils"
)
const (
defaultConnectTimeout = 5 * time.Second
defaultSubmitTimeout = 10 * time.Second
defaultHTTPTimeout = 10 * time.Second
defaultWindowSize = 16
)
type Manager struct {
APIBaseURL string
HTTPClient *http.Client
mu sync.Mutex
conns map[string]*connectionPool
}
func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
if err := validateSubmitCommand(cmd); err != nil {
result := submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error())
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err
}
pool, err := m.connectionFor(cmd)
if err != nil {
result := submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error())
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err
}
result, err := pool.submit(ctx, cmd)
if err != nil {
if postErr := m.post(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err
}
if err := m.post(ctx, "/gateway/events/submit-result", result); err != nil {
return result, err
}
return result, nil
}
func (m *Manager) connectionFor(cmd queue.SubmitCommand) (*connectionPool, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.HTTPClient == nil {
m.HTTPClient = &http.Client{Timeout: defaultHTTPTimeout}
}
if m.conns == nil {
m.conns = make(map[string]*connectionPool)
}
pool := m.conns[cmd.ChannelID]
if pool == nil || !pool.matches(cmd.Upstream) {
if pool != nil {
pool.close()
}
pool = &connectionPool{
channelID: cmd.ChannelID,
config: normalizeUpstreamConfig(cmd.Upstream),
apiBaseURL: m.APIBaseURL,
httpClient: m.HTTPClient,
}
m.conns[cmd.ChannelID] = pool
}
if err := pool.ensureConnected(); err != nil {
delete(m.conns, cmd.ChannelID)
return nil, err
}
return pool, nil
}
func (m *Manager) post(ctx context.Context, path string, payload any) error {
client := m.HTTPClient
if client == nil {
client = &http.Client{Timeout: defaultHTTPTimeout}
}
return postJSON(ctx, client, m.APIBaseURL, path, payload)
}
type connectionPool struct {
channelID string
config queue.UpstreamConfig
apiBaseURL string
httpClient *http.Client
mu sync.Mutex
conns []*connection
next int
}
func (p *connectionPool) matches(config queue.UpstreamConfig) bool {
return p.config == normalizeUpstreamConfig(config)
}
func (p *connectionPool) ensureConnected() error {
p.mu.Lock()
defer p.mu.Unlock()
desired := p.config.DesiredConnections
if desired <= 0 {
desired = 1
}
for len(p.conns) < desired {
index := len(p.conns)
conn := &connection{
channelID: p.channelID,
config: p.config,
index: index,
apiBaseURL: p.apiBaseURL,
httpClient: p.httpClient,
window: make(chan struct{}, p.config.WindowSize),
pending: make(map[uint32]chan submitPartResponse),
tracker: make(map[uint64]queue.SubmitCommand),
longUplink: make(map[string]*longUplinkAssembly),
}
if err := conn.ensureConnected(); err != nil {
conn.close()
p.closeLocked()
return err
}
p.conns = append(p.conns, conn)
}
return nil
}
func (p *connectionPool) submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content)
if err != nil {
result := submitResult(cmd, 0, "", "rejected", "ENCODE_FAILED", err.Error())
return result, err
}
var firstSequence uint32
var firstGatewayMessageID string
segments := make([]queue.SubmitSegmentResult, 0, len(parts))
for _, part := range parts {
conn, release, err := p.acquireConnection(ctx)
if err != nil {
result := submitResult(cmd, 0, "", "timeout", "WINDOW_TIMEOUT", err.Error())
result.Segments = segments
return result, err
}
seq, gatewayMessageID, result, err := conn.submitPart(ctx, cmd, part)
release()
segments = append(segments, submitSegmentResult(part, seq, gatewayMessageID, result))
if firstSequence == 0 {
firstSequence = seq
}
if firstGatewayMessageID == "" {
firstGatewayMessageID = gatewayMessageID
}
if err != nil {
result.Segments = segments
return result, err
}
if result.SubmitStatus != "accepted" {
result.Segments = segments
return result, nil
}
}
result := submitResult(cmd, firstSequence, firstGatewayMessageID, "accepted", "", "")
result.Segments = segments
return result, nil
}
func (p *connectionPool) acquireConnection(ctx context.Context) (*connection, func(), error) {
waitCtx, cancel := context.WithTimeout(ctx, defaultSubmitTimeout)
defer cancel()
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
if conn, release := p.tryAcquireConnection(); conn != nil {
if err := conn.ensureConnected(); err != nil {
release()
select {
case <-waitCtx.Done():
return nil, nil, waitCtx.Err()
case <-ticker.C:
continue
}
}
return conn, release, nil
}
select {
case <-waitCtx.Done():
return nil, nil, waitCtx.Err()
case <-ticker.C:
}
}
}
func (p *connectionPool) tryAcquireConnection() (*connection, func()) {
p.mu.Lock()
defer p.mu.Unlock()
if len(p.conns) == 0 {
return nil, nil
}
for i := 0; i < len(p.conns); i++ {
index := (p.next + i) % len(p.conns)
conn := p.conns[index]
if conn.tryAcquireWindow() {
p.next = (index + 1) % len(p.conns)
return conn, conn.releaseWindow
}
}
return nil, nil
}
func (p *connectionPool) close() {
p.mu.Lock()
defer p.mu.Unlock()
p.closeLocked()
}
func (p *connectionPool) closeLocked() {
for _, conn := range p.conns {
conn.close()
}
p.conns = nil
}
type connection struct {
channelID string
config queue.UpstreamConfig
index int
apiBaseURL string
httpClient *http.Client
mu sync.Mutex
sendMu sync.Mutex
client *cmpp.Client
window chan struct{}
pending map[uint32]chan submitPartResponse
tracker map[uint64]queue.SubmitCommand
longUplink map[string]*longUplinkAssembly
readOnce sync.Once
closed bool
}
type submitPartResponse struct {
rsp *cmpp.Cmpp3SubmitRspPkt
err error
}
func (c *connection) matches(config queue.UpstreamConfig) bool {
return c.config == normalizeUpstreamConfig(config)
}
func (c *connection) ensureConnected() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.client != nil && !c.closed {
return nil
}
client := cmpp.NewClient(protocolVersion(c.config.CMPPVersion))
addr := fmt.Sprintf("%s:%d", c.config.GatewayHost, c.config.GatewayPort)
if err := client.Connect(addr, c.config.Account, c.config.PasswordCipher, defaultConnectTimeout); err != nil {
client.Disconnect()
return err
}
c.client = client
c.closed = false
go c.readLoop()
return nil
}
func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) {
rspCh := make(chan submitPartResponse, 1)
pkt := &cmpp.Cmpp3SubmitReqPkt{
PkTotal: part.PkTotal,
PkNumber: part.PkNumber,
TpUdhi: part.TpUdhi,
RegisteredDelivery: uint8(cmd.CMPP.RegisteredDelivery),
MsgLevel: 1,
ServiceId: cmd.CMPP.ServiceID,
FeeUserType: uint8(defaultInt(cmd.CMPP.FeeUserType, 2)),
FeeTerminalId: cmd.PhoneNumber,
MsgFmt: uint8(cmd.CMPP.MsgFmt),
MsgSrc: c.config.Account,
FeeType: defaultString(cmd.CMPP.FeeType, "02"),
FeeCode: defaultString(cmd.CMPP.FeeCode, "0"),
SrcId: cmd.CMPP.SrcID,
DestUsrTl: 1,
DestTerminalId: []string{cmd.PhoneNumber},
MsgLength: uint8(len(part.MsgContent)),
MsgContent: part.MsgContent,
}
c.sendMu.Lock()
seq, err := c.client.SendReqPkt(pkt)
c.sendMu.Unlock()
if err != nil {
c.close()
result := submitResult(cmd, 0, "", "timeout", "SEND_FAILED", err.Error())
return 0, "", result, err
}
c.mu.Lock()
c.pending[seq] = rspCh
c.mu.Unlock()
defer func() {
c.mu.Lock()
delete(c.pending, seq)
c.mu.Unlock()
}()
waitCtx, cancel := context.WithTimeout(ctx, defaultSubmitTimeout)
defer cancel()
select {
case <-waitCtx.Done():
result := submitResult(cmd, seq, "", "timeout", "SUBMIT_TIMEOUT", waitCtx.Err().Error())
return seq, "", result, waitCtx.Err()
case rsp := <-rspCh:
if rsp.err != nil {
result := submitResult(cmd, seq, "", "timeout", "CONNECTION_LOST", rsp.err.Error())
return seq, "", result, rsp.err
}
if rsp.rsp == nil {
err := fmt.Errorf("submit response is empty")
result := submitResult(cmd, seq, "", "timeout", "EMPTY_SUBMIT_RESPONSE", err.Error())
return seq, "", result, err
}
gatewayMessageID := fmt.Sprint(rsp.rsp.MsgId)
status := "accepted"
errorCode := ""
errorMessage := ""
if rsp.rsp.Result != 0 {
status = "rejected"
errorCode = fmt.Sprint(rsp.rsp.Result)
errorMessage = fmt.Sprintf("upstream submit rejected with result %d", rsp.rsp.Result)
}
if rsp.rsp.Result == 0 {
c.mu.Lock()
c.tracker[rsp.rsp.MsgId] = cmd
c.mu.Unlock()
}
return seq, gatewayMessageID, submitResult(cmd, seq, gatewayMessageID, status, errorCode, errorMessage), nil
}
}
func (c *connection) tryAcquireWindow() bool {
if c.window == nil {
c.window = make(chan struct{}, defaultWindowSize)
}
select {
case c.window <- struct{}{}:
return true
default:
return false
}
}
func (c *connection) releaseWindow() {
if c.window == nil {
return
}
select {
case <-c.window:
default:
}
}
func (c *connection) readLoop() {
for {
pkt, err := c.client.RecvAndUnpackPkt(time.Second)
if err != nil {
c.mu.Lock()
closed := c.closed
c.mu.Unlock()
if closed {
return
}
if isTemporaryReadTimeout(err) {
continue
}
c.handleConnectionLoss(err)
continue
}
switch p := pkt.(type) {
case *cmpp.Cmpp3SubmitRspPkt:
c.mu.Lock()
ch := c.pending[p.SeqId]
c.mu.Unlock()
if ch != nil {
ch <- submitPartResponse{rsp: p}
}
case *cmpp.Cmpp3DeliverReqPkt:
c.handleDeliver(p)
case *cmpp.CmppActiveTestReqPkt:
_ = c.client.SendRspPkt(&cmpp.CmppActiveTestRspPkt{}, p.SeqId)
}
}
}
func (c *connection) handleDeliver(pkt *cmpp.Cmpp3DeliverReqPkt) {
_ = c.client.SendRspPkt(&cmpp.Cmpp3DeliverRspPkt{MsgId: pkt.MsgId, Result: 0}, pkt.SeqId)
if pkt.RegisterDelivery == 1 {
var receipt cmpp.CmppReceiptPkt
if err := receipt.Unpack([]byte(pkt.MsgContent)); err != nil {
return
}
cmd, ok := c.commandFor(receipt.MsgId)
if !ok {
cmd, ok = c.commandFor(pkt.MsgId)
}
traceID := fmt.Sprintf("receipt-%d", receipt.MsgId)
messageID := fmt.Sprintf("receipt-%d", receipt.MsgId)
channelID := c.channelID
if ok {
traceID = cmd.TraceID
messageID = cmd.MessageID
channelID = cmd.ChannelID
}
event := queue.ReceiptEvent{
Envelope: queue.Envelope{
SchemaVersion: queue.SchemaVersion,
MessageType: queue.MessageTypeReceiptEvent,
TraceID: traceID,
MessageID: messageID,
ChannelID: channelID,
CreatedAt: time.Now().UTC(),
},
SequenceID: pkt.SeqId,
GatewayMessageID: fmt.Sprint(receipt.MsgId),
PhoneNumber: strings.TrimSpace(receipt.DestTerminalId),
ReceiptStatus: receiptStatus(receipt.Stat),
RawStatus: strings.TrimSpace(receipt.Stat),
DeliveredAt: time.Now().UTC(),
}
_ = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/receipt", event)
return
}
content, complete, err := c.decodeUplinkContent(pkt)
if err != nil {
return
}
if !complete {
return
}
cmd, _ := c.commandFor(pkt.MsgId)
event := queue.UplinkEvent{
Envelope: queue.Envelope{
SchemaVersion: queue.SchemaVersion,
MessageType: queue.MessageTypeUplinkEvent,
TraceID: cmd.TraceID,
MessageID: cmd.MessageID,
ChannelID: c.channelID,
CreatedAt: time.Now().UTC(),
},
SequenceID: pkt.SeqId,
PhoneNumber: strings.TrimSpace(pkt.SrcTerminalId),
DestID: strings.TrimSpace(pkt.DestId),
Content: content,
ReceivedAt: time.Now().UTC(),
}
_ = postJSON(context.Background(), c.httpClient, c.apiBaseURL, "/gateway/events/uplink", event)
}
func (c *connection) decodeUplinkContent(pkt *cmpp.Cmpp3DeliverReqPkt) (string, bool, error) {
if pkt.TpUdhi != 1 {
content, err := decodeContent(pkt.MsgFmt, pkt.MsgContent)
return content, true, err
}
ref, total, number, payload, ok := parseConcatSegment(pkt.MsgContent)
if !ok {
content, err := decodeContent(pkt.MsgFmt, pkt.MsgContent)
return content, true, err
}
key := fmt.Sprintf("%s:%s:%s:%d:%d", c.channelID, strings.TrimSpace(pkt.SrcTerminalId), strings.TrimSpace(pkt.DestId), ref, total)
c.mu.Lock()
if c.longUplink == nil {
c.longUplink = make(map[string]*longUplinkAssembly)
}
pruneLongUplinkAssemblies(c.longUplink, time.Now(), 10*time.Minute)
content, complete, err := assembleLongUplink(c.longUplink, key, pkt.MsgFmt, total, number, payload)
c.mu.Unlock()
return content, complete, err
}
func (c *connection) commandFor(gatewayMsgID uint64) (queue.SubmitCommand, bool) {
c.mu.Lock()
defer c.mu.Unlock()
cmd, ok := c.tracker[gatewayMsgID]
return cmd, ok
}
func (c *connection) close() {
c.handleConnectionLoss(fmt.Errorf("connection closed"))
}
func (c *connection) handleConnectionLoss(err error) {
c.mu.Lock()
if c.closed && c.client == nil {
c.mu.Unlock()
return
}
c.closed = true
pending := c.pending
c.pending = make(map[uint32]chan submitPartResponse)
if c.client != nil {
c.client.Disconnect()
c.client = nil
}
c.mu.Unlock()
for _, ch := range pending {
select {
case ch <- submitPartResponse{err: err}:
default:
}
}
}
func submitResult(cmd queue.SubmitCommand, sequenceID uint32, gatewayMessageID string, status string, code string, message string) queue.SubmitResult {
if gatewayMessageID == "" {
gatewayMessageID = fmt.Sprintf("GW-%s-%d", cmd.SubmitID, time.Now().UnixNano())
}
return queue.SubmitResult{
Envelope: queue.Envelope{
SchemaVersion: queue.SchemaVersion,
MessageType: queue.MessageTypeSubmitResult,
TraceID: cmd.TraceID,
MessageID: cmd.MessageID,
ChannelID: cmd.ChannelID,
CreatedAt: time.Now().UTC(),
},
SequenceID: sequenceID,
GatewayMessageID: gatewayMessageID,
SubmitStatus: status,
ErrorCode: code,
ErrorMessage: message,
SubmittedAt: time.Now().UTC(),
}
}
func submitSegmentResult(part submitPart, sequenceID uint32, gatewayMessageID string, result queue.SubmitResult) queue.SubmitSegmentResult {
if gatewayMessageID == "" {
gatewayMessageID = result.GatewayMessageID
}
return queue.SubmitSegmentResult{
SegmentTotal: int(part.PkTotal),
SegmentIndex: int(part.PkNumber),
SequenceID: sequenceID,
GatewayMessageID: gatewayMessageID,
SubmitStatus: result.SubmitStatus,
ErrorCode: result.ErrorCode,
ErrorMessage: result.ErrorMessage,
SubmittedAt: result.SubmittedAt,
}
}
func validateSubmitCommand(cmd queue.SubmitCommand) error {
if cmd.MessageType != queue.MessageTypeSubmitCommand {
return fmt.Errorf("unsupported messageType %q", cmd.MessageType)
}
if cmd.MessageID == "" || cmd.ChannelID == "" || cmd.SubmitID == "" {
return fmt.Errorf("messageId, channelId and submitId are required")
}
if cmd.Upstream.GatewayHost == "" || cmd.Upstream.GatewayPort <= 0 {
return fmt.Errorf("upstream gatewayHost and gatewayPort are required")
}
if cmd.Upstream.Account == "" || cmd.Upstream.PasswordCipher == "" {
return fmt.Errorf("upstream account and passwordCipher are required")
}
if len(cmd.PhoneNumber) == 0 || len(cmd.Content) == 0 {
return fmt.Errorf("phoneNumber and content are required")
}
return nil
}
func normalizeUpstreamConfig(config queue.UpstreamConfig) queue.UpstreamConfig {
if config.DesiredConnections <= 0 {
config.DesiredConnections = 1
}
if config.WindowSize <= 0 {
config.WindowSize = defaultWindowSize
}
return config
}
func isTemporaryReadTimeout(err error) bool {
var netErr net.Error
return errors.As(err, &netErr) && netErr.Timeout()
}
func postJSON(ctx context.Context, client *http.Client, apiBaseURL string, path string, payload any) error {
if client == nil {
client = &http.Client{Timeout: defaultHTTPTimeout}
}
body, err := json.Marshal(payload)
if err != nil {
return err
}
base := strings.TrimRight(apiBaseURL, "/")
if base == "" {
base = "http://127.0.0.1:3000/api"
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+path, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("api returned %s", resp.Status)
}
return nil
}
func encodeContent(format int, content string) (string, error) {
switch format {
case 8:
return cmpputils.Utf8ToUcs2(content)
case 15:
return cmpputils.Utf8ToGB18030(content)
default:
return content, nil
}
}
func decodeContent(format uint8, content string) (string, error) {
switch format {
case 8:
return cmpputils.Ucs2ToUtf8(content)
case 15:
return cmpputils.GB18030ToUtf8(content)
default:
return content, nil
}
}
func protocolVersion(version string) cmpp.Type {
if strings.HasPrefix(version, "2") {
return cmpp.V20
}
return cmpp.V30
}
func receiptStatus(stat string) string {
switch strings.ToUpper(strings.TrimSpace(stat)) {
case "DELIVRD":
return "delivered"
case "":
return "unknown"
default:
return "undelivered"
}
}
func defaultString(value string, fallback string) string {
if value == "" {
return fallback
}
return value
}
func defaultInt(value int, fallback int) int {
if value == 0 {
return fallback
}
return value
}
gateway/internal/upstream/pool_test.go
+60
View File
@@ -0,0 +1,60 @@
package upstream
import (
"testing"
"cmpp-platform/gateway/internal/queue"
)
func TestConnectionPoolAcquiresAcrossConnections(t *testing.T) {
pool := &connectionPool{
conns: []*connection{
{window: make(chan struct{}, 1)},
{window: make(chan struct{}, 1)},
},
}
first, releaseFirst := pool.tryAcquireConnection()
if first == nil {
t.Fatalf("expected first connection")
}
second, releaseSecond := pool.tryAcquireConnection()
if second == nil {
t.Fatalf("expected second connection")
}
if first == second {
t.Fatalf("expected pool to use another connection when the first window is full")
}
third, _ := pool.tryAcquireConnection()
if third != nil {
t.Fatalf("expected nil connection while all windows are full")
}
releaseFirst()
reacquired, releaseReacquired := pool.tryAcquireConnection()
if reacquired == nil {
t.Fatalf("expected a connection after releasing a window")
}
releaseReacquired()
releaseSecond()
}
func TestNormalizeUpstreamConfigDefaults(t *testing.T) {
config := normalizeUpstreamConfig(queueUpstreamConfigForTest())
if config.DesiredConnections != 1 {
t.Fatalf("DesiredConnections = %d, want 1", config.DesiredConnections)
}
if config.WindowSize != defaultWindowSize {
t.Fatalf("WindowSize = %d, want %d", config.WindowSize, defaultWindowSize)
}
}
func queueUpstreamConfigForTest() queue.UpstreamConfig {
return queue.UpstreamConfig{
GatewayHost: "127.0.0.1",
GatewayPort: 17890,
Account: "account",
PasswordCipher: "secret",
CMPPVersion: "3.0",
}
}