hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete cmpp gateway delivery recovery workflows

Browse Source
This commit is contained in:
hectorzhao
2026-07-08 16:30:06 +08:00
parent cc628d0214
commit 8144f08652
60 changed files with 8901 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/internal/inbound/presence.go
+182
View File
@@ -0,0 +1,182 @@
package inbound
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
const defaultPresenceTTL = 3 * time.Minute
type PresenceStore interface {
TouchAccount(ctx context.Context, snapshot DownstreamPresence) error
RemoveAccount(ctx context.Context, account string) error
ListAccounts(ctx context.Context) ([]DownstreamPresence, error)
}
type DownstreamPresence struct {
Account string `json:"account"`
SrcID string `json:"srcId,omitempty"`
RemoteIP string `json:"remoteIp,omitempty"`
GatewayInstanceID string `json:"gatewayInstanceId,omitempty"`
State string `json:"state"`
ConnectedAt time.Time `json:"connectedAt"`
UpdatedAt time.Time `json:"updatedAt"`
LastSubmitAt time.Time `json:"lastSubmitAt,omitempty"`
LastDeliverAt time.Time `json:"lastDeliverAt,omitempty"`
}
type RedisPresenceStore struct {
Client *redis.Client
TTL time.Duration
Prefix string
}
func NewRedisPresenceStore(redisURL string) (*RedisPresenceStore, error) {
if redisURL == "" {
redisURL = "redis://127.0.0.1:6379"
}
options, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}
return &RedisPresenceStore{
Client: redis.NewClient(options),
TTL: defaultPresenceTTL,
Prefix: "gateway:downstream:presence",
}, nil
}
func (s *RedisPresenceStore) TouchAccount(ctx context.Context, snapshot DownstreamPresence) error {
if s == nil || s.Client == nil || snapshot.Account == "" {
return nil
}
if snapshot.UpdatedAt.IsZero() {
snapshot.UpdatedAt = time.Now().UTC()
}
if snapshot.ConnectedAt.IsZero() {
snapshot.ConnectedAt = snapshot.UpdatedAt
}
payload, err := json.Marshal(snapshot)
if err != nil {
return err
}
ttl := s.ttl()
pipe := s.Client.TxPipeline()
pipe.Set(ctx, s.accountKey(snapshot.Account), payload, ttl)
pipe.SAdd(ctx, s.accountsKey(), snapshot.Account)
pipe.Expire(ctx, s.accountsKey(), ttl*4)
_, err = pipe.Exec(ctx)
return err
}
func (s *RedisPresenceStore) RemoveAccount(ctx context.Context, account string) error {
if s == nil || s.Client == nil || account == "" {
return nil
}
pipe := s.Client.TxPipeline()
pipe.Del(ctx, s.accountKey(account))
pipe.SRem(ctx, s.accountsKey(), account)
_, err := pipe.Exec(ctx)
return err
}
func (s *RedisPresenceStore) ListAccounts(ctx context.Context) ([]DownstreamPresence, error) {
if s == nil || s.Client == nil {
return nil, nil
}
accounts, err := s.Client.SMembers(ctx, s.accountsKey()).Result()
if err != nil {
return nil, err
}
result := make([]DownstreamPresence, 0, len(accounts))
for _, account := range accounts {
payload, getErr := s.Client.Get(ctx, s.accountKey(account)).Bytes()
if getErr == redis.Nil {
_ = s.Client.SRem(ctx, s.accountsKey(), account).Err()
continue
}
if getErr != nil {
return nil, getErr
}
var snapshot DownstreamPresence
if err := json.Unmarshal(payload, &snapshot); err != nil {
return nil, fmt.Errorf("decode presence %s: %w", account, err)
}
result = append(result, snapshot)
}
return result, nil
}
func (s *RedisPresenceStore) ttl() time.Duration {
if s != nil && s.TTL > 0 {
return s.TTL
}
return defaultPresenceTTL
}
func (s *RedisPresenceStore) accountKey(account string) string {
return fmt.Sprintf("%s:account:%s", s.prefix(), account)
}
func (s *RedisPresenceStore) accountsKey() string {
return fmt.Sprintf("%s:accounts", s.prefix())
}
func (s *RedisPresenceStore) prefix() string {
if s != nil && s.Prefix != "" {
return s.Prefix
}
return "gateway:downstream:presence"
}
func ListRecoveryCandidates(ctx context.Context, store PresenceStore) ([]DownstreamPresence, error) {
candidateMap := map[string]DownstreamPresence{}
if store != nil {
snapshots, err := store.ListAccounts(ctx)
if err != nil {
return nil, err
}
for _, snapshot := range snapshots {
account := strings.TrimSpace(snapshot.Account)
if account == "" {
continue
}
candidateMap[account] = snapshot
}
}
for _, account := range onlineAccounts() {
account = strings.TrimSpace(account)
if account == "" {
continue
}
current := candidateMap[account]
current.Account = account
if strings.TrimSpace(current.State) == "" {
current.State = "connected"
}
if current.UpdatedAt.IsZero() {
current.UpdatedAt = time.Now().UTC()
}
candidateMap[account] = current
}
result := make([]DownstreamPresence, 0, len(candidateMap))
for _, snapshot := range candidateMap {
result = append(result, snapshot)
}
sort.Slice(result, func(i, j int) bool {
if result[i].UpdatedAt.Equal(result[j].UpdatedAt) {
return result[i].Account < result[j].Account
}
return result[i].UpdatedAt.After(result[j].UpdatedAt)
})
return result, nil
}
gateway/internal/inbound/presence_test.go
+95
View File
@@ -0,0 +1,95 @@
package inbound
import (
"context"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
)
func TestRedisPresenceStoreTouchListAndRemove(t *testing.T) {
mr := miniredis.RunT(t)
store, err := NewRedisPresenceStore("redis://" + mr.Addr())
if err != nil {
t.Fatalf("new presence store: %v", err)
}
store.TTL = time.Minute
store.Prefix = "test:presence"
now := time.Now().UTC().Truncate(time.Second)
err = store.TouchAccount(context.Background(), DownstreamPresence{
Account: "100001",
SrcID: "10690000",
RemoteIP: "127.0.0.1",
GatewayInstanceID: "gateway-a",
State: "connected",
ConnectedAt: now,
UpdatedAt: now,
LastSubmitAt: now,
})
if err != nil {
t.Fatalf("touch account: %v", err)
}
accounts, err := store.ListAccounts(context.Background())
if err != nil {
t.Fatalf("list accounts: %v", err)
}
if len(accounts) != 1 {
t.Fatalf("accounts len = %d, want 1", len(accounts))
}
if accounts[0].Account != "100001" || accounts[0].GatewayInstanceID != "gateway-a" || accounts[0].State != "connected" {
t.Fatalf("unexpected presence snapshot: %+v", accounts[0])
}
if err := store.RemoveAccount(context.Background(), "100001"); err != nil {
t.Fatalf("remove account: %v", err)
}
accounts, err = store.ListAccounts(context.Background())
if err != nil {
t.Fatalf("list accounts after remove: %v", err)
}
if len(accounts) != 0 {
t.Fatalf("accounts len after remove = %d, want 0", len(accounts))
}
}
func TestListRecoveryCandidatesMergesPresenceAndInMemory(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
store := &memoryPresenceStore{
snapshots: map[string]DownstreamPresence{
"100001": {
Account: "100001",
GatewayInstanceID: "gateway-a",
State: "connected",
UpdatedAt: time.Now().UTC().Add(-time.Minute),
},
},
}
downstreamRegistry.Lock()
downstreamRegistry.byAccount["100002"] = &downstreamSession{account: "100002"}
downstreamRegistry.Unlock()
candidates, err := ListRecoveryCandidates(context.Background(), store)
if err != nil {
t.Fatalf("list recovery candidates: %v", err)
}
if len(candidates) != 2 {
t.Fatalf("candidates len = %d, want 2", len(candidates))
}
accounts := map[string]DownstreamPresence{}
for _, item := range candidates {
accounts[item.Account] = item
}
if _, ok := accounts["100001"]; !ok {
t.Fatal("expected redis presence candidate 100001")
}
if snapshot, ok := accounts["100002"]; !ok || snapshot.State != "connected" {
t.Fatalf("expected in-memory candidate 100002 connected, got %+v", snapshot)
}
}
gateway/internal/inbound/recovery.go
+323
View File
@@ -0,0 +1,323 @@
package inbound
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/redis/go-redis/v9"
)
const (
defaultRecoveryLockTTL = 30 * time.Second
defaultRecoveryBackoffBase = 30 * time.Second
defaultRecoveryBackoffMax = 10 * time.Minute
)
var ErrRecoveryLockLost = errors.New("recovery lock lost")
type RecoveryStore interface {
StartAccountRecovery(ctx context.Context, account string, instanceID string) (RecoveryStartDecision, error)
CompleteAccountRecovery(ctx context.Context, status DownstreamRecoveryStatus) error
GetAccountRecoveryStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error)
ListRecoveryStatuses(ctx context.Context) ([]DownstreamRecoveryStatus, error)
}
type RecoveryStartDecision struct {
Allowed bool
SkipReason string
Status DownstreamRecoveryStatus
}
type DownstreamRecoveryStatus struct {
Account string `json:"account"`
GatewayInstanceID string `json:"gatewayInstanceId,omitempty"`
State string `json:"state"`
LockOwner string `json:"lockOwner,omitempty"`
LockToken string `json:"lockToken,omitempty"`
LockAcquiredAt time.Time `json:"lockAcquiredAt,omitempty"`
LockExpiresAt time.Time `json:"lockExpiresAt,omitempty"`
LastAttemptAt time.Time `json:"lastAttemptAt,omitempty"`
LastSuccessAt time.Time `json:"lastSuccessAt,omitempty"`
LastFailureAt time.Time `json:"lastFailureAt,omitempty"`
NextRetryAt time.Time `json:"nextRetryAt,omitempty"`
AttemptCount int `json:"attemptCount"`
FailureCategory string `json:"failureCategory,omitempty"`
LastError string `json:"lastError,omitempty"`
LastSkipReason string `json:"lastSkipReason,omitempty"`
}
type RedisRecoveryStore struct {
Client *redis.Client
Prefix string
}
func NewRedisRecoveryStore(redisURL string) (*RedisRecoveryStore, error) {
if redisURL == "" {
redisURL = "redis://127.0.0.1:6379"
}
options, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}
return &RedisRecoveryStore{
Client: redis.NewClient(options),
Prefix: "gateway:downstream:recovery",
}, nil
}
func (s *RedisRecoveryStore) StartAccountRecovery(ctx context.Context, account string, instanceID string) (RecoveryStartDecision, error) {
if s == nil || s.Client == nil || account == "" {
return RecoveryStartDecision{Allowed: true}, nil
}
status, err := s.getStatus(ctx, account)
if err != nil {
return RecoveryStartDecision{}, err
}
now := time.Now().UTC()
if !status.NextRetryAt.IsZero() && status.NextRetryAt.After(now) {
status.LastSkipReason = "backoff"
status.FailureCategory = "backoff"
return RecoveryStartDecision{Allowed: false, SkipReason: "backoff", Status: status}, nil
}
lockToken := newRecoveryLockToken(instanceID)
acquired, err := s.Client.SetNX(ctx, s.lockKey(account), lockToken, defaultRecoveryLockTTL).Result()
if err != nil {
return RecoveryStartDecision{}, err
}
if !acquired {
status.LockToken, _ = s.Client.Get(ctx, s.lockKey(account)).Result()
status.LockOwner = recoveryLockOwner(status.LockToken)
status.LockExpiresAt = lockExpiresAt(ctx, s.Client, s.lockKey(account), now)
status.LastSkipReason = "locked"
status.FailureCategory = "lock_contended"
return RecoveryStartDecision{Allowed: false, SkipReason: "locked", Status: status}, nil
}
status.Account = account
status.GatewayInstanceID = instanceID
status.State = "running"
status.LockOwner = instanceID
status.LockToken = lockToken
status.LockAcquiredAt = now
status.LockExpiresAt = now.Add(defaultRecoveryLockTTL)
status.LastAttemptAt = now
status.LastSkipReason = ""
status.FailureCategory = ""
if err := s.saveStatus(ctx, status); err != nil {
_ = s.deleteLockIfOwned(ctx, account, lockToken)
return RecoveryStartDecision{}, err
}
return RecoveryStartDecision{Allowed: true, Status: status}, nil
}
func (s *RedisRecoveryStore) CompleteAccountRecovery(ctx context.Context, status DownstreamRecoveryStatus) error {
if s == nil || s.Client == nil || status.Account == "" {
return nil
}
now := time.Now().UTC()
if status.LastAttemptAt.IsZero() {
status.LastAttemptAt = now
}
if status.LockOwner == "" {
status.LockOwner = status.GatewayInstanceID
}
switch status.State {
case "success":
status.LastSuccessAt = now
status.AttemptCount = 0
status.NextRetryAt = time.Time{}
status.LastError = ""
status.FailureCategory = ""
case "waiting_connection", "failed", "partial":
status.AttemptCount++
if status.State == "failed" {
status.LastFailureAt = now
}
if status.FailureCategory == "" {
status.FailureCategory = classifyRecoveryFailure(status)
}
status.NextRetryAt = now.Add(recoveryBackoffDelay(status.AttemptCount))
default:
status.State = "unknown"
}
payload, err := json.Marshal(status)
if err != nil {
return err
}
result, err := s.Client.Eval(ctx, completeRecoveryScript, []string{s.lockKey(status.Account), s.statusKey(status.Account), s.accountsKey()}, status.LockToken, payload, int64((24*time.Hour)/time.Millisecond), status.Account).Int()
if err != nil {
return err
}
if result == 0 {
return ErrRecoveryLockLost
}
return nil
}
func classifyRecoveryFailure(status DownstreamRecoveryStatus) string {
switch status.State {
case "waiting_connection":
return "client_disconnected"
case "partial":
return "partial_delivery_failed"
case "failed":
if status.LastSkipReason == "backoff" {
return "backoff"
}
if status.LastSkipReason == "locked" {
return "lock_contended"
}
if status.LastError != "" {
return "flush_failed"
}
}
return "unknown"
}
func (s *RedisRecoveryStore) ListRecoveryStatuses(ctx context.Context) ([]DownstreamRecoveryStatus, error) {
if s == nil || s.Client == nil {
return nil, nil
}
accounts, err := s.Client.SMembers(ctx, s.accountsKey()).Result()
if err != nil {
return nil, err
}
result := make([]DownstreamRecoveryStatus, 0, len(accounts))
for _, account := range accounts {
status, getErr := s.getStatus(ctx, account)
if getErr == redis.Nil {
_ = s.Client.SRem(ctx, s.accountsKey(), account).Err()
continue
}
if getErr != nil {
return nil, getErr
}
result = append(result, status)
}
return result, nil
}
func (s *RedisRecoveryStore) GetAccountRecoveryStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error) {
if s == nil || s.Client == nil || account == "" {
return DownstreamRecoveryStatus{Account: account}, nil
}
return s.getStatus(ctx, account)
}
func (s *RedisRecoveryStore) getStatus(ctx context.Context, account string) (DownstreamRecoveryStatus, error) {
payload, err := s.Client.Get(ctx, s.statusKey(account)).Bytes()
if err != nil {
if err == redis.Nil {
return DownstreamRecoveryStatus{Account: account}, nil
}
return DownstreamRecoveryStatus{}, err
}
var status DownstreamRecoveryStatus
if err := json.Unmarshal(payload, &status); err != nil {
return DownstreamRecoveryStatus{}, fmt.Errorf("decode recovery status %s: %w", account, err)
}
if status.Account == "" {
status.Account = account
}
return status, nil
}
func (s *RedisRecoveryStore) saveStatus(ctx context.Context, status DownstreamRecoveryStatus) error {
payload, err := json.Marshal(status)
if err != nil {
return err
}
pipe := s.Client.TxPipeline()
pipe.Set(ctx, s.statusKey(status.Account), payload, 24*time.Hour)
pipe.SAdd(ctx, s.accountsKey(), status.Account)
pipe.Expire(ctx, s.accountsKey(), 24*time.Hour)
_, err = pipe.Exec(ctx)
return err
}
func (s *RedisRecoveryStore) deleteLockIfOwned(ctx context.Context, account string, token string) error {
if token == "" {
return nil
}
_, err := s.Client.Eval(ctx, deleteLockIfOwnedScript, []string{s.lockKey(account)}, token).Result()
return err
}
func (s *RedisRecoveryStore) statusKey(account string) string {
return fmt.Sprintf("%s:status:%s", s.prefix(), account)
}
func (s *RedisRecoveryStore) lockKey(account string) string {
return fmt.Sprintf("%s:lock:%s", s.prefix(), account)
}
func (s *RedisRecoveryStore) accountsKey() string {
return fmt.Sprintf("%s:accounts", s.prefix())
}
func (s *RedisRecoveryStore) prefix() string {
if s != nil && s.Prefix != "" {
return s.Prefix
}
return "gateway:downstream:recovery"
}
func recoveryBackoffDelay(attemptCount int) time.Duration {
if attemptCount <= 0 {
return defaultRecoveryBackoffBase
}
delay := defaultRecoveryBackoffBase
for step := 1; step < attemptCount; step++ {
delay *= 2
if delay >= defaultRecoveryBackoffMax {
return defaultRecoveryBackoffMax
}
}
if delay > defaultRecoveryBackoffMax {
return defaultRecoveryBackoffMax
}
return delay
}
func newRecoveryLockToken(instanceID string) string {
return fmt.Sprintf("%s:%d", instanceID, time.Now().UTC().UnixNano())
}
func recoveryLockOwner(token string) string {
for index, char := range token {
if char == ':' {
return token[:index]
}
}
return token
}
func lockExpiresAt(ctx context.Context, client *redis.Client, key string, now time.Time) time.Time {
ttl, err := client.TTL(ctx, key).Result()
if err != nil || ttl <= 0 {
return time.Time{}
}
return now.Add(ttl)
}
const completeRecoveryScript = `
local current = redis.call("GET", KEYS[1])
if current ~= ARGV[1] then
return 0
end
redis.call("SET", KEYS[2], ARGV[2], "PX", ARGV[3])
redis.call("SADD", KEYS[3], ARGV[4])
redis.call("PEXPIRE", KEYS[3], ARGV[3])
redis.call("DEL", KEYS[1])
return 1
`
const deleteLockIfOwnedScript = `
local current = redis.call("GET", KEYS[1])
if current == ARGV[1] then
return redis.call("DEL", KEYS[1])
end
return 0
`
gateway/internal/inbound/recovery_test.go
+101
View File
@@ -0,0 +1,101 @@
package inbound
import (
"context"
"errors"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
)
func TestRedisRecoveryStoreBackoffAndStatuses(t *testing.T) {
mr := miniredis.RunT(t)
store, err := NewRedisRecoveryStore("redis://" + mr.Addr())
if err != nil {
t.Fatalf("new recovery store: %v", err)
}
store.Prefix = "test:recovery"
decision, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
if err != nil {
t.Fatalf("start recovery: %v", err)
}
if !decision.Allowed {
t.Fatalf("expected recovery allowed, got %+v", decision)
}
if decision.Status.LockOwner != "gateway-a" || decision.Status.LockToken == "" || decision.Status.LockExpiresAt.IsZero() {
t.Fatalf("missing recovery lock metadata: %+v", decision.Status)
}
status := decision.Status
status.State = "waiting_connection"
if err := store.CompleteAccountRecovery(context.Background(), status); err != nil {
t.Fatalf("complete recovery: %v", err)
}
statuses, err := store.ListRecoveryStatuses(context.Background())
if err != nil {
t.Fatalf("list recovery statuses: %v", err)
}
if len(statuses) != 1 || statuses[0].State != "waiting_connection" || statuses[0].AttemptCount != 1 {
t.Fatalf("unexpected statuses: %+v", statuses)
}
if statuses[0].FailureCategory != "client_disconnected" {
t.Fatalf("failure category = %q, want client_disconnected", statuses[0].FailureCategory)
}
if statuses[0].NextRetryAt.IsZero() {
t.Fatalf("expected next retry at after waiting connection: %+v", statuses[0])
}
decision, err = store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
if err != nil {
t.Fatalf("start recovery second time: %v", err)
}
if decision.Allowed || decision.SkipReason != "backoff" {
t.Fatalf("expected backoff skip, got %+v", decision)
}
if decision.Status.FailureCategory != "backoff" {
t.Fatalf("skip failure category = %q, want backoff", decision.Status.FailureCategory)
}
}
func TestRedisRecoveryStoreDoesNotReleaseLockOwnedByAnotherGateway(t *testing.T) {
mr := miniredis.RunT(t)
store, err := NewRedisRecoveryStore("redis://" + mr.Addr())
if err != nil {
t.Fatalf("new recovery store: %v", err)
}
store.Prefix = "test:recovery:takeover"
first, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-a")
if err != nil {
t.Fatalf("start first recovery: %v", err)
}
if !first.Allowed {
t.Fatalf("expected first recovery allowed, got %+v", first)
}
mr.FastForward(defaultRecoveryLockTTL + time.Second)
second, err := store.StartAccountRecovery(context.Background(), "100001", "gateway-b")
if err != nil {
t.Fatalf("start second recovery: %v", err)
}
if !second.Allowed {
t.Fatalf("expected second recovery to take over expired lock, got %+v", second)
}
stale := first.Status
stale.State = "success"
err = store.CompleteAccountRecovery(context.Background(), stale)
if !errors.Is(err, ErrRecoveryLockLost) {
t.Fatalf("stale completion error = %v, want ErrRecoveryLockLost", err)
}
lockValue, err := store.Client.Get(context.Background(), store.lockKey("100001")).Result()
if err != nil {
t.Fatalf("load current lock: %v", err)
}
if lockValue != second.Status.LockToken {
t.Fatalf("current lock was changed by stale completion: got %q want %q", lockValue, second.Status.LockToken)
}
}
gateway/internal/inbound/server.go
+534 -3
View File
@@ -11,6 +11,7 @@ import (
"net"
"net/http"
"strings"
"sync"
"time"
cmpp "github.com/bigwhite/gocmpp"
@@ -20,9 +21,13 @@ import (
const defaultHTTPTimeout = 10 * time.Second
type Server struct {
Addr string
APIBaseURL string
HTTPClient *http.Client
Addr string
APIBaseURL string
HTTPClient *http.Client
PendingFlushInterval time.Duration
PresenceStore PresenceStore
RecoveryStore RecoveryStore
GatewayInstanceID string
}
type authRequest struct {
@@ -49,6 +54,56 @@ type submitResponse struct {
type authResponse struct {
PasswordCipher string `json:"passwordCipher"`
ApplicationID string `json:"applicationId"`
TenantID string `json:"tenantId"`
Account string `json:"account"`
}
type DownstreamReceipt struct {
DeliveryID string `json:"deliveryId,omitempty"`
Account string `json:"account,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
MessageID string `json:"messageId"`
GatewayMessageID string `json:"gatewayMessageId,omitempty"`
PhoneNumber string `json:"phoneNumber,omitempty"`
ReceiptStatus string `json:"receiptStatus"`
RawStatus string `json:"rawStatus,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
DeliveredAt string `json:"deliveredAt,omitempty"`
}
type DownstreamUplink struct {
DeliveryID string `json:"deliveryId,omitempty"`
Account string `json:"account,omitempty"`
ApplicationID string `json:"applicationId,omitempty"`
MessageID string `json:"messageId,omitempty"`
PhoneNumber string `json:"phoneNumber"`
DestID string `json:"destId"`
Content string `json:"content"`
ReceivedAt string `json:"receivedAt,omitempty"`
}
type downstreamSession struct {
messageID string
account string
srcID string
phoneNumber string
gatewayMsgID uint64
remoteIP string
connectedAt time.Time
conn *cmpp.Conn
mu *sync.Mutex
presence PresenceStore
instanceID string
}
var downstreamRegistry = struct {
sync.RWMutex
byMessageID map[string]*downstreamSession
byAccount map[string]*downstreamSession
}{
byMessageID: make(map[string]*downstreamSession),
byAccount: make(map[string]*downstreamSession),
}
func (s Server) ListenAndServe() error {
@@ -56,6 +111,9 @@ func (s Server) ListenAndServe() error {
if addr == "" {
addr = ":17890"
}
s.logRecoveryCandidates(log.Default())
go s.recoverPendingCandidates(log.Default())
go s.runPendingFlusher(log.Default())
return cmpp.ListenAndServe(addr, cmpp.V30, 30*time.Second, 3, nil,
cmpp.HandlerFunc(s.handleLogin),
cmpp.HandlerFunc(s.handleSubmit),
@@ -83,6 +141,18 @@ func (s Server) handleLogin(response *cmpp.Response, packet *cmpp.Packet, logger
authSource := []byte(req.AuthSrc)
authISMG := md5.Sum(bytes.Join([][]byte{{byte(resp.Status)}, authSource, []byte(auth.PasswordCipher)}, nil))
resp.AuthIsmg = string(authISMG[:])
session := downstreamSession{
account: strings.TrimSpace(defaultString(auth.Account, account)),
srcID: strings.TrimSpace(auth.Account),
remoteIP: remoteIP(packet.Conn.Conn.RemoteAddr()),
connectedAt: time.Now().UTC(),
conn: packet.Conn,
mu: &sync.Mutex{},
presence: s.PresenceStore,
instanceID: s.gatewayInstanceID(),
}
rememberAccount(session)
go s.flushPending(defaultString(auth.Account, account), logger)
logger.Printf("cmpp inbound account=%s login ok remote=%s", account, packet.Conn.Conn.RemoteAddr())
return false, nil
}
@@ -120,6 +190,19 @@ func (s Server) handleSubmit(response *cmpp.Response, packet *cmpp.Packet, logge
}
resp.MsgId = messageIDFrom(result.MessageID, req.SeqId)
resp.Result = 0
rememberDownstream(downstreamSession{
messageID: result.MessageID,
account: account,
srcID: strings.TrimSpace(req.SrcId),
phoneNumber: phone,
gatewayMsgID: resp.MsgId,
remoteIP: remoteIP(packet.Conn.Conn.RemoteAddr()),
connectedAt: time.Now().UTC(),
conn: packet.Conn,
mu: &sync.Mutex{},
presence: s.PresenceStore,
instanceID: s.gatewayInstanceID(),
})
return false, nil
}
@@ -142,6 +225,82 @@ func (s Server) submit(remote net.Addr, payload submitRequest) (submitResponse,
return result, err
}
type pendingDeliveryRequest struct {
Account string `json:"account"`
Limit int `json:"limit,omitempty"`
}
type pendingDelivery struct {
ID string `json:"id"`
DeliveryType string `json:"deliveryType"`
Payload json.RawMessage `json:"payload"`
}
type pendingFlushResult struct {
Account string
Deliveries int
DeliveredCount int
FailedCount int
WaitingCount int
LastError string
}
func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushResult, error) {
result := pendingFlushResult{Account: account}
if account == "" {
return result, nil
}
var deliveries []pendingDelivery
if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100}, &deliveries); err != nil {
logger.Printf("cmpp inbound pending delivery fetch failed account=%s err=%v", account, err)
result.LastError = err.Error()
return result, err
}
result.Deliveries = len(deliveries)
for _, delivery := range deliveries {
delivered, err := s.pushPendingDelivery(account, delivery)
if err != nil {
result.FailedCount++
result.LastError = err.Error()
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]string{
"id": delivery.ID,
"errorMessage": err.Error(),
}, nil)
continue
}
if delivered {
result.DeliveredCount++
_ = s.post(context.Background(), "/gateway/events/downstream/delivered", map[string]string{"id": delivery.ID}, nil)
continue
}
result.WaitingCount++
}
return result, nil
}
func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (bool, error) {
switch delivery.DeliveryType {
case "receipt":
var event DownstreamReceipt
if err := json.Unmarshal(delivery.Payload, &event); err != nil {
return false, err
}
event.DeliveryID = delivery.ID
event.Account = defaultString(event.Account, account)
return PushReceipt(event)
case "uplink":
var event DownstreamUplink
if err := json.Unmarshal(delivery.Payload, &event); err != nil {
return false, err
}
event.DeliveryID = delivery.ID
event.Account = defaultString(event.Account, account)
return PushUplink(event)
default:
return false, fmt.Errorf("unsupported downstream delivery type %q", delivery.DeliveryType)
}
}
func (s Server) post(ctx context.Context, path string, payload any, result any) error {
client := s.HTTPClient
if client == nil {
@@ -210,3 +369,375 @@ func messageIDFrom(value string, seq uint32) uint64 {
}
return result
}
func rememberDownstream(session downstreamSession) {
if session.messageID == "" || session.conn == nil {
return
}
session.touchPresence("connected", true, false)
downstreamRegistry.Lock()
downstreamRegistry.byMessageID[session.messageID] = &session
if session.account != "" {
downstreamRegistry.byAccount[session.account] = &session
}
downstreamRegistry.Unlock()
}
func rememberAccount(session downstreamSession) {
if session.account == "" || session.conn == nil {
return
}
session.touchPresence("connected", false, false)
downstreamRegistry.Lock()
downstreamRegistry.byAccount[session.account] = &session
downstreamRegistry.Unlock()
}
func forgetDownstream(session *downstreamSession) {
if session == nil {
return
}
downstreamRegistry.Lock()
if session.messageID != "" {
if current := downstreamRegistry.byMessageID[session.messageID]; current == session {
delete(downstreamRegistry.byMessageID, session.messageID)
}
}
if session.account != "" {
if current := downstreamRegistry.byAccount[session.account]; current == session {
delete(downstreamRegistry.byAccount, session.account)
}
}
downstreamRegistry.Unlock()
_ = session.removePresence()
}
func (s Server) runPendingFlusher(logger *log.Logger) {
ticker := time.NewTicker(s.pendingFlushInterval())
defer ticker.Stop()
for range ticker.C {
s.flushOnlineAccounts(logger)
s.recoverPendingCandidates(logger)
}
}
func (s Server) flushOnlineAccounts(logger *log.Logger) {
for _, account := range onlineAccounts() {
_, _ = s.flushPending(account, logger)
}
}
func (s Server) recoverPendingCandidates(logger *log.Logger) {
candidates, err := ListRecoveryCandidates(context.Background(), s.PresenceStore)
if err != nil {
logger.Printf("cmpp inbound recovery candidate refresh failed err=%v", err)
return
}
for _, candidate := range candidates {
account := strings.TrimSpace(candidate.Account)
if account == "" {
continue
}
activeRecovery := DownstreamRecoveryStatus{
Account: account,
GatewayInstanceID: s.gatewayInstanceID(),
}
if s.RecoveryStore != nil {
decision, recoveryErr := s.RecoveryStore.StartAccountRecovery(context.Background(), account, s.gatewayInstanceID())
if recoveryErr != nil {
logger.Printf("cmpp inbound recovery start failed account=%s err=%v", account, recoveryErr)
continue
}
if !decision.Allowed {
logger.Printf("cmpp inbound recovery skipped account=%s reason=%s", account, decision.SkipReason)
decision.Status.Account = account
decision.Status.GatewayInstanceID = s.gatewayInstanceID()
decision.Status.State = defaultString(decision.Status.State, "failed")
decision.Status.LastSkipReason = defaultString(decision.Status.LastSkipReason, decision.SkipReason)
if decision.Status.FailureCategory == "" {
decision.Status.FailureCategory = recoveryFailureCategory(decision.Status.State, "", decision.Status.LastSkipReason)
}
s.syncRecoveryStatus(logger, account, decision.Status)
continue
}
activeRecovery = decision.Status
activeRecovery.Account = account
activeRecovery.GatewayInstanceID = s.gatewayInstanceID()
}
result, flushErr := s.flushPending(account, logger)
if s.RecoveryStore != nil {
status := DownstreamRecoveryStatus{
Account: account,
GatewayInstanceID: s.gatewayInstanceID(),
LockToken: activeRecovery.LockToken,
LockOwner: defaultString(activeRecovery.LockOwner, s.gatewayInstanceID()),
LockAcquiredAt: activeRecovery.LockAcquiredAt,
LockExpiresAt: activeRecovery.LockExpiresAt,
LastAttemptAt: activeRecovery.LastAttemptAt,
}
switch {
case flushErr != nil:
status.State = "failed"
status.LastError = flushErr.Error()
status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason)
case result.WaitingCount > 0 && result.DeliveredCount == 0 && result.FailedCount == 0:
status.State = "waiting_connection"
status.LastError = "downstream client is not connected"
status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason)
case result.FailedCount > 0 && result.DeliveredCount > 0:
status.State = "partial"
status.LastError = result.LastError
status.FailureCategory = recoveryFailureCategory(status.State, status.LastError, status.LastSkipReason)
default:
status.State = "success"
status.LastError = ""
status.FailureCategory = ""
}
if err := s.RecoveryStore.CompleteAccountRecovery(context.Background(), status); err != nil {
logger.Printf("cmpp inbound recovery completion failed account=%s err=%v", account, err)
if err == ErrRecoveryLockLost {
status.State = "failed"
status.LastSkipReason = "lock_lost"
status.FailureCategory = "lock_lost"
s.syncRecoveryStatus(logger, account, status)
}
} else if persisted, err := s.RecoveryStore.GetAccountRecoveryStatus(context.Background(), account); err != nil {
logger.Printf("cmpp inbound recovery status fetch failed account=%s err=%v", account, err)
} else {
s.syncRecoveryStatus(logger, account, persisted)
}
}
}
}
func (s Server) syncRecoveryStatus(logger *log.Logger, account string, status DownstreamRecoveryStatus) {
if err := s.post(context.Background(), "/gateway/events/downstream/recovery-status", map[string]any{
"account": status.Account,
"gatewayInstanceId": status.GatewayInstanceID,
"state": status.State,
"lockOwner": status.LockOwner,
"lockExpiresAt": formatRFC3339Nano(status.LockExpiresAt),
"lastAttemptAt": formatRFC3339Nano(status.LastAttemptAt),
"lastSuccessAt": formatRFC3339Nano(status.LastSuccessAt),
"lastFailureAt": formatRFC3339Nano(status.LastFailureAt),
"nextRetryAt": formatRFC3339Nano(status.NextRetryAt),
"attemptCount": status.AttemptCount,
"failureCategory": status.FailureCategory,
"lastError": status.LastError,
"lastSkipReason": status.LastSkipReason,
}, nil); err != nil {
logger.Printf("cmpp inbound recovery status sync failed account=%s err=%v", account, err)
}
}
func recoveryFailureCategory(state string, lastError string, lastSkipReason string) string {
if state == "success" || state == "running" {
return ""
}
if lastSkipReason == "backoff" {
return "backoff"
}
if lastSkipReason == "locked" {
return "lock_contended"
}
if lastSkipReason == "lock_lost" {
return "lock_lost"
}
if state == "waiting_connection" {
return "client_disconnected"
}
if state == "partial" {
return "partial_delivery_failed"
}
if state == "failed" && strings.TrimSpace(lastError) != "" {
return "flush_failed"
}
return "unknown"
}
func onlineAccounts() []string {
downstreamRegistry.RLock()
defer downstreamRegistry.RUnlock()
accounts := make([]string, 0, len(downstreamRegistry.byAccount))
for account := range downstreamRegistry.byAccount {
if strings.TrimSpace(account) != "" {
accounts = append(accounts, account)
}
}
return accounts
}
func PushReceipt(event DownstreamReceipt) (bool, error) {
session := findSession(event.MessageID, event.Account)
if session == nil {
return false, nil
}
stat := strings.TrimSpace(event.RawStatus)
if stat == "" {
stat = cmppReceiptStatus(event.ReceiptStatus)
}
when := time.Now()
if event.DeliveredAt != "" {
if parsed, err := time.Parse(time.RFC3339Nano, event.DeliveredAt); err == nil {
when = parsed
}
}
receipt := &cmpp.CmppReceiptPkt{
MsgId: session.gatewayMsgID,
Stat: stat,
SubmitTime: when.Format("0601021504"),
DoneTime: when.Format("0601021504"),
DestTerminalId: defaultString(event.PhoneNumber, session.phoneNumber),
SmscSequence: uint32(time.Now().UnixNano() & 0xffffffff),
}
receiptBytes, err := receipt.Pack()
if err != nil {
return false, err
}
deliver := &cmpp.Cmpp3DeliverReqPkt{
MsgId: session.gatewayMsgID,
DestId: session.srcID,
ServiceId: "cmpp",
MsgFmt: 0,
SrcTerminalId: defaultString(event.PhoneNumber, session.phoneNumber),
RegisterDelivery: 1,
MsgLength: uint8(cmpp.CmppReceiptPktLen),
MsgContent: string(receiptBytes),
}
return sendDownstream(session, deliver)
}
func PushUplink(event DownstreamUplink) (bool, error) {
session := findSession(event.MessageID, event.Account)
if session == nil {
return false, nil
}
content, err := cmpputils.Utf8ToUcs2(event.Content)
if err != nil {
return false, err
}
deliver := &cmpp.Cmpp3DeliverReqPkt{
MsgId: messageIDFrom(defaultString(event.MessageID, event.Account), uint32(time.Now().UnixNano())),
DestId: defaultString(event.DestID, session.srcID),
ServiceId: "cmpp",
MsgFmt: 8,
SrcTerminalId: event.PhoneNumber,
RegisterDelivery: 0,
MsgLength: uint8(len(content)),
MsgContent: content,
}
return sendDownstream(session, deliver)
}
func findSession(messageID string, account string) *downstreamSession {
downstreamRegistry.RLock()
defer downstreamRegistry.RUnlock()
if messageID != "" {
if session := downstreamRegistry.byMessageID[messageID]; session != nil {
return session
}
}
if account != "" {
return downstreamRegistry.byAccount[account]
}
return nil
}
func sendDownstream(session *downstreamSession, deliver *cmpp.Cmpp3DeliverReqPkt) (bool, error) {
session.mu.Lock()
defer session.mu.Unlock()
if err := session.conn.SendPkt(deliver, <-session.conn.SeqId); err != nil {
forgetDownstream(session)
return false, err
}
session.touchPresence("connected", false, true)
return true, nil
}
func (s Server) pendingFlushInterval() time.Duration {
if s.PendingFlushInterval > 0 {
return s.PendingFlushInterval
}
return time.Minute
}
func (s Server) logRecoveryCandidates(logger *log.Logger) {
candidates, err := ListRecoveryCandidates(context.Background(), s.PresenceStore)
if err != nil {
logger.Printf("cmpp inbound recovery candidates load failed err=%v", err)
return
}
if len(candidates) == 0 {
logger.Printf("cmpp inbound recovery candidates loaded count=0")
return
}
accounts := make([]string, 0, len(candidates))
for _, item := range candidates {
if strings.TrimSpace(item.Account) != "" {
accounts = append(accounts, item.Account)
}
}
logger.Printf("cmpp inbound recovery candidates loaded count=%d accounts=%s", len(candidates), strings.Join(accounts, ","))
}
func cmppReceiptStatus(status string) string {
switch strings.ToLower(strings.TrimSpace(status)) {
case "delivered":
return "DELIVRD"
case "unknown":
return "UNKNOWN"
default:
return "UNDELIV"
}
}
func defaultString(value string, fallback string) string {
if value == "" {
return fallback
}
return value
}
func formatRFC3339Nano(value time.Time) string {
if value.IsZero() {
return ""
}
return value.UTC().Format(time.RFC3339Nano)
}
func (s Server) gatewayInstanceID() string {
if strings.TrimSpace(s.GatewayInstanceID) != "" {
return strings.TrimSpace(s.GatewayInstanceID)
}
return "gateway-1"
}
func (session downstreamSession) touchPresence(state string, includeSubmit bool, includeDeliver bool) {
if session.presence == nil || strings.TrimSpace(session.account) == "" {
return
}
now := time.Now().UTC()
snapshot := DownstreamPresence{
Account: strings.TrimSpace(session.account),
SrcID: strings.TrimSpace(session.srcID),
RemoteIP: strings.TrimSpace(session.remoteIP),
GatewayInstanceID: strings.TrimSpace(session.instanceID),
State: defaultString(strings.TrimSpace(state), "connected"),
ConnectedAt: session.connectedAt,
UpdatedAt: now,
}
if includeSubmit {
snapshot.LastSubmitAt = now
}
if includeDeliver {
snapshot.LastDeliverAt = now
}
_ = session.presence.TouchAccount(context.Background(), snapshot)
}
func (session downstreamSession) removePresence() error {
if session.presence == nil || strings.TrimSpace(session.account) == "" {
return nil
}
return session.presence.RemoveAccount(context.Background(), strings.TrimSpace(session.account))
}
gateway/internal/inbound/server_test.go
+259
View File
@@ -1,10 +1,13 @@
package inbound
import (
"context"
"encoding/json"
"log"
"net"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
@@ -12,6 +15,63 @@ import (
cmpputils "github.com/bigwhite/gocmpp/utils"
)
type memoryPresenceStore struct {
snapshots map[string]DownstreamPresence
removed []string
}
type memoryRecoveryStore struct {
decisions map[string]RecoveryStartDecision
completed []DownstreamRecoveryStatus
}
func (m *memoryPresenceStore) TouchAccount(_ context.Context, snapshot DownstreamPresence) error {
if m.snapshots == nil {
m.snapshots = map[string]DownstreamPresence{}
}
m.snapshots[snapshot.Account] = snapshot
return nil
}
func (m *memoryPresenceStore) RemoveAccount(_ context.Context, account string) error {
delete(m.snapshots, account)
m.removed = append(m.removed, account)
return nil
}
func (m *memoryPresenceStore) ListAccounts(_ context.Context) ([]DownstreamPresence, error) {
result := make([]DownstreamPresence, 0, len(m.snapshots))
for _, snapshot := range m.snapshots {
result = append(result, snapshot)
}
return result, nil
}
func (m *memoryRecoveryStore) StartAccountRecovery(_ context.Context, account string, _ string) (RecoveryStartDecision, error) {
if decision, ok := m.decisions[account]; ok {
return decision, nil
}
return RecoveryStartDecision{Allowed: true}, nil
}
func (m *memoryRecoveryStore) CompleteAccountRecovery(_ context.Context, status DownstreamRecoveryStatus) error {
m.completed = append(m.completed, status)
return nil
}
func (m *memoryRecoveryStore) ListRecoveryStatuses(_ context.Context) ([]DownstreamRecoveryStatus, error) {
return append([]DownstreamRecoveryStatus(nil), m.completed...), nil
}
func (m *memoryRecoveryStore) GetAccountRecoveryStatus(_ context.Context, account string) (DownstreamRecoveryStatus, error) {
for _, item := range m.completed {
if item.Account == account {
return item, nil
}
}
return DownstreamRecoveryStatus{Account: account}, nil
}
func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
account := "100001"
password := "secret-hash"
@@ -29,6 +89,8 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
t.Fatalf("decode submit: %v", err)
}
_ = json.NewEncoder(w).Encode(submitResponse{Accepted: true, MessageID: "MSG-1"})
case "/api/gateway/events/downstream/pending":
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
default:
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
@@ -76,6 +138,26 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
if rsp.Result != 0 || rsp.MsgId == 0 {
t.Fatalf("unexpected submit response: %+v", rsp)
}
delivered, err := PushReceipt(DownstreamReceipt{
MessageID: "MSG-1",
PhoneNumber: "13500002696",
ReceiptStatus: "delivered",
DeliveredAt: time.Now().UTC().Format(time.RFC3339Nano),
})
if err != nil || !delivered {
t.Fatalf("push receipt delivered=%v err=%v", delivered, err)
}
deliver := recvDeliver(t, client)
if deliver.RegisterDelivery != 1 {
t.Fatalf("expected receipt deliver, got %+v", deliver)
}
var receipt cmpp.CmppReceiptPkt
if err := receipt.Unpack([]byte(deliver.MsgContent)); err != nil {
t.Fatalf("unpack pushed receipt: %v", err)
}
if receipt.Stat != "DELIVRD" || receipt.DestTerminalId != "13500002696" {
t.Fatalf("unexpected pushed receipt: %+v", receipt)
}
if gotAuth.Account != account || gotAuth.AuthSource == "" || gotAuth.RemoteIP == "" {
t.Fatalf("unexpected auth payload: %+v", gotAuth)
}
@@ -84,6 +166,183 @@ func TestInboundServerAuthenticatesAndSubmits(t *testing.T) {
}
}
func TestFlushOnlineAccountsFetchesPendingDeliveries(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
account := "100001"
calls := 0
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/gateway/events/downstream/pending":
calls++
var payload pendingDeliveryRequest
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("decode pending request: %v", err)
}
if payload.Account != account {
t.Fatalf("unexpected account: %+v", payload)
}
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
default:
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
}))
defer api.Close()
downstreamRegistry.Lock()
downstreamRegistry.byAccount[account] = &downstreamSession{account: account}
downstreamRegistry.Unlock()
server := Server{APIBaseURL: api.URL + "/api"}
server.flushOnlineAccounts(log.Default())
if calls != 1 {
t.Fatalf("pending fetch calls = %d, want 1", calls)
}
}
func TestRecoverPendingCandidatesFetchesPresenceAccounts(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
account := "100009"
calls := 0
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/gateway/events/downstream/pending":
calls++
var payload pendingDeliveryRequest
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("decode pending request: %v", err)
}
if payload.Account != account {
t.Fatalf("unexpected account: %+v", payload)
}
_ = json.NewEncoder(w).Encode([]pendingDelivery{})
default:
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
}))
defer api.Close()
store := &memoryPresenceStore{
snapshots: map[string]DownstreamPresence{
account: {
Account: account,
GatewayInstanceID: "gateway-a",
State: "connected",
UpdatedAt: time.Now().UTC(),
},
},
}
server := Server{APIBaseURL: api.URL + "/api", PresenceStore: store}
server.recoverPendingCandidates(log.Default())
if calls != 1 {
t.Fatalf("recovery pending fetch calls = %d, want 1", calls)
}
}
func TestRecoverPendingCandidatesWritesWaitingConnectionStatus(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
account := "100010"
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/gateway/events/downstream/pending":
_ = json.NewEncoder(w).Encode([]pendingDelivery{{
ID: "delivery-1",
DeliveryType: "receipt",
Payload: json.RawMessage(`{"messageId":"MSG-404","phoneNumber":"13800000001","receiptStatus":"delivered"}`),
}})
case "/api/gateway/events/downstream/recovery-status":
w.WriteHeader(http.StatusOK)
default:
t.Fatalf("unexpected api path: %s", r.URL.Path)
}
}))
defer api.Close()
recovery := &memoryRecoveryStore{}
store := &memoryPresenceStore{
snapshots: map[string]DownstreamPresence{
account: {
Account: account,
GatewayInstanceID: "gateway-a",
State: "connected",
UpdatedAt: time.Now().UTC(),
},
},
}
server := Server{APIBaseURL: api.URL + "/api", PresenceStore: store, RecoveryStore: recovery}
server.recoverPendingCandidates(log.Default())
if len(recovery.completed) != 1 {
t.Fatalf("completed recovery statuses = %d, want 1", len(recovery.completed))
}
if recovery.completed[0].State != "waiting_connection" {
t.Fatalf("unexpected recovery status: %+v", recovery.completed[0])
}
}
func TestRememberAndForgetAccountUpdatesPresenceStore(t *testing.T) {
resetDownstreamRegistry()
defer resetDownstreamRegistry()
store := &memoryPresenceStore{}
session := downstreamSession{
account: "100001",
srcID: "10690000",
remoteIP: "127.0.0.1",
connectedAt: time.Now().UTC(),
conn: &cmpp.Conn{},
mu: &sync.Mutex{},
presence: store,
instanceID: "gateway-a",
}
rememberAccount(session)
snapshot, ok := store.snapshots["100001"]
if !ok {
t.Fatal("expected presence snapshot to be stored")
}
if snapshot.Account != "100001" || snapshot.GatewayInstanceID != "gateway-a" || snapshot.State != "connected" {
t.Fatalf("unexpected snapshot: %+v", snapshot)
}
forgetDownstream(downstreamRegistry.byAccount["100001"])
if len(store.removed) != 1 || store.removed[0] != "100001" {
t.Fatalf("unexpected removed accounts: %+v", store.removed)
}
}
func recvDeliver(t *testing.T, client *cmpp.Client) *cmpp.Cmpp3DeliverReqPkt {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
packet, err := client.RecvAndUnpackPkt(200 * time.Millisecond)
if err != nil {
continue
}
if deliver, ok := packet.(*cmpp.Cmpp3DeliverReqPkt); ok {
return deliver
}
}
t.Fatal("timed out waiting deliver request")
return nil
}
func resetDownstreamRegistry() {
downstreamRegistry.Lock()
defer downstreamRegistry.Unlock()
downstreamRegistry.byAccount = make(map[string]*downstreamSession)
downstreamRegistry.byMessageID = make(map[string]*downstreamSession)
}
func reserveTCPAddr(t *testing.T) string {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")