hectorzhao/lislgosms
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: add report material workflows and gateway safeguards

Browse Source
This commit is contained in:
hectorzhao
2026-07-15 18:23:48 +08:00
parent cf9f4ce4cd
commit 7091a8bed4
41 changed files with 3606 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/internal/submitworker/worker.go
+22 -5
View File
@@ -9,9 +9,11 @@ import (
"log"
"net/http"
"strings"
"sync"
"time"
"cmpp-platform/gateway/internal/queue"
"cmpp-platform/gateway/internal/ratelimit"
"cmpp-platform/gateway/internal/upstream"
"github.com/redis/go-redis/v9"
@@ -28,6 +30,7 @@ const (
type Worker struct {
Redis *redis.Client
Upstream *upstream.Manager
Limiter ratelimit.Limiter
Submit func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error)
ReportDeadLetter func(context.Context, DeadLetterEvent) error
Stream string
@@ -64,7 +67,7 @@ func New(redisURL string, manager *upstream.Manager) (*Worker, error) {
if err != nil {
return nil, err
}
return &Worker{Redis: client, Upstream: manager}, nil
return &Worker{Redis: client, Upstream: manager, Limiter: ratelimit.NewWithClient(client)}, nil
}
func (w *Worker) Run(ctx context.Context) error {
@@ -163,12 +166,18 @@ func (w *Worker) recoverPending(ctx context.Context) error {
}
func (w *Worker) processMessages(ctx context.Context, messages []redis.XMessage) error {
var group sync.WaitGroup
for _, message := range messages {
if err := w.processMessage(ctx, message); err != nil {
w.logf("gateway submit worker message %s failed: %v", message.ID, err)
continue
}
message := message
group.Add(1)
go func() {
defer group.Done()
if err := w.processMessage(ctx, message); err != nil {
w.logf("gateway submit worker message %s failed: %v", message.ID, err)
}
}()
}
group.Wait()
return nil
}
@@ -178,6 +187,9 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
return w.deadLetterMalformedMessage(ctx, message, err)
}
if err := w.handleCommand(ctx, command); err != nil {
if ctx.Err() != nil {
return ctx.Err()
}
attempts, attemptsErr := w.incrementFailureAttempt(ctx, message.ID)
if attemptsErr != nil {
w.logf("gateway submit worker increment failure %s failed: %v", message.ID, attemptsErr)
@@ -194,6 +206,11 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
}
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
if w.Limiter != nil {
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
return err
}
}
submit := w.Submit
if submit == nil {
if w.Upstream == nil {
gateway/internal/submitworker/worker_test.go
+81 -2
View File
@@ -3,12 +3,26 @@ package submitworker
import (
"context"
"testing"
"time"
"cmpp-platform/gateway/internal/queue"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
)
type recordingLimiter struct {
channelID string
rate int
called bool
}
func (l *recordingLimiter) Wait(_ context.Context, channelID string, rate int) (time.Duration, error) {
l.called = true
l.channelID = channelID
l.rate = rate
return 0, nil
}
func TestCommandFromStreamValuesParsesSubmitCommand(t *testing.T) {
command, err := CommandFromStreamValues(map[string]interface{}{
"messageType": "SubmitCommand",
@@ -70,20 +84,22 @@ func TestCommandFromStreamValuesRejectsMissingData(t *testing.T) {
func TestHandleMessageUsesInjectedSubmit(t *testing.T) {
var got queue.SubmitCommand
limiter := &recordingLimiter{}
worker := &Worker{
Limiter: limiter,
Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
got = command
return queue.SubmitResult{SubmitStatus: "accepted"}, nil
},
}
if err := worker.handleCommand(context.Background(), queue.SubmitCommand{
Envelope: queue.Envelope{MessageID: "msg-worker-0002"},
Envelope: queue.Envelope{MessageID: "msg-worker-0002", ChannelID: "channel-1"},
SubmitID: "submit-2",
PhoneNumber: "13800138000",
Content: "hello",
Upstream: queue.UpstreamConfig{GatewayHost: "127.0.0.1", GatewayPort: 17890, Account: "account-a", PasswordCipher: "secret", CMPPVersion: "3.0"},
CMPP: queue.CMPP{ServiceID: "SMS", SrcID: "10690000", RegisteredDelivery: 1, MsgFmt: 8},
Route: queue.Route{ChannelCode: "CMPP-A"},
Route: queue.Route{ChannelCode: "CMPP-A", RateLimitPerSecond: 320},
Retry: queue.Retry{Attempt: 0, MaxAttempts: 1},
ApplicationID: "app-1",
TenantID: "tenant-1",
@@ -93,6 +109,9 @@ func TestHandleMessageUsesInjectedSubmit(t *testing.T) {
if got.MessageID != "msg-worker-0002" || got.SubmitID != "submit-2" {
t.Fatalf("unexpected command: %+v", got)
}
if !limiter.called || limiter.channelID != "channel-1" || limiter.rate != 320 {
t.Fatalf("unexpected limiter call: %+v", limiter)
}
}
func TestMinIdleDefaultsToThirtySeconds(t *testing.T) {
@@ -102,6 +121,52 @@ func TestMinIdleDefaultsToThirtySeconds(t *testing.T) {
}
}
func TestProcessMessagesDoesNotLetOneChannelBlockAnother(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
startedA := make(chan struct{})
startedB := make(chan struct{})
releaseA := make(chan struct{})
worker := &Worker{
Redis: client,
Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
switch command.ChannelID {
case "channel-a":
close(startedA)
<-releaseA
case "channel-b":
close(startedB)
}
return queue.SubmitResult{SubmitStatus: "accepted"}, nil
},
}
messages := []redis.XMessage{
{ID: "1-0", Values: submitCommandValues("message-a", "channel-a")},
{ID: "2-0", Values: submitCommandValues("message-b", "channel-b")},
}
done := make(chan struct{})
go func() {
_ = worker.processMessages(context.Background(), messages)
close(done)
}()
select {
case <-startedA:
case <-time.After(time.Second):
t.Fatal("channel-a did not start")
}
select {
case <-startedB:
case <-time.After(200 * time.Millisecond):
t.Fatal("channel-b was blocked by channel-a")
}
close(releaseA)
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("message batch did not complete")
}
}
func TestProcessMessageDeadLettersAfterMaxFailures(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
@@ -178,3 +243,17 @@ func TestProcessMessageDeadLettersAfterMaxFailures(t *testing.T) {
t.Fatalf("failure attempt key was not cleared")
}
}
func submitCommandValues(messageID string, channelID string) map[string]interface{} {
return map[string]interface{}{
"data": `{
"schemaVersion":"v1","messageType":"SubmitCommand","traceId":"trace-1",
"messageId":"` + messageID + `","channelId":"` + channelID + `","submitId":"submit-1",
"tenantId":"tenant-1","applicationId":"app-1","phoneNumber":"13800138000","content":"hello",
"route":{"channelCode":"CMPP-A","rateLimitPerSecond":100},
"cmpp":{"serviceId":"SMS","srcId":"10690000","registeredDelivery":1,"msgFmt":8},
"upstream":{"gatewayHost":"127.0.0.1","gatewayPort":17890,"account":"sp","passwordCipher":"secret","cmppVersion":"3.0"},
"retry":{"attempt":0,"maxAttempts":1}
}`,
}
}