perf: expand gateway capacity and prevent receipt replay
This commit is contained in:
hectorzhao
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -15,6 +16,8 @@ import (
|
||||
type pendingDeliveryRequest struct {
|
||||
Account string `json:"account"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
ClaimID string `json:"claimId"`
|
||||
LeaseMS int `json:"leaseMs"`
|
||||
}
|
||||
|
||||
type pendingDelivery struct {
|
||||
@@ -22,6 +25,7 @@ type pendingDelivery struct {
|
||||
DeliveryType string `json:"deliveryType"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ClaimID string `json:"claimId"`
|
||||
}
|
||||
|
||||
type pendingFlushResult struct {
|
||||
@@ -33,25 +37,96 @@ type pendingFlushResult struct {
|
||||
LastError string
|
||||
}
|
||||
|
||||
type pendingFlushCall struct {
|
||||
done chan struct{}
|
||||
result pendingFlushResult
|
||||
err error
|
||||
}
|
||||
|
||||
var pendingFlushSingleflight = struct {
|
||||
sync.Mutex
|
||||
byAccount map[string]*pendingFlushCall
|
||||
}{byAccount: make(map[string]*pendingFlushCall)}
|
||||
|
||||
type pendingFlushSchedule struct {
|
||||
first time.Time
|
||||
timer *time.Timer
|
||||
}
|
||||
|
||||
var pendingFlushDebouncer = struct {
|
||||
sync.Mutex
|
||||
byAccount map[string]*pendingFlushSchedule
|
||||
}{byAccount: make(map[string]*pendingFlushSchedule)}
|
||||
|
||||
func (s Server) schedulePendingFlush(account string, logger *log.Logger) {
|
||||
account = strings.TrimSpace(account)
|
||||
if account == "" {
|
||||
return
|
||||
}
|
||||
pendingFlushDebouncer.Lock()
|
||||
if scheduled := pendingFlushDebouncer.byAccount[account]; scheduled != nil {
|
||||
if time.Since(scheduled.first) < 250*time.Millisecond {
|
||||
scheduled.timer.Reset(25 * time.Millisecond)
|
||||
}
|
||||
pendingFlushDebouncer.Unlock()
|
||||
return
|
||||
}
|
||||
scheduled := &pendingFlushSchedule{first: time.Now()}
|
||||
scheduled.timer = time.AfterFunc(25*time.Millisecond, func() {
|
||||
pendingFlushDebouncer.Lock()
|
||||
if pendingFlushDebouncer.byAccount[account] == scheduled {
|
||||
delete(pendingFlushDebouncer.byAccount, account)
|
||||
}
|
||||
pendingFlushDebouncer.Unlock()
|
||||
if _, err := s.flushPending(account, logger); err != nil {
|
||||
logger.Printf("cmpp inbound event=scheduled_pending_flush_failed account=%s error=%q", account, err.Error())
|
||||
}
|
||||
})
|
||||
pendingFlushDebouncer.byAccount[account] = scheduled
|
||||
pendingFlushDebouncer.Unlock()
|
||||
}
|
||||
|
||||
func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushResult, error) {
|
||||
account = strings.TrimSpace(account)
|
||||
result := pendingFlushResult{Account: account}
|
||||
if account == "" {
|
||||
return result, nil
|
||||
}
|
||||
pendingFlushSingleflight.Lock()
|
||||
if active := pendingFlushSingleflight.byAccount[account]; active != nil {
|
||||
pendingFlushSingleflight.Unlock()
|
||||
<-active.done
|
||||
return active.result, active.err
|
||||
}
|
||||
call := &pendingFlushCall{done: make(chan struct{})}
|
||||
pendingFlushSingleflight.byAccount[account] = call
|
||||
pendingFlushSingleflight.Unlock()
|
||||
call.result, call.err = s.flushPendingClaimed(account, logger)
|
||||
pendingFlushSingleflight.Lock()
|
||||
delete(pendingFlushSingleflight.byAccount, account)
|
||||
close(call.done)
|
||||
pendingFlushSingleflight.Unlock()
|
||||
return call.result, call.err
|
||||
}
|
||||
|
||||
func (s Server) flushPendingClaimed(account string, logger *log.Logger) (pendingFlushResult, error) {
|
||||
result := pendingFlushResult{Account: account}
|
||||
claimID := fmt.Sprintf("%s:%s:%d", s.gatewayInstanceID(), account, time.Now().UnixNano())
|
||||
var deliveries []pendingDelivery
|
||||
if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100}, &deliveries); err != nil {
|
||||
if err := s.post(context.Background(), "/gateway/events/downstream/pending", pendingDeliveryRequest{Account: account, Limit: 100, ClaimID: claimID, LeaseMS: 30000}, &deliveries); err != nil {
|
||||
logger.Printf("cmpp inbound pending delivery fetch failed account=%s err=%v", account, err)
|
||||
result.LastError = err.Error()
|
||||
return result, err
|
||||
}
|
||||
result.Deliveries = len(deliveries)
|
||||
for _, delivery := range deliveries {
|
||||
delivery.ClaimID = defaultString(delivery.ClaimID, claimID)
|
||||
sendResult, err := s.pushPendingDelivery(account, delivery)
|
||||
if err != nil {
|
||||
result.FailedCount++
|
||||
result.LastError = err.Error()
|
||||
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
|
||||
"id": delivery.ID, "errorMessage": err.Error(), "failureType": "send_failed",
|
||||
"id": delivery.ID, "claimId": delivery.ClaimID, "errorMessage": err.Error(), "failureType": "send_failed",
|
||||
"connectionId": sendResult.ConnectionID, "sequenceId": sendResult.SequenceID,
|
||||
"messageId": sendResult.MessageID, "sentAt": sendResult.SentAt,
|
||||
}, nil)
|
||||
@@ -63,6 +138,7 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
|
||||
}
|
||||
if sendResult.ReasonCode == "SUBMIT_RESPONSE_PENDING" {
|
||||
result.WaitingCount++
|
||||
_ = s.releasePendingClaim(delivery, sendResult)
|
||||
continue
|
||||
}
|
||||
errorMessage := defaultString(sendResult.ErrorMessage, "gateway did not complete downstream delivery")
|
||||
@@ -74,8 +150,12 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
|
||||
result.FailedCount++
|
||||
}
|
||||
result.LastError = errorMessage
|
||||
if sendResult.Retryable {
|
||||
_ = s.releasePendingClaim(delivery, sendResult)
|
||||
continue
|
||||
}
|
||||
_ = s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
|
||||
"id": delivery.ID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode),
|
||||
"id": delivery.ID, "claimId": delivery.ClaimID, "errorMessage": errorMessageWithCode(errorMessage, sendResult.ReasonCode),
|
||||
"failureType": failureType, "connectionId": sendResult.ConnectionID,
|
||||
"sequenceId": sendResult.SequenceID, "messageId": sendResult.MessageID, "sentAt": sendResult.SentAt,
|
||||
}, nil)
|
||||
@@ -83,6 +163,14 @@ func (s Server) flushPending(account string, logger *log.Logger) (pendingFlushRe
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s Server) releasePendingClaim(delivery pendingDelivery, sendResult DownstreamSendResult) error {
|
||||
return s.post(context.Background(), "/gateway/events/downstream/failed", map[string]any{
|
||||
"id": delivery.ID, "claimId": delivery.ClaimID,
|
||||
"errorMessage": errorMessageWithCode(defaultString(sendResult.ErrorMessage, "Gateway released downstream delivery claim"), sendResult.ReasonCode),
|
||||
"failureType": "claim_released",
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (DownstreamSendResult, error) {
|
||||
switch delivery.DeliveryType {
|
||||
case "receipt":
|
||||
@@ -91,6 +179,7 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D
|
||||
return DownstreamSendResult{}, err
|
||||
}
|
||||
event.DeliveryID = delivery.ID
|
||||
event.ClaimID = delivery.ClaimID
|
||||
event.Account = defaultString(event.Account, account)
|
||||
allowRecovery := !delivery.CreatedAt.IsZero() && time.Since(delivery.CreatedAt) >= 5*time.Second
|
||||
return pushReceiptWithResult(event, allowRecovery)
|
||||
@@ -100,6 +189,7 @@ func (s Server) pushPendingDelivery(account string, delivery pendingDelivery) (D
|
||||
return DownstreamSendResult{}, err
|
||||
}
|
||||
event.DeliveryID = delivery.ID
|
||||
event.ClaimID = delivery.ClaimID
|
||||
event.Account = defaultString(event.Account, account)
|
||||
return PushUplinkWithResult(event)
|
||||
default:
|
||||
|
||||
Reference in New Issue
Block a user