perf(cmpp): decouple supplier result callbacks
This commit is contained in:
@@ -43,5 +43,9 @@ GATEWAY_CMPP_ADDR=127.0.0.1:7890
|
|||||||
GATEWAY_CMPP_USER=900001
|
GATEWAY_CMPP_USER=900001
|
||||||
GATEWAY_CMPP_PASSWORD=888888
|
GATEWAY_CMPP_PASSWORD=888888
|
||||||
GATEWAY_SUBMIT_WORKER_CONCURRENCY=64
|
GATEWAY_SUBMIT_WORKER_CONCURRENCY=64
|
||||||
|
GATEWAY_SUBMIT_RESULT_STREAM=gateway.submit.results
|
||||||
|
GATEWAY_SUBMIT_RESULT_GROUP=cmpp-api-callback
|
||||||
|
GATEWAY_SUBMIT_RESULT_CONSUMER=gateway-1
|
||||||
|
GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY=8
|
||||||
GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64
|
GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64
|
||||||
CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30
|
CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
ALTER TABLE "SmsSubmitRecord"
|
||||||
|
ADD COLUMN "resultEventId" TEXT,
|
||||||
|
ADD COLUMN "resultProcessedAt" TIMESTAMP(3);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX "SmsSubmitRecord_resultEventId_key"
|
||||||
|
ON "SmsSubmitRecord"("resultEventId");
|
||||||
@@ -1760,6 +1760,8 @@ model SmsSubmitRecord {
|
|||||||
sequenceId Int?
|
sequenceId Int?
|
||||||
gatewayMessageId String?
|
gatewayMessageId String?
|
||||||
submitStatus String @default("queued")
|
submitStatus String @default("queued")
|
||||||
|
resultEventId String? @unique
|
||||||
|
resultProcessedAt DateTime?
|
||||||
costUnitPrice BigInt @default(0)
|
costUnitPrice BigInt @default(0)
|
||||||
costAmountCents BigInt @default(0)
|
costAmountCents BigInt @default(0)
|
||||||
errorCode String?
|
errorCode String?
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ export interface GatewayInboundSingleSubmitResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface GatewaySubmitResultDto {
|
export interface GatewaySubmitResultDto {
|
||||||
|
eventId?: string;
|
||||||
traceId?: string;
|
traceId?: string;
|
||||||
messageId: string;
|
messageId: string;
|
||||||
channelId: string;
|
channelId: string;
|
||||||
@@ -81,6 +82,7 @@ export interface GatewaySubmitResultDto {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface GatewaySubmitSegmentResultDto {
|
export interface GatewaySubmitSegmentResultDto {
|
||||||
|
eventId?: string;
|
||||||
traceId?: string;
|
traceId?: string;
|
||||||
messageId: string;
|
messageId: string;
|
||||||
channelId: string;
|
channelId: string;
|
||||||
|
|||||||
@@ -2216,6 +2216,46 @@ describe('SendChainService', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('treats a redelivered Outbox aggregate event as idempotent', async () => {
|
||||||
|
const { service, prisma, billing } = createService();
|
||||||
|
const baseSubmit = {
|
||||||
|
id: 'submit-1',
|
||||||
|
messageRecordId: 'record-1',
|
||||||
|
channelId: 'channel-1',
|
||||||
|
submitId: 'SUB-1',
|
||||||
|
submitStatus: 'accepted',
|
||||||
|
};
|
||||||
|
prisma.smsSubmitRecord.findUnique
|
||||||
|
.mockResolvedValueOnce({ ...baseSubmit, resultEventId: null })
|
||||||
|
.mockResolvedValueOnce({ ...baseSubmit, resultEventId: 'submit:SUB-1:aggregate' });
|
||||||
|
const event = {
|
||||||
|
eventId: 'submit:SUB-1:aggregate',
|
||||||
|
messageId: 'MSG-1',
|
||||||
|
channelId: 'channel-1',
|
||||||
|
submitId: 'SUB-1',
|
||||||
|
sequenceId: 7,
|
||||||
|
gatewayMessageId: 'GW-1',
|
||||||
|
submitStatus: 'accepted' as const,
|
||||||
|
submittedAt: '2026-07-01T10:00:00.000Z',
|
||||||
|
};
|
||||||
|
|
||||||
|
await service.handleSubmitResult(event);
|
||||||
|
await service.handleSubmitResult(event);
|
||||||
|
|
||||||
|
expect(billing.charge).toHaveBeenCalledTimes(1);
|
||||||
|
expect(billing.release).toHaveBeenCalledTimes(1);
|
||||||
|
expect(prisma.smsSubmitRecord.updateMany).toHaveBeenCalledWith({
|
||||||
|
where: {
|
||||||
|
id: 'submit-1',
|
||||||
|
OR: [{ resultEventId: null }, { resultEventId: 'submit:SUB-1:aggregate' }],
|
||||||
|
},
|
||||||
|
data: {
|
||||||
|
resultEventId: 'submit:SUB-1:aggregate',
|
||||||
|
resultProcessedAt: new Date('2026-07-01T10:00:00.000Z'),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('rejects a legacy aggregate SubmitResult when it cannot match one submit attempt uniquely', async () => {
|
it('rejects a legacy aggregate SubmitResult when it cannot match one submit attempt uniquely', async () => {
|
||||||
const { service, prisma } = createService();
|
const { service, prisma } = createService();
|
||||||
prisma.smsSubmitRecord.findMany.mockResolvedValue([
|
prisma.smsSubmitRecord.findMany.mockResolvedValue([
|
||||||
|
|||||||
@@ -128,6 +128,12 @@ export class SendGatewayResultService {
|
|||||||
async handleSubmitResult(data: GatewaySubmitResultDto) {
|
async handleSubmitResult(data: GatewaySubmitResultDto) {
|
||||||
const message = await this.facade.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
|
const message = await this.facade.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
|
||||||
const submitRecord = await this.facade.resolveSubmitRecordForGatewayResult(message.id, data);
|
const submitRecord = await this.facade.resolveSubmitRecordForGatewayResult(message.id, data);
|
||||||
|
if (data.eventId && submitRecord.resultEventId) {
|
||||||
|
if (submitRecord.resultEventId !== data.eventId) {
|
||||||
|
throw new BadRequestException('Gateway SubmitResult eventId conflicts with the submit attempt');
|
||||||
|
}
|
||||||
|
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||||
|
}
|
||||||
const effectiveData = { ...data, submitId: submitRecord.submitId };
|
const effectiveData = { ...data, submitId: submitRecord.submitId };
|
||||||
const batchTask = message.batchTaskId
|
const batchTask = message.batchTaskId
|
||||||
? await this.prisma.smsBatchTask.findUnique({ where: { id: message.batchTaskId }, select: { sourceType: true } })
|
? await this.prisma.smsBatchTask.findUnique({ where: { id: message.batchTaskId }, select: { sourceType: true } })
|
||||||
@@ -147,6 +153,7 @@ export class SendGatewayResultService {
|
|||||||
await this.facade.recordSubmitSegments(message, effectiveData, submittedAt);
|
await this.facade.recordSubmitSegments(message, effectiveData, submittedAt);
|
||||||
const isStandaloneChannelTest = !message.tenantId && !message.batchTaskId;
|
const isStandaloneChannelTest = !message.tenantId && !message.batchTaskId;
|
||||||
if (message.submitId && effectiveData.submitId !== message.submitId) {
|
if (message.submitId && effectiveData.submitId !== message.submitId) {
|
||||||
|
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
|
||||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||||
}
|
}
|
||||||
const status = data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed';
|
const status = data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed';
|
||||||
@@ -166,6 +173,7 @@ export class SendGatewayResultService {
|
|||||||
);
|
);
|
||||||
if (retried) {
|
if (retried) {
|
||||||
await this.facade.refreshTaskProgress(businessMessage.batchTaskId);
|
await this.facade.refreshTaskProgress(businessMessage.batchTaskId);
|
||||||
|
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
|
||||||
return retried;
|
return retried;
|
||||||
}
|
}
|
||||||
await this.facade.releaseMessageReservation(businessMessage, data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结');
|
await this.facade.releaseMessageReservation(businessMessage, data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结');
|
||||||
@@ -218,9 +226,29 @@ export class SendGatewayResultService {
|
|||||||
if (message.batchTaskId) {
|
if (message.batchTaskId) {
|
||||||
await this.facade.refreshTaskProgress(message.batchTaskId);
|
await this.facade.refreshTaskProgress(message.batchTaskId);
|
||||||
}
|
}
|
||||||
|
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
|
||||||
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async markSubmitResultProcessed(submitRecordId: string, eventId: string | undefined, processedAt: Date) {
|
||||||
|
if (!eventId) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const updated = await this.prisma.smsSubmitRecord.updateMany({
|
||||||
|
where: {
|
||||||
|
id: submitRecordId,
|
||||||
|
OR: [{ resultEventId: null }, { resultEventId: eventId }],
|
||||||
|
},
|
||||||
|
data: {
|
||||||
|
resultEventId: eventId,
|
||||||
|
resultProcessedAt: processedAt,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (updated.count === 0) {
|
||||||
|
throw new BadRequestException('Gateway SubmitResult eventId conflicts with the submit attempt');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async resolveSubmitRecordForGatewayResult(messageRecordId: string, data: GatewaySubmitResultDto) {
|
async resolveSubmitRecordForGatewayResult(messageRecordId: string, data: GatewaySubmitResultDto) {
|
||||||
if (data.submitId) {
|
if (data.submitId) {
|
||||||
const exact = await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: data.submitId } });
|
const exact = await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: data.submitId } });
|
||||||
|
|||||||
@@ -1153,5 +1153,6 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
|
|||||||
- CMPP性能分段沿用上述边界:业务模块只在原调用边界提交固定阶段名、成功标志和单调时钟耗时;指标模块拒绝未知阶段。`supplier_rtt`在供应商连接调用结束时立即停止,API结果回写另计`api_callback`,防止后续优化依据混杂总耗时。
|
- CMPP性能分段沿用上述边界:业务模块只在原调用边界提交固定阶段名、成功标志和单调时钟耗时;指标模块拒绝未知阶段。`supplier_rtt`在供应商连接调用结束时立即停止,API结果回写另计`api_callback`,防止后续优化依据混杂总耗时。
|
||||||
- V2提交工作池只归`gateway/internal/submitworker/`治理:Redis领取、全局槽位、在途消息ID和逐条ACK不能渗入上游连接池;`gateway/internal/upstream/`继续只负责通道连接、窗口和供应商协议往返。这样Worker吞吐调优不会改写CMPP连接状态机,连接池也不能自行确认Redis消息。
|
- V2提交工作池只归`gateway/internal/submitworker/`治理:Redis领取、全局槽位、在途消息ID和逐条ACK不能渗入上游连接池;`gateway/internal/upstream/`继续只负责通道连接、窗口和供应商协议往返。这样Worker吞吐调优不会改写CMPP连接状态机,连接池也不能自行确认Redis消息。
|
||||||
- V3客户入站窗口只归`gateway/internal/inbound/`与项目内受控的`third_party/gocmpp`服务循环治理:API认证只返回应用窗口,inbound会话负责收紧窗口,协议服务循环负责受限派发和断线等待;不得把客户入站槽位与`submitworker`供应商槽位或`upstream`供应商窗口合并成同一并发计数。
|
- V3客户入站窗口只归`gateway/internal/inbound/`与项目内受控的`third_party/gocmpp`服务循环治理:API认证只返回应用窗口,inbound会话负责收紧窗口,协议服务循环负责受限派发和断线等待;不得把客户入站槽位与`submitworker`供应商槽位或`upstream`供应商窗口合并成同一并发计数。
|
||||||
|
- V4供应商结果异步边界只归`gateway/internal/resultoutbox/`治理:`upstream`在每个真实分片SubmitResp后只调用持久化接口,`submitworker`只负责聚合结果入Outbox与命令ACK的原子边界,Outbox回调Worker独立控制API并发和PEL恢复。API的`SmsSubmitRecord.resultEventId`是跨重启持久幂等事实;不得把HTTP回调重新放回供应商连接池或Submit工作槽,也不得让Outbox承担业务计费、补发或状态机判断。
|
||||||
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
|
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
|
||||||
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
|
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"schemaVersion": "v1",
|
||||||
|
"eventId": "submit:submit-20260820-0001:aggregate",
|
||||||
|
"eventType": "submit_result",
|
||||||
|
"path": "/gateway/events/submit-result",
|
||||||
|
"traceId": "trace-20260820-0001",
|
||||||
|
"messageId": "message-20260820-0001",
|
||||||
|
"channelId": "channel-test-1",
|
||||||
|
"submitId": "submit-20260820-0001",
|
||||||
|
"payload": {
|
||||||
|
"eventId": "submit:submit-20260820-0001:aggregate",
|
||||||
|
"schemaVersion": "v1",
|
||||||
|
"messageType": "SubmitResult",
|
||||||
|
"traceId": "trace-20260820-0001",
|
||||||
|
"messageId": "message-20260820-0001",
|
||||||
|
"channelId": "channel-test-1",
|
||||||
|
"createdAt": "2026-08-20T05:00:00Z",
|
||||||
|
"sequenceId": 1,
|
||||||
|
"gatewayMessageId": "1000000000000001",
|
||||||
|
"submitStatus": "accepted",
|
||||||
|
"submittedAt": "2026-08-20T05:00:00Z"
|
||||||
|
},
|
||||||
|
"createdAt": "2026-08-20T05:00:00Z"
|
||||||
|
}
|
||||||
@@ -6,7 +6,8 @@
|
|||||||
{ "$ref": "#/$defs/SubmitCommand" },
|
{ "$ref": "#/$defs/SubmitCommand" },
|
||||||
{ "$ref": "#/$defs/SubmitResult" },
|
{ "$ref": "#/$defs/SubmitResult" },
|
||||||
{ "$ref": "#/$defs/ReceiptEvent" },
|
{ "$ref": "#/$defs/ReceiptEvent" },
|
||||||
{ "$ref": "#/$defs/UplinkEvent" }
|
{ "$ref": "#/$defs/UplinkEvent" },
|
||||||
|
{ "$ref": "#/$defs/SubmitResultOutboxEvent" }
|
||||||
],
|
],
|
||||||
"$defs": {
|
"$defs": {
|
||||||
"Envelope": {
|
"Envelope": {
|
||||||
@@ -139,6 +140,26 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"SubmitResultOutboxEvent": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["schemaVersion", "eventId", "eventType", "path", "messageId", "channelId", "submitId", "payload", "createdAt"],
|
||||||
|
"properties": {
|
||||||
|
"schemaVersion": { "const": "v1" },
|
||||||
|
"eventId": { "type": "string", "pattern": "^submit:.+:(aggregate|segment:[1-9][0-9]*)$" },
|
||||||
|
"eventType": { "enum": ["submit_result", "submit_segment_result"] },
|
||||||
|
"path": { "enum": ["/gateway/events/submit-result", "/gateway/events/submit-segment-result"] },
|
||||||
|
"traceId": { "type": "string" },
|
||||||
|
"messageId": { "type": "string", "minLength": 1 },
|
||||||
|
"channelId": { "type": "string", "minLength": 1 },
|
||||||
|
"submitId": { "type": "string", "minLength": 1 },
|
||||||
|
"payload": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["eventId"],
|
||||||
|
"properties": { "eventId": { "type": "string", "minLength": 1 } }
|
||||||
|
},
|
||||||
|
"createdAt": { "type": "string", "format": "date-time" }
|
||||||
|
}
|
||||||
|
},
|
||||||
"ReceiptEvent": {
|
"ReceiptEvent": {
|
||||||
"allOf": [
|
"allOf": [
|
||||||
{ "$ref": "#/$defs/Envelope" },
|
{ "$ref": "#/$defs/Envelope" },
|
||||||
|
|||||||
@@ -17,7 +17,7 @@
|
|||||||
{
|
{
|
||||||
"name": "handleSubmitResult",
|
"name": "handleSubmitResult",
|
||||||
"file": "send-gateway-result.service.ts",
|
"file": "send-gateway-result.service.ts",
|
||||||
"bodySha256": "6184170748eb3934496b060ea3b6704f1d1a831e7ffde7e7f8b73a7c223a869c"
|
"bodySha256": "9414f8d4e5e1c3cf6d0747e47ce01e101e142772af4f56900932bc6d4ac11ec7"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "resolveSubmitRecordForGatewayResult",
|
"name": "resolveSubmitRecordForGatewayResult",
|
||||||
|
|||||||
@@ -173,7 +173,7 @@
|
|||||||
"name": "Manager",
|
"name": "Manager",
|
||||||
"kind": "type",
|
"kind": "type",
|
||||||
"file": "manager.go",
|
"file": "manager.go",
|
||||||
"sha256": "c7ead8b037bc87ad9800650ce191f5fcee9648b8805a133b9506743c762ea608"
|
"sha256": "b08726201c34da87ae735abd7fc56d2b904104d4b7f6a6a6365263ec5b0a43bc"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "defaultChannelConnectionID",
|
"name": "defaultChannelConnectionID",
|
||||||
@@ -389,7 +389,7 @@
|
|||||||
"kind": "func",
|
"kind": "func",
|
||||||
"receiver": "Manager",
|
"receiver": "Manager",
|
||||||
"file": "submit.go",
|
"file": "submit.go",
|
||||||
"sha256": "a25d30aeb87c1fe123929330fdabc99261e63d5b51e1cb6207197c43a15de07d"
|
"sha256": "9be92b0d32626ce93d0b2d721bf008305c4b9e5919e287d49b91c6f2474ddeb1"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "submitPart",
|
"name": "submitPart",
|
||||||
@@ -410,7 +410,7 @@
|
|||||||
"kind": "func",
|
"kind": "func",
|
||||||
"receiver": "connectionPool",
|
"receiver": "connectionPool",
|
||||||
"file": "submit.go",
|
"file": "submit.go",
|
||||||
"sha256": "e3dbf2d433ef4ff95b2fe7a8494b17c6a8e185bdace017cce9c7d2e8c785bfb3"
|
"sha256": "577b7f942f4b95ea3868d7cf9aadae800df8edda0e7ca94a205b4124be58e9e3"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "defaultInt",
|
"name": "defaultInt",
|
||||||
|
|||||||
@@ -240,6 +240,7 @@
|
|||||||
9. Gateway 必须按通道连接和窗口容量控制并发,处理窗口满、SMSC 慢响应、sequence 回绕、连接断开时的在途消息状态。
|
9. Gateway 必须按通道连接和窗口容量控制并发,处理窗口满、SMSC 慢响应、sequence 回绕、连接断开时的在途消息状态。
|
||||||
10. Gateway 必须对每个物理通道执行 Redis 分布式 TPS 限速。`SmsChannel.rateLimitPerSecond` 是单通道上限,不是平台总上限;同一通道被多个通道组或多个 Gateway 实例使用时共享同一额度,不同通道独立计数。通道连接命令下发的配置值是最终上限,提交命令携带的值只能进一步降低、不能放大该上限。超流速消息必须继续保留在 Redis Stream pending 中等待可用时隙,不能因等待直接标记发送失败;Gateway 重启后仍可由 consumer group 恢复。worker 必须使用持续补位的全局有界工作池,任一任务完成后立即领取后续消息,不得以“读取10条、等待整批结束”形成批次屏障;每条消息只在自身提交结果已持久回传或完成死信处理后独立`XACK`。全局并发默认64、可由`GATEWAY_SUBMIT_WORKER_CONCURRENCY`配置且上限1024;单通道实际并发仍由 Redis 限速、连接数和 CMPP 窗口共同约束。恢复pending时必须防止超过`MinIdle`的在途消息被同一进程重复提交。
|
10. Gateway 必须对每个物理通道执行 Redis 分布式 TPS 限速。`SmsChannel.rateLimitPerSecond` 是单通道上限,不是平台总上限;同一通道被多个通道组或多个 Gateway 实例使用时共享同一额度,不同通道独立计数。通道连接命令下发的配置值是最终上限,提交命令携带的值只能进一步降低、不能放大该上限。超流速消息必须继续保留在 Redis Stream pending 中等待可用时隙,不能因等待直接标记发送失败;Gateway 重启后仍可由 consumer group 恢复。worker 必须使用持续补位的全局有界工作池,任一任务完成后立即领取后续消息,不得以“读取10条、等待整批结束”形成批次屏障;每条消息只在自身提交结果已持久回传或完成死信处理后独立`XACK`。全局并发默认64、可由`GATEWAY_SUBMIT_WORKER_CONCURRENCY`配置且上限1024;单通道实际并发仍由 Redis 限速、连接数和 CMPP 窗口共同约束。恢复pending时必须防止超过`MinIdle`的在途消息被同一进程重复提交。
|
||||||
11. Gateway 不承担业务审核、计费、签名报备、通道组路由、黑名单或敏感词判断;这些由 NestJS 完成,Gateway 只执行已授权通道提交与协议事件回传。
|
11. Gateway 不承担业务审核、计费、签名报备、通道组路由、黑名单或敏感词判断;这些由 NestJS 完成,Gateway 只执行已授权通道提交与协议事件回传。
|
||||||
|
12. 供应商SubmitResp及长短信分片结果必须先进入共享Redis的幂等Outbox,再由独立有界回调Worker异步上报API;供应商Submit工作槽只等待连接窗口、限速和真实供应商往返,不得等待API结果回写。每个分片结果必须在继续发送下一分片前写入Outbox;聚合结果入Outbox与原`gateway.submit.commands`消息`XACK`必须原子完成。事件ID按submitId和分片序号确定生成,重复发布不得产生第二个事件;API必须按事件ID持久幂等,重复回调不得重复扣费、释放余额、补发或推进状态。回调失败保留在Outbox PEL并可在进程重启后恢复,成功后逐条ACK并删除;去重键和并发必须有界配置,不得形成无限内存或Stream归档。
|
||||||
|
|
||||||
#### 4.8.2 下游客户 CMPP 接入能力
|
#### 4.8.2 下游客户 CMPP 接入能力
|
||||||
|
|
||||||
@@ -2100,6 +2101,7 @@
|
|||||||
- Gateway供应商下发必须分别记录`stream_wait/rate_limit_wait/connection_wait/supplier_rtt/api_callback`;其中`supplier_rtt`只覆盖供应商连接上的Submit请求与SubmitResp往返,不得包含结果回写API的耗时。阶段名和结果值必须使用代码固定白名单,不得添加手机号、企业、应用、通道、消息、任务或连接ID标签。
|
- Gateway供应商下发必须分别记录`stream_wait/rate_limit_wait/connection_wait/supplier_rtt/api_callback`;其中`supplier_rtt`只覆盖供应商连接上的Submit请求与SubmitResp往返,不得包含结果回写API的耗时。阶段名和结果值必须使用代码固定白名单,不得添加手机号、企业、应用、通道、消息、任务或连接ID标签。
|
||||||
- V2有界工作池必须暴露配置槽位数和当前在途槽位数,使用固定`state=configured|in_flight`标签;该指标用于区分Worker容量耗尽与供应商窗口/限速等待,不得增加通道或消息标签。
|
- V2有界工作池必须暴露配置槽位数和当前在途槽位数,使用固定`state=configured|in_flight`标签;该指标用于区分Worker容量耗尽与供应商窗口/限速等待,不得增加通道或消息标签。
|
||||||
- V3入站并发必须只并发Submit业务处理,连接认证保持串行先完成,心跳和Deliver ACK不得被长耗时Submit阻塞;每个SubmitResp继续使用原请求Sequence_Id关联,允许按实际完成顺序返回。同一连接关闭时必须先等待已接受的在途处理收尾,再清理会话和回执映射,避免迟到处理重新注册已断开的连接。`cmpp_gateway_inbound_submit_slots{state=configured|in_flight}`只暴露全部在线连接的聚合窗口与在途数量,不得增加账号、应用、连接或消息标签。
|
- V3入站并发必须只并发Submit业务处理,连接认证保持串行先完成,心跳和Deliver ACK不得被长耗时Submit阻塞;每个SubmitResp继续使用原请求Sequence_Id关联,允许按实际完成顺序返回。同一连接关闭时必须先等待已接受的在途处理收尾,再清理会话和回执映射,避免迟到处理重新注册已断开的连接。`cmpp_gateway_inbound_submit_slots{state=configured|in_flight}`只暴露全部在线连接的聚合窗口与在途数量,不得增加账号、应用、连接或消息标签。
|
||||||
|
- V4结果Outbox必须暴露独立回调Worker的configured/in_flight槽位和Outbox pending/lag,仍只使用固定状态标签。`api_callback`从V4起只在Outbox回调Worker计时,不再混入供应商Submit工作槽;压测结束必须同时核对命令Stream和结果Outbox均`pending=0/lag=0`,并证明API回调故障时供应商槽继续释放、结果不丢失且恢复后只处理一次。
|
||||||
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
|
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
|
||||||
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
|
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
|
||||||
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
|
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
|
||||||
|
|||||||
@@ -61,6 +61,10 @@ CMPP_PUBLIC_HOST=8.160.169.106
|
|||||||
CMPP_PUBLIC_PORT=17890
|
CMPP_PUBLIC_PORT=17890
|
||||||
GATEWAY_STARTUP_RECONNECT_DELAY_MS=1000
|
GATEWAY_STARTUP_RECONNECT_DELAY_MS=1000
|
||||||
GATEWAY_SUBMIT_WORKER_CONCURRENCY=64
|
GATEWAY_SUBMIT_WORKER_CONCURRENCY=64
|
||||||
|
GATEWAY_SUBMIT_RESULT_STREAM=gateway.submit.results
|
||||||
|
GATEWAY_SUBMIT_RESULT_GROUP=cmpp-api-callback
|
||||||
|
GATEWAY_SUBMIT_RESULT_CONSUMER=gateway-1
|
||||||
|
GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY=8
|
||||||
GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64
|
GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64
|
||||||
OBJECT_STORAGE_DRIVER=minio
|
OBJECT_STORAGE_DRIVER=minio
|
||||||
OBJECT_STORAGE_LOCAL_ROOT=/var/lib/cmpp-platform/object-storage
|
OBJECT_STORAGE_LOCAL_ROOT=/var/lib/cmpp-platform/object-storage
|
||||||
@@ -85,6 +89,8 @@ Gateway 的最终 TPS 防线依赖与 API 相同的 Redis。通道连接时会
|
|||||||
|
|
||||||
V3起客户CMPP入站Submit按应用`cmppWindowSize`在单连接内受限并发,全局上限`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`缺省64、最大1024。发布后必须先以已认证测试连接确认`cmpp_gateway_inbound_submit_slots{state="configured"}`等于应用有效窗口,再执行阶梯压测;不得通过调高全局值绕过应用窗口,也不得在未验证Sequence_Id关联、心跳/ACK活性和断线清理时直接提高生产并发。
|
V3起客户CMPP入站Submit按应用`cmppWindowSize`在单连接内受限并发,全局上限`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`缺省64、最大1024。发布后必须先以已认证测试连接确认`cmpp_gateway_inbound_submit_slots{state="configured"}`等于应用有效窗口,再执行阶梯压测;不得通过调高全局值绕过应用窗口,也不得在未验证Sequence_Id关联、心跳/ACK活性和断线清理时直接提高生产并发。
|
||||||
|
|
||||||
|
V4起供应商分片和聚合结果写入`gateway.submit.results`幂等Outbox,由默认8槽、最大1024的独立回调Worker上报API。聚合结果XADD与原命令XACK由Redis Lua原子执行;回调成功后结果事件XACK+XDEL,失败事件留在PEL。发布时必须保留同一Redis数据和AOF,不得清理`gateway.submit.results`、其consumer group或`gateway.submit.results:dedupe:*`;调整`GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY`前须核对API/PostgreSQL承载能力。第91条向前兼容migration增加`SmsSubmitRecord.resultEventId/resultProcessedAt`,用于回调跨重启幂等,不删除历史字段或数据。
|
||||||
|
|
||||||
服务重启顺序必须是 Gateway 在前、API 在后。API 启动后等待 `GATEWAY_STARTUP_RECONNECT_DELAY_MS`(默认 1 秒),从 PostgreSQL 读取全部 active 通道并重新下发真实连接命令,同时恢复 Gateway 内存连接池和 Redis 权威 TPS key;禁止沿用数据库中重启前的 connected 状态冒充当前连接。
|
服务重启顺序必须是 Gateway 在前、API 在后。API 启动后等待 `GATEWAY_STARTUP_RECONNECT_DELAY_MS`(默认 1 秒),从 PostgreSQL 读取全部 active 通道并重新下发真实连接命令,同时恢复 Gateway 内存连接池和 Redis 权威 TPS key;禁止沿用数据库中重启前的 connected 状态冒充当前连接。
|
||||||
|
|
||||||
日报任务默认启用,并由 `REPORT_REFRESH_INTERVAL_MS` 每小时检查一次北京时间业务日是否变化;每个业务日只执行一次 T-4 至 T-1 重算。服务重启后也会自动补跑最近四个完整自然日,确保 72 小时回执更新反映到对账和利润报表。
|
日报任务默认启用,并由 `REPORT_REFRESH_INTERVAL_MS` 每小时检查一次北京时间业务日是否变化;每个业务日只执行一次 T-4 至 T-1 重算。服务重启后也会自动补跑最近四个完整自然日,确保 72 小时回执更新反映到对账和利润报表。
|
||||||
@@ -145,9 +151,10 @@ curl http://127.0.0.1:8090/health
|
|||||||
curl http://127.0.0.1:12026/
|
curl http://127.0.0.1:12026/
|
||||||
redis-cli -h 127.0.0.1 -p 6379 ping
|
redis-cli -h 127.0.0.1 -p 6379 ping
|
||||||
pg_isready -d "$(grep '^DATABASE_URL=' /etc/cmpp-platform/cmpp-platform.env | cut -d= -f2-)"
|
pg_isready -d "$(grep '^DATABASE_URL=' /etc/cmpp-platform/cmpp-platform.env | cut -d= -f2-)"
|
||||||
grep -E '^(API_ENABLE_SEND_WORKER|API_SEND_WORKER_CONCURRENCY|GATEWAY_SUBMIT_WORKER_CONCURRENCY|GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY)=' /etc/cmpp-platform/cmpp-platform.env
|
grep -E '^(API_ENABLE_SEND_WORKER|API_SEND_WORKER_CONCURRENCY|GATEWAY_SUBMIT_WORKER_CONCURRENCY|GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY|GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY)=' /etc/cmpp-platform/cmpp-platform.env
|
||||||
redis-cli --scan --pattern 'rate:gateway:channel:*'
|
redis-cli --scan --pattern 'rate:gateway:channel:*'
|
||||||
redis-cli XINFO GROUPS gateway.submit.commands
|
redis-cli XINFO GROUPS gateway.submit.commands
|
||||||
|
redis-cli XINFO GROUPS gateway.submit.results
|
||||||
```
|
```
|
||||||
|
|
||||||
## 回滚
|
## 回滚
|
||||||
|
|||||||
@@ -4682,6 +4682,14 @@ npm run verify:phase8
|
|||||||
| TC-CMPP-PERF-V3-004 | 断线在途清理 | Submit进入API后由客户端断开TCP,随后让API处理完成 | Gateway等待已接受处理收尾后再执行连接关闭回调;会话、Submit barrier和消息映射最终清理,不重新出现幽灵连接,不发生panic |
|
| TC-CMPP-PERF-V3-004 | 断线在途清理 | Submit进入API后由客户端断开TCP,随后让API处理完成 | Gateway等待已接受处理收尾后再执行连接关闭回调;会话、Submit barrier和消息映射最终清理,不重新出现幽灵连接,不发生panic |
|
||||||
| TC-CMPP-PERF-V3-005 | 入站槽位聚合指标 | 建立不同窗口的测试连接并制造部分在途Submit后抓取metrics | `cmpp_gateway_inbound_submit_slots`的configured等于在线连接有效窗口合计、in_flight等于当前业务处理数;只有固定state标签 |
|
| TC-CMPP-PERF-V3-005 | 入站槽位聚合指标 | 建立不同窗口的测试连接并制造部分在途Submit后抓取metrics | `cmpp_gateway_inbound_submit_slots`的configured等于在线连接有效窗口合计、in_flight等于当前业务处理数;只有固定state标签 |
|
||||||
| TC-CMPP-PERF-V3-006 | V3阶梯容量复测 | 在与V2相同的隔离真实后端/数据库/Redis/供应商模拟器中执行10、20、30、40、50条/秒各60秒 | 与V2按同口径比较SubmitResp P50/P95/P99、API阶段、Stream pending/lag、供应商吞吐和资源;遇P95超过5秒、持续积压或服务异常立即停止,未执行档位不记为通过 |
|
| TC-CMPP-PERF-V3-006 | V3阶梯容量复测 | 在与V2相同的隔离真实后端/数据库/Redis/供应商模拟器中执行10、20、30、40、50条/秒各60秒 | 与V2按同口径比较SubmitResp P50/P95/P99、API阶段、Stream pending/lag、供应商吞吐和资源;遇P95超过5秒、持续积压或服务异常立即停止,未执行档位不记为通过 |
|
||||||
|
| TC-CMPP-PERF-V4-001 | 分片结果幂等入Outbox | 对同一submitId和segmentIndex重复发布两次分片结果 | `gateway.submit.results`只新增一个确定性eventId事件;下一分片仅在前一分片Outbox写入成功后发送 |
|
||||||
|
| TC-CMPP-PERF-V4-002 | 聚合结果与命令ACK原子性 | 让供应商返回成功,在聚合结果写入与命令ACK边界注入Redis失败并重启Gateway | 不存在“命令已ACK但结果Outbox缺失”状态;重试同一发布脚本不会产生重复聚合事件 |
|
||||||
|
| TC-CMPP-PERF-V4-003 | API回调不占供应商槽 | API submit-result接口延迟10秒,同时持续让供应商快速返回SubmitResp | 供应商Worker槽在结果写入Outbox后立即释放;API延迟只增加独立回调Worker和Outbox积压,不降低供应商Submit槽可继续补位的能力 |
|
||||||
|
| TC-CMPP-PERF-V4-004 | 回调失败恢复与逐条ACK | 结果回调第一次返回503,随后恢复201并重启回调Worker | 失败事件保留PEL且未删除;恢复后重新投递,成功时单事件原子ACK+删除,其他事件不受整批等待 |
|
||||||
|
| TC-CMPP-PERF-V4-005 | API持久幂等 | 对同一聚合eventId重复回调两次,并检查提交记录、计费、余额、重试和任务进度 | `SmsSubmitRecord.resultEventId`只记录一次;第二次直接返回当前结果,不重复扣费、释放、补发或推进状态;不同eventId占用同一submit尝试时拒绝 |
|
||||||
|
| TC-CMPP-PERF-V4-006 | Outbox有界指标 | 制造回调在途和积压后抓取Gateway metrics | 回调Worker configured/in_flight与真实槽位一致,Outbox pending/lag与Redis consumer group一致,指标不含手机号、消息、submit、企业、应用或通道标签 |
|
||||||
|
| TC-CMPP-PERF-V4-007 | 双Stream发布后排空 | 完成一档隔离压测并等待异步处理结束 | `gateway.submit.commands`和`gateway.submit.results`均为`pending=0/lag=0`,结果Outbox成功事件已删除,无Gateway Submit死信;数据库业务数与客户端完全一致 |
|
||||||
|
| TC-CMPP-PERF-V4-008 | V4阶梯容量复测 | 在V3相同隔离供应商模拟器与真实API/PostgreSQL/Redis/Gateway中依次执行10、20、30、40、50条/秒各60秒 | 对比V3的SubmitResp分位、供应商RTT、命令Stream、结果Outbox、API阶段和资源;任一档出现拒绝、连接错误、P95超过5秒、双Stream持续增长或服务异常立即停止升档 |
|
||||||
| TC-GLOBAL-ALERT-001 | 铃铛分域预警菜单 | 准备签名清退未读消息和安全待处置告警后点击右上角铃铛 | 弹层分开显示“签名清退预警”和“安全检测与封禁”,分别展示真实数量和摘要,角标等于两项之和 |
|
| TC-GLOBAL-ALERT-001 | 铃铛分域预警菜单 | 准备签名清退未读消息和安全待处置告警后点击右上角铃铛 | 弹层分开显示“签名清退预警”和“安全检测与封禁”,分别展示真实数量和摘要,角标等于两项之和 |
|
||||||
| TC-GLOBAL-ALERT-002 | 预警菜单跳转 | 分别点击铃铛中的两个菜单项 | 签名项跳转`/admin/signature-retirement`,安全项跳转`/admin/security-detection`,弹层关闭且对应页面读取真实后端数据 |
|
| TC-GLOBAL-ALERT-002 | 预警菜单跳转 | 分别点击铃铛中的两个菜单项 | 签名项跳转`/admin/signature-retirement`,安全项跳转`/admin/security-detection`,弹层关闭且对应页面读取真实后端数据 |
|
||||||
| TC-GLOBAL-ALERT-003 | 域间故障隔离与轻量轮询 | 分别让一个汇总接口失败并观察30秒轮询请求 | 失败域显示0且另一域数据保留;安全预警使用专用汇总接口,不调用完整overview、规则、代理状态或告警大列表 |
|
| TC-GLOBAL-ALERT-003 | 域间故障隔离与轻量轮询 | 分别让一个汇总接口失败并观察30秒轮询请求 | 失败域显示0且另一域数据保留;安全预警使用专用汇总接口,不调用完整overview、规则、代理状态或告警大列表 |
|
||||||
|
|||||||
@@ -3747,3 +3747,17 @@ git diff --check
|
|||||||
- 优先级隔离仍未通过:40条/秒priority P95约3912ms、normal P95约3911ms,且两次SubmitResp拒绝都发生在priority连接;全部5996条均为mobile,联通/电信及六通道容量仍待号段识别修复后测试。原始目录名继续沿用既有`lg-v2-*`脚本标签,但被测运行标识和代码均为V3修复版,正式结论以运行标识为准。
|
- 优先级隔离仍未通过:40条/秒priority P95约3912ms、normal P95约3911ms,且两次SubmitResp拒绝都发生在priority连接;全部5996条均为mobile,联通/电信及六通道容量仍待号段识别修复后测试。原始目录名继续沿用既有`lg-v2-*`脚本标签,但被测运行标识和代码均为V3修复版,正式结论以运行标识为准。
|
||||||
- V3回归已通过Gateway全量`go test ./... -count=1`、`go vet ./...`、项目内gocmpp测试/vet、API TypeScript正式编译、真实隔离Redis上的SendChain 113/113、4份Stream契约、R6 102声明/14项关键测试、R7、SendChain R10及`git diff --check`。Linux测试机补跑`-race`时因网络无法下载仅供测试的`miniredis`和`golang.org/x/text`依赖而阻塞,未伪报通过,也未伪造外部依赖。
|
- V3回归已通过Gateway全量`go test ./... -count=1`、`go vet ./...`、项目内gocmpp测试/vet、API TypeScript正式编译、真实隔离Redis上的SendChain 113/113、4份Stream契约、R6 102声明/14项关键测试、R7、SendChain R10及`git diff --check`。Linux测试机补跑`-race`时因网络无法下载仅供测试的`miniredis`和`golang.org/x/text`依赖而阻塞,未伪报通过,也未伪造外部依赖。
|
||||||
- 本轮只使用隔离供应商模拟器,没有发送、补发或重投真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接。`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续作为受保护项排除提交。
|
- 本轮只使用隔离供应商模拟器,没有发送、补发或重投真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接。`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续作为受保护项排除提交。
|
||||||
|
|
||||||
|
# 2026-08-20 CMPP压测优化V4:供应商结果幂等Outbox、测试环境发布与阶梯复测
|
||||||
|
|
||||||
|
- V4将供应商分片结果和聚合结果先写入Redis Stream `gateway.submit.results`,API回调由独立持续有界工作池执行,供应商Submit工作槽不再等待API往返。分片事件ID为`submit:<submitId>:segment:<index>`,聚合事件ID为`submit:<submitId>:aggregate`;Lua脚本保证分片去重键与XADD原子、聚合XADD与原命令XACK原子、成功回调XACK与XDEL原子。失败事件保留在PEL并由`XAUTOCLAIM`恢复,回收阈值30秒高于10秒HTTP超时,避免多Gateway实例在回调仍执行时并发重领。
|
||||||
|
- 第91条向前兼容migration `20260820130000_add_submit_result_idempotency`为`SmsSubmitRecord`增加唯一可空`resultEventId`和`resultProcessedAt`。API重复收到同一已完成事件直接返回当前消息,冲突事件ID拒绝;测试覆盖重复回调不重复计费、重试或下游业务副作用。新增第5类Gateway队列契约示例,Outbox指标覆盖回调Worker槽位和结果Stream pending/lag。
|
||||||
|
- 本地验证通过:Gateway全量`go test ./... -count=1`及`go vet ./...`,API 42套/483项断言全部通过,API与前端TypeScript正式编译、Vite生产构建、5份队列契约、R0/R6/R7及SendChain R10结构门禁、`git diff --check`均通过。全量Jest在断言完成后仍因仓库既有异步句柄不自行退出,本轮未把人工终止后的进程伪报为完整退出码0;使用本机真实Redis补跑,不伪造外部依赖。
|
||||||
|
- 仅发布到`100.93.204.60`虚拟机测试环境;预生产`8.160.169.106`只读复核标识保持`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`,未发布、未回退、未压测。测试环境发布前恢复资产位于`/opt/cmpp-platform-backups/v4-20260820T052241Z`,包含PostgreSQL自定义格式备份、运行源码、环境和systemd配置;`pg_restore --list`、源码tar可读性及`SHA256SUMS`全部通过。
|
||||||
|
- 最终运行包`outputs/cmpp-v4-runtime-final-20260820-141323.tar.gz`共827项、1641927字节,本地与测试机SHA-256均为`a63885439ed53d53a3a012048d5fd1a9aa67772503fd778ec510ec6a38a9944f`,排除依赖、构建产物、`outputs`、`*.tsbuildinfo`、`pnpm-lock.yaml`和空文件`=`。测试环境运行标识为`485af688d21ee0bdb98281f4c20d151d69f7889f+workspace.v4.a63885439ed5`;第91条migration仅应用一次,API、Gateway、安全代理、MinIO、PostgreSQL和Redis健康,最终供应商连接6/6,两条Stream均`pending=0/lag=0`。
|
||||||
|
- 首轮默认32槽10条/秒恰逢重启积压回执回放,599/599受理但回执2536、P95=3294ms;积压排空后再测仍为P95=671ms。把测试配置收敛为8槽后,10条/秒599/599、P50/P95/P99=`39/146/304ms`、回执603。基于该对照,代码、示例环境和发布文档的默认值同步改为8;32槽不作为推荐配置。
|
||||||
|
- 8槽正式阶梯结果:20条/秒1199/1199受理、P50/P95/P99=`45/1129/1259ms`;30条/秒1799/1799受理、`1081/1626/1885ms`;40条/秒2399/2399受理、`2049/2330/2386ms`。三档均无连接错误;相对V3,20/30/40条/秒P95分别由1427/2033/3912ms降至1129/1626/2330ms,且40条/秒从2条10秒超时改进为零拒绝,确认安全档位由30提升到40条/秒。
|
||||||
|
- 解耦证据:20条/秒命令Stream未投递lag峰值0,结果Outbox峰值`pending=8/lag=242`后归零;30条/秒峰值约`pending=8/lag=1076`;40条/秒峰值约`pending=9/lag=1639`,压测结束后约36秒归零并连续保持0/0。40条/秒时供应商命令与结果回调已分开排队,结果回调积压不再占用供应商Submit槽;但Outbox回落时间已成为容量判定的一部分,不能只看客户SubmitResp。
|
||||||
|
- 50条/秒触发停止线:因客户端背压只生成2790条而非约2999条,2787条成功、3条等待API满10秒后返回result=9,P50/P95/P99=`6765/7236/7789ms`,throttled ticks=3184。结束后命令Stream一度`pending=64/lag=489`、结果Outbox`pending=8/lag=1690`;命令约1分钟内、结果随后约20秒内归零。日志显示API饱和时协议遥测和连接状态回调超时,并暴露既有`CmppDownstreamConnection`创建/更新的P2002/P2025竞态;未出现`resultEventId`唯一键冲突或Outbox事件失败。停止后未继续上探。
|
||||||
|
- 整个窗口客户侧共9984条业务提交,真实PostgreSQL按`queuedAt`精确新增9984条;窗口内7922条实际供应商提交记录全部具有非空且互不重复的`resultEventId`,重复事件组0、Gateway Submit死信0。提交结果为accepted 7392、rejected 147、timeout 383;客户有限回执收集窗口和重连积压回执不替代数据库/Stream对账。最终确认40条/秒为当前测试环境安全档,50条/秒瓶颈转为API/数据库同步入站及遥测争用,后续应继续V1的重复查询/零散写入合并和P1分阶段指标分析。
|
||||||
|
- 本轮只连接隔离供应商模拟器`100.91.249.119:17900`,没有发送、补发或重投真实短信,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。压测原始目录沿用既有`lg-v2-*`名称,但被测运行标识与结论均为V4;完整V4报告保存在短信平台测试项目。受保护的`api/tsconfig.build.tsbuildinfo`、`tsconfig.tsbuildinfo`、`outputs/`、`pnpm-lock.yaml`和空文件`=`继续排除提交、不删除、不归因。
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
- 暴露健康检查和最小指标。
|
- 暴露健康检查和最小指标。
|
||||||
- Redis Stream Submit Worker 使用持续补位的有界工作池并逐条 ACK;默认并发64,可用`GATEWAY_SUBMIT_WORKER_CONCURRENCY`调整,最大1024。供应商通道的真实上限仍由TPS限速、连接数和CMPP窗口共同决定。
|
- Redis Stream Submit Worker 使用持续补位的有界工作池并逐条 ACK;默认并发64,可用`GATEWAY_SUBMIT_WORKER_CONCURRENCY`调整,最大1024。供应商通道的真实上限仍由TPS限速、连接数和CMPP窗口共同决定。
|
||||||
- 客户CMPP入站Submit按认证接口返回的应用`cmppWindowSize`在单连接内并发,Gateway再以`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`实施默认64、最大1024的全局单连接保护。登录保持串行,心跳和Deliver ACK不等待慢Submit;SubmitResp依靠Sequence_Id关联,允许按完成顺序返回。
|
- 客户CMPP入站Submit按认证接口返回的应用`cmppWindowSize`在单连接内并发,Gateway再以`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`实施默认64、最大1024的全局单连接保护。登录保持串行,心跳和Deliver ACK不等待慢Submit;SubmitResp依靠Sequence_Id关联,允许按完成顺序返回。
|
||||||
|
- 供应商SubmitResp先写入Redis Stream幂等Outbox`gateway.submit.results`,聚合结果入Outbox与原Submit命令ACK使用同一Lua脚本;独立默认8槽回调Worker再调用API,供应商工作槽不等待API。每个事件使用确定性`eventId`和7天Redis去重键,API在`SmsSubmitRecord.resultEventId`完成持久幂等;成功回调后Outbox事件原子ACK+删除,失败事件留在PEL恢复。
|
||||||
|
|
||||||
## 建议骨架
|
## 建议骨架
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ import (
|
|||||||
"cmpp-platform/gateway/internal/health"
|
"cmpp-platform/gateway/internal/health"
|
||||||
"cmpp-platform/gateway/internal/inbound"
|
"cmpp-platform/gateway/internal/inbound"
|
||||||
platformmetrics "cmpp-platform/gateway/internal/metrics"
|
platformmetrics "cmpp-platform/gateway/internal/metrics"
|
||||||
|
"cmpp-platform/gateway/internal/queue"
|
||||||
"cmpp-platform/gateway/internal/ratelimit"
|
"cmpp-platform/gateway/internal/ratelimit"
|
||||||
|
"cmpp-platform/gateway/internal/resultoutbox"
|
||||||
"cmpp-platform/gateway/internal/submitworker"
|
"cmpp-platform/gateway/internal/submitworker"
|
||||||
"cmpp-platform/gateway/internal/upstream"
|
"cmpp-platform/gateway/internal/upstream"
|
||||||
|
|
||||||
@@ -32,6 +34,7 @@ func main() {
|
|||||||
apiBaseURL := os.Getenv("API_BASE_URL")
|
apiBaseURL := os.Getenv("API_BASE_URL")
|
||||||
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
|
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
|
||||||
var worker *submitworker.Worker
|
var worker *submitworker.Worker
|
||||||
|
var resultOutbox *resultoutbox.Outbox
|
||||||
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
|
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("gateway channel rate limiter init failed: %v", err)
|
log.Fatalf("gateway channel rate limiter init failed: %v", err)
|
||||||
@@ -70,12 +73,26 @@ func main() {
|
|||||||
worker.Consumer = getenv("GATEWAY_SUBMIT_CONSUMER", "gateway-1")
|
worker.Consumer = getenv("GATEWAY_SUBMIT_CONSUMER", "gateway-1")
|
||||||
worker.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_WORKER_CONCURRENCY", 64)
|
worker.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_WORKER_CONCURRENCY", 64)
|
||||||
worker.APIBaseURL = apiBaseURL
|
worker.APIBaseURL = apiBaseURL
|
||||||
|
resultOutbox = resultoutbox.New(worker.Redis)
|
||||||
|
resultOutbox.Stream = getenv("GATEWAY_SUBMIT_RESULT_STREAM", "gateway.submit.results")
|
||||||
|
resultOutbox.Group = getenv("GATEWAY_SUBMIT_RESULT_GROUP", "cmpp-api-callback")
|
||||||
|
resultOutbox.Consumer = getenv("GATEWAY_SUBMIT_RESULT_CONSUMER", "gateway-1")
|
||||||
|
resultOutbox.APIBaseURL = apiBaseURL
|
||||||
|
resultOutbox.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY", 8)
|
||||||
|
worker.ResultOutbox = resultOutbox
|
||||||
|
upstreamManager.SubmitSegmentPublisher = resultOutbox
|
||||||
go func() {
|
go func() {
|
||||||
log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer)
|
log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer)
|
||||||
if err := worker.Run(context.Background()); err != nil {
|
if err := worker.Run(context.Background()); err != nil {
|
||||||
log.Printf("gateway submit worker stopped: %v", err)
|
log.Printf("gateway submit worker stopped: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
go func() {
|
||||||
|
log.Printf("cmpp gateway result Outbox consuming stream=%s group=%s consumer=%s", resultOutbox.StreamName(), resultOutbox.GroupName(), resultOutbox.Consumer)
|
||||||
|
if err := resultOutbox.Run(context.Background()); err != nil {
|
||||||
|
log.Printf("gateway result Outbox worker stopped: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,6 +108,11 @@ func main() {
|
|||||||
snapshot.SubmitWorkerConcurrency = worker.ConfiguredConcurrency()
|
snapshot.SubmitWorkerConcurrency = worker.ConfiguredConcurrency()
|
||||||
snapshot.SubmitWorkerInFlight = worker.InFlight()
|
snapshot.SubmitWorkerInFlight = worker.InFlight()
|
||||||
}
|
}
|
||||||
|
if resultOutbox != nil {
|
||||||
|
snapshot.ResultWorkerUp = true
|
||||||
|
snapshot.ResultWorkerConcurrency = resultOutbox.ConfiguredConcurrency()
|
||||||
|
snapshot.ResultWorkerInFlight = resultOutbox.InFlight()
|
||||||
|
}
|
||||||
snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight = inbound.SubmitSlotSnapshot()
|
snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight = inbound.SubmitSlotSnapshot()
|
||||||
if worker == nil || worker.Redis == nil {
|
if worker == nil || worker.Redis == nil {
|
||||||
return snapshot
|
return snapshot
|
||||||
@@ -117,12 +139,38 @@ func main() {
|
|||||||
snapshot.QueueOldestAgeSeconds = max(0, time.Since(time.UnixMilli(milliseconds)).Seconds())
|
snapshot.QueueOldestAgeSeconds = max(0, time.Since(time.UnixMilli(milliseconds)).Seconds())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if resultOutbox != nil {
|
||||||
|
resultPending, resultErr := worker.Redis.XPending(ctx, resultOutbox.StreamName(), resultOutbox.GroupName()).Result()
|
||||||
|
if resultErr == nil {
|
||||||
|
snapshot.ResultQueueAvailable = true
|
||||||
|
snapshot.ResultQueuePending = resultPending.Count
|
||||||
|
}
|
||||||
|
resultGroups, resultErr := worker.Redis.XInfoGroups(ctx, resultOutbox.StreamName()).Result()
|
||||||
|
if resultErr == nil {
|
||||||
|
for _, group := range resultGroups {
|
||||||
|
if group.Name == resultOutbox.GroupName() {
|
||||||
|
snapshot.ResultQueueLag = group.Lag
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
return snapshot
|
return snapshot
|
||||||
}))
|
}))
|
||||||
control.Register(mux, control.Server{
|
control.Register(mux, control.Server{
|
||||||
APIBaseURL: apiBaseURL,
|
APIBaseURL: apiBaseURL,
|
||||||
Upstream: upstreamManager,
|
Upstream: upstreamManager,
|
||||||
Limiter: channelLimiter,
|
Limiter: channelLimiter,
|
||||||
|
Submit: func(ctx context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||||
|
result, submitErr := upstreamManager.Submit(ctx, command)
|
||||||
|
if resultOutbox == nil {
|
||||||
|
return result, submitErr
|
||||||
|
}
|
||||||
|
if publishErr := resultOutbox.PublishSubmitResult(ctx, command, result); publishErr != nil {
|
||||||
|
return result, publishErr
|
||||||
|
}
|
||||||
|
return result, submitErr
|
||||||
|
},
|
||||||
RecoveryCandidates: func(ctx context.Context) ([]inbound.DownstreamPresence, error) {
|
RecoveryCandidates: func(ctx context.Context) ([]inbound.DownstreamPresence, error) {
|
||||||
return inbound.ListRecoveryCandidates(ctx, presenceStore)
|
return inbound.ListRecoveryCandidates(ctx, presenceStore)
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -43,12 +43,18 @@ type Snapshot struct {
|
|||||||
SubmitWorkerUp bool
|
SubmitWorkerUp bool
|
||||||
SubmitWorkerConcurrency int
|
SubmitWorkerConcurrency int
|
||||||
SubmitWorkerInFlight int64
|
SubmitWorkerInFlight int64
|
||||||
|
ResultWorkerUp bool
|
||||||
|
ResultWorkerConcurrency int
|
||||||
|
ResultWorkerInFlight int64
|
||||||
InboundSubmitConcurrency int
|
InboundSubmitConcurrency int
|
||||||
InboundSubmitInFlight int64
|
InboundSubmitInFlight int64
|
||||||
QueueAvailable bool
|
QueueAvailable bool
|
||||||
QueuePending int64
|
QueuePending int64
|
||||||
QueueLag int64
|
QueueLag int64
|
||||||
QueueOldestAgeSeconds float64
|
QueueOldestAgeSeconds float64
|
||||||
|
ResultQueueAvailable bool
|
||||||
|
ResultQueuePending int64
|
||||||
|
ResultQueueLag int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type SnapshotFunc func(context.Context) Snapshot
|
type SnapshotFunc func(context.Context) Snapshot
|
||||||
@@ -118,12 +124,18 @@ func Handler(load SnapshotFunc) http.Handler {
|
|||||||
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
|
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
|
||||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
|
||||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight)
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_result_callback_worker_up Whether the asynchronous result callback worker was initialized.\n# TYPE cmpp_gateway_result_callback_worker_up gauge\ncmpp_gateway_result_callback_worker_up %d\n", boolNumber(snapshot.ResultWorkerUp))
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_result_callback_worker_slots Configured and active result callback worker slots.\n# TYPE cmpp_gateway_result_callback_worker_slots gauge\ncmpp_gateway_result_callback_worker_slots{state=\"configured\"} %d\ncmpp_gateway_result_callback_worker_slots{state=\"in_flight\"} %d\n", snapshot.ResultWorkerConcurrency, snapshot.ResultWorkerInFlight)
|
||||||
fmt.Fprintf(response, "# HELP cmpp_gateway_inbound_submit_slots Configured and active authenticated client Submit slots.\n# TYPE cmpp_gateway_inbound_submit_slots gauge\ncmpp_gateway_inbound_submit_slots{state=\"configured\"} %d\ncmpp_gateway_inbound_submit_slots{state=\"in_flight\"} %d\n", snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight)
|
fmt.Fprintf(response, "# HELP cmpp_gateway_inbound_submit_slots Configured and active authenticated client Submit slots.\n# TYPE cmpp_gateway_inbound_submit_slots gauge\ncmpp_gateway_inbound_submit_slots{state=\"configured\"} %d\ncmpp_gateway_inbound_submit_slots{state=\"in_flight\"} %d\n", snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight)
|
||||||
if snapshot.QueueAvailable {
|
if snapshot.QueueAvailable {
|
||||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending)
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending)
|
||||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag)
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag)
|
||||||
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_oldest_pending_age_seconds Age of the oldest pending entry.\n# TYPE cmpp_gateway_submit_queue_oldest_pending_age_seconds gauge\ncmpp_gateway_submit_queue_oldest_pending_age_seconds %f\n", snapshot.QueueOldestAgeSeconds)
|
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_oldest_pending_age_seconds Age of the oldest pending entry.\n# TYPE cmpp_gateway_submit_queue_oldest_pending_age_seconds gauge\ncmpp_gateway_submit_queue_oldest_pending_age_seconds %f\n", snapshot.QueueOldestAgeSeconds)
|
||||||
}
|
}
|
||||||
|
if snapshot.ResultQueueAvailable {
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_result_outbox_pending Pending result callbacks owned by the consumer group.\n# TYPE cmpp_gateway_result_outbox_pending gauge\ncmpp_gateway_result_outbox_pending %d\n", snapshot.ResultQueuePending)
|
||||||
|
fmt.Fprintf(response, "# HELP cmpp_gateway_result_outbox_lag Undelivered result callbacks for the consumer group.\n# TYPE cmpp_gateway_result_outbox_lag gauge\ncmpp_gateway_result_outbox_lag %d\n", snapshot.ResultQueueLag)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
|
|||||||
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
Handler(func(context.Context) Snapshot {
|
Handler(func(context.Context) Snapshot {
|
||||||
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, SubmitWorkerConcurrency: 64, SubmitWorkerInFlight: 7, InboundSubmitConcurrency: 96, InboundSubmitInFlight: 9, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12}
|
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, SubmitWorkerConcurrency: 64, SubmitWorkerInFlight: 7, ResultWorkerUp: true, ResultWorkerConcurrency: 32, ResultWorkerInFlight: 5, InboundSubmitConcurrency: 96, InboundSubmitInFlight: 9, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12, ResultQueueAvailable: true, ResultQueuePending: 6, ResultQueueLag: 8}
|
||||||
}).ServeHTTP(response, request)
|
}).ServeHTTP(response, request)
|
||||||
|
|
||||||
body := response.Body.String()
|
body := response.Body.String()
|
||||||
@@ -28,6 +28,10 @@ func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
|
|||||||
`cmpp_gateway_submit_stage_duration_seconds_count{stage="rate_limit_wait",result="success"} 1`,
|
`cmpp_gateway_submit_stage_duration_seconds_count{stage="rate_limit_wait",result="success"} 1`,
|
||||||
`cmpp_gateway_submit_worker_slots{state="configured"} 64`,
|
`cmpp_gateway_submit_worker_slots{state="configured"} 64`,
|
||||||
`cmpp_gateway_submit_worker_slots{state="in_flight"} 7`,
|
`cmpp_gateway_submit_worker_slots{state="in_flight"} 7`,
|
||||||
|
`cmpp_gateway_result_callback_worker_slots{state="configured"} 32`,
|
||||||
|
`cmpp_gateway_result_callback_worker_slots{state="in_flight"} 5`,
|
||||||
|
`cmpp_gateway_result_outbox_pending 6`,
|
||||||
|
`cmpp_gateway_result_outbox_lag 8`,
|
||||||
`cmpp_gateway_inbound_submit_slots{state="configured"} 96`,
|
`cmpp_gateway_inbound_submit_slots{state="configured"} 96`,
|
||||||
`cmpp_gateway_inbound_submit_slots{state="in_flight"} 9`,
|
`cmpp_gateway_inbound_submit_slots{state="in_flight"} 9`,
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -0,0 +1,260 @@
|
|||||||
|
package resultoutbox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"cmpp-platform/gateway/internal/queue"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultStream = "gateway.submit.results"
|
||||||
|
defaultGroup = "cmpp-api-callback"
|
||||||
|
defaultConsumer = "gateway-1"
|
||||||
|
defaultDedupeTTL = 7 * 24 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
var publishScript = redis.NewScript(`
|
||||||
|
local inserted = redis.call('SET', KEYS[2], '1', 'NX', 'EX', ARGV[1])
|
||||||
|
if inserted then
|
||||||
|
redis.call('XADD', KEYS[1], '*', 'data', ARGV[2])
|
||||||
|
return 1
|
||||||
|
end
|
||||||
|
return 0
|
||||||
|
`)
|
||||||
|
|
||||||
|
var publishAndAckScript = redis.NewScript(`
|
||||||
|
local inserted = redis.call('SET', KEYS[2], '1', 'NX', 'EX', ARGV[1])
|
||||||
|
if inserted then
|
||||||
|
redis.call('XADD', KEYS[1], '*', 'data', ARGV[2])
|
||||||
|
end
|
||||||
|
redis.call('XACK', KEYS[3], ARGV[3], ARGV[4])
|
||||||
|
if inserted then return 1 end
|
||||||
|
return 0
|
||||||
|
`)
|
||||||
|
|
||||||
|
type Event struct {
|
||||||
|
SchemaVersion string `json:"schemaVersion"`
|
||||||
|
EventID string `json:"eventId"`
|
||||||
|
EventType string `json:"eventType"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
TraceID string `json:"traceId,omitempty"`
|
||||||
|
MessageID string `json:"messageId"`
|
||||||
|
ChannelID string `json:"channelId"`
|
||||||
|
SubmitID string `json:"submitId"`
|
||||||
|
Payload json.RawMessage `json:"payload"`
|
||||||
|
CreatedAt time.Time `json:"createdAt"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Outbox struct {
|
||||||
|
Redis *redis.Client
|
||||||
|
Stream string
|
||||||
|
Group string
|
||||||
|
Consumer string
|
||||||
|
DedupeTTL time.Duration
|
||||||
|
APIBaseURL string
|
||||||
|
HTTPTimeout time.Duration
|
||||||
|
Concurrency int
|
||||||
|
MinIdle time.Duration
|
||||||
|
inFlight atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(client *redis.Client) *Outbox {
|
||||||
|
return &Outbox{Redis: client}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) PublishSubmitSegment(ctx context.Context, command queue.SubmitCommand, segment queue.SubmitSegmentResult) error {
|
||||||
|
payload := struct {
|
||||||
|
queue.Envelope
|
||||||
|
SubmitID string `json:"submitId,omitempty"`
|
||||||
|
queue.SubmitSegmentResult
|
||||||
|
}{
|
||||||
|
Envelope: command.Envelope,
|
||||||
|
SubmitID: command.SubmitID,
|
||||||
|
SubmitSegmentResult: segment,
|
||||||
|
}
|
||||||
|
event, err := newEvent(
|
||||||
|
fmt.Sprintf("submit:%s:segment:%d", command.SubmitID, segment.SegmentIndex),
|
||||||
|
"submit_segment_result",
|
||||||
|
"/gateway/events/submit-segment-result",
|
||||||
|
command,
|
||||||
|
payload,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return o.publish(ctx, event)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) PublishSubmitResultAndAck(
|
||||||
|
ctx context.Context,
|
||||||
|
commandStream string,
|
||||||
|
commandGroup string,
|
||||||
|
commandMessageID string,
|
||||||
|
command queue.SubmitCommand,
|
||||||
|
result queue.SubmitResult,
|
||||||
|
) error {
|
||||||
|
if o.Redis == nil {
|
||||||
|
return fmt.Errorf("result Outbox Redis client is required")
|
||||||
|
}
|
||||||
|
event, err := newEvent(
|
||||||
|
fmt.Sprintf("submit:%s:aggregate", command.SubmitID),
|
||||||
|
"submit_result",
|
||||||
|
"/gateway/events/submit-result",
|
||||||
|
command,
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(event)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// XADD and command XACK share one Redis script so a process crash cannot leave
|
||||||
|
// an acknowledged supplier command without its aggregate result in the Outbox.
|
||||||
|
_, err = publishAndAckScript.Run(
|
||||||
|
ctx,
|
||||||
|
o.Redis,
|
||||||
|
[]string{o.stream(), o.dedupeKey(event.EventID), commandStream},
|
||||||
|
int64(o.dedupeTTL().Seconds()),
|
||||||
|
string(data),
|
||||||
|
commandGroup,
|
||||||
|
commandMessageID,
|
||||||
|
).Result()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) PublishSubmitResult(ctx context.Context, command queue.SubmitCommand, result queue.SubmitResult) error {
|
||||||
|
event, err := newEvent(
|
||||||
|
fmt.Sprintf("submit:%s:aggregate", command.SubmitID),
|
||||||
|
"submit_result",
|
||||||
|
"/gateway/events/submit-result",
|
||||||
|
command,
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return o.publish(ctx, event)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) publish(ctx context.Context, event Event) error {
|
||||||
|
if o.Redis == nil {
|
||||||
|
return fmt.Errorf("result Outbox Redis client is required")
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(event)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = publishScript.Run(
|
||||||
|
ctx,
|
||||||
|
o.Redis,
|
||||||
|
[]string{o.stream(), o.dedupeKey(event.EventID)},
|
||||||
|
int64(o.dedupeTTL().Seconds()),
|
||||||
|
string(data),
|
||||||
|
).Result()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func newEvent(eventID string, eventType string, path string, command queue.SubmitCommand, payload any) (Event, error) {
|
||||||
|
data, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return Event{}, err
|
||||||
|
}
|
||||||
|
var object map[string]interface{}
|
||||||
|
if err := json.Unmarshal(data, &object); err != nil {
|
||||||
|
return Event{}, err
|
||||||
|
}
|
||||||
|
// The API stores the deterministic ID on the submit attempt after successful
|
||||||
|
// processing, making callback redelivery idempotent across Gateway restarts.
|
||||||
|
object["eventId"] = eventID
|
||||||
|
data, err = json.Marshal(object)
|
||||||
|
if err != nil {
|
||||||
|
return Event{}, err
|
||||||
|
}
|
||||||
|
return Event{
|
||||||
|
SchemaVersion: queue.SchemaVersion,
|
||||||
|
EventID: eventID,
|
||||||
|
EventType: eventType,
|
||||||
|
Path: path,
|
||||||
|
TraceID: command.TraceID,
|
||||||
|
MessageID: command.MessageID,
|
||||||
|
ChannelID: command.ChannelID,
|
||||||
|
SubmitID: command.SubmitID,
|
||||||
|
Payload: data,
|
||||||
|
CreatedAt: time.Now().UTC(),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func EventFromStreamValues(values map[string]interface{}) (Event, error) {
|
||||||
|
raw, ok := values["data"]
|
||||||
|
if !ok {
|
||||||
|
return Event{}, fmt.Errorf("result Outbox data field is required")
|
||||||
|
}
|
||||||
|
var data string
|
||||||
|
switch value := raw.(type) {
|
||||||
|
case string:
|
||||||
|
data = value
|
||||||
|
case []byte:
|
||||||
|
data = string(value)
|
||||||
|
default:
|
||||||
|
data = fmt.Sprint(value)
|
||||||
|
}
|
||||||
|
var event Event
|
||||||
|
if err := json.Unmarshal([]byte(data), &event); err != nil {
|
||||||
|
return Event{}, err
|
||||||
|
}
|
||||||
|
if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" || event.SubmitID == "" {
|
||||||
|
return Event{}, fmt.Errorf("invalid result Outbox envelope")
|
||||||
|
}
|
||||||
|
if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" {
|
||||||
|
return Event{}, fmt.Errorf("unsupported result Outbox path %q", event.Path)
|
||||||
|
}
|
||||||
|
if len(event.Payload) == 0 {
|
||||||
|
return Event{}, fmt.Errorf("result Outbox payload is required")
|
||||||
|
}
|
||||||
|
return event, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) stream() string {
|
||||||
|
if strings.TrimSpace(o.Stream) != "" {
|
||||||
|
return o.Stream
|
||||||
|
}
|
||||||
|
return defaultStream
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) group() string {
|
||||||
|
if strings.TrimSpace(o.Group) != "" {
|
||||||
|
return o.Group
|
||||||
|
}
|
||||||
|
return defaultGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) consumer() string {
|
||||||
|
if strings.TrimSpace(o.Consumer) != "" {
|
||||||
|
return o.Consumer
|
||||||
|
}
|
||||||
|
return defaultConsumer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) dedupeTTL() time.Duration {
|
||||||
|
if o.DedupeTTL > 0 {
|
||||||
|
return o.DedupeTTL
|
||||||
|
}
|
||||||
|
return defaultDedupeTTL
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) dedupeKey(eventID string) string {
|
||||||
|
return o.stream() + ":dedupe:" + eventID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) StreamName() string { return o.stream() }
|
||||||
|
func (o *Outbox) GroupName() string { return o.group() }
|
||||||
|
func (o *Outbox) InFlight() int64 { return o.inFlight.Load() }
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package resultoutbox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"cmpp-platform/gateway/internal/queue"
|
||||||
|
|
||||||
|
"github.com/alicebob/miniredis/v2"
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPublishSubmitSegmentIsIdempotent(t *testing.T) {
|
||||||
|
mr := miniredis.RunT(t)
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||||
|
outbox := New(client)
|
||||||
|
command := testCommand()
|
||||||
|
segment := queue.SubmitSegmentResult{SegmentTotal: 1, SegmentIndex: 1, SequenceID: 7, GatewayMessageID: "88", SubmitStatus: "accepted"}
|
||||||
|
|
||||||
|
if err := outbox.PublishSubmitSegment(context.Background(), command, segment); err != nil {
|
||||||
|
t.Fatalf("first publish: %v", err)
|
||||||
|
}
|
||||||
|
if err := outbox.PublishSubmitSegment(context.Background(), command, segment); err != nil {
|
||||||
|
t.Fatalf("duplicate publish: %v", err)
|
||||||
|
}
|
||||||
|
if got := client.XLen(context.Background(), outbox.StreamName()).Val(); got != 1 {
|
||||||
|
t.Fatalf("stream length = %d, want 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublishAggregateAndCommandAckAreAtomicAndIdempotent(t *testing.T) {
|
||||||
|
mr := miniredis.RunT(t)
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||||
|
ctx := context.Background()
|
||||||
|
commandStream := "gateway.submit.commands"
|
||||||
|
commandGroup := "cmpp-gateway"
|
||||||
|
if err := client.XGroupCreateMkStream(ctx, commandStream, commandGroup, "0").Err(); err != nil {
|
||||||
|
t.Fatalf("create command group: %v", err)
|
||||||
|
}
|
||||||
|
commandID, err := client.XAdd(ctx, &redis.XAddArgs{Stream: commandStream, Values: map[string]interface{}{"data": "command"}}).Result()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("add command: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: commandGroup, Consumer: "gateway-1", Streams: []string{commandStream, ">"}, Count: 1}).Result(); err != nil {
|
||||||
|
t.Fatalf("claim command: %v", err)
|
||||||
|
}
|
||||||
|
outbox := New(client)
|
||||||
|
command := testCommand()
|
||||||
|
result := queue.SubmitResult{Envelope: command.Envelope, SubmitID: command.SubmitID, GatewayMessageID: "99", SubmitStatus: "accepted"}
|
||||||
|
|
||||||
|
for attempt := 0; attempt < 2; attempt++ {
|
||||||
|
if err := outbox.PublishSubmitResultAndAck(ctx, commandStream, commandGroup, commandID, command, result); err != nil {
|
||||||
|
t.Fatalf("publish attempt %d: %v", attempt+1, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pending, err := client.XPending(ctx, commandStream, commandGroup).Result()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("command pending: %v", err)
|
||||||
|
}
|
||||||
|
if pending.Count != 0 {
|
||||||
|
t.Fatalf("command pending = %d, want 0", pending.Count)
|
||||||
|
}
|
||||||
|
if got := client.XLen(ctx, outbox.StreamName()).Val(); got != 1 {
|
||||||
|
t.Fatalf("result stream length = %d, want 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCallbackWorkerRetriesAndOnlyDeletesAfterSuccess(t *testing.T) {
|
||||||
|
mr := miniredis.RunT(t)
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||||
|
var calls atomic.Int32
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||||
|
if request.Header.Get("X-CMPP-Result-Event-ID") == "" {
|
||||||
|
t.Error("missing result event id header")
|
||||||
|
}
|
||||||
|
if calls.Add(1) == 1 {
|
||||||
|
http.Error(response, "temporary failure", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
response.WriteHeader(http.StatusCreated)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
outbox := New(client)
|
||||||
|
outbox.APIBaseURL = server.URL
|
||||||
|
outbox.MinIdle = 10 * time.Millisecond
|
||||||
|
outbox.Concurrency = 1
|
||||||
|
command := testCommand()
|
||||||
|
if err := outbox.PublishSubmitSegment(context.Background(), command, queue.SubmitSegmentResult{
|
||||||
|
SegmentTotal: 1, SegmentIndex: 1, SequenceID: 7, GatewayMessageID: "88", SubmitStatus: "accepted",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("publish segment: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- outbox.Run(ctx) }()
|
||||||
|
deadline := time.Now().Add(5 * time.Second)
|
||||||
|
for calls.Load() < 2 || client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("calls=%d streamLength=%d", calls.Load(), client.XLen(context.Background(), outbox.StreamName()).Val())
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("worker did not stop")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testCommand() queue.SubmitCommand {
|
||||||
|
return queue.SubmitCommand{
|
||||||
|
Envelope: queue.Envelope{
|
||||||
|
SchemaVersion: queue.SchemaVersion,
|
||||||
|
MessageType: queue.MessageTypeSubmitCommand,
|
||||||
|
TraceID: "trace-1",
|
||||||
|
MessageID: "message-1",
|
||||||
|
ChannelID: "channel-1",
|
||||||
|
CreatedAt: time.Now().UTC(),
|
||||||
|
},
|
||||||
|
SubmitID: "submit-1",
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package resultoutbox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"cmpp-platform/gateway/internal/metrics"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Eight callbacks matched the test VM's API/PostgreSQL capacity through
|
||||||
|
// 40 rps. A larger default caused callback bursts to contend with inbound
|
||||||
|
// persistence; operators can still raise it after measuring both queues.
|
||||||
|
defaultConcurrency = 8
|
||||||
|
// Keep the reclaim threshold above the callback timeout. Otherwise another
|
||||||
|
// Gateway replica could reclaim a still-running callback and execute the same
|
||||||
|
// business transition concurrently before the API stores its idempotency ID.
|
||||||
|
defaultMinIdle = 30 * time.Second
|
||||||
|
defaultBlock = 2 * time.Second
|
||||||
|
defaultHTTPTimeout = 10 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
var acknowledgeAndDeleteScript = redis.NewScript(`
|
||||||
|
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
||||||
|
redis.call('XDEL', KEYS[1], ARGV[2])
|
||||||
|
return 1
|
||||||
|
`)
|
||||||
|
|
||||||
|
func (o *Outbox) Run(ctx context.Context) error {
|
||||||
|
if o.Redis == nil {
|
||||||
|
return fmt.Errorf("result Outbox Redis client is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(o.APIBaseURL) == "" {
|
||||||
|
return fmt.Errorf("result Outbox API base URL is required")
|
||||||
|
}
|
||||||
|
if err := o.ensureGroup(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pool := newCallbackPool(ctx, o, o.concurrency())
|
||||||
|
defer pool.wait()
|
||||||
|
for {
|
||||||
|
if err := o.recoverPending(ctx, pool); err != nil && ctx.Err() == nil {
|
||||||
|
log.Printf("gateway result Outbox pending recovery failed: %v", err)
|
||||||
|
sleep(ctx, time.Second)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
if err := o.consumeOnce(ctx, pool); err != nil && ctx.Err() == nil {
|
||||||
|
log.Printf("gateway result Outbox consume failed: %v", err)
|
||||||
|
sleep(ctx, time.Second)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) ensureGroup(ctx context.Context) error {
|
||||||
|
err := o.Redis.XGroupCreateMkStream(ctx, o.stream(), o.group(), "0").Err()
|
||||||
|
if err == nil || strings.Contains(err.Error(), "BUSYGROUP") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) consumeOnce(ctx context.Context, pool *callbackPool) error {
|
||||||
|
available, err := pool.waitForCapacity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
streams, err := o.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||||
|
Group: o.group(), Consumer: o.consumer(), Streams: []string{o.stream(), ">"},
|
||||||
|
Count: int64(available), Block: defaultBlock,
|
||||||
|
}).Result()
|
||||||
|
if errors.Is(err, redis.Nil) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, stream := range streams {
|
||||||
|
for _, message := range stream.Messages {
|
||||||
|
if !pool.dispatch(message) {
|
||||||
|
return fmt.Errorf("gateway result Outbox capacity accounting mismatch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) recoverPending(ctx context.Context, pool *callbackPool) error {
|
||||||
|
if pool.available() == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
messages, _, err := o.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{
|
||||||
|
Stream: o.stream(), Group: o.group(), Consumer: o.consumer(), MinIdle: o.minIdle(),
|
||||||
|
Start: "0-0", Count: int64(pool.available()),
|
||||||
|
}).Result()
|
||||||
|
if errors.Is(err, redis.Nil) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, message := range messages {
|
||||||
|
if !pool.dispatch(message) {
|
||||||
|
return fmt.Errorf("gateway result Outbox recovery capacity accounting mismatch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) processMessage(ctx context.Context, message redis.XMessage) error {
|
||||||
|
event, err := EventFromStreamValues(message.Values)
|
||||||
|
if err != nil {
|
||||||
|
// Malformed internal events cannot be delivered. Keep them pending for operator
|
||||||
|
// evidence instead of ACKing and silently losing a supplier result.
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
startedAt := time.Now()
|
||||||
|
err = o.post(ctx, event)
|
||||||
|
metrics.ObserveSubmitStage("api_callback", err == nil, time.Since(startedAt))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Result events have a bounded dedupe key, so successful callbacks can be
|
||||||
|
// ACKed and deleted atomically instead of turning the Outbox into an archive.
|
||||||
|
return acknowledgeAndDeleteScript.Run(
|
||||||
|
ctx,
|
||||||
|
o.Redis,
|
||||||
|
[]string{o.stream()},
|
||||||
|
o.group(),
|
||||||
|
message.ID,
|
||||||
|
).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) post(ctx context.Context, event Event) error {
|
||||||
|
client := &http.Client{Timeout: o.httpTimeout()}
|
||||||
|
req, err := http.NewRequestWithContext(
|
||||||
|
ctx,
|
||||||
|
http.MethodPost,
|
||||||
|
strings.TrimRight(o.APIBaseURL, "/")+event.Path,
|
||||||
|
bytes.NewReader(event.Payload),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("X-CMPP-Result-Event-ID", event.EventID)
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||||
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||||
|
return fmt.Errorf("result callback returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type callbackPool struct {
|
||||||
|
ctx context.Context
|
||||||
|
outbox *Outbox
|
||||||
|
slots chan struct{}
|
||||||
|
completed chan struct{}
|
||||||
|
group sync.WaitGroup
|
||||||
|
mu sync.Mutex
|
||||||
|
active map[string]struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newCallbackPool(ctx context.Context, outbox *Outbox, concurrency int) *callbackPool {
|
||||||
|
return &callbackPool{
|
||||||
|
ctx: ctx, outbox: outbox, slots: make(chan struct{}, concurrency), completed: make(chan struct{}, concurrency), active: make(map[string]struct{}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *callbackPool) available() int { return cap(p.slots) - len(p.slots) }
|
||||||
|
|
||||||
|
func (p *callbackPool) waitForCapacity(ctx context.Context) (int, error) {
|
||||||
|
for p.available() == 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return 0, ctx.Err()
|
||||||
|
case <-p.completed:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return p.available(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *callbackPool) dispatch(message redis.XMessage) bool {
|
||||||
|
p.mu.Lock()
|
||||||
|
if _, exists := p.active[message.ID]; exists {
|
||||||
|
p.mu.Unlock()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case p.slots <- struct{}{}:
|
||||||
|
p.active[message.ID] = struct{}{}
|
||||||
|
p.outbox.inFlight.Add(1)
|
||||||
|
p.group.Add(1)
|
||||||
|
p.mu.Unlock()
|
||||||
|
case <-p.ctx.Done():
|
||||||
|
p.mu.Unlock()
|
||||||
|
return false
|
||||||
|
default:
|
||||||
|
p.mu.Unlock()
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
p.mu.Lock()
|
||||||
|
delete(p.active, message.ID)
|
||||||
|
p.mu.Unlock()
|
||||||
|
<-p.slots
|
||||||
|
p.outbox.inFlight.Add(-1)
|
||||||
|
select {
|
||||||
|
case p.completed <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
p.group.Done()
|
||||||
|
}()
|
||||||
|
if err := p.outbox.processMessage(p.ctx, message); err != nil {
|
||||||
|
log.Printf("gateway result Outbox event %s failed: %v", message.ID, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *callbackPool) wait() { p.group.Wait() }
|
||||||
|
|
||||||
|
func (o *Outbox) concurrency() int {
|
||||||
|
if o.Concurrency > 0 {
|
||||||
|
return min(o.Concurrency, 1024)
|
||||||
|
}
|
||||||
|
return defaultConcurrency
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) minIdle() time.Duration {
|
||||||
|
if o.MinIdle > 0 {
|
||||||
|
return o.MinIdle
|
||||||
|
}
|
||||||
|
return defaultMinIdle
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) httpTimeout() time.Duration {
|
||||||
|
if o.HTTPTimeout > 0 {
|
||||||
|
return o.HTTPTimeout
|
||||||
|
}
|
||||||
|
return defaultHTTPTimeout
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *Outbox) ConfiguredConcurrency() int { return o.concurrency() }
|
||||||
|
|
||||||
|
func sleep(ctx context.Context, duration time.Duration) {
|
||||||
|
timer := time.NewTimer(duration)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-timer.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,6 +36,7 @@ type Worker struct {
|
|||||||
Limiter ratelimit.Limiter
|
Limiter ratelimit.Limiter
|
||||||
Submit func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error)
|
Submit func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error)
|
||||||
ReportDeadLetter func(context.Context, DeadLetterEvent) error
|
ReportDeadLetter func(context.Context, DeadLetterEvent) error
|
||||||
|
ResultOutbox SubmitResultOutbox
|
||||||
Stream string
|
Stream string
|
||||||
Group string
|
Group string
|
||||||
Consumer string
|
Consumer string
|
||||||
@@ -50,6 +51,17 @@ type Worker struct {
|
|||||||
inFlight atomic.Int64
|
inFlight atomic.Int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type SubmitResultOutbox interface {
|
||||||
|
PublishSubmitResultAndAck(
|
||||||
|
context.Context,
|
||||||
|
string,
|
||||||
|
string,
|
||||||
|
string,
|
||||||
|
queue.SubmitCommand,
|
||||||
|
queue.SubmitResult,
|
||||||
|
) error
|
||||||
|
}
|
||||||
|
|
||||||
type DeadLetterEvent struct {
|
type DeadLetterEvent struct {
|
||||||
StreamMessageID string `json:"streamMessageId"`
|
StreamMessageID string `json:"streamMessageId"`
|
||||||
TraceID string `json:"traceId,omitempty"`
|
TraceID string `json:"traceId,omitempty"`
|
||||||
@@ -82,6 +94,9 @@ func (w *Worker) Run(ctx context.Context) error {
|
|||||||
if w.Upstream == nil {
|
if w.Upstream == nil {
|
||||||
return fmt.Errorf("upstream manager is required")
|
return fmt.Errorf("upstream manager is required")
|
||||||
}
|
}
|
||||||
|
if w.ResultOutbox == nil {
|
||||||
|
return fmt.Errorf("submit result Outbox is required")
|
||||||
|
}
|
||||||
pool := newMessageWorkPool(ctx, w, w.concurrency())
|
pool := newMessageWorkPool(ctx, w, w.concurrency())
|
||||||
defer pool.wait()
|
defer pool.wait()
|
||||||
for {
|
for {
|
||||||
@@ -292,7 +307,8 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
|
|||||||
if !command.CreatedAt.IsZero() {
|
if !command.CreatedAt.IsZero() {
|
||||||
metrics.ObserveSubmitStage("stream_wait", true, time.Since(command.CreatedAt))
|
metrics.ObserveSubmitStage("stream_wait", true, time.Since(command.CreatedAt))
|
||||||
}
|
}
|
||||||
if err := w.handleCommand(ctx, command); err != nil {
|
result, err := w.executeCommand(ctx, command)
|
||||||
|
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
@@ -308,29 +324,50 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
|
|||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return w.ackAndClearFailure(ctx, message.ID)
|
if err := w.ResultOutbox.PublishSubmitResultAndAck(
|
||||||
|
ctx,
|
||||||
|
w.stream(),
|
||||||
|
w.group(),
|
||||||
|
message.ID,
|
||||||
|
command,
|
||||||
|
result,
|
||||||
|
); err != nil {
|
||||||
|
return fmt.Errorf("persist aggregate submit result: %w", err)
|
||||||
|
}
|
||||||
|
// The command was ACKed atomically with the aggregate result event. Clearing the
|
||||||
|
// auxiliary retry counter may lag without affecting delivery correctness.
|
||||||
|
w.clearFailure(ctx, message.ID)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
|
func (w *Worker) executeCommand(ctx context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||||
startedAt := time.Now()
|
startedAt := time.Now()
|
||||||
limitStartedAt := time.Now()
|
limitStartedAt := time.Now()
|
||||||
if w.Limiter != nil {
|
if w.Limiter != nil {
|
||||||
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
|
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
|
||||||
metrics.ObserveSubmitStage("rate_limit_wait", false, time.Since(limitStartedAt))
|
metrics.ObserveSubmitStage("rate_limit_wait", false, time.Since(limitStartedAt))
|
||||||
return err
|
return queue.SubmitResult{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
metrics.ObserveSubmitStage("rate_limit_wait", true, time.Since(limitStartedAt))
|
metrics.ObserveSubmitStage("rate_limit_wait", true, time.Since(limitStartedAt))
|
||||||
submit := w.Submit
|
submit := w.Submit
|
||||||
if submit == nil {
|
if submit == nil {
|
||||||
if w.Upstream == nil {
|
if w.Upstream == nil {
|
||||||
return fmt.Errorf("upstream manager is required")
|
return queue.SubmitResult{}, fmt.Errorf("upstream manager is required")
|
||||||
}
|
}
|
||||||
submit = w.Upstream.Submit
|
submit = w.Upstream.Submit
|
||||||
}
|
}
|
||||||
result, err := submit(ctx, command)
|
result, err := submit(ctx, command)
|
||||||
accepted := err == nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted")
|
accepted := err == nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted")
|
||||||
metrics.ObserveSubmit(accepted, time.Since(startedAt))
|
metrics.ObserveSubmit(accepted, time.Since(startedAt))
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleCommand remains a narrow compatibility seam for focused tests and the
|
||||||
|
// control path. Stream consumption uses executeCommand so it can persist the
|
||||||
|
// exact terminal result before acknowledging the command.
|
||||||
|
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
|
||||||
|
result, err := w.executeCommand(ctx, command)
|
||||||
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -456,10 +493,14 @@ func (w *Worker) ackAndClearFailure(ctx context.Context, messageID string) error
|
|||||||
if err := w.Redis.XAck(ctx, w.stream(), w.group(), messageID).Err(); err != nil {
|
if err := w.Redis.XAck(ctx, w.stream(), w.group(), messageID).Err(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
w.clearFailure(ctx, messageID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Worker) clearFailure(ctx context.Context, messageID string) {
|
||||||
if err := w.Redis.HDel(ctx, w.failureAttemptsKey(), messageID).Err(); err != nil {
|
if err := w.Redis.HDel(ctx, w.failureAttemptsKey(), messageID).Err(); err != nil {
|
||||||
w.logf("gateway submit worker clear failure %s failed: %v", messageID, err)
|
w.logf("gateway submit worker clear failure %s failed: %v", messageID, err)
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Worker) stream() string {
|
func (w *Worker) stream() string {
|
||||||
|
|||||||
@@ -17,6 +17,24 @@ type recordingLimiter struct {
|
|||||||
called bool
|
called bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type acknowledgingResultOutbox struct {
|
||||||
|
client *redis.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o acknowledgingResultOutbox) PublishSubmitResultAndAck(
|
||||||
|
ctx context.Context,
|
||||||
|
commandStream string,
|
||||||
|
commandGroup string,
|
||||||
|
commandMessageID string,
|
||||||
|
_ queue.SubmitCommand,
|
||||||
|
_ queue.SubmitResult,
|
||||||
|
) error {
|
||||||
|
if o.client == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return o.client.XAck(ctx, commandStream, commandGroup, commandMessageID).Err()
|
||||||
|
}
|
||||||
|
|
||||||
func (l *recordingLimiter) Wait(_ context.Context, channelID string, rate int) (time.Duration, error) {
|
func (l *recordingLimiter) Wait(_ context.Context, channelID string, rate int) (time.Duration, error) {
|
||||||
l.called = true
|
l.called = true
|
||||||
l.channelID = channelID
|
l.channelID = channelID
|
||||||
@@ -140,6 +158,7 @@ func TestMessageWorkPoolContinuouslyRefillsWithoutWaitingForSlowSibling(t *testi
|
|||||||
releaseA := make(chan struct{})
|
releaseA := make(chan struct{})
|
||||||
worker := &Worker{
|
worker := &Worker{
|
||||||
Redis: client,
|
Redis: client,
|
||||||
|
ResultOutbox: acknowledgingResultOutbox{client: client},
|
||||||
Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
|
Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||||
switch command.ChannelID {
|
switch command.ChannelID {
|
||||||
case "channel-a":
|
case "channel-a":
|
||||||
@@ -188,7 +207,7 @@ func TestMessageWorkPoolContinuouslyRefillsWithoutWaitingForSlowSibling(t *testi
|
|||||||
func TestMessageWorkPoolAcknowledgesFastMessageBeforeSlowSiblingCompletes(t *testing.T) {
|
func TestMessageWorkPoolAcknowledgesFastMessageBeforeSlowSiblingCompletes(t *testing.T) {
|
||||||
mr := miniredis.RunT(t)
|
mr := miniredis.RunT(t)
|
||||||
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||||
worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway"}
|
worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", ResultOutbox: acknowledgingResultOutbox{client: client}}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
if err := worker.ensureGroup(ctx); err != nil {
|
if err := worker.ensureGroup(ctx); err != nil {
|
||||||
t.Fatalf("ensureGroup: %v", err)
|
t.Fatalf("ensureGroup: %v", err)
|
||||||
@@ -246,7 +265,7 @@ func TestMessageWorkPoolAcknowledgesFastMessageBeforeSlowSiblingCompletes(t *tes
|
|||||||
func TestPendingRecoveryDoesNotDuplicateAnActiveOrAlreadyAcknowledgedMessage(t *testing.T) {
|
func TestPendingRecoveryDoesNotDuplicateAnActiveOrAlreadyAcknowledgedMessage(t *testing.T) {
|
||||||
mr := miniredis.RunT(t)
|
mr := miniredis.RunT(t)
|
||||||
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
||||||
worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", MinIdle: time.Millisecond}
|
worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", MinIdle: time.Millisecond, ResultOutbox: acknowledgingResultOutbox{client: client}}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
if err := worker.ensureGroup(ctx); err != nil {
|
if err := worker.ensureGroup(ctx); err != nil {
|
||||||
t.Fatalf("ensureGroup: %v", err)
|
t.Fatalf("ensureGroup: %v", err)
|
||||||
|
|||||||
@@ -24,11 +24,19 @@ const (
|
|||||||
type Manager struct {
|
type Manager struct {
|
||||||
APIBaseURL string
|
APIBaseURL string
|
||||||
HTTPClient *http.Client
|
HTTPClient *http.Client
|
||||||
|
SubmitSegmentPublisher SubmitSegmentPublisher
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
conns map[string]*connectionPool
|
conns map[string]*connectionPool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SubmitSegmentPublisher persists each supplier response before the next long-message
|
||||||
|
// segment is sent. The boundary is intentionally storage-only: HTTP callbacks belong to
|
||||||
|
// the result Outbox worker and must not consume a supplier Submit window slot.
|
||||||
|
type SubmitSegmentPublisher interface {
|
||||||
|
PublishSubmitSegment(context.Context, queue.SubmitCommand, queue.SubmitSegmentResult) error
|
||||||
|
}
|
||||||
|
|
||||||
type ConnectionState struct {
|
type ConnectionState struct {
|
||||||
ChannelID string `json:"channelId"`
|
ChannelID string `json:"channelId"`
|
||||||
ConnectionID string `json:"connectionId"`
|
ConnectionID string `json:"connectionId"`
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
cmpp "github.com/bigwhite/gocmpp"
|
cmpp "github.com/bigwhite/gocmpp"
|
||||||
cmpputils "github.com/bigwhite/gocmpp/utils"
|
cmpputils "github.com/bigwhite/gocmpp/utils"
|
||||||
"log"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -17,58 +16,30 @@ import (
|
|||||||
|
|
||||||
func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
|
func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
|
||||||
if err := validateSubmitCommand(cmd); err != nil {
|
if err := validateSubmitCommand(cmd); err != nil {
|
||||||
result := submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error())
|
return submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error()), err
|
||||||
if postErr := m.postSubmitCallback(ctx, "/gateway/events/submit-result", result); postErr != nil {
|
|
||||||
return result, postErr
|
|
||||||
}
|
|
||||||
return result, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pool, err := m.connectionFor(cmd)
|
pool, err := m.connectionFor(cmd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
result := submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error())
|
return submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error()), err
|
||||||
if postErr := m.postSubmitCallback(ctx, "/gateway/events/submit-result", result); postErr != nil {
|
|
||||||
return result, postErr
|
|
||||||
}
|
|
||||||
return result, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := pool.submit(ctx, cmd, func(segment queue.SubmitSegmentResult) {
|
result, err := pool.submit(ctx, cmd, func(segment queue.SubmitSegmentResult) error {
|
||||||
payload := struct {
|
if m.SubmitSegmentPublisher == nil {
|
||||||
queue.Envelope
|
return fmt.Errorf("submit segment result publisher is required")
|
||||||
SubmitID string `json:"submitId,omitempty"`
|
|
||||||
queue.SubmitSegmentResult
|
|
||||||
}{
|
|
||||||
Envelope: cmd.Envelope,
|
|
||||||
SubmitID: cmd.SubmitID,
|
|
||||||
SubmitSegmentResult: segment,
|
|
||||||
}
|
|
||||||
callbackCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
||||||
postErr := m.postSubmitCallback(callbackCtx, "/gateway/events/submit-segment-result", payload)
|
|
||||||
cancel()
|
|
||||||
if postErr != nil {
|
|
||||||
log.Printf(
|
|
||||||
"protocol_event protocol=cmpp direction=gateway_to_api event=submit_segment_result status=forward_failed channel_id=%s message_id=%s segment=%d/%d error=%q",
|
|
||||||
cmd.ChannelID, cmd.MessageID, segment.SegmentIndex, segment.SegmentTotal, postErr,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
// A segment result must reach durable local storage before the next segment.
|
||||||
|
// This cannot make the supplier/Redis boundary globally atomic, but it avoids
|
||||||
|
// holding the supplier slot for an API round trip and minimizes untracked sends.
|
||||||
|
return m.SubmitSegmentPublisher.PublishSubmitSegment(ctx, cmd, segment)
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
if postErr := m.postSubmitCallback(ctx, "/gateway/events/submit-result", result); postErr != nil {
|
|
||||||
return result, postErr
|
|
||||||
}
|
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
if err := m.postSubmitCallback(ctx, "/gateway/events/submit-result", result); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *connectionPool) submit(
|
func (p *connectionPool) submit(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
cmd queue.SubmitCommand,
|
cmd queue.SubmitCommand,
|
||||||
onSegment func(queue.SubmitSegmentResult),
|
onSegment func(queue.SubmitSegmentResult) error,
|
||||||
) (queue.SubmitResult, error) {
|
) (queue.SubmitResult, error) {
|
||||||
parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content)
|
parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -95,7 +66,10 @@ func (p *connectionPool) submit(
|
|||||||
segment := submitSegmentResult(part, seq, gatewayMessageID, result)
|
segment := submitSegmentResult(part, seq, gatewayMessageID, result)
|
||||||
segments = append(segments, segment)
|
segments = append(segments, segment)
|
||||||
if onSegment != nil {
|
if onSegment != nil {
|
||||||
onSegment(segment)
|
if publishErr := onSegment(segment); publishErr != nil {
|
||||||
|
result.Segments = segments
|
||||||
|
return result, publishErr
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if firstSequence == 0 {
|
if firstSequence == 0 {
|
||||||
firstSequence = seq
|
firstSequence = seq
|
||||||
@@ -118,13 +92,6 @@ func (p *connectionPool) submit(
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *Manager) postSubmitCallback(ctx context.Context, path string, payload any) error {
|
|
||||||
startedAt := time.Now()
|
|
||||||
err := m.post(ctx, path, payload)
|
|
||||||
metrics.ObserveSubmitStage("api_callback", err == nil, time.Since(startedAt))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) {
|
func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) {
|
||||||
rspCh := make(chan submitPartResponse, 1)
|
rspCh := make(chan submitPartResponse, 1)
|
||||||
pkt := c.submitRequestPacket(cmd, part)
|
pkt := c.submitRequestPacket(cmd, part)
|
||||||
|
|||||||
@@ -40,6 +40,17 @@ function assertIsoDateTime(value, field, file) {
|
|||||||
function validateExample(fileName) {
|
function validateExample(fileName) {
|
||||||
const fullPath = join(examplesDir, fileName);
|
const fullPath = join(examplesDir, fileName);
|
||||||
const payload = JSON.parse(readFileSync(fullPath, 'utf8'));
|
const payload = JSON.parse(readFileSync(fullPath, 'utf8'));
|
||||||
|
if (fileName === 'submit-result-outbox-event.json') {
|
||||||
|
for (const field of ['schemaVersion', 'eventId', 'eventType', 'path', 'messageId', 'channelId', 'submitId', 'payload', 'createdAt']) {
|
||||||
|
assert(Object.hasOwn(payload, field), `${fileName}: missing ${field}`);
|
||||||
|
}
|
||||||
|
assert(payload.schemaVersion === 'v1', `${fileName}: schemaVersion must be v1`);
|
||||||
|
assert(['submit_result', 'submit_segment_result'].includes(payload.eventType), `${fileName}: unsupported eventType`);
|
||||||
|
assert(['/gateway/events/submit-result', '/gateway/events/submit-segment-result'].includes(payload.path), `${fileName}: unsupported path`);
|
||||||
|
assert(payload.payload?.eventId === payload.eventId, `${fileName}: payload.eventId must match eventId`);
|
||||||
|
assertIsoDateTime(payload.createdAt, 'createdAt', fileName);
|
||||||
|
return `${fileName}: ${payload.eventType} ok`;
|
||||||
|
}
|
||||||
const type = payload.messageType;
|
const type = payload.messageType;
|
||||||
|
|
||||||
assert(validTypes.has(type), `${fileName}: unsupported messageType ${type}`);
|
assert(validTypes.has(type), `${fileName}: unsupported messageType ${type}`);
|
||||||
|
|||||||
Reference in New Issue
Block a user