perf(cmpp): decouple supplier result callbacks

This commit is contained in:
hectorzhao
2026-08-20 14:19:42 +08:00
parent 485af688d2
commit 67b760a599
27 changed files with 1000 additions and 66 deletions
+4
View File
@@ -43,5 +43,9 @@ GATEWAY_CMPP_ADDR=127.0.0.1:7890
GATEWAY_CMPP_USER=900001 GATEWAY_CMPP_USER=900001
GATEWAY_CMPP_PASSWORD=888888 GATEWAY_CMPP_PASSWORD=888888
GATEWAY_SUBMIT_WORKER_CONCURRENCY=64 GATEWAY_SUBMIT_WORKER_CONCURRENCY=64
GATEWAY_SUBMIT_RESULT_STREAM=gateway.submit.results
GATEWAY_SUBMIT_RESULT_GROUP=cmpp-api-callback
GATEWAY_SUBMIT_RESULT_CONSUMER=gateway-1
GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY=8
GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64 GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64
CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30 CMPP_DOWNSTREAM_ACK_TIMEOUT_SECONDS=30
@@ -0,0 +1,6 @@
ALTER TABLE "SmsSubmitRecord"
ADD COLUMN "resultEventId" TEXT,
ADD COLUMN "resultProcessedAt" TIMESTAMP(3);
CREATE UNIQUE INDEX "SmsSubmitRecord_resultEventId_key"
ON "SmsSubmitRecord"("resultEventId");
+2
View File
@@ -1760,6 +1760,8 @@ model SmsSubmitRecord {
sequenceId Int? sequenceId Int?
gatewayMessageId String? gatewayMessageId String?
submitStatus String @default("queued") submitStatus String @default("queued")
resultEventId String? @unique
resultProcessedAt DateTime?
costUnitPrice BigInt @default(0) costUnitPrice BigInt @default(0)
costAmountCents BigInt @default(0) costAmountCents BigInt @default(0)
errorCode String? errorCode String?
@@ -58,6 +58,7 @@ export interface GatewayInboundSingleSubmitResult {
} }
export interface GatewaySubmitResultDto { export interface GatewaySubmitResultDto {
eventId?: string;
traceId?: string; traceId?: string;
messageId: string; messageId: string;
channelId: string; channelId: string;
@@ -81,6 +82,7 @@ export interface GatewaySubmitResultDto {
} }
export interface GatewaySubmitSegmentResultDto { export interface GatewaySubmitSegmentResultDto {
eventId?: string;
traceId?: string; traceId?: string;
messageId: string; messageId: string;
channelId: string; channelId: string;
@@ -2216,6 +2216,46 @@ describe('SendChainService', () => {
}); });
}); });
it('treats a redelivered Outbox aggregate event as idempotent', async () => {
const { service, prisma, billing } = createService();
const baseSubmit = {
id: 'submit-1',
messageRecordId: 'record-1',
channelId: 'channel-1',
submitId: 'SUB-1',
submitStatus: 'accepted',
};
prisma.smsSubmitRecord.findUnique
.mockResolvedValueOnce({ ...baseSubmit, resultEventId: null })
.mockResolvedValueOnce({ ...baseSubmit, resultEventId: 'submit:SUB-1:aggregate' });
const event = {
eventId: 'submit:SUB-1:aggregate',
messageId: 'MSG-1',
channelId: 'channel-1',
submitId: 'SUB-1',
sequenceId: 7,
gatewayMessageId: 'GW-1',
submitStatus: 'accepted' as const,
submittedAt: '2026-07-01T10:00:00.000Z',
};
await service.handleSubmitResult(event);
await service.handleSubmitResult(event);
expect(billing.charge).toHaveBeenCalledTimes(1);
expect(billing.release).toHaveBeenCalledTimes(1);
expect(prisma.smsSubmitRecord.updateMany).toHaveBeenCalledWith({
where: {
id: 'submit-1',
OR: [{ resultEventId: null }, { resultEventId: 'submit:SUB-1:aggregate' }],
},
data: {
resultEventId: 'submit:SUB-1:aggregate',
resultProcessedAt: new Date('2026-07-01T10:00:00.000Z'),
},
});
});
it('rejects a legacy aggregate SubmitResult when it cannot match one submit attempt uniquely', async () => { it('rejects a legacy aggregate SubmitResult when it cannot match one submit attempt uniquely', async () => {
const { service, prisma } = createService(); const { service, prisma } = createService();
prisma.smsSubmitRecord.findMany.mockResolvedValue([ prisma.smsSubmitRecord.findMany.mockResolvedValue([
@@ -128,6 +128,12 @@ export class SendGatewayResultService {
async handleSubmitResult(data: GatewaySubmitResultDto) { async handleSubmitResult(data: GatewaySubmitResultDto) {
const message = await this.facade.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId); const message = await this.facade.requireMessageByGatewayEvent(data.messageId, data.gatewayMessageId);
const submitRecord = await this.facade.resolveSubmitRecordForGatewayResult(message.id, data); const submitRecord = await this.facade.resolveSubmitRecordForGatewayResult(message.id, data);
if (data.eventId && submitRecord.resultEventId) {
if (submitRecord.resultEventId !== data.eventId) {
throw new BadRequestException('Gateway SubmitResult eventId conflicts with the submit attempt');
}
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
}
const effectiveData = { ...data, submitId: submitRecord.submitId }; const effectiveData = { ...data, submitId: submitRecord.submitId };
const batchTask = message.batchTaskId const batchTask = message.batchTaskId
? await this.prisma.smsBatchTask.findUnique({ where: { id: message.batchTaskId }, select: { sourceType: true } }) ? await this.prisma.smsBatchTask.findUnique({ where: { id: message.batchTaskId }, select: { sourceType: true } })
@@ -147,6 +153,7 @@ export class SendGatewayResultService {
await this.facade.recordSubmitSegments(message, effectiveData, submittedAt); await this.facade.recordSubmitSegments(message, effectiveData, submittedAt);
const isStandaloneChannelTest = !message.tenantId && !message.batchTaskId; const isStandaloneChannelTest = !message.tenantId && !message.batchTaskId;
if (message.submitId && effectiveData.submitId !== message.submitId) { if (message.submitId && effectiveData.submitId !== message.submitId) {
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
} }
const status = data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed'; const status = data.submitStatus === 'accepted' ? 'submitted' : data.submitStatus === 'timeout' ? 'timeout' : 'submit_failed';
@@ -166,6 +173,7 @@ export class SendGatewayResultService {
); );
if (retried) { if (retried) {
await this.facade.refreshTaskProgress(businessMessage.batchTaskId); await this.facade.refreshTaskProgress(businessMessage.batchTaskId);
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
return retried; return retried;
} }
await this.facade.releaseMessageReservation(businessMessage, data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结'); await this.facade.releaseMessageReservation(businessMessage, data.submitStatus === 'timeout' ? '提交超时释放冻结' : '提交失败释放冻结');
@@ -218,9 +226,29 @@ export class SendGatewayResultService {
if (message.batchTaskId) { if (message.batchTaskId) {
await this.facade.refreshTaskProgress(message.batchTaskId); await this.facade.refreshTaskProgress(message.batchTaskId);
} }
await this.markSubmitResultProcessed(submitRecord.id, data.eventId, submittedAt);
return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } }); return this.prisma.smsMessageRecord.findUnique({ where: { id: message.id } });
} }
private async markSubmitResultProcessed(submitRecordId: string, eventId: string | undefined, processedAt: Date) {
if (!eventId) {
return;
}
const updated = await this.prisma.smsSubmitRecord.updateMany({
where: {
id: submitRecordId,
OR: [{ resultEventId: null }, { resultEventId: eventId }],
},
data: {
resultEventId: eventId,
resultProcessedAt: processedAt,
},
});
if (updated.count === 0) {
throw new BadRequestException('Gateway SubmitResult eventId conflicts with the submit attempt');
}
}
async resolveSubmitRecordForGatewayResult(messageRecordId: string, data: GatewaySubmitResultDto) { async resolveSubmitRecordForGatewayResult(messageRecordId: string, data: GatewaySubmitResultDto) {
if (data.submitId) { if (data.submitId) {
const exact = await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: data.submitId } }); const exact = await this.prisma.smsSubmitRecord.findUnique({ where: { submitId: data.submitId } });
+1
View File
@@ -1153,5 +1153,6 @@ global,不能错误归入client。`client-signature-*`、发送页、企业认
- CMPP性能分段沿用上述边界:业务模块只在原调用边界提交固定阶段名、成功标志和单调时钟耗时;指标模块拒绝未知阶段。`supplier_rtt`在供应商连接调用结束时立即停止,API结果回写另计`api_callback`,防止后续优化依据混杂总耗时。 - CMPP性能分段沿用上述边界:业务模块只在原调用边界提交固定阶段名、成功标志和单调时钟耗时;指标模块拒绝未知阶段。`supplier_rtt`在供应商连接调用结束时立即停止,API结果回写另计`api_callback`,防止后续优化依据混杂总耗时。
- V2提交工作池只归`gateway/internal/submitworker/`治理:Redis领取、全局槽位、在途消息ID和逐条ACK不能渗入上游连接池;`gateway/internal/upstream/`继续只负责通道连接、窗口和供应商协议往返。这样Worker吞吐调优不会改写CMPP连接状态机,连接池也不能自行确认Redis消息。 - V2提交工作池只归`gateway/internal/submitworker/`治理:Redis领取、全局槽位、在途消息ID和逐条ACK不能渗入上游连接池;`gateway/internal/upstream/`继续只负责通道连接、窗口和供应商协议往返。这样Worker吞吐调优不会改写CMPP连接状态机,连接池也不能自行确认Redis消息。
- V3客户入站窗口只归`gateway/internal/inbound/`与项目内受控的`third_party/gocmpp`服务循环治理:API认证只返回应用窗口,inbound会话负责收紧窗口,协议服务循环负责受限派发和断线等待;不得把客户入站槽位与`submitworker`供应商槽位或`upstream`供应商窗口合并成同一并发计数。 - V3客户入站窗口只归`gateway/internal/inbound/`与项目内受控的`third_party/gocmpp`服务循环治理:API认证只返回应用窗口,inbound会话负责收紧窗口,协议服务循环负责受限派发和断线等待;不得把客户入站槽位与`submitworker`供应商槽位或`upstream`供应商窗口合并成同一并发计数。
- V4供应商结果异步边界只归`gateway/internal/resultoutbox/`治理:`upstream`在每个真实分片SubmitResp后只调用持久化接口,`submitworker`只负责聚合结果入Outbox与命令ACK的原子边界,Outbox回调Worker独立控制API并发和PEL恢复。API的`SmsSubmitRecord.resultEventId`是跨重启持久幂等事实;不得把HTTP回调重新放回供应商连接池或Submit工作槽,也不得让Outbox承担业务计费、补发或状态机判断。
- `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。 - `api/src/infrastructure-monitoring/`是运营端监控聚合与固定阈值应用边界:只消费代码白名单 PromQL,并通过版本化 PostgreSQL 单例、promtool 校验和原子规则热加载管理数值阈值;Exporter安装、端口隔离、固定规则模板和权限仍归`tools/monitoring/`治理。
- 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。 - 活动告警已读也归该边界:Prometheus保留告警事实,Prisma仅持久化逐管理员、逐触发周期的阅读状态;全局布局只消费轻量未读汇总,不复制指纹、activeAt或用户隔离逻辑。
@@ -0,0 +1,24 @@
{
"schemaVersion": "v1",
"eventId": "submit:submit-20260820-0001:aggregate",
"eventType": "submit_result",
"path": "/gateway/events/submit-result",
"traceId": "trace-20260820-0001",
"messageId": "message-20260820-0001",
"channelId": "channel-test-1",
"submitId": "submit-20260820-0001",
"payload": {
"eventId": "submit:submit-20260820-0001:aggregate",
"schemaVersion": "v1",
"messageType": "SubmitResult",
"traceId": "trace-20260820-0001",
"messageId": "message-20260820-0001",
"channelId": "channel-test-1",
"createdAt": "2026-08-20T05:00:00Z",
"sequenceId": 1,
"gatewayMessageId": "1000000000000001",
"submitStatus": "accepted",
"submittedAt": "2026-08-20T05:00:00Z"
},
"createdAt": "2026-08-20T05:00:00Z"
}
@@ -6,7 +6,8 @@
{ "$ref": "#/$defs/SubmitCommand" }, { "$ref": "#/$defs/SubmitCommand" },
{ "$ref": "#/$defs/SubmitResult" }, { "$ref": "#/$defs/SubmitResult" },
{ "$ref": "#/$defs/ReceiptEvent" }, { "$ref": "#/$defs/ReceiptEvent" },
{ "$ref": "#/$defs/UplinkEvent" } { "$ref": "#/$defs/UplinkEvent" },
{ "$ref": "#/$defs/SubmitResultOutboxEvent" }
], ],
"$defs": { "$defs": {
"Envelope": { "Envelope": {
@@ -139,6 +140,26 @@
} }
] ]
}, },
"SubmitResultOutboxEvent": {
"type": "object",
"required": ["schemaVersion", "eventId", "eventType", "path", "messageId", "channelId", "submitId", "payload", "createdAt"],
"properties": {
"schemaVersion": { "const": "v1" },
"eventId": { "type": "string", "pattern": "^submit:.+:(aggregate|segment:[1-9][0-9]*)$" },
"eventType": { "enum": ["submit_result", "submit_segment_result"] },
"path": { "enum": ["/gateway/events/submit-result", "/gateway/events/submit-segment-result"] },
"traceId": { "type": "string" },
"messageId": { "type": "string", "minLength": 1 },
"channelId": { "type": "string", "minLength": 1 },
"submitId": { "type": "string", "minLength": 1 },
"payload": {
"type": "object",
"required": ["eventId"],
"properties": { "eventId": { "type": "string", "minLength": 1 } }
},
"createdAt": { "type": "string", "format": "date-time" }
}
},
"ReceiptEvent": { "ReceiptEvent": {
"allOf": [ "allOf": [
{ "$ref": "#/$defs/Envelope" }, { "$ref": "#/$defs/Envelope" },
@@ -17,7 +17,7 @@
{ {
"name": "handleSubmitResult", "name": "handleSubmitResult",
"file": "send-gateway-result.service.ts", "file": "send-gateway-result.service.ts",
"bodySha256": "6184170748eb3934496b060ea3b6704f1d1a831e7ffde7e7f8b73a7c223a869c" "bodySha256": "9414f8d4e5e1c3cf6d0747e47ce01e101e142772af4f56900932bc6d4ac11ec7"
}, },
{ {
"name": "resolveSubmitRecordForGatewayResult", "name": "resolveSubmitRecordForGatewayResult",
+3 -3
View File
@@ -173,7 +173,7 @@
"name": "Manager", "name": "Manager",
"kind": "type", "kind": "type",
"file": "manager.go", "file": "manager.go",
"sha256": "c7ead8b037bc87ad9800650ce191f5fcee9648b8805a133b9506743c762ea608" "sha256": "b08726201c34da87ae735abd7fc56d2b904104d4b7f6a6a6365263ec5b0a43bc"
}, },
{ {
"name": "defaultChannelConnectionID", "name": "defaultChannelConnectionID",
@@ -389,7 +389,7 @@
"kind": "func", "kind": "func",
"receiver": "Manager", "receiver": "Manager",
"file": "submit.go", "file": "submit.go",
"sha256": "a25d30aeb87c1fe123929330fdabc99261e63d5b51e1cb6207197c43a15de07d" "sha256": "9be92b0d32626ce93d0b2d721bf008305c4b9e5919e287d49b91c6f2474ddeb1"
}, },
{ {
"name": "submitPart", "name": "submitPart",
@@ -410,7 +410,7 @@
"kind": "func", "kind": "func",
"receiver": "connectionPool", "receiver": "connectionPool",
"file": "submit.go", "file": "submit.go",
"sha256": "e3dbf2d433ef4ff95b2fe7a8494b17c6a8e185bdace017cce9c7d2e8c785bfb3" "sha256": "577b7f942f4b95ea3868d7cf9aadae800df8edda0e7ca94a205b4124be58e9e3"
}, },
{ {
"name": "defaultInt", "name": "defaultInt",
@@ -240,6 +240,7 @@
9. Gateway 必须按通道连接和窗口容量控制并发,处理窗口满、SMSC 慢响应、sequence 回绕、连接断开时的在途消息状态。 9. Gateway 必须按通道连接和窗口容量控制并发,处理窗口满、SMSC 慢响应、sequence 回绕、连接断开时的在途消息状态。
10. Gateway 必须对每个物理通道执行 Redis 分布式 TPS 限速。`SmsChannel.rateLimitPerSecond` 是单通道上限,不是平台总上限;同一通道被多个通道组或多个 Gateway 实例使用时共享同一额度,不同通道独立计数。通道连接命令下发的配置值是最终上限,提交命令携带的值只能进一步降低、不能放大该上限。超流速消息必须继续保留在 Redis Stream pending 中等待可用时隙,不能因等待直接标记发送失败;Gateway 重启后仍可由 consumer group 恢复。worker 必须使用持续补位的全局有界工作池,任一任务完成后立即领取后续消息,不得以“读取10条、等待整批结束”形成批次屏障;每条消息只在自身提交结果已持久回传或完成死信处理后独立`XACK`。全局并发默认64、可由`GATEWAY_SUBMIT_WORKER_CONCURRENCY`配置且上限1024;单通道实际并发仍由 Redis 限速、连接数和 CMPP 窗口共同约束。恢复pending时必须防止超过`MinIdle`的在途消息被同一进程重复提交。 10. Gateway 必须对每个物理通道执行 Redis 分布式 TPS 限速。`SmsChannel.rateLimitPerSecond` 是单通道上限,不是平台总上限;同一通道被多个通道组或多个 Gateway 实例使用时共享同一额度,不同通道独立计数。通道连接命令下发的配置值是最终上限,提交命令携带的值只能进一步降低、不能放大该上限。超流速消息必须继续保留在 Redis Stream pending 中等待可用时隙,不能因等待直接标记发送失败;Gateway 重启后仍可由 consumer group 恢复。worker 必须使用持续补位的全局有界工作池,任一任务完成后立即领取后续消息,不得以“读取10条、等待整批结束”形成批次屏障;每条消息只在自身提交结果已持久回传或完成死信处理后独立`XACK`。全局并发默认64、可由`GATEWAY_SUBMIT_WORKER_CONCURRENCY`配置且上限1024;单通道实际并发仍由 Redis 限速、连接数和 CMPP 窗口共同约束。恢复pending时必须防止超过`MinIdle`的在途消息被同一进程重复提交。
11. Gateway 不承担业务审核、计费、签名报备、通道组路由、黑名单或敏感词判断;这些由 NestJS 完成,Gateway 只执行已授权通道提交与协议事件回传。 11. Gateway 不承担业务审核、计费、签名报备、通道组路由、黑名单或敏感词判断;这些由 NestJS 完成,Gateway 只执行已授权通道提交与协议事件回传。
12. 供应商SubmitResp及长短信分片结果必须先进入共享Redis的幂等Outbox,再由独立有界回调Worker异步上报API;供应商Submit工作槽只等待连接窗口、限速和真实供应商往返,不得等待API结果回写。每个分片结果必须在继续发送下一分片前写入Outbox;聚合结果入Outbox与原`gateway.submit.commands`消息`XACK`必须原子完成。事件ID按submitId和分片序号确定生成,重复发布不得产生第二个事件;API必须按事件ID持久幂等,重复回调不得重复扣费、释放余额、补发或推进状态。回调失败保留在Outbox PEL并可在进程重启后恢复,成功后逐条ACK并删除;去重键和并发必须有界配置,不得形成无限内存或Stream归档。
#### 4.8.2 下游客户 CMPP 接入能力 #### 4.8.2 下游客户 CMPP 接入能力
@@ -2100,6 +2101,7 @@
- Gateway供应商下发必须分别记录`stream_wait/rate_limit_wait/connection_wait/supplier_rtt/api_callback`;其中`supplier_rtt`只覆盖供应商连接上的Submit请求与SubmitResp往返,不得包含结果回写API的耗时。阶段名和结果值必须使用代码固定白名单,不得添加手机号、企业、应用、通道、消息、任务或连接ID标签。 - Gateway供应商下发必须分别记录`stream_wait/rate_limit_wait/connection_wait/supplier_rtt/api_callback`;其中`supplier_rtt`只覆盖供应商连接上的Submit请求与SubmitResp往返,不得包含结果回写API的耗时。阶段名和结果值必须使用代码固定白名单,不得添加手机号、企业、应用、通道、消息、任务或连接ID标签。
- V2有界工作池必须暴露配置槽位数和当前在途槽位数,使用固定`state=configured|in_flight`标签;该指标用于区分Worker容量耗尽与供应商窗口/限速等待,不得增加通道或消息标签。 - V2有界工作池必须暴露配置槽位数和当前在途槽位数,使用固定`state=configured|in_flight`标签;该指标用于区分Worker容量耗尽与供应商窗口/限速等待,不得增加通道或消息标签。
- V3入站并发必须只并发Submit业务处理,连接认证保持串行先完成,心跳和Deliver ACK不得被长耗时Submit阻塞;每个SubmitResp继续使用原请求Sequence_Id关联,允许按实际完成顺序返回。同一连接关闭时必须先等待已接受的在途处理收尾,再清理会话和回执映射,避免迟到处理重新注册已断开的连接。`cmpp_gateway_inbound_submit_slots{state=configured|in_flight}`只暴露全部在线连接的聚合窗口与在途数量,不得增加账号、应用、连接或消息标签。 - V3入站并发必须只并发Submit业务处理,连接认证保持串行先完成,心跳和Deliver ACK不得被长耗时Submit阻塞;每个SubmitResp继续使用原请求Sequence_Id关联,允许按实际完成顺序返回。同一连接关闭时必须先等待已接受的在途处理收尾,再清理会话和回执映射,避免迟到处理重新注册已断开的连接。`cmpp_gateway_inbound_submit_slots{state=configured|in_flight}`只暴露全部在线连接的聚合窗口与在途数量,不得增加账号、应用、连接或消息标签。
- V4结果Outbox必须暴露独立回调Worker的configured/in_flight槽位和Outbox pending/lag,仍只使用固定状态标签。`api_callback`从V4起只在Outbox回调Worker计时,不再混入供应商Submit工作槽;压测结束必须同时核对命令Stream和结果Outbox均`pending=0/lag=0`,并证明API回调故障时供应商槽继续释放、结果不丢失且恢复后只处理一次。
- 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。 - 系统监控标题说明必须明确标注数据来自 Prometheus;“服务关键指标”位于趋势/核心服务区域之后、活动告警之前,并提供统一的“告警阈值设置”入口。安全检测页不重复渲染大号标题,说明文字必须明确标注使用 Fail2ban。
- 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。 - 告警阈值仅开放固定指标的警告/严重数值,不允许前端提交 PromQL、标签、文件路径或持续时间;必须满足警告值小于严重值。配置以 PostgreSQL 保存版本、期望值、生效值和应用状态,经 `promtool check rules` 校验、同目录原子替换和 Prometheus 热加载成功后才标记生效,失败保留上一生效规则并展示原因。
- 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。 - 右上角预警中心增加“系统监控告警”,通过独立轻量接口统计 Prometheus 当前 firing/pending 告警及严重数,跳转系统监控活动告警区;任一预警域失败不得清空其他域。
+8 -1
View File
@@ -61,6 +61,10 @@ CMPP_PUBLIC_HOST=8.160.169.106
CMPP_PUBLIC_PORT=17890 CMPP_PUBLIC_PORT=17890
GATEWAY_STARTUP_RECONNECT_DELAY_MS=1000 GATEWAY_STARTUP_RECONNECT_DELAY_MS=1000
GATEWAY_SUBMIT_WORKER_CONCURRENCY=64 GATEWAY_SUBMIT_WORKER_CONCURRENCY=64
GATEWAY_SUBMIT_RESULT_STREAM=gateway.submit.results
GATEWAY_SUBMIT_RESULT_GROUP=cmpp-api-callback
GATEWAY_SUBMIT_RESULT_CONSUMER=gateway-1
GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY=8
GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64 GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY=64
OBJECT_STORAGE_DRIVER=minio OBJECT_STORAGE_DRIVER=minio
OBJECT_STORAGE_LOCAL_ROOT=/var/lib/cmpp-platform/object-storage OBJECT_STORAGE_LOCAL_ROOT=/var/lib/cmpp-platform/object-storage
@@ -85,6 +89,8 @@ Gateway 的最终 TPS 防线依赖与 API 相同的 Redis。通道连接时会
V3起客户CMPP入站Submit按应用`cmppWindowSize`在单连接内受限并发,全局上限`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`缺省64、最大1024。发布后必须先以已认证测试连接确认`cmpp_gateway_inbound_submit_slots{state="configured"}`等于应用有效窗口,再执行阶梯压测;不得通过调高全局值绕过应用窗口,也不得在未验证Sequence_Id关联、心跳/ACK活性和断线清理时直接提高生产并发。 V3起客户CMPP入站Submit按应用`cmppWindowSize`在单连接内受限并发,全局上限`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`缺省64、最大1024。发布后必须先以已认证测试连接确认`cmpp_gateway_inbound_submit_slots{state="configured"}`等于应用有效窗口,再执行阶梯压测;不得通过调高全局值绕过应用窗口,也不得在未验证Sequence_Id关联、心跳/ACK活性和断线清理时直接提高生产并发。
V4起供应商分片和聚合结果写入`gateway.submit.results`幂等Outbox,由默认8槽、最大1024的独立回调Worker上报API。聚合结果XADD与原命令XACK由Redis Lua原子执行;回调成功后结果事件XACK+XDEL,失败事件留在PEL。发布时必须保留同一Redis数据和AOF,不得清理`gateway.submit.results`、其consumer group或`gateway.submit.results:dedupe:*`;调整`GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY`前须核对API/PostgreSQL承载能力。第91条向前兼容migration增加`SmsSubmitRecord.resultEventId/resultProcessedAt`,用于回调跨重启幂等,不删除历史字段或数据。
服务重启顺序必须是 Gateway 在前、API 在后。API 启动后等待 `GATEWAY_STARTUP_RECONNECT_DELAY_MS`(默认 1 秒),从 PostgreSQL 读取全部 active 通道并重新下发真实连接命令,同时恢复 Gateway 内存连接池和 Redis 权威 TPS key;禁止沿用数据库中重启前的 connected 状态冒充当前连接。 服务重启顺序必须是 Gateway 在前、API 在后。API 启动后等待 `GATEWAY_STARTUP_RECONNECT_DELAY_MS`(默认 1 秒),从 PostgreSQL 读取全部 active 通道并重新下发真实连接命令,同时恢复 Gateway 内存连接池和 Redis 权威 TPS key;禁止沿用数据库中重启前的 connected 状态冒充当前连接。
日报任务默认启用,并由 `REPORT_REFRESH_INTERVAL_MS` 每小时检查一次北京时间业务日是否变化;每个业务日只执行一次 T-4 至 T-1 重算。服务重启后也会自动补跑最近四个完整自然日,确保 72 小时回执更新反映到对账和利润报表。 日报任务默认启用,并由 `REPORT_REFRESH_INTERVAL_MS` 每小时检查一次北京时间业务日是否变化;每个业务日只执行一次 T-4 至 T-1 重算。服务重启后也会自动补跑最近四个完整自然日,确保 72 小时回执更新反映到对账和利润报表。
@@ -145,9 +151,10 @@ curl http://127.0.0.1:8090/health
curl http://127.0.0.1:12026/ curl http://127.0.0.1:12026/
redis-cli -h 127.0.0.1 -p 6379 ping redis-cli -h 127.0.0.1 -p 6379 ping
pg_isready -d "$(grep '^DATABASE_URL=' /etc/cmpp-platform/cmpp-platform.env | cut -d= -f2-)" pg_isready -d "$(grep '^DATABASE_URL=' /etc/cmpp-platform/cmpp-platform.env | cut -d= -f2-)"
grep -E '^(API_ENABLE_SEND_WORKER|API_SEND_WORKER_CONCURRENCY|GATEWAY_SUBMIT_WORKER_CONCURRENCY|GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY)=' /etc/cmpp-platform/cmpp-platform.env grep -E '^(API_ENABLE_SEND_WORKER|API_SEND_WORKER_CONCURRENCY|GATEWAY_SUBMIT_WORKER_CONCURRENCY|GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY|GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY)=' /etc/cmpp-platform/cmpp-platform.env
redis-cli --scan --pattern 'rate:gateway:channel:*' redis-cli --scan --pattern 'rate:gateway:channel:*'
redis-cli XINFO GROUPS gateway.submit.commands redis-cli XINFO GROUPS gateway.submit.commands
redis-cli XINFO GROUPS gateway.submit.results
``` ```
## 回滚 ## 回滚
+8
View File
@@ -4682,6 +4682,14 @@ npm run verify:phase8
| TC-CMPP-PERF-V3-004 | 断线在途清理 | Submit进入API后由客户端断开TCP,随后让API处理完成 | Gateway等待已接受处理收尾后再执行连接关闭回调;会话、Submit barrier和消息映射最终清理,不重新出现幽灵连接,不发生panic | | TC-CMPP-PERF-V3-004 | 断线在途清理 | Submit进入API后由客户端断开TCP,随后让API处理完成 | Gateway等待已接受处理收尾后再执行连接关闭回调;会话、Submit barrier和消息映射最终清理,不重新出现幽灵连接,不发生panic |
| TC-CMPP-PERF-V3-005 | 入站槽位聚合指标 | 建立不同窗口的测试连接并制造部分在途Submit后抓取metrics | `cmpp_gateway_inbound_submit_slots`的configured等于在线连接有效窗口合计、in_flight等于当前业务处理数;只有固定state标签 | | TC-CMPP-PERF-V3-005 | 入站槽位聚合指标 | 建立不同窗口的测试连接并制造部分在途Submit后抓取metrics | `cmpp_gateway_inbound_submit_slots`的configured等于在线连接有效窗口合计、in_flight等于当前业务处理数;只有固定state标签 |
| TC-CMPP-PERF-V3-006 | V3阶梯容量复测 | 在与V2相同的隔离真实后端/数据库/Redis/供应商模拟器中执行10、20、30、40、50条/秒各60秒 | 与V2按同口径比较SubmitResp P50/P95/P99、API阶段、Stream pending/lag、供应商吞吐和资源;遇P95超过5秒、持续积压或服务异常立即停止,未执行档位不记为通过 | | TC-CMPP-PERF-V3-006 | V3阶梯容量复测 | 在与V2相同的隔离真实后端/数据库/Redis/供应商模拟器中执行10、20、30、40、50条/秒各60秒 | 与V2按同口径比较SubmitResp P50/P95/P99、API阶段、Stream pending/lag、供应商吞吐和资源;遇P95超过5秒、持续积压或服务异常立即停止,未执行档位不记为通过 |
| TC-CMPP-PERF-V4-001 | 分片结果幂等入Outbox | 对同一submitId和segmentIndex重复发布两次分片结果 | `gateway.submit.results`只新增一个确定性eventId事件;下一分片仅在前一分片Outbox写入成功后发送 |
| TC-CMPP-PERF-V4-002 | 聚合结果与命令ACK原子性 | 让供应商返回成功,在聚合结果写入与命令ACK边界注入Redis失败并重启Gateway | 不存在“命令已ACK但结果Outbox缺失”状态;重试同一发布脚本不会产生重复聚合事件 |
| TC-CMPP-PERF-V4-003 | API回调不占供应商槽 | API submit-result接口延迟10秒,同时持续让供应商快速返回SubmitResp | 供应商Worker槽在结果写入Outbox后立即释放;API延迟只增加独立回调Worker和Outbox积压,不降低供应商Submit槽可继续补位的能力 |
| TC-CMPP-PERF-V4-004 | 回调失败恢复与逐条ACK | 结果回调第一次返回503,随后恢复201并重启回调Worker | 失败事件保留PEL且未删除;恢复后重新投递,成功时单事件原子ACK+删除,其他事件不受整批等待 |
| TC-CMPP-PERF-V4-005 | API持久幂等 | 对同一聚合eventId重复回调两次,并检查提交记录、计费、余额、重试和任务进度 | `SmsSubmitRecord.resultEventId`只记录一次;第二次直接返回当前结果,不重复扣费、释放、补发或推进状态;不同eventId占用同一submit尝试时拒绝 |
| TC-CMPP-PERF-V4-006 | Outbox有界指标 | 制造回调在途和积压后抓取Gateway metrics | 回调Worker configured/in_flight与真实槽位一致,Outbox pending/lag与Redis consumer group一致,指标不含手机号、消息、submit、企业、应用或通道标签 |
| TC-CMPP-PERF-V4-007 | 双Stream发布后排空 | 完成一档隔离压测并等待异步处理结束 | `gateway.submit.commands``gateway.submit.results`均为`pending=0/lag=0`,结果Outbox成功事件已删除,无Gateway Submit死信;数据库业务数与客户端完全一致 |
| TC-CMPP-PERF-V4-008 | V4阶梯容量复测 | 在V3相同隔离供应商模拟器与真实API/PostgreSQL/Redis/Gateway中依次执行10、20、30、40、50条/秒各60秒 | 对比V3的SubmitResp分位、供应商RTT、命令Stream、结果Outbox、API阶段和资源;任一档出现拒绝、连接错误、P95超过5秒、双Stream持续增长或服务异常立即停止升档 |
| TC-GLOBAL-ALERT-001 | 铃铛分域预警菜单 | 准备签名清退未读消息和安全待处置告警后点击右上角铃铛 | 弹层分开显示“签名清退预警”和“安全检测与封禁”,分别展示真实数量和摘要,角标等于两项之和 | | TC-GLOBAL-ALERT-001 | 铃铛分域预警菜单 | 准备签名清退未读消息和安全待处置告警后点击右上角铃铛 | 弹层分开显示“签名清退预警”和“安全检测与封禁”,分别展示真实数量和摘要,角标等于两项之和 |
| TC-GLOBAL-ALERT-002 | 预警菜单跳转 | 分别点击铃铛中的两个菜单项 | 签名项跳转`/admin/signature-retirement`,安全项跳转`/admin/security-detection`,弹层关闭且对应页面读取真实后端数据 | | TC-GLOBAL-ALERT-002 | 预警菜单跳转 | 分别点击铃铛中的两个菜单项 | 签名项跳转`/admin/signature-retirement`,安全项跳转`/admin/security-detection`,弹层关闭且对应页面读取真实后端数据 |
| TC-GLOBAL-ALERT-003 | 域间故障隔离与轻量轮询 | 分别让一个汇总接口失败并观察30秒轮询请求 | 失败域显示0且另一域数据保留;安全预警使用专用汇总接口,不调用完整overview、规则、代理状态或告警大列表 | | TC-GLOBAL-ALERT-003 | 域间故障隔离与轻量轮询 | 分别让一个汇总接口失败并观察30秒轮询请求 | 失败域显示0且另一域数据保留;安全预警使用专用汇总接口,不调用完整overview、规则、代理状态或告警大列表 |
+14
View File
@@ -3747,3 +3747,17 @@ git diff --check
- 优先级隔离仍未通过:40条/秒priority P95约3912ms、normal P95约3911ms,且两次SubmitResp拒绝都发生在priority连接;全部5996条均为mobile,联通/电信及六通道容量仍待号段识别修复后测试。原始目录名继续沿用既有`lg-v2-*`脚本标签,但被测运行标识和代码均为V3修复版,正式结论以运行标识为准。 - 优先级隔离仍未通过:40条/秒priority P95约3912ms、normal P95约3911ms,且两次SubmitResp拒绝都发生在priority连接;全部5996条均为mobile,联通/电信及六通道容量仍待号段识别修复后测试。原始目录名继续沿用既有`lg-v2-*`脚本标签,但被测运行标识和代码均为V3修复版,正式结论以运行标识为准。
- V3回归已通过Gateway全量`go test ./... -count=1``go vet ./...`、项目内gocmpp测试/vet、API TypeScript正式编译、真实隔离Redis上的SendChain 113/113、4份Stream契约、R6 102声明/14项关键测试、R7、SendChain R10及`git diff --check`。Linux测试机补跑`-race`时因网络无法下载仅供测试的`miniredis``golang.org/x/text`依赖而阻塞,未伪报通过,也未伪造外部依赖。 - V3回归已通过Gateway全量`go test ./... -count=1``go vet ./...`、项目内gocmpp测试/vet、API TypeScript正式编译、真实隔离Redis上的SendChain 113/113、4份Stream契约、R6 102声明/14项关键测试、R7、SendChain R10及`git diff --check`。Linux测试机补跑`-race`时因网络无法下载仅供测试的`miniredis``golang.org/x/text`依赖而阻塞,未伪报通过,也未伪造外部依赖。
- 本轮只使用隔离供应商模拟器,没有发送、补发或重投真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接。`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/``pnpm-lock.yaml`和空文件`=`继续作为受保护项排除提交。 - 本轮只使用隔离供应商模拟器,没有发送、补发或重投真实短信,没有修改通道账号、密码、启停状态、企业余额或客户连接。`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/``pnpm-lock.yaml`和空文件`=`继续作为受保护项排除提交。
# 2026-08-20 CMPP压测优化V4:供应商结果幂等Outbox、测试环境发布与阶梯复测
- V4将供应商分片结果和聚合结果先写入Redis Stream `gateway.submit.results`,API回调由独立持续有界工作池执行,供应商Submit工作槽不再等待API往返。分片事件ID为`submit:<submitId>:segment:<index>`,聚合事件ID为`submit:<submitId>:aggregate`;Lua脚本保证分片去重键与XADD原子、聚合XADD与原命令XACK原子、成功回调XACK与XDEL原子。失败事件保留在PEL并由`XAUTOCLAIM`恢复,回收阈值30秒高于10秒HTTP超时,避免多Gateway实例在回调仍执行时并发重领。
- 第91条向前兼容migration `20260820130000_add_submit_result_idempotency``SmsSubmitRecord`增加唯一可空`resultEventId``resultProcessedAt`。API重复收到同一已完成事件直接返回当前消息,冲突事件ID拒绝;测试覆盖重复回调不重复计费、重试或下游业务副作用。新增第5类Gateway队列契约示例,Outbox指标覆盖回调Worker槽位和结果Stream pending/lag。
- 本地验证通过:Gateway全量`go test ./... -count=1``go vet ./...`,API 42套/483项断言全部通过,API与前端TypeScript正式编译、Vite生产构建、5份队列契约、R0/R6/R7及SendChain R10结构门禁、`git diff --check`均通过。全量Jest在断言完成后仍因仓库既有异步句柄不自行退出,本轮未把人工终止后的进程伪报为完整退出码0;使用本机真实Redis补跑,不伪造外部依赖。
- 仅发布到`100.93.204.60`虚拟机测试环境;预生产`8.160.169.106`只读复核标识保持`433b2ee56f6016ad8afff1bac73f510b8fd53083+gateway-v2.cd7bb8d05e7b`,未发布、未回退、未压测。测试环境发布前恢复资产位于`/opt/cmpp-platform-backups/v4-20260820T052241Z`,包含PostgreSQL自定义格式备份、运行源码、环境和systemd配置;`pg_restore --list`、源码tar可读性及`SHA256SUMS`全部通过。
- 最终运行包`outputs/cmpp-v4-runtime-final-20260820-141323.tar.gz`共827项、1641927字节,本地与测试机SHA-256均为`a63885439ed53d53a3a012048d5fd1a9aa67772503fd778ec510ec6a38a9944f`,排除依赖、构建产物、`outputs``*.tsbuildinfo``pnpm-lock.yaml`和空文件`=`。测试环境运行标识为`485af688d21ee0bdb98281f4c20d151d69f7889f+workspace.v4.a63885439ed5`;第91条migration仅应用一次,API、Gateway、安全代理、MinIO、PostgreSQL和Redis健康,最终供应商连接6/6,两条Stream均`pending=0/lag=0`
- 首轮默认32槽10条/秒恰逢重启积压回执回放,599/599受理但回执2536、P95=3294ms;积压排空后再测仍为P95=671ms。把测试配置收敛为8槽后,10条/秒599/599、P50/P95/P99=`39/146/304ms`、回执603。基于该对照,代码、示例环境和发布文档的默认值同步改为8;32槽不作为推荐配置。
- 8槽正式阶梯结果:20条/秒1199/1199受理、P50/P95/P99=`45/1129/1259ms`30条/秒1799/1799受理、`1081/1626/1885ms`40条/秒2399/2399受理、`2049/2330/2386ms`。三档均无连接错误;相对V3,20/30/40条/秒P95分别由1427/2033/3912ms降至1129/1626/2330ms,且40条/秒从2条10秒超时改进为零拒绝,确认安全档位由30提升到40条/秒。
- 解耦证据:20条/秒命令Stream未投递lag峰值0,结果Outbox峰值`pending=8/lag=242`后归零;30条/秒峰值约`pending=8/lag=1076`40条/秒峰值约`pending=9/lag=1639`,压测结束后约36秒归零并连续保持0/0。40条/秒时供应商命令与结果回调已分开排队,结果回调积压不再占用供应商Submit槽;但Outbox回落时间已成为容量判定的一部分,不能只看客户SubmitResp。
- 50条/秒触发停止线:因客户端背压只生成2790条而非约2999条,2787条成功、3条等待API满10秒后返回result=9P50/P95/P99=`6765/7236/7789ms`throttled ticks=3184。结束后命令Stream一度`pending=64/lag=489`、结果Outbox`pending=8/lag=1690`;命令约1分钟内、结果随后约20秒内归零。日志显示API饱和时协议遥测和连接状态回调超时,并暴露既有`CmppDownstreamConnection`创建/更新的P2002/P2025竞态;未出现`resultEventId`唯一键冲突或Outbox事件失败。停止后未继续上探。
- 整个窗口客户侧共9984条业务提交,真实PostgreSQL按`queuedAt`精确新增9984条;窗口内7922条实际供应商提交记录全部具有非空且互不重复的`resultEventId`,重复事件组0、Gateway Submit死信0。提交结果为accepted 7392、rejected 147、timeout 383;客户有限回执收集窗口和重连积压回执不替代数据库/Stream对账。最终确认40条/秒为当前测试环境安全档,50条/秒瓶颈转为API/数据库同步入站及遥测争用,后续应继续V1的重复查询/零散写入合并和P1分阶段指标分析。
- 本轮只连接隔离供应商模拟器`100.91.249.119:17900`,没有发送、补发或重投真实短信,没有修改真实通道账号、密码、启停状态、企业余额或客户连接。压测原始目录沿用既有`lg-v2-*`名称,但被测运行标识与结论均为V4;完整V4报告保存在短信平台测试项目。受保护的`api/tsconfig.build.tsbuildinfo``tsconfig.tsbuildinfo``outputs/``pnpm-lock.yaml`和空文件`=`继续排除提交、不删除、不归因。
+1
View File
@@ -13,6 +13,7 @@
- 暴露健康检查和最小指标。 - 暴露健康检查和最小指标。
- Redis Stream Submit Worker 使用持续补位的有界工作池并逐条 ACK;默认并发64,可用`GATEWAY_SUBMIT_WORKER_CONCURRENCY`调整,最大1024。供应商通道的真实上限仍由TPS限速、连接数和CMPP窗口共同决定。 - Redis Stream Submit Worker 使用持续补位的有界工作池并逐条 ACK;默认并发64,可用`GATEWAY_SUBMIT_WORKER_CONCURRENCY`调整,最大1024。供应商通道的真实上限仍由TPS限速、连接数和CMPP窗口共同决定。
- 客户CMPP入站Submit按认证接口返回的应用`cmppWindowSize`在单连接内并发,Gateway再以`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`实施默认64、最大1024的全局单连接保护。登录保持串行,心跳和Deliver ACK不等待慢SubmitSubmitResp依靠Sequence_Id关联,允许按完成顺序返回。 - 客户CMPP入站Submit按认证接口返回的应用`cmppWindowSize`在单连接内并发,Gateway再以`GATEWAY_CMPP_INBOUND_MAX_CONCURRENCY`实施默认64、最大1024的全局单连接保护。登录保持串行,心跳和Deliver ACK不等待慢SubmitSubmitResp依靠Sequence_Id关联,允许按完成顺序返回。
- 供应商SubmitResp先写入Redis Stream幂等Outbox`gateway.submit.results`,聚合结果入Outbox与原Submit命令ACK使用同一Lua脚本;独立默认8槽回调Worker再调用API,供应商工作槽不等待API。每个事件使用确定性`eventId`和7天Redis去重键,API在`SmsSubmitRecord.resultEventId`完成持久幂等;成功回调后Outbox事件原子ACK+删除,失败事件留在PEL恢复。
## 建议骨架 ## 建议骨架
+48
View File
@@ -13,7 +13,9 @@ import (
"cmpp-platform/gateway/internal/health" "cmpp-platform/gateway/internal/health"
"cmpp-platform/gateway/internal/inbound" "cmpp-platform/gateway/internal/inbound"
platformmetrics "cmpp-platform/gateway/internal/metrics" platformmetrics "cmpp-platform/gateway/internal/metrics"
"cmpp-platform/gateway/internal/queue"
"cmpp-platform/gateway/internal/ratelimit" "cmpp-platform/gateway/internal/ratelimit"
"cmpp-platform/gateway/internal/resultoutbox"
"cmpp-platform/gateway/internal/submitworker" "cmpp-platform/gateway/internal/submitworker"
"cmpp-platform/gateway/internal/upstream" "cmpp-platform/gateway/internal/upstream"
@@ -32,6 +34,7 @@ func main() {
apiBaseURL := os.Getenv("API_BASE_URL") apiBaseURL := os.Getenv("API_BASE_URL")
upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL} upstreamManager := &upstream.Manager{APIBaseURL: apiBaseURL}
var worker *submitworker.Worker var worker *submitworker.Worker
var resultOutbox *resultoutbox.Outbox
channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL")) channelLimiter, err := ratelimit.New(os.Getenv("REDIS_URL"))
if err != nil { if err != nil {
log.Fatalf("gateway channel rate limiter init failed: %v", err) log.Fatalf("gateway channel rate limiter init failed: %v", err)
@@ -70,12 +73,26 @@ func main() {
worker.Consumer = getenv("GATEWAY_SUBMIT_CONSUMER", "gateway-1") worker.Consumer = getenv("GATEWAY_SUBMIT_CONSUMER", "gateway-1")
worker.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_WORKER_CONCURRENCY", 64) worker.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_WORKER_CONCURRENCY", 64)
worker.APIBaseURL = apiBaseURL worker.APIBaseURL = apiBaseURL
resultOutbox = resultoutbox.New(worker.Redis)
resultOutbox.Stream = getenv("GATEWAY_SUBMIT_RESULT_STREAM", "gateway.submit.results")
resultOutbox.Group = getenv("GATEWAY_SUBMIT_RESULT_GROUP", "cmpp-api-callback")
resultOutbox.Consumer = getenv("GATEWAY_SUBMIT_RESULT_CONSUMER", "gateway-1")
resultOutbox.APIBaseURL = apiBaseURL
resultOutbox.Concurrency = positiveEnvInt("GATEWAY_SUBMIT_RESULT_WORKER_CONCURRENCY", 8)
worker.ResultOutbox = resultOutbox
upstreamManager.SubmitSegmentPublisher = resultOutbox
go func() { go func() {
log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer) log.Printf("cmpp gateway submit worker consuming stream=%s group=%s consumer=%s", worker.Stream, worker.Group, worker.Consumer)
if err := worker.Run(context.Background()); err != nil { if err := worker.Run(context.Background()); err != nil {
log.Printf("gateway submit worker stopped: %v", err) log.Printf("gateway submit worker stopped: %v", err)
} }
}() }()
go func() {
log.Printf("cmpp gateway result Outbox consuming stream=%s group=%s consumer=%s", resultOutbox.StreamName(), resultOutbox.GroupName(), resultOutbox.Consumer)
if err := resultOutbox.Run(context.Background()); err != nil {
log.Printf("gateway result Outbox worker stopped: %v", err)
}
}()
} }
} }
@@ -91,6 +108,11 @@ func main() {
snapshot.SubmitWorkerConcurrency = worker.ConfiguredConcurrency() snapshot.SubmitWorkerConcurrency = worker.ConfiguredConcurrency()
snapshot.SubmitWorkerInFlight = worker.InFlight() snapshot.SubmitWorkerInFlight = worker.InFlight()
} }
if resultOutbox != nil {
snapshot.ResultWorkerUp = true
snapshot.ResultWorkerConcurrency = resultOutbox.ConfiguredConcurrency()
snapshot.ResultWorkerInFlight = resultOutbox.InFlight()
}
snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight = inbound.SubmitSlotSnapshot() snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight = inbound.SubmitSlotSnapshot()
if worker == nil || worker.Redis == nil { if worker == nil || worker.Redis == nil {
return snapshot return snapshot
@@ -117,12 +139,38 @@ func main() {
snapshot.QueueOldestAgeSeconds = max(0, time.Since(time.UnixMilli(milliseconds)).Seconds()) snapshot.QueueOldestAgeSeconds = max(0, time.Since(time.UnixMilli(milliseconds)).Seconds())
} }
} }
if resultOutbox != nil {
resultPending, resultErr := worker.Redis.XPending(ctx, resultOutbox.StreamName(), resultOutbox.GroupName()).Result()
if resultErr == nil {
snapshot.ResultQueueAvailable = true
snapshot.ResultQueuePending = resultPending.Count
}
resultGroups, resultErr := worker.Redis.XInfoGroups(ctx, resultOutbox.StreamName()).Result()
if resultErr == nil {
for _, group := range resultGroups {
if group.Name == resultOutbox.GroupName() {
snapshot.ResultQueueLag = group.Lag
break
}
}
}
}
return snapshot return snapshot
})) }))
control.Register(mux, control.Server{ control.Register(mux, control.Server{
APIBaseURL: apiBaseURL, APIBaseURL: apiBaseURL,
Upstream: upstreamManager, Upstream: upstreamManager,
Limiter: channelLimiter, Limiter: channelLimiter,
Submit: func(ctx context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
result, submitErr := upstreamManager.Submit(ctx, command)
if resultOutbox == nil {
return result, submitErr
}
if publishErr := resultOutbox.PublishSubmitResult(ctx, command, result); publishErr != nil {
return result, publishErr
}
return result, submitErr
},
RecoveryCandidates: func(ctx context.Context) ([]inbound.DownstreamPresence, error) { RecoveryCandidates: func(ctx context.Context) ([]inbound.DownstreamPresence, error) {
return inbound.ListRecoveryCandidates(ctx, presenceStore) return inbound.ListRecoveryCandidates(ctx, presenceStore)
}, },
+12
View File
@@ -43,12 +43,18 @@ type Snapshot struct {
SubmitWorkerUp bool SubmitWorkerUp bool
SubmitWorkerConcurrency int SubmitWorkerConcurrency int
SubmitWorkerInFlight int64 SubmitWorkerInFlight int64
ResultWorkerUp bool
ResultWorkerConcurrency int
ResultWorkerInFlight int64
InboundSubmitConcurrency int InboundSubmitConcurrency int
InboundSubmitInFlight int64 InboundSubmitInFlight int64
QueueAvailable bool QueueAvailable bool
QueuePending int64 QueuePending int64
QueueLag int64 QueueLag int64
QueueOldestAgeSeconds float64 QueueOldestAgeSeconds float64
ResultQueueAvailable bool
ResultQueuePending int64
ResultQueueLag int64
} }
type SnapshotFunc func(context.Context) Snapshot type SnapshotFunc func(context.Context) Snapshot
@@ -118,12 +124,18 @@ func Handler(load SnapshotFunc) http.Handler {
fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected) fmt.Fprintf(response, "# HELP cmpp_gateway_downstream_connections Authenticated client connections.\n# TYPE cmpp_gateway_downstream_connections gauge\ncmpp_gateway_downstream_connections %d\n", snapshot.DownstreamConnected)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp)) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_up Whether the submit worker was initialized.\n# TYPE cmpp_gateway_submit_worker_up gauge\ncmpp_gateway_submit_worker_up %d\n", boolNumber(snapshot.SubmitWorkerUp))
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_worker_slots Configured and active bounded submit worker slots.\n# TYPE cmpp_gateway_submit_worker_slots gauge\ncmpp_gateway_submit_worker_slots{state=\"configured\"} %d\ncmpp_gateway_submit_worker_slots{state=\"in_flight\"} %d\n", snapshot.SubmitWorkerConcurrency, snapshot.SubmitWorkerInFlight)
fmt.Fprintf(response, "# HELP cmpp_gateway_result_callback_worker_up Whether the asynchronous result callback worker was initialized.\n# TYPE cmpp_gateway_result_callback_worker_up gauge\ncmpp_gateway_result_callback_worker_up %d\n", boolNumber(snapshot.ResultWorkerUp))
fmt.Fprintf(response, "# HELP cmpp_gateway_result_callback_worker_slots Configured and active result callback worker slots.\n# TYPE cmpp_gateway_result_callback_worker_slots gauge\ncmpp_gateway_result_callback_worker_slots{state=\"configured\"} %d\ncmpp_gateway_result_callback_worker_slots{state=\"in_flight\"} %d\n", snapshot.ResultWorkerConcurrency, snapshot.ResultWorkerInFlight)
fmt.Fprintf(response, "# HELP cmpp_gateway_inbound_submit_slots Configured and active authenticated client Submit slots.\n# TYPE cmpp_gateway_inbound_submit_slots gauge\ncmpp_gateway_inbound_submit_slots{state=\"configured\"} %d\ncmpp_gateway_inbound_submit_slots{state=\"in_flight\"} %d\n", snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight) fmt.Fprintf(response, "# HELP cmpp_gateway_inbound_submit_slots Configured and active authenticated client Submit slots.\n# TYPE cmpp_gateway_inbound_submit_slots gauge\ncmpp_gateway_inbound_submit_slots{state=\"configured\"} %d\ncmpp_gateway_inbound_submit_slots{state=\"in_flight\"} %d\n", snapshot.InboundSubmitConcurrency, snapshot.InboundSubmitInFlight)
if snapshot.QueueAvailable { if snapshot.QueueAvailable {
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_pending Pending entries owned by the consumer group.\n# TYPE cmpp_gateway_submit_queue_pending gauge\ncmpp_gateway_submit_queue_pending %d\n", snapshot.QueuePending)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_lag Undelivered entries for the consumer group.\n# TYPE cmpp_gateway_submit_queue_lag gauge\ncmpp_gateway_submit_queue_lag %d\n", snapshot.QueueLag)
fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_oldest_pending_age_seconds Age of the oldest pending entry.\n# TYPE cmpp_gateway_submit_queue_oldest_pending_age_seconds gauge\ncmpp_gateway_submit_queue_oldest_pending_age_seconds %f\n", snapshot.QueueOldestAgeSeconds) fmt.Fprintf(response, "# HELP cmpp_gateway_submit_queue_oldest_pending_age_seconds Age of the oldest pending entry.\n# TYPE cmpp_gateway_submit_queue_oldest_pending_age_seconds gauge\ncmpp_gateway_submit_queue_oldest_pending_age_seconds %f\n", snapshot.QueueOldestAgeSeconds)
} }
if snapshot.ResultQueueAvailable {
fmt.Fprintf(response, "# HELP cmpp_gateway_result_outbox_pending Pending result callbacks owned by the consumer group.\n# TYPE cmpp_gateway_result_outbox_pending gauge\ncmpp_gateway_result_outbox_pending %d\n", snapshot.ResultQueuePending)
fmt.Fprintf(response, "# HELP cmpp_gateway_result_outbox_lag Undelivered result callbacks for the consumer group.\n# TYPE cmpp_gateway_result_outbox_lag gauge\ncmpp_gateway_result_outbox_lag %d\n", snapshot.ResultQueueLag)
}
}) })
} }
+5 -1
View File
@@ -16,7 +16,7 @@ func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
request := httptest.NewRequest(http.MethodGet, "/metrics", nil) request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
response := httptest.NewRecorder() response := httptest.NewRecorder()
Handler(func(context.Context) Snapshot { Handler(func(context.Context) Snapshot {
return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, SubmitWorkerConcurrency: 64, SubmitWorkerInFlight: 7, InboundSubmitConcurrency: 96, InboundSubmitInFlight: 9, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12} return Snapshot{UpstreamDesired: 2, UpstreamConnected: 1, DownstreamConnected: 3, SubmitWorkerUp: true, SubmitWorkerConcurrency: 64, SubmitWorkerInFlight: 7, ResultWorkerUp: true, ResultWorkerConcurrency: 32, ResultWorkerInFlight: 5, InboundSubmitConcurrency: 96, InboundSubmitInFlight: 9, QueueAvailable: true, QueuePending: 4, QueueLag: 5, QueueOldestAgeSeconds: 12, ResultQueueAvailable: true, ResultQueuePending: 6, ResultQueueLag: 8}
}).ServeHTTP(response, request) }).ServeHTTP(response, request)
body := response.Body.String() body := response.Body.String()
@@ -28,6 +28,10 @@ func TestMetricsHandlerExportsOnlyAggregateGatewayState(t *testing.T) {
`cmpp_gateway_submit_stage_duration_seconds_count{stage="rate_limit_wait",result="success"} 1`, `cmpp_gateway_submit_stage_duration_seconds_count{stage="rate_limit_wait",result="success"} 1`,
`cmpp_gateway_submit_worker_slots{state="configured"} 64`, `cmpp_gateway_submit_worker_slots{state="configured"} 64`,
`cmpp_gateway_submit_worker_slots{state="in_flight"} 7`, `cmpp_gateway_submit_worker_slots{state="in_flight"} 7`,
`cmpp_gateway_result_callback_worker_slots{state="configured"} 32`,
`cmpp_gateway_result_callback_worker_slots{state="in_flight"} 5`,
`cmpp_gateway_result_outbox_pending 6`,
`cmpp_gateway_result_outbox_lag 8`,
`cmpp_gateway_inbound_submit_slots{state="configured"} 96`, `cmpp_gateway_inbound_submit_slots{state="configured"} 96`,
`cmpp_gateway_inbound_submit_slots{state="in_flight"} 9`, `cmpp_gateway_inbound_submit_slots{state="in_flight"} 9`,
} { } {
+260
View File
@@ -0,0 +1,260 @@
package resultoutbox
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"time"
"cmpp-platform/gateway/internal/queue"
"github.com/redis/go-redis/v9"
)
const (
defaultStream = "gateway.submit.results"
defaultGroup = "cmpp-api-callback"
defaultConsumer = "gateway-1"
defaultDedupeTTL = 7 * 24 * time.Hour
)
var publishScript = redis.NewScript(`
local inserted = redis.call('SET', KEYS[2], '1', 'NX', 'EX', ARGV[1])
if inserted then
redis.call('XADD', KEYS[1], '*', 'data', ARGV[2])
return 1
end
return 0
`)
var publishAndAckScript = redis.NewScript(`
local inserted = redis.call('SET', KEYS[2], '1', 'NX', 'EX', ARGV[1])
if inserted then
redis.call('XADD', KEYS[1], '*', 'data', ARGV[2])
end
redis.call('XACK', KEYS[3], ARGV[3], ARGV[4])
if inserted then return 1 end
return 0
`)
type Event struct {
SchemaVersion string `json:"schemaVersion"`
EventID string `json:"eventId"`
EventType string `json:"eventType"`
Path string `json:"path"`
TraceID string `json:"traceId,omitempty"`
MessageID string `json:"messageId"`
ChannelID string `json:"channelId"`
SubmitID string `json:"submitId"`
Payload json.RawMessage `json:"payload"`
CreatedAt time.Time `json:"createdAt"`
}
type Outbox struct {
Redis *redis.Client
Stream string
Group string
Consumer string
DedupeTTL time.Duration
APIBaseURL string
HTTPTimeout time.Duration
Concurrency int
MinIdle time.Duration
inFlight atomic.Int64
}
func New(client *redis.Client) *Outbox {
return &Outbox{Redis: client}
}
func (o *Outbox) PublishSubmitSegment(ctx context.Context, command queue.SubmitCommand, segment queue.SubmitSegmentResult) error {
payload := struct {
queue.Envelope
SubmitID string `json:"submitId,omitempty"`
queue.SubmitSegmentResult
}{
Envelope: command.Envelope,
SubmitID: command.SubmitID,
SubmitSegmentResult: segment,
}
event, err := newEvent(
fmt.Sprintf("submit:%s:segment:%d", command.SubmitID, segment.SegmentIndex),
"submit_segment_result",
"/gateway/events/submit-segment-result",
command,
payload,
)
if err != nil {
return err
}
return o.publish(ctx, event)
}
func (o *Outbox) PublishSubmitResultAndAck(
ctx context.Context,
commandStream string,
commandGroup string,
commandMessageID string,
command queue.SubmitCommand,
result queue.SubmitResult,
) error {
if o.Redis == nil {
return fmt.Errorf("result Outbox Redis client is required")
}
event, err := newEvent(
fmt.Sprintf("submit:%s:aggregate", command.SubmitID),
"submit_result",
"/gateway/events/submit-result",
command,
result,
)
if err != nil {
return err
}
data, err := json.Marshal(event)
if err != nil {
return err
}
// XADD and command XACK share one Redis script so a process crash cannot leave
// an acknowledged supplier command without its aggregate result in the Outbox.
_, err = publishAndAckScript.Run(
ctx,
o.Redis,
[]string{o.stream(), o.dedupeKey(event.EventID), commandStream},
int64(o.dedupeTTL().Seconds()),
string(data),
commandGroup,
commandMessageID,
).Result()
return err
}
func (o *Outbox) PublishSubmitResult(ctx context.Context, command queue.SubmitCommand, result queue.SubmitResult) error {
event, err := newEvent(
fmt.Sprintf("submit:%s:aggregate", command.SubmitID),
"submit_result",
"/gateway/events/submit-result",
command,
result,
)
if err != nil {
return err
}
return o.publish(ctx, event)
}
func (o *Outbox) publish(ctx context.Context, event Event) error {
if o.Redis == nil {
return fmt.Errorf("result Outbox Redis client is required")
}
data, err := json.Marshal(event)
if err != nil {
return err
}
_, err = publishScript.Run(
ctx,
o.Redis,
[]string{o.stream(), o.dedupeKey(event.EventID)},
int64(o.dedupeTTL().Seconds()),
string(data),
).Result()
return err
}
func newEvent(eventID string, eventType string, path string, command queue.SubmitCommand, payload any) (Event, error) {
data, err := json.Marshal(payload)
if err != nil {
return Event{}, err
}
var object map[string]interface{}
if err := json.Unmarshal(data, &object); err != nil {
return Event{}, err
}
// The API stores the deterministic ID on the submit attempt after successful
// processing, making callback redelivery idempotent across Gateway restarts.
object["eventId"] = eventID
data, err = json.Marshal(object)
if err != nil {
return Event{}, err
}
return Event{
SchemaVersion: queue.SchemaVersion,
EventID: eventID,
EventType: eventType,
Path: path,
TraceID: command.TraceID,
MessageID: command.MessageID,
ChannelID: command.ChannelID,
SubmitID: command.SubmitID,
Payload: data,
CreatedAt: time.Now().UTC(),
}, nil
}
func EventFromStreamValues(values map[string]interface{}) (Event, error) {
raw, ok := values["data"]
if !ok {
return Event{}, fmt.Errorf("result Outbox data field is required")
}
var data string
switch value := raw.(type) {
case string:
data = value
case []byte:
data = string(value)
default:
data = fmt.Sprint(value)
}
var event Event
if err := json.Unmarshal([]byte(data), &event); err != nil {
return Event{}, err
}
if event.SchemaVersion != queue.SchemaVersion || event.EventID == "" || event.MessageID == "" || event.SubmitID == "" {
return Event{}, fmt.Errorf("invalid result Outbox envelope")
}
if event.Path != "/gateway/events/submit-result" && event.Path != "/gateway/events/submit-segment-result" {
return Event{}, fmt.Errorf("unsupported result Outbox path %q", event.Path)
}
if len(event.Payload) == 0 {
return Event{}, fmt.Errorf("result Outbox payload is required")
}
return event, nil
}
func (o *Outbox) stream() string {
if strings.TrimSpace(o.Stream) != "" {
return o.Stream
}
return defaultStream
}
func (o *Outbox) group() string {
if strings.TrimSpace(o.Group) != "" {
return o.Group
}
return defaultGroup
}
func (o *Outbox) consumer() string {
if strings.TrimSpace(o.Consumer) != "" {
return o.Consumer
}
return defaultConsumer
}
func (o *Outbox) dedupeTTL() time.Duration {
if o.DedupeTTL > 0 {
return o.DedupeTTL
}
return defaultDedupeTTL
}
func (o *Outbox) dedupeKey(eventID string) string {
return o.stream() + ":dedupe:" + eventID
}
func (o *Outbox) StreamName() string { return o.stream() }
func (o *Outbox) GroupName() string { return o.group() }
func (o *Outbox) InFlight() int64 { return o.inFlight.Load() }
@@ -0,0 +1,129 @@
package resultoutbox
import (
"context"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"cmpp-platform/gateway/internal/queue"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
)
func TestPublishSubmitSegmentIsIdempotent(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
outbox := New(client)
command := testCommand()
segment := queue.SubmitSegmentResult{SegmentTotal: 1, SegmentIndex: 1, SequenceID: 7, GatewayMessageID: "88", SubmitStatus: "accepted"}
if err := outbox.PublishSubmitSegment(context.Background(), command, segment); err != nil {
t.Fatalf("first publish: %v", err)
}
if err := outbox.PublishSubmitSegment(context.Background(), command, segment); err != nil {
t.Fatalf("duplicate publish: %v", err)
}
if got := client.XLen(context.Background(), outbox.StreamName()).Val(); got != 1 {
t.Fatalf("stream length = %d, want 1", got)
}
}
func TestPublishAggregateAndCommandAckAreAtomicAndIdempotent(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
ctx := context.Background()
commandStream := "gateway.submit.commands"
commandGroup := "cmpp-gateway"
if err := client.XGroupCreateMkStream(ctx, commandStream, commandGroup, "0").Err(); err != nil {
t.Fatalf("create command group: %v", err)
}
commandID, err := client.XAdd(ctx, &redis.XAddArgs{Stream: commandStream, Values: map[string]interface{}{"data": "command"}}).Result()
if err != nil {
t.Fatalf("add command: %v", err)
}
if _, err := client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: commandGroup, Consumer: "gateway-1", Streams: []string{commandStream, ">"}, Count: 1}).Result(); err != nil {
t.Fatalf("claim command: %v", err)
}
outbox := New(client)
command := testCommand()
result := queue.SubmitResult{Envelope: command.Envelope, SubmitID: command.SubmitID, GatewayMessageID: "99", SubmitStatus: "accepted"}
for attempt := 0; attempt < 2; attempt++ {
if err := outbox.PublishSubmitResultAndAck(ctx, commandStream, commandGroup, commandID, command, result); err != nil {
t.Fatalf("publish attempt %d: %v", attempt+1, err)
}
}
pending, err := client.XPending(ctx, commandStream, commandGroup).Result()
if err != nil {
t.Fatalf("command pending: %v", err)
}
if pending.Count != 0 {
t.Fatalf("command pending = %d, want 0", pending.Count)
}
if got := client.XLen(ctx, outbox.StreamName()).Val(); got != 1 {
t.Fatalf("result stream length = %d, want 1", got)
}
}
func TestCallbackWorkerRetriesAndOnlyDeletesAfterSuccess(t *testing.T) {
mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.Header.Get("X-CMPP-Result-Event-ID") == "" {
t.Error("missing result event id header")
}
if calls.Add(1) == 1 {
http.Error(response, "temporary failure", http.StatusServiceUnavailable)
return
}
response.WriteHeader(http.StatusCreated)
}))
defer server.Close()
outbox := New(client)
outbox.APIBaseURL = server.URL
outbox.MinIdle = 10 * time.Millisecond
outbox.Concurrency = 1
command := testCommand()
if err := outbox.PublishSubmitSegment(context.Background(), command, queue.SubmitSegmentResult{
SegmentTotal: 1, SegmentIndex: 1, SequenceID: 7, GatewayMessageID: "88", SubmitStatus: "accepted",
}); err != nil {
t.Fatalf("publish segment: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- outbox.Run(ctx) }()
deadline := time.Now().Add(5 * time.Second)
for calls.Load() < 2 || client.XLen(context.Background(), outbox.StreamName()).Val() != 0 {
if time.Now().After(deadline) {
t.Fatalf("calls=%d streamLength=%d", calls.Load(), client.XLen(context.Background(), outbox.StreamName()).Val())
}
time.Sleep(10 * time.Millisecond)
}
cancel()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("worker did not stop")
}
}
func testCommand() queue.SubmitCommand {
return queue.SubmitCommand{
Envelope: queue.Envelope{
SchemaVersion: queue.SchemaVersion,
MessageType: queue.MessageTypeSubmitCommand,
TraceID: "trace-1",
MessageID: "message-1",
ChannelID: "channel-1",
CreatedAt: time.Now().UTC(),
},
SubmitID: "submit-1",
}
}
+275
View File
@@ -0,0 +1,275 @@
package resultoutbox
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"log"
"net/http"
"strings"
"sync"
"time"
"cmpp-platform/gateway/internal/metrics"
"github.com/redis/go-redis/v9"
)
const (
// Eight callbacks matched the test VM's API/PostgreSQL capacity through
// 40 rps. A larger default caused callback bursts to contend with inbound
// persistence; operators can still raise it after measuring both queues.
defaultConcurrency = 8
// Keep the reclaim threshold above the callback timeout. Otherwise another
// Gateway replica could reclaim a still-running callback and execute the same
// business transition concurrently before the API stores its idempotency ID.
defaultMinIdle = 30 * time.Second
defaultBlock = 2 * time.Second
defaultHTTPTimeout = 10 * time.Second
)
var acknowledgeAndDeleteScript = redis.NewScript(`
redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
redis.call('XDEL', KEYS[1], ARGV[2])
return 1
`)
func (o *Outbox) Run(ctx context.Context) error {
if o.Redis == nil {
return fmt.Errorf("result Outbox Redis client is required")
}
if strings.TrimSpace(o.APIBaseURL) == "" {
return fmt.Errorf("result Outbox API base URL is required")
}
if err := o.ensureGroup(ctx); err != nil {
return err
}
pool := newCallbackPool(ctx, o, o.concurrency())
defer pool.wait()
for {
if err := o.recoverPending(ctx, pool); err != nil && ctx.Err() == nil {
log.Printf("gateway result Outbox pending recovery failed: %v", err)
sleep(ctx, time.Second)
continue
}
if ctx.Err() != nil {
return ctx.Err()
}
if err := o.consumeOnce(ctx, pool); err != nil && ctx.Err() == nil {
log.Printf("gateway result Outbox consume failed: %v", err)
sleep(ctx, time.Second)
continue
}
if ctx.Err() != nil {
return ctx.Err()
}
}
}
func (o *Outbox) ensureGroup(ctx context.Context) error {
err := o.Redis.XGroupCreateMkStream(ctx, o.stream(), o.group(), "0").Err()
if err == nil || strings.Contains(err.Error(), "BUSYGROUP") {
return nil
}
return err
}
func (o *Outbox) consumeOnce(ctx context.Context, pool *callbackPool) error {
available, err := pool.waitForCapacity(ctx)
if err != nil {
return err
}
streams, err := o.Redis.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: o.group(), Consumer: o.consumer(), Streams: []string{o.stream(), ">"},
Count: int64(available), Block: defaultBlock,
}).Result()
if errors.Is(err, redis.Nil) {
return nil
}
if err != nil {
return err
}
for _, stream := range streams {
for _, message := range stream.Messages {
if !pool.dispatch(message) {
return fmt.Errorf("gateway result Outbox capacity accounting mismatch")
}
}
}
return nil
}
func (o *Outbox) recoverPending(ctx context.Context, pool *callbackPool) error {
if pool.available() == 0 {
return nil
}
messages, _, err := o.Redis.XAutoClaim(ctx, &redis.XAutoClaimArgs{
Stream: o.stream(), Group: o.group(), Consumer: o.consumer(), MinIdle: o.minIdle(),
Start: "0-0", Count: int64(pool.available()),
}).Result()
if errors.Is(err, redis.Nil) {
return nil
}
if err != nil {
return err
}
for _, message := range messages {
if !pool.dispatch(message) {
return fmt.Errorf("gateway result Outbox recovery capacity accounting mismatch")
}
}
return nil
}
func (o *Outbox) processMessage(ctx context.Context, message redis.XMessage) error {
event, err := EventFromStreamValues(message.Values)
if err != nil {
// Malformed internal events cannot be delivered. Keep them pending for operator
// evidence instead of ACKing and silently losing a supplier result.
return err
}
startedAt := time.Now()
err = o.post(ctx, event)
metrics.ObserveSubmitStage("api_callback", err == nil, time.Since(startedAt))
if err != nil {
return err
}
// Result events have a bounded dedupe key, so successful callbacks can be
// ACKed and deleted atomically instead of turning the Outbox into an archive.
return acknowledgeAndDeleteScript.Run(
ctx,
o.Redis,
[]string{o.stream()},
o.group(),
message.ID,
).Err()
}
func (o *Outbox) post(ctx context.Context, event Event) error {
client := &http.Client{Timeout: o.httpTimeout()}
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
strings.TrimRight(o.APIBaseURL, "/")+event.Path,
bytes.NewReader(event.Payload),
)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-CMPP-Result-Event-ID", event.EventID)
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return fmt.Errorf("result callback returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
return nil
}
type callbackPool struct {
ctx context.Context
outbox *Outbox
slots chan struct{}
completed chan struct{}
group sync.WaitGroup
mu sync.Mutex
active map[string]struct{}
}
func newCallbackPool(ctx context.Context, outbox *Outbox, concurrency int) *callbackPool {
return &callbackPool{
ctx: ctx, outbox: outbox, slots: make(chan struct{}, concurrency), completed: make(chan struct{}, concurrency), active: make(map[string]struct{}),
}
}
func (p *callbackPool) available() int { return cap(p.slots) - len(p.slots) }
func (p *callbackPool) waitForCapacity(ctx context.Context) (int, error) {
for p.available() == 0 {
select {
case <-ctx.Done():
return 0, ctx.Err()
case <-p.completed:
}
}
return p.available(), nil
}
func (p *callbackPool) dispatch(message redis.XMessage) bool {
p.mu.Lock()
if _, exists := p.active[message.ID]; exists {
p.mu.Unlock()
return true
}
select {
case p.slots <- struct{}{}:
p.active[message.ID] = struct{}{}
p.outbox.inFlight.Add(1)
p.group.Add(1)
p.mu.Unlock()
case <-p.ctx.Done():
p.mu.Unlock()
return false
default:
p.mu.Unlock()
return false
}
go func() {
defer func() {
p.mu.Lock()
delete(p.active, message.ID)
p.mu.Unlock()
<-p.slots
p.outbox.inFlight.Add(-1)
select {
case p.completed <- struct{}{}:
default:
}
p.group.Done()
}()
if err := p.outbox.processMessage(p.ctx, message); err != nil {
log.Printf("gateway result Outbox event %s failed: %v", message.ID, err)
}
}()
return true
}
func (p *callbackPool) wait() { p.group.Wait() }
func (o *Outbox) concurrency() int {
if o.Concurrency > 0 {
return min(o.Concurrency, 1024)
}
return defaultConcurrency
}
func (o *Outbox) minIdle() time.Duration {
if o.MinIdle > 0 {
return o.MinIdle
}
return defaultMinIdle
}
func (o *Outbox) httpTimeout() time.Duration {
if o.HTTPTimeout > 0 {
return o.HTTPTimeout
}
return defaultHTTPTimeout
}
func (o *Outbox) ConfiguredConcurrency() int { return o.concurrency() }
func sleep(ctx context.Context, duration time.Duration) {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
+47 -6
View File
@@ -36,6 +36,7 @@ type Worker struct {
Limiter ratelimit.Limiter Limiter ratelimit.Limiter
Submit func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error) Submit func(context.Context, queue.SubmitCommand) (queue.SubmitResult, error)
ReportDeadLetter func(context.Context, DeadLetterEvent) error ReportDeadLetter func(context.Context, DeadLetterEvent) error
ResultOutbox SubmitResultOutbox
Stream string Stream string
Group string Group string
Consumer string Consumer string
@@ -50,6 +51,17 @@ type Worker struct {
inFlight atomic.Int64 inFlight atomic.Int64
} }
type SubmitResultOutbox interface {
PublishSubmitResultAndAck(
context.Context,
string,
string,
string,
queue.SubmitCommand,
queue.SubmitResult,
) error
}
type DeadLetterEvent struct { type DeadLetterEvent struct {
StreamMessageID string `json:"streamMessageId"` StreamMessageID string `json:"streamMessageId"`
TraceID string `json:"traceId,omitempty"` TraceID string `json:"traceId,omitempty"`
@@ -82,6 +94,9 @@ func (w *Worker) Run(ctx context.Context) error {
if w.Upstream == nil { if w.Upstream == nil {
return fmt.Errorf("upstream manager is required") return fmt.Errorf("upstream manager is required")
} }
if w.ResultOutbox == nil {
return fmt.Errorf("submit result Outbox is required")
}
pool := newMessageWorkPool(ctx, w, w.concurrency()) pool := newMessageWorkPool(ctx, w, w.concurrency())
defer pool.wait() defer pool.wait()
for { for {
@@ -292,7 +307,8 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
if !command.CreatedAt.IsZero() { if !command.CreatedAt.IsZero() {
metrics.ObserveSubmitStage("stream_wait", true, time.Since(command.CreatedAt)) metrics.ObserveSubmitStage("stream_wait", true, time.Since(command.CreatedAt))
} }
if err := w.handleCommand(ctx, command); err != nil { result, err := w.executeCommand(ctx, command)
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
if ctx.Err() != nil { if ctx.Err() != nil {
return ctx.Err() return ctx.Err()
} }
@@ -308,29 +324,50 @@ func (w *Worker) processMessage(ctx context.Context, message redis.XMessage) err
} }
return err return err
} }
return w.ackAndClearFailure(ctx, message.ID) if err := w.ResultOutbox.PublishSubmitResultAndAck(
ctx,
w.stream(),
w.group(),
message.ID,
command,
result,
); err != nil {
return fmt.Errorf("persist aggregate submit result: %w", err)
}
// The command was ACKed atomically with the aggregate result event. Clearing the
// auxiliary retry counter may lag without affecting delivery correctness.
w.clearFailure(ctx, message.ID)
return nil
} }
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error { func (w *Worker) executeCommand(ctx context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
startedAt := time.Now() startedAt := time.Now()
limitStartedAt := time.Now() limitStartedAt := time.Now()
if w.Limiter != nil { if w.Limiter != nil {
if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil { if _, err := w.Limiter.Wait(ctx, command.ChannelID, command.Route.RateLimitPerSecond); err != nil {
metrics.ObserveSubmitStage("rate_limit_wait", false, time.Since(limitStartedAt)) metrics.ObserveSubmitStage("rate_limit_wait", false, time.Since(limitStartedAt))
return err return queue.SubmitResult{}, err
} }
} }
metrics.ObserveSubmitStage("rate_limit_wait", true, time.Since(limitStartedAt)) metrics.ObserveSubmitStage("rate_limit_wait", true, time.Since(limitStartedAt))
submit := w.Submit submit := w.Submit
if submit == nil { if submit == nil {
if w.Upstream == nil { if w.Upstream == nil {
return fmt.Errorf("upstream manager is required") return queue.SubmitResult{}, fmt.Errorf("upstream manager is required")
} }
submit = w.Upstream.Submit submit = w.Upstream.Submit
} }
result, err := submit(ctx, command) result, err := submit(ctx, command)
accepted := err == nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") accepted := err == nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted")
metrics.ObserveSubmit(accepted, time.Since(startedAt)) metrics.ObserveSubmit(accepted, time.Since(startedAt))
return result, err
}
// handleCommand remains a narrow compatibility seam for focused tests and the
// control path. Stream consumption uses executeCommand so it can persist the
// exact terminal result before acknowledging the command.
func (w *Worker) handleCommand(ctx context.Context, command queue.SubmitCommand) error {
result, err := w.executeCommand(ctx, command)
if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") { if err != nil && (result.SubmitStatus == "" || result.SubmitStatus == "accepted") {
return err return err
} }
@@ -456,10 +493,14 @@ func (w *Worker) ackAndClearFailure(ctx context.Context, messageID string) error
if err := w.Redis.XAck(ctx, w.stream(), w.group(), messageID).Err(); err != nil { if err := w.Redis.XAck(ctx, w.stream(), w.group(), messageID).Err(); err != nil {
return err return err
} }
w.clearFailure(ctx, messageID)
return nil
}
func (w *Worker) clearFailure(ctx context.Context, messageID string) {
if err := w.Redis.HDel(ctx, w.failureAttemptsKey(), messageID).Err(); err != nil { if err := w.Redis.HDel(ctx, w.failureAttemptsKey(), messageID).Err(); err != nil {
w.logf("gateway submit worker clear failure %s failed: %v", messageID, err) w.logf("gateway submit worker clear failure %s failed: %v", messageID, err)
} }
return nil
} }
func (w *Worker) stream() string { func (w *Worker) stream() string {
+21 -2
View File
@@ -17,6 +17,24 @@ type recordingLimiter struct {
called bool called bool
} }
type acknowledgingResultOutbox struct {
client *redis.Client
}
func (o acknowledgingResultOutbox) PublishSubmitResultAndAck(
ctx context.Context,
commandStream string,
commandGroup string,
commandMessageID string,
_ queue.SubmitCommand,
_ queue.SubmitResult,
) error {
if o.client == nil {
return nil
}
return o.client.XAck(ctx, commandStream, commandGroup, commandMessageID).Err()
}
func (l *recordingLimiter) Wait(_ context.Context, channelID string, rate int) (time.Duration, error) { func (l *recordingLimiter) Wait(_ context.Context, channelID string, rate int) (time.Duration, error) {
l.called = true l.called = true
l.channelID = channelID l.channelID = channelID
@@ -140,6 +158,7 @@ func TestMessageWorkPoolContinuouslyRefillsWithoutWaitingForSlowSibling(t *testi
releaseA := make(chan struct{}) releaseA := make(chan struct{})
worker := &Worker{ worker := &Worker{
Redis: client, Redis: client,
ResultOutbox: acknowledgingResultOutbox{client: client},
Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) { Submit: func(_ context.Context, command queue.SubmitCommand) (queue.SubmitResult, error) {
switch command.ChannelID { switch command.ChannelID {
case "channel-a": case "channel-a":
@@ -188,7 +207,7 @@ func TestMessageWorkPoolContinuouslyRefillsWithoutWaitingForSlowSibling(t *testi
func TestMessageWorkPoolAcknowledgesFastMessageBeforeSlowSiblingCompletes(t *testing.T) { func TestMessageWorkPoolAcknowledgesFastMessageBeforeSlowSiblingCompletes(t *testing.T) {
mr := miniredis.RunT(t) mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway"} worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", ResultOutbox: acknowledgingResultOutbox{client: client}}
ctx := context.Background() ctx := context.Background()
if err := worker.ensureGroup(ctx); err != nil { if err := worker.ensureGroup(ctx); err != nil {
t.Fatalf("ensureGroup: %v", err) t.Fatalf("ensureGroup: %v", err)
@@ -246,7 +265,7 @@ func TestMessageWorkPoolAcknowledgesFastMessageBeforeSlowSiblingCompletes(t *tes
func TestPendingRecoveryDoesNotDuplicateAnActiveOrAlreadyAcknowledgedMessage(t *testing.T) { func TestPendingRecoveryDoesNotDuplicateAnActiveOrAlreadyAcknowledgedMessage(t *testing.T) {
mr := miniredis.RunT(t) mr := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", MinIdle: time.Millisecond} worker := &Worker{Redis: client, Stream: "gateway.submit.commands", Group: "cmpp-gateway", MinIdle: time.Millisecond, ResultOutbox: acknowledgingResultOutbox{client: client}}
ctx := context.Background() ctx := context.Background()
if err := worker.ensureGroup(ctx); err != nil { if err := worker.ensureGroup(ctx); err != nil {
t.Fatalf("ensureGroup: %v", err) t.Fatalf("ensureGroup: %v", err)
+8
View File
@@ -24,11 +24,19 @@ const (
type Manager struct { type Manager struct {
APIBaseURL string APIBaseURL string
HTTPClient *http.Client HTTPClient *http.Client
SubmitSegmentPublisher SubmitSegmentPublisher
mu sync.Mutex mu sync.Mutex
conns map[string]*connectionPool conns map[string]*connectionPool
} }
// SubmitSegmentPublisher persists each supplier response before the next long-message
// segment is sent. The boundary is intentionally storage-only: HTTP callbacks belong to
// the result Outbox worker and must not consume a supplier Submit window slot.
type SubmitSegmentPublisher interface {
PublishSubmitSegment(context.Context, queue.SubmitCommand, queue.SubmitSegmentResult) error
}
type ConnectionState struct { type ConnectionState struct {
ChannelID string `json:"channelId"` ChannelID string `json:"channelId"`
ConnectionID string `json:"connectionId"` ConnectionID string `json:"connectionId"`
+14 -47
View File
@@ -7,7 +7,6 @@ import (
"fmt" "fmt"
cmpp "github.com/bigwhite/gocmpp" cmpp "github.com/bigwhite/gocmpp"
cmpputils "github.com/bigwhite/gocmpp/utils" cmpputils "github.com/bigwhite/gocmpp/utils"
"log"
"strings" "strings"
"time" "time"
) )
@@ -17,58 +16,30 @@ import (
func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) { func (m *Manager) Submit(ctx context.Context, cmd queue.SubmitCommand) (queue.SubmitResult, error) {
if err := validateSubmitCommand(cmd); err != nil { if err := validateSubmitCommand(cmd); err != nil {
result := submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error()) return submitResult(cmd, 0, "", "rejected", "INVALID_COMMAND", err.Error()), err
if postErr := m.postSubmitCallback(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err
} }
pool, err := m.connectionFor(cmd) pool, err := m.connectionFor(cmd)
if err != nil { if err != nil {
result := submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error()) return submitResult(cmd, 0, "", "rejected", "CONNECT_FAILED", err.Error()), err
if postErr := m.postSubmitCallback(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err
} }
result, err := pool.submit(ctx, cmd, func(segment queue.SubmitSegmentResult) { result, err := pool.submit(ctx, cmd, func(segment queue.SubmitSegmentResult) error {
payload := struct { if m.SubmitSegmentPublisher == nil {
queue.Envelope return fmt.Errorf("submit segment result publisher is required")
SubmitID string `json:"submitId,omitempty"`
queue.SubmitSegmentResult
}{
Envelope: cmd.Envelope,
SubmitID: cmd.SubmitID,
SubmitSegmentResult: segment,
}
callbackCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
postErr := m.postSubmitCallback(callbackCtx, "/gateway/events/submit-segment-result", payload)
cancel()
if postErr != nil {
log.Printf(
"protocol_event protocol=cmpp direction=gateway_to_api event=submit_segment_result status=forward_failed channel_id=%s message_id=%s segment=%d/%d error=%q",
cmd.ChannelID, cmd.MessageID, segment.SegmentIndex, segment.SegmentTotal, postErr,
)
} }
// A segment result must reach durable local storage before the next segment.
// This cannot make the supplier/Redis boundary globally atomic, but it avoids
// holding the supplier slot for an API round trip and minimizes untracked sends.
return m.SubmitSegmentPublisher.PublishSubmitSegment(ctx, cmd, segment)
}) })
if err != nil {
if postErr := m.postSubmitCallback(ctx, "/gateway/events/submit-result", result); postErr != nil {
return result, postErr
}
return result, err return result, err
} }
if err := m.postSubmitCallback(ctx, "/gateway/events/submit-result", result); err != nil {
return result, err
}
return result, nil
}
func (p *connectionPool) submit( func (p *connectionPool) submit(
ctx context.Context, ctx context.Context,
cmd queue.SubmitCommand, cmd queue.SubmitCommand,
onSegment func(queue.SubmitSegmentResult), onSegment func(queue.SubmitSegmentResult) error,
) (queue.SubmitResult, error) { ) (queue.SubmitResult, error) {
parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content) parts, err := splitSubmitContent(cmd.CMPP.MsgFmt, cmd.Content)
if err != nil { if err != nil {
@@ -95,7 +66,10 @@ func (p *connectionPool) submit(
segment := submitSegmentResult(part, seq, gatewayMessageID, result) segment := submitSegmentResult(part, seq, gatewayMessageID, result)
segments = append(segments, segment) segments = append(segments, segment)
if onSegment != nil { if onSegment != nil {
onSegment(segment) if publishErr := onSegment(segment); publishErr != nil {
result.Segments = segments
return result, publishErr
}
} }
if firstSequence == 0 { if firstSequence == 0 {
firstSequence = seq firstSequence = seq
@@ -118,13 +92,6 @@ func (p *connectionPool) submit(
return result, nil return result, nil
} }
func (m *Manager) postSubmitCallback(ctx context.Context, path string, payload any) error {
startedAt := time.Now()
err := m.post(ctx, path, payload)
metrics.ObserveSubmitStage("api_callback", err == nil, time.Since(startedAt))
return err
}
func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) { func (c *connection) submitPart(ctx context.Context, cmd queue.SubmitCommand, part submitPart) (uint32, string, queue.SubmitResult, error) {
rspCh := make(chan submitPartResponse, 1) rspCh := make(chan submitPartResponse, 1)
pkt := c.submitRequestPacket(cmd, part) pkt := c.submitRequestPacket(cmd, part)
@@ -40,6 +40,17 @@ function assertIsoDateTime(value, field, file) {
function validateExample(fileName) { function validateExample(fileName) {
const fullPath = join(examplesDir, fileName); const fullPath = join(examplesDir, fileName);
const payload = JSON.parse(readFileSync(fullPath, 'utf8')); const payload = JSON.parse(readFileSync(fullPath, 'utf8'));
if (fileName === 'submit-result-outbox-event.json') {
for (const field of ['schemaVersion', 'eventId', 'eventType', 'path', 'messageId', 'channelId', 'submitId', 'payload', 'createdAt']) {
assert(Object.hasOwn(payload, field), `${fileName}: missing ${field}`);
}
assert(payload.schemaVersion === 'v1', `${fileName}: schemaVersion must be v1`);
assert(['submit_result', 'submit_segment_result'].includes(payload.eventType), `${fileName}: unsupported eventType`);
assert(['/gateway/events/submit-result', '/gateway/events/submit-segment-result'].includes(payload.path), `${fileName}: unsupported path`);
assert(payload.payload?.eventId === payload.eventId, `${fileName}: payload.eventId must match eventId`);
assertIsoDateTime(payload.createdAt, 'createdAt', fileName);
return `${fileName}: ${payload.eventType} ok`;
}
const type = payload.messageType; const type = payload.messageType;
assert(validTypes.has(type), `${fileName}: unsupported messageType ${type}`); assert(validTypes.has(type), `${fileName}: unsupported messageType ${type}`);