From b7ae1aa41afdc41ff083bff26e3b8fca34de74b3 Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Wed, 26 Aug 2026 14:04:44 +0800 Subject: [PATCH] fix: restore channels after gateway restart --- docs/production-deployment.md | 1 + docs/testing-progress.md | 1 + tools/deploy/production-deploy.sh | 8 ++++---- tools/deploy/verify-production-deployment.mjs | 7 +++++++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/production-deployment.md b/docs/production-deployment.md index bf7b4e5..45021ab 100644 --- a/docs/production-deployment.md +++ b/docs/production-deployment.md @@ -189,3 +189,4 @@ bash tools/deploy/production-deploy.sh - 第三阶段要求环境显式设置`API_INBOUND_WORKFLOW_BATCH_ENABLED=true`和正整数`API_INBOUND_WORKFLOW_BATCH_SIZE`，初始建议64且不得大于Worker业务槽的可解释倍数。批次增大前必须核对PostgreSQL参数数量、单事务持续时间、Worker RSS和租约时长；付费短信仍走逐条账务锁，不能用零计费批量结果替代付费链路验收。 - 单企业微批阶段要求显式设置`API_INBOUND_WORKFLOW_BATCH_WAIT_MS=40`、`API_INBOUND_WORKFLOW_TARGET_BATCH_SIZE=32`，目标不得超过批次上限；发布脚本拒绝负等待、超过250ms或目标越界。API还必须设置`API_HTTP_KEEP_ALIVE_TIMEOUT_MS=120000`和更大的`API_HTTP_HEADERS_TIMEOUT_MS=125000`，确保服务端keep-alive长于Gateway 90秒空闲池；发布后用响应头回读实际timeout并检查Gateway日志无loopback reset。 - `GATEWAY_CALLBACK_BATCH_ENABLED`默认关闭，只有独立Gateway callback进程已启用、`GATEWAY_CALLBACK_API_BASE_URL`指向其回环端口且批量路由健康检查通过时才允许显式设为`true`。主API回调模式必须保持`false`；发布后同时检查`gateway.submit.results`的`pending/lag`，任何持续增长均视为发布失败，禁止手工ACK掩盖回调未持久化。 +- 服务重启顺序必须为“独立callback（如启用）→ Gateway → API → Send Worker”。API必须在Gateway已健康后重启，利用启动同步重新下发活动供应商通道；发布后`cmpp_gateway_upstream_connections{state="desired"}`与`connected`必须恢复到发布前活动连接数，不能只凭Gateway `/health` 判定发布成功。 diff --git a/docs/testing-progress.md b/docs/testing-progress.md index 0dda4cc..9fa329c 100644 --- a/docs/testing-progress.md +++ b/docs/testing-progress.md @@ -7,6 +7,7 @@ - 数据库由90条推进至94条migration，实际新增Submit结果幂等、入站Inbox、Submit Outbox和上行eventId四条迁移。API、独立Send Worker、Gateway、安全代理、Fail2ban、PostgreSQL、Redis、Nginx均active，API/Gateway/前端健康，Inbox最终无pending/processing，Submit命令Stream为`pending=0/lag=0`。 - 发布后发现主API回调模式下未显式配置批量开关时，Gateway却默认启用批量回调；批量路由只存在于独立callback进程，导致`gateway.submit.results`一度累积到97条pending。预生产显式设置`GATEWAY_CALLBACK_BATCH_ENABLED=false`并重启Gateway后，PEL按幂等逐条回调从97持续降至0并连续六次保持`pending=0/lag=0`，未手工ACK或删除事件。 - 最小代码修复将批量回调改为仅在环境变量严格等于`true`时启用，并增加默认关闭单元测试、示例环境和发布门禁说明。全程未进行压力测试，未由测试人员主动发送、补发或重投短信，未修改通道账号、启停状态、企业余额、白名单、临时号段或测试账号；发布窗口存在预生产客户自然流量，均按真实链路处理。多Gateway P2未实施。 +- 回调修复提交`8170f727a3c547e3c51d7308a08680ecac9ae1f0`已推送并热修复到预生产；热修复备份为`/opt/cmpp-platform-backups/releases/20260826T135700CST-before-8170f72-gateway`，包含当前PostgreSQL custom dump、运行源码、系统配置、旧Gateway二进制和部署标记，SHA-256、`pg_restore --list`及tar可读性均通过。热修复后又发现Gateway单独重启会使供应商连接变成`desired=0/connected=0`；在不修改通道配置的前提下按正确顺序重启API后恢复为`9/9`。发布脚本已改为独立callback（如启用）→Gateway→API→Send Worker，并增加顺序门禁，避免后续仅健康接口正常但供应商通道未恢复。 > 环境命名：`8.160.169.106:12026`（Web/API）和 `8.160.169.106:17890`（CMPP 入站）实例统一定义为“预生产环境”；`100.93.204.60`统一定义为“虚拟机测试环境”或“测试机”。“虚拟机”不得再用于指代预生产。历史记录中涉及这两个实例的验证和部署按其明确IP归属理解；`production-deploy.sh`、`NODE_ENV=production`及正式生产安全/备份规范保留原有技术语义，不代表测试机或预生产为正式生产。 diff --git a/tools/deploy/production-deploy.sh b/tools/deploy/production-deploy.sh index c1bf35e..cf2d57a 100644 --- a/tools/deploy/production-deploy.sh +++ b/tools/deploy/production-deploy.sh @@ -241,8 +241,6 @@ else systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx fi systemctl restart cmpp-security-agent -systemctl restart cmpp-api -systemctl restart cmpp-send-worker if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then systemctl enable --now cmpp-submit-outbox systemctl restart cmpp-submit-outbox @@ -261,9 +259,11 @@ if [[ "${GATEWAY_PROTOCOL_LOG_STREAM_ENABLED:-false}" == "true" ]]; then else systemctl disable --now cmpp-protocol-log-worker 2>/dev/null || true fi -# Gateway is restarted after the callback listener so supplier events never point -# at a callback port that has not completed Nest/Prisma initialization. +# Gateway is restarted after the callback listener, then API is restarted after +# Gateway so API startup reconciliation always restores the active channel set. systemctl restart cmpp-gateway +systemctl restart cmpp-api +systemctl restart cmpp-send-worker systemctl restart nginx echo "[deploy] Health checks" diff --git a/tools/deploy/verify-production-deployment.mjs b/tools/deploy/verify-production-deployment.mjs index 13a7d0b..dd5329a 100644 --- a/tools/deploy/verify-production-deployment.mjs +++ b/tools/deploy/verify-production-deployment.mjs @@ -58,4 +58,11 @@ if (!workerMain.includes("process.env.API_WORKER_METRICS_HOST?.trim() || '127.0. throw new Error('send worker metrics must default to loopback'); } +const gatewayRestart = deploy.indexOf('systemctl restart cmpp-gateway'); +const apiRestart = deploy.indexOf('systemctl restart cmpp-api'); +const workerRestart = deploy.indexOf('systemctl restart cmpp-send-worker'); +if (gatewayRestart < 0 || apiRestart < gatewayRestart || workerRestart < apiRestart) { + throw new Error('production deploy must restart Gateway before API and the send worker so API startup restores active channels'); +} + console.log('Production deployment verified: Nginx/API guards and the split durable send worker contract are present.');