From 394949f9f61b401fd1d223339c938c35c873b4dc Mon Sep 17 00:00:00 2001 From: hectorzhao Date: Tue, 25 Aug 2026 17:35:18 +0800 Subject: [PATCH] fix: align api keepalive with gateway pool --- api/src/http-server-timeouts.spec.ts | 20 ++++++++++++++++++ api/src/http-server-timeouts.ts | 21 +++++++++++++++++++ api/src/main.ts | 4 +++- .../first-version-development-requirements.md | 1 + docs/system-functional-test-cases.md | 1 + tools/deploy/production-bootstrap.sh | 4 ++++ tools/deploy/production-deploy.sh | 5 +++++ tools/deploy/verify-production-deployment.mjs | 9 ++++++++ 8 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 api/src/http-server-timeouts.spec.ts create mode 100644 api/src/http-server-timeouts.ts diff --git a/api/src/http-server-timeouts.spec.ts b/api/src/http-server-timeouts.spec.ts new file mode 100644 index 0000000..04ef34b --- /dev/null +++ b/api/src/http-server-timeouts.spec.ts @@ -0,0 +1,20 @@ +import { configureApiHttpServerTimeouts } from './http-server-timeouts'; + +describe('configureApiHttpServerTimeouts', () => { + it('keeps the API connection alive longer than the Gateway idle pool', () => { + const server = { keepAliveTimeout: 0, headersTimeout: 0 }; + expect(configureApiHttpServerTimeouts(server, {})).toEqual({ + keepAliveTimeoutMs: 120_000, + headersTimeoutMs: 125_000, + }); + expect(server).toEqual({ keepAliveTimeout: 120_000, headersTimeout: 125_000 }); + }); + + it('keeps headers timeout above a configured keep-alive timeout', () => { + const server = { keepAliveTimeout: 0, headersTimeout: 0 }; + expect(configureApiHttpServerTimeouts(server, { + API_HTTP_KEEP_ALIVE_TIMEOUT_MS: '90000', + API_HTTP_HEADERS_TIMEOUT_MS: '1000', + })).toEqual({ keepAliveTimeoutMs: 90_000, headersTimeoutMs: 91_000 }); + }); +}); diff --git a/api/src/http-server-timeouts.ts b/api/src/http-server-timeouts.ts new file mode 100644 index 0000000..ca9f02f --- /dev/null +++ b/api/src/http-server-timeouts.ts @@ -0,0 +1,21 @@ +import type { Server } from 'node:http'; + +const DEFAULT_KEEP_ALIVE_TIMEOUT_MS = 120_000; +const DEFAULT_HEADERS_TIMEOUT_MS = 125_000; + +export function configureApiHttpServerTimeouts( + server: Pick, + env: NodeJS.ProcessEnv, +) { + const keepAliveTimeoutMs = positiveInteger(env.API_HTTP_KEEP_ALIVE_TIMEOUT_MS, DEFAULT_KEEP_ALIVE_TIMEOUT_MS); + const configuredHeadersTimeoutMs = positiveInteger(env.API_HTTP_HEADERS_TIMEOUT_MS, DEFAULT_HEADERS_TIMEOUT_MS); + const headersTimeoutMs = Math.max(configuredHeadersTimeoutMs, keepAliveTimeoutMs + 1_000); + server.keepAliveTimeout = keepAliveTimeoutMs; + server.headersTimeout = headersTimeoutMs; + return { keepAliveTimeoutMs, headersTimeoutMs }; +} + +function positiveInteger(value: string | undefined, fallback: number) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} diff --git a/api/src/main.ts b/api/src/main.ts index 6f731cb..a6a2db7 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -7,6 +7,7 @@ import { AppModule } from './app.module'; import { MetricsService } from './metrics/metrics.service'; import { OpenApiModule } from './open-api/open-api.module'; import { configureHttpBodyParsers } from './http-body-limits'; +import { configureApiHttpServerTimeouts } from './http-server-timeouts'; Object.defineProperty(BigInt.prototype, 'toJSON', { configurable: true, @@ -43,7 +44,8 @@ async function bootstrap() { const port = Number(process.env.API_PORT ?? 3000); // 生产环境只允许 Nginx 访问管理 API；显式绑定回环，避免默认的全网卡监听绕过入口鉴权与限流。 const host = process.env.API_HOST?.trim() || '127.0.0.1'; - await app.listen(port, host); + const apiServer = await app.listen(port, host); + configureApiHttpServerTimeouts(apiServer, process.env); const metrics = app.get(MetricsService); const metricsHost = process.env.API_METRICS_HOST?.trim() || '127.0.0.1'; diff --git a/docs/first-version-development-requirements.md b/docs/first-version-development-requirements.md index b03561e..1ddbe85 100644 --- a/docs/first-version-development-requirements.md +++ b/docs/first-version-development-requirements.md @@ -2167,3 +2167,4 @@ - 正在处理的企业必须从下一次领取候选中排除，避免同企业并发事务重新争抢账户锁；租约过期恢复、优先级/FIFO、`FOR UPDATE SKIP LOCKED`和逐请求幂等语义保持不变。 - 原生SQL对Prisma的无时区时间列统一使用`NOW() AT TIME ZONE 'UTC'`，覆盖Submit Outbox领取、租约、发布、重试及关联消息更新时间，禁止用数据库会话时区污染时延审计。 - 验收必须保持正价，分别执行单企业100/150 TPS和至少两个独立企业合计200 TPS，按非补发首次供应商Submit、回执、上行、计费、主备补发、业务拦截及全队列排空对账；触发拒绝、连接错误、持续积压、数据库异常或账务不一致立即停止。多Gateway P2不在本阶段范围。 +- Gateway到主API的回环HTTP连接池空闲时长不得超过API服务端keep-alive生命周期；API默认keep-alive 120秒、headers timeout 125秒，并由发布环境显式设置和校验，防止负载期复用已被Node关闭的连接而把本可受理的Submit误回Result 9。 diff --git a/docs/system-functional-test-cases.md b/docs/system-functional-test-cases.md index 2b3ea60..bbfabbd 100644 --- a/docs/system-functional-test-cases.md +++ b/docs/system-functional-test-cases.md @@ -4885,3 +4885,4 @@ npm run verify:phase8 | TC-CMPP-PHASE5-014 | 不同企业正价并行 | 至少两个独立企业合计200 TPS；各企业账户冻结串行且企业间并行，单价均为325，分别对账消息、账单、首提和最终状态 | | TC-CMPP-PHASE5-015 | Outbox UTC时间语义 | Asia/Shanghai数据库会话下领取、租约、发布、重试均写UTC无时区值；publishedAt-createdAt不再出现约8小时偏差 | | TC-CMPP-PHASE5-016 | 微批发布边界与停止线 | 仅单Gateway；分别验证单企业100/150与多企业200，发生拒绝、连接错误、持续积压、数据库异常或账务不一致立即停止 | +| TC-CMPP-PHASE5-017 | Gateway到API长连接生命周期 | API keep-alive 120秒大于Gateway连接池90秒，headers timeout更大；跨越Node原默认5秒空闲边界后继续压测，不得出现loopback connection reset或Result 9 | diff --git a/tools/deploy/production-bootstrap.sh b/tools/deploy/production-bootstrap.sh index 11ccbac..27d3528 100644 --- a/tools/deploy/production-bootstrap.sh +++ b/tools/deploy/production-bootstrap.sh @@ -7,6 +7,8 @@ BRANCH="${BRANCH:-main}" PUBLIC_HTTP_PORT="${PUBLIC_HTTP_PORT:-12026}" API_PORT="${API_PORT:-3000}" API_HOST="${API_HOST:-127.0.0.1}" +API_HTTP_KEEP_ALIVE_TIMEOUT_MS="${API_HTTP_KEEP_ALIVE_TIMEOUT_MS:-120000}" +API_HTTP_HEADERS_TIMEOUT_MS="${API_HTTP_HEADERS_TIMEOUT_MS:-125000}" API_METRICS_HOST="${API_METRICS_HOST:-127.0.0.1}" API_METRICS_PORT="${API_METRICS_PORT:-9464}" API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}" @@ -200,6 +202,8 @@ write_env() { NODE_ENV=production API_PORT=${API_PORT} API_HOST=${API_HOST} +API_HTTP_KEEP_ALIVE_TIMEOUT_MS=${API_HTTP_KEEP_ALIVE_TIMEOUT_MS} +API_HTTP_HEADERS_TIMEOUT_MS=${API_HTTP_HEADERS_TIMEOUT_MS} API_METRICS_HOST=${API_METRICS_HOST} API_METRICS_PORT=${API_METRICS_PORT} API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER} diff --git a/tools/deploy/production-deploy.sh b/tools/deploy/production-deploy.sh index 23d14e5..c1bf35e 100644 --- a/tools/deploy/production-deploy.sh +++ b/tools/deploy/production-deploy.sh @@ -19,6 +19,11 @@ set -a source "$ENV_FILE" set +a +if [[ ! "${API_HTTP_KEEP_ALIVE_TIMEOUT_MS:-}" =~ ^[1-9][0-9]*$ || ! "${API_HTTP_HEADERS_TIMEOUT_MS:-}" =~ ^[1-9][0-9]*$ || "${API_HTTP_HEADERS_TIMEOUT_MS}" -le "${API_HTTP_KEEP_ALIVE_TIMEOUT_MS}" ]]; then + echo "API_HTTP_KEEP_ALIVE_TIMEOUT_MS must be positive and API_HTTP_HEADERS_TIMEOUT_MS must be greater in $ENV_FILE." >&2 + exit 1 +fi + if [[ "${API_ENABLE_SEND_WORKER:-}" != "true" ]]; then echo "API_ENABLE_SEND_WORKER=true is required in $ENV_FILE; refusing to deploy with SMS sending disabled." >&2 exit 1 diff --git a/tools/deploy/verify-production-deployment.mjs b/tools/deploy/verify-production-deployment.mjs index 4474dc3..13a7d0b 100644 --- a/tools/deploy/verify-production-deployment.mjs +++ b/tools/deploy/verify-production-deployment.mjs @@ -4,6 +4,7 @@ import { resolve } from 'node:path'; const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8'); const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8'); const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8'); +const apiTimeouts = readFileSync(resolve(import.meta.dirname, '../../api/src/http-server-timeouts.ts'), 'utf8'); const workerMain = readFileSync(resolve(import.meta.dirname, '../../api/src/send-worker.ts'), 'utf8'); const required = [ 'compression_config=/etc/nginx/conf.d/cmpp-compression.conf', @@ -26,6 +27,14 @@ for (const marker of ['API_HOST="${API_HOST:-127.0.0.1}"', 'API_HOST=${API_HOST} if (!apiMain.includes("process.env.API_HOST?.trim() || '127.0.0.1'") || !apiMain.includes('app.listen(port, host)')) { throw new Error('NestJS API must bind to API_HOST and default to loopback'); } +for (const marker of ['API_HTTP_KEEP_ALIVE_TIMEOUT_MS', 'API_HTTP_HEADERS_TIMEOUT_MS']) { + if (!`${deploy}\n${bootstrap}\n${apiTimeouts}`.includes(marker)) { + throw new Error(`production deployment is missing the API keep-alive contract: ${marker}`); + } +} +if (!apiMain.includes('configureApiHttpServerTimeouts(apiServer, process.env)')) { + throw new Error('NestJS API must apply the explicit keep-alive contract to its HTTP server'); +} for (const marker of [ 'CMPP_INBOUND_FAST_PATH_ENABLED=true',