perf(cmpp): add durable inbound fast path

2026-08-20 17:34:45 +08:00
parent 26ef67fb6a
commit 0b63bcd74e
29 changed files with 1136 additions and 87 deletions
@@ -11,6 +11,13 @@ API_METRICS_HOST="${API_METRICS_HOST:-127.0.0.1}"
 API_METRICS_PORT="${API_METRICS_PORT:-9464}"
 API_ENABLE_SEND_WORKER="${API_ENABLE_SEND_WORKER:-true}"
 API_SEND_WORKER_CONCURRENCY="${API_SEND_WORKER_CONCURRENCY:-50}"
+API_WORKER_METRICS_HOST="${API_WORKER_METRICS_HOST:-127.0.0.1}"
+API_WORKER_METRICS_PORT="${API_WORKER_METRICS_PORT:-9465}"
+CMPP_INBOUND_FAST_PATH_ENABLED="${CMPP_INBOUND_FAST_PATH_ENABLED:-true}"
+CMPP_INBOUND_WORKFLOW_WORKER_ENABLED="${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED:-true}"
+API_INBOUND_WORKFLOW_CONCURRENCY="${API_INBOUND_WORKFLOW_CONCURRENCY:-32}"
+API_INBOUND_WORKFLOW_POLL_INTERVAL_MS="${API_INBOUND_WORKFLOW_POLL_INTERVAL_MS:-100}"
+API_INBOUND_WORKFLOW_STALE_SECONDS="${API_INBOUND_WORKFLOW_STALE_SECONDS:-300}"
 GATEWAY_CONTROL_ADDR="${GATEWAY_CONTROL_ADDR:-127.0.0.1:8090}"
 GATEWAY_CMPP_ADDR="${GATEWAY_CMPP_ADDR:-0.0.0.0:17890}"
 CMPP_PUBLIC_HOST="${CMPP_PUBLIC_HOST:-8.160.169.106}"
@@ -182,7 +189,7 @@ SQL

 write_env() {
  log "Writing production environment"
-  mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
+  mkdir -p /etc/cmpp-platform "$APP_DIR" "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway" "$APP_DIR/backups" "$OBJECT_STORAGE_LOCAL_ROOT"
  cat >/etc/cmpp-platform/cmpp-platform.env <<EOF
 NODE_ENV=production
 API_PORT=${API_PORT}
@@ -191,6 +198,13 @@ API_METRICS_HOST=${API_METRICS_HOST}
 API_METRICS_PORT=${API_METRICS_PORT}
 API_ENABLE_SEND_WORKER=${API_ENABLE_SEND_WORKER}
 API_SEND_WORKER_CONCURRENCY=${API_SEND_WORKER_CONCURRENCY}
+API_WORKER_METRICS_HOST=${API_WORKER_METRICS_HOST}
+API_WORKER_METRICS_PORT=${API_WORKER_METRICS_PORT}
+CMPP_INBOUND_FAST_PATH_ENABLED=${CMPP_INBOUND_FAST_PATH_ENABLED}
+CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED}
+API_INBOUND_WORKFLOW_CONCURRENCY=${API_INBOUND_WORKFLOW_CONCURRENCY}
+API_INBOUND_WORKFLOW_POLL_INTERVAL_MS=${API_INBOUND_WORKFLOW_POLL_INTERVAL_MS}
+API_INBOUND_WORKFLOW_STALE_SECONDS=${API_INBOUND_WORKFLOW_STALE_SECONDS}
 DATABASE_URL=postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:5432/${DB_NAME}?schema=public
 REDIS_HOST=127.0.0.1
 REDIS_PORT=6379
@@ -265,12 +279,34 @@ User=cmpp-api
 Group=cmpp-security
 WorkingDirectory=${APP_DIR}/api
 EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
+Environment=CMPP_PROCESS_ROLE=api
 ExecStart=${node_bin} dist/main.js
 Restart=always
 RestartSec=5
 StandardOutput=append:${APP_DIR}/logs/api/stdout.log
 StandardError=append:${APP_DIR}/logs/api/stderr.log

+[Install]
+WantedBy=multi-user.target
+EOF
+
+  cat >/etc/systemd/system/cmpp-send-worker.service <<EOF
+[Unit]
+Description=CMPP durable send workflow worker
+After=network.target postgresql.service redis.service cmpp-minio.service
+
+[Service]
+User=cmpp-api
+Group=cmpp-security
+WorkingDirectory=${APP_DIR}/api
+EnvironmentFile=/etc/cmpp-platform/cmpp-platform.env
+Environment=CMPP_PROCESS_ROLE=worker
+ExecStart=${node_bin} dist/send-worker.js
+Restart=always
+RestartSec=5
+StandardOutput=append:${APP_DIR}/logs/send-worker/stdout.log
+StandardError=append:${APP_DIR}/logs/send-worker/stderr.log
+
 [Install]
 WantedBy=multi-user.target
 EOF
@@ -29,6 +29,16 @@ if [[ ! "${API_SEND_WORKER_CONCURRENCY:-}" =~ ^[1-9][0-9]*$ ]]; then
  exit 1
 fi

+if [[ "${CMPP_INBOUND_FAST_PATH_ENABLED:-}" != "true" || "${CMPP_INBOUND_WORKFLOW_WORKER_ENABLED:-}" != "true" ]]; then
+  echo "CMPP_INBOUND_FAST_PATH_ENABLED=true and CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true are required in $ENV_FILE." >&2
+  exit 1
+fi
+
+if [[ ! "${API_INBOUND_WORKFLOW_CONCURRENCY:-}" =~ ^[1-9][0-9]*$ ]]; then
+  echo "API_INBOUND_WORKFLOW_CONCURRENCY must be a positive integer in $ENV_FILE." >&2
+  exit 1
+fi
+
 if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then
  echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2
  exit 1
@@ -64,7 +74,35 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro
 chmod 600 "$ADMIN_CREDENTIAL_FILE" || true

 echo "[deploy] Ensuring runtime log directories"
-install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/gateway"
+install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway"
+
+echo "[deploy] Installing split API and send-worker services"
+node_bin="$(command -v node)"
+install -d -m 0755 /etc/systemd/system/cmpp-api.service.d
+cat >/etc/systemd/system/cmpp-api.service.d/process-role.conf <<'EOF'
+[Service]
+Environment=CMPP_PROCESS_ROLE=api
+EOF
+cat >/etc/systemd/system/cmpp-send-worker.service <<EOF
+[Unit]
+Description=CMPP durable send workflow worker
+After=network.target postgresql.service redis.service cmpp-minio.service
+
+[Service]
+User=cmpp-api
+Group=cmpp-security
+WorkingDirectory=$APP_DIR/api
+EnvironmentFile=$ENV_FILE
+Environment=CMPP_PROCESS_ROLE=worker
+ExecStart=$node_bin dist/send-worker.js
+Restart=always
+RestartSec=5
+StandardOutput=append:$APP_DIR/logs/send-worker/stdout.log
+StandardError=append:$APP_DIR/logs/send-worker/stderr.log
+
+[Install]
+WantedBy=multi-user.target
+EOF

 echo "[deploy] Installing restricted security boundary"
 bash "$APP_DIR/tools/security/install-security-agent.sh"
@@ -90,15 +128,16 @@ echo "[deploy] Restarting services"
 systemctl daemon-reload
 if [[ "${OBJECT_STORAGE_DRIVER:-minio}" == "local" ]]; then
  systemctl disable --now cmpp-minio 2>/dev/null || true
-  systemctl enable --now cmpp-api cmpp-gateway nginx
+  systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
 else
  systemctl enable --now cmpp-minio
  systemctl restart cmpp-minio
-  systemctl enable --now cmpp-api cmpp-gateway nginx
+  systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
 fi
 systemctl restart cmpp-gateway
 systemctl restart cmpp-security-agent
 systemctl restart cmpp-api
+systemctl restart cmpp-send-worker
 systemctl restart nginx

 echo "[deploy] Health checks"
@@ -117,6 +156,7 @@ wait_for_http() {
  return 1
 }
 wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health"
+wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics"
 wait_for_http "Gateway" "http://127.0.0.1:8090/health"
 redis-cli -h "${REDIS_HOST:-127.0.0.1}" -p "${REDIS_PORT:-6379}" ping >/dev/null
 pg_isready -d "${DATABASE_URL%%\?*}" >/dev/null
@@ -4,6 +4,7 @@ import { resolve } from 'node:path';
 const deploy = readFileSync(resolve(import.meta.dirname, 'production-deploy.sh'), 'utf8');
 const bootstrap = readFileSync(resolve(import.meta.dirname, 'production-bootstrap.sh'), 'utf8');
 const apiMain = readFileSync(resolve(import.meta.dirname, '../../api/src/main.ts'), 'utf8');
+const workerMain = readFileSync(resolve(import.meta.dirname, '../../api/src/send-worker.ts'), 'utf8');
 const required = [
  'compression_config=/etc/nginx/conf.d/cmpp-compression.conf',
  ': >"$compression_config"',
@@ -26,4 +27,22 @@ if (!apiMain.includes("process.env.API_HOST?.trim() || '127.0.0.1'") || !apiMain
  throw new Error('NestJS API must bind to API_HOST and default to loopback');
 }

-console.log('Production deployment verified: Nginx compression is idempotent and NestJS defaults to loopback.');
+for (const marker of [
+  'CMPP_INBOUND_FAST_PATH_ENABLED=true',
+  'CMPP_INBOUND_WORKFLOW_WORKER_ENABLED=true',
+  'API_INBOUND_WORKFLOW_CONCURRENCY',
+  'cmpp-send-worker.service',
+  'Environment=CMPP_PROCESS_ROLE=api',
+  'Environment=CMPP_PROCESS_ROLE=worker',
+  'dist/send-worker.js',
+  'API_WORKER_METRICS_PORT',
+]) {
+  if (!`${deploy}\n${bootstrap}`.includes(marker)) {
+    throw new Error(`production deployment is missing the durable inbound worker contract: ${marker}`);
+  }
+}
+if (!workerMain.includes("process.env.API_WORKER_METRICS_HOST?.trim() || '127.0.0.1'")) {
+  throw new Error('send worker metrics must default to loopback');
+}
+
+console.log('Production deployment verified: Nginx/API guards and the split durable send worker contract are present.');
@@ -182,7 +182,7 @@ groups:
  - name: cmpp-core-services
    rules:
      - alert: CmppCoreServiceInactive
-        expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
+        expr: node_systemd_unit_state{name=~"cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service",state="active"} == 0
        for: 2m
        labels:
          severity: critical
@@ -200,6 +200,16 @@ groups:
        for: 2m
        labels: { severity: critical, service: api }
        annotations: { summary: "API指标采集不可用", description: "Prometheus连续2分钟无法读取API内部指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: CmppSendWorkerMetricsDown
+        expr: up{job="cmpp-send-worker"} == 0
+        for: 2m
+        labels: { severity: critical, service: send-worker }
+        annotations: { summary: "发送Worker指标采集不可用", description: "Prometheus连续2分钟无法读取独立发送Worker指标端点。", currentValue: "{{ $value }}", threshold: "up = 1" }
+      - alert: CmppInboundWorkflowBacklogCritical
+        expr: cmpp_worker_inbound_workflow_oldest_pending_age_seconds > 120
+        for: 2m
+        labels: { severity: critical, service: send-worker }
+        annotations: { summary: "CMPP耐久Inbox严重积压", description: "最旧待处理CMPP Inbox连续2分钟超过120秒。", currentValue: "{{ printf \"%.0f\" $value }}s", threshold: "120s" }
      - alert: CmppApiHttpErrorRateWarning
        expr: (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) > 0.01) and (sum(rate(cmpp_api_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(cmpp_api_http_requests_total[5m])), 0.001) <= 0.05) and sum(increase(cmpp_api_http_requests_total{status=~"5.."}[5m])) >= 5
        for: 5m
@@ -68,7 +68,7 @@ EOF
 cat >/etc/systemd/system/prometheus-node-exporter.service.d/cmpp-monitoring.conf <<EOF
 [Service]
 ExecStart=
-ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
+ExecStart=${node_exporter_bin} --web.listen-address=127.0.0.1:9100 --collector.systemd --collector.systemd.unit-include='cmpp-api\\.service|cmpp-send-worker\\.service|cmpp-gateway\\.service|postgresql\\.service|redis(-server)?\\.service|cmpp-minio\\.service|nginx\\.service' --collector.filesystem.mount-points-exclude='^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)'
 EOF

 log "Validating Prometheus configuration before restart"
@@ -24,6 +24,10 @@ scrape_configs:
    static_configs:
      - targets: [127.0.0.1:9464]

+  - job_name: cmpp-send-worker
+    static_configs:
+      - targets: [127.0.0.1:9465]
+
  - job_name: cmpp-gateway
    metrics_path: /metrics
    static_configs:
@@ -13,6 +13,7 @@ usermod -a -G cmpp-security cmpp-api
 install -d -o root -g cmpp-security -m 0770 /run/cmpp-security-agent
 install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
 install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
+install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/send-worker"
 [[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage

 sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
@@ -29,6 +30,18 @@ table inet cmpp_security {
  chain input { type filter hook input priority -10; policy accept; ip saddr @blocked_ipv4 drop; ip6 saddr @blocked_ipv6 drop; }
 }
 EOF
+
+install -d -m 0755 /etc/systemd/system/cmpp-send-worker.service.d
+cat >/etc/systemd/system/cmpp-send-worker.service.d/security-boundary.conf <<EOF
+[Service]
+User=cmpp-api
+Group=cmpp-security
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=true
+ReadWritePaths=$APP_DIR/logs/send-worker /var/lib/cmpp-platform/object-storage
+EOF
 grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf
 nft -c -f /etc/nftables.conf
 nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
@@ -54,4 +67,4 @@ fail2ban-client -t
 nginx -t
 systemctl daemon-reload
 systemctl enable cmpp-security-agent
-echo "Security boundary installed. Restart cmpp-security-agent and cmpp-api only in the approved release window."
+echo "Security boundary installed. Restart cmpp-security-agent, cmpp-api and cmpp-send-worker only in the approved release window."