perf: batch gateway submits and isolate callbacks
This commit is contained in:
hectorzhao
@@ -46,6 +46,21 @@ if [[ ! "${API_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ || ! "${API_WORKER_DB_POOL_MAX:-
|
||||
echo "API_DB_POOL_MAX and API_WORKER_DB_POOL_MAX must be positive integers in $ENV_FILE." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" && ! "${API_OUTBOX_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "API_OUTBOX_DB_POOL_MAX must be a positive integer when the separate Submit Outbox publisher is enabled." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
|
||||
if [[ ! "${API_CALLBACK_DB_POOL_MAX:-}" =~ ^[1-9][0-9]*$ || ! "${API_CALLBACK_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "API_CALLBACK_DB_POOL_MAX and API_CALLBACK_PORT must be positive integers when the separate Gateway callback is enabled." >&2
|
||||
exit 1
|
||||
fi
|
||||
expected_callback_url="http://127.0.0.1:${API_CALLBACK_PORT}/api"
|
||||
if [[ "${GATEWAY_CALLBACK_API_BASE_URL:-}" != "$expected_callback_url" ]]; then
|
||||
echo "GATEWAY_CALLBACK_API_BASE_URL must equal $expected_callback_url for the loopback callback process." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${CMPP_PUBLIC_HOST:-}" || ! "${CMPP_PUBLIC_PORT:-}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "CMPP_PUBLIC_HOST and a positive CMPP_PUBLIC_PORT are required in $ENV_FILE; these are the customer-facing CMPP endpoint." >&2
|
||||
@@ -82,7 +97,7 @@ PROD_ADMIN_CREDENTIAL_FILE="$ADMIN_CREDENTIAL_FILE" node tools/deploy/ensure-pro
|
||||
chmod 600 "$ADMIN_CREDENTIAL_FILE" || true
|
||||
|
||||
echo "[deploy] Ensuring runtime log directories"
|
||||
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/gateway"
|
||||
install -d -m 0755 "$APP_DIR/logs/api" "$APP_DIR/logs/send-worker" "$APP_DIR/logs/submit-outbox" "$APP_DIR/logs/gateway-callback" "$APP_DIR/logs/gateway"
|
||||
|
||||
echo "[deploy] Installing split API and send-worker services"
|
||||
node_bin="$(command -v node)"
|
||||
@@ -108,6 +123,46 @@ RestartSec=5
|
||||
StandardOutput=append:$APP_DIR/logs/send-worker/stdout.log
|
||||
StandardError=append:$APP_DIR/logs/send-worker/stderr.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
cat >/etc/systemd/system/cmpp-submit-outbox.service <<EOF
|
||||
[Unit]
|
||||
Description=CMPP PostgreSQL Submit Outbox publisher
|
||||
After=network.target postgresql.service redis.service
|
||||
|
||||
[Service]
|
||||
User=cmpp-api
|
||||
Group=cmpp-security
|
||||
WorkingDirectory=$APP_DIR/api
|
||||
EnvironmentFile=$ENV_FILE
|
||||
Environment=CMPP_PROCESS_ROLE=outbox
|
||||
ExecStart=$node_bin dist/submit-outbox-worker.js
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:$APP_DIR/logs/submit-outbox/stdout.log
|
||||
StandardError=append:$APP_DIR/logs/submit-outbox/stderr.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
cat >/etc/systemd/system/cmpp-gateway-callback.service <<EOF
|
||||
[Unit]
|
||||
Description=CMPP Gateway result, receipt and billing callback API
|
||||
After=network.target postgresql.service redis.service
|
||||
|
||||
[Service]
|
||||
User=cmpp-api
|
||||
Group=cmpp-security
|
||||
WorkingDirectory=$APP_DIR/api
|
||||
EnvironmentFile=$ENV_FILE
|
||||
Environment=CMPP_PROCESS_ROLE=callback
|
||||
ExecStart=$node_bin dist/gateway-callback.js
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
StandardOutput=append:$APP_DIR/logs/gateway-callback/stdout.log
|
||||
StandardError=append:$APP_DIR/logs/gateway-callback/stderr.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
@@ -142,10 +197,24 @@ else
|
||||
systemctl restart cmpp-minio
|
||||
systemctl enable --now cmpp-api cmpp-send-worker cmpp-gateway nginx
|
||||
fi
|
||||
systemctl restart cmpp-gateway
|
||||
systemctl restart cmpp-security-agent
|
||||
systemctl restart cmpp-api
|
||||
systemctl restart cmpp-send-worker
|
||||
if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
|
||||
systemctl enable --now cmpp-submit-outbox
|
||||
systemctl restart cmpp-submit-outbox
|
||||
else
|
||||
systemctl disable --now cmpp-submit-outbox 2>/dev/null || true
|
||||
fi
|
||||
if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
|
||||
systemctl enable --now cmpp-gateway-callback
|
||||
systemctl restart cmpp-gateway-callback
|
||||
else
|
||||
systemctl disable --now cmpp-gateway-callback 2>/dev/null || true
|
||||
fi
|
||||
# Gateway is restarted after the callback listener so supplier events never point
|
||||
# at a callback port that has not completed Nest/Prisma initialization.
|
||||
systemctl restart cmpp-gateway
|
||||
systemctl restart nginx
|
||||
|
||||
echo "[deploy] Health checks"
|
||||
@@ -165,6 +234,13 @@ wait_for_http() {
|
||||
}
|
||||
wait_for_http "API" "http://127.0.0.1:${API_PORT:-3000}/api/health"
|
||||
wait_for_http "Send worker metrics" "http://127.0.0.1:${API_WORKER_METRICS_PORT:-9465}/metrics"
|
||||
if [[ "${SEND_SUBMIT_OUTBOX_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
|
||||
wait_for_http "Submit Outbox metrics" "http://127.0.0.1:${API_OUTBOX_METRICS_PORT:-9467}/metrics"
|
||||
fi
|
||||
if [[ "${GATEWAY_CALLBACK_SEPARATE_PROCESS_ENABLED:-false}" == "true" ]]; then
|
||||
wait_for_http "Gateway callback" "http://127.0.0.1:${API_CALLBACK_PORT}/api/health"
|
||||
wait_for_http "Gateway callback metrics" "http://127.0.0.1:${API_CALLBACK_METRICS_PORT:-9468}/metrics"
|
||||
fi
|
||||
wait_for_http "Gateway" "http://127.0.0.1:8090/health"
|
||||
redis-cli -h "${REDIS_HOST:-127.0.0.1}" -p "${REDIS_PORT:-6379}" ping >/dev/null
|
||||
pg_isready -d "${DATABASE_URL%%\?*}" >/dev/null
|
||||
|
||||
@@ -14,6 +14,8 @@ install -d -o root -g cmpp-security -m 0770 /run/cmpp-security-agent
|
||||
install -d -o root -g cmpp-security -m 0750 /var/lib/cmpp-security-agent
|
||||
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/api"
|
||||
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/send-worker"
|
||||
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/submit-outbox"
|
||||
install -d -o cmpp-api -g cmpp-security -m 0750 "$APP_DIR/logs/gateway-callback"
|
||||
[[ -d /var/lib/cmpp-platform/object-storage ]] && chown -R cmpp-api:cmpp-security /var/lib/cmpp-platform/object-storage
|
||||
|
||||
sed "s#@CMPP_SECURITY_AGENT_BIN@#$agent_binary#g" "$APP_DIR/deploy/security/cmpp-report-only.conf" >/etc/fail2ban/action.d/cmpp-report-only.conf
|
||||
@@ -42,6 +44,29 @@ ProtectHome=true
|
||||
ProtectSystem=true
|
||||
ReadWritePaths=$APP_DIR/logs/send-worker /var/lib/cmpp-platform/object-storage
|
||||
EOF
|
||||
install -d -m 0755 /etc/systemd/system/cmpp-submit-outbox.service.d
|
||||
cat >/etc/systemd/system/cmpp-submit-outbox.service.d/security-boundary.conf <<EOF
|
||||
[Service]
|
||||
User=cmpp-api
|
||||
Group=cmpp-security
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=true
|
||||
ReadWritePaths=$APP_DIR/logs/submit-outbox
|
||||
EOF
|
||||
install -d -m 0755 /etc/systemd/system/cmpp-gateway-callback.service.d
|
||||
cat >/etc/systemd/system/cmpp-gateway-callback.service.d/security-boundary.conf <<EOF
|
||||
[Service]
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
ReadWritePaths=$APP_DIR/logs/gateway-callback
|
||||
EOF
|
||||
grep -q 'cmpp-security.nft' /etc/nftables.conf || printf '\ninclude "/etc/nftables.d/cmpp-security.nft"\n' >>/etc/nftables.conf
|
||||
nft -c -f /etc/nftables.conf
|
||||
nft list table inet cmpp_security >/dev/null 2>&1 || nft -f /etc/nftables.d/cmpp-security.nft
|
||||
|
||||
Reference in New Issue
Block a user