feat: remediate HTTP API reliability and developer documentation
CSS quality / css-quality (push) Has been cancelled

This commit is contained in:
hectorzhao
2026-09-14 12:48:10 +08:00
parent d13ca0713a
commit f0e843436c
33 changed files with 3332 additions and 452 deletions
+187 -162
View File
@@ -1,3 +1,5 @@
import { OpenApiRecovery } from './open-api.recovery';
import { HTTP_REQUEST_CONTEXT } from '../send-chain/send-chain.contracts';
import {
BadRequestException,
ConflictException,
@@ -14,7 +16,7 @@ import {
} from '@nestjs/common';
import { Prisma } from '@prisma/client';
import { Queue, Worker } from 'bullmq';
import { createHash, createHmac, randomBytes, randomUUID } from 'node:crypto';
import { createHmac, randomBytes, randomUUID } from 'node:crypto';
import { lookup } from 'node:dns/promises';
import { isIP } from 'node:net';
import { request as httpRequest } from 'node:http';
@@ -24,8 +26,18 @@ import { SendChainService } from '../send-chain/send-chain.service';
import { decryptSecret, encryptSecret } from './open-api.crypto';
import type { OpenApiAuthContext } from './open-api.types';
import { ProtocolLogsService } from '../protocol-logs/protocol-logs.service';
import { publicOpenApiFailure, webhookJobId } from './open-api.protocol';
import { automaticDeliveryMode } from './delivery-mode';
export const OPEN_API_WEBHOOK_TRANSPORT = Symbol('open-api-webhook-transport');
export type OpenApiWebhookTransport = (
url: string,
body: string,
headers: Record<string, string>,
timeoutMs: number,
requireHttps: boolean,
) => Promise<{ status: number; body: string }>;
const WEBHOOK_QUEUE = 'http-webhook-delivery';
const RETRY_DELAYS_SECONDS = [0, 60, 300, 900, 3600, 21600, 86400];
@@ -58,11 +70,14 @@ export type HttpConfigInput = {
export class OpenApiService implements OnModuleInit, OnModuleDestroy {
private queue?: Queue<{ deliveryId: string }>;
private worker?: Worker<{ deliveryId: string }>;
private recovery?: OpenApiRecovery;
private recoveryTimer?: ReturnType<typeof setInterval>;
constructor(
private readonly prisma: PrismaService,
@Inject(forwardRef(() => SendChainService)) private readonly sendChain: SendChainService,
@Optional() private readonly protocolLogs?: ProtocolLogsService,
@Optional() @Inject(OPEN_API_WEBHOOK_TRANSPORT) private readonly webhookTransport?: OpenApiWebhookTransport,
) {}
onModuleInit() {
@@ -72,6 +87,9 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
// Delivery remains owned by the main API process so callback DB/HTTP capacity
// cannot be consumed by slow customer webhook endpoints.
if (process.env.CMPP_PROCESS_ROLE === 'callback') return;
this.recovery = new OpenApiRecovery(this.prisma, this.sendChain, this.queue);
this.recoveryTimer = setInterval(() => void this.recovery?.tick(), 15_000);
this.recoveryTimer.unref?.();
this.worker = new Worker(WEBHOOK_QUEUE, (job) => this.deliverWebhook(job.data.deliveryId), {
connection,
concurrency: 10,
@@ -79,6 +97,8 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
}
async onModuleDestroy() {
if (this.recoveryTimer) clearInterval(this.recoveryTimer);
await this.recovery?.close();
await this.worker?.close();
await this.queue?.close();
}
@@ -262,10 +282,17 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
) {
if (!auth.config.sendEnabled)
throw new ForbiddenException({ code: 'SEND_NOT_ENABLED', message: '该应用未开通HTTP短信发送' });
const mobile = String(input.mobile ?? '').trim();
if (
typeof input.mobile !== 'string' ||
typeof input.content !== 'string' ||
(input.clientMessageId != null &&
(typeof input.clientMessageId !== 'string' || Array.from(input.clientMessageId).length > 128))
) {
throw new BadRequestException({ code: 'PARAMETER_INVALID', message: '请求字段类型或长度非法' });
}
const mobile = input.mobile.trim();
const content = String(input.content ?? '');
if (!/^1[3-9]\d{9}$/.test(mobile))
throw new BadRequestException({ code: 'MOBILE_INVALID', message: '手机号格式非法' });
if (!/^1\d{10}$/.test(mobile)) throw new BadRequestException({ code: 'MOBILE_INVALID', message: '手机号格式非法' });
if (!content.trim()) throw new BadRequestException({ code: 'CONTENT_REQUIRED', message: '短信内容不能为空' });
const idempotencyKey = String(meta.idempotencyKey ?? '').trim();
if (!/^[A-Za-z0-9._:-]{8,128}$/.test(idempotencyKey))
@@ -283,8 +310,17 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
message: '同一Idempotency-Key对应的请求内容不一致',
});
if (existing.status === 'completed' && existing.responseBody) return existing.responseBody;
if (existing.status === 'failed' && existing.responseBody && existing.httpStatus)
if (['failed', 'requires_review'].includes(existing.status) && existing.responseBody && existing.httpStatus)
throw new HttpException(existing.responseBody as Record<string, unknown>, existing.httpStatus);
if (
existing.status === 'requires_review' ||
(existing.createdAt && Date.now() - existing.createdAt.getTime() > 600_000)
)
throw new ConflictException({
code: 'REQUEST_REQUIRES_REVIEW',
message: '请求结果待核对,请提供requestId联系支持,勿更换幂等键重发',
requestId: existing.requestId,
});
throw new ConflictException({ code: 'REQUEST_PROCESSING', message: '同一请求正在处理中,请稍后查询' });
}
if (input.clientMessageId) {
@@ -326,14 +362,15 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
message: '同一Idempotency-Key对应的请求内容不一致',
});
if (raced?.status === 'completed' && raced.responseBody) return raced.responseBody;
if (raced?.status === 'failed' && raced.responseBody && raced.httpStatus)
if (raced && ['failed', 'requires_review'].includes(raced.status) && raced.responseBody && raced.httpStatus)
throw new HttpException(raced.responseBody as Record<string, unknown>, raced.httpStatus);
throw new ConflictException({ code: 'REQUEST_PROCESSING', message: '同一请求正在处理中,请稍后查询' });
}
throw error;
}
try {
const task = await this.sendChain.createHttpBatchTask({
await this.sendChain.createHttpBatchTask({
[HTTP_REQUEST_CONTEXT]: { id: request.id, requestId },
tenantId: auth.application.tenantId,
applicationId: auth.application.id,
content,
@@ -342,50 +379,19 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
userAgent: meta.userAgent,
clientMessageId: input.clientMessageId,
});
const message = task.messages?.[0];
if (task.status === 'rejected' || message?.status === 'rejected') {
throw new UnprocessableEntityException({
code: 'SEND_REJECTED',
message: message?.errorMessage ?? task.rejectReason ?? '短信未通过业务校验',
});
const frozen = await this.prisma.openApiRequest.findUnique({ where: { id: request.id } });
if (frozen?.status === 'completed' && frozen.responseBody) {
void this.recovery?.tick();
return frozen.responseBody;
}
const response = {
code: 'ACCEPTED',
requestId,
messageId: message?.messageId,
clientMessageId: input.clientMessageId ?? null,
status: message?.status ?? task.status,
acceptedAt: new Date().toISOString(),
};
await this.prisma.openApiRequest.update({
where: { id: request.id },
data: {
status: 'completed',
httpStatus: 202,
businessCode: 'ACCEPTED',
responseBody: response,
messageRecordId: message?.id,
durationMs: Date.now() - startedAt,
completedAt: new Date(),
},
});
this.protocolLogs?.record({
protocol: 'http',
direction: 'client_to_platform',
eventType: 'send_request',
status: 'accepted',
tenantId: auth.application.tenantId,
applicationId: auth.application.id,
messageId: message?.messageId,
requestId,
phone: mobile,
resultCode: 'ACCEPTED',
durationMs: Date.now() - startedAt,
payloadBytes: Buffer.byteLength(content, 'utf8'),
detail: { clientMessageId: input.clientMessageId },
});
return response;
if (frozen?.status === 'failed' && frozen.responseBody && frozen.httpStatus)
throw new HttpException(frozen.responseBody as Record<string, unknown>, frozen.httpStatus);
throw new Error('HTTP acceptance snapshot was not committed');
} catch (error) {
const frozen = await this.prisma.openApiRequest.findUnique({ where: { id: request.id } });
if (frozen?.status === 'completed' && frozen.responseBody) return frozen.responseBody;
if (frozen?.status === 'failed' && frozen.responseBody && frozen.httpStatus)
throw new HttpException(frozen.responseBody as Record<string, unknown>, frozen.httpStatus);
let outwardError = error;
if (error instanceof HttpException && error.getStatus() === 400) {
const response = error.getResponse();
@@ -399,7 +405,7 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
await this.prisma.openApiRequest.update({
where: { id: request.id },
data: {
status: 'failed',
status: failure.httpStatus >= 500 ? 'requires_review' : 'failed',
httpStatus: failure.httpStatus,
businessCode: failure.code,
responseBody: failure.responseBody,
@@ -407,20 +413,8 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
completedAt: new Date(),
},
});
this.protocolLogs?.record({
protocol: 'http',
direction: 'client_to_platform',
eventType: 'send_request',
status: 'failed',
tenantId: auth.application.tenantId,
applicationId: auth.application.id,
requestId,
phone: mobile,
resultCode: failure.code,
durationMs: Date.now() - startedAt,
payloadBytes: Buffer.byteLength(content, 'utf8'),
});
throw outwardError;
throw new HttpException(failure.responseBody as Record<string, unknown>, failure.httpStatus);
}
}
@@ -451,6 +445,10 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
async listUplinks(auth: OpenApiAuthContext, query: Record<string, string | undefined>) {
if (!auth.config.uplinkQueryEnabled)
throw new ForbiddenException({ code: 'UPLINK_QUERY_NOT_ENABLED', message: '该应用未开通上行查询' });
for (const value of Object.values(query)) {
if (value !== undefined && typeof value !== 'string')
throw new BadRequestException({ code: 'PARAMETER_INVALID', message: '查询参数必须为单个字符串' });
}
const endTime = query.endTime ? new Date(query.endTime) : new Date();
const startTime = query.startTime ? new Date(query.startTime) : new Date(endTime.getTime() - 24 * 3600_000);
if (!Number.isFinite(startTime.getTime()) || !Number.isFinite(endTime.getTime()) || startTime > endTime)
@@ -460,7 +458,12 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
code: 'TIME_RANGE_TOO_LARGE',
message: `单次查询不能超过${auth.config.maxQueryRangeDays}`,
});
const limit = Math.min(Math.max(Number(query.limit) || 50, 1), auth.config.maxPageSize);
if (
query.limit !== undefined &&
(!/^\d+$/.test(query.limit) || !Number.isSafeInteger(Number(query.limit)) || Number(query.limit) < 1)
)
throw new BadRequestException({ code: 'LIMIT_INVALID', message: 'limit必须为正整数' });
const limit = Math.min(Number(query.limit ?? 50), auth.config.maxPageSize);
const cursor = decodeCursor(query.cursor);
const rows = await this.prisma.smsUplinkMessage.findMany({
where: {
@@ -482,8 +485,6 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
phoneNumber: true,
destId: true,
content: true,
matchStatus: true,
matchReason: true,
receivedAt: true,
},
orderBy: [{ receivedAt: 'desc' }, { id: 'desc' }],
@@ -499,7 +500,22 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
if (!auth.config.uplinkQueryEnabled)
throw new ForbiddenException({ code: 'UPLINK_QUERY_NOT_ENABLED', message: '该应用未开通上行查询' });
const row = await this.prisma.smsUplinkMessage.findFirst({
where: { id: uplinkId, applicationId: auth.application.id, matchStatus: 'matched' },
where: {
id: uplinkId,
applicationId: auth.application.id,
tenantId: auth.application.tenantId,
matchStatus: 'matched',
},
select: {
id: true,
messageId: true,
phoneNumber: true,
destId: true,
content: true,
receivedAt: true,
tenantId: true,
applicationId: true,
},
});
if (!row) throw new NotFoundException({ code: 'UPLINK_NOT_FOUND', message: '上行记录不存在' });
return row;
@@ -532,30 +548,32 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
: data.eventType === 'uplink' && data.uplinkMessageId
? `evt_uplink_${data.uplinkMessageId}`
: `evt_${randomUUID()}`;
const event = await this.prisma.httpWebhookEvent.upsert({
where: { eventId },
update: {},
create: {
eventId,
tenantId: data.tenantId,
applicationId: data.applicationId,
eventType: data.eventType,
messageRecordId: data.messageRecordId,
messageId: data.messageId,
uplinkMessageId: data.uplinkMessageId,
payload: data.payload as Prisma.InputJsonValue,
},
const delivery = await this.prisma.$transaction(async (tx) => {
const event = await tx.httpWebhookEvent.upsert({
where: { eventId },
update: {},
create: {
eventId,
tenantId: data.tenantId,
applicationId: data.applicationId!,
eventType: data.eventType,
messageRecordId: data.messageRecordId,
messageId: data.messageId,
uplinkMessageId: data.uplinkMessageId,
payload: data.payload as Prisma.InputJsonValue,
},
});
return tx.httpWebhookDelivery.upsert({
where: { eventId_endpointId: { eventId: event.id, endpointId: endpoint.id } },
update: {},
create: { eventId: event.id, endpointId: endpoint.id, recoveryVersion: 1 },
});
});
const delivery = await this.prisma.httpWebhookDelivery.upsert({
where: { eventId_endpointId: { eventId: event.id, endpointId: endpoint.id } },
update: {},
create: { eventId: event.id, endpointId: endpoint.id },
});
if (delivery.status === 'delivered') return delivery;
if (delivery.status !== 'pending' || delivery.recoveryVersion !== 1) return delivery;
await this.queue?.add(
'deliver',
{ deliveryId: delivery.id },
{ jobId: delivery.id, removeOnComplete: 1000, removeOnFail: 1000 },
{ jobId: webhookJobId(delivery.id, 1), removeOnComplete: 1000, removeOnFail: 1000 },
);
return delivery;
}
@@ -603,14 +621,27 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
where: { id: deliveryId, event: { applicationId } },
});
if (!delivery) throw new NotFoundException('Webhook投递记录不存在');
await this.prisma.httpWebhookDelivery.update({
where: { id: delivery.id },
data: { status: 'pending', nextRetryAt: null, lastError: null },
const reset = await this.prisma.httpWebhookDelivery.updateMany({
where: {
id: delivery.id,
status: { in: ['pending', 'retrying', 'failed'] },
attemptCount: delivery.attemptCount,
OR: [{ leaseUntil: null }, { leaseUntil: { lt: new Date() } }],
},
data: {
status: 'pending',
nextRetryAt: null,
lastError: null,
recoveryVersion: 1,
leaseToken: null,
leaseUntil: null,
},
});
if (!reset.count) throw new ConflictException('回调正在投递或已成功,不能重投');
await this.queue?.add(
'deliver',
{ deliveryId },
{ jobId: `${deliveryId}:manual:${Date.now()}`, removeOnComplete: 1000, removeOnFail: 1000 },
{ jobId: webhookJobId(deliveryId, Date.now()), removeOnComplete: 1000, removeOnFail: 1000 },
);
return { id: deliveryId, status: 'pending' };
}
@@ -620,11 +651,32 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
where: { id: deliveryId },
include: { event: true, endpoint: true },
});
if (!delivery || delivery.status === 'delivered') return;
if (!delivery || !['pending', 'retrying', 'delivering'].includes(delivery.status)) return;
if (delivery.nextRetryAt && delivery.nextRetryAt.getTime() > Date.now()) return;
const config = await this.prisma.smsApplicationHttpConfig.findUnique({
where: { applicationId: delivery.event.applicationId },
});
if (!config) return;
if (
!config?.enabled ||
delivery.endpoint.status !== 'active' ||
!(delivery.event.eventType === 'receipt' ? config.receiptWebhookEnabled : config.uplinkWebhookEnabled)
)
return;
const leaseToken = randomUUID();
const claimed = await this.prisma.httpWebhookDelivery.updateMany({
where: {
id: deliveryId,
status: delivery.status,
attemptCount: delivery.attemptCount,
OR: [{ leaseUntil: null }, { leaseUntil: { lt: new Date() } }],
},
data: {
status: 'delivering',
leaseToken,
leaseUntil: new Date(Date.now() + config.webhookTimeoutSeconds * 1000 + 60_000),
},
});
if (!claimed.count) return;
const attemptNo = delivery.attemptCount + 1;
const timestamp = String(Math.floor(Date.now() / 1000));
const body = JSON.stringify({
@@ -641,7 +693,7 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
let responseSummary: string | undefined;
let errorMessage: string | undefined;
try {
const response = await postWebhook(
const response = await (this.webhookTransport ?? postWebhook)(
delivery.endpoint.url,
body,
{
@@ -665,22 +717,6 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
responseStatus === 408 ||
responseStatus === 429 ||
(responseStatus !== undefined && responseStatus >= 500);
await this.prisma.httpWebhookAttempt.create({
data: {
deliveryId,
attemptNo,
responseStatus,
responseSummary,
errorMessage,
durationMs: Date.now() - startedAt,
requestHeaders: {
'x-event-id': delivery.event.eventId,
'x-event-type': delivery.event.eventType,
'x-timestamp': timestamp,
'x-signature': 'sha256=***',
},
},
});
this.protocolLogs?.record({
protocol: 'http',
direction: 'platform_to_client',
@@ -696,56 +732,53 @@ export class OpenApiService implements OnModuleInit, OnModuleDestroy {
retryCount: attemptNo - 1,
detail: { deliveryId, attemptNo, error: errorMessage },
});
if (success) {
await this.prisma.httpWebhookDelivery.update({
where: { id: deliveryId },
data: {
status: 'delivered',
attemptCount: attemptNo,
lastHttpStatus: responseStatus,
lastError: null,
deliveredAt: new Date(),
nextRetryAt: null,
},
});
return;
}
const maxAttempts = Math.min(config.webhookMaxAttempts, RETRY_DELAYS_SECONDS.length);
if (config.webhookRetryEnabled && retryable && attemptNo < maxAttempts) {
const delaySeconds = RETRY_DELAYS_SECONDS[attemptNo] ?? RETRY_DELAYS_SECONDS.at(-1)!;
const nextRetryAt = new Date(Date.now() + delaySeconds * 1000);
await this.prisma.httpWebhookDelivery.update({
where: { id: deliveryId },
const willRetry = !success && config.webhookRetryEnabled && retryable && attemptNo < maxAttempts;
const delaySeconds = RETRY_DELAYS_SECONDS[attemptNo] ?? RETRY_DELAYS_SECONDS.at(-1)!;
const nextRetryAt = willRetry ? new Date(Date.now() + delaySeconds * 1000) : null;
await this.prisma.$transaction(async (tx) => {
const updated = await tx.httpWebhookDelivery.updateMany({
where: { id: deliveryId, leaseToken },
data: {
status: 'retrying',
status: success ? 'delivered' : willRetry ? 'retrying' : 'failed',
attemptCount: attemptNo,
lastHttpStatus: responseStatus,
lastError: errorMessage ?? `HTTP ${responseStatus}`,
lastError: success ? null : (errorMessage ?? 'HTTP ' + responseStatus),
deliveredAt: success ? new Date() : null,
nextRetryAt,
leaseToken: null,
leaseUntil: null,
},
});
if (!updated.count) return;
await tx.httpWebhookAttempt.create({
data: {
deliveryId,
attemptNo,
responseStatus,
responseSummary,
errorMessage,
durationMs: Date.now() - startedAt,
requestHeaders: {
'x-event-id': delivery.event.eventId,
'x-event-type': delivery.event.eventType,
'x-timestamp': timestamp,
'x-signature': 'sha256=***',
},
},
});
});
if (willRetry)
await this.queue?.add(
'deliver',
{ deliveryId },
{
jobId: `${deliveryId}:${attemptNo + 1}`,
jobId: webhookJobId(deliveryId, attemptNo + 1),
delay: delaySeconds * 1000,
removeOnComplete: 1000,
removeOnFail: 1000,
},
);
return;
}
await this.prisma.httpWebhookDelivery.update({
where: { id: deliveryId },
data: {
status: 'failed',
attemptCount: attemptNo,
lastHttpStatus: responseStatus,
lastError: errorMessage ?? `HTTP ${responseStatus}`,
nextRetryAt: null,
},
});
}
private async requireApplication(applicationId: string, tenantId?: string) {
@@ -783,23 +816,11 @@ function httpApiPublicOrigin() {
}
function normalizeOpenApiFailure(error: unknown) {
if (error instanceof HttpException) {
const value = error.getResponse();
const object = typeof value === 'object' && value ? (value as Record<string, unknown>) : {};
const rawMessage = object.message ?? error.message;
return {
httpStatus: error.getStatus(),
code: String(object.code ?? 'SEND_REJECTED'),
responseBody: {
code: String(object.code ?? 'SEND_REJECTED'),
message: Array.isArray(rawMessage) ? rawMessage.join('') : String(rawMessage),
} as Prisma.InputJsonValue,
};
}
const failure = publicOpenApiFailure(error);
return {
httpStatus: 500,
code: 'INTERNAL_ERROR',
responseBody: { code: 'INTERNAL_ERROR', message: 'Internal server error' } as Prisma.InputJsonValue,
httpStatus: failure.status,
code: failure.code,
responseBody: { code: failure.code, message: failure.message } as Prisma.InputJsonValue,
};
}
@@ -968,7 +989,11 @@ function encodeCursor(receivedAt: Date, id: string) {
function decodeCursor(value?: string) {
if (!value) return null;
try {
const [date, id] = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')) as [string, string];
if (value.length > 2048 || !/^[A-Za-z0-9_-]+$/.test(value)) throw new Error();
const parsed: unknown = JSON.parse(Buffer.from(value, 'base64url').toString('utf8'));
if (!Array.isArray(parsed) || parsed.length !== 2 || typeof parsed[0] !== 'string' || typeof parsed[1] !== 'string')
throw new Error();
const [date, id] = parsed;
const receivedAt = new Date(date);
if (!id || !Number.isFinite(receivedAt.getTime())) throw new Error();
return { receivedAt, id };