Files
lislgosms/tools/testing/verify-report-refresh.mjs
hectorzhao ebb185b22b
CSS quality / css-quality (push) Has been cancelled
fix: prevent daily report refresh timeouts
2026-09-08 17:16:43 +08:00

401 lines
20 KiB
JavaScript

/**
* Real PostgreSQL report integration, isolated from all messaging lifecycles.
* Build API first. Set REPORT_TEST_DATABASE_URL to a disposable local database
* named cmpp_report_test_*. The caller owns database/server creation and shutdown.
* This script creates and drops only its unique schema; it never uses DATABASE_URL.
*/
import assert from 'node:assert/strict';
import { createRequire } from 'node:module';
import { randomUUID } from 'node:crypto';
const require = createRequire(new URL('../../api/package.json', import.meta.url));
const { Pool } = require('pg');
const { PrismaClient } = require('@prisma/client');
const { PrismaPg } = require('@prisma/adapter-pg');
const { ReportsService } = require('./dist/reports/reports.service.js');
const { ReportsController } = require('./dist/reports/reports.controller.js');
const { Module, Logger } = require('@nestjs/common');
const { NestFactory } = require('@nestjs/core');
const connectionString = process.env.REPORT_TEST_DATABASE_URL;
assert.ok(connectionString, 'Set REPORT_TEST_DATABASE_URL explicitly; DATABASE_URL is never used');
const target = new URL(connectionString);
assert.ok(['postgres:', 'postgresql:'].includes(target.protocol), 'PostgreSQL URL required');
assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(target.hostname), 'Only loopback PostgreSQL is allowed');
assert.match(
decodeURIComponent(target.pathname.slice(1)),
/^cmpp_report_test_[a-z0-9_]+$/,
'Dedicated disposable database name required',
);
assert.equal(target.search, '', 'URL query overrides are forbidden');
process.env.REPORT_DAILY_REFRESH_ENABLED = 'false';
delete process.env.REPORT_REFRESH_TRANSACTION_TIMEOUT_MS;
Logger.overrideLogger(false);
const schema = `report_refresh_${randomUUID().replaceAll('-', '')}`;
const admin = new Pool({ connectionString, max: 2 });
const pool = new Pool({ connectionString, max: 4, options: `-c search_path=${schema} -c timezone=UTC` });
const prisma = new PrismaClient({ adapter: new PrismaPg(pool, { schema, disposeExternalPool: true }) });
const service = new ReportsService(prisma);
const checks = [];
let slowDayElapsedMs;
const now = new Date('2026-09-08T03:00:00Z');
const days = ['2026-09-04', '2026-09-05', '2026-09-06', '2026-09-07'];
const tables = ['DailyReconciliationReport', 'DailyProfitReport', 'DailyQualityReport'];
let app;
let schemaCreated = false;
async function check(name, operation) {
await operation();
checks.push(name);
}
async function snapshot(date, includeTimestamps = false) {
const result = {};
for (const table of tables) {
const expression = includeTimestamps ? 'to_jsonb(row)' : "to_jsonb(row) - 'generatedAt' - 'updatedAt'";
const rows = await pool.query(
`SELECT ${expression} AS value FROM "${table}" row WHERE "reportDate" = $1::date ORDER BY id`,
[date],
);
result[table] = rows.rows.map(({ value }) => value);
}
return result;
}
async function installFailureTrigger(body) {
await pool.query(`CREATE FUNCTION fail_report_test() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
IF NEW."reportDate" = DATE '2026-09-04' THEN ${body} END IF;
RETURN NEW;
END $$;
CREATE TRIGGER fail_report_test BEFORE INSERT ON "DailyQualityReport" FOR EACH ROW EXECUTE FUNCTION fail_report_test()`);
}
async function removeFailureTrigger() {
await pool.query('DROP TRIGGER fail_report_test ON "DailyQualityReport"; DROP FUNCTION fail_report_test()');
}
async function assertPartialFailure() {
await assert.rejects(service.refreshRollingWindow(now), (error) => {
assert.match(error.message, /failed dates: 2026-09-04/);
assert.match(error.message, /refreshed dates: 2026-09-05, 2026-09-06, 2026-09-07/);
return true;
});
}
try {
const identity = await admin.query('SELECT current_database() AS database, host(inet_server_addr()) AS address');
assert.equal(identity.rows[0].database, decodeURIComponent(target.pathname.slice(1)));
assert.ok(['127.0.0.1', '::1'].includes(identity.rows[0].address), 'Server must actually be loopback');
await admin.query(`CREATE SCHEMA "${schema}"`);
schemaCreated = true;
// Minimal source tables expose exactly the columns read by reporting SQL.
// Report table columns/types and uniqueness mirror the production Prisma model.
await pool.query(`
CREATE TABLE "Tenant" (id text PRIMARY KEY, name text NOT NULL);
CREATE TABLE "SmsApplication" (id text PRIMARY KEY, name text NOT NULL);
CREATE TABLE "SmsChannel" (id text PRIMARY KEY, name text NOT NULL);
CREATE TABLE "SmsSignature" (id text PRIMARY KEY, name text NOT NULL);
CREATE TABLE "SmsDrainageInfo" (id text PRIMARY KEY, "siteName" text NOT NULL);
CREATE TABLE "SmsMessageRecord" (
id text PRIMARY KEY, "tenantId" text NOT NULL, "applicationId" text NOT NULL,
"signatureId" text, "drainageInfoId" text, "billingUnits" integer NOT NULL,
status text, "receiptStatus" text, "queuedAt" timestamp(3) NOT NULL,
"submittedAt" timestamp(3), "deliveredAt" timestamp(3), "unitPrice" bigint NOT NULL, "submitId" text
);
CREATE INDEX ON "SmsMessageRecord" ("queuedAt");
CREATE TABLE "SmsSubmitRecord" (
id text PRIMARY KEY, "messageRecordId" text NOT NULL, "channelId" text NOT NULL,
"gatewayMessageId" text, "submitId" text, "submitStatus" text NOT NULL,
"costUnitPrice" bigint NOT NULL, "submittedAt" timestamp(3), "createdAt" timestamp(3) NOT NULL
);
CREATE INDEX ON "SmsSubmitRecord" ("messageRecordId");
CREATE TABLE "SmsMessageSegmentAudit" (id text PRIMARY KEY, "submitRecordId" text NOT NULL, "receiptStatus" text);
CREATE INDEX ON "SmsMessageSegmentAudit" ("submitRecordId");
CREATE TABLE "SmsReceiptRecord" (id text PRIMARY KEY, "gatewayMessageId" text, "channelId" text, "receiptStatus" text, "deliveredAt" timestamp(3));
CREATE INDEX ON "SmsReceiptRecord" ("channelId", "gatewayMessageId");
CREATE TABLE "DailyReconciliationReport" (
id text PRIMARY KEY, "reportDate" date NOT NULL, "tenantId" text NOT NULL, "tenantName" text NOT NULL,
"applicationId" text NOT NULL, "applicationName" text NOT NULL,
"submittedUnits" integer NOT NULL, "sentUnits" integer NOT NULL, "unknownUnits" integer NOT NULL,
"successUnits" integer NOT NULL, "failedUnits" integer NOT NULL,
"generatedAt" timestamp(3) NOT NULL, "updatedAt" timestamp(3) NOT NULL,
UNIQUE ("reportDate", "tenantId", "applicationId")
);
CREATE TABLE "DailyProfitReport" (
id text PRIMARY KEY, "reportDate" date NOT NULL, "dimensionType" text NOT NULL, "dimensionId" text NOT NULL, "dimensionName" text NOT NULL,
"tenantId" text, "tenantName" text, "applicationId" text, "channelId" text,
"submittedUnits" integer NOT NULL, "sentUnits" integer NOT NULL, "unknownUnits" integer NOT NULL,
"successUnits" integer NOT NULL, "failedUnits" integer NOT NULL,
"revenueCents" bigint NOT NULL, "refundCents" bigint NOT NULL, "costCents" bigint NOT NULL,
"profitCents" bigint NOT NULL, "profitRateBps" integer NOT NULL,
"generatedAt" timestamp(3) NOT NULL, "updatedAt" timestamp(3) NOT NULL,
UNIQUE ("reportDate", "dimensionType", "dimensionId")
);
CREATE TABLE "DailyQualityReport" (
id text PRIMARY KEY, "reportDate" date NOT NULL, "dimensionType" text NOT NULL, "dimensionId" text NOT NULL, "dimensionName" text NOT NULL,
"tenantId" text, "tenantName" text, "applicationId" text, "channelId" text, "signatureId" text, "drainageInfoId" text,
"submittedUnits" integer NOT NULL, "sentUnits" integer NOT NULL, "unknownUnits" integer NOT NULL,
"successUnits" integer NOT NULL, "failedUnits" integer NOT NULL, "successRateBps" integer NOT NULL, "avgArrivalMs" integer,
"generatedAt" timestamp(3) NOT NULL, "updatedAt" timestamp(3) NOT NULL,
UNIQUE ("reportDate", "dimensionType", "dimensionId")
);
INSERT INTO "Tenant" VALUES ('t1', 'Report Test One'), ('t2', 'Report Test Two');
INSERT INTO "SmsApplication" VALUES ('a1', 'Test Application One'), ('a2', 'Test Application Two');
INSERT INTO "SmsChannel" VALUES ('c1', 'Test Channel One'), ('c2', 'Test Channel Two');
INSERT INTO "SmsSignature" VALUES ('sig1', 'Test Signature'), ('sig2', 'Second Test Signature');
INSERT INTO "SmsDrainageInfo" VALUES ('drain1', 'Test Site'), ('drain2', 'Second Test Site');
`);
const messages = [
['cross', '2026-09-03T16:00:00.000Z', 3, 'delivered', 1000, 'cross_retry'],
['legacy', '2026-09-04T02:00:00.000Z', 2, 'delivered', 500, 'legacy_submit'],
['failed', '2026-09-04T03:00:00.000Z', 1, 'failed', 800, 'failed_submit'],
['unknown', '2026-09-04T04:00:00.000Z', 1, 'unknown', 800, 'unknown_submit'],
['before', '2026-09-03T15:59:59.999Z', 7, 'delivered', 900, 'before_submit'],
['next', '2026-09-04T16:00:00.000Z', 1, 'delivered', 1000, 'next_submit'],
['six', '2026-09-06T01:00:00.000Z', 1, 'delivered', 1000, 'six_submit'],
['seven', '2026-09-07T01:00:00.000Z', 1, 'delivered', 1000, 'seven_submit'],
['today', '2026-09-07T16:00:00.000Z', 11, 'delivered', 900, 'today_submit'],
['other', '2026-09-04T02:00:00.000Z', 1, 'delivered', 2000, null],
];
for (const [id, date, units, status, price, submitId] of messages) {
await pool.query(
`INSERT INTO "SmsMessageRecord" VALUES ($1,$2,$3,$9,$10,$4,$5,NULL,$6,$6,$6::timestamp + INTERVAL '1 second',$7,$8)`,
[
id,
id === 'other' ? 't2' : 't1',
id === 'other' ? 'a2' : 'a1',
units,
status,
date,
price,
submitId,
id === 'other' ? 'sig2' : 'sig1',
id === 'other' ? 'drain2' : 'drain1',
],
);
}
const submits = [
['cross_first', 'cross', 'c1', '2026-09-04T01:00:00Z', 100, ['delivered', 'undelivered', 'undelivered'], true],
['cross_retry', 'cross', 'c2', '2026-09-05T01:00:00Z', 200, ['delivered', 'delivered', 'undelivered'], true],
['legacy_submit', 'legacy', 'c1', '2026-09-04T02:00:00Z', 150, [], true],
['failed_submit', 'failed', 'c1', '2026-09-04T03:00:00Z', 50, ['undelivered'], true],
['unknown_submit', 'unknown', 'c1', '2026-09-04T04:00:00Z', 70, ['unknown'], false],
['before_submit', 'before', 'c1', '2026-09-03T15:59:59Z', 99, ['delivered'], true],
['next_submit', 'next', 'c1', '2026-09-04T16:00:00Z', 100, ['delivered'], true],
['six_submit', 'six', 'c1', '2026-09-06T01:00:00Z', 100, ['delivered'], true],
['seven_submit', 'seven', 'c1', '2026-09-07T01:00:00Z', 100, ['delivered'], true],
['today_submit', 'today', 'c1', '2026-09-07T16:00:00Z', 99, ['delivered'], true],
];
for (const [id, messageId, channel, date, price, audits, delivered] of submits) {
await pool.query('INSERT INTO "SmsSubmitRecord" VALUES ($1,$2,$3,$1,$1,\'accepted\',$4,$5,$5)', [
id,
messageId,
channel,
price,
date,
]);
for (const [index, status] of audits.entries()) {
await pool.query('INSERT INTO "SmsMessageSegmentAudit" VALUES ($1,$2,$3)', [`${id}_${index}`, id, status]);
}
if (delivered)
await pool.query(
"INSERT INTO \"SmsReceiptRecord\" VALUES ($1,$1,$2,'delivered',$3::timestamp + INTERVAL '1 second')",
[id, channel, date],
);
}
await pool.query(
'INSERT INTO "SmsReceiptRecord" SELECT \'duplicate_legacy\', "gatewayMessageId", "channelId", "receiptStatus", "deliveredAt" FROM "SmsReceiptRecord" WHERE id=\'legacy_submit\'',
);
await check('seven report dimensions and Beijing complete-day boundaries', async () => {
assert.deepEqual(await service.refreshRollingWindow(now), { refreshedDates: days });
const stored = await snapshot(days[0]);
assert.equal(stored.DailyReconciliationReport.length, 2);
assert.deepEqual([...new Set(stored.DailyProfitReport.map((row) => row.dimensionType))].sort(), [
'application',
'channel',
]);
assert.deepEqual([...new Set(stored.DailyQualityReport.map((row) => row.dimensionType))].sort(), [
'application',
'channel',
'drainage',
'signature',
]);
const appRow = stored.DailyReconciliationReport.find((row) => row.applicationId === 'a1');
assert.deepEqual(
[appRow.submittedUnits, appRow.sentUnits, appRow.successUnits, appRow.failedUnits, appRow.unknownUnits],
[7, 7, 5, 1, 1],
);
for (const table of tables) {
const dates = await pool.query(`SELECT DISTINCT "reportDate"::text AS date FROM "${table}" ORDER BY date`);
assert.deepEqual(
dates.rows.map((row) => row.date),
days,
);
}
});
await check('cross-day retry cost, segment priority, legacy fallback and final-only revenue', async () => {
const appRow = (await snapshot(days[0])).DailyProfitReport.find((row) => row.dimensionId === 'a1');
assert.deepEqual(
[appRow.revenueCents, appRow.costCents, appRow.profitCents, appRow.profitRateBps],
[4000, 800, 3200, 8000],
);
const dayFourChannel = (await snapshot(days[0])).DailyProfitReport.find((row) => row.dimensionId === 'c1');
assert.deepEqual([dayFourChannel.revenueCents, dayFourChannel.costCents], [1000, 400]);
const retryChannel = (await snapshot(days[1])).DailyProfitReport.find((row) => row.dimensionId === 'c2');
assert.deepEqual([retryChannel.revenueCents, retryChannel.costCents], [3000, 400]);
const nextApp = (await snapshot(days[1])).DailyProfitReport.find((row) => row.dimensionId === 'a1');
assert.deepEqual([nextApp.revenueCents, nextApp.costCents, nextApp.submittedUnits], [1000, 100, 1]);
});
await check('repeated refresh is idempotent', async () => {
const before = await Promise.all(days.map((date) => snapshot(date)));
await service.refreshRollingWindow(now);
assert.deepEqual(await Promise.all(days.map((date) => snapshot(date))), before);
});
await check('real SQL failure rolls back all three tables and later days still refresh', async () => {
const before = await snapshot(days[0], true);
const later = await snapshot(days[1], true);
await pool.query('UPDATE "SmsMessageRecord" SET "unitPrice"=1100 WHERE id=\'cross\'');
await installFailureTrigger("RAISE EXCEPTION 'intentional isolated report failure';");
await assertPartialFailure();
assert.deepEqual(await snapshot(days[0], true), before);
assert.notDeepEqual(await snapshot(days[1], true), later);
await removeFailureTrigger();
await service.refreshRollingWindow(now);
const recovered = (await snapshot(days[0])).DailyProfitReport.find((row) => row.dimensionId === 'a1');
assert.equal(recovered.revenueCents, 4300);
await pool.query('UPDATE "SmsMessageRecord" SET "unitPrice"=1000 WHERE id=\'cross\'');
await service.refreshRollingWindow(now);
});
await check('database transaction day lock protects prior reports and permits retry', async () => {
const before = await snapshot(days[0], true);
const holder = await pool.connect();
try {
await holder.query('BEGIN');
await holder.query('SELECT pg_advisory_xact_lock($1::integer, 20260904)', [0x434d5052]);
await assertPartialFailure();
assert.deepEqual(await snapshot(days[0], true), before);
} finally {
await holder.query('ROLLBACK');
holder.release();
}
assert.deepEqual(await service.refreshRollingWindow(now), { refreshedDates: days });
});
await check('day transaction exceeding the old five-second limit completes within the new budget', async () => {
await installFailureTrigger(
'IF NEW."dimensionType" = \'application\' AND NEW."dimensionId" = \'a1\' THEN PERFORM pg_sleep(5.2); END IF;',
);
try {
const start = performance.now();
await service.refreshBusinessDay({
key: days[0],
reportDate: new Date('2026-09-04T00:00:00Z'),
startAt: new Date('2026-09-03T16:00:00Z'),
endAt: new Date('2026-09-04T16:00:00Z'),
});
slowDayElapsedMs = Math.round(performance.now() - start);
assert.ok(slowDayElapsedMs >= 5200);
} finally {
await removeFailureTrigger();
}
});
await check('bounded real statement timeout rolls back and recovers', async () => {
const before = await snapshot(days[0], true);
await installFailureTrigger('PERFORM pg_sleep(0.3);');
process.env.REPORT_REFRESH_TRANSACTION_TIMEOUT_MS = '200';
try {
await assertPartialFailure();
assert.deepEqual(await snapshot(days[0], true), before);
} finally {
delete process.env.REPORT_REFRESH_TRANSACTION_TIMEOUT_MS;
await removeFailureTrigger();
}
assert.deepEqual(await service.refreshRollingWindow(now), { refreshedDates: days });
});
await check('real ReportsController HTTP reads match PostgreSQL filtering pagination and exports', async () => {
// Harness exposes only the real reporting controller on an ephemeral loopback
// port. Full application authentication and UI are outside this integration.
class ReportTestModule {}
Module({ controllers: [ReportsController], providers: [{ provide: ReportsService, useValue: service }] })(
ReportTestModule,
);
app = await NestFactory.create(ReportTestModule, { logger: false });
app.use((_request, response, next) => {
response.app.set('json replacer', (_key, value) => (typeof value === 'bigint' ? Number(value) : value));
next();
});
await app.listen(0, '127.0.0.1');
const base = await app.getUrl();
const read = async (path) => {
const response = await fetch(`${base}/admin/reports/${path}`);
assert.equal(response.status, 200);
return response.json();
};
const recon = await read('reconciliation?dateFrom=2026-09-04&dateTo=2026-09-04&tenantId=t1&pageSize=1');
assert.equal(recon.total, 1);
assert.equal(recon.items[0].applicationId, 'a1');
assert.deepEqual(recon.summary, {
submittedUnits: 7,
sentUnits: 7,
unknownUnits: 1,
successUnits: 5,
failedUnits: 1,
});
const profit = await read('profit?dateFrom=2026-09-04&dateTo=2026-09-04&dimensionType=application&tenantId=t1');
assert.deepEqual([profit.summary.revenueCents, profit.summary.costCents], [4000, 800]);
assert.equal('refundCents' in profit.items[0], false);
for (const dimension of ['application', 'channel', 'signature', 'drainage']) {
const result = await read(`quality?dateFrom=2026-09-04&dateTo=2026-09-04&dimensionType=${dimension}&pageSize=1`);
const expected = await pool.query(
'SELECT count(*)::integer AS total, sum("sentUnits")::integer AS sent FROM "DailyQualityReport" WHERE "reportDate"=\'2026-09-04\' AND "dimensionType"=$1',
[dimension],
);
assert.equal(result.total, expected.rows[0].total);
assert.equal(result.summary.sentUnits, expected.rows[0].sent);
assert.equal(result.items.length, 1);
}
const empty = await read('reconciliation?tenantId=missing');
assert.equal(empty.total, 0);
for (const report of ['reconciliation', 'profit', 'quality']) {
const response = await fetch(
`${base}/admin/reports/${report}/export?dateFrom=2026-09-04&dateTo=2026-09-04&tenantId=t1`,
);
assert.equal(response.status, 200);
assert.match(response.headers.get('content-type'), /text\/csv/);
assert.ok((await response.text()).split('\n').length > 1);
}
});
console.log(
JSON.stringify(
{
passed: checks.length,
checks,
sourceMessages: messages.length,
sourceSubmits: submits.length,
reportDates: days,
slowDayElapsedMs,
expectedApplicationDayFour: { revenue: 4000, cost: 800, profit: 3200 },
schemaCleanup: 'performed in finally',
environment: 'isolated local PostgreSQL; no application or messaging lifecycle',
},
null,
2,
),
);
} finally {
if (app) await app.close();
service.onModuleDestroy();
await prisma.$disconnect();
if (schemaCreated) await admin.query(`DROP SCHEMA "${schema}" CASCADE`);
await admin.end();
}