fix: prevent daily report refresh timeouts

2026-09-08 17:16:43 +08:00
parent 2a9d03be2e
commit ebb185b22b
7 changed files with 1031 additions and 106 deletions
@@ -0,0 +1,400 @@
+/**
+ * Real PostgreSQL report integration, isolated from all messaging lifecycles.
+ * Build API first. Set REPORT_TEST_DATABASE_URL to a disposable local database
+ * named cmpp_report_test_*. The caller owns database/server creation and shutdown.
+ * This script creates and drops only its unique schema; it never uses DATABASE_URL.
+ */
+import assert from 'node:assert/strict';
+import { createRequire } from 'node:module';
+import { randomUUID } from 'node:crypto';
+
+const require = createRequire(new URL('../../api/package.json', import.meta.url));
+const { Pool } = require('pg');
+const { PrismaClient } = require('@prisma/client');
+const { PrismaPg } = require('@prisma/adapter-pg');
+const { ReportsService } = require('./dist/reports/reports.service.js');
+const { ReportsController } = require('./dist/reports/reports.controller.js');
+const { Module, Logger } = require('@nestjs/common');
+const { NestFactory } = require('@nestjs/core');
+
+const connectionString = process.env.REPORT_TEST_DATABASE_URL;
+assert.ok(connectionString, 'Set REPORT_TEST_DATABASE_URL explicitly; DATABASE_URL is never used');
+const target = new URL(connectionString);
+assert.ok(['postgres:', 'postgresql:'].includes(target.protocol), 'PostgreSQL URL required');
+assert.ok(['127.0.0.1', 'localhost', '[::1]'].includes(target.hostname), 'Only loopback PostgreSQL is allowed');
+assert.match(
+  decodeURIComponent(target.pathname.slice(1)),
+  /^cmpp_report_test_[a-z0-9_]+$/,
+  'Dedicated disposable database name required',
+);
+assert.equal(target.search, '', 'URL query overrides are forbidden');
+
+process.env.REPORT_DAILY_REFRESH_ENABLED = 'false';
+delete process.env.REPORT_REFRESH_TRANSACTION_TIMEOUT_MS;
+Logger.overrideLogger(false);
+const schema = `report_refresh_${randomUUID().replaceAll('-', '')}`;
+const admin = new Pool({ connectionString, max: 2 });
+const pool = new Pool({ connectionString, max: 4, options: `-c search_path=${schema} -c timezone=UTC` });
+const prisma = new PrismaClient({ adapter: new PrismaPg(pool, { schema, disposeExternalPool: true }) });
+const service = new ReportsService(prisma);
+const checks = [];
+let slowDayElapsedMs;
+const now = new Date('2026-09-08T03:00:00Z');
+const days = ['2026-09-04', '2026-09-05', '2026-09-06', '2026-09-07'];
+const tables = ['DailyReconciliationReport', 'DailyProfitReport', 'DailyQualityReport'];
+let app;
+let schemaCreated = false;
+
+async function check(name, operation) {
+  await operation();
+  checks.push(name);
+}
+
+async function snapshot(date, includeTimestamps = false) {
+  const result = {};
+  for (const table of tables) {
+    const expression = includeTimestamps ? 'to_jsonb(row)' : "to_jsonb(row) - 'generatedAt' - 'updatedAt'";
+    const rows = await pool.query(
+      `SELECT ${expression} AS value FROM "${table}" row WHERE "reportDate" = $1::date ORDER BY id`,
+      [date],
+    );
+    result[table] = rows.rows.map(({ value }) => value);
+  }
+  return result;
+}
+
+async function installFailureTrigger(body) {
+  await pool.query(`CREATE FUNCTION fail_report_test() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN
+    IF NEW."reportDate" = DATE '2026-09-04' THEN ${body} END IF;
+    RETURN NEW;
+  END $$;
+  CREATE TRIGGER fail_report_test BEFORE INSERT ON "DailyQualityReport" FOR EACH ROW EXECUTE FUNCTION fail_report_test()`);
+}
+
+async function removeFailureTrigger() {
+  await pool.query('DROP TRIGGER fail_report_test ON "DailyQualityReport"; DROP FUNCTION fail_report_test()');
+}
+
+async function assertPartialFailure() {
+  await assert.rejects(service.refreshRollingWindow(now), (error) => {
+    assert.match(error.message, /failed dates: 2026-09-04/);
+    assert.match(error.message, /refreshed dates: 2026-09-05, 2026-09-06, 2026-09-07/);
+    return true;
+  });
+}
+
+try {
+  const identity = await admin.query('SELECT current_database() AS database, host(inet_server_addr()) AS address');
+  assert.equal(identity.rows[0].database, decodeURIComponent(target.pathname.slice(1)));
+  assert.ok(['127.0.0.1', '::1'].includes(identity.rows[0].address), 'Server must actually be loopback');
+  await admin.query(`CREATE SCHEMA "${schema}"`);
+  schemaCreated = true;
+  // Minimal source tables expose exactly the columns read by reporting SQL.
+  // Report table columns/types and uniqueness mirror the production Prisma model.
+  await pool.query(`
+    CREATE TABLE "Tenant" (id text PRIMARY KEY, name text NOT NULL);
+    CREATE TABLE "SmsApplication" (id text PRIMARY KEY, name text NOT NULL);
+    CREATE TABLE "SmsChannel" (id text PRIMARY KEY, name text NOT NULL);
+    CREATE TABLE "SmsSignature" (id text PRIMARY KEY, name text NOT NULL);
+    CREATE TABLE "SmsDrainageInfo" (id text PRIMARY KEY, "siteName" text NOT NULL);
+    CREATE TABLE "SmsMessageRecord" (
+      id text PRIMARY KEY, "tenantId" text NOT NULL, "applicationId" text NOT NULL,
+      "signatureId" text, "drainageInfoId" text, "billingUnits" integer NOT NULL,
+      status text, "receiptStatus" text, "queuedAt" timestamp(3) NOT NULL,
+      "submittedAt" timestamp(3), "deliveredAt" timestamp(3), "unitPrice" bigint NOT NULL, "submitId" text
+    );
+    CREATE INDEX ON "SmsMessageRecord" ("queuedAt");
+    CREATE TABLE "SmsSubmitRecord" (
+      id text PRIMARY KEY, "messageRecordId" text NOT NULL, "channelId" text NOT NULL,
+      "gatewayMessageId" text, "submitId" text, "submitStatus" text NOT NULL,
+      "costUnitPrice" bigint NOT NULL, "submittedAt" timestamp(3), "createdAt" timestamp(3) NOT NULL
+    );
+    CREATE INDEX ON "SmsSubmitRecord" ("messageRecordId");
+    CREATE TABLE "SmsMessageSegmentAudit" (id text PRIMARY KEY, "submitRecordId" text NOT NULL, "receiptStatus" text);
+    CREATE INDEX ON "SmsMessageSegmentAudit" ("submitRecordId");
+    CREATE TABLE "SmsReceiptRecord" (id text PRIMARY KEY, "gatewayMessageId" text, "channelId" text, "receiptStatus" text, "deliveredAt" timestamp(3));
+    CREATE INDEX ON "SmsReceiptRecord" ("channelId", "gatewayMessageId");
+    CREATE TABLE "DailyReconciliationReport" (
+      id text PRIMARY KEY, "reportDate" date NOT NULL, "tenantId" text NOT NULL, "tenantName" text NOT NULL,
+      "applicationId" text NOT NULL, "applicationName" text NOT NULL,
+      "submittedUnits" integer NOT NULL, "sentUnits" integer NOT NULL, "unknownUnits" integer NOT NULL,
+      "successUnits" integer NOT NULL, "failedUnits" integer NOT NULL,
+      "generatedAt" timestamp(3) NOT NULL, "updatedAt" timestamp(3) NOT NULL,
+      UNIQUE ("reportDate", "tenantId", "applicationId")
+    );
+    CREATE TABLE "DailyProfitReport" (
+      id text PRIMARY KEY, "reportDate" date NOT NULL, "dimensionType" text NOT NULL, "dimensionId" text NOT NULL, "dimensionName" text NOT NULL,
+      "tenantId" text, "tenantName" text, "applicationId" text, "channelId" text,
+      "submittedUnits" integer NOT NULL, "sentUnits" integer NOT NULL, "unknownUnits" integer NOT NULL,
+      "successUnits" integer NOT NULL, "failedUnits" integer NOT NULL,
+      "revenueCents" bigint NOT NULL, "refundCents" bigint NOT NULL, "costCents" bigint NOT NULL,
+      "profitCents" bigint NOT NULL, "profitRateBps" integer NOT NULL,
+      "generatedAt" timestamp(3) NOT NULL, "updatedAt" timestamp(3) NOT NULL,
+      UNIQUE ("reportDate", "dimensionType", "dimensionId")
+    );
+    CREATE TABLE "DailyQualityReport" (
+      id text PRIMARY KEY, "reportDate" date NOT NULL, "dimensionType" text NOT NULL, "dimensionId" text NOT NULL, "dimensionName" text NOT NULL,
+      "tenantId" text, "tenantName" text, "applicationId" text, "channelId" text, "signatureId" text, "drainageInfoId" text,
+      "submittedUnits" integer NOT NULL, "sentUnits" integer NOT NULL, "unknownUnits" integer NOT NULL,
+      "successUnits" integer NOT NULL, "failedUnits" integer NOT NULL, "successRateBps" integer NOT NULL, "avgArrivalMs" integer,
+      "generatedAt" timestamp(3) NOT NULL, "updatedAt" timestamp(3) NOT NULL,
+      UNIQUE ("reportDate", "dimensionType", "dimensionId")
+    );
+    INSERT INTO "Tenant" VALUES ('t1', 'Report Test One'), ('t2', 'Report Test Two');
+    INSERT INTO "SmsApplication" VALUES ('a1', 'Test Application One'), ('a2', 'Test Application Two');
+    INSERT INTO "SmsChannel" VALUES ('c1', 'Test Channel One'), ('c2', 'Test Channel Two');
+    INSERT INTO "SmsSignature" VALUES ('sig1', 'Test Signature'), ('sig2', 'Second Test Signature');
+    INSERT INTO "SmsDrainageInfo" VALUES ('drain1', 'Test Site'), ('drain2', 'Second Test Site');
+  `);
+  const messages = [
+    ['cross', '2026-09-03T16:00:00.000Z', 3, 'delivered', 1000, 'cross_retry'],
+    ['legacy', '2026-09-04T02:00:00.000Z', 2, 'delivered', 500, 'legacy_submit'],
+    ['failed', '2026-09-04T03:00:00.000Z', 1, 'failed', 800, 'failed_submit'],
+    ['unknown', '2026-09-04T04:00:00.000Z', 1, 'unknown', 800, 'unknown_submit'],
+    ['before', '2026-09-03T15:59:59.999Z', 7, 'delivered', 900, 'before_submit'],
+    ['next', '2026-09-04T16:00:00.000Z', 1, 'delivered', 1000, 'next_submit'],
+    ['six', '2026-09-06T01:00:00.000Z', 1, 'delivered', 1000, 'six_submit'],
+    ['seven', '2026-09-07T01:00:00.000Z', 1, 'delivered', 1000, 'seven_submit'],
+    ['today', '2026-09-07T16:00:00.000Z', 11, 'delivered', 900, 'today_submit'],
+    ['other', '2026-09-04T02:00:00.000Z', 1, 'delivered', 2000, null],
+  ];
+  for (const [id, date, units, status, price, submitId] of messages) {
+    await pool.query(
+      `INSERT INTO "SmsMessageRecord" VALUES ($1,$2,$3,$9,$10,$4,$5,NULL,$6,$6,$6::timestamp + INTERVAL '1 second',$7,$8)`,
+      [
+        id,
+        id === 'other' ? 't2' : 't1',
+        id === 'other' ? 'a2' : 'a1',
+        units,
+        status,
+        date,
+        price,
+        submitId,
+        id === 'other' ? 'sig2' : 'sig1',
+        id === 'other' ? 'drain2' : 'drain1',
+      ],
+    );
+  }
+  const submits = [
+    ['cross_first', 'cross', 'c1', '2026-09-04T01:00:00Z', 100, ['delivered', 'undelivered', 'undelivered'], true],
+    ['cross_retry', 'cross', 'c2', '2026-09-05T01:00:00Z', 200, ['delivered', 'delivered', 'undelivered'], true],
+    ['legacy_submit', 'legacy', 'c1', '2026-09-04T02:00:00Z', 150, [], true],
+    ['failed_submit', 'failed', 'c1', '2026-09-04T03:00:00Z', 50, ['undelivered'], true],
+    ['unknown_submit', 'unknown', 'c1', '2026-09-04T04:00:00Z', 70, ['unknown'], false],
+    ['before_submit', 'before', 'c1', '2026-09-03T15:59:59Z', 99, ['delivered'], true],
+    ['next_submit', 'next', 'c1', '2026-09-04T16:00:00Z', 100, ['delivered'], true],
+    ['six_submit', 'six', 'c1', '2026-09-06T01:00:00Z', 100, ['delivered'], true],
+    ['seven_submit', 'seven', 'c1', '2026-09-07T01:00:00Z', 100, ['delivered'], true],
+    ['today_submit', 'today', 'c1', '2026-09-07T16:00:00Z', 99, ['delivered'], true],
+  ];
+  for (const [id, messageId, channel, date, price, audits, delivered] of submits) {
+    await pool.query('INSERT INTO "SmsSubmitRecord" VALUES ($1,$2,$3,$1,$1,\'accepted\',$4,$5,$5)', [
+      id,
+      messageId,
+      channel,
+      price,
+      date,
+    ]);
+    for (const [index, status] of audits.entries()) {
+      await pool.query('INSERT INTO "SmsMessageSegmentAudit" VALUES ($1,$2,$3)', [`${id}_${index}`, id, status]);
+    }
+    if (delivered)
+      await pool.query(
+        "INSERT INTO \"SmsReceiptRecord\" VALUES ($1,$1,$2,'delivered',$3::timestamp + INTERVAL '1 second')",
+        [id, channel, date],
+      );
+  }
+  await pool.query(
+    'INSERT INTO "SmsReceiptRecord" SELECT \'duplicate_legacy\', "gatewayMessageId", "channelId", "receiptStatus", "deliveredAt" FROM "SmsReceiptRecord" WHERE id=\'legacy_submit\'',
+  );
+
+  await check('seven report dimensions and Beijing complete-day boundaries', async () => {
+    assert.deepEqual(await service.refreshRollingWindow(now), { refreshedDates: days });
+    const stored = await snapshot(days[0]);
+    assert.equal(stored.DailyReconciliationReport.length, 2);
+    assert.deepEqual([...new Set(stored.DailyProfitReport.map((row) => row.dimensionType))].sort(), [
+      'application',
+      'channel',
+    ]);
+    assert.deepEqual([...new Set(stored.DailyQualityReport.map((row) => row.dimensionType))].sort(), [
+      'application',
+      'channel',
+      'drainage',
+      'signature',
+    ]);
+    const appRow = stored.DailyReconciliationReport.find((row) => row.applicationId === 'a1');
+    assert.deepEqual(
+      [appRow.submittedUnits, appRow.sentUnits, appRow.successUnits, appRow.failedUnits, appRow.unknownUnits],
+      [7, 7, 5, 1, 1],
+    );
+    for (const table of tables) {
+      const dates = await pool.query(`SELECT DISTINCT "reportDate"::text AS date FROM "${table}" ORDER BY date`);
+      assert.deepEqual(
+        dates.rows.map((row) => row.date),
+        days,
+      );
+    }
+  });
+
+  await check('cross-day retry cost, segment priority, legacy fallback and final-only revenue', async () => {
+    const appRow = (await snapshot(days[0])).DailyProfitReport.find((row) => row.dimensionId === 'a1');
+    assert.deepEqual(
+      [appRow.revenueCents, appRow.costCents, appRow.profitCents, appRow.profitRateBps],
+      [4000, 800, 3200, 8000],
+    );
+    const dayFourChannel = (await snapshot(days[0])).DailyProfitReport.find((row) => row.dimensionId === 'c1');
+    assert.deepEqual([dayFourChannel.revenueCents, dayFourChannel.costCents], [1000, 400]);
+    const retryChannel = (await snapshot(days[1])).DailyProfitReport.find((row) => row.dimensionId === 'c2');
+    assert.deepEqual([retryChannel.revenueCents, retryChannel.costCents], [3000, 400]);
+    const nextApp = (await snapshot(days[1])).DailyProfitReport.find((row) => row.dimensionId === 'a1');
+    assert.deepEqual([nextApp.revenueCents, nextApp.costCents, nextApp.submittedUnits], [1000, 100, 1]);
+  });
+
+  await check('repeated refresh is idempotent', async () => {
+    const before = await Promise.all(days.map((date) => snapshot(date)));
+    await service.refreshRollingWindow(now);
+    assert.deepEqual(await Promise.all(days.map((date) => snapshot(date))), before);
+  });
+
+  await check('real SQL failure rolls back all three tables and later days still refresh', async () => {
+    const before = await snapshot(days[0], true);
+    const later = await snapshot(days[1], true);
+    await pool.query('UPDATE "SmsMessageRecord" SET "unitPrice"=1100 WHERE id=\'cross\'');
+    await installFailureTrigger("RAISE EXCEPTION 'intentional isolated report failure';");
+    await assertPartialFailure();
+    assert.deepEqual(await snapshot(days[0], true), before);
+    assert.notDeepEqual(await snapshot(days[1], true), later);
+    await removeFailureTrigger();
+    await service.refreshRollingWindow(now);
+    const recovered = (await snapshot(days[0])).DailyProfitReport.find((row) => row.dimensionId === 'a1');
+    assert.equal(recovered.revenueCents, 4300);
+    await pool.query('UPDATE "SmsMessageRecord" SET "unitPrice"=1000 WHERE id=\'cross\'');
+    await service.refreshRollingWindow(now);
+  });
+
+  await check('database transaction day lock protects prior reports and permits retry', async () => {
+    const before = await snapshot(days[0], true);
+    const holder = await pool.connect();
+    try {
+      await holder.query('BEGIN');
+      await holder.query('SELECT pg_advisory_xact_lock($1::integer, 20260904)', [0x434d5052]);
+      await assertPartialFailure();
+      assert.deepEqual(await snapshot(days[0], true), before);
+    } finally {
+      await holder.query('ROLLBACK');
+      holder.release();
+    }
+    assert.deepEqual(await service.refreshRollingWindow(now), { refreshedDates: days });
+  });
+
+  await check('day transaction exceeding the old five-second limit completes within the new budget', async () => {
+    await installFailureTrigger(
+      'IF NEW."dimensionType" = \'application\' AND NEW."dimensionId" = \'a1\' THEN PERFORM pg_sleep(5.2); END IF;',
+    );
+    try {
+      const start = performance.now();
+      await service.refreshBusinessDay({
+        key: days[0],
+        reportDate: new Date('2026-09-04T00:00:00Z'),
+        startAt: new Date('2026-09-03T16:00:00Z'),
+        endAt: new Date('2026-09-04T16:00:00Z'),
+      });
+      slowDayElapsedMs = Math.round(performance.now() - start);
+      assert.ok(slowDayElapsedMs >= 5200);
+    } finally {
+      await removeFailureTrigger();
+    }
+  });
+
+  await check('bounded real statement timeout rolls back and recovers', async () => {
+    const before = await snapshot(days[0], true);
+    await installFailureTrigger('PERFORM pg_sleep(0.3);');
+    process.env.REPORT_REFRESH_TRANSACTION_TIMEOUT_MS = '200';
+    try {
+      await assertPartialFailure();
+      assert.deepEqual(await snapshot(days[0], true), before);
+    } finally {
+      delete process.env.REPORT_REFRESH_TRANSACTION_TIMEOUT_MS;
+      await removeFailureTrigger();
+    }
+    assert.deepEqual(await service.refreshRollingWindow(now), { refreshedDates: days });
+  });
+
+  await check('real ReportsController HTTP reads match PostgreSQL filtering pagination and exports', async () => {
+    // Harness exposes only the real reporting controller on an ephemeral loopback
+    // port. Full application authentication and UI are outside this integration.
+    class ReportTestModule {}
+    Module({ controllers: [ReportsController], providers: [{ provide: ReportsService, useValue: service }] })(
+      ReportTestModule,
+    );
+    app = await NestFactory.create(ReportTestModule, { logger: false });
+    app.use((_request, response, next) => {
+      response.app.set('json replacer', (_key, value) => (typeof value === 'bigint' ? Number(value) : value));
+      next();
+    });
+    await app.listen(0, '127.0.0.1');
+    const base = await app.getUrl();
+    const read = async (path) => {
+      const response = await fetch(`${base}/admin/reports/${path}`);
+      assert.equal(response.status, 200);
+      return response.json();
+    };
+    const recon = await read('reconciliation?dateFrom=2026-09-04&dateTo=2026-09-04&tenantId=t1&pageSize=1');
+    assert.equal(recon.total, 1);
+    assert.equal(recon.items[0].applicationId, 'a1');
+    assert.deepEqual(recon.summary, {
+      submittedUnits: 7,
+      sentUnits: 7,
+      unknownUnits: 1,
+      successUnits: 5,
+      failedUnits: 1,
+    });
+    const profit = await read('profit?dateFrom=2026-09-04&dateTo=2026-09-04&dimensionType=application&tenantId=t1');
+    assert.deepEqual([profit.summary.revenueCents, profit.summary.costCents], [4000, 800]);
+    assert.equal('refundCents' in profit.items[0], false);
+    for (const dimension of ['application', 'channel', 'signature', 'drainage']) {
+      const result = await read(`quality?dateFrom=2026-09-04&dateTo=2026-09-04&dimensionType=${dimension}&pageSize=1`);
+      const expected = await pool.query(
+        'SELECT count(*)::integer AS total, sum("sentUnits")::integer AS sent FROM "DailyQualityReport" WHERE "reportDate"=\'2026-09-04\' AND "dimensionType"=$1',
+        [dimension],
+      );
+      assert.equal(result.total, expected.rows[0].total);
+      assert.equal(result.summary.sentUnits, expected.rows[0].sent);
+      assert.equal(result.items.length, 1);
+    }
+    const empty = await read('reconciliation?tenantId=missing');
+    assert.equal(empty.total, 0);
+    for (const report of ['reconciliation', 'profit', 'quality']) {
+      const response = await fetch(
+        `${base}/admin/reports/${report}/export?dateFrom=2026-09-04&dateTo=2026-09-04&tenantId=t1`,
+      );
+      assert.equal(response.status, 200);
+      assert.match(response.headers.get('content-type'), /text\/csv/);
+      assert.ok((await response.text()).split('\n').length > 1);
+    }
+  });
+
+  console.log(
+    JSON.stringify(
+      {
+        passed: checks.length,
+        checks,
+        sourceMessages: messages.length,
+        sourceSubmits: submits.length,
+        reportDates: days,
+        slowDayElapsedMs,
+        expectedApplicationDayFour: { revenue: 4000, cost: 800, profit: 3200 },
+        schemaCleanup: 'performed in finally',
+        environment: 'isolated local PostgreSQL; no application or messaging lifecycle',
+      },
+      null,
+      2,
+    ),
+  );
+} finally {
+  if (app) await app.close();
+  service.onModuleDestroy();
+  await prisma.$disconnect();
+  if (schemaCreated) await admin.query(`DROP SCHEMA "${schema}" CASCADE`);
+  await admin.end();
+}