perf: expand gateway capacity and prevent receipt replay

api/src/protocol-log-worker.ts

@@ -0,0 +1,59 @@
+import { NestFactory } from '@nestjs/core';
+import IORedis from 'ioredis';
+import { ProtocolLogWorkerModule } from './protocol-log-worker.module';
+import { ProtocolLogsService, type ProtocolLogInput } from './protocol-logs/protocol-logs.service';
+
+const STREAM = process.env.GATEWAY_PROTOCOL_LOG_STREAM ?? 'gateway.protocol.logs';
+const GROUP = process.env.GATEWAY_PROTOCOL_LOG_GROUP ?? 'cmpp-protocol-log-writer';
+const CONSUMER = process.env.GATEWAY_PROTOCOL_LOG_CONSUMER ?? `protocol-log-${process.pid}`;
+const BATCH_SIZE = boundedEnv('PROTOCOL_LOG_STREAM_BATCH_SIZE', 250, 100, 500);
+
+async function bootstrap() {
+  process.env.CMPP_PROCESS_ROLE = 'protocol-log-worker';
+  const app = await NestFactory.createApplicationContext(ProtocolLogWorkerModule, { logger: ['log', 'warn', 'error'] });
+  const logs = app.get(ProtocolLogsService);
+  const redis = new IORedis(process.env.REDIS_URL ?? 'redis://127.0.0.1:6379', { maxRetriesPerRequest: null });
+  try { await redis.xgroup('CREATE', STREAM, GROUP, '0', 'MKSTREAM'); } catch (error) {
+    if (!String(error).includes('BUSYGROUP')) throw error;
+  }
+  let stopping = false;
+  const stop = () => { stopping = true; };
+  process.on('SIGTERM', stop); process.on('SIGINT', stop);
+  while (!stopping) {
+    const claimed = await redis.xautoclaim(STREAM, GROUP, CONSUMER, 30_000, '0-0', 'COUNT', BATCH_SIZE) as unknown as [string, Array<[string, string[]]>];
+    let messages = claimed[1] ?? [];
+    if (!messages.length) {
+      const reply = await redis.xreadgroup('GROUP', GROUP, CONSUMER, 'COUNT', BATCH_SIZE, 'BLOCK', 2000, 'STREAMS', STREAM, '>') as unknown as Array<[string, Array<[string, string[]]>]> | null;
+      if (!reply) continue;
+      messages = reply[0]?.[1] ?? [];
+    }
+    const accepted: string[] = [];
+    const parsed: ProtocolLogInput[] = [];
+    for (const [id, fields] of messages) {
+      const dataIndex = fields.indexOf('data');
+      try {
+        if (dataIndex < 0) throw new Error('data field missing');
+        parsed.push(JSON.parse(fields[dataIndex + 1]) as ProtocolLogInput);
+        accepted.push(id);
+      } catch (error) {
+        console.error(`protocol log stream event ${id} is invalid`, error);
+		await redis.xadd(`${STREAM}.dead`, '*', 'sourceId', id, 'error', String(error), 'data', dataIndex >= 0 ? fields[dataIndex + 1] : '');
+		await redis.xack(STREAM, GROUP, id); await redis.xdel(STREAM, id);
+      }
+    }
+    logs.recordMany(parsed);
+    if (accepted.length && await logs.flushNow()) {
+      const pipeline = redis.pipeline();
+      for (const id of accepted) pipeline.xack(STREAM, GROUP, id).xdel(STREAM, id);
+      await pipeline.exec();
+    }
+  }
+  await logs.flushNow(); await redis.quit(); await app.close();
+}
+
+function boundedEnv(name: string, fallback: number, minimum: number, maximum: number) {
+  const value = Number(process.env[name] ?? fallback);
+  return Number.isInteger(value) && value >= minimum && value <= maximum ? value : fallback;
+}
+
+void bootstrap().catch((error) => { console.error(error); process.exitCode = 1; });