feat: add caller connection and answer rate analytics

packages/auth/src/index.ts

@@ -4,6 +4,8 @@ export const ACCESS_TOKEN_TYPE = 'Bearer';
 export const PASSWORD_ALGO_ARGON2ID = 'argon2id';
 
 export type PermissionKey =
+  | 'caller_analytics.view'
+  | 'caller_analytics.view_all'
   | 'dashboard.view'
   | 'active_calls.view'
   | 'active_calls.manage'

packages/database/src/caller-analytics-model.ts

@@ -0,0 +1,130 @@
+import { createHash } from 'node:crypto';
+
+export const ANALYTICS_VERSION = 'caller-analytics-v1.1';
+export const ANALYTICS_STREAM = 'stream:caller_analytics';
+export const ANALYTICS_GROUP = 'caller-analytics-workers';
+export type AnalyticsKind = 'START' | 'ATTEMPT' | 'PROGRESS' | 'ACCEPTED' | 'DURATION' | 'END' | 'RECONCILE' | 'UNKNOWN';
+export interface AnalyticsEvent {
+  eventId: string; callUid: string; callId: string; attemptId: string;
+  kind: AnalyticsKind; at: number; startedAt: number;
+  customerId: string; customerGatewayId: string; vendorId: string; vendorGatewayId: string;
+  caller: string; landingCaller: string; callee: string; city: string; carrier: string;
+  code: number; talkMs: number | null; source: string; sequence: number;
+}
+export interface AnalyticsLeg {
+  event: AnalyticsEvent; first180At: number | null; first183At: number | null;
+  acceptedAt: number | null; endedAt: number | null; finalCode: number;
+  talkMs: number; durationAt: number; durationFinal: boolean; unknown: boolean;
+}
+export interface AnalyticsCall { start: AnalyticsEvent; legs: Record<string, AnalyticsLeg>; }
+export interface AnalyticsCounts {
+  totalCalls: number; connectedCalls: number; answeredCalls: number;
+  failedCalls: number; pendingCalls: number; unknownCalls: number; activeCalls: number; talkMs: number;
+}
+export const COUNT_KEYS = ['totalCalls', 'connectedCalls', 'answeredCalls', 'failedCalls', 'pendingCalls', 'unknownCalls', 'activeCalls', 'talkMs'] as const;
+export const emptyCounts = (): AnalyticsCounts => ({ totalCalls: 0, connectedCalls: 0, answeredCalls: 0, failedCalls: 0, pendingCalls: 0, unknownCalls: 0, activeCalls: 0, talkMs: 0 });
+export function analyticsHash(value: unknown): string { return createHash('sha256').update(JSON.stringify(value)).digest('hex'); }
+
+export function parseAnalyticsEvent(fields: string[]): AnalyticsEvent {
+  const f: Record<string, string> = {};
+  for (let i = 0; i < fields.length; i += 2) f[fields[i]] = fields[i + 1];
+  const text = (key: string, max = 255): string => {
+    const value = f[key] ?? '';
+    if (value.length > max || [...value].some(c => c.charCodeAt(0) < 32)) throw new Error(`Invalid ${key}`);
+    return value === 'none' ? '' : value;
+  };
+  const num = (key: string, fallback = 0): number => {
+    const value = f[key] === undefined ? fallback : Number(f[key]);
+    if (!Number.isSafeInteger(value) || value < 0) throw new Error(`Invalid ${key}`);
+    return value;
+  };
+  const kind = text('kind') as AnalyticsKind;
+  if (f.version !== '1' || !['START', 'ATTEMPT', 'PROGRESS', 'ACCEPTED', 'DURATION', 'END', 'RECONCILE', 'UNKNOWN'].includes(kind)) throw new Error('Invalid analytics schema');
+  const at = num('at'); const startedAt = num('startedAt');
+  if (at < 1_700_000_000_000 || at > Date.now() + 60_000 || startedAt > at || startedAt < 1_700_000_000_000) throw new Error('Invalid event time');
+  const event: AnalyticsEvent = {
+    eventId: text('eventId', 255), callUid: text('callUid', 255), callId: text('callId'), attemptId: text('attemptId', 80),
+    kind, at, startedAt, customerId: text('customerId', 32), customerGatewayId: text('customerGatewayId', 32),
+    vendorId: text('vendorId', 32), vendorGatewayId: text('vendorGatewayId', 32),
+    caller: text('caller', 64), landingCaller: text('landingCaller', 64), callee: text('callee', 64),
+    city: text('city', 12), carrier: text('carrier', 20), code: num('code'),
+    talkMs: f.talkMs === undefined || f.talkMs === '-1' ? null : num('talkMs'), source: text('source', 64), sequence: num('sequence')
+  };
+  if (!event.eventId || !event.callUid || !event.callId || !event.customerId || !event.caller) throw new Error('Missing call identity');
+  if (event.code > 699 || (kind === 'PROGRESS' && ![100, 180, 181, 182, 183].includes(event.code))) throw new Error('Invalid response code');
+  if (kind === 'RECONCILE' && event.source !== 'verified-dialog-final') throw new Error('Unverified CDR cannot reconcile');
+  return event;
+}
+
+const earliest = (a: number | null, b: number): number => a === null ? b : Math.min(a, b);
+export function foldAnalytics(call: AnalyticsCall | null, event: AnalyticsEvent): AnalyticsCall {
+  const result: AnalyticsCall = call ? structuredClone(call) : { start: event, legs: {} };
+  if (result.start.customerId !== event.customerId || result.start.caller !== event.caller) throw new Error('Conflicting call identity');
+  if (event.startedAt < result.start.startedAt || (event.kind === 'START' && event.startedAt === result.start.startedAt)) result.start = event;
+  if (event.kind === 'START') return result;
+  const key = event.attemptId || 'platform';
+  const leg: AnalyticsLeg = result.legs[key] ?? {
+    event, first180At: null, first183At: null, acceptedAt: null, endedAt: null,
+    finalCode: 0, talkMs: 0, durationAt: 0, durationFinal: false, unknown: event.kind !== 'ATTEMPT'
+  };
+  if (event.kind === 'ATTEMPT') { leg.event = event; leg.unknown = false; }
+  if (event.kind === 'PROGRESS' && event.code === 180) leg.first180At = earliest(leg.first180At, event.at);
+  if (event.kind === 'PROGRESS' && event.code === 183) leg.first183At = earliest(leg.first183At, event.at);
+  if (event.kind === 'ACCEPTED' && event.code >= 200 && event.code < 300) leg.acceptedAt = earliest(leg.acceptedAt, event.at);
+  if (event.kind === 'UNKNOWN') leg.unknown = true;
+  if (event.kind === 'END' || event.kind === 'RECONCILE') {
+    if (leg.endedAt === null || event.at >= leg.endedAt) { leg.endedAt = event.at; leg.finalCode = event.code; }
+    // END carries the producer's accumulated phase flags through separate replayable events.
+    if (event.source === 'verified-dialog-final' || event.source === 'platform-final') leg.unknown = false;
+    if (event.source === 'verified-dialog-final' && leg.acceptedAt !== null && event.talkMs === null) {
+      leg.talkMs = Math.max(0, event.at - leg.acceptedAt); leg.durationAt = event.at; leg.durationFinal = true;
+    }
+  }
+  if (event.kind === 'ACCEPTED' && leg.endedAt !== null && !leg.durationFinal) {
+    leg.talkMs = Math.max(0, leg.endedAt - event.at); leg.durationAt = leg.endedAt; leg.durationFinal = true;
+  }
+  if (event.talkMs !== null && ['DURATION', 'END', 'RECONCILE'].includes(event.kind)) {
+    const final = event.kind !== 'DURATION';
+    if ((!leg.durationFinal || final) && event.at >= leg.durationAt) {
+      leg.talkMs = event.talkMs; leg.durationAt = event.at; leg.durationFinal = final;
+    }
+  }
+  result.legs[key] = leg;
+  return result;
+}
+
+export interface AnalyticsProjection extends AnalyticsCounts {
+  id: string; callKey: string; view: 'original' | 'landing'; startedAt: number;
+  customerId: string; customerGatewayId: string; vendorId: string; vendorGatewayId: string;
+  caller: string; callee: string; city: string; carrier: string; payload: AnalyticsCall | AnalyticsLeg;
+}
+function legCounts(leg: AnalyticsLeg): AnalyticsCounts {
+  const connected = leg.first180At !== null || leg.first183At !== null || leg.acceptedAt !== null || leg.talkMs > 0;
+  const unknown = !connected && leg.unknown;
+  return { totalCalls: 1, connectedCalls: +connected, answeredCalls: +(leg.talkMs > 0), failedCalls: +(!connected && !unknown && leg.endedAt !== null), pendingCalls: +(!connected && !unknown && leg.endedAt === null), unknownCalls: +unknown, activeCalls: +(leg.endedAt === null), talkMs: leg.talkMs };
+}
+export function projectAnalytics(call: AnalyticsCall): AnalyticsProjection[] {
+  const e = call.start; const key = analyticsHash(e.callUid);
+  const legs = Object.values(call.legs); const counts = legs.map(legCounts);
+  const connected = counts.some(c => c.connectedCalls > 0); const active = !legs.length || counts.some(c => c.activeCalls > 0);
+  const unknown = !connected && counts.some(c => c.unknownCalls > 0);
+  const base = { customerId: e.customerId, customerGatewayId: e.customerGatewayId, callee: e.callee, city: e.city, carrier: e.carrier, callKey: key };
+  return [{ ...base, id: analyticsHash([key, 'original']), view: 'original', startedAt: e.startedAt, caller: e.caller,
+    vendorId: '', vendorGatewayId: '', payload: call, totalCalls: 1, connectedCalls: +connected,
+    answeredCalls: +(counts.some(c => c.answeredCalls > 0)), failedCalls: +(!connected && !unknown && !active),
+    pendingCalls: +(!connected && !unknown && active), unknownCalls: +unknown, activeCalls: +active,
+    talkMs: counts.reduce((sum, c) => sum + c.talkMs, 0)
+  }, ...legs.filter(l => l.event.attemptId && l.event.vendorGatewayId).map(leg => ({
+    ...base, ...legCounts(leg), id: analyticsHash([key, leg.event.attemptId]), view: 'landing' as const,
+    startedAt: leg.event.startedAt, caller: leg.event.landingCaller || leg.event.caller,
+    vendorId: leg.event.vendorId, vendorGatewayId: leg.event.vendorGatewayId, payload: leg
+  }))];
+}
+
+export function analyticsRates(counts: AnalyticsCounts) {
+  const ratio = (n: number, d: number) => d ? Number((n / d * 100).toFixed(4)) : null;
+  return { ...counts, notConnectedCalls: counts.failedCalls + counts.pendingCalls,
+    connectionRate: counts.unknownCalls ? null : ratio(counts.connectedCalls, counts.totalCalls),
+    overallAnswerRate: ratio(counts.answeredCalls, counts.totalCalls),
+    connectedAnswerRate: counts.unknownCalls ? null : ratio(counts.answeredCalls, counts.connectedCalls) };
+}

packages/database/src/caller-analytics-store.ts

@@ -0,0 +1,52 @@
+import { Prisma, PrismaClient } from '@prisma/client';
+import { analyticsHash, foldAnalytics, projectAnalytics, COUNT_KEYS, type AnalyticsCall, type AnalyticsEvent, type AnalyticsProjection } from './caller-analytics-model.js';
+
+type Tx = Prisma.TransactionClient;
+const json = <T>(value: T | string): T => typeof value === 'string' ? JSON.parse(value) as T : value;
+export class CallerAnalyticsStore {
+  constructor(private readonly db: PrismaClient) {}
+
+  async consume(event: AnalyticsEvent): Promise<'processed' | 'duplicate'> {
+    // The legacy router puts diagnostic markers in customer_id for unauthenticated traffic.
+    // Such traffic is outside the business population and must not create analysis calls.
+    if (!await this.db.customer.findUnique({ where: { id: event.customerId }, select: { id: true } })) return 'duplicate';
+    return this.db.$transaction(async tx => {
+      const callKey = analyticsHash(event.callUid);
+      await tx.$executeRaw`INSERT IGNORE INTO caller_analysis_calls (id, payload, updated_at) VALUES (${callKey}, NULL, NOW(3))`;
+      const rows = await tx.$queryRaw<Array<{ payload: AnalyticsCall | string | null }>>`SELECT payload FROM caller_analysis_calls WHERE id=${callKey} FOR UPDATE`;
+      const eventKey = analyticsHash(event.eventId);
+      const previous = await tx.$queryRaw<Array<{ id: string }>>`SELECT id FROM caller_analysis_events WHERE id=${eventKey}`;
+      if (previous.length) return 'duplicate';
+      const old = rows[0].payload ? json<AnalyticsCall>(rows[0].payload) : null;
+      const next = foldAnalytics(old, event);
+      const before = new Map((old ? projectAnalytics(old) : []).map(row => [row.id, row]));
+      for (const row of projectAnalytics(next)) {
+        const prior = before.get(row.id);
+        if (prior) await this.bucket(tx, prior, -1);
+        await this.bucket(tx, row, 1);
+        await this.state(tx, row);
+      }
+      await tx.$executeRaw`UPDATE caller_analysis_calls SET payload=${JSON.stringify(next)}, updated_at=NOW(3) WHERE id=${callKey}`;
+      await tx.$executeRaw`INSERT INTO caller_analysis_events (id, call_key, occurred_at, payload, created_at) VALUES (${eventKey}, ${callKey}, ${new Date(event.at)}, ${JSON.stringify(event)}, NOW(3))`;
+      return 'processed';
+    }, { timeout: 15000 });
+  }
+
+  private async state(tx: Tx, row: AnalyticsProjection) {
+    await tx.$executeRaw`INSERT INTO caller_analysis_states
+      (id, call_key, view, started_at, customer_id, customer_gateway_id, vendor_id, vendor_gateway_id, caller, callee, city, carrier, counts, payload, updated_at)
+      VALUES (${row.id}, ${row.callKey}, ${row.view}, ${new Date(row.startedAt)}, ${row.customerId}, ${row.customerGatewayId}, ${row.vendorId}, ${row.vendorGatewayId}, ${row.caller}, ${row.callee}, ${row.city}, ${row.carrier}, ${JSON.stringify(Object.fromEntries(COUNT_KEYS.map(k => [k, row[k]])))}, ${JSON.stringify(row.payload)}, NOW(3))
+      ON DUPLICATE KEY UPDATE started_at=VALUES(started_at), customer_gateway_id=VALUES(customer_gateway_id), vendor_id=VALUES(vendor_id), vendor_gateway_id=VALUES(vendor_gateway_id), caller=VALUES(caller), callee=VALUES(callee), city=VALUES(city), carrier=VALUES(carrier), counts=VALUES(counts), payload=VALUES(payload), updated_at=NOW(3)`;
+  }
+
+  private async bucket(tx: Tx, row: AnalyticsProjection, sign: number) {
+    const minute = new Date(Math.floor(row.startedAt / 60000) * 60000);
+    const id = analyticsHash([minute.toISOString(), row.view, row.customerId, row.customerGatewayId, row.vendorId, row.vendorGatewayId, row.caller, row.city, row.carrier]);
+    const delta = JSON.stringify(Object.fromEntries(COUNT_KEYS.map(k => [k, sign * row[k]])));
+    const additions = Prisma.join(COUNT_KEYS.flatMap(k => [Prisma.sql`${`$.${k}`}`, Prisma.sql`CAST(JSON_UNQUOTE(JSON_EXTRACT(counts, ${`$.${k}`})) AS SIGNED) + ${sign * row[k]}`]));
+    await tx.$executeRaw(Prisma.sql`INSERT INTO caller_analysis_minute_buckets
+      (id, started_at, view, customer_id, customer_gateway_id, vendor_id, vendor_gateway_id, caller, city, carrier, counts)
+      VALUES (${id}, ${minute}, ${row.view}, ${row.customerId}, ${row.customerGatewayId}, ${row.vendorId}, ${row.vendorGatewayId}, ${row.caller}, ${row.city}, ${row.carrier}, ${delta})
+      ON DUPLICATE KEY UPDATE counts=JSON_OBJECT(${additions})`);
+  }
+}

packages/database/src/caller-analytics.spec.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from 'vitest';
+import { analyticsRates, foldAnalytics, projectAnalytics, type AnalyticsEvent, type AnalyticsCall } from './caller-analytics-model.js';
+const start = Date.UTC(2026, 7, 31, 0);
+const event = (kind: AnalyticsEvent['kind'], extra: Partial<AnalyticsEvent> = {}): AnalyticsEvent => ({ eventId: kind, callUid: 'u1', callId: 'c1', attemptId: '1', kind, at: start + 1000, startedAt: start, customerId: 'c', customerGatewayId: 'cg', vendorId: 'v', vendorGatewayId: 'vg', caller: '123', landingCaller: '456', callee: '789', city: 'city', carrier: 'UNKNOWN', code: 0, talkMs: null, source: 'opensips', sequence: 0, ...extra });
+function run(events: AnalyticsEvent[]) { return projectAnalytics(events.reduce<AnalyticsCall | null>((c, e) => foldAnalytics(c, e), null)!); }
+describe('caller analytics business state', () => {
+  it.each([180,183])('counts %i as connected even if final call fails', code => {
+    const [r] = run([event('ATTEMPT'), event('PROGRESS', { code }), event('END', { code: 487, source: 'verified-dialog-final' })]);
+    expect(analyticsRates(r)).toMatchObject({ totalCalls: 1, connectedCalls: 1, answeredCalls: 0, failedCalls: 0, notConnectedCalls: 0, connectionRate: 100, overallAnswerRate: 0, connectedAnswerRate: 0 });
+  });
+  it.each([100,181,182])('does not treat %i as connected', code => {
+    const [r] = run([event('ATTEMPT'), event('PROGRESS', { code }), event('END', { code: 486, source: 'verified-dialog-final' })]);
+    expect(analyticsRates(r)).toMatchObject({ connectedCalls: 0, failedCalls: 1, connectedAnswerRate: null });
+  });
+  it('counts positive sub-second duration, not just 200', () => {
+    const [zero] = run([event('ATTEMPT'), event('ACCEPTED', { code: 200 })]);
+    expect(zero.answeredCalls).toBe(0);
+    const [positive] = run([event('ATTEMPT'), event('ACCEPTED', { code: 200 }), event('END', { at: start + 1500, source: 'verified-dialog-final' })]);
+    expect(positive).toMatchObject({ connectedCalls: 1, answeredCalls: 1, talkMs: 500 });
+  });
+  it('preserves zero actual duration', () => {
+    const [r] = run([event('ATTEMPT'), event('ACCEPTED', { code: 200 }), event('END', { talkMs: 0, source: 'verified-dialog-final' })]);
+    expect(r.answeredCalls).toBe(0);
+  });
+  it('is insensitive to duplicate progress and delayed attempt', () => {
+    const e = event('PROGRESS', { code: 183 });
+    const [r] = run([e, e, event('END', { source: 'verified-dialog-final', code: 487 }), event('ATTEMPT')]);
+    expect(r).toMatchObject({ totalCalls: 1, connectedCalls: 1, answeredCalls: 0, activeCalls: 0 });
+  });
+  it('corrects provisional duration and rejects late provisional overwrite', () => {
+    const [r] = run([event('ATTEMPT'), event('ACCEPTED', { code: 200 }), event('DURATION', { at: start + 2000, talkMs: 1000 }), event('RECONCILE', { at: start + 3000, talkMs: 0, source: 'verified-dialog-final' }), event('DURATION', { at: start + 4000, talkMs: 3000 })]);
+    expect(r).toMatchObject({ connectedCalls: 1, answeredCalls: 0, talkMs: 0 });
+  });
+  it('keeps original call separate from multiple landing attempts', () => {
+    const rows = run([event('ATTEMPT'), event('END', { code: 486 }), event('ATTEMPT', { attemptId: '2' }), event('PROGRESS', { attemptId: '2', code: 180 })]);
+    expect(rows).toHaveLength(3); expect(rows[0]).toMatchObject({ totalCalls: 1, connectedCalls: 1 });
+    expect(rows.slice(1).reduce((n,r) => n+r.totalCalls,0)).toBe(2);
+  });
+  it('distinguishes pending and unknown', () => {
+    expect(run([event('ATTEMPT')])[0]).toMatchObject({ pendingCalls: 1, failedCalls: 0 });
+    expect(run([event('UNKNOWN')])[0]).toMatchObject({ unknownCalls: 1, failedCalls: 0, pendingCalls: 0 });
+  });
+  it('does not merge different customer identities', () => {
+    expect(() => run([event('ATTEMPT'), event('PROGRESS', { customerId: 'other', code: 180 })])).toThrow('Conflicting');
+  });
+  it('calculates the three different business rates', () => {
+    expect(analyticsRates({ totalCalls:100, connectedCalls:60, answeredCalls:30, failedCalls:25, pendingCalls:15, unknownCalls:0, activeCalls:15, talkMs:0 })).toMatchObject({ notConnectedCalls:40, connectionRate:60, overallAnswerRate:30, connectedAnswerRate:50 });
+  });
+});

packages/database/src/index.ts

@@ -1,3 +1,6 @@
 export { Prisma, PrismaClient } from '@prisma/client';
 
 export const DATABASE_PROVIDER = 'mysql';
+
+export * from './caller-analytics-model.js';
+export * from './caller-analytics-store.js';