feat(recording): consolidate recording pipeline on server B

This commit is contained in:
hectorzhao
2026-08-27 16:57:45 +08:00
parent 32bba1bd84
commit be53848dee
12 changed files with 260 additions and 22 deletions
+14 -5
View File
@@ -1,13 +1,15 @@
import { createLogger } from '@lisglosips/observability';
import { PrismaClient } from '@lisglosips/database';
import { PrismaRecordingStore, RecordingTransferService, SshRecordingClient } from './transfer.js';
import { LocalRecordingClient, PrismaRecordingStore, RecordingTransferService, SshRecordingClient } from './transfer.js';
const serviceName = process.env.LISGLOSIPS_SERVICE_NAME ?? 'worker-recording';
const logger = createLogger(serviceName, process.env.LISGLOSIPS_LOG_LEVEL ?? 'info');
const sourceMode = process.env.RECORDING_SOURCE_MODE ?? 'ssh';
const remoteHost = process.env.RECORDING_REMOTE_HOST ?? 'lisglosips-a';
const sshConfig = process.env.RECORDING_SSH_CONFIG;
const remoteReadyDir = process.env.RECORDING_REMOTE_READY_DIR ?? '/dev/shm/voip_rec/ready';
const localReadyDir = process.env.RECORDING_LOCAL_READY_DIR ?? '/dev/shm/voip_rec/ready';
const localRoot = process.env.RECORDING_LOCAL_ROOT ?? '/data/recordings';
const scanIntervalMs = Number.parseInt(process.env.RECORDING_SCAN_INTERVAL_MS ?? '10000', 10);
const maxFilesPerScan = Number.parseInt(process.env.RECORDING_MAX_FILES_PER_SCAN ?? '50', 10);
@@ -17,8 +19,14 @@ let shuttingDown = false;
async function main(): Promise<void> {
const prisma = new PrismaClient();
const remote = new SshRecordingClient({ host: remoteHost, sshConfig, remoteReadyDir });
const service = new RecordingTransferService(remote, new PrismaRecordingStore(prisma), {
if (sourceMode !== 'ssh' && sourceMode !== 'local') {
throw new Error(`Unsupported RECORDING_SOURCE_MODE: ${sourceMode}`);
}
const source =
sourceMode === 'local'
? new LocalRecordingClient({ readyDir: localReadyDir })
: new SshRecordingClient({ host: remoteHost, sshConfig, remoteReadyDir });
const service = new RecordingTransferService(source, new PrismaRecordingStore(prisma), {
localRoot,
deleteSourceAfterCopy,
maxFilesPerScan
@@ -27,8 +35,9 @@ async function main(): Promise<void> {
await prisma.$connect();
logger.info(
{
remoteHost,
remoteReadyDir,
sourceMode,
sourceDir: sourceMode === 'local' ? localReadyDir : remoteReadyDir,
...(sourceMode === 'ssh' ? { remoteHost } : {}),
localRoot,
scanIntervalMs,
maxFilesPerScan,
+31 -1
View File
@@ -1,4 +1,4 @@
import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
@@ -6,6 +6,7 @@ import { describe, expect, it } from 'vitest';
import {
callIdFromStorageKey,
LocalRecordingClient,
normalizeRemoteRelativePath,
type PulledRecordingFile,
type RecordingStore,
@@ -90,6 +91,35 @@ describe('recording transfer', () => {
}
});
it('lists, copies, and deletes ready files from a local source directory', async () => {
const root = await mkdtemp(path.join(os.tmpdir(), 'lisglosips-local-rec-'));
const readyDir = path.join(root, 'ready');
const outputDir = path.join(root, 'recordings');
const sourcePath = path.join(readyDir, '2026', '08', '27', 'call.wav.ready');
const client = new LocalRecordingClient({ readyDir });
const store = new MemoryStore();
const service = new RecordingTransferService(client, store, {
localRoot: outputDir,
deleteSourceAfterCopy: true,
maxFilesPerScan: 10
});
try {
await mkdir(path.dirname(sourcePath), { recursive: true });
await writeFile(sourcePath, 'local-audio');
await writeFile(path.join(readyDir, 'ignored.part'), 'incomplete');
await expect(client.listReadyFiles()).resolves.toEqual([
{ relativePath: '2026/08/27/call.wav.ready', bytes: 11 }
]);
await expect(service.scanOnce()).resolves.toEqual({ moved: 1, skipped: 0, failed: 0 });
await expect(readFile(path.join(outputDir, '2026', '08', '27', 'call.wav'), 'utf8')).resolves.toBe('local-audio');
await expect(stat(sourcePath)).rejects.toThrow();
} finally {
await rm(root, { recursive: true, force: true });
}
});
it('keeps source file when local size verification fails', async () => {
const root = await mkdtemp(path.join(os.tmpdir(), 'lisglosips-rec-'));
const remote = new MemoryRemote(new Map([['bad.wav.ready', Buffer.from('abc')]]));
+53 -1
View File
@@ -1,6 +1,6 @@
import { createHash, randomUUID } from 'node:crypto';
import { createReadStream, createWriteStream } from 'node:fs';
import { mkdir, rename, rm, stat } from 'node:fs/promises';
import { mkdir, readdir, rename, rm, stat } from 'node:fs/promises';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { pipeline } from 'node:stream/promises';
@@ -149,6 +149,31 @@ export interface SshRecordingClientOptions {
remoteReadyDir: string;
}
export interface LocalRecordingClientOptions {
readyDir: string;
}
/** Reads finalized recordings from a directory on the same host as the worker. */
export class LocalRecordingClient implements RemoteRecordingClient {
constructor(private readonly options: LocalRecordingClientOptions) {}
async listReadyFiles(): Promise<RemoteRecordingFile[]> {
const files: RemoteRecordingFile[] = [];
await walkReadyFiles(this.options.readyDir, '', files);
return files.sort((left, right) => left.relativePath.localeCompare(right.relativePath));
}
async pullFile(file: RemoteRecordingFile, localPath: string): Promise<void> {
const sourcePath = safeJoin(this.options.readyDir, normalizeRemoteRelativePath(file.relativePath));
await pipeline(createReadStream(sourcePath), createWriteStream(localPath, { mode: 0o640 }));
}
async deleteFile(file: RemoteRecordingFile): Promise<void> {
const sourcePath = safeJoin(this.options.readyDir, normalizeRemoteRelativePath(file.relativePath));
await rm(sourcePath);
}
}
export class SshRecordingClient implements RemoteRecordingClient {
constructor(private readonly options: SshRecordingClientOptions) {}
@@ -225,6 +250,33 @@ function safeJoin(root: string, relativePath: string): string {
return resolvedPath;
}
async function walkReadyFiles(root: string, relativeDir: string, files: RemoteRecordingFile[]): Promise<void> {
const directory = relativeDir ? safeJoin(root, relativeDir) : path.resolve(root);
let entries;
try {
entries = await readdir(directory, { withFileTypes: true });
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
return;
}
throw error;
}
for (const entry of entries) {
const relativePath = relativeDir ? `${relativeDir}/${entry.name}` : entry.name;
if (entry.isDirectory()) {
await walkReadyFiles(root, relativePath, files);
continue;
}
if (!entry.isFile() || !entry.name.endsWith('.ready')) {
continue;
}
const normalized = normalizeRemoteRelativePath(relativePath);
const fileStat = await stat(safeJoin(root, normalized));
files.push({ relativePath: normalized, bytes: fileStat.size });
}
}
function remotePathFor(baseDir: string, relativePath: string): string {
return `${baseDir.replace(/\/+$/, '')}/${normalizeRemoteRelativePath(relativePath)}`;
}
@@ -0,0 +1,28 @@
# Server B 单机服务收敛实施方案
日期:2026-08-27
范围:停止 B 到 A 的录音 SSH 依赖,当前生产服务全部留在 B。
## 结论
B 已运行 OpenSIPS、RTPEngine、RTPEngine Recording Daemon、Nginx、API、MySQL、Redis、CDR Worker、Recording Worker 与 Config PublisherCPU、内存和磁盘空间满足当前负载。现有单机化缺口只有录音 Worker 仍配置为 SSH 访问 AA 不可达会导致该 Worker 周期性失败重启。
## 改动
1. Recording Worker 新增 `local` 源模式,递归扫描 B 的 `.ready` 文件,保留大小校验、SHA-256、原子落盘、数据库 upsert 与成功后删除源文件语义。
2. B 启用录音 finalizer timer,将 Recording Daemon 生成的稳定 WAV/MP3 从 `incoming` 原子推进到 `ready`
3. Worker systemd 增加 `rtpengine` 附加组,只开放 ready 目录和持久化目录权限。
4. B 生产环境改为 `RECORDING_SOURCE_MODE=local`;旧 SSH 文件不立即删除,只作短期回滚材料。
5. 不修改 TLS、数据库结构、SIP/RTP 监听地址和防火墙,不迁移历史录音,不停止不可达的 A。
## 验收
- 定向单元测试和 TypeScript 构建通过。
- finalizer timer、Recording Worker、OpenSIPS、RTPEngine、API、Nginx、MySQL、Redis 均 active。
- Recording Worker 启动日志显示 `sourceMode=local`,稳定运行超过原失败周期,且无 SSH 子进程。
- API readiness、首页和发布 preflight 通过。
- 真实新通话录音应在下一次受控呼叫时补做:`incoming -> ready -> /data/recordings -> recordings READY -> 鉴权播放`
## 回滚
发布前备份 Worker env、systemd drop-in、finalizer 脚本和 unit。异常时停用 finalizer timer,恢复备份配置并将 release symlink 切回上一版本;不删除 B 已落盘录音。
+41 -4
View File
@@ -2,6 +2,21 @@
> 任务:S24 - 录音搬运与播放
> 完成时间:2026-06-21 20:20 +08:00
> 当前模式:S56 - Server B 本机录音闭环(2026-08-27
## 0. 当前生产架构(S56
录音不再经过 B 到 A 的专用 SSH。Server B 上的 RTPEngine Recording Daemon 先写入
`/dev/shm/voip_rec/incoming`,本机 finalizer 将稳定文件原子改名为
`/dev/shm/voip_rec/ready/**/*.ready`Recording Worker 再复制、校验并持久化到
`/data/recordings`,最后写入数据库并删除本机 `.ready` 源文件。
```text
B RTPEngine -> B incoming -> B finalizer -> B ready
-> B Recording Worker -> B /data/recordings + MySQL
```
旧的 B -> A SSH 配置仅作为短期回滚材料保留,不参与当前服务。S24 以下章节保留为历史实施记录。
## 1. 目标
@@ -26,8 +41,23 @@ S24 完成录音闭环的最小能力:
## 3. Worker 环境变量
S56 当前生产配置:
```text
DATABASE_URL
RECORDING_SOURCE_MODE=local
RECORDING_LOCAL_READY_DIR=/dev/shm/voip_rec/ready
RECORDING_LOCAL_ROOT=/data/recordings
RECORDING_SCAN_INTERVAL_MS=10000
RECORDING_MAX_FILES_PER_SCAN=50
RECORDING_DELETE_SOURCE_AFTER_COPY=true
```
`RECORDING_SOURCE_MODE=ssh` 仍受代码支持,以下变量只用于回滚到旧架构:
```text
DATABASE_URL
RECORDING_SOURCE_MODE=ssh
RECORDING_REMOTE_HOST=lisglosips-a
RECORDING_SSH_CONFIG=/etc/lisglosips/recording/ssh_config
RECORDING_REMOTE_READY_DIR=/dev/shm/voip_rec/ready
@@ -37,12 +67,16 @@ RECORDING_MAX_FILES_PER_SCAN=50
RECORDING_DELETE_SOURCE_AFTER_COPY=true
```
本地开发可使用 `.codex-private/ssh/config`;生产或 B 端 systemd 应使用 `/etc/lisglosips/recording/` 下的受控 key 和 known_hosts。
SSH 模式本地开发可使用 `.codex-private/ssh/config`;生产回滚时才使用 `/etc/lisglosips/recording/` 下的受控 key 和 known_hosts。
## 4. 服务器变更
### Server B
- S56 新增 `lisglosips-recording-finalize.timer`,每 5 秒检查本机稳定录音文件。
- Recording Worker 以 `lisglo-recorder` 身份运行,通过 `SupplementaryGroups=rtpengine` 访问 ready 目录。
- systemd 只开放 `/dev/shm/voip_rec/ready``/data/recordings` 所需写权限。
- Worker 使用 `RECORDING_SOURCE_MODE=local`,运行期不再启动 `ssh` 子进程。
- 新增 `/etc/lisglosips/recording/a_pull_ed25519`,权限 `0640 root:lisglosips`
- 新增 `/etc/lisglosips/recording/known_hosts`
- 录音持久化目录仍为 `/data/recordings`,由 `lisglo-recorder:lisglosips` 管理。
@@ -109,7 +143,7 @@ location /_recordings/ {
## 7. 当前限制
B 当前 `/opt/lisglosips/current` 仍是 S05 placeholder release,未切换到完整 monorepo API/Worker release。因此 S24 本次完成代码、构建、服务器私网拉取链路和 Nginx 内部播放保护验证;正式 `lisglosips@recording-worker.service` 随完整应用 release 发布时启用
S24 记载的 placeholder 限制已经失效;B 已运行完整 monorepo API/Worker release。S56 不迁移 A 的历史录音,也不自动删除旧 SSH key;旧凭据待本地闭环稳定运行并完成真实录音验收后再单独清理
## 8. 回滚
@@ -125,8 +159,11 @@ sudo chmod 0750 /dev/shm/voip_rec/ready
Server B
```bash
sudo rm -rf /etc/lisglosips/recording
sudo rm -rf /data/recordings/s24-smoke
sudo systemctl disable --now lisglosips-recording-finalize.timer
sudo cp /var/backups/lisglosips-s56/<timestamp>/recording-worker.env /etc/lisglosips/recording-worker.env
sudo cp /var/backups/lisglosips-s56/<timestamp>/10-s28-recording.conf /etc/systemd/system/lisglosips@recording-worker.service.d/10-s28-recording.conf
sudo systemctl daemon-reload
sudo systemctl restart lisglosips@recording-worker
```
代码回滚:
+11 -11
View File
@@ -35,8 +35,8 @@
| 环境 | 用途 | 说明 |
| --- | --- | --- |
| 本地开发 | 单元、服务、API e2e、前端构建 | Windows 工作区,Node.js 22+pnpm 10.33.0Vitest |
| Server A | SIP/RTP/录音/HEP/Exporter | OpenSIPS、RTPEngine、录音 tmpfs、Redis 热路径 |
| Server B | Web/API/Worker/DB/监控 | Nginx、API、Redis、MySQL、Worker、HOMER、Prometheus/Grafana |
| Server A | 历史节点/回滚参考 | S56 起不承载当前生产服务,不参与录音传输 |
| Server B | SIP/RTP/录音/Web/API/Worker/DB/监控 | OpenSIPS、RTPEngine、录音 tmpfs、Nginx、API、Redis、MySQL、Worker、HOMER、Prometheus/Grafana |
| Server T | 客户与落地模拟 | SIP 注册、呼叫、失败场景、并发呼叫脚本 |
| 阿里云迁移环境 | 上线前复验 | 生产网络、安全组、正式 TLS、数据盘、备份恢复和灰度呼叫必须重新执行 |
@@ -1331,18 +1331,18 @@ corepack pnpm@10.33.0 exec vitest run apps/worker-recording/src/transfer.spec.ts
- 只读质检账号 `quality_viewer_001`:包含 `quality.view`,不包含 `quality.manage``recordings.play`
- 至少 1 条成功通话 CDR,关联 READY 录音 `rec_ready_seed_001`storageKey 指向 B 本地录音数据盘。
- 至少 1 条 PENDING 录音 `rec_pending_seed_001`,用于播放和质检不可用状态。
- A`/dev/shm/voip_rec` 或测试目录可生成 ready 文件,B 侧录音存储目录可写
- B`/dev/shm/voip_rec` 可生成 ready 文件,`/data/recordings` 可写;Worker 配置为本地源模式
#### REC-001 录音搬运成功
| 字段 | 内容 |
| --- | --- |
| 优先级 | P0 |
| 目的 | 验证 Recording Worker 从 A 拉取 ready 文件,校验大小/hash 后写入 B,并安全删除源文件。 |
| 前置条件 | A 侧存在测试录音文件和 ready 标记;B 侧 Recording Worker 可通过私网访问 A;关联 CDR/callId 可匹配。 |
| 目的 | 验证 Recording Worker 从 B 本机读取 ready 文件,校验大小/hash 后持久化,并安全删除 tmpfs 源文件。 |
| 前置条件 | B 侧存在测试录音文件和 ready 标记;本机 finalizer 与 Recording Worker active;关联 CDR/callId 可匹配。 |
| 测试数据 | 文件名包含 Call-ID;ready 内容包含相对路径和 size;文件大小固定,例如 55758 bytes。 |
| 步骤 | 1. 在 A 侧生成测试录音 ready 文件。2. 启动或触发 Recording Worker。3. 查询 B 侧录音文件、数据库 recordings 表和 Worker 日志。4. 检查 A 源文件和 ready 文件。 |
| 预期结果 | B 文件存在,大小/hash 与 A 源文件一致;recording 状态为 `READY`storageKey 为安全相对路径;A 源文件在校验成功后删除;日志记录成功搬运。 |
| 步骤 | 1. 在 B 侧生成测试录音,由 finalizer 形成 ready 文件。2. 启动或触发 Recording Worker。3. 查询 B 持久化文件、数据库 recordings 表和 Worker 日志。4. 检查 B ready 文件。 |
| 预期结果 | B 持久化文件存在,大小/hash 与 ready 源文件一致;recording 状态为 `READY`storageKey 为安全相对路径;B ready 源文件在校验成功后删除;日志记录成功搬运,进程树中无录音 SSH。 |
| 数据检查 | recordings 记录包含 callId、rawCdrId 或关联键、fileSize、checksum、storageKey、readyAt;状态从 PENDING 变为 READY。 |
| 安全检查 | Worker 只处理配置目录下文件;不会跟随任意绝对路径或 `../` 路径。 |
@@ -1351,13 +1351,13 @@ corepack pnpm@10.33.0 exec vitest run apps/worker-recording/src/transfer.spec.ts
| 字段 | 内容 |
| --- | --- |
| 优先级 | P0 |
| 目的 | 验证复制后大小/hash 不一致时不删除 A 源文件,避免录音丢失。 |
| 目的 | 验证复制后大小/hash 不一致时不删除 B 本机 ready 源文件,避免录音丢失。 |
| 前置条件 | 可构造错误 ready size、截断文件或模拟 B 写入失败。 |
| 测试数据 | ready size 与实际文件大小不一致。 |
| 步骤 | 1. 生成 size 不一致的 ready 文件。2. 执行 Recording Worker。3. 查询 A/B 文件和 recordings 状态。 |
| 预期结果 | Worker 标记失败或保持待重试;A 源文件保留;B 不产生 READY 记录;错误日志可定位。 |
| 步骤 | 1. 生成 size 不一致的 ready 文件。2. 执行 Recording Worker。3. 查询 B ready/持久化文件和 recordings 状态。 |
| 预期结果 | Worker 标记失败或保持待重试;B ready 源文件保留;持久化目录不产生有效成品且数据库不误标 READY;错误日志可定位。 |
| 数据检查 | recordings 不应被误标记 READY;如生成 FAILED 状态,应记录失败原因和重试信息。 |
| 安全检查 | 失败日志不泄露 SSH 私钥、完整内部路径或连接凭据。 |
| 安全检查 | 失败日志不泄露数据库连接凭据;Worker 不访问配置目录之外的文件。 |
#### REC-003 播放接口鉴权与权限
@@ -0,0 +1,10 @@
LISGLOSIPS_ENTRYPOINT=apps/worker-recording/dist/main.js
LISGLOSIPS_SERVICE_NAME=worker-recording
LISGLOSIPS_LOG_LEVEL=info
DATABASE_URL=mysql://user:password@127.0.0.1:3306/lisglosips
RECORDING_SOURCE_MODE=local
RECORDING_LOCAL_READY_DIR=/dev/shm/voip_rec/ready
RECORDING_LOCAL_ROOT=/data/recordings
RECORDING_SCAN_INTERVAL_MS=10000
RECORDING_MAX_FILES_PER_SCAN=50
RECORDING_DELETE_SOURCE_AFTER_COPY=true
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
set -euo pipefail
BASE_DIR=${1:-/dev/shm/voip_rec}
INCOMING_DIR="$BASE_DIR/incoming"
READY_DIR="$BASE_DIR/ready"
FAILED_DIR="$BASE_DIR/failed"
MIN_AGE_SECONDS=${MIN_AGE_SECONDS:-15}
mkdir -p "$INCOMING_DIR" "$READY_DIR" "$FAILED_DIR"
chmod 0750 "$BASE_DIR" "$INCOMING_DIR" "$READY_DIR" "$FAILED_DIR"
find "$INCOMING_DIR" -type f \( -name '*.wav' -o -name '*.mp3' \) -print0 |
while IFS= read -r -d '' source; do
if [ ! -s "$source" ]; then
continue
fi
now=$(date +%s)
mtime=$(stat -c %Y "$source")
if [ $((now - mtime)) -lt "$MIN_AGE_SECONDS" ]; then
continue
fi
if command -v fuser >/dev/null 2>&1 && fuser -s -- "$source"; then
continue
fi
rel=${source#"$INCOMING_DIR"/}
part="$READY_DIR/$rel.part"
ready="$READY_DIR/$rel.ready"
failed="$FAILED_DIR/$rel"
mkdir -p "$(dirname "$part")" "$(dirname "$failed")"
chmod 0750 "$(dirname "$part")" "$(dirname "$failed")"
if [ -e "$ready" ] || [ -e "$part" ]; then
mv -- "$source" "$failed.$(date -u +%Y%m%dT%H%M%SZ).duplicate"
continue
fi
mv -- "$source" "$part"
chmod 0640 "$part"
mv -- "$part" "$ready"
done
@@ -0,0 +1,5 @@
[Service]
User=lisglo-recorder
Group=lisglosips
SupplementaryGroups=rtpengine
ReadWritePaths=/data/recordings /dev/shm/voip_rec/ready /run/lisglosips /run/lisglo-recorder
@@ -0,0 +1,15 @@
[Unit]
Description=Finalize local RTPEngine recordings for LisgloSIPS
After=rtpengine-recording-daemon.service
Requires=rtpengine-recording-daemon.service
[Service]
Type=oneshot
User=rtpengine
Group=rtpengine
ExecStart=/usr/local/sbin/lisglosips-recording-finalize /dev/shm/voip_rec
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/dev/shm/voip_rec
@@ -0,0 +1,12 @@
[Unit]
Description=Finalize local RTPEngine recordings every five seconds
[Timer]
OnBootSec=10s
OnUnitActiveSec=5s
AccuracySec=1s
Unit=lisglosips-recording-finalize.service
Persistent=true
[Install]
WantedBy=timers.target
+1
View File
@@ -71,6 +71,7 @@ const copyEntries = [
['packages/redis/dist', 'packages/redis/dist', true],
['packages/redis/node_modules', 'packages/redis/node_modules', true],
['infra/server-b/s30', 'infra/server-b/s30', true],
['infra/server-b/s56', 'infra/server-b/s56', true],
['infra/server-a/s28/lisglosips_hotpath.lua', 'infra/server-a/s28/lisglosips_hotpath.lua', true],
['scripts/phase2-gateway-migration.mjs', 'scripts/phase2-gateway-migration.mjs', true]
];