diff --git a/docs/SERVER_B_AUTOSTART_RUNBOOK.md b/docs/SERVER_B_AUTOSTART_RUNBOOK.md new file mode 100644 index 0000000..2f7f53b --- /dev/null +++ b/docs/SERVER_B_AUTOSTART_RUNBOOK.md @@ -0,0 +1,38 @@ +# Server B 单机服务开机自启 Runbook + +## 方案 + +生产进程继续由各自的 systemd unit 管理,使用 `lisglosips.target` 编组整套单 B 服务。`lisglosips-start-b` 是人工启动和开机后验收脚本,不直接常驻托管业务进程。 + +这种方式比 `rc.local`、桌面启动项或后台 shell 脚本更适合生产环境:systemd 能分别记录日志、限制权限、处理依赖、实施失败重启,并准确显示单个服务状态。 + +## 自启范围 + +- 基础设施:Tailscale、MySQL、Redis、PostgreSQL、Nginx。 +- 通信:OpenSIPS、RTPEngine、RTPEngine Recording、Redis AUTH Proxy、Redis 热路径加载。 +- 应用:API、CDR Worker、Recording Worker、Config Publisher。 +- 信令与监控:HEP、HOMER、Prometheus、Grafana、Node/MySQL/PostgreSQL/Redis Exporter。 +- 定时任务:每日数据备份、每 5 秒录音整理。 + +## 安装 + +```bash +sudo /opt/lisglosips/current/infra/server-b/s58/deploy-autostart.sh +``` + +安装流程会备份已有文件、安装 target 和开机检查服务、启用四个应用实例、启动整套 target,并运行 API ready、Nginx、OpenSIPS 和 preflight 检查。失败时自动撤销本次新建的 enable 链接并恢复旧文件。 + +## 日常操作 + +```bash +sudo systemctl start lisglosips.target +sudo /usr/local/sbin/lisglosips-start-b +systemctl status lisglosips.target lisglosips-boot-check.service +systemctl --failed +``` + +不建议使用 `kill`、`nohup` 或重复启动脚本托管进程。单个服务变更仍应使用 `systemctl restart `,避免无关服务中断。 + +## 验收 + +安装后检查 `lisglosips.target`、`lisglosips-boot-check.service`、API、CDR Worker、Recording Worker、Config Publisher 是否 enabled。首次真正重启验收应安排维护窗口,记录重启时间、所有服务状态、API ready、当前通话 MI、录音入库和配置发布结果。 diff --git a/infra/ops/startup/lisglosips-start-b.sh b/infra/ops/startup/lisglosips-start-b.sh index 9f11ff1..e78a5e1 100644 --- a/infra/ops/startup/lisglosips-start-b.sh +++ b/infra/ops/startup/lisglosips-start-b.sh @@ -8,37 +8,58 @@ fi echo "== LisgloSIPS B startup ==" hostname -systemctl start \ - mysql \ - redis-server \ - nginx \ - heplify-server \ - lisglosips-prometheus \ - grafana-server +systemctl start lisglosips.target -systemctl start \ - lisglosips@api \ - lisglosips@cdr-worker \ - lisglosips@recording-worker \ +required_services=( + tailscaled + mysql + redis-server + postgresql + nginx + opensips + rtpengine-daemon + rtpengine-recording-daemon + lisglosips-redis-auth-proxy + lisglosips@api + lisglosips@cdr-worker + lisglosips@recording-worker lisglosips@config-publisher + heplify-server + homer-app + lisglosips-prometheus + grafana-server + lisglosips-node-exporter + lisglosips-mysqld-exporter + lisglosips-postgres-exporter + lisglosips-redis-exporter +) + +for attempt in $(seq 1 30); do + missing=() + for unit in "${required_services[@]}"; do + systemctl --quiet is-active "$unit" || missing+=("$unit") + done + [ "${#missing[@]}" -eq 0 ] && break + [ "$attempt" -eq 30 ] && { + printf 'startup timeout; inactive services: %s\n' "${missing[*]}" >&2 + exit 1 + } + sleep 2 +done nginx -t +opensips -C -f /etc/opensips/opensips.cfg >/dev/null echo "== release ==" readlink -f /opt/lisglosips/current echo "== services ==" -systemctl is-active \ - mysql \ - redis-server \ - nginx \ - lisglosips@api \ - lisglosips@cdr-worker \ - lisglosips@recording-worker \ - lisglosips@config-publisher \ - heplify-server \ - lisglosips-prometheus \ - grafana-server +systemctl is-active "${required_services[@]}" + +echo "== timers ==" +systemctl is-active lisglosips-backup.timer lisglosips-recording-finalize.timer +systemctl is-enabled lisglosips.target lisglosips-backup.timer lisglosips-recording-finalize.timer \ + lisglosips@api lisglosips@cdr-worker lisglosips@recording-worker lisglosips@config-publisher echo "== api ready ==" curl -fsS http://127.0.0.1:3000/api/v2/health/ready @@ -49,4 +70,3 @@ echo "== preflight ==" echo "== failed units ==" systemctl --failed --no-pager - diff --git a/infra/server-b/s58/deploy-autostart.sh b/infra/server-b/s58/deploy-autostart.sh new file mode 100644 index 0000000..bb7ff4c --- /dev/null +++ b/infra/server-b/s58/deploy-autostart.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +release_root=${1:-/opt/lisglosips/current} +stamp=$(date -u +%Y%m%dT%H%M%SZ) +backup=/var/backups/lisglosips-s58/$stamp + +test -x "$release_root/infra/ops/startup/lisglosips-start-b.sh" +test -f "$release_root/infra/server-b/s58/systemd/lisglosips.target" +test -f "$release_root/infra/server-b/s58/systemd/lisglosips-boot-check.service" + +install -d -m 0700 "$backup" +for path in \ + /usr/local/sbin/lisglosips-start-b \ + /etc/systemd/system/lisglosips.target \ + /etc/systemd/system/lisglosips-boot-check.service; do + [ ! -e "$path" ] || cp -a "$path" "$backup/$(basename "$path").previous" +done + +changed=0 +rollback() { + rc=$? + if [ "$changed" -eq 1 ]; then + echo "phase=rollback rc=$rc" + systemctl disable lisglosips-boot-check.service lisglosips.target \ + lisglosips@recording-worker.service lisglosips@config-publisher.service 2>/dev/null || true + for name in lisglosips-start-b lisglosips.target lisglosips-boot-check.service; do + previous="$backup/$name.previous" + case "$name" in + lisglosips-start-b) target=/usr/local/sbin/lisglosips-start-b ;; + *) target=/etc/systemd/system/$name ;; + esac + if [ -e "$previous" ]; then cp -a "$previous" "$target"; else rm -f "$target"; fi + done + systemctl daemon-reload + fi + exit "$rc" +} +trap rollback ERR + +echo "phase=install" +install -o root -g root -m 0755 "$release_root/infra/ops/startup/lisglosips-start-b.sh" /usr/local/sbin/lisglosips-start-b +install -o root -g root -m 0644 "$release_root/infra/server-b/s58/systemd/lisglosips.target" /etc/systemd/system/lisglosips.target +install -o root -g root -m 0644 "$release_root/infra/server-b/s58/systemd/lisglosips-boot-check.service" /etc/systemd/system/lisglosips-boot-check.service + +changed=1 +systemctl daemon-reload +systemctl enable lisglosips.target lisglosips-boot-check.service \ + lisglosips@api.service lisglosips@cdr-worker.service \ + lisglosips@recording-worker.service lisglosips@config-publisher.service + +echo "phase=start-and-verify" +systemctl start lisglosips.target +systemctl restart lisglosips-boot-check.service +test "$(systemctl is-enabled lisglosips.target)" = enabled +test "$(systemctl is-enabled lisglosips@recording-worker.service)" = enabled +test "$(systemctl is-enabled lisglosips@config-publisher.service)" = enabled +test "$(systemctl is-active lisglosips@recording-worker.service)" = active +test "$(systemctl is-active lisglosips@config-publisher.service)" = active + +printf 'BACKUP=%s\nTARGET=%s\n' "$backup" "$(systemctl is-enabled lisglosips.target)" +changed=0 +trap - ERR diff --git a/infra/server-b/s58/systemd/lisglosips-boot-check.service b/infra/server-b/s58/systemd/lisglosips-boot-check.service new file mode 100644 index 0000000..0241dfe --- /dev/null +++ b/infra/server-b/s58/systemd/lisglosips-boot-check.service @@ -0,0 +1,13 @@ +[Unit] +Description=Validate LisgloSIPS after Server B boot +Requires=lisglosips.target +After=lisglosips.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/lisglosips-start-b +RemainAfterExit=yes +TimeoutStartSec=90 + +[Install] +WantedBy=multi-user.target diff --git a/infra/server-b/s58/systemd/lisglosips.target b/infra/server-b/s58/systemd/lisglosips.target new file mode 100644 index 0000000..c53132f --- /dev/null +++ b/infra/server-b/s58/systemd/lisglosips.target @@ -0,0 +1,16 @@ +[Unit] +Description=LisgloSIPS single-server stack on Server B +Wants=network-online.target tailscaled.service +Wants=mysql.service redis-server.service postgresql.service +Wants=nginx.service opensips.service rtpengine-daemon.service rtpengine-recording-daemon.service +Wants=lisglosips-redis-auth-proxy.service lisglosips-redis-hotpath-load.service +Wants=lisglosips@api.service lisglosips@cdr-worker.service lisglosips@recording-worker.service lisglosips@config-publisher.service +Wants=heplify-server.service homer-app.service +Wants=lisglosips-prometheus.service grafana-server.service +Wants=lisglosips-node-exporter.service lisglosips-mysqld-exporter.service lisglosips-postgres-exporter.service lisglosips-redis-exporter.service +Wants=lisglosips-backup.timer lisglosips-recording-finalize.timer +After=network-online.target tailscaled.service +After=mysql.service redis-server.service postgresql.service + +[Install] +WantedBy=multi-user.target