feat: add Server B systemd startup target

2026-08-30 21:11:22 +08:00
parent 6da5782ac7
commit 406b3744aa
5 changed files with 173 additions and 23 deletions
@@ -0,0 +1,38 @@
+# Server B 单机服务开机自启 Runbook
+
+## 方案
+
+生产进程继续由各自的 systemd unit 管理，使用 `lisglosips.target` 编组整套单 B 服务。`lisglosips-start-b` 是人工启动和开机后验收脚本，不直接常驻托管业务进程。
+
+这种方式比 `rc.local`、桌面启动项或后台 shell 脚本更适合生产环境：systemd 能分别记录日志、限制权限、处理依赖、实施失败重启，并准确显示单个服务状态。
+
+## 自启范围
+
+- 基础设施：Tailscale、MySQL、Redis、PostgreSQL、Nginx。
+- 通信：OpenSIPS、RTPEngine、RTPEngine Recording、Redis AUTH Proxy、Redis 热路径加载。
+- 应用：API、CDR Worker、Recording Worker、Config Publisher。
+- 信令与监控：HEP、HOMER、Prometheus、Grafana、Node/MySQL/PostgreSQL/Redis Exporter。
+- 定时任务：每日数据备份、每 5 秒录音整理。
+
+## 安装
+
+```bash
+sudo /opt/lisglosips/current/infra/server-b/s58/deploy-autostart.sh
+```
+
+安装流程会备份已有文件、安装 target 和开机检查服务、启用四个应用实例、启动整套 target，并运行 API ready、Nginx、OpenSIPS 和 preflight 检查。失败时自动撤销本次新建的 enable 链接并恢复旧文件。
+
+## 日常操作
+
+```bash
+sudo systemctl start lisglosips.target
+sudo /usr/local/sbin/lisglosips-start-b
+systemctl status lisglosips.target lisglosips-boot-check.service
+systemctl --failed
+```
+
+不建议使用 `kill`、`nohup` 或重复启动脚本托管进程。单个服务变更仍应使用 `systemctl restart <unit>`，避免无关服务中断。
+
+## 验收
+
+安装后检查 `lisglosips.target`、`lisglosips-boot-check.service`、API、CDR Worker、Recording Worker、Config Publisher 是否 enabled。首次真正重启验收应安排维护窗口，记录重启时间、所有服务状态、API ready、当前通话 MI、录音入库和配置发布结果。
@@ -8,37 +8,58 @@ fi
 echo "== LisgloSIPS B startup =="
 hostname

-systemctl start \
-  mysql \
-  redis-server \
-  nginx \
-  heplify-server \
-  lisglosips-prometheus \
-  grafana-server
+systemctl start lisglosips.target

-systemctl start \
-  lisglosips@api \
-  lisglosips@cdr-worker \
-  lisglosips@recording-worker \
+required_services=(
+  tailscaled
+  mysql
+  redis-server
+  postgresql
+  nginx
+  opensips
+  rtpengine-daemon
+  rtpengine-recording-daemon
+  lisglosips-redis-auth-proxy
+  lisglosips@api
+  lisglosips@cdr-worker
+  lisglosips@recording-worker
  lisglosips@config-publisher
+  heplify-server
+  homer-app
+  lisglosips-prometheus
+  grafana-server
+  lisglosips-node-exporter
+  lisglosips-mysqld-exporter
+  lisglosips-postgres-exporter
+  lisglosips-redis-exporter
+)
+
+for attempt in $(seq 1 30); do
+  missing=()
+  for unit in "${required_services[@]}"; do
+    systemctl --quiet is-active "$unit" || missing+=("$unit")
+  done
+  [ "${#missing[@]}" -eq 0 ] && break
+  [ "$attempt" -eq 30 ] && {
+    printf 'startup timeout; inactive services: %s\n' "${missing[*]}" >&2
+    exit 1
+  }
+  sleep 2
+done

 nginx -t
+opensips -C -f /etc/opensips/opensips.cfg >/dev/null

 echo "== release =="
 readlink -f /opt/lisglosips/current

 echo "== services =="
-systemctl is-active \
-  mysql \
-  redis-server \
-  nginx \
-  lisglosips@api \
-  lisglosips@cdr-worker \
-  lisglosips@recording-worker \
-  lisglosips@config-publisher \
-  heplify-server \
-  lisglosips-prometheus \
-  grafana-server
+systemctl is-active "${required_services[@]}"
+
+echo "== timers =="
+systemctl is-active lisglosips-backup.timer lisglosips-recording-finalize.timer
+systemctl is-enabled lisglosips.target lisglosips-backup.timer lisglosips-recording-finalize.timer \
+  lisglosips@api lisglosips@cdr-worker lisglosips@recording-worker lisglosips@config-publisher

 echo "== api ready =="
 curl -fsS http://127.0.0.1:3000/api/v2/health/ready
@@ -49,4 +70,3 @@ echo "== preflight =="

 echo "== failed units =="
 systemctl --failed --no-pager
-
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+release_root=${1:-/opt/lisglosips/current}
+stamp=$(date -u +%Y%m%dT%H%M%SZ)
+backup=/var/backups/lisglosips-s58/$stamp
+
+test -x "$release_root/infra/ops/startup/lisglosips-start-b.sh"
+test -f "$release_root/infra/server-b/s58/systemd/lisglosips.target"
+test -f "$release_root/infra/server-b/s58/systemd/lisglosips-boot-check.service"
+
+install -d -m 0700 "$backup"
+for path in \
+  /usr/local/sbin/lisglosips-start-b \
+  /etc/systemd/system/lisglosips.target \
+  /etc/systemd/system/lisglosips-boot-check.service; do
+  [ ! -e "$path" ] || cp -a "$path" "$backup/$(basename "$path").previous"
+done
+
+changed=0
+rollback() {
+  rc=$?
+  if [ "$changed" -eq 1 ]; then
+    echo "phase=rollback rc=$rc"
+    systemctl disable lisglosips-boot-check.service lisglosips.target \
+      lisglosips@recording-worker.service lisglosips@config-publisher.service 2>/dev/null || true
+    for name in lisglosips-start-b lisglosips.target lisglosips-boot-check.service; do
+      previous="$backup/$name.previous"
+      case "$name" in
+        lisglosips-start-b) target=/usr/local/sbin/lisglosips-start-b ;;
+        *) target=/etc/systemd/system/$name ;;
+      esac
+      if [ -e "$previous" ]; then cp -a "$previous" "$target"; else rm -f "$target"; fi
+    done
+    systemctl daemon-reload
+  fi
+  exit "$rc"
+}
+trap rollback ERR
+
+echo "phase=install"
+install -o root -g root -m 0755 "$release_root/infra/ops/startup/lisglosips-start-b.sh" /usr/local/sbin/lisglosips-start-b
+install -o root -g root -m 0644 "$release_root/infra/server-b/s58/systemd/lisglosips.target" /etc/systemd/system/lisglosips.target
+install -o root -g root -m 0644 "$release_root/infra/server-b/s58/systemd/lisglosips-boot-check.service" /etc/systemd/system/lisglosips-boot-check.service
+
+changed=1
+systemctl daemon-reload
+systemctl enable lisglosips.target lisglosips-boot-check.service \
+  lisglosips@api.service lisglosips@cdr-worker.service \
+  lisglosips@recording-worker.service lisglosips@config-publisher.service
+
+echo "phase=start-and-verify"
+systemctl start lisglosips.target
+systemctl restart lisglosips-boot-check.service
+test "$(systemctl is-enabled lisglosips.target)" = enabled
+test "$(systemctl is-enabled lisglosips@recording-worker.service)" = enabled
+test "$(systemctl is-enabled lisglosips@config-publisher.service)" = enabled
+test "$(systemctl is-active lisglosips@recording-worker.service)" = active
+test "$(systemctl is-active lisglosips@config-publisher.service)" = active
+
+printf 'BACKUP=%s\nTARGET=%s\n' "$backup" "$(systemctl is-enabled lisglosips.target)"
+changed=0
+trap - ERR
@@ -0,0 +1,13 @@
+[Unit]
+Description=Validate LisgloSIPS after Server B boot
+Requires=lisglosips.target
+After=lisglosips.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/lisglosips-start-b
+RemainAfterExit=yes
+TimeoutStartSec=90
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,16 @@
+[Unit]
+Description=LisgloSIPS single-server stack on Server B
+Wants=network-online.target tailscaled.service
+Wants=mysql.service redis-server.service postgresql.service
+Wants=nginx.service opensips.service rtpengine-daemon.service rtpengine-recording-daemon.service
+Wants=lisglosips-redis-auth-proxy.service lisglosips-redis-hotpath-load.service
+Wants=lisglosips@api.service lisglosips@cdr-worker.service lisglosips@recording-worker.service lisglosips@config-publisher.service
+Wants=heplify-server.service homer-app.service
+Wants=lisglosips-prometheus.service grafana-server.service
+Wants=lisglosips-node-exporter.service lisglosips-mysqld-exporter.service lisglosips-postgres-exporter.service lisglosips-redis-exporter.service
+Wants=lisglosips-backup.timer lisglosips-recording-finalize.timer
+After=network-online.target tailscaled.service
+After=mysql.service redis-server.service postgresql.service
+
+[Install]
+WantedBy=multi-user.target