hectorzhao/lisglosips
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: complete caller analytics reliability remediation

Browse Source
This commit is contained in:
hectorzhao
2026-09-01 10:03:29 +08:00
parent a116926867
commit 7f8825cf3a
25 changed files with 850 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/server-b/caller-analytics/deploy.sh
+48 -12
View File
@@ -15,25 +15,48 @@ test ! -e "$new"
curl -fsS -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"dlg_list","params":[]}' http://127.0.0.1:8888/mi | python3 -c 'import sys,json; assert json.load(sys.stdin)["result"]["Dialogs"]==[], "Active calls: retry after calls end"'
systemctl start lisglosips-backup.service
install -d -m 0700 "$backup"
if systemctl --quiet is-active lisglosips-caller-analytics-redis.service && test -f /etc/lisglosips/caller-analytics-redis.secret; then
export REDISCLI_AUTH
REDISCLI_AUTH=$(cat /etc/lisglosips/caller-analytics-redis.secret)
redis-cli -h 127.0.0.1 -p 6381 --no-auth-warning SAVE >/dev/null
unset REDISCLI_AUTH
cp -a /var/lib/redis-caller-analytics "$backup/redis-caller-analytics"
fi
cp -a /etc/opensips/opensips.cfg "$backup/opensips.cfg"
printf '%s\n' "$old" > "$backup/previous-release"
for file in /etc/lisglosips/caller-analytics.env /etc/rsyslog.d/35-caller-analytics.conf; do
managed_files=(/etc/lisglosips/caller-analytics.env /etc/rsyslog.d/35-caller-analytics.conf /etc/redis/caller-analytics.conf /etc/systemd/system/lisglosips-caller-analytics-redis.service /etc/logrotate.d/lisglosips-caller-analytics /etc/systemd/system/lisglosips-caller-analytics-maintenance.service /etc/systemd/system/lisglosips-caller-analytics-maintenance.timer /etc/systemd/system/lisglosips.target)
for file in "${managed_files[@]}"; do
if test -f "$file"; then cp -a "$file" "$backup/$(basename "$file")"; fi
done
changed=0
was_running=$(systemctl is-active lisglosips@caller-analytics || true)
was_enabled=$(systemctl is-enabled lisglosips@caller-analytics 2>/dev/null || true)
redis_was_running=$(systemctl is-active lisglosips-caller-analytics-redis 2>/dev/null || true)
redis_was_enabled=$(systemctl is-enabled lisglosips-caller-analytics-redis 2>/dev/null || true)
maintenance_was_running=$(systemctl is-active lisglosips-caller-analytics-maintenance.timer 2>/dev/null || true)
maintenance_was_enabled=$(systemctl is-enabled lisglosips-caller-analytics-maintenance.timer 2>/dev/null || true)
secret_existed=0
test -f /etc/lisglosips/caller-analytics-redis.secret && secret_existed=1
rollback() {
rc=$?
if test "$changed" = 1; then
systemctl stop lisglosips@caller-analytics || true
systemctl stop lisglosips-caller-analytics-maintenance.timer lisglosips-caller-analytics-redis || true
cp -a "$backup/opensips.cfg" /etc/opensips/opensips.cfg
for file in /etc/lisglosips/caller-analytics.env /etc/rsyslog.d/35-caller-analytics.conf; do
for file in "${managed_files[@]}"; do
if test -f "$backup/$(basename "$file")"; then cp -a "$backup/$(basename "$file")" "$file"; else rm -f "$file"; fi
done
ln -sfn "$old" /opt/lisglosips/current
systemctl restart rsyslog || true
systemctl daemon-reload || true
systemctl restart opensips lisglosips@api || true
if test "$was_running" = active; then systemctl start lisglosips@caller-analytics || true; fi
if test "$was_enabled" != enabled; then systemctl disable lisglosips@caller-analytics || true; fi
if test "$redis_was_running" = active; then systemctl start lisglosips-caller-analytics-redis || true; fi
if test "$redis_was_enabled" != enabled; then systemctl disable lisglosips-caller-analytics-redis || true; fi
if test "$maintenance_was_running" = active; then systemctl start lisglosips-caller-analytics-maintenance.timer || true; fi
if test "$maintenance_was_enabled" != enabled; then systemctl disable lisglosips-caller-analytics-maintenance.timer || true; fi
if test "$secret_existed" = 0; then rm -f /etc/lisglosips/caller-analytics-redis.secret; fi
fi
echo "FAILED rc=$rc backup=$backup (additive analytics tables retained)"
exit "$rc"
@@ -44,8 +67,7 @@ tar -xzf "$archive" -C "$new"
printf '%s\n' "$commit" > "$new/.deployed-commit"
printf '%s\n' "$old" > "$new/.delta-base-release"
source_cfg="$backup/opensips.cfg"
if grep -q 'CRA1|' "$source_cfg"; then
# A code-only redeploy keeps previously reviewed instrumentation unchanged.
if grep -q 'CRA_INSTRUMENTATION_VERSION=2' "$source_cfg"; then
cp "$source_cfg" "$new/opensips-candidate.cfg"
else
/usr/bin/node "$new/scripts/instrument-caller-analytics.mjs" "$source_cfg" "$new/opensips-candidate.cfg"
@@ -68,27 +90,41 @@ chown syslog:lisglosips /var/log/lisglosips/caller-analytics.log
chmod 0640 /var/log/lisglosips/caller-analytics.log
install -m 0644 "$new/infra/server-b/caller-analytics/rsyslog.conf" /etc/rsyslog.d/35-caller-analytics.conf
rsyslogd -N1
if ! test -f /etc/lisglosips/caller-analytics.env; then
grep -E '^(DATABASE_URL|REDIS_URL)=' /etc/lisglosips/cdr-worker.env > /etc/lisglosips/caller-analytics.env
printf '%s\n' 'LISGLOSIPS_ENTRYPOINT=apps/worker-cdr/dist/analytics-main.js' 'LISGLOSIPS_SERVICE_NAME=caller-analytics' "ANALYTICS_CAPTURE_SINCE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /etc/lisglosips/caller-analytics.env
chown root:lisglosips /etc/lisglosips/caller-analytics.env
chmod 0640 /etc/lisglosips/caller-analytics.env
fi
changed=1
install -d -o redis -g redis -m 0750 /var/lib/redis-caller-analytics
if ! test -f /etc/lisglosips/caller-analytics-redis.secret; then openssl rand -hex 32 > /etc/lisglosips/caller-analytics-redis.secret; chown root:lisglosips /etc/lisglosips/caller-analytics-redis.secret; chmod 0640 /etc/lisglosips/caller-analytics-redis.secret; fi
analytics_redis_password=$(cat /etc/lisglosips/caller-analytics-redis.secret)
sed "s/@@ANALYTICS_REDIS_PASSWORD@@/$analytics_redis_password/" "$new/infra/server-b/caller-analytics/redis.conf" > /etc/redis/caller-analytics.conf
chown root:redis /etc/redis/caller-analytics.conf; chmod 0640 /etc/redis/caller-analytics.conf
install -m 0644 "$new/infra/server-b/caller-analytics/redis.service" /etc/systemd/system/lisglosips-caller-analytics-redis.service
install -m 0644 "$new/infra/server-b/caller-analytics/logrotate.conf" /etc/logrotate.d/lisglosips-caller-analytics
install -m 0644 "$new/infra/server-b/caller-analytics/maintenance.service" /etc/systemd/system/lisglosips-caller-analytics-maintenance.service
install -m 0644 "$new/infra/server-b/caller-analytics/maintenance.timer" /etc/systemd/system/lisglosips-caller-analytics-maintenance.timer
install -m 0644 "$new/infra/server-b/s58/systemd/lisglosips.target" /etc/systemd/system/lisglosips.target
systemctl daemon-reload
systemctl enable --now lisglosips-caller-analytics-redis.service
encoded_redis_password=$(/usr/bin/node -e 'console.log(encodeURIComponent(process.argv[1]))' "$analytics_redis_password")
grep '^DATABASE_URL=' /etc/lisglosips/cdr-worker.env > /etc/lisglosips/caller-analytics.env.new
printf '%s\n' "REDIS_URL=redis://:$encoded_redis_password@127.0.0.1:6381/0" 'LISGLOSIPS_ENTRYPOINT=apps/worker-cdr/dist/analytics-main.js' 'LISGLOSIPS_SERVICE_NAME=caller-analytics' 'ANALYTICS_STREAM_MAXLEN=1000000' "ANALYTICS_CAPTURE_SINCE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /etc/lisglosips/caller-analytics.env.new
install -o root -g lisglosips -m 0640 /etc/lisglosips/caller-analytics.env.new /etc/lisglosips/caller-analytics.env
rm -f /etc/lisglosips/caller-analytics.env.new
unset analytics_redis_password encoded_redis_password
chown -R root:lisglosips "$new/apps/api/dist" "$new/apps/worker-cdr/dist" "$new/packages/database/dist" "$new/packages/auth/dist" "$new/public" "$new/scripts" "$new/infra/server-b/caller-analytics"
chmod -R u=rwX,g=rX,o= "$new/apps/api/dist" "$new/apps/worker-cdr/dist" "$new/packages/database/dist" "$new/packages/auth/dist" "$new/public" "$new/scripts" "$new/infra/server-b/caller-analytics"
changed=1
systemctl restart rsyslog
install -o root -g root -m 0644 "$new/opensips-candidate.cfg" /etc/opensips/opensips.cfg
ln -sfn "$new" /opt/lisglosips/current
systemctl restart opensips lisglosips@api
systemctl enable --now lisglosips@caller-analytics
systemctl restart lisglosips@caller-analytics
systemctl enable --now lisglosips-caller-analytics-maintenance.timer
for i in $(seq 1 15); do
if curl -fsS http://127.0.0.1:3000/api/v2/health/ready >/dev/null; then break; fi
sleep 1
done
curl -fsS http://127.0.0.1:3000/api/v2/health/ready
systemctl is-active opensips lisglosips@api lisglosips@cdr-worker lisglosips@caller-analytics
systemctl is-active opensips lisglosips@api lisglosips@cdr-worker lisglosips@caller-analytics lisglosips-caller-analytics-redis
systemctl is-enabled lisglosips-caller-analytics-maintenance.timer
test "$(cat /opt/lisglosips/current/.deployed-commit)" = "$commit"
changed=0
trap - ERR
infra/server-b/caller-analytics/logrotate.conf
+11
View File
@@ -0,0 +1,11 @@
/var/log/lisglosips/caller-analytics.log {
daily
rotate 14
size 50M
compress
delaycompress
copytruncate
missingok
notifempty
su syslog lisglosips
}
infra/server-b/caller-analytics/maintenance.service
+15
View File
@@ -0,0 +1,15 @@
[Unit]
Description=LisgloSIPS caller analytics retention maintenance
After=mysql.service lisglosips-caller-analytics-redis.service
[Service]
Type=oneshot
User=lisglosips
Group=lisglosips
WorkingDirectory=/opt/lisglosips/current
EnvironmentFile=/etc/lisglosips/caller-analytics.env
ExecStart=/usr/bin/node scripts/maintain-caller-analytics.mjs --apply
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
infra/server-b/caller-analytics/maintenance.timer
+10
View File
@@ -0,0 +1,10 @@
[Unit]
Description=Daily caller analytics retention maintenance
[Timer]
OnCalendar=*-*-* 03:35:00
Persistent=true
RandomizedDelaySec=600
[Install]
WantedBy=timers.target
infra/server-b/caller-analytics/redis.conf
+19
View File
@@ -0,0 +1,19 @@
bind 127.0.0.1
protected-mode yes
port 6381
dir /var/lib/redis-caller-analytics
dbfilename dump.rdb
appendonly yes
appendfilename "appendonly.aof"
appenddirname "appendonlydir"
appendfsync everysec
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
maxmemory 256mb
maxmemory-policy noeviction
tcp-keepalive 60
requirepass @@ANALYTICS_REDIS_PASSWORD@@
pidfile /run/redis-caller-analytics/redis.pid
logfile /var/log/redis/redis-caller-analytics.log
infra/server-b/caller-analytics/redis.service
+21
View File
@@ -0,0 +1,21 @@
[Unit]
Description=LisgloSIPS isolated caller analytics Redis
After=network.target
[Service]
Type=notify
User=redis
Group=redis
RuntimeDirectory=redis-caller-analytics
RuntimeDirectoryMode=0750
ExecStart=/usr/bin/redis-server /etc/redis/caller-analytics.conf --supervised systemd
ExecStop=/bin/kill -s TERM $MAINPID
Restart=on-failure
RestartSec=2s
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ReadWritePaths=/var/lib/redis-caller-analytics /var/log/redis /run/redis-caller-analytics
[Install]
WantedBy=multi-user.target
infra/server-b/s58/systemd/lisglosips.target
+2 -2
View File
@@ -1,14 +1,14 @@
[Unit]
Description=LisgloSIPS single-server stack on Server B
Wants=network-online.target tailscaled.service
Wants=mysql.service redis-server.service postgresql.service
Wants=mysql.service redis-server.service lisglosips-caller-analytics-redis.service postgresql.service
Wants=nginx.service opensips.service rtpengine-daemon.service rtpengine-recording-daemon.service
Wants=lisglosips-redis-auth-proxy.service lisglosips-redis-hotpath-load.service
Wants=lisglosips@api.service lisglosips@cdr-worker.service lisglosips@recording-worker.service lisglosips@config-publisher.service
Wants=heplify-server.service homer-app.service
Wants=lisglosips-prometheus.service grafana-server.service
Wants=lisglosips-node-exporter.service lisglosips-mysqld-exporter.service lisglosips-postgres-exporter.service lisglosips-redis-exporter.service
Wants=lisglosips-backup.timer lisglosips-recording-finalize.timer
Wants=lisglosips-backup.timer lisglosips-recording-finalize.timer lisglosips-caller-analytics-maintenance.timer
After=network-online.target tailscaled.service
After=mysql.service redis-server.service postgresql.service