#!/usr/bin/env bash set -Eeuo pipefail commit=${1:?commit required} archive=${2:?archive required} expected=${3:?sha256 required} [[ "$commit" =~ ^[0-9a-f]{40}$ ]] test "$(sha256sum "$archive" | awk '{print $1}')" = "$expected" stamp=$(date -u +%Y%m%dT%H%M%SZ) old=$(readlink -f /opt/lisglosips/current) new=/opt/lisglosips/releases/caller-analytics-$stamp backup=/var/backups/lisglosips-caller-analytics/$stamp test -d "$old" test ! -e "$new" # Refuse to interrupt calls, including unrelated test calls. curl -fsS -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"dlg_list","params":[]}' http://127.0.0.1:8888/mi | python3 -c 'import sys,json; assert json.load(sys.stdin)["result"]["Dialogs"]==[], "Active calls: retry after calls end"' systemctl start lisglosips-backup.service install -d -m 0700 "$backup" if systemctl --quiet is-active lisglosips-caller-analytics-redis.service && test -f /etc/lisglosips/caller-analytics-redis.secret; then export REDISCLI_AUTH REDISCLI_AUTH=$(cat /etc/lisglosips/caller-analytics-redis.secret) redis-cli -h 127.0.0.1 -p 6381 --no-auth-warning SAVE >/dev/null unset REDISCLI_AUTH cp -a /var/lib/redis-caller-analytics "$backup/redis-caller-analytics" fi cp -a /etc/opensips/opensips.cfg "$backup/opensips.cfg" checkpoint_file=/var/lib/lisglosips/caller-analytics-offset.json if test -f "$checkpoint_file"; then cp -a "$checkpoint_file" "$backup/caller-analytics-offset.json"; fi printf '%s\n' "$old" > "$backup/previous-release" managed_files=(/etc/lisglosips/caller-analytics.env /etc/rsyslog.d/35-caller-analytics.conf /etc/redis/caller-analytics.conf /etc/systemd/system/lisglosips-caller-analytics-redis.service /etc/logrotate.d/lisglosips-caller-analytics /etc/systemd/system/lisglosips-caller-analytics-maintenance.service /etc/systemd/system/lisglosips-caller-analytics-maintenance.timer /etc/systemd/system/lisglosips.target) for file in "${managed_files[@]}"; do if test -f "$file"; then cp -a "$file" "$backup/$(basename "$file")"; fi done changed=0 was_running=$(systemctl is-active lisglosips@caller-analytics || true) was_enabled=$(systemctl is-enabled lisglosips@caller-analytics 2>/dev/null || true) redis_was_running=$(systemctl is-active lisglosips-caller-analytics-redis 2>/dev/null || true) redis_was_enabled=$(systemctl is-enabled lisglosips-caller-analytics-redis 2>/dev/null || true) maintenance_was_running=$(systemctl is-active lisglosips-caller-analytics-maintenance.timer 2>/dev/null || true) maintenance_was_enabled=$(systemctl is-enabled lisglosips-caller-analytics-maintenance.timer 2>/dev/null || true) secret_existed=0 test -f /etc/lisglosips/caller-analytics-redis.secret && secret_existed=1 rollback() { rc=$? if test "$changed" = 1; then systemctl stop lisglosips@caller-analytics || true systemctl stop lisglosips-caller-analytics-maintenance.timer lisglosips-caller-analytics-redis || true cp -a "$backup/opensips.cfg" /etc/opensips/opensips.cfg if test -f "$backup/caller-analytics-offset.json"; then cp -a "$backup/caller-analytics-offset.json" "$checkpoint_file"; fi for file in "${managed_files[@]}"; do if test -f "$backup/$(basename "$file")"; then cp -a "$backup/$(basename "$file")" "$file"; else rm -f "$file"; fi done ln -sfn "$old" /opt/lisglosips/current systemctl restart rsyslog || true systemctl daemon-reload || true systemctl restart opensips lisglosips@api || true if test "$was_running" = active; then systemctl start lisglosips@caller-analytics || true; fi if test "$was_enabled" != enabled; then systemctl disable lisglosips@caller-analytics || true; fi if test "$redis_was_running" = active; then systemctl start lisglosips-caller-analytics-redis || true; fi if test "$redis_was_enabled" != enabled; then systemctl disable lisglosips-caller-analytics-redis || true; fi if test "$maintenance_was_running" = active; then systemctl start lisglosips-caller-analytics-maintenance.timer || true; fi if test "$maintenance_was_enabled" != enabled; then systemctl disable lisglosips-caller-analytics-maintenance.timer || true; fi if test "$secret_existed" = 0; then rm -f /etc/lisglosips/caller-analytics-redis.secret; fi fi echo "FAILED rc=$rc backup=$backup (additive analytics tables retained)" exit "$rc" } trap rollback ERR cp -a --reflink=auto "$old" "$new" tar -xzf "$archive" -C "$new" printf '%s\n' "$commit" > "$new/.deployed-commit" printf '%s\n' "$old" > "$new/.delta-base-release" source_cfg="$backup/opensips.cfg" if grep -q 'CRA_INSTRUMENTATION_VERSION=3' "$source_cfg"; then cp "$source_cfg" "$new/opensips-candidate.cfg" else /usr/bin/node "$new/scripts/instrument-caller-analytics.mjs" "$source_cfg" "$new/opensips-candidate.cfg" fi python3 "$new/infra/server-b/caller-analytics/verify-hotpath.py" "$new/opensips-candidate.cfg" systemctl start lisglosips-redis-hotpath-load.service opensips -C -f "$new/opensips-candidate.cfg" set -a source /etc/lisglosips/secrets/mysql-migrate.env set +a export DATABASE_URL DATABASE_URL=$(/usr/bin/node -e 'console.log(`mysql://${encodeURIComponent(process.env.MYSQL_USER)}:${encodeURIComponent(process.env.MYSQL_PASSWORD)}@${process.env.MYSQL_HOST}:${process.env.MYSQL_PORT}/${process.env.MYSQL_DATABASE}`)') # Previous delta releases stripped executable bits from bundled native engines. find "$new/node_modules/.pnpm" -path '*/@prisma/engines/schema-engine-debian-openssl-3.0.x' -type f -exec chmod 0750 {} \; /usr/bin/node "$new/node_modules/prisma/build/index.js" migrate deploy --schema "$new/prisma/schema.prisma" unset DATABASE_URL MYSQL_PASSWORD install -d -o syslog -g lisglosips -m 0750 /var/log/lisglosips touch /var/log/lisglosips/caller-analytics.log chown syslog:lisglosips /var/log/lisglosips/caller-analytics.log chmod 0640 /var/log/lisglosips/caller-analytics.log install -m 0644 "$new/infra/server-b/caller-analytics/rsyslog.conf" /etc/rsyslog.d/35-caller-analytics.conf rsyslogd -N1 changed=1 install -d -o redis -g redis -m 0750 /var/lib/redis-caller-analytics if ! test -f /etc/lisglosips/caller-analytics-redis.secret; then openssl rand -hex 32 > /etc/lisglosips/caller-analytics-redis.secret; chown root:lisglosips /etc/lisglosips/caller-analytics-redis.secret; chmod 0640 /etc/lisglosips/caller-analytics-redis.secret; fi analytics_redis_password=$(cat /etc/lisglosips/caller-analytics-redis.secret) sed "s/@@ANALYTICS_REDIS_PASSWORD@@/$analytics_redis_password/" "$new/infra/server-b/caller-analytics/redis.conf" > /etc/redis/caller-analytics.conf chown root:redis /etc/redis/caller-analytics.conf; chmod 0640 /etc/redis/caller-analytics.conf install -m 0644 "$new/infra/server-b/caller-analytics/redis.service" /etc/systemd/system/lisglosips-caller-analytics-redis.service install -m 0644 "$new/infra/server-b/caller-analytics/logrotate.conf" /etc/logrotate.d/lisglosips-caller-analytics install -m 0644 "$new/infra/server-b/caller-analytics/maintenance.service" /etc/systemd/system/lisglosips-caller-analytics-maintenance.service install -m 0644 "$new/infra/server-b/caller-analytics/maintenance.timer" /etc/systemd/system/lisglosips-caller-analytics-maintenance.timer install -m 0644 "$new/infra/server-b/s58/systemd/lisglosips.target" /etc/systemd/system/lisglosips.target systemctl daemon-reload systemctl enable --now lisglosips-caller-analytics-redis.service encoded_redis_password=$(/usr/bin/node -e 'console.log(encodeURIComponent(process.argv[1]))' "$analytics_redis_password") grep '^DATABASE_URL=' /etc/lisglosips/cdr-worker.env > /etc/lisglosips/caller-analytics.env.new printf '%s\n' "REDIS_URL=redis://:$encoded_redis_password@127.0.0.1:6381/0" 'LISGLOSIPS_ENTRYPOINT=apps/worker-cdr/dist/analytics-main.js' 'LISGLOSIPS_SERVICE_NAME=caller-analytics' 'ANALYTICS_STREAM_MAXLEN=1000000' "ANALYTICS_CAPTURE_SINCE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /etc/lisglosips/caller-analytics.env.new install -o root -g lisglosips -m 0640 /etc/lisglosips/caller-analytics.env.new /etc/lisglosips/caller-analytics.env rm -f /etc/lisglosips/caller-analytics.env.new unset analytics_redis_password encoded_redis_password chown -R root:lisglosips "$new/apps/api/dist" "$new/apps/worker-cdr/dist" "$new/packages/database/dist" "$new/packages/auth/dist" "$new/public" "$new/scripts" "$new/infra/server-b/caller-analytics" chmod -R u=rwX,g=rX,o= "$new/apps/api/dist" "$new/apps/worker-cdr/dist" "$new/packages/database/dist" "$new/packages/auth/dist" "$new/public" "$new/scripts" "$new/infra/server-b/caller-analytics" systemctl restart rsyslog install -o root -g root -m 0644 "$new/opensips-candidate.cfg" /etc/opensips/opensips.cfg ln -sfn "$new" /opt/lisglosips/current systemctl stop lisglosips@caller-analytics # This is a coordinated producer restart after the spool is quiescent. Reset # only the per-process reconciliation baseline; preserve any pre-existing gap. if test -f "$checkpoint_file"; then python3 - "$checkpoint_file" <<'PY' import json, os, sys p=sys.argv[1]; st=os.stat(p) with open(p,encoding='utf-8') as f: data=json.load(f) spool='/var/log/lisglosips/caller-analytics.log' assert int(data.get('offset',-1)) == os.path.getsize(spool), 'analytics spool is not quiescent' produced=data.get('lastProducerCount'); collected=data.get('eventsSinceProducerStart') assert produced is None or produced == collected, 'producer/collector counters differ before planned restart' data.pop('lastProducerCount',None); data['eventsSinceProducerStart']=0 t=p+'.deploy-tmp' with open(t,'w',encoding='utf-8') as f: json.dump(data,f,separators=(',',':')) os.chown(t,st.st_uid,st.st_gid); os.chmod(t,st.st_mode); os.replace(t,p) PY fi systemctl restart opensips lisglosips@api systemctl enable --now lisglosips@caller-analytics systemctl restart lisglosips@caller-analytics systemctl enable --now lisglosips-caller-analytics-maintenance.timer for i in $(seq 1 15); do if curl -fsS http://127.0.0.1:3000/api/v2/health/ready >/dev/null; then break; fi sleep 1 done curl -fsS http://127.0.0.1:3000/api/v2/health/ready systemctl is-active opensips lisglosips@api lisglosips@cdr-worker lisglosips@caller-analytics lisglosips-caller-analytics-redis systemctl is-enabled lisglosips-caller-analytics-maintenance.timer test "$(cat /opt/lisglosips/current/.deployed-commit)" = "$commit" changed=0 trap - ERR printf '\nRELEASE=%s\nBACKUP=%s\nCOMMIT=%s\n' "$new" "$backup" "$commit"