go-sip/deploys/test/nonprod-call-evidence.sh

#!/usr/bin/env bash
# Capture-first entrypoint for every non-production mixed/real call attempt.
set -euo pipefail

usage() {
  cat >&2 <<'EOF'
usage: nonprod-call-evidence.sh --trunk TRUNK --target NUMBER [options] -- COMMAND [ARG...]

Options:
  --environment NAME       Defaults to AGENT_ENVIRONMENT or development; production is refused.
  --call-id ID             Evidence directory suffix (default: UTC timestamp).
  --evidence-dir DIR       Evidence root (default: /var/lib/sip-go-agent/evidence/<call-id>).
  --interface IFACE        Capture interface (default: default-route interface; any fallback).
  --run-as USER            Run COMMAND as this non-root user (default: rogee).
  --asterisk-scope SCOPE   system (default) or explicitly configured user service.
  --sip-port PORT          SIP UDP port (default: 5060).
  --rtp-start PORT         RTP range start (default: 10000).
  --rtp-end PORT           RTP range end (default: 10800).
  --preflight-only         Start/stop capture and diagnostics without a call; do not require packets.
  --proof-root DIR          Live capture arm directory (default: /run/sip-go-agent/nonprod-armed).
EOF
  exit 2
}

[[ "$(id -u)" == 0 ]] || { echo 'must run as root for tcpdump and Asterisk diagnostics' >&2; exit 1; }

environment="${AGENT_ENVIRONMENT:-development}"
call_id="$(date -u +%Y%m%dT%H%M%SZ)"
evidence_dir=""
recording_dir="/var/lib/sip-go-agent/recordings"
interface="any"
run_as="rogee"
asterisk_scope="system"
sip_port=5060
rtp_start=10000
rtp_end=10800
trunk=""
target=""
call_command=()
preflight_only=0
proof_root="/run/sip-go-agent/nonprod-armed"
proof_file=""
proof_created=0

while (($#)); do
  case "$1" in
    --environment) [[ $# -ge 2 ]] || usage; environment=$2; shift 2 ;;
    --call-id) [[ $# -ge 2 ]] || usage; call_id=$2; shift 2 ;;
    --evidence-dir) [[ $# -ge 2 ]] || usage; evidence_dir=$2; shift 2 ;;
    --recording-dir) [[ $# -ge 2 ]] || usage; recording_dir=$2; shift 2 ;;
    --interface) [[ $# -ge 2 ]] || usage; interface=$2; shift 2 ;;
    --run-as) [[ $# -ge 2 ]] || usage; run_as=$2; shift 2 ;;
    --asterisk-scope) [[ $# -ge 2 ]] || usage; asterisk_scope=$2; shift 2 ;;
    --sip-port) [[ $# -ge 2 ]] || usage; sip_port=$2; shift 2 ;;
    --rtp-start) [[ $# -ge 2 ]] || usage; rtp_start=$2; shift 2 ;;
    --rtp-end) [[ $# -ge 2 ]] || usage; rtp_end=$2; shift 2 ;;
    --preflight-only) preflight_only=1; shift ;;
    --proof-root) [[ $# -ge 2 ]] || usage; proof_root=$2; shift 2 ;;
    --trunk) [[ $# -ge 2 ]] || usage; trunk=$2; shift 2 ;;
    --target) [[ $# -ge 2 ]] || usage; target=$2; shift 2 ;;
    --) shift; call_command=("$@"); break ;;
    -h|--help) usage ;;
    *) echo "unknown option: $1" >&2; usage ;;
  esac
done

case "$environment" in
  development|mock|mixed|real|nonprod-real) ;;
  production) echo 'production requires the separate production gate' >&2; exit 1 ;;
  *) echo 'invalid non-production environment' >&2; exit 1 ;;
esac
[[ "$asterisk_scope" == system || "$asterisk_scope" == user ]] || { echo 'invalid Asterisk service scope' >&2; exit 1; }
[[ "$call_id" =~ ^[A-Za-z0-9._-]+$ ]] || { echo 'invalid call id' >&2; exit 1; }
[[ "$trunk" =~ ^[A-Za-z0-9._-]{1,128}$ ]] || { echo 'invalid SaaS trunk identifier' >&2; exit 1; }
[[ "$target" =~ ^[0-9]{1,32}$ ]] || { echo 'SaaS event target is not an original numeric dial route' >&2; exit 1; }
[[ "$proof_root" =~ ^/[A-Za-z0-9._/-]+$ ]] || { echo 'invalid proof root path' >&2; exit 1; }
[[ ${#call_command[@]} -gt 0 ]] || { echo 'call command is required after --' >&2; exit 1; }
[[ "$interface" =~ ^[A-Za-z0-9_.:-]+$ ]] || { echo 'invalid capture interface' >&2; exit 1; }
[[ "$sip_port" =~ ^[0-9]+$ && "$rtp_start" =~ ^[0-9]+$ && "$rtp_end" =~ ^[0-9]+$ ]] || { echo 'invalid port' >&2; exit 1; }
# "any" includes both provider SIP and the local Asterisk ExternalMedia RTP.
# Reducing it to the default-route NIC silently omits loopback media.
if [[ -z "$evidence_dir" ]]; then
  evidence_dir="/var/lib/sip-go-agent/evidence/$call_id"
fi
install -d -m 0700 "$evidence_dir"
umask 077
exec > >(tee "$evidence_dir/entrypoint.log") 2>&1
install -d -o "$run_as" -g "$run_as" -m 0700 "$recording_dir"
touch "$evidence_dir/recording-start.marker"

asterisk_bin="${ASTERISK_BIN:-/usr/sbin/asterisk}"
if [[ "$asterisk_scope" == user ]]; then
  [[ -n "${ASTERISK_BIN:-}" && -n "${ASTERISK_CONFIG:-}" && -r "$ASTERISK_CONFIG" ]] || { echo 'explicit user Asterisk configuration required; fail-closed'; exit 1; }
  asterisk_lib="${ASTERISK_LIBRARY_PATH:-$(dirname "$asterisk_bin")/../lib}"
  [[ -d "$asterisk_lib" ]] || { echo 'user Asterisk runtime libraries unavailable; fail-closed'; exit 1; }
  asterisk_user_uid="$(id -u "$run_as")" || { echo 'user Asterisk service account unavailable; fail-closed'; exit 1; }
fi
tcpdump_bin="${TCPDUMP_BIN:-$(command -v tcpdump || true)}"
[[ -x "$asterisk_bin" ]] || { echo 'Asterisk CLI unavailable; fail-closed'; exit 1; }
[[ -n "$tcpdump_bin" && -x "$tcpdump_bin" ]] || { echo 'tcpdump unavailable; fail-closed'; exit 1; }
command -v runuser >/dev/null || { echo 'runuser unavailable; fail-closed'; exit 1; }
command -v python3 >/dev/null || { echo 'python3 unavailable for SIP evidence summary; fail-closed'; exit 1; }

# A successful one-packet probe or a timeout after opening the capture proves
# that the binary can open a raw capture socket; permission errors fail closed.
probe_status=0
timeout 2s "$tcpdump_bin" -i "$interface" -nn -c 1 -w /dev/null >/dev/null 2>"$evidence_dir/tcpdump-preflight.log" || probe_status=$?
if [[ "$probe_status" != 0 && "$probe_status" != 124 ]]; then
  echo "tcpdump CAP_NET_RAW preflight failed: status=$probe_status" >&2
  exit 1
fi

started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
printf '{"environment":"%s","call_id":"%s","trunk":"%s","target":"%s","interface":"%s","sip_port":%s,"rtp_start":%s,"rtp_end":%s,"started_at":"%s"}\n' \
  "$environment" "$call_id" "$trunk" "$target" "$interface" "$sip_port" "$rtp_start" "$rtp_end" "$started_at" >"$evidence_dir/metadata.json"

redact() {
  sed -E 's/(password|secret|token|authorization|api[_-]?key)[^[:space:]]*/\1=<redacted>/Ig'
}

asterisk_service() {
  if [[ "$asterisk_scope" == user ]]; then
    runuser -u "$run_as" -- env XDG_RUNTIME_DIR="/run/user/$asterisk_user_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$asterisk_user_uid/bus" systemctl --user "$@" go-sip-asterisk.service
  else
    systemctl "$@" asterisk.service
  fi
}
asterisk_cli() {
  if [[ "$asterisk_scope" == user ]]; then
    runuser -u "$run_as" -- env LD_LIBRARY_PATH="$asterisk_lib" "$asterisk_bin" -C "$ASTERISK_CONFIG" -rx "$1"
  else
    "$asterisk_bin" -rx "$1"
  fi
}
asterisk_journal() {
  if [[ "$asterisk_scope" == user ]]; then
    runuser -u "$run_as" -- env XDG_RUNTIME_DIR="/run/user/$asterisk_user_uid" DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$asterisk_user_uid/bus" journalctl --user -u go-sip-asterisk.service "$@"
  else
    journalctl -u asterisk.service "$@"
  fi
}

if ! asterisk_service is-enabled >"$evidence_dir/asterisk-enabled.txt" 2>&1 ||
   ! grep -qx enabled "$evidence_dir/asterisk-enabled.txt"; then
  echo 'Asterisk service must be enabled and active; fail-closed' >&2
  exit 1
fi
if ! asterisk_service is-active >"$evidence_dir/asterisk-active.txt" 2>&1 ||
   ! grep -qx active "$evidence_dir/asterisk-active.txt"; then
  echo 'Asterisk service must be enabled and active; fail-closed' >&2
  exit 1
fi
uname -a >"$evidence_dir/uname.txt"
cat /etc/os-release >"$evidence_dir/os-release.txt"
ip -brief address >"$evidence_dir/ip-address.txt"
ss -lunp >"$evidence_dir/udp-listeners.txt" 2>&1 || ss -lun >"$evidence_dir/udp-listeners.txt"
asterisk_cli "module show like res_ari.so" 2>&1 | redact >"$evidence_dir/ari-module-status.txt"
asterisk_cli "http show status" 2>&1 | redact >"$evidence_dir/ari-http-status.txt"
if ! grep -Eq 'res_ari\.so.*Running' "$evidence_dir/ari-module-status.txt" ||
   ! grep -Fq 'Server Enabled and Bound' "$evidence_dir/ari-http-status.txt" ||
   ! grep -Fq '/ari/' "$evidence_dir/ari-http-status.txt"; then
  echo 'ARI module or HTTP route unavailable; fail-closed' >&2
  exit 1
fi
asterisk_cli "pjsip show endpoint $trunk" 2>&1 | redact >"$evidence_dir/pjsip-endpoint.txt"
if ! grep -Eq 'Endpoint:[[:space:]]*' "$evidence_dir/pjsip-endpoint.txt" || ! grep -Fq "$trunk" "$evidence_dir/pjsip-endpoint.txt"; then
  echo 'PJSIP endpoint unavailable; fail-closed' >&2
  exit 1
fi
asterisk_cli "pjsip show contacts" 2>&1 | redact >"$evidence_dir/pjsip-contacts-before.txt"
asterisk_cli "core show channels verbose" 2>&1 | redact >"$evidence_dir/channels-before.txt"
if ! sha256sum /opt/sip-go-agent/current/sip-go-agent /etc/sip-go-agent/artifacts/*.json /etc/sip-go-agent/ai/*.json >"$evidence_dir/installed-sha256.txt" 2>&1; then
  echo 'installed package/config SHA-256 unavailable; fail-closed' >&2
  exit 1
fi

logger_enabled=0
capture_pid=""
write_sip_summary() {
  python3 - "$evidence_dir/asterisk-journal.txt" "$evidence_dir/call-output.private" >"$evidence_dir/sip-summary.json" <<'PY'
import json
import re
import sys
from pathlib import Path

journal = Path(sys.argv[1]).read_text(errors="replace") if Path(sys.argv[1]).exists() else ""
call_output = Path(sys.argv[2]).read_text(errors="replace") if Path(sys.argv[2]).exists() else ""
responses = []
methods = []
reason_headers = []
timeline = []
direction = None
current_response = None
sdp = {"present": False, "audio_ports": [], "codecs": [], "payload_types": [], "ptime": [], "directions": []}

def timestamp(line):
    return line.split(" asterisk", 1)[0].strip() if " asterisk" in line else None

for line in journal.splitlines():
    ts = timestamp(line)
    if "Transmitting SIP request" in line:
        direction = "outbound"
    elif "Received SIP request" in line or "Received SIP response" in line:
        direction = "inbound"
    status = re.search(r"SIP/2\.0\s+(\d{3})(?:\s+(.+?))?\s*$", line)
    if status:
        item = {"timestamp": ts, "direction": direction, "code": int(status.group(1)), "reason": (status.group(2) or "").strip(), "cseq_method": None}
        responses.append(item)
        current_response = item
        timeline.append({"timestamp": ts, "direction": direction, "event": f"{item['code']} {item['reason']}".strip()})
    method = re.search(r"\b(INVITE|ACK|BYE|CANCEL)\s+(sip:\S+)\s+SIP/2\.0", line)
    if method:
        current_response = None
        item = {"timestamp": ts, "direction": direction, "method": method.group(1), "request_uri": method.group(2)}
        methods.append(item)
        timeline.append({"timestamp": ts, "direction": direction, "event": method.group(1), "request_uri": method.group(2)})
    cseq = re.search(r"\bCSeq:\s*\d+\s+([A-Za-z]+)", line)
    if cseq and current_response is not None and current_response["cseq_method"] is None:
        current_response["cseq_method"] = cseq.group(1).upper()
    reason = re.search(r"\bReason:\s*(.+)$", line)
    if reason:
        value = reason.group(1).strip()
        reason_headers.append(value)
        q850 = re.search(r"Q\.850\s*;\s*cause\s*=\s*(\d+)", value, re.I)
    else:
        q850 = None
    if "v=0" in line:
        sdp["present"] = True
    audio = re.search(r"m=audio\s+(\d+)", line)
    if audio:
        sdp["audio_ports"].append(int(audio.group(1)))
    rtpmap = re.search(r"a=rtpmap:(\d+)\s+([^\s]+)", line)
    if rtpmap:
        sdp["payload_types"].append(int(rtpmap.group(1)))
        sdp["codecs"].append(rtpmap.group(2))
    ptime = re.search(r"a=ptime:\s*(\d+)", line)
    if ptime:
        sdp["ptime"].append(int(ptime.group(1)))
    media_direction = re.search(r"a=(sendrecv|sendonly|recvonly|inactive)\s*$", line)
    if media_direction:
        sdp["directions"].append(media_direction.group(1))

hangup = re.search(r"cause=(\d+)", call_output)
invite_responses = [item for item in responses if item.get("cseq_method") in (None, "INVITE")]
final_response = next((item for item in reversed(invite_responses) if item["code"] >= 200), None)
summary = {
    "responses": responses,
    "invite_responses": invite_responses,
    "final_response": final_response,
    "methods": methods,
    "bye": [item for item in methods if item["method"] == "BYE"],
    "cancel": [item for item in methods if item["method"] == "CANCEL"],
    "reason_headers": reason_headers,
    "q850": next((re.search(r"Q\.850\s*;\s*cause\s*=\s*(\d+)", value, re.I).group(1) for value in reason_headers if re.search(r"Q\.850\s*;\s*cause\s*=\s*(\d+)", value, re.I)), None),
    "asterisk_hangup_cause": int(hangup.group(1)) if hangup else None,
    "sdp": sdp,
    "timeline": timeline[:200],
    "stasis_start_seen": "StasisStart" in journal,
    "stasis_end_seen": "StasisEnd" in journal,
}
print(json.dumps(summary, ensure_ascii=False, sort_keys=True, indent=2))
PY
}
stop_capture() {
  if [[ -n "$capture_pid" ]]; then
    # Let libpcap drain packets already accepted by the kernel before SIGINT;
    # short INVITE/404 calls otherwise can leave a header-only pcap.
    sleep 2
  fi
  if [[ -n "$capture_pid" ]] && kill -0 "$capture_pid" 2>/dev/null; then
    kill -INT "$capture_pid" 2>/dev/null || true
    for _ in 1 2 3 4 5; do
      kill -0 "$capture_pid" 2>/dev/null || break
      sleep 1
    done
    kill -TERM "$capture_pid" 2>/dev/null || true
    wait "$capture_pid" 2>/dev/null || true
  fi
  capture_pid=""
}
cleanup() {
  local call_exit=$? evidence_failed=0
  trap - EXIT
  set +e
  if ((proof_created)) && ! rm -f -- "$proof_file"; then
    printf 'capture arm removal failed\n' >>"$evidence_dir/diagnostic-errors.txt"
    evidence_failed=1
  fi
  stop_capture
  if ((logger_enabled)) && ! asterisk_cli "pjsip set logger off" >"$evidence_dir/pjsip-logger-off.txt" 2>&1; then
    printf 'PJSIP logger stop failed\n' >>"$evidence_dir/diagnostic-errors.txt"
    evidence_failed=1
  fi
  if ! date -u +%Y-%m-%dT%H:%M:%SZ >"$evidence_dir/ended-at.txt"; then
    printf 'end timestamp unavailable\n' >>"$evidence_dir/diagnostic-errors.txt"
    evidence_failed=1
  fi
  if [[ -f "$evidence_dir/capture.pcap" ]]; then
    if ! sha256sum "$evidence_dir/capture.pcap" >"$evidence_dir/capture.pcap.sha256"; then
      printf 'capture SHA-256 unavailable\n' >>"$evidence_dir/diagnostic-errors.txt"
      evidence_failed=1
    fi
  elif ((logger_enabled)); then
    printf 'capture file missing\n' >>"$evidence_dir/diagnostic-errors.txt"
    evidence_failed=1
  fi
  if ! find "$recording_dir" -maxdepth 1 -type f -newer "$evidence_dir/recording-start.marker" -print0 | xargs -0r sha256sum >"$evidence_dir/recordings.sha256"; then
    printf 'recording SHA-256 unavailable\n' >>"$evidence_dir/diagnostic-errors.txt"
    evidence_failed=1
  fi
  if ! asterisk_journal --since "$started_at" --no-pager 2>/dev/null | redact >"$evidence_dir/asterisk-journal.txt"; then
    printf 'Asterisk journal unavailable\n' >>"$evidence_dir/diagnostic-errors.txt"
    evidence_failed=1
  fi
  if ! write_sip_summary; then
    printf '{"error":"sip summary unavailable"}\n' >"$evidence_dir/sip-summary.json"
    printf 'SIP summary unavailable\n' >>"$evidence_dir/diagnostic-errors.txt"
    evidence_failed=1
  fi
  if ! chown -R "$run_as:$run_as" "$evidence_dir" 2>/dev/null; then
    printf 'restricted evidence ownership update failed\n' >>"$evidence_dir/diagnostic-errors.txt"
    evidence_failed=1
  fi
  if ((evidence_failed)); then
    echo 'required post-call evidence unavailable; fail-closed' >&2
    if ((call_exit == 0)); then exit 1; fi
  fi
  exit "$call_exit"
}
trap cleanup EXIT
asterisk_cli "pjsip set logger on" >"$evidence_dir/pjsip-logger-on.txt" 2>&1 || { echo 'cannot enable PJSIP logger; fail-closed' >&2; exit 1; }
logger_enabled=1

"$tcpdump_bin" -i "$interface" -nn -s0 -U -w "$evidence_dir/capture.pcap" \
  "udp port $sip_port or (udp portrange $rtp_start-$rtp_end)" >"$evidence_dir/tcpdump.log" 2>&1 &
capture_pid=$!
sleep 1
kill -0 "$capture_pid" 2>/dev/null || { echo 'tcpdump exited before call; fail-closed' >&2; exit 1; }
printf '%s\n' "capture_started=$evidence_dir/capture.pcap"
if ((preflight_only)); then
  sleep 1
  stop_capture
  printf 'call_exit=0\ncapture_packets=0\ncapture_status=0\npreflight_only=1\n' >"$evidence_dir/result.txt"
  exit 0
fi

# The Agent checks this root-owned, call-specific live capture arm before any
# originate. A stale arm is never overwritten; the trap removes it first.
install -d -o root -g "$run_as" -m 0750 -- "$proof_root"
proof_file="$proof_root/$call_id.active"
[[ ! -e "$proof_file" ]] || { echo 'stale or concurrent capture arm; fail-closed' >&2; exit 1; }
proof_tmp="$(mktemp --tmpdir="$proof_root" ".$call_id.XXXXXX")"
printf '%s\t%s\t%s\n' "$capture_pid" "$trunk" "$target" >"$proof_tmp"
chown "root:$run_as" "$proof_tmp"
chmod 0640 "$proof_tmp"
ln -- "$proof_tmp" "$proof_file" || { rm -f -- "$proof_tmp"; echo 'capture arm already exists; fail-closed' >&2; exit 1; }
proof_created=1
rm -f -- "$proof_tmp"
call_status=0
set +e
runuser -u "$run_as" -- "${call_command[@]}" >"$evidence_dir/call-output.private" 2>&1
call_status=$?
set -e
printf '%s\n' "call_exit=$call_status"
stop_capture
asterisk_cli "pjsip show contacts" 2>&1 | redact >"$evidence_dir/pjsip-contacts-after.txt"
asterisk_cli "core show channels verbose" 2>&1 | redact >"$evidence_dir/channels-after.txt"
capture_packets="$(awk '/ packets captured/{print $1; exit}' "$evidence_dir/tcpdump.log" 2>/dev/null || true)"
[[ "$capture_packets" =~ ^[0-9]+$ ]] || capture_packets=0
capture_status=0
if ((capture_packets == 0)); then capture_status=2; fi
printf 'call_exit=%s\ncapture_packets=%s\ncapture_status=%s\n' "$call_status" "$capture_packets" "$capture_status" >"$evidence_dir/result.txt"
if ((call_status != 0)); then exit "$call_status"; fi
exit "$capture_status"