489 lines
26 KiB
Python
489 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""Private-input remote operations for the single approved preproduction node."""
|
|
import base64
|
|
import configparser
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import socket
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
UNITS = ('go-sip-asterisk.service', 'go-sip-agent.service', 'go-sip-dispatcher.service')
|
|
|
|
|
|
def paths(home):
|
|
return {'home': home, 'config': home / '.config/go-sip', 'state': home / '.local/share/go-sip', 'releases': home / '.local/opt/go-sip/releases', 'current': home / '.local/opt/go-sip/current', 'backups': home / '.local/share/go-sip-backups', 'units': home / '.config/systemd/user'}
|
|
|
|
|
|
def run(args, **kwargs):
|
|
r = subprocess.run(args, capture_output=True, text=True, **kwargs)
|
|
if r.returncode:
|
|
raise ValueError('command failed: ' + Path(args[0]).name)
|
|
return r.stdout.strip()
|
|
|
|
|
|
def mkdir(p):
|
|
if p.is_symlink():
|
|
raise ValueError('symlink directory refused')
|
|
p.mkdir(parents=True, mode=0o700, exist_ok=True)
|
|
if p.stat().st_uid != os.getuid():
|
|
raise ValueError('directory owner mismatch')
|
|
p.chmod(0o700)
|
|
|
|
|
|
def write_private(p, data):
|
|
if p.is_symlink():
|
|
raise ValueError('symlink file refused')
|
|
tmp = p.with_name(p.name + '.new')
|
|
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
with os.fdopen(fd, 'w') as stream:
|
|
stream.write(data)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(tmp, p)
|
|
p.chmod(0o600)
|
|
|
|
|
|
def private_text(p):
|
|
s = p.stat()
|
|
if p.is_symlink() or s.st_uid != os.getuid() or stat.S_IMODE(s.st_mode) != 0o600:
|
|
raise ValueError('private file ownership or permissions invalid')
|
|
return p.read_text()
|
|
|
|
|
|
def read_env(p):
|
|
result = {}
|
|
for raw in private_text(p).splitlines():
|
|
raw = raw.strip()
|
|
if not raw or raw.startswith('#'):
|
|
continue
|
|
if '=' not in raw:
|
|
raise ValueError('invalid runtime env')
|
|
k, v = raw.split('=', 1)
|
|
if k in result:
|
|
raise ValueError('duplicate runtime env key')
|
|
parts = shlex.split(v)
|
|
if len(parts) != 1:
|
|
raise ValueError('ambiguous runtime env value')
|
|
result[k] = parts[0]
|
|
return result
|
|
|
|
|
|
def unit_properties(unit):
|
|
r = subprocess.run(['systemctl', '--user', 'show', unit, '--property=LoadState,ActiveState,SubState,UnitFileState,FragmentPath,EnvironmentFiles,ExecMainStatus'], capture_output=True, text=True)
|
|
if r.returncode and 'LoadState=not-found' not in r.stdout.splitlines():
|
|
raise ValueError('systemd status inspection failed')
|
|
return dict(line.split('=', 1) for line in r.stdout.splitlines() if '=' in line)
|
|
|
|
|
|
def legacy_env(unit, home):
|
|
meta = unit_properties(unit)
|
|
fragment = Path(meta['FragmentPath'])
|
|
if not fragment.is_file() or fragment.parent != home / '.config/systemd/user':
|
|
raise ValueError('legacy unit ownership ambiguous')
|
|
references = re.findall(r'(\S+) \(ignore_errors=(?:yes|no)\)', meta['EnvironmentFiles'])
|
|
if len(references) != 1:
|
|
raise ValueError('legacy environment references ambiguous')
|
|
return read_env(Path(references[0]))
|
|
|
|
|
|
def render_roles(agent, dispatcher, source, p):
|
|
common = {'MTLS_CA_FILE', 'MTLS_CERT_FILE', 'MTLS_KEY_FILE', 'MTLS_PEER_CERT_FINGERPRINTS'}
|
|
agent_keys = common | {'AGENT_ID', 'CELL_ID', 'DISPATCHER_GRPC_SERVER_NAME', 'ASTERISK_CONFIG_DIR', 'ASTERISK_BIN', 'ASTERISK_LIBRARY_DIR', 'AGENT_HEP_LISTEN_ADDR'}
|
|
roles = {'agent': {k: v for k, v in agent.items() if k in agent_keys}, 'dispatcher': {k: v for k, v in dispatcher.items() if k in common}}
|
|
roles['agent'].update({'AGENT_SESSION_PATH': str(p['state'] / 'agent-session.json'), 'AGENT_RECOVERY_ROOT': str(p['state'] / 'agent-recovery'), 'DISPATCHER_ID': source['dispatcher_id'], 'DISPATCHER_GRPC_ENDPOINT': '127.0.0.1:19444', 'AGENT_GRPC_LISTEN': '127.0.0.1:19443', 'AGENT_EVIDENCE_ROOT': str(p['backups'] / 'debug-calls')})
|
|
roles['dispatcher'].update({'DISPATCHER_ID': source['dispatcher_id'], 'SAAS_BASE_URL': source['saas_url'], 'DISPATCHER_SECRET_KEY': source['secret'], 'RABBITMQ_URL': source['mq_url'], 'DISPATCHER_SQLITE_PATH': str(p['state'] / 'dispatcher.sqlite'), 'DISPATCHER_GRPC_LISTEN': '127.0.0.1:19444', 'DISPATCHER_AGENT_ENDPOINTS_FILE': str(p['config'] / 'agent-endpoints.json'), 'DISPATCHER_OSS_CONFIG_FILE': str(p['config'] / 'oss.json')})
|
|
return roles
|
|
|
|
|
|
def reset_requested(payload):
|
|
return payload.get('reset_state') is True
|
|
|
|
|
|
def require_reset(requested, confirmed, channels):
|
|
if not requested or not confirmed or channels != 0:
|
|
raise ValueError('reset requires explicit authorization and zero active channels')
|
|
|
|
|
|
def dependency_state(report):
|
|
if report.get('success'):
|
|
return 'verified'
|
|
if report.get('phase') in ('sip', 'providers', 'tasks', 'task_and_quota') and report.get('http_status', 0) in (0, 404, 502, 503, 504):
|
|
return 'saas_pending'
|
|
if report.get('phase', '').startswith('mq_'):
|
|
return 'mq_failed'
|
|
return 'dependency_failed'
|
|
|
|
|
|
def ari(home, endpoint='channels'):
|
|
root = home / '.config/go-sip-asterisk'
|
|
a = configparser.ConfigParser(interpolation=None)
|
|
h = configparser.ConfigParser(interpolation=None)
|
|
a.read_string(private_text(root / 'ari.conf'))
|
|
h.read_string(private_text(root / 'http.conf'))
|
|
secret = private_text(root / 'ari-secret').strip()
|
|
users = [s for s in a.sections() if s != 'general' and a.get(s, 'password', fallback=None) == secret]
|
|
if len(users) != 1 or h.get('general', 'enabled').lower() != 'yes' or h.get('general', 'bindaddr') not in ('127.0.0.1', '0.0.0.0'):
|
|
raise ValueError('native ARI configuration uncertain')
|
|
port = int(h.get('general', 'bindport'))
|
|
authorization = base64.b64encode((users[0] + ':' + secret).encode()).decode()
|
|
req = urllib.request.Request('http://127.0.0.1:' + str(port) + '/ari/' + endpoint, headers={'Authorization': 'Basic ' + authorization})
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
with opener.open(req, timeout=10) as response:
|
|
data = json.load(response)
|
|
if not isinstance(data, list):
|
|
raise ValueError('unexpected native ARI response')
|
|
return data
|
|
|
|
|
|
def stop_for_change(p, legacy=False):
|
|
if legacy:
|
|
text = run(['systemctl', '--user', 'list-unit-files', '--type=service', '--no-legend'])
|
|
units = [line.split()[0] for line in text.splitlines() if line.split()[0].startswith('go-sip-nonprod-') and any(s in line.split()[0] for s in ('dispatcher', 'agent', 'saas'))]
|
|
else:
|
|
units = ['go-sip-dispatcher.service', 'go-sip-agent.service']
|
|
dispatchers = [u for u in units if 'dispatcher' in u or 'saas' in u]
|
|
for unit in dispatchers:
|
|
if unit_properties(unit).get('FragmentPath'):
|
|
run(['systemctl', '--user', 'stop', unit])
|
|
if ari(p['home']):
|
|
raise ValueError('active calls: Agent remains running; no data changes allowed')
|
|
for unit in units:
|
|
if 'agent' in unit and unit_properties(unit).get('FragmentPath'):
|
|
run(['systemctl', '--user', 'stop', unit])
|
|
return units
|
|
|
|
|
|
def backup_dir(p, source, label):
|
|
mkdir(p['backups'])
|
|
tag = datetime.datetime.now(datetime.UTC).strftime('%Y%m%dT%H%M%S%fZ')
|
|
target = p['backups'] / (label + '-' + tag + '.tar')
|
|
if source.is_symlink() or source == p['home'] or not source.is_dir():
|
|
raise ValueError('unsafe archive source')
|
|
run(['sudo', '-n', 'tar', '--one-file-system', '-C', str(source.parent), '-cf', str(target), source.name], timeout=180)
|
|
run(['sudo', '-n', 'chown', str(os.getuid()) + ':' + str(os.getgid()), str(target)])
|
|
target.chmod(0o600)
|
|
if target.stat().st_size < 512:
|
|
raise ValueError('archive is incomplete')
|
|
# tar's own reread validates the archive before any source removal.
|
|
run(['tar', '-tf', str(target)], timeout=120)
|
|
with target.open('rb') as stream:
|
|
digest = hashlib.file_digest(stream, 'sha256').hexdigest()
|
|
write_private(target.with_suffix('.json'), json.dumps({'source': str(source), 'archive': str(target), 'sha256': digest, 'created_at': tag}, indent=2))
|
|
return digest
|
|
|
|
|
|
def bootstrap(p, source):
|
|
for key in ('config', 'state', 'releases', 'backups', 'units'):
|
|
mkdir(p[key])
|
|
baseline = p['config'] / 'host-baseline.json'
|
|
if baseline.exists():
|
|
old = json.loads(private_text(baseline))
|
|
agent, dispatcher = old['agent'], old['dispatcher']
|
|
else:
|
|
agent = legacy_env('go-sip-nonprod-agent-20261004.service', p['home'])
|
|
dispatcher = legacy_env('go-sip-nonprod-dispatcher-20261004.service', p['home'])
|
|
# This only imports installed Asterisk/TLS/platform metadata. Business
|
|
# addresses and credentials are replaced from current root sources.
|
|
for d in (agent, dispatcher):
|
|
for k in list(d):
|
|
if k.startswith(('SAAS_', 'RABBITMQ_', 'OSS_')) or 'MOCK' in k or 'FIXTURE' in k:
|
|
del d[k]
|
|
tls = p['config'] / 'tls'
|
|
mkdir(tls)
|
|
for role, d in (('agent', agent), ('dispatcher', dispatcher)):
|
|
for key in ('MTLS_CA_FILE', 'MTLS_CERT_FILE', 'MTLS_KEY_FILE'):
|
|
old_file = Path(d[key])
|
|
data = private_text(old_file)
|
|
new_file = tls / (role + '-' + old_file.name)
|
|
write_private(new_file, data)
|
|
d[key] = str(new_file)
|
|
write_private(baseline, json.dumps({'agent': agent, 'dispatcher': dispatcher}, indent=2))
|
|
endpoint_file = p['config'] / 'agent-endpoints.json'
|
|
if endpoint_file.exists():
|
|
inventory = json.loads(private_text(endpoint_file))
|
|
else:
|
|
inventory = json.loads(private_text(Path(dispatcher['DISPATCHER_AGENT_ENDPOINTS_FILE'])))
|
|
if len(inventory) != 1 or inventory[0]['agent_id'] != agent['AGENT_ID'] or inventory[0]['cell_id'] != agent['CELL_ID']:
|
|
raise ValueError('single Agent endpoint identity mismatch')
|
|
inventory[0]['address'] = '127.0.0.1:19443'
|
|
write_private(endpoint_file, json.dumps(inventory, indent=2))
|
|
roles = render_roles(agent, dispatcher, source, p)
|
|
# Read native mirror destination, rather than guessing which old Agent port
|
|
# currently receives HEP. No static transport or trunk changes are made.
|
|
hep = p['home'] / '.config/go-sip-asterisk/hep.conf'
|
|
if hep.exists():
|
|
cfg = configparser.ConfigParser(interpolation=None)
|
|
cfg.read(hep)
|
|
if cfg.has_option('general', 'capture_address'):
|
|
roles['agent']['AGENT_HEP_LISTEN_ADDR'] = cfg.get('general', 'capture_address')
|
|
o = source['oss']
|
|
endpoint = o['Endpoint']
|
|
if '://' not in endpoint:
|
|
endpoint = 'https://' + endpoint
|
|
oss = {'dispatcher_id': source['dispatcher_id'], 'oss': {'bucket': o['bucket'], 'endpoint': endpoint, 'region': o['Region'], 'object_prefix': 'call-recordings', 'access_key_id_env': 'OSS_ACCESS_KEY_ID', 'access_key_secret_env': 'OSS_ACCESS_KEY_SECRET', 'max_asset_bytes': 64 * 1024 * 1024}}
|
|
roles['agent']['AGENT_OSS_ALLOWED_HOST'] = o['bucket'] + '.' + urllib.parse.urlsplit(endpoint).hostname
|
|
roles['dispatcher']['OSS_ACCESS_KEY_ID'] = o['RAM.accessKeyId']
|
|
roles['dispatcher']['OSS_ACCESS_KEY_SECRET'] = o['RAM.accessKeySecret']
|
|
for role, d in roles.items():
|
|
for v in d.values():
|
|
if any(c in str(v) for c in '\x00\r\n'):
|
|
raise ValueError('invalid multiline runtime value')
|
|
write_private(p['config'] / (role + '.env'), ''.join(k + '=' + json.dumps(str(v), ensure_ascii=False) + '\n' for k, v in sorted(d.items())))
|
|
write_private(p['config'] / 'oss.json', json.dumps(oss, indent=2))
|
|
return roles
|
|
|
|
|
|
def host_check(p):
|
|
os_release = dict(line.split('=', 1) for line in Path('/etc/os-release').read_text().splitlines() if '=' in line)
|
|
if os_release.get('ID', '').strip('"') != 'debian' or os_release.get('VERSION_ID', '').strip('"') != '13' or run(['uname', '-m']) != 'x86_64':
|
|
raise ValueError('host is not Debian 13 amd64')
|
|
if os.getuid() == 0 or p['home'].name != 'rogee':
|
|
raise ValueError('runtime user must be rogee')
|
|
for tool in ('python3', 'openssl', 'tcpdump', 'tshark'):
|
|
if not shutil.which(tool):
|
|
raise ValueError('required host tool missing: ' + tool)
|
|
run(['sudo', '-n', 'true'])
|
|
channels = len(ari(p['home']))
|
|
if channels:
|
|
raise ValueError('active native calls prevent deployment')
|
|
native = unit_properties('go-sip-asterisk.service')
|
|
if native.get('ActiveState') != 'active' or native.get('UnitFileState') != 'enabled':
|
|
raise ValueError('native Asterisk is not enabled and active')
|
|
return {'os': 'debian13', 'architecture': 'amd64', 'runtime_user': 'rogee', 'active_channels': channels}
|
|
|
|
|
|
def retire(p, payload):
|
|
host_check(p)
|
|
bootstrap(p, payload['source'])
|
|
units = stop_for_change(p, legacy=True)
|
|
archived = []
|
|
for unit in units:
|
|
meta = unit_properties(unit)
|
|
fragment = Path(meta['FragmentPath'])
|
|
if fragment.parent != p['units'] or fragment.is_symlink() or 'sip' not in fragment.read_text():
|
|
raise ValueError('legacy unit ownership unclear')
|
|
run(['systemctl', '--user', 'disable', unit])
|
|
obsolete_binaries = set()
|
|
for unit in units:
|
|
detail = run(['systemctl', '--user', 'show', unit, '--property=ExecStart', '--value'])
|
|
match = re.search(r'path=(\S+) ;', detail)
|
|
if match:
|
|
executable = Path(match.group(1))
|
|
if executable.is_file() and not executable.is_symlink() and executable.is_relative_to(p['home']) and executable.name in ('sip-go-agent', 'saas-mock'):
|
|
obsolete_binaries.add(executable)
|
|
artifact_backup = p['backups'] / ('legacy-artifacts-' + datetime.datetime.now(datetime.UTC).strftime('%Y%m%dT%H%M%SZ'))
|
|
mkdir(artifact_backup)
|
|
for old in obsolete_binaries:
|
|
target = artifact_backup / (hashlib.sha256(str(old).encode()).hexdigest()[:12] + '-' + old.name)
|
|
shutil.copy2(old, target); target.chmod(0o600)
|
|
if hashlib.sha256(old.read_bytes()).digest() != hashlib.sha256(target.read_bytes()).digest():
|
|
raise ValueError('legacy artifact backup mismatch')
|
|
old.unlink()
|
|
# The registered host uses its project-owned system RabbitMQ, not Docker.
|
|
local_mq = False
|
|
ctl = Path('/usr/sbin/rabbitmqctl')
|
|
if ctl.exists() and run(['systemctl', 'show', 'rabbitmq-server.service', '--property=ActiveState', '--value']) == 'active':
|
|
legacy = legacy_env('go-sip-nonprod-dispatcher-20261004.service', p['home'])
|
|
old_vhost = urllib.parse.unquote(urllib.parse.urlsplit(legacy['RABBITMQ_URL']).path[1:]) or '/'
|
|
def query(args):
|
|
return json.loads(run(['sudo', '-n', str(ctl), '-q', *args, '--formatter=json'], timeout=30))
|
|
vhosts = [v['name'] for v in query(['list_vhosts', 'name'])]
|
|
connections = query(['list_connections', 'vhost'])
|
|
if any(v not in ('/', old_vhost) for v in vhosts) or query(['list_queues', '-p', '/', 'name']) or any(c['vhost'] != old_vhost for c in connections):
|
|
raise ValueError('local RabbitMQ contains unowned resources; retained')
|
|
# Correlate a live unique connection, not merely IP addresses: public
|
|
# addresses/NAT must not make us uninstall the configured real broker.
|
|
marker = 'go-sip-retirement-check-' + uuid.uuid4().hex
|
|
config = dict(payload['source']['probe'], connection_name=marker, hold_mq_ms=10000)
|
|
probe = p['home'] / '.local/share/go-sip-tools/preprod-probe'
|
|
child = subprocess.Popen([str(probe)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
child.stdin.write(json.dumps(config)); child.stdin.close()
|
|
time.sleep(1)
|
|
connected = query(['list_connections', 'client_properties'])
|
|
target_is_local = any(marker in json.dumps(c) for c in connected)
|
|
still_held = child.poll() is None
|
|
child.stdout.read(); child.wait(timeout=30)
|
|
if not still_held or target_is_local:
|
|
raise ValueError('configured RabbitMQ target may be local; no broker removal permitted')
|
|
run(['sudo', '-n', 'systemctl', 'stop', 'rabbitmq-server.service'])
|
|
backup_dir(p, Path('/var/lib/rabbitmq'), 'mock-rabbitmq-data')
|
|
backup_dir(p, Path('/etc/rabbitmq'), 'mock-rabbitmq-config')
|
|
run(['sudo', '-n', 'systemctl', 'disable', 'rabbitmq-server.service'])
|
|
run(['sudo', '-n', 'apt-get', 'remove', '-y', 'rabbitmq-server'], timeout=180)
|
|
run(['sudo', '-n', 'rm', '-rf', '--', '/var/lib/rabbitmq', '/etc/rabbitmq'])
|
|
local_mq = True
|
|
# Archive legacy project data/config before removal; never touch all ~/.local.
|
|
for parent in (p['home'] / '.local/share', p['home'] / '.config'):
|
|
for source in sorted(parent.glob('go-sip-nonprod-*')):
|
|
if source.is_dir() and not source.is_symlink():
|
|
digest = backup_dir(p, source, 'legacy')
|
|
archived.append(digest)
|
|
run(['sudo', '-n', 'rm', '-rf', '--', str(source)])
|
|
unit_backup = p['backups'] / ('legacy-units-' + datetime.datetime.now(datetime.UTC).strftime('%Y%m%dT%H%M%SZ'))
|
|
mkdir(unit_backup)
|
|
for unit in units:
|
|
fragment = Path(unit_properties(unit)['FragmentPath'])
|
|
shutil.copy2(fragment, unit_backup / fragment.name)
|
|
(unit_backup / fragment.name).chmod(0o600)
|
|
fragment.unlink()
|
|
run(['systemctl', '--user', 'daemon-reload'])
|
|
evidence = {'ok': True, 'legacy_units_removed': len(units), 'legacy_binaries_removed': len(obsolete_binaries), 'legacy_directories_archived': len(archived), 'archive_hashes': archived, 'mock_broker_removed': local_mq, 'active_channels': 0, 'external_services_modified': False}
|
|
write_private(p['backups'] / 'retirement.json', json.dumps(evidence, indent=2))
|
|
return evidence
|
|
|
|
|
|
def validate_environments(probe, roles):
|
|
for role, values in roles.items():
|
|
environment = dict(os.environ, **values)
|
|
checked = subprocess.run([str(probe), '-environment', role], env=environment, capture_output=True, text=True, timeout=15)
|
|
try:
|
|
report = json.loads(checked.stdout)
|
|
except json.JSONDecodeError:
|
|
raise ValueError('environment inspection failed') from None
|
|
if checked.returncode or not report.get('success'):
|
|
raise ValueError('invalid ' + role + ' configuration at ' + report.get('phase', 'unknown'))
|
|
|
|
|
|
def install(payload, p):
|
|
facts = host_check(p)
|
|
source = payload['source']
|
|
# Changing ownership of existing durable data is never a normal deployment.
|
|
old_env = p['config'] / 'dispatcher.env'
|
|
if old_env.exists() and (p['state'] / 'dispatcher.sqlite').exists():
|
|
previous = read_env(old_env).get('DISPATCHER_ID')
|
|
if previous != source['dispatcher_id'] and not reset_requested(payload):
|
|
raise ValueError('Dispatcher identity change requires an explicitly confirmed reset')
|
|
stop_for_change(p)
|
|
if reset_requested(payload):
|
|
require_reset(True, payload.get('confirm_reset') is True, len(ari(p['home'])))
|
|
if p['state'].exists() and any(p['state'].iterdir()):
|
|
backup_dir(p, p['state'], 'reset')
|
|
shutil.rmtree(p['state'])
|
|
roles = bootstrap(p, source)
|
|
mkdir(p['state'] / 'agent-recovery')
|
|
release = p['releases'] / payload['commit']
|
|
mkdir(release)
|
|
staging = Path(payload['staging'])
|
|
if staging.parent != p['home'] / '.local/share/go-sip-tools' or staging.is_symlink():
|
|
raise ValueError('unexpected uploaded staging path')
|
|
for name, expected in payload['hashes'].items():
|
|
if name not in ('sip-go-agent', 'preprod-probe'):
|
|
raise ValueError('unexpected artifact')
|
|
candidate = staging / name
|
|
if candidate.is_symlink() or hashlib.sha256(candidate.read_bytes()).hexdigest() != expected:
|
|
raise ValueError('uploaded artifact checksum mismatch')
|
|
shutil.copy2(candidate, release / name)
|
|
(release / name).chmod(0o700)
|
|
validate_environments(release / 'preprod-probe', roles)
|
|
write_private(release / 'manifest.json', json.dumps({'commit': payload['commit'], 'hashes': payload['hashes'], 'production_approval': False}, indent=2))
|
|
current = p['current']
|
|
if current.exists() and not current.is_symlink():
|
|
raise ValueError('current artifact pointer is not a symlink')
|
|
tmp = current.with_name('current.new')
|
|
if tmp.exists() or tmp.is_symlink():
|
|
raise ValueError('unfinished artifact switch exists')
|
|
tmp.symlink_to(release)
|
|
os.replace(tmp, current)
|
|
for role in ('agent', 'dispatcher'):
|
|
unit = '[Unit]\nDescription=go-sip real preproduction ' + role + '\nAfter=go-sip-asterisk.service network-online.target\n\n[Service]\nType=simple\nEnvironmentFile=' + str(p['config'] / (role + '.env')) + '\nExecStart=' + str(current / 'sip-go-agent') + ' ' + role + ' --mode nonprod-real\nRestart=on-failure\nRestartSec=10\nUMask=0077\n\n[Install]\nWantedBy=default.target\n'
|
|
write_private(p['units'] / ('go-sip-' + role + '.service'), unit)
|
|
run(['systemctl', '--user', 'daemon-reload'])
|
|
run(['sudo', '-n', 'loginctl', 'enable-linger', p['home'].name])
|
|
run(['systemctl', '--user', 'enable', *UNITS])
|
|
# MQ is checked independently even when real SaaS has not started.
|
|
probe = subprocess.run([str(release / 'preprod-probe')], input=json.dumps(source['probe']), text=True, capture_output=True, timeout=110)
|
|
try:
|
|
dependency = json.loads(probe.stdout)
|
|
except json.JSONDecodeError:
|
|
raise ValueError('redacted external probe failed') from None
|
|
state = dependency_state(dependency)
|
|
if state not in ('verified', 'saas_pending') or not dependency.get('mq_connected') or not dependency.get('mq_connection_closed'):
|
|
raise ValueError('independent dependency failure; not attributable to unavailable SaaS')
|
|
write_private(p['config'] / 'dependency-status.json', json.dumps(dependency, indent=2))
|
|
run(['systemctl', '--user', 'start', 'go-sip-agent.service'])
|
|
time.sleep(2)
|
|
if unit_properties('go-sip-agent.service').get('ActiveState') != 'active':
|
|
raise ValueError('Agent startup failed independently of SaaS')
|
|
run(['systemctl', '--user', 'start', 'go-sip-dispatcher.service'])
|
|
time.sleep(2)
|
|
result = status(p)
|
|
result.update({'host': facts, 'dependency_state': state, 'reset_performed': reset_requested(payload), 'commit': payload['commit']})
|
|
write_private(p['backups'] / 'last-deployment.json', json.dumps(result, indent=2))
|
|
return result
|
|
|
|
|
|
def status(p):
|
|
units = {u: {k: v for k, v in unit_properties(u).items() if k in ('ActiveState', 'SubState', 'UnitFileState', 'ExecMainStatus')} for u in UNITS}
|
|
manifest = p['current'] / 'manifest.json'
|
|
release = json.loads(private_text(manifest)) if manifest.exists() else None
|
|
matches = False
|
|
if release:
|
|
matches = all(hashlib.sha256((p['current'] / name).read_bytes()).hexdigest() == digest for name, digest in release['hashes'].items())
|
|
listen = False
|
|
try:
|
|
with socket.create_connection(('127.0.0.1', 19443), timeout=3):
|
|
listen = True
|
|
except OSError:
|
|
pass # Report an explicit false, never substitute a successful listener.
|
|
dep = p['config'] / 'dependency-status.json'
|
|
dependency = dependency_state(json.loads(private_text(dep))) if dep.exists() else 'not_checked'
|
|
healthy = matches and listen and all(units[u]['ActiveState'] == 'active' and units[u]['UnitFileState'] == 'enabled' for u in UNITS[:2])
|
|
dispatcher = units[UNITS[2]]
|
|
waiting_reason = None
|
|
if dependency == 'saas_pending':
|
|
logs = run(['journalctl', '--user', '-u', 'go-sip-dispatcher.service', '_COMM=sip-go-agent', '-n', '1', '--no-pager', '-o', 'cat'])
|
|
if re.search(r'HTTP(?: status)?[ :]+(?:404|502|503|504)', logs):
|
|
waiting_reason = 'saas_http_unavailable'
|
|
elif 'NOT_FOUND' in logs and ('queue' in logs or 'exchange' in logs):
|
|
waiting_reason = 'saas_topology_pending'
|
|
elif '/internal/v1/dispatcher/' in logs and any(v in logs.lower() for v in ('timeout', 'connection refused', 'no such host')):
|
|
waiting_reason = 'saas_network_unavailable'
|
|
acceptable = dispatcher['UnitFileState'] == 'enabled' and (dispatcher['ActiveState'] == 'active' or (waiting_reason is not None and dispatcher['ActiveState'] in ('activating', 'failed')))
|
|
return {'ok': healthy and acceptable, 'services': units, 'artifact_hashes_match': matches, 'agent_listener': listen, 'active_channels': len(ari(p['home'])), 'dependency_state': dependency, 'business_ready': dependency == 'verified' and dispatcher['ActiveState'] == 'active', 'commit': release['commit'] if release else None, 'waiting_reason': waiting_reason, 'real_reboot_verified': False}
|
|
|
|
|
|
def main():
|
|
payload = json.load(sys.stdin)
|
|
p = paths(Path.home())
|
|
action = payload['action']
|
|
if action == 'bootstrap':
|
|
host_check(p)
|
|
bootstrap(p, payload['source'])
|
|
result = {'ok': True, 'fixed_paths_configured': True}
|
|
elif action == 'retire-mocks':
|
|
result = retire(p, payload)
|
|
elif action == 'deploy':
|
|
result = install(payload, p)
|
|
elif action == 'status':
|
|
result = status(p)
|
|
elif action == 'stop':
|
|
stop_for_change(p)
|
|
result = {'ok': True, 'stopped': True, 'data_modified': False}
|
|
elif action == 'start':
|
|
run(['systemctl', '--user', 'start', *UNITS])
|
|
time.sleep(2)
|
|
result = status(p)
|
|
else:
|
|
raise ValueError('unsupported operation')
|
|
print(json.dumps(result, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception as e:
|
|
print(json.dumps({'ok': False, 'error_class': type(e).__name__, 'reason': str(e) if isinstance(e, ValueError) else 'operation failed; no successful deployment claimed'}))
|
|
raise SystemExit(1)
|