Files
rust-discovery/deploy/cloud_lifecycle_acceptance.py
T
2026-11-13 17:01:14 +01:00

235 lines
16 KiB
Python

#!/usr/bin/env python3
"""Native TLS lifecycle on the existing Cloud VM, passive test identity only.
Never replaces the permanent device, bypasses its unknown fence or executes a
job. Rotation uses the existing development trigger; revocation uses real UI.
"""
import base64
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
import cloud_apt_retry as apt
cloud = apt.cloud
require = cloud.require
STATE = cloud.ROOT/'devstate/cloud-lifecycle'
def prepare_vm(payload):
import signal
os.umask(0o077)
ident = str(uuid.UUID(payload['id']))
binary = Path('/usr/local/libexec/rmm-agent-remote')
require(hashlib.sha256(binary.read_bytes()).hexdigest() == payload['binary_sha256'], 'reviewed native binary')
parent = Path('/var/lib/rmm-agent-remote/lifecycle')
if not parent.exists(): parent.mkdir(mode=0o700); os.chown(parent,993,984)
require(not parent.is_symlink() and parent.stat().st_uid == 993 and parent.stat().st_mode&0o777 == 0o700, 'private masked lifecycle parent')
root = parent/ident; root.mkdir(mode=0o700); os.chown(root,993,984)
def create(path, data):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchown(f.fileno(),993,984)
create(root/'enroll.token',payload['token'].encode())
create(root/'ca.pem',base64.b64decode(payload['ca'],validate=True))
wrong = root/'untrusted-ca.pem'
subprocess.run(['/usr/bin/openssl','req','-x509','-newkey','rsa:2048','-nodes','-days','1',
'-keyout',str(root/'untrusted-ca.key'),'-out',str(wrong),'-subj','/CN=RMM_LIFECYCLE_TEST_UNTRUSTED'],capture_output=True,check=True,timeout=10)
for p in (wrong,root/'untrusted-ca.key'):os.chown(p,993,984);p.chmod(0o600)
negatives={}
for name,server,ca in (('hostname','wrong-name.invalid',root/'ca.pem'),('untrusted-ca','discovery.ease-technologies.de',wrong)):
state=root/name;state.mkdir(mode=0o700);os.chown(state,993,984)
env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',HOME=str(root),RMM_AGENT_STATE=str(state),
RMM_CA_FILE=str(ca),RMM_SERVER_NAME=server,RMM_CONNECT_ADDR='88.99.80.202:443',
RMM_ENROLLMENT_TOKEN_FILE=str(root/'enroll.token'),RMM_EXTENDED_INVENTORY='0')
proc=subprocess.Popen(['/usr/sbin/runuser','-u','rmm-agent','--',str(binary)],env=env,
stdout=subprocess.DEVNULL,stderr=subprocess.PIPE,start_new_session=True)
try:_,error=proc.communicate(timeout=8)
except subprocess.TimeoutExpired:
os.killpg(proc.pid,signal.SIGTERM);proc.communicate(timeout=5)
raise RuntimeError('negative TLS process did not reject') from None
require(proc.returncode != 0 and not (state/'identity.json').exists(), 'native invalid trust/name must reject before enrollment')
negatives[name]=dict(rejected=True,exit_code=proc.returncode,stderr_sha256=hashlib.sha256(error).hexdigest(),identity_created=False)
state=root/'passive';state.mkdir(mode=0o700);os.chown(state,993,984)
unit='rmm-lifecycle-'+ident+'.service';path=Path('/etc/systemd/system')/unit
text=('[Unit]\nDescription=RMM passive native lifecycle acceptance\nAfter=network-online.target time-sync.target\n'
'[Service]\nType=simple\nUser=rmm-agent\nGroup=rmm-agent\nUMask=0077\nRestart=no\nRuntimeMaxSec=900\n'
'Environment=RMM_AGENT_STATE='+str(state)+'\nEnvironment=RMM_CA_FILE='+str(root/'ca.pem')+'\n'
'Environment=RMM_SERVER_NAME=discovery.ease-technologies.de\nEnvironment=RMM_CONNECT_ADDR=88.99.80.202:443\n'
'Environment=RMM_ENROLLMENT_TOKEN_FILE='+str(root/'enroll.token')+'\nEnvironment=RMM_EXTENDED_INVENTORY=0\n'
'UnsetEnvironment=RMM_JOB_BINDING_FILE RMM_JOB_SIGNING_KEY_BOOTSTRAP RMM_HELPER_SOCKET RMM_APT_HELPER_SOCKET\n'
'ExecStart='+str(binary)+'\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\n'
'ProtectKernelTunables=true\nProtectKernelModules=true\nProtectControlGroups=true\nRestrictSUIDSGID=true\n'
'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK\nCapabilityBoundingSet=\nAmbientCapabilities=\n'
'ReadWritePaths='+str(state)+'\n')
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o644)
with os.fdopen(fd,'w') as f:f.write(text);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),0o644)
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=10)
return dict(id=ident,root=str(root),unit=unit,unit_sha256=hashlib.sha256(text.encode()).hexdigest(),negative_tls=negatives,passive_only=True)
def start_vm(payload):
ident=str(uuid.UUID(payload['id']));unit='rmm-lifecycle-'+ident+'.service';path=Path('/etc/systemd/system')/unit
require(hashlib.sha256(path.read_bytes()).hexdigest()==payload['unit_sha256'],'own lifecycle unit only')
subprocess.run(['systemctl','start',unit],capture_output=True,check=True,timeout=10)
return dict(started=True,passive_only=True)
def vm_state(payload):
ident=str(uuid.UUID(payload['id']));root=Path('/var/lib/rmm-agent-remote/lifecycle')/ident;state=root/'passive'
p=state/'identity.json'
if not p.exists():return dict(enrolled=False)
identity=json.loads(p.read_text());selector=json.loads((state/'certificates/current.json').read_text())
cert=state/'certificates/versions'/str(uuid.UUID(selector['version']))/'device.pem'
der=subprocess.run(['openssl','x509','-in',str(cert),'-outform','DER'],capture_output=True,check=True,timeout=10).stdout
return dict(enrolled=True,identity={k:identity[k] for k in ('agent_id','asset_id')},selector=selector,
fingerprint=hashlib.sha256(der).hexdigest(),token_removed=not (root/'enroll.token').exists())
def rotation_switch(payload):
ident=str(uuid.UUID(payload['id']));unit='rmm-lifecycle-'+ident+'.service';path=Path('/etc/systemd/system')/unit
require(hashlib.sha256(path.read_bytes()).hexdigest()==payload['unit_sha256'],'own lifecycle unit only')
directory=Path(str(path)+'.d');trigger=directory/'10-rotation-test.conf'
text='[Service]\nEnvironment=RMM_AGENT_PROFILE=development\nEnvironment=RMM_ROTATION_TEST_AFTER_SECONDS=1\n'
if payload['enabled']:
root=Path('/var/lib/rmm-agent-remote/lifecycle')/ident
require((root/'passive/identity.json').is_file(),'enrolled identity required before token cleanup')
token=root/'enroll.token'
require(not token.is_symlink() and token.stat().st_uid==993 and token.stat().st_mode&0o777==0o600,'own private enrollment token')
token.unlink()
directory.mkdir(mode=0o755)
fd=os.open(trigger,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o644)
with os.fdopen(fd,'w') as f:f.write(text);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),0o644)
else:
require(not trigger.is_symlink() and trigger.stat().st_uid==0 and trigger.read_text()==text,'own exact trigger only')
trigger.unlink()
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=10)
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=10)
return dict(rotation_test_enabled=payload['enabled'])
def token_unused(payload):
root=Path('/opt/rust-discovery');sha=payload['sha256'];require(len(sha)==64 and all(c in '0123456789abcdef' for c in sha),'token hash only')
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
value=subprocess.run(cmd,input=("SELECT consumed_at IS NULL FROM enrollment_tokens WHERE token_hash='"+sha+"';").encode(),capture_output=True,check=True).stdout.strip()
return dict(unused=value==b't')
def start():
os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True);m=cloud.contract
ident=str(uuid.uuid4());folder=STATE/ident;folder.mkdir(mode=0o700)
permanent=apt.prior.identity();old=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
unknown=m.protected_json(apt.STATE/'apt_install_retry/checkpoint.json')['plan']['jobs'][0]
before=cloud.bootstrap.rpc('linux-vm.conf',apt.stored_attempt_evidence,dict(job_id=unknown['job_id'],attempt_id=unknown['attempt_id']))
material=cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='enroll'))
result=cloud.bootstrap.rpc('linux-vm.conf',prepare_vm,dict(id=ident,binary_sha256=old['binary_sha256'],token=material['token'],ca=material['ca']))
m.write_private(folder/'bootstrap.json',result)
unused=cloud.bootstrap.rpc('server.conf',token_unused,dict(sha256=hashlib.sha256(material['token'].encode()).hexdigest()))
require(unused['unused'],'negative TLS must not consume enrollment')
m.write_private(folder/'negative-token-unused.json',unused)
cloud.bootstrap.rpc('linux-vm.conf',start_vm,result)
backend=cloud.Backend();deadline=time.monotonic()+50
while True:
native=cloud.bootstrap.rpc('linux-vm.conf',vm_state,dict(id=ident))
if native['enrolled']:
args=dict(tenant_id=cloud.TENANT,asset_id=native['identity']['asset_id'])
device=backend.call('inventory.get_device',args)
if device['presence']=='online' and device['last_authenticated_certificate_sha256']==native['fingerprint']:break
require(time.monotonic()<deadline,'passive lifecycle enrollment incomplete; retain checkpoint');time.sleep(2)
require(native['identity']['agent_id']!=permanent['agent_id'] and set(device['capabilities'])=={'linux.base/1','heartbeat/1'},'separate strictly passive identity')
m.write_private(folder/'before-rotation.json',dict(native=native,device=device))
cloud.bootstrap.rpc('linux-vm.conf',rotation_switch,dict(result,enabled=True))
deadline=time.monotonic()+80
try:
while True:
rotated=cloud.bootstrap.rpc('linux-vm.conf',vm_state,dict(id=ident))
device_after=backend.call('inventory.get_device',args)
if rotated['fingerprint']!=native['fingerprint'] and device_after['last_authenticated_certificate_sha256']==rotated['fingerprint']:break
require(time.monotonic()<deadline,'native rotation incomplete; no fabricated success');time.sleep(2)
finally:
cloud.bootstrap.rpc('linux-vm.conf',rotation_switch,dict(result,enabled=False))
require(rotated['identity']==native['identity'] and rotated['selector']['enrollment_generation']==native['selector']['enrollment_generation'] and
rotated['token_removed'] and device_after['certificate_sha256']==rotated['fingerprint'],'native same-identity rotation and actual new mTLS peer')
m.write_private(folder/'rotation.json',dict(before=native,after=rotated,device=device_after,verified=True))
after=cloud.bootstrap.rpc('linux-vm.conf',apt.stored_attempt_evidence,dict(job_id=unknown['job_id'],attempt_id=unknown['attempt_id']))
require(before==after,'permanent unknown evidence changed during lifecycle test')
m.write_private(folder/'permanent-unknown-preserved.json',dict(evidence=after))
print(dict(id=ident,negative_tls_passed=True,rotation_verified=True,permanent_unknown_preserved=True,revocation_pending=True))
def lifecycle_status(payload):
ident=str(uuid.UUID(payload['id']));unit='rmm-lifecycle-'+ident+'.service'
root=Path('/var/lib/rmm-agent-remote/lifecycle')/ident
require(json.loads((root/'passive/identity.json').read_text())['agent_id']==payload['agent_id'],'own passive identity only')
raw=subprocess.run(['journalctl','--unit',unit,'--no-pager','-o','cat','-n','300'],capture_output=True,check=True,timeout=10).stdout
active=subprocess.run(['systemctl','show',unit,'-p','ActiveState','--value'],capture_output=True,check=True,timeout=10).stdout.strip().decode()
if payload.get('stop'):
subprocess.run(['systemctl','stop',unit],capture_output=True,check=True,timeout=10)
return dict(active_before=active,retry_messages=raw.count(b'channel unavailable; retrying with jitter'),logs_sha256=hashlib.sha256(raw).hexdigest(),
stopped=bool(payload.get('stop')))
def revocation_audit(payload):
root=Path('/opt/rust-discovery');asset=str(uuid.UUID(payload['asset_id']))
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
query=("SELECT json_build_object('principal',principal,'origin',data->>'approval_origin',"
"'simulation',data->'testsimulation') FROM audit_events WHERE tenant_id='11111111-1111-4111-8111-111111111111' "
"AND event_type='device_revoked' AND data->'decision'->>'asset_id'='"+asset+"';")
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
return json.loads(raw)
def revoke(ident):
ident=str(uuid.UUID(ident));folder=STATE/ident;m=cloud.contract
bootstrap=m.protected_json(folder/'bootstrap.json');rotation=m.protected_json(folder/'rotation.json')
require(rotation['verified'],'actual rotation required')
identity=rotation['after']['identity'];target=dict(tenant_id=cloud.TENANT,**identity)
backend=cloud.Backend();device=backend.call('inventory.get_device',{k:target[k] for k in ('tenant_id','asset_id')})
require(device['presence']=='online' and set(device['capabilities'])=={'linux.base/1','heartbeat/1'},'only independent passive test identity')
before=cloud.bootstrap.rpc('linux-vm.conf',lifecycle_status,dict(id=ident,agent_id=identity['agent_id']))
expected=folder/'revoke-ui.json';expected_value=dict(profile='dev/test',expected_principal='simulated-testprincipal-a',**target)
if expected.exists():require(m.protected_json(expected)==expected_value,'same existing administrative intent only')
else:m.write_private(expected,expected_value)
cloud.prepare_browser()
result=subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_lifecycle_revoke.cjs'),str(expected)],capture_output=True,timeout=90)
if result.returncode:print(result.stdout.decode('ascii',errors='replace').strip())
require(result.returncode==0,'revocation UI incomplete; do not repeat blindly')
revoked=backend.call('inventory.get_device',{k:target[k] for k in ('tenant_id','asset_id')})
require(revoked['presence']=='revoked','actual revocation required')
deadline=time.monotonic()+40
while True:
after=cloud.bootstrap.rpc('linux-vm.conf',lifecycle_status,dict(id=ident,agent_id=identity['agent_id']))
if after['retry_messages']>before['retry_messages']:break
require(time.monotonic()<deadline,'revoked channel not observed closed');time.sleep(2)
after_device=backend.call('inventory.get_device',{k:target[k] for k in ('tenant_id','asset_id')})
require(after_device['last_contact_at']==revoked['last_contact_at'] and
after_device['last_authenticated_at']==revoked['last_authenticated_at'],'revoked identity authenticated again')
audit=cloud.bootstrap.rpc('server.conf',revocation_audit,target)
require(audit==dict(principal='simulated-testprincipal-a',origin='dev_test_browser_simulation',simulation=True),'actual simulated UI revocation audit')
gate=cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,agent_id=identity['agent_id']))
require(gate==dict(jobs=0,outbox=0,permits=0,unresolved=0),'strictly passive lifecycle zero job gate')
stopped=cloud.bootstrap.rpc('linux-vm.conf',lifecycle_status,dict(id=ident,agent_id=identity['agent_id'],stop=True))
unknown=m.protected_json(apt.STATE/'apt_install_retry/checkpoint.json')['plan']['jobs'][0]
preserved=cloud.bootstrap.rpc('linux-vm.conf',apt.stored_attempt_evidence,dict(job_id=unknown['job_id'],attempt_id=unknown['attempt_id']))
require(preserved==m.protected_json(folder/'permanent-unknown-preserved.json')['evidence'],'permanent unknown changed')
m.write_private(folder/'revoked.json',dict(verified=True,device=after_device,audit=audit,gate=gate,native_before=before,native_after=after,stopped=stopped))
print(dict(revocation_verified=True,simulated_ui=True,revoked_channel_closed=True,zero_jobs=True,test_agent_stopped=True,permanent_unknown_preserved=True))
def main():
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=('start','revoke'),default='start',nargs='?');parser.add_argument('--id')
args=parser.parse_args()
if args.action=='start':start()
else:revoke(args.id)
if __name__=='__main__':
try:main()
except Exception as error:
print('CLOUD_LIFECYCLE_INCOMPLETE:'+type(error).__name__);raise SystemExit(2) from None