Files
rust-discovery/deploy/vm_runtime_upgrade.py
T
BuTzZ 6c3203d725
Rust development slice / linux (push) Failing after 1m10s
feat: snapshot Linux-first RMM MVP and pause handover
2026-11-12 22:06:25 +01:00

316 lines
23 KiB
Python

#!/usr/bin/env python3
"""Versioned final-source build/install/probe. No RMM job or package mutation.
Existing identity/journals stay in place. Only manifest-proven own deployment
files/empty old provider stores are moved to root-private history. Failures retain
reports/backups and never reset a journal, grant, acceptance or safety flag.
"""
import argparse
import base64
from contextlib import closing
import hashlib
import inspect
import json
import os
from pathlib import Path
import re
import shutil
import sqlite3
import subprocess
import sys
import uuid
import vm_job_bootstrap as previous
from vm_passive_test import execute, require, remote_python
ROOT=previous.ROOT
BASE=ROOT/'devstate/vm-final'
BOUND=('crates/agent','crates/agent-linux','crates/privileged-helper','crates/domain','crates/protocol',
'tools/apt-transaction/apt_transaction.py','tools/apt-transaction/hook','deploy/linux/rmm-apt-helper.service',
'Cargo.toml','Cargo.lock','rust-toolchain.toml')
def sha(data):return hashlib.sha256(data).hexdigest()
def bound_sources(root):
values={}
for relative in BOUND:
path=root/relative
paths=path.rglob('*') if path.is_dir() else [path]
for item in paths:
if item.is_file() and not any(part in ('target','__pycache__','.git') for part in item.parts):
require(not item.is_symlink(),'bound source symlink')
values[str(item.relative_to(root))]=sha(item.read_bytes())
return values
def build():
os.umask(0o077);BASE.mkdir(mode=0o700,parents=True,exist_ok=True)
run=BASE/uuid.uuid4().hex;run.mkdir(mode=0o700)
context=run/'context';context.mkdir(mode=0o700)
for name in ('Cargo.toml','Cargo.lock','rust-toolchain.toml'):shutil.copyfile(ROOT/name,context/name)
for name in ('crates','tools/apt-transaction','deploy/linux','migrations'):
shutil.copytree(ROOT/name,context/name,ignore=shutil.ignore_patterns('target','__pycache__','*.pyc','.git','devstate','credentials','secrets'))
sources=bound_sources(context)
require(sources==bound_sources(ROOT),'source changed during build snapshot')
(context/'Dockerfile').write_text('FROM rust:1.99.1-bookworm\nWORKDIR /src\nCOPY . .\nRUN cargo build --locked --release -p rmm-agent -p rmm-privileged-helper\n')
image='rmm-vm-final:'+run.name
subprocess.run(['docker','build','-t',image,str(context)],check=True)
container=execute(['docker','create',image]).decode().strip()
artifacts={}
try:
for name in ('rmm-agent','rmm-privileged-helper'):
path=run/name;execute(['docker','cp',container+':/src/target/release/'+name,str(path)]);path.chmod(0o700)
data=path.read_bytes();require(data[:6]==b'\x7fELF\x02\x01' and data[18:20]==b'\x3e\x00','wrong ELF target')
versions=execute(['readelf','--version-info',str(path)]).decode()
glibc=max(tuple(map(int,x.split('.'))) for x in re.findall(r'GLIBC_(\d+\.\d+)',versions))
require(glibc<=(2,39),'unsupported VM glibc')
dynamic=execute(['readelf','-d',str(path)]).decode()
needed=re.findall(r'\(NEEDED\).*\[(.*?)\]',dynamic)
require('libssl.so.3' in needed and 'libssl.so.1.1' not in needed,'OpenSSL-3 build required')
artifacts[name]=dict(sha256=sha(data),glibc='.'.join(map(str,glibc)),needed=needed)
finally:execute(['docker','rm',container])
receipt=dict(build_id=run.name,sources=sources,artifacts=artifacts,own_verification='locked Bookworm release build and ELF/dependency inspection',
source_owner_reported_tests=dict(rust=115,python_apt=56,installer=2,rerun_by_this_build=False))
(run/'build.json').write_text(json.dumps(receipt,indent=2))
# Pointer is not trust: install rechecks every source/artifact and old VM hash.
pointer=BASE/'current.json';temporary=BASE/('.current-'+run.name)
temporary.write_text(json.dumps(dict(build_id=run.name)));temporary.chmod(0o600);os.replace(temporary,pointer)
print('PASS final Bookworm agent/helper build; receipt directory: devstate/vm-final/'+run.name)
def remote_upgrade(payload):
import fcntl
import pwd
os.umask(0o077)
require(os.geteuid()==0,'root installation required')
account=pwd.getpwnam('rmm-agent');require((account.pw_uid,account.pw_gid)==(993,984),'dedicated UID/GID drift')
for unit in ('rmm-agent.service','rmm-helper-dev-test.service','rmm-apt-helper.service'):
status=subprocess.run(['systemctl','is-active',unit],capture_output=True,text=True)
require(status.stdout.strip() not in ('active','activating','deactivating'),'existing active service; no stop performed')
ports=subprocess.run(['ss','-H','-ltn','sport = :18443'],capture_output=True,check=True).stdout
require(not ports,'existing reverse listener')
require(not Path('/run/rmm-helper/control.sock').exists() and not Path('/run/rmm-apt-helper/helper.sock').exists(),'existing helper socket')
state=Path('/var/lib/rmm-agent');helper_state=Path('/var/lib/rmm-helper')
for path in (state,helper_state):
require(path.is_dir() and not path.is_symlink() and path.stat().st_mode&0o777==0o700,'unsafe preserved state')
lock=(state/'agent.lock').open('rb');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
for database in (state/'journal.sqlite',state/'consumption.sqlite',helper_state/'consumption.sqlite'):
with closing(sqlite3.connect(database.as_uri()+'?mode=ro',uri=True)) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','preserved SQLite corruption')
tables={row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if 'job_attempts' in tables:
require(db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','expired','cancelled')").fetchone()[0]==0,'unresolved agent attempt')
if 'consumed' in tables:require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0]==0,'unresolved consumption')
for process in Path('/proc').glob('[0-9]*'):
try:require(Path(os.readlink(process/'exe')).name not in ('rmm-agent','rmm-privileged-helper'),'existing runtime process')
except (FileNotFoundError,PermissionError):pass
for descriptor in (process/'fd').glob('*'):
try:
target=os.readlink(descriptor)
require(not (target.startswith(str(state)+'/') or target.startswith(str(helper_state)+'/'))
or (process.name==str(os.getpid()) and target==str(state/'agent.lock')),'active journal file handle')
except (FileNotFoundError,PermissionError):pass
before=manifest(state);helper_before=manifest(helper_state)
binding=json.loads(Path('/etc/rmm/agent-binding.json').read_text())
require(binding==payload['binding'],'authenticated current binding differs; no automatic recovery')
old_key=json.loads(Path('/etc/rmm/helper-signing-key-v1.json').read_text())
require(old_key==payload['key'],'independent helper key changed')
for path,expected in payload['expected_old'].items():
p=Path(path);require(p.is_file() and not p.is_symlink() and p.stat().st_uid==0 and sha(p.read_bytes())==expected,'unexpected old deployment bytes')
store=Path('/var/lib/rmm-apt-transaction');code=Path('/usr/lib/rmm-apt-transaction')
require({p.name for p in code.iterdir()}=={'apt_transaction.py','hook'},'foreign provider files')
require({p.name for p in store.iterdir()}=={'archives','baselines','history','install-metadata.json'},'provider store already used or changed')
require(all(not any((store/name).iterdir()) for name in ('archives','baselines','history')),'nonempty provider history; do not move acceptance')
old_metadata=json.loads((store/'install-metadata.json').read_text())
require(old_metadata==dict(schema_version=1,provider_sha256=payload['old_tool_pins']['apt_transaction.py'],hook_sha256=payload['old_tool_pins']['hook']), 'unexpected old install metadata')
history=Path('/var/lib/rmm-bootstrap-history')
if not history.exists():history.mkdir(mode=0o700)
require(not history.is_symlink() and history.stat().st_uid==0 and history.stat().st_mode&0o777==0o700,'unsafe history parent')
backup=history/payload['build_id'];backup.mkdir(mode=0o700)
require(all(Path(path).stat().st_dev==backup.stat().st_dev for path in list(payload['expected_old'])+[str(store),str(code)]),'backup requires same filesystem')
report=dict(stage='validated',backups={},created=[],agent_before=before,helper_before=helper_before,completed=False)
def save():
temp=backup/'.report.tmp';fd=os.open(temp,os.O_WRONLY|os.O_CREAT|os.O_TRUNC|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'w') as stream:json.dump(report,stream,indent=2);stream.flush();os.fsync(stream.fileno())
os.replace(temp,backup/'report.json')
def retain(path):
path=Path(path);destination=backup/(str(path).lstrip('/').replace('/','__'))
require(not destination.exists(),'backup collision')
os.rename(path,destination);report['backups'][str(path)]=str(destination);save()
def create(path,data,mode,uid=0,gid=0):
path=Path(path);fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as stream:
stream.write(data);stream.flush();os.fsync(stream.fileno());os.fchown(stream.fileno(),uid,gid);os.fchmod(stream.fileno(),mode)
report['created'].append(str(path));save()
try:
save()
# Loader-check both newly staged binaries BEFORE replacing an old file.
staged={}
for name,encoded in payload['binaries'].items():
raw=base64.b64decode(encoded,validate=True);require(sha(raw)==payload['artifact_hashes'][name],'artifact changed in transit')
path=backup/name;create(path,raw,0o755)
subprocess.run(['/lib64/ld-linux-x86-64.so.2','--list',str(path)],capture_output=True,check=True);staged[name]=path
for path in payload['replace_files']:retain(path)
retain(store);retain(code)
for name,path in staged.items():
destination=Path('/usr/local/libexec')/name
require(not destination.exists(),'binary destination occupied after retain')
os.rename(path,destination);report['created'].append(str(destination));save()
# Correct restrictive umask only for reviewed public executable payloads.
subprocess.run(['/usr/local/libexec/rmm-privileged-helper','--install-apt-provider'],capture_output=True,check=True,umask=0o022)
report['stage']='provider_installed';save()
provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')
require(sha(provider.read_bytes())==payload['tool_pins']['apt_transaction.py'],'installed provider source mismatch')
command=['/usr/bin/python3','-I',str(provider)]
runtime=json.loads(subprocess.run(command+['inspect-runtime'],capture_output=True,check=True).stdout)
canonical=json.dumps(runtime,sort_keys=True,separators=(',',':'),ensure_ascii=True).encode()
config_bytes=subprocess.run(command+['print-config'],capture_output=True,check=True).stdout
report.update(runtime=runtime,runtime_sha256=sha(canonical),apt_config_sha256=sha(config_bytes))
general=dict(profile='general',binding=dict(binding,public_key_hex=''),signing_key_file='/etc/rmm/helper-signing-key-v1.json',
agent_uid=993,socket_gid=984,socket='/run/rmm-helper/control.sock',state='/var/lib/rmm-helper',
allowed_services=['rmm-mvp-test.service'],allow_root_scripts=False,allow_reboot=False,apt_provider=None)
apt=dict(general,profile='apt_only',socket='/run/rmm-apt-helper/helper.sock',allowed_services=[],apt_provider=dict(schema_version=1,
execution_profile='dev-test',allow_fixture_bootstrap=True,allowed_recipes=['rmm-apt-fixture-install-1.0','rmm-apt-fixture-update-1.1'],
runtime_sha256=report['runtime_sha256'],apt_config_sha256=report['apt_config_sha256']))
create('/etc/rmm/helper-general-v2.json',json.dumps(general).encode(),0o600)
create('/etc/rmm/apt-helper.json',json.dumps(apt).encode(),0o600)
create('/etc/systemd/system/rmm-helper-dev-test.service',payload['general_unit'].encode(),0o644)
create('/etc/systemd/system/rmm-agent.service',payload['agent_unit'].encode(),0o644)
for name,config_file,mode in [('general','/etc/rmm/helper-general-v2.json','--inspect-general-profile'),('apt','/etc/rmm/apt-helper.json','--inspect-apt-profile')]:
result=subprocess.run(['/usr/local/libexec/rmm-privileged-helper',mode],env=dict(os.environ,RMM_HELPER_CONFIG=config_file),capture_output=True,check=True)
proof=json.loads(result.stdout)
require(proof['identity']['agent_id']==binding['agent_id'] and proof['identity']['signing_key_id']==payload['key']['key_id'],'inspected helper identity differs')
create('/etc/rmm/'+name+'-helper-proof-v2.json',result.stdout,0o640,0,984)
report[name+'_pin']=proof
# No actual service content: one root-private zero-byte writable inode
# is exclusively reserved for a later explicitly UI-approved setup job.
create('/etc/systemd/system/rmm-mvp-test.service',b'',0o600)
dropin=b'[Service]\nEnvironment=RMM_JOB_BINDING_FILE=/etc/rmm/agent-binding.json\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper/control.sock\nEnvironment=RMM_HELPER_POLICY_FILE=/etc/rmm/general-helper-proof-v2.json\nEnvironment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\nEnvironment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm/apt-helper-proof-v2.json\nEnvironment=RMM_JOB_SIGNING_KEY_BOOTSTRAP=1\nEnvironment=RMM_EXTENDED_INVENTORY=1\n'
create('/etc/systemd/system/rmm-agent.service.d/20-runtime-v2.conf',dropin,0o644)
create('/etc/rmm/log-units-v2.json',b'["rmm-mvp-test.service"]\n',0o600,993,984)
create('/etc/systemd/system/rmm-agent.service.d/21-log-units-v2.conf',b'[Service]\nEnvironment=RMM_LOG_UNITS_FILE=/etc/rmm/log-units-v2.json\n',0o644)
for unit in ('rmm-agent.service','rmm-helper-dev-test.service','rmm-apt-helper.service'):
subprocess.run(['systemd-analyze','verify','/etc/systemd/system/'+unit],capture_output=True,check=True)
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True)
report['agent_after']=manifest(state);report['helper_after']=manifest(helper_state)
require(before==report['agent_after'] and helper_before==report['helper_after'],'installation changed retained identity or SQLite bytes')
report.update(completed=True,stage='installed_not_started',journals_and_identity_unchanged=True)
except Exception as error:report['error']=sanitized_error(error);raise
finally:save();lock.close()
return report
def remote_probe():
os.umask(0o077)
config=Path('/etc/rmm/apt-helper.json');require(config.is_file(),'missing reviewed APT config')
before=manifest(Path('/var/lib/rmm-agent'))
report=dict(stage='preseal',package_effects=False,service_business_effects=False)
try:
# Explicit administrative fixture bootstrap: fixed known archives and
# private seals only. Never invoke apt/dpkg install or acceptance here.
command=['/usr/bin/python3','-I','/usr/lib/rmm-apt-transaction/apt_transaction.py']
preparation=subprocess.run(command+['provision-fixture-store'],capture_output=True,check=True)
report['fixture_preparation']=json.loads(preparation.stdout)
report['stage']='native_dedicated_unit_probe'
result=subprocess.run(['systemctl','start','rmm-apt-helper.service'],capture_output=True)
report['start_exit']=result.returncode
report['unit_state']=subprocess.run(['systemctl','show','rmm-apt-helper.service','--property=ActiveState,SubState,Result,ExecMainStatus'],capture_output=True,text=True,check=True).stdout.splitlines()
path=Path('/var/lib/rmm-apt-transaction/native-readiness.json')
report['native_readiness_present']=path.is_file()
if path.is_file():report['native_readiness']=json.loads(path.read_text())
require(result.returncode==0 and path.is_file(),'native probe failed; no job may proceed')
with closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
report['root_consumed_count']=db.execute('SELECT count(*) FROM consumed').fetchone()[0]
require(report['root_consumed_count']==0,'unexpected root consumption; do not stop/retry')
report['agent_state_unchanged']=manifest(Path('/var/lib/rmm-agent'))==before
require(report['agent_state_unchanged'],'probe changed agent state')
report['completed']=True
except Exception as error:report['error']=sanitized_error(error);raise
finally:
with closing(sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True)) as db:
unsettled=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0]
if unsettled==0:
subprocess.run(['systemctl','stop','rmm-apt-helper.service'],capture_output=True,check=True);report['helper_stopped']=True
path=Path('/etc/rmm/runtime-v2-probe-report.json')
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'w') as stream:json.dump(report,stream,indent=2);stream.flush();os.fsync(stream.fileno())
return report
def program(invocation):
common='\n\n'.join(inspect.getsource(function) for function in (previous.manifest,previous.sanitized_error,sha,remote_upgrade,remote_probe))
return ('import base64,json,os,subprocess,sqlite3,uuid\nfrom pathlib import Path\nfrom contextlib import closing\nimport hashlib\n'
'def require(value,message):\n if not value: raise RuntimeError(message)\n'+common+'\ntry:\n print(json.dumps({"result":'+invocation+'}))\nexcept Exception as error:\n print(json.dumps({"error":sanitized_error(error)}))\n')
def remote(code):
with previous.connection() as (ssh,target,env):
result=subprocess.run(ssh+[target,'python3 -'],env=env,input=code.encode(),capture_output=True,timeout=180)
require(result.returncode==0,'SSH/bootstrap transport failed; private stderr withheld')
response=json.loads(result.stdout)
if 'error' in response:
print('RUNTIME_UPGRADE_ERROR='+json.dumps(response['error']));raise RuntimeError('remote phase failed; inspect private retained report')
return response['result']
def install():
sys.path.insert(0,str(ROOT/'tests'))
from backend_contract import request
from local_integration import STATE,A,sql
current=json.loads((BASE/'current.json').read_text());run=BASE/current['build_id'];receipt=json.loads((run/'build.json').read_text())
require(receipt['sources']==bound_sources(ROOT),'bound source changed since build')
agent_old=json.loads((STATE/'vm-jobs/bootstrap.json').read_text());root_old=json.loads((STATE/'vm-root/result.json').read_text())
agent=str(uuid.UUID(agent_old['agent_id']))
require(sql(f"SELECT count(*) FROM jobs WHERE agent_id='{agent}' AND state NOT IN ('succeeded','failed','expired','cancelled');")=='0','unresolved server jobs')
operator=(STATE/'operator-a.token').read_text().strip()
status,_,device=request(f'/api/v1/tenants/{A}/devices/{agent_old["asset_id"]}',token=operator)
require(status==200 and device['agent_id']==agent,'preserved server identity missing')
status,_,key=request('/api/v1/job-signing-key',token=operator)
require(status==200 and key['algorithm']=='Ed25519' and key['schema_version']==1 and sha(bytes.fromhex(key['public_key_hex']))==key['key_id'],'authenticated signing key invalid')
binding={field:device[field] for field in ('asset_id','agent_id','enrollment_generation','assignment_version')}
binding.update(tenant_id=A,public_key_hex=key['public_key_hex'])
general_old=(ROOT/'deploy/linux/rmm-helper-dev-test.service').read_bytes()
general=general_old.decode().replace('/etc/rmm/helper-dev-test-v1.json','/etc/rmm/helper-general-v2.json')
general=general.replace('SystemCallFilter=~@mount @reboot @swap @raw-io @module @clock @debug',
'SystemCallFilter=~@mount @reboot @swap @raw-io @module @clock @debug\nSystemCallFilter=adjtimex')
# General remains without SYS_ADMIN/MKNOD. Only a precreated empty target
# inode (not all /etc) may be written by a later authorized fixed setup job.
general=general.replace('ProtectSystem=strict','ProtectSystem=strict\nReadWritePaths=/etc/systemd/system/rmm-mvp-test.service')
old_agent_unit=(ROOT/'devstate/vm-jobs/docker-agent-unit-old.txt')
# Old unit was the exact pre-runtime unit deployed by the earlier bootstrap.
original_agent=(ROOT/'deploy/linux/rmm-agent.service').read_text().replace('Environment=RMM_TRANSPORT=auto\n','').replace('Environment=RMM_EXTENDED_INVENTORY=1\n','')
original_agent=original_agent.replace('Environment=RMM_CA_FILE=/etc/rmm/ca.pem\n','Environment=RMM_CA_FILE=/etc/rmm/ca.pem\nEnvironment=RMM_JOB_BINDING_FILE=/etc/rmm/agent-binding.json\n')
expected={'/usr/local/libexec/rmm-agent':agent_old['artifact_sha256'],
'/usr/local/libexec/rmm-privileged-helper':root_old['helper_sha256'],
'/usr/lib/rmm-apt-transaction/apt_transaction.py':root_old['tool_pins']['apt_transaction.py'],
'/usr/lib/rmm-apt-transaction/hook':root_old['tool_pins']['hook'],
'/etc/systemd/system/rmm-helper-dev-test.service':sha(general_old),
'/etc/systemd/system/rmm-agent.service':sha(original_agent.encode())}
payload=dict(build_id=run.name,binding=binding,key=key,expected_old=expected,
old_tool_pins=root_old['tool_pins'],replace_files=['/usr/local/libexec/rmm-agent','/usr/local/libexec/rmm-privileged-helper',
'/etc/systemd/system/rmm-helper-dev-test.service','/etc/systemd/system/rmm-agent.service'],
binaries={name:base64.b64encode((run/name).read_bytes()).decode() for name in ('rmm-agent','rmm-privileged-helper')},
artifact_hashes={name:value['sha256'] for name,value in receipt['artifacts'].items()},
tool_pins={name:receipt['sources']['tools/apt-transaction/'+name] for name in ('apt_transaction.py','hook')},
general_unit=general,agent_unit=(ROOT/'deploy/linux/rmm-agent.service').read_text())
result=remote(program('remote_upgrade('+repr(payload)+')'))
(run/'install.json').write_text(json.dumps(result,indent=2))
print('PASS versioned runtime upgrade; identities/journals byte-preserved; services not started')
if __name__=='__main__':
os.umask(0o077)
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('mode',choices=('build','install','probe'))
args=parser.parse_args()
try:
if args.mode=='build':build()
elif args.mode=='install':install()
else:
current=json.loads((BASE/'current.json').read_text());run=BASE/current['build_id']
result=remote(program('remote_probe()'));(run/'probe.json').write_text(json.dumps(result,indent=2))
print('PASS actual dedicated-profile native probe; no package/RMM effects; helper stopped')
except Exception as error:
print('RUNTIME_PHASE_INCOMPLETE='+json.dumps(previous.sanitized_error(error)));raise SystemExit(2) from None