feat: qualify bound configure-only recovery runtime on cloud VM
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 18:07:46 +01:00
parent b24314da9e
commit 2230235dfb
21 changed files with 1749 additions and 80 deletions
+12 -4
View File
@@ -12,14 +12,15 @@ function readPrivate(file) {
return JSON.parse(fs.readFileSync(file, 'utf8'));
}
function checkAction(p, expected) {
const native = ['apt_install_retry', 'apt_native_update'].includes(expected.case);
const version = expected.case === 'apt_install_retry' ? '1.0' : '1.1';
const recovery = expected.case === 'apt_configure_recovery';
const native = recovery || ['apt_install_retry', 'apt_native_update'].includes(expected.case);
const version = recovery || expected.case === 'apt_install_retry' ? '1.0' : '1.1';
const recipe = version === '1.0' ? 'rmm-apt-fixture-install-1.0' : 'rmm-apt-fixture-update-1.1';
const action = p.action; const m = action.apt_manifest; const tx = m.resolved_apt_transaction;
assert(action.action === (native ? 'apt_native_acceptance' : expected.case));
assert(p.execution_identity === 'root' && p.max_runtime_seconds === 300 && p.max_output_bytes === 65536);
assert(m.type === 'rmm.apt.helper-action' && m.execution_profile === 'dev-test' && m.action === (native ? 'apt-native-acceptance' : 'apt-execute'));
assert(m.schema_version === (expected.case === 'apt_install_retry' ? 3 : 2));
assert(m.schema_version === (recovery ? 4 : expected.case === 'apt_install_retry' ? 3 : 2));
assert(tx.recipe_id === recipe && tx.recipe_version === '1' && tx.no_reboot === true && tx.packages.length === 1);
const pkg = tx.packages[0];
const sha = version === '1.0' ? '74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867' : 'ef67c9fe0ba1443fcfe465768e1204763063c369a2a1da04af6027d8a9c00491';
@@ -27,6 +28,13 @@ function checkAction(p, expected) {
assert(crypto.createHash('sha256').update(Buffer.from(m.transaction_bytes)).digest('hex') === m.transaction_sha256);
assert(equal(JSON.parse(Buffer.from(m.transaction_bytes)), tx));
assert(equal(m, expected.request.parameters.apt_manifest));
if (recovery) {
assert(tx.schema_version === 2 && tx.operation === 'configure_only' && pkg.old_version === '1.0');
assert(m.native_retry === undefined && m.native_recovery.schema_version === 1);
assert(m.native_recovery.previous_grant_id === '31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0');
assert(m.native_recovery.previous_attempt_id !== p.attempt_id);
assert(m.native_recovery.partial_proof_sha256 === 'dc3bf3870ea77c12f9291bac712c53ae72194eaec980a8646155a7fb5c0dbe21');
} else assert(tx.schema_version === 1 && tx.operation === undefined);
if (expected.case === 'apt_install_retry') {
assert(m.native_retry.schema_version === 1 && m.native_retry.bootstrap_marker_sha256 === 'ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053');
assert(m.native_retry.previous_attempt_id !== p.attempt_id);
@@ -34,7 +42,7 @@ function checkAction(p, expected) {
}
(async () => {
const file = path.resolve(process.argv[2]); const expected = readPrivate(file);
assert(expected.profile === 'dev/test' && ['apt_install_retry', 'apt_native_update', 'ensure_package', 'apply_updates'].includes(expected.case) && expected.expected_principal === 'simulated-testprincipal-a');
assert(expected.profile === 'dev/test' && ['apt_configure_recovery', 'apt_install_retry', 'apt_native_update', 'ensure_package', 'apply_updates'].includes(expected.case) && expected.expected_principal === 'simulated-testprincipal-a');
const login = readPrivate(path.join(root, 'credentials/server-browser-login.json'));
const browsers = path.join(root, 'web/node_modules/.cache/playwright-browsers');
const version = fs.readdirSync(browsers).find(x => /^chromium-/.test(x));
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env python3
"""Exact configure-only recovery followed by separate fixture upgrade/noops."""
import argparse
import datetime as dt
import hashlib
import json
import os
import time
import uuid
import cloud_apt_retry as prior
import cloud_root_evidence
import cloud_root_policy
cloud=prior.cloud
require=cloud.require
BASE=cloud.ROOT/'devstate/cloud-apt-recovery'
STATE=BASE/'cases'
CASES=('apt_configure_recovery','apt_native_update','ensure_package','apply_updates')
def recovery_request(device,target):
m=cloud.contract;data=prior.adapters.planning_from_device(device,recovery=True)
closed=m.protected_json(BASE/'readonly-central-result.json')['result']
reference=dict(schema_version=1,previous_grant_id=closed['result']['grant_id'],previous_attempt_id=closed['attempt_id'],
partial_proof_sha256=closed['result']['evidence']['apt']['reconciliation_proof_sha256'])
require(closed['state']=='failed' and closed['result']['error_code']=='APT_PARTIAL_UNCONFIGURED_VERIFIED' and
data['native_recovery']==reference,'exact previously closed partial reference')
pin=prior.adapters.PINS['1.0'];start=m.now();receipt=data['prefetch'][pin]
require(receipt['schema_version']==1 and receipt['sha256']==pin and receipt['size']==20672 and
receipt['sealed_at_ms']<=int(start.timestamp()*1000),'actual pregrant archive receipt')
tx=dict(schema_version=2,operation='configure_only',recipe_id='rmm-apt-fixture-install-1.0',recipe_version='1',
platform='ubuntu-24.04-amd64',resolver_config_version='rmm-apt-local-v1',no_reboot=True,
status_fingerprint=data['status_fingerprint'],baseline_sha256=data['baseline_sha256'],packages=[
dict(name='rmm-mvp-test',architecture='all',old_version='1.0',new_version='1.0',origin='rmm-test',sha256=pin,
size=20672,archive_path='/var/lib/rmm-apt-transaction/archives/'+pin+'.deb')])
raw=prior.adapters.canonical(tx)
manifest=dict(type='rmm.apt.helper-action',schema_version=4,action='apt-native-acceptance',execution_profile='dev-test',
transaction_sha256=hashlib.sha256(raw).hexdigest(),resolved_apt_transaction=tx,transaction_bytes=list(raw),
native_recovery=reference,test_result_id='configure-recovery-'+uuid.uuid4().hex,
**{k:data[k] for k in ('provider_sha256','hook_sha256','apt_config_sha256','runtime_sha256')})
m.validate_apt_manifest(manifest)
return dict(tenant_id=target['tenant_id'],action='apt_native_acceptance',targets=dict(asset_ids=[target['asset_id']]),
parameters=dict(execution_identity='root',timeout_seconds=300,max_output_bytes=65536,
recipe_id=tx['recipe_id'],recipe_version='1',apt_manifest=manifest),
schedule=m.schedule(start,start+dt.timedelta(minutes=15)),rollout=dict(max_parallel=1,halt_after_failures=1),
reason='Configure ONLY the exactly proven unpacked fixture1.0; new attempt and separate recovery record; no unpack, pending packages, reinstall or reboot')
def enable():
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
ready=m.protected_json(BASE/'configure-runtime.json')
require(ready['completed'] and ready['readiness_only'] and ready['preserved_state'] and not ready['native_acceptance'],
'qualified native probe, no invented package acceptance')
target=prior.prior.identity();pin=ready['apt_pin'];identity=pin['identity']
require(all(identity[k]==target[k] for k in target),'same permanent Cloud VM')
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=True,
agent_id=target['agent_id'],binary_sha256=ready['helper_sha256']))
m.write_private(BASE/'recovery-endpoint-enabled.json',endpoint)
peers=cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,target)
require(peers['ledger']['faults']==0 and peers['apt']['provider_stack_sha256']==pin['provider_stack_sha256'],
'actual fixed helper peer and stack')
scope={k:identity[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')}
scope.update(principals=['simulated-testprincipal-a'],helper_policy_sha256=pin['provider_stack_sha256'],
acceptance_evidence_id='cloud-configure-recovery-probe:'+ready['readiness_sha256'],allowed_services=[],allow_root_scripts=False,allow_reboot=False,
apt=dict(allow_native_acceptance=True,allow_execute=True,runtime_sha256=ready['runtime_sha256'],
apt_config_sha256=ready['apt_config_sha256'],provider_sha256=ready['provider_sha256'],
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
policy=dict(schema_version=1,profile='development',expires_at=(m.now()+dt.timedelta(hours=1)).isoformat(),scopes=[scope])
old=m.protected_json(cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
record=cloud.bootstrap.rpc('server.conf',prior.replace_policy,dict(id=str(uuid.uuid4()),previous_sha256=old['policy_sha256'],policy=policy))
m.write_private(BASE/'recovery-policy.json',dict(record,policy=policy))
print('Fixed APT runtime endpoint and expiring limits enabled; no job or approval created')
def accept(case):
m=cloud.contract;target=prior.prior.identity()
require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
# A maintenance restart only refreshes passive inventory from the preserved
# agent; it never issues a package diagnostic or executes a job.
deadline=time.monotonic()+90
while True:
device=cloud.Backend().call('inventory.get_device',{k:target[k] for k in ('tenant_id','asset_id')})
try:
data=prior.adapters.planning_from_device(device,recovery=case=='apt_configure_recovery')
break
except m.GuardError:
require(time.monotonic()<deadline,'fresh stored exact planning unavailable; no job submitted');time.sleep(3)
request=recovery_request(device,target) if case=='apt_configure_recovery' else None
prior.accept(case,request=request,folder_root=STATE)
def close():
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
record=m.protected_json(BASE/'recovery-policy.json')
closed=cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=record['policy_sha256'],backup=record['backup']))
m.write_private(BASE/'recovery-policy-closed.json',closed)
ready=m.protected_json(BASE/'configure-runtime.json')
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=False,
agent_id=prior.prior.identity()['agent_id'],binary_sha256=ready['helper_sha256']))
m.write_private(BASE/'recovery-endpoint-disabled.json',endpoint)
print('APT test limits closed and endpoint stopped; original histories retained')
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('action',choices=('enable','accept','finish','close'));p.add_argument('--case',choices=CASES)
args=p.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
if args.action in ('accept','finish'):
require(args.case is not None,'one fixed case required')
if args.action=='accept':accept(args.case)
else:prior.finish(args.case,folder_root=STATE)
else:(enable if args.action=='enable' else close)()
if __name__=='__main__':
try:main()
except Exception as error:
message=str(error)
print(message if message.startswith('remote bootstrap incomplete: ') else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+332
View File
@@ -0,0 +1,332 @@
#!/usr/bin/env python3
"""Versioned Cloud runtime upgrade for the one signed, closed APT partial.
No package recovery, approval or enrollment. Original active hook, once markers,
signed histories and SQLite state survive the maintenance/probe byte-for-byte.
The APT endpoint remains absent; only the copied-database native probe runs.
"""
import base64
import argparse
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import sys
import uuid
import cloud_apt_reconcile_upgrade as previous
cloud = previous.cloud
require = cloud.require
STATE = previous.STATE
PARTIAL_GRANT = '31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0'
PARTIAL_PROOF = 'dc3bf3870ea77c12f9291bac712c53ae72194eaec980a8646155a7fb5c0dbe21'
PINS = dict(helper='ab7bbc87c75f68c1aa3d35941db9a2ece8d398f92a67e655b1d420bbd3287bd9',
agent='44ae91151ff08a186ce4dff587697eb604cb7c0a22f0c9d618724a9ba8a67234',
provider='5ff260f75cb2959678f6be729d3a806a4ea29febd5ae4003317fed41afc25327')
def closed_policy(payload):
import datetime as dt
path = Path('/opt/rust-discovery/devstate/root-policy/policy.json')
require(not path.is_symlink() and path.stat().st_uid == 10001 and path.stat().st_mode & 0o777 == 0o600, 'safe Root policy required')
raw = path.read_bytes(); policy = json.loads(raw)
require(policy['schema_version'] == 1 and policy['profile'] == 'development' and
dt.datetime.fromisoformat(policy['expires_at']) <= dt.datetime.now(dt.timezone.utc), 'Root policy must stay closed')
return dict(closed=True, policy_sha256=hashlib.sha256(raw).hexdigest())
def installed_metadata(payload):
pins = {'/usr/local/libexec/rmm-privileged-helper': payload['helper'],
'/usr/local/libexec/rmm-agent-remote': payload['agent'],
'/usr/lib/rmm-apt-transaction/apt_transaction.py': payload['provider']}
names = list(pins) + ['/etc/rmm/apt-helper.json', '/etc/rmm-remote/apt-helper-proof.json',
'/var/lib/rmm-apt-transaction/install-metadata.json']
files = {}
for name in names:
p = Path(name)
require(not p.is_symlink() and p.is_file() and p.stat().st_uid == 0 and p.stat().st_mode & 0o022 == 0, 'safe installed file required')
files[name] = hashlib.sha256(p.read_bytes()).hexdigest()
require(all(files[name] == pin for name, pin in pins.items()), 'reviewed readonly runtime required')
return dict(files=files)
def upgrade(payload):
import ast
import fcntl
import sqlite3
import time
import traceback
os.umask(0o077)
base = Path('/var/lib/rmm-apt-transaction'); state = Path('/var/lib/rmm-agent-remote')
ledger = Path('/var/lib/rmm-helper/consumption.sqlite')
helper = Path('/usr/local/libexec/rmm-privileged-helper'); agent = Path('/usr/local/libexec/rmm-agent-remote')
provider = Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')
config = Path('/etc/rmm/apt-helper.json'); proof_file = Path('/etc/rmm-remote/apt-helper-proof.json')
metadata = base/'install-metadata.json'; readiness_file = base/'native-readiness.json'
dropin = Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
reduced = ('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
def run(args, **kwargs):
return subprocess.run(args, capture_output=True, check=True, timeout=90, **kwargs).stdout
def sha(raw): return hashlib.sha256(raw).hexdigest()
def canonical(value): return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()
def fsync_dir(path):
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try: os.fsync(fd)
finally: os.close(fd)
def create(path, raw, mode=0o600, gid=0):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw); stream.flush(); os.fchown(stream.fileno(), 0, gid)
os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno())
fsync_dir(path.parent)
def manifest(path):
require(not path.is_symlink(), 'preserved directory symlink')
if not path.exists(): return {'__absent__':True}
require(path.is_dir(), 'preserved directory required')
result = {}
for p in [path, *sorted(path.rglob('*'))]:
s = p.lstat(); require(not p.is_symlink() and (p.is_dir() or p.is_file()), 'preserved regular state required')
result[str(p.relative_to(path))] = dict(dev=s.st_dev, inode=s.st_ino, mode=s.st_mode,
uid=s.st_uid, gid=s.st_gid, sha256=sha(p.read_bytes()) if p.is_file() else None)
return result
def guard():
require(payload['partial_grant'] == '31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0' and
payload['partial_proof_sha256'] == 'dc3bf3870ea77c12f9291bac712c53ae72194eaec980a8646155a7fb5c0dbe21', 'one reviewed closed partial only')
with sqlite3.connect(ledger.as_uri()+'?mode=ro', uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'Root ledger integrity')
require({r[0] for r in db.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants']) and
db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0 and
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'exact known closed Root attempts')
db.row_factory = sqlite3.Row
rows = db.execute('SELECT * FROM apt_partial_reconciliations').fetchall()
require(len(rows) == 1, 'one immutable partial proof required'); row = dict(rows[0])
expected = payload['proof_row']
for key in ('grant_id', 'original_request_sha256', 'previous_outcome_sha256', 'proof_sha256', 'checked_at_ms'):
require(row[key] == expected[key], 'immutable partial binding drift')
for key in ('previous_outcome', 'proof', 'classified_outcome'):
require(sha(row[key]) == expected[key]['sha256'] and len(row[key]) == expected[key]['size'], 'immutable partial bytes drift')
require(row['grant_id'] == payload['partial_grant'] and sha(row['proof']) == payload['partial_proof_sha256'], 'closed proof hash required')
proof = json.loads(row['proof']); result = db.execute('SELECT c.result,e.outcome,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE grant_id=?', (payload['partial_grant'],)).fetchone()
classified = json.loads(row['classified_outcome'])
require(json.loads(result['result']) == json.loads(result['outcome']) == classified and
classified['state'] == 'failed' and classified['error_code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED', 'normal signed RootSync closure required')
saved = json.loads(result['start'])
# save_start nests Request through serde_json::Value (sorted keys),
# whereas the immutable proof hashes serde's Request struct codec.
request_fields = ('job_bytes','permit_bytes','signature','plan_hash','authorization_id')
require(set(saved['request']) == set(request_fields), 'original Request fields required')
request_bytes = json.dumps({key:saved['request'][key] for key in request_fields},
separators=(',', ':'), ensure_ascii=False).encode()
require(sha(request_bytes) == row['original_request_sha256'], 'original signed request hash required')
require(proof['grant_id'] == payload['partial_grant'] and proof['attempt_id'] == payload['partial_attempt'] and
proof['snapshot']['rmm-mvp-test:all'] == {'version':'1.0','status':'install ok unpacked'}, 'exact partial fixture required')
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro', uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0 and
db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no unresolved or faulted agent attempt')
require(Path('/usr/share/rmm-mvp-test/version').read_bytes() == b'1.0\n', 'existing unpacked fixture payload required')
require(not list(base.glob('acceptance-*.json')) and not list(base.glob('pending-acceptance-*.json')) and
not list(base.glob('recovery-*.json')) and not list(base.glob('pending-recovery-*.json')), 'no existing acceptance or recovery attempt')
return proof
def bound_partial(proof):
keys = 'transaction_sha256 job_payload_sha256 effect_armed_ledger_id grant_id job_id attempt_id authorization_id plan_sha256'.split()
value = json.loads(run(['/usr/bin/python3','-I',str(provider),'verify-partial'], input=canonical({k:proof[k] for k in keys})))
require(value['effect_state'] == 'known_failed' and value['code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED' and
{k:v for k,v in value['proof'].items() if k != 'checked_at_ms'} ==
{k:v for k,v in proof.items() if k != 'checked_at_ms'}, 'fresh original-bound partial state required')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'permanent Cloud identity required')
require(not dropin.is_symlink() and dropin.read_text() == reduced, 'APT endpoint must remain absent')
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT helper must be stopped')
for name, pin in payload['old_files'].items():
p = Path(name); require(not p.is_symlink() and p.stat().st_uid == 0 and p.stat().st_mode & 0o022 == 0 and sha(p.read_bytes()) == pin, 'installed file drift')
require(Path('/etc/systemd/system/rmm-apt-helper.service').read_text() == payload['unit'], 'exact reviewed APT service required')
markers = {base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json':'ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053',
base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json':'a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3'}
require({p.name for p in base.glob('bootstrap-*.json')} == {p.name for p in markers} and
all(sha(p.read_bytes()) == pin for p,pin in markers.items()), 'original/retry tombstones required')
partial = guard(); bound_partial(partial)
raw = {name:base64.b64decode(payload[name], validate=True) for name in ('helper','agent','provider')}
pins = {'helper':'ab7bbc87c75f68c1aa3d35941db9a2ece8d398f92a67e655b1d420bbd3287bd9',
'agent':'44ae91151ff08a186ce4dff587697eb604cb7c0a22f0c9d618724a9ba8a67234',
'provider':'5ff260f75cb2959678f6be729d3a806a4ea29febd5ae4003317fed41afc25327'}
require(all(sha(raw[name]) == pin == payload[name+'_sha256'] for name,pin in pins.items()), 'qualified exact recovery runtime required')
for name in ('helper','agent'):
require(raw[name][:6] == b'\x7fELF\x02\x01' and raw[name][18:20] == b'\x3e\x00', 'Linux x64 binary required')
ast.parse(raw['provider'])
if payload.get('preflight_only') is True:
return dict(stage='preflight_validated', completed=False, preflight_only=True,
package_effects=False, maintenance_mutations=False, expected_grants=len(payload['expected_grants']),
partial_proof_sha256=payload['partial_proof_sha256'], apt_endpoint_enabled=False)
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-configure-recovery-'+payload['id']); backup.mkdir(mode=0o700)
report = dict(stage='validated', completed=False, backup=str(backup), package_effects=False,
apt_endpoint_enabled=False, partial_proof_sha256=payload['partial_proof_sha256'], backups={})
def save():
next_path = backup/'.report-next.json'; create(next_path,json.dumps(report,indent=2).encode())
os.replace(next_path,backup/'report.json'); fsync_dir(backup)
def replace(path, staged=None):
target = backup/str(path).lstrip('/').replace('/','__')
require(not path.is_symlink() and path.stat().st_dev == backup.stat().st_dev and not target.exists(), 'exclusive same-filesystem versioned replacement')
os.rename(path,target); fsync_dir(path.parent); fsync_dir(backup)
report['backups'][str(path)] = str(target); save()
if staged is not None:
os.rename(staged,path); fsync_dir(path.parent); fsync_dir(backup)
lock = None
try:
save()
for name in ('helper','agent','provider'): create(backup/(name+'-next'), raw[name], 0o755)
for name in ('helper','agent'): run(['/lib64/ld-linux-x86-64.so.2','--list',str(backup/(name+'-next'))])
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
partial = guard(); bound_partial(partial)
preserved = {str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent)}
preserved_base = manifest(base)
witnesses = {}
for p in (Path('/var/lib/dpkg/status'),Path('/usr/share/rmm-mvp-test/version'),dropin,
Path('/etc/systemd/system/rmm-apt-helper.service'),Path('/usr/lib/rmm-apt-transaction/hook'),
Path('/etc/rmm-remote/general-helper-proof.json'),Path('/etc/rmm-remote/helper-general.json')):
s = p.stat(); witnesses[str(p)] = (s.st_dev,s.st_ino,sha(p.read_bytes()))
general_bytes = Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
general_config = Path('/etc/rmm-remote/helper-general.json').read_bytes()
old_metadata = json.loads(metadata.read_text()); old_config = json.loads(config.read_text())
require(old_metadata['schema_version'] == 2 and old_metadata['provider_sha256'] == sha(provider.read_bytes()) and
old_metadata['hook_sha256'] == sha(Path('/usr/lib/rmm-apt-transaction/hook').read_bytes()) and
old_metadata['unit_sha256'] == sha(payload['unit'].encode()), 'old install metadata required')
for path,name in ((helper,'helper'),(agent,'agent'),(provider,'provider')): replace(path,backup/(name+'-next'))
create(backup/'metadata-next',json.dumps(dict(old_metadata,provider_sha256=sha(raw['provider']))).encode()); replace(metadata,backup/'metadata-next')
command = ['/usr/bin/python3','-I',str(provider)]
config_bytes = run(command+['print-config']); config_sha = sha(config_bytes)
require(config_sha == payload['reviewed_config_sha256'] and all(line in config_bytes for line in (
b'DPkg::ConfigurePending "false";', b'DPkg::TriggersPending "false";', b'dpkg::selection::remove::approved "false";')), 'reviewed exact configure/no-pending settings')
runtime = json.loads(run(command+['inspect-runtime'])); runtime_sha = sha(canonical(runtime))
old_config['apt_provider'].update(runtime_sha256=runtime_sha,apt_config_sha256=config_sha)
create(backup/'config-next',json.dumps(old_config).encode()); replace(config,backup/'config-next')
pin = json.loads(run([str(helper),'--inspect-apt-profile'],env=dict(os.environ,RMM_HELPER_CONFIG=str(config))))
require(pin['identity'] == json.loads(general_bytes)['identity'], 'same helper identity required')
create(backup/'proof-next',json.dumps(pin).encode(),0o640,984); replace(proof_file,backup/'proof-next')
general = json.loads(run([str(helper),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(general == json.loads(general_bytes) and Path('/etc/rmm-remote/helper-general.json').read_bytes() == general_config, 'General helper policy unchanged')
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'identity/journal/Root ledger bytes or inode changed')
def preserved_original_base():
current = manifest(base)
require(all(current.get(name) == value for name,value in preserved_base.items() if name not in ('install-metadata.json','native-readiness.json')), 'original APT state/archive/hook/probe changed')
for name,value in witnesses.items():
p=Path(name);s=p.stat();require((s.st_dev,s.st_ino,sha(p.read_bytes())) == value, 'original package/control file bytes or inode changed')
return current
preserved_original_base(); bound_partial(guard())
report.update(stage='installed_preserved',preserved_state=True,helper_sha256=pins['helper'],agent_sha256=pins['agent'],
provider_sha256=pins['provider'],runtime_sha256=runtime_sha,apt_config_sha256=config_sha,apt_pin=pin,general_pin=general)
save()
# No client can reach this endpoint; it exists only for the service's
# ExecStartPre copied-database probe, then is stopped before restart.
run(['systemd-analyze','verify','/etc/systemd/system/rmm-apt-helper.service'])
replace(readiness_file)
# Daemon startup itself takes executor.lock to open the existing DB.
# Installation is complete; agent/General remain stopped during probe.
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
run(['systemctl','start','rmm-apt-helper.service'])
for _ in range(3):
status = run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','-p','SubState','-p','NRestarts']).decode()
require('ActiveState=active\n' in status and 'SubState=running\n' in status and 'NRestarts=0\n' in status, 'stable native probe helper required')
time.sleep(1)
run(['systemctl','stop','rmm-apt-helper.service'])
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
ready = json.loads(readiness_file.read_text())
require(ready['runtime'] == runtime and ready['readiness_only'] is True and ready['native_acceptance'] is False and
ready['host_status_sha256'] == sha(Path('/var/lib/dpkg/status').read_bytes()), 'exact new copied-database probe required')
workspace = base/'probes'/str(uuid.UUID(ready['probe_id']))
require((workspace/'dpkg/status').read_bytes() == Path('/var/lib/dpkg/status').read_bytes() and
(workspace/'apt.conf').read_bytes().count(b'dpkg::selection::remove::approved "false";') == 1,
'partial copied status and configure fix required')
current = preserved_original_base()
require(all(name in preserved_base or name.startswith('probes/'+ready['probe_id']+'/') or name == 'probes/'+ready['probe_id']
for name in current), 'only generated readiness probe metadata may be added')
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'journal/ledger bytes or inode changed during probe')
bound_partial(guard())
require(dropin.read_text() == reduced and Path('/etc/rmm-remote/general-helper-proof.json').read_bytes() == general_bytes, 'endpoint/General proof unchanged')
report.update(stage='native_probe_preserved',readiness_only=True,native_acceptance=False,
journals_preserved_before_restart=True,readiness_sha256=sha(readiness_file.read_bytes()),probe_id=ready['probe_id'])
save()
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT endpoint must remain stopped')
report.update(stage='qualified_runtime_started',completed=True);save()
except Exception as error:
report.update(error_class=type(error).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
save()
finally:
if lock is not None: fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
subprocess.run(['systemctl','stop','rmm-apt-helper.service'],capture_output=True,timeout=30)
return report
def build_payload():
m = cloud.contract
checkpoint = m.protected_json(previous.apt.STATE/'apt_install_retry/checkpoint.json')
readonly = m.protected_json(STATE/'readonly-result-first.json')
require(readonly['unresolved'] == readonly['faults'] == 0 and readonly['original_result']['state'] == 'failed' and
readonly['original_result']['error_code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED', 'actual closed partial evidence required')
require(len(readonly['proof']) == 1 and readonly['proof'][0]['grant_id'] == PARTIAL_GRANT and readonly['proof'][0]['proof_sha256'] == PARTIAL_PROOF, 'one reviewed actual partial proof')
folders = [cloud.ROOT/'devstate/cloud-root/cases'/n for n in ('service_setup','service_start','service_stop','apt_native_install')]
folders += [previous.apt.prior.STATE/n for n in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
query = cloud.ROOT/'devstate/cloud-root/cases/query_logs/checkpoint.json'
if query.exists():
require(m.protected_json(query)['verified'] is True, 'actual completed query_logs checkpoint required')
folders.append(query.parent)
expected = [str(uuid.UUID(m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'])) for p in folders] + [PARTIAL_GRANT]
require(len(expected) == len(set(expected)) and len(expected) in (9,10), 'exact known checkpoint grants required')
raw = {name:(STATE/filename).read_bytes() for name,filename in (
('helper','helper-configure-recovery'),('agent','agent-configure-recovery'),('provider','provider-configure-recovery.py'))}
require(all(hashlib.sha256(raw[name]).hexdigest() == pin for name,pin in PINS.items()), 'qualified artifacts changed')
require(raw['provider'] == (cloud.ROOT/'tools/apt-transaction/apt_transaction.py').read_bytes(), 'source/qualified provider mismatch')
spec = importlib.util.spec_from_file_location('reviewed_recovery_provider',cloud.ROOT/'tools/apt-transaction/apt_transaction.py')
module = importlib.util.module_from_spec(spec);sys.modules[spec.name]=module;spec.loader.exec_module(module)
return dict(id=str(uuid.uuid4()),agent_id=previous.apt.prior.identity()['agent_id'],expected_grants=expected,
partial_grant=PARTIAL_GRANT,partial_attempt=checkpoint['plan']['jobs'][0]['attempt_id'],partial_proof_sha256=PARTIAL_PROOF,
proof_row=readonly['proof'][0],unit=(cloud.ROOT/'deploy/linux/rmm-apt-helper.service').read_text(),
reviewed_config_sha256=hashlib.sha256(module.apt_config()).hexdigest(),
**{name:base64.b64encode(data).decode() for name,data in raw.items()}, **{name+'_sha256':pin for name,pin in PINS.items()})
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--preflight-only', action='store_true')
args=parser.parse_args()
os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
require(not (STATE/'configure-runtime.json').exists(), 'existing qualified runtime retained; no implicit repeat')
require(previous.apt.prior.evidence('gate')['unresolved'] == 0, 'no unresolved central job')
policy = cloud.bootstrap.rpc('server.conf',closed_policy,{})
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
old = cloud.bootstrap.rpc('linux-vm.conf',installed_metadata,dict(
helper='5ad30cdf6e258df1efd05958ecdb6545e76568d84389a2800ed449b01118fc03',
agent=installed['binary_sha256'],provider='edd1950db26b3ca0b2c1335ff65bf88d5a83e3c946159cbcbb56d8f562252d70'))
payload = dict(build_payload(),old_files=old['files'],preflight_only=args.preflight_only)
report = cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload)
if args.preflight_only:
require(report.get('preflight_only') is True and report.get('maintenance_mutations') is False, 'fixed preflight response required')
print(report);return
cloud.contract.write_private(STATE/('runtime-upgrade-'+payload['id']+'.json'),dict(report,closed_policy=policy))
require(report['completed'], 'preserve incomplete maintenance report; never repeat automatically')
previous.record_agent_upgrade(report,installed)
cloud.contract.write_private(STATE/'configure-runtime.json',report)
print({k:report[k] for k in ('completed','stage','preserved_state','journals_preserved_before_restart','readiness_only','apt_endpoint_enabled')})
if __name__ == '__main__':
try: main()
except Exception as error:
print('CLOUD_APT_RECOVERY_UPGRADE_INCOMPLETE:'+type(error).__name__)
prefix='remote bootstrap incomplete: '
if isinstance(error,RuntimeError) and str(error).startswith(prefix):
try:
detail=json.loads(str(error)[len(prefix):])
safe=dict(error_class=str(detail['error_class']),frames=[
dict(function=str(frame['function']),line=int(frame['line'])) for frame in detail['frames']])
print('REMOTE_BOOTSTRAP_INCOMPLETE:'+json.dumps(safe))
except (ValueError,TypeError,KeyError):pass
raise SystemExit(2) from None
+15 -9
View File
@@ -180,14 +180,15 @@ def enable():
print('APT-only expiring limits installed; no job or approval created')
def accept(case):
def accept(case, *, request=None, folder_root=STATE):
m = cloud.contract; target = prior.identity(); backend = cloud.Backend()
require(prior.evidence('gate')['unresolved'] == 0, 'unresolved central job; no new attempt')
device = backend.call('inventory.get_device', {k:target[k] for k in ('tenant_id','asset_id')})
require(device['presence'] == 'online', 'online cloud VM required')
planning = adapters.planning_from_device(device)
source_case = 'apt_native_install' if case == 'apt_install_retry' else case
request = m.case_input(source_case, target, planning=planning)
if request is None:
planning = adapters.planning_from_device(device)
source_case = 'apt_native_install' if case == 'apt_install_retry' else case
request = m.case_input(source_case, target, planning=planning)
if case == 'apt_install_retry':
old = m.protected_json(cloud.ROOT/'devstate/cloud-root/cases/apt_native_install/checkpoint.json')
require(old['result']['state'] == 'failed' and old['result']['result']['error_code'] == 'HOOK_NOT_HANDED_OFF', 'original definitive pre-hook failure')
@@ -197,7 +198,7 @@ def accept(case):
bootstrap_marker_sha256='ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053'))
request['reason'] = 'One new UI-approved fixture attempt: preserve original failed attempt, verify known no-handoff history, separate retry tombstone; never reboot'
require({'rmm.sync.v1','helper.root.v1','root:'+request['action'],request['action']} <= set(device['capabilities']), 'native root capability required')
folder = STATE/case; folder.mkdir(mode=0o700, exist_ok=False)
folder = folder_root/case; folder.mkdir(mode=0o700, exist_ok=False)
common = dict(profile='dev/test', case=case, expected_principal='simulated-testprincipal-a', **target)
prepare = folder/'prepare.json'; output = folder/'prepared.json'
m.write_private(prepare, dict(common, phase='prepare', request=request, output_file=str(output)))
@@ -216,11 +217,11 @@ def accept(case):
m.write_private(folder/'checkpoint.json', checkpoint)
approval = folder/'approve.json'; m.write_private(approval, dict(checkpoint, phase='approve'))
print(case+': pending 0/0/0 verified; separate actual UI approval', flush=True); browser(approval)
finish(case)
finish(case,folder_root=folder_root)
def finish(case):
m = cloud.contract; folder = STATE/case; checkpoint = m.protected_json(folder/'checkpoint.json')
def finish(case, *, folder_root=STATE):
m = cloud.contract; folder = folder_root/case; checkpoint = m.protected_json(folder/'checkpoint.json')
job = checkpoint['plan']['jobs'][0]; backend = cloud.Backend(); end = time.monotonic()+420
while True:
result = backend.call('jobs.get', dict(tenant_id=checkpoint['tenant_id'], job_id=job['job_id']))
@@ -239,7 +240,12 @@ def finish(case):
package = manifest['resolved_apt_transaction']['packages'][0]
require(apt['packages'] == [dict(name='rmm-mvp-test', architecture='all', version=package['new_version'])] and
apt['fixture_payload_sha256'] == hashlib.sha256((package['new_version']+'\n').encode()).hexdigest(), 'native version/payload proof')
if case in ('apt_install_retry','apt_native_update'):
if case=='apt_configure_recovery':
require(apt.get('native_recovery') is True and apt.get('partial_proof_sha256')==manifest['native_recovery']['partial_proof_sha256'] and
len(apt.get('recovery_record_sha256',''))==64 and 'native_record_sha256' not in apt and
apt['test_result_id']==manifest['test_result_id'] and apt['noProductionEvidence'] is True,
'separate configure-only recovery record; never fresh installation acceptance')
elif case in ('apt_install_retry','apt_native_update'):
require(apt['test_acceptance'] is True and apt['noProductionEvidence'] is True and apt['test_result_id'] == manifest['test_result_id'] and
len(apt['native_record_sha256']) == 64, 'real scoped native acceptance required')
else:
+15 -4
View File
@@ -186,21 +186,32 @@ def validate_apt_manifest(value):
"""Pure format verification, NEVER evidence that live planning/acceptance ran."""
fields = {'type', 'schema_version', 'action', 'execution_profile', 'transaction_sha256', 'resolved_apt_transaction',
'transaction_bytes', 'provider_sha256', 'hook_sha256', 'apt_config_sha256', 'runtime_sha256', 'test_result_id'}
require(set(value) == fields and value['type'] == 'rmm.apt.helper-action' and value['schema_version'] == 2
recovery=value.get('schema_version')==4
if recovery:fields.add('native_recovery')
require(set(value) == fields and value['type'] == 'rmm.apt.helper-action' and value['schema_version'] in (2,4)
and value['action'] in ('apt-native-acceptance', 'apt-execute') and value['execution_profile'] == 'dev-test', 'apt_manifest_schema')
if recovery:
reference=value['native_recovery']
require(value['action']=='apt-native-acceptance' and set(reference)=={'schema_version','previous_grant_id','previous_attempt_id','partial_proof_sha256'}
and reference['schema_version']==1 and all(str(uuid.UUID(reference[k]))==reference[k] and uuid.UUID(reference[k]).int!=0
for k in ('previous_grant_id','previous_attempt_id')) and re.fullmatch('[a-f0-9]{64}',reference['partial_proof_sha256']),
'typed native recovery reference')
raw = value['transaction_bytes']
require(isinstance(raw, list) and len(raw) <= 65536 and all(type(n) is int and 0 <= n < 256 for n in raw), 'apt_exact_transaction_bytes')
raw = bytes(raw)
require(hashlib.sha256(raw).hexdigest() == value['transaction_sha256'], 'apt_transaction_hash')
transaction = json.loads(raw, object_pairs_hook=strict_object)
require(transaction == value['resolved_apt_transaction'] and set(transaction) == {'schema_version', 'recipe_id', 'recipe_version', 'platform',
'resolver_config_version', 'no_reboot', 'status_fingerprint', 'baseline_sha256', 'packages'}, 'apt_full_typed_transaction')
require(type(transaction['schema_version']) is int and transaction['schema_version'] == 1 and transaction['recipe_version'] == '1' and transaction['platform'] == 'ubuntu-24.04-amd64'
txfields={'schema_version','recipe_id','recipe_version','platform','resolver_config_version','no_reboot','status_fingerprint','baseline_sha256','packages'}
if recovery:txfields.add('operation')
require(transaction == value['resolved_apt_transaction'] and set(transaction) == txfields, 'apt_full_typed_transaction')
require(type(transaction['schema_version']) is int and transaction['schema_version'] == (2 if recovery else 1) and transaction['recipe_version'] == '1' and transaction['platform'] == 'ubuntu-24.04-amd64'
and transaction['resolver_config_version'] == 'rmm-apt-local-v1' and transaction['no_reboot'] is True, 'apt_fixture_profile_no_reboot')
if recovery:require(transaction['operation']=='configure_only' and transaction['recipe_id']=='rmm-apt-fixture-install-1.0','same fixture configure-only mode')
pins = {'rmm-apt-fixture-install-1.0': (None, '1.0', '74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867'),
'rmm-apt-fixture-update-1.1': ('1.0', '1.1', 'ef67c9fe0ba1443fcfe465768e1204763063c369a2a1da04af6027d8a9c00491')}
require(transaction['recipe_id'] in pins and len(transaction['packages']) == 1, 'apt_fixed_single_package_recipe')
old, new, pin = pins[transaction['recipe_id']]
if recovery:old='1.0'
package = transaction['packages'][0]
require(package == dict(name='rmm-mvp-test', architecture='all', old_version=old, new_version=new, origin='rmm-test',
sha256=pin, size=20672, archive_path='/var/lib/rmm-apt-transaction/archives/' + pin + '.deb'), 'apt_pinned_fixture_archive')
+9 -2
View File
@@ -28,15 +28,22 @@ def digest(value):
return hashlib.sha256(value).hexdigest()
def planning_from_device(device):
def planning_from_device(device,recovery=False):
"""Only already stored passive observation, never an SSH package scan."""
observations=(device.get('inventory') or {}).get('observations',[])
observation=next((x for x in observations if x.get('scope')=='linux.apt.planning'),None)
scope='linux.apt.recovery-planning' if recovery else 'linux.apt.planning'
observation=next((x for x in observations if x.get('scope')==scope),None)
m.require(observation is not None,'not_configured:stored_linux.apt.planning_observation_missing')
m.require(observation.get('quality')=='valid' and observation.get('completeness')=='complete'
and observation.get('truncated') is False and 0 <= (m.now()-m.timestamp(observation['observed_at'])).total_seconds()<90,
'not_configured:fresh_complete_apt_planning_required')
data=observation.get('data',{})
if recovery:
m.require(observation['source_id']=='root-helper:apt_recovery_planning_v1' and
data.get('fixture_package')==dict(version='1.0',status='install ok unpacked') and
set(data.get('native_recovery',{}))=={'schema_version','previous_grant_id','previous_attempt_id','partial_proof_sha256'},
'dedicated exact partial recovery observation required')
else:m.require(data.get('native_recovery') is None,'normal planning cannot use partial recovery')
missing=[key for key in PLANNING_FIELDS if key not in data]
m.require(not missing,'not_configured:apt_planning_fields_missing:'+','.join(missing))
m.require(data['dpkg_updates_empty'] is True and data['pending_triggers_empty'] is True,'not_configured:dpkg_dirty')