278 lines
20 KiB
Python
278 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Single explicitly signed native recovery, then separate fixed fixture jobs.
|
|
|
|
No direct approval, package-manager SSH execution, history reset or implicit
|
|
retry. Existing checkpoints may only be read to finish/audit that SAME job.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
import cloud_apt_followup as prior
|
|
import cloud_root_evidence
|
|
import cloud_root_policy
|
|
import vm_mvp_adapters as adapters
|
|
|
|
cloud = prior.cloud
|
|
require = cloud.require
|
|
STATE = cloud.ROOT / 'devstate/cloud-apt-retry'
|
|
CASES = ('apt_install_retry', 'apt_native_update', 'ensure_package', 'apply_updates')
|
|
|
|
|
|
def upgrade_agent(payload):
|
|
"""Parser compatibility upgrade only; same identity/journal, no execution."""
|
|
import sqlite3
|
|
os.umask(0o077)
|
|
state = Path('/var/lib/rmm-agent-remote'); binary = Path('/usr/local/libexec/rmm-agent-remote')
|
|
require(not binary.is_symlink() and binary.stat().st_uid == 0 and binary.stat().st_mode&0o022 == 0 and
|
|
hashlib.sha256(binary.read_bytes()).hexdigest() == payload['previous_sha256'], 'reviewed prior agent required')
|
|
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed identity required')
|
|
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro', uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
|
|
db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0,
|
|
'unresolved native job')
|
|
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-agent-schema3-'+payload['id']); backup.mkdir(mode=0o700)
|
|
raw = base64.b64decode(payload['binary'], validate=True)
|
|
require(hashlib.sha256(raw).hexdigest() == payload['sha256'] and raw[:6] == b'\x7fELF\x02\x01' and raw[18:20] == b'\x3e\x00', 'reviewed x64 artifact')
|
|
staged = backup/'agent-next'
|
|
with staged.open('xb') as f: f.write(raw); f.flush(); os.fsync(f.fileno()); os.fchmod(f.fileno(), 0o755)
|
|
subprocess.run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)], capture_output=True, check=True, timeout=15)
|
|
subprocess.run(['systemctl','stop','rmm-agent-remote.service'], capture_output=True, check=True, timeout=20)
|
|
def manifest():
|
|
result = {}
|
|
for p in state.rglob('*'):
|
|
require(not p.is_symlink(), 'agent state symlink')
|
|
if p.is_file(): result[str(p.relative_to(state))] = hashlib.sha256(p.read_bytes()).hexdigest()
|
|
return result
|
|
before = manifest()
|
|
require(binary.stat().st_dev == backup.stat().st_dev, 'same-filesystem preserved binary')
|
|
os.rename(binary, backup/'previous-agent'); os.rename(staged, binary)
|
|
require(before == manifest(), 'identity/journal bytes changed during upgrade')
|
|
subprocess.run(['systemctl','start','rmm-agent-remote.service'], capture_output=True, check=True, timeout=20)
|
|
return dict(binary_sha256=payload['sha256'], state_preserved_before_start=True, package_effects=False, backup=str(backup))
|
|
|
|
|
|
def stored_attempt_evidence(payload):
|
|
"""Read only already persisted evidence; no live collector/package command."""
|
|
import sqlite3
|
|
import re
|
|
base = Path('/var/lib/rmm-apt-transaction'); active = base/'active'
|
|
context = json.loads((active/'context.json').read_text())
|
|
require(context['job_id'] == payload['job_id'] and context['attempt_id'] == payload['attempt_id'], 'exact stored attempt required')
|
|
result = dict(files={}, records={}, context=context)
|
|
for name in ('hook-armed.json','native-prechecks.json','sandbox-ready.json','sandbox-result.json','launch.json'):
|
|
path = active/name
|
|
result['files'][name] = dict(exists=path.exists(), symlink=path.is_symlink())
|
|
if path.is_file() and not path.is_symlink(): result['records'][name] = json.loads(path.read_text())
|
|
for name in ('bootstrap-attempt-rmm-apt-fixture-install-1.0.json','bootstrap-retry-rmm-apt-fixture-install-1.0.json',
|
|
'acceptance-rmm-apt-fixture-install-1.0.json','pending-acceptance-rmm-apt-fixture-install-1.0.json'):
|
|
path = base/name
|
|
result['files'][name] = dict(exists=path.exists(), symlink=path.is_symlink(), sha256=hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None)
|
|
with sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro', uri=True) as db:
|
|
row = db.execute('SELECT c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?', (context['grant_id'],)).fetchone()
|
|
result['ledger'] = dict(result=json.loads(row[0]) if row[0] else None, outcome=json.loads(row[1]) if row[1] else None,
|
|
consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0], unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0])
|
|
result['logs'] = {}
|
|
for name in ('dpkg.log','term.log','history.log'):
|
|
path = base/'log'/name
|
|
if path.is_file() and not path.is_symlink():
|
|
raw = path.read_bytes(); require(len(raw) <= 1024*1024, 'bounded stored log')
|
|
text = raw.decode('utf-8', errors='replace')
|
|
result['logs'][name] = dict(sha256=hashlib.sha256(raw).hexdigest(), size=len(raw),
|
|
fixture_states=re.findall(r'\bstatus (not-installed|half-installed|unpacked|half-configured|installed|triggers-pending|triggers-awaited) rmm-mvp-test(?::all)? (1\.[01])\b', text),
|
|
configure_record=bool(re.search(r'\bconfigure rmm-mvp-test(?::all)?\b', text)),
|
|
setup_message='Setting up rmm-mvp-test (1.0)' in text,
|
|
dpkg_error='dpkg: error' in text)
|
|
return result
|
|
|
|
|
|
def switch_endpoint(payload):
|
|
import sqlite3
|
|
os.umask(0o077)
|
|
binary = Path('/usr/local/libexec/rmm-privileged-helper')
|
|
require(hashlib.sha256(binary.read_bytes()).hexdigest() == payload['binary_sha256'], 'reviewed helper required')
|
|
state = Path('/var/lib/rmm-agent-remote')
|
|
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed cloud identity')
|
|
for database, query in ((state/'journal.sqlite', "SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')"),
|
|
(Path('/var/lib/rmm-helper/consumption.sqlite'), 'SELECT count(*) FROM consumed WHERE result IS NULL')):
|
|
with sqlite3.connect(database.as_uri()+'?mode=ro', uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute(query).fetchone()[0] == 0, 'unresolved native attempt')
|
|
path = Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
|
|
full = ('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
|
|
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
|
|
'Environment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\n'
|
|
'Environment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm-remote/apt-helper-proof.json\n'
|
|
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
|
|
reduced = ''.join(line+'\n' for line in full.splitlines() if not line.startswith('Environment=RMM_APT_HELPER_'))
|
|
previous, updated = (reduced, full) if payload['enabled'] else (full, reduced)
|
|
require(not path.is_symlink() and path.stat().st_uid == 0 and path.read_text() == previous, 'known drop-in required')
|
|
backup = Path('/var/lib/rmm-bootstrap-history') / ('cloud-apt-endpoint-'+payload['id'])
|
|
backup.mkdir(mode=0o700)
|
|
with (backup/'previous.conf').open('xb') as f:
|
|
f.write(previous.encode()); f.flush(); os.fsync(f.fileno())
|
|
if payload['enabled']:
|
|
subprocess.run(['systemctl', 'start', 'rmm-apt-helper.service'], capture_output=True, check=True, timeout=30)
|
|
next_path = path.with_suffix('.retry-next')
|
|
fd = os.open(next_path, os.O_CREAT|os.O_EXCL|os.O_WRONLY|os.O_NOFOLLOW, 0o644)
|
|
with os.fdopen(fd, 'w') as f:
|
|
f.write(updated); f.flush(); os.fsync(f.fileno()); os.fchmod(f.fileno(), 0o644)
|
|
os.replace(next_path, path)
|
|
subprocess.run(['systemctl', 'daemon-reload'], capture_output=True, check=True, timeout=30)
|
|
subprocess.run(['systemctl', 'restart', 'rmm-agent-remote.service'], capture_output=True, check=True, timeout=30)
|
|
if not payload['enabled']:
|
|
subprocess.run(['systemctl', 'stop', 'rmm-apt-helper.service'], capture_output=True, check=True, timeout=30)
|
|
return dict(enabled=payload['enabled'], package_effects=False, backup=str(backup))
|
|
|
|
|
|
def replace_policy(payload):
|
|
import datetime as dt
|
|
p = Path('/opt/rust-discovery/devstate/root-policy/policy.json')
|
|
require(not p.is_symlink() and p.stat().st_uid == 10001 and p.stat().st_mode&0o777 == 0o600, 'safe policy required')
|
|
old = p.read_bytes()
|
|
require(hashlib.sha256(old).hexdigest() == payload['previous_sha256'] and
|
|
dt.datetime.fromisoformat(json.loads(old)['expires_at']) < dt.datetime.now(dt.timezone.utc), 'previous policy must be closed')
|
|
policy = payload['policy']; require(len(policy['scopes']) == 1, 'one APT target only')
|
|
scope = policy['scopes'][0]
|
|
require(not scope['allow_root_scripts'] and not scope['allow_reboot'] and not scope['allowed_services'] and
|
|
scope['principals'] == ['simulated-testprincipal-a'] and scope['apt']['allow_native_acceptance'] and
|
|
scope['apt']['allow_execute'], 'APT-only simulation limits')
|
|
backup = p.parents[1] / ('root-policy-bootstrap-'+payload['id']); backup.mkdir(mode=0o700)
|
|
with (backup/'previous-policy.json').open('xb') as f:
|
|
f.write(old); f.flush(); os.fsync(f.fileno())
|
|
next_path = p.with_suffix('.retry-next')
|
|
fd = os.open(next_path, os.O_CREAT|os.O_EXCL|os.O_WRONLY|os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'w') as f:
|
|
json.dump(policy, f); f.flush(); os.fsync(f.fileno()); os.fchown(f.fileno(), 10001, 10001)
|
|
os.replace(next_path, p)
|
|
return dict(policy_sha256=hashlib.sha256(p.read_bytes()).hexdigest(), backup=str(backup), approvals_created=False)
|
|
|
|
|
|
def enable():
|
|
m = cloud.contract; require(prior.evidence('gate')['unresolved'] == 0, 'unresolved central job')
|
|
reports = [m.protected_json(p) for p in prior.STATE.glob('sandbox-*.json')]
|
|
code_sha = hashlib.sha256((cloud.ROOT/'tools/apt-transaction/apt_transaction.py').read_bytes()).hexdigest()
|
|
# Exactly the reviewed current compiled provider, never an old readiness.
|
|
ready = next(r for r in reports if r['completed'] and r.get('provider_sha256') == code_sha)
|
|
target = prior.identity(); binding = ready['apt_pin']['identity']
|
|
require(all(binding[k] == target[k] for k in target), 'native helper target binding')
|
|
helper = (prior.STATE/'helper-native-retry').read_bytes()
|
|
endpoint = cloud.bootstrap.rpc('linux-vm.conf', switch_endpoint, dict(id=str(uuid.uuid4()), enabled=True,
|
|
agent_id=target['agent_id'], binary_sha256=hashlib.sha256(helper).hexdigest()))
|
|
m.write_private(STATE/'endpoint-enabled.json', endpoint)
|
|
peers = cloud.bootstrap.rpc('linux-vm.conf', cloud_root_evidence.inspect_helpers, target)
|
|
require(peers['apt']['fixture_acceptance_ready'] and peers['ledger']['faults'] == 0, 'actual shared-ledger native readiness')
|
|
scope = {k:binding[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')}
|
|
scope.update(principals=['simulated-testprincipal-a'], helper_policy_sha256=ready['apt_pin']['provider_stack_sha256'],
|
|
acceptance_evidence_id='cloud-signed-single-retry:'+ready['runtime_sha256'], allowed_services=[], allow_root_scripts=False, allow_reboot=False,
|
|
apt=dict(allow_native_acceptance=True, allow_execute=True, runtime_sha256=ready['runtime_sha256'],
|
|
apt_config_sha256=ready['apt_config_sha256'], provider_sha256=code_sha,
|
|
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
|
|
policy = dict(schema_version=1, profile='development', expires_at=(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=2)).isoformat(), scopes=[scope])
|
|
old = m.protected_json(prior.STATE/'diagnostic-policy-closed.json')
|
|
result = cloud.bootstrap.rpc('server.conf', replace_policy, dict(id=str(uuid.uuid4()), previous_sha256=old['policy_sha256'], policy=policy))
|
|
m.write_private(STATE/'policy.json', dict(result, policy=policy, peers=peers))
|
|
print('APT-only expiring limits installed; no job or approval created')
|
|
|
|
|
|
def accept(case):
|
|
m = cloud.contract; target = prior.identity(); backend = cloud.Backend()
|
|
require(prior.evidence('gate')['unresolved'] == 0, 'unresolved central job; no new attempt')
|
|
device = backend.call('inventory.get_device', {k:target[k] for k in ('tenant_id','asset_id')})
|
|
require(device['presence'] == 'online', 'online cloud VM required')
|
|
planning = adapters.planning_from_device(device)
|
|
source_case = 'apt_native_install' if case == 'apt_install_retry' else case
|
|
request = m.case_input(source_case, target, planning=planning)
|
|
if case == 'apt_install_retry':
|
|
old = m.protected_json(cloud.ROOT/'devstate/cloud-root/cases/apt_native_install/checkpoint.json')
|
|
require(old['result']['state'] == 'failed' and old['result']['result']['error_code'] == 'HOOK_NOT_HANDED_OFF', 'original definitive pre-hook failure')
|
|
manifest = request['parameters']['apt_manifest']
|
|
manifest.update(schema_version=3, native_retry=dict(schema_version=1,
|
|
previous_grant_id=old['result']['result']['grant_id'], previous_attempt_id=old['job']['attempt_id'],
|
|
bootstrap_marker_sha256='ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053'))
|
|
request['reason'] = 'One new UI-approved fixture attempt: preserve original failed attempt, verify known no-handoff history, separate retry tombstone; never reboot'
|
|
require({'rmm.sync.v1','helper.root.v1','root:'+request['action'],request['action']} <= set(device['capabilities']), 'native root capability required')
|
|
folder = STATE/case; folder.mkdir(mode=0o700, exist_ok=False)
|
|
common = dict(profile='dev/test', case=case, expected_principal='simulated-testprincipal-a', **target)
|
|
prepare = folder/'prepare.json'; output = folder/'prepared.json'
|
|
m.write_private(prepare, dict(common, phase='prepare', request=request, output_file=str(output)))
|
|
cloud.prepare_browser()
|
|
def browser(file):
|
|
result = subprocess.run(['node', str(cloud.ROOT/'deploy/browser/cloud_apt_acceptance.cjs'), str(file)], capture_output=True, timeout=120)
|
|
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint, never repeat automatically')
|
|
print(case+': actual UI preparation', flush=True); browser(prepare)
|
|
prepared = m.protected_json(output); plan = prepared['plan']; operation = prepared['operation']; job = plan['jobs'][0]
|
|
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'immutable single job')
|
|
before = prior.evidence('gate', job_id=job['job_id'])
|
|
pending = backend.call('jobs.get', dict(tenant_id=target['tenant_id'], operation_id=operation['operation_id']))
|
|
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0 and pending['execution_state'] is None and
|
|
pending['plan_hash'] == plan['plan_hash'] and all(before[k] == 0 for k in ('jobs','outbox','permits')), 'pending zero execution artifacts')
|
|
checkpoint = dict(common, plan=plan, request=request, operation_id=operation['operation_id'], preapproval=before, stage='pending_approval')
|
|
m.write_private(folder/'checkpoint.json', checkpoint)
|
|
approval = folder/'approve.json'; m.write_private(approval, dict(checkpoint, phase='approve'))
|
|
print(case+': pending 0/0/0 verified; separate actual UI approval', flush=True); browser(approval)
|
|
finish(case)
|
|
|
|
|
|
def finish(case):
|
|
m = cloud.contract; folder = STATE/case; checkpoint = m.protected_json(folder/'checkpoint.json')
|
|
job = checkpoint['plan']['jobs'][0]; backend = cloud.Backend(); end = time.monotonic()+420
|
|
while True:
|
|
result = backend.call('jobs.get', dict(tenant_id=checkpoint['tenant_id'], job_id=job['job_id']))
|
|
if result['state'] in m.TERMINAL or result['state'] in ('unknown','awaiting_reboot'): break
|
|
require(time.monotonic() < end, 'unconfirmed effect; no retry'); time.sleep(3)
|
|
checkpoint.update(stage=result['state'], result=result)
|
|
def save():
|
|
path = folder/'checkpoint.json'; path.write_text(json.dumps(checkpoint, indent=2)); path.chmod(0o600)
|
|
save()
|
|
after = prior.evidence('gate', job_id=job['job_id']); audit = prior.evidence('audit', operation_id=checkpoint['operation_id'])
|
|
require([after[k] for k in ('jobs','outbox','permits')] == [1,1,1] and audit == dict(origin='dev_test_browser_simulation', simulation=True,
|
|
principal='simulated-testprincipal-a', plan_hash=checkpoint['plan']['plan_hash']), 'single attempt and simulated UI audit')
|
|
checkpoint.update(postapproval=after, audit=audit); save()
|
|
require(result['state'] == 'succeeded' and result['result']['evidence']['postconditions_passed'], 'native job failed; retain outcome, never repeat')
|
|
apt = result['result']['evidence']['apt']; manifest = checkpoint['request']['parameters']['apt_manifest']
|
|
package = manifest['resolved_apt_transaction']['packages'][0]
|
|
require(apt['packages'] == [dict(name='rmm-mvp-test', architecture='all', version=package['new_version'])] and
|
|
apt['fixture_payload_sha256'] == hashlib.sha256((package['new_version']+'\n').encode()).hexdigest(), 'native version/payload proof')
|
|
if case in ('apt_install_retry','apt_native_update'):
|
|
require(apt['test_acceptance'] is True and apt['noProductionEvidence'] is True and apt['test_result_id'] == manifest['test_result_id'] and
|
|
len(apt['native_record_sha256']) == 64, 'real scoped native acceptance required')
|
|
else:
|
|
require(apt['changed'] is False and apt['already_desired'] is True and apt['transaction_executed'] is False and
|
|
apt['postcondition_already_met'] is True, 'separate exact desired-state no-change result required')
|
|
checkpoint['verified'] = True; save()
|
|
print('PASS '+case+': real cloud VM, actual simulated UI, 0/0/0 -> 1/1/1')
|
|
|
|
|
|
def close():
|
|
require(prior.evidence('gate')['unresolved'] == 0, 'unresolved central job')
|
|
previous = cloud.contract.protected_json(STATE/'policy.json')
|
|
result = cloud.bootstrap.rpc('server.conf', cloud_root_policy.close_policy, dict(expected_sha256=previous['policy_sha256'], backup=previous['backup']))
|
|
cloud.contract.write_private(STATE/'policy-closed.json', result)
|
|
endpoint = cloud.bootstrap.rpc('linux-vm.conf', switch_endpoint, dict(id=str(uuid.uuid4()), enabled=False,
|
|
agent_id=prior.identity()['agent_id'], binary_sha256=hashlib.sha256((prior.STATE/'helper-native-retry').read_bytes()).hexdigest()))
|
|
cloud.contract.write_private(STATE/'endpoint-disabled.json', endpoint)
|
|
print('APT scope closed and endpoint stopped; all attempt history preserved')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('enable','accept','finish','close')); parser.add_argument('--case', choices=CASES)
|
|
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, exist_ok=True)
|
|
if args.action in ('accept','finish'):
|
|
require(args.case is not None, 'fixed case required')
|
|
(accept if args.action == 'accept' else finish)(args.case)
|
|
else: (enable if args.action == 'enable' else close)()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try: main()
|
|
except Exception as error:
|
|
print('CLOUD_APT_RETRY_INCOMPLETE:'+type(error).__name__)
|
|
raise SystemExit(2) from None
|