feat: preserve failed APT history for one signed UI-approved recovery
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 15:50:34 +01:00
parent a5c2dce4fb
commit fd7225600b
18 changed files with 1345 additions and 34 deletions
+97
View File
@@ -0,0 +1,97 @@
// Actual UI only. No injected fetch, request context or direct approval API.
const fs = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const { isDeepStrictEqual: equal } = require('node:util');
const { chromium } = require('../../web/node_modules/playwright');
const root = path.resolve(__dirname, '../..');
const assert = v => { if (!v) throw new Error('APT UI contract mismatch'); };
function readPrivate(file) {
const s = fs.lstatSync(file);
assert(s.isFile() && !s.isSymbolicLink() && s.uid === process.getuid() && (s.mode & 0o777) === 0o600 && s.size < 1048576);
return JSON.parse(fs.readFileSync(file, 'utf8'));
}
function checkAction(p, expected) {
const native = ['apt_install_retry', 'apt_native_update'].includes(expected.case);
const version = expected.case === 'apt_install_retry' ? '1.0' : '1.1';
const recipe = version === '1.0' ? 'rmm-apt-fixture-install-1.0' : 'rmm-apt-fixture-update-1.1';
const action = p.action; const m = action.apt_manifest; const tx = m.resolved_apt_transaction;
assert(action.action === (native ? 'apt_native_acceptance' : expected.case));
assert(p.execution_identity === 'root' && p.max_runtime_seconds === 300 && p.max_output_bytes === 65536);
assert(m.type === 'rmm.apt.helper-action' && m.execution_profile === 'dev-test' && m.action === (native ? 'apt-native-acceptance' : 'apt-execute'));
assert(m.schema_version === (expected.case === 'apt_install_retry' ? 3 : 2));
assert(tx.recipe_id === recipe && tx.recipe_version === '1' && tx.no_reboot === true && tx.packages.length === 1);
const pkg = tx.packages[0];
const sha = version === '1.0' ? '74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867' : 'ef67c9fe0ba1443fcfe465768e1204763063c369a2a1da04af6027d8a9c00491';
assert(pkg.name === 'rmm-mvp-test' && pkg.architecture === 'all' && pkg.origin === 'rmm-test' && pkg.new_version === version && pkg.sha256 === sha && pkg.size === 20672);
assert(crypto.createHash('sha256').update(Buffer.from(m.transaction_bytes)).digest('hex') === m.transaction_sha256);
assert(equal(JSON.parse(Buffer.from(m.transaction_bytes)), tx));
assert(equal(m, expected.request.parameters.apt_manifest));
if (expected.case === 'apt_install_retry') {
assert(m.native_retry.schema_version === 1 && m.native_retry.bootstrap_marker_sha256 === 'ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053');
assert(m.native_retry.previous_attempt_id !== p.attempt_id);
}
}
(async () => {
const file = path.resolve(process.argv[2]); const expected = readPrivate(file);
assert(expected.profile === 'dev/test' && ['apt_install_retry', 'apt_native_update', 'ensure_package', 'apply_updates'].includes(expected.case) && expected.expected_principal === 'simulated-testprincipal-a');
const login = readPrivate(path.join(root, 'credentials/server-browser-login.json'));
const browsers = path.join(root, 'web/node_modules/.cache/playwright-browsers');
const version = fs.readdirSync(browsers).find(x => /^chromium-/.test(x));
const executablePath = ['chrome-linux/chrome', 'chrome-linux64/chrome'].map(x => path.join(browsers, version, x)).find(x => fs.existsSync(x));
const browser = await chromium.launch({ headless: true, executablePath, env: { ...process.env, HOME: path.join(root, 'web/node_modules/.cache/cloud-acceptance/home') } });
try {
const page = await browser.newPage({ ignoreHTTPSErrors: false, timezoneId: 'UTC' }); page.setDefaultTimeout(15000);
let approvals = 0;
page.on('request', r => { if (new URL(r.url()).pathname.endsWith('/authorizations')) approvals++; });
await page.goto('https://localhost:18443/');
await page.getByLabel('Benutzername').fill(login.username); await page.getByLabel('Passwort', { exact: true }).fill(login.password);
const response = page.waitForResponse(r => new URL(r.url()).pathname === '/auth/session' && r.status() === 200);
await page.getByRole('button', { name: 'Anmelden', exact: true }).click();
const session = await (await response).json();
assert(session.testsimulation === true && session.approval_origin === 'dev_test_browser_simulation' && session.principal === expected.expected_principal);
await page.getByRole('status').filter({ hasText: 'SIMULIERT' }).waitFor();
const detail = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Unveränderlicher Plan / Genehmigungsdetail', exact: true }) });
if (expected.phase === 'prepare') {
await page.getByRole('checkbox', { name: 'Ziel ' + expected.asset_id, exact: true }).check();
await page.getByLabel('Aktion', { exact: true }).selectOption(expected.request.action);
await page.getByLabel('Aktionsparameter', { exact: true }).fill(JSON.stringify(expected.request.parameters));
await page.getByLabel('Ausführungsidentität', { exact: true }).selectOption('root');
await page.getByLabel('Laufzeitlimit (s)', { exact: true }).fill('300'); await page.getByLabel('Ausgabelimit (Bytes)', { exact: true }).fill('65536');
const start = new Date(Math.ceil(Date.now() / 60000) * 60000);
await page.getByLabel('Frühester Start (lokale Zeit)', { exact: true }).fill(start.toISOString().slice(0, 16));
await page.getByLabel('Startfrist / Ablauf (lokale Zeit)', { exact: true }).fill(new Date(start.getTime() + 15 * 60000).toISOString().slice(0, 16));
await page.getByLabel('Maximal parallel', { exact: true }).fill('1'); await page.getByLabel('Stopp nach Fehlern', { exact: true }).fill('1');
await page.getByLabel('Begründung', { exact: true }).fill(expected.request.reason);
await page.getByRole('button', { name: 'Plan erstellen · noch keine Ausführung' }).click();
await detail.locator('pre').waitFor(); const plan = JSON.parse(await detail.locator('pre').innerText());
assert(plan.jobs.length === 1); checkAction(plan.jobs[0].payload, expected);
assert(crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
await detail.getByRole('button', { name: 'Antrag absenden · pending_approval' }).click();
const op = page.locator('section').filter({ has: page.getByRole('heading', { name: /^Antrag \/ Ausführung / }) });
await op.locator('pre').first().filter({ hasText: '"job_count": 0' }).waitFor(); const operation = JSON.parse(await op.locator('pre').first().innerText());
assert(operation.approval_state === 'pending_approval' && operation.execution_state === null && approvals === 0);
assert(path.dirname(path.resolve(expected.output_file)) === path.dirname(file));
fs.writeFileSync(expected.output_file, JSON.stringify({ plan, operation }), { flag: 'wx', mode: 0o600 });
} else {
assert(expected.phase === 'approve'); await page.getByRole('button', { name: expected.operation_id, exact: true }).click();
const op = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Antrag / Ausführung ' + expected.operation_id, exact: true }) });
const pending = JSON.parse(await op.locator('pre').first().innerText());
assert(pending.approval_state === 'pending_approval' && pending.job_count === 0 && pending.execution_state === null);
await op.getByRole('button', { name: 'Unveränderlichen Plan zur Prüfung öffnen' }).click();
const plan = JSON.parse(await detail.locator('pre').innerText());
assert(equal(plan, expected.plan) && plan.jobs.length === 1 && plan.resolved_targets.length === 1);
assert(crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
const p = plan.jobs[0].payload; checkAction(p, expected);
assert(p.agent_id === expected.agent_id && p.asset_id === expected.asset_id && p.tenant_id === expected.tenant_id && Date.parse(p.start_before) > Date.now());
const button = detail.getByRole('button', { name: 'SIMULIERT: Diesen festen Plan ausdrücklich genehmigen', exact: true }); assert(await button.isDisabled());
await detail.getByRole('checkbox', { name: 'Ich habe sämtliche festen Ziele', exact: false }).check(); await button.click();
await op.locator('p.warning').filter({ hasText: 'Antrag: approved' }).waitFor();
assert(approvals === 1);
}
await page.getByRole('button', { name: 'Abmelden', exact: true }).click();
} finally { await browser.close(); }
})().catch(error => {
const line = error.stack?.match(/cloud_apt_acceptance\.cjs:(\d+):\d+/)?.[1] || 'unknown';
console.log('CLOUD_APT_UI_INCOMPLETE:' + error.constructor.name + ':source_line=' + line); process.exitCode = 1;
});
+210
View File
@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""Single explicitly signed native recovery, then separate fixed fixture jobs.
No direct approval, package-manager SSH execution, history reset or implicit
retry. Existing checkpoints may only be read to finish/audit that SAME job.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
import cloud_apt_followup as prior
import cloud_root_evidence
import cloud_root_policy
import vm_mvp_adapters as adapters
cloud = prior.cloud
require = cloud.require
STATE = cloud.ROOT / 'devstate/cloud-apt-retry'
CASES = ('apt_install_retry', 'apt_native_update', 'ensure_package', 'apply_updates')
def switch_endpoint(payload):
import sqlite3
os.umask(0o077)
binary = Path('/usr/local/libexec/rmm-privileged-helper')
require(hashlib.sha256(binary.read_bytes()).hexdigest() == payload['binary_sha256'], 'reviewed helper required')
state = Path('/var/lib/rmm-agent-remote')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed cloud identity')
for database, query in ((state/'journal.sqlite', "SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')"),
(Path('/var/lib/rmm-helper/consumption.sqlite'), 'SELECT count(*) FROM consumed WHERE result IS NULL')):
with sqlite3.connect(database.as_uri()+'?mode=ro', uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute(query).fetchone()[0] == 0, 'unresolved native attempt')
path = Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
full = ('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
'Environment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\n'
'Environment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm-remote/apt-helper-proof.json\n'
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
reduced = ''.join(line+'\n' for line in full.splitlines() if not line.startswith('Environment=RMM_APT_HELPER_'))
previous, updated = (reduced, full) if payload['enabled'] else (full, reduced)
require(not path.is_symlink() and path.stat().st_uid == 0 and path.read_text() == previous, 'known drop-in required')
backup = Path('/var/lib/rmm-bootstrap-history') / ('cloud-apt-endpoint-'+payload['id'])
backup.mkdir(mode=0o700)
with (backup/'previous.conf').open('xb') as f:
f.write(previous.encode()); f.flush(); os.fsync(f.fileno())
if payload['enabled']:
subprocess.run(['systemctl', 'start', 'rmm-apt-helper.service'], capture_output=True, check=True, timeout=30)
next_path = path.with_suffix('.retry-next')
fd = os.open(next_path, os.O_CREAT|os.O_EXCL|os.O_WRONLY|os.O_NOFOLLOW, 0o644)
with os.fdopen(fd, 'w') as f:
f.write(updated); f.flush(); os.fsync(f.fileno()); os.fchmod(f.fileno(), 0o644)
os.replace(next_path, path)
subprocess.run(['systemctl', 'daemon-reload'], capture_output=True, check=True, timeout=30)
subprocess.run(['systemctl', 'restart', 'rmm-agent-remote.service'], capture_output=True, check=True, timeout=30)
if not payload['enabled']:
subprocess.run(['systemctl', 'stop', 'rmm-apt-helper.service'], capture_output=True, check=True, timeout=30)
return dict(enabled=payload['enabled'], package_effects=False, backup=str(backup))
def replace_policy(payload):
import datetime as dt
p = Path('/opt/rust-discovery/devstate/root-policy/policy.json')
require(not p.is_symlink() and p.stat().st_uid == 10001 and p.stat().st_mode&0o777 == 0o600, 'safe policy required')
old = p.read_bytes()
require(hashlib.sha256(old).hexdigest() == payload['previous_sha256'] and
dt.datetime.fromisoformat(json.loads(old)['expires_at']) < dt.datetime.now(dt.timezone.utc), 'previous policy must be closed')
policy = payload['policy']; require(len(policy['scopes']) == 1, 'one APT target only')
scope = policy['scopes'][0]
require(not scope['allow_root_scripts'] and not scope['allow_reboot'] and not scope['allowed_services'] and
scope['principals'] == ['simulated-testprincipal-a'] and scope['apt']['allow_native_acceptance'] and
scope['apt']['allow_execute'], 'APT-only simulation limits')
backup = p.parents[1] / ('root-policy-bootstrap-'+payload['id']); backup.mkdir(mode=0o700)
with (backup/'previous-policy.json').open('xb') as f:
f.write(old); f.flush(); os.fsync(f.fileno())
next_path = p.with_suffix('.retry-next')
fd = os.open(next_path, os.O_CREAT|os.O_EXCL|os.O_WRONLY|os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'w') as f:
json.dump(policy, f); f.flush(); os.fsync(f.fileno()); os.fchown(f.fileno(), 10001, 10001)
os.replace(next_path, p)
return dict(policy_sha256=hashlib.sha256(p.read_bytes()).hexdigest(), backup=str(backup), approvals_created=False)
def enable():
m = cloud.contract; require(prior.evidence('gate')['unresolved'] == 0, 'unresolved central job')
reports = [m.protected_json(p) for p in prior.STATE.glob('sandbox-*.json')]
code_sha = hashlib.sha256((cloud.ROOT/'tools/apt-transaction/apt_transaction.py').read_bytes()).hexdigest()
# Exactly the reviewed current compiled provider, never an old readiness.
ready = next(r for r in reports if r['completed'] and r.get('provider_sha256') == code_sha)
target = prior.identity(); binding = ready['apt_pin']['identity']
require(all(binding[k] == target[k] for k in target), 'native helper target binding')
helper = (prior.STATE/'helper-native-retry').read_bytes()
endpoint = cloud.bootstrap.rpc('linux-vm.conf', switch_endpoint, dict(id=str(uuid.uuid4()), enabled=True,
agent_id=target['agent_id'], binary_sha256=hashlib.sha256(helper).hexdigest()))
m.write_private(STATE/'endpoint-enabled.json', endpoint)
peers = cloud.bootstrap.rpc('linux-vm.conf', cloud_root_evidence.inspect_helpers, target)
require(peers['apt']['fixture_acceptance_ready'] and peers['ledger']['faults'] == 0, 'actual shared-ledger native readiness')
scope = {k:binding[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')}
scope.update(principals=['simulated-testprincipal-a'], helper_policy_sha256=ready['apt_pin']['provider_stack_sha256'],
acceptance_evidence_id='cloud-signed-single-retry:'+ready['runtime_sha256'], allowed_services=[], allow_root_scripts=False, allow_reboot=False,
apt=dict(allow_native_acceptance=True, allow_execute=True, runtime_sha256=ready['runtime_sha256'],
apt_config_sha256=ready['apt_config_sha256'], provider_sha256=code_sha,
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
policy = dict(schema_version=1, profile='development', expires_at=(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=2)).isoformat(), scopes=[scope])
old = m.protected_json(prior.STATE/'diagnostic-policy-closed.json')
result = cloud.bootstrap.rpc('server.conf', replace_policy, dict(id=str(uuid.uuid4()), previous_sha256=old['policy_sha256'], policy=policy))
m.write_private(STATE/'policy.json', dict(result, policy=policy, peers=peers))
print('APT-only expiring limits installed; no job or approval created')
def accept(case):
m = cloud.contract; target = prior.identity(); backend = cloud.Backend()
require(prior.evidence('gate')['unresolved'] == 0, 'unresolved central job; no new attempt')
device = backend.call('inventory.get_device', {k:target[k] for k in ('tenant_id','asset_id')})
require(device['presence'] == 'online', 'online cloud VM required')
planning = adapters.planning_from_device(device)
source_case = 'apt_native_install' if case == 'apt_install_retry' else case
request = m.case_input(source_case, target, planning=planning)
if case == 'apt_install_retry':
old = m.protected_json(cloud.ROOT/'devstate/cloud-root/cases/apt_native_install/checkpoint.json')
require(old['result']['state'] == 'failed' and old['result']['result']['error_code'] == 'HOOK_NOT_HANDED_OFF', 'original definitive pre-hook failure')
manifest = request['parameters']['apt_manifest']
manifest.update(schema_version=3, native_retry=dict(schema_version=1,
previous_grant_id=old['result']['result']['grant_id'], previous_attempt_id=old['job']['attempt_id'],
bootstrap_marker_sha256='ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053'))
request['reason'] = 'One new UI-approved fixture attempt: preserve original failed attempt, verify known no-handoff history, separate retry tombstone; never reboot'
require({'rmm.sync.v1','helper.root.v1','root:'+request['action'],request['action']} <= set(device['capabilities']), 'native root capability required')
folder = STATE/case; folder.mkdir(mode=0o700, exist_ok=False)
common = dict(profile='dev/test', case=case, expected_principal='simulated-testprincipal-a', **target)
prepare = folder/'prepare.json'; output = folder/'prepared.json'
m.write_private(prepare, dict(common, phase='prepare', request=request, output_file=str(output)))
cloud.prepare_browser()
def browser(file):
result = subprocess.run(['node', str(cloud.ROOT/'deploy/browser/cloud_apt_acceptance.cjs'), str(file)], capture_output=True, timeout=120)
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint, never repeat automatically')
print(case+': actual UI preparation', flush=True); browser(prepare)
prepared = m.protected_json(output); plan = prepared['plan']; operation = prepared['operation']; job = plan['jobs'][0]
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'immutable single job')
before = prior.evidence('gate', job_id=job['job_id'])
pending = backend.call('jobs.get', dict(tenant_id=target['tenant_id'], operation_id=operation['operation_id']))
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0 and pending['execution_state'] is None and
pending['plan_hash'] == plan['plan_hash'] and all(before[k] == 0 for k in ('jobs','outbox','permits')), 'pending zero execution artifacts')
checkpoint = dict(common, plan=plan, request=request, operation_id=operation['operation_id'], preapproval=before, stage='pending_approval')
m.write_private(folder/'checkpoint.json', checkpoint)
approval = folder/'approve.json'; m.write_private(approval, dict(checkpoint, phase='approve'))
print(case+': pending 0/0/0 verified; separate actual UI approval', flush=True); browser(approval)
finish(case)
def finish(case):
m = cloud.contract; folder = STATE/case; checkpoint = m.protected_json(folder/'checkpoint.json')
job = checkpoint['plan']['jobs'][0]; backend = cloud.Backend(); end = time.monotonic()+420
while True:
result = backend.call('jobs.get', dict(tenant_id=checkpoint['tenant_id'], job_id=job['job_id']))
if result['state'] in m.TERMINAL or result['state'] in ('unknown','awaiting_reboot'): break
require(time.monotonic() < end, 'unconfirmed effect; no retry'); time.sleep(3)
checkpoint.update(stage=result['state'], result=result)
def save():
path = folder/'checkpoint.json'; path.write_text(json.dumps(checkpoint, indent=2)); path.chmod(0o600)
save()
after = prior.evidence('gate', job_id=job['job_id']); audit = prior.evidence('audit', operation_id=checkpoint['operation_id'])
require([after[k] for k in ('jobs','outbox','permits')] == [1,1,1] and audit == dict(origin='dev_test_browser_simulation', simulation=True,
principal='simulated-testprincipal-a', plan_hash=checkpoint['plan']['plan_hash']), 'single attempt and simulated UI audit')
checkpoint.update(postapproval=after, audit=audit); save()
require(result['state'] == 'succeeded' and result['result']['evidence']['postconditions_passed'], 'native job failed; retain outcome, never repeat')
apt = result['result']['evidence']['apt']; manifest = checkpoint['request']['parameters']['apt_manifest']
package = manifest['resolved_apt_transaction']['packages'][0]
require(apt['packages'] == [dict(name='rmm-mvp-test', architecture='all', version=package['new_version'])] and
apt['fixture_payload_sha256'] == hashlib.sha256((package['new_version']+'\n').encode()).hexdigest(), 'native version/payload proof')
if case in ('apt_install_retry','apt_native_update'):
require(apt['test_acceptance'] is True and apt['noProductionEvidence'] is True and apt['test_result_id'] == manifest['test_result_id'] and
len(apt['native_record_sha256']) == 64, 'real scoped native acceptance required')
else:
require(apt['changed'] is False and apt['already_desired'] is True and apt['transaction_executed'] is False and
apt['postcondition_already_met'] is True, 'separate exact desired-state no-change result required')
checkpoint['verified'] = True; save()
print('PASS '+case+': real cloud VM, actual simulated UI, 0/0/0 -> 1/1/1')
def close():
require(prior.evidence('gate')['unresolved'] == 0, 'unresolved central job')
previous = cloud.contract.protected_json(STATE/'policy.json')
result = cloud.bootstrap.rpc('server.conf', cloud_root_policy.close_policy, dict(expected_sha256=previous['policy_sha256'], backup=previous['backup']))
cloud.contract.write_private(STATE/'policy-closed.json', result)
endpoint = cloud.bootstrap.rpc('linux-vm.conf', switch_endpoint, dict(id=str(uuid.uuid4()), enabled=False,
agent_id=prior.identity()['agent_id'], binary_sha256=hashlib.sha256((prior.STATE/'helper-native-retry').read_bytes()).hexdigest()))
cloud.contract.write_private(STATE/'endpoint-disabled.json', endpoint)
print('APT scope closed and endpoint stopped; all attempt history preserved')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('enable','accept','finish','close')); parser.add_argument('--case', choices=CASES)
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, exist_ok=True)
if args.action in ('accept','finish'):
require(args.case is not None, 'fixed case required')
(accept if args.action == 'accept' else finish)(args.case)
else: (enable if args.action == 'enable' else close)()
if __name__ == '__main__':
try: main()
except Exception as error:
print('CLOUD_APT_RETRY_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+14 -7
View File
@@ -98,7 +98,8 @@ def upgrade(payload):
require(general==old_general,'general policy changed')
require(all(manifest(Path(p))==v for p,v in preserved.items()) and markers[0].read_bytes()==marker and
(ledger.stat().st_dev,ledger.stat().st_ino)==inode,'preserved identity/attempt/ledger changed')
report.update(stage='installed_preserved',preserved_state=True,runtime_sha256=runtime_sha,apt_config_sha256=config_sha,apt_pin=pin,general_pin=general);save()
report.update(stage='installed_preserved',preserved_state=True,runtime_sha256=runtime_sha,apt_config_sha256=config_sha,apt_pin=pin,general_pin=general,
provider_sha256=sha(code),helper_sha256=sha(raw));save()
fcntl.flock(lockfd,fcntl.LOCK_UN);os.close(lockfd);lockfd=None
run(['systemd-analyze','verify','/etc/systemd/system/rmm-apt-helper.service'])
run(['systemctl','daemon-reload'])
@@ -135,17 +136,23 @@ def installed_metadata(payload):
def main():
parser=argparse.ArgumentParser();parser.add_argument('--local-file-fix',action='store_true');args=parser.parse_args()
parser=argparse.ArgumentParser();group=parser.add_mutually_exclusive_group()
group.add_argument('--local-file-fix',action='store_true')
group.add_argument('--native-retry',action='store_true',help='One versioned signed-retry upgrade; never a package retry itself')
args=parser.parse_args()
os.umask(0o077);m=cloud.contract
require(followup.evidence('gate')['unresolved']==0,'unresolved central job')
diagnosis=m.protected_json(followup.STATE/'diagnosis/checkpoint.json');require(diagnosis.get('verified') is True,'diagnostic evidence required')
reports=list(followup.STATE.glob('sandbox-*.json'))
if args.local_file_fix:
require(len(reports)==1,'exact prior successful sandbox upgrade required; never repeat migration')
prior=m.protected_json(reports[0]);require(prior['completed'] is True,'previous migration incomplete')
pins=m.protected_json(followup.STATE/'provider-before-local-file-fix.json')
if args.local_file_fix or args.native_retry:
require(len(reports)==(2 if args.native_retry else 1),'exact prior successful sandbox upgrades required; never repeat migration')
previous=[m.protected_json(p) for p in reports]
require(all(p['completed'] is True for p in previous),'previous migration incomplete')
pins=m.protected_json(followup.STATE/('provider-before-native-retry.json' if args.native_retry else 'provider-before-local-file-fix.json'))
candidates=[p for p in previous if p['runtime_sha256']==pins.get('runtime_sha256')] if args.native_retry else previous
require(len(candidates)==1,'reviewed prior runtime required');prior=candidates[0]
old=cloud.bootstrap.rpc('linux-vm.conf',installed_metadata,dict(pins,runtime_sha256=prior['runtime_sha256'],apt_pin=prior['apt_pin']))
artifact=(followup.STATE/'helper-local-file-fixed').read_bytes()
artifact=(followup.STATE/('helper-native-retry' if args.native_retry else 'helper-local-file-fixed')).read_bytes()
folders=[cloud.ROOT/'devstate/cloud-root/cases'/name for name in ('service_setup','service_start','service_stop','apt_native_install')]
folders += [followup.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
expected=[str(uuid.UUID(m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'])) for p in folders]
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Versioned cloud source/build/deploy, no DB/PKI reset or job approval."""
import argparse
import base64
import hashlib
import io
import json
import os
from pathlib import Path
import tarfile
import uuid
import cloud_mvp_acceptance as cloud
STATE = cloud.ROOT/'devstate/server-upgrades'
require = cloud.require
def stage(payload):
import io
import tarfile
root = Path('/opt/rust-discovery')
backup = root/'devstate'/('source-upgrade-'+payload['id']); backup.mkdir(mode=0o700)
raw = base64.b64decode(payload['tar'], validate=True)
require(hashlib.sha256(raw).hexdigest() == payload['sha256'], 'source archive hash')
def allowed(name):
return name in ('Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile',
'deploy/linux/rmm-apt-helper.service','tools/apt-transaction/apt_transaction.py',
'tools/apt-transaction/hook') or name.startswith(('crates/','migrations/'))
old_image = subprocess.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}'], capture_output=True, check=True).stdout.decode().strip()
manifest = {}; previous = {}
with tarfile.open(fileobj=io.BytesIO(raw), mode='r:') as archive:
for member in archive.getmembers():
name = member.name
require(member.isfile() and allowed(name) and not Path(name).is_absolute() and '..' not in Path(name).parts and name not in manifest, 'source path allowlist')
dest = root/name; data = archive.extractfile(member).read()
require(len(data) <= 2*1024*1024 and not dest.is_symlink(), 'source size/path')
for parent in dest.parents:
require(not parent.is_symlink(), 'source parent symlink')
if parent == root: break
if dest.exists():
old = dest.read_bytes(); previous[name] = hashlib.sha256(old).hexdigest()
retained = backup/'source'/name; retained.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
with retained.open('xb') as f: f.write(old); f.flush(); os.fsync(f.fileno())
else: previous[name] = None
manifest[name] = hashlib.sha256(data).hexdigest()
dest.parent.mkdir(mode=0o755, parents=True, exist_ok=True)
next_path = dest.with_name(dest.name+'.upgrade-next')
fd = os.open(next_path, os.O_CREAT|os.O_EXCL|os.O_WRONLY|os.O_NOFOLLOW, 0o644)
with os.fdopen(fd, 'wb') as f: f.write(data); f.flush(); os.fsync(f.fileno()); os.fchmod(f.fileno(), 0o644)
os.replace(next_path, dest)
# Build asynchronously. Captured compiler output remains root-private.
worker = backup/'build.py'
worker.write_text('''import json,os,subprocess
from pathlib import Path
os.umask(0o077)
root=Path('/opt/rust-discovery'); here=Path(__file__).parent
with (here/'build.log').open('xb') as log:
result=subprocess.run(['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),'build','migrate'],stdout=log,stderr=subprocess.STDOUT)
(here/'build-result.json').write_text(json.dumps({'exit_code':result.returncode}))
'''); worker.chmod(0o600)
record = dict(id=payload['id'], source_sha256=payload['sha256'], files=manifest, previous=previous, old_image=old_image, backup=str(backup))
(backup/'source.json').write_text(json.dumps(record)); (backup/'source.json').chmod(0o600)
unit = 'rmm-source-build-'+payload['id']
result = subprocess.run(['systemd-run','--unit='+unit,'--property=UMask=0077','--property=WorkingDirectory='+str(root),
'/usr/bin/python3',str(worker)], capture_output=True, timeout=15)
require(result.returncode == 0, 'source build launch incomplete')
return dict(record, unit=unit, launched=True, deployed=False)
def status(payload):
root = Path('/opt/rust-discovery'); backup = root/'devstate'/('source-upgrade-'+payload['id'])
record = json.loads((backup/'source.json').read_text())
result = backup/'build-result.json'
return dict(id=payload['id'], finished=result.exists(), exit_code=json.loads(result.read_text())['exit_code'] if result.exists() else None,
source_unchanged=all(hashlib.sha256((root/name).read_bytes()).hexdigest() == sha for name,sha in record['files'].items()))
def deploy(payload):
root = Path('/opt/rust-discovery'); backup = root/'devstate'/('source-upgrade-'+payload['id'])
record = json.loads((backup/'source.json').read_text())
require(json.loads((backup/'build-result.json').read_text())['exit_code'] == 0, 'build must pass')
require(all(hashlib.sha256((root/name).read_bytes()).hexdigest() == sha for name,sha in record['files'].items()), 'source drift')
require(not (backup/'deploy-result.json').exists(), 'deployment already recorded')
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')]
# Existing migration image runs without rebuild; preserves PG volume/PKI.
migrate = subprocess.run(cmd+['run','--rm','--no-deps','migrate'], capture_output=True, timeout=25)
require(migrate.returncode == 0, 'migration failed')
result = subprocess.run(cmd+['up','-d','--no-deps','api','gateway'], capture_output=True, timeout=25)
outcome = dict(deployed=result.returncode == 0, existing_state_preserved=True, approvals_created=False)
(backup/'deploy-result.json').write_text(json.dumps(outcome)); (backup/'deploy-result.json').chmod(0o600)
return outcome
def main():
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('stage','status','deploy')); parser.add_argument('--id')
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
if args.action == 'stage':
names = ['Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile','deploy/linux/rmm-apt-helper.service',
'tools/apt-transaction/apt_transaction.py','tools/apt-transaction/hook']
names += [str(p.relative_to(cloud.ROOT)) for base in ('crates','migrations') for p in (cloud.ROOT/base).rglob('*')
if p.is_file() and not p.is_symlink() and 'target' not in p.parts and '__pycache__' not in p.parts and p.suffix in ('.rs','.toml','.sql')]
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode='w:') as archive:
for name in sorted(set(names)):
path = cloud.ROOT/name; require(not path.is_symlink(), 'source symlink')
archive.add(path, arcname=name, recursive=False)
data = stream.getvalue(); ident = str(uuid.uuid4())
result = cloud.bootstrap.rpc('server.conf', stage, dict(id=ident, tar=base64.b64encode(data).decode(), sha256=hashlib.sha256(data).hexdigest()))
cloud.contract.write_private(STATE/(ident+'.json'), result)
print(dict(id=ident, launched=result['launched'], deployed=False))
else:
ident = str(uuid.UUID(args.id)); cloud.contract.protected_json(STATE/(ident+'.json'))
if args.action == 'deploy':
target = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
gate = cloud.bootstrap.rpc('server.conf', cloud.server, dict(kind='gate', tenant_id=cloud.TENANT, agent_id=target['agent_id']))
require(gate['unresolved'] == 0, 'unresolved cloud job; no server switch')
result = cloud.bootstrap.rpc('server.conf', status if args.action == 'status' else deploy, dict(id=ident))
print(result)
if __name__ == '__main__':
try: main()
except Exception as error:
print('CLOUD_SERVER_UPGRADE_INCOMPLETE:'+type(error).__name__); raise SystemExit(2) from None