fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 19:11:45 +01:00
parent da7eb8109c
commit c512bd6fb2
36 changed files with 2556 additions and 38 deletions
+102
View File
@@ -0,0 +1,102 @@
// Actual visible dev/test UI only; no API/DB approval or injected requests.
const fs = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const { isDeepStrictEqual: equal } = require('node:util');
const { chromium } = require('../../web/node_modules/playwright');
const root = path.resolve(__dirname, '../..');
const assert = value => { if (!value) throw new Error('Crash UI contract mismatch'); };
function readPrivate(file) {
const metadata = fs.lstatSync(file);
assert(metadata.isFile() && !metadata.isSymbolicLink() && metadata.uid === process.getuid()
&& (metadata.mode & 0o777) === 0o600 && metadata.size < 1048576);
return JSON.parse(fs.readFileSync(file, 'utf8'));
}
function checkJob(plan, expected, script) {
assert(plan.jobs.length === 1 && plan.resolved_targets.length === 1
&& crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
const job = plan.jobs[0], p = job.payload;
assert(p.tenant_id === expected.tenant_id && p.agent_id === expected.agent_id && p.asset_id === expected.asset_id
&& p.execution_identity === 'root' && p.max_runtime_seconds === 120 && p.max_output_bytes === 65536
&& Date.parse(p.start_before) > Date.now());
assert(equal(p.action, { action: 'run_script', interpreter: 'posix_sh', script_utf8: script,
arguments: [], expected_stdout: 'RMM_CRASH_COUNT=1\n' }));
}
(async () => {
const file = path.resolve(process.argv[2]), expected = readPrivate(file);
assert(expected.profile === 'dev/test' && expected.case === 'agent_crash'
&& expected.expected_principal === 'simulated-testprincipal-a' && typeof expected.read_only === 'boolean'
&& /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(expected.id));
const source = fs.readFileSync(path.join(root, 'deploy/linux/agent-crash-counter.py'), 'utf8');
const script = 'exec /usr/bin/python3 -I - ' + (expected.read_only ? 'read' : 'effect') + ' ' + expected.id
+ " <<'RMM_AGENT_CRASH_COUNTER'\n" + source + '\nRMM_AGENT_CRASH_COUNTER\n';
const login = readPrivate(path.join(root, 'credentials/server-browser-login.json'));
const browsers = path.join(root, 'web/node_modules/.cache/playwright-browsers');
const version = fs.readdirSync(browsers).find(x => /^chromium-/.test(x));
const executablePath = ['chrome-linux/chrome', 'chrome-linux64/chrome'].map(x => path.join(browsers, version, x)).find(x => fs.existsSync(x));
const browser = await chromium.launch({ headless: true, executablePath,
env: { ...process.env, HOME: path.join(root, 'web/node_modules/.cache/cloud-acceptance/home') } });
try {
const page = await browser.newPage({ ignoreHTTPSErrors: false, timezoneId: 'UTC' });page.setDefaultTimeout(20000);
let approvals = 0;
page.on('request', r => { if (new URL(r.url()).pathname.endsWith('/authorizations')) approvals++; });
await page.goto('https://localhost:18443/');
await page.getByLabel('Benutzername').fill(login.username);
await page.getByLabel('Passwort', { exact: true }).fill(login.password);
const response = page.waitForResponse(r => new URL(r.url()).pathname === '/auth/session' && r.status() === 200);
await page.getByRole('button', { name: 'Anmelden', exact: true }).click();
const session = await (await response).json();
assert(session.testsimulation === true && session.approval_origin === 'dev_test_browser_simulation' && session.principal === expected.expected_principal);
await page.getByRole('status').filter({ hasText: 'SIMULIERT' }).waitFor();
const detail = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Unveränderlicher Plan / Genehmigungsdetail', exact: true }) });
if (expected.phase === 'prepare') {
assert(expected.request.parameters.variants.length === 1 && expected.request.parameters.variants[0].script_utf8 === script
&& expected.request.parameters.expected_stdout === 'RMM_CRASH_COUNT=1\n');
await page.getByRole('checkbox', { name: 'Ziel ' + expected.asset_id, exact: true }).check();
await page.getByLabel('Aktion', { exact: true }).selectOption('run_script');
await page.getByRole('combobox', { name: 'Interpreter', exact: true }).selectOption('posix_sh');
await page.getByLabel('Exakte UTF-8 Skriptbytes', { exact: false }).fill(script);
await page.getByLabel('Erwartete stdout-Ausgabe', { exact: false }).fill('RMM_CRASH_COUNT=1\n');
await page.getByLabel('Ausführungsidentität', { exact: true }).selectOption('root');
await page.getByLabel('Laufzeitlimit (s)', { exact: true }).fill('120');
await page.getByLabel('Ausgabelimit (Bytes)', { exact: true }).fill('65536');
const start = new Date(Math.ceil(Date.now() / 60000) * 60000);
await page.getByLabel('Frühester Start (lokale Zeit)', { exact: true }).fill(start.toISOString().slice(0, 16));
await page.getByLabel('Startfrist / Ablauf (lokale Zeit)', { exact: true }).fill(new Date(start.getTime() + 15 * 60000).toISOString().slice(0, 16));
await page.getByLabel('Maximal parallel', { exact: true }).fill('1');
await page.getByLabel('Stopp nach Fehlern', { exact: true }).fill('1');
await page.getByLabel('Begründung', { exact: true }).fill(expected.request.reason);
await page.getByRole('button', { name: 'Plan erstellen · noch keine Ausführung' }).click();
const plan = JSON.parse(await detail.locator('pre').innerText());checkJob(plan, expected, script);
await detail.getByRole('button', { name: 'Antrag absenden · pending_approval' }).click();
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: /^Antrag \/ Ausführung / }) });
await operation.locator('pre').first().filter({ hasText: '"job_count": 0' }).waitFor();
const pending = JSON.parse(await operation.locator('pre').first().innerText());
assert(pending.approval_state === 'pending_approval' && pending.execution_state === null && pending.job_count === 0 && approvals === 0);
assert(path.dirname(path.resolve(expected.output_file)) === path.dirname(file));
fs.writeFileSync(expected.output_file, JSON.stringify({ plan, operation: pending }), { flag: 'wx', mode: 0o600 });
} else {
assert(expected.phase === 'approve');
await page.getByRole('button', { name: expected.operation_id, exact: true }).click();
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Antrag / Ausführung ' + expected.operation_id, exact: true }) });
const pending = JSON.parse(await operation.locator('pre').first().innerText());
assert(pending.approval_state === 'pending_approval' && pending.execution_state === null && pending.job_count === 0 && approvals === 0);
await operation.getByRole('button', { name: 'Unveränderlichen Plan zur Prüfung öffnen', exact: true }).click();
const plan = JSON.parse(await detail.locator('pre').innerText());
const { approval_state: currentState, ...currentPlan } = plan;
const { approval_state: preparedState, ...preparedPlan } = expected.plan;
assert(currentState === 'pending_approval' && preparedState === null && equal(currentPlan, preparedPlan));
checkJob(plan, expected, script);assert(equal(plan.jobs[0], expected.job));
const approve = detail.getByRole('button', { name: 'SIMULIERT: Diesen festen Plan ausdrücklich genehmigen', exact: true });
assert(await approve.isDisabled());
await detail.getByRole('checkbox', { name: 'Ich habe sämtliche festen Ziele', exact: false }).check();
await approve.click();
await operation.locator('p.warning').filter({ hasText: 'Antrag: approved' }).waitFor();
assert(approvals === 1);
}
await page.getByRole('button', { name: 'Abmelden', exact: true }).click();
} finally { await browser.close(); }
})().catch(error => {
const line = error.stack?.match(/cloud_crash_acceptance\.cjs:(\d+):\d+/)?.[1] || 'unknown';
console.log('CLOUD_CRASH_UI_INCOMPLETE:' + error.constructor.name + ':source_line=' + line);process.exitCode = 1;
});
@@ -0,0 +1,81 @@
// Actual development UI approval/revocation only; no API client or injected fetch.
const fs = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');
const { isDeepStrictEqual: equal } = require('node:util');
const { chromium } = require('../../web/node_modules/playwright');
const root = path.resolve(__dirname, '../..');
const assert = value => { if (!value) throw new Error('Offline UI contract mismatch'); };
function readPrivate(file) {
const s = fs.lstatSync(file);
assert(s.isFile() && !s.isSymbolicLink() && s.uid === process.getuid() && (s.mode & 0o777) === 0o600 && s.size < 1048576);
return JSON.parse(fs.readFileSync(file, 'utf8'));
}
function counterScript(id) {
assert(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(id));
return 'set -eu\numask 077\nd=/var/lib/rmm-agent-remote/offline-' + id + '\n'
+ 'test ! -e "$d"\nmkdir "$d"\nprintf \'%s\\n\' \'1\' > "$d/count"\n'
+ 'test "$(cat "$d/count")" = 1\nprintf \'%s\\n\' \'RMM_OFFLINE_COUNT=1\'\n';
}
(async () => {
const expected = readPrivate(path.resolve(process.argv[2]));
assert(expected.profile === 'dev/test' && ['offline_wait', 'expired_offline', 'revoked_before_start'].includes(expected.case)
&& ['approve', 'revoke'].includes(expected.phase) && expected.expected_principal === 'simulated-testprincipal-a');
const login = readPrivate(path.join(root, 'credentials/server-browser-login.json'));
const browsers = path.join(root, 'web/node_modules/.cache/playwright-browsers');
const version = fs.readdirSync(browsers).find(x => /^chromium-/.test(x));
const executablePath = ['chrome-linux/chrome', 'chrome-linux64/chrome'].map(x => path.join(browsers, version, x)).find(x => fs.existsSync(x));
const browser = await chromium.launch({ headless: true, executablePath, env: { ...process.env, HOME: path.join(root, 'web/node_modules/.cache/cloud-acceptance/home') } });
try {
const page = await browser.newPage({ ignoreHTTPSErrors: false, timezoneId: 'UTC' }); page.setDefaultTimeout(15000);
let approvals = 0, revocations = 0;
page.on('request', r => { const p = new URL(r.url()).pathname; if (p.endsWith('/authorizations')) approvals++; if (p.endsWith('/revocation') && r.method() === 'POST') revocations++; });
await page.goto('https://localhost:18443/');
await page.getByLabel('Benutzername').fill(login.username); await page.getByLabel('Passwort', { exact: true }).fill(login.password);
const response = page.waitForResponse(r => new URL(r.url()).pathname === '/auth/session' && r.status() === 200);
await page.getByRole('button', { name: 'Anmelden', exact: true }).click();
const session = await (await response).json();
assert(session.testsimulation === true && session.approval_origin === 'dev_test_browser_simulation' && session.principal === expected.expected_principal);
await page.getByRole('status').filter({ hasText: 'SIMULIERT' }).waitFor();
assert(await page.getByLabel('Mandant', { exact: true }).inputValue() === expected.tenant_id);
await page.getByRole('button', { name: expected.operation_id, exact: true }).click();
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Antrag / Ausführung ' + expected.operation_id, exact: true }) });
const state = JSON.parse(await operation.locator('pre').first().innerText());
assert(state.approval_state === (expected.phase === 'approve' ? 'pending_approval' : 'approved'));
if (expected.phase === 'approve') assert(state.job_count === 0 && state.execution_state === null);
await operation.getByRole('button', { name: 'Unveränderlichen Plan zur Prüfung öffnen', exact: true }).click();
const detail = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Unveränderlicher Plan / Genehmigungsdetail', exact: true }) });
const plan = JSON.parse(await detail.locator('pre').innerText());
const { approval_state: actualApproval, ...actual } = plan;
const { approval_state: plannedApproval, ...planned } = expected.plan;
assert(equal(actual, planned) && plan.jobs.length === 1 && plan.resolved_targets.length === 1
&& crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
const job = plan.jobs[0], p = job.payload;
assert(equal(job, expected.job) && p.agent_id === expected.agent_id && p.asset_id === expected.asset_id && p.tenant_id === expected.tenant_id
&& p.execution_identity === 'agent_service' && p.max_runtime_seconds === 60 && p.max_output_bytes === 65536);
assert(equal(p.action, { action: 'run_script', interpreter: 'posix_sh', script_utf8: counterScript(expected.id), arguments: [], expected_stdout: 'RMM_OFFLINE_COUNT=1\n' }));
assert(Date.parse(p.start_before) - Date.parse(p.not_before) === 130000 && Date.parse(p.start_before) > Date.now());
if (expected.phase === 'approve') {
assert(actualApproval === 'pending_approval' && plannedApproval === null && Date.parse(p.not_before) > Date.now() + 90000);
const approve = detail.getByRole('button', { name: 'SIMULIERT: Diesen festen Plan ausdrücklich genehmigen', exact: true });
assert(await approve.isDisabled()); await detail.getByRole('checkbox', { name: 'Ich habe sämtliche festen Ziele', exact: false }).check(); await approve.click();
await operation.locator('p.warning').filter({ hasText: 'Antrag: approved' }).waitFor();
assert(approvals === 1 && revocations === 0);
} else {
assert(expected.case === 'revoked_before_start' && actualApproval === 'approved' && session.permissions.includes('plans:revoke') && Date.parse(p.not_before) > Date.now() + 1000);
const control = detail.locator('details').filter({ has: page.locator('summary').filter({ hasText: /^Planberechtigung widerrufen$/ }) });
await control.locator('summary').click();
await control.getByLabel('Grund für Planberechtigung widerrufen', { exact: true }).fill('Explicit dev/test offline acceptance: revoke this exact future plan before any start');
const nonce = await control.locator('code').innerText();
await control.getByLabel('Bestätigungskennung eingeben', { exact: true }).fill(nonce);
await control.getByRole('checkbox', { name: 'Ich bestätige diese konkrete administrative Wirkung ausdrücklich.' }).check();
await control.getByRole('button', { name: 'Planberechtigung widerrufen · ausdrücklich ausführen', exact: true }).click();
await control.locator('pre').filter({ hasText: '"approval_state": "revoked"' }).waitFor();
assert(approvals === 0 && revocations === 1);
}
await page.getByRole('button', { name: 'Abmelden', exact: true }).click();
} finally { await browser.close(); }
})().catch(error => {
const line = error.stack?.match(/cloud_offline_acceptance\.cjs:(\d+):\d+/)?.[1] || 'unknown';
console.log('CLOUD_OFFLINE_UI_INCOMPLETE:' + error.constructor.name + ':source_line=' + line); process.exitCode = 1;
});
+52 -13
View File
@@ -19,6 +19,15 @@ STATE=BASE/'cases'
CASES=('apt_configure_recovery','apt_native_update','ensure_package','apply_updates')
def runtime_session():
latest=BASE/'journal-capacity-runtime.json'
if latest.exists():
report=cloud.contract.protected_json(latest)
require(report['completed'] and report['preserved_state'],'qualified preserved capacity runtime')
return report,BASE/'capacity-recovery'
return cloud.contract.protected_json(BASE/'configure-runtime.json'),BASE
def recovery_request(device,target):
m=cloud.contract;data=prior.adapters.planning_from_device(device,recovery=True)
closed=m.protected_json(BASE/'readonly-central-result.json')['result']
@@ -49,14 +58,14 @@ def recovery_request(device,target):
def enable():
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
ready=m.protected_json(BASE/'configure-runtime.json')
ready,session=runtime_session();session.mkdir(mode=0o700,exist_ok=True)
require(ready['completed'] and ready['readiness_only'] and ready['preserved_state'] and not ready['native_acceptance'],
'qualified native probe, no invented package acceptance')
target=prior.prior.identity();pin=ready['apt_pin'];identity=pin['identity']
require(all(identity[k]==target[k] for k in target),'same permanent Cloud VM')
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=True,
agent_id=target['agent_id'],binary_sha256=ready['helper_sha256']))
m.write_private(BASE/'recovery-endpoint-enabled.json',endpoint)
m.write_private(session/'recovery-endpoint-enabled.json',endpoint)
peers=cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,target)
require(peers['ledger']['faults']==0 and peers['apt']['provider_stack_sha256']==pin['provider_stack_sha256'],
'actual fixed helper peer and stack')
@@ -67,15 +76,43 @@ def enable():
apt_config_sha256=ready['apt_config_sha256'],provider_sha256=ready['provider_sha256'],
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
policy=dict(schema_version=1,profile='development',expires_at=(m.now()+dt.timedelta(hours=1)).isoformat(),scopes=[scope])
old=m.protected_json(cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
old=m.protected_json(BASE/'recovery-policy-closed.json' if session!=BASE else cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
record=cloud.bootstrap.rpc('server.conf',prior.replace_policy,dict(id=str(uuid.uuid4()),previous_sha256=old['policy_sha256'],policy=policy))
m.write_private(BASE/'recovery-policy.json',dict(record,policy=policy))
m.write_private(session/'recovery-policy.json',dict(record,policy=policy))
print('Fixed APT runtime endpoint and expiring limits enabled; no job or approval created')
def accept(case):
def unsubmitted_preparation(payload):
import re
root=Path('/opt/rust-discovery');ident=payload['test_result_id']
require(isinstance(ident,str) and re.fullmatch('[A-Za-z0-9_-]{1,128}',ident),'fixed preparation test ID')
query="SELECT count(*) FROM operation_plans WHERE tenant_id='11111111-1111-4111-8111-111111111111' AND position('"+ident+"' in convert_from(payload,'UTF8'))>0;"
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
count=int(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip())
return dict(persisted_plans=count)
def accept(case,resume_unsubmitted=False):
m=cloud.contract;target=prior.prior.identity()
require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
ready,session=runtime_session();cases=session/'cases';cases.mkdir(mode=0o700,exist_ok=True)
if session!=BASE and case=='apt_configure_recovery':
original=m.protected_json(STATE/case/'checkpoint-capacity-terminal.json')
old_job=original['plan']['jobs'][0]
old_result=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=old_job['job_id']))
counts=prior.prior.evidence('gate',job_id=old_job['job_id'])
require(original['not_started_capacity_block'] and old_result['state'] in ('expired','cancelled') and counts['permits']==0,
'preserve original approved no-start; new attempt requires fresh UI approval')
folder=cases/case
if resume_unsubmitted:
require(folder.is_dir() and {p.name for p in folder.iterdir()}=={'prepare.json'},'only an unsubmitted UI preparation may be archived')
old=m.protected_json(folder/'prepare.json')
require(old['case']==case and old['phase']=='prepare' and all(old[k]==target[k] for k in target),'same fixed prior preparation target')
untouched=cloud.bootstrap.rpc('server.conf',unsubmitted_preparation,dict(test_result_id=old['request']['parameters']['apt_manifest']['test_result_id']))
require(untouched==dict(persisted_plans=0),'existing plan must be reviewed, never silently replaced')
archived=cases/(case+'.unsubmitted-ui-'+uuid.uuid4().hex)
os.rename(folder,archived);m.write_private(archived/'not-submitted.json',untouched)
# A maintenance restart only refreshes passive inventory from the preserved
# agent; it never issues a package diagnostic or executes a job.
deadline=time.monotonic()+90
@@ -87,28 +124,30 @@ def accept(case):
except m.GuardError:
require(time.monotonic()<deadline,'fresh stored exact planning unavailable; no job submitted');time.sleep(3)
request=recovery_request(device,target) if case=='apt_configure_recovery' else None
prior.accept(case,request=request,folder_root=STATE)
if request is not None and session!=BASE:
request['reason']+='; prior fixed plan expired with zero grants at the old journal capacity; preserve it and authorize this fresh plan separately'
prior.accept(case,request=request,folder_root=cases)
def close():
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
record=m.protected_json(BASE/'recovery-policy.json')
ready,session=runtime_session();record=m.protected_json(session/'recovery-policy.json')
closed=cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=record['policy_sha256'],backup=record['backup']))
m.write_private(BASE/'recovery-policy-closed.json',closed)
ready=m.protected_json(BASE/'configure-runtime.json')
m.write_private(session/'recovery-policy-closed.json',closed)
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=False,
agent_id=prior.prior.identity()['agent_id'],binary_sha256=ready['helper_sha256']))
m.write_private(BASE/'recovery-endpoint-disabled.json',endpoint)
m.write_private(session/'recovery-endpoint-disabled.json',endpoint)
print('APT test limits closed and endpoint stopped; original histories retained')
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('action',choices=('enable','accept','finish','close'));p.add_argument('--case',choices=CASES)
p.add_argument('--resume-unsubmitted-ui',action='store_true',help='Preserve a failed preparation only after proving no plan was persisted')
args=p.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
if args.action in ('accept','finish'):
require(args.case is not None,'one fixed case required')
if args.action=='accept':accept(args.case)
else:prior.finish(args.case,folder_root=STATE)
if args.action=='accept':accept(args.case,args.resume_unsubmitted_ui)
else:prior.finish(args.case,folder_root=runtime_session()[1]/'cases')
else:(enable if args.action=='enable' else close)()
@@ -116,5 +155,5 @@ if __name__=='__main__':
try:main()
except Exception as error:
message=str(error)
print(message if message.startswith('remote bootstrap incomplete: ') else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
print(message if message.startswith(('remote bootstrap incomplete: ','APT_UI_INCOMPLETE:source_lines=')) else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+4 -1
View File
@@ -205,7 +205,10 @@ def accept(case, *, request=None, folder_root=STATE):
cloud.prepare_browser()
def browser(file):
result = subprocess.run(['node', str(cloud.ROOT/'deploy/browser/cloud_apt_acceptance.cjs'), str(file)], capture_output=True, timeout=120)
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint, never repeat automatically')
if result.returncode!=0:
import re
lines=re.findall(rb'CLOUD_APT_UI_INCOMPLETE:[A-Za-z]+:source_line=([0-9]+|unknown)',result.stdout)
raise RuntimeError('APT_UI_INCOMPLETE:source_lines='+','.join(x.decode('ascii') for x in lines))
print(case+': actual UI preparation', flush=True); browser(prepare)
prepared = m.protected_json(output); plan = prepared['plan']; operation = prepared['operation']; job = plan['jobs'][0]
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'immutable single job')
+7
View File
@@ -119,6 +119,13 @@ def status_vm(payload):
result[name]=dict(faults=db.execute('SELECT count(*) FROM '+table+' WHERE fault IS NOT NULL').fetchone()[0],
receipts=db.execute('SELECT count(*) FROM checkpoint_repair_receipts').fetchone()[0] if 'checkpoint_repair_receipts' in tables else 0,
integrity_ok=db.execute('PRAGMA integrity_check').fetchone()[0]=='ok')
if payload.get('job_id'):
job=str(uuid.UUID(payload['job_id']));attempt=str(uuid.UUID(payload['attempt_id']))
if name=='agent':
result[name]['attempts']=[dict(state=s,revision=r,grant_id=g) for s,r,g in db.execute(
'SELECT state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt))]
else:
result[name]['attempt_consumed']=db.execute('SELECT count(*) FROM consumed WHERE job=? AND attempt=?',(job,attempt)).fetchone()[0]
if name=='helper':
result[name].update(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0])
+262
View File
@@ -0,0 +1,262 @@
#!/usr/bin/env python3
"""One bounded Cloud agent SIGKILL; independent root helper closes its outcome.
Requires fixed dev/test UI approval twice, a qualified agent and known closed
root-policy hash. No API/DB approval, ledger edits, retry or unsigned recovery.
Do not execute alongside another VM/runtime/approval operation.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import time
import uuid
import cloud_delivery_acceptance as delivery
import cloud_apt_followup as prior
import cloud_root_evidence
import cloud_root_policy
cloud = delivery.cloud
require = cloud.contract.require
STATE = cloud.ROOT/'devstate/cloud-crash'
def script(identifier, read_only=False):
identifier = str(uuid.UUID(identifier))
source = (cloud.ROOT/'deploy/linux/agent-crash-counter.py').read_text()
return ("exec /usr/bin/python3 -I - "+('read' if read_only else 'effect')+' '+identifier+
" <<'RMM_AGENT_CRASH_COUNTER'\n"+source+'\nRMM_AGENT_CRASH_COUNTER\n')
def vm(payload):
"""Read retained evidence; optionally signal only the exact bound MainPID."""
import signal
import sqlite3
import stat
state = Path('/var/lib/rmm-agent-remote')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'permanent identity')
for path,key in ((Path('/usr/local/libexec/rmm-agent-remote'),'agent_sha256'),
(Path('/usr/local/libexec/rmm-privileged-helper'),'helper_sha256')):
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == payload[key],'qualified pinned artifact')
def process(unit, expected_sha):
raw = subprocess.run(['systemctl','show',unit,'-p','MainPID','--value'],capture_output=True,check=True,timeout=10).stdout
pid = int(raw);require(pid > 1,'active pinned service')
require(hashlib.sha256(Path('/proc/'+str(pid)+'/exe').read_bytes()).hexdigest() == expected_sha,'running service artifact')
start = Path('/proc/'+str(pid)+'/stat').read_text().rsplit(')',1)[1].split()[19]
return dict(pid=pid,start=start)
processes = dict(agent=process('rmm-agent-remote.service',payload['agent_sha256']),
helper=process('rmm-helper-cloud.service',payload['helper_sha256']))
result = dict(processes=processes)
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','native journal integrity')
result['native_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
if payload.get('job_id'):
job = str(uuid.UUID(payload['job_id']));attempt = str(uuid.UUID(payload['attempt_id']))
rows = db.execute('SELECT state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
result['attempts'] = [dict(state=r[0],revision=r[1],grant_id=r[2]) for r in rows]
result['agent_last_decision'] = db.execute('SELECT last_decision FROM reconcile_meta').fetchone()[0]
helper = Path('/var/lib/rmm-helper/consumption.sqlite')
with sqlite3.connect(helper.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','root ledger integrity')
result['ledger'] = dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],
faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0])
if payload.get('job_id'):
rows = db.execute('SELECT c.grant_id,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.job=? AND c.attempt=?',(job,attempt)).fetchall()
result['root_attempts'] = []
for grant,finished,start,outcome in rows:
stored = json.loads(start);request = stored['request'];permit = json.loads(bytes(request['permit_bytes']))
require(permit['job_id'] == job and permit['attempt_id'] == attempt and permit['agent_id'] == payload['agent_id']
and permit['grant_id'] == grant and hashlib.sha256(bytes(request['job_bytes'])).hexdigest() == payload['job_payload_sha256']
and request['plan_hash'] == payload['plan_sha256'],'exact durable root start binding')
result['root_attempts'].append(dict(grant_id=grant,finished=json.loads(finished) if finished else None,
outcome=json.loads(outcome) if outcome else None,start_sha256=hashlib.sha256(start).hexdigest()))
head = db.execute('SELECT last_decision FROM helper_sync_meta').fetchone()[0]
if head:
row = db.execute('SELECT response FROM helper_sync_decisions WHERE decision_id=?',(head,)).fetchone()
if row:
signed = json.loads(row[0]);decision = json.loads(bytes(signed['decision_bytes']))
selected = [a for a in decision['attempts'] if a['job_id'] == job and a['attempt_id'] == attempt]
result['signed_sync'] = dict(decision_id=head,ready=decision['ready'],helper_ready=decision['helper_ready'],
payload_sha256=hashlib.sha256(bytes(signed['decision_bytes'])).hexdigest(),signature_sha256=hashlib.sha256(bytes(signed['signature'])).hexdigest(),
selected=selected)
if payload.get('job_id'):
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
result['agent_consumed'] = [dict(grant_id=g,finished=json.loads(o) if o else None) for g,o in db.execute('SELECT grant_id,result FROM consumed WHERE job=? AND attempt=?',(job,attempt))]
identifier = str(uuid.UUID(payload['id']));directory = helper.parent/('crash-'+identifier)
require(not directory.is_symlink(),'counter symlink')
result['counter'] = None
if directory.exists():
require(directory.stat().st_uid == 0 and stat.S_IMODE(directory.stat().st_mode) == 0o700,'private root counter')
path = directory/'count'
if path.exists():
fd = os.open(path,os.O_RDONLY|os.O_NOFOLLOW)
with os.fdopen(fd,'rb') as stream:
metadata = os.fstat(stream.fileno());require(stat.S_ISREG(metadata.st_mode) and metadata.st_uid == 0
and metadata.st_nlink == 1 and stat.S_IMODE(metadata.st_mode) == 0o600 and metadata.st_size <= 16,'fixed root counter file')
raw = stream.read();result['counter'] = int(raw)
if payload['mode'] == 'kill':
require(processes == payload['processes'],'exact original process instance')
require(len(result['attempts']) == len(result['root_attempts']) == len(result['agent_consumed']) == 1
and result['attempts'][0]['state'] == 'running' and result['root_attempts'][0]['finished'] is None
and result['root_attempts'][0]['outcome'] is None and result['counter'] == 1
and result['ledger'] == dict(consumed=payload['baseline_consumed']+1,unresolved=1,faults=0),'one actual running root effect; deterministic independent closure')
require(result['attempts'][0]['grant_id'] == result['root_attempts'][0]['grant_id'] == result['agent_consumed'][0]['grant_id'],'one consumed grant on both sides')
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-agent-crash-'+identifier)
backup.mkdir(mode=0o700)
fd = os.open(backup/'before-signal.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'w') as stream:json.dump(result,stream);stream.flush();os.fsync(stream.fileno())
# pidfd prevents a PID-reuse race. No unit stop, cgroup/process-tree kill,
# helper signal or package-manager intervention exists here.
fd = os.pidfd_open(processes['agent']['pid'])
try:
require(process('rmm-agent-remote.service',payload['agent_sha256']) == processes['agent'],'same agent before signal')
signal.pidfd_send_signal(fd,signal.SIGKILL)
finally: os.close(fd)
require(process('rmm-helper-cloud.service',payload['helper_sha256']) == processes['helper'],'independent helper survived')
return dict(before=result,agent_signal='SIGKILL',helper_unchanged=True,backup=str(backup))
require(payload['mode'] == 'inspect','fixed fault operation')
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
parser.add_argument('--expected-agent-sha256',required=True)
parser.add_argument('--expected-helper-sha256',required=True)
parser.add_argument('--expected-root-policy-sha256',required=True)
args = parser.parse_args();os.umask(0o077)
for value in (args.expected_agent_sha256,args.expected_helper_sha256,args.expected_root_policy_sha256):
require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified artifact/config pin')
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
cloud.prepare_browser()
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
settings = dict(tenant_id=cloud.TENANT,**identity)
backend = cloud.Backend()
def evidence(kind,**extra):return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind=kind,tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**extra))
require(evidence('gate')['unresolved'] == 0,'previous central effects must be closed')
identifier = str(uuid.uuid4())
scope = dict(id=identifier,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,helper_sha256=args.expected_helper_sha256)
initial = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(scope,mode='inspect'))
require(initial['native_unresolved'] == initial['ledger']['unresolved'] == initial['ledger']['faults'] == 0
and initial['counter'] is None,'native/history must be settled; untouched unique counter')
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
require(observed['sync_ready'] and not observed['restore_blocked'],'signed sync before new root plan')
peers = cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,dict(settings,general_only=True))
pin = peers['general'];binding = pin['identity']
root_scope = {k:binding[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')}
root_scope.update(principals=['simulated-testprincipal-a'],helper_policy_sha256=pin['policy_sha256'],
acceptance_evidence_id='cloud-agent-crash-independent-helper:'+identifier,allowed_services=[],allow_root_scripts=True,allow_reboot=False)
policy = dict(schema_version=1,profile='development',expires_at=(cloud.contract.now()+dt.timedelta(minutes=15)).isoformat(),scopes=[root_scope])
folder = STATE/identifier;folder.mkdir(mode=0o700,parents=True)
checkpoint = dict(profile='dev/test',case='agent_crash',expected_principal='simulated-testprincipal-a',**settings,
id=identifier,initial=initial,qualified=scope,stage='prepared_no_job',no_retry=True)
path = folder/'checkpoint.json'
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
save()
opened = cloud.bootstrap.rpc('server.conf',prior.narrow_policy,dict(id=identifier,expected_sha256=args.expected_root_policy_sha256,policy=policy))
checkpoint['policy_open'] = opened;save()
def browser(value, name):
file = folder/(name+'.json');cloud.contract.write_private(file,value)
process = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_crash_acceptance.cjs'),str(file)],capture_output=True,timeout=120)
require(process.returncode == 0,'actual UI incomplete; preserve pending plan; never resubmit/approve again')
def authorize(read_only):
# A busy helper may have temporarily removed root capabilities from the
# new process's hello. Wait for genuine measured readvertisement/Sync;
# never plan a new effect against a stale capability or readiness flag.
deadline = time.monotonic()+100
while True:
device = backend.call('inventory.get_device',dict(tenant_id=cloud.TENANT,asset_id=identity['asset_id']))
current = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
if (device['agent_id'] == identity['agent_id'] and device['presence'] == 'online'
and {'run_script','root:run_script','helper.root.v1','helper.policy.sha256:'+pin['policy_sha256']} <= set(device['capabilities'])
and current['sync_ready'] and not current['restore_blocked']):break
require(time.monotonic() < deadline,'fresh root capability and signed Sync before new plan');time.sleep(2)
request = cloud.contract.case_input('service_setup',settings)
request['parameters'].update(variants=[dict(os_family='linux',interpreter='posix_sh',encoding='utf-8',script_utf8=script(identifier,read_only))],
expected_stdout='RMM_CRASH_COUNT=1\n',timeout_seconds=120,max_output_bytes=65536)
request['reason'] = 'Explicit bounded Cloud agent crash '+('independent final counter read' if read_only else 'one root counter effect, helper remains alive for90s')
common = dict(profile='dev/test',case='agent_crash',expected_principal=checkpoint['expected_principal'],**settings,id=identifier,read_only=read_only,request=request)
name = 'postcheck' if read_only else 'effect'
output = folder/(name+'-prepared.json')
browser(dict(common,phase='prepare',output_file=str(output)),name+'-prepare')
prepared = cloud.contract.protected_json(output);plan=prepared['plan'];pending=prepared['operation']
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'],'one immutable fixed plan')
job = plan['jobs'][0];gate = evidence('gate',job_id=job['job_id'])
stored = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,operation_id=pending['operation_id']))
require(stored['approval_state'] == 'pending_approval' and stored['job_count'] == 0 and stored['execution_state'] is None
and stored['plan_hash'] == plan['plan_hash'] and all(gate[k] == 0 for k in ('jobs','outbox','permits')),'pending zero artifacts')
record = dict(plan=plan,job=job,operation_id=pending['operation_id'],preapproval=gate,request=request)
checkpoint[name] = record;save()
browser(dict(common,phase='approve',**record),name+'-approve')
return record
effect = authorize(False);job=effect['job']
fixed = dict(scope,job_id=job['job_id'],attempt_id=job['payload']['attempt_id'],job_payload_sha256=job['job_payload_hash'],plan_sha256=effect['plan']['plan_hash'])
deadline = time.monotonic()+90
while True:
before = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
if before['counter'] == 1 and before['root_attempts'] and before['root_attempts'][0]['outcome'] is None:break
require(time.monotonic() < deadline,'counter/running marker missing; never kill outside bound window');time.sleep(1)
killed = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='kill',processes=initial['processes'],baseline_consumed=initial['ledger']['consumed']))
checkpoint['signal'] = killed;save()
deadline = time.monotonic()+100
while True:
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
if result['state'] == 'unknown':
unknown = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
if unknown['root_attempts'][0]['outcome'] is None and unknown['attempts'][0]['state'] == 'unknown':break
require(result['state'] not in cloud.contract.TERMINAL and time.monotonic() < deadline,'actual durable and central unknown not observed; preserve same attempt');time.sleep(1)
require(result['result']['error_code'] == 'AGENT_PROCESS_INTERRUPTED' and unknown['counter'] == 1
and unknown['processes']['agent'] != initial['processes']['agent']
and unknown['processes']['helper'] == initial['processes']['helper'],'actual new agent process; independent live helper')
checkpoint['unknown'] = dict(central=result,native=unknown);save()
deadline = time.monotonic()+180
while True:
final = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
native = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
sync = native.get('signed_sync',{})
if (final['state'] in cloud.contract.TERMINAL and native['ledger']['unresolved'] == 0 and sync.get('ready') and sync.get('helper_ready')
and native['agent_last_decision'] == sync.get('decision_id')):break
require(time.monotonic() < deadline,'same-attempt definitive reconciliation missing; never unlock/retry');time.sleep(2)
require(final['state'] == 'succeeded' and final['result']['exit_code'] == 0 and bytes(final['result']['stdout']) == b'RMM_CRASH_COUNT=1\n'
and final['result']['evidence']['postconditions_passed'] and native['counter'] == 1
and len(native['root_attempts']) == len(native['agent_consumed']) == len(native['attempts']) == 1
and native['ledger'] == dict(consumed=initial['ledger']['consumed']+1,unresolved=0,faults=0),'one actual helper completion and consumption')
selected = sync['selected'];grant = killed['before']['root_attempts'][0]['grant_id']
require(len(selected) == 1 and selected[0]['grant_id'] == grant and selected[0]['unlock'] is True
and selected[0]['outcome_state'] == 'succeeded' and selected[0]['reason'] == 'DEFINITIVE_OUTCOME_RECONCILED'
and selected[0]['revision'] > unknown['attempts'][0]['revision']
and selected[0]['job_payload_hash'] == fixed['job_payload_sha256'] and selected[0]['plan_hash'] == fixed['plan_sha256']
and native['root_attempts'][0]['grant_id'] == native['agent_consumed'][0]['grant_id'] == grant,'exact existing signed Sync closure')
checkpoint.update(final=final,native=native,stage='reconciled');save()
def audit(record):
record['postapproval'] = evidence('gate',job_id=record['job']['job_id'])
record['audit'] = evidence('audit',operation_id=record['operation_id']);save()
require([record['postapproval'][k] for k in ('jobs','outbox','permits')] == [1,1,1]
and record['audit'] == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=record['plan']['plan_hash']),'one authorized immutable attempt')
audit(effect)
postcheck = authorize(True);deadline=time.monotonic()+90
while True:
checked = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=postcheck['job']['job_id']))
if checked['state'] in cloud.contract.TERMINAL or checked['state'] == 'unknown':break
require(time.monotonic() < deadline,'independent read outcome missing');time.sleep(2)
postcheck['result'] = checked;save()
require(checked['state'] == 'succeeded' and checked['result']['exit_code'] == 0
and checked['result']['evidence']['postconditions_passed'] and bytes(checked['result']['stdout']) == b'RMM_CRASH_COUNT=1\n','independent final counter still one')
audit(postcheck)
require(evidence('gate')['unresolved'] == 0,'all effects must be closed before policy close')
closed = cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=opened['policy_sha256'],backup=opened['backup']))
checkpoint.update(policy_closed=closed,verified=True,stage='native_dev_test_accepted');save()
print('PASS bounded Cloud agent crash: durable+central unknown, same-attempt root outcome and signed Sync, one effect, separate UI-approved counter1; dev/test only')
if __name__ == '__main__':
try:main()
except Exception as error:
print('CLOUD_CRASH_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
raise SystemExit(2) from None
@@ -31,6 +31,9 @@ def vm_pending(payload):
events.append(dict(message_id=ident,agent_id=e[4].decode(),snapshot_id=snap[1].decode(),
observed_at_ms=e[9],sequence=e[8],data_sha256=hashlib.sha256(snap[6]).hexdigest(),
payload_sha256=hashlib.sha256(b''.join(raw for number,raw in parts if number!=6)).hexdigest(),
collection_started_at=data.get('collection_started_at'),collection_completed_at=data.get('collection_completed_at'),
observations=[dict(scope=o.get('scope'),source_id=o.get('source_id'),quality=o.get('quality'),observed_at=o.get('observed_at'),
bytes=len(json.dumps(o,separators=(',',':')).encode())) for o in data.get('observations',[])],
recovery_scope_present=any(o.get('scope')=='linux.apt.recovery-planning' for o in data.get('observations',[]))))
return dict(events=events,other_unacked=other)
+355
View File
@@ -0,0 +1,355 @@
#!/usr/bin/env python3
"""Preserving Cloud journal-capacity maintenance; never a job/history reset.
The copy-only mode changes only its own private disposable database copy.
Installation requires a separate explicit option and qualified artifact hashes.
"""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import re
import uuid
import cloud_apt_recovery_upgrade as previous
cloud = previous.cloud
require = cloud.require
STATE = previous.STATE
def copy_reproducer(payload):
import sqlite3
os.umask(0o077)
source = Path('/var/lib/rmm-helper/consumption.sqlite')
state = Path('/var/lib/rmm-agent-remote')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent Cloud identity')
require(not source.is_symlink() and source.stat().st_uid == 0 and source.stat().st_mode&0o777 == 0o600, 'private original ledger')
def fingerprint(db):
h = hashlib.sha256()
tables = db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name").fetchall()
for name,sql in tables:
require(name.replace('_','').isalnum(), 'fixed stored table name')
h.update(json.dumps([name,sql],separators=(',',':')).encode())
for row in db.execute('SELECT * FROM "'+name+'" ORDER BY rowid'):
data = [dict(blob_sha256=hashlib.sha256(v).hexdigest(),length=len(v)) if isinstance(v,bytes) else v for v in row]
h.update(json.dumps(data,separators=(',',':'),ensure_ascii=True).encode())
return h.hexdigest()
def identity():
s = source.stat()
return dict(device=s.st_dev,inode=s.st_ino,size=s.st_size,sha256=hashlib.sha256(source.read_bytes()).hexdigest())
before = identity()
with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True) as original:
require(original.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'original integrity')
require({r[0] for r in original.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants'])
and len(payload['expected_grants']) == 10
and original.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0
and original.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0,
'exact ten known closed Root consumptions')
original_fingerprint = fingerprint(original)
folder = Path('/var/lib/rmm-bootstrap-history')/('journal-capacity-copy-'+str(uuid.UUID(payload['id'])))
folder.mkdir(mode=0o700)
copy = folder/'ledger-copy.sqlite'
fd = os.open(copy,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600);os.close(fd)
with sqlite3.connect(copy) as duplicate:
original.backup(duplicate)
require(fingerprint(duplicate) == original_fingerprint, 'consistent private SQLite backup')
page_size = duplicate.execute('PRAGMA page_size').fetchone()[0]
page_count = duplicate.execute('PRAGMA page_count').fetchone()[0]
old = duplicate.execute('PRAGMA max_page_count=16384').fetchone()[0]
require(page_size == 4096 and old == 16384 and page_count <= old, 'actual bounded old capacity')
failed = None
try:
duplicate.execute('BEGIN IMMEDIATE')
duplicate.execute('CREATE TABLE journal_capacity_probe(payload BLOB NOT NULL)')
duplicate.execute('INSERT INTO journal_capacity_probe VALUES(zeroblob(524288))')
except sqlite3.DatabaseError as error:
failed = getattr(error,'sqlite_errorcode',None)
finally:
duplicate.rollback()
require(failed == sqlite3.SQLITE_FULL, 'old limit must fail with SQLITE_FULL13 on the copied actual DB')
require(fingerprint(duplicate) == original_fingerprint, 'old-capacity test rolled back every row/schema change')
bounded = duplicate.execute('PRAGMA max_page_count=65536').fetchone()[0]
require(bounded == 65536, 'bounded 256 MiB new capacity')
duplicate.execute('BEGIN IMMEDIATE')
duplicate.execute('CREATE TABLE journal_capacity_probe(payload BLOB NOT NULL)')
duplicate.execute('INSERT INTO journal_capacity_probe VALUES(zeroblob(524288))')
require(duplicate.execute('SELECT length(payload) FROM journal_capacity_probe').fetchone()[0] == 524288, 'same copied-DB insertion passes at the new finite cap')
duplicate.rollback()
require(fingerprint(duplicate) == original_fingerprint and duplicate.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'new-capacity test rolled back every row/schema change')
after = identity()
require(after == before, 'original DB bytes, size and inode must remain unchanged')
report = dict(verified=True,backup=str(folder),old_cap_pages=16384,new_cap_pages=65536,page_size=page_size,page_count=page_count,
old_error_code=failed,probe_bytes=524288,new_insert_passed=True,copy_original_content_sha256=original_fingerprint,
original_before=before,original_after=after,original_unchanged=True,rows_and_schema_rolled_back=True,
original_package_effects=False,original_sql_mutations=False)
fd = os.open(folder/'report.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'w') as stream:json.dump(report,stream,indent=2);stream.flush();os.fsync(stream.fileno())
return report
def install_capacity(payload):
import fcntl
import sqlite3
import time
import traceback
os.umask(0o077)
base = Path('/var/lib/rmm-apt-transaction'); state = Path('/var/lib/rmm-agent-remote')
ledger = Path('/var/lib/rmm-helper/consumption.sqlite')
helper = Path('/usr/local/libexec/rmm-privileged-helper'); agent = Path('/usr/local/libexec/rmm-agent-remote')
provider = Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')
config = Path('/etc/rmm/apt-helper.json'); proof_file = Path('/etc/rmm-remote/apt-helper-proof.json')
metadata = base/'install-metadata.json'; readiness_file = base/'native-readiness.json'
dropin = Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
reduced = ('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
def run(args, **kwargs):
return subprocess.run(args, capture_output=True, check=True, timeout=90, **kwargs).stdout
def sha(raw): return hashlib.sha256(raw).hexdigest()
def canonical(value): return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()
def fsync_dir(path):
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try: os.fsync(fd)
finally: os.close(fd)
def create(path, raw, mode=0o600, gid=0):
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
with os.fdopen(fd, 'wb') as stream:
stream.write(raw); stream.flush(); os.fchown(stream.fileno(), 0, gid)
os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno())
fsync_dir(path.parent)
def manifest(path):
require(not path.is_symlink(), 'preserved directory symlink')
if not path.exists(): return {'__absent__':True}
require(path.is_dir(), 'preserved directory required')
result = {}
for p in [path, *sorted(path.rglob('*'))]:
s = p.lstat(); require(not p.is_symlink() and (p.is_dir() or p.is_file()), 'preserved regular state required')
result[str(p.relative_to(path))] = dict(dev=s.st_dev, inode=s.st_ino, mode=s.st_mode,
uid=s.st_uid, gid=s.st_gid, sha256=sha(p.read_bytes()) if p.is_file() else None)
return result
def guard():
require(len(payload['expected_grants']) == 10, 'exact ten original closed Root grants')
require(payload['partial_grant'] == '31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0' and
payload['partial_proof_sha256'] == 'dc3bf3870ea77c12f9291bac712c53ae72194eaec980a8646155a7fb5c0dbe21', 'one reviewed closed partial only')
with sqlite3.connect(ledger.as_uri()+'?mode=ro', uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'Root ledger integrity')
require({r[0] for r in db.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants']) and
db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0 and
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'exact known closed Root attempts')
db.row_factory = sqlite3.Row
rows = db.execute('SELECT * FROM apt_partial_reconciliations').fetchall()
require(len(rows) == 1, 'one immutable partial proof required'); row = dict(rows[0])
expected = payload['proof_row']
for key in ('grant_id', 'original_request_sha256', 'previous_outcome_sha256', 'proof_sha256', 'checked_at_ms'):
require(row[key] == expected[key], 'immutable partial binding drift')
for key in ('previous_outcome', 'proof', 'classified_outcome'):
require(sha(row[key]) == expected[key]['sha256'] and len(row[key]) == expected[key]['size'], 'immutable partial bytes drift')
require(row['grant_id'] == payload['partial_grant'] and sha(row['proof']) == payload['partial_proof_sha256'], 'closed proof hash required')
proof = json.loads(row['proof']); result = db.execute('SELECT c.result,e.outcome,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE grant_id=?', (payload['partial_grant'],)).fetchone()
classified = json.loads(row['classified_outcome'])
require(json.loads(result['result']) == json.loads(result['outcome']) == classified and
classified['state'] == 'failed' and classified['error_code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED', 'normal signed RootSync closure required')
saved = json.loads(result['start'])
# save_start nests Request through serde_json::Value (sorted keys),
# whereas the immutable proof hashes serde's Request struct codec.
request_fields = ('job_bytes','permit_bytes','signature','plan_hash','authorization_id')
require(set(saved['request']) == set(request_fields), 'original Request fields required')
request_bytes = json.dumps({key:saved['request'][key] for key in request_fields},
separators=(',', ':'), ensure_ascii=False).encode()
require(sha(request_bytes) == row['original_request_sha256'], 'original signed request hash required')
require(proof['grant_id'] == payload['partial_grant'] and proof['attempt_id'] == payload['partial_attempt'] and
proof['snapshot']['rmm-mvp-test:all'] == {'version':'1.0','status':'install ok unpacked'}, 'exact partial fixture required')
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro', uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0 and
db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no unresolved or faulted agent attempt')
require(Path('/usr/share/rmm-mvp-test/version').read_bytes() == b'1.0\n', 'existing unpacked fixture payload required')
require(not list(base.glob('acceptance-*.json')) and not list(base.glob('pending-acceptance-*.json')) and
not list(base.glob('recovery-*.json')) and not list(base.glob('pending-recovery-*.json')), 'no existing acceptance or recovery attempt')
return proof
def bound_partial(proof):
keys = 'transaction_sha256 job_payload_sha256 effect_armed_ledger_id grant_id job_id attempt_id authorization_id plan_sha256'.split()
value = json.loads(run(['/usr/bin/python3','-I',str(provider),'verify-partial'], input=canonical({k:proof[k] for k in keys})))
require(value['effect_state'] == 'known_failed' and value['code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED' and
{k:v for k,v in value['proof'].items() if k != 'checked_at_ms'} ==
{k:v for k,v in proof.items() if k != 'checked_at_ms'}, 'fresh original-bound partial state required')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'permanent Cloud identity required')
require(not dropin.is_symlink() and dropin.read_text() == reduced, 'APT endpoint must remain absent')
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT helper must be stopped')
for name, pin in payload['old_files'].items():
p = Path(name); require(not p.is_symlink() and p.stat().st_uid == 0 and p.stat().st_mode & 0o022 == 0 and sha(p.read_bytes()) == pin, 'installed file drift')
require(Path('/etc/systemd/system/rmm-apt-helper.service').read_text() == payload['unit'], 'exact reviewed APT service required')
markers = {base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json':'ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053',
base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json':'a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3'}
require({p.name for p in base.glob('bootstrap-*.json')} == {p.name for p in markers} and
all(sha(p.read_bytes()) == pin for p,pin in markers.items()), 'original/retry tombstones required')
partial = guard(); bound_partial(partial)
raw = {name:base64.b64decode(payload[name], validate=True) for name in ('helper','agent')}
pins = {'helper':payload['helper_sha256'],'agent':payload['agent_sha256'],
'provider':'5ff260f75cb2959678f6be729d3a806a4ea29febd5ae4003317fed41afc25327'}
require(all(sha(raw[name]) == pins[name] for name in ('helper','agent')) and
sha(provider.read_bytes()) == pins['provider'], 'qualified exact capacity binaries and unchanged provider required')
for name in ('helper','agent'):
require(raw[name][:6] == b'\x7fELF\x02\x01' and raw[name][18:20] == b'\x3e\x00', 'Linux x64 binary required')
if payload.get('preflight_only') is True:
return dict(stage='preflight_validated', completed=False, preflight_only=True,
package_effects=False, maintenance_mutations=False, expected_grants=len(payload['expected_grants']),
partial_proof_sha256=payload['partial_proof_sha256'], apt_endpoint_enabled=False)
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-journal-capacity-'+payload['id']); backup.mkdir(mode=0o700)
report = dict(stage='validated', completed=False, backup=str(backup), package_effects=False,
apt_endpoint_enabled=False, partial_proof_sha256=payload['partial_proof_sha256'], backups={})
def save():
next_path = backup/'.report-next.json'; create(next_path,json.dumps(report,indent=2).encode())
os.replace(next_path,backup/'report.json'); fsync_dir(backup)
def replace(path, staged=None):
target = backup/str(path).lstrip('/').replace('/','__')
require(not path.is_symlink() and path.stat().st_dev == backup.stat().st_dev and not target.exists(), 'exclusive same-filesystem versioned replacement')
os.rename(path,target); fsync_dir(path.parent); fsync_dir(backup)
report['backups'][str(path)] = str(target); save()
if staged is not None:
os.rename(staged,path); fsync_dir(path.parent); fsync_dir(backup)
lock = None
try:
save()
for name in ('helper','agent'): create(backup/(name+'-next'), raw[name], 0o755)
for name in ('helper','agent'): run(['/lib64/ld-linux-x86-64.so.2','--list',str(backup/(name+'-next'))])
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
partial = guard(); bound_partial(partial)
preserved = {str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent)}
preserved_base = manifest(base)
witnesses = {}
for p in (Path('/var/lib/dpkg/status'),Path('/usr/share/rmm-mvp-test/version'),dropin,
Path('/etc/systemd/system/rmm-apt-helper.service'),Path('/usr/lib/rmm-apt-transaction/hook'),
Path('/etc/rmm-remote/general-helper-proof.json'),Path('/etc/rmm-remote/helper-general.json'),provider,metadata):
s = p.stat(); witnesses[str(p)] = (s.st_dev,s.st_ino,sha(p.read_bytes()))
general_bytes = Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
general_config = Path('/etc/rmm-remote/helper-general.json').read_bytes()
old_metadata = json.loads(metadata.read_text()); old_config = json.loads(config.read_text())
require(old_metadata['schema_version'] == 2 and old_metadata['provider_sha256'] == sha(provider.read_bytes()) and
old_metadata['hook_sha256'] == sha(Path('/usr/lib/rmm-apt-transaction/hook').read_bytes()) and
old_metadata['unit_sha256'] == sha(payload['unit'].encode()), 'old install metadata required')
for path,name in ((helper,'helper'),(agent,'agent')): replace(path,backup/(name+'-next'))
command = ['/usr/bin/python3','-I',str(provider)]
config_bytes = run(command+['print-config']); config_sha = sha(config_bytes)
require(config_sha == payload['reviewed_config_sha256'] and all(line in config_bytes for line in (
b'DPkg::ConfigurePending "false";', b'DPkg::TriggersPending "false";', b'dpkg::selection::remove::approved "false";')), 'reviewed exact configure/no-pending settings')
runtime = json.loads(run(command+['inspect-runtime'])); runtime_sha = sha(canonical(runtime))
old_config['apt_provider'].update(runtime_sha256=runtime_sha,apt_config_sha256=config_sha)
create(backup/'config-next',json.dumps(old_config).encode()); replace(config,backup/'config-next')
pin = json.loads(run([str(helper),'--inspect-apt-profile'],env=dict(os.environ,RMM_HELPER_CONFIG=str(config))))
require(pin['identity'] == json.loads(general_bytes)['identity'], 'same helper identity required')
create(backup/'proof-next',json.dumps(pin).encode(),0o640,984); replace(proof_file,backup/'proof-next')
general = json.loads(run([str(helper),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(general == json.loads(general_bytes) and Path('/etc/rmm-remote/helper-general.json').read_bytes() == general_config, 'General helper policy unchanged')
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'identity/journal/Root ledger bytes or inode changed')
def preserved_original_base():
current = manifest(base)
require(all(current.get(name) == value for name,value in preserved_base.items() if name not in ('install-metadata.json','native-readiness.json')), 'original APT state/archive/hook/probe changed')
for name,value in witnesses.items():
p=Path(name);s=p.stat();require((s.st_dev,s.st_ino,sha(p.read_bytes())) == value, 'original package/control file bytes or inode changed')
return current
preserved_original_base(); bound_partial(guard())
report.update(stage='installed_preserved',preserved_state=True,helper_sha256=pins['helper'],agent_sha256=pins['agent'],journal_page_size=4096,journal_max_page_count=65536,journal_capacity_bytes=268435456,
provider_sha256=pins['provider'],runtime_sha256=runtime_sha,apt_config_sha256=config_sha,apt_pin=pin,general_pin=general)
save()
# No client can reach this endpoint; it exists only for the service's
# ExecStartPre copied-database probe, then is stopped before restart.
run(['systemd-analyze','verify','/etc/systemd/system/rmm-apt-helper.service'])
replace(readiness_file)
# Daemon startup itself takes executor.lock to open the existing DB.
# Installation is complete; agent/General remain stopped during probe.
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
run(['systemctl','start','rmm-apt-helper.service'])
for _ in range(3):
status = run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','-p','SubState','-p','NRestarts']).decode()
require('ActiveState=active\n' in status and 'SubState=running\n' in status and 'NRestarts=0\n' in status, 'stable native probe helper required')
time.sleep(1)
run(['systemctl','stop','rmm-apt-helper.service'])
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
ready = json.loads(readiness_file.read_text())
require(ready['runtime'] == runtime and ready['readiness_only'] is True and ready['native_acceptance'] is False and
ready['host_status_sha256'] == sha(Path('/var/lib/dpkg/status').read_bytes()), 'exact new copied-database probe required')
workspace = base/'probes'/str(uuid.UUID(ready['probe_id']))
require((workspace/'dpkg/status').read_bytes() == Path('/var/lib/dpkg/status').read_bytes() and
(workspace/'apt.conf').read_bytes().count(b'dpkg::selection::remove::approved "false";') == 1,
'partial copied status and configure fix required')
current = preserved_original_base()
require(all(name in preserved_base or name.startswith('probes/'+ready['probe_id']+'/') or name == 'probes/'+ready['probe_id']
for name in current), 'only generated readiness probe metadata may be added')
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'journal/ledger bytes or inode changed during probe')
bound_partial(guard())
require(dropin.read_text() == reduced and Path('/etc/rmm-remote/general-helper-proof.json').read_bytes() == general_bytes, 'endpoint/General proof unchanged')
report.update(stage='native_probe_preserved',readiness_only=True,native_acceptance=False,
journals_preserved_before_restart=True,readiness_sha256=sha(readiness_file.read_bytes()),probe_id=ready['probe_id'])
save()
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT endpoint must remain stopped')
report.update(stage='qualified_runtime_started',completed=True);save()
except Exception as error:
report.update(error_class=type(error).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
save()
finally:
if lock is not None: fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
subprocess.run(['systemctl','stop','rmm-apt-helper.service'],capture_output=True,timeout=30)
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--copy-only',action='store_true')
mode.add_argument('--preflight-only',action='store_true')
mode.add_argument('--install-reviewed-artifacts',action='store_true')
parser.add_argument('--agent-sha256');parser.add_argument('--helper-sha256')
args = parser.parse_args();os.umask(0o077)
require(previous.previous.apt.prior.evidence('gate')['unresolved'] == 0, 'no unsettled central job')
payload = previous.build_payload()
require(len(payload['expected_grants']) == 10, 'exact completed Root/log history')
if args.copy_only:
report = cloud.bootstrap.rpc('linux-vm.conf',copy_reproducer,dict(id=str(uuid.uuid4()),agent_id=payload['agent_id'],expected_grants=payload['expected_grants']))
cloud.contract.write_private(STATE/('journal-capacity-copy-'+Path(report['backup']).name.removeprefix('journal-capacity-copy-')+'.json'),report)
print({k:report[k] for k in ('verified','old_error_code','new_insert_passed','original_unchanged','rows_and_schema_rolled_back','original_sql_mutations')})
return
require(not (STATE/'journal-capacity-runtime.json').exists(), 'existing capacity runtime retained; no automatic maintenance repeat')
reviewed = {'agent':'2ce885cf5e6881e77383482b7d3a87a02c6847aa26b82e5fd70d2ebd8a45360d',
'helper':'332eae004d9416234eabf7ed3d5c9ece5440d1e0c764ea63357961c16b673b6b'}
for name,pin in reviewed.items():
require(getattr(args,name+'_sha256') == pin and re.fullmatch('[0-9a-f]{64}',pin), 'independently reviewed capacity artifact pin required')
raw = (STATE/(name+'-journal-capacity')).read_bytes()
require(hashlib.sha256(raw).hexdigest() == pin, 'qualified capacity artifact drift')
payload[name] = base64.b64encode(raw).decode();payload[name+'_sha256'] = pin
payload.pop('provider')
previous_runtime = cloud.contract.protected_json(STATE/'configure-runtime.json')
require(previous_runtime['completed'] and previous_runtime['helper_sha256'] == previous.PINS['helper']
and previous_runtime['agent_sha256'] == previous.PINS['agent'] and previous_runtime['provider_sha256'] == previous.PINS['provider'], 'exact independently qualified previous runtime')
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
require(installed['binary_sha256'] == previous.PINS['agent'], 'old permanent agent record required')
policy = cloud.bootstrap.rpc('server.conf',previous.closed_policy,{})
old = cloud.bootstrap.rpc('linux-vm.conf',previous.installed_metadata,previous.PINS)
payload.update(old_files=old['files'],preflight_only=args.preflight_only)
report = cloud.bootstrap.rpc('linux-vm.conf',install_capacity,payload)
if args.preflight_only:
require(report.get('preflight_only') and report.get('maintenance_mutations') is False, 'read-only maintenance preflight required')
cloud.contract.write_private(STATE/('journal-capacity-preflight-'+payload['id']+'.json'),dict(report,closed_policy=policy))
print(report);return
cloud.contract.write_private(STATE/('journal-capacity-install-'+payload['id']+'.json'),dict(report,closed_policy=policy))
require(report['completed'] and report['preserved_state'] and report['journals_preserved_before_restart'] and report['provider_sha256'] == previous.PINS['provider'], 'preserving bounded-capacity installation incomplete; retain report and do not repeat')
previous.previous.record_agent_upgrade(report,installed)
cloud.contract.write_private(STATE/'journal-capacity-runtime.json',report)
print({k:report[k] for k in ('completed','stage','preserved_state','journals_preserved_before_restart','readiness_only','apt_endpoint_enabled')})
if __name__ == '__main__':
try:main()
except Exception as error:
print('CLOUD_JOURNAL_CAPACITY_INCOMPLETE:'+type(error).__name__)
if str(error).startswith('remote bootstrap incomplete: '):print(str(error))
raise SystemExit(2) from None
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env python3
"""Cloud-only active-agent offline/start-window/revocation acceptance.
Requires explicit execution. Plans start pending with zero executable artifacts;
approval and plan revocation use actual dev/test WebUI controls. A narrowly
scoped systemd BPF network fault affects only the existing Cloud agent unit.
SSH is used solely for that transport bootstrap and retained-state evidence.
No enrollment, package action, journal reset, approval API or automatic retry.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
import cloud_mvp_acceptance as cloud
CASES = ('offline_wait', 'expired_offline', 'revoked_before_start')
SERVER_IP = '88.99.80.202'
STATE = cloud.ROOT/'devstate/cloud-offline'
require = cloud.contract.require
def counter_script(identifier):
identifier = str(uuid.UUID(identifier))
return ('set -eu\numask 077\nd=/var/lib/rmm-agent-remote/offline-'+identifier+'\n'
'test ! -e "$d"\nmkdir "$d"\n'
'printf \'%s\\n\' \'1\' > "$d/count"\n'
'test "$(cat "$d/count")" = 1\n'
'printf \'%s\\n\' \'RMM_OFFLINE_COUNT=1\'\n')
def server_evidence(payload):
"""Fixed SELECTs only, never creates/revokes/approves an operation."""
root = Path('/opt/rust-discovery')
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
job = str(uuid.UUID(payload['job_id'])); operation = str(uuid.UUID(payload['operation_id']))
plan = str(uuid.UUID(payload['plan_id']))
generation = int(payload['previous_generation']); require(generation > 0, 'previous fenced channel')
import datetime as dt
after = dt.datetime.fromisoformat(payload['after'].replace('Z','+00:00')).isoformat() if payload.get('after') else '1970-01-01T00:00:00+00:00'
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND job_id='"+job+"'"
query = ("SELECT json_build_object('observed_at',now(),'state',(SELECT state FROM jobs WHERE "+scope+"),"
"'start_reserved',COALESCE((SELECT start_reserved FROM jobs WHERE "+scope+"),false),"
"'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
"'approval_state',(SELECT approval_state FROM approval_requests WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"'),"
"'revoked_authorizations',(SELECT count(*) FROM authorizations WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"' AND revoked_at IS NOT NULL),"
"'starts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data)),'[]'::json) FROM audit_events "
"WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"'),"
"'fresh_signed_sync_before_start',EXISTS(SELECT 1 FROM sync_checkpoints c WHERE c.tenant_id='"+tenant+"' AND c.agent_id='"+agent+"' "
"AND c.connection_generation>"+str(generation)+" AND (convert_from(c.payload,'UTF8')::jsonb->>'ready')::boolean "
"AND c.created_at>='"+after+"'::timestamptz AND c.created_at<=(SELECT min(created_at) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')),"
"'revocation',(SELECT json_build_object('principal',principal,'origin',data->>'approval_origin',"
"'simulation',data->'testsimulation','interactive_session',data->>'session_hash' IS NOT NULL,"
"'plan_id',data->'decision'->>'plan_id','operation_id',data->'decision'->>'operation_id') "
"FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='plan_revoked' AND data->'decision'->>'plan_id'='"+plan+"' ORDER BY created_at DESC LIMIT 1));")
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
def vm_network(payload):
"""One operational transport fault, plus reads of existing durable evidence."""
import datetime as dt
import errno
import fcntl
import sqlite3
import stat
identifier = str(uuid.UUID(payload['id']))
phase = payload['phase']; require(phase in ('preflight','window'), 'fixed network phase')
action = payload['action']; require(action in ('state','block','restore'), 'fixed network operation')
state = Path('/var/lib/rmm-agent-remote'); helper = Path('/var/lib/rmm-helper')
unit = 'rmm-agent-remote.service'; account = pwd.getpwnam('rmm-agent')
require(os.geteuid() == 0 and account.pw_uid == 993 and account.pw_gid == 993, 'fixed unprivileged Cloud agent')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed Cloud identity')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['binary_sha256'], 'reviewed current agent binary')
def properties():
raw = subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','MainPID','-p','ControlGroup','-p','IPAddressAllow'],capture_output=True,check=True,timeout=10).stdout.decode()
return dict(line.split('=',1) for line in raw.splitlines())
def snapshot():
result = dict(agent_service=properties())
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'native journal integrity')
result['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
result['agent_fault'] = db.execute('SELECT fault FROM reconcile_meta WHERE id=1').fetchone()[0]
if payload.get('job_id'):
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
rows = db.execute('SELECT state,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
result['attempts'] = [dict(state=r[0],grant_id=r[1]) for r in rows]
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(job,attempt)).fetchone()[0]
else:
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
result['consumed'] = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(payload.get('job_id',''),payload.get('attempt_id',''))).fetchall()
with sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'helper ledger integrity')
result['helper_fault'] = db.execute('SELECT fault FROM helper_sync_meta WHERE id=1').fetchone()[0]
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
serial = [[dict(blob_sha256=hashlib.sha256(v).hexdigest(),length=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows]
result['helper_ledger_sha256'] = hashlib.sha256(json.dumps(serial,separators=(',',':')).encode()).hexdigest()
result['helper_consumed'] = len(rows); result['helper_unresolved'] = sum(row[5] is None for row in rows)
effect_directory = state/('offline-'+identifier)
if effect_directory.exists():
m = effect_directory.lstat();require(stat.S_ISDIR(m.st_mode) and m.st_uid == 993 and m.st_mode&0o777 == 0o700, 'fixed stored effect directory')
else: require(not effect_directory.is_symlink(), 'stored effect directory symlink')
marker = effect_directory/'count'
if marker.exists():
m = marker.lstat(); require(stat.S_ISREG(m.st_mode) and m.st_uid == account.pw_uid and m.st_mode&0o777 == 0o600 and m.st_size <= 16, 'fixed stored counter evidence')
result['count'] = marker.read_text()
else:
require(not marker.is_symlink(), 'stored counter symlink'); result['count'] = None
return result
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
require(not directory.is_symlink(), 'network dropin directory')
directory.mkdir(mode=0o755,exist_ok=True)
require(directory.stat().st_uid == 0 and directory.stat().st_mode&0o022 == 0, 'network dropin ownership')
dropin = directory/'96-cloud-offline-acceptance.conf'
expected = ('# Cloud MVP offline '+identifier+' '+phase+'\n[Service]\nIPAddressDeny=88.99.80.202/32\n').encode()
require(not dropin.is_symlink(), 'network dropin symlink')
timer = 'rmm-offline-rollback-'+identifier+'-'+phase
before = snapshot()
require(before['agent_fault'] is None and before['helper_fault'] is None and before['helper_unresolved'] == 0 and before['other_unresolved'] == 0, 'unrelated or unknown work remains fenced')
if payload.get('helper_ledger_sha256'):
require(before['helper_ledger_sha256'] == payload['helper_ledger_sha256'], 'existing Root ledger changed')
if action == 'state': return before
if action == 'restore':
if dropin.exists():
require(dropin.stat().st_uid == 0 and dropin.read_bytes() == expected, 'exact network fault ownership')
dropin.unlink()
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
subprocess.run(['systemctl','stop',timer+'.timer'],capture_output=True,timeout=10)
return dict(restored=True,after=snapshot())
require(not dropin.exists() and before['agent_service']['ActiveState'] == 'active' and not before['agent_service']['IPAddressAllow'], 'clean active unit and no allow override')
require(before['count'] is None and not before['consumed'], 'no prior effect/consumption')
if phase == 'window':
start = dt.datetime.fromisoformat(payload['not_before'].replace('Z','+00:00'))
require(dt.datetime.now(dt.timezone.utc) < start-dt.timedelta(seconds=60) and before['attempts'] == [dict(state='accepted',grant_id='')], 'fixed future accepted attempt with zero grant')
else:
require(not payload.get('job_id'), 'preflight has no job')
# Arm bounded automatic transport rollback before creating the deny rule.
# The timer can remove only these exact bytes and never touches job state.
rollback_dir = Path('/var/lib/rmm-offline-bootstrap')
require(not rollback_dir.is_symlink(), 'rollback directory symlink')
rollback_dir.mkdir(mode=0o700,exist_ok=True)
require(rollback_dir.stat().st_uid == 0 and rollback_dir.stat().st_mode&0o777 == 0o700, 'private bootstrap rollback directory')
rollback = rollback_dir/(identifier+'-'+phase+'.py')
code = ('import os,subprocess\nfrom pathlib import Path\np=Path('+repr(str(dropin))+')\n'
'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(expected)+':\n'
' p.unlink()\n subprocess.run(["systemctl","daemon-reload"],check=True)\n'
' subprocess.run(["systemctl","restart","rmm-agent-remote.service"],check=True)\n')
fd = os.open(rollback,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'w') as stream: stream.write(code);stream.flush();os.fsync(stream.fileno())
subprocess.run(['systemd-run','--quiet','--unit='+timer,'--on-active=420s','--property=Type=oneshot','/usr/bin/python3',str(rollback)],capture_output=True,check=True,timeout=10)
# Quiesce this accepted/unstarted process only while applying the tested
# policy. It must be running again BEFORE not_before; a stopped-agent
# interval never counts toward the active offline acceptance window.
subprocess.run(['systemctl','stop',unit],capture_output=True,check=True,timeout=20)
stopped = snapshot()
require(stopped['agent_service']['ActiveState'] == 'inactive' and not stopped['consumed'] and stopped['count'] is None, 'quiescent unstarted network setup')
fd = os.open(dropin,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o644)
with os.fdopen(fd,'wb') as stream: stream.write(expected);stream.flush();os.fsync(stream.fileno())
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
after = snapshot(); require(after['agent_service']['ActiveState'] == 'active' and int(after['agent_service']['MainPID']) > 0 and after['count'] is None and not after['consumed'], 'active offline agent and zero effect')
if phase == 'window': require(dt.datetime.now(dt.timezone.utc) < start, 'agent must be active offline before earliest start')
group = after['agent_service']['ControlGroup']
require((group == '/system.slice/rmm-agent-remote.service' or group.startswith('/system.slice/rmm-agent-remote.service/')) and '..' not in group, 'actual agent cgroup')
cgroup = Path('/sys/fs/cgroup'+group)/'cgroup.procs'
require(cgroup.is_file(), 'unified cgroup transport enforcement required')
# This separate bootstrap child moves ONLY ITS OWN PID, drops privilege and
# attempts TCP only. No agent credential, TLS override or appdata is used.
probe = ('import os,socket,errno,json\nfrom pathlib import Path\n'
'Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n'
'os.setgroups([]);os.setgid(993);os.setuid(993)\n'
'try:\n s=socket.create_connection(("88.99.80.202",443),2);s.close();result={"connected":True}\n'
'except OSError as e: result={"connected":False,"errno":e.errno}\n'
'print(json.dumps(result))\n')
result = subprocess.run(['/usr/bin/python3','-I','-c',probe],capture_output=True,check=True,timeout=4)
blocked = json.loads(result.stdout)
require(blocked.get('connected') is False and blocked.get('errno') in (errno.EACCES,errno.EPERM), 'actual same-cgroup BPF denial must be EACCES/EPERM; timeout is not proof')
after = snapshot()
require(after['helper_ledger_sha256'] == before['helper_ledger_sha256'], 'Root ledger preserved during network bootstrap')
return dict(blocked=True,probe=blocked,after=after,rollback_after_seconds=420,unit=unit,server_ip='88.99.80.202',port=443)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
parser.add_argument('--case',choices=CASES,required=True)
args = parser.parse_args(); os.umask(0o077)
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(), 'notebook is development only')
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
identity = installed['identity']; settings = dict(tenant_id=cloud.TENANT,**identity)
backend = cloud.Backend(); identifier = str(uuid.uuid4())
folder = STATE/identifier; folder.mkdir(mode=0o700,parents=True)
checkpoint = dict(profile='dev/test',case=args.case,id=identifier,expected_principal='simulated-testprincipal-a',**settings,no_retry=True,stage='preflight')
serial = 0
def save():
nonlocal serial
cloud.contract.write_private(folder/('%03d-%s.json'%(serial,checkpoint['stage'])),checkpoint);serial += 1
def gate(job_id=None):
return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**({'job_id':job_id} if job_id else {})))
def native(action,phase='window',**extra):
p = dict(action=action,phase=phase,id=identifier,agent_id=identity['agent_id'],binary_sha256=installed['binary_sha256'],**extra)
if 'job' in checkpoint and phase == 'window': p.update(job_id=checkpoint['job']['job_id'],attempt_id=checkpoint['job']['payload']['attempt_id'])
return cloud.bootstrap.rpc('linux-vm.conf',vm_network,p)
def healthy(previous_generation=None):
end = time.monotonic()+100
while True:
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
if observed.get('sync_ready') and not observed.get('restore_blocked') and (previous_generation is None or observed['connection_generation'] > previous_generation): return observed
require(time.monotonic() < end, 'ordinary signed reconnect did not become healthy');time.sleep(2)
require(gate()['unresolved'] == 0, 'unresolved central work remains fenced')
checkpoint['initial_channel'] = healthy(); initial = native('state','preflight')
ledger_sha = initial['helper_ledger_sha256']; checkpoint['initial_native'] = initial;save()
checkpoint['network_preflight'] = native('block','preflight',helper_ledger_sha256=ledger_sha);save()
checkpoint['preflight_restored'] = native('restore','preflight',helper_ledger_sha256=ledger_sha);save()
checkpoint['healthy_before_plan'] = healthy(checkpoint['initial_channel']['connection_generation'])
start = cloud.contract.now()+dt.timedelta(seconds=150); end = start+dt.timedelta(seconds=130)
request = cloud.contract.case_input('hello',settings,start=start)
request['parameters']['variants'][0]['script_utf8'] = counter_script(identifier)
request['parameters']['expected_stdout'] = 'RMM_OFFLINE_COUNT=1\n'
request['schedule'] = cloud.contract.schedule(start,end)
request['reason'] = 'Explicit dev/test active Cloud agent '+args.case+'; one fixed attempt, no implicit retry'
plan = backend.call('operations.plan',dict(tenant_id=cloud.TENANT,input=request)); require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'one immutable fixed job')
job = plan['jobs'][0]; pending = backend.call('operations.submit',dict(tenant_id=cloud.TENANT,plan_id=plan['plan_id'],idempotency_key='offline-'+identifier))
checkpoint.update(plan=plan,job=job,request=request,operation_id=pending['operation_id'],stage='pending_approval',preapproval=gate(job['job_id']))
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0 and pending['execution_state'] is None and all(checkpoint['preapproval'][k] == 0 for k in ('jobs','outbox','permits')), 'pending zero execution artifacts');save()
def browser(phase):
cloud.prepare_browser(); path = folder/('ui-'+phase+'.json'); cloud.contract.write_private(path,dict(checkpoint,phase=phase))
result = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_offline_acceptance.cjs'),str(path)],capture_output=True,timeout=90)
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint and never repeat approval')
browser('approve'); checkpoint['stage'] = 'approved';save()
deadline = time.monotonic()+45
while True:
server = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id'])); retained = native('state',helper_ledger_sha256=ledger_sha)
require(gate(job['job_id'])['permits'] == 0 and not retained['consumed'] and retained['count'] is None, 'future attempt already has a grant/effect')
if server['state'] == 'accepted' and retained['attempts'] == [dict(state='accepted',grant_id='')]: break
require(time.monotonic() < deadline, 'future offer was not durably accepted');time.sleep(2)
checkpoint.update(accepted=server,native_accepted=retained,stage='accepted_unstarted');save()
checkpoint['network_fault'] = native('block',not_before=job['payload']['not_before'],helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'active_offline';save()
channel = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
resume = cloud.contract.timestamp(job['payload']['start_before'] if args.case == 'expired_offline' else job['payload']['not_before'])+dt.timedelta(seconds=10)
if args.case == 'revoked_before_start': browser('revoke');checkpoint['ui_revoked'] = True;save()
sequence = checkpoint['network_fault']['after']['sequence']; windows = []
while cloud.contract.now() < resume:
retained = native('state',helper_ledger_sha256=ledger_sha); counts = gate(job['job_id'])
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
require(retained['agent_service']['ActiveState'] == 'active' and retained['agent_service']['MainPID'] == checkpoint['network_fault']['after']['agent_service']['MainPID']
and not retained['consumed'] and retained['count'] is None and counts['permits'] == 0 and observed['last_contact_at'] == channel['last_contact_at'], 'active offline window had a grant/effect/contact or agent exited')
windows.append(dict(observed_at=cloud.contract.now().isoformat(),sequence=retained['sequence'],grants=counts['permits'],count=retained['count'],active=True))
checkpoint['offline_window'] = windows;save();time.sleep(5)
require(windows and windows[-1]['sequence'] > sequence, 'running offline agent did not persist reconnect attempts')
checkpoint['server_offline_tail'] = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation']))
require(checkpoint['server_offline_tail']['grants'] == 0 and not checkpoint['server_offline_tail']['start_reserved'], 'no start immediately before network restoration')
checkpoint['network_restored'] = native('restore',helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'network_restored';save()
checkpoint['healthy_reconnect'] = healthy(channel['connection_generation'])
deadline = time.monotonic()+100
while True:
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
if result['state'] in cloud.contract.TERMINAL or result['state'] in ('unknown','awaiting_reboot'): break
require(time.monotonic() < deadline, 'terminal native result missing; no retry');time.sleep(2)
expected = {'offline_wait':'succeeded','expired_offline':'expired','revoked_before_start':'cancelled'}[args.case]
require(result['state'] == expected, 'actual terminal state differs from bounded case')
native_final = native('state',helper_ledger_sha256=ledger_sha)
wanted = 1 if args.case == 'offline_wait' else 0
final_gate = gate(job['job_id']); require([final_gate[k] for k in ('jobs','outbox','permits')] == [1,1,wanted] and final_gate['unresolved'] == 0, 'exact job/outbox/start-grant counts')
require(len(native_final['consumed']) == wanted and native_final['count'] == ('1\n' if wanted else None), 'exact native consumption and stored effect counter')
if wanted: require(result['result']['exit_code'] == 0 and bytes(result['result']['stdout']) == b'RMM_OFFLINE_COUNT=1\n' and result['result']['evidence']['postconditions_passed'], 'native one-effect postcondition')
audit = cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='audit',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],operation_id=pending['operation_id']))
require(audit == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=plan['plan_hash']), 'exact interactive approval audit')
details = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation'],after=checkpoint['server_offline_tail']['observed_at']))
require(details['grants'] == wanted and len(details['starts']) == wanted and details['start_reserved'] == bool(wanted), 'retained start reservation evidence')
if wanted: require(details['fresh_signed_sync_before_start'], 'fresh healthy signed reconnect must precede the one start reservation')
if args.case == 'revoked_before_start':
require(details['approval_state'] == 'revoked' and details['revoked_authorizations'] == 1 and details['revocation'] == dict(principal=checkpoint['expected_principal'],origin='dev_test_browser_simulation',simulation=True,interactive_session=True,plan_id=plan['plan_id'],operation_id=pending['operation_id']), 'actual interactive plan revocation audit')
checkpoint.update(result=result,native_final=native_final,postapproval=final_gate,audit=audit,server_evidence=details,stage='native_dev_test_accepted',verified=True);save()
print('PASS Cloud '+args.case+': active offline BPF verified, WebUI gate, exact grants/effect='+str(wanted))
if __name__ == '__main__':
try: main()
except Exception as error:
print('CLOUD_OFFLINE_INCOMPLETE:'+ (str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
raise SystemExit(2) from None
+40
View File
@@ -3,6 +3,27 @@
def service_processes(payload):
result={}
# Same read-only kernel clock-readiness query required by the runtime;
# modes stays zero. No time correction or trust override exists here.
import ctypes
class TimexPrefix(ctypes.Structure):
_fields_=[('modes',ctypes.c_uint),('offset',ctypes.c_long),('freq',ctypes.c_long),
('maxerror',ctypes.c_long),('esterror',ctypes.c_long),('status',ctypes.c_int),('remaining',ctypes.c_byte*256)]
require(ctypes.sizeof(ctypes.c_long)==8 and TimexPrefix.maxerror.offset==24 and TimexPrefix.status.offset==40,'reviewed Linux x64 timex layout')
clock=TimexPrefix();libc=ctypes.CDLL(None,use_errno=True);query=libc.adjtimex
query.argtypes=[ctypes.POINTER(TimexPrefix)];query.restype=ctypes.c_int
status=query(ctypes.byref(clock))
result['kernel_clock']=dict(status=status,unsynchronized=bool(clock.status&0x40),maxerror_us=clock.maxerror,
trusted=status>=0 and status!=5 and not clock.status&0x40 and 0<=clock.maxerror<=5_000_000,modes=clock.modes)
storage=os.statvfs('/var/lib/rmm-helper')
result['bootstrap_storage']=dict(available_bytes=storage.f_bavail*storage.f_frsize,available_inodes=storage.f_favail,
helper_database_bytes=Path('/var/lib/rmm-helper/consumption.sqlite').stat().st_size,
agent_database_bytes=Path('/var/lib/rmm-agent-remote/journal.sqlite').stat().st_size)
lock=Path('/var/lib/rmm-helper/executor.lock').stat()
result['executor_lock_pids']=[]
for line in Path('/proc/locks').read_text().splitlines():
parts=line.split()
if len(parts)>5 and parts[5].split(':')[-1]==str(lock.st_ino):result['executor_lock_pids'].append(parts[4])
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
values=subprocess.run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts'],
capture_output=True,check=True,timeout=10).stdout.decode()
@@ -23,3 +44,22 @@ def service_processes(payload):
if children.exists():pending.extend(int(s) for s in children.read_text().split())
result[name]=dict(service=fields,processes=processes)
return result
def stored_bootstrap_error_codes(payload):
"""Only allowlisted stored daemon errors during this runtime deployment."""
known=('CLOCK_UNTRUSTED','HELPER_VALIDATION_OR_IO_FAILURE','HELPER_PIN_PROFILE_MISMATCH',
'SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE','HELPER_CHECKPOINTS_DID_NOT_CONVERGE',
'HELPER_MANUAL_RECOVERY_REQUIRED','LOCAL_CHECKPOINT_ROLLBACK','SYNC_MANUAL_RECOVERY_REQUIRED')
result={}
for unit in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
process=subprocess.run(['/usr/bin/journalctl','--no-pager','--quiet','--output=json','--unit',unit,
'--since','2026-11-13 17:34:00 UTC','--lines','100'],capture_output=True,check=True,timeout=10)
require(len(process.stdout)<=256*1024,'bounded deployment error metadata')
counts={k:0 for k in known};times=[]
for line in process.stdout.splitlines():
row=json.loads(line);message=row.get('MESSAGE','')
for code in known:
if code in message:counts[code]+=1;times.append(row.get('__REALTIME_TIMESTAMP'))
result[unit]=dict(counts={k:v for k,v in counts.items() if v},first=min(times) if times else None,last=max(times) if times else None)
return result
+7
View File
@@ -95,6 +95,9 @@ def deploy(payload):
require(built['exit_code'] == 0, 'build must pass')
require(all(hashlib.sha256((root/name).read_bytes()).hexdigest() == sha for name,sha in record['files'].items()), 'source drift')
require(not (backup/'deploy-result.json').exists(), 'deployment already recorded')
if payload.get('previous_ui_index_sha256'):
require(hashlib.sha256((root/'web/dist/index.html').read_bytes()).hexdigest()==payload['previous_ui_index_sha256'],
'independently known previous UI index required')
if payload.get('closed_policy_sha256'):
import datetime as dt
policy = (root/'devstate/root-policy/policy.json').read_bytes()
@@ -142,6 +145,7 @@ def deploy(payload):
def main():
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('stage','status','deploy')); parser.add_argument('--id')
parser.add_argument('--preserve-known-unknown', action='store_true', help='preserve the exact fenced APT attempt while upgrading its control path')
parser.add_argument('--expected-ui-index-sha256',help='CAS the independently known previous UI build before publication')
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
if args.action == 'stage':
names = ['Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile','deploy/linux/rmm-apt-helper.service',
@@ -161,6 +165,9 @@ def main():
ident = str(uuid.UUID(args.id)); cloud.contract.protected_json(STATE/(ident+'.json'))
before = None; payload = dict(id=ident)
if args.action == 'deploy':
if args.expected_ui_index_sha256:
require(len(args.expected_ui_index_sha256)==64 and all(c in '0123456789abcdef' for c in args.expected_ui_index_sha256),'fixed previous UI hash')
payload['previous_ui_index_sha256']=args.expected_ui_index_sha256
target = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
gate = cloud.bootstrap.rpc('server.conf', cloud.server, dict(kind='gate', tenant_id=cloud.TENANT, agent_id=target['agent_id']))
if args.preserve_known_unknown:
+190
View File
@@ -0,0 +1,190 @@
#!/usr/bin/env python3
"""Passive Cloud auto-WSS→HTTPS fallback with two actual mTLS WSS rejections.
No RMM job, plan, approval or new identity. Only explicit, bounded dev/test
transport fault configuration and preserving agent/service reconnects exist.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import re
import time
import uuid
import cloud_mvp_acceptance as cloud
require = cloud.contract.require
STATE = cloud.ROOT/'devstate/cloud-transport'
def auto_mode(payload):
import sqlite3
state = Path('/var/lib/rmm-agent-remote')
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'fixed permanent identity')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['agent_sha256'],'known qualified agent')
for path,query in ((state/'journal.sqlite',"SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')"),
(Path('/var/lib/rmm-helper/consumption.sqlite'),'SELECT count(*) FROM consumed WHERE result IS NULL')):
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute(query).fetchone()[0] == 0,'all native effects must be settled')
unit = Path('/etc/systemd/system/rmm-agent-remote.service')
require(unit.read_text().splitlines().count('Environment=RMM_TRANSPORT=auto') == 1,'known automatic base service')
require('RMM_TRANSPORT=' not in Path('/etc/rmm-remote/agent.env').read_text(),'no environment-file transport override')
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
selected = directory/'95-cloud-acceptance-transport.conf'
for path in directory.glob('*.conf'):
require(not path.is_symlink(),'transport drop-in symlink')
require(path == selected or 'RMM_TRANSPORT=' not in path.read_text(),'unknown transport override')
changed = selected.exists()
if changed:
require(selected.stat().st_uid == 0 and selected.read_text() in
('[Service]\nEnvironment=RMM_TRANSPORT=wss\n','[Service]\nEnvironment=RMM_TRANSPORT=https\n'),'known existing test override')
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-auto-transport-'+str(uuid.UUID(payload['id'])))
backup.mkdir(mode=0o700)
os.rename(selected,backup/'previous-transport.conf')
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=10)
subprocess.run(['systemctl','restart','rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
return dict(mode='auto',preserved_identity=True,removed_test_override=changed)
def configure_server(payload):
root = Path('/opt/rust-discovery');compose = root/'deploy/server.compose.yaml'
directory = root/'devstate/root-policy';path = directory/'transport-fault.json'
require(not compose.is_symlink() and hashlib.sha256(compose.read_bytes()).hexdigest() == payload['compose_sha256'],'independently known compose required')
require(not directory.is_symlink() and directory.is_dir() and directory.stat().st_uid == 10001
and directory.stat().st_mode & 0o777 == 0o700,'existing private read-only mount')
require(not path.is_symlink(),'transport config symlink')
old = path.read_bytes() if path.exists() else None
actual_sha = hashlib.sha256(old).hexdigest() if old is not None else None
expected = payload['expected_config_sha256']
# Cleanup may accept either of two PREVIOUSLY known hashes when an SSH
# response was lost; it never adopts an arbitrary hash read from the host.
require(actual_sha in expected if isinstance(expected,list) else actual_sha == expected,'known prior transport config')
tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']))
require(tenant == '11111111-1111-4111-8111-111111111111','fixed development tenant')
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(compose)]
query = "SELECT count(*) FROM jobs WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND state NOT IN ('succeeded','failed','cancelled','expired');"
require(subprocess.run(cmd+['exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'],
input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip() == b'0','no open Cloud jobs')
enabled = payload['enabled'];require(type(enabled) is bool,'explicit opt-in')
backup = root/'devstate'/('transport-fault-'+str(uuid.UUID(payload['id'])))
backup.mkdir(mode=0o700)
def create(target,raw,uid=0):
fd = os.open(target,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'wb') as stream:
stream.write(raw);stream.flush();os.fsync(stream.fileno());os.fchown(stream.fileno(),uid,uid)
if old is not None:create(backup/'transport-fault-before.json',old)
if enabled:
value = payload['config'];require(value['tenant_id'] == tenant and value['agent_id'] == agent,'exact scope')
raw = json.dumps(value,separators=(',',':')).encode()
create(directory/'transport-fault.next',raw,10001);os.replace(directory/'transport-fault.next',path)
else:raw = old
overlay = backup/'transport.compose.yaml'
# Literal overlay; no existing .env secret is read, returned or replaced.
body = 'services:\n'
for role in ('api','gateway'):
body += ' '+role+':\n environment:\n RMM_DEV_TRANSPORT_FAULTS: "'+('true' if enabled else 'false')+'"\n'
if enabled:body += ' RMM_DEV_TRANSPORT_FAULT_FILE: /root-policy/transport-fault.json\n'
create(overlay,body.encode())
cmd += ['-f',str(overlay)]
subprocess.run(cmd+['config','-q'],capture_output=True,check=True,timeout=15)
completed = subprocess.run(cmd+['up','-d','--no-deps','api','gateway'],capture_output=True,timeout=25).returncode == 0
return dict(deployed=completed,enabled=enabled,backup=str(backup),config_sha256=hashlib.sha256(raw).hexdigest() if raw is not None else None)
def server_evidence(payload):
root = Path('/opt/rust-discovery');tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']));fault = str(uuid.UUID(payload['fault_id']))
require(tenant == '11111111-1111-4111-8111-111111111111','fixed test tenant')
query = ("SELECT json_build_object('receipts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data) ORDER BY created_at),'[]'::json) "
"FROM audit_events WHERE tenant_id='"+tenant+"' AND principal='dev_transport_fault_adapter' AND data->>'fault_id'='"+fault+"'),"
"'device',(SELECT json_build_object('agent_id',agent_id,'asset_id',asset_id,'enrollment_generation',enrollment_generation,'assignment_version',assignment_version,"
"'transport',transport,'connection_generation',connection_generation,'last_contact_at',last_contact_at,'last_authenticated_at',last_authenticated_at,"
"'certificate_sha256',last_authenticated_certificate_sha256,'sync_ready',sync_ready,'restore_blocked',restore_blocked) "
"FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'));")
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout)
def runtime_flags(payload):
root = Path('/opt/rust-discovery')
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')]
result = {}
for role in ('api','gateway'):
ident = subprocess.run(cmd+['ps','-q',role],capture_output=True,check=True,timeout=10).stdout.decode().strip()
require(ident and '\n' not in ident,'one running role')
values = json.loads(subprocess.run(['docker','inspect','--format','{{json .Config.Env}}',ident],capture_output=True,check=True,timeout=10).stdout)
keys = ('RMM_PROFILE','RMM_DEV_AUTH','RMM_DEV_TRANSPORT_FAULTS','RMM_DEV_TRANSPORT_FAULT_FILE')
result[role] = {v.split('=',1)[0]:v.split('=',1)[1] for v in values if v.split('=',1)[0] in keys}
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--execute-dev-test-transport',action='store_true',required=True)
parser.add_argument('--expected-agent-sha256',required=True)
parser.add_argument('--expected-compose-sha256',required=True)
parser.add_argument('--expected-config-sha256')
args = parser.parse_args();os.umask(0o077)
for value in (args.expected_agent_sha256,args.expected_compose_sha256):require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified pin')
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
scope = dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'])
def gate():return cloud.bootstrap.rpc('server.conf',cloud.server,dict(scope,kind='gate'))
before = gate();require(before['unresolved'] == 0,'earlier effects must be closed')
ident = str(uuid.uuid4());folder = STATE/ident;folder.mkdir(mode=0o700,parents=True)
evidence_scope = dict(scope,fault_id=ident)
auto = cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=ident,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256))
checkpoint = dict(profile='dev/test',testsimulation=True,case='actual_auto_fallback',id=ident,identity=identity,preapproval_counts=before,auto_mode=auto,no_jobs_created=True)
path = folder/'checkpoint.json'
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
def wait_transport(transport,after=None):
end=time.monotonic()+150
while True:
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);device=observed['device']
if (device['transport'] == transport and device['sync_ready'] and not device['restore_blocked'] and device['last_contact_at']
and (after is None or cloud.contract.timestamp(device['last_contact_at']) > after)):return observed
require(time.monotonic() < end,'fresh actual transport heartbeat and signed Sync missing');time.sleep(2)
save();checkpoint['initial_wss']=wait_transport('wss');save()
now=cloud.contract.now();config=dict(schema_version=1,fault_id=ident,**scope,issued_at=now.isoformat(),expires_at=(now+dt.timedelta(minutes=10)).isoformat())
raw=json.dumps(config,separators=(',',':')).encode();checkpoint['config']=config;save()
enabled=None;complete=False;attempted=False
try:
attempted=True
enabled=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=ident,enabled=True,**scope,compose_sha256=args.expected_compose_sha256,
expected_config_sha256=args.expected_config_sha256,config=config))
checkpoint['enabled']=enabled;save();require(enabled['deployed'],'transport opt-in incomplete')
end=time.monotonic()+150
while True:
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);receipts=observed['receipts'];device=observed['device']
if (len(receipts) == 2 and device['transport'] == 'https' and device['sync_ready'] and not device['restore_blocked']
and device['last_contact_at'] and cloud.contract.timestamp(device['last_contact_at']) > cloud.contract.timestamp(receipts[1]['created_at'])):break
require(time.monotonic() < end,'two authenticated WSS503 and actual HTTPS heartbeat missing');time.sleep(2)
require({r['data']['slot'] for r in receipts} == {1,2} and len({r['data']['handshake_id'] for r in receipts}) == 2
and all(r['data']['status'] == 503 and r['data']['agent_id'] == identity['agent_id'] and r['data']['tenant_id'] == cloud.TENANT
and r['data']['config_sha256'] == hashlib.sha256(raw).hexdigest() and r['data']['certificate_sha256'] == device['certificate_sha256']
and r['data']['testsimulation'] is True for r in receipts),'actual exact peer fault receipts')
require(all(device[k] == identity[k] for k in ('agent_id','asset_id','enrollment_generation','assignment_version')),'permanent identity unchanged')
checkpoint['https']=observed;save();complete=True
finally:
if attempted:
closed=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=str(uuid.uuid4()),enabled=False,**scope,compose_sha256=args.expected_compose_sha256,
expected_config_sha256=[hashlib.sha256(raw).hexdigest(),args.expected_config_sha256]))
checkpoint['disabled']=closed;save();require(closed['deployed'],'transport fault cleanup incomplete')
flags=cloud.bootstrap.rpc('server.conf',runtime_flags,{})
require(len(flags) == 2 and all(v.get('RMM_DEV_TRANSPORT_FAULTS') in (None,'false') and 'RMM_DEV_TRANSPORT_FAULT_FILE' not in v for v in flags.values()),'both real roles disabled')
checkpoint['disabled_runtime']=flags;save()
checkpoint['restored_wss']=wait_transport('wss',cloud.contract.now());save()
require(complete,'fallback not proven')
after=gate();require(after == before,'passive fallback must leave job/outbox/grant counts unchanged')
checkpoint.update(post_counts=after,verified=True,stage='native_dev_test_accepted');save()
print('PASS passive Cloud auto WSS→HTTPS: two authenticated503, fresh HTTPS heartbeat+Sync, normal WSS restored, identity and job counts preserved')
if __name__ == '__main__':
try:main()
except Exception as error:
print('CLOUD_TRANSPORT_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
raise SystemExit(2) from None
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Exact dev/test root script payload. Never installed as a daemon/provider."""
import os
from pathlib import Path
import stat
import sys
import time
import uuid
assert os.geteuid() == 0 and len(sys.argv) == 3
operation, identifier = sys.argv[1:]
assert operation in ('effect', 'read') and str(uuid.UUID(identifier)) == identifier
directory = Path('/var/lib/rmm-helper')/('crash-'+identifier)
for parent in directory.parents:
metadata = parent.lstat()
assert stat.S_ISDIR(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_mode & 0o022 == 0
os.umask(0o077)
if operation == 'effect' and not directory.exists(): directory.mkdir(mode=0o700)
metadata = directory.lstat()
assert stat.S_ISDIR(metadata.st_mode) and metadata.st_uid == 0 and stat.S_IMODE(metadata.st_mode) == 0o700
counter = directory/'count'
def read_counter():
fd = os.open(counter, os.O_RDONLY | os.O_NOFOLLOW)
with os.fdopen(fd, 'r') as stream:
metadata = os.fstat(stream.fileno())
assert stat.S_ISREG(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_nlink == 1
assert stat.S_IMODE(metadata.st_mode) == 0o600 and metadata.st_size <= 16
value = stream.read()
assert value.endswith('\n') and value[:-1].isdigit()
return int(value)
if operation == 'effect':
lock = directory/'lock';lock.mkdir(mode=0o700)
try:
value = read_counter() if counter.exists() else 0
assert 0 <= value < 100
fd = os.open(directory/'count.next',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'w') as stream:
stream.write(str(value+1)+'\n');stream.flush();os.fsync(stream.fileno())
os.replace(directory/'count.next',counter)
fd = os.open(directory,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
try: os.fsync(fd)
finally: os.close(fd)
finally: lock.rmdir()
# Independent helper remains alive after only the agent process dies.
time.sleep(90)
assert read_counter() == 1
print('RMM_CRASH_COUNT=1')