fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
// Actual visible dev/test UI only; no API/DB approval or injected requests.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
const { isDeepStrictEqual: equal } = require('node:util');
|
||||
const { chromium } = require('../../web/node_modules/playwright');
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const assert = value => { if (!value) throw new Error('Crash UI contract mismatch'); };
|
||||
function readPrivate(file) {
|
||||
const metadata = fs.lstatSync(file);
|
||||
assert(metadata.isFile() && !metadata.isSymbolicLink() && metadata.uid === process.getuid()
|
||||
&& (metadata.mode & 0o777) === 0o600 && metadata.size < 1048576);
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
}
|
||||
function checkJob(plan, expected, script) {
|
||||
assert(plan.jobs.length === 1 && plan.resolved_targets.length === 1
|
||||
&& crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
|
||||
const job = plan.jobs[0], p = job.payload;
|
||||
assert(p.tenant_id === expected.tenant_id && p.agent_id === expected.agent_id && p.asset_id === expected.asset_id
|
||||
&& p.execution_identity === 'root' && p.max_runtime_seconds === 120 && p.max_output_bytes === 65536
|
||||
&& Date.parse(p.start_before) > Date.now());
|
||||
assert(equal(p.action, { action: 'run_script', interpreter: 'posix_sh', script_utf8: script,
|
||||
arguments: [], expected_stdout: 'RMM_CRASH_COUNT=1\n' }));
|
||||
}
|
||||
(async () => {
|
||||
const file = path.resolve(process.argv[2]), expected = readPrivate(file);
|
||||
assert(expected.profile === 'dev/test' && expected.case === 'agent_crash'
|
||||
&& expected.expected_principal === 'simulated-testprincipal-a' && typeof expected.read_only === 'boolean'
|
||||
&& /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(expected.id));
|
||||
const source = fs.readFileSync(path.join(root, 'deploy/linux/agent-crash-counter.py'), 'utf8');
|
||||
const script = 'exec /usr/bin/python3 -I - ' + (expected.read_only ? 'read' : 'effect') + ' ' + expected.id
|
||||
+ " <<'RMM_AGENT_CRASH_COUNTER'\n" + source + '\nRMM_AGENT_CRASH_COUNTER\n';
|
||||
const login = readPrivate(path.join(root, 'credentials/server-browser-login.json'));
|
||||
const browsers = path.join(root, 'web/node_modules/.cache/playwright-browsers');
|
||||
const version = fs.readdirSync(browsers).find(x => /^chromium-/.test(x));
|
||||
const executablePath = ['chrome-linux/chrome', 'chrome-linux64/chrome'].map(x => path.join(browsers, version, x)).find(x => fs.existsSync(x));
|
||||
const browser = await chromium.launch({ headless: true, executablePath,
|
||||
env: { ...process.env, HOME: path.join(root, 'web/node_modules/.cache/cloud-acceptance/home') } });
|
||||
try {
|
||||
const page = await browser.newPage({ ignoreHTTPSErrors: false, timezoneId: 'UTC' });page.setDefaultTimeout(20000);
|
||||
let approvals = 0;
|
||||
page.on('request', r => { if (new URL(r.url()).pathname.endsWith('/authorizations')) approvals++; });
|
||||
await page.goto('https://localhost:18443/');
|
||||
await page.getByLabel('Benutzername').fill(login.username);
|
||||
await page.getByLabel('Passwort', { exact: true }).fill(login.password);
|
||||
const response = page.waitForResponse(r => new URL(r.url()).pathname === '/auth/session' && r.status() === 200);
|
||||
await page.getByRole('button', { name: 'Anmelden', exact: true }).click();
|
||||
const session = await (await response).json();
|
||||
assert(session.testsimulation === true && session.approval_origin === 'dev_test_browser_simulation' && session.principal === expected.expected_principal);
|
||||
await page.getByRole('status').filter({ hasText: 'SIMULIERT' }).waitFor();
|
||||
const detail = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Unveränderlicher Plan / Genehmigungsdetail', exact: true }) });
|
||||
if (expected.phase === 'prepare') {
|
||||
assert(expected.request.parameters.variants.length === 1 && expected.request.parameters.variants[0].script_utf8 === script
|
||||
&& expected.request.parameters.expected_stdout === 'RMM_CRASH_COUNT=1\n');
|
||||
await page.getByRole('checkbox', { name: 'Ziel ' + expected.asset_id, exact: true }).check();
|
||||
await page.getByLabel('Aktion', { exact: true }).selectOption('run_script');
|
||||
await page.getByRole('combobox', { name: 'Interpreter', exact: true }).selectOption('posix_sh');
|
||||
await page.getByLabel('Exakte UTF-8 Skriptbytes', { exact: false }).fill(script);
|
||||
await page.getByLabel('Erwartete stdout-Ausgabe', { exact: false }).fill('RMM_CRASH_COUNT=1\n');
|
||||
await page.getByLabel('Ausführungsidentität', { exact: true }).selectOption('root');
|
||||
await page.getByLabel('Laufzeitlimit (s)', { exact: true }).fill('120');
|
||||
await page.getByLabel('Ausgabelimit (Bytes)', { exact: true }).fill('65536');
|
||||
const start = new Date(Math.ceil(Date.now() / 60000) * 60000);
|
||||
await page.getByLabel('Frühester Start (lokale Zeit)', { exact: true }).fill(start.toISOString().slice(0, 16));
|
||||
await page.getByLabel('Startfrist / Ablauf (lokale Zeit)', { exact: true }).fill(new Date(start.getTime() + 15 * 60000).toISOString().slice(0, 16));
|
||||
await page.getByLabel('Maximal parallel', { exact: true }).fill('1');
|
||||
await page.getByLabel('Stopp nach Fehlern', { exact: true }).fill('1');
|
||||
await page.getByLabel('Begründung', { exact: true }).fill(expected.request.reason);
|
||||
await page.getByRole('button', { name: 'Plan erstellen · noch keine Ausführung' }).click();
|
||||
const plan = JSON.parse(await detail.locator('pre').innerText());checkJob(plan, expected, script);
|
||||
await detail.getByRole('button', { name: 'Antrag absenden · pending_approval' }).click();
|
||||
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: /^Antrag \/ Ausführung / }) });
|
||||
await operation.locator('pre').first().filter({ hasText: '"job_count": 0' }).waitFor();
|
||||
const pending = JSON.parse(await operation.locator('pre').first().innerText());
|
||||
assert(pending.approval_state === 'pending_approval' && pending.execution_state === null && pending.job_count === 0 && approvals === 0);
|
||||
assert(path.dirname(path.resolve(expected.output_file)) === path.dirname(file));
|
||||
fs.writeFileSync(expected.output_file, JSON.stringify({ plan, operation: pending }), { flag: 'wx', mode: 0o600 });
|
||||
} else {
|
||||
assert(expected.phase === 'approve');
|
||||
await page.getByRole('button', { name: expected.operation_id, exact: true }).click();
|
||||
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Antrag / Ausführung ' + expected.operation_id, exact: true }) });
|
||||
const pending = JSON.parse(await operation.locator('pre').first().innerText());
|
||||
assert(pending.approval_state === 'pending_approval' && pending.execution_state === null && pending.job_count === 0 && approvals === 0);
|
||||
await operation.getByRole('button', { name: 'Unveränderlichen Plan zur Prüfung öffnen', exact: true }).click();
|
||||
const plan = JSON.parse(await detail.locator('pre').innerText());
|
||||
const { approval_state: currentState, ...currentPlan } = plan;
|
||||
const { approval_state: preparedState, ...preparedPlan } = expected.plan;
|
||||
assert(currentState === 'pending_approval' && preparedState === null && equal(currentPlan, preparedPlan));
|
||||
checkJob(plan, expected, script);assert(equal(plan.jobs[0], expected.job));
|
||||
const approve = detail.getByRole('button', { name: 'SIMULIERT: Diesen festen Plan ausdrücklich genehmigen', exact: true });
|
||||
assert(await approve.isDisabled());
|
||||
await detail.getByRole('checkbox', { name: 'Ich habe sämtliche festen Ziele', exact: false }).check();
|
||||
await approve.click();
|
||||
await operation.locator('p.warning').filter({ hasText: 'Antrag: approved' }).waitFor();
|
||||
assert(approvals === 1);
|
||||
}
|
||||
await page.getByRole('button', { name: 'Abmelden', exact: true }).click();
|
||||
} finally { await browser.close(); }
|
||||
})().catch(error => {
|
||||
const line = error.stack?.match(/cloud_crash_acceptance\.cjs:(\d+):\d+/)?.[1] || 'unknown';
|
||||
console.log('CLOUD_CRASH_UI_INCOMPLETE:' + error.constructor.name + ':source_line=' + line);process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
// Actual development UI approval/revocation only; no API client or injected fetch.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const crypto = require('node:crypto');
|
||||
const { isDeepStrictEqual: equal } = require('node:util');
|
||||
const { chromium } = require('../../web/node_modules/playwright');
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const assert = value => { if (!value) throw new Error('Offline UI contract mismatch'); };
|
||||
function readPrivate(file) {
|
||||
const s = fs.lstatSync(file);
|
||||
assert(s.isFile() && !s.isSymbolicLink() && s.uid === process.getuid() && (s.mode & 0o777) === 0o600 && s.size < 1048576);
|
||||
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
}
|
||||
function counterScript(id) {
|
||||
assert(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(id));
|
||||
return 'set -eu\numask 077\nd=/var/lib/rmm-agent-remote/offline-' + id + '\n'
|
||||
+ 'test ! -e "$d"\nmkdir "$d"\nprintf \'%s\\n\' \'1\' > "$d/count"\n'
|
||||
+ 'test "$(cat "$d/count")" = 1\nprintf \'%s\\n\' \'RMM_OFFLINE_COUNT=1\'\n';
|
||||
}
|
||||
(async () => {
|
||||
const expected = readPrivate(path.resolve(process.argv[2]));
|
||||
assert(expected.profile === 'dev/test' && ['offline_wait', 'expired_offline', 'revoked_before_start'].includes(expected.case)
|
||||
&& ['approve', 'revoke'].includes(expected.phase) && expected.expected_principal === 'simulated-testprincipal-a');
|
||||
const login = readPrivate(path.join(root, 'credentials/server-browser-login.json'));
|
||||
const browsers = path.join(root, 'web/node_modules/.cache/playwright-browsers');
|
||||
const version = fs.readdirSync(browsers).find(x => /^chromium-/.test(x));
|
||||
const executablePath = ['chrome-linux/chrome', 'chrome-linux64/chrome'].map(x => path.join(browsers, version, x)).find(x => fs.existsSync(x));
|
||||
const browser = await chromium.launch({ headless: true, executablePath, env: { ...process.env, HOME: path.join(root, 'web/node_modules/.cache/cloud-acceptance/home') } });
|
||||
try {
|
||||
const page = await browser.newPage({ ignoreHTTPSErrors: false, timezoneId: 'UTC' }); page.setDefaultTimeout(15000);
|
||||
let approvals = 0, revocations = 0;
|
||||
page.on('request', r => { const p = new URL(r.url()).pathname; if (p.endsWith('/authorizations')) approvals++; if (p.endsWith('/revocation') && r.method() === 'POST') revocations++; });
|
||||
await page.goto('https://localhost:18443/');
|
||||
await page.getByLabel('Benutzername').fill(login.username); await page.getByLabel('Passwort', { exact: true }).fill(login.password);
|
||||
const response = page.waitForResponse(r => new URL(r.url()).pathname === '/auth/session' && r.status() === 200);
|
||||
await page.getByRole('button', { name: 'Anmelden', exact: true }).click();
|
||||
const session = await (await response).json();
|
||||
assert(session.testsimulation === true && session.approval_origin === 'dev_test_browser_simulation' && session.principal === expected.expected_principal);
|
||||
await page.getByRole('status').filter({ hasText: 'SIMULIERT' }).waitFor();
|
||||
assert(await page.getByLabel('Mandant', { exact: true }).inputValue() === expected.tenant_id);
|
||||
await page.getByRole('button', { name: expected.operation_id, exact: true }).click();
|
||||
const operation = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Antrag / Ausführung ' + expected.operation_id, exact: true }) });
|
||||
const state = JSON.parse(await operation.locator('pre').first().innerText());
|
||||
assert(state.approval_state === (expected.phase === 'approve' ? 'pending_approval' : 'approved'));
|
||||
if (expected.phase === 'approve') assert(state.job_count === 0 && state.execution_state === null);
|
||||
await operation.getByRole('button', { name: 'Unveränderlichen Plan zur Prüfung öffnen', exact: true }).click();
|
||||
const detail = page.locator('section').filter({ has: page.getByRole('heading', { name: 'Unveränderlicher Plan / Genehmigungsdetail', exact: true }) });
|
||||
const plan = JSON.parse(await detail.locator('pre').innerText());
|
||||
const { approval_state: actualApproval, ...actual } = plan;
|
||||
const { approval_state: plannedApproval, ...planned } = expected.plan;
|
||||
assert(equal(actual, planned) && plan.jobs.length === 1 && plan.resolved_targets.length === 1
|
||||
&& crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
|
||||
const job = plan.jobs[0], p = job.payload;
|
||||
assert(equal(job, expected.job) && p.agent_id === expected.agent_id && p.asset_id === expected.asset_id && p.tenant_id === expected.tenant_id
|
||||
&& p.execution_identity === 'agent_service' && p.max_runtime_seconds === 60 && p.max_output_bytes === 65536);
|
||||
assert(equal(p.action, { action: 'run_script', interpreter: 'posix_sh', script_utf8: counterScript(expected.id), arguments: [], expected_stdout: 'RMM_OFFLINE_COUNT=1\n' }));
|
||||
assert(Date.parse(p.start_before) - Date.parse(p.not_before) === 130000 && Date.parse(p.start_before) > Date.now());
|
||||
if (expected.phase === 'approve') {
|
||||
assert(actualApproval === 'pending_approval' && plannedApproval === null && Date.parse(p.not_before) > Date.now() + 90000);
|
||||
const approve = detail.getByRole('button', { name: 'SIMULIERT: Diesen festen Plan ausdrücklich genehmigen', exact: true });
|
||||
assert(await approve.isDisabled()); await detail.getByRole('checkbox', { name: 'Ich habe sämtliche festen Ziele', exact: false }).check(); await approve.click();
|
||||
await operation.locator('p.warning').filter({ hasText: 'Antrag: approved' }).waitFor();
|
||||
assert(approvals === 1 && revocations === 0);
|
||||
} else {
|
||||
assert(expected.case === 'revoked_before_start' && actualApproval === 'approved' && session.permissions.includes('plans:revoke') && Date.parse(p.not_before) > Date.now() + 1000);
|
||||
const control = detail.locator('details').filter({ has: page.locator('summary').filter({ hasText: /^Planberechtigung widerrufen$/ }) });
|
||||
await control.locator('summary').click();
|
||||
await control.getByLabel('Grund für Planberechtigung widerrufen', { exact: true }).fill('Explicit dev/test offline acceptance: revoke this exact future plan before any start');
|
||||
const nonce = await control.locator('code').innerText();
|
||||
await control.getByLabel('Bestätigungskennung eingeben', { exact: true }).fill(nonce);
|
||||
await control.getByRole('checkbox', { name: 'Ich bestätige diese konkrete administrative Wirkung ausdrücklich.' }).check();
|
||||
await control.getByRole('button', { name: 'Planberechtigung widerrufen · ausdrücklich ausführen', exact: true }).click();
|
||||
await control.locator('pre').filter({ hasText: '"approval_state": "revoked"' }).waitFor();
|
||||
assert(approvals === 0 && revocations === 1);
|
||||
}
|
||||
await page.getByRole('button', { name: 'Abmelden', exact: true }).click();
|
||||
} finally { await browser.close(); }
|
||||
})().catch(error => {
|
||||
const line = error.stack?.match(/cloud_offline_acceptance\.cjs:(\d+):\d+/)?.[1] || 'unknown';
|
||||
console.log('CLOUD_OFFLINE_UI_INCOMPLETE:' + error.constructor.name + ':source_line=' + line); process.exitCode = 1;
|
||||
});
|
||||
@@ -19,6 +19,15 @@ STATE=BASE/'cases'
|
||||
CASES=('apt_configure_recovery','apt_native_update','ensure_package','apply_updates')
|
||||
|
||||
|
||||
def runtime_session():
|
||||
latest=BASE/'journal-capacity-runtime.json'
|
||||
if latest.exists():
|
||||
report=cloud.contract.protected_json(latest)
|
||||
require(report['completed'] and report['preserved_state'],'qualified preserved capacity runtime')
|
||||
return report,BASE/'capacity-recovery'
|
||||
return cloud.contract.protected_json(BASE/'configure-runtime.json'),BASE
|
||||
|
||||
|
||||
def recovery_request(device,target):
|
||||
m=cloud.contract;data=prior.adapters.planning_from_device(device,recovery=True)
|
||||
closed=m.protected_json(BASE/'readonly-central-result.json')['result']
|
||||
@@ -49,14 +58,14 @@ def recovery_request(device,target):
|
||||
|
||||
def enable():
|
||||
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
ready=m.protected_json(BASE/'configure-runtime.json')
|
||||
ready,session=runtime_session();session.mkdir(mode=0o700,exist_ok=True)
|
||||
require(ready['completed'] and ready['readiness_only'] and ready['preserved_state'] and not ready['native_acceptance'],
|
||||
'qualified native probe, no invented package acceptance')
|
||||
target=prior.prior.identity();pin=ready['apt_pin'];identity=pin['identity']
|
||||
require(all(identity[k]==target[k] for k in target),'same permanent Cloud VM')
|
||||
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=True,
|
||||
agent_id=target['agent_id'],binary_sha256=ready['helper_sha256']))
|
||||
m.write_private(BASE/'recovery-endpoint-enabled.json',endpoint)
|
||||
m.write_private(session/'recovery-endpoint-enabled.json',endpoint)
|
||||
peers=cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,target)
|
||||
require(peers['ledger']['faults']==0 and peers['apt']['provider_stack_sha256']==pin['provider_stack_sha256'],
|
||||
'actual fixed helper peer and stack')
|
||||
@@ -67,15 +76,43 @@ def enable():
|
||||
apt_config_sha256=ready['apt_config_sha256'],provider_sha256=ready['provider_sha256'],
|
||||
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
|
||||
policy=dict(schema_version=1,profile='development',expires_at=(m.now()+dt.timedelta(hours=1)).isoformat(),scopes=[scope])
|
||||
old=m.protected_json(cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
||||
old=m.protected_json(BASE/'recovery-policy-closed.json' if session!=BASE else cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
||||
record=cloud.bootstrap.rpc('server.conf',prior.replace_policy,dict(id=str(uuid.uuid4()),previous_sha256=old['policy_sha256'],policy=policy))
|
||||
m.write_private(BASE/'recovery-policy.json',dict(record,policy=policy))
|
||||
m.write_private(session/'recovery-policy.json',dict(record,policy=policy))
|
||||
print('Fixed APT runtime endpoint and expiring limits enabled; no job or approval created')
|
||||
|
||||
|
||||
def accept(case):
|
||||
def unsubmitted_preparation(payload):
|
||||
import re
|
||||
root=Path('/opt/rust-discovery');ident=payload['test_result_id']
|
||||
require(isinstance(ident,str) and re.fullmatch('[A-Za-z0-9_-]{1,128}',ident),'fixed preparation test ID')
|
||||
query="SELECT count(*) FROM operation_plans WHERE tenant_id='11111111-1111-4111-8111-111111111111' AND position('"+ident+"' in convert_from(payload,'UTF8'))>0;"
|
||||
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
count=int(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip())
|
||||
return dict(persisted_plans=count)
|
||||
|
||||
|
||||
def accept(case,resume_unsubmitted=False):
|
||||
m=cloud.contract;target=prior.prior.identity()
|
||||
require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
ready,session=runtime_session();cases=session/'cases';cases.mkdir(mode=0o700,exist_ok=True)
|
||||
if session!=BASE and case=='apt_configure_recovery':
|
||||
original=m.protected_json(STATE/case/'checkpoint-capacity-terminal.json')
|
||||
old_job=original['plan']['jobs'][0]
|
||||
old_result=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=old_job['job_id']))
|
||||
counts=prior.prior.evidence('gate',job_id=old_job['job_id'])
|
||||
require(original['not_started_capacity_block'] and old_result['state'] in ('expired','cancelled') and counts['permits']==0,
|
||||
'preserve original approved no-start; new attempt requires fresh UI approval')
|
||||
folder=cases/case
|
||||
if resume_unsubmitted:
|
||||
require(folder.is_dir() and {p.name for p in folder.iterdir()}=={'prepare.json'},'only an unsubmitted UI preparation may be archived')
|
||||
old=m.protected_json(folder/'prepare.json')
|
||||
require(old['case']==case and old['phase']=='prepare' and all(old[k]==target[k] for k in target),'same fixed prior preparation target')
|
||||
untouched=cloud.bootstrap.rpc('server.conf',unsubmitted_preparation,dict(test_result_id=old['request']['parameters']['apt_manifest']['test_result_id']))
|
||||
require(untouched==dict(persisted_plans=0),'existing plan must be reviewed, never silently replaced')
|
||||
archived=cases/(case+'.unsubmitted-ui-'+uuid.uuid4().hex)
|
||||
os.rename(folder,archived);m.write_private(archived/'not-submitted.json',untouched)
|
||||
# A maintenance restart only refreshes passive inventory from the preserved
|
||||
# agent; it never issues a package diagnostic or executes a job.
|
||||
deadline=time.monotonic()+90
|
||||
@@ -87,28 +124,30 @@ def accept(case):
|
||||
except m.GuardError:
|
||||
require(time.monotonic()<deadline,'fresh stored exact planning unavailable; no job submitted');time.sleep(3)
|
||||
request=recovery_request(device,target) if case=='apt_configure_recovery' else None
|
||||
prior.accept(case,request=request,folder_root=STATE)
|
||||
if request is not None and session!=BASE:
|
||||
request['reason']+='; prior fixed plan expired with zero grants at the old journal capacity; preserve it and authorize this fresh plan separately'
|
||||
prior.accept(case,request=request,folder_root=cases)
|
||||
|
||||
|
||||
def close():
|
||||
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
record=m.protected_json(BASE/'recovery-policy.json')
|
||||
ready,session=runtime_session();record=m.protected_json(session/'recovery-policy.json')
|
||||
closed=cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=record['policy_sha256'],backup=record['backup']))
|
||||
m.write_private(BASE/'recovery-policy-closed.json',closed)
|
||||
ready=m.protected_json(BASE/'configure-runtime.json')
|
||||
m.write_private(session/'recovery-policy-closed.json',closed)
|
||||
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=False,
|
||||
agent_id=prior.prior.identity()['agent_id'],binary_sha256=ready['helper_sha256']))
|
||||
m.write_private(BASE/'recovery-endpoint-disabled.json',endpoint)
|
||||
m.write_private(session/'recovery-endpoint-disabled.json',endpoint)
|
||||
print('APT test limits closed and endpoint stopped; original histories retained')
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__);p.add_argument('action',choices=('enable','accept','finish','close'));p.add_argument('--case',choices=CASES)
|
||||
p.add_argument('--resume-unsubmitted-ui',action='store_true',help='Preserve a failed preparation only after proving no plan was persisted')
|
||||
args=p.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
|
||||
if args.action in ('accept','finish'):
|
||||
require(args.case is not None,'one fixed case required')
|
||||
if args.action=='accept':accept(args.case)
|
||||
else:prior.finish(args.case,folder_root=STATE)
|
||||
if args.action=='accept':accept(args.case,args.resume_unsubmitted_ui)
|
||||
else:prior.finish(args.case,folder_root=runtime_session()[1]/'cases')
|
||||
else:(enable if args.action=='enable' else close)()
|
||||
|
||||
|
||||
@@ -116,5 +155,5 @@ if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
message=str(error)
|
||||
print(message if message.startswith('remote bootstrap incomplete: ') else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
|
||||
print(message if message.startswith(('remote bootstrap incomplete: ','APT_UI_INCOMPLETE:source_lines=')) else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
|
||||
raise SystemExit(2) from None
|
||||
|
||||
@@ -205,7 +205,10 @@ def accept(case, *, request=None, folder_root=STATE):
|
||||
cloud.prepare_browser()
|
||||
def browser(file):
|
||||
result = subprocess.run(['node', str(cloud.ROOT/'deploy/browser/cloud_apt_acceptance.cjs'), str(file)], capture_output=True, timeout=120)
|
||||
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint, never repeat automatically')
|
||||
if result.returncode!=0:
|
||||
import re
|
||||
lines=re.findall(rb'CLOUD_APT_UI_INCOMPLETE:[A-Za-z]+:source_line=([0-9]+|unknown)',result.stdout)
|
||||
raise RuntimeError('APT_UI_INCOMPLETE:source_lines='+','.join(x.decode('ascii') for x in lines))
|
||||
print(case+': actual UI preparation', flush=True); browser(prepare)
|
||||
prepared = m.protected_json(output); plan = prepared['plan']; operation = prepared['operation']; job = plan['jobs'][0]
|
||||
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'immutable single job')
|
||||
|
||||
@@ -119,6 +119,13 @@ def status_vm(payload):
|
||||
result[name]=dict(faults=db.execute('SELECT count(*) FROM '+table+' WHERE fault IS NOT NULL').fetchone()[0],
|
||||
receipts=db.execute('SELECT count(*) FROM checkpoint_repair_receipts').fetchone()[0] if 'checkpoint_repair_receipts' in tables else 0,
|
||||
integrity_ok=db.execute('PRAGMA integrity_check').fetchone()[0]=='ok')
|
||||
if payload.get('job_id'):
|
||||
job=str(uuid.UUID(payload['job_id']));attempt=str(uuid.UUID(payload['attempt_id']))
|
||||
if name=='agent':
|
||||
result[name]['attempts']=[dict(state=s,revision=r,grant_id=g) for s,r,g in db.execute(
|
||||
'SELECT state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt))]
|
||||
else:
|
||||
result[name]['attempt_consumed']=db.execute('SELECT count(*) FROM consumed WHERE job=? AND attempt=?',(job,attempt)).fetchone()[0]
|
||||
if name=='helper':
|
||||
result[name].update(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
|
||||
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0])
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One bounded Cloud agent SIGKILL; independent root helper closes its outcome.
|
||||
|
||||
Requires fixed dev/test UI approval twice, a qualified agent and known closed
|
||||
root-policy hash. No API/DB approval, ledger edits, retry or unsigned recovery.
|
||||
Do not execute alongside another VM/runtime/approval operation.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import cloud_delivery_acceptance as delivery
|
||||
import cloud_apt_followup as prior
|
||||
import cloud_root_evidence
|
||||
import cloud_root_policy
|
||||
|
||||
cloud = delivery.cloud
|
||||
require = cloud.contract.require
|
||||
STATE = cloud.ROOT/'devstate/cloud-crash'
|
||||
|
||||
|
||||
def script(identifier, read_only=False):
|
||||
identifier = str(uuid.UUID(identifier))
|
||||
source = (cloud.ROOT/'deploy/linux/agent-crash-counter.py').read_text()
|
||||
return ("exec /usr/bin/python3 -I - "+('read' if read_only else 'effect')+' '+identifier+
|
||||
" <<'RMM_AGENT_CRASH_COUNTER'\n"+source+'\nRMM_AGENT_CRASH_COUNTER\n')
|
||||
|
||||
|
||||
def vm(payload):
|
||||
"""Read retained evidence; optionally signal only the exact bound MainPID."""
|
||||
import signal
|
||||
import sqlite3
|
||||
import stat
|
||||
state = Path('/var/lib/rmm-agent-remote')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'permanent identity')
|
||||
for path,key in ((Path('/usr/local/libexec/rmm-agent-remote'),'agent_sha256'),
|
||||
(Path('/usr/local/libexec/rmm-privileged-helper'),'helper_sha256')):
|
||||
require(not path.is_symlink() and hashlib.sha256(path.read_bytes()).hexdigest() == payload[key],'qualified pinned artifact')
|
||||
def process(unit, expected_sha):
|
||||
raw = subprocess.run(['systemctl','show',unit,'-p','MainPID','--value'],capture_output=True,check=True,timeout=10).stdout
|
||||
pid = int(raw);require(pid > 1,'active pinned service')
|
||||
require(hashlib.sha256(Path('/proc/'+str(pid)+'/exe').read_bytes()).hexdigest() == expected_sha,'running service artifact')
|
||||
start = Path('/proc/'+str(pid)+'/stat').read_text().rsplit(')',1)[1].split()[19]
|
||||
return dict(pid=pid,start=start)
|
||||
processes = dict(agent=process('rmm-agent-remote.service',payload['agent_sha256']),
|
||||
helper=process('rmm-helper-cloud.service',payload['helper_sha256']))
|
||||
result = dict(processes=processes)
|
||||
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','native journal integrity')
|
||||
result['native_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
|
||||
if payload.get('job_id'):
|
||||
job = str(uuid.UUID(payload['job_id']));attempt = str(uuid.UUID(payload['attempt_id']))
|
||||
rows = db.execute('SELECT state,revision,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
|
||||
result['attempts'] = [dict(state=r[0],revision=r[1],grant_id=r[2]) for r in rows]
|
||||
result['agent_last_decision'] = db.execute('SELECT last_decision FROM reconcile_meta').fetchone()[0]
|
||||
helper = Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
with sqlite3.connect(helper.as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok','root ledger integrity')
|
||||
result['ledger'] = dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
|
||||
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],
|
||||
faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0])
|
||||
if payload.get('job_id'):
|
||||
rows = db.execute('SELECT c.grant_id,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.job=? AND c.attempt=?',(job,attempt)).fetchall()
|
||||
result['root_attempts'] = []
|
||||
for grant,finished,start,outcome in rows:
|
||||
stored = json.loads(start);request = stored['request'];permit = json.loads(bytes(request['permit_bytes']))
|
||||
require(permit['job_id'] == job and permit['attempt_id'] == attempt and permit['agent_id'] == payload['agent_id']
|
||||
and permit['grant_id'] == grant and hashlib.sha256(bytes(request['job_bytes'])).hexdigest() == payload['job_payload_sha256']
|
||||
and request['plan_hash'] == payload['plan_sha256'],'exact durable root start binding')
|
||||
result['root_attempts'].append(dict(grant_id=grant,finished=json.loads(finished) if finished else None,
|
||||
outcome=json.loads(outcome) if outcome else None,start_sha256=hashlib.sha256(start).hexdigest()))
|
||||
head = db.execute('SELECT last_decision FROM helper_sync_meta').fetchone()[0]
|
||||
if head:
|
||||
row = db.execute('SELECT response FROM helper_sync_decisions WHERE decision_id=?',(head,)).fetchone()
|
||||
if row:
|
||||
signed = json.loads(row[0]);decision = json.loads(bytes(signed['decision_bytes']))
|
||||
selected = [a for a in decision['attempts'] if a['job_id'] == job and a['attempt_id'] == attempt]
|
||||
result['signed_sync'] = dict(decision_id=head,ready=decision['ready'],helper_ready=decision['helper_ready'],
|
||||
payload_sha256=hashlib.sha256(bytes(signed['decision_bytes'])).hexdigest(),signature_sha256=hashlib.sha256(bytes(signed['signature'])).hexdigest(),
|
||||
selected=selected)
|
||||
if payload.get('job_id'):
|
||||
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
result['agent_consumed'] = [dict(grant_id=g,finished=json.loads(o) if o else None) for g,o in db.execute('SELECT grant_id,result FROM consumed WHERE job=? AND attempt=?',(job,attempt))]
|
||||
identifier = str(uuid.UUID(payload['id']));directory = helper.parent/('crash-'+identifier)
|
||||
require(not directory.is_symlink(),'counter symlink')
|
||||
result['counter'] = None
|
||||
if directory.exists():
|
||||
require(directory.stat().st_uid == 0 and stat.S_IMODE(directory.stat().st_mode) == 0o700,'private root counter')
|
||||
path = directory/'count'
|
||||
if path.exists():
|
||||
fd = os.open(path,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd,'rb') as stream:
|
||||
metadata = os.fstat(stream.fileno());require(stat.S_ISREG(metadata.st_mode) and metadata.st_uid == 0
|
||||
and metadata.st_nlink == 1 and stat.S_IMODE(metadata.st_mode) == 0o600 and metadata.st_size <= 16,'fixed root counter file')
|
||||
raw = stream.read();result['counter'] = int(raw)
|
||||
if payload['mode'] == 'kill':
|
||||
require(processes == payload['processes'],'exact original process instance')
|
||||
require(len(result['attempts']) == len(result['root_attempts']) == len(result['agent_consumed']) == 1
|
||||
and result['attempts'][0]['state'] == 'running' and result['root_attempts'][0]['finished'] is None
|
||||
and result['root_attempts'][0]['outcome'] is None and result['counter'] == 1
|
||||
and result['ledger'] == dict(consumed=payload['baseline_consumed']+1,unresolved=1,faults=0),'one actual running root effect; deterministic independent closure')
|
||||
require(result['attempts'][0]['grant_id'] == result['root_attempts'][0]['grant_id'] == result['agent_consumed'][0]['grant_id'],'one consumed grant on both sides')
|
||||
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-agent-crash-'+identifier)
|
||||
backup.mkdir(mode=0o700)
|
||||
fd = os.open(backup/'before-signal.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream:json.dump(result,stream);stream.flush();os.fsync(stream.fileno())
|
||||
# pidfd prevents a PID-reuse race. No unit stop, cgroup/process-tree kill,
|
||||
# helper signal or package-manager intervention exists here.
|
||||
fd = os.pidfd_open(processes['agent']['pid'])
|
||||
try:
|
||||
require(process('rmm-agent-remote.service',payload['agent_sha256']) == processes['agent'],'same agent before signal')
|
||||
signal.pidfd_send_signal(fd,signal.SIGKILL)
|
||||
finally: os.close(fd)
|
||||
require(process('rmm-helper-cloud.service',payload['helper_sha256']) == processes['helper'],'independent helper survived')
|
||||
return dict(before=result,agent_signal='SIGKILL',helper_unchanged=True,backup=str(backup))
|
||||
require(payload['mode'] == 'inspect','fixed fault operation')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
|
||||
parser.add_argument('--expected-agent-sha256',required=True)
|
||||
parser.add_argument('--expected-helper-sha256',required=True)
|
||||
parser.add_argument('--expected-root-policy-sha256',required=True)
|
||||
args = parser.parse_args();os.umask(0o077)
|
||||
for value in (args.expected_agent_sha256,args.expected_helper_sha256,args.expected_root_policy_sha256):
|
||||
require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified artifact/config pin')
|
||||
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
|
||||
cloud.prepare_browser()
|
||||
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
||||
settings = dict(tenant_id=cloud.TENANT,**identity)
|
||||
backend = cloud.Backend()
|
||||
def evidence(kind,**extra):return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind=kind,tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**extra))
|
||||
require(evidence('gate')['unresolved'] == 0,'previous central effects must be closed')
|
||||
identifier = str(uuid.uuid4())
|
||||
scope = dict(id=identifier,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256,helper_sha256=args.expected_helper_sha256)
|
||||
initial = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(scope,mode='inspect'))
|
||||
require(initial['native_unresolved'] == initial['ledger']['unresolved'] == initial['ledger']['faults'] == 0
|
||||
and initial['counter'] is None,'native/history must be settled; untouched unique counter')
|
||||
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
require(observed['sync_ready'] and not observed['restore_blocked'],'signed sync before new root plan')
|
||||
peers = cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,dict(settings,general_only=True))
|
||||
pin = peers['general'];binding = pin['identity']
|
||||
root_scope = {k:binding[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')}
|
||||
root_scope.update(principals=['simulated-testprincipal-a'],helper_policy_sha256=pin['policy_sha256'],
|
||||
acceptance_evidence_id='cloud-agent-crash-independent-helper:'+identifier,allowed_services=[],allow_root_scripts=True,allow_reboot=False)
|
||||
policy = dict(schema_version=1,profile='development',expires_at=(cloud.contract.now()+dt.timedelta(minutes=15)).isoformat(),scopes=[root_scope])
|
||||
folder = STATE/identifier;folder.mkdir(mode=0o700,parents=True)
|
||||
checkpoint = dict(profile='dev/test',case='agent_crash',expected_principal='simulated-testprincipal-a',**settings,
|
||||
id=identifier,initial=initial,qualified=scope,stage='prepared_no_job',no_retry=True)
|
||||
path = folder/'checkpoint.json'
|
||||
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
|
||||
save()
|
||||
opened = cloud.bootstrap.rpc('server.conf',prior.narrow_policy,dict(id=identifier,expected_sha256=args.expected_root_policy_sha256,policy=policy))
|
||||
checkpoint['policy_open'] = opened;save()
|
||||
def browser(value, name):
|
||||
file = folder/(name+'.json');cloud.contract.write_private(file,value)
|
||||
process = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_crash_acceptance.cjs'),str(file)],capture_output=True,timeout=120)
|
||||
require(process.returncode == 0,'actual UI incomplete; preserve pending plan; never resubmit/approve again')
|
||||
def authorize(read_only):
|
||||
# A busy helper may have temporarily removed root capabilities from the
|
||||
# new process's hello. Wait for genuine measured readvertisement/Sync;
|
||||
# never plan a new effect against a stale capability or readiness flag.
|
||||
deadline = time.monotonic()+100
|
||||
while True:
|
||||
device = backend.call('inventory.get_device',dict(tenant_id=cloud.TENANT,asset_id=identity['asset_id']))
|
||||
current = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
if (device['agent_id'] == identity['agent_id'] and device['presence'] == 'online'
|
||||
and {'run_script','root:run_script','helper.root.v1','helper.policy.sha256:'+pin['policy_sha256']} <= set(device['capabilities'])
|
||||
and current['sync_ready'] and not current['restore_blocked']):break
|
||||
require(time.monotonic() < deadline,'fresh root capability and signed Sync before new plan');time.sleep(2)
|
||||
request = cloud.contract.case_input('service_setup',settings)
|
||||
request['parameters'].update(variants=[dict(os_family='linux',interpreter='posix_sh',encoding='utf-8',script_utf8=script(identifier,read_only))],
|
||||
expected_stdout='RMM_CRASH_COUNT=1\n',timeout_seconds=120,max_output_bytes=65536)
|
||||
request['reason'] = 'Explicit bounded Cloud agent crash '+('independent final counter read' if read_only else 'one root counter effect, helper remains alive for90s')
|
||||
common = dict(profile='dev/test',case='agent_crash',expected_principal=checkpoint['expected_principal'],**settings,id=identifier,read_only=read_only,request=request)
|
||||
name = 'postcheck' if read_only else 'effect'
|
||||
output = folder/(name+'-prepared.json')
|
||||
browser(dict(common,phase='prepare',output_file=str(output)),name+'-prepare')
|
||||
prepared = cloud.contract.protected_json(output);plan=prepared['plan'];pending=prepared['operation']
|
||||
require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'],'one immutable fixed plan')
|
||||
job = plan['jobs'][0];gate = evidence('gate',job_id=job['job_id'])
|
||||
stored = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,operation_id=pending['operation_id']))
|
||||
require(stored['approval_state'] == 'pending_approval' and stored['job_count'] == 0 and stored['execution_state'] is None
|
||||
and stored['plan_hash'] == plan['plan_hash'] and all(gate[k] == 0 for k in ('jobs','outbox','permits')),'pending zero artifacts')
|
||||
record = dict(plan=plan,job=job,operation_id=pending['operation_id'],preapproval=gate,request=request)
|
||||
checkpoint[name] = record;save()
|
||||
browser(dict(common,phase='approve',**record),name+'-approve')
|
||||
return record
|
||||
effect = authorize(False);job=effect['job']
|
||||
fixed = dict(scope,job_id=job['job_id'],attempt_id=job['payload']['attempt_id'],job_payload_sha256=job['job_payload_hash'],plan_sha256=effect['plan']['plan_hash'])
|
||||
deadline = time.monotonic()+90
|
||||
while True:
|
||||
before = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
||||
if before['counter'] == 1 and before['root_attempts'] and before['root_attempts'][0]['outcome'] is None:break
|
||||
require(time.monotonic() < deadline,'counter/running marker missing; never kill outside bound window');time.sleep(1)
|
||||
killed = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='kill',processes=initial['processes'],baseline_consumed=initial['ledger']['consumed']))
|
||||
checkpoint['signal'] = killed;save()
|
||||
deadline = time.monotonic()+100
|
||||
while True:
|
||||
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
||||
if result['state'] == 'unknown':
|
||||
unknown = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
||||
if unknown['root_attempts'][0]['outcome'] is None and unknown['attempts'][0]['state'] == 'unknown':break
|
||||
require(result['state'] not in cloud.contract.TERMINAL and time.monotonic() < deadline,'actual durable and central unknown not observed; preserve same attempt');time.sleep(1)
|
||||
require(result['result']['error_code'] == 'AGENT_PROCESS_INTERRUPTED' and unknown['counter'] == 1
|
||||
and unknown['processes']['agent'] != initial['processes']['agent']
|
||||
and unknown['processes']['helper'] == initial['processes']['helper'],'actual new agent process; independent live helper')
|
||||
checkpoint['unknown'] = dict(central=result,native=unknown);save()
|
||||
deadline = time.monotonic()+180
|
||||
while True:
|
||||
final = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
||||
native = cloud.bootstrap.rpc('linux-vm.conf',vm,dict(fixed,mode='inspect'))
|
||||
sync = native.get('signed_sync',{})
|
||||
if (final['state'] in cloud.contract.TERMINAL and native['ledger']['unresolved'] == 0 and sync.get('ready') and sync.get('helper_ready')
|
||||
and native['agent_last_decision'] == sync.get('decision_id')):break
|
||||
require(time.monotonic() < deadline,'same-attempt definitive reconciliation missing; never unlock/retry');time.sleep(2)
|
||||
require(final['state'] == 'succeeded' and final['result']['exit_code'] == 0 and bytes(final['result']['stdout']) == b'RMM_CRASH_COUNT=1\n'
|
||||
and final['result']['evidence']['postconditions_passed'] and native['counter'] == 1
|
||||
and len(native['root_attempts']) == len(native['agent_consumed']) == len(native['attempts']) == 1
|
||||
and native['ledger'] == dict(consumed=initial['ledger']['consumed']+1,unresolved=0,faults=0),'one actual helper completion and consumption')
|
||||
selected = sync['selected'];grant = killed['before']['root_attempts'][0]['grant_id']
|
||||
require(len(selected) == 1 and selected[0]['grant_id'] == grant and selected[0]['unlock'] is True
|
||||
and selected[0]['outcome_state'] == 'succeeded' and selected[0]['reason'] == 'DEFINITIVE_OUTCOME_RECONCILED'
|
||||
and selected[0]['revision'] > unknown['attempts'][0]['revision']
|
||||
and selected[0]['job_payload_hash'] == fixed['job_payload_sha256'] and selected[0]['plan_hash'] == fixed['plan_sha256']
|
||||
and native['root_attempts'][0]['grant_id'] == native['agent_consumed'][0]['grant_id'] == grant,'exact existing signed Sync closure')
|
||||
checkpoint.update(final=final,native=native,stage='reconciled');save()
|
||||
def audit(record):
|
||||
record['postapproval'] = evidence('gate',job_id=record['job']['job_id'])
|
||||
record['audit'] = evidence('audit',operation_id=record['operation_id']);save()
|
||||
require([record['postapproval'][k] for k in ('jobs','outbox','permits')] == [1,1,1]
|
||||
and record['audit'] == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=record['plan']['plan_hash']),'one authorized immutable attempt')
|
||||
audit(effect)
|
||||
postcheck = authorize(True);deadline=time.monotonic()+90
|
||||
while True:
|
||||
checked = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=postcheck['job']['job_id']))
|
||||
if checked['state'] in cloud.contract.TERMINAL or checked['state'] == 'unknown':break
|
||||
require(time.monotonic() < deadline,'independent read outcome missing');time.sleep(2)
|
||||
postcheck['result'] = checked;save()
|
||||
require(checked['state'] == 'succeeded' and checked['result']['exit_code'] == 0
|
||||
and checked['result']['evidence']['postconditions_passed'] and bytes(checked['result']['stdout']) == b'RMM_CRASH_COUNT=1\n','independent final counter still one')
|
||||
audit(postcheck)
|
||||
require(evidence('gate')['unresolved'] == 0,'all effects must be closed before policy close')
|
||||
closed = cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=opened['policy_sha256'],backup=opened['backup']))
|
||||
checkpoint.update(policy_closed=closed,verified=True,stage='native_dev_test_accepted');save()
|
||||
print('PASS bounded Cloud agent crash: durable+central unknown, same-attempt root outcome and signed Sync, one effect, separate UI-approved counter1; dev/test only')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
print('CLOUD_CRASH_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
@@ -31,6 +31,9 @@ def vm_pending(payload):
|
||||
events.append(dict(message_id=ident,agent_id=e[4].decode(),snapshot_id=snap[1].decode(),
|
||||
observed_at_ms=e[9],sequence=e[8],data_sha256=hashlib.sha256(snap[6]).hexdigest(),
|
||||
payload_sha256=hashlib.sha256(b''.join(raw for number,raw in parts if number!=6)).hexdigest(),
|
||||
collection_started_at=data.get('collection_started_at'),collection_completed_at=data.get('collection_completed_at'),
|
||||
observations=[dict(scope=o.get('scope'),source_id=o.get('source_id'),quality=o.get('quality'),observed_at=o.get('observed_at'),
|
||||
bytes=len(json.dumps(o,separators=(',',':')).encode())) for o in data.get('observations',[])],
|
||||
recovery_scope_present=any(o.get('scope')=='linux.apt.recovery-planning' for o in data.get('observations',[]))))
|
||||
return dict(events=events,other_unacked=other)
|
||||
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preserving Cloud journal-capacity maintenance; never a job/history reset.
|
||||
|
||||
The copy-only mode changes only its own private disposable database copy.
|
||||
Installation requires a separate explicit option and qualified artifact hashes.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import cloud_apt_recovery_upgrade as previous
|
||||
|
||||
cloud = previous.cloud
|
||||
require = cloud.require
|
||||
STATE = previous.STATE
|
||||
|
||||
|
||||
def copy_reproducer(payload):
|
||||
import sqlite3
|
||||
os.umask(0o077)
|
||||
source = Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
state = Path('/var/lib/rmm-agent-remote')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed permanent Cloud identity')
|
||||
require(not source.is_symlink() and source.stat().st_uid == 0 and source.stat().st_mode&0o777 == 0o600, 'private original ledger')
|
||||
def fingerprint(db):
|
||||
h = hashlib.sha256()
|
||||
tables = db.execute("SELECT name,sql FROM sqlite_master WHERE type='table' ORDER BY name").fetchall()
|
||||
for name,sql in tables:
|
||||
require(name.replace('_','').isalnum(), 'fixed stored table name')
|
||||
h.update(json.dumps([name,sql],separators=(',',':')).encode())
|
||||
for row in db.execute('SELECT * FROM "'+name+'" ORDER BY rowid'):
|
||||
data = [dict(blob_sha256=hashlib.sha256(v).hexdigest(),length=len(v)) if isinstance(v,bytes) else v for v in row]
|
||||
h.update(json.dumps(data,separators=(',',':'),ensure_ascii=True).encode())
|
||||
return h.hexdigest()
|
||||
def identity():
|
||||
s = source.stat()
|
||||
return dict(device=s.st_dev,inode=s.st_ino,size=s.st_size,sha256=hashlib.sha256(source.read_bytes()).hexdigest())
|
||||
before = identity()
|
||||
with sqlite3.connect(source.as_uri()+'?mode=ro',uri=True) as original:
|
||||
require(original.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'original integrity')
|
||||
require({r[0] for r in original.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants'])
|
||||
and len(payload['expected_grants']) == 10
|
||||
and original.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0
|
||||
and original.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0,
|
||||
'exact ten known closed Root consumptions')
|
||||
original_fingerprint = fingerprint(original)
|
||||
folder = Path('/var/lib/rmm-bootstrap-history')/('journal-capacity-copy-'+str(uuid.UUID(payload['id'])))
|
||||
folder.mkdir(mode=0o700)
|
||||
copy = folder/'ledger-copy.sqlite'
|
||||
fd = os.open(copy,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600);os.close(fd)
|
||||
with sqlite3.connect(copy) as duplicate:
|
||||
original.backup(duplicate)
|
||||
require(fingerprint(duplicate) == original_fingerprint, 'consistent private SQLite backup')
|
||||
page_size = duplicate.execute('PRAGMA page_size').fetchone()[0]
|
||||
page_count = duplicate.execute('PRAGMA page_count').fetchone()[0]
|
||||
old = duplicate.execute('PRAGMA max_page_count=16384').fetchone()[0]
|
||||
require(page_size == 4096 and old == 16384 and page_count <= old, 'actual bounded old capacity')
|
||||
failed = None
|
||||
try:
|
||||
duplicate.execute('BEGIN IMMEDIATE')
|
||||
duplicate.execute('CREATE TABLE journal_capacity_probe(payload BLOB NOT NULL)')
|
||||
duplicate.execute('INSERT INTO journal_capacity_probe VALUES(zeroblob(524288))')
|
||||
except sqlite3.DatabaseError as error:
|
||||
failed = getattr(error,'sqlite_errorcode',None)
|
||||
finally:
|
||||
duplicate.rollback()
|
||||
require(failed == sqlite3.SQLITE_FULL, 'old limit must fail with SQLITE_FULL13 on the copied actual DB')
|
||||
require(fingerprint(duplicate) == original_fingerprint, 'old-capacity test rolled back every row/schema change')
|
||||
bounded = duplicate.execute('PRAGMA max_page_count=65536').fetchone()[0]
|
||||
require(bounded == 65536, 'bounded 256 MiB new capacity')
|
||||
duplicate.execute('BEGIN IMMEDIATE')
|
||||
duplicate.execute('CREATE TABLE journal_capacity_probe(payload BLOB NOT NULL)')
|
||||
duplicate.execute('INSERT INTO journal_capacity_probe VALUES(zeroblob(524288))')
|
||||
require(duplicate.execute('SELECT length(payload) FROM journal_capacity_probe').fetchone()[0] == 524288, 'same copied-DB insertion passes at the new finite cap')
|
||||
duplicate.rollback()
|
||||
require(fingerprint(duplicate) == original_fingerprint and duplicate.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'new-capacity test rolled back every row/schema change')
|
||||
after = identity()
|
||||
require(after == before, 'original DB bytes, size and inode must remain unchanged')
|
||||
report = dict(verified=True,backup=str(folder),old_cap_pages=16384,new_cap_pages=65536,page_size=page_size,page_count=page_count,
|
||||
old_error_code=failed,probe_bytes=524288,new_insert_passed=True,copy_original_content_sha256=original_fingerprint,
|
||||
original_before=before,original_after=after,original_unchanged=True,rows_and_schema_rolled_back=True,
|
||||
original_package_effects=False,original_sql_mutations=False)
|
||||
fd = os.open(folder/'report.json',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream:json.dump(report,stream,indent=2);stream.flush();os.fsync(stream.fileno())
|
||||
return report
|
||||
|
||||
|
||||
|
||||
def install_capacity(payload):
|
||||
import fcntl
|
||||
import sqlite3
|
||||
import time
|
||||
import traceback
|
||||
os.umask(0o077)
|
||||
base = Path('/var/lib/rmm-apt-transaction'); state = Path('/var/lib/rmm-agent-remote')
|
||||
ledger = Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
helper = Path('/usr/local/libexec/rmm-privileged-helper'); agent = Path('/usr/local/libexec/rmm-agent-remote')
|
||||
provider = Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')
|
||||
config = Path('/etc/rmm/apt-helper.json'); proof_file = Path('/etc/rmm-remote/apt-helper-proof.json')
|
||||
metadata = base/'install-metadata.json'; readiness_file = base/'native-readiness.json'
|
||||
dropin = Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
|
||||
reduced = ('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
|
||||
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
|
||||
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
|
||||
def run(args, **kwargs):
|
||||
return subprocess.run(args, capture_output=True, check=True, timeout=90, **kwargs).stdout
|
||||
def sha(raw): return hashlib.sha256(raw).hexdigest()
|
||||
def canonical(value): return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()
|
||||
def fsync_dir(path):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try: os.fsync(fd)
|
||||
finally: os.close(fd)
|
||||
def create(path, raw, mode=0o600, gid=0):
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(raw); stream.flush(); os.fchown(stream.fileno(), 0, gid)
|
||||
os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno())
|
||||
fsync_dir(path.parent)
|
||||
def manifest(path):
|
||||
require(not path.is_symlink(), 'preserved directory symlink')
|
||||
if not path.exists(): return {'__absent__':True}
|
||||
require(path.is_dir(), 'preserved directory required')
|
||||
result = {}
|
||||
for p in [path, *sorted(path.rglob('*'))]:
|
||||
s = p.lstat(); require(not p.is_symlink() and (p.is_dir() or p.is_file()), 'preserved regular state required')
|
||||
result[str(p.relative_to(path))] = dict(dev=s.st_dev, inode=s.st_ino, mode=s.st_mode,
|
||||
uid=s.st_uid, gid=s.st_gid, sha256=sha(p.read_bytes()) if p.is_file() else None)
|
||||
return result
|
||||
def guard():
|
||||
require(len(payload['expected_grants']) == 10, 'exact ten original closed Root grants')
|
||||
require(payload['partial_grant'] == '31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0' and
|
||||
payload['partial_proof_sha256'] == 'dc3bf3870ea77c12f9291bac712c53ae72194eaec980a8646155a7fb5c0dbe21', 'one reviewed closed partial only')
|
||||
with sqlite3.connect(ledger.as_uri()+'?mode=ro', uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'Root ledger integrity')
|
||||
require({r[0] for r in db.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants']) and
|
||||
db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0] == 0 and
|
||||
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'exact known closed Root attempts')
|
||||
db.row_factory = sqlite3.Row
|
||||
rows = db.execute('SELECT * FROM apt_partial_reconciliations').fetchall()
|
||||
require(len(rows) == 1, 'one immutable partial proof required'); row = dict(rows[0])
|
||||
expected = payload['proof_row']
|
||||
for key in ('grant_id', 'original_request_sha256', 'previous_outcome_sha256', 'proof_sha256', 'checked_at_ms'):
|
||||
require(row[key] == expected[key], 'immutable partial binding drift')
|
||||
for key in ('previous_outcome', 'proof', 'classified_outcome'):
|
||||
require(sha(row[key]) == expected[key]['sha256'] and len(row[key]) == expected[key]['size'], 'immutable partial bytes drift')
|
||||
require(row['grant_id'] == payload['partial_grant'] and sha(row['proof']) == payload['partial_proof_sha256'], 'closed proof hash required')
|
||||
proof = json.loads(row['proof']); result = db.execute('SELECT c.result,e.outcome,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE grant_id=?', (payload['partial_grant'],)).fetchone()
|
||||
classified = json.loads(row['classified_outcome'])
|
||||
require(json.loads(result['result']) == json.loads(result['outcome']) == classified and
|
||||
classified['state'] == 'failed' and classified['error_code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED', 'normal signed RootSync closure required')
|
||||
saved = json.loads(result['start'])
|
||||
# save_start nests Request through serde_json::Value (sorted keys),
|
||||
# whereas the immutable proof hashes serde's Request struct codec.
|
||||
request_fields = ('job_bytes','permit_bytes','signature','plan_hash','authorization_id')
|
||||
require(set(saved['request']) == set(request_fields), 'original Request fields required')
|
||||
request_bytes = json.dumps({key:saved['request'][key] for key in request_fields},
|
||||
separators=(',', ':'), ensure_ascii=False).encode()
|
||||
require(sha(request_bytes) == row['original_request_sha256'], 'original signed request hash required')
|
||||
require(proof['grant_id'] == payload['partial_grant'] and proof['attempt_id'] == payload['partial_attempt'] and
|
||||
proof['snapshot']['rmm-mvp-test:all'] == {'version':'1.0','status':'install ok unpacked'}, 'exact partial fixture required')
|
||||
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro', uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
|
||||
db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0] == 0 and
|
||||
db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'no unresolved or faulted agent attempt')
|
||||
require(Path('/usr/share/rmm-mvp-test/version').read_bytes() == b'1.0\n', 'existing unpacked fixture payload required')
|
||||
require(not list(base.glob('acceptance-*.json')) and not list(base.glob('pending-acceptance-*.json')) and
|
||||
not list(base.glob('recovery-*.json')) and not list(base.glob('pending-recovery-*.json')), 'no existing acceptance or recovery attempt')
|
||||
return proof
|
||||
def bound_partial(proof):
|
||||
keys = 'transaction_sha256 job_payload_sha256 effect_armed_ledger_id grant_id job_id attempt_id authorization_id plan_sha256'.split()
|
||||
value = json.loads(run(['/usr/bin/python3','-I',str(provider),'verify-partial'], input=canonical({k:proof[k] for k in keys})))
|
||||
require(value['effect_state'] == 'known_failed' and value['code'] == 'APT_PARTIAL_UNCONFIGURED_VERIFIED' and
|
||||
{k:v for k,v in value['proof'].items() if k != 'checked_at_ms'} ==
|
||||
{k:v for k,v in proof.items() if k != 'checked_at_ms'}, 'fresh original-bound partial state required')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'permanent Cloud identity required')
|
||||
require(not dropin.is_symlink() and dropin.read_text() == reduced, 'APT endpoint must remain absent')
|
||||
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT helper must be stopped')
|
||||
for name, pin in payload['old_files'].items():
|
||||
p = Path(name); require(not p.is_symlink() and p.stat().st_uid == 0 and p.stat().st_mode & 0o022 == 0 and sha(p.read_bytes()) == pin, 'installed file drift')
|
||||
require(Path('/etc/systemd/system/rmm-apt-helper.service').read_text() == payload['unit'], 'exact reviewed APT service required')
|
||||
markers = {base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json':'ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053',
|
||||
base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json':'a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3'}
|
||||
require({p.name for p in base.glob('bootstrap-*.json')} == {p.name for p in markers} and
|
||||
all(sha(p.read_bytes()) == pin for p,pin in markers.items()), 'original/retry tombstones required')
|
||||
partial = guard(); bound_partial(partial)
|
||||
raw = {name:base64.b64decode(payload[name], validate=True) for name in ('helper','agent')}
|
||||
pins = {'helper':payload['helper_sha256'],'agent':payload['agent_sha256'],
|
||||
'provider':'5ff260f75cb2959678f6be729d3a806a4ea29febd5ae4003317fed41afc25327'}
|
||||
require(all(sha(raw[name]) == pins[name] for name in ('helper','agent')) and
|
||||
sha(provider.read_bytes()) == pins['provider'], 'qualified exact capacity binaries and unchanged provider required')
|
||||
for name in ('helper','agent'):
|
||||
require(raw[name][:6] == b'\x7fELF\x02\x01' and raw[name][18:20] == b'\x3e\x00', 'Linux x64 binary required')
|
||||
if payload.get('preflight_only') is True:
|
||||
return dict(stage='preflight_validated', completed=False, preflight_only=True,
|
||||
package_effects=False, maintenance_mutations=False, expected_grants=len(payload['expected_grants']),
|
||||
partial_proof_sha256=payload['partial_proof_sha256'], apt_endpoint_enabled=False)
|
||||
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-journal-capacity-'+payload['id']); backup.mkdir(mode=0o700)
|
||||
report = dict(stage='validated', completed=False, backup=str(backup), package_effects=False,
|
||||
apt_endpoint_enabled=False, partial_proof_sha256=payload['partial_proof_sha256'], backups={})
|
||||
def save():
|
||||
next_path = backup/'.report-next.json'; create(next_path,json.dumps(report,indent=2).encode())
|
||||
os.replace(next_path,backup/'report.json'); fsync_dir(backup)
|
||||
def replace(path, staged=None):
|
||||
target = backup/str(path).lstrip('/').replace('/','__')
|
||||
require(not path.is_symlink() and path.stat().st_dev == backup.stat().st_dev and not target.exists(), 'exclusive same-filesystem versioned replacement')
|
||||
os.rename(path,target); fsync_dir(path.parent); fsync_dir(backup)
|
||||
report['backups'][str(path)] = str(target); save()
|
||||
if staged is not None:
|
||||
os.rename(staged,path); fsync_dir(path.parent); fsync_dir(backup)
|
||||
lock = None
|
||||
try:
|
||||
save()
|
||||
for name in ('helper','agent'): create(backup/(name+'-next'), raw[name], 0o755)
|
||||
for name in ('helper','agent'): run(['/lib64/ld-linux-x86-64.so.2','--list',str(backup/(name+'-next'))])
|
||||
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
|
||||
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
partial = guard(); bound_partial(partial)
|
||||
preserved = {str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent)}
|
||||
preserved_base = manifest(base)
|
||||
witnesses = {}
|
||||
for p in (Path('/var/lib/dpkg/status'),Path('/usr/share/rmm-mvp-test/version'),dropin,
|
||||
Path('/etc/systemd/system/rmm-apt-helper.service'),Path('/usr/lib/rmm-apt-transaction/hook'),
|
||||
Path('/etc/rmm-remote/general-helper-proof.json'),Path('/etc/rmm-remote/helper-general.json'),provider,metadata):
|
||||
s = p.stat(); witnesses[str(p)] = (s.st_dev,s.st_ino,sha(p.read_bytes()))
|
||||
general_bytes = Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
|
||||
general_config = Path('/etc/rmm-remote/helper-general.json').read_bytes()
|
||||
old_metadata = json.loads(metadata.read_text()); old_config = json.loads(config.read_text())
|
||||
require(old_metadata['schema_version'] == 2 and old_metadata['provider_sha256'] == sha(provider.read_bytes()) and
|
||||
old_metadata['hook_sha256'] == sha(Path('/usr/lib/rmm-apt-transaction/hook').read_bytes()) and
|
||||
old_metadata['unit_sha256'] == sha(payload['unit'].encode()), 'old install metadata required')
|
||||
for path,name in ((helper,'helper'),(agent,'agent')): replace(path,backup/(name+'-next'))
|
||||
command = ['/usr/bin/python3','-I',str(provider)]
|
||||
config_bytes = run(command+['print-config']); config_sha = sha(config_bytes)
|
||||
require(config_sha == payload['reviewed_config_sha256'] and all(line in config_bytes for line in (
|
||||
b'DPkg::ConfigurePending "false";', b'DPkg::TriggersPending "false";', b'dpkg::selection::remove::approved "false";')), 'reviewed exact configure/no-pending settings')
|
||||
runtime = json.loads(run(command+['inspect-runtime'])); runtime_sha = sha(canonical(runtime))
|
||||
old_config['apt_provider'].update(runtime_sha256=runtime_sha,apt_config_sha256=config_sha)
|
||||
create(backup/'config-next',json.dumps(old_config).encode()); replace(config,backup/'config-next')
|
||||
pin = json.loads(run([str(helper),'--inspect-apt-profile'],env=dict(os.environ,RMM_HELPER_CONFIG=str(config))))
|
||||
require(pin['identity'] == json.loads(general_bytes)['identity'], 'same helper identity required')
|
||||
create(backup/'proof-next',json.dumps(pin).encode(),0o640,984); replace(proof_file,backup/'proof-next')
|
||||
general = json.loads(run([str(helper),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
|
||||
require(general == json.loads(general_bytes) and Path('/etc/rmm-remote/helper-general.json').read_bytes() == general_config, 'General helper policy unchanged')
|
||||
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'identity/journal/Root ledger bytes or inode changed')
|
||||
def preserved_original_base():
|
||||
current = manifest(base)
|
||||
require(all(current.get(name) == value for name,value in preserved_base.items() if name not in ('install-metadata.json','native-readiness.json')), 'original APT state/archive/hook/probe changed')
|
||||
for name,value in witnesses.items():
|
||||
p=Path(name);s=p.stat();require((s.st_dev,s.st_ino,sha(p.read_bytes())) == value, 'original package/control file bytes or inode changed')
|
||||
return current
|
||||
preserved_original_base(); bound_partial(guard())
|
||||
report.update(stage='installed_preserved',preserved_state=True,helper_sha256=pins['helper'],agent_sha256=pins['agent'],journal_page_size=4096,journal_max_page_count=65536,journal_capacity_bytes=268435456,
|
||||
provider_sha256=pins['provider'],runtime_sha256=runtime_sha,apt_config_sha256=config_sha,apt_pin=pin,general_pin=general)
|
||||
save()
|
||||
# No client can reach this endpoint; it exists only for the service's
|
||||
# ExecStartPre copied-database probe, then is stopped before restart.
|
||||
run(['systemd-analyze','verify','/etc/systemd/system/rmm-apt-helper.service'])
|
||||
replace(readiness_file)
|
||||
# Daemon startup itself takes executor.lock to open the existing DB.
|
||||
# Installation is complete; agent/General remain stopped during probe.
|
||||
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
|
||||
run(['systemctl','start','rmm-apt-helper.service'])
|
||||
for _ in range(3):
|
||||
status = run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','-p','SubState','-p','NRestarts']).decode()
|
||||
require('ActiveState=active\n' in status and 'SubState=running\n' in status and 'NRestarts=0\n' in status, 'stable native probe helper required')
|
||||
time.sleep(1)
|
||||
run(['systemctl','stop','rmm-apt-helper.service'])
|
||||
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
ready = json.loads(readiness_file.read_text())
|
||||
require(ready['runtime'] == runtime and ready['readiness_only'] is True and ready['native_acceptance'] is False and
|
||||
ready['host_status_sha256'] == sha(Path('/var/lib/dpkg/status').read_bytes()), 'exact new copied-database probe required')
|
||||
workspace = base/'probes'/str(uuid.UUID(ready['probe_id']))
|
||||
require((workspace/'dpkg/status').read_bytes() == Path('/var/lib/dpkg/status').read_bytes() and
|
||||
(workspace/'apt.conf').read_bytes().count(b'dpkg::selection::remove::approved "false";') == 1,
|
||||
'partial copied status and configure fix required')
|
||||
current = preserved_original_base()
|
||||
require(all(name in preserved_base or name.startswith('probes/'+ready['probe_id']+'/') or name == 'probes/'+ready['probe_id']
|
||||
for name in current), 'only generated readiness probe metadata may be added')
|
||||
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'journal/ledger bytes or inode changed during probe')
|
||||
bound_partial(guard())
|
||||
require(dropin.read_text() == reduced and Path('/etc/rmm-remote/general-helper-proof.json').read_bytes() == general_bytes, 'endpoint/General proof unchanged')
|
||||
report.update(stage='native_probe_preserved',readiness_only=True,native_acceptance=False,
|
||||
journals_preserved_before_restart=True,readiness_sha256=sha(readiness_file.read_bytes()),probe_id=ready['probe_id'])
|
||||
save()
|
||||
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
|
||||
run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
|
||||
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT endpoint must remain stopped')
|
||||
report.update(stage='qualified_runtime_started',completed=True);save()
|
||||
except Exception as error:
|
||||
report.update(error_class=type(error).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
|
||||
save()
|
||||
finally:
|
||||
if lock is not None: fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
|
||||
subprocess.run(['systemctl','stop','rmm-apt-helper.service'],capture_output=True,timeout=30)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--copy-only',action='store_true')
|
||||
mode.add_argument('--preflight-only',action='store_true')
|
||||
mode.add_argument('--install-reviewed-artifacts',action='store_true')
|
||||
parser.add_argument('--agent-sha256');parser.add_argument('--helper-sha256')
|
||||
args = parser.parse_args();os.umask(0o077)
|
||||
require(previous.previous.apt.prior.evidence('gate')['unresolved'] == 0, 'no unsettled central job')
|
||||
payload = previous.build_payload()
|
||||
require(len(payload['expected_grants']) == 10, 'exact completed Root/log history')
|
||||
if args.copy_only:
|
||||
report = cloud.bootstrap.rpc('linux-vm.conf',copy_reproducer,dict(id=str(uuid.uuid4()),agent_id=payload['agent_id'],expected_grants=payload['expected_grants']))
|
||||
cloud.contract.write_private(STATE/('journal-capacity-copy-'+Path(report['backup']).name.removeprefix('journal-capacity-copy-')+'.json'),report)
|
||||
print({k:report[k] for k in ('verified','old_error_code','new_insert_passed','original_unchanged','rows_and_schema_rolled_back','original_sql_mutations')})
|
||||
return
|
||||
require(not (STATE/'journal-capacity-runtime.json').exists(), 'existing capacity runtime retained; no automatic maintenance repeat')
|
||||
reviewed = {'agent':'2ce885cf5e6881e77383482b7d3a87a02c6847aa26b82e5fd70d2ebd8a45360d',
|
||||
'helper':'332eae004d9416234eabf7ed3d5c9ece5440d1e0c764ea63357961c16b673b6b'}
|
||||
for name,pin in reviewed.items():
|
||||
require(getattr(args,name+'_sha256') == pin and re.fullmatch('[0-9a-f]{64}',pin), 'independently reviewed capacity artifact pin required')
|
||||
raw = (STATE/(name+'-journal-capacity')).read_bytes()
|
||||
require(hashlib.sha256(raw).hexdigest() == pin, 'qualified capacity artifact drift')
|
||||
payload[name] = base64.b64encode(raw).decode();payload[name+'_sha256'] = pin
|
||||
payload.pop('provider')
|
||||
previous_runtime = cloud.contract.protected_json(STATE/'configure-runtime.json')
|
||||
require(previous_runtime['completed'] and previous_runtime['helper_sha256'] == previous.PINS['helper']
|
||||
and previous_runtime['agent_sha256'] == previous.PINS['agent'] and previous_runtime['provider_sha256'] == previous.PINS['provider'], 'exact independently qualified previous runtime')
|
||||
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
||||
require(installed['binary_sha256'] == previous.PINS['agent'], 'old permanent agent record required')
|
||||
policy = cloud.bootstrap.rpc('server.conf',previous.closed_policy,{})
|
||||
old = cloud.bootstrap.rpc('linux-vm.conf',previous.installed_metadata,previous.PINS)
|
||||
payload.update(old_files=old['files'],preflight_only=args.preflight_only)
|
||||
report = cloud.bootstrap.rpc('linux-vm.conf',install_capacity,payload)
|
||||
if args.preflight_only:
|
||||
require(report.get('preflight_only') and report.get('maintenance_mutations') is False, 'read-only maintenance preflight required')
|
||||
cloud.contract.write_private(STATE/('journal-capacity-preflight-'+payload['id']+'.json'),dict(report,closed_policy=policy))
|
||||
print(report);return
|
||||
cloud.contract.write_private(STATE/('journal-capacity-install-'+payload['id']+'.json'),dict(report,closed_policy=policy))
|
||||
require(report['completed'] and report['preserved_state'] and report['journals_preserved_before_restart'] and report['provider_sha256'] == previous.PINS['provider'], 'preserving bounded-capacity installation incomplete; retain report and do not repeat')
|
||||
previous.previous.record_agent_upgrade(report,installed)
|
||||
cloud.contract.write_private(STATE/'journal-capacity-runtime.json',report)
|
||||
print({k:report[k] for k in ('completed','stage','preserved_state','journals_preserved_before_restart','readiness_only','apt_endpoint_enabled')})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
print('CLOUD_JOURNAL_CAPACITY_INCOMPLETE:'+type(error).__name__)
|
||||
if str(error).startswith('remote bootstrap incomplete: '):print(str(error))
|
||||
raise SystemExit(2) from None
|
||||
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cloud-only active-agent offline/start-window/revocation acceptance.
|
||||
|
||||
Requires explicit execution. Plans start pending with zero executable artifacts;
|
||||
approval and plan revocation use actual dev/test WebUI controls. A narrowly
|
||||
scoped systemd BPF network fault affects only the existing Cloud agent unit.
|
||||
SSH is used solely for that transport bootstrap and retained-state evidence.
|
||||
No enrollment, package action, journal reset, approval API or automatic retry.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import cloud_mvp_acceptance as cloud
|
||||
|
||||
CASES = ('offline_wait', 'expired_offline', 'revoked_before_start')
|
||||
SERVER_IP = '88.99.80.202'
|
||||
STATE = cloud.ROOT/'devstate/cloud-offline'
|
||||
require = cloud.contract.require
|
||||
|
||||
|
||||
def counter_script(identifier):
|
||||
identifier = str(uuid.UUID(identifier))
|
||||
return ('set -eu\numask 077\nd=/var/lib/rmm-agent-remote/offline-'+identifier+'\n'
|
||||
'test ! -e "$d"\nmkdir "$d"\n'
|
||||
'printf \'%s\\n\' \'1\' > "$d/count"\n'
|
||||
'test "$(cat "$d/count")" = 1\n'
|
||||
'printf \'%s\\n\' \'RMM_OFFLINE_COUNT=1\'\n')
|
||||
|
||||
|
||||
def server_evidence(payload):
|
||||
"""Fixed SELECTs only, never creates/revokes/approves an operation."""
|
||||
root = Path('/opt/rust-discovery')
|
||||
tenant = str(uuid.UUID(payload['tenant_id'])); agent = str(uuid.UUID(payload['agent_id']))
|
||||
job = str(uuid.UUID(payload['job_id'])); operation = str(uuid.UUID(payload['operation_id']))
|
||||
plan = str(uuid.UUID(payload['plan_id']))
|
||||
generation = int(payload['previous_generation']); require(generation > 0, 'previous fenced channel')
|
||||
import datetime as dt
|
||||
after = dt.datetime.fromisoformat(payload['after'].replace('Z','+00:00')).isoformat() if payload.get('after') else '1970-01-01T00:00:00+00:00'
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111', 'fixed development tenant')
|
||||
scope = "tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND job_id='"+job+"'"
|
||||
query = ("SELECT json_build_object('observed_at',now(),'state',(SELECT state FROM jobs WHERE "+scope+"),"
|
||||
"'start_reserved',COALESCE((SELECT start_reserved FROM jobs WHERE "+scope+"),false),"
|
||||
"'grants',(SELECT count(*) FROM execution_permits WHERE tenant_id='"+tenant+"' AND job_id='"+job+"'),"
|
||||
"'approval_state',(SELECT approval_state FROM approval_requests WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"'),"
|
||||
"'revoked_authorizations',(SELECT count(*) FROM authorizations WHERE tenant_id='"+tenant+"' AND operation_id='"+operation+"' AND revoked_at IS NOT NULL),"
|
||||
"'starts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data)),'[]'::json) FROM audit_events "
|
||||
"WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"'),"
|
||||
"'fresh_signed_sync_before_start',EXISTS(SELECT 1 FROM sync_checkpoints c WHERE c.tenant_id='"+tenant+"' AND c.agent_id='"+agent+"' "
|
||||
"AND c.connection_generation>"+str(generation)+" AND (convert_from(c.payload,'UTF8')::jsonb->>'ready')::boolean "
|
||||
"AND c.created_at>='"+after+"'::timestamptz AND c.created_at<=(SELECT min(created_at) FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='start_reserved' AND data->>'job_id'='"+job+"')),"
|
||||
"'revocation',(SELECT json_build_object('principal',principal,'origin',data->>'approval_origin',"
|
||||
"'simulation',data->'testsimulation','interactive_session',data->>'session_hash' IS NOT NULL,"
|
||||
"'plan_id',data->'decision'->>'plan_id','operation_id',data->'decision'->>'operation_id') "
|
||||
"FROM audit_events WHERE tenant_id='"+tenant+"' AND event_type='plan_revoked' AND data->'decision'->>'plan_id'='"+plan+"' ORDER BY created_at DESC LIMIT 1));")
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=20).stdout)
|
||||
|
||||
|
||||
def vm_network(payload):
|
||||
"""One operational transport fault, plus reads of existing durable evidence."""
|
||||
import datetime as dt
|
||||
import errno
|
||||
import fcntl
|
||||
import sqlite3
|
||||
import stat
|
||||
identifier = str(uuid.UUID(payload['id']))
|
||||
phase = payload['phase']; require(phase in ('preflight','window'), 'fixed network phase')
|
||||
action = payload['action']; require(action in ('state','block','restore'), 'fixed network operation')
|
||||
state = Path('/var/lib/rmm-agent-remote'); helper = Path('/var/lib/rmm-helper')
|
||||
unit = 'rmm-agent-remote.service'; account = pwd.getpwnam('rmm-agent')
|
||||
require(os.geteuid() == 0 and account.pw_uid == 993 and account.pw_gid == 993, 'fixed unprivileged Cloud agent')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'fixed Cloud identity')
|
||||
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['binary_sha256'], 'reviewed current agent binary')
|
||||
def properties():
|
||||
raw = subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','MainPID','-p','ControlGroup','-p','IPAddressAllow'],capture_output=True,check=True,timeout=10).stdout.decode()
|
||||
return dict(line.split('=',1) for line in raw.splitlines())
|
||||
def snapshot():
|
||||
result = dict(agent_service=properties())
|
||||
with sqlite3.connect((state/'journal.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'native journal integrity')
|
||||
result['sequence'] = db.execute('SELECT seq FROM counter WHERE id=1').fetchone()[0]
|
||||
result['agent_fault'] = db.execute('SELECT fault FROM reconcile_meta WHERE id=1').fetchone()[0]
|
||||
if payload.get('job_id'):
|
||||
job = str(uuid.UUID(payload['job_id'])); attempt = str(uuid.UUID(payload['attempt_id']))
|
||||
rows = db.execute('SELECT state,grant_id FROM job_attempts WHERE job=? AND attempt=?',(job,attempt)).fetchall()
|
||||
result['attempts'] = [dict(state=r[0],grant_id=r[1]) for r in rows]
|
||||
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE NOT(job=? AND attempt=?) AND state NOT IN ('succeeded','failed','cancelled','expired')",(job,attempt)).fetchone()[0]
|
||||
else:
|
||||
result['other_unresolved'] = db.execute("SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchone()[0]
|
||||
with sqlite3.connect((state/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
result['consumed'] = db.execute('SELECT grant_id FROM consumed WHERE job=? AND attempt=?',(payload.get('job_id',''),payload.get('attempt_id',''))).fetchall()
|
||||
with sqlite3.connect((helper/'consumption.sqlite').as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok', 'helper ledger integrity')
|
||||
result['helper_fault'] = db.execute('SELECT fault FROM helper_sync_meta WHERE id=1').fetchone()[0]
|
||||
rows = db.execute('SELECT c.grant_id,c.agent,c.generation,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c LEFT JOIN execution_evidence e USING(grant_id) ORDER BY c.grant_id').fetchall()
|
||||
serial = [[dict(blob_sha256=hashlib.sha256(v).hexdigest(),length=len(v)) if isinstance(v,bytes) else v for v in row] for row in rows]
|
||||
result['helper_ledger_sha256'] = hashlib.sha256(json.dumps(serial,separators=(',',':')).encode()).hexdigest()
|
||||
result['helper_consumed'] = len(rows); result['helper_unresolved'] = sum(row[5] is None for row in rows)
|
||||
effect_directory = state/('offline-'+identifier)
|
||||
if effect_directory.exists():
|
||||
m = effect_directory.lstat();require(stat.S_ISDIR(m.st_mode) and m.st_uid == 993 and m.st_mode&0o777 == 0o700, 'fixed stored effect directory')
|
||||
else: require(not effect_directory.is_symlink(), 'stored effect directory symlink')
|
||||
marker = effect_directory/'count'
|
||||
if marker.exists():
|
||||
m = marker.lstat(); require(stat.S_ISREG(m.st_mode) and m.st_uid == account.pw_uid and m.st_mode&0o777 == 0o600 and m.st_size <= 16, 'fixed stored counter evidence')
|
||||
result['count'] = marker.read_text()
|
||||
else:
|
||||
require(not marker.is_symlink(), 'stored counter symlink'); result['count'] = None
|
||||
return result
|
||||
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
|
||||
require(not directory.is_symlink(), 'network dropin directory')
|
||||
directory.mkdir(mode=0o755,exist_ok=True)
|
||||
require(directory.stat().st_uid == 0 and directory.stat().st_mode&0o022 == 0, 'network dropin ownership')
|
||||
dropin = directory/'96-cloud-offline-acceptance.conf'
|
||||
expected = ('# Cloud MVP offline '+identifier+' '+phase+'\n[Service]\nIPAddressDeny=88.99.80.202/32\n').encode()
|
||||
require(not dropin.is_symlink(), 'network dropin symlink')
|
||||
timer = 'rmm-offline-rollback-'+identifier+'-'+phase
|
||||
before = snapshot()
|
||||
require(before['agent_fault'] is None and before['helper_fault'] is None and before['helper_unresolved'] == 0 and before['other_unresolved'] == 0, 'unrelated or unknown work remains fenced')
|
||||
if payload.get('helper_ledger_sha256'):
|
||||
require(before['helper_ledger_sha256'] == payload['helper_ledger_sha256'], 'existing Root ledger changed')
|
||||
if action == 'state': return before
|
||||
if action == 'restore':
|
||||
if dropin.exists():
|
||||
require(dropin.stat().st_uid == 0 and dropin.read_bytes() == expected, 'exact network fault ownership')
|
||||
dropin.unlink()
|
||||
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
|
||||
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
|
||||
subprocess.run(['systemctl','stop',timer+'.timer'],capture_output=True,timeout=10)
|
||||
return dict(restored=True,after=snapshot())
|
||||
require(not dropin.exists() and before['agent_service']['ActiveState'] == 'active' and not before['agent_service']['IPAddressAllow'], 'clean active unit and no allow override')
|
||||
require(before['count'] is None and not before['consumed'], 'no prior effect/consumption')
|
||||
if phase == 'window':
|
||||
start = dt.datetime.fromisoformat(payload['not_before'].replace('Z','+00:00'))
|
||||
require(dt.datetime.now(dt.timezone.utc) < start-dt.timedelta(seconds=60) and before['attempts'] == [dict(state='accepted',grant_id='')], 'fixed future accepted attempt with zero grant')
|
||||
else:
|
||||
require(not payload.get('job_id'), 'preflight has no job')
|
||||
# Arm bounded automatic transport rollback before creating the deny rule.
|
||||
# The timer can remove only these exact bytes and never touches job state.
|
||||
rollback_dir = Path('/var/lib/rmm-offline-bootstrap')
|
||||
require(not rollback_dir.is_symlink(), 'rollback directory symlink')
|
||||
rollback_dir.mkdir(mode=0o700,exist_ok=True)
|
||||
require(rollback_dir.stat().st_uid == 0 and rollback_dir.stat().st_mode&0o777 == 0o700, 'private bootstrap rollback directory')
|
||||
rollback = rollback_dir/(identifier+'-'+phase+'.py')
|
||||
code = ('import os,subprocess\nfrom pathlib import Path\np=Path('+repr(str(dropin))+')\n'
|
||||
'if p.exists() and not p.is_symlink() and p.stat().st_uid==0 and p.read_bytes()=='+repr(expected)+':\n'
|
||||
' p.unlink()\n subprocess.run(["systemctl","daemon-reload"],check=True)\n'
|
||||
' subprocess.run(["systemctl","restart","rmm-agent-remote.service"],check=True)\n')
|
||||
fd = os.open(rollback,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream: stream.write(code);stream.flush();os.fsync(stream.fileno())
|
||||
subprocess.run(['systemd-run','--quiet','--unit='+timer,'--on-active=420s','--property=Type=oneshot','/usr/bin/python3',str(rollback)],capture_output=True,check=True,timeout=10)
|
||||
# Quiesce this accepted/unstarted process only while applying the tested
|
||||
# policy. It must be running again BEFORE not_before; a stopped-agent
|
||||
# interval never counts toward the active offline acceptance window.
|
||||
subprocess.run(['systemctl','stop',unit],capture_output=True,check=True,timeout=20)
|
||||
stopped = snapshot()
|
||||
require(stopped['agent_service']['ActiveState'] == 'inactive' and not stopped['consumed'] and stopped['count'] is None, 'quiescent unstarted network setup')
|
||||
fd = os.open(dropin,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o644)
|
||||
with os.fdopen(fd,'wb') as stream: stream.write(expected);stream.flush();os.fsync(stream.fileno())
|
||||
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=15)
|
||||
subprocess.run(['systemctl','restart',unit],capture_output=True,check=True,timeout=20)
|
||||
after = snapshot(); require(after['agent_service']['ActiveState'] == 'active' and int(after['agent_service']['MainPID']) > 0 and after['count'] is None and not after['consumed'], 'active offline agent and zero effect')
|
||||
if phase == 'window': require(dt.datetime.now(dt.timezone.utc) < start, 'agent must be active offline before earliest start')
|
||||
group = after['agent_service']['ControlGroup']
|
||||
require((group == '/system.slice/rmm-agent-remote.service' or group.startswith('/system.slice/rmm-agent-remote.service/')) and '..' not in group, 'actual agent cgroup')
|
||||
cgroup = Path('/sys/fs/cgroup'+group)/'cgroup.procs'
|
||||
require(cgroup.is_file(), 'unified cgroup transport enforcement required')
|
||||
# This separate bootstrap child moves ONLY ITS OWN PID, drops privilege and
|
||||
# attempts TCP only. No agent credential, TLS override or appdata is used.
|
||||
probe = ('import os,socket,errno,json\nfrom pathlib import Path\n'
|
||||
'Path('+repr(str(cgroup))+').write_text(str(os.getpid()))\n'
|
||||
'os.setgroups([]);os.setgid(993);os.setuid(993)\n'
|
||||
'try:\n s=socket.create_connection(("88.99.80.202",443),2);s.close();result={"connected":True}\n'
|
||||
'except OSError as e: result={"connected":False,"errno":e.errno}\n'
|
||||
'print(json.dumps(result))\n')
|
||||
result = subprocess.run(['/usr/bin/python3','-I','-c',probe],capture_output=True,check=True,timeout=4)
|
||||
blocked = json.loads(result.stdout)
|
||||
require(blocked.get('connected') is False and blocked.get('errno') in (errno.EACCES,errno.EPERM), 'actual same-cgroup BPF denial must be EACCES/EPERM; timeout is not proof')
|
||||
after = snapshot()
|
||||
require(after['helper_ledger_sha256'] == before['helper_ledger_sha256'], 'Root ledger preserved during network bootstrap')
|
||||
return dict(blocked=True,probe=blocked,after=after,rollback_after_seconds=420,unit=unit,server_ip='88.99.80.202',port=443)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--execute-dev-test-ui',action='store_true',required=True)
|
||||
parser.add_argument('--case',choices=CASES,required=True)
|
||||
args = parser.parse_args(); os.umask(0o077)
|
||||
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(), 'notebook is development only')
|
||||
installed = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
||||
identity = installed['identity']; settings = dict(tenant_id=cloud.TENANT,**identity)
|
||||
backend = cloud.Backend(); identifier = str(uuid.uuid4())
|
||||
folder = STATE/identifier; folder.mkdir(mode=0o700,parents=True)
|
||||
checkpoint = dict(profile='dev/test',case=args.case,id=identifier,expected_principal='simulated-testprincipal-a',**settings,no_retry=True,stage='preflight')
|
||||
serial = 0
|
||||
def save():
|
||||
nonlocal serial
|
||||
cloud.contract.write_private(folder/('%03d-%s.json'%(serial,checkpoint['stage'])),checkpoint);serial += 1
|
||||
def gate(job_id=None):
|
||||
return cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],**({'job_id':job_id} if job_id else {})))
|
||||
def native(action,phase='window',**extra):
|
||||
p = dict(action=action,phase=phase,id=identifier,agent_id=identity['agent_id'],binary_sha256=installed['binary_sha256'],**extra)
|
||||
if 'job' in checkpoint and phase == 'window': p.update(job_id=checkpoint['job']['job_id'],attempt_id=checkpoint['job']['payload']['attempt_id'])
|
||||
return cloud.bootstrap.rpc('linux-vm.conf',vm_network,p)
|
||||
def healthy(previous_generation=None):
|
||||
end = time.monotonic()+100
|
||||
while True:
|
||||
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
if observed.get('sync_ready') and not observed.get('restore_blocked') and (previous_generation is None or observed['connection_generation'] > previous_generation): return observed
|
||||
require(time.monotonic() < end, 'ordinary signed reconnect did not become healthy');time.sleep(2)
|
||||
require(gate()['unresolved'] == 0, 'unresolved central work remains fenced')
|
||||
checkpoint['initial_channel'] = healthy(); initial = native('state','preflight')
|
||||
ledger_sha = initial['helper_ledger_sha256']; checkpoint['initial_native'] = initial;save()
|
||||
checkpoint['network_preflight'] = native('block','preflight',helper_ledger_sha256=ledger_sha);save()
|
||||
checkpoint['preflight_restored'] = native('restore','preflight',helper_ledger_sha256=ledger_sha);save()
|
||||
checkpoint['healthy_before_plan'] = healthy(checkpoint['initial_channel']['connection_generation'])
|
||||
start = cloud.contract.now()+dt.timedelta(seconds=150); end = start+dt.timedelta(seconds=130)
|
||||
request = cloud.contract.case_input('hello',settings,start=start)
|
||||
request['parameters']['variants'][0]['script_utf8'] = counter_script(identifier)
|
||||
request['parameters']['expected_stdout'] = 'RMM_OFFLINE_COUNT=1\n'
|
||||
request['schedule'] = cloud.contract.schedule(start,end)
|
||||
request['reason'] = 'Explicit dev/test active Cloud agent '+args.case+'; one fixed attempt, no implicit retry'
|
||||
plan = backend.call('operations.plan',dict(tenant_id=cloud.TENANT,input=request)); require(len(plan['jobs']) == 1 and hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'one immutable fixed job')
|
||||
job = plan['jobs'][0]; pending = backend.call('operations.submit',dict(tenant_id=cloud.TENANT,plan_id=plan['plan_id'],idempotency_key='offline-'+identifier))
|
||||
checkpoint.update(plan=plan,job=job,request=request,operation_id=pending['operation_id'],stage='pending_approval',preapproval=gate(job['job_id']))
|
||||
require(pending['approval_state'] == 'pending_approval' and pending['job_count'] == 0 and pending['execution_state'] is None and all(checkpoint['preapproval'][k] == 0 for k in ('jobs','outbox','permits')), 'pending zero execution artifacts');save()
|
||||
def browser(phase):
|
||||
cloud.prepare_browser(); path = folder/('ui-'+phase+'.json'); cloud.contract.write_private(path,dict(checkpoint,phase=phase))
|
||||
result = subprocess.run(['node',str(cloud.ROOT/'deploy/browser/cloud_offline_acceptance.cjs'),str(path)],capture_output=True,timeout=90)
|
||||
require(result.returncode == 0, 'actual UI incomplete; preserve checkpoint and never repeat approval')
|
||||
browser('approve'); checkpoint['stage'] = 'approved';save()
|
||||
deadline = time.monotonic()+45
|
||||
while True:
|
||||
server = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id'])); retained = native('state',helper_ledger_sha256=ledger_sha)
|
||||
require(gate(job['job_id'])['permits'] == 0 and not retained['consumed'] and retained['count'] is None, 'future attempt already has a grant/effect')
|
||||
if server['state'] == 'accepted' and retained['attempts'] == [dict(state='accepted',grant_id='')]: break
|
||||
require(time.monotonic() < deadline, 'future offer was not durably accepted');time.sleep(2)
|
||||
checkpoint.update(accepted=server,native_accepted=retained,stage='accepted_unstarted');save()
|
||||
checkpoint['network_fault'] = native('block',not_before=job['payload']['not_before'],helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'active_offline';save()
|
||||
channel = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
resume = cloud.contract.timestamp(job['payload']['start_before'] if args.case == 'expired_offline' else job['payload']['not_before'])+dt.timedelta(seconds=10)
|
||||
if args.case == 'revoked_before_start': browser('revoke');checkpoint['ui_revoked'] = True;save()
|
||||
sequence = checkpoint['network_fault']['after']['sequence']; windows = []
|
||||
while cloud.contract.now() < resume:
|
||||
retained = native('state',helper_ledger_sha256=ledger_sha); counts = gate(job['job_id'])
|
||||
observed = cloud.bootstrap.rpc('server.conf',cloud.bootstrap.server,dict(action='observe',agent_id=identity['agent_id']))
|
||||
require(retained['agent_service']['ActiveState'] == 'active' and retained['agent_service']['MainPID'] == checkpoint['network_fault']['after']['agent_service']['MainPID']
|
||||
and not retained['consumed'] and retained['count'] is None and counts['permits'] == 0 and observed['last_contact_at'] == channel['last_contact_at'], 'active offline window had a grant/effect/contact or agent exited')
|
||||
windows.append(dict(observed_at=cloud.contract.now().isoformat(),sequence=retained['sequence'],grants=counts['permits'],count=retained['count'],active=True))
|
||||
checkpoint['offline_window'] = windows;save();time.sleep(5)
|
||||
require(windows and windows[-1]['sequence'] > sequence, 'running offline agent did not persist reconnect attempts')
|
||||
checkpoint['server_offline_tail'] = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation']))
|
||||
require(checkpoint['server_offline_tail']['grants'] == 0 and not checkpoint['server_offline_tail']['start_reserved'], 'no start immediately before network restoration')
|
||||
checkpoint['network_restored'] = native('restore',helper_ledger_sha256=ledger_sha);checkpoint['stage'] = 'network_restored';save()
|
||||
checkpoint['healthy_reconnect'] = healthy(channel['connection_generation'])
|
||||
deadline = time.monotonic()+100
|
||||
while True:
|
||||
result = backend.call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
||||
if result['state'] in cloud.contract.TERMINAL or result['state'] in ('unknown','awaiting_reboot'): break
|
||||
require(time.monotonic() < deadline, 'terminal native result missing; no retry');time.sleep(2)
|
||||
expected = {'offline_wait':'succeeded','expired_offline':'expired','revoked_before_start':'cancelled'}[args.case]
|
||||
require(result['state'] == expected, 'actual terminal state differs from bounded case')
|
||||
native_final = native('state',helper_ledger_sha256=ledger_sha)
|
||||
wanted = 1 if args.case == 'offline_wait' else 0
|
||||
final_gate = gate(job['job_id']); require([final_gate[k] for k in ('jobs','outbox','permits')] == [1,1,wanted] and final_gate['unresolved'] == 0, 'exact job/outbox/start-grant counts')
|
||||
require(len(native_final['consumed']) == wanted and native_final['count'] == ('1\n' if wanted else None), 'exact native consumption and stored effect counter')
|
||||
if wanted: require(result['result']['exit_code'] == 0 and bytes(result['result']['stdout']) == b'RMM_OFFLINE_COUNT=1\n' and result['result']['evidence']['postconditions_passed'], 'native one-effect postcondition')
|
||||
audit = cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='audit',tenant_id=cloud.TENANT,agent_id=identity['agent_id'],operation_id=pending['operation_id']))
|
||||
require(audit == dict(origin='dev_test_browser_simulation',simulation=True,principal=checkpoint['expected_principal'],plan_hash=plan['plan_hash']), 'exact interactive approval audit')
|
||||
details = cloud.bootstrap.rpc('server.conf',server_evidence,dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'],job_id=job['job_id'],operation_id=pending['operation_id'],plan_id=plan['plan_id'],previous_generation=channel['connection_generation'],after=checkpoint['server_offline_tail']['observed_at']))
|
||||
require(details['grants'] == wanted and len(details['starts']) == wanted and details['start_reserved'] == bool(wanted), 'retained start reservation evidence')
|
||||
if wanted: require(details['fresh_signed_sync_before_start'], 'fresh healthy signed reconnect must precede the one start reservation')
|
||||
if args.case == 'revoked_before_start':
|
||||
require(details['approval_state'] == 'revoked' and details['revoked_authorizations'] == 1 and details['revocation'] == dict(principal=checkpoint['expected_principal'],origin='dev_test_browser_simulation',simulation=True,interactive_session=True,plan_id=plan['plan_id'],operation_id=pending['operation_id']), 'actual interactive plan revocation audit')
|
||||
checkpoint.update(result=result,native_final=native_final,postapproval=final_gate,audit=audit,server_evidence=details,stage='native_dev_test_accepted',verified=True);save()
|
||||
print('PASS Cloud '+args.case+': active offline BPF verified, WebUI gate, exact grants/effect='+str(wanted))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try: main()
|
||||
except Exception as error:
|
||||
print('CLOUD_OFFLINE_INCOMPLETE:'+ (str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
@@ -3,6 +3,27 @@
|
||||
|
||||
def service_processes(payload):
|
||||
result={}
|
||||
# Same read-only kernel clock-readiness query required by the runtime;
|
||||
# modes stays zero. No time correction or trust override exists here.
|
||||
import ctypes
|
||||
class TimexPrefix(ctypes.Structure):
|
||||
_fields_=[('modes',ctypes.c_uint),('offset',ctypes.c_long),('freq',ctypes.c_long),
|
||||
('maxerror',ctypes.c_long),('esterror',ctypes.c_long),('status',ctypes.c_int),('remaining',ctypes.c_byte*256)]
|
||||
require(ctypes.sizeof(ctypes.c_long)==8 and TimexPrefix.maxerror.offset==24 and TimexPrefix.status.offset==40,'reviewed Linux x64 timex layout')
|
||||
clock=TimexPrefix();libc=ctypes.CDLL(None,use_errno=True);query=libc.adjtimex
|
||||
query.argtypes=[ctypes.POINTER(TimexPrefix)];query.restype=ctypes.c_int
|
||||
status=query(ctypes.byref(clock))
|
||||
result['kernel_clock']=dict(status=status,unsynchronized=bool(clock.status&0x40),maxerror_us=clock.maxerror,
|
||||
trusted=status>=0 and status!=5 and not clock.status&0x40 and 0<=clock.maxerror<=5_000_000,modes=clock.modes)
|
||||
storage=os.statvfs('/var/lib/rmm-helper')
|
||||
result['bootstrap_storage']=dict(available_bytes=storage.f_bavail*storage.f_frsize,available_inodes=storage.f_favail,
|
||||
helper_database_bytes=Path('/var/lib/rmm-helper/consumption.sqlite').stat().st_size,
|
||||
agent_database_bytes=Path('/var/lib/rmm-agent-remote/journal.sqlite').stat().st_size)
|
||||
lock=Path('/var/lib/rmm-helper/executor.lock').stat()
|
||||
result['executor_lock_pids']=[]
|
||||
for line in Path('/proc/locks').read_text().splitlines():
|
||||
parts=line.split()
|
||||
if len(parts)>5 and parts[5].split(':')[-1]==str(lock.st_ino):result['executor_lock_pids'].append(parts[4])
|
||||
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
|
||||
values=subprocess.run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts'],
|
||||
capture_output=True,check=True,timeout=10).stdout.decode()
|
||||
@@ -23,3 +44,22 @@ def service_processes(payload):
|
||||
if children.exists():pending.extend(int(s) for s in children.read_text().split())
|
||||
result[name]=dict(service=fields,processes=processes)
|
||||
return result
|
||||
|
||||
|
||||
def stored_bootstrap_error_codes(payload):
|
||||
"""Only allowlisted stored daemon errors during this runtime deployment."""
|
||||
known=('CLOCK_UNTRUSTED','HELPER_VALIDATION_OR_IO_FAILURE','HELPER_PIN_PROFILE_MISMATCH',
|
||||
'SYNC_OR_TRANSPORT_DEPENDENCY_UNAVAILABLE','HELPER_CHECKPOINTS_DID_NOT_CONVERGE',
|
||||
'HELPER_MANUAL_RECOVERY_REQUIRED','LOCAL_CHECKPOINT_ROLLBACK','SYNC_MANUAL_RECOVERY_REQUIRED')
|
||||
result={}
|
||||
for unit in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
|
||||
process=subprocess.run(['/usr/bin/journalctl','--no-pager','--quiet','--output=json','--unit',unit,
|
||||
'--since','2026-11-13 17:34:00 UTC','--lines','100'],capture_output=True,check=True,timeout=10)
|
||||
require(len(process.stdout)<=256*1024,'bounded deployment error metadata')
|
||||
counts={k:0 for k in known};times=[]
|
||||
for line in process.stdout.splitlines():
|
||||
row=json.loads(line);message=row.get('MESSAGE','')
|
||||
for code in known:
|
||||
if code in message:counts[code]+=1;times.append(row.get('__REALTIME_TIMESTAMP'))
|
||||
result[unit]=dict(counts={k:v for k,v in counts.items() if v},first=min(times) if times else None,last=max(times) if times else None)
|
||||
return result
|
||||
|
||||
@@ -95,6 +95,9 @@ def deploy(payload):
|
||||
require(built['exit_code'] == 0, 'build must pass')
|
||||
require(all(hashlib.sha256((root/name).read_bytes()).hexdigest() == sha for name,sha in record['files'].items()), 'source drift')
|
||||
require(not (backup/'deploy-result.json').exists(), 'deployment already recorded')
|
||||
if payload.get('previous_ui_index_sha256'):
|
||||
require(hashlib.sha256((root/'web/dist/index.html').read_bytes()).hexdigest()==payload['previous_ui_index_sha256'],
|
||||
'independently known previous UI index required')
|
||||
if payload.get('closed_policy_sha256'):
|
||||
import datetime as dt
|
||||
policy = (root/'devstate/root-policy/policy.json').read_bytes()
|
||||
@@ -142,6 +145,7 @@ def deploy(payload):
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('stage','status','deploy')); parser.add_argument('--id')
|
||||
parser.add_argument('--preserve-known-unknown', action='store_true', help='preserve the exact fenced APT attempt while upgrading its control path')
|
||||
parser.add_argument('--expected-ui-index-sha256',help='CAS the independently known previous UI build before publication')
|
||||
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if args.action == 'stage':
|
||||
names = ['Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile','deploy/linux/rmm-apt-helper.service',
|
||||
@@ -161,6 +165,9 @@ def main():
|
||||
ident = str(uuid.UUID(args.id)); cloud.contract.protected_json(STATE/(ident+'.json'))
|
||||
before = None; payload = dict(id=ident)
|
||||
if args.action == 'deploy':
|
||||
if args.expected_ui_index_sha256:
|
||||
require(len(args.expected_ui_index_sha256)==64 and all(c in '0123456789abcdef' for c in args.expected_ui_index_sha256),'fixed previous UI hash')
|
||||
payload['previous_ui_index_sha256']=args.expected_ui_index_sha256
|
||||
target = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
||||
gate = cloud.bootstrap.rpc('server.conf', cloud.server, dict(kind='gate', tenant_id=cloud.TENANT, agent_id=target['agent_id']))
|
||||
if args.preserve_known_unknown:
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Passive Cloud auto-WSS→HTTPS fallback with two actual mTLS WSS rejections.
|
||||
|
||||
No RMM job, plan, approval or new identity. Only explicit, bounded dev/test
|
||||
transport fault configuration and preserving agent/service reconnects exist.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import cloud_mvp_acceptance as cloud
|
||||
|
||||
require = cloud.contract.require
|
||||
STATE = cloud.ROOT/'devstate/cloud-transport'
|
||||
|
||||
|
||||
def auto_mode(payload):
|
||||
import sqlite3
|
||||
state = Path('/var/lib/rmm-agent-remote')
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'],'fixed permanent identity')
|
||||
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest() == payload['agent_sha256'],'known qualified agent')
|
||||
for path,query in ((state/'journal.sqlite',"SELECT count(*) FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')"),
|
||||
(Path('/var/lib/rmm-helper/consumption.sqlite'),'SELECT count(*) FROM consumed WHERE result IS NULL')):
|
||||
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and db.execute(query).fetchone()[0] == 0,'all native effects must be settled')
|
||||
unit = Path('/etc/systemd/system/rmm-agent-remote.service')
|
||||
require(unit.read_text().splitlines().count('Environment=RMM_TRANSPORT=auto') == 1,'known automatic base service')
|
||||
require('RMM_TRANSPORT=' not in Path('/etc/rmm-remote/agent.env').read_text(),'no environment-file transport override')
|
||||
directory = Path('/etc/systemd/system/rmm-agent-remote.service.d')
|
||||
selected = directory/'95-cloud-acceptance-transport.conf'
|
||||
for path in directory.glob('*.conf'):
|
||||
require(not path.is_symlink(),'transport drop-in symlink')
|
||||
require(path == selected or 'RMM_TRANSPORT=' not in path.read_text(),'unknown transport override')
|
||||
changed = selected.exists()
|
||||
if changed:
|
||||
require(selected.stat().st_uid == 0 and selected.read_text() in
|
||||
('[Service]\nEnvironment=RMM_TRANSPORT=wss\n','[Service]\nEnvironment=RMM_TRANSPORT=https\n'),'known existing test override')
|
||||
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-auto-transport-'+str(uuid.UUID(payload['id'])))
|
||||
backup.mkdir(mode=0o700)
|
||||
os.rename(selected,backup/'previous-transport.conf')
|
||||
subprocess.run(['systemctl','daemon-reload'],capture_output=True,check=True,timeout=10)
|
||||
subprocess.run(['systemctl','restart','rmm-agent-remote.service'],capture_output=True,check=True,timeout=20)
|
||||
return dict(mode='auto',preserved_identity=True,removed_test_override=changed)
|
||||
|
||||
|
||||
def configure_server(payload):
|
||||
root = Path('/opt/rust-discovery');compose = root/'deploy/server.compose.yaml'
|
||||
directory = root/'devstate/root-policy';path = directory/'transport-fault.json'
|
||||
require(not compose.is_symlink() and hashlib.sha256(compose.read_bytes()).hexdigest() == payload['compose_sha256'],'independently known compose required')
|
||||
require(not directory.is_symlink() and directory.is_dir() and directory.stat().st_uid == 10001
|
||||
and directory.stat().st_mode & 0o777 == 0o700,'existing private read-only mount')
|
||||
require(not path.is_symlink(),'transport config symlink')
|
||||
old = path.read_bytes() if path.exists() else None
|
||||
actual_sha = hashlib.sha256(old).hexdigest() if old is not None else None
|
||||
expected = payload['expected_config_sha256']
|
||||
# Cleanup may accept either of two PREVIOUSLY known hashes when an SSH
|
||||
# response was lost; it never adopts an arbitrary hash read from the host.
|
||||
require(actual_sha in expected if isinstance(expected,list) else actual_sha == expected,'known prior transport config')
|
||||
tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']))
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111','fixed development tenant')
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(compose)]
|
||||
query = "SELECT count(*) FROM jobs WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"' AND state NOT IN ('succeeded','failed','cancelled','expired');"
|
||||
require(subprocess.run(cmd+['exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm'],
|
||||
input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip() == b'0','no open Cloud jobs')
|
||||
enabled = payload['enabled'];require(type(enabled) is bool,'explicit opt-in')
|
||||
backup = root/'devstate'/('transport-fault-'+str(uuid.UUID(payload['id'])))
|
||||
backup.mkdir(mode=0o700)
|
||||
def create(target,raw,uid=0):
|
||||
fd = os.open(target,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'wb') as stream:
|
||||
stream.write(raw);stream.flush();os.fsync(stream.fileno());os.fchown(stream.fileno(),uid,uid)
|
||||
if old is not None:create(backup/'transport-fault-before.json',old)
|
||||
if enabled:
|
||||
value = payload['config'];require(value['tenant_id'] == tenant and value['agent_id'] == agent,'exact scope')
|
||||
raw = json.dumps(value,separators=(',',':')).encode()
|
||||
create(directory/'transport-fault.next',raw,10001);os.replace(directory/'transport-fault.next',path)
|
||||
else:raw = old
|
||||
overlay = backup/'transport.compose.yaml'
|
||||
# Literal overlay; no existing .env secret is read, returned or replaced.
|
||||
body = 'services:\n'
|
||||
for role in ('api','gateway'):
|
||||
body += ' '+role+':\n environment:\n RMM_DEV_TRANSPORT_FAULTS: "'+('true' if enabled else 'false')+'"\n'
|
||||
if enabled:body += ' RMM_DEV_TRANSPORT_FAULT_FILE: /root-policy/transport-fault.json\n'
|
||||
create(overlay,body.encode())
|
||||
cmd += ['-f',str(overlay)]
|
||||
subprocess.run(cmd+['config','-q'],capture_output=True,check=True,timeout=15)
|
||||
completed = subprocess.run(cmd+['up','-d','--no-deps','api','gateway'],capture_output=True,timeout=25).returncode == 0
|
||||
return dict(deployed=completed,enabled=enabled,backup=str(backup),config_sha256=hashlib.sha256(raw).hexdigest() if raw is not None else None)
|
||||
|
||||
|
||||
def server_evidence(payload):
|
||||
root = Path('/opt/rust-discovery');tenant = str(uuid.UUID(payload['tenant_id']));agent = str(uuid.UUID(payload['agent_id']));fault = str(uuid.UUID(payload['fault_id']))
|
||||
require(tenant == '11111111-1111-4111-8111-111111111111','fixed test tenant')
|
||||
query = ("SELECT json_build_object('receipts',(SELECT COALESCE(json_agg(json_build_object('created_at',created_at,'data',data) ORDER BY created_at),'[]'::json) "
|
||||
"FROM audit_events WHERE tenant_id='"+tenant+"' AND principal='dev_transport_fault_adapter' AND data->>'fault_id'='"+fault+"'),"
|
||||
"'device',(SELECT json_build_object('agent_id',agent_id,'asset_id',asset_id,'enrollment_generation',enrollment_generation,'assignment_version',assignment_version,"
|
||||
"'transport',transport,'connection_generation',connection_generation,'last_contact_at',last_contact_at,'last_authenticated_at',last_authenticated_at,"
|
||||
"'certificate_sha256',last_authenticated_certificate_sha256,'sync_ready',sync_ready,'restore_blocked',restore_blocked) "
|
||||
"FROM agent_installations WHERE tenant_id='"+tenant+"' AND agent_id='"+agent+"'));")
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
return json.loads(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout)
|
||||
|
||||
|
||||
def runtime_flags(payload):
|
||||
root = Path('/opt/rust-discovery')
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')]
|
||||
result = {}
|
||||
for role in ('api','gateway'):
|
||||
ident = subprocess.run(cmd+['ps','-q',role],capture_output=True,check=True,timeout=10).stdout.decode().strip()
|
||||
require(ident and '\n' not in ident,'one running role')
|
||||
values = json.loads(subprocess.run(['docker','inspect','--format','{{json .Config.Env}}',ident],capture_output=True,check=True,timeout=10).stdout)
|
||||
keys = ('RMM_PROFILE','RMM_DEV_AUTH','RMM_DEV_TRANSPORT_FAULTS','RMM_DEV_TRANSPORT_FAULT_FILE')
|
||||
result[role] = {v.split('=',1)[0]:v.split('=',1)[1] for v in values if v.split('=',1)[0] in keys}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--execute-dev-test-transport',action='store_true',required=True)
|
||||
parser.add_argument('--expected-agent-sha256',required=True)
|
||||
parser.add_argument('--expected-compose-sha256',required=True)
|
||||
parser.add_argument('--expected-config-sha256')
|
||||
args = parser.parse_args();os.umask(0o077)
|
||||
for value in (args.expected_agent_sha256,args.expected_compose_sha256):require(re.fullmatch('[0-9a-f]{64}',value) is not None,'independently qualified pin')
|
||||
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
|
||||
identity = cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
||||
scope = dict(tenant_id=cloud.TENANT,agent_id=identity['agent_id'])
|
||||
def gate():return cloud.bootstrap.rpc('server.conf',cloud.server,dict(scope,kind='gate'))
|
||||
before = gate();require(before['unresolved'] == 0,'earlier effects must be closed')
|
||||
ident = str(uuid.uuid4());folder = STATE/ident;folder.mkdir(mode=0o700,parents=True)
|
||||
evidence_scope = dict(scope,fault_id=ident)
|
||||
auto = cloud.bootstrap.rpc('linux-vm.conf',auto_mode,dict(id=ident,agent_id=identity['agent_id'],agent_sha256=args.expected_agent_sha256))
|
||||
checkpoint = dict(profile='dev/test',testsimulation=True,case='actual_auto_fallback',id=ident,identity=identity,preapproval_counts=before,auto_mode=auto,no_jobs_created=True)
|
||||
path = folder/'checkpoint.json'
|
||||
def save():path.write_text(json.dumps(checkpoint,indent=2));path.chmod(0o600)
|
||||
def wait_transport(transport,after=None):
|
||||
end=time.monotonic()+150
|
||||
while True:
|
||||
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);device=observed['device']
|
||||
if (device['transport'] == transport and device['sync_ready'] and not device['restore_blocked'] and device['last_contact_at']
|
||||
and (after is None or cloud.contract.timestamp(device['last_contact_at']) > after)):return observed
|
||||
require(time.monotonic() < end,'fresh actual transport heartbeat and signed Sync missing');time.sleep(2)
|
||||
save();checkpoint['initial_wss']=wait_transport('wss');save()
|
||||
now=cloud.contract.now();config=dict(schema_version=1,fault_id=ident,**scope,issued_at=now.isoformat(),expires_at=(now+dt.timedelta(minutes=10)).isoformat())
|
||||
raw=json.dumps(config,separators=(',',':')).encode();checkpoint['config']=config;save()
|
||||
enabled=None;complete=False;attempted=False
|
||||
try:
|
||||
attempted=True
|
||||
enabled=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=ident,enabled=True,**scope,compose_sha256=args.expected_compose_sha256,
|
||||
expected_config_sha256=args.expected_config_sha256,config=config))
|
||||
checkpoint['enabled']=enabled;save();require(enabled['deployed'],'transport opt-in incomplete')
|
||||
end=time.monotonic()+150
|
||||
while True:
|
||||
observed=cloud.bootstrap.rpc('server.conf',server_evidence,evidence_scope);receipts=observed['receipts'];device=observed['device']
|
||||
if (len(receipts) == 2 and device['transport'] == 'https' and device['sync_ready'] and not device['restore_blocked']
|
||||
and device['last_contact_at'] and cloud.contract.timestamp(device['last_contact_at']) > cloud.contract.timestamp(receipts[1]['created_at'])):break
|
||||
require(time.monotonic() < end,'two authenticated WSS503 and actual HTTPS heartbeat missing');time.sleep(2)
|
||||
require({r['data']['slot'] for r in receipts} == {1,2} and len({r['data']['handshake_id'] for r in receipts}) == 2
|
||||
and all(r['data']['status'] == 503 and r['data']['agent_id'] == identity['agent_id'] and r['data']['tenant_id'] == cloud.TENANT
|
||||
and r['data']['config_sha256'] == hashlib.sha256(raw).hexdigest() and r['data']['certificate_sha256'] == device['certificate_sha256']
|
||||
and r['data']['testsimulation'] is True for r in receipts),'actual exact peer fault receipts')
|
||||
require(all(device[k] == identity[k] for k in ('agent_id','asset_id','enrollment_generation','assignment_version')),'permanent identity unchanged')
|
||||
checkpoint['https']=observed;save();complete=True
|
||||
finally:
|
||||
if attempted:
|
||||
closed=cloud.bootstrap.rpc('server.conf',configure_server,dict(id=str(uuid.uuid4()),enabled=False,**scope,compose_sha256=args.expected_compose_sha256,
|
||||
expected_config_sha256=[hashlib.sha256(raw).hexdigest(),args.expected_config_sha256]))
|
||||
checkpoint['disabled']=closed;save();require(closed['deployed'],'transport fault cleanup incomplete')
|
||||
flags=cloud.bootstrap.rpc('server.conf',runtime_flags,{})
|
||||
require(len(flags) == 2 and all(v.get('RMM_DEV_TRANSPORT_FAULTS') in (None,'false') and 'RMM_DEV_TRANSPORT_FAULT_FILE' not in v for v in flags.values()),'both real roles disabled')
|
||||
checkpoint['disabled_runtime']=flags;save()
|
||||
checkpoint['restored_wss']=wait_transport('wss',cloud.contract.now());save()
|
||||
require(complete,'fallback not proven')
|
||||
after=gate();require(after == before,'passive fallback must leave job/outbox/grant counts unchanged')
|
||||
checkpoint.update(post_counts=after,verified=True,stage='native_dev_test_accepted');save()
|
||||
print('PASS passive Cloud auto WSS→HTTPS: two authenticated503, fresh HTTPS heartbeat+Sync, normal WSS restored, identity and job counts preserved')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
print('CLOUD_TRANSPORT_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exact dev/test root script payload. Never installed as a daemon/provider."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
assert os.geteuid() == 0 and len(sys.argv) == 3
|
||||
operation, identifier = sys.argv[1:]
|
||||
assert operation in ('effect', 'read') and str(uuid.UUID(identifier)) == identifier
|
||||
directory = Path('/var/lib/rmm-helper')/('crash-'+identifier)
|
||||
for parent in directory.parents:
|
||||
metadata = parent.lstat()
|
||||
assert stat.S_ISDIR(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_mode & 0o022 == 0
|
||||
os.umask(0o077)
|
||||
if operation == 'effect' and not directory.exists(): directory.mkdir(mode=0o700)
|
||||
metadata = directory.lstat()
|
||||
assert stat.S_ISDIR(metadata.st_mode) and metadata.st_uid == 0 and stat.S_IMODE(metadata.st_mode) == 0o700
|
||||
counter = directory/'count'
|
||||
def read_counter():
|
||||
fd = os.open(counter, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
with os.fdopen(fd, 'r') as stream:
|
||||
metadata = os.fstat(stream.fileno())
|
||||
assert stat.S_ISREG(metadata.st_mode) and metadata.st_uid == 0 and metadata.st_nlink == 1
|
||||
assert stat.S_IMODE(metadata.st_mode) == 0o600 and metadata.st_size <= 16
|
||||
value = stream.read()
|
||||
assert value.endswith('\n') and value[:-1].isdigit()
|
||||
return int(value)
|
||||
if operation == 'effect':
|
||||
lock = directory/'lock';lock.mkdir(mode=0o700)
|
||||
try:
|
||||
value = read_counter() if counter.exists() else 0
|
||||
assert 0 <= value < 100
|
||||
fd = os.open(directory/'count.next',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream:
|
||||
stream.write(str(value+1)+'\n');stream.flush();os.fsync(stream.fileno())
|
||||
os.replace(directory/'count.next',counter)
|
||||
fd = os.open(directory,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW)
|
||||
try: os.fsync(fd)
|
||||
finally: os.close(fd)
|
||||
finally: lock.rmdir()
|
||||
# Independent helper remains alive after only the agent process dies.
|
||||
time.sleep(90)
|
||||
assert read_counter() == 1
|
||||
print('RMM_CRASH_COUNT=1')
|
||||
Reference in New Issue
Block a user