fix: reconcile configured APT effect without retroactive acceptance and retain host reboot evidence
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -73,6 +73,7 @@ function checkAction(p, expected) {
|
||||
await page.getByLabel('Begründung', { exact: true }).fill(expected.request.reason);
|
||||
await page.getByRole('button', { name: 'Plan erstellen · noch keine Ausführung' }).click();
|
||||
await detail.locator('pre').waitFor(); const plan = JSON.parse(await detail.locator('pre').innerText());
|
||||
assert(JSON.parse(Buffer.from(plan.plan_payload_bytes)).reason === expected.request.reason);
|
||||
assert(plan.jobs.length === 1); checkAction(plan.jobs[0].payload, expected);
|
||||
assert(crypto.createHash('sha256').update(Buffer.from(plan.plan_payload_bytes)).digest('hex') === plan.plan_hash);
|
||||
await detail.getByRole('button', { name: 'Antrag absenden · pending_approval' }).click();
|
||||
@@ -88,6 +89,7 @@ function checkAction(p, expected) {
|
||||
assert(pending.approval_state === 'pending_approval' && pending.job_count === 0 && pending.execution_state === null);
|
||||
await op.getByRole('button', { name: 'Unveränderlichen Plan zur Prüfung öffnen' }).click();
|
||||
const plan = JSON.parse(await detail.locator('pre').innerText());
|
||||
assert(JSON.parse(Buffer.from(plan.plan_payload_bytes)).reason === expected.request.reason);
|
||||
// Submission changes request metadata, never the fixed plan bytes/jobs.
|
||||
// Require the new pending state explicitly and compare EVERY other field.
|
||||
const { approval_state: currentState, ...currentPlan } = plan;
|
||||
|
||||
@@ -20,6 +20,13 @@ CASES=('apt_configure_recovery','apt_native_update','ensure_package','apply_upda
|
||||
|
||||
|
||||
def runtime_session():
|
||||
configured=BASE/'configured-reconcile-runtime.json'
|
||||
if configured.exists():
|
||||
report=cloud.contract.protected_json(configured)
|
||||
require(report['completed'] and report['preserved_state'] and report['readiness_only'] and
|
||||
not report['native_acceptance'] and report['unknown_grant']=='33775478-15b0-46ec-b059-44f500dbb0b6',
|
||||
'qualified preserving configured-reconciliation runtime')
|
||||
return report,BASE/'configured-reconcile-runtime'
|
||||
latest=BASE/'journal-capacity-runtime.json'
|
||||
if latest.exists():
|
||||
report=cloud.contract.protected_json(latest)
|
||||
@@ -76,7 +83,15 @@ def enable():
|
||||
apt_config_sha256=ready['apt_config_sha256'],provider_sha256=ready['provider_sha256'],
|
||||
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
|
||||
policy=dict(schema_version=1,profile='development',expires_at=(m.now()+dt.timedelta(hours=1)).isoformat(),scopes=[scope])
|
||||
old=m.protected_json(BASE/'recovery-policy-closed.json' if session!=BASE else cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
||||
if session==BASE/'configured-reconcile-runtime':
|
||||
old=m.protected_json(BASE/'capacity-recovery/recovery-policy-closed-unknown.json')
|
||||
prior_checkpoint=m.protected_json(BASE/'capacity-recovery/cases/apt_configure_recovery/checkpoint.json')
|
||||
closed=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=prior_checkpoint['plan']['jobs'][0]['job_id']))
|
||||
require(closed['state']=='failed' and closed['result']['error_code']=='APT_RECOVERY_CONFIGURED_ACCEPTANCE_REBOOT_FLAG_VERIFIED' and
|
||||
closed['result']['grant_id']=='33775478-15b0-46ec-b059-44f500dbb0b6',
|
||||
'ordinary signed Sync must first close the exact historical configured effect as failed')
|
||||
else:
|
||||
old=m.protected_json(BASE/'recovery-policy-closed.json' if session!=BASE else cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
||||
record=cloud.bootstrap.rpc('server.conf',prior.replace_policy,dict(id=str(uuid.uuid4()),previous_sha256=old['policy_sha256'],policy=policy))
|
||||
m.write_private(session/'recovery-policy.json',dict(record,policy=policy))
|
||||
print('Fixed APT runtime endpoint and expiring limits enabled; no job or approval created')
|
||||
@@ -97,6 +112,8 @@ def accept(case,resume_unsubmitted=False):
|
||||
m=cloud.contract;target=prior.prior.identity()
|
||||
require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
ready,session=runtime_session();cases=session/'cases';cases.mkdir(mode=0o700,exist_ok=True)
|
||||
require(session!=BASE/'configured-reconcile-runtime' or case!='apt_configure_recovery',
|
||||
'configured historical1.0 is never executed again; next action is separate native1.1 upgrade')
|
||||
if session!=BASE and case=='apt_configure_recovery':
|
||||
original=m.protected_json(STATE/case/'checkpoint-capacity-terminal.json')
|
||||
old_job=original['plan']['jobs'][0]
|
||||
@@ -123,8 +140,10 @@ def accept(case,resume_unsubmitted=False):
|
||||
break
|
||||
except m.GuardError:
|
||||
require(time.monotonic()<deadline,'fresh stored exact planning unavailable; no job submitted');time.sleep(3)
|
||||
request=recovery_request(device,target) if case=='apt_configure_recovery' else None
|
||||
if request is not None and session!=BASE:
|
||||
request=recovery_request(device,target) if case=='apt_configure_recovery' else m.case_input(case,target,planning=data)
|
||||
if data['host_reboot_required']:
|
||||
request['reason']+='; Host reboot already required; flag unchanged; no reboot in this job'
|
||||
if request is not None and session==BASE/'capacity-recovery':
|
||||
request['reason']+='; prior fixed plan expired with zero grants at the old journal capacity; preserve it and authorize this fresh plan separately'
|
||||
prior.accept(case,request=request,folder_root=cases)
|
||||
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Preserved witnesses of the one already approved configure-only attempt."""
|
||||
|
||||
|
||||
def maintenance_status(payload):
|
||||
root=Path('/var/lib/rmm-bootstrap-history')
|
||||
reports=[]
|
||||
for folder in sorted(root.glob('cloud-configured-reconcile-*')):
|
||||
require(not folder.is_symlink() and folder.stat().st_uid==0,'known private maintenance directory')
|
||||
path=folder/'report.json'
|
||||
if path.exists():
|
||||
require(not path.is_symlink() and path.stat().st_size<=1024*1024,'bounded maintenance report')
|
||||
reports.append(dict(path=str(path),report=json.loads(path.read_text())))
|
||||
require(len(reports)<=3,'bounded configured maintenance reports')
|
||||
states={}
|
||||
for unit in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
|
||||
raw=subprocess.run(['systemctl','show',unit,'-p','ActiveState','-p','SubState','-p','MainPID'],capture_output=True,check=True,timeout=10).stdout.decode()
|
||||
states[unit]=dict(line.split('=',1) for line in raw.splitlines())
|
||||
hashes={name:hashlib.sha256(Path(path).read_bytes()).hexdigest() for name,path in (
|
||||
('agent','/usr/local/libexec/rmm-agent-remote'),('helper','/usr/local/libexec/rmm-privileged-helper'),
|
||||
('provider','/usr/lib/rmm-apt-transaction/apt_transaction.py'))}
|
||||
return dict(reports=reports,services=states,installed_sha256=hashes)
|
||||
|
||||
|
||||
def stored_configured(payload):
|
||||
import stat
|
||||
base=Path('/var/lib/rmm-apt-transaction');active=base/'active'
|
||||
context=json.loads((active/'context.json').read_text())
|
||||
require(all(context[k]==str(uuid.UUID(payload[k])) for k in ('job_id','attempt_id','grant_id')),'same already approved configure attempt')
|
||||
require(context['grant_id']=='33775478-15b0-46ec-b059-44f500dbb0b6','one preserved configure attempt')
|
||||
def witness(path):
|
||||
require(not path.is_symlink(),'witness symlink')
|
||||
if not path.exists():return dict(exists=False)
|
||||
s=path.stat();require(stat.S_ISREG(s.st_mode) and s.st_uid==0 and s.st_size<=1024*1024,'bounded original root file')
|
||||
return dict(exists=True,sha256=hashlib.sha256(path.read_bytes()).hexdigest(),size=s.st_size,
|
||||
inode=s.st_ino,mtime_ns=s.st_mtime_ns,ctime_ns=s.st_ctime_ns)
|
||||
names=('context.json','signed-action.json','expectation.json','baseline.json','partial-proof.json',
|
||||
'hook-armed.json','sandbox-result.json','native-prechecks.json','sandbox-ready.json','launch.json')
|
||||
files={name:witness(active/name) for name in names}
|
||||
original='31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0'
|
||||
for name in ('recovery-attempt-'+original+'.json','recovery-'+original+'.json','pending-recovery-'+original+'.json',
|
||||
'bootstrap-attempt-rmm-apt-fixture-install-1.0.json','bootstrap-retry-rmm-apt-fixture-install-1.0.json'):
|
||||
files[name]=witness(base/name)
|
||||
return dict(files=files,marker_current=witness(Path('/run/reboot-required')),
|
||||
same_boot=Path('/proc/sys/kernel/random/boot_id').read_text().strip()==context['boot_id'],
|
||||
marker_observation_phase='after_original_effect_only',original_state_changed=False)
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read/audit ONLY the existing 337 configured attempt; never execute/unlock."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
import cloud_mvp_acceptance as cloud
|
||||
|
||||
GRANT='33775478-15b0-46ec-b059-44f500dbb0b6'
|
||||
PRIOR='31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0'
|
||||
PARTIAL='dc3bf3870ea77c12f9291bac712c53ae72194eaec980a8646155a7fb5c0dbe21'
|
||||
CODE='APT_RECOVERY_CONFIGURED_ACCEPTANCE_REBOOT_FLAG_VERIFIED'
|
||||
STATE=cloud.ROOT/'devstate/cloud-apt-recovery/capacity-recovery'
|
||||
|
||||
|
||||
def stored(payload):
|
||||
import sqlite3
|
||||
grant='33775478-15b0-46ec-b059-44f500dbb0b6'
|
||||
code='APT_RECOVERY_CONFIGURED_ACCEPTANCE_REBOOT_FLAG_VERIFIED'
|
||||
path=Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','root ledger integrity')
|
||||
row=db.execute('SELECT c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?',(grant,)).fetchone()
|
||||
require(row is not None,'one exact retained configured attempt')
|
||||
job,attempt,result,start,outcome=row
|
||||
require(job==payload['job_id'] and attempt==payload['attempt_id'],'original job/attempt')
|
||||
names=[r[1] for r in db.execute('PRAGMA table_info(apt_effect_reconciliations)')]
|
||||
proofrow=db.execute('SELECT * FROM apt_effect_reconciliations WHERE grant_id=?',(grant,)).fetchone() if names else None
|
||||
record=dict(zip(names,proofrow)) if proofrow else None
|
||||
original=json.loads(record['previous_outcome']) if record else None
|
||||
proof=json.loads(record['proof']) if record else None
|
||||
request=json.loads(start)['request']
|
||||
# Match the original Request struct codec, without reading any key.
|
||||
request_raw=json.dumps({k:request[k] for k in ('job_bytes','permit_bytes','signature','plan_hash','authorization_id')},separators=(',',':')).encode()
|
||||
binding=bool(record and hashlib.sha256(request_raw).hexdigest()==record['original_request_sha256']
|
||||
and hashlib.sha256(record['previous_outcome']).hexdigest()==record['previous_outcome_sha256']
|
||||
and hashlib.sha256(record['proof']).hexdigest()==record['proof_sha256']
|
||||
and original['state']=='unknown' and original['error_code']=='FIXTURE_UNEXPECTED_REBOOT_REQUIRED'
|
||||
and proof['grant_id']==grant and proof['job_id']==job and proof['attempt_id']==attempt
|
||||
and proof['job_payload_sha256']==hashlib.sha256(bytes(request['job_bytes'])).hexdigest()
|
||||
and proof['effect_complete'] is True and proof['postconditions_passed'] is False
|
||||
and proof['reboot_required'] is True and proof['host_reboot_required'] is True)
|
||||
classified=json.loads(outcome) if outcome else None;finished=json.loads(result) if result else None
|
||||
closed=bool(binding and finished==classified==json.loads(record['classified_outcome'])
|
||||
and finished['state']=='failed' and finished['error_code']==code
|
||||
and finished['evidence']['effect_complete'] is True and finished['evidence']['postconditions_passed'] is False
|
||||
and finished['reboot_required'] is True and finished['evidence']['apt']['host_reboot_required_before'] is None)
|
||||
partials=[dict(grant_id=g,proof_sha256=s) for g,s in db.execute('SELECT grant_id,proof_sha256 FROM apt_partial_reconciliations')]
|
||||
stats=dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0])
|
||||
result=dict(ledger=stats,classified=classified,original_unknown=original,proof=proof,proof_sha256=record['proof_sha256'] if record else None,
|
||||
immutable_binding_verified=binding,closed_by_normal_sync=closed,result_bytes_equal_outcome=result==outcome,partial_rows=partials,
|
||||
effect_reconciliation_rows=db.execute('SELECT count(*) FROM apt_effect_reconciliations').fetchone()[0] if names else 0)
|
||||
base=Path('/var/lib/rmm-apt-transaction')
|
||||
result['forbidden_acceptance_records']={name:(base/name).exists() for name in
|
||||
('acceptance-rmm-apt-fixture-install-1.0.json','pending-acceptance-rmm-apt-fixture-install-1.0.json',
|
||||
'recovery-31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0.json','pending-recovery-31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0.json')}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
os.umask(0o077)
|
||||
checkpoint=cloud.contract.protected_json(STATE/'cases/apt_configure_recovery/checkpoint.json')
|
||||
job=checkpoint['plan']['jobs'][0]
|
||||
report=cloud.bootstrap.rpc('linux-vm.conf',stored,dict(job_id=job['job_id'],attempt_id=job['payload']['attempt_id']))
|
||||
identity=cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
|
||||
counts=cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,agent_id=identity['agent_id']))
|
||||
result=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
|
||||
cloud.contract.write_private(STATE/('configured-closure-'+uuid.uuid4().hex+'.json'),dict(native=report,counts=counts,central=result))
|
||||
cloud.contract.require(report['immutable_binding_verified'] and report['closed_by_normal_sync'] and report['result_bytes_equal_outcome']
|
||||
and report['ledger']==dict(consumed=11,unresolved=0,faults=0) and report['partial_rows']==[dict(grant_id=PRIOR,proof_sha256=PARTIAL)]
|
||||
and report['effect_reconciliation_rows']==1 and not any(report['forbidden_acceptance_records'].values())
|
||||
and counts==dict(jobs=20,outbox=20,permits=18,unresolved=0) and result['state']=='failed' and result['result']['error_code']==CODE,
|
||||
'same-attempt classified failure and normal signed closure not yet proven')
|
||||
print('PASS exact337 failed acceptance/effect complete, truthful host rebootrequired, immutable oldunknown/partial proof, normal Sync closed Root11/0/fault0 central20/20/18/0; no success/retry')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
print('CONFIGURED_CLOSURE_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
@@ -0,0 +1,406 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Preserve UNKNOWN337 while installing its narrowly bound readonly verifier.
|
||||
|
||||
Only bootstrap runtime/probe maintenance; no package effect, new grant, retry,
|
||||
acceptance record, identity reset or direct consumed/result update. General and
|
||||
agent restart only after immutable state and copied-dpkg probe qualification.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import cloud_apt_recovery_upgrade as previous
|
||||
|
||||
cloud = previous.cloud
|
||||
require = cloud.require
|
||||
STATE = previous.STATE
|
||||
UNKNOWN = '33775478-15b0-46ec-b059-44f500dbb0b6'
|
||||
OLD = dict(helper='332eae004d9416234eabf7ed3d5c9ece5440d1e0c764ea63357961c16b673b6b',
|
||||
agent='2ce885cf5e6881e77383482b7d3a87a02c6847aa26b82e5fd70d2ebd8a45360d',
|
||||
provider='5ff260f75cb2959678f6be729d3a806a4ea29febd5ae4003317fed41afc25327')
|
||||
|
||||
|
||||
def upgrade_configured(payload):
|
||||
import ast
|
||||
import fcntl
|
||||
import sqlite3
|
||||
import time
|
||||
import traceback
|
||||
os.umask(0o077)
|
||||
base = Path('/var/lib/rmm-apt-transaction'); state = Path('/var/lib/rmm-agent-remote')
|
||||
ledger = Path('/var/lib/rmm-helper/consumption.sqlite')
|
||||
helper = Path('/usr/local/libexec/rmm-privileged-helper'); agent = Path('/usr/local/libexec/rmm-agent-remote')
|
||||
provider = Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')
|
||||
config = Path('/etc/rmm/apt-helper.json'); proof_file = Path('/etc/rmm-remote/apt-helper-proof.json')
|
||||
metadata = base/'install-metadata.json'; readiness_file = base/'native-readiness.json'
|
||||
dropin = Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
|
||||
reduced = ('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
|
||||
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
|
||||
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
|
||||
full = reduced.replace('Environment=RMM_LOG_UNITS_FILE=',
|
||||
'Environment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\n'
|
||||
'Environment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm-remote/apt-helper-proof.json\n'
|
||||
'Environment=RMM_LOG_UNITS_FILE=')
|
||||
def run(args, **kwargs):
|
||||
return subprocess.run(args, capture_output=True, check=True, timeout=90, **kwargs).stdout
|
||||
def sha(raw): return hashlib.sha256(raw).hexdigest()
|
||||
def canonical(value): return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode()
|
||||
def fsync_dir(path):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
||||
try: os.fsync(fd)
|
||||
finally: os.close(fd)
|
||||
def create(path, raw, mode=0o600, gid=0):
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(raw); stream.flush(); os.fchown(stream.fileno(), 0, gid)
|
||||
os.fchmod(stream.fileno(), mode); os.fsync(stream.fileno())
|
||||
fsync_dir(path.parent)
|
||||
def manifest(path):
|
||||
require(not path.is_symlink(), 'preserved directory symlink')
|
||||
if not path.exists(): return {'__absent__':True}
|
||||
require(path.is_dir(), 'preserved directory required')
|
||||
result = {}
|
||||
for p in [path, *sorted(path.rglob('*'))]:
|
||||
s = p.lstat(); require(not p.is_symlink() and (p.is_dir() or p.is_file()), 'preserved regular state required')
|
||||
result[str(p.relative_to(path))] = dict(dev=s.st_dev, inode=s.st_ino, mode=s.st_mode,
|
||||
uid=s.st_uid, gid=s.st_gid, sha256=sha(p.read_bytes()) if p.is_file() else None)
|
||||
return result
|
||||
def guard():
|
||||
require(len(payload['expected_grants']) == 11 and payload['unknown_grant'] ==
|
||||
'33775478-15b0-46ec-b059-44f500dbb0b6', 'exact reviewed eleven Root grants')
|
||||
with sqlite3.connect(ledger.as_uri()+'?mode=ro', uri=True) as db:
|
||||
db.row_factory = sqlite3.Row
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
|
||||
{r[0] for r in db.execute('SELECT grant_id FROM consumed')} == set(payload['expected_grants']) and
|
||||
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0] == 0,
|
||||
'exact preserved Root history without faults')
|
||||
rows = db.execute('SELECT c.grant_id,c.result,e.outcome,e.start FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
|
||||
require(len(rows) == 1 and rows[0]['grant_id'] == payload['unknown_grant'], 'only known configure unknown may remain')
|
||||
outcome = json.loads(rows[0]['outcome'])
|
||||
require(outcome == payload['unknown_outcome'] and outcome['state'] == 'unknown' and
|
||||
outcome['error_code'] == 'FIXTURE_UNEXPECTED_REBOOT_REQUIRED', 'original unknown bytes must retain their meaning')
|
||||
saved = json.loads(rows[0]['start']); request = saved['request']
|
||||
job = json.loads(bytes(request['job_bytes']))
|
||||
require(job['job_id'] == payload['context']['job_id'] and
|
||||
job['attempt_id'] == payload['context']['attempt_id'] and
|
||||
job['agent_id'] == payload['agent_id'], 'original saved request binding')
|
||||
unknown_witness = dict(outcome_sha256=sha(rows[0]['outcome']),start_sha256=sha(rows[0]['start']))
|
||||
rows = db.execute('SELECT * FROM apt_partial_reconciliations').fetchall()
|
||||
require(len(rows) == 1, 'one immutable old partial proof required')
|
||||
row = dict(rows[0]); expected = payload['proof_row']
|
||||
for key in ('grant_id','original_request_sha256','previous_outcome_sha256','proof_sha256','checked_at_ms'):
|
||||
require(row[key] == expected[key], 'old partial binding drift')
|
||||
for key in ('previous_outcome','proof','classified_outcome'):
|
||||
require(sha(row[key]) == expected[key]['sha256'] and len(row[key]) == expected[key]['size'], 'old partial bytes drift')
|
||||
require(row['grant_id'] == payload['partial_grant'] and sha(row['proof']) == payload['partial_proof_sha256'], 'original partial proof unchanged')
|
||||
old = db.execute('SELECT c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?',(payload['partial_grant'],)).fetchone()
|
||||
require(old['result'] is not None and json.loads(old['result']) == json.loads(old['outcome']) == json.loads(row['classified_outcome']), 'old partial stays signed-Sync closed')
|
||||
if db.execute("SELECT count(*) FROM sqlite_master WHERE type='table' AND name='apt_effect_reconciliations'").fetchone()[0]:
|
||||
require(db.execute('SELECT count(*) FROM apt_effect_reconciliations').fetchone()[0] == 0, 'configure classification must not already exist')
|
||||
journal = state/'journal.sqlite'; sidecars = [Path(str(journal)+s) for s in ('-wal','-shm')]
|
||||
existing_sidecars = {p for p in sidecars if p.exists()}
|
||||
owner = journal.stat()
|
||||
with sqlite3.connect(journal.as_uri()+'?mode=ro', uri=True) as db:
|
||||
require(db.execute('PRAGMA integrity_check').fetchone()[0] == 'ok' and
|
||||
db.execute('SELECT count(*) FROM reconcile_meta WHERE fault IS NOT NULL').fetchone()[0] == 0, 'agent journal integrity without fault')
|
||||
rows = db.execute("SELECT job,attempt,state,grant_id FROM job_attempts WHERE state NOT IN ('succeeded','failed','cancelled','expired')").fetchall()
|
||||
require(rows == [(payload['context']['job_id'],payload['context']['attempt_id'],'unknown',payload['unknown_grant'])], 'only bound known agent unknown may remain')
|
||||
# Read-only SQLite can create WAL sidecars. Correct only our newly
|
||||
# created fixed-name files, never existing state or recursive ownership.
|
||||
for p in sidecars:
|
||||
if p not in existing_sidecars and p.exists():
|
||||
require(not p.is_symlink() and p.stat().st_uid == 0, 'new fixed SQLite sidecar owner')
|
||||
os.chown(p,owner.st_uid,owner.st_gid)
|
||||
if p.exists(): require(not p.is_symlink() and p.stat().st_uid == owner.st_uid, 'agent SQLite sidecar ownership preserved')
|
||||
active = base/'active'
|
||||
for name,expected in payload['file_witnesses'].items():
|
||||
p = (base if name.startswith(('bootstrap-','recovery-','pending-recovery-')) else active)/name
|
||||
require(not p.is_symlink(), 'historical witness symlink')
|
||||
require(p.exists() == expected['exists'], 'historical file existence changed')
|
||||
if expected['exists']:
|
||||
s = p.stat()
|
||||
require(s.st_ino == expected['inode'] and s.st_size == expected['size']
|
||||
and s.st_mtime_ns == expected['mtime_ns'] and s.st_ctime_ns == expected['ctime_ns']
|
||||
and sha(p.read_bytes()) == expected['sha256'], 'historical exact witness bytes/inode changed')
|
||||
pending = Path('/run/reboot-required'); expected = payload['host_marker_witness']
|
||||
require(not pending.is_symlink() and pending.exists() == expected['exists'], 'host pending reboot marker changed')
|
||||
if expected['exists']:
|
||||
s = pending.stat()
|
||||
require(s.st_ino == expected['inode'] and s.st_size == expected['size']
|
||||
and s.st_mtime_ns == expected['mtime_ns'] and s.st_ctime_ns == expected['ctime_ns']
|
||||
and sha(pending.read_bytes()) == expected['sha256'], 'host pending reboot marker must remain unchanged')
|
||||
ctx = json.loads((active/'context.json').read_bytes())
|
||||
require(ctx == payload['context'] and ctx['grant_id'] == payload['unknown_grant'], 'original active configure context')
|
||||
require(Path('/proc/sys/kernel/random/boot_id').read_text().strip() == ctx['boot_id'], 'original configure boot must remain current')
|
||||
for name,key in (('signed-action.json','job_payload_sha256'),('expectation.json','transaction_sha256')):
|
||||
require(sha((active/name).read_bytes()) == ctx[key], 'original active signed bytes')
|
||||
for name,expected in payload['records'].items():
|
||||
require(json.loads((active/name).read_bytes()) == expected, 'original configure native record unchanged')
|
||||
require(payload['records']['sandbox-result.json'] == dict(transaction_sha256=ctx['transaction_sha256'],apt_exit_code=0,reboot_required=False), 'completed native sandbox APT0 required')
|
||||
action = json.loads((active/'signed-action.json').read_bytes())['action']['apt_manifest']
|
||||
tx = json.loads((active/'expectation.json').read_bytes())
|
||||
ref = action['native_recovery']
|
||||
require(action['schema_version'] == 4 and action['action'] == 'apt-native-acceptance' and
|
||||
tx['schema_version'] == 2 and tx['operation'] == 'configure_only' and
|
||||
tx['recipe_id'] == 'rmm-apt-fixture-install-1.0' and len(tx['packages']) == 1 and
|
||||
tx['packages'][0]['old_version'] == tx['packages'][0]['new_version'] == '1.0' and
|
||||
ref == dict(schema_version=1,previous_grant_id=payload['partial_grant'],previous_attempt_id=payload['partial_attempt'],partial_proof_sha256=payload['partial_proof_sha256']), 'exact historical schema4 configure-only reference')
|
||||
require(sha((active/'baseline.json').read_bytes()) == tx['baseline_sha256'] and
|
||||
sha((active/'partial-proof.json').read_bytes()) == payload['partial_proof_sha256'], 'original partial baseline and staged proof')
|
||||
marker = base/('recovery-attempt-'+payload['partial_grant']+'.json')
|
||||
require(json.loads(marker.read_bytes()) == dict(transaction_sha256=ctx['transaction_sha256'],job_payload_sha256=ctx['job_payload_sha256'],grant_id=ctx['grant_id'],attempt_id=ctx['attempt_id'],native_recovery=ref), 'original recovery once marker required')
|
||||
require(Path('/usr/share/rmm-mvp-test/version').read_bytes() == b'1.0\n', 'existing fixture payload remains1.0')
|
||||
fixture = []
|
||||
for paragraph in Path('/var/lib/dpkg/status').read_text().split('\n\n'):
|
||||
fields = dict(line.split(': ',1) for line in paragraph.splitlines() if ': ' in line and not line[0].isspace())
|
||||
if fields.get('Package') == 'rmm-mvp-test': fixture.append(fields)
|
||||
require(len(fixture) == 1 and all(fixture[0].get(k) == v for k,v in
|
||||
dict(Architecture='all',Version='1.0',Status='install ok installed').items()), 'existing installed1.0 status required for copied probe')
|
||||
require(not list(base.glob('acceptance-*.json')) and not list(base.glob('pending-acceptance-*.json')), 'no fabricated normal acceptance')
|
||||
require({p.name for p in base.glob('recovery-*.json')} == {marker.name} and
|
||||
not list(base.glob('pending-recovery-*.json')), 'only original recovery attempt, no recovery acceptance')
|
||||
return unknown_witness
|
||||
require(json.loads((state/'identity.json').read_text())['agent_id'] == payload['agent_id'], 'permanent Cloud identity required')
|
||||
require(not dropin.is_symlink() and dropin.read_text() in (full,reduced), 'only reviewed endpoint env allowed')
|
||||
old_endpoint = dropin.read_text() == full
|
||||
active_state = run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip()
|
||||
main_pid = int(run(['systemctl','show','rmm-apt-helper.service','-p','MainPID','--value']).strip())
|
||||
require((active_state == b'inactive' and main_pid == 0) or (active_state == b'active' and main_pid > 0), 'known APT helper state required')
|
||||
cgroup = Path('/sys/fs/cgroup/system.slice/rmm-apt-helper.service')
|
||||
pids = {int(pid) for p in cgroup.rglob('cgroup.procs') for pid in p.read_text().split()} if cgroup.exists() else set()
|
||||
require(pids == ({main_pid} if main_pid else set()), 'no native sandbox/package child may remain')
|
||||
for name, pin in payload['old_files'].items():
|
||||
p = Path(name); require(not p.is_symlink() and p.stat().st_uid == 0 and p.stat().st_mode & 0o022 == 0 and sha(p.read_bytes()) == pin, 'installed file drift')
|
||||
require(Path('/etc/systemd/system/rmm-apt-helper.service').read_text() == payload['unit'], 'exact reviewed APT service required')
|
||||
markers = {base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json':'ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053',
|
||||
base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json':'a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3'}
|
||||
require({p.name for p in base.glob('bootstrap-*.json')} == {p.name for p in markers} and
|
||||
all(sha(p.read_bytes()) == pin for p,pin in markers.items()), 'original/retry tombstones required')
|
||||
original_unknown = guard()
|
||||
raw = {name:base64.b64decode(payload[name], validate=True) for name in ('helper','agent','provider')}
|
||||
pins = {name:payload[name+'_sha256'] for name in ('helper','agent','provider')}
|
||||
require(all(sha(raw[name]) == pin == payload[name+'_sha256'] for name,pin in pins.items()), 'qualified exact recovery runtime required')
|
||||
for name in ('helper','agent'):
|
||||
require(raw[name][:6] == b'\x7fELF\x02\x01' and raw[name][18:20] == b'\x3e\x00', 'Linux x64 binary required')
|
||||
ast.parse(raw['provider'])
|
||||
if payload.get('preflight_only') is True:
|
||||
return dict(stage='preflight_validated', completed=False, preflight_only=True,
|
||||
package_effects=False, maintenance_mutations=False, expected_grants=len(payload['expected_grants']),
|
||||
partial_proof_sha256=payload['partial_proof_sha256'], apt_endpoint_enabled=old_endpoint,
|
||||
unknown_grant=payload['unknown_grant'],original_unknown=original_unknown)
|
||||
backup = Path('/var/lib/rmm-bootstrap-history')/('cloud-configured-reconcile-'+payload['id']); backup.mkdir(mode=0o700)
|
||||
report = dict(stage='validated', completed=False, backup=str(backup), package_effects=False,
|
||||
apt_endpoint_enabled=old_endpoint, partial_proof_sha256=payload['partial_proof_sha256'], backups={})
|
||||
def save():
|
||||
next_path = backup/'.report-next.json'; create(next_path,json.dumps(report,indent=2).encode())
|
||||
os.replace(next_path,backup/'report.json'); fsync_dir(backup)
|
||||
def replace(path, staged=None):
|
||||
target = backup/str(path).lstrip('/').replace('/','__')
|
||||
require(not path.is_symlink() and path.stat().st_dev == backup.stat().st_dev and not target.exists(), 'exclusive same-filesystem versioned replacement')
|
||||
os.rename(path,target); fsync_dir(path.parent); fsync_dir(backup)
|
||||
report['backups'][str(path)] = str(target); save()
|
||||
if staged is not None:
|
||||
os.rename(staged,path); fsync_dir(path.parent); fsync_dir(backup)
|
||||
lock = None
|
||||
try:
|
||||
save()
|
||||
for name in ('helper','agent','provider'): create(backup/(name+'-next'), raw[name], 0o755)
|
||||
for name in ('helper','agent'): run(['/lib64/ld-linux-x86-64.so.2','--list',str(backup/(name+'-next'))])
|
||||
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
|
||||
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
require(guard() == original_unknown, 'unknown outcome/start bytes unchanged')
|
||||
if old_endpoint:
|
||||
create(backup/'dropin-next',reduced.encode(),0o644); replace(dropin,backup/'dropin-next')
|
||||
run(['systemctl','daemon-reload'])
|
||||
report['apt_endpoint_enabled'] = False; save()
|
||||
preserved = {str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent)}
|
||||
preserved_base = manifest(base)
|
||||
witnesses = {}
|
||||
for p in (Path('/var/lib/dpkg/status'),Path('/usr/share/rmm-mvp-test/version'),dropin,
|
||||
Path('/etc/systemd/system/rmm-apt-helper.service'),Path('/usr/lib/rmm-apt-transaction/hook'),
|
||||
Path('/etc/rmm-remote/general-helper-proof.json'),Path('/etc/rmm-remote/helper-general.json')):
|
||||
s = p.stat(); witnesses[str(p)] = (s.st_dev,s.st_ino,sha(p.read_bytes()))
|
||||
general_bytes = Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
|
||||
general_config = Path('/etc/rmm-remote/helper-general.json').read_bytes()
|
||||
old_metadata = json.loads(metadata.read_text()); old_config = json.loads(config.read_text())
|
||||
require(old_metadata['schema_version'] == 2 and old_metadata['provider_sha256'] == sha(provider.read_bytes()) and
|
||||
old_metadata['hook_sha256'] == sha(Path('/usr/lib/rmm-apt-transaction/hook').read_bytes()) and
|
||||
old_metadata['unit_sha256'] == sha(payload['unit'].encode()), 'old install metadata required')
|
||||
for path,name in ((helper,'helper'),(agent,'agent'),(provider,'provider')): replace(path,backup/(name+'-next'))
|
||||
create(backup/'metadata-next',json.dumps(dict(old_metadata,provider_sha256=sha(raw['provider']))).encode()); replace(metadata,backup/'metadata-next')
|
||||
command = ['/usr/bin/python3','-I',str(provider)]
|
||||
config_bytes = run(command+['print-config']); config_sha = sha(config_bytes)
|
||||
require(config_sha == payload['reviewed_config_sha256'] and all(line in config_bytes for line in (
|
||||
b'DPkg::ConfigurePending "false";', b'DPkg::TriggersPending "false";', b'dpkg::selection::remove::approved "false";')), 'reviewed exact configure/no-pending settings')
|
||||
runtime = json.loads(run(command+['inspect-runtime'])); runtime_sha = sha(canonical(runtime))
|
||||
old_config['apt_provider'].update(runtime_sha256=runtime_sha,apt_config_sha256=config_sha)
|
||||
create(backup/'config-next',json.dumps(old_config).encode()); replace(config,backup/'config-next')
|
||||
pin = json.loads(run([str(helper),'--inspect-apt-profile'],env=dict(os.environ,RMM_HELPER_CONFIG=str(config))))
|
||||
require(pin['identity'] == json.loads(general_bytes)['identity'], 'same helper identity required')
|
||||
create(backup/'proof-next',json.dumps(pin).encode(),0o640,984); replace(proof_file,backup/'proof-next')
|
||||
general = json.loads(run([str(helper),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
|
||||
require(general == json.loads(general_bytes) and Path('/etc/rmm-remote/helper-general.json').read_bytes() == general_config, 'General helper policy unchanged')
|
||||
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'identity/journal/Root ledger bytes or inode changed')
|
||||
def preserved_original_base():
|
||||
current = manifest(base)
|
||||
require(all(current.get(name) == value for name,value in preserved_base.items() if name not in ('install-metadata.json','native-readiness.json')), 'original APT state/archive/hook/probe changed')
|
||||
for name,value in witnesses.items():
|
||||
p=Path(name);s=p.stat();require((s.st_dev,s.st_ino,sha(p.read_bytes())) == value, 'original package/control file bytes or inode changed')
|
||||
return current
|
||||
preserved_original_base(); require(guard() == original_unknown, 'unknown outcome/start bytes unchanged')
|
||||
report.update(stage='installed_preserved',preserved_state=True,helper_sha256=pins['helper'],agent_sha256=pins['agent'],
|
||||
unknown_grant=payload['unknown_grant'],original_unknown=original_unknown,provider_sha256=pins['provider'],runtime_sha256=runtime_sha,apt_config_sha256=config_sha,apt_pin=pin,general_pin=general)
|
||||
save()
|
||||
# No client can reach this endpoint; it exists only for the service's
|
||||
# ExecStartPre copied-database probe, then is stopped before restart.
|
||||
run(['systemd-analyze','verify','/etc/systemd/system/rmm-apt-helper.service'])
|
||||
replace(readiness_file)
|
||||
# Daemon startup itself takes executor.lock to open the existing DB.
|
||||
# Installation is complete; agent/General remain stopped during probe.
|
||||
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
|
||||
run(['systemctl','start','rmm-apt-helper.service'])
|
||||
for _ in range(3):
|
||||
status = run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','-p','SubState','-p','NRestarts']).decode()
|
||||
require('ActiveState=active\n' in status and 'SubState=running\n' in status and 'NRestarts=0\n' in status, 'stable native probe helper required')
|
||||
time.sleep(1)
|
||||
run(['systemctl','stop','rmm-apt-helper.service'])
|
||||
lock = os.open(ledger.parent/'executor.lock', os.O_RDWR | os.O_NOFOLLOW)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
ready = json.loads(readiness_file.read_text())
|
||||
require(ready['runtime'] == runtime and ready['readiness_only'] is True and ready['native_acceptance'] is False and
|
||||
ready['host_status_sha256'] == sha(Path('/var/lib/dpkg/status').read_bytes()), 'exact new copied-database probe required')
|
||||
workspace = base/'probes'/str(uuid.UUID(ready['probe_id']))
|
||||
require((workspace/'dpkg/status').read_bytes() == Path('/var/lib/dpkg/status').read_bytes() and
|
||||
(workspace/'apt.conf').read_bytes().count(b'dpkg::selection::remove::approved "false";') == 1,
|
||||
'installed copied status and configure fix required')
|
||||
current = preserved_original_base()
|
||||
require(all(name in preserved_base or name.startswith('probes/'+ready['probe_id']+'/') or name == 'probes/'+ready['probe_id']
|
||||
for name in current), 'only generated readiness probe metadata may be added')
|
||||
for name,values in preserved.items(): require(manifest(Path(name)) == values, 'journal/ledger bytes or inode changed during probe')
|
||||
require(guard() == original_unknown, 'unknown outcome/start bytes unchanged')
|
||||
require(dropin.read_text() == reduced and Path('/etc/rmm-remote/general-helper-proof.json').read_bytes() == general_bytes, 'endpoint/General proof unchanged')
|
||||
report.update(stage='native_probe_preserved',readiness_only=True,native_acceptance=False,
|
||||
journals_preserved_before_restart=True,readiness_sha256=sha(readiness_file.read_bytes()),probe_id=ready['probe_id'])
|
||||
save()
|
||||
fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock);lock=None
|
||||
run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
|
||||
require(run(['systemctl','show','rmm-apt-helper.service','-p','ActiveState','--value']).strip() == b'inactive', 'APT endpoint must remain stopped')
|
||||
report.update(stage='qualified_runtime_started',completed=True);save()
|
||||
except Exception as error:
|
||||
report.update(error_class=type(error).__name__,frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])
|
||||
save()
|
||||
finally:
|
||||
if lock is not None: fcntl.flock(lock,fcntl.LOCK_UN);os.close(lock)
|
||||
subprocess.run(['systemctl','stop','rmm-apt-helper.service'],capture_output=True,timeout=30)
|
||||
return report
|
||||
|
||||
|
||||
def build_payload(args):
|
||||
m = cloud.contract
|
||||
installed = m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
|
||||
old = m.protected_json(STATE/'journal-capacity-runtime.json')
|
||||
require(old['completed'] and all(old[n+'_sha256'] == pin for n,pin in OLD.items())
|
||||
and installed['binary_sha256'] == OLD['agent'], 'known qualified capacity runtime required')
|
||||
stored = m.protected_json(STATE/'configure-attempt-stored.json')
|
||||
witnesses = m.protected_json(STATE/'configure-attempt-witnesses.json')
|
||||
require(witnesses['same_boot'] is True and witnesses['original_state_changed'] is False,
|
||||
'unchanged same-boot historical witnesses required')
|
||||
context = stored['context']; outcome = stored['ledger']['outcome']
|
||||
require(context['grant_id'] == UNKNOWN and stored['ledger']['consumed'] == 11
|
||||
and stored['ledger']['unresolved'] == 1 and stored['ledger']['result'] is None
|
||||
and outcome['state'] == 'unknown' and outcome['error_code'] == 'FIXTURE_UNEXPECTED_REBOOT_REQUIRED',
|
||||
'exact original persisted configure unknown')
|
||||
checkpoint = m.protected_json(STATE/'capacity-recovery/cases/apt_configure_recovery/checkpoint.json')
|
||||
job = checkpoint['plan']['jobs'][0]
|
||||
require(checkpoint['result']['state'] == 'unknown' and checkpoint['result']['result']['grant_id'] == UNKNOWN
|
||||
and job['job_id'] == context['job_id']
|
||||
and job['attempt_id'] == context['attempt_id'], 'bound actual UI configure attempt')
|
||||
readonly = m.protected_json(STATE/'readonly-result-first.json')
|
||||
require(readonly['unresolved'] == readonly['faults'] == 0 and len(readonly['proof']) == 1
|
||||
and readonly['proof'][0]['grant_id'] == previous.PARTIAL_GRANT
|
||||
and readonly['proof'][0]['proof_sha256'] == previous.PARTIAL_PROOF, 'original closed partial proof')
|
||||
folders = [cloud.ROOT/'devstate/cloud-root/cases'/n for n in ('service_setup','service_start','service_stop','apt_native_install','query_logs')]
|
||||
folders += [previous.previous.apt.prior.STATE/n for n in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
|
||||
grants = [str(uuid.UUID(m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'])) for p in folders]
|
||||
grants += [previous.PARTIAL_GRANT, UNKNOWN]
|
||||
require(len(grants) == len(set(grants)) == 11, 'exact original ten grants plus configure337')
|
||||
raw = {}; pins = {}; qualifications = {}
|
||||
for name in ('helper','agent','provider'):
|
||||
pin = getattr(args,name+'_sha256')
|
||||
require(isinstance(pin,str) and re.fullmatch('[0-9a-f]{64}',pin) and pin != OLD[name], 'independently supplied new qualified artifact pin')
|
||||
path = STATE/(name+'-configured-reconcile'+('.py' if name == 'provider' else ''))
|
||||
raw[name] = path.read_bytes(); pins[name] = pin
|
||||
require(hashlib.sha256(raw[name]).hexdigest() == pin, 'qualified artifact bytes drift')
|
||||
if name != 'provider':
|
||||
qualification = m.protected_json(path.with_name(path.name+'.json'))
|
||||
require(qualification['binary_sha256'] == pin and qualification['release_build'] is True
|
||||
and qualification['architecture'] == 'x86_64-unknown-linux-gnu'
|
||||
and qualification['provider_sha256'] == args.provider_sha256
|
||||
and qualification['source_files'], 'bounded release qualification required')
|
||||
for filename,sha in qualification['source_files'].items():
|
||||
source = (cloud.ROOT/filename).resolve()
|
||||
require(source.is_relative_to(cloud.ROOT.resolve()) and
|
||||
hashlib.sha256(source.read_bytes()).hexdigest() == sha, 'qualified source snapshot drift')
|
||||
qualifications[name] = qualification
|
||||
require(raw['provider'] == (cloud.ROOT/'tools/apt-transaction/apt_transaction.py').read_bytes(), 'qualified provider/source bytes mismatch')
|
||||
spec = importlib.util.spec_from_file_location('reviewed_configured_reconcile_provider',cloud.ROOT/'tools/apt-transaction/apt_transaction.py')
|
||||
module = importlib.util.module_from_spec(spec); sys.modules[spec.name] = module; spec.loader.exec_module(module)
|
||||
return dict(id=str(uuid.uuid4()),agent_id=installed['identity']['agent_id'],expected_grants=grants,
|
||||
unknown_grant=UNKNOWN,unknown_outcome=outcome,context=context,records=stored['records'],
|
||||
file_witnesses=witnesses['files'],host_marker_witness=witnesses['marker_current'],
|
||||
partial_grant=previous.PARTIAL_GRANT,partial_proof_sha256=previous.PARTIAL_PROOF,
|
||||
partial_attempt=checkpoint['request']['parameters']['apt_manifest']['native_recovery']['previous_attempt_id'],
|
||||
proof_row=readonly['proof'][0],unit=(cloud.ROOT/'deploy/linux/rmm-apt-helper.service').read_text(),
|
||||
reviewed_config_sha256=hashlib.sha256(module.apt_config()).hexdigest(),
|
||||
qualifications=qualifications,
|
||||
**{name:base64.b64encode(data).decode() for name,data in raw.items()},
|
||||
**{name+'_sha256':pin for name,pin in pins.items()}), installed
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--preflight-only',action='store_true')
|
||||
mode.add_argument('--install-reviewed-artifacts',action='store_true')
|
||||
for name in ('helper','agent','provider'): parser.add_argument('--'+name+'-sha256',required=True)
|
||||
args = parser.parse_args(); os.umask(0o077)
|
||||
require(not (STATE/'configured-reconcile-runtime.json').exists(), 'existing runtime pointer retained; never repeat maintenance automatically')
|
||||
payload,installed = build_payload(args)
|
||||
policy = cloud.bootstrap.rpc('server.conf',previous.closed_policy,{})
|
||||
expected_policy = cloud.contract.protected_json(STATE/'capacity-recovery/recovery-policy-closed-unknown.json')
|
||||
require(expected_policy['closed'] is True and expected_policy['policy_sha256'] ==
|
||||
'06e4484cd95b88c77bf1674c84d867f852ee71b3083d3697f6609867b54893f7'
|
||||
and policy['policy_sha256'] == expected_policy['policy_sha256'], 'exact reviewed deny-only policy must remain closed')
|
||||
gate = cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,
|
||||
agent_id=payload['agent_id'],job_id=payload['context']['job_id']))
|
||||
require(gate == dict(jobs=1,outbox=1,permits=1,unresolved=1), 'only exact approved configure unknown may remain')
|
||||
old = cloud.bootstrap.rpc('linux-vm.conf',previous.installed_metadata,OLD)
|
||||
payload.update(old_files=old['files'],preflight_only=args.preflight_only)
|
||||
report = cloud.bootstrap.rpc('linux-vm.conf',upgrade_configured,payload)
|
||||
cloud.contract.write_private(STATE/('configured-reconcile-'+('preflight-' if args.preflight_only else 'install-')+payload['id']+'.json'),dict(report,closed_policy=policy,central_gate=gate,qualification=payload['qualifications']))
|
||||
if args.preflight_only:
|
||||
require(report.get('preflight_only') is True and report.get('maintenance_mutations') is False, 'read-only preflight response required')
|
||||
print('PASS reviewed configured-reconciliation preflight; original unknown retained'); return
|
||||
require(report['completed'] and report['preserved_state'] and report['journals_preserved_before_restart']
|
||||
and report['unknown_grant'] == UNKNOWN and report['provider_sha256'] == args.provider_sha256,
|
||||
'preserving installation incomplete; retain report, never repeat automatically')
|
||||
previous.previous.record_agent_upgrade(report,installed)
|
||||
cloud.contract.write_private(STATE/'configured-reconcile-runtime.json',report)
|
||||
print('PASS preserving configured-reconciliation runtime; APT endpoint stopped, normal signed RootSync still required')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
print('CLOUD_CONFIGURED_RECONCILE_INCOMPLETE:'+type(error).__name__)
|
||||
if str(error).startswith('remote bootstrap incomplete: '): print(str(error))
|
||||
raise SystemExit(2) from None
|
||||
@@ -93,8 +93,19 @@ def main():
|
||||
operations = []; submit_contexts = {}
|
||||
for path in args.checkpoint:
|
||||
checkpoint, source = witness(path)
|
||||
if checkpoint.get('case') == 'reboot_last' and checkpoint.get('verified') is not True:
|
||||
final, final_source = witness(Path(path).with_name('result.json'))
|
||||
job = checkpoint['plan']['jobs'][0]
|
||||
require(final.get('verified') is True and final['result']['state'] == 'succeeded'
|
||||
and final['result']['job_id'] == job['job_id'] and final['result']['attempt_id'] == job['attempt_id']
|
||||
and final['result']['result']['evidence']['postconditions_passed'] is True
|
||||
and final['result']['result']['evidence']['reconciled_after_reboot'] is True
|
||||
and final['central']['grant_id'] == final['result']['result']['grant_id'],
|
||||
'separate verified reboot result is not bound to original checkpoint')
|
||||
source['completion'] = final_source
|
||||
checkpoint = dict(checkpoint, verified=True, audit=final['audit'], postapproval=final['postapproval'])
|
||||
require(checkpoint.get('verified') is True, 'native checkpoint not verified')
|
||||
plan = checkpoint['plan']; job = checkpoint['job']
|
||||
plan = checkpoint['plan']; job = checkpoint.get('job', plan['jobs'][0])
|
||||
require(hashlib.sha256(bytes(plan['plan_payload_bytes'])).hexdigest() == plan['plan_hash'], 'immutable plan bytes mismatch')
|
||||
require(plan['resolved_targets'] == [identity] and len(plan['jobs']) == 1
|
||||
and plan['jobs'][0] == job and job['agent_id'] == identity['agent_id']
|
||||
@@ -130,20 +141,34 @@ def main():
|
||||
and cloud.contract.timestamp(receipt['observed_at']) < context['valid_until'],
|
||||
'live existing-submit receipt not bound to selected operation')
|
||||
receipt = dict(source=source, observation=receipt)
|
||||
required_cases = {'hello','refresh_inventory','service_setup','service_start','service_stop','offline_wait'}
|
||||
successful_cases = {o['case'] for o in operations if o['operation']['execution_state'] == 'succeeded'
|
||||
and o['operation']['counts'] == dict(succeeded=1)}
|
||||
reference_complete = (required_cases <= successful_cases and receipt is not None
|
||||
and all(o['operation']['execution_state'] in ('succeeded','failed','cancelled') for o in operations))
|
||||
health = backend.call('monitoring.get_health', dict(tenant_id=cloud.TENANT))
|
||||
# Keep the health projection's honest timestamps/coverage without repeating
|
||||
# every device's full inventory in this bounded private reference report.
|
||||
health['items'] = [{k: d.get(k) for k in ('agent_id','asset_id','presence','health','quality',
|
||||
'completeness','last_contact_at','observed_at','received_at','source_id')}
|
||||
for d in health['items']]
|
||||
report = dict(schema_version=1, observed_at=cloud.contract.now().isoformat(),
|
||||
target=dict(tenant_id=cloud.TENANT,**identity), verified=reference_complete,
|
||||
scope='one explicitly selected real Linux Cloud VM; no multi-device concurrency proof',
|
||||
tenants=tenants, list_pages=pages, selected_targets=[identity], selected_device=device,
|
||||
other_devices=[{k: d.get(k) for k in ('agent_id', 'asset_id', 'presence', 'source_id', 'observed_at', 'received_at')}
|
||||
for d in devices if d['agent_id'] != identity['agent_id']],
|
||||
health=backend.call('monitoring.get_health', dict(tenant_id=cloud.TENANT)),
|
||||
health=health,
|
||||
operations=operations, live_submit_idempotency=receipt,
|
||||
idempotency_status='observed' if receipt else 'not_attached',
|
||||
no_new_plans_jobs_or_approvals=True)
|
||||
output.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
encoded = (json.dumps(report, indent=2)+'\n').encode()
|
||||
require(len(encoded) <= 1024*1024, 'bounded protected reference report size')
|
||||
fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, 'w') as stream:
|
||||
json.dump(report, stream, indent=2); stream.write('\n'); stream.flush(); os.fsync(stream.fileno())
|
||||
print('PASS stored MCP reference: one fixed Linux target, '+str(len(operations))+' existing operations; idempotency '+report['idempotency_status'])
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(encoded); stream.flush(); os.fsync(stream.fileno())
|
||||
print('Stored MCP reference saved: one fixed Linux target, '+str(len(operations))+' existing operations; verified='+str(reference_complete)+'; idempotency '+report['idempotency_status'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -262,6 +262,7 @@ def close(ident):
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=('enable','accept','finish','close'));parser.add_argument('--id');parser.add_argument('--prerequisites');args=parser.parse_args();os.umask(0o077)
|
||||
require(not (Path.home()/'.local/share/rust-discovery/remote-agent').exists(),'notebook development only')
|
||||
if args.action=='enable':require(args.prerequisites,'explicit completed native prerequisite manifest required');enable(args.prerequisites)
|
||||
else:require(args.id,'one existing bound reboot session required');{'accept':accept,'finish':finish,'close':close}[args.action](args.id)
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ RECIPES = {'apt_native_install':'rmm-apt-fixture-install-1.0','ensure_package':'
|
||||
PINS = {'1.0':'74036ae81223deab3aae994fd2a2a695af408741777e1ce17dcb4e32f1235867',
|
||||
'1.1':'ef67c9fe0ba1443fcfe465768e1204763063c369a2a1da04af6027d8a9c00491'}
|
||||
PLANNING_FIELDS = ('fixture_package','baseline_sha256','sealed_baseline_sha256','status_fingerprint',
|
||||
'dpkg_updates_empty','pending_triggers_empty','runtime','runtime_sha256',
|
||||
'dpkg_updates_empty','pending_triggers_empty','host_reboot_required','runtime','runtime_sha256',
|
||||
'provider_sha256','hook_sha256','apt_config_sha256','prefetch')
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ def planning_from_device(device,recovery=False):
|
||||
missing=[key for key in PLANNING_FIELDS if key not in data]
|
||||
m.require(not missing,'not_configured:apt_planning_fields_missing:'+','.join(missing))
|
||||
m.require(data['dpkg_updates_empty'] is True and data['pending_triggers_empty'] is True,'not_configured:dpkg_dirty')
|
||||
m.require(type(data['host_reboot_required']) is bool,'not_configured:actual_host_reboot_flag_required')
|
||||
m.require(data['baseline_sha256']==data['sealed_baseline_sha256'] and __import__('re').fullmatch('[a-f0-9]{64}',data['baseline_sha256']),
|
||||
'not_configured:actual_sealed_baseline_binding')
|
||||
if 'baseline' in data:
|
||||
|
||||
Reference in New Issue
Block a user