fix: bind noop acceptance to post-upgrade passive baseline and preserve failed attempt
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 20:08:52 +01:00
parent aee9d72d8b
commit 335486cd1f
4 changed files with 264 additions and 3 deletions
+93 -3
View File
@@ -108,12 +108,92 @@ def unsubmitted_preparation(payload):
return dict(persisted_plans=count)
def accept(case,resume_unsubmitted=False):
def no_change_source(cases,target):
"""A new passive sample must follow an independently read verified upgrade."""
m=cloud.contract
checkpoint=m.protected_json(cases/'apt_native_update/checkpoint.json')
require(checkpoint.get('verified') is True,'verified native upgrade required before no-change acceptance')
job=checkpoint['plan']['jobs'][0]
current=cloud.Backend().call('jobs.get',dict(tenant_id=target['tenant_id'],job_id=job['job_id']))
require(current==checkpoint['result'] and current['state']=='succeeded' and
current['result']['evidence']['postconditions_passed'] is True,
'same authoritative completed native upgrade')
apt=current['result']['evidence']['apt']
require(apt['packages']==[dict(name='rmm-mvp-test',architecture='all',version='1.1')] and
apt['test_acceptance'] is True and len(apt['native_record_sha256'])==64,
'accepted native fixture1.1 postcondition required')
return apt['snapshot_sha256'],m.now()
def require_no_change_planning(device,data,snapshot_sha256,after):
m=cloud.contract
observation=next(x for x in device['inventory']['observations'] if x['scope']=='linux.apt.planning')
m.require(m.timestamp(observation['observed_at'])>=after and
data['fixture_package']==dict(version='1.1',status='install ok installed') and
data['baseline_sha256']==snapshot_sha256,
'not_configured:post_upgrade_passive_fixture1.1_baseline_required')
def no_effect_retry_proof(payload):
import sqlite3
require(payload['case'] in ('ensure_package','apply_updates'),'fixed no-change case')
require(all(str(uuid.UUID(payload[k]))==payload[k] for k in ('grant_id','job_id','attempt_id')),'canonical fixed identifiers')
with sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','ledger integrity')
require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0]==0 and
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,
'closed fault-free Root ledger')
row=db.execute('SELECT c.job,c.attempt,c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?',(payload['grant_id'],)).fetchone()
require(row is not None and row[:2]==(payload['job_id'],payload['attempt_id']) and row[2]==row[3],
'same terminal consumed no-effect attempt')
result=json.loads(row[2])
require(result['state']=='failed' and result['error_code']=='STATUS_FINGERPRINT' and
result['evidence']['postconditions_passed'] is False and result['evidence']['apt']=={},
'definitive pre-hook status mismatch only')
base=Path('/var/lib/rmm-apt-transaction');active=base/'active'
if not active.exists() or json.loads((active/'context.json').read_bytes())['grant_id']!=payload['grant_id']:
active=base/'history'/payload['grant_id']
require(active.is_dir() and not active.is_symlink(),'original handoff retained')
context=json.loads((active/'context.json').read_bytes())
require(all(context[k]==payload[k] for k in ('grant_id','job_id','attempt_id')) and
not any((active/name).exists() for name in ('hook-armed.json','launch.json','sandbox-ready.json','sandbox-result.json',
'native-prechecks.json','host-reboot-before.json','host-namespaces.json')),
'original status failure never launched sandbox or armed hook')
return dict(verified_no_effect=True,grant_id=payload['grant_id'],job_id=payload['job_id'],attempt_id=payload['attempt_id'],
result_sha256=hashlib.sha256(row[2].encode() if isinstance(row[2],str) else row[2]).hexdigest(),
context_sha256=hashlib.sha256((active/'context.json').read_bytes()).hexdigest())
def no_change_retry_folder(case,cases,target):
m=cloud.contract;old=m.protected_json(cases/case/'checkpoint.json')
require(case in ('ensure_package','apply_updates') and old['stage']=='failed' and
all(old[k]==target[k] for k in target),'same fixed terminal no-change case')
job=old['plan']['jobs'][0]
current=cloud.Backend().call('jobs.get',dict(tenant_id=target['tenant_id'],job_id=job['job_id']))
counts=prior.prior.evidence('gate',job_id=job['job_id'])
require(current==old['result'] and current['state']=='failed' and current['result']['error_code']=='STATUS_FINGERPRINT' and
counts==old['postapproval']==dict(jobs=1,outbox=1,permits=1,unresolved=0),
'original approved single terminal status mismatch retained')
proof=cloud.bootstrap.rpc('linux-vm.conf',no_effect_retry_proof,dict(case=case,grant_id=current['result']['grant_id'],
job_id=job['job_id'],attempt_id=job['payload']['attempt_id']))
require(proof['verified_no_effect'] is True,'readonly original no-effect proof')
new=cases/'no-change-retry';new.mkdir(mode=0o700,exist_ok=True)
m.write_private(new/(case+'-prior-no-effect.json'),dict(checkpoint_sha256=hashlib.sha256((cases/case/'checkpoint.json').read_bytes()).hexdigest(),proof=proof))
return new
def accept(case,resume_unsubmitted=False,new_noop_after_definitive_no_effect=False):
m=cloud.contract;target=prior.prior.identity()
require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
ready,session=runtime_session();cases=session/'cases';cases.mkdir(mode=0o700,exist_ok=True)
require(session!=BASE/'configured-reconcile-runtime' or case!='apt_configure_recovery',
'configured historical1.0 is never executed again; next action is separate native1.1 upgrade')
no_change=case in ('ensure_package','apply_updates')
require(not new_noop_after_definitive_no_effect or (no_change and session==BASE/'configured-reconcile-runtime' and not resume_unsubmitted),
'explicit no-effect retry only for qualified runtime no-change cases')
source=no_change_source(cases,target) if no_change else None
if new_noop_after_definitive_no_effect:
cases=no_change_retry_folder(case,cases,target)
if session!=BASE and case=='apt_configure_recovery':
original=m.protected_json(STATE/case/'checkpoint-capacity-terminal.json')
old_job=original['plan']['jobs'][0]
@@ -137,10 +217,17 @@ def accept(case,resume_unsubmitted=False):
device=cloud.Backend().call('inventory.get_device',{k:target[k] for k in ('tenant_id','asset_id')})
try:
data=prior.adapters.planning_from_device(device,recovery=case=='apt_configure_recovery')
if source:require_no_change_planning(device,data,*source)
break
except m.GuardError:
require(time.monotonic()<deadline,'fresh stored exact planning unavailable; no job submitted');time.sleep(3)
request=recovery_request(device,target) if case=='apt_configure_recovery' else m.case_input(case,target,planning=data)
if source:
observation=next(x for x in device['inventory']['observations'] if x['scope']=='linux.apt.planning')
m.write_private(cases/(case+'-post-upgrade-planning.json'),dict(snapshot_sha256=source[0],after=source[1].isoformat(),
observed_at=observation['observed_at'],baseline_sha256=data['baseline_sha256'],status_fingerprint=data['status_fingerprint'],
fixture_package=data['fixture_package']))
request['reason']+='; Verify already installed fixture1.1 without executing a transaction; fresh baseline after accepted native upgrade'
if data['host_reboot_required']:
request['reason']+='; Host reboot already required; flag unchanged; no reboot in this job'
if request is not None and session==BASE/'capacity-recovery':
@@ -162,11 +249,14 @@ def close():
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('action',choices=('enable','accept','finish','close'));p.add_argument('--case',choices=CASES)
p.add_argument('--resume-unsubmitted-ui',action='store_true',help='Preserve a failed preparation only after proving no plan was persisted')
p.add_argument('--new-noop-after-definitive-no-effect',action='store_true',help='New UI plan in a separate folder only after proving the retained STATUS_FINGERPRINT attempt had no effect')
args=p.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
if args.action in ('accept','finish'):
require(args.case is not None,'one fixed case required')
if args.action=='accept':accept(args.case,args.resume_unsubmitted_ui)
else:prior.finish(args.case,folder_root=runtime_session()[1]/'cases')
if args.action=='accept':accept(args.case,args.resume_unsubmitted_ui,args.new_noop_after_definitive_no_effect)
else:
require(not args.new_noop_after_definitive_no_effect or args.case in ('ensure_package','apply_updates'),'fixed no-change result only')
prior.finish(args.case,folder_root=runtime_session()[1]/'cases'/('no-change-retry' if args.new_noop_after_definitive_no_effect else ''))
else:(enable if args.action=='enable' else close)()
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""Read only the exact failed b7fbd NoOp and retained native12 history.
No package scan/command, service operation, API/DB write, execution retry,
grant or approval. A saved report does not authorize a future attempt.
"""
import hashlib
import json
import os
from pathlib import Path
import uuid
import cloud_mvp_acceptance as cloud
STATE=cloud.ROOT/'devstate/cloud-apt-recovery/configured-reconcile-runtime'
GRANT='b7fbd250-c382-4d2f-950d-ae6001e4c48e'
NATIVE='6b68c65b-8c0c-4cab-964d-e12a2d7f0626'
ACCEPTANCE='a88d6b10bbf6c36eaa8dee87eb1a381f2becc36297db8df2b4189791b9165506'
PARTIAL='dc3bf3870ea77c12f9291bac712c53ae72194eaec980a8646155a7fb5c0dbe21'
CONFIGURED_PROOF='27f784e135eeaf1403bdfa7f86374dc32ae7ebb9d7e92ca2405e8440a0172b56'
def stored(payload):
import sqlite3
import stat
grant='b7fbd250-c382-4d2f-950d-ae6001e4c48e'
native='6b68c65b-8c0c-4cab-964d-e12a2d7f0626'
base=Path('/var/lib/rmm-apt-transaction')
def read(path,limit=1024*1024):
for parent in path.parents:
metadata=parent.lstat();require(stat.S_ISDIR(metadata.st_mode) and metadata.st_uid==0 and metadata.st_mode&0o022==0,'trusted root witness parents')
fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW)
with os.fdopen(fd,'rb') as stream:
metadata=os.fstat(stream.fileno());require(stat.S_ISREG(metadata.st_mode) and metadata.st_uid==0 and metadata.st_mode&0o022==0
and metadata.st_size<=limit,'bounded retained root witness')
return stream.read()
def witness(path):
require(not path.is_symlink(),'witness symlink')
if not path.exists():return dict(exists=False)
raw=read(path);metadata=path.stat()
return dict(exists=True,sha256=hashlib.sha256(raw).hexdigest(),size=len(raw),inode=metadata.st_ino)
root=Path('/var/lib/rmm-helper/consumption.sqlite')
with sqlite3.connect(root.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','retained ledger integrity')
row=db.execute('SELECT c.rowid,c.agent,c.job,c.attempt,c.result,e.start,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?',(grant,)).fetchone()
require(row is not None,'exact existing NoOp row')
rowid,agent,job,attempt,finished,start,outcome=row
require(job==payload['job_id'] and attempt==payload['attempt_id'] and agent==payload['agent_id'],'fixed old NoOp attempt')
saved=json.loads(start);request=saved['request'];permit=json.loads(bytes(request['permit_bytes']))
action=json.loads(bytes(request['job_bytes']))
binding=bool(permit['grant_id']==grant and permit['job_id']==job and permit['attempt_id']==attempt
and request['plan_hash']==payload['plan_sha256'] and action['job_id']==job and action['attempt_id']==attempt
and action['action']['action']=='ensure_package' and action['execution_identity']=='root'
and hashlib.sha256(bytes(request['job_bytes'])).hexdigest()==payload['job_payload_sha256'])
result=json.loads(finished) if finished else None;last=json.loads(outcome) if outcome else None
failed=bool(result==last and result and result['state']=='failed' and result['error_code']=='STATUS_FINGERPRINT'
and result['evidence']['postconditions_passed'] is False
and result['evidence']['effect_armed_ledger_id']=='apt-'+grant+'-'+str(rowid))
ledger=dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],
faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0])
partial=[dict(grant_id=g,proof_sha256=h,proof_hash_valid=hashlib.sha256(raw).hexdigest()==h)
for g,h,raw in db.execute('SELECT grant_id,proof_sha256,proof FROM apt_partial_reconciliations')]
effect=[]
for g,h,raw,oldhash,old in db.execute('SELECT grant_id,proof_sha256,proof,previous_outcome_sha256,previous_outcome FROM apt_effect_reconciliations'):
previous=json.loads(old)
effect.append(dict(grant_id=g,proof_sha256=h,proof_hash_valid=hashlib.sha256(raw).hexdigest()==h,
previous_unknown_hash_valid=hashlib.sha256(old).hexdigest()==oldhash,previous_state=previous['state'],previous_code=previous['error_code']))
native_row=db.execute('SELECT c.rowid,c.job,c.attempt,c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?',(native,)).fetchone()
native_result=json.loads(native_row[3]) if native_row and native_row[3] else None
native_status=dict(rowid=native_row[0],job_id=native_row[1],attempt_id=native_row[2],result=native_result,
result_equals_outcome=bool(native_row[3] and native_row[3]==native_row[4])) if native_row else None
names=('context.json','signed-action.json','expectation.json','baseline.json','launch.json','hook-armed.json',
'native-prechecks.json','sandbox-ready.json','sandbox-result.json','host-reboot-before.json','host-namespaces.json')
locations={};raw_contexts={}
for name,path in (('active',base/'active'),('target_history',base/'history'/grant),('native_history',base/'history'/native)):
require(not path.is_symlink(),'history symlink')
if not path.exists():locations[name]=dict(exists=False);continue
context=json.loads(read(path/'context.json'));raw_contexts[name]=context
files={n:witness(path/n) for n in names}
locations[name]=dict(exists=True,grant_id=context['grant_id'],job_id=context['job_id'],attempt_id=context['attempt_id'],files=files)
targets=[name for name,context in raw_contexts.items() if context['grant_id']==grant]
no_child=bool(binding and failed and targets)
for name in targets:
context=raw_contexts[name];files=locations[name]['files']
no_child=bool(no_child and context['job_id']==job and context['attempt_id']==attempt
and context['job_payload_sha256']==payload['job_payload_sha256'] and context['plan_sha256']==payload['plan_sha256']
and all(not files[n]['exists'] for n in ('launch.json','hook-armed.json','native-prechecks.json','sandbox-ready.json','sandbox-result.json','host-reboot-before.json','host-namespaces.json')))
acceptance=witness(base/'acceptance-rmm-apt-fixture-update-1.1.json')
# Verify that recorded hashes/ownership did not change during this read.
for name,location in locations.items():
if not location['exists']:continue
path=base/'active' if name=='active' else base/'history'/(grant if name=='target_history' else native)
require({n:witness(path/n) for n in names}==location['files'],'retained witness changed during evidence read')
require(witness(base/'acceptance-rmm-apt-fixture-update-1.1.json')==acceptance,'native acceptance changed during evidence read')
return dict(schema_version=1,grant_id=grant,target_binding_verified=binding,root_rowid=rowid,stored_result=result,stored_outcome=last,
result_bytes_equal_outcome=finished==outcome,definitive_status_fingerprint_failure=failed,ledger=ledger,locations=locations,
target_context_locations=targets,verified_no_apt_child_launch=no_child,
active_still_original_native=raw_contexts.get('active',{}).get('grant_id')==native,native12=native_status,
native_acceptance=acceptance,partial_rows=partial,effect_rows=effect,
evidence_kind='retained files and readonly SQLite only; no live package scan')
def main():
os.umask(0o077)
checkpoint=cloud.contract.protected_json(STATE/'cases/ensure_package/checkpoint.json')
job=checkpoint['plan']['jobs'][0];identity=cloud.contract.protected_json(cloud.ROOT/'devstate/remote-vm.json')['identity']
report=cloud.bootstrap.rpc('linux-vm.conf',stored,dict(agent_id=identity['agent_id'],job_id=job['job_id'],attempt_id=job['payload']['attempt_id'],
plan_sha256=checkpoint['plan']['plan_hash'],job_payload_sha256=job['job_payload_hash']))
counts=cloud.bootstrap.rpc('server.conf',cloud.server,dict(kind='gate',tenant_id=cloud.TENANT,agent_id=identity['agent_id']))
result=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=job['job_id']))
path=STATE/('noop-preflight-'+uuid.uuid4().hex+'.json')
cloud.contract.write_private(path,dict(native=report,central_counts=counts,central=result))
cloud.contract.require(report['target_binding_verified'] and report['definitive_status_fingerprint_failure'] and report['root_rowid']==13
and report['result_bytes_equal_outcome'] and report['verified_no_apt_child_launch'] and report['ledger']==dict(consumed=13,unresolved=0,faults=0)
and report['native_acceptance'].get('sha256')==ACCEPTANCE and report['native12']['result']['state']=='succeeded'
and report['native12']['result_equals_outcome'] and report['partial_rows']==[dict(grant_id='31ef5e5e-5dbd-4428-a9a1-cbd15de2f5a0',proof_sha256=PARTIAL,proof_hash_valid=True)]
and report['effect_rows']==[dict(grant_id='33775478-15b0-46ec-b059-44f500dbb0b6',proof_sha256=CONFIGURED_PROOF,proof_hash_valid=True,previous_unknown_hash_valid=True,
previous_state='unknown',previous_code='FIXTURE_UNEXPECTED_REBOOT_REQUIRED')]
and counts['unresolved']==0 and result['state']=='failed' and result['result']['error_code']=='STATUS_FINGERPRINT',
'same failed attempt no-child and retained history not proven; no automatic retry')
print('PASS old b7fbd failed STATUS_FINGERPRINT, Root13/0/fault0, no targetAPTchild launch, native12 acceptance and old31/337 proofs preserved; new UI plan still required')
print('protected_report='+str(path.relative_to(cloud.ROOT)))
if __name__=='__main__':
try:main()
except Exception as error:
print('NOOP_PREFLIGHT_INCOMPLETE:'+(str(error) if isinstance(error,cloud.contract.GuardError) else type(error).__name__))
raise SystemExit(2) from None
+27
View File
@@ -120,3 +120,30 @@ is not a passed noop and is never reactivated. The minimal next preparation
must bind a newly stored passive installed1.1 observation to the accepted
post-snapshot; any new attempt needs its own pending0/0/0 and actual WebUI click.
No provider/helper change or replacement of the accepted1.1 record is needed.
Actual read-only preflight then proved the exact terminal Root13 outcome,
active handoff13 with no launch/hook/prechecks/child result, and retained native12
acceptance plus original31/337 proofs. Root13/0unresolved/0faults. Protected
report`configured-reconcile-runtime/noop-preflight-d140758efc3e421b91550d3dd52d5d7a.json`.
The SDK fix is implemented and locally qualified: **29 pure harness guard
tests passed**. A no-change preparation now independently reads the completed,
verified native upgrade, then waits for a passive observation collected after
that read. It requires installed fixture `1.1` and the exact full baseline hash
from the native upgrade's postconditions. The earlier observation's under
90-second age is insufficient. A saved preparation witness records that time
barrier, observation time and baseline/fingerprint.
The explicit `--new-noop-after-definitive-no-effect` option additionally checks
the old immutable failed plan, its single authoritative grant and a read-only
Root-ledger/context proof that the `STATUS_FINGERPRINT` failure occurred before
launch or hook arming. It creates a separate
`cases/no-change-retry/ensure_package/` directory; the original checkpoint is
retained. The next actual browser flow still creates a new pending plan and
requires its own approval. These commands are **prepared, not executed by this
source qualification**:
```sh
python3 deploy/cloud_apt_recovery.py accept --case ensure_package --new-noop-after-definitive-no-effect
python3 deploy/cloud_apt_recovery.py finish --case ensure_package --new-noop-after-definitive-no-effect
```
+15
View File
@@ -19,9 +19,24 @@ import vm_mvp_acceptance as m
import vm_mvp_adapters as advanced
import vm_mvp_runtime as native
import vm_runtime_upgrade as upgrade
import cloud_apt_recovery as apt_recovery
class Guards(unittest.TestCase):
def test_no_change_planning_follows_accepted_upgrade_with_exact_post_snapshot(self):
barrier=m.now();snapshot='b'*64
observation=dict(scope='linux.apt.planning',observed_at=(barrier+dt.timedelta(seconds=1)).isoformat())
device=dict(inventory=dict(observations=[observation]))
data=dict(fixture_package=dict(version='1.1',status='install ok installed'),baseline_sha256=snapshot)
apt_recovery.require_no_change_planning(device,data,snapshot,barrier)
for changed in (dict(fixture_package=dict(version='1.0',status='install ok installed')),
dict(baseline_sha256='a'*64)):
with self.assertRaises(m.GuardError):
apt_recovery.require_no_change_planning(device,dict(data,**changed),snapshot,barrier)
observation['observed_at']=(barrier-dt.timedelta(seconds=1)).isoformat()
with self.assertRaises(m.GuardError):
apt_recovery.require_no_change_planning(device,data,snapshot,barrier)
def settings(self):
return m.config(dict(profile='dev/test', origin=m.ORIGIN, connect_address='127.0.0.1:18443', ca_file=str(m.CA),
tenant_id='11111111-1111-4111-8111-111111111111', asset_id='22222222-2222-4222-8222-222222222222',