269 lines
19 KiB
Python
269 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Exact configure-only recovery followed by separate fixture upgrade/noops."""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import time
|
|
import uuid
|
|
|
|
import cloud_apt_retry as prior
|
|
import cloud_root_evidence
|
|
import cloud_root_policy
|
|
|
|
cloud=prior.cloud
|
|
require=cloud.require
|
|
BASE=cloud.ROOT/'devstate/cloud-apt-recovery'
|
|
STATE=BASE/'cases'
|
|
CASES=('apt_configure_recovery','apt_native_update','ensure_package','apply_updates')
|
|
|
|
|
|
def runtime_session():
|
|
configured=BASE/'configured-reconcile-runtime.json'
|
|
if configured.exists():
|
|
report=cloud.contract.protected_json(configured)
|
|
require(report['completed'] and report['preserved_state'] and report['readiness_only'] and
|
|
not report['native_acceptance'] and report['unknown_grant']=='33775478-15b0-46ec-b059-44f500dbb0b6',
|
|
'qualified preserving configured-reconciliation runtime')
|
|
return report,BASE/'configured-reconcile-runtime'
|
|
latest=BASE/'journal-capacity-runtime.json'
|
|
if latest.exists():
|
|
report=cloud.contract.protected_json(latest)
|
|
require(report['completed'] and report['preserved_state'],'qualified preserved capacity runtime')
|
|
return report,BASE/'capacity-recovery'
|
|
return cloud.contract.protected_json(BASE/'configure-runtime.json'),BASE
|
|
|
|
|
|
def recovery_request(device,target):
|
|
m=cloud.contract;data=prior.adapters.planning_from_device(device,recovery=True)
|
|
closed=m.protected_json(BASE/'readonly-central-result.json')['result']
|
|
reference=dict(schema_version=1,previous_grant_id=closed['result']['grant_id'],previous_attempt_id=closed['attempt_id'],
|
|
partial_proof_sha256=closed['result']['evidence']['apt']['reconciliation_proof_sha256'])
|
|
require(closed['state']=='failed' and closed['result']['error_code']=='APT_PARTIAL_UNCONFIGURED_VERIFIED' and
|
|
data['native_recovery']==reference,'exact previously closed partial reference')
|
|
pin=prior.adapters.PINS['1.0'];start=m.now();receipt=data['prefetch'][pin]
|
|
require(receipt['schema_version']==1 and receipt['sha256']==pin and receipt['size']==20672 and
|
|
receipt['sealed_at_ms']<=int(start.timestamp()*1000),'actual pregrant archive receipt')
|
|
tx=dict(schema_version=2,operation='configure_only',recipe_id='rmm-apt-fixture-install-1.0',recipe_version='1',
|
|
platform='ubuntu-24.04-amd64',resolver_config_version='rmm-apt-local-v1',no_reboot=True,
|
|
status_fingerprint=data['status_fingerprint'],baseline_sha256=data['baseline_sha256'],packages=[
|
|
dict(name='rmm-mvp-test',architecture='all',old_version='1.0',new_version='1.0',origin='rmm-test',sha256=pin,
|
|
size=20672,archive_path='/var/lib/rmm-apt-transaction/archives/'+pin+'.deb')])
|
|
raw=prior.adapters.canonical(tx)
|
|
manifest=dict(type='rmm.apt.helper-action',schema_version=4,action='apt-native-acceptance',execution_profile='dev-test',
|
|
transaction_sha256=hashlib.sha256(raw).hexdigest(),resolved_apt_transaction=tx,transaction_bytes=list(raw),
|
|
native_recovery=reference,test_result_id='configure-recovery-'+uuid.uuid4().hex,
|
|
**{k:data[k] for k in ('provider_sha256','hook_sha256','apt_config_sha256','runtime_sha256')})
|
|
m.validate_apt_manifest(manifest)
|
|
return dict(tenant_id=target['tenant_id'],action='apt_native_acceptance',targets=dict(asset_ids=[target['asset_id']]),
|
|
parameters=dict(execution_identity='root',timeout_seconds=300,max_output_bytes=65536,
|
|
recipe_id=tx['recipe_id'],recipe_version='1',apt_manifest=manifest),
|
|
schedule=m.schedule(start,start+dt.timedelta(minutes=15)),rollout=dict(max_parallel=1,halt_after_failures=1),
|
|
reason='Configure ONLY the exactly proven unpacked fixture1.0; new attempt and separate recovery record; no unpack, pending packages, reinstall or reboot')
|
|
|
|
|
|
def enable():
|
|
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
|
ready,session=runtime_session();session.mkdir(mode=0o700,exist_ok=True)
|
|
require(ready['completed'] and ready['readiness_only'] and ready['preserved_state'] and not ready['native_acceptance'],
|
|
'qualified native probe, no invented package acceptance')
|
|
target=prior.prior.identity();pin=ready['apt_pin'];identity=pin['identity']
|
|
require(all(identity[k]==target[k] for k in target),'same permanent Cloud VM')
|
|
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=True,
|
|
agent_id=target['agent_id'],binary_sha256=ready['helper_sha256']))
|
|
m.write_private(session/'recovery-endpoint-enabled.json',endpoint)
|
|
peers=cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,target)
|
|
require(peers['ledger']['faults']==0 and peers['apt']['provider_stack_sha256']==pin['provider_stack_sha256'],
|
|
'actual fixed helper peer and stack')
|
|
scope={k:identity[k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')}
|
|
scope.update(principals=['simulated-testprincipal-a'],helper_policy_sha256=pin['provider_stack_sha256'],
|
|
acceptance_evidence_id='cloud-configure-recovery-probe:'+ready['readiness_sha256'],allowed_services=[],allow_root_scripts=False,allow_reboot=False,
|
|
apt=dict(allow_native_acceptance=True,allow_execute=True,runtime_sha256=ready['runtime_sha256'],
|
|
apt_config_sha256=ready['apt_config_sha256'],provider_sha256=ready['provider_sha256'],
|
|
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
|
|
policy=dict(schema_version=1,profile='development',expires_at=(m.now()+dt.timedelta(hours=1)).isoformat(),scopes=[scope])
|
|
if session==BASE/'configured-reconcile-runtime':
|
|
old=m.protected_json(BASE/'capacity-recovery/recovery-policy-closed-unknown.json')
|
|
prior_checkpoint=m.protected_json(BASE/'capacity-recovery/cases/apt_configure_recovery/checkpoint.json')
|
|
closed=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=prior_checkpoint['plan']['jobs'][0]['job_id']))
|
|
require(closed['state']=='failed' and closed['result']['error_code']=='APT_RECOVERY_CONFIGURED_ACCEPTANCE_REBOOT_FLAG_VERIFIED' and
|
|
closed['result']['grant_id']=='33775478-15b0-46ec-b059-44f500dbb0b6',
|
|
'ordinary signed Sync must first close the exact historical configured effect as failed')
|
|
else:
|
|
old=m.protected_json(BASE/'recovery-policy-closed.json' if session!=BASE else cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
|
record=cloud.bootstrap.rpc('server.conf',prior.replace_policy,dict(id=str(uuid.uuid4()),previous_sha256=old['policy_sha256'],policy=policy))
|
|
m.write_private(session/'recovery-policy.json',dict(record,policy=policy))
|
|
print('Fixed APT runtime endpoint and expiring limits enabled; no job or approval created')
|
|
|
|
|
|
def unsubmitted_preparation(payload):
|
|
import re
|
|
root=Path('/opt/rust-discovery');ident=payload['test_result_id']
|
|
require(isinstance(ident,str) and re.fullmatch('[A-Za-z0-9_-]{1,128}',ident),'fixed preparation test ID')
|
|
query="SELECT count(*) FROM operation_plans WHERE tenant_id='11111111-1111-4111-8111-111111111111' AND position('"+ident+"' in convert_from(payload,'UTF8'))>0;"
|
|
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
|
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
|
count=int(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip())
|
|
return dict(persisted_plans=count)
|
|
|
|
|
|
def no_change_source(cases,target):
|
|
"""A new passive sample must follow an independently read verified upgrade."""
|
|
m=cloud.contract
|
|
checkpoint=m.protected_json(cases/'apt_native_update/checkpoint.json')
|
|
require(checkpoint.get('verified') is True,'verified native upgrade required before no-change acceptance')
|
|
job=checkpoint['plan']['jobs'][0]
|
|
current=cloud.Backend().call('jobs.get',dict(tenant_id=target['tenant_id'],job_id=job['job_id']))
|
|
require(current==checkpoint['result'] and current['state']=='succeeded' and
|
|
current['result']['evidence']['postconditions_passed'] is True,
|
|
'same authoritative completed native upgrade')
|
|
apt=current['result']['evidence']['apt']
|
|
require(apt['packages']==[dict(name='rmm-mvp-test',architecture='all',version='1.1')] and
|
|
apt['test_acceptance'] is True and len(apt['native_record_sha256'])==64,
|
|
'accepted native fixture1.1 postcondition required')
|
|
return apt['snapshot_sha256'],m.now()
|
|
|
|
|
|
def require_no_change_planning(device,data,snapshot_sha256,after):
|
|
m=cloud.contract
|
|
observation=next(x for x in device['inventory']['observations'] if x['scope']=='linux.apt.planning')
|
|
m.require(m.timestamp(observation['observed_at'])>=after and
|
|
data['fixture_package']==dict(version='1.1',status='install ok installed') and
|
|
data['baseline_sha256']==snapshot_sha256,
|
|
'not_configured:post_upgrade_passive_fixture1.1_baseline_required')
|
|
|
|
|
|
def no_effect_retry_proof(payload):
|
|
import sqlite3
|
|
require(payload['case'] in ('ensure_package','apply_updates'),'fixed no-change case')
|
|
require(all(str(uuid.UUID(payload[k]))==payload[k] for k in ('grant_id','job_id','attempt_id')),'canonical fixed identifiers')
|
|
with sqlite3.connect('file:/var/lib/rmm-helper/consumption.sqlite?mode=ro',uri=True) as db:
|
|
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','ledger integrity')
|
|
require(db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0]==0 and
|
|
db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,
|
|
'closed fault-free Root ledger')
|
|
row=db.execute('SELECT c.job,c.attempt,c.result,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.grant_id=?',(payload['grant_id'],)).fetchone()
|
|
require(row is not None and row[:2]==(payload['job_id'],payload['attempt_id']) and row[2]==row[3],
|
|
'same terminal consumed no-effect attempt')
|
|
result=json.loads(row[2])
|
|
require(result['state']=='failed' and result['error_code']=='STATUS_FINGERPRINT' and
|
|
result['evidence']['postconditions_passed'] is False and result['evidence']['apt']=={},
|
|
'definitive pre-hook status mismatch only')
|
|
base=Path('/var/lib/rmm-apt-transaction');active=base/'active'
|
|
if not active.exists() or json.loads((active/'context.json').read_bytes())['grant_id']!=payload['grant_id']:
|
|
active=base/'history'/payload['grant_id']
|
|
require(active.is_dir() and not active.is_symlink(),'original handoff retained')
|
|
context=json.loads((active/'context.json').read_bytes())
|
|
require(all(context[k]==payload[k] for k in ('grant_id','job_id','attempt_id')) and
|
|
not any((active/name).exists() for name in ('hook-armed.json','launch.json','sandbox-ready.json','sandbox-result.json',
|
|
'native-prechecks.json','host-reboot-before.json','host-namespaces.json')),
|
|
'original status failure never launched sandbox or armed hook')
|
|
return dict(verified_no_effect=True,grant_id=payload['grant_id'],job_id=payload['job_id'],attempt_id=payload['attempt_id'],
|
|
result_sha256=hashlib.sha256(row[2].encode() if isinstance(row[2],str) else row[2]).hexdigest(),
|
|
context_sha256=hashlib.sha256((active/'context.json').read_bytes()).hexdigest())
|
|
|
|
|
|
def no_change_retry_folder(case,cases,target):
|
|
m=cloud.contract;old=m.protected_json(cases/case/'checkpoint.json')
|
|
require(case in ('ensure_package','apply_updates') and old['stage']=='failed' and
|
|
all(old[k]==target[k] for k in target),'same fixed terminal no-change case')
|
|
job=old['plan']['jobs'][0]
|
|
current=cloud.Backend().call('jobs.get',dict(tenant_id=target['tenant_id'],job_id=job['job_id']))
|
|
counts=prior.prior.evidence('gate',job_id=job['job_id'])
|
|
require(current==old['result'] and current['state']=='failed' and current['result']['error_code']=='STATUS_FINGERPRINT' and
|
|
counts==old['postapproval']==dict(jobs=1,outbox=1,permits=1,unresolved=0),
|
|
'original approved single terminal status mismatch retained')
|
|
proof=cloud.bootstrap.rpc('linux-vm.conf',no_effect_retry_proof,dict(case=case,grant_id=current['result']['grant_id'],
|
|
job_id=job['job_id'],attempt_id=job['payload']['attempt_id']))
|
|
require(proof['verified_no_effect'] is True,'readonly original no-effect proof')
|
|
new=cases/'no-change-retry';new.mkdir(mode=0o700,exist_ok=True)
|
|
m.write_private(new/(case+'-prior-no-effect.json'),dict(checkpoint_sha256=hashlib.sha256((cases/case/'checkpoint.json').read_bytes()).hexdigest(),proof=proof))
|
|
return new
|
|
|
|
|
|
def accept(case,resume_unsubmitted=False,new_noop_after_definitive_no_effect=False):
|
|
m=cloud.contract;target=prior.prior.identity()
|
|
require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
|
ready,session=runtime_session();cases=session/'cases';cases.mkdir(mode=0o700,exist_ok=True)
|
|
require(session!=BASE/'configured-reconcile-runtime' or case!='apt_configure_recovery',
|
|
'configured historical1.0 is never executed again; next action is separate native1.1 upgrade')
|
|
no_change=case in ('ensure_package','apply_updates')
|
|
require(not new_noop_after_definitive_no_effect or (no_change and session==BASE/'configured-reconcile-runtime' and not resume_unsubmitted),
|
|
'explicit no-effect retry only for qualified runtime no-change cases')
|
|
source=no_change_source(cases,target) if no_change else None
|
|
if new_noop_after_definitive_no_effect:
|
|
cases=no_change_retry_folder(case,cases,target)
|
|
if session!=BASE and case=='apt_configure_recovery':
|
|
original=m.protected_json(STATE/case/'checkpoint-capacity-terminal.json')
|
|
old_job=original['plan']['jobs'][0]
|
|
old_result=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=old_job['job_id']))
|
|
counts=prior.prior.evidence('gate',job_id=old_job['job_id'])
|
|
require(original['not_started_capacity_block'] and old_result['state'] in ('expired','cancelled') and counts['permits']==0,
|
|
'preserve original approved no-start; new attempt requires fresh UI approval')
|
|
folder=cases/case
|
|
if resume_unsubmitted:
|
|
require(folder.is_dir() and {p.name for p in folder.iterdir()}=={'prepare.json'},'only an unsubmitted UI preparation may be archived')
|
|
old=m.protected_json(folder/'prepare.json')
|
|
require(old['case']==case and old['phase']=='prepare' and all(old[k]==target[k] for k in target),'same fixed prior preparation target')
|
|
untouched=cloud.bootstrap.rpc('server.conf',unsubmitted_preparation,dict(test_result_id=old['request']['parameters']['apt_manifest']['test_result_id']))
|
|
require(untouched==dict(persisted_plans=0),'existing plan must be reviewed, never silently replaced')
|
|
archived=cases/(case+'.unsubmitted-ui-'+uuid.uuid4().hex)
|
|
os.rename(folder,archived);m.write_private(archived/'not-submitted.json',untouched)
|
|
# A maintenance restart only refreshes passive inventory from the preserved
|
|
# agent; it never issues a package diagnostic or executes a job.
|
|
deadline=time.monotonic()+90
|
|
while True:
|
|
device=cloud.Backend().call('inventory.get_device',{k:target[k] for k in ('tenant_id','asset_id')})
|
|
try:
|
|
data=prior.adapters.planning_from_device(device,recovery=case=='apt_configure_recovery')
|
|
if source:require_no_change_planning(device,data,*source)
|
|
break
|
|
except m.GuardError:
|
|
require(time.monotonic()<deadline,'fresh stored exact planning unavailable; no job submitted');time.sleep(3)
|
|
request=recovery_request(device,target) if case=='apt_configure_recovery' else m.case_input(case,target,planning=data)
|
|
if source:
|
|
observation=next(x for x in device['inventory']['observations'] if x['scope']=='linux.apt.planning')
|
|
m.write_private(cases/(case+'-post-upgrade-planning.json'),dict(snapshot_sha256=source[0],after=source[1].isoformat(),
|
|
observed_at=observation['observed_at'],baseline_sha256=data['baseline_sha256'],status_fingerprint=data['status_fingerprint'],
|
|
fixture_package=data['fixture_package']))
|
|
request['reason']+='; Verify already installed fixture1.1 without executing a transaction; fresh baseline after accepted native upgrade'
|
|
if data['host_reboot_required']:
|
|
request['reason']+='; Host reboot already required; flag unchanged; no reboot in this job'
|
|
if request is not None and session==BASE/'capacity-recovery':
|
|
request['reason']+='; prior fixed plan expired with zero grants at the old journal capacity; preserve it and authorize this fresh plan separately'
|
|
prior.accept(case,request=request,folder_root=cases)
|
|
|
|
|
|
def close():
|
|
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
|
ready,session=runtime_session();record=m.protected_json(session/'recovery-policy.json')
|
|
closed=cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=record['policy_sha256'],backup=record['backup']))
|
|
m.write_private(session/'recovery-policy-closed.json',closed)
|
|
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=False,
|
|
agent_id=prior.prior.identity()['agent_id'],binary_sha256=ready['helper_sha256']))
|
|
m.write_private(session/'recovery-endpoint-disabled.json',endpoint)
|
|
print('APT test limits closed and endpoint stopped; original histories retained')
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser(description=__doc__);p.add_argument('action',choices=('enable','accept','finish','close'));p.add_argument('--case',choices=CASES)
|
|
p.add_argument('--resume-unsubmitted-ui',action='store_true',help='Preserve a failed preparation only after proving no plan was persisted')
|
|
p.add_argument('--new-noop-after-definitive-no-effect',action='store_true',help='New UI plan in a separate folder only after proving the retained STATUS_FINGERPRINT attempt had no effect')
|
|
args=p.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
|
|
if args.action in ('accept','finish'):
|
|
require(args.case is not None,'one fixed case required')
|
|
if args.action=='accept':accept(args.case,args.resume_unsubmitted_ui,args.new_noop_after_definitive_no_effect)
|
|
else:
|
|
require(not args.new_noop_after_definitive_no_effect or args.case in ('ensure_package','apply_updates'),'fixed no-change result only')
|
|
prior.finish(args.case,folder_root=runtime_session()[1]/'cases'/('no-change-retry' if args.new_noop_after_definitive_no_effect else ''))
|
|
else:(enable if args.action=='enable' else close)()
|
|
|
|
|
|
if __name__=='__main__':
|
|
try:main()
|
|
except Exception as error:
|
|
message=str(error)
|
|
print(message if message.startswith(('remote bootstrap incomplete: ','APT_UI_INCOMPLETE:source_lines=')) else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
|
|
raise SystemExit(2) from None
|