fix: retain complete histories with bounded journal capacity and native recovery
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -19,6 +19,15 @@ STATE=BASE/'cases'
|
||||
CASES=('apt_configure_recovery','apt_native_update','ensure_package','apply_updates')
|
||||
|
||||
|
||||
def runtime_session():
|
||||
latest=BASE/'journal-capacity-runtime.json'
|
||||
if latest.exists():
|
||||
report=cloud.contract.protected_json(latest)
|
||||
require(report['completed'] and report['preserved_state'],'qualified preserved capacity runtime')
|
||||
return report,BASE/'capacity-recovery'
|
||||
return cloud.contract.protected_json(BASE/'configure-runtime.json'),BASE
|
||||
|
||||
|
||||
def recovery_request(device,target):
|
||||
m=cloud.contract;data=prior.adapters.planning_from_device(device,recovery=True)
|
||||
closed=m.protected_json(BASE/'readonly-central-result.json')['result']
|
||||
@@ -49,14 +58,14 @@ def recovery_request(device,target):
|
||||
|
||||
def enable():
|
||||
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
ready=m.protected_json(BASE/'configure-runtime.json')
|
||||
ready,session=runtime_session();session.mkdir(mode=0o700,exist_ok=True)
|
||||
require(ready['completed'] and ready['readiness_only'] and ready['preserved_state'] and not ready['native_acceptance'],
|
||||
'qualified native probe, no invented package acceptance')
|
||||
target=prior.prior.identity();pin=ready['apt_pin'];identity=pin['identity']
|
||||
require(all(identity[k]==target[k] for k in target),'same permanent Cloud VM')
|
||||
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=True,
|
||||
agent_id=target['agent_id'],binary_sha256=ready['helper_sha256']))
|
||||
m.write_private(BASE/'recovery-endpoint-enabled.json',endpoint)
|
||||
m.write_private(session/'recovery-endpoint-enabled.json',endpoint)
|
||||
peers=cloud.bootstrap.rpc('linux-vm.conf',cloud_root_evidence.inspect_helpers,target)
|
||||
require(peers['ledger']['faults']==0 and peers['apt']['provider_stack_sha256']==pin['provider_stack_sha256'],
|
||||
'actual fixed helper peer and stack')
|
||||
@@ -67,15 +76,43 @@ def enable():
|
||||
apt_config_sha256=ready['apt_config_sha256'],provider_sha256=ready['provider_sha256'],
|
||||
hook_sha256=hashlib.sha256((cloud.ROOT/'tools/apt-transaction/hook').read_bytes()).hexdigest()))
|
||||
policy=dict(schema_version=1,profile='development',expires_at=(m.now()+dt.timedelta(hours=1)).isoformat(),scopes=[scope])
|
||||
old=m.protected_json(cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
||||
old=m.protected_json(BASE/'recovery-policy-closed.json' if session!=BASE else cloud.ROOT/'devstate/cloud-logs/policy-closed.json')
|
||||
record=cloud.bootstrap.rpc('server.conf',prior.replace_policy,dict(id=str(uuid.uuid4()),previous_sha256=old['policy_sha256'],policy=policy))
|
||||
m.write_private(BASE/'recovery-policy.json',dict(record,policy=policy))
|
||||
m.write_private(session/'recovery-policy.json',dict(record,policy=policy))
|
||||
print('Fixed APT runtime endpoint and expiring limits enabled; no job or approval created')
|
||||
|
||||
|
||||
def accept(case):
|
||||
def unsubmitted_preparation(payload):
|
||||
import re
|
||||
root=Path('/opt/rust-discovery');ident=payload['test_result_id']
|
||||
require(isinstance(ident,str) and re.fullmatch('[A-Za-z0-9_-]{1,128}',ident),'fixed preparation test ID')
|
||||
query="SELECT count(*) FROM operation_plans WHERE tenant_id='11111111-1111-4111-8111-111111111111' AND position('"+ident+"' in convert_from(payload,'UTF8'))>0;"
|
||||
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
count=int(subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip())
|
||||
return dict(persisted_plans=count)
|
||||
|
||||
|
||||
def accept(case,resume_unsubmitted=False):
|
||||
m=cloud.contract;target=prior.prior.identity()
|
||||
require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
ready,session=runtime_session();cases=session/'cases';cases.mkdir(mode=0o700,exist_ok=True)
|
||||
if session!=BASE and case=='apt_configure_recovery':
|
||||
original=m.protected_json(STATE/case/'checkpoint-capacity-terminal.json')
|
||||
old_job=original['plan']['jobs'][0]
|
||||
old_result=cloud.Backend().call('jobs.get',dict(tenant_id=cloud.TENANT,job_id=old_job['job_id']))
|
||||
counts=prior.prior.evidence('gate',job_id=old_job['job_id'])
|
||||
require(original['not_started_capacity_block'] and old_result['state'] in ('expired','cancelled') and counts['permits']==0,
|
||||
'preserve original approved no-start; new attempt requires fresh UI approval')
|
||||
folder=cases/case
|
||||
if resume_unsubmitted:
|
||||
require(folder.is_dir() and {p.name for p in folder.iterdir()}=={'prepare.json'},'only an unsubmitted UI preparation may be archived')
|
||||
old=m.protected_json(folder/'prepare.json')
|
||||
require(old['case']==case and old['phase']=='prepare' and all(old[k]==target[k] for k in target),'same fixed prior preparation target')
|
||||
untouched=cloud.bootstrap.rpc('server.conf',unsubmitted_preparation,dict(test_result_id=old['request']['parameters']['apt_manifest']['test_result_id']))
|
||||
require(untouched==dict(persisted_plans=0),'existing plan must be reviewed, never silently replaced')
|
||||
archived=cases/(case+'.unsubmitted-ui-'+uuid.uuid4().hex)
|
||||
os.rename(folder,archived);m.write_private(archived/'not-submitted.json',untouched)
|
||||
# A maintenance restart only refreshes passive inventory from the preserved
|
||||
# agent; it never issues a package diagnostic or executes a job.
|
||||
deadline=time.monotonic()+90
|
||||
@@ -87,28 +124,30 @@ def accept(case):
|
||||
except m.GuardError:
|
||||
require(time.monotonic()<deadline,'fresh stored exact planning unavailable; no job submitted');time.sleep(3)
|
||||
request=recovery_request(device,target) if case=='apt_configure_recovery' else None
|
||||
prior.accept(case,request=request,folder_root=STATE)
|
||||
if request is not None and session!=BASE:
|
||||
request['reason']+='; prior fixed plan expired with zero grants at the old journal capacity; preserve it and authorize this fresh plan separately'
|
||||
prior.accept(case,request=request,folder_root=cases)
|
||||
|
||||
|
||||
def close():
|
||||
m=cloud.contract;require(prior.prior.evidence('gate')['unresolved']==0,'no unresolved attempt')
|
||||
record=m.protected_json(BASE/'recovery-policy.json')
|
||||
ready,session=runtime_session();record=m.protected_json(session/'recovery-policy.json')
|
||||
closed=cloud.bootstrap.rpc('server.conf',cloud_root_policy.close_policy,dict(expected_sha256=record['policy_sha256'],backup=record['backup']))
|
||||
m.write_private(BASE/'recovery-policy-closed.json',closed)
|
||||
ready=m.protected_json(BASE/'configure-runtime.json')
|
||||
m.write_private(session/'recovery-policy-closed.json',closed)
|
||||
endpoint=cloud.bootstrap.rpc('linux-vm.conf',prior.switch_endpoint,dict(id=str(uuid.uuid4()),enabled=False,
|
||||
agent_id=prior.prior.identity()['agent_id'],binary_sha256=ready['helper_sha256']))
|
||||
m.write_private(BASE/'recovery-endpoint-disabled.json',endpoint)
|
||||
m.write_private(session/'recovery-endpoint-disabled.json',endpoint)
|
||||
print('APT test limits closed and endpoint stopped; original histories retained')
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__);p.add_argument('action',choices=('enable','accept','finish','close'));p.add_argument('--case',choices=CASES)
|
||||
p.add_argument('--resume-unsubmitted-ui',action='store_true',help='Preserve a failed preparation only after proving no plan was persisted')
|
||||
args=p.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
|
||||
if args.action in ('accept','finish'):
|
||||
require(args.case is not None,'one fixed case required')
|
||||
if args.action=='accept':accept(args.case)
|
||||
else:prior.finish(args.case,folder_root=STATE)
|
||||
if args.action=='accept':accept(args.case,args.resume_unsubmitted_ui)
|
||||
else:prior.finish(args.case,folder_root=runtime_session()[1]/'cases')
|
||||
else:(enable if args.action=='enable' else close)()
|
||||
|
||||
|
||||
@@ -116,5 +155,5 @@ if __name__=='__main__':
|
||||
try:main()
|
||||
except Exception as error:
|
||||
message=str(error)
|
||||
print(message if message.startswith('remote bootstrap incomplete: ') else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
|
||||
print(message if message.startswith(('remote bootstrap incomplete: ','APT_UI_INCOMPLETE:source_lines=')) else 'CLOUD_APT_RECOVERY_INCOMPLETE:'+type(error).__name__)
|
||||
raise SystemExit(2) from None
|
||||
|
||||
Reference in New Issue
Block a user