fix: retain sync anchors and verify bound APT partial postconditions
Rust development slice / linux (push) Failing after 6s

This commit is contained in:
2026-11-13 17:20:50 +01:00
parent 435c4453fe
commit 998f16fde4
13 changed files with 1469 additions and 16 deletions
+154
View File
@@ -0,0 +1,154 @@
#!/usr/bin/env python3
"""Preserving helper upgrade for the original APT bound readonly recheck.
APT execution endpoint stays stopped. General helper snapshots may prove the
existing failed partial postcondition; only existing signed Sync closes it.
No package command, new grant, approval or consumption reset exists here.
"""
import base64
import hashlib
import json
import os
from pathlib import Path
import uuid
import cloud_apt_retry as apt
cloud=apt.cloud
require=cloud.require
STATE=cloud.ROOT/'devstate/cloud-apt-recovery'
def upgrade(payload):
import fcntl
import sqlite3
os.umask(0o077)
base=Path('/var/lib/rmm-apt-transaction');state=Path('/var/lib/rmm-agent-remote')
ledger=Path('/var/lib/rmm-helper/consumption.sqlite');binary=Path('/usr/local/libexec/rmm-privileged-helper')
provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py');metadata=base/'install-metadata.json'
dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
full=('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
'Environment=RMM_APT_HELPER_SOCKET=/run/rmm-apt-helper/helper.sock\n'
'Environment=RMM_APT_HELPER_PROOF_FILE=/etc/rmm-remote/apt-helper-proof.json\n'
'Environment=RMM_LOG_UNITS_FILE=/etc/rmm-remote/log-units.json\n')
reduced=''.join(line+'\n' for line in full.splitlines() if not line.startswith('Environment=RMM_APT_HELPER_'))
def run(args,**kw):return subprocess.run(args,capture_output=True,check=True,timeout=20,**kw).stdout
def sha(data):return hashlib.sha256(data).hexdigest()
def manifest(path):
result={}
for p in path.rglob('*'):
require(not p.is_symlink(),'preserved state symlink')
if p.is_file():result[str(p.relative_to(path))]=sha(p.read_bytes())
return result
def guard():
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','ledger integrity')
require({r[0] for r in db.execute('SELECT grant_id FROM consumed')}==set(payload['expected_grants']),'exact preserved grants')
rows=db.execute('SELECT c.grant_id,e.outcome FROM consumed c JOIN execution_evidence e USING(grant_id) WHERE c.result IS NULL').fetchall()
require(len(rows)==1 and rows[0][0]==payload['unknown_grant'],'one exact existing unknown')
outcome=json.loads(rows[0][1]);require(outcome['state']=='unknown' and outcome['error_code']=='DPKG_INCONSISTENT','exact known partial outcome')
require(db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0]==0,'no sticky faults')
return sha(rows[0][1])
guard()
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'permanent identity preserved')
require(sha(binary.read_bytes())==payload['old_helper_sha256'] and sha(provider.read_bytes())==payload['old_provider_sha256'],'reviewed old artifacts required')
require(dropin.read_text()==full and not dropin.is_symlink(),'known active endpoint drop-in required')
original=base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json'
retry=base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json'
require(sha(original.read_bytes())=='ec84aa4df8065f0b54b282bd273a6f342f07a6d47312289608389d05c7fb6053' and
sha(retry.read_bytes())=='a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3','original and retry tombstones required')
require(not list(base.glob('acceptance-*.json')),'no existing acceptance expected')
raw=base64.b64decode(payload['binary'],validate=True);code=base64.b64decode(payload['provider'],validate=True)
require(sha(raw)==payload['helper_sha256'] and sha(code)==payload['provider_sha256'],'reviewed new artifacts')
backup=Path('/var/lib/rmm-bootstrap-history')/('cloud-readonly-reconcile-'+payload['id']);backup.mkdir(mode=0o700)
def create(path,data,mode=0o600):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),mode)
report=dict(stage='validated',completed=False,package_effects=False,backup=str(backup))
def save():
next_path=backup/'.report-next.json';create(next_path,json.dumps(report,indent=2).encode());os.replace(next_path,backup/'report.json')
save();staged=backup/'helper-next';create(staged,raw,0o755)
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)])
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
fd=os.open(ledger.parent/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
try:
original_unknown=guard();inode=(ledger.stat().st_dev,ledger.stat().st_ino)
preserved={str(p):manifest(p) for p in (state,Path('/var/lib/rmm-agent'),ledger.parent,base/'active',base/'history',base/'archives',base/'baselines',base/'log',base/'state')}
witnesses={str(p):sha(p.read_bytes()) for p in (original,retry,base/'native-readiness.json',Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'))}
old_meta=json.loads(metadata.read_text());require(old_meta['provider_sha256']==payload['old_provider_sha256'],'old installation metadata')
general=Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
for p in (binary,provider,metadata,dropin):
target=backup/str(p).lstrip('/').replace('/','__')
require(not p.is_symlink() and p.stat().st_dev==backup.stat().st_dev,'same-filesystem versioned backup')
os.rename(p,target)
os.rename(staged,binary);create(provider,code,0o755)
create(metadata,json.dumps(dict(old_meta,provider_sha256=sha(code))).encode());create(dropin,reduced.encode(),0o644)
inspected=json.loads(run([str(binary),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(inspected==json.loads(general),'general policy unchanged')
require(all(manifest(Path(p))==values for p,values in preserved.items()) and
all(sha(Path(p).read_bytes())==value for p,value in witnesses.items()) and
(ledger.stat().st_dev,ledger.stat().st_ino)==inode and guard()==original_unknown,'identity/journal/unknown evidence drift')
report.update(stage='installed_preserved',preserved_state=True,original_unknown_sha256=original_unknown,
helper_sha256=sha(raw),provider_sha256=sha(code),apt_endpoint_enabled=False);save()
finally:
fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
run(['systemctl','daemon-reload']);run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
report.update(stage='started_bound_readonly_recheck',completed=True);save()
return report
def result_evidence(payload):
import sqlite3
ledger=Path('/var/lib/rmm-helper/consumption.sqlite')
with sqlite3.connect(ledger.as_uri()+'?mode=ro',uri=True) as db:
row=db.execute('SELECT result FROM consumed WHERE grant_id=?',(payload['grant_id'],)).fetchone()
tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
proof=[]
if 'apt_partial_reconciliations' in tables:
columns=[r[1] for r in db.execute('PRAGMA table_info(apt_partial_reconciliations)')]
proof=[dict(zip(columns,r)) for r in db.execute('SELECT * FROM apt_partial_reconciliations')]
# Opaque witness bytes remain private, JSON-safe for protected file.
for item in proof:
for k,v in list(item.items()):
if isinstance(v,bytes):item[k]=dict(sha256=hashlib.sha256(v).hexdigest(),size=len(v))
return dict(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0],
faults=db.execute('SELECT count(*) FROM helper_sync_meta WHERE fault IS NOT NULL').fetchone()[0],
fault_codes=[r[0] for r in db.execute('SELECT fault FROM helper_sync_meta WHERE fault IS NOT NULL')],
original_result=json.loads(row[0]) if row and row[0] else None,proof=proof,
apt_active=subprocess.run(['systemctl','is-active','rmm-apt-helper.service'],capture_output=True).stdout.strip().decode())
def installation_status(payload):
root=Path('/var/lib/rmm-bootstrap-history')
reports=[]
for p in sorted(root.glob('cloud-readonly-reconcile-*/report.json')):
reports.append(json.loads(p.read_text()))
services={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True).stdout.strip().decode()
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')}
hashes={name:hashlib.sha256(Path(name).read_bytes()).hexdigest() for name in
('/usr/local/libexec/rmm-privileged-helper','/usr/lib/rmm-apt-transaction/apt_transaction.py')}
return dict(reports=reports,services=services,hashes=hashes)
def main():
m=cloud.contract;os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True)
checkpoint=m.protected_json(apt.STATE/'apt_install_retry/checkpoint.json');grant=checkpoint['result']['result']['grant_id']
require(apt.prior.evidence('gate')==dict(jobs=13,outbox=13,permits=12,unresolved=1),'exact closed-policy unknown gate')
folders=[cloud.ROOT/'devstate/cloud-root/cases'/name for name in ('service_setup','service_start','service_stop','apt_native_install')]
folders += [apt.prior.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
expected=[m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'] for p in folders]+[grant]
ident=str(uuid.uuid4());raw=(STATE/'helper-readonly-reconcile').read_bytes();code=(STATE/'provider-readonly-reconcile.py').read_bytes()
payload=dict(id=ident,agent_id=apt.prior.identity()['agent_id'],expected_grants=expected,unknown_grant=grant,
old_helper_sha256='91ffd13a4a0a6713bf6d1aa07ccc72e1bea737ce5533fbec87b40a9f274cf35c',
old_provider_sha256='465ac5ed61185dd72f4600c49e0da5b9da16f75aa3686f7c278f8667a6564945',
binary=base64.b64encode(raw).decode(),provider=base64.b64encode(code).decode(),
helper_sha256=hashlib.sha256(raw).hexdigest(),provider_sha256=hashlib.sha256(code).hexdigest())
report=cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload);m.write_private(STATE/('install-'+ident+'.json'),report)
print({k:report[k] for k in ('completed','stage','preserved_state','package_effects','apt_endpoint_enabled')})
if __name__=='__main__':
try:main()
except Exception as error:
print('CLOUD_APT_RECONCILE_UPGRADE_INCOMPLETE:'+type(error).__name__);raise SystemExit(2) from None
+67
View File
@@ -0,0 +1,67 @@
"""Read-only stored checkpoint metadata; no effects or trust-state reset."""
def vm_chain(payload):
import sqlite3
def wire(raw):
result={};i=0
def number():
nonlocal i
value=0;shift=0
while True:
require(i<len(raw) and shift<70,'stored protobuf bounds');b=raw[i];i+=1;value|=(b&127)<<shift
if not b&128:return value
shift+=7
while i<len(raw):
key=number();field=key>>3;kind=key&7
if kind==0:value=number()
elif kind==2:
length=number();require(i+length<=len(raw),'stored protobuf length');value=raw[i:i+length];i+=length
else:raise RuntimeError('stored protobuf wire type')
result.setdefault(field,[]).append(value)
return result
def request(raw):
fields=wire(raw);result={}
for n,k in ((1,'request_id'),(2,'journal_id'),(4,'previous_decision_id'),(9,'helper_journal_id'),(11,'helper_previous_decision_id'),(12,'previous_request_id')):
result[k]=fields.get(n,[b''])[0].decode('ascii')
for n,k in ((3,'checkpoint'),(10,'helper_checkpoint')):result[k]=fields.get(n,[0])[0]
return result
def decision(value):
names=('decision_id','request_id','journal_id','checkpoint','helper_journal_id','helper_checkpoint','ready','helper_ready','reason','issued_at','connection_generation')
return {k:value.get(k) for k in names}
result={}
path=Path('/var/lib/rmm-agent-remote/journal.sqlite')
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','agent journal integrity')
names=[r[1] for r in db.execute('PRAGMA table_info(reconcile_meta)')]
result['agent_meta']=dict(zip(names,db.execute('SELECT * FROM reconcile_meta').fetchone()))
requests=[]
for ident,generation,raw,response in db.execute('SELECT id,generation,request,response FROM reconcile_requests ORDER BY rowid DESC LIMIT 12'):
item=dict(id=ident,generation=generation,request=request(raw),response_sha256=hashlib.sha256(response).hexdigest() if response else None)
if response:item['decision']=decision(json.loads(wire(response)[1][0]))
requests.append(item)
result['agent_requests']=requests
path=Path('/var/lib/rmm-helper/consumption.sqlite')
with sqlite3.connect(path.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','helper ledger integrity')
names=[r[1] for r in db.execute('PRAGMA table_info(helper_sync_meta)')]
result['helper_meta']=dict(zip(names,db.execute('SELECT * FROM helper_sync_meta').fetchone()))
decisions=[]
for ident,checkpoint,response,raw in db.execute('SELECT decision_id,checkpoint,response,request FROM helper_sync_decisions ORDER BY rowid DESC LIMIT 12'):
signed=json.loads(response);value=json.loads(bytes(signed['decision_bytes']))
decisions.append(dict(id=ident,checkpoint=checkpoint,request=request(raw),decision=decision(value)))
result['helper_decisions']=decisions
return result
def server_chain(payload):
root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id']))
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
scope="agent_id='"+agent+"' AND tenant_id='11111111-1111-4111-8111-111111111111'"
query=("SELECT json_build_object('meta',(SELECT row_to_json(s) FROM agent_sync_state s WHERE "+scope+"),"
"'decisions',(SELECT json_agg(row_to_json(d)) FROM (SELECT request_id,decision_id,journal_id,checkpoint,connection_generation,request_hash,created_at,"
"convert_from(payload,'UTF8')::jsonb->>'reason' AS reason,convert_from(payload,'UTF8')::jsonb->>'ready' AS ready,"
"convert_from(payload,'UTF8')::jsonb->>'helper_ready' AS helper_ready FROM sync_checkpoints WHERE "+scope+" ORDER BY created_at DESC LIMIT 20) d));")
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True).stdout.strip()
return json.loads(raw)