fix: repair proven unsent sync chain with signed archived receipt
Rust development slice / linux (push) Failing after 7s

This commit is contained in:
2026-11-13 17:43:38 +01:00
parent 998f16fde4
commit 359233c64d
21 changed files with 3030 additions and 30 deletions
+13 -2
View File
@@ -25,6 +25,7 @@ def upgrade(payload):
base=Path('/var/lib/rmm-apt-transaction');state=Path('/var/lib/rmm-agent-remote')
ledger=Path('/var/lib/rmm-helper/consumption.sqlite');binary=Path('/usr/local/libexec/rmm-privileged-helper')
provider=Path('/usr/lib/rmm-apt-transaction/apt_transaction.py');metadata=base/'install-metadata.json'
agent_binary=Path('/usr/local/libexec/rmm-agent-remote')
dropin=Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf')
full=('[Service]\nEnvironment=RMM_HELPER_SOCKET=/run/rmm-helper-cloud/control.sock\n'
'Environment=RMM_HELPER_POLICY_FILE=/etc/rmm-remote/general-helper-proof.json\n'
@@ -52,6 +53,7 @@ def upgrade(payload):
guard()
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'permanent identity preserved')
require(sha(binary.read_bytes())==payload['old_helper_sha256'] and sha(provider.read_bytes())==payload['old_provider_sha256'],'reviewed old artifacts required')
require(sha(agent_binary.read_bytes())==payload['old_agent_sha256'],'reviewed permanent agent required')
require(dropin.read_text()==full and not dropin.is_symlink(),'known active endpoint drop-in required')
original=base/'bootstrap-attempt-rmm-apt-fixture-install-1.0.json'
retry=base/'bootstrap-retry-rmm-apt-fixture-install-1.0.json'
@@ -59,7 +61,9 @@ def upgrade(payload):
sha(retry.read_bytes())=='a118072812553dca1a8ffee84550be0b2d514d555ad7cbcfc4722675a211a0e3','original and retry tombstones required')
require(not list(base.glob('acceptance-*.json')),'no existing acceptance expected')
raw=base64.b64decode(payload['binary'],validate=True);code=base64.b64decode(payload['provider'],validate=True)
agent_raw=base64.b64decode(payload['agent_binary'],validate=True)
require(sha(raw)==payload['helper_sha256'] and sha(code)==payload['provider_sha256'],'reviewed new artifacts')
require(sha(agent_raw)==payload['agent_sha256'],'qualified sync-prevention agent')
backup=Path('/var/lib/rmm-bootstrap-history')/('cloud-readonly-reconcile-'+payload['id']);backup.mkdir(mode=0o700)
def create(path,data,mode=0o600):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
@@ -68,7 +72,9 @@ def upgrade(payload):
def save():
next_path=backup/'.report-next.json';create(next_path,json.dumps(report,indent=2).encode());os.replace(next_path,backup/'report.json')
save();staged=backup/'helper-next';create(staged,raw,0o755)
staged_agent=backup/'agent-next';create(staged_agent,agent_raw,0o755)
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged)])
run(['/lib64/ld-linux-x86-64.so.2','--list',str(staged_agent)])
run(['systemctl','stop','rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'])
fd=os.open(ledger.parent/'executor.lock',os.O_RDWR|os.O_NOFOLLOW);fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
try:
@@ -77,11 +83,11 @@ def upgrade(payload):
witnesses={str(p):sha(p.read_bytes()) for p in (original,retry,base/'native-readiness.json',Path('/etc/rmm/apt-helper.json'),Path('/etc/rmm-remote/apt-helper-proof.json'))}
old_meta=json.loads(metadata.read_text());require(old_meta['provider_sha256']==payload['old_provider_sha256'],'old installation metadata')
general=Path('/etc/rmm-remote/general-helper-proof.json').read_bytes()
for p in (binary,provider,metadata,dropin):
for p in (binary,agent_binary,provider,metadata,dropin):
target=backup/str(p).lstrip('/').replace('/','__')
require(not p.is_symlink() and p.stat().st_dev==backup.stat().st_dev,'same-filesystem versioned backup')
os.rename(p,target)
os.rename(staged,binary);create(provider,code,0o755)
os.rename(staged,binary);os.rename(staged_agent,agent_binary);create(provider,code,0o755)
create(metadata,json.dumps(dict(old_meta,provider_sha256=sha(code))).encode());create(dropin,reduced.encode(),0o644)
inspected=json.loads(run([str(binary),'--inspect-general-profile'],env=dict(os.environ,RMM_HELPER_CONFIG='/etc/rmm-remote/helper-general.json')))
require(inspected==json.loads(general),'general policy unchanged')
@@ -90,6 +96,7 @@ def upgrade(payload):
(ledger.stat().st_dev,ledger.stat().st_ino)==inode and guard()==original_unknown,'identity/journal/unknown evidence drift')
report.update(stage='installed_preserved',preserved_state=True,original_unknown_sha256=original_unknown,
helper_sha256=sha(raw),provider_sha256=sha(code),apt_endpoint_enabled=False);save()
report['agent_sha256']=sha(agent_raw);save()
finally:
fcntl.flock(fd,fcntl.LOCK_UN);os.close(fd)
run(['systemctl','daemon-reload']);run(['systemctl','start','rmm-helper-cloud.service','rmm-agent-remote.service'])
@@ -139,12 +146,16 @@ def main():
folders += [apt.prior.STATE/name for name in ('diagnosis','diagnosis-acquisition','diagnosis-hook','diagnosis-local-hook')]
expected=[m.protected_json(p/'checkpoint.json')['result']['result']['grant_id'] for p in folders]+[grant]
ident=str(uuid.uuid4());raw=(STATE/'helper-readonly-reconcile').read_bytes();code=(STATE/'provider-readonly-reconcile.py').read_bytes()
agent_raw=(STATE/'agent-sync-prevention').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
payload=dict(id=ident,agent_id=apt.prior.identity()['agent_id'],expected_grants=expected,unknown_grant=grant,
old_helper_sha256='91ffd13a4a0a6713bf6d1aa07ccc72e1bea737ce5533fbec87b40a9f274cf35c',
old_provider_sha256='465ac5ed61185dd72f4600c49e0da5b9da16f75aa3686f7c278f8667a6564945',
binary=base64.b64encode(raw).decode(),provider=base64.b64encode(code).decode(),
agent_binary=base64.b64encode(agent_raw).decode(),agent_sha256=hashlib.sha256(agent_raw).hexdigest(),old_agent_sha256=installed['binary_sha256'],
helper_sha256=hashlib.sha256(raw).hexdigest(),provider_sha256=hashlib.sha256(code).hexdigest())
report=cloud.bootstrap.rpc('linux-vm.conf',upgrade,payload);m.write_private(STATE/('install-'+ident+'.json'),report)
require(report['completed'] and report['agent_sha256']==payload['agent_sha256'],'qualified preserving installation')
m.write_private(cloud.ROOT/'devstate/remote-vm.json',dict(installed,binary_sha256=report['agent_sha256']))
print({k:report[k] for k in ('completed','stage','preserved_state','package_effects','apt_endpoint_enabled')})
+158
View File
@@ -0,0 +1,158 @@
#!/usr/bin/env python3
"""One signed development control-chain repair, never a job or grant repair."""
import argparse
import base64
import hashlib
import json
import os
from pathlib import Path
import uuid
import cloud_apt_reconcile_upgrade as upgrade
cloud=upgrade.cloud
require=cloud.require
STATE=cloud.ROOT/'devstate/checkpoint-chain-repair'
def prepared_json(path):
"""The bounded full historical witness exceeds ordinary 1 MiB DTOs."""
import stat
fd=os.open(path,os.O_RDONLY|os.O_NOFOLLOW)
with os.fdopen(fd,'r') as stream:
info=os.fstat(stream.fileno())
require(stat.S_ISREG(info.st_mode) and info.st_uid==os.getuid() and
stat.S_IMODE(info.st_mode)==0o600 and info.st_nlink==1 and info.st_size<=45*1024*1024,
'private bounded repair witness')
return json.load(stream)
def prepare_vm(payload):
import re
import shutil
import sqlite3
os.umask(0o077)
ident=str(uuid.UUID(payload['id']));state=Path('/var/lib/rmm-agent-remote');helper=Path('/var/lib/rmm-helper')
require(hashlib.sha256(Path('/usr/local/libexec/rmm-agent-remote').read_bytes()).hexdigest()==payload['agent_sha256'],'reviewed permanent agent')
require(json.loads((state/'identity.json').read_text())['agent_id']==payload['agent_id'],'fixed permanent device')
base=Path('/var/lib/rmm-apt-transaction');result=json.loads((base/'active/sandbox-result.json').read_text())
require(result['apt_exit_code']==0 and not result['reboot_required'],'ended native child required; never stop package manager')
root=Path('/var/lib/rmm-bootstrap-history')/('checkpoint-chain-repair-'+ident);root.mkdir(mode=0o700)
def create(path,data,mode=0o600):
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,mode)
with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno());os.fchmod(f.fileno(),mode)
tool=base64.b64decode(payload['tool'],validate=True)
require(hashlib.sha256(tool).hexdigest()==payload['tool_sha256'],'qualified repair tool')
create(root/'checkpoint-repair',tool,0o755)
subprocess.run(['/lib64/ld-linux-x86-64.so.2','--list',str(root/'checkpoint-repair')],capture_output=True,check=True,timeout=10)
binding=json.loads((state/'binding.json').read_text())
config=json.loads(Path('/etc/rmm-remote/helper-general.json').read_text())
pin=json.loads(Path('/etc/rmm-remote/helper-signing-key.json').read_text())
require(all(binding[k]==config['binding'][k] for k in ('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')) and
binding['public_key_hex']==pin['public_key_hex'],'independent root trust binding')
create(root/'binding.json',json.dumps(binding).encode())
units=('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')
before={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True,check=True).stdout.strip().decode() for name in units}
subprocess.run(['systemctl','stop',*units],capture_output=True,check=True,timeout=25)
prepared=False;report=dict(id=ident,vm_dir=str(root),tool_sha256=payload['tool_sha256'],prepared=False,package_effects=False)
try:
paths=(state/'journal.sqlite',state/'consumption.sqlite',helper/'consumption.sqlite')
hashes={str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}
for index,p in enumerate(paths):
with sqlite3.connect(p.as_uri()+'?mode=ro',uri=True) as db:
require(db.execute('PRAGMA integrity_check').fetchone()[0]=='ok','preserved database integrity')
create(root/('database-before-'+str(index)+'.sqlite'),p.read_bytes())
process=subprocess.run([str(root/'checkpoint-repair'),'prepare',str(root/'binding.json'),str(root/'evidence.json')],
env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',RMM_CHECKPOINT_REPAIR_PROFILE='development'),capture_output=True,timeout=25)
create(root/'prepare.stderr',process.stderr)
require(all(hashlib.sha256(p.read_bytes()).hexdigest()==hashes[str(p)] for p in paths),'prepare must not change databases')
report.update(exit_code=process.returncode,error_codes=[x.decode() for x in re.findall(rb'\b(?:REPAIR_|CHECKPOINT_)[A-Z0-9_]+\b',process.stderr)],preserved_database_bytes=True)
if process.returncode==0:
raw=(root/'evidence.json').read_bytes();require(len(raw)<=32*1024*1024,'bounded evidence')
report.update(prepared=True,evidence_sha256=hashlib.sha256(raw).hexdigest(),evidence_b64=base64.b64encode(raw).decode())
prepared=True
finally:
create(root/'report.json',json.dumps({k:v for k,v in report.items() if k!='evidence_b64'}).encode())
if not prepared:
for unit in units:
if before[unit]=='active':subprocess.run(['systemctl','start',unit],capture_output=True,timeout=20)
return report
def issue_server(payload):
root=Path('/opt/rust-discovery')
raw=base64.b64decode(payload['evidence_b64'],validate=True)
require(hashlib.sha256(raw).hexdigest()==payload['evidence_sha256'],'original exact witness bytes')
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
'exec','-T','api','/usr/local/bin/rmm-server','repair-unsent-sync-request']
result=subprocess.run(cmd,input=raw,capture_output=True,timeout=30)
require(result.returncode==0,'signed control receipt issuance failed')
value=json.loads(result.stdout);require(set(value)=={'payload','signature'},'only signed control receipt output')
return value
def apply_vm(payload):
import re
os.umask(0o077)
ident=str(uuid.UUID(payload['id']));root=Path('/var/lib/rmm-bootstrap-history')/('checkpoint-chain-repair-'+ident)
require(hashlib.sha256((root/'evidence.json').read_bytes()).hexdigest()==payload['evidence_sha256'] and
hashlib.sha256((root/'checkpoint-repair').read_bytes()).hexdigest()==payload['tool_sha256'],'original repair witnesses/tool')
signed=json.dumps(payload['receipt'],separators=(',',':')).encode();path=root/'receipt.json'
if path.exists():require(path.read_bytes()==signed,'identical existing receipt only')
else:
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'wb') as f:f.write(signed);f.flush();os.fsync(f.fileno())
result=subprocess.run([str(root/'checkpoint-repair'),'apply',str(root/'binding.json'),str(root/'evidence.json'),str(path)],
env=dict(PATH='/usr/sbin:/usr/bin:/sbin:/bin',RMM_CHECKPOINT_REPAIR_PROFILE='development'),capture_output=True,timeout=25)
fd=os.open(root/'apply.stderr',os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'wb') as f:f.write(result.stderr)
return dict(applied=result.returncode==0,exit_code=result.returncode,error_codes=[x.decode() for x in re.findall(rb'\b(?:REPAIR_|CHECKPOINT_)[A-Z0-9_]+\b',result.stderr)],
ready_lease_issued=False,jobs_created=False,grants_created=False)
def status_vm(payload):
import sqlite3
result={}
for name,path,table in (('agent','/var/lib/rmm-agent-remote/journal.sqlite','reconcile_meta'),
('helper','/var/lib/rmm-helper/consumption.sqlite','helper_sync_meta')):
with sqlite3.connect(Path(path).as_uri()+'?mode=ro',uri=True) as db:
tables={r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
result[name]=dict(faults=db.execute('SELECT count(*) FROM '+table+' WHERE fault IS NOT NULL').fetchone()[0],
receipts=db.execute('SELECT count(*) FROM checkpoint_repair_receipts').fetchone()[0] if 'checkpoint_repair_receipts' in tables else 0,
integrity_ok=db.execute('PRAGMA integrity_check').fetchone()[0]=='ok')
if name=='helper':
result[name].update(consumed=db.execute('SELECT count(*) FROM consumed').fetchone()[0],
unresolved=db.execute('SELECT count(*) FROM consumed WHERE result IS NULL').fetchone()[0])
result['services']={name:subprocess.run(['systemctl','show',name,'-p','ActiveState','--value'],capture_output=True,check=True).stdout.strip().decode()
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service')}
return result
def main():
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=('prepare','issue','apply'));parser.add_argument('--id')
args=parser.parse_args();os.umask(0o077);STATE.mkdir(mode=0o700,exist_ok=True);m=cloud.contract
if args.action=='prepare':
require(upgrade.apt.prior.evidence('gate')==dict(jobs=13,outbox=13,permits=12,unresolved=1),'exact no-new-execution gate')
ident=str(uuid.uuid4());folder=STATE/ident;folder.mkdir(mode=0o700)
tool=(upgrade.STATE/'checkpoint-repair').read_bytes();installed=m.protected_json(cloud.ROOT/'devstate/remote-vm.json')
result=cloud.bootstrap.rpc('linux-vm.conf',prepare_vm,dict(id=ident,agent_id=installed['identity']['agent_id'],agent_sha256=installed['binary_sha256'],
tool_sha256=hashlib.sha256(tool).hexdigest(),tool=base64.b64encode(tool).decode()))
m.write_private(folder/'prepared.json',result)
print({k:result.get(k) for k in ('id','prepared','exit_code','error_codes','preserved_database_bytes')})
require(result['prepared'],'stored witness preparation incomplete; no fault changes')
else:
ident=str(uuid.UUID(args.id));folder=STATE/ident;prepared=prepared_json(folder/'prepared.json');require(prepared['prepared'],'qualified original witnesses')
if args.action=='issue':
result=cloud.bootstrap.rpc('server.conf',issue_server,prepared);m.write_private(folder/'receipt.json',result)
print(dict(signed_control_receipt=True,execution_permission=False))
else:
result=cloud.bootstrap.rpc('linux-vm.conf',apply_vm,dict(prepared,receipt=m.protected_json(folder/'receipt.json')))
m.write_private(folder/'applied.json',result);print(result)
if __name__=='__main__':
try:main()
except Exception as error:
message=str(error)
if message.startswith('remote bootstrap incomplete: '):print(message)
else:print('CLOUD_CHECKPOINT_REPAIR_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None
+3 -3
View File
@@ -124,7 +124,7 @@ def deploy(payload):
def main():
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('action', choices=('stage','status','deploy')); parser.add_argument('--id')
parser.add_argument('--preserve-known-unknown', action='store_true', help='Windows inventory migration only; preserve exact fenced APT evidence')
parser.add_argument('--preserve-known-unknown', action='store_true', help='preserve the exact fenced APT attempt while upgrading its control path')
args = parser.parse_args(); os.umask(0o077); STATE.mkdir(mode=0o700, parents=True, exist_ok=True)
if args.action == 'stage':
names = ['Cargo.toml','Cargo.lock','rust-toolchain.toml','deploy/Dockerfile','deploy/linux/rmm-apt-helper.service',
@@ -160,7 +160,7 @@ def main():
before['ledger']['result'] is None and before['ledger']['outcome']['error_code'] == 'DPKG_INCONSISTENT', 'preserve exact native unknown')
closed = cloud.contract.protected_json(apt.STATE/'policy-closed.json')
payload['closed_policy_sha256'] = closed['policy_sha256']
require((cloud.ROOT/'migrations/0013_windows_inventory_scope.sql').is_file(), 'additive Windows migration only')
require((cloud.ROOT/'migrations/0013_windows_inventory_scope.sql').is_file(), 'retain existing additive schema')
else: require(gate['unresolved'] == 0, 'unresolved cloud job; no server switch')
if args.action == 'deploy':
stream = io.BytesIO()
@@ -177,7 +177,7 @@ def main():
gate_after = cloud.bootstrap.rpc('server.conf', cloud.server, dict(kind='gate', tenant_id=cloud.TENANT, agent_id=target['agent_id']))
current = cloud.Backend().call('jobs.get', dict(tenant_id=cloud.TENANT, job_id=job['job_id']))
require(before == after and gate_after == gate and current['state'] == 'unknown' and current['attempt_id'] == job['attempt_id'],
'unknown evidence or execution artifacts changed during Windows migration')
'unknown evidence or execution artifacts changed during control-path upgrade')
cloud.contract.write_private(STATE/(ident+'.unknown-preserved.json'), dict(evidence=after, gate=gate_after, result=current))
result['unknown_attempt_preserved'] = True
print(result)