fix: accept exact recovery inventory and preserve native durable replay
Rust development slice / linux (push) Failing after 6s
Rust development slice / linux (push) Failing after 6s
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
"""Read preserved inventory identity/hash only; no collection or queue edits."""
|
||||
|
||||
|
||||
def vm_pending(payload):
|
||||
import sqlite3
|
||||
def wire(raw):
|
||||
result={};parts=[];i=0
|
||||
def number():
|
||||
nonlocal i
|
||||
value=shift=0
|
||||
while True:
|
||||
require(i<len(raw) and shift<70,'bounded preserved protobuf')
|
||||
b=raw[i];i+=1;value|=(b&127)<<shift
|
||||
if not b&128:return value
|
||||
shift+=7
|
||||
while i<len(raw):
|
||||
start=i;key=number();field=key>>3;kind=key&7
|
||||
if kind==0:value=number()
|
||||
elif kind==2:
|
||||
length=number();require(i+length<=len(raw),'bounded protobuf field');value=raw[i:i+length];i+=length
|
||||
else:raise RuntimeError('unexpected preserved protobuf type')
|
||||
result[field]=value;parts.append((field,raw[start:i]))
|
||||
return result,parts
|
||||
with sqlite3.connect('file:/var/lib/rmm-agent-remote/journal.sqlite?mode=ro',uri=True) as db:
|
||||
events=[];other=0
|
||||
for ident,raw in db.execute('SELECT id,payload FROM job_events'):
|
||||
require(len(raw)<=1024*1024,'bounded durable message');e,parts=wire(raw)
|
||||
if 13 not in e:other+=1;continue
|
||||
snap,_=wire(e[13]);data=json.loads(snap[6]);require(hashlib.sha256(snap[6]).digest()==snap[7],'original data hash')
|
||||
require(e[3].decode()==ident,'original message identity')
|
||||
events.append(dict(message_id=ident,agent_id=e[4].decode(),snapshot_id=snap[1].decode(),
|
||||
observed_at_ms=e[9],sequence=e[8],data_sha256=hashlib.sha256(snap[6]).hexdigest(),
|
||||
payload_sha256=hashlib.sha256(b''.join(raw for number,raw in parts if number!=6)).hexdigest(),
|
||||
recovery_scope_present=any(o.get('scope')=='linux.apt.recovery-planning' for o in data.get('observations',[]))))
|
||||
return dict(events=events,other_unacked=other)
|
||||
|
||||
|
||||
def server_stored(payload):
|
||||
root=Path('/opt/rust-discovery');agent=str(uuid.UUID(payload['agent_id']));message=str(uuid.UUID(payload['message_id']))
|
||||
query="SELECT row_to_json(x) FROM (SELECT message_id,snapshot_id,payload_sha256,(extract(epoch FROM observed_at)*1000)::bigint AS observed_at_ms FROM observations WHERE tenant_id='11111111-1111-4111-8111-111111111111' AND agent_id='"+agent+"' AND message_id='"+message+"') x;"
|
||||
cmd=['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),
|
||||
'exec','-T','postgres','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm']
|
||||
raw=subprocess.run(cmd,input=query.encode(),capture_output=True,check=True,timeout=15).stdout.strip()
|
||||
return json.loads(raw) if raw else {}
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Operational process state for the three project services; no customer scan."""
|
||||
|
||||
|
||||
def service_processes(payload):
|
||||
result={}
|
||||
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service'):
|
||||
values=subprocess.run(['systemctl','show',name,'-p','ActiveState','-p','SubState','-p','MainPID','-p','NRestarts'],
|
||||
capture_output=True,check=True,timeout=10).stdout.decode()
|
||||
fields=dict(line.split('=',1) for line in values.splitlines() if '=' in line)
|
||||
pid=int(fields['MainPID']);processes=[]
|
||||
# Kernel scheduling metadata only; never command lines, environment,
|
||||
# descriptors, credentials, arbitrary process memory or customer logs.
|
||||
if pid:
|
||||
pending=[pid]
|
||||
while pending and len(processes)<32:
|
||||
current=pending.pop(0);root=Path('/proc')/str(current)
|
||||
if not root.exists():continue
|
||||
processes.append(dict(pid=current,comm=(root/'comm').read_text().strip(),wchan=(root/'wchan').read_text().strip(),
|
||||
state=next((s.split(':',1)[1].strip() for s in (root/'status').read_text().splitlines() if s.startswith('State:')),None)))
|
||||
processes[-1]['threads']=[dict(tid=int(p.name),comm=(p/'comm').read_text().strip(),wchan=(p/'wchan').read_text().strip())
|
||||
for p in sorted((root/'task').iterdir())[:32] if p.is_dir()]
|
||||
children=root/'task'/str(current)/'children'
|
||||
if children.exists():pending.extend(int(s) for s in children.read_text().split())
|
||||
result[name]=dict(service=fields,processes=processes)
|
||||
return result
|
||||
@@ -51,15 +51,20 @@ def stage(payload):
|
||||
os.replace(next_path, dest)
|
||||
# Build asynchronously. Captured compiler output remains root-private.
|
||||
worker = backup/'build.py'
|
||||
image_tag='rust-discovery-server:upgrade-'+payload['id']
|
||||
worker.write_text('''import json,os,subprocess
|
||||
from pathlib import Path
|
||||
os.umask(0o077)
|
||||
root=Path('/opt/rust-discovery'); here=Path(__file__).parent
|
||||
record=json.loads((here/'source.json').read_text())
|
||||
with (here/'build.log').open('xb') as log:
|
||||
result=subprocess.run(['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml'),'build','migrate'],stdout=log,stderr=subprocess.STDOUT)
|
||||
(here/'build-result.json').write_text(json.dumps({'exit_code':result.returncode}))
|
||||
result=subprocess.run(['docker','build','--file',str(root/'deploy/Dockerfile'),'--tag',record['image_tag'],str(root)],stdout=log,stderr=subprocess.STDOUT)
|
||||
outcome={'exit_code':result.returncode}
|
||||
if result.returncode==0:
|
||||
outcome['image_id']=subprocess.run(['docker','image','inspect',record['image_tag'],'--format','{{.Id}}'],capture_output=True,check=True).stdout.decode().strip()
|
||||
(here/'build-result.json').write_text(json.dumps(outcome))
|
||||
'''); worker.chmod(0o600)
|
||||
record = dict(id=payload['id'], source_sha256=payload['sha256'], files=manifest, previous=previous, old_image=old_image, backup=str(backup))
|
||||
record = dict(id=payload['id'], source_sha256=payload['sha256'], files=manifest, previous=previous, old_image=old_image, backup=str(backup),image_tag=image_tag)
|
||||
(backup/'source.json').write_text(json.dumps(record)); (backup/'source.json').chmod(0o600)
|
||||
unit = 'rmm-source-build-'+payload['id']
|
||||
result = subprocess.run(['systemd-run','--unit='+unit,'--property=UMask=0077','--property=WorkingDirectory='+str(root),
|
||||
@@ -72,8 +77,13 @@ def status(payload):
|
||||
root = Path('/opt/rust-discovery'); backup = root/'devstate'/('source-upgrade-'+payload['id'])
|
||||
record = json.loads((backup/'source.json').read_text())
|
||||
result = backup/'build-result.json'
|
||||
unit=subprocess.run(['systemctl','show','rmm-source-build-'+payload['id'],'-p','ActiveState','-p','SubState','-p','ExecMainStatus'],capture_output=True,check=True,timeout=10).stdout.decode()
|
||||
current=subprocess.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}'],capture_output=True,check=True,timeout=10).stdout.decode().strip()
|
||||
log=backup/'build.log'
|
||||
return dict(id=payload['id'], finished=result.exists(), exit_code=json.loads(result.read_text())['exit_code'] if result.exists() else None,
|
||||
source_unchanged=all(hashlib.sha256((root/name).read_bytes()).hexdigest() == sha for name,sha in record['files'].items()))
|
||||
source_unchanged=all(hashlib.sha256((root/name).read_bytes()).hexdigest() == sha for name,sha in record['files'].items()),
|
||||
service=dict(line.split('=',1) for line in unit.splitlines() if '=' in line),build_log_bytes=log.stat().st_size if log.exists() else 0,
|
||||
build_has_separate_image_tag='image_tag' in record,runtime_tag_unchanged=current==record['old_image'] if not (backup/'deploy-result.json').exists() else None)
|
||||
|
||||
|
||||
def deploy(payload):
|
||||
@@ -81,7 +91,8 @@ def deploy(payload):
|
||||
import tarfile
|
||||
root = Path('/opt/rust-discovery'); backup = root/'devstate'/('source-upgrade-'+payload['id'])
|
||||
record = json.loads((backup/'source.json').read_text())
|
||||
require(json.loads((backup/'build-result.json').read_text())['exit_code'] == 0, 'build must pass')
|
||||
built=json.loads((backup/'build-result.json').read_text())
|
||||
require(built['exit_code'] == 0, 'build must pass')
|
||||
require(all(hashlib.sha256((root/name).read_bytes()).hexdigest() == sha for name,sha in record['files'].items()), 'source drift')
|
||||
require(not (backup/'deploy-result.json').exists(), 'deployment already recorded')
|
||||
if payload.get('closed_policy_sha256'):
|
||||
@@ -113,6 +124,12 @@ def deploy(payload):
|
||||
with os.fdopen(fd, 'wb') as f: f.write(data); f.flush(); os.fsync(f.fileno()); os.fchmod(f.fileno(), 0o644)
|
||||
os.replace(next_path, dest)
|
||||
cmd = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')]
|
||||
if 'image_tag' in record:
|
||||
inspected=subprocess.run(['docker','image','inspect',record['image_tag'],'--format','{{.Id}}'],capture_output=True,check=True,timeout=10).stdout.decode().strip()
|
||||
require(inspected==built['image_id'] and inspected.startswith('sha256:'),'fixed qualified image required')
|
||||
# Only this explicit, gated deployment may move the runtime tag. A
|
||||
# concurrently finishing source build cannot alter running services.
|
||||
subprocess.run(['docker','tag',inspected,'rust-discovery-server:local'],capture_output=True,check=True,timeout=10)
|
||||
# Existing migration image runs without rebuild; preserves PG volume/PKI.
|
||||
migrate = subprocess.run(cmd+['run','--rm','--no-deps','migrate'], capture_output=True, timeout=25)
|
||||
require(migrate.returncode == 0, 'migration failed')
|
||||
|
||||
Reference in New Issue
Block a user