Files
rust-discovery/tests/backend_sync.py
T
BuTzZ 6c3203d725
Rust development slice / linux (push) Failing after 1m10s
feat: snapshot Linux-first RMM MVP and pause handover
2026-11-12 22:06:25 +01:00

165 lines
13 KiB
Python

#!/usr/bin/env python3
"""Real mTLS sync/signature/checkpoint/key/inventory denials, no approvals/effects."""
import datetime
import hashlib
import json
import os
from pathlib import Path
import ssl
import subprocess
import tempfile
import uuid
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
from local_integration import LocalHTTPS, CA, STATE, A, B, field, envelope, token, sql, check
from backend_transport import fields, batch, csr, context, exchange
from backend_contract import request
def get(path, ctx, headers=None):
c=LocalHTTPS("rmm-dev.internal",18443,context=ctx,timeout=10)
c.request("GET",path,headers=headers or {})
r=c.getresponse(); result=(r.status,r.read());c.close();return result
def verify(_folder, key, payload, signature):
# Host CLI is OpenSSL 1.1.1, whose pkeyutl lacks -rawin. Verify with the
# installed independent Python Ed25519 implementation, not a skipped check.
try:
Ed25519PublicKey.from_public_bytes(bytes.fromhex(key["public_key_hex"])).verify(signature,payload)
return True
except (InvalidSignature,ValueError):
return False
class Flow:
def __init__(self, agent, ctx):
self.agent,self.ctx,self.sequence,self.generation=agent,ctx,0,0
def send(self, tag, body):
self.sequence+=1
return exchange("/agent/v1/messages",batch(envelope(self.agent,self.sequence,self.generation,tag,body)),self.ctx)
def hello(self):
self.generation=0
status,raw=self.send(10,field(1,"sync-fixture")+field(2,"rmm.sync.v1")+field(2,"run_script"))
assert status==200,"sync-aware hello failed"
self.generation=fields(fields(fields(raw)[2][0])[11][0])[1][0]
def sync(self, body):
status,raw=self.send(27,body)
if status!=200:return status,None,None
result=fields(fields(fields(raw)[2][0])[28][0])
return status,result[1][0],result[2][0]
def sync_body(journal, checkpoint, previous="", request_id=None, previous_request="",helper=None,helper_previous="",attempts=()):
request_id=request_id or str(uuid.uuid4())
body=field(1,request_id)+field(2,journal)+field(3,checkpoint)
if previous:body+=field(4,previous)
body+=field(5,1)+b"".join(field(6,attempt) for attempt in attempts)+field(8,1)
if helper:body+=field(9,helper)+field(10,checkpoint)
if helper_previous:body+=field(11,helper_previous)
if previous_request:body+=field(12,previous_request)
return request_id,body
def snapshot(value):
raw=json.dumps(value,separators=(",",":")).encode()
return b"".join([field(1,str(uuid.uuid4())),field(2,"linux.base"),field(3,1),field(5,1),field(6,raw),field(7,hashlib.sha256(raw).digest()),field(8,"partial")])
def main():
with tempfile.TemporaryDirectory(prefix="backend-sync-",dir=STATE) as folder:
folder=Path(folder)
_,_,enrolled=request("/agent/v1/enroll","POST",{"token":token(A),"csr_pem":csr(folder,"device")})
agent,asset=enrolled["agent_id"],enrolled["asset_id"]
ctx=context(folder,"device",enrolled["certificate_pem"])
flow=Flow(agent,ctx)
try:
status,raw=get("/api/v1/job-signing-key",ctx);key=json.loads(raw)
check(status==200 and set(key)=={"key_id","algorithm","schema_version","public_key_hex"} and key["algorithm"]=="Ed25519" and key["schema_version"]==1 and hashlib.sha256(bytes.fromhex(key["public_key_hex"])).hexdigest()==key["key_id"],"registered device mTLS bootstrap gets exact stable signing-key profile")
check(get("/api/v1/job-signing-key",ctx,{"Authorization":"Bearer invalid"})[0]==401,"explicit invalid bearer cannot fall back to mTLS key identity")
flow.hello()
check(sql(f"SELECT sync_required AND NOT sync_ready FROM agent_installations WHERE agent_id='{agent}';")=="t","negotiated sync gate closes on connection establishment")
journal,helper=str(uuid.uuid4()),str(uuid.uuid4())
rid,body=sync_body(journal,1,helper=helper)
status,payload,signature=flow.sync(body);assert status==200
decision=json.loads(payload)
check(decision["ready"] and decision["helper_ready"] and decision["tenant_id"]==A and decision["asset_id"]==asset and decision["agent_id"]==agent and decision["connection_generation"]==flow.generation and decision["request_hash"]==hashlib.sha256(body).hexdigest(),"authoritative empty-journal sync opens only bound current generation")
check(verify(folder,key,payload,signature),"independent Ed25519 verification of exact reconcile decision bytes")
check(not verify(folder,key,payload+b" ",signature),"modified reconcile decision bytes fail independent verification")
status,replayed,replayed_signature=flow.sync(body)
check(status==200 and replayed==payload and replayed_signature==signature,"same sync ID/content returns one persisted signed checkpoint")
_,changed=sync_body(journal,2,request_id=rid,helper=helper)
check(flow.sync(changed)[0]==409,"same sync ID with changed checkpoint denied")
prior_generation=flow.generation;flow.hello()
check(flow.generation>prior_generation and sql(f"SELECT sync_ready FROM agent_installations WHERE agent_id='{agent}';")=="f","clean reconnect fences old readiness until a fresh decision")
check(flow.sync(body)[0]==409,"old-generation signed checkpoint cannot be replayed as current readiness")
_,body2=sync_body(journal,2,decision["decision_id"],helper=helper,helper_previous=decision["decision_id"])
status,payload2,sig2=flow.sync(body2);assert status==200
decision2=json.loads(payload2)
check(decision2["ready"] and decision2["helper_ready"] and verify(folder,key,payload2,sig2),"consistent agent/helper journals regain readiness after clean reconnect")
lost_id,lost=sync_body(journal,3,decision2["decision_id"],helper=helper,helper_previous=decision2["decision_id"])
status,lost_payload,_=flow.sync(lost);assert status==200
lost_decision=json.loads(lost_payload)
flow.hello()
_,recovery=sync_body(journal,3,previous_request=lost_id,helper=helper,helper_previous=decision2["decision_id"])
status,payload3,_=flow.sync(recovery);assert status==200
recovered=json.loads(payload3)
check(recovered["ready"],"lost response recovery uses durable previous request without replacement grant")
now=datetime.datetime.now(datetime.timezone.utc).isoformat()
inventory={"os":"Linux fixture","hostname":"synthetic-sync","architecture":"x86_64","root_total_bytes":2,"root_available_bytes":1,"root_available_inodes":1,"collection_started_at":now,"collection_completed_at":now,"observations":[{"scope":"linux.smart","source_id":"linux.smart","observed_at":now,"quality":"unsupported","completeness":"unknown","truncated":False,"error_code":"NOT_SUPPORTED","data":None}]}
check(flow.send(13,snapshot(inventory))[0]==200,"additive RFC3339 observation metadata accepted through durable mTLS ingest")
operator=(STATE/"operator-a.token").read_text().strip()
stored=request(f"/api/v1/tenants/{A}/devices/{asset}",token=operator)[2]["inventory"]
check(stored["observations"]==inventory["observations"],"stored extended metadata retains unsupported/unknown distinctions exactly")
inventory["observations"][0]["data"]="x"*32768
check(flow.send(13,snapshot(inventory))[0]==409,"oversized individual observation fails before durable ACK")
inventory["observations"][0]["data"]=None;inventory["observations"][0]["received_at"]=now
check(flow.send(13,snapshot(inventory))[0]==409,"device cannot assert server receipt metadata")
artifact=str(uuid.uuid4());job=str(uuid.uuid4());attempt=str(uuid.uuid4())
sql(f"INSERT INTO artifacts(tenant_id,artifact_id,sha256,size_bytes,media_type) VALUES('{B}','{artifact}','{hashlib.sha256(b'foreign fixture').hexdigest()}',15,'application/octet-stream');")
path=f"/agent/v1/artifacts/{artifact}?job_id={job}&attempt_id={attempt}"
check(get(path,ctx)[0]==404,"device cannot retrieve an unapproved or foreign artifact reference")
check(get(path,ctx,{"Authorization":"Bearer "+operator})[0]==401,"agent artifact route cannot switch to bearer authority")
check(flow.send(23,field(1,b'{"execution_identity":"root"}')+field(2,b'x'*64))[0]==409,"agent-supplied unsigned execution permit is never server authority")
archive=b"synthetic sealed archive bytes "+uuid.uuid4().hex.encode();archive_hash=hashlib.sha256(archive).hexdigest();own_artifact=str(uuid.uuid4())
archive_dir=STATE/"jobs"/"artifacts"/A;archive_dir.mkdir(mode=0o700,parents=True,exist_ok=True)
with open(os.open(archive_dir/archive_hash,os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o600),"wb") as f:f.write(archive)
sql(f"INSERT INTO artifacts(tenant_id,artifact_id,sha256,size_bytes,media_type) VALUES('{A}','{own_artifact}','{archive_hash}',{len(archive)},'application/octet-stream');")
when=datetime.datetime.now(datetime.timezone.utc)
plan_input={"targets":{"asset_ids":[asset]},"action":{"action":"run_script","interpreter":"posix_sh","script_utf8":"true\n","arguments":[],"expected_stdout":""},"execution_identity":"agent_service","schedule":{"not_before":when.isoformat(),"start_before":(when+datetime.timedelta(hours=1)).isoformat()},"max_runtime_seconds":60,"max_output_bytes":1024,"rollout":{"max_parallel":1,"halt_after_failures":1},"reason":"artifact reference fixture; never execute","artifact_ids":[own_artifact]}
status,_,plan=request(f"/api/v1/tenants/{A}/operation-plans","POST",plan_input,operator);assert status==200
check(plan["plan_payload"]["artifact_references"][0]["artifact"]["sha256"]==archive_hash and hashlib.sha256(bytes(plan["plan_payload_bytes"])).hexdigest()==plan["plan_hash"],"plan binds verified actual artifact bytes to fixed job/attempt")
reserved=plan["jobs"][0]
scoped_path=f"/agent/v1/artifacts/{own_artifact}?job_id={reserved['job_id']}&attempt_id={reserved['attempt_id']}"
check(get(scoped_path,ctx)[0]==404,"reserved plan IDs do not authorize artifact access before approval/job creation")
_,_,missing_device=request("/agent/v1/enroll","POST",{"token":token(A),"csr_pem":csr(folder,"missing")})
missing_agent=missing_device["agent_id"]
try:
missing=Flow(missing_agent,context(folder,"missing",missing_device["certificate_pem"]));missing.hello();missing_journal=str(uuid.uuid4())
fake=field(1,reserved["job_id"])+field(2,reserved["attempt_id"])+field(3,reserved["job_payload_hash"])+field(5,"unconsumed")+field(6,"accepted")
_,report=sync_body(missing_journal,1,attempts=[fake]);status,fault,_=missing.sync(report);assert status==200;fault=json.loads(fault)
check(not fault["ready"] and sql(f"SELECT sync_recovery_required FROM agent_installations WHERE agent_id='{missing_agent}';")=="t","missing/unapproved server job creates durable non-executable reconciliation fault")
_,omit=sync_body(missing_journal,2,fault["decision_id"]);status,blocked,_=missing.sync(omit)
check(status==200 and not json.loads(blocked)["ready"],"omitting an unresolved missing attempt cannot reset readiness")
finally:sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{missing_agent}';")
# Only own synchronization metadata is removed to simulate a restored
# server checkpoint. No jobs, approvals, grants, or VM records touched.
sql(f"DELETE FROM agent_sync_state WHERE agent_id='{agent}';")
_,rollback=sync_body(journal,4,recovered["decision_id"],helper=helper,helper_previous=recovered["decision_id"])
status,denied,_=flow.sync(rollback);assert status==200
check(not json.loads(denied)["ready"] and sql(f"SELECT restore_blocked FROM agent_installations WHERE agent_id='{agent}';")=="t","missing authoritative server checkpoint persists fail-closed restore gate")
_,blank=sync_body(str(uuid.uuid4()),1)
check(not json.loads(flow.sync(blank)[1])["ready"],"fresh empty journal cannot bypass known server checkpoint history")
flow.sequence=0;flow.hello()
_,regressed=sync_body(journal,5)
check(not json.loads(flow.sync(regressed)[1])["ready"] and sql(f"SELECT sequence_regressed FROM agent_installations WHERE agent_id='{agent}';")=="t","journal sequence rollback never yields blind readiness")
check(sql(f"SELECT count(*) FROM execution_permits e JOIN jobs j USING(tenant_id,job_id,attempt_id) WHERE j.agent_id='{agent}';")=="0","sync never creates execution grants")
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{agent}';")
check(get("/api/v1/job-signing-key",ctx)[0]==401,"revoked device cannot bootstrap the signing key")
finally:
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{agent}';")
print("Backend sync/key/inventory/restore tests passed locally; no approvals or VM/root effects executed.")
if __name__=="__main__":main()