217 lines
16 KiB
Python
217 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Actual local mTLS HTTPS/WSS, rotation/fencing and independent housekeeping.
|
|
No VM access, approval, execution, or mutable existing-device fixtures.
|
|
"""
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import ssl
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from local_integration import LocalHTTPS, WebSocket, field, envelope, token, sql, check, CA, STATE, A, B
|
|
from backend_contract import request
|
|
|
|
|
|
def fields(data):
|
|
result = {}
|
|
offset = 0
|
|
def integer():
|
|
nonlocal offset
|
|
value = shift = 0
|
|
while True:
|
|
assert offset < len(data) and shift < 70, "malformed protobuf integer"
|
|
byte = data[offset]; offset += 1
|
|
value |= (byte & 127) << shift
|
|
if byte < 128:
|
|
return value
|
|
shift += 7
|
|
while offset < len(data):
|
|
key = integer(); tag, kind = key >> 3, key & 7
|
|
if kind == 0:
|
|
value = integer()
|
|
else:
|
|
assert kind == 2, "unsupported protobuf fixture field"
|
|
length = integer(); value = data[offset:offset+length]; offset += length
|
|
assert offset <= len(data), "malformed protobuf fixture length"
|
|
result.setdefault(tag, []).append(value)
|
|
return result
|
|
|
|
|
|
def exchange(path, body, context, content_type="application/x-protobuf"):
|
|
c = LocalHTTPS("rmm-dev.internal", 18443, context=context, timeout=30)
|
|
c.request("POST", path, body=body, headers={"Content-Type": content_type})
|
|
r = c.getresponse(); status, raw = r.status, r.read(); c.close()
|
|
return status, raw
|
|
|
|
|
|
def batch(e):
|
|
return field(1, 1) + field(2, e)
|
|
|
|
|
|
def csr(folder, name):
|
|
subprocess.run(["openssl", "req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", "-keyout", str(folder/(name+".key")), "-out", str(folder/(name+".csr")), "-subj", "/CN=backend-transport-fixture"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
return (folder/(name+".csr")).read_text()
|
|
|
|
|
|
def context(folder, name, pem):
|
|
(folder/(name+".pem")).write_text(pem)
|
|
ctx = ssl.create_default_context(cafile=str(CA))
|
|
ctx.load_cert_chain(folder/(name+".pem"), folder/(name+".key"))
|
|
return ctx
|
|
|
|
|
|
def hello(agent, sequence, ctx):
|
|
status, raw = exchange("/agent/v1/messages", batch(envelope(agent, sequence, 0, 10, field(1, "backend-transport")+field(2, "run_script"))), ctx)
|
|
assert status == 200, "HTTPS hello failed"
|
|
response = fields(fields(raw)[2][0])
|
|
return fields(response[11][0])[1][0]
|
|
|
|
|
|
def mtls_get(path, ctx):
|
|
client=LocalHTTPS("rmm-dev.internal",18443,context=ctx,timeout=10)
|
|
try:
|
|
client.request("GET",path)
|
|
response=client.getresponse();response.read();return response.status
|
|
finally:
|
|
client.close()
|
|
|
|
|
|
def certificate_hash(pem):
|
|
return hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest()
|
|
|
|
|
|
def public_device(asset, viewer):
|
|
status,_,device=request(f"/api/v1/tenants/{A}/devices/{asset}",token=viewer)
|
|
assert status==200,"own public device detail unavailable"
|
|
forbidden={"private_key","private_key_pem","csr_pem","certificate_pem","token","enrollment_token","renewal_csr_sha256","password"}
|
|
assert not forbidden.intersection(device),"private enrollment/credential fields leaked into device DTO"
|
|
return device
|
|
|
|
|
|
def main():
|
|
trust = ssl.create_default_context(cafile=str(CA))
|
|
with tempfile.TemporaryDirectory(prefix="backend-https-", dir=STATE) as folder:
|
|
folder = Path(folder)
|
|
initial = csr(folder, "old")
|
|
status, _, enrolled = request("/agent/v1/enroll", "POST", {"token": token(A), "csr_pem": initial})
|
|
assert status == 200
|
|
agent, asset = enrolled["agent_id"], enrolled["asset_id"]
|
|
viewer=(STATE/"viewer-a.token").read_text().strip()
|
|
viewer_b=(STATE/"viewer-b.token").read_text().strip()
|
|
old_hash=certificate_hash(enrolled["certificate_pem"])
|
|
old = context(folder, "old", enrolled["certificate_pem"])
|
|
ws = None
|
|
try:
|
|
initial_device=public_device(asset,viewer)
|
|
check(initial_device["certificate_sha256"]==old_hash and initial_device["last_authenticated_certificate_sha256"] is None and initial_device["last_authenticated_at"] is None,"issued certificate pin is public but issuance does not invent TLS-use evidence")
|
|
cursor=None;listed=None
|
|
for _ in range(100):
|
|
path=f"/api/v1/tenants/{A}/devices?limit=500"+("&cursor="+cursor if cursor else "")
|
|
list_status,_,page=request(path,token=viewer);assert list_status==200
|
|
listed=next((item for item in page["items"] if item["agent_id"]==agent),None)
|
|
if listed or not page["next_cursor"]:break
|
|
cursor=page["next_cursor"]
|
|
check(listed is not None and listed["certificate_sha256"]==old_hash and listed["last_authenticated_certificate_sha256"] is None,"device list exposes the same public certificate evidence contract")
|
|
check(request(f"/api/v1/tenants/{A}/devices/{asset}",token=viewer_b)[0]==404 and request(f"/api/v1/tenants/{B}/devices/{asset}",token=viewer_b)[0]==404,"certificate evidence remains tenant-scoped and foreign assets are 404")
|
|
check(exchange("/agent/v1/messages", batch(envelope(agent,1,0,10,field(1,"x"))),trust)[0] == 401, "HTTPS messages require real mTLS identity")
|
|
check(public_device(asset,viewer)["last_authenticated_at"] is None,"anonymous/forged device claim creates no authentication evidence")
|
|
generation = hello(agent, 1, old)
|
|
check(generation > 0, "HTTPS hello selects fenced connection generation")
|
|
authenticated=public_device(asset,viewer)
|
|
check(authenticated["last_authenticated_certificate_sha256"]==old_hash and authenticated["last_authenticated_at"] is not None and authenticated["last_contact_at"] is None,"actual old-certificate mTLS Hello records authentication, not heartbeat presence")
|
|
heartbeat = envelope(agent, 2, generation, 12, field(1, str(uuid.uuid4())))
|
|
status, raw = exchange("/agent/v1/messages", batch(heartbeat), old)
|
|
check(status == 200 and 14 in fields(fields(raw)[2][0]), "HTTPS heartbeat durable contract returns concrete ACK")
|
|
before = sql(f"SELECT last_contact_at FROM agent_installations WHERE agent_id='{agent}';")
|
|
check(exchange("/agent/v1/messages", batch(heartbeat), old)[0] == 409, "HTTPS fresh heartbeat replay denied")
|
|
check(sql(f"SELECT last_contact_at FROM agent_installations WHERE agent_id='{agent}';") == before, "replayed HTTPS heartbeat never refreshes presence")
|
|
check(exchange("/agent/v1/messages", b"{}", old, "application/json")[0] == 415, "HTTPS protobuf content type enforced")
|
|
check(exchange("/agent/v1/messages", field(1,1)+b"".join(field(2,heartbeat) for _ in range(17)), old)[0] == 400, "HTTPS batch count bounded before ingest")
|
|
fresh = envelope(agent,3,generation,12,field(1,str(uuid.uuid4())))
|
|
started = time.monotonic()
|
|
status, raw = exchange("/agent/v1/poll",field(1,fresh)+field(2,1),old)
|
|
check(status == 200 and len(fields(raw)[2]) == 1 and time.monotonic()-started >= .8, "bounded long poll returns an empty-work ACK after wait")
|
|
# A real WSS connection takes ownership from HTTPS.
|
|
ws = WebSocket(old); assert ws.status == 101
|
|
ws.send(envelope(agent,4,0,10,field(1,"backend-transport")))
|
|
_, wire = ws.receive(); wss_generation = fields(fields(wire)[11][0])[1][0]
|
|
check(wss_generation > generation, "WSS transport takeover increments generation")
|
|
check(exchange("/agent/v1/messages",batch(envelope(agent,5,generation,12,field(1,str(uuid.uuid4())))),old)[0] == 409, "old HTTPS channel is fenced by WSS takeover")
|
|
generation = hello(agent,6,old)
|
|
ws.send(envelope(agent,7,wss_generation,12,field(1,str(uuid.uuid4()))))
|
|
check(ws.receive()[0] == 8, "HTTPS takeover fences stale WSS control traffic")
|
|
ws.close(); ws = None
|
|
# The scheduler must expire a pending request without any agent poll.
|
|
operator=(STATE/"operator-a.token").read_text().strip()
|
|
now=datetime.datetime.now(datetime.timezone.utc)
|
|
plan_input={"targets":{"asset_ids":[asset]},"action":{"action":"run_script","interpreter":"posix_sh","script_utf8":"true\n","arguments":[],"expected_stdout":""},"execution_identity":"agent_service","schedule":{"not_before":now.isoformat(),"start_before":(now+datetime.timedelta(seconds=4)).isoformat()},"max_runtime_seconds":1,"max_output_bytes":1024,"rollout":{"max_parallel":1,"halt_after_failures":1},"reason":"offline housekeeping fixture; never execute"}
|
|
base=f"/api/v1/tenants/{A}"
|
|
status,_,plan=request(base+"/operation-plans","POST",plan_input,operator); assert status == 200
|
|
status,_,pending=request(base+f"/operation-plans/{plan['plan_id']}/submit","POST",{"idempotency_key":"housekeeping-"+uuid.uuid4().hex},operator); assert status == 200
|
|
deadline=time.monotonic()+8
|
|
while time.monotonic()<deadline:
|
|
expired=sql(f"SELECT approval_state FROM approval_requests WHERE operation_id='{pending['operation_id']}';")
|
|
if expired == "expired": break
|
|
time.sleep(.25)
|
|
check(expired == "expired", "background scheduler expires offline pending request without read/poll side effects")
|
|
check(sql(f"SELECT count(*) FROM jobs WHERE operation_id='{pending['operation_id']}';") == "0", "expiry never creates executable jobs")
|
|
# Renewing keys does not replace asset or enrollment generation.
|
|
contact_before_renewal=public_device(asset,viewer)["last_contact_at"]
|
|
check(mtls_get("/api/v1/job-signing-key",old)==200,"device key bootstrap authenticates the actual TLS peer")
|
|
check(public_device(asset,viewer)["last_contact_at"]==contact_before_renewal,"key GET does not refresh heartbeat presence")
|
|
missing_artifact=f"/agent/v1/artifacts/{uuid.uuid4()}?job_id={uuid.uuid4()}&attempt_id={uuid.uuid4()}"
|
|
check(mtls_get(missing_artifact,old)==404 and public_device(asset,viewer)["last_contact_at"]==contact_before_renewal,"artifact authentication does not refresh heartbeat presence")
|
|
renewal=csr(folder,"new")
|
|
status,raw=exchange("/agent/v1/certificates/renew",json.dumps({"csr_pem":renewal}).encode(),old,"application/json")
|
|
check(status == 200, "current mTLS identity can rotate a valid new CSR")
|
|
renewed=json.loads(raw); new=context(folder,"new",renewed["certificate_pem"])
|
|
new_hash=certificate_hash(renewed["certificate_pem"])
|
|
issued_new=public_device(asset,viewer)
|
|
check(new_hash!=old_hash and issued_new["certificate_sha256"]==new_hash and issued_new["last_authenticated_certificate_sha256"]==old_hash,"renewal updates issued pin while actual authenticated peer remains the old certificate")
|
|
check(issued_new["last_contact_at"]==contact_before_renewal,"certificate renewal does not refresh heartbeat presence")
|
|
status,repeated=exchange("/agent/v1/certificates/renew",json.dumps({"csr_pem":renewal}).encode(),old,"application/json")
|
|
check(status == 200 and json.loads(repeated)["certificate_pem"] == renewed["certificate_pem"], "lost renewal response can be recovered idempotently during overlap")
|
|
overlap=public_device(asset,viewer)
|
|
check(overlap["certificate_sha256"]==new_hash and overlap["last_authenticated_certificate_sha256"]==old_hash,"lost-response recovery using old overlap is not misreported as new-certificate use")
|
|
check(sql(f"SELECT asset_id::text||':'||enrollment_generation FROM agent_installations WHERE agent_id='{agent}';") == asset+":1", "certificate rotation preserves device assignment/identity generation")
|
|
generation=hello(agent,8,new)
|
|
used_new=public_device(asset,viewer)
|
|
check(used_new["certificate_sha256"]==new_hash and used_new["last_authenticated_certificate_sha256"]==new_hash and used_new["enrollment_generation"]==1 and used_new["agent_id"]==agent,"verified new-certificate Hello changes actual peer evidence without identity replacement")
|
|
check(datetime.datetime.fromisoformat(used_new["last_authenticated_at"].replace("Z","+00:00"))>=datetime.datetime.fromisoformat(overlap["last_authenticated_at"].replace("Z","+00:00")) and used_new["last_contact_at"]==contact_before_renewal,"new peer authentication has a separate time and cannot fake a heartbeat")
|
|
sql(f"UPDATE agent_installations SET previous_certificate_until=now()-interval '1 second' WHERE agent_id='{agent}';")
|
|
check(exchange("/agent/v1/messages",batch(envelope(agent,9,0,10,field(1,"old"))),old)[0] == 401, "expired previous certificate binding rejected without disabling TLS")
|
|
check(public_device(asset,viewer)["last_authenticated_at"]==used_new["last_authenticated_at"],"rejected expired-overlap certificate cannot change authentication evidence")
|
|
subprocess.run(["openssl","req","-x509","-newkey","ec","-pkeyopt","ec_paramgen_curve:prime256v1","-nodes","-keyout",str(folder/"invalid.key"),"-out",str(folder/"invalid.pem"),"-days","1","-subj","/CN=untrusted-client-fixture"],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
invalid=ssl.create_default_context(cafile=str(CA));invalid.load_cert_chain(folder/"invalid.pem",folder/"invalid.key")
|
|
try:
|
|
invalid_status=mtls_get("/api/v1/job-signing-key",invalid)
|
|
assert invalid_status==401,"untrusted client certificate was accepted"
|
|
except (ssl.SSLError,ConnectionResetError,ConnectionAbortedError,BrokenPipeError):
|
|
pass
|
|
check(public_device(asset,viewer)["last_authenticated_at"]==used_new["last_authenticated_at"],"TLS-invalid client certificate cannot create authentication evidence")
|
|
hello(agent,1,new)
|
|
check(sql(f"SELECT restore_blocked FROM agent_installations WHERE agent_id='{agent}';") == "t", "regressed journal sequence persistently blocks new effects for reconciliation")
|
|
ws=WebSocket(new); assert ws.status == 101
|
|
ws.send(envelope(agent,10,0,10,field(1,"revocation-fixture"))); assert ws.receive()[0] == 2
|
|
before_revocation=public_device(asset,viewer)
|
|
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{agent}';")
|
|
ws.sock.settimeout(4)
|
|
check(ws.receive()[0] == 8, "persistent revocation actively closes idle WSS channel")
|
|
check(exchange("/agent/v1/messages",batch(envelope(agent,11,0,10,field(1,"revoked"))),new)[0] == 401, "revoked identity cannot reconnect using rotated certificate")
|
|
after_revocation=public_device(asset,viewer)
|
|
check(after_revocation["last_authenticated_at"]==before_revocation["last_authenticated_at"] and after_revocation["last_authenticated_certificate_sha256"]==new_hash,"revoked peer cannot overwrite retained public authentication evidence")
|
|
check(sql(f"SELECT count(*) FROM agent_installations WHERE agent_id='{agent}';","rmm_app")=="0","certificate evidence remains RLS-protected without tenant context")
|
|
sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); UPDATE agent_installations SET last_authenticated_at=now() WHERE agent_id='{agent}';","rmm_app",fail=True)
|
|
print("PASS application has no general UPDATE grant on authentication-evidence columns")
|
|
finally:
|
|
if ws: ws.close()
|
|
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{agent}';")
|
|
print("Backend transport/rotation/housekeeping passed locally; no approval, execution or VM acceptance performed.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|