#!/usr/bin/env python3 """Real local TLS/PostgreSQL/agent checks. Does not read VM credentials.""" import concurrent.futures import base64 import hashlib import http.client import json import os from pathlib import Path import secrets import socket import ssl import struct import subprocess import tempfile import time ROOT = Path(__file__).resolve().parents[1] STATE = ROOT / "devstate" CA = STATE / "pki/ca.pem" A = "11111111-1111-4111-8111-111111111111" B = "22222222-2222-4222-8222-222222222222" def sql(query, user="postgres", fail=False): result = subprocess.run(["docker", "compose", "--env-file", str(STATE / ".env"), "-f", str(ROOT / "deploy/compose.yaml"), "-p", "rust-discovery-dev", "exec", "-T", "postgres", "psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", "rmm"], input=query, text=True, capture_output=True) assert (result.returncode != 0) if fail else (result.returncode == 0), "database assertion failed (details suppressed)" return result.stdout.strip() class LocalHTTPS(http.client.HTTPSConnection): def connect(self): tcp = socket.create_connection(("127.0.0.1", 18443), timeout=10) self.sock = self._context.wrap_socket(tcp, server_hostname=self.host) def request(path, token=None, body=None, context=None, hostname="rmm-dev.internal"): client = LocalHTTPS(hostname, 18443, context=context or ssl.create_default_context(cafile=str(CA)), timeout=10) headers = {} if token: headers["Authorization"] = "Bearer " + token if body is not None: headers["Content-Type"] = "application/json" client.request("POST" if body is not None else "GET", path, body=json.dumps(body) if body is not None else None, headers=headers) response = client.getresponse() data = response.read() client.close() return response.status, data def token(tenant): raw = secrets.token_hex(32) digest = hashlib.sha256(raw.encode()).hexdigest() sql(f"INSERT INTO enrollment_tokens VALUES('{digest}','{tenant}',now()+interval '15 minutes',NULL);") return raw def check(condition, message): assert condition, message print("PASS", message) def varint(number): value = bytearray() while number > 127: value.append((number & 127) | 128) number >>= 7 value.append(number) return bytes(value) def field(tag, value): if isinstance(value, int): return varint(tag << 3) + varint(value) value = value.encode() if isinstance(value, str) else value return varint((tag << 3) | 2) + varint(len(value)) + value def envelope(agent, sequence, generation, payload_tag, payload, major=1, observed=None): import uuid return b"".join([field(1, major), field(3, str(uuid.uuid4())), field(4, agent), field(5, 1), field(6, generation), field(7, "33333333-3333-4333-8333-333333333333"), field(8, sequence), field(9, observed if observed is not None else int(time.time() * 1000)), field(payload_tag, payload)]) class WebSocket: def __init__(self, context): tcp = socket.create_connection(("127.0.0.1", 18443), timeout=10) self.sock = context.wrap_socket(tcp, server_hostname="rmm-dev.internal") key = base64.b64encode(secrets.token_bytes(16)).decode() self.sock.sendall(("GET /agent/v1/connect HTTP/1.1\r\nHost: rmm-dev.internal:18443\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: " + key + "\r\nX-Client-Cert: forged\r\n\r\n").encode()) headers = b"" while not headers.endswith(b"\r\n\r\n"): byte = self.sock.recv(1) assert byte and len(headers) < 16384 headers += byte self.status = int(headers.split(b" ")[1]) def send(self, payload): mask = secrets.token_bytes(4) size = len(payload) length = bytes([size | 128]) if size < 126 else (bytes([126 | 128]) + struct.pack("!H", size)) if size < 65536 else (bytes([127 | 128]) + struct.pack("!Q", size)) self.sock.sendall(b"\x82" + length + mask + bytes(x ^ mask[i % 4] for i, x in enumerate(payload))) def exact(self, count): result = b"" while len(result) < count: chunk = self.sock.recv(count - len(result)) if not chunk: raise EOFError() result += chunk return result def receive(self): opcode, size = self.exact(2) size &= 127 if size == 126: size = struct.unpack("!H", self.exact(2))[0] elif size == 127: size = struct.unpack("!Q", self.exact(8))[0] assert size <= 1048576 return opcode & 15, self.exact(size) def close(self): self.sock.close() def rejected_frame(context, payload): ws = WebSocket(context) try: assert ws.status == 101 try: # The server may reject an oversized frame from its header before # the client has finished sending the body. ws.send(payload) opcode, _ = ws.receive() assert opcode == 8, "invalid protocol input was not closed" except (EOFError, ssl.SSLError, ConnectionResetError, BrokenPipeError): pass finally: ws.close() def main(): production = subprocess.run([str(ROOT / "target/debug/rmm-server")], env=dict(os.environ, RMM_PROFILE="production", RMM_DEV_AUTH="true"), capture_output=True) check(production.returncode != 0 and b"production forbids dev auth" in production.stderr, "production rejects explicit dev auth before connecting") viewer_a = (STATE / "viewer-a.token").read_text().strip() viewer_b = (STATE / "viewer-b.token").read_text().strip() deadline = time.monotonic() + 15 while True: try: request("/healthz") break except (ConnectionRefusedError, ConnectionResetError): if time.monotonic() >= deadline: raise time.sleep(0.25) check(request("/healthz")[0] == 200, "trusted TLS health") for context, hostname in [(ssl.create_default_context(), "rmm-dev.internal"), (ssl.create_default_context(cafile=str(CA)), "wrong.internal")]: try: request("/healthz", context=context, hostname=hostname) except ssl.SSLCertVerificationError: pass else: raise AssertionError("TLS validation did not reject invalid trust/name") print("PASS untrusted CA and wrong DNS name rejected") check(request("/api/v1/tenants")[0] == 401, "anonymous API denied") check(request("/api/v1/tenants", "invalid")[0] == 401, "invalid bearer denied") status, error = request("/api/v1/tenants", "invalid") check(status == 401 and json.loads(error)["code"] == "UNAUTHENTICATED" and "correlation_id" in json.loads(error), "structured errors without request echo") for bearer, tenant, foreign in [(viewer_a, A, B), (viewer_b, B, A)]: status, data = request("/api/v1/tenants", bearer) check(status == 200 and [x["tenant_id"] for x in json.loads(data)["items"]] == [tenant], "viewer sees exactly assigned tenant") check(request(f"/api/v1/tenants/{foreign}/devices", bearer)[0] == 404, "foreign tenant denied") check(sql("SELECT count(*) FROM tenants;", "rmm_app") == "0", "RLS denies missing context") result = sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); SELECT count(*) FROM tenants; ROLLBACK; SELECT count(*) FROM tenants;", "rmm_app") check(result.splitlines()[-2:] == ["1", "0"], "transaction-local tenant reset after rollback") sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); INSERT INTO assets VALUES('{B}','{secrets.token_hex(16)}');", "rmm_app", fail=True) print("PASS RLS rejects cross-tenant insert") role = sql("SELECT rolsuper OR rolbypassrls FROM pg_roles WHERE rolname='rmm_app';") check(role == "f", "application role has neither superuser nor BYPASSRLS") check(sql("SELECT count(*) FROM pg_tables WHERE schemaname='public' AND tableowner='rmm_app';") == "0", "application owns no tables") with tempfile.TemporaryDirectory(prefix="rmm-e2e-", dir=STATE) as folder: folder = Path(folder) subprocess.run(["openssl", "req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", "-keyout", str(folder / "device.key"), "-out", str(folder / "device.csr"), "-subj", "/CN=e2e"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) csr = (folder / "device.csr").read_text() value = token(A) # Invalid CSR rolls consumption back. check(request("/agent/v1/enroll", body={"token": value, "csr_pem": "invalid"})[0] == 400, "invalid CSR rejected") body = {"token": value, "csr_pem": csr} with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: replies = list(pool.map(lambda _: request("/agent/v1/enroll", body=body), range(2))) check(sorted(x[0] for x in replies) == [200, 401], "enrollment token consumed atomically exactly once") enrolled = json.loads(next(data for status, data in replies if status == 200)) (folder / "device.pem").write_text(enrolled["certificate_pem"]) context = ssl.create_default_context(cafile=str(CA)) context.load_cert_chain(folder / "device.pem", folder / "device.key") check(request("/healthz", context=context)[0] == 200, "issued client certificate passes TLS validation") anonymous = WebSocket(ssl.create_default_context(cafile=str(CA))) check(anonymous.status == 401, "WSS denies missing peer identity despite forged certificate header") anonymous.close() rejected_frame(context, envelope(enrolled["agent_id"], 1, 0, 10, field(1, "test"), major=2)) print("PASS unknown protocol major closes channel") rejected_frame(context, envelope("44444444-4444-4444-8444-444444444444", 1, 0, 10, field(1, "test"))) print("PASS TLS device cannot claim another agent ID") ws = WebSocket(context) try: check(ws.status == 101, "registered certificate upgrades to WSS") ws.send(envelope(enrolled["agent_id"], 1, 0, 10, field(1, "test"))) check(ws.receive()[0] == 2, "server hello is binary protobuf") generation = int(sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{enrolled['agent_id']}';")) ws.send(envelope(enrolled["agent_id"], 2, generation, 12, field(1, "55555555-5555-4555-8555-555555555555"), observed=int(time.time() * 1000) - 600000)) check(ws.receive()[0] == 8, "stale buffered heartbeat closes channel") check(sql(f"SELECT last_contact_at IS NULL FROM agent_installations WHERE agent_id='{enrolled['agent_id']}';") == "t", "stale heartbeat cannot mark presence online") finally: ws.close() rejected_frame(context, b"x" * (1048576 + 1)) print("PASS oversized WebSocket message rejected") # Actual agent enrollment, WSS, inventory and heartbeat, with bounded wait. agent_state = folder / "agent" token_path = folder / "enroll.token" token_path.write_text(token(A)) token_path.chmod(0o600) env = dict(os.environ, RMM_AGENT_STATE=str(agent_state), RMM_CA_FILE=str(CA), RMM_ENROLLMENT_TOKEN_FILE=str(token_path),RMM_EXTENDED_INVENTORY="1") with open(folder / "agent.log", "wb") as log: agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log) try: deadline = time.monotonic() + 55 observed = None while time.monotonic() < deadline: status, data = request(f"/api/v1/tenants/{A}/devices", viewer_a) assert status == 200 items = json.loads(data)["items"] # Never select/revoke a concurrently running VM or another # integration worker's device merely because it is online. own_identity = agent_state / "identity.json" own = json.loads(own_identity.read_text()) if own_identity.exists() else None observed = None if own: detail_status, detail = request(f"/api/v1/tenants/{A}/devices/{own['asset_id']}", viewer_a) candidate = json.loads(detail) if detail_status == 200 else None if candidate and candidate["agent_id"] == own["agent_id"] and candidate["inventory"] and candidate["presence"] == "online": observed = candidate if observed: break assert agent.poll() is None, "agent exited (logs remain private)" time.sleep(1) check(observed is not None, "real Linux agent enrolled, mTLS WSS inventory stored and heartbeat online") check(observed["inventory"]["root_available_bytes"] > 0 and observed["observed_at"] and observed["received_at"], "real disk value and observation timestamps") check(isinstance(observed["inventory"].get("observations"),list) and len(observed["inventory"]["observations"])>0 and observed["inventory"].get("collection_completed_at"),"real native extended collectors stored with bounded per-source metadata") status, data = request(f"/api/v1/tenants/{B}/devices", viewer_b) check(status == 200 and observed["agent_id"] not in data.decode(), "tenant B cannot see tenant A telemetry") check((agent_state / "device.key").stat().st_mode & 0o777 == 0o600, "local private device key is 0600") agent_id = observed["agent_id"] generation = sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{agent_id}';") agent.terminate() agent.wait(timeout=10) agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log) deadline = time.monotonic() + 15 while time.monotonic() < deadline: current = sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{agent_id}';") if int(current) > int(generation): break time.sleep(1) check(int(current) > int(generation), "restart reuses local identity and fences old channel") sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{agent_id}';") before = sql(f"SELECT count(*) FROM observations WHERE agent_id='{agent_id}';") agent.terminate() agent.wait(timeout=10) agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log) time.sleep(3) check(sql(f"SELECT count(*) FROM observations WHERE agent_id='{agent_id}';") == before, "revoked identity cannot ingest after reconnect") finally: agent.terminate() agent.wait(timeout=10) if (agent_state / "identity.json").exists(): identity = json.loads((agent_state / "identity.json").read_text()) sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{identity['agent_id']}';") sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{enrolled['agent_id']}';") check(sql("SELECT count(*) FROM information_schema.tables WHERE table_name IN ('jobs','outbox','execution_permits');") == "3", "durable job/outbox/grant schema present") check(sql("SELECT count(*) FROM jobs j LEFT JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id LEFT JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE a.authorization_id IS NULL;") == "0", "no executable job exists without durable human authorization") print("All local integration checks passed. VM/systemd acceptance NOT performed.") if __name__ == "__main__": main()