Files
rust-discovery/tests/local_integration.py
T
BuTzZ 6c3203d725
Rust development slice / linux (push) Failing after 1m10s
feat: snapshot Linux-first RMM MVP and pause handover
2026-11-12 22:06:25 +01:00

285 lines
16 KiB
Python

#!/usr/bin/env python3
"""Real local TLS/PostgreSQL/agent checks. Does not read VM credentials."""
import concurrent.futures
import base64
import hashlib
import http.client
import json
import os
from pathlib import Path
import secrets
import socket
import ssl
import struct
import subprocess
import tempfile
import time
ROOT = Path(__file__).resolve().parents[1]
STATE = ROOT / "devstate"
CA = STATE / "pki/ca.pem"
A = "11111111-1111-4111-8111-111111111111"
B = "22222222-2222-4222-8222-222222222222"
def sql(query, user="postgres", fail=False):
result = subprocess.run(["docker", "compose", "--env-file", str(STATE / ".env"), "-f", str(ROOT / "deploy/compose.yaml"), "-p", "rust-discovery-dev", "exec", "-T", "postgres", "psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", "rmm"], input=query, text=True, capture_output=True)
assert (result.returncode != 0) if fail else (result.returncode == 0), "database assertion failed (details suppressed)"
return result.stdout.strip()
class LocalHTTPS(http.client.HTTPSConnection):
def connect(self):
tcp = socket.create_connection(("127.0.0.1", 18443), timeout=10)
self.sock = self._context.wrap_socket(tcp, server_hostname=self.host)
def request(path, token=None, body=None, context=None, hostname="rmm-dev.internal"):
client = LocalHTTPS(hostname, 18443, context=context or ssl.create_default_context(cafile=str(CA)), timeout=10)
headers = {}
if token:
headers["Authorization"] = "Bearer " + token
if body is not None:
headers["Content-Type"] = "application/json"
client.request("POST" if body is not None else "GET", path, body=json.dumps(body) if body is not None else None, headers=headers)
response = client.getresponse()
data = response.read()
client.close()
return response.status, data
def token(tenant):
raw = secrets.token_hex(32)
digest = hashlib.sha256(raw.encode()).hexdigest()
sql(f"INSERT INTO enrollment_tokens VALUES('{digest}','{tenant}',now()+interval '15 minutes',NULL);")
return raw
def check(condition, message):
assert condition, message
print("PASS", message)
def varint(number):
value = bytearray()
while number > 127:
value.append((number & 127) | 128)
number >>= 7
value.append(number)
return bytes(value)
def field(tag, value):
if isinstance(value, int):
return varint(tag << 3) + varint(value)
value = value.encode() if isinstance(value, str) else value
return varint((tag << 3) | 2) + varint(len(value)) + value
def envelope(agent, sequence, generation, payload_tag, payload, major=1, observed=None):
import uuid
return b"".join([field(1, major), field(3, str(uuid.uuid4())), field(4, agent), field(5, 1), field(6, generation), field(7, "33333333-3333-4333-8333-333333333333"), field(8, sequence), field(9, observed if observed is not None else int(time.time() * 1000)), field(payload_tag, payload)])
class WebSocket:
def __init__(self, context):
tcp = socket.create_connection(("127.0.0.1", 18443), timeout=10)
self.sock = context.wrap_socket(tcp, server_hostname="rmm-dev.internal")
key = base64.b64encode(secrets.token_bytes(16)).decode()
self.sock.sendall(("GET /agent/v1/connect HTTP/1.1\r\nHost: rmm-dev.internal:18443\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: " + key + "\r\nX-Client-Cert: forged\r\n\r\n").encode())
headers = b""
while not headers.endswith(b"\r\n\r\n"):
byte = self.sock.recv(1)
assert byte and len(headers) < 16384
headers += byte
self.status = int(headers.split(b" ")[1])
def send(self, payload):
mask = secrets.token_bytes(4)
size = len(payload)
length = bytes([size | 128]) if size < 126 else (bytes([126 | 128]) + struct.pack("!H", size)) if size < 65536 else (bytes([127 | 128]) + struct.pack("!Q", size))
self.sock.sendall(b"\x82" + length + mask + bytes(x ^ mask[i % 4] for i, x in enumerate(payload)))
def exact(self, count):
result = b""
while len(result) < count:
chunk = self.sock.recv(count - len(result))
if not chunk:
raise EOFError()
result += chunk
return result
def receive(self):
opcode, size = self.exact(2)
size &= 127
if size == 126:
size = struct.unpack("!H", self.exact(2))[0]
elif size == 127:
size = struct.unpack("!Q", self.exact(8))[0]
assert size <= 1048576
return opcode & 15, self.exact(size)
def close(self):
self.sock.close()
def rejected_frame(context, payload):
ws = WebSocket(context)
try:
assert ws.status == 101
try:
# The server may reject an oversized frame from its header before
# the client has finished sending the body.
ws.send(payload)
opcode, _ = ws.receive()
assert opcode == 8, "invalid protocol input was not closed"
except (EOFError, ssl.SSLError, ConnectionResetError, BrokenPipeError):
pass
finally:
ws.close()
def main():
production = subprocess.run([str(ROOT / "target/debug/rmm-server")], env=dict(os.environ, RMM_PROFILE="production", RMM_DEV_AUTH="true"), capture_output=True)
check(production.returncode != 0 and b"production forbids dev auth" in production.stderr, "production rejects explicit dev auth before connecting")
viewer_a = (STATE / "viewer-a.token").read_text().strip()
viewer_b = (STATE / "viewer-b.token").read_text().strip()
deadline = time.monotonic() + 15
while True:
try:
request("/healthz")
break
except (ConnectionRefusedError, ConnectionResetError):
if time.monotonic() >= deadline:
raise
time.sleep(0.25)
check(request("/healthz")[0] == 200, "trusted TLS health")
for context, hostname in [(ssl.create_default_context(), "rmm-dev.internal"), (ssl.create_default_context(cafile=str(CA)), "wrong.internal")]:
try:
request("/healthz", context=context, hostname=hostname)
except ssl.SSLCertVerificationError:
pass
else:
raise AssertionError("TLS validation did not reject invalid trust/name")
print("PASS untrusted CA and wrong DNS name rejected")
check(request("/api/v1/tenants")[0] == 401, "anonymous API denied")
check(request("/api/v1/tenants", "invalid")[0] == 401, "invalid bearer denied")
status, error = request("/api/v1/tenants", "invalid")
check(status == 401 and json.loads(error)["code"] == "UNAUTHENTICATED" and "correlation_id" in json.loads(error), "structured errors without request echo")
for bearer, tenant, foreign in [(viewer_a, A, B), (viewer_b, B, A)]:
status, data = request("/api/v1/tenants", bearer)
check(status == 200 and [x["tenant_id"] for x in json.loads(data)["items"]] == [tenant], "viewer sees exactly assigned tenant")
check(request(f"/api/v1/tenants/{foreign}/devices", bearer)[0] == 404, "foreign tenant denied")
check(sql("SELECT count(*) FROM tenants;", "rmm_app") == "0", "RLS denies missing context")
result = sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); SELECT count(*) FROM tenants; ROLLBACK; SELECT count(*) FROM tenants;", "rmm_app")
check(result.splitlines()[-2:] == ["1", "0"], "transaction-local tenant reset after rollback")
sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); INSERT INTO assets VALUES('{B}','{secrets.token_hex(16)}');", "rmm_app", fail=True)
print("PASS RLS rejects cross-tenant insert")
role = sql("SELECT rolsuper OR rolbypassrls FROM pg_roles WHERE rolname='rmm_app';")
check(role == "f", "application role has neither superuser nor BYPASSRLS")
check(sql("SELECT count(*) FROM pg_tables WHERE schemaname='public' AND tableowner='rmm_app';") == "0", "application owns no tables")
with tempfile.TemporaryDirectory(prefix="rmm-e2e-", dir=STATE) as folder:
folder = Path(folder)
subprocess.run(["openssl", "req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", "-keyout", str(folder / "device.key"), "-out", str(folder / "device.csr"), "-subj", "/CN=e2e"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
csr = (folder / "device.csr").read_text()
value = token(A)
# Invalid CSR rolls consumption back.
check(request("/agent/v1/enroll", body={"token": value, "csr_pem": "invalid"})[0] == 400, "invalid CSR rejected")
body = {"token": value, "csr_pem": csr}
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
replies = list(pool.map(lambda _: request("/agent/v1/enroll", body=body), range(2)))
check(sorted(x[0] for x in replies) == [200, 401], "enrollment token consumed atomically exactly once")
enrolled = json.loads(next(data for status, data in replies if status == 200))
(folder / "device.pem").write_text(enrolled["certificate_pem"])
context = ssl.create_default_context(cafile=str(CA))
context.load_cert_chain(folder / "device.pem", folder / "device.key")
check(request("/healthz", context=context)[0] == 200, "issued client certificate passes TLS validation")
anonymous = WebSocket(ssl.create_default_context(cafile=str(CA)))
check(anonymous.status == 401, "WSS denies missing peer identity despite forged certificate header")
anonymous.close()
rejected_frame(context, envelope(enrolled["agent_id"], 1, 0, 10, field(1, "test"), major=2))
print("PASS unknown protocol major closes channel")
rejected_frame(context, envelope("44444444-4444-4444-8444-444444444444", 1, 0, 10, field(1, "test")))
print("PASS TLS device cannot claim another agent ID")
ws = WebSocket(context)
try:
check(ws.status == 101, "registered certificate upgrades to WSS")
ws.send(envelope(enrolled["agent_id"], 1, 0, 10, field(1, "test")))
check(ws.receive()[0] == 2, "server hello is binary protobuf")
generation = int(sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{enrolled['agent_id']}';"))
ws.send(envelope(enrolled["agent_id"], 2, generation, 12, field(1, "55555555-5555-4555-8555-555555555555"), observed=int(time.time() * 1000) - 600000))
check(ws.receive()[0] == 8, "stale buffered heartbeat closes channel")
check(sql(f"SELECT last_contact_at IS NULL FROM agent_installations WHERE agent_id='{enrolled['agent_id']}';") == "t", "stale heartbeat cannot mark presence online")
finally:
ws.close()
rejected_frame(context, b"x" * (1048576 + 1))
print("PASS oversized WebSocket message rejected")
# Actual agent enrollment, WSS, inventory and heartbeat, with bounded wait.
agent_state = folder / "agent"
token_path = folder / "enroll.token"
token_path.write_text(token(A))
token_path.chmod(0o600)
env = dict(os.environ, RMM_AGENT_STATE=str(agent_state), RMM_CA_FILE=str(CA), RMM_ENROLLMENT_TOKEN_FILE=str(token_path),RMM_EXTENDED_INVENTORY="1")
with open(folder / "agent.log", "wb") as log:
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
try:
deadline = time.monotonic() + 55
observed = None
while time.monotonic() < deadline:
status, data = request(f"/api/v1/tenants/{A}/devices", viewer_a)
assert status == 200
items = json.loads(data)["items"]
# Never select/revoke a concurrently running VM or another
# integration worker's device merely because it is online.
own_identity = agent_state / "identity.json"
own = json.loads(own_identity.read_text()) if own_identity.exists() else None
observed = None
if own:
detail_status, detail = request(f"/api/v1/tenants/{A}/devices/{own['asset_id']}", viewer_a)
candidate = json.loads(detail) if detail_status == 200 else None
if candidate and candidate["agent_id"] == own["agent_id"] and candidate["inventory"] and candidate["presence"] == "online":
observed = candidate
if observed:
break
assert agent.poll() is None, "agent exited (logs remain private)"
time.sleep(1)
check(observed is not None, "real Linux agent enrolled, mTLS WSS inventory stored and heartbeat online")
check(observed["inventory"]["root_available_bytes"] > 0 and observed["observed_at"] and observed["received_at"], "real disk value and observation timestamps")
check(isinstance(observed["inventory"].get("observations"),list) and len(observed["inventory"]["observations"])>0 and observed["inventory"].get("collection_completed_at"),"real native extended collectors stored with bounded per-source metadata")
status, data = request(f"/api/v1/tenants/{B}/devices", viewer_b)
check(status == 200 and observed["agent_id"] not in data.decode(), "tenant B cannot see tenant A telemetry")
check((agent_state / "device.key").stat().st_mode & 0o777 == 0o600, "local private device key is 0600")
agent_id = observed["agent_id"]
generation = sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{agent_id}';")
agent.terminate()
agent.wait(timeout=10)
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
current = sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{agent_id}';")
if int(current) > int(generation):
break
time.sleep(1)
check(int(current) > int(generation), "restart reuses local identity and fences old channel")
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{agent_id}';")
before = sql(f"SELECT count(*) FROM observations WHERE agent_id='{agent_id}';")
agent.terminate()
agent.wait(timeout=10)
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
time.sleep(3)
check(sql(f"SELECT count(*) FROM observations WHERE agent_id='{agent_id}';") == before, "revoked identity cannot ingest after reconnect")
finally:
agent.terminate()
agent.wait(timeout=10)
if (agent_state / "identity.json").exists():
identity = json.loads((agent_state / "identity.json").read_text())
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{identity['agent_id']}';")
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{enrolled['agent_id']}';")
check(sql("SELECT count(*) FROM information_schema.tables WHERE table_name IN ('jobs','outbox','execution_permits');") == "3", "durable job/outbox/grant schema present")
check(sql("SELECT count(*) FROM jobs j LEFT JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id LEFT JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE a.authorization_id IS NULL;") == "0", "no executable job exists without durable human authorization")
print("All local integration checks passed. VM/systemd acceptance NOT performed.")
if __name__ == "__main__":
main()