285 lines
16 KiB
Python
285 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Real local TLS/PostgreSQL/agent checks. Does not read VM credentials."""
|
|
import concurrent.futures
|
|
import base64
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import socket
|
|
import ssl
|
|
import struct
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
STATE = ROOT / "devstate"
|
|
CA = STATE / "pki/ca.pem"
|
|
A = "11111111-1111-4111-8111-111111111111"
|
|
B = "22222222-2222-4222-8222-222222222222"
|
|
|
|
|
|
def sql(query, user="postgres", fail=False):
|
|
result = subprocess.run(["docker", "compose", "--env-file", str(STATE / ".env"), "-f", str(ROOT / "deploy/compose.yaml"), "-p", "rust-discovery-dev", "exec", "-T", "postgres", "psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", "rmm"], input=query, text=True, capture_output=True)
|
|
assert (result.returncode != 0) if fail else (result.returncode == 0), "database assertion failed (details suppressed)"
|
|
return result.stdout.strip()
|
|
|
|
|
|
class LocalHTTPS(http.client.HTTPSConnection):
|
|
def connect(self):
|
|
tcp = socket.create_connection(("127.0.0.1", 18443), timeout=10)
|
|
self.sock = self._context.wrap_socket(tcp, server_hostname=self.host)
|
|
|
|
|
|
def request(path, token=None, body=None, context=None, hostname="rmm-dev.internal"):
|
|
client = LocalHTTPS(hostname, 18443, context=context or ssl.create_default_context(cafile=str(CA)), timeout=10)
|
|
headers = {}
|
|
if token:
|
|
headers["Authorization"] = "Bearer " + token
|
|
if body is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
client.request("POST" if body is not None else "GET", path, body=json.dumps(body) if body is not None else None, headers=headers)
|
|
response = client.getresponse()
|
|
data = response.read()
|
|
client.close()
|
|
return response.status, data
|
|
|
|
|
|
def token(tenant):
|
|
raw = secrets.token_hex(32)
|
|
digest = hashlib.sha256(raw.encode()).hexdigest()
|
|
sql(f"INSERT INTO enrollment_tokens VALUES('{digest}','{tenant}',now()+interval '15 minutes',NULL);")
|
|
return raw
|
|
|
|
|
|
def check(condition, message):
|
|
assert condition, message
|
|
print("PASS", message)
|
|
|
|
|
|
def varint(number):
|
|
value = bytearray()
|
|
while number > 127:
|
|
value.append((number & 127) | 128)
|
|
number >>= 7
|
|
value.append(number)
|
|
return bytes(value)
|
|
|
|
|
|
def field(tag, value):
|
|
if isinstance(value, int):
|
|
return varint(tag << 3) + varint(value)
|
|
value = value.encode() if isinstance(value, str) else value
|
|
return varint((tag << 3) | 2) + varint(len(value)) + value
|
|
|
|
|
|
def envelope(agent, sequence, generation, payload_tag, payload, major=1, observed=None):
|
|
import uuid
|
|
return b"".join([field(1, major), field(3, str(uuid.uuid4())), field(4, agent), field(5, 1), field(6, generation), field(7, "33333333-3333-4333-8333-333333333333"), field(8, sequence), field(9, observed if observed is not None else int(time.time() * 1000)), field(payload_tag, payload)])
|
|
|
|
|
|
class WebSocket:
|
|
def __init__(self, context):
|
|
tcp = socket.create_connection(("127.0.0.1", 18443), timeout=10)
|
|
self.sock = context.wrap_socket(tcp, server_hostname="rmm-dev.internal")
|
|
key = base64.b64encode(secrets.token_bytes(16)).decode()
|
|
self.sock.sendall(("GET /agent/v1/connect HTTP/1.1\r\nHost: rmm-dev.internal:18443\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: " + key + "\r\nX-Client-Cert: forged\r\n\r\n").encode())
|
|
headers = b""
|
|
while not headers.endswith(b"\r\n\r\n"):
|
|
byte = self.sock.recv(1)
|
|
assert byte and len(headers) < 16384
|
|
headers += byte
|
|
self.status = int(headers.split(b" ")[1])
|
|
|
|
def send(self, payload):
|
|
mask = secrets.token_bytes(4)
|
|
size = len(payload)
|
|
length = bytes([size | 128]) if size < 126 else (bytes([126 | 128]) + struct.pack("!H", size)) if size < 65536 else (bytes([127 | 128]) + struct.pack("!Q", size))
|
|
self.sock.sendall(b"\x82" + length + mask + bytes(x ^ mask[i % 4] for i, x in enumerate(payload)))
|
|
|
|
def exact(self, count):
|
|
result = b""
|
|
while len(result) < count:
|
|
chunk = self.sock.recv(count - len(result))
|
|
if not chunk:
|
|
raise EOFError()
|
|
result += chunk
|
|
return result
|
|
|
|
def receive(self):
|
|
opcode, size = self.exact(2)
|
|
size &= 127
|
|
if size == 126:
|
|
size = struct.unpack("!H", self.exact(2))[0]
|
|
elif size == 127:
|
|
size = struct.unpack("!Q", self.exact(8))[0]
|
|
assert size <= 1048576
|
|
return opcode & 15, self.exact(size)
|
|
|
|
def close(self):
|
|
self.sock.close()
|
|
|
|
|
|
def rejected_frame(context, payload):
|
|
ws = WebSocket(context)
|
|
try:
|
|
assert ws.status == 101
|
|
try:
|
|
# The server may reject an oversized frame from its header before
|
|
# the client has finished sending the body.
|
|
ws.send(payload)
|
|
opcode, _ = ws.receive()
|
|
assert opcode == 8, "invalid protocol input was not closed"
|
|
except (EOFError, ssl.SSLError, ConnectionResetError, BrokenPipeError):
|
|
pass
|
|
finally:
|
|
ws.close()
|
|
|
|
|
|
def main():
|
|
production = subprocess.run([str(ROOT / "target/debug/rmm-server")], env=dict(os.environ, RMM_PROFILE="production", RMM_DEV_AUTH="true"), capture_output=True)
|
|
check(production.returncode != 0 and b"production forbids dev auth" in production.stderr, "production rejects explicit dev auth before connecting")
|
|
viewer_a = (STATE / "viewer-a.token").read_text().strip()
|
|
viewer_b = (STATE / "viewer-b.token").read_text().strip()
|
|
deadline = time.monotonic() + 15
|
|
while True:
|
|
try:
|
|
request("/healthz")
|
|
break
|
|
except (ConnectionRefusedError, ConnectionResetError):
|
|
if time.monotonic() >= deadline:
|
|
raise
|
|
time.sleep(0.25)
|
|
check(request("/healthz")[0] == 200, "trusted TLS health")
|
|
for context, hostname in [(ssl.create_default_context(), "rmm-dev.internal"), (ssl.create_default_context(cafile=str(CA)), "wrong.internal")]:
|
|
try:
|
|
request("/healthz", context=context, hostname=hostname)
|
|
except ssl.SSLCertVerificationError:
|
|
pass
|
|
else:
|
|
raise AssertionError("TLS validation did not reject invalid trust/name")
|
|
print("PASS untrusted CA and wrong DNS name rejected")
|
|
check(request("/api/v1/tenants")[0] == 401, "anonymous API denied")
|
|
check(request("/api/v1/tenants", "invalid")[0] == 401, "invalid bearer denied")
|
|
status, error = request("/api/v1/tenants", "invalid")
|
|
check(status == 401 and json.loads(error)["code"] == "UNAUTHENTICATED" and "correlation_id" in json.loads(error), "structured errors without request echo")
|
|
for bearer, tenant, foreign in [(viewer_a, A, B), (viewer_b, B, A)]:
|
|
status, data = request("/api/v1/tenants", bearer)
|
|
check(status == 200 and [x["tenant_id"] for x in json.loads(data)["items"]] == [tenant], "viewer sees exactly assigned tenant")
|
|
check(request(f"/api/v1/tenants/{foreign}/devices", bearer)[0] == 404, "foreign tenant denied")
|
|
check(sql("SELECT count(*) FROM tenants;", "rmm_app") == "0", "RLS denies missing context")
|
|
result = sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); SELECT count(*) FROM tenants; ROLLBACK; SELECT count(*) FROM tenants;", "rmm_app")
|
|
check(result.splitlines()[-2:] == ["1", "0"], "transaction-local tenant reset after rollback")
|
|
sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); INSERT INTO assets VALUES('{B}','{secrets.token_hex(16)}');", "rmm_app", fail=True)
|
|
print("PASS RLS rejects cross-tenant insert")
|
|
role = sql("SELECT rolsuper OR rolbypassrls FROM pg_roles WHERE rolname='rmm_app';")
|
|
check(role == "f", "application role has neither superuser nor BYPASSRLS")
|
|
check(sql("SELECT count(*) FROM pg_tables WHERE schemaname='public' AND tableowner='rmm_app';") == "0", "application owns no tables")
|
|
with tempfile.TemporaryDirectory(prefix="rmm-e2e-", dir=STATE) as folder:
|
|
folder = Path(folder)
|
|
subprocess.run(["openssl", "req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", "-keyout", str(folder / "device.key"), "-out", str(folder / "device.csr"), "-subj", "/CN=e2e"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
csr = (folder / "device.csr").read_text()
|
|
value = token(A)
|
|
# Invalid CSR rolls consumption back.
|
|
check(request("/agent/v1/enroll", body={"token": value, "csr_pem": "invalid"})[0] == 400, "invalid CSR rejected")
|
|
body = {"token": value, "csr_pem": csr}
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
|
replies = list(pool.map(lambda _: request("/agent/v1/enroll", body=body), range(2)))
|
|
check(sorted(x[0] for x in replies) == [200, 401], "enrollment token consumed atomically exactly once")
|
|
enrolled = json.loads(next(data for status, data in replies if status == 200))
|
|
(folder / "device.pem").write_text(enrolled["certificate_pem"])
|
|
context = ssl.create_default_context(cafile=str(CA))
|
|
context.load_cert_chain(folder / "device.pem", folder / "device.key")
|
|
check(request("/healthz", context=context)[0] == 200, "issued client certificate passes TLS validation")
|
|
anonymous = WebSocket(ssl.create_default_context(cafile=str(CA)))
|
|
check(anonymous.status == 401, "WSS denies missing peer identity despite forged certificate header")
|
|
anonymous.close()
|
|
rejected_frame(context, envelope(enrolled["agent_id"], 1, 0, 10, field(1, "test"), major=2))
|
|
print("PASS unknown protocol major closes channel")
|
|
rejected_frame(context, envelope("44444444-4444-4444-8444-444444444444", 1, 0, 10, field(1, "test")))
|
|
print("PASS TLS device cannot claim another agent ID")
|
|
ws = WebSocket(context)
|
|
try:
|
|
check(ws.status == 101, "registered certificate upgrades to WSS")
|
|
ws.send(envelope(enrolled["agent_id"], 1, 0, 10, field(1, "test")))
|
|
check(ws.receive()[0] == 2, "server hello is binary protobuf")
|
|
generation = int(sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{enrolled['agent_id']}';"))
|
|
ws.send(envelope(enrolled["agent_id"], 2, generation, 12, field(1, "55555555-5555-4555-8555-555555555555"), observed=int(time.time() * 1000) - 600000))
|
|
check(ws.receive()[0] == 8, "stale buffered heartbeat closes channel")
|
|
check(sql(f"SELECT last_contact_at IS NULL FROM agent_installations WHERE agent_id='{enrolled['agent_id']}';") == "t", "stale heartbeat cannot mark presence online")
|
|
finally:
|
|
ws.close()
|
|
rejected_frame(context, b"x" * (1048576 + 1))
|
|
print("PASS oversized WebSocket message rejected")
|
|
# Actual agent enrollment, WSS, inventory and heartbeat, with bounded wait.
|
|
agent_state = folder / "agent"
|
|
token_path = folder / "enroll.token"
|
|
token_path.write_text(token(A))
|
|
token_path.chmod(0o600)
|
|
env = dict(os.environ, RMM_AGENT_STATE=str(agent_state), RMM_CA_FILE=str(CA), RMM_ENROLLMENT_TOKEN_FILE=str(token_path),RMM_EXTENDED_INVENTORY="1")
|
|
with open(folder / "agent.log", "wb") as log:
|
|
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
|
|
try:
|
|
deadline = time.monotonic() + 55
|
|
observed = None
|
|
while time.monotonic() < deadline:
|
|
status, data = request(f"/api/v1/tenants/{A}/devices", viewer_a)
|
|
assert status == 200
|
|
items = json.loads(data)["items"]
|
|
# Never select/revoke a concurrently running VM or another
|
|
# integration worker's device merely because it is online.
|
|
own_identity = agent_state / "identity.json"
|
|
own = json.loads(own_identity.read_text()) if own_identity.exists() else None
|
|
observed = None
|
|
if own:
|
|
detail_status, detail = request(f"/api/v1/tenants/{A}/devices/{own['asset_id']}", viewer_a)
|
|
candidate = json.loads(detail) if detail_status == 200 else None
|
|
if candidate and candidate["agent_id"] == own["agent_id"] and candidate["inventory"] and candidate["presence"] == "online":
|
|
observed = candidate
|
|
if observed:
|
|
break
|
|
assert agent.poll() is None, "agent exited (logs remain private)"
|
|
time.sleep(1)
|
|
check(observed is not None, "real Linux agent enrolled, mTLS WSS inventory stored and heartbeat online")
|
|
check(observed["inventory"]["root_available_bytes"] > 0 and observed["observed_at"] and observed["received_at"], "real disk value and observation timestamps")
|
|
check(isinstance(observed["inventory"].get("observations"),list) and len(observed["inventory"]["observations"])>0 and observed["inventory"].get("collection_completed_at"),"real native extended collectors stored with bounded per-source metadata")
|
|
status, data = request(f"/api/v1/tenants/{B}/devices", viewer_b)
|
|
check(status == 200 and observed["agent_id"] not in data.decode(), "tenant B cannot see tenant A telemetry")
|
|
check((agent_state / "device.key").stat().st_mode & 0o777 == 0o600, "local private device key is 0600")
|
|
agent_id = observed["agent_id"]
|
|
generation = sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{agent_id}';")
|
|
agent.terminate()
|
|
agent.wait(timeout=10)
|
|
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
|
|
deadline = time.monotonic() + 15
|
|
while time.monotonic() < deadline:
|
|
current = sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{agent_id}';")
|
|
if int(current) > int(generation):
|
|
break
|
|
time.sleep(1)
|
|
check(int(current) > int(generation), "restart reuses local identity and fences old channel")
|
|
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{agent_id}';")
|
|
before = sql(f"SELECT count(*) FROM observations WHERE agent_id='{agent_id}';")
|
|
agent.terminate()
|
|
agent.wait(timeout=10)
|
|
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
|
|
time.sleep(3)
|
|
check(sql(f"SELECT count(*) FROM observations WHERE agent_id='{agent_id}';") == before, "revoked identity cannot ingest after reconnect")
|
|
finally:
|
|
agent.terminate()
|
|
agent.wait(timeout=10)
|
|
if (agent_state / "identity.json").exists():
|
|
identity = json.loads((agent_state / "identity.json").read_text())
|
|
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{identity['agent_id']}';")
|
|
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{enrolled['agent_id']}';")
|
|
check(sql("SELECT count(*) FROM information_schema.tables WHERE table_name IN ('jobs','outbox','execution_permits');") == "3", "durable job/outbox/grant schema present")
|
|
check(sql("SELECT count(*) FROM jobs j LEFT JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id LEFT JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE a.authorization_id IS NULL;") == "0", "no executable job exists without durable human authorization")
|
|
print("All local integration checks passed. VM/systemd acceptance NOT performed.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|