feat: snapshot Linux-first RMM MVP and pause handover
Rust development slice / linux (push) Failing after 1m10s
Rust development slice / linux (push) Failing after 1m10s
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real local TLS/PostgreSQL/agent checks. Does not read VM credentials."""
|
||||
import concurrent.futures
|
||||
import base64
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import socket
|
||||
import ssl
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STATE = ROOT / "devstate"
|
||||
CA = STATE / "pki/ca.pem"
|
||||
A = "11111111-1111-4111-8111-111111111111"
|
||||
B = "22222222-2222-4222-8222-222222222222"
|
||||
|
||||
|
||||
def sql(query, user="postgres", fail=False):
|
||||
result = subprocess.run(["docker", "compose", "--env-file", str(STATE / ".env"), "-f", str(ROOT / "deploy/compose.yaml"), "-p", "rust-discovery-dev", "exec", "-T", "postgres", "psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", user, "-d", "rmm"], input=query, text=True, capture_output=True)
|
||||
assert (result.returncode != 0) if fail else (result.returncode == 0), "database assertion failed (details suppressed)"
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
class LocalHTTPS(http.client.HTTPSConnection):
|
||||
def connect(self):
|
||||
tcp = socket.create_connection(("127.0.0.1", 18443), timeout=10)
|
||||
self.sock = self._context.wrap_socket(tcp, server_hostname=self.host)
|
||||
|
||||
|
||||
def request(path, token=None, body=None, context=None, hostname="rmm-dev.internal"):
|
||||
client = LocalHTTPS(hostname, 18443, context=context or ssl.create_default_context(cafile=str(CA)), timeout=10)
|
||||
headers = {}
|
||||
if token:
|
||||
headers["Authorization"] = "Bearer " + token
|
||||
if body is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
client.request("POST" if body is not None else "GET", path, body=json.dumps(body) if body is not None else None, headers=headers)
|
||||
response = client.getresponse()
|
||||
data = response.read()
|
||||
client.close()
|
||||
return response.status, data
|
||||
|
||||
|
||||
def token(tenant):
|
||||
raw = secrets.token_hex(32)
|
||||
digest = hashlib.sha256(raw.encode()).hexdigest()
|
||||
sql(f"INSERT INTO enrollment_tokens VALUES('{digest}','{tenant}',now()+interval '15 minutes',NULL);")
|
||||
return raw
|
||||
|
||||
|
||||
def check(condition, message):
|
||||
assert condition, message
|
||||
print("PASS", message)
|
||||
|
||||
|
||||
def varint(number):
|
||||
value = bytearray()
|
||||
while number > 127:
|
||||
value.append((number & 127) | 128)
|
||||
number >>= 7
|
||||
value.append(number)
|
||||
return bytes(value)
|
||||
|
||||
|
||||
def field(tag, value):
|
||||
if isinstance(value, int):
|
||||
return varint(tag << 3) + varint(value)
|
||||
value = value.encode() if isinstance(value, str) else value
|
||||
return varint((tag << 3) | 2) + varint(len(value)) + value
|
||||
|
||||
|
||||
def envelope(agent, sequence, generation, payload_tag, payload, major=1, observed=None):
|
||||
import uuid
|
||||
return b"".join([field(1, major), field(3, str(uuid.uuid4())), field(4, agent), field(5, 1), field(6, generation), field(7, "33333333-3333-4333-8333-333333333333"), field(8, sequence), field(9, observed if observed is not None else int(time.time() * 1000)), field(payload_tag, payload)])
|
||||
|
||||
|
||||
class WebSocket:
|
||||
def __init__(self, context):
|
||||
tcp = socket.create_connection(("127.0.0.1", 18443), timeout=10)
|
||||
self.sock = context.wrap_socket(tcp, server_hostname="rmm-dev.internal")
|
||||
key = base64.b64encode(secrets.token_bytes(16)).decode()
|
||||
self.sock.sendall(("GET /agent/v1/connect HTTP/1.1\r\nHost: rmm-dev.internal:18443\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: " + key + "\r\nX-Client-Cert: forged\r\n\r\n").encode())
|
||||
headers = b""
|
||||
while not headers.endswith(b"\r\n\r\n"):
|
||||
byte = self.sock.recv(1)
|
||||
assert byte and len(headers) < 16384
|
||||
headers += byte
|
||||
self.status = int(headers.split(b" ")[1])
|
||||
|
||||
def send(self, payload):
|
||||
mask = secrets.token_bytes(4)
|
||||
size = len(payload)
|
||||
length = bytes([size | 128]) if size < 126 else (bytes([126 | 128]) + struct.pack("!H", size)) if size < 65536 else (bytes([127 | 128]) + struct.pack("!Q", size))
|
||||
self.sock.sendall(b"\x82" + length + mask + bytes(x ^ mask[i % 4] for i, x in enumerate(payload)))
|
||||
|
||||
def exact(self, count):
|
||||
result = b""
|
||||
while len(result) < count:
|
||||
chunk = self.sock.recv(count - len(result))
|
||||
if not chunk:
|
||||
raise EOFError()
|
||||
result += chunk
|
||||
return result
|
||||
|
||||
def receive(self):
|
||||
opcode, size = self.exact(2)
|
||||
size &= 127
|
||||
if size == 126:
|
||||
size = struct.unpack("!H", self.exact(2))[0]
|
||||
elif size == 127:
|
||||
size = struct.unpack("!Q", self.exact(8))[0]
|
||||
assert size <= 1048576
|
||||
return opcode & 15, self.exact(size)
|
||||
|
||||
def close(self):
|
||||
self.sock.close()
|
||||
|
||||
|
||||
def rejected_frame(context, payload):
|
||||
ws = WebSocket(context)
|
||||
try:
|
||||
assert ws.status == 101
|
||||
try:
|
||||
# The server may reject an oversized frame from its header before
|
||||
# the client has finished sending the body.
|
||||
ws.send(payload)
|
||||
opcode, _ = ws.receive()
|
||||
assert opcode == 8, "invalid protocol input was not closed"
|
||||
except (EOFError, ssl.SSLError, ConnectionResetError, BrokenPipeError):
|
||||
pass
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
|
||||
def main():
|
||||
production = subprocess.run([str(ROOT / "target/debug/rmm-server")], env=dict(os.environ, RMM_PROFILE="production", RMM_DEV_AUTH="true"), capture_output=True)
|
||||
check(production.returncode != 0 and b"production forbids dev auth" in production.stderr, "production rejects explicit dev auth before connecting")
|
||||
viewer_a = (STATE / "viewer-a.token").read_text().strip()
|
||||
viewer_b = (STATE / "viewer-b.token").read_text().strip()
|
||||
deadline = time.monotonic() + 15
|
||||
while True:
|
||||
try:
|
||||
request("/healthz")
|
||||
break
|
||||
except (ConnectionRefusedError, ConnectionResetError):
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
time.sleep(0.25)
|
||||
check(request("/healthz")[0] == 200, "trusted TLS health")
|
||||
for context, hostname in [(ssl.create_default_context(), "rmm-dev.internal"), (ssl.create_default_context(cafile=str(CA)), "wrong.internal")]:
|
||||
try:
|
||||
request("/healthz", context=context, hostname=hostname)
|
||||
except ssl.SSLCertVerificationError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("TLS validation did not reject invalid trust/name")
|
||||
print("PASS untrusted CA and wrong DNS name rejected")
|
||||
check(request("/api/v1/tenants")[0] == 401, "anonymous API denied")
|
||||
check(request("/api/v1/tenants", "invalid")[0] == 401, "invalid bearer denied")
|
||||
status, error = request("/api/v1/tenants", "invalid")
|
||||
check(status == 401 and json.loads(error)["code"] == "UNAUTHENTICATED" and "correlation_id" in json.loads(error), "structured errors without request echo")
|
||||
for bearer, tenant, foreign in [(viewer_a, A, B), (viewer_b, B, A)]:
|
||||
status, data = request("/api/v1/tenants", bearer)
|
||||
check(status == 200 and [x["tenant_id"] for x in json.loads(data)["items"]] == [tenant], "viewer sees exactly assigned tenant")
|
||||
check(request(f"/api/v1/tenants/{foreign}/devices", bearer)[0] == 404, "foreign tenant denied")
|
||||
check(sql("SELECT count(*) FROM tenants;", "rmm_app") == "0", "RLS denies missing context")
|
||||
result = sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); SELECT count(*) FROM tenants; ROLLBACK; SELECT count(*) FROM tenants;", "rmm_app")
|
||||
check(result.splitlines()[-2:] == ["1", "0"], "transaction-local tenant reset after rollback")
|
||||
sql(f"BEGIN; SELECT set_config('app.tenant_id','{A}',true); INSERT INTO assets VALUES('{B}','{secrets.token_hex(16)}');", "rmm_app", fail=True)
|
||||
print("PASS RLS rejects cross-tenant insert")
|
||||
role = sql("SELECT rolsuper OR rolbypassrls FROM pg_roles WHERE rolname='rmm_app';")
|
||||
check(role == "f", "application role has neither superuser nor BYPASSRLS")
|
||||
check(sql("SELECT count(*) FROM pg_tables WHERE schemaname='public' AND tableowner='rmm_app';") == "0", "application owns no tables")
|
||||
with tempfile.TemporaryDirectory(prefix="rmm-e2e-", dir=STATE) as folder:
|
||||
folder = Path(folder)
|
||||
subprocess.run(["openssl", "req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", "-keyout", str(folder / "device.key"), "-out", str(folder / "device.csr"), "-subj", "/CN=e2e"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
csr = (folder / "device.csr").read_text()
|
||||
value = token(A)
|
||||
# Invalid CSR rolls consumption back.
|
||||
check(request("/agent/v1/enroll", body={"token": value, "csr_pem": "invalid"})[0] == 400, "invalid CSR rejected")
|
||||
body = {"token": value, "csr_pem": csr}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
replies = list(pool.map(lambda _: request("/agent/v1/enroll", body=body), range(2)))
|
||||
check(sorted(x[0] for x in replies) == [200, 401], "enrollment token consumed atomically exactly once")
|
||||
enrolled = json.loads(next(data for status, data in replies if status == 200))
|
||||
(folder / "device.pem").write_text(enrolled["certificate_pem"])
|
||||
context = ssl.create_default_context(cafile=str(CA))
|
||||
context.load_cert_chain(folder / "device.pem", folder / "device.key")
|
||||
check(request("/healthz", context=context)[0] == 200, "issued client certificate passes TLS validation")
|
||||
anonymous = WebSocket(ssl.create_default_context(cafile=str(CA)))
|
||||
check(anonymous.status == 401, "WSS denies missing peer identity despite forged certificate header")
|
||||
anonymous.close()
|
||||
rejected_frame(context, envelope(enrolled["agent_id"], 1, 0, 10, field(1, "test"), major=2))
|
||||
print("PASS unknown protocol major closes channel")
|
||||
rejected_frame(context, envelope("44444444-4444-4444-8444-444444444444", 1, 0, 10, field(1, "test")))
|
||||
print("PASS TLS device cannot claim another agent ID")
|
||||
ws = WebSocket(context)
|
||||
try:
|
||||
check(ws.status == 101, "registered certificate upgrades to WSS")
|
||||
ws.send(envelope(enrolled["agent_id"], 1, 0, 10, field(1, "test")))
|
||||
check(ws.receive()[0] == 2, "server hello is binary protobuf")
|
||||
generation = int(sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{enrolled['agent_id']}';"))
|
||||
ws.send(envelope(enrolled["agent_id"], 2, generation, 12, field(1, "55555555-5555-4555-8555-555555555555"), observed=int(time.time() * 1000) - 600000))
|
||||
check(ws.receive()[0] == 8, "stale buffered heartbeat closes channel")
|
||||
check(sql(f"SELECT last_contact_at IS NULL FROM agent_installations WHERE agent_id='{enrolled['agent_id']}';") == "t", "stale heartbeat cannot mark presence online")
|
||||
finally:
|
||||
ws.close()
|
||||
rejected_frame(context, b"x" * (1048576 + 1))
|
||||
print("PASS oversized WebSocket message rejected")
|
||||
# Actual agent enrollment, WSS, inventory and heartbeat, with bounded wait.
|
||||
agent_state = folder / "agent"
|
||||
token_path = folder / "enroll.token"
|
||||
token_path.write_text(token(A))
|
||||
token_path.chmod(0o600)
|
||||
env = dict(os.environ, RMM_AGENT_STATE=str(agent_state), RMM_CA_FILE=str(CA), RMM_ENROLLMENT_TOKEN_FILE=str(token_path),RMM_EXTENDED_INVENTORY="1")
|
||||
with open(folder / "agent.log", "wb") as log:
|
||||
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
|
||||
try:
|
||||
deadline = time.monotonic() + 55
|
||||
observed = None
|
||||
while time.monotonic() < deadline:
|
||||
status, data = request(f"/api/v1/tenants/{A}/devices", viewer_a)
|
||||
assert status == 200
|
||||
items = json.loads(data)["items"]
|
||||
# Never select/revoke a concurrently running VM or another
|
||||
# integration worker's device merely because it is online.
|
||||
own_identity = agent_state / "identity.json"
|
||||
own = json.loads(own_identity.read_text()) if own_identity.exists() else None
|
||||
observed = None
|
||||
if own:
|
||||
detail_status, detail = request(f"/api/v1/tenants/{A}/devices/{own['asset_id']}", viewer_a)
|
||||
candidate = json.loads(detail) if detail_status == 200 else None
|
||||
if candidate and candidate["agent_id"] == own["agent_id"] and candidate["inventory"] and candidate["presence"] == "online":
|
||||
observed = candidate
|
||||
if observed:
|
||||
break
|
||||
assert agent.poll() is None, "agent exited (logs remain private)"
|
||||
time.sleep(1)
|
||||
check(observed is not None, "real Linux agent enrolled, mTLS WSS inventory stored and heartbeat online")
|
||||
check(observed["inventory"]["root_available_bytes"] > 0 and observed["observed_at"] and observed["received_at"], "real disk value and observation timestamps")
|
||||
check(isinstance(observed["inventory"].get("observations"),list) and len(observed["inventory"]["observations"])>0 and observed["inventory"].get("collection_completed_at"),"real native extended collectors stored with bounded per-source metadata")
|
||||
status, data = request(f"/api/v1/tenants/{B}/devices", viewer_b)
|
||||
check(status == 200 and observed["agent_id"] not in data.decode(), "tenant B cannot see tenant A telemetry")
|
||||
check((agent_state / "device.key").stat().st_mode & 0o777 == 0o600, "local private device key is 0600")
|
||||
agent_id = observed["agent_id"]
|
||||
generation = sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{agent_id}';")
|
||||
agent.terminate()
|
||||
agent.wait(timeout=10)
|
||||
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
current = sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{agent_id}';")
|
||||
if int(current) > int(generation):
|
||||
break
|
||||
time.sleep(1)
|
||||
check(int(current) > int(generation), "restart reuses local identity and fences old channel")
|
||||
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{agent_id}';")
|
||||
before = sql(f"SELECT count(*) FROM observations WHERE agent_id='{agent_id}';")
|
||||
agent.terminate()
|
||||
agent.wait(timeout=10)
|
||||
agent = subprocess.Popen([str(ROOT / "target/debug/rmm-agent")], env=env, stdout=log, stderr=log)
|
||||
time.sleep(3)
|
||||
check(sql(f"SELECT count(*) FROM observations WHERE agent_id='{agent_id}';") == before, "revoked identity cannot ingest after reconnect")
|
||||
finally:
|
||||
agent.terminate()
|
||||
agent.wait(timeout=10)
|
||||
if (agent_state / "identity.json").exists():
|
||||
identity = json.loads((agent_state / "identity.json").read_text())
|
||||
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{identity['agent_id']}';")
|
||||
sql(f"UPDATE agent_installations SET revoked_at=now() WHERE agent_id='{enrolled['agent_id']}';")
|
||||
check(sql("SELECT count(*) FROM information_schema.tables WHERE table_name IN ('jobs','outbox','execution_permits');") == "3", "durable job/outbox/grant schema present")
|
||||
check(sql("SELECT count(*) FROM jobs j LEFT JOIN operations o ON o.tenant_id=j.tenant_id AND o.operation_id=j.operation_id LEFT JOIN authorizations a ON a.tenant_id=o.tenant_id AND a.authorization_id=o.authorization_id WHERE a.authorization_id IS NULL;") == "0", "no executable job exists without durable human authorization")
|
||||
print("All local integration checks passed. VM/systemd acceptance NOT performed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user