Files
rust-discovery/tests/customer_transfer.py
T
kevin_janssen 2378a54bd4
Rust development slice / linux (push) Failing after 7s
Record production customer transfer availability
2026-11-18 21:47:27 +01:00

286 lines
15 KiB
Python

#!/usr/bin/env python3
"""Exercise a Linux pairing switch against isolated PostgreSQL and real TLS.
Pass the built rmm-server and rmm-agent binaries. Only disposable fixtures,
random local login credentials and loopback ports are used.
"""
import base64
import contextlib
import hashlib
import hmac
import http.client
import json
import os
from pathlib import Path
import secrets
import signal
import socket
import ssl
import subprocess
import sys
import tempfile
import time
import uuid
from local_access_database import command
ROOT = Path(__file__).resolve().parents[1]
A = "11111111-1111-4111-8111-111111111111"
B = "22222222-2222-4222-8222-222222222222"
IMAGE = "postgres:17.6-bookworm"
HOST = "rmm-transfer.test"
def until(probe, seconds=80):
for _ in range(seconds * 4):
value = probe()
if value:
return value
time.sleep(0.25)
raise AssertionError(f"Timed out waiting for {probe.__name__}")
def run(server, agent, folder, db, api):
pg_password = secrets.token_hex(24)
command(["docker", "run", "-d", "--name", db, "-p", "127.0.0.1::5432",
"-e", "POSTGRES_PASSWORD=" + pg_password, IMAGE])
def sql(statement, role="postgres"):
return command(["docker", "exec", "-i", db, "psql", "-XqAt", "-v",
"ON_ERROR_STOP=1", "-U", role, "-d", "postgres"],
statement.encode()).decode().strip()
until(lambda: b"init process complete" in command(["docker", "logs", db]), 30)
port = int(command(["docker", "port", db, "5432/tcp"]).decode().strip().split(":")[-1])
app_password = secrets.token_hex(24)
sql(f"""CREATE ROLE rmm_migrator LOGIN NOSUPERUSER NOBYPASSRLS;
CREATE ROLE rmm_app LOGIN NOINHERIT NOSUPERUSER NOBYPASSRLS PASSWORD '{app_password}';
ALTER DATABASE postgres OWNER TO rmm_migrator;
ALTER SCHEMA public OWNER TO rmm_migrator;""")
for migration in sorted((ROOT / "migrations").glob("*.sql")):
sql(migration.read_text(), "rmm_migrator")
users, login_data = [], {}
for username, home in [("owner", A), ("sourceonly", A), ("targetonly", B)]:
password = secrets.token_urlsafe(26)
secret = secrets.token_bytes(20)
salt = secrets.token_bytes(32)
principal = f"local:{uuid.uuid4()}"
digest = hashlib.scrypt(password.encode(), salt=salt, n=2**17, r=8, p=1,
maxmem=256 * 1024**2, dklen=64)
users.append({"username": username, "principal": principal, "tenant_id": home,
"disabled": False, "password_scrypt": {"salt_hex": salt.hex(), "hash_hex": digest.hex()},
"totp_secret_base32": base64.b32encode(secret).decode()})
login_data[username] = password, secret, home
sql(f"""INSERT INTO memberships(tenant_id,principal,kind,permissions) VALUES
('{home}','{principal}','human',ARRAY['inventory:read','devices:revoke','devices:enroll','tokens:manage']);""")
sql(f"SELECT provision_customer_admin('{A}','{users[0]['principal']}');")
local_auth = folder / "users.json"
local_auth.write_text(json.dumps({"schema_version": 1, "users": users}))
local_auth.chmod(0o600)
command(["docker", "run", "--rm", "--network=none", "-v", f"{folder}:/fixture",
"--entrypoint", "chown", IMAGE, "10001:10001", "/fixture/users.json"])
pki = folder / "pki"
pki.mkdir(mode=0o700)
command(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
"-nodes", "-days", "1", "-subj", "/CN=Transfer Fixture CA",
"-addext", "basicConstraints=critical,CA:TRUE", "-out", str(pki / "ca.pem"), "-keyout", str(pki / "ca.key")])
command(["openssl", "req", "-new", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:P-256",
"-nodes", "-subj", f"/CN={HOST}", "-addext", f"subjectAltName=DNS:{HOST}",
"-out", str(pki / "server.csr"), "-keyout", str(pki / "server.key")])
command(["openssl", "x509", "-req", "-in", str(pki / "server.csr"), "-CA", str(pki / "ca.pem"),
"-CAkey", str(pki / "ca.key"), "-set_serial", "2", "-days", "1", "-copy_extensions", "copy", "-out", str(pki / "server.pem")])
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
api_port = listener.getsockname()[1]
origin = f"https://{HOST}:{api_port}"
command(["docker", "run", "-d", "--name", api, "--network=host",
"-v", f"{folder}:/fixture", "-v", f"{server}:/rmm-server:ro",
"-e", "RMM_LISTEN_ADDR=127.0.0.1:" + str(api_port), "-e", "RMM_PROFILE=production",
"-e", "RMM_SERVER_ROLE=all", "-e", "RMM_TLS_DIR=/fixture/pki",
"-e", "RMM_LOCAL_AUTH_FILE=/fixture/users.json", "-e", "RMM_UI_ORIGIN=" + origin,
"-e", "RMM_JOB_KEY_FILE=/fixture/ed25519.pem",
"-e", f"DATABASE_URL=postgresql://rmm_app:{app_password}@127.0.0.1:{port}/postgres",
"--entrypoint", "/rmm-server", IMAGE])
context = ssl.create_default_context(cafile=pki / "ca.pem")
class HTTPS(http.client.HTTPSConnection):
def connect(self):
sock = socket.create_connection(("127.0.0.1", self.port), self.timeout)
self.sock = self._context.wrap_socket(sock, server_hostname=self.host)
def request(path, body=None, headers=None, tls=context):
conn = HTTPS(HOST, api_port, timeout=8, context=tls)
try:
h = {"Origin": origin, **(headers or {})}
if body is not None:
h["Content-Type"] = "application/json"
conn.request("POST" if body is not None else "GET", path, json.dumps(body) if body is not None else None, h)
result = conn.getresponse()
data = result.read()
try:
data = json.loads(data)
except ValueError:
pass
return result.status, data, result.getheader("Set-Cookie")
finally:
conn.close()
def healthy():
try:
return request("/healthz")[0] == 200
except OSError:
return False
until(healthy, 15)
def login(username):
password, secret, tenant = login_data[username]
code = hmac.digest(secret, (int(time.time()) // 30).to_bytes(8, "big"), "sha1")
offset = code[-1] & 15
totp = f"{(int.from_bytes(code[offset:offset+4], 'big') & 0x7fffffff) % 1000000:06d}"
status, _, cookie = request("/auth/local/login", {"username": username, "password": password, "totp_code": totp})
assert status == 200, ("fixture login", status)
headers = {"Cookie": cookie.split(";")[0], "X-RMM-Tenant": tenant}
status, session, _ = request("/auth/session", headers=headers)
assert status == 200
headers["X-CSRF-Token"] = session["csrf_token"]
return headers
owner, source_only, target_only = map(login, ("owner", "sourceonly", "targetonly"))
token = secrets.token_hex(32)
token_path = folder / "enrollment"
token_path.write_text(token)
token_path.chmod(0o600)
sql(f"INSERT INTO enrollment_tokens VALUES('{hashlib.sha256(token.encode()).hexdigest()}','{A}',now()+interval '15 minutes',NULL)")
state = folder / "state"
state.mkdir(mode=0o700)
guard = folder / "release.lock"
guard.touch(mode=0o644)
config = {
**os.environ, "RMM_AGENT_STATE": str(state), "RMM_CA_FILE": str(pki / "ca.pem"),
"RMM_CONNECT_ADDR": f"127.0.0.1:{api_port}", "RMM_SERVER_NAME": HOST, "RMM_TRANSPORT": "https",
"RMM_ENROLLMENT_TENANT": A, "RMM_ENROLLMENT_TOKEN_FILE": str(token_path),
"RMM_JOB_SIGNING_KEY_BOOTSTRAP": "1", "RMM_RELEASE_LOCK": str(guard),
"RMM_INSTALLED_RELEASE_VERSION": "transfer-fixture",
}
with (folder / "agent.log").open("w") as log:
proc = subprocess.Popen([agent], env=config, stdout=log, stderr=log)
try:
def current_source():
status, listing, _ = request(f"/api/v1/tenants/{A}/devices?view=current", headers=owner)
assert status == 200
return next((row for row in listing["items"]
if row.get("agent_version") == "transfer-fixture" and row.get("hostname")), None)
old = until(current_source, 30)
old_asset, old_agent = old["asset_id"], old["agent_id"]
old_context = ssl.create_default_context(cafile=pki / "ca.pem")
old_context.load_cert_chain(state / "device.pem", state / "device.key")
path = f"/api/v1/tenants/{A}/devices/{old_asset}"
transfer = path + "/customer-transfer"
query = "?agent_id=" + old_agent
suffix = lambda id, action: transfer + f"/{id}/{action}" + query
body = {"target_tenant_id": B}
assert request(transfer + query, body, source_only)[0] in (403, 404)
assert request(transfer + query, body, target_only)[0] == 404
assert request("/agent/v1/customer-transfer")[0] == 401
assert request(path.replace(A, B) + query, headers=target_only)[0] == 404
print("PASS limited sessions, direct IDs and unauthenticated device route")
proc.send_signal(signal.SIGSTOP)
try:
status, pending, _ = request(transfer + query, body, owner)
assert status == 200, (status, pending)
assert pending["state"] == "pending"
assert request(suffix(pending["transfer_id"], "complete"), {}, owner)[0] == 409
assert request(suffix(pending["transfer_id"], "cancel"), {}, owner)[0] == 204
status, pending, _ = request(transfer + query, body, owner)
assert status == 200 and pending["state"] == "pending"
finally:
proc.send_signal(signal.SIGCONT)
def target_ready():
status, dto, _ = request(transfer + query, headers=owner)
assert status == 200, (status, dto)
move = dto["transfer"]
return move if move["state"] == "ready" else None
moved = until(target_ready)
assert moved["target_hostname"] == old["hostname"]
target_agent, target_asset = moved["target_agent_id"], moved["target_asset_id"]
assert target_agent != old_agent and target_asset != old_asset
local = json.loads((state / "identity.json").read_text())
assert local["agent_id"] == target_agent
assert not list(state.glob("customer-move-*-prior-journal.sqlite"))
assert proc.poll() is None
print("PASS the existing Linux process received a signed handoff and reported fresh target inventory")
route = suffix(moved["transfer_id"], "complete")
assert request(route, {}, owner)[0] == 200
assert request(route, {}, owner)[0] == 200
assert request("/agent/v1/customer-transfer", tls=old_context)[0] == 401
status, historic, _ = request(f"/api/v1/tenants/{A}/devices?view=history", headers=source_only)
assert status == 200 and any(x["agent_id"] == old_agent and x["lifecycle"] == "revoked" for x in historic["items"])
status, detail, _ = request(f"/api/v1/tenants/{B}/devices/{target_asset}?agent_id={target_agent}", headers=target_only)
assert status == 200 and detail["tenant_id"] == B
assert request(path + query, headers=target_only)[0] == 404
assert request(f"/api/v1/tenants/{B}/devices/{target_asset}", headers=source_only)[0] == 404
assert sql("SELECT count(*) FROM jobs;") == "0"
assert sql(f"SELECT count(*) FROM observations WHERE tenant_id='{A}' AND agent_id='{old_agent}';") != "0"
print("PASS source certificate revoked, source history preserved and target inventory isolated")
def generation():
return int(sql(f"SELECT connection_generation FROM agent_installations WHERE agent_id='{target_agent}';"))
previous_generation = generation()
proc.terminate()
proc.wait(timeout=12)
proc = subprocess.Popen([agent], env=config, stdout=log, stderr=log)
until(lambda: generation() > previous_generation, 30)
active = json.loads((state / "identity.json").read_text())
assert active["agent_id"] == target_agent
assert request("/agent/v1/customer-transfer", tls=old_context)[0] == 401
print("PASS restart preserves the new pairing using the original installer configuration")
tokens = f"/api/v1/tenants/{A}/mcp-tokens"
for audience, allowed_tenants in [("selected", [A]), ("all", [])]:
status, issued, _ = request(tokens, {
"name": f"transfer fixture {audience}", "audience": audience,
"tenant_ids": allowed_tenants, "scopes": ["inventory:read"],
}, owner)
assert status == 200, ("token issuance", status)
bearer = {"Authorization": "Bearer " + issued["token"]}
status, fleet, _ = request("/api/v1/devices?view=current", headers=bearer)
assert status == 200
assert (target_agent in {row["agent_id"] for row in fleet["items"]}) == (audience == "all")
destination = f"/api/v1/tenants/{B}/devices/{target_asset}?agent_id={target_agent}"
assert request(destination, headers={**bearer, "X-RMM-Tenant": B})[0] == (200 if audience == "all" else 404)
assert request(transfer + query, headers={**bearer, "X-RMM-Tenant": A})[0] == 403
assert request(f"{tokens}/{issued['token_id']}/revoke", {}, owner)[0] == 200
assert request("/api/v1/devices?view=current", headers=bearer)[0] == 401
print("PASS scoped and global bearer lists, direct IDs, interactive transfer and token revocation")
finally:
proc.terminate()
proc.wait(timeout=12)
if __name__ == "__main__":
server, agent = (Path(arg).resolve(strict=True) for arg in sys.argv[1:])
suffix = uuid.uuid4().hex[:10]
db, api = f"rmm-transfer-db-{suffix}", f"rmm-transfer-api-{suffix}"
with tempfile.TemporaryDirectory(prefix="rmm-transfer-") as tmp:
directory = Path(tmp)
try:
run(server, agent, directory, db, api)
except Exception:
for name in (api,):
log = subprocess.run(["docker", "logs", "--tail", "15", name], capture_output=True, text=True)
print(log.stderr)
with contextlib.suppress(FileNotFoundError):
print((directory / "agent.log").read_text()[-1600:])
raise
finally:
subprocess.run(["docker", "rm", "-f", api, db], capture_output=True)