From 689f326460c02c5c589340c7573499ed84a725d1 Mon Sep 17 00:00:00 2001 From: Kevin Janssen Date: Tue, 17 Nov 2026 15:52:39 +0100 Subject: [PATCH] Track embedded migrations and verify the deployed schema and binary --- crates/persistence/build.rs | 4 + tests/backend_contract.py | 16 +++- tools/dev_deploy_server.py | 167 ++++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 crates/persistence/build.rs create mode 100644 tools/dev_deploy_server.py diff --git a/crates/persistence/build.rs b/crates/persistence/build.rs new file mode 100644 index 0000000..9cb0bb6 --- /dev/null +++ b/crates/persistence/build.rs @@ -0,0 +1,4 @@ +fn main() { + // sqlx embeds migrations. Cargo must also notice newly added SQL files. + println!("cargo:rerun-if-changed=../../migrations"); +} diff --git a/tests/backend_contract.py b/tests/backend_contract.py index 98e0f82..9a82426 100644 --- a/tests/backend_contract.py +++ b/tests/backend_contract.py @@ -47,7 +47,7 @@ def main(): viewer = (STATE / "viewer-a.token").read_text().strip() operator_b = (STATE / "operator-b.token").read_text().strip() asset, agent = str(uuid.uuid4()), str(uuid.uuid4()) - sql(f"INSERT INTO assets VALUES('{A}','{asset}'); INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,capabilities) VALUES('{A}','{agent}','{asset}','{secrets.token_hex(32)}','[\"run_script\",\"query_logs\"]');") + sql(f"INSERT INTO assets VALUES('{A}','{asset}'); INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,capabilities) VALUES('{A}','{agent}','{asset}','{secrets.token_hex(32)}','[\"linux.base/1\",\"run_script\",\"query_logs\"]');") base = f"/api/v1/tenants/{A}" try: now = datetime.datetime.now(datetime.timezone.utc) @@ -120,13 +120,25 @@ def main(): check(request("/mcp", "POST", {"jsonrpc": "2.0", "method": "notifications/initialized"}, operator, h)[0] == 202, "MCP initialized notification") result = request("/mcp", "POST", {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, operator, h)[2] names = [x["name"] for x in result["result"]["tools"]] - check(len(names) == 10 and not any("approv" in n or "authoriz" in n for n in names), "ten real MCP tools and no approval tool") + check({"tenants.list","inventory.list_devices","inventory.get_device","updates.list_releases","monitoring.get_health","logs.query","operations.plan","operations.submit","jobs.get","jobs.cancel","sessions.request","scripts.list","scripts.publish","scripts.plan"} <= set(names) and not any("approv" in n or "authoriz" in n for n in names), "inventory, operations and script library exposed through MCP with no approval tool") call = {"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "inventory.list_devices", "arguments": {"tenant_id": B}}} result = request("/mcp", "POST", call, operator, h)[2]["result"] check(result["isError"] and result["structuredContent"]["code"] == "NOT_FOUND", "MCP enforces same tenant boundary") call["params"] = {"name": "jobs.get", "arguments": {"tenant_id": A, "operation_id": pending["operation_id"]}} result = request("/mcp", "POST", call, operator, h)[2]["result"]["structuredContent"] check(result["approval_state"] == "pending_approval" and result["job_count"] == 0, "MCP jobs.get sees same persisted pending request") + library_input = {"name": "contract-" + uuid.uuid4().hex, "expected_version": 0, + "interpreter": "posix_sh", "execution_identity": "agent_service", "script_utf8": "printf 'catalog version 1\\n'\n"} + call["params"] = {"name": "scripts.publish", "arguments": {"tenant_id": A, "input": library_input}} + stored = request("/mcp", "POST", call, operator, h)[2]["result"]["structuredContent"] + check(stored["version"] == 1 and stored["script_sha256"] == hashlib.sha256(library_input["script_utf8"].encode()).hexdigest(), + "MCP publishes an immutable named revision") + selection = {"targets": plan_input["targets"], "schedule": {"not_before": now.isoformat(), "start_before": end.isoformat()}, + "rollout": plan_input["rollout"], "reason": "library contract", "max_runtime_seconds": 60, "max_output_bytes": 4096} + call["params"] = {"name": "scripts.plan", "arguments": {"tenant_id": A, "revision_id": stored["revision_id"], "input": selection}} + library_plan = request("/mcp", "POST", call, operator, h)[2]["result"]["structuredContent"] + reloaded = request(base + "/operation-plans/" + library_plan["plan_id"], token=operator)[2] + check(reloaded["plan_payload"]["script_revision"]["revision_id"] == stored["revision_id"], "WebUI reads the same immutable MCP revision plan") call["params"]={"name":"operations.plan","arguments":{"tenant_id":A,"input":plan_input}} mcp_plan=request("/mcp","POST",call,operator,h)[2]["result"]["structuredContent"] check(mcp_plan["jobs"][0]["asset_id"]==asset,"MCP planning uses same exact DTO and fixed job contract") diff --git a/tools/dev_deploy_server.py b/tools/dev_deploy_server.py new file mode 100644 index 0000000..1c44051 --- /dev/null +++ b/tools/dev_deploy_server.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Activate a tools/build.py ARM64 server and matching WebUI on the dev server.""" +import argparse +import base64 +import hashlib +import json +from pathlib import Path +import re +import subprocess +import sys + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def write_atomic(path, data, mode=0o644): + import os + import tempfile + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as stream: + tmp = Path(stream.name) + try: + stream.write(data) + stream.flush() + os.fchmod(stream.fileno(), mode) + os.replace(tmp, path) + finally: + tmp.unlink(missing_ok=True) + + +def web_files(revision): + """Build the exact Git revision using the local locked frontend dependencies.""" + import tempfile + root = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as folder: + work = Path(folder) + archive = work / 'src.tar' + subprocess.run(['git','archive','-o',archive,revision,'--','web','tests/browser'],cwd=root,check=True) + subprocess.run(['tar','-xf',archive,'-C',work],check=True) + (work/'web/node_modules').symlink_to(root/'web/node_modules',target_is_directory=True) + subprocess.run(['npm','run','build'],cwd=work/'web',check=True) + dist = work/'web/dist' + return {str(file.relative_to(dist)):base64.b64encode(file.read_bytes()).decode() + for file in dist.rglob('*') if file.is_file()} + + +def activate(payload): + import fcntl + import time + revision = payload['revision'] + if not re.fullmatch('[0-9a-f]{40}', revision): + raise ValueError('Invalid Git revision') + def run(*command): + return subprocess.check_output(list(command),text=True).strip() + def inspect(role): + return json.loads(run('docker','inspect',f'rust-discovery-server-{role}-1'))[0] + root = Path('/opt/rust-discovery') + work = root/'devstate/mvp-deploy' + work.mkdir(exist_ok=True,mode=0o700) + role_dir = work/revision + role_dir.mkdir(exist_ok=True,mode=0o700) + with (work/'deployment.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX) + matches = list((Path.home()/'rmm-dev-build/server-arm64/artifacts'/revision).glob('*/manifest.json')) + if len(matches) != 1: + raise RuntimeError('Expected one ARM64 build manifest for this revision') + manifest = json.loads(matches[0].read_bytes()) + binary = (matches[0].parent/'rmm-server').read_bytes() + if (manifest['revision'] != revision or manifest['target'] != 'aarch64-unknown-linux-gnu' + or sha(binary) != manifest['sha256']['rmm-server'] or sha(matches[0].read_bytes()) != matches[0].parent.name): + raise RuntimeError('Server build digest mismatch') + active = inspect('api') + if inspect('gateway')['Image'] != active['Image']: + raise RuntimeError('API and Gateway have different base images') + labels = active['Config']['Labels'] + if labels['com.docker.compose.project'] != 'rust-discovery-server': + raise RuntimeError('Unexpected compose project') + files = [Path(x) for x in labels['com.docker.compose.project.config_files'].split(',')] + base_files = [f for f in files if not (f.parent.parent == work and f.name == 'image.compose.yaml')] + compose = ['docker','compose','--project-name','rust-discovery-server','--project-directory',str(root/'deploy'), + '--env-file',str(root/'devstate/.env')] + for path in base_files: + if not path.is_file() or not path.resolve().is_relative_to(root): + raise ValueError('Missing deployment overlay') + compose.extend(['-f',str(path)]) + tag='rmm-dev-server:'+revision + context=role_dir/'image' + context.mkdir(exist_ok=True) + base_tag='rmm-dev-base:'+active['Image'].split(':')[1] + run('docker','tag',active['Image'],base_tag) + write_atomic(context/'Dockerfile',f'FROM {base_tag}\nCOPY rmm-server /usr/local/bin/rmm-server\n'.encode()) + write_atomic(context/'rmm-server',binary,0o555) + if subprocess.run(['docker','image','inspect',tag],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode: + run('docker','build','--network=none','--pull=false','-t',tag,str(context)) + overlay=role_dir/'image.compose.yaml' + write_atomic(overlay,('services:\n'+''.join(f' {r}:\n image: {tag}\n pull_policy: never\n' for r in ('migrate','api','gateway'))).encode()) + for path in payload['overlays']: + file=(root/'deploy')/path + if not re.fullmatch(r'[\w.-]+\.compose\.yaml',path) or not file.is_file(): + raise ValueError('Dev compose overlay must exist in the deployment directory') + if file not in base_files: + compose.extend(['-f',str(file)]) + compose.extend(['-f',str(overlay)]) + config=json.loads(run(*compose,'config','--format','json'))['services'] + for role,port in [('api','18443'),('gateway','18444')]: + svc=config[role] + if (svc['ports'][0]['host_ip']!='127.0.0.1' or str(svc['ports'][0]['published'])!=port or len(svc['ports'])!=1 + or svc['user']!='10001:10001' or not svc['read_only']): + raise RuntimeError('Dev admin/gateway boundary changed') + if config['postgres'].get('ports'): + raise RuntimeError('Unexpected public database') + dist=next(Path(m['Source']) for m in active['Mounts'] if m['Destination']=='/web') + assets={name:base64.b64decode(data,validate=True) for name,data in payload['web'].items()} + if 'index.html' not in assets or any(Path(n).is_absolute() or '..' in Path(n).parts for n in assets): + raise ValueError('Invalid WebUI asset set') + for name,data in assets.items(): + if name!='index.html': + write_atomic(dist/name,data) + run(*compose,'run','--no-deps','-T','--rm','migrate') + sql = ['docker','exec','rust-discovery-server-postgres-1','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm','-c'] + recorded = dict(line.split('|') for line in run(*sql,"SELECT version,encode(checksum,'hex') FROM _sqlx_migrations WHERE success").splitlines()) + if recorded != payload['migrations']: + raise RuntimeError('Applied migrations differ from the build revision') + run(*compose,'up','-d','--no-deps','--no-build','--pull','never','api','gateway') + for role,host,port,ca in [('api','localhost','18443','api-pki'),('gateway','discovery.ease-technologies.de','18444','pki')]: + for _ in range(25): + res=subprocess.run(['curl','--noproxy','*','--resolve',f'{host}:{port}:127.0.0.1','--cacert',str(root/f'devstate/{ca}/ca.pem'), + '--max-time','2','--silent','--fail',f'https://{host}:{port}/healthz'],capture_output=True) + if res.returncode==0 and res.stdout==b'ok': + break + time.sleep(1) + else: + raise RuntimeError(role+' health check failed') + live=run('docker','exec','rust-discovery-server-'+role+'-1','sha256sum','/usr/local/bin/rmm-server').split()[0] + if live!=sha(binary): + raise RuntimeError(role+' running digest differs') + write_atomic(dist/'index.html',assets['index.html']) + index=subprocess.check_output(['curl','--noproxy','*','--cacert',str(root/'devstate/api-pki/ca.pem'), + '--silent','--fail','https://localhost:18443/']) + if index!=assets['index.html']: + raise RuntimeError('Served index differs') + print(json.dumps(dict(revision=revision,server_sha256=sha(binary),web_index_sha256=sha(index), + image=inspect('api')['Image'],api_health='ok',gateway_health='ok'))) + + +def main(): + from dev_stage import ssh_prefix + import inspect + import shlex + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--ref',required=True) + parser.add_argument('--overlay',action='append',default=[],help='Additional existing compose file in remote deploy/') + args=parser.parse_args() + revision=subprocess.check_output(['git','rev-parse','--verify','--end-of-options',args.ref+'^{commit}'],text=True).strip() + web=web_files(revision) + source=Path(inspect.getfile(activate)).read_text().rsplit('\nif __name__ ==', 1)[0] + names=subprocess.check_output(['git','ls-tree','--name-only','-r',revision,'migrations'],text=True).splitlines() + migrations={str(int(Path(n).name.split('_',1)[0])):hashlib.sha384(subprocess.check_output(['git','show',revision+':'+n])).hexdigest() + for n in names if n.endswith('.sql')} + payload=json.dumps(dict(revision=revision,web=web,overlays=args.overlay,migrations=migrations)) + cmd=source+'\nactivate(json.loads('+repr(payload)+'))\n' + subprocess.run(ssh_prefix('server')+[shlex.join(['python3','-I','-'])],input=cmd,text=True,check=True) + + +if __name__ == '__main__': + main()