Track embedded migrations and verify the deployed schema and binary

This commit is contained in:
2026-11-17 15:52:39 +01:00
parent 77c747fc82
commit 689f326460
3 changed files with 185 additions and 2 deletions
+4
View File
@@ -0,0 +1,4 @@
fn main() {
// sqlx embeds migrations. Cargo must also notice newly added SQL files.
println!("cargo:rerun-if-changed=../../migrations");
}
+14 -2
View File
@@ -47,7 +47,7 @@ def main():
viewer = (STATE / "viewer-a.token").read_text().strip()
operator_b = (STATE / "operator-b.token").read_text().strip()
asset, agent = str(uuid.uuid4()), str(uuid.uuid4())
sql(f"INSERT INTO assets VALUES('{A}','{asset}'); INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,capabilities) VALUES('{A}','{agent}','{asset}','{secrets.token_hex(32)}','[\"run_script\",\"query_logs\"]');")
sql(f"INSERT INTO assets VALUES('{A}','{asset}'); INSERT INTO agent_installations(tenant_id,agent_id,asset_id,certificate_sha256,capabilities) VALUES('{A}','{agent}','{asset}','{secrets.token_hex(32)}','[\"linux.base/1\",\"run_script\",\"query_logs\"]');")
base = f"/api/v1/tenants/{A}"
try:
now = datetime.datetime.now(datetime.timezone.utc)
@@ -120,13 +120,25 @@ def main():
check(request("/mcp", "POST", {"jsonrpc": "2.0", "method": "notifications/initialized"}, operator, h)[0] == 202, "MCP initialized notification")
result = request("/mcp", "POST", {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, operator, h)[2]
names = [x["name"] for x in result["result"]["tools"]]
check(len(names) == 10 and not any("approv" in n or "authoriz" in n for n in names), "ten real MCP tools and no approval tool")
check({"tenants.list","inventory.list_devices","inventory.get_device","updates.list_releases","monitoring.get_health","logs.query","operations.plan","operations.submit","jobs.get","jobs.cancel","sessions.request","scripts.list","scripts.publish","scripts.plan"} <= set(names) and not any("approv" in n or "authoriz" in n for n in names), "inventory, operations and script library exposed through MCP with no approval tool")
call = {"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "inventory.list_devices", "arguments": {"tenant_id": B}}}
result = request("/mcp", "POST", call, operator, h)[2]["result"]
check(result["isError"] and result["structuredContent"]["code"] == "NOT_FOUND", "MCP enforces same tenant boundary")
call["params"] = {"name": "jobs.get", "arguments": {"tenant_id": A, "operation_id": pending["operation_id"]}}
result = request("/mcp", "POST", call, operator, h)[2]["result"]["structuredContent"]
check(result["approval_state"] == "pending_approval" and result["job_count"] == 0, "MCP jobs.get sees same persisted pending request")
library_input = {"name": "contract-" + uuid.uuid4().hex, "expected_version": 0,
"interpreter": "posix_sh", "execution_identity": "agent_service", "script_utf8": "printf 'catalog version 1\\n'\n"}
call["params"] = {"name": "scripts.publish", "arguments": {"tenant_id": A, "input": library_input}}
stored = request("/mcp", "POST", call, operator, h)[2]["result"]["structuredContent"]
check(stored["version"] == 1 and stored["script_sha256"] == hashlib.sha256(library_input["script_utf8"].encode()).hexdigest(),
"MCP publishes an immutable named revision")
selection = {"targets": plan_input["targets"], "schedule": {"not_before": now.isoformat(), "start_before": end.isoformat()},
"rollout": plan_input["rollout"], "reason": "library contract", "max_runtime_seconds": 60, "max_output_bytes": 4096}
call["params"] = {"name": "scripts.plan", "arguments": {"tenant_id": A, "revision_id": stored["revision_id"], "input": selection}}
library_plan = request("/mcp", "POST", call, operator, h)[2]["result"]["structuredContent"]
reloaded = request(base + "/operation-plans/" + library_plan["plan_id"], token=operator)[2]
check(reloaded["plan_payload"]["script_revision"]["revision_id"] == stored["revision_id"], "WebUI reads the same immutable MCP revision plan")
call["params"]={"name":"operations.plan","arguments":{"tenant_id":A,"input":plan_input}}
mcp_plan=request("/mcp","POST",call,operator,h)[2]["result"]["structuredContent"]
check(mcp_plan["jobs"][0]["asset_id"]==asset,"MCP planning uses same exact DTO and fixed job contract")
+167
View File
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""Activate a tools/build.py ARM64 server and matching WebUI on the dev server."""
import argparse
import base64
import hashlib
import json
from pathlib import Path
import re
import subprocess
import sys
def sha(data):
return hashlib.sha256(data).hexdigest()
def write_atomic(path, data, mode=0o644):
import os
import tempfile
path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as stream:
tmp = Path(stream.name)
try:
stream.write(data)
stream.flush()
os.fchmod(stream.fileno(), mode)
os.replace(tmp, path)
finally:
tmp.unlink(missing_ok=True)
def web_files(revision):
"""Build the exact Git revision using the local locked frontend dependencies."""
import tempfile
root = Path(__file__).resolve().parents[1]
with tempfile.TemporaryDirectory() as folder:
work = Path(folder)
archive = work / 'src.tar'
subprocess.run(['git','archive','-o',archive,revision,'--','web','tests/browser'],cwd=root,check=True)
subprocess.run(['tar','-xf',archive,'-C',work],check=True)
(work/'web/node_modules').symlink_to(root/'web/node_modules',target_is_directory=True)
subprocess.run(['npm','run','build'],cwd=work/'web',check=True)
dist = work/'web/dist'
return {str(file.relative_to(dist)):base64.b64encode(file.read_bytes()).decode()
for file in dist.rglob('*') if file.is_file()}
def activate(payload):
import fcntl
import time
revision = payload['revision']
if not re.fullmatch('[0-9a-f]{40}', revision):
raise ValueError('Invalid Git revision')
def run(*command):
return subprocess.check_output(list(command),text=True).strip()
def inspect(role):
return json.loads(run('docker','inspect',f'rust-discovery-server-{role}-1'))[0]
root = Path('/opt/rust-discovery')
work = root/'devstate/mvp-deploy'
work.mkdir(exist_ok=True,mode=0o700)
role_dir = work/revision
role_dir.mkdir(exist_ok=True,mode=0o700)
with (work/'deployment.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
matches = list((Path.home()/'rmm-dev-build/server-arm64/artifacts'/revision).glob('*/manifest.json'))
if len(matches) != 1:
raise RuntimeError('Expected one ARM64 build manifest for this revision')
manifest = json.loads(matches[0].read_bytes())
binary = (matches[0].parent/'rmm-server').read_bytes()
if (manifest['revision'] != revision or manifest['target'] != 'aarch64-unknown-linux-gnu'
or sha(binary) != manifest['sha256']['rmm-server'] or sha(matches[0].read_bytes()) != matches[0].parent.name):
raise RuntimeError('Server build digest mismatch')
active = inspect('api')
if inspect('gateway')['Image'] != active['Image']:
raise RuntimeError('API and Gateway have different base images')
labels = active['Config']['Labels']
if labels['com.docker.compose.project'] != 'rust-discovery-server':
raise RuntimeError('Unexpected compose project')
files = [Path(x) for x in labels['com.docker.compose.project.config_files'].split(',')]
base_files = [f for f in files if not (f.parent.parent == work and f.name == 'image.compose.yaml')]
compose = ['docker','compose','--project-name','rust-discovery-server','--project-directory',str(root/'deploy'),
'--env-file',str(root/'devstate/.env')]
for path in base_files:
if not path.is_file() or not path.resolve().is_relative_to(root):
raise ValueError('Missing deployment overlay')
compose.extend(['-f',str(path)])
tag='rmm-dev-server:'+revision
context=role_dir/'image'
context.mkdir(exist_ok=True)
base_tag='rmm-dev-base:'+active['Image'].split(':')[1]
run('docker','tag',active['Image'],base_tag)
write_atomic(context/'Dockerfile',f'FROM {base_tag}\nCOPY rmm-server /usr/local/bin/rmm-server\n'.encode())
write_atomic(context/'rmm-server',binary,0o555)
if subprocess.run(['docker','image','inspect',tag],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode:
run('docker','build','--network=none','--pull=false','-t',tag,str(context))
overlay=role_dir/'image.compose.yaml'
write_atomic(overlay,('services:\n'+''.join(f' {r}:\n image: {tag}\n pull_policy: never\n' for r in ('migrate','api','gateway'))).encode())
for path in payload['overlays']:
file=(root/'deploy')/path
if not re.fullmatch(r'[\w.-]+\.compose\.yaml',path) or not file.is_file():
raise ValueError('Dev compose overlay must exist in the deployment directory')
if file not in base_files:
compose.extend(['-f',str(file)])
compose.extend(['-f',str(overlay)])
config=json.loads(run(*compose,'config','--format','json'))['services']
for role,port in [('api','18443'),('gateway','18444')]:
svc=config[role]
if (svc['ports'][0]['host_ip']!='127.0.0.1' or str(svc['ports'][0]['published'])!=port or len(svc['ports'])!=1
or svc['user']!='10001:10001' or not svc['read_only']):
raise RuntimeError('Dev admin/gateway boundary changed')
if config['postgres'].get('ports'):
raise RuntimeError('Unexpected public database')
dist=next(Path(m['Source']) for m in active['Mounts'] if m['Destination']=='/web')
assets={name:base64.b64decode(data,validate=True) for name,data in payload['web'].items()}
if 'index.html' not in assets or any(Path(n).is_absolute() or '..' in Path(n).parts for n in assets):
raise ValueError('Invalid WebUI asset set')
for name,data in assets.items():
if name!='index.html':
write_atomic(dist/name,data)
run(*compose,'run','--no-deps','-T','--rm','migrate')
sql = ['docker','exec','rust-discovery-server-postgres-1','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm','-c']
recorded = dict(line.split('|') for line in run(*sql,"SELECT version,encode(checksum,'hex') FROM _sqlx_migrations WHERE success").splitlines())
if recorded != payload['migrations']:
raise RuntimeError('Applied migrations differ from the build revision')
run(*compose,'up','-d','--no-deps','--no-build','--pull','never','api','gateway')
for role,host,port,ca in [('api','localhost','18443','api-pki'),('gateway','discovery.ease-technologies.de','18444','pki')]:
for _ in range(25):
res=subprocess.run(['curl','--noproxy','*','--resolve',f'{host}:{port}:127.0.0.1','--cacert',str(root/f'devstate/{ca}/ca.pem'),
'--max-time','2','--silent','--fail',f'https://{host}:{port}/healthz'],capture_output=True)
if res.returncode==0 and res.stdout==b'ok':
break
time.sleep(1)
else:
raise RuntimeError(role+' health check failed')
live=run('docker','exec','rust-discovery-server-'+role+'-1','sha256sum','/usr/local/bin/rmm-server').split()[0]
if live!=sha(binary):
raise RuntimeError(role+' running digest differs')
write_atomic(dist/'index.html',assets['index.html'])
index=subprocess.check_output(['curl','--noproxy','*','--cacert',str(root/'devstate/api-pki/ca.pem'),
'--silent','--fail','https://localhost:18443/'])
if index!=assets['index.html']:
raise RuntimeError('Served index differs')
print(json.dumps(dict(revision=revision,server_sha256=sha(binary),web_index_sha256=sha(index),
image=inspect('api')['Image'],api_health='ok',gateway_health='ok')))
def main():
from dev_stage import ssh_prefix
import inspect
import shlex
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--ref',required=True)
parser.add_argument('--overlay',action='append',default=[],help='Additional existing compose file in remote deploy/')
args=parser.parse_args()
revision=subprocess.check_output(['git','rev-parse','--verify','--end-of-options',args.ref+'^{commit}'],text=True).strip()
web=web_files(revision)
source=Path(inspect.getfile(activate)).read_text().rsplit('\nif __name__ ==', 1)[0]
names=subprocess.check_output(['git','ls-tree','--name-only','-r',revision,'migrations'],text=True).splitlines()
migrations={str(int(Path(n).name.split('_',1)[0])):hashlib.sha384(subprocess.check_output(['git','show',revision+':'+n])).hexdigest()
for n in names if n.endswith('.sql')}
payload=json.dumps(dict(revision=revision,web=web,overlays=args.overlay,migrations=migrations))
cmd=source+'\nactivate(json.loads('+repr(payload)+'))\n'
subprocess.run(ssh_prefix('server')+[shlex.join(['python3','-I','-'])],input=cmd,text=True,check=True)
if __name__ == '__main__':
main()