#!/usr/bin/env python3 """Activate a tools/build.py ARM64 server and matching WebUI on the dev server.""" import argparse import base64 import hashlib import json from pathlib import Path import re import subprocess import sys def sha(data): return hashlib.sha256(data).hexdigest() def write_atomic(path, data, mode=0o644): import os import tempfile path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as stream: tmp = Path(stream.name) try: stream.write(data) stream.flush() os.fchmod(stream.fileno(), mode) os.replace(tmp, path) finally: tmp.unlink(missing_ok=True) def web_files(revision): """Build the exact Git revision using the local locked frontend dependencies.""" import tempfile root = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory() as folder: work = Path(folder) archive = work / 'src.tar' subprocess.run(['git','archive','-o',archive,revision,'--','web','tests/browser'],cwd=root,check=True) subprocess.run(['tar','-xf',archive,'-C',work],check=True) (work/'web/node_modules').symlink_to(root/'web/node_modules',target_is_directory=True) subprocess.run(['npm','run','build'],cwd=work/'web',check=True) dist = work/'web/dist' return {str(file.relative_to(dist)):base64.b64encode(file.read_bytes()).decode() for file in dist.rglob('*') if file.is_file()} def activate(payload): import fcntl import time revision = payload['revision'] if not re.fullmatch('[0-9a-f]{40}', revision): raise ValueError('Invalid Git revision') def run(*command): return subprocess.check_output(list(command),text=True).strip() def inspect(role): return json.loads(run('docker','inspect',f'rust-discovery-server-{role}-1'))[0] root = Path('/opt/rust-discovery') work = root/'devstate/mvp-deploy' work.mkdir(exist_ok=True,mode=0o700) role_dir = work/revision role_dir.mkdir(exist_ok=True,mode=0o700) with (work/'deployment.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX) matches = list((Path.home()/'rmm-dev-build/server-arm64/artifacts'/revision).glob('*/manifest.json')) if len(matches) != 1: raise RuntimeError('Expected one ARM64 build manifest for this revision') manifest = json.loads(matches[0].read_bytes()) binary = (matches[0].parent/'rmm-server').read_bytes() if (manifest['revision'] != revision or manifest['target'] != 'aarch64-unknown-linux-gnu' or sha(binary) != manifest['sha256']['rmm-server'] or sha(matches[0].read_bytes()) != matches[0].parent.name): raise RuntimeError('Server build digest mismatch') active = inspect('api') if inspect('gateway')['Image'] != active['Image']: raise RuntimeError('API and Gateway have different base images') labels = active['Config']['Labels'] if labels['com.docker.compose.project'] != 'rust-discovery-server': raise RuntimeError('Unexpected compose project') files = [Path(x) for x in labels['com.docker.compose.project.config_files'].split(',')] base_files = [f for f in files if not (f.parent.parent == work and f.name == 'image.compose.yaml')] compose = ['docker','compose','--project-name','rust-discovery-server','--project-directory',str(root/'deploy'), '--env-file',str(root/'devstate/.env')] for path in base_files: if not path.is_file() or not path.resolve().is_relative_to(root): raise ValueError('Missing deployment overlay') compose.extend(['-f',str(path)]) tag='rmm-dev-server:'+revision context=role_dir/'image' context.mkdir(exist_ok=True) base_tag='rmm-dev-base:'+active['Image'].split(':')[1] run('docker','tag',active['Image'],base_tag) write_atomic(context/'Dockerfile',f'FROM {base_tag}\nCOPY rmm-server /usr/local/bin/rmm-server\n'.encode()) write_atomic(context/'rmm-server',binary,0o555) if subprocess.run(['docker','image','inspect',tag],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode: run('docker','build','--network=none','--pull=false','-t',tag,str(context)) overlay=role_dir/'image.compose.yaml' write_atomic(overlay,('services:\n'+''.join(f' {r}:\n image: {tag}\n pull_policy: never\n' for r in ('migrate','api','gateway'))).encode()) for path, content in payload['overlays'].items(): file = (root / 'deploy') / path if not re.fullmatch(r'server\.[\w.-]+\.compose\.yaml', path) or file.is_symlink(): raise ValueError('Invalid development overlay name') write_atomic(file, base64.b64decode(content, validate=True)) if file not in base_files: compose.extend(['-f',str(file)]) compose.extend(['-f',str(overlay)]) config=json.loads(run(*compose,'config','--format','json'))['services'] for role,port in [('api','18443'),('gateway','18444')]: svc=config[role] if (svc['ports'][0]['host_ip']!='127.0.0.1' or str(svc['ports'][0]['published'])!=port or len(svc['ports'])!=1 or svc['user']!='10001:10001' or not svc['read_only']): raise RuntimeError('Dev admin/gateway boundary changed') if config['postgres'].get('ports'): raise RuntimeError('Unexpected public database') dist=next(Path(m['Source']) for m in active['Mounts'] if m['Destination']=='/web') assets={name:base64.b64decode(data,validate=True) for name,data in payload['web'].items()} if 'index.html' not in assets or any(Path(n).is_absolute() or '..' in Path(n).parts for n in assets): raise ValueError('Invalid WebUI asset set') for name,data in assets.items(): if name!='index.html': write_atomic(dist/name,data) run(*compose,'run','--no-deps','-T','--rm','migrate') sql = ['docker','exec','rust-discovery-server-postgres-1','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm','-c'] recorded = dict(line.split('|') for line in run(*sql,"SELECT version,encode(checksum,'hex') FROM _sqlx_migrations WHERE success").splitlines()) if recorded != payload['migrations']: raise RuntimeError('Applied migrations differ from the build revision') run(*compose,'up','-d','--no-deps','--no-build','--pull','never','api','gateway') for role,host,port,ca in [('api','localhost','18443','api-pki'),('gateway','discovery.ease-technologies.de','18444','pki')]: for _ in range(25): res=subprocess.run(['curl','--noproxy','*','--resolve',f'{host}:{port}:127.0.0.1','--cacert',str(root/f'devstate/{ca}/ca.pem'), '--max-time','2','--silent','--fail',f'https://{host}:{port}/healthz'],capture_output=True) if res.returncode==0 and res.stdout==b'ok': break time.sleep(1) else: raise RuntimeError(role+' health check failed') live=run('docker','exec','rust-discovery-server-'+role+'-1','sha256sum','/usr/local/bin/rmm-server').split()[0] if live!=sha(binary): raise RuntimeError(role+' running digest differs') write_atomic(dist/'index.html',assets['index.html']) index=subprocess.check_output(['curl','--noproxy','*','--cacert',str(root/'devstate/api-pki/ca.pem'), '--silent','--fail','https://localhost:18443/']) if index!=assets['index.html']: raise RuntimeError('Served index differs') print(json.dumps(dict(revision=revision,server_sha256=sha(binary),web_index_sha256=sha(index), image=inspect('api')['Image'],api_health='ok',gateway_health='ok'))) def main(): from dev_stage import ssh_prefix import inspect import shlex parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--ref',required=True) parser.add_argument('--overlay',action='append',default=[],help='Deploy and use this compose overlay from the selected Git revision') args=parser.parse_args() revision=subprocess.check_output(['git','rev-parse','--verify','--end-of-options',args.ref+'^{commit}'],text=True).strip() web=web_files(revision) source=Path(inspect.getfile(activate)).read_text().rsplit('\nif __name__ ==', 1)[0] names=subprocess.check_output(['git','ls-tree','--name-only','-r',revision,'migrations'],text=True).splitlines() migrations={str(int(Path(n).name.split('_',1)[0])):hashlib.sha384(subprocess.check_output(['git','show',revision+':'+n])).hexdigest() for n in names if n.endswith('.sql')} overlays = {} for name in args.overlay: if not re.fullmatch(r'server\.[\w.-]+\.compose\.yaml', name): parser.error('Expected a server overlay filename from deploy/') raw = subprocess.check_output(['git', 'show', revision + ':deploy/' + name]) overlays[name] = base64.b64encode(raw).decode() payload=json.dumps(dict(revision=revision,web=web,overlays=overlays,migrations=migrations)) cmd=source+'\nactivate(json.loads('+repr(payload)+'))\n' subprocess.run(ssh_prefix('server')+[shlex.join(['python3','-I','-'])],input=cmd,text=True,check=True) if __name__ == '__main__': main()