#!/usr/bin/env python3 """Activate a tools/build.py ARM64 server and matching WebUI on the dev server.""" import argparse import base64 import hashlib import json from pathlib import Path import re import subprocess def sha(data): return hashlib.sha256(data).hexdigest() def write_atomic(path, data, mode=0o644): import os import tempfile path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as stream: tmp = Path(stream.name) try: stream.write(data) stream.flush() os.fchmod(stream.fileno(), mode) os.replace(tmp, path) finally: tmp.unlink(missing_ok=True) def pack_web(dist): if not (dist / 'index.html').is_file(): raise ValueError('A built WebUI directory with index.html is required') files = {} for file in dist.rglob('*'): if file.is_symlink(): raise ValueError('Web build contains a symlink') if file.is_file(): files[file.relative_to(dist).as_posix()] = base64.b64encode(file.read_bytes()).decode() return files def stage_web(dist, files): if 'index.html' not in files: raise ValueError('Web build has no index.html') for name, encoded in files.items(): path = dist / name if Path(name).is_absolute() or '..' in Path(name).parts or path.resolve() != path: raise ValueError('Invalid WebUI asset path') if name != 'index.html': write_atomic(path, base64.b64decode(encoded, validate=True)) return base64.b64decode(files['index.html'], validate=True) def publish_index(root, dist, index): write_atomic(dist / 'index.html', index) actual = subprocess.check_output(['curl','--noproxy','*','--max-time','10','--cacert', str(root/'devstate/api-pki/ca.pem'),'--silent','--fail','https://localhost:18443/']) if actual != index: raise RuntimeError('Served index differs') return sha(actual) def web_files(revision): """Build the exact Git revision using the local locked frontend dependencies.""" import tempfile root = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory() as folder: work = Path(folder) archive = work / 'src.tar' subprocess.run(['git','archive','-o',archive,revision,'--','web','tests/browser'],cwd=root,check=True) subprocess.run(['tar','-xf',archive,'-C',work],check=True) (work/'web/node_modules').symlink_to(root/'web/node_modules',target_is_directory=True) subprocess.run(['npm','run','build'],cwd=work/'web',check=True) return pack_web(work / 'web/dist') def activate(payload): import fcntl import time revision = payload['revision'] if not payload['web_only'] and not re.fullmatch('[0-9a-f]{40}', revision): raise ValueError('Invalid Git revision') def run(*command): return subprocess.check_output(list(command),text=True).strip() def inspect(role): return json.loads(run('docker','inspect',f'rust-discovery-server-{role}-1'))[0] root = Path('/opt/rust-discovery') work = root/'devstate/mvp-deploy' work.mkdir(exist_ok=True,mode=0o700) with (work/'deployment.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX) active = inspect('api') dist = next(Path(m['Source']) for m in active['Mounts'] if m['Destination'] == '/web') if dist != root / 'web/dist' or active['Config']['Labels']['com.docker.compose.project'] != 'rust-discovery-server': raise RuntimeError('Unexpected development WebUI deployment') index = stage_web(dist, payload['web']) if payload['web_only']: print(json.dumps(dict(revision=revision, web_index_sha256=publish_index(root, dist, index), api_container=active['Id'], web_only=True))) return role_dir = work / revision role_dir.mkdir(exist_ok=True, mode=0o700) matches = list((Path.home()/'rmm-dev-build/server-arm64/artifacts'/revision).glob('*/manifest.json')) if len(matches) != 1: raise RuntimeError('Expected one ARM64 build manifest for this revision') manifest = json.loads(matches[0].read_bytes()) binary = (matches[0].parent/'rmm-server').read_bytes() if (manifest['revision'] != revision or manifest['target'] != 'aarch64-unknown-linux-gnu' or sha(binary) != manifest['sha256']['rmm-server'] or sha(matches[0].read_bytes()) != matches[0].parent.name): raise RuntimeError('Server build digest mismatch') if inspect('gateway')['Image'] != active['Image']: raise RuntimeError('API and Gateway have different base images') labels = active['Config']['Labels'] if labels['com.docker.compose.project'] != 'rust-discovery-server': raise RuntimeError('Unexpected compose project') files = [Path(x) for x in labels['com.docker.compose.project.config_files'].split(',')] base_files = [f for f in files if not (f.parent.parent == work and f.name == 'image.compose.yaml')] compose = ['docker','compose','--project-name','rust-discovery-server','--project-directory',str(root/'deploy'), '--env-file',str(root/'devstate/.env')] for path in base_files: if not path.is_file() or not path.resolve().is_relative_to(root): raise ValueError('Missing deployment overlay') compose.extend(['-f',str(path)]) tag='rmm-dev-server:'+revision context=role_dir/'image' context.mkdir(exist_ok=True) # Reuse the deployment's runtime base across edits. Building on each # previous dev image would retain a full obsolete server in every layer. base_label = 'eu.incoso.rust-discovery.dev-runtime-base' base_id = labels.get(base_label, active['Image']) if not re.fullmatch(r'sha256:[0-9a-f]{64}', base_id): raise ValueError('Invalid development runtime image identifier') base_tag = 'rmm-dev-base:' + base_id.split(':')[1] run('docker','tag',base_id,base_tag) dockerfile = f'FROM {base_tag}\nLABEL {base_label}={base_id}\nCOPY rmm-server /usr/local/bin/rmm-server\n' write_atomic(context/'Dockerfile', dockerfile.encode()) write_atomic(context/'rmm-server',binary,0o555) if subprocess.run(['docker','image','inspect',tag],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode: run('docker','build','--network=none','--pull=false','-t',tag,str(context)) overlay=role_dir/'image.compose.yaml' write_atomic(overlay,('services:\n'+''.join(f' {r}:\n image: {tag}\n pull_policy: never\n' for r in ('migrate','api','gateway'))).encode()) for path, content in payload['overlays'].items(): file = (root / 'deploy') / path if not re.fullmatch(r'server\.[\w.-]+\.compose\.yaml', path) or file.is_symlink(): raise ValueError('Invalid development overlay name') write_atomic(file, base64.b64decode(content, validate=True)) if file not in base_files: compose.extend(['-f',str(file)]) compose.extend(['-f',str(overlay)]) config=json.loads(run(*compose,'config','--format','json'))['services'] for role,port in [('api','18443'),('gateway','18444')]: svc=config[role] if (svc['ports'][0]['host_ip']!='127.0.0.1' or str(svc['ports'][0]['published'])!=port or len(svc['ports'])!=1 or svc['user']!='10001:10001' or not svc['read_only']): raise RuntimeError('Dev admin/gateway boundary changed') if config['postgres'].get('ports'): raise RuntimeError('Unexpected public database') run(*compose,'run','--no-deps','-T','--rm','migrate') sql = ['docker','exec','rust-discovery-server-postgres-1','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm','-c'] recorded = dict(line.split('|') for line in run(*sql,"SELECT version,encode(checksum,'hex') FROM _sqlx_migrations WHERE success").splitlines()) if recorded != payload['migrations']: raise RuntimeError('Applied migrations differ from the build revision') run(*compose,'up','-d','--no-deps','--no-build','--pull','never','api','gateway') for role,host,port,ca in [('api','localhost','18443','api-pki'),('gateway','discovery.ease-technologies.de','18444','pki')]: for _ in range(25): res=subprocess.run(['curl','--noproxy','*','--resolve',f'{host}:{port}:127.0.0.1','--cacert',str(root/f'devstate/{ca}/ca.pem'), '--max-time','2','--silent','--fail',f'https://{host}:{port}/healthz'],capture_output=True) if res.returncode==0 and res.stdout==b'ok': break time.sleep(1) else: raise RuntimeError(role+' health check failed') live=run('docker','exec','rust-discovery-server-'+role+'-1','sha256sum','/usr/local/bin/rmm-server').split()[0] if live!=sha(binary): raise RuntimeError(role+' running digest differs') print(json.dumps(dict(revision=revision,server_sha256=sha(binary),web_index_sha256=publish_index(root, dist, index), image=inspect('api')['Image'],api_health='ok',gateway_health='ok'))) def main(): from dev_stage import ssh_prefix import inspect import shlex parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--ref', help='Exact Git revision for the server and WebUI') parser.add_argument('--web-only', action='store_true', help='Only activate built web assets') parser.add_argument('--web-dist', type=Path, help='Use an existing WebUI build with --web-only') parser.add_argument('--overlay',action='append',default=[],help='Deploy and use this compose overlay from the selected Git revision') args=parser.parse_args() if args.web_dist and (not args.web_only or args.ref) or args.web_only and args.overlay: parser.error('Choose --web-only with either --web-dist or --ref and no compose overlay') if not args.web_dist and not args.ref: parser.error('--ref or --web-only --web-dist is required') revision = None if args.web_dist else subprocess.check_output( ['git','rev-parse','--verify','--end-of-options',args.ref+'^{commit}'],text=True).strip() web = pack_web(args.web_dist.resolve(strict=True)) if args.web_dist else web_files(revision) source=Path(inspect.getfile(activate)).read_text().rsplit('\nif __name__ ==', 1)[0] names=subprocess.check_output(['git','ls-tree','--name-only','-r',revision,'migrations'],text=True).splitlines() if not args.web_only else [] migrations={str(int(Path(n).name.split('_',1)[0])):hashlib.sha384(subprocess.check_output(['git','show',revision+':'+n])).hexdigest() for n in names if n.endswith('.sql')} overlays = {} for name in args.overlay: if not re.fullmatch(r'server\.[\w.-]+\.compose\.yaml', name): parser.error('Expected a server overlay filename from deploy/') raw = subprocess.check_output(['git', 'show', revision + ':deploy/' + name]) overlays[name] = base64.b64encode(raw).decode() payload=json.dumps(dict(revision=revision,web=web,web_only=args.web_only,overlays=overlays,migrations=migrations)) cmd=source+'\nactivate(json.loads('+repr(payload)+'))\n' subprocess.run(ssh_prefix('server')+[shlex.join(['python3','-I','-'])],input=cmd,text=True,check=True) if __name__ == '__main__': main()