Track embedded migrations and verify the deployed schema and binary
This commit is contained in:
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Activate a tools/build.py ARM64 server and matching WebUI on the dev server."""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def write_atomic(path, data, mode=0o644):
|
||||
import os
|
||||
import tempfile
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as stream:
|
||||
tmp = Path(stream.name)
|
||||
try:
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
os.fchmod(stream.fileno(), mode)
|
||||
os.replace(tmp, path)
|
||||
finally:
|
||||
tmp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def web_files(revision):
|
||||
"""Build the exact Git revision using the local locked frontend dependencies."""
|
||||
import tempfile
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
work = Path(folder)
|
||||
archive = work / 'src.tar'
|
||||
subprocess.run(['git','archive','-o',archive,revision,'--','web','tests/browser'],cwd=root,check=True)
|
||||
subprocess.run(['tar','-xf',archive,'-C',work],check=True)
|
||||
(work/'web/node_modules').symlink_to(root/'web/node_modules',target_is_directory=True)
|
||||
subprocess.run(['npm','run','build'],cwd=work/'web',check=True)
|
||||
dist = work/'web/dist'
|
||||
return {str(file.relative_to(dist)):base64.b64encode(file.read_bytes()).decode()
|
||||
for file in dist.rglob('*') if file.is_file()}
|
||||
|
||||
|
||||
def activate(payload):
|
||||
import fcntl
|
||||
import time
|
||||
revision = payload['revision']
|
||||
if not re.fullmatch('[0-9a-f]{40}', revision):
|
||||
raise ValueError('Invalid Git revision')
|
||||
def run(*command):
|
||||
return subprocess.check_output(list(command),text=True).strip()
|
||||
def inspect(role):
|
||||
return json.loads(run('docker','inspect',f'rust-discovery-server-{role}-1'))[0]
|
||||
root = Path('/opt/rust-discovery')
|
||||
work = root/'devstate/mvp-deploy'
|
||||
work.mkdir(exist_ok=True,mode=0o700)
|
||||
role_dir = work/revision
|
||||
role_dir.mkdir(exist_ok=True,mode=0o700)
|
||||
with (work/'deployment.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
matches = list((Path.home()/'rmm-dev-build/server-arm64/artifacts'/revision).glob('*/manifest.json'))
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError('Expected one ARM64 build manifest for this revision')
|
||||
manifest = json.loads(matches[0].read_bytes())
|
||||
binary = (matches[0].parent/'rmm-server').read_bytes()
|
||||
if (manifest['revision'] != revision or manifest['target'] != 'aarch64-unknown-linux-gnu'
|
||||
or sha(binary) != manifest['sha256']['rmm-server'] or sha(matches[0].read_bytes()) != matches[0].parent.name):
|
||||
raise RuntimeError('Server build digest mismatch')
|
||||
active = inspect('api')
|
||||
if inspect('gateway')['Image'] != active['Image']:
|
||||
raise RuntimeError('API and Gateway have different base images')
|
||||
labels = active['Config']['Labels']
|
||||
if labels['com.docker.compose.project'] != 'rust-discovery-server':
|
||||
raise RuntimeError('Unexpected compose project')
|
||||
files = [Path(x) for x in labels['com.docker.compose.project.config_files'].split(',')]
|
||||
base_files = [f for f in files if not (f.parent.parent == work and f.name == 'image.compose.yaml')]
|
||||
compose = ['docker','compose','--project-name','rust-discovery-server','--project-directory',str(root/'deploy'),
|
||||
'--env-file',str(root/'devstate/.env')]
|
||||
for path in base_files:
|
||||
if not path.is_file() or not path.resolve().is_relative_to(root):
|
||||
raise ValueError('Missing deployment overlay')
|
||||
compose.extend(['-f',str(path)])
|
||||
tag='rmm-dev-server:'+revision
|
||||
context=role_dir/'image'
|
||||
context.mkdir(exist_ok=True)
|
||||
base_tag='rmm-dev-base:'+active['Image'].split(':')[1]
|
||||
run('docker','tag',active['Image'],base_tag)
|
||||
write_atomic(context/'Dockerfile',f'FROM {base_tag}\nCOPY rmm-server /usr/local/bin/rmm-server\n'.encode())
|
||||
write_atomic(context/'rmm-server',binary,0o555)
|
||||
if subprocess.run(['docker','image','inspect',tag],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode:
|
||||
run('docker','build','--network=none','--pull=false','-t',tag,str(context))
|
||||
overlay=role_dir/'image.compose.yaml'
|
||||
write_atomic(overlay,('services:\n'+''.join(f' {r}:\n image: {tag}\n pull_policy: never\n' for r in ('migrate','api','gateway'))).encode())
|
||||
for path in payload['overlays']:
|
||||
file=(root/'deploy')/path
|
||||
if not re.fullmatch(r'[\w.-]+\.compose\.yaml',path) or not file.is_file():
|
||||
raise ValueError('Dev compose overlay must exist in the deployment directory')
|
||||
if file not in base_files:
|
||||
compose.extend(['-f',str(file)])
|
||||
compose.extend(['-f',str(overlay)])
|
||||
config=json.loads(run(*compose,'config','--format','json'))['services']
|
||||
for role,port in [('api','18443'),('gateway','18444')]:
|
||||
svc=config[role]
|
||||
if (svc['ports'][0]['host_ip']!='127.0.0.1' or str(svc['ports'][0]['published'])!=port or len(svc['ports'])!=1
|
||||
or svc['user']!='10001:10001' or not svc['read_only']):
|
||||
raise RuntimeError('Dev admin/gateway boundary changed')
|
||||
if config['postgres'].get('ports'):
|
||||
raise RuntimeError('Unexpected public database')
|
||||
dist=next(Path(m['Source']) for m in active['Mounts'] if m['Destination']=='/web')
|
||||
assets={name:base64.b64decode(data,validate=True) for name,data in payload['web'].items()}
|
||||
if 'index.html' not in assets or any(Path(n).is_absolute() or '..' in Path(n).parts for n in assets):
|
||||
raise ValueError('Invalid WebUI asset set')
|
||||
for name,data in assets.items():
|
||||
if name!='index.html':
|
||||
write_atomic(dist/name,data)
|
||||
run(*compose,'run','--no-deps','-T','--rm','migrate')
|
||||
sql = ['docker','exec','rust-discovery-server-postgres-1','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm','-c']
|
||||
recorded = dict(line.split('|') for line in run(*sql,"SELECT version,encode(checksum,'hex') FROM _sqlx_migrations WHERE success").splitlines())
|
||||
if recorded != payload['migrations']:
|
||||
raise RuntimeError('Applied migrations differ from the build revision')
|
||||
run(*compose,'up','-d','--no-deps','--no-build','--pull','never','api','gateway')
|
||||
for role,host,port,ca in [('api','localhost','18443','api-pki'),('gateway','discovery.ease-technologies.de','18444','pki')]:
|
||||
for _ in range(25):
|
||||
res=subprocess.run(['curl','--noproxy','*','--resolve',f'{host}:{port}:127.0.0.1','--cacert',str(root/f'devstate/{ca}/ca.pem'),
|
||||
'--max-time','2','--silent','--fail',f'https://{host}:{port}/healthz'],capture_output=True)
|
||||
if res.returncode==0 and res.stdout==b'ok':
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError(role+' health check failed')
|
||||
live=run('docker','exec','rust-discovery-server-'+role+'-1','sha256sum','/usr/local/bin/rmm-server').split()[0]
|
||||
if live!=sha(binary):
|
||||
raise RuntimeError(role+' running digest differs')
|
||||
write_atomic(dist/'index.html',assets['index.html'])
|
||||
index=subprocess.check_output(['curl','--noproxy','*','--cacert',str(root/'devstate/api-pki/ca.pem'),
|
||||
'--silent','--fail','https://localhost:18443/'])
|
||||
if index!=assets['index.html']:
|
||||
raise RuntimeError('Served index differs')
|
||||
print(json.dumps(dict(revision=revision,server_sha256=sha(binary),web_index_sha256=sha(index),
|
||||
image=inspect('api')['Image'],api_health='ok',gateway_health='ok')))
|
||||
|
||||
|
||||
def main():
|
||||
from dev_stage import ssh_prefix
|
||||
import inspect
|
||||
import shlex
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--ref',required=True)
|
||||
parser.add_argument('--overlay',action='append',default=[],help='Additional existing compose file in remote deploy/')
|
||||
args=parser.parse_args()
|
||||
revision=subprocess.check_output(['git','rev-parse','--verify','--end-of-options',args.ref+'^{commit}'],text=True).strip()
|
||||
web=web_files(revision)
|
||||
source=Path(inspect.getfile(activate)).read_text().rsplit('\nif __name__ ==', 1)[0]
|
||||
names=subprocess.check_output(['git','ls-tree','--name-only','-r',revision,'migrations'],text=True).splitlines()
|
||||
migrations={str(int(Path(n).name.split('_',1)[0])):hashlib.sha384(subprocess.check_output(['git','show',revision+':'+n])).hexdigest()
|
||||
for n in names if n.endswith('.sql')}
|
||||
payload=json.dumps(dict(revision=revision,web=web,overlays=args.overlay,migrations=migrations))
|
||||
cmd=source+'\nactivate(json.loads('+repr(payload)+'))\n'
|
||||
subprocess.run(ssh_prefix('server')+[shlex.join(['python3','-I','-'])],input=cmd,text=True,check=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user