Files
rust-discovery/deploy/server_deploy_221.py
T
2026-11-16 17:46:54 +01:00

165 lines
15 KiB
Python

#!/usr/bin/env python3
"""Deploy the scoped Windows221 closure build; preserve runtime, jobs and broker."""
import argparse
import inspect
import json
from pathlib import Path
import server_build_only as build
def remote():
from pathlib import Path
import subprocess,json,os,hashlib,stat,time,ssl,http.client,socket,tempfile
p=PARAMETERS;root=Path('/opt/rust-discovery');case=root/'devstate'/p['case'];pkg=case/'package'
def sha(raw):return hashlib.sha256(raw).hexdigest()
def encoded(v):return json.dumps(v,sort_keys=True,separators=(',',':')).encode()
def run(args,timeout=40):return subprocess.check_output(args,stderr=subprocess.DEVNULL,text=True,timeout=timeout).strip()
def once(path,value):
raw=encoded(value)
with path.open('xb') as stream:stream.write(raw);stream.flush();os.fsync(stream.fileno())
def docker(role):return json.loads(run(['docker','inspect','rust-discovery-server-'+role+'-1']))[0]
def actual(c):return dict(id=c['Id'],image=c['Image'],started_at=c['State']['StartedAt'],restart_count=c['RestartCount'],running=c['State']['Running'])
def shape(c):
conf,host=c['Config'],c['HostConfig']
value={'user':conf['User'],'env':sorted(conf['Env']),'entrypoint':conf['Entrypoint'],'cmd':conf['Cmd'],'working_dir':conf['WorkingDir'],
'mounts':sorted(c['Mounts'],key=lambda m:m['Destination']),'binds':sorted(host.get('Binds')or[]),'networks':sorted(c['NetworkSettings']['Networks']),
'labels':{k:v for k,v in (conf.get('Labels')or{}).items() if not k.startswith('com.docker.compose.')},
'host':{k:host.get(k) for k in ('ReadonlyRootfs','PortBindings','CapDrop','CapAdd','SecurityOpt','RestartPolicy','Privileged','NetworkMode','IpcMode','PidMode','Devices','UsernsMode')}}
return sha(encoded(value))
def sql(query):return run(['docker','exec','rust-discovery-server-postgres-1','psql','-XqAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm','-c',query],15)
def migrations():
values=sql("SELECT version,encode(checksum,'hex'),success FROM _sqlx_migrations ORDER BY version").splitlines()
actual_map={}
for row in values:
version,checksum,success=row.split('|');assert success=='t';actual_map[int(version)]=checksum
expected={int(x.name[:4]):hashlib.sha384(x.read_bytes()).hexdigest() for x in (pkg/'source/migrations').glob('*.sql')}
assert actual_map==expected or actual_map=={k:v for k,v in expected.items() if k<21}
return max(actual_map)
def counts():
return json.loads(sql("SELECT json_build_object('jobs',(SELECT count(*) FROM jobs),'grants',(SELECT count(*) FROM execution_permits),'authorizations',(SELECT count(*) FROM authorizations),'outbox',(SELECT count(*) FROM outbox),'echo_completion',(SELECT count(*) FROM windows_echo_completions),'windows_jobs',(SELECT count(*) FROM jobs WHERE agent_id='63ea15b2-0643-403d-b5b4-cba1288dbea3'),'windows_grants',(SELECT count(*) FROM execution_permits p JOIN jobs j USING(tenant_id,job_id,attempt_id) WHERE j.agent_id='63ea15b2-0643-403d-b5b4-cba1288dbea3'))"))
def windows():
return json.loads(sql("SELECT json_build_object('generation',i.connection_generation,'boot',i.connection_boot_id,'revoked',i.revoked_at,'journal',s.journal_id,'checkpoint',s.checkpoint,'decision',s.decision_id,'helper_journal',s.helper_journal_id,'helper_checkpoint',s.helper_checkpoint) FROM agent_installations i JOIN agent_sync_state s USING(tenant_id,agent_id) WHERE i.agent_id='63ea15b2-0643-403d-b5b4-cba1288dbea3'"))
def passive_units():
return {name:run(['systemctl','show',name,'--property=MainPID,ActiveState,SubState,ExecMainStartTimestampMonotonic,NRestarts']) for name in ('rmm-time-broker.service','chrony.service')}
def protected_metadata():
records={}
for base in ('devstate/pki','devstate/api-pki','devstate/jobs','devstate/root-policy'):
path=root/base
for item in [path,*sorted(path.rglob('*'))]:
meta=item.lstat();assert not item.is_symlink() and (stat.S_ISDIR(meta.st_mode)or stat.S_ISREG(meta.st_mode))
records[str(item)]={k:getattr(meta,'st_'+k) for k in ('dev','ino','uid','gid','mode','nlink','size','mtime_ns','ctime_ns')}
for item in (Path('/run/rmm-time-source'),Path('/run/rmm-time-source/broker.sock')):
if item.exists():
meta=item.lstat();records[str(item)]={k:getattr(meta,'st_'+k)for k in('dev','ino','uid','gid','mode','nlink')}
return sha(encoded(records))
def assert_role(role,c):
port={'api':'18443','gateway':'18444'}[role];conf,host=c['Config'],c['HostConfig']
assert c['State']['Running'] and c['RestartCount']==0 and conf['User']=='10001:10001'
assert host['PortBindings']=={'18443/tcp':[{'HostIp':'127.0.0.1','HostPort':port}]} and host['ReadonlyRootfs'] and host['CapDrop']==['ALL'] and host['SecurityOpt']==['no-new-privileges:true']
if role=='gateway':assert any(m['Destination']=='/run/rmm-time-source' and m['Source']=='/run/rmm-time-source' and m['RW']is False for m in c['Mounts'])
def request(role,path):
if role=='api':connection=http.client.HTTPSConnection('localhost',18443,context=ssl.create_default_context(cafile=str(root/'devstate/api-pki/ca.pem')),timeout=3)
else:
class Checked(http.client.HTTPSConnection):
def connect(self):self.sock=self._context.wrap_socket(socket.create_connection(('127.0.0.1',18444),timeout=self.timeout),server_hostname='discovery.ease-technologies.de')
connection=Checked('discovery.ease-technologies.de',18444,context=ssl.create_default_context(cafile=str(root/'devstate/pki/ca.pem')),timeout=3)
try:connection.request('GET',path);response=connection.getresponse();return response.status,response.read(2*1024*1024)
finally:connection.close()
assert os.uname().machine=='aarch64' and os.geteuid()==0
assert not any(n in os.environ for n in ('DOCKER_HOST','DOCKER_CONTEXT','COMPOSE_FILE','COMPOSE_PROJECT_NAME','COMPOSE_PROFILES'))
assert sha((pkg/'manifest.json').read_bytes())==p['manifest']
manifest=json.loads((pkg/'manifest.json').read_text());built=json.loads((case/'build.receipt.json').read_text())
assert built['exit_code']==0 and built['image']==p['image'] and built['architecture']=='arm64' and built['runtime_unchanged']
assert all(sha((pkg/n).read_bytes())==h for n,h in manifest['file_hashes'].items())
before_path=case/'deployment221-before.json'
if p['action']=='preflight':
assert not before_path.exists()
current={role:docker(role)for role in('api','gateway','postgres')}
assert {r:actual(c)for r,c in current.items()}==built['runtime_after']
for role in('api','gateway'):assert_role(role,current[role])
assert current['postgres']['HostConfig']['PortBindings']in({},None)
labels=current['api']['Config']['Labels'];assert labels['com.docker.compose.project']=='rust-discovery-server'
files=labels['com.docker.compose.project.config_files'].split(',');assert len(files) in (7,8)
assert all(Path(f).is_file() and not Path(f).is_symlink() and Path(f).is_relative_to(root)for f in files)
base=['docker','compose','--project-name','rust-discovery-server','--project-directory',str(root/'deploy'),'--env-file',str(root/'devstate/.env')]
for f in files:base+=['-f',f]
old=json.loads(run(base+['config','--format','json']))
overlay=case/'deployment221-images.compose.yaml';raw=('services:\n'+''.join(' '+role+':\n image: '+p['image']+'\n pull_policy: never\n'for role in('api','gateway','migrate'))).encode()
if overlay.exists():assert overlay.read_bytes()==raw
else:
with overlay.open('xb')as stream:stream.write(raw)
compose=base+['-f',str(overlay)];new=json.loads(run(compose+['config','--format','json']))
expected=json.loads(json.dumps(old))
for role in('api','gateway','migrate'):expected['services'][role]['image']=p['image'];expected['services'][role]['pull_policy']='never'
assert expected==new,'Unexpected compose semantics change'
candidate=json.loads(run(['docker','image','inspect',p['image']]))[0]['Config']
for role in('api','gateway'):
service=new['services'][role];env=dict(x.split('=',1)for x in(candidate.get('Env')or[]));env.update(service.get('environment')or{})
assert sorted(k+'='+str(v)for k,v in env.items())==sorted(current[role]['Config']['Env'])
assert service.get('user',candidate.get('User',''))==current[role]['Config']['User']
assert (candidate['Entrypoint']if service.get('entrypoint')is None else service['entrypoint'])==current[role]['Config']['Entrypoint']
assert (candidate.get('Cmd')if service.get('command')is None else service['command'])==current[role]['Config']['Cmd']
schema_before=migrations();assert schema_before in (20,21)
w=windows();assert w['generation']>=221 and w['boot']=='a9ad0895-5a96-8f2c-8fb6-cfdc2d5225ea'and w['decision']=='0442c732-8226-4c25-9615-7f007e24b269'and w['checkpoint']==10784 and w['helper_checkpoint']==413 and w['revoked']is None
state={'actual':{r:actual(c)for r,c in current.items()},'shape':{r:shape(c)for r,c in current.items()},'compose':compose,'compose_sha256':sha(encoded(new)),'files_sha256':{f:sha(Path(f).read_bytes())for f in files},'overlay_sha256':sha(raw),'counts':counts(),'windows':w,'protected_sha256':protected_metadata(),'passive_units':passive_units(),'web_before':{str(f.relative_to(root/'web/dist')):sha(f.read_bytes())for f in(root/'web/dist').rglob('*')if f.is_file()}}
once(before_path,state);return {'preflight':True,'migration':schema_before,'image':p['image'],'compose_only_image_changes':True,'environment_identical':True,'loopback_only':True,'windows':w,'counts':state['counts'],'root_uid':10001,'secrets_logged':False}
before=json.loads(before_path.read_text());compose=before['compose']
assert all(sha(Path(f).read_bytes())==h for f,h in before['files_sha256'].items())
assert sha(Path(compose[-1]).read_bytes())==before['overlay_sha256']
assert sha(encoded(json.loads(run(compose+['config','--format','json']))))==before['compose_sha256']
assert actual(docker('postgres'))==before['actual']['postgres'] and shape(docker('postgres'))==before['shape']['postgres']
assert passive_units()==before['passive_units'] and protected_metadata()==before['protected_sha256']
assert counts()==before['counts'] and windows()==before['windows']
def check_apps(new):
for role in('api','gateway'):
c=docker(role);assert_role(role,c);assert shape(c)==before['shape'][role]
assert (c['Image']==p['image']and c['Id']!=before['actual'][role]['id'])if new else actual(c)==before['actual'][role]
if p['action']=='migrate':
assert migrations()==20;check_apps(False);once(case/'deployment221-migration-intent.json',{'image':p['image'],'migration':21})
log=case/'deployment221-migration.log'
with log.open('xb')as stream:r=subprocess.run(compose+['run','--no-deps','-T','--rm','--name','rmm-server221-migrate-21','migrate'],stdout=stream,stderr=subprocess.STDOUT,timeout=60)
assert r.returncode==0 and migrations()==21;check_apps(False)
assert sql('SELECT count(*) FROM windows_system_nonstart_reviews')=='0' and sql('SELECT count(*) FROM windows_system_nonstart_receipts')=='0'
elif p['action']=='apps':
assert migrations()==21;check_apps(False);once(case/'deployment221-apps-intent.json',{'image':p['image'],'before':before['actual']})
with(case/'deployment221-apps.log').open('xb')as stream:r=subprocess.run(compose+['up','-d','--no-deps','--no-build','--pull','never','api','gateway'],stdout=stream,stderr=subprocess.STDOUT,timeout=60)
assert r.returncode==0
for _ in range(30):
try:
assert request('api','/healthz')==(200,b'ok')and request('gateway','/healthz')==(200,b'ok');break
except Exception:time.sleep(.25)
else:raise RuntimeError('TLS-validated application health did not become ready')
check_apps(True)
for role in('api','gateway'):assert run(['docker','exec','rust-discovery-server-'+role+'-1','sha256sum','/usr/local/bin/rmm-server']).split()[0]==built['binaries']['rmm-server']['sha256']
elif p['action']=='web':
assert migrations()==21;check_apps(True);once(case/'deployment221-web-intent.json',{'web':manifest['web']})
current={str(f.relative_to(root/'web/dist')):sha(f.read_bytes())for f in(root/'web/dist').rglob('*')if f.is_file()};assert current==before['web_before']
for name,h in sorted(manifest['web'].items(),key=lambda v:v[0]=='index.html'):
src=pkg/'web'/name;assert sha(src.read_bytes())==h;dst=root/'web/dist'/name;assert not dst.is_symlink()and dst.parent.resolve()==dst.parent
if dst.exists()and sha(dst.read_bytes())==h:continue
if name!='index.html':assert not dst.exists()
with tempfile.NamedTemporaryFile(dir=dst.parent,delete=False)as stream:
temp=Path(stream.name);os.fchmod(stream.fileno(),0o644);stream.write(src.read_bytes());stream.flush();os.fsync(stream.fileno())
os.replace(temp,dst)
assert request('api','/')[0]==200 and sha(request('api','/')[1])==manifest['web']['index.html']
elif p['action']=='verify':
assert migrations()==21;check_apps(True)
assert all(sha((root/'web/dist'/n).read_bytes())==h for n,h in manifest['web'].items())
assert request('api','/healthz')==(200,b'ok')and request('gateway','/healthz')==(200,b'ok')
assert request('api','/api/v1/tenants/11111111-1111-4111-8111-111111111111/windows-system-nonstart-221')[0]==401
else:raise RuntimeError('unknown fixed deployment action')
assert counts()==before['counts'] and windows()==before['windows'] and passive_units()==before['passive_units'] and protected_metadata()==before['protected_sha256']
assert actual(docker('postgres'))==before['actual']['postgres']
result={'action':p['action'],'image':p['image'],'migration':migrations(),'api':actual(docker('api')),'gateway':actual(docker('gateway')),'postgres_unchanged':True,'broker_chrony_unchanged':True,'protected_pki_jobkey_policy_metadata_unchanged':True,'counts':counts(),'windows':windows(),'environment_mounts_uid_security_loopback_unchanged':True,'job_authority_created':False}
once(case/('deployment221-'+p['action']+'-result.json'),result);return result
def main():
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=('preflight','migrate','apps','web','verify'));parser.add_argument('--build-folder',required=True,type=Path);args=parser.parse_args()
folder=args.build_folder.resolve();assert folder.is_relative_to(build.ROOT/'devstate/server-build-only')
sealed=json.loads((folder/'sealed.json').read_text());receipt=json.loads((folder/'local-build-receipt.json').read_text());assert sealed['release'].startswith('server221-') and receipt['release']==sealed['release'] and receipt['exit_code']==0 and receipt['runtime_unchanged']
p={'case':sealed['release'],'image':receipt['image'],'manifest':sealed['manifest_sha256'],'action':args.action}
code=inspect.getsource(remote).replace('PARAMETERS',repr(p))+'\ntry:\n print(__import__("json").dumps(remote()))\nexcept Exception as error:\n import traceback\n print(__import__("json").dumps({"error_class":type(error).__name__,"frames":[{"function":f.name,"line":f.lineno}for f in traceback.extract_tb(error.__traceback__)]}))\n'
result=build.ssh(code,timeout=90)
if 'error_class'in result:print(json.dumps(result));raise RuntimeError('Deployment incomplete; redacted frame diagnostics above')
target=folder/('local-deployment-'+args.action+'.json');build.write(target,json.dumps(result,sort_keys=True,indent=2).encode());print(json.dumps(result))
if __name__=='__main__':main()