diff --git a/deploy/server_time_gateway_continue.py b/deploy/server_time_gateway_continue.py new file mode 100644 index 0000000..b1440c5 --- /dev/null +++ b/deploy/server_time_gateway_continue.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Only the unentered fixed gateway backend effects of original fb87; no installed-effect replay.""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import stat +import sys + +OLD_SHA = 'e14aeabfd7d3a8ab298985706dc26014e0ad540a1c3ef40cf57e1a65b77b7f3b' +OLD_SOURCE = Path('/var/lib/rmm-time-source-operators') / OLD_SHA / 'gateway_deploy.py' +PIN_CONTROLLER_SHA = 'fac85dd0399009af36f625232daa5b67ab7fda391dbf95229d4237c307b142ca' +PIN_CONTROLLER = Path('/var/lib/rmm-time-source-operators') / PIN_CONTROLLER_SHA / 'pin_bootstrap.py' +PROFILE = 'rmm.server-time-gateway.backend-continuation.v1' +PHASE = 'continue-backend-v1' +DIAGNOSTIC_RECEIPT_SHA = 'a06256380d2febeddb506ffe17a614325b25c25464f45b4d022c82f644a0fc0f' +RECORDS = ('intent.json','before.json','tag-intent.json','tag-exit.json','tag.log', + 'migrate-intent.json','migrate-exit.json','migrate.log','recreate-intent.json','recreate-exit.json', + 'recreate.log','after.json','result.json','failure.json') + +class GuardDenied(RuntimeError):pass + +def require(ok,reason): + if not ok:raise GuardDenied(reason) + +def sha(raw):return hashlib.sha256(raw).hexdigest() + +ORIGINAL_RECORDS = {'before.json': {'metadata': {'ctime_ns': 1794745703017552725, + 'dev': 2049, + 'gid': 0, + 'ino': 417059, + 'mode': 33152, + 'mtime_ns': 1794745703017552725, + 'nlink': 1, + 'size': 14614, + 'uid': 0}, + 'sha256': '7d3b172db67321a49f103e2893a445259eda6e14fdb98ff3aa91c50eb41793a1'}, + 'failure.json': {'metadata': {'ctime_ns': 1794745703413559883, + 'dev': 2049, + 'gid': 0, + 'ino': 417066, + 'mode': 33152, + 'mtime_ns': 1794745703413559883, + 'nlink': 1, + 'size': 229, + 'uid': 0}, + 'sha256': '5d0fe5d630394ea4d7d6554526c585cac473ae181baf1895d869573940ba4fd5'}, + 'intent.json': {'metadata': {'ctime_ns': 1794745703014552671, + 'dev': 2049, + 'gid': 0, + 'ino': 417058, + 'mode': 33152, + 'mtime_ns': 1794745703014552671, + 'nlink': 1, + 'size': 1621, + 'uid': 0}, + 'sha256': '579a74c0f997b9c0b8fe2899929a42f3a009fd86ef6f62b11bfb2733bf9a7496'}, + 'overlay-installed.json': {'metadata': {'ctime_ns': 1794745703320558202, + 'dev': 2049, + 'gid': 0, + 'ino': 417063, + 'mode': 33152, + 'mtime_ns': 1794745703320558202, + 'nlink': 1, + 'size': 1684, + 'uid': 0}, + 'sha256': 'fa5b871d5bc259d9a2fdeedeb3a259ee56fe8f064f9588cda4eae8ec28133551'}, + 'overlay-intent.json': {'metadata': {'ctime_ns': 1794745703313558076, + 'dev': 2049, + 'gid': 0, + 'ino': 417062, + 'mode': 33152, + 'mtime_ns': 1794745703313558076, + 'nlink': 1, + 'size': 1602, + 'uid': 0}, + 'sha256': 'baad163ee43a3570ddec5cbc73ddc2ac7a1273c8f3fbb2f0c99752348e605cbc'}, + 'reservation.json': {'metadata': {'ctime_ns': 1794745703310558021, + 'dev': 2049, + 'gid': 0, + 'ino': 417061, + 'mode': 33152, + 'mtime_ns': 1794745703310558021, + 'nlink': 1, + 'size': 1592, + 'uid': 0}, + 'sha256': '5b1cd41a9ea91e1833c572f57c07bd32082e8f7170843d688c93fe269d3d9300'}, + 'upgrade-tool-installed.json': {'metadata': {'ctime_ns': 1794745703332558419, + 'dev': 2049, + 'gid': 0, + 'ino': 417065, + 'mode': 33152, + 'mtime_ns': 1794745703332558419, + 'nlink': 1, + 'size': 1686, + 'uid': 0}, + 'sha256': '5e976330fc1fa838432a3ed404bd0d999087becc5933a91e33df0339f590a47d'}, + 'upgrade-tool-intent.json': {'metadata': {'ctime_ns': 1794745703325558293, + 'dev': 2049, + 'gid': 0, + 'ino': 417064, + 'mode': 33152, + 'mtime_ns': 1794745703325558293, + 'nlink': 1, + 'size': 1593, + 'uid': 0}, + 'sha256': '1a9a689b6c3241ca7ce7c6c156f073937a854b6b5d37d6f9a3cfc199a9c4645c'}} +ORIGINAL_MISSING = ('after.json', 'migrate-exit.json', 'migrate-intent.json', 'migrate.log', 'old-upgrade-tool.bin', 'recreate-exit.json', 'recreate-intent.json', 'recreate.log', 'result.json', 'tag-exit.json', 'tag-intent.json', 'tag.log') + +def load_pin_controller(): + fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + for part in PIN_CONTROLLER.parent.parts[1:]: + info = os.fstat(fd) + require(info.st_uid == 0 and info.st_mode & 0o022 == 0, 'protected_controller_ancestry') + next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd) + os.close(fd) + fd = next_fd + info = os.fstat(fd) + require(info.st_uid == 0 and info.st_mode & 0o022 == 0, 'protected_controller_parent') + file_fd = os.open(PIN_CONTROLLER.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=fd) + with os.fdopen(file_fd, 'rb') as stream: + before = os.fstat(stream.fileno()) + require(stat.S_ISREG(before.st_mode) and before.st_uid == before.st_gid == 0 and + before.st_nlink == 1 and stat.S_IMODE(before.st_mode) == 0o600 and + before.st_size <= 128 * 1024, 'reviewed_root_controller_file') + raw = stream.read(128 * 1024 + 1) + stable = lambda s: tuple(getattr(s, 'st_' + k) for k in + ('dev', 'ino', 'uid', 'gid', 'mode', 'nlink', 'size', 'mtime_ns', 'ctime_ns')) + require(stable(before) == stable(os.fstat(stream.fileno())) == + stable(os.stat(PIN_CONTROLLER.name, dir_fd=fd, follow_symlinks=False)) and + len(raw) == before.st_size and sha(raw) == PIN_CONTROLLER_SHA, 'exact_pin_controller_bytes') + finally: + os.close(fd) + pin = importlib.util.module_from_spec(importlib.util.spec_from_loader('rmm_peer_pin_guard', loader=None)) + pin.__file__ = str(PIN_CONTROLLER) + exec(compile(raw, str(PIN_CONTROLLER), 'exec'), pin.__dict__) + held = pin.read_file(PIN_CONTROLLER, 128 * 1024) + require(held[0] == raw, 'same_loaded_pin_controller') + return pin, lambda: require(pin.read_file(PIN_CONTROLLER, 128 * 1024) == held, 'pin_controller_unchanged') + + +def canonical_containers(value): + require(type(value) is dict and set(value)=={'api','gateway','postgres'},'exact_three_role_container_observation') + result={} + for role,row in value.items(): + require(type(row) is list and len(row)==12 and type(row[4]) is list,'fixed_docker_metadata_shape') + destinations=[] + for mount in row[4]: + require(type(mount) is dict and type(mount.get('Destination')) is str + and mount['Destination'].startswith('/'),'known_absolute_mount_destination') + destinations.append(mount['Destination']) + require(len(destinations)==len(set(destinations)),'unique_docker_mount_destinations') + result[role]=list(row) + result[role][4]=sorted(row[4],key=lambda mount:mount['Destination']) + return result + + +def stable_chrony(binding): + value=json.loads(json.dumps(binding)) + # Exactly the two incidental directory-entry timestamps excluded by the + # original pin.observation_key. Every process/image/socket field remains. + del value['directory_metadata']['mtime_ns'] + del value['directory_metadata']['ctime_ns'] + return value + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('command',choices=('host-preflight','host-continue','host-read')) + args=parser.parse_args() + require(sys.flags.isolated==1 and os.geteuid()==os.getegid()==0 and os.uname().machine=='aarch64', + 'isolated_root_cloud_arm64_only') + require(not any(name in os.environ for name in ('DOCKER_HOST','DOCKER_CONTEXT','COMPOSE_PROJECT_NAME','COMPOSE_FILE','COMPOSE_PROFILES')), + 'fixed_local_docker_compose_context') + os.umask(0o077) + pin,pin_guard=load_pin_controller() + raw,old_meta=pin.read_file(OLD_SOURCE,131072);require(sha(raw)==OLD_SHA,'exact_original_gateway_controller') + old=importlib.util.module_from_spec(importlib.util.spec_from_loader('rmm_retained_gateway_deploy',loader=None)) + old.__file__=str(OLD_SOURCE);exec(compile(raw,str(OLD_SOURCE),'exec'),old.__dict__) + here=old.CASE/PHASE + if args.command=='host-read': + records={} + for name in RECORDS: + try: + data,metadata=pin.read_file(here/name,8*1024*1024 if name.endswith('.log') else 1024*1024) + records[name]=dict(sha256=sha(data),metadata=metadata,record=pin.parsed(data) if name.endswith('.json') else None) + except FileNotFoundError:records[name]=None + print(pin.encoded(dict(profile=PROFILE,case=old.CASE_ID,phase=PHASE,read_only=True,records=records)).decode(),end='');return + operator,operator_guard=pin.load_operator() + own=pin.read_file(Path(__file__),131072) + require(Path(__file__).name=='gateway_continue.py' and Path(__file__).parent.name==sha(own[0]),'fixed_continuation_source') + original={} + for name,expected in ORIGINAL_RECORDS.items(): + retained=pin.read_file(old.CASE/name,1024*1024) + require(sha(retained[0])==expected['sha256'] and retained[1]==expected['metadata'],'exact_retained_original_phase_record') + original[name]=retained + original_fd=pin.directory(old.CASE);phase=-1;broker=None + try: + for name in ORIGINAL_MISSING:pin.absence(original_fd,name) + pin.absence(original_fd,PHASE) + failure=pin.parsed(original['failure.json'][0]) + require(failure['fixed_tag']=='last_original_runtime_and_migrations_cas' + and failure['read_original_case_only'] is True,'proven_original_failure_before_backend_effects') + before_record=pin.parsed(original['before.json'][0]);baseline=before_record['witness'] + require(before_record['controller_sha256']==OLD_SHA and before_record['case']==old.CASE_ID + and before_record['image_id']==old.IMAGE and before_record['stage']==old.STAGE_ID,'same_original_fixed_gateway_plan') + reservation=pin.read_file(old.STAGE/'time-gateway-deploy-intent.json',65536) + require(reservation[0]==original['reservation.json'][0],'same_original_stage_reservation_no_generic_retry') + installed_overlay=pin.read_file(old.ROOT/'deploy/server.time-broker.compose.yaml',4096,0o644) + installed_tool=pin.read_file(old.ROOT/'deploy/cloud_server_upgrade.py',131072,0o644) + require(sha(installed_overlay[0])==old.OVERLAY_SHA and installed_overlay[1]==pin.parsed(original['overlay-installed.json'][0])['metadata'] + and sha(installed_tool[0])==old.UPGRADE_SHA and installed_tool[1]==pin.parsed(original['upgrade-tool-installed.json'][0])['metadata'], + 'only_verify_original_installed_effects_exact_bytes_and_inodes') + source=pin.read_file(old.STAGE/'source.json',1024*1024);built=pin.read_file(old.STAGE/'build-result.json',65536) + compose=pin.read_file(old.ROOT/'deploy/server.compose.yaml',8192,0o644) + require(sha(source[0])==old.SOURCE_RECORD_SHA and sha(built[0])==old.BUILD_RESULT_SHA + and sha(compose[0])==old.COMPOSE_SHA,'unchanged_original_qualified_source_build_compose') + source_value=pin.parsed(source[0]) + require(all(sha(pin.read_file(old.ROOT/name,2*1024*1024,0o644)[0])==digest for name,digest in source_value['files'].items()) + and len(source_value['files'])==209,'same_original_published_source_files') + signature=pin.read_file(old.BROKER_START.parent/'broker-signature-acceptance-v1/result.json',65536) + require(sha(signature[0])==old.BROKER_SIGNATURE_RESULT_SHA,'same_original_actual_positive_signature_proof') + public=pin.read_file(Path('/etc/rmm-time-source/time-public-key.json'),8192) + require(sha(public[0])==old.TIME_PUBLIC_PIN_SHA,'same_actual_public_time_pin') + broker=old.BrokerRuntime(pin,operator) + cmd=['docker','compose','--env-file',str(old.ROOT/'devstate/.env'),'-f',str(old.ROOT/'deploy/server.compose.yaml'), + '-f',str(old.ROOT/'deploy/server.time-broker.compose.yaml')] + expected=canonical_containers(baseline['containers']) + def common(): + pin_guard();operator_guard();broker.verify() + require(pin.read_file(OLD_SOURCE,131072)==(raw,old_meta) and pin.read_file(Path(__file__),131072)==own,'retained_reviewed_controller_sources') + for name,retained in original.items():require(pin.read_file(old.CASE/name,1024*1024)==retained,'original_failed_phase_records_unchanged') + for name in ORIGINAL_MISSING:pin.absence(original_fd,name) + require(pin.read_file(old.STAGE/'time-gateway-deploy-intent.json',65536)==reservation + and pin.read_file(old.STAGE/'source.json',1024*1024)==source and pin.read_file(old.STAGE/'build-result.json',65536)==built + and pin.read_file(old.ROOT/'deploy/server.compose.yaml',8192,0o644)==compose,'same_original_reservation_build_and_compose') + require(pin.read_file(old.ROOT/'deploy/server.time-broker.compose.yaml',4096,0o644)==installed_overlay + and pin.read_file(old.ROOT/'deploy/cloud_server_upgrade.py',131072,0o644)==installed_tool,'installed_overlay_tool_never_repeated_or_replaced') + require(old.file_tree(old.ROOT/'web/dist',True)==baseline['web'] + and {name:old.file_tree(Path(name)) for name in baseline['key_metadata']}==baseline['key_metadata'], + 'same_original_ui_pki_key_policy_metadata') + require(pin.read_file(old.BROKER_START.parent/'broker-signature-acceptance-v1/result.json',65536)==signature + and pin.read_file(Path('/etc/rmm-time-source/time-public-key.json'),8192)==public,'original_signature_and_time_key_unchanged') + # ecb.preflight deliberately requires old Runtime6f. Reuse its + # complete independent Chrony getter without changing that guard. + current=operator.chrony_observation();previous=baseline['actual'] + boot=operator.regular(Path('/proc/sys/kernel/random/boot_id'),128).decode().strip() + builder=json.loads(operator.run(['docker','image','inspect','--format', + '[{{json .Id}},{{json .Architecture}},{{json .Os}}]',operator.BUILDER])) + require(boot==previous['boot_id'] and dict(zip(('Id','Architecture','Os'),builder,strict=True))==previous['builder'] + and current['pin']==previous['chrony_observation'] + and current['fixed_images']['daemon']==previous['chrony_image_metadata'], + 'same_original_actual_boot_chrony_builder_image') + require(stable_chrony(current)==stable_chrony(previous['chrony_binding']), + 'same_original_actual_chrony_process_cgroup_socket') + return current + common() + require(canonical_containers(old.containers(operator))==expected + and old.migrations(operator,cmd)==baseline['migrations']==list(range(1,15)), + 'only_nondeterministic_mount_order_changed_before_backend_continuation') + require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.OLD_IMAGE, + 'retained_original_runtime_tag_cas') + require(json.loads(operator.run(['docker','image','inspect',old.IMAGE,'--format','[{{json .Id}},{{json .Architecture}},{{json .Os}}]'])) + ==[old.IMAGE,'arm64','linux'],'same_existing_qualified_native_image') + binding=dict(profile=PROFILE,case=old.CASE_ID,phase=PHASE,controller_sha256=sha(own[0]),original_controller_sha256=OLD_SHA, + original_before_sha256=ORIGINAL_RECORDS['before.json']['sha256'],original_failure_sha256=ORIGINAL_RECORDS['failure.json']['sha256'], + mount_order_diagnostic_receipt_sha256=DIAGNOSTIC_RECEIPT_SHA,image_id=old.IMAGE,stage=old.STAGE_ID, + installed_overlay_sha256=old.OVERLAY_SHA,installed_tool_sha256=old.UPGRADE_SHA) + if args.command=='host-preflight': + print(pin.encoded(dict(binding,read_only=True,backend_unentered=True,canonical_containers=expected,migrations=baseline['migrations'])).decode(),end='');return + os.mkdir(PHASE,0o700,dir_fd=original_fd);os.fsync(original_fd);phase=pin.directory(here) + save=lambda name,value:old.create(pin,phase,name,value) + save('intent.json',dict(binding,once=True,no_repeat_installed_effects=True,no_retry=True)) + save('before.json',dict(binding,containers=expected,migrations=baseline['migrations'],broker=broker.verify())) + def effect(name,argv,timeout): + common();save(name+'-intent.json',dict(binding,argv=argv,once=True)) + code=operator.long_command(argv,here/(name+'.log'),timeout) + save(name+'-exit.json',dict(binding,exit_code=code));require(code==0,'backend_effect_failed_original_continuation_read_only') + require(canonical_containers(old.containers(operator))==expected and old.migrations(operator,cmd)==list(range(1,15)), + 'exact_original_runtime_and_schema_before_first_backend_effect') + require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.OLD_IMAGE,'last_runtime_tag_cas') + effect('tag',['docker','tag',old.IMAGE,'rust-discovery-server:local'],10) + require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.IMAGE,'qualified_runtime_tag_after_once') + effect('migrate',cmd+['run','--rm','--no-deps','migrate'],25) + require(canonical_containers(old.containers(operator))==expected and old.migrations(operator,cmd)==list(range(1,16)), + 'only_migration_fifteen_before_recreate') + effect('recreate',cmd+['up','-d','--no-deps','--force-recreate','--no-build','api','gateway'],25) + after=canonical_containers(old.containers(operator)) + require(after['postgres']==expected['postgres'],'original_postgres_container_volume_generation_unchanged') + for role in ('api','gateway'): + previous,current=expected[role],after[role] + require(current[0]!=previous[0] and current[1]==old.IMAGE and current[2:4]==previous[2:4] + and current[5:9]==previous[5:9],'only_qualified_role_image_generation_changed') + mounts=current[4] + if role=='gateway': + added=[mount for mount in mounts if mount.get('Destination')=='/run/rmm-time-source'] + require(len(added)==1 and added[0]['Source']=='/run/rmm-time-source' and added[0]['Type']=='bind' + and added[0]['RW'] is False,'one_actual_readonly_gateway_time_mount') + mounts=[mount for mount in mounts if mount.get('Destination')!='/run/rmm-time-source'] + require(mounts==previous[4],'all_original_mount_semantics_retained_no_api_time_mount') + require(old.migrations(operator,cmd)==list(range(1,16)),'actual_additive_schema_through_fifteen') + actual_chrony=common() + save('after.json',dict(binding,containers=after,migrations=list(range(1,16)),broker=broker.verify(),chrony=actual_chrony)) + save('result.json',dict(binding,deployed=True,installed_effects_repeated=False,source_rebuilt=False,ui_rewritten=False, + existing_pki_keys_policy_pg_container_preserved=True,broker_chrony_not_restarted=True, + gateway_only_readonly_broker_mount=True, + rmm_jobs_created=False,approvals_created=False,immutable_corehistory_witness='external_root_before_after_required')) + print(pin.encoded(dict(binding,deployed=True)).decode(),end='') + except Exception as error: + if phase>=0: + tag=str(error) if isinstance(error,GuardDenied) and 0=0:os.close(fd) + + +if __name__=='__main__': + try:main() + except Exception as error: + tag=str(error) if isinstance(error,GuardDenied) and 0