Files
rust-discovery/deploy/server_time_gateway_continue.py
T
2026-11-15 13:36:50 +01:00

330 lines
22 KiB
Python

#!/usr/bin/env python3
"""Only the unentered fixed gateway backend effects of original fb87; no installed-effect replay."""
import argparse
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import stat
import sys
OLD_SHA = 'e14aeabfd7d3a8ab298985706dc26014e0ad540a1c3ef40cf57e1a65b77b7f3b'
OLD_SOURCE = Path('/var/lib/rmm-time-source-operators') / OLD_SHA / 'gateway_deploy.py'
PIN_CONTROLLER_SHA = 'fac85dd0399009af36f625232daa5b67ab7fda391dbf95229d4237c307b142ca'
PIN_CONTROLLER = Path('/var/lib/rmm-time-source-operators') / PIN_CONTROLLER_SHA / 'pin_bootstrap.py'
PROFILE = 'rmm.server-time-gateway.backend-continuation.v1'
PHASE = 'continue-backend-v1'
DIAGNOSTIC_RECEIPT_SHA = 'a06256380d2febeddb506ffe17a614325b25c25464f45b4d022c82f644a0fc0f'
RECORDS = ('intent.json','before.json','tag-intent.json','tag-exit.json','tag.log',
'migrate-intent.json','migrate-exit.json','migrate.log','recreate-intent.json','recreate-exit.json',
'recreate.log','after.json','result.json','failure.json')
class GuardDenied(RuntimeError):pass
def require(ok,reason):
if not ok:raise GuardDenied(reason)
def sha(raw):return hashlib.sha256(raw).hexdigest()
ORIGINAL_RECORDS = {'before.json': {'metadata': {'ctime_ns': 1794745703017552725,
'dev': 2049,
'gid': 0,
'ino': 417059,
'mode': 33152,
'mtime_ns': 1794745703017552725,
'nlink': 1,
'size': 14614,
'uid': 0},
'sha256': '7d3b172db67321a49f103e2893a445259eda6e14fdb98ff3aa91c50eb41793a1'},
'failure.json': {'metadata': {'ctime_ns': 1794745703413559883,
'dev': 2049,
'gid': 0,
'ino': 417066,
'mode': 33152,
'mtime_ns': 1794745703413559883,
'nlink': 1,
'size': 229,
'uid': 0},
'sha256': '5d0fe5d630394ea4d7d6554526c585cac473ae181baf1895d869573940ba4fd5'},
'intent.json': {'metadata': {'ctime_ns': 1794745703014552671,
'dev': 2049,
'gid': 0,
'ino': 417058,
'mode': 33152,
'mtime_ns': 1794745703014552671,
'nlink': 1,
'size': 1621,
'uid': 0},
'sha256': '579a74c0f997b9c0b8fe2899929a42f3a009fd86ef6f62b11bfb2733bf9a7496'},
'overlay-installed.json': {'metadata': {'ctime_ns': 1794745703320558202,
'dev': 2049,
'gid': 0,
'ino': 417063,
'mode': 33152,
'mtime_ns': 1794745703320558202,
'nlink': 1,
'size': 1684,
'uid': 0},
'sha256': 'fa5b871d5bc259d9a2fdeedeb3a259ee56fe8f064f9588cda4eae8ec28133551'},
'overlay-intent.json': {'metadata': {'ctime_ns': 1794745703313558076,
'dev': 2049,
'gid': 0,
'ino': 417062,
'mode': 33152,
'mtime_ns': 1794745703313558076,
'nlink': 1,
'size': 1602,
'uid': 0},
'sha256': 'baad163ee43a3570ddec5cbc73ddc2ac7a1273c8f3fbb2f0c99752348e605cbc'},
'reservation.json': {'metadata': {'ctime_ns': 1794745703310558021,
'dev': 2049,
'gid': 0,
'ino': 417061,
'mode': 33152,
'mtime_ns': 1794745703310558021,
'nlink': 1,
'size': 1592,
'uid': 0},
'sha256': '5b1cd41a9ea91e1833c572f57c07bd32082e8f7170843d688c93fe269d3d9300'},
'upgrade-tool-installed.json': {'metadata': {'ctime_ns': 1794745703332558419,
'dev': 2049,
'gid': 0,
'ino': 417065,
'mode': 33152,
'mtime_ns': 1794745703332558419,
'nlink': 1,
'size': 1686,
'uid': 0},
'sha256': '5e976330fc1fa838432a3ed404bd0d999087becc5933a91e33df0339f590a47d'},
'upgrade-tool-intent.json': {'metadata': {'ctime_ns': 1794745703325558293,
'dev': 2049,
'gid': 0,
'ino': 417064,
'mode': 33152,
'mtime_ns': 1794745703325558293,
'nlink': 1,
'size': 1593,
'uid': 0},
'sha256': '1a9a689b6c3241ca7ce7c6c156f073937a854b6b5d37d6f9a3cfc199a9c4645c'}}
ORIGINAL_MISSING = ('after.json', 'migrate-exit.json', 'migrate-intent.json', 'migrate.log', 'old-upgrade-tool.bin', 'recreate-exit.json', 'recreate-intent.json', 'recreate.log', 'result.json', 'tag-exit.json', 'tag-intent.json', 'tag.log')
def load_pin_controller():
fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
for part in PIN_CONTROLLER.parent.parts[1:]:
info = os.fstat(fd)
require(info.st_uid == 0 and info.st_mode & 0o022 == 0, 'protected_controller_ancestry')
next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
os.close(fd)
fd = next_fd
info = os.fstat(fd)
require(info.st_uid == 0 and info.st_mode & 0o022 == 0, 'protected_controller_parent')
file_fd = os.open(PIN_CONTROLLER.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=fd)
with os.fdopen(file_fd, 'rb') as stream:
before = os.fstat(stream.fileno())
require(stat.S_ISREG(before.st_mode) and before.st_uid == before.st_gid == 0 and
before.st_nlink == 1 and stat.S_IMODE(before.st_mode) == 0o600 and
before.st_size <= 128 * 1024, 'reviewed_root_controller_file')
raw = stream.read(128 * 1024 + 1)
stable = lambda s: tuple(getattr(s, 'st_' + k) for k in
('dev', 'ino', 'uid', 'gid', 'mode', 'nlink', 'size', 'mtime_ns', 'ctime_ns'))
require(stable(before) == stable(os.fstat(stream.fileno())) ==
stable(os.stat(PIN_CONTROLLER.name, dir_fd=fd, follow_symlinks=False)) and
len(raw) == before.st_size and sha(raw) == PIN_CONTROLLER_SHA, 'exact_pin_controller_bytes')
finally:
os.close(fd)
pin = importlib.util.module_from_spec(importlib.util.spec_from_loader('rmm_peer_pin_guard', loader=None))
pin.__file__ = str(PIN_CONTROLLER)
exec(compile(raw, str(PIN_CONTROLLER), 'exec'), pin.__dict__)
held = pin.read_file(PIN_CONTROLLER, 128 * 1024)
require(held[0] == raw, 'same_loaded_pin_controller')
return pin, lambda: require(pin.read_file(PIN_CONTROLLER, 128 * 1024) == held, 'pin_controller_unchanged')
def canonical_containers(value):
require(type(value) is dict and set(value)=={'api','gateway','postgres'},'exact_three_role_container_observation')
result={}
for role,row in value.items():
require(type(row) is list and len(row)==12 and type(row[4]) is list,'fixed_docker_metadata_shape')
destinations=[]
for mount in row[4]:
require(type(mount) is dict and type(mount.get('Destination')) is str
and mount['Destination'].startswith('/'),'known_absolute_mount_destination')
destinations.append(mount['Destination'])
require(len(destinations)==len(set(destinations)),'unique_docker_mount_destinations')
result[role]=list(row)
result[role][4]=sorted(row[4],key=lambda mount:mount['Destination'])
return result
def stable_chrony(binding):
value=json.loads(json.dumps(binding))
# Exactly the two incidental directory-entry timestamps excluded by the
# original pin.observation_key. Every process/image/socket field remains.
del value['directory_metadata']['mtime_ns']
del value['directory_metadata']['ctime_ns']
return value
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('command',choices=('host-preflight','host-continue','host-read'))
args=parser.parse_args()
require(sys.flags.isolated==1 and os.geteuid()==os.getegid()==0 and os.uname().machine=='aarch64',
'isolated_root_cloud_arm64_only')
require(not any(name in os.environ for name in ('DOCKER_HOST','DOCKER_CONTEXT','COMPOSE_PROJECT_NAME','COMPOSE_FILE','COMPOSE_PROFILES')),
'fixed_local_docker_compose_context')
os.umask(0o077)
pin,pin_guard=load_pin_controller()
raw,old_meta=pin.read_file(OLD_SOURCE,131072);require(sha(raw)==OLD_SHA,'exact_original_gateway_controller')
old=importlib.util.module_from_spec(importlib.util.spec_from_loader('rmm_retained_gateway_deploy',loader=None))
old.__file__=str(OLD_SOURCE);exec(compile(raw,str(OLD_SOURCE),'exec'),old.__dict__)
here=old.CASE/PHASE
if args.command=='host-read':
records={}
for name in RECORDS:
try:
data,metadata=pin.read_file(here/name,8*1024*1024 if name.endswith('.log') else 1024*1024)
records[name]=dict(sha256=sha(data),metadata=metadata,record=pin.parsed(data) if name.endswith('.json') else None)
except FileNotFoundError:records[name]=None
print(pin.encoded(dict(profile=PROFILE,case=old.CASE_ID,phase=PHASE,read_only=True,records=records)).decode(),end='');return
operator,operator_guard=pin.load_operator()
own=pin.read_file(Path(__file__),131072)
require(Path(__file__).name=='gateway_continue.py' and Path(__file__).parent.name==sha(own[0]),'fixed_continuation_source')
original={}
for name,expected in ORIGINAL_RECORDS.items():
retained=pin.read_file(old.CASE/name,1024*1024)
require(sha(retained[0])==expected['sha256'] and retained[1]==expected['metadata'],'exact_retained_original_phase_record')
original[name]=retained
original_fd=pin.directory(old.CASE);phase=-1;broker=None
try:
for name in ORIGINAL_MISSING:pin.absence(original_fd,name)
pin.absence(original_fd,PHASE)
failure=pin.parsed(original['failure.json'][0])
require(failure['fixed_tag']=='last_original_runtime_and_migrations_cas'
and failure['read_original_case_only'] is True,'proven_original_failure_before_backend_effects')
before_record=pin.parsed(original['before.json'][0]);baseline=before_record['witness']
require(before_record['controller_sha256']==OLD_SHA and before_record['case']==old.CASE_ID
and before_record['image_id']==old.IMAGE and before_record['stage']==old.STAGE_ID,'same_original_fixed_gateway_plan')
reservation=pin.read_file(old.STAGE/'time-gateway-deploy-intent.json',65536)
require(reservation[0]==original['reservation.json'][0],'same_original_stage_reservation_no_generic_retry')
installed_overlay=pin.read_file(old.ROOT/'deploy/server.time-broker.compose.yaml',4096,0o644)
installed_tool=pin.read_file(old.ROOT/'deploy/cloud_server_upgrade.py',131072,0o644)
require(sha(installed_overlay[0])==old.OVERLAY_SHA and installed_overlay[1]==pin.parsed(original['overlay-installed.json'][0])['metadata']
and sha(installed_tool[0])==old.UPGRADE_SHA and installed_tool[1]==pin.parsed(original['upgrade-tool-installed.json'][0])['metadata'],
'only_verify_original_installed_effects_exact_bytes_and_inodes')
source=pin.read_file(old.STAGE/'source.json',1024*1024);built=pin.read_file(old.STAGE/'build-result.json',65536)
compose=pin.read_file(old.ROOT/'deploy/server.compose.yaml',8192,0o644)
require(sha(source[0])==old.SOURCE_RECORD_SHA and sha(built[0])==old.BUILD_RESULT_SHA
and sha(compose[0])==old.COMPOSE_SHA,'unchanged_original_qualified_source_build_compose')
source_value=pin.parsed(source[0])
require(all(sha(pin.read_file(old.ROOT/name,2*1024*1024,0o644)[0])==digest for name,digest in source_value['files'].items())
and len(source_value['files'])==209,'same_original_published_source_files')
signature=pin.read_file(old.BROKER_START.parent/'broker-signature-acceptance-v1/result.json',65536)
require(sha(signature[0])==old.BROKER_SIGNATURE_RESULT_SHA,'same_original_actual_positive_signature_proof')
public=pin.read_file(Path('/etc/rmm-time-source/time-public-key.json'),8192)
require(sha(public[0])==old.TIME_PUBLIC_PIN_SHA,'same_actual_public_time_pin')
broker=old.BrokerRuntime(pin,operator)
cmd=['docker','compose','--env-file',str(old.ROOT/'devstate/.env'),'-f',str(old.ROOT/'deploy/server.compose.yaml'),
'-f',str(old.ROOT/'deploy/server.time-broker.compose.yaml')]
expected=canonical_containers(baseline['containers'])
def common():
pin_guard();operator_guard();broker.verify()
require(pin.read_file(OLD_SOURCE,131072)==(raw,old_meta) and pin.read_file(Path(__file__),131072)==own,'retained_reviewed_controller_sources')
for name,retained in original.items():require(pin.read_file(old.CASE/name,1024*1024)==retained,'original_failed_phase_records_unchanged')
for name in ORIGINAL_MISSING:pin.absence(original_fd,name)
require(pin.read_file(old.STAGE/'time-gateway-deploy-intent.json',65536)==reservation
and pin.read_file(old.STAGE/'source.json',1024*1024)==source and pin.read_file(old.STAGE/'build-result.json',65536)==built
and pin.read_file(old.ROOT/'deploy/server.compose.yaml',8192,0o644)==compose,'same_original_reservation_build_and_compose')
require(pin.read_file(old.ROOT/'deploy/server.time-broker.compose.yaml',4096,0o644)==installed_overlay
and pin.read_file(old.ROOT/'deploy/cloud_server_upgrade.py',131072,0o644)==installed_tool,'installed_overlay_tool_never_repeated_or_replaced')
require(old.file_tree(old.ROOT/'web/dist',True)==baseline['web']
and {name:old.file_tree(Path(name)) for name in baseline['key_metadata']}==baseline['key_metadata'],
'same_original_ui_pki_key_policy_metadata')
require(pin.read_file(old.BROKER_START.parent/'broker-signature-acceptance-v1/result.json',65536)==signature
and pin.read_file(Path('/etc/rmm-time-source/time-public-key.json'),8192)==public,'original_signature_and_time_key_unchanged')
# ecb.preflight deliberately requires old Runtime6f. Reuse its
# complete independent Chrony getter without changing that guard.
current=operator.chrony_observation();previous=baseline['actual']
boot=operator.regular(Path('/proc/sys/kernel/random/boot_id'),128).decode().strip()
builder=json.loads(operator.run(['docker','image','inspect','--format',
'[{{json .Id}},{{json .Architecture}},{{json .Os}}]',operator.BUILDER]))
require(boot==previous['boot_id'] and dict(zip(('Id','Architecture','Os'),builder,strict=True))==previous['builder']
and current['pin']==previous['chrony_observation']
and current['fixed_images']['daemon']==previous['chrony_image_metadata'],
'same_original_actual_boot_chrony_builder_image')
require(stable_chrony(current)==stable_chrony(previous['chrony_binding']),
'same_original_actual_chrony_process_cgroup_socket')
return current
common()
require(canonical_containers(old.containers(operator))==expected
and old.migrations(operator,cmd)==baseline['migrations']==list(range(1,15)),
'only_nondeterministic_mount_order_changed_before_backend_continuation')
require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.OLD_IMAGE,
'retained_original_runtime_tag_cas')
require(json.loads(operator.run(['docker','image','inspect',old.IMAGE,'--format','[{{json .Id}},{{json .Architecture}},{{json .Os}}]']))
==[old.IMAGE,'arm64','linux'],'same_existing_qualified_native_image')
binding=dict(profile=PROFILE,case=old.CASE_ID,phase=PHASE,controller_sha256=sha(own[0]),original_controller_sha256=OLD_SHA,
original_before_sha256=ORIGINAL_RECORDS['before.json']['sha256'],original_failure_sha256=ORIGINAL_RECORDS['failure.json']['sha256'],
mount_order_diagnostic_receipt_sha256=DIAGNOSTIC_RECEIPT_SHA,image_id=old.IMAGE,stage=old.STAGE_ID,
installed_overlay_sha256=old.OVERLAY_SHA,installed_tool_sha256=old.UPGRADE_SHA)
if args.command=='host-preflight':
print(pin.encoded(dict(binding,read_only=True,backend_unentered=True,canonical_containers=expected,migrations=baseline['migrations'])).decode(),end='');return
os.mkdir(PHASE,0o700,dir_fd=original_fd);os.fsync(original_fd);phase=pin.directory(here)
save=lambda name,value:old.create(pin,phase,name,value)
save('intent.json',dict(binding,once=True,no_repeat_installed_effects=True,no_retry=True))
save('before.json',dict(binding,containers=expected,migrations=baseline['migrations'],broker=broker.verify()))
def effect(name,argv,timeout):
common();save(name+'-intent.json',dict(binding,argv=argv,once=True))
code=operator.long_command(argv,here/(name+'.log'),timeout)
save(name+'-exit.json',dict(binding,exit_code=code));require(code==0,'backend_effect_failed_original_continuation_read_only')
require(canonical_containers(old.containers(operator))==expected and old.migrations(operator,cmd)==list(range(1,15)),
'exact_original_runtime_and_schema_before_first_backend_effect')
require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.OLD_IMAGE,'last_runtime_tag_cas')
effect('tag',['docker','tag',old.IMAGE,'rust-discovery-server:local'],10)
require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.IMAGE,'qualified_runtime_tag_after_once')
effect('migrate',cmd+['run','--rm','--no-deps','migrate'],25)
require(canonical_containers(old.containers(operator))==expected and old.migrations(operator,cmd)==list(range(1,16)),
'only_migration_fifteen_before_recreate')
effect('recreate',cmd+['up','-d','--no-deps','--force-recreate','--no-build','api','gateway'],25)
after=canonical_containers(old.containers(operator))
require(after['postgres']==expected['postgres'],'original_postgres_container_volume_generation_unchanged')
for role in ('api','gateway'):
previous,current=expected[role],after[role]
require(current[0]!=previous[0] and current[1]==old.IMAGE and current[2:4]==previous[2:4]
and current[5:9]==previous[5:9],'only_qualified_role_image_generation_changed')
mounts=current[4]
if role=='gateway':
added=[mount for mount in mounts if mount.get('Destination')=='/run/rmm-time-source']
require(len(added)==1 and added[0]['Source']=='/run/rmm-time-source' and added[0]['Type']=='bind'
and added[0]['RW'] is False,'one_actual_readonly_gateway_time_mount')
mounts=[mount for mount in mounts if mount.get('Destination')!='/run/rmm-time-source']
require(mounts==previous[4],'all_original_mount_semantics_retained_no_api_time_mount')
require(old.migrations(operator,cmd)==list(range(1,16)),'actual_additive_schema_through_fifteen')
actual_chrony=common()
save('after.json',dict(binding,containers=after,migrations=list(range(1,16)),broker=broker.verify(),chrony=actual_chrony))
save('result.json',dict(binding,deployed=True,installed_effects_repeated=False,source_rebuilt=False,ui_rewritten=False,
existing_pki_keys_policy_pg_container_preserved=True,broker_chrony_not_restarted=True,
gateway_only_readonly_broker_mount=True,
rmm_jobs_created=False,approvals_created=False,immutable_corehistory_witness='external_root_before_after_required'))
print(pin.encoded(dict(binding,deployed=True)).decode(),end='')
except Exception as error:
if phase>=0:
tag=str(error) if isinstance(error,GuardDenied) and 0<len(str(error))<=128 and all(c in 'abcdefghijklmnopqrstuvwxyz0123456789_' for c in str(error)) else 'unexpected_gateway_continuation_exception'
old.create(pin,phase,'failure.json',dict(profile=PROFILE,failed=True,fixed_tag=tag,read_original_continuation_only=True))
raise
finally:
if broker is not None:broker.close()
for fd in (phase,original_fd):
if fd>=0:os.close(fd)
if __name__=='__main__':
try:main()
except Exception as error:
tag=str(error) if isinstance(error,GuardDenied) and 0<len(str(error))<=128 and all(c in 'abcdefghijklmnopqrstuvwxyz0123456789_' for c in str(error)) else 'unexpected_gateway_continuation_exception'
print(json.dumps(dict(profile=PROFILE,failed=True,fixed_tag=tag,read_original_continuation_only=True)))
raise SystemExit(1)