330 lines
22 KiB
Python
330 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Only the unentered fixed gateway backend effects of original fb87; no installed-effect replay."""
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import stat
|
|
import sys
|
|
|
|
OLD_SHA = 'e14aeabfd7d3a8ab298985706dc26014e0ad540a1c3ef40cf57e1a65b77b7f3b'
|
|
OLD_SOURCE = Path('/var/lib/rmm-time-source-operators') / OLD_SHA / 'gateway_deploy.py'
|
|
PIN_CONTROLLER_SHA = 'fac85dd0399009af36f625232daa5b67ab7fda391dbf95229d4237c307b142ca'
|
|
PIN_CONTROLLER = Path('/var/lib/rmm-time-source-operators') / PIN_CONTROLLER_SHA / 'pin_bootstrap.py'
|
|
PROFILE = 'rmm.server-time-gateway.backend-continuation.v1'
|
|
PHASE = 'continue-backend-v1'
|
|
DIAGNOSTIC_RECEIPT_SHA = 'a06256380d2febeddb506ffe17a614325b25c25464f45b4d022c82f644a0fc0f'
|
|
RECORDS = ('intent.json','before.json','tag-intent.json','tag-exit.json','tag.log',
|
|
'migrate-intent.json','migrate-exit.json','migrate.log','recreate-intent.json','recreate-exit.json',
|
|
'recreate.log','after.json','result.json','failure.json')
|
|
|
|
class GuardDenied(RuntimeError):pass
|
|
|
|
def require(ok,reason):
|
|
if not ok:raise GuardDenied(reason)
|
|
|
|
def sha(raw):return hashlib.sha256(raw).hexdigest()
|
|
|
|
ORIGINAL_RECORDS = {'before.json': {'metadata': {'ctime_ns': 1794745703017552725,
|
|
'dev': 2049,
|
|
'gid': 0,
|
|
'ino': 417059,
|
|
'mode': 33152,
|
|
'mtime_ns': 1794745703017552725,
|
|
'nlink': 1,
|
|
'size': 14614,
|
|
'uid': 0},
|
|
'sha256': '7d3b172db67321a49f103e2893a445259eda6e14fdb98ff3aa91c50eb41793a1'},
|
|
'failure.json': {'metadata': {'ctime_ns': 1794745703413559883,
|
|
'dev': 2049,
|
|
'gid': 0,
|
|
'ino': 417066,
|
|
'mode': 33152,
|
|
'mtime_ns': 1794745703413559883,
|
|
'nlink': 1,
|
|
'size': 229,
|
|
'uid': 0},
|
|
'sha256': '5d0fe5d630394ea4d7d6554526c585cac473ae181baf1895d869573940ba4fd5'},
|
|
'intent.json': {'metadata': {'ctime_ns': 1794745703014552671,
|
|
'dev': 2049,
|
|
'gid': 0,
|
|
'ino': 417058,
|
|
'mode': 33152,
|
|
'mtime_ns': 1794745703014552671,
|
|
'nlink': 1,
|
|
'size': 1621,
|
|
'uid': 0},
|
|
'sha256': '579a74c0f997b9c0b8fe2899929a42f3a009fd86ef6f62b11bfb2733bf9a7496'},
|
|
'overlay-installed.json': {'metadata': {'ctime_ns': 1794745703320558202,
|
|
'dev': 2049,
|
|
'gid': 0,
|
|
'ino': 417063,
|
|
'mode': 33152,
|
|
'mtime_ns': 1794745703320558202,
|
|
'nlink': 1,
|
|
'size': 1684,
|
|
'uid': 0},
|
|
'sha256': 'fa5b871d5bc259d9a2fdeedeb3a259ee56fe8f064f9588cda4eae8ec28133551'},
|
|
'overlay-intent.json': {'metadata': {'ctime_ns': 1794745703313558076,
|
|
'dev': 2049,
|
|
'gid': 0,
|
|
'ino': 417062,
|
|
'mode': 33152,
|
|
'mtime_ns': 1794745703313558076,
|
|
'nlink': 1,
|
|
'size': 1602,
|
|
'uid': 0},
|
|
'sha256': 'baad163ee43a3570ddec5cbc73ddc2ac7a1273c8f3fbb2f0c99752348e605cbc'},
|
|
'reservation.json': {'metadata': {'ctime_ns': 1794745703310558021,
|
|
'dev': 2049,
|
|
'gid': 0,
|
|
'ino': 417061,
|
|
'mode': 33152,
|
|
'mtime_ns': 1794745703310558021,
|
|
'nlink': 1,
|
|
'size': 1592,
|
|
'uid': 0},
|
|
'sha256': '5b1cd41a9ea91e1833c572f57c07bd32082e8f7170843d688c93fe269d3d9300'},
|
|
'upgrade-tool-installed.json': {'metadata': {'ctime_ns': 1794745703332558419,
|
|
'dev': 2049,
|
|
'gid': 0,
|
|
'ino': 417065,
|
|
'mode': 33152,
|
|
'mtime_ns': 1794745703332558419,
|
|
'nlink': 1,
|
|
'size': 1686,
|
|
'uid': 0},
|
|
'sha256': '5e976330fc1fa838432a3ed404bd0d999087becc5933a91e33df0339f590a47d'},
|
|
'upgrade-tool-intent.json': {'metadata': {'ctime_ns': 1794745703325558293,
|
|
'dev': 2049,
|
|
'gid': 0,
|
|
'ino': 417064,
|
|
'mode': 33152,
|
|
'mtime_ns': 1794745703325558293,
|
|
'nlink': 1,
|
|
'size': 1593,
|
|
'uid': 0},
|
|
'sha256': '1a9a689b6c3241ca7ce7c6c156f073937a854b6b5d37d6f9a3cfc199a9c4645c'}}
|
|
ORIGINAL_MISSING = ('after.json', 'migrate-exit.json', 'migrate-intent.json', 'migrate.log', 'old-upgrade-tool.bin', 'recreate-exit.json', 'recreate-intent.json', 'recreate.log', 'result.json', 'tag-exit.json', 'tag-intent.json', 'tag.log')
|
|
|
|
def load_pin_controller():
|
|
fd = os.open('/', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
try:
|
|
for part in PIN_CONTROLLER.parent.parts[1:]:
|
|
info = os.fstat(fd)
|
|
require(info.st_uid == 0 and info.st_mode & 0o022 == 0, 'protected_controller_ancestry')
|
|
next_fd = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
os.close(fd)
|
|
fd = next_fd
|
|
info = os.fstat(fd)
|
|
require(info.st_uid == 0 and info.st_mode & 0o022 == 0, 'protected_controller_parent')
|
|
file_fd = os.open(PIN_CONTROLLER.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=fd)
|
|
with os.fdopen(file_fd, 'rb') as stream:
|
|
before = os.fstat(stream.fileno())
|
|
require(stat.S_ISREG(before.st_mode) and before.st_uid == before.st_gid == 0 and
|
|
before.st_nlink == 1 and stat.S_IMODE(before.st_mode) == 0o600 and
|
|
before.st_size <= 128 * 1024, 'reviewed_root_controller_file')
|
|
raw = stream.read(128 * 1024 + 1)
|
|
stable = lambda s: tuple(getattr(s, 'st_' + k) for k in
|
|
('dev', 'ino', 'uid', 'gid', 'mode', 'nlink', 'size', 'mtime_ns', 'ctime_ns'))
|
|
require(stable(before) == stable(os.fstat(stream.fileno())) ==
|
|
stable(os.stat(PIN_CONTROLLER.name, dir_fd=fd, follow_symlinks=False)) and
|
|
len(raw) == before.st_size and sha(raw) == PIN_CONTROLLER_SHA, 'exact_pin_controller_bytes')
|
|
finally:
|
|
os.close(fd)
|
|
pin = importlib.util.module_from_spec(importlib.util.spec_from_loader('rmm_peer_pin_guard', loader=None))
|
|
pin.__file__ = str(PIN_CONTROLLER)
|
|
exec(compile(raw, str(PIN_CONTROLLER), 'exec'), pin.__dict__)
|
|
held = pin.read_file(PIN_CONTROLLER, 128 * 1024)
|
|
require(held[0] == raw, 'same_loaded_pin_controller')
|
|
return pin, lambda: require(pin.read_file(PIN_CONTROLLER, 128 * 1024) == held, 'pin_controller_unchanged')
|
|
|
|
|
|
def canonical_containers(value):
|
|
require(type(value) is dict and set(value)=={'api','gateway','postgres'},'exact_three_role_container_observation')
|
|
result={}
|
|
for role,row in value.items():
|
|
require(type(row) is list and len(row)==12 and type(row[4]) is list,'fixed_docker_metadata_shape')
|
|
destinations=[]
|
|
for mount in row[4]:
|
|
require(type(mount) is dict and type(mount.get('Destination')) is str
|
|
and mount['Destination'].startswith('/'),'known_absolute_mount_destination')
|
|
destinations.append(mount['Destination'])
|
|
require(len(destinations)==len(set(destinations)),'unique_docker_mount_destinations')
|
|
result[role]=list(row)
|
|
result[role][4]=sorted(row[4],key=lambda mount:mount['Destination'])
|
|
return result
|
|
|
|
|
|
def stable_chrony(binding):
|
|
value=json.loads(json.dumps(binding))
|
|
# Exactly the two incidental directory-entry timestamps excluded by the
|
|
# original pin.observation_key. Every process/image/socket field remains.
|
|
del value['directory_metadata']['mtime_ns']
|
|
del value['directory_metadata']['ctime_ns']
|
|
return value
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('command',choices=('host-preflight','host-continue','host-read'))
|
|
args=parser.parse_args()
|
|
require(sys.flags.isolated==1 and os.geteuid()==os.getegid()==0 and os.uname().machine=='aarch64',
|
|
'isolated_root_cloud_arm64_only')
|
|
require(not any(name in os.environ for name in ('DOCKER_HOST','DOCKER_CONTEXT','COMPOSE_PROJECT_NAME','COMPOSE_FILE','COMPOSE_PROFILES')),
|
|
'fixed_local_docker_compose_context')
|
|
os.umask(0o077)
|
|
pin,pin_guard=load_pin_controller()
|
|
raw,old_meta=pin.read_file(OLD_SOURCE,131072);require(sha(raw)==OLD_SHA,'exact_original_gateway_controller')
|
|
old=importlib.util.module_from_spec(importlib.util.spec_from_loader('rmm_retained_gateway_deploy',loader=None))
|
|
old.__file__=str(OLD_SOURCE);exec(compile(raw,str(OLD_SOURCE),'exec'),old.__dict__)
|
|
here=old.CASE/PHASE
|
|
if args.command=='host-read':
|
|
records={}
|
|
for name in RECORDS:
|
|
try:
|
|
data,metadata=pin.read_file(here/name,8*1024*1024 if name.endswith('.log') else 1024*1024)
|
|
records[name]=dict(sha256=sha(data),metadata=metadata,record=pin.parsed(data) if name.endswith('.json') else None)
|
|
except FileNotFoundError:records[name]=None
|
|
print(pin.encoded(dict(profile=PROFILE,case=old.CASE_ID,phase=PHASE,read_only=True,records=records)).decode(),end='');return
|
|
operator,operator_guard=pin.load_operator()
|
|
own=pin.read_file(Path(__file__),131072)
|
|
require(Path(__file__).name=='gateway_continue.py' and Path(__file__).parent.name==sha(own[0]),'fixed_continuation_source')
|
|
original={}
|
|
for name,expected in ORIGINAL_RECORDS.items():
|
|
retained=pin.read_file(old.CASE/name,1024*1024)
|
|
require(sha(retained[0])==expected['sha256'] and retained[1]==expected['metadata'],'exact_retained_original_phase_record')
|
|
original[name]=retained
|
|
original_fd=pin.directory(old.CASE);phase=-1;broker=None
|
|
try:
|
|
for name in ORIGINAL_MISSING:pin.absence(original_fd,name)
|
|
pin.absence(original_fd,PHASE)
|
|
failure=pin.parsed(original['failure.json'][0])
|
|
require(failure['fixed_tag']=='last_original_runtime_and_migrations_cas'
|
|
and failure['read_original_case_only'] is True,'proven_original_failure_before_backend_effects')
|
|
before_record=pin.parsed(original['before.json'][0]);baseline=before_record['witness']
|
|
require(before_record['controller_sha256']==OLD_SHA and before_record['case']==old.CASE_ID
|
|
and before_record['image_id']==old.IMAGE and before_record['stage']==old.STAGE_ID,'same_original_fixed_gateway_plan')
|
|
reservation=pin.read_file(old.STAGE/'time-gateway-deploy-intent.json',65536)
|
|
require(reservation[0]==original['reservation.json'][0],'same_original_stage_reservation_no_generic_retry')
|
|
installed_overlay=pin.read_file(old.ROOT/'deploy/server.time-broker.compose.yaml',4096,0o644)
|
|
installed_tool=pin.read_file(old.ROOT/'deploy/cloud_server_upgrade.py',131072,0o644)
|
|
require(sha(installed_overlay[0])==old.OVERLAY_SHA and installed_overlay[1]==pin.parsed(original['overlay-installed.json'][0])['metadata']
|
|
and sha(installed_tool[0])==old.UPGRADE_SHA and installed_tool[1]==pin.parsed(original['upgrade-tool-installed.json'][0])['metadata'],
|
|
'only_verify_original_installed_effects_exact_bytes_and_inodes')
|
|
source=pin.read_file(old.STAGE/'source.json',1024*1024);built=pin.read_file(old.STAGE/'build-result.json',65536)
|
|
compose=pin.read_file(old.ROOT/'deploy/server.compose.yaml',8192,0o644)
|
|
require(sha(source[0])==old.SOURCE_RECORD_SHA and sha(built[0])==old.BUILD_RESULT_SHA
|
|
and sha(compose[0])==old.COMPOSE_SHA,'unchanged_original_qualified_source_build_compose')
|
|
source_value=pin.parsed(source[0])
|
|
require(all(sha(pin.read_file(old.ROOT/name,2*1024*1024,0o644)[0])==digest for name,digest in source_value['files'].items())
|
|
and len(source_value['files'])==209,'same_original_published_source_files')
|
|
signature=pin.read_file(old.BROKER_START.parent/'broker-signature-acceptance-v1/result.json',65536)
|
|
require(sha(signature[0])==old.BROKER_SIGNATURE_RESULT_SHA,'same_original_actual_positive_signature_proof')
|
|
public=pin.read_file(Path('/etc/rmm-time-source/time-public-key.json'),8192)
|
|
require(sha(public[0])==old.TIME_PUBLIC_PIN_SHA,'same_actual_public_time_pin')
|
|
broker=old.BrokerRuntime(pin,operator)
|
|
cmd=['docker','compose','--env-file',str(old.ROOT/'devstate/.env'),'-f',str(old.ROOT/'deploy/server.compose.yaml'),
|
|
'-f',str(old.ROOT/'deploy/server.time-broker.compose.yaml')]
|
|
expected=canonical_containers(baseline['containers'])
|
|
def common():
|
|
pin_guard();operator_guard();broker.verify()
|
|
require(pin.read_file(OLD_SOURCE,131072)==(raw,old_meta) and pin.read_file(Path(__file__),131072)==own,'retained_reviewed_controller_sources')
|
|
for name,retained in original.items():require(pin.read_file(old.CASE/name,1024*1024)==retained,'original_failed_phase_records_unchanged')
|
|
for name in ORIGINAL_MISSING:pin.absence(original_fd,name)
|
|
require(pin.read_file(old.STAGE/'time-gateway-deploy-intent.json',65536)==reservation
|
|
and pin.read_file(old.STAGE/'source.json',1024*1024)==source and pin.read_file(old.STAGE/'build-result.json',65536)==built
|
|
and pin.read_file(old.ROOT/'deploy/server.compose.yaml',8192,0o644)==compose,'same_original_reservation_build_and_compose')
|
|
require(pin.read_file(old.ROOT/'deploy/server.time-broker.compose.yaml',4096,0o644)==installed_overlay
|
|
and pin.read_file(old.ROOT/'deploy/cloud_server_upgrade.py',131072,0o644)==installed_tool,'installed_overlay_tool_never_repeated_or_replaced')
|
|
require(old.file_tree(old.ROOT/'web/dist',True)==baseline['web']
|
|
and {name:old.file_tree(Path(name)) for name in baseline['key_metadata']}==baseline['key_metadata'],
|
|
'same_original_ui_pki_key_policy_metadata')
|
|
require(pin.read_file(old.BROKER_START.parent/'broker-signature-acceptance-v1/result.json',65536)==signature
|
|
and pin.read_file(Path('/etc/rmm-time-source/time-public-key.json'),8192)==public,'original_signature_and_time_key_unchanged')
|
|
# ecb.preflight deliberately requires old Runtime6f. Reuse its
|
|
# complete independent Chrony getter without changing that guard.
|
|
current=operator.chrony_observation();previous=baseline['actual']
|
|
boot=operator.regular(Path('/proc/sys/kernel/random/boot_id'),128).decode().strip()
|
|
builder=json.loads(operator.run(['docker','image','inspect','--format',
|
|
'[{{json .Id}},{{json .Architecture}},{{json .Os}}]',operator.BUILDER]))
|
|
require(boot==previous['boot_id'] and dict(zip(('Id','Architecture','Os'),builder,strict=True))==previous['builder']
|
|
and current['pin']==previous['chrony_observation']
|
|
and current['fixed_images']['daemon']==previous['chrony_image_metadata'],
|
|
'same_original_actual_boot_chrony_builder_image')
|
|
require(stable_chrony(current)==stable_chrony(previous['chrony_binding']),
|
|
'same_original_actual_chrony_process_cgroup_socket')
|
|
return current
|
|
common()
|
|
require(canonical_containers(old.containers(operator))==expected
|
|
and old.migrations(operator,cmd)==baseline['migrations']==list(range(1,15)),
|
|
'only_nondeterministic_mount_order_changed_before_backend_continuation')
|
|
require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.OLD_IMAGE,
|
|
'retained_original_runtime_tag_cas')
|
|
require(json.loads(operator.run(['docker','image','inspect',old.IMAGE,'--format','[{{json .Id}},{{json .Architecture}},{{json .Os}}]']))
|
|
==[old.IMAGE,'arm64','linux'],'same_existing_qualified_native_image')
|
|
binding=dict(profile=PROFILE,case=old.CASE_ID,phase=PHASE,controller_sha256=sha(own[0]),original_controller_sha256=OLD_SHA,
|
|
original_before_sha256=ORIGINAL_RECORDS['before.json']['sha256'],original_failure_sha256=ORIGINAL_RECORDS['failure.json']['sha256'],
|
|
mount_order_diagnostic_receipt_sha256=DIAGNOSTIC_RECEIPT_SHA,image_id=old.IMAGE,stage=old.STAGE_ID,
|
|
installed_overlay_sha256=old.OVERLAY_SHA,installed_tool_sha256=old.UPGRADE_SHA)
|
|
if args.command=='host-preflight':
|
|
print(pin.encoded(dict(binding,read_only=True,backend_unentered=True,canonical_containers=expected,migrations=baseline['migrations'])).decode(),end='');return
|
|
os.mkdir(PHASE,0o700,dir_fd=original_fd);os.fsync(original_fd);phase=pin.directory(here)
|
|
save=lambda name,value:old.create(pin,phase,name,value)
|
|
save('intent.json',dict(binding,once=True,no_repeat_installed_effects=True,no_retry=True))
|
|
save('before.json',dict(binding,containers=expected,migrations=baseline['migrations'],broker=broker.verify()))
|
|
def effect(name,argv,timeout):
|
|
common();save(name+'-intent.json',dict(binding,argv=argv,once=True))
|
|
code=operator.long_command(argv,here/(name+'.log'),timeout)
|
|
save(name+'-exit.json',dict(binding,exit_code=code));require(code==0,'backend_effect_failed_original_continuation_read_only')
|
|
require(canonical_containers(old.containers(operator))==expected and old.migrations(operator,cmd)==list(range(1,15)),
|
|
'exact_original_runtime_and_schema_before_first_backend_effect')
|
|
require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.OLD_IMAGE,'last_runtime_tag_cas')
|
|
effect('tag',['docker','tag',old.IMAGE,'rust-discovery-server:local'],10)
|
|
require(operator.run(['docker','image','inspect','rust-discovery-server:local','--format','{{.Id}}']).decode().strip()==old.IMAGE,'qualified_runtime_tag_after_once')
|
|
effect('migrate',cmd+['run','--rm','--no-deps','migrate'],25)
|
|
require(canonical_containers(old.containers(operator))==expected and old.migrations(operator,cmd)==list(range(1,16)),
|
|
'only_migration_fifteen_before_recreate')
|
|
effect('recreate',cmd+['up','-d','--no-deps','--force-recreate','--no-build','api','gateway'],25)
|
|
after=canonical_containers(old.containers(operator))
|
|
require(after['postgres']==expected['postgres'],'original_postgres_container_volume_generation_unchanged')
|
|
for role in ('api','gateway'):
|
|
previous,current=expected[role],after[role]
|
|
require(current[0]!=previous[0] and current[1]==old.IMAGE and current[2:4]==previous[2:4]
|
|
and current[5:9]==previous[5:9],'only_qualified_role_image_generation_changed')
|
|
mounts=current[4]
|
|
if role=='gateway':
|
|
added=[mount for mount in mounts if mount.get('Destination')=='/run/rmm-time-source']
|
|
require(len(added)==1 and added[0]['Source']=='/run/rmm-time-source' and added[0]['Type']=='bind'
|
|
and added[0]['RW'] is False,'one_actual_readonly_gateway_time_mount')
|
|
mounts=[mount for mount in mounts if mount.get('Destination')!='/run/rmm-time-source']
|
|
require(mounts==previous[4],'all_original_mount_semantics_retained_no_api_time_mount')
|
|
require(old.migrations(operator,cmd)==list(range(1,16)),'actual_additive_schema_through_fifteen')
|
|
actual_chrony=common()
|
|
save('after.json',dict(binding,containers=after,migrations=list(range(1,16)),broker=broker.verify(),chrony=actual_chrony))
|
|
save('result.json',dict(binding,deployed=True,installed_effects_repeated=False,source_rebuilt=False,ui_rewritten=False,
|
|
existing_pki_keys_policy_pg_container_preserved=True,broker_chrony_not_restarted=True,
|
|
gateway_only_readonly_broker_mount=True,
|
|
rmm_jobs_created=False,approvals_created=False,immutable_corehistory_witness='external_root_before_after_required'))
|
|
print(pin.encoded(dict(binding,deployed=True)).decode(),end='')
|
|
except Exception as error:
|
|
if phase>=0:
|
|
tag=str(error) if isinstance(error,GuardDenied) and 0<len(str(error))<=128 and all(c in 'abcdefghijklmnopqrstuvwxyz0123456789_' for c in str(error)) else 'unexpected_gateway_continuation_exception'
|
|
old.create(pin,phase,'failure.json',dict(profile=PROFILE,failed=True,fixed_tag=tag,read_original_continuation_only=True))
|
|
raise
|
|
finally:
|
|
if broker is not None:broker.close()
|
|
for fd in (phase,original_fd):
|
|
if fd>=0:os.close(fd)
|
|
|
|
|
|
if __name__=='__main__':
|
|
try:main()
|
|
except Exception as error:
|
|
tag=str(error) if isinstance(error,GuardDenied) and 0<len(str(error))<=128 and all(c in 'abcdefghijklmnopqrstuvwxyz0123456789_' for c in str(error)) else 'unexpected_gateway_continuation_exception'
|
|
print(json.dumps(dict(profile=PROFILE,failed=True,fixed_tag=tag,read_original_continuation_only=True)))
|
|
raise SystemExit(1)
|
|
|