Complete bounded Linux MVP M0-M5 with verified cloud closeout and handover
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only closeout of the existing finite Linux dev/test acceptance."""
|
||||
import concurrent.futures
|
||||
import argparse
|
||||
import hashlib
|
||||
|
||||
import cloud_reboot_acceptance as reboot
|
||||
import cloud_mvp_final_readback as metadata
|
||||
|
||||
cloud = reboot.cloud
|
||||
require = cloud.contract.require
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--verify-existing-observations', action='store_true')
|
||||
args = parser.parse_args()
|
||||
folder = reboot.STATE/'d015c3ef-f3aa-4522-a3c2-fcb6099df17b'
|
||||
read = cloud.contract.protected_json
|
||||
session = read(folder/'before.json'); checkpoint = read(folder/'checkpoint.json')
|
||||
result = read(folder/'result.json'); closed = read(folder/'closed.json')
|
||||
startup = read(folder/'startup-retry-result.json'); intent = read(folder/'startup-retry-intent.json')
|
||||
apt = read(folder/'post-reboot-apt-recovery.json')
|
||||
mcp = read(cloud.ROOT/'devstate/cloud-mcp-reference-final.json')
|
||||
require(result['verified'] and closed['verified'] and startup['completed']
|
||||
and startup['automatic_recovery'] and startup['state_preserved']
|
||||
and apt['completed'] and apt['apt_runtime_ready'] and apt['preserved_state']
|
||||
and apt['readiness_only'] and not apt['native_acceptance'] and not apt['package_effects']
|
||||
and mcp['verified'] and mcp['no_new_plans_jobs_or_approvals']
|
||||
and len(mcp['operations']) == 7 and all(o['operation']['counts'] == {'succeeded': 1} for o in mcp['operations']),
|
||||
'all existing native acceptance receipts required')
|
||||
pins = reboot.effective_pins(folder, session, checkpoint)
|
||||
unit_hash = hashlib.sha256((cloud.ROOT/'deploy/linux/rmm-helper-cloud.service').read_bytes()).hexdigest()
|
||||
require(startup['old_unit_sha256'] == pins['general_unit_sha256']
|
||||
and startup['new_unit_sha256'] == unit_hash, 'independently qualified startup-only unit delta')
|
||||
pins = dict(pins, general_unit_sha256=unit_hash)
|
||||
target = session['target']
|
||||
calls = {
|
||||
'vm': ('linux-vm.conf', metadata.vm, {'startup_retry_id': intent['id']}),
|
||||
'server': ('server.conf', metadata.server, {'candidate_id': '7be1f330-4bbd-4801-b72a-c35ce7ac479e'}),
|
||||
'native': ('linux-vm.conf', reboot.vm_state, dict(pins, **{k:checkpoint['job'][k] for k in ('job_id','attempt_id')})),
|
||||
'central_counts': ('server.conf', cloud.server, dict(kind='gate', tenant_id=target['tenant_id'], agent_id=target['agent_id'])),
|
||||
'policy': ('server.conf', reboot.server_policy, {'action':'state'}),
|
||||
}
|
||||
output = cloud.ROOT/'devstate/cloud-mvp-final-readback.json'
|
||||
if args.verify_existing_observations:
|
||||
capture = read(output)
|
||||
require(capture['target'] == target and capture['verified'] is False,
|
||||
'same preserved native observations before report finalization')
|
||||
observed = {name:capture[name] for name in calls}
|
||||
observed_at = capture['observed_at']
|
||||
else:
|
||||
require(not output.exists(), 'preserve earlier final observations')
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
|
||||
pending = {name:pool.submit(cloud.bootstrap.rpc, *call) for name,call in calls.items()}
|
||||
observed = {name:future.result() for name,future in pending.items()}
|
||||
observed_at = cloud.contract.now().isoformat()
|
||||
cloud.contract.write_private(output, dict(verified=False, observed_at=observed_at, target=target, **observed))
|
||||
vm = observed['vm']; native = observed['native']; server = observed['server']
|
||||
require(observed['central_counts'] == dict(jobs=30,outbox=30,permits=26,unresolved=0)
|
||||
and observed['policy']['closed'], 'exact unchanged counts and closed central Root scope')
|
||||
require(native['boot_id'] == result['native']['boot_id']
|
||||
and native['root_unresolved'] == native['other_unresolved'] == 0
|
||||
and native['root_rows'] == result['native']['root_rows']
|
||||
and native['database_inodes'] == result['native']['database_inodes']
|
||||
and native['signed_sync']['ready'] and native['signed_sync']['helper_ready']
|
||||
and native['agent_last_decision'] == native['signed_sync']['decision_id']
|
||||
and native['signed_sync']['selected'][0]['outcome_state'] == 'succeeded', 'all 18 original Root rows, inodes and normal ready signed Sync')
|
||||
for name in ('rmm-agent-remote.service','rmm-helper-cloud.service'):
|
||||
require(vm['units'][name]['show_exit_code'] == 0 and vm['units'][name]['properties']['ActiveState'] == 'active'
|
||||
and vm['units'][name]['properties']['UnitFileState'] == 'enabled', 'current own services active and enabled')
|
||||
for name in ('rmm-apt-helper.service','rmm-agent.service','rmm-helper-dev-test.service'):
|
||||
require(vm['units'][name]['properties']['ActiveState'] == 'inactive'
|
||||
and vm['units'][name]['properties']['UnitFileState'] == 'disabled', 'APT and historical workers inactive and disabled')
|
||||
require(vm['agent_process']['uids'] == [993]*4 and vm['agent_process']['gids'] == [984]*4
|
||||
and vm['public_key_cache_match'] and vm['general_allow_reboot'] is False
|
||||
and vm['general_config_sha256'] == closed['helper']['config_sha256']
|
||||
and vm['general_pin_sha256'] == closed['helper']['pin_sha256']
|
||||
and vm['general_unit_sha256'] == unit_hash and not vm['startup_probe_override_exists']
|
||||
and not vm['apt_endpoint_present']
|
||||
and vm['rollback_timer']['properties']['ActiveState'] == 'inactive'
|
||||
and vm['units']['rmm-helper-cloud.service']['properties']['Restart'] == 'on-failure'
|
||||
and vm['units']['rmm-helper-cloud.service']['properties']['RestartUSec'] == '5s', 'actual unprivileged Agent and restored closed policy; no temporary startup fault')
|
||||
require(vm['artifact_sha256'] == dict(agent=pins['agent_sha256'],helper=pins['helper_sha256'],provider=apt['provider_sha256']), 'actual qualified artifacts')
|
||||
image = server['candidate_built_image_id']
|
||||
require(server['candidate_build_exit_code'] == 0 and image == server['candidate_tag_image_id'] == server['runtime_tag_image_id'], 'qualified native server image remains deployed')
|
||||
for name, role in server['roles'].items():
|
||||
require(role['State']['Running'] and not role['State']['Restarting'] and not role['State']['OOMKilled']
|
||||
and role['State']['Status'] == 'running', 'actual running server roles')
|
||||
if name == 'postgres':
|
||||
require(not role['PortBindings'] and role['HealthStatus'] == 'healthy', 'database healthy with no published port')
|
||||
continue
|
||||
require(role['Image'] == image and role['User'] == '10001:10001', 'qualified unprivileged server role')
|
||||
bindings = role['PortBindings']
|
||||
require(set(bindings) == {'18443/tcp'} and len(bindings['18443/tcp']) == 1
|
||||
and bindings['18443/tcp'][0] == dict(HostIp='127.0.0.1',HostPort='18443' if name == 'api' else '18444'), 'administrative and gateway Dockerports strictly loopback')
|
||||
flags = role['flags']
|
||||
require(flags['RMM_PROFILE'] == 'development' and flags['RMM_DEV_AUTH'] == 'true'
|
||||
and all(flags[k] in (None,'false','0') for k in ('RMM_DEV_DELIVERY_FAULTS','RMM_DEV_TRANSPORT_FAULTS'))
|
||||
and all(flags[k] is None for k in ('RMM_DEV_DELIVERY_FAULT_FILE','RMM_DEV_TRANSPORT_FAULT_FILE')), 'explicit dev/test profile with delivery and transport fault hooks disabled')
|
||||
completion = cloud.ROOT/'devstate/cloud-mvp-final-closeout.json'
|
||||
cloud.contract.write_private(completion, dict(verified=True, observed_at=observed_at,
|
||||
verified_at=cloud.contract.now().isoformat(), target=target,
|
||||
observation=dict(path=str(output.relative_to(cloud.ROOT)),
|
||||
sha256=hashlib.sha256(output.read_bytes()).hexdigest()), **observed))
|
||||
print('PASS final Linux dev/test readback: 30/30/26/0, Root18 closed, qualified services/images/pins, loopback administration, all temporary fault scopes closed')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
message = str(error)
|
||||
print(message if message.startswith('remote bootstrap incomplete: ') else 'MVP_CLOSEOUT_INCOMPLETE:' + (message if isinstance(error, cloud.contract.GuardError) else type(error).__name__))
|
||||
raise SystemExit(2) from None
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Fixed read-only final metadata; callers assert the actual returned state."""
|
||||
|
||||
|
||||
def vm(payload):
|
||||
import stat
|
||||
retry = str(uuid.UUID(payload['startup_retry_id']))
|
||||
def owned(path,owner,limit):
|
||||
require(not path.is_symlink(), 'fixed metadata file symlink')
|
||||
s = path.stat()
|
||||
require(stat.S_ISREG(s.st_mode) and s.st_uid == owner and not s.st_mode&0o022
|
||||
and s.st_size <= limit, 'owned bounded metadata file')
|
||||
return path.read_bytes()
|
||||
def service(name):
|
||||
answer = subprocess.run(['systemctl','show',name,'-p','ActiveState','-p','SubState',
|
||||
'-p','UnitFileState','-p','MainPID','-p','Restart','-p','RestartUSec','-p','NRestarts'],
|
||||
capture_output=True,timeout=10)
|
||||
return dict(properties=dict(line.split('=',1) for line in answer.stdout.decode().splitlines()),
|
||||
show_exit_code=answer.returncode)
|
||||
names = ('rmm-agent-remote.service','rmm-helper-cloud.service','rmm-apt-helper.service',
|
||||
'rmm-agent.service','rmm-helper-dev-test.service')
|
||||
units = {name:service(name) for name in names}
|
||||
account = pwd.getpwnam('rmm-agent'); state = Path('/var/lib/rmm-agent-remote')
|
||||
config_raw = owned(Path('/etc/rmm-remote/helper-general.json'),0,16384)
|
||||
pin_raw = owned(Path('/etc/rmm-remote/general-helper-proof.json'),0,16384)
|
||||
general = json.loads(config_raw); pin = json.loads(pin_raw)
|
||||
cached = json.loads(owned(state/'job-signing-key.json',account.pw_uid,4096))
|
||||
helper_key = json.loads(owned(Path('/etc/rmm-remote/helper-signing-key.json'),0,4096))
|
||||
binding = json.loads(owned(state/'binding.json',account.pw_uid,16384))
|
||||
try: raw_key = bytes.fromhex(cached.get('public_key_hex',''))
|
||||
except (TypeError,ValueError): raw_key = b''
|
||||
match = (set(cached) == {'algorithm','schema_version','public_key_hex','key_id'}
|
||||
and cached == helper_key and cached.get('algorithm') == 'Ed25519'
|
||||
and type(cached.get('schema_version')) is int and cached['schema_version'] == 1
|
||||
and len(raw_key) == 32 and cached.get('key_id') == hashlib.sha256(raw_key).hexdigest()
|
||||
and binding.get('public_key_hex') == cached.get('public_key_hex')
|
||||
and general.get('signing_key_file') == '/etc/rmm-remote/helper-signing-key.json'
|
||||
and general.get('binding',{}).get('public_key_hex') in ('',cached.get('public_key_hex'))
|
||||
and pin.get('identity',{}).get('signing_key_id') == cached.get('key_id')
|
||||
and all(binding.get(k) == general.get('binding',{}).get(k) == pin.get('identity',{}).get(k)
|
||||
and binding.get(k) is not None for k in
|
||||
('tenant_id','asset_id','agent_id','enrollment_generation','assignment_version')))
|
||||
pid = int(units['rmm-agent-remote.service']['properties'].get('MainPID','0'))
|
||||
process = dict(pid=pid,uids=None,gids=None)
|
||||
if pid > 0:
|
||||
try:
|
||||
lines = Path('/proc/'+str(pid)+'/status').read_text().splitlines()
|
||||
for label,key in (('Uid:','uids'),('Gid:','gids')):
|
||||
selected = [line for line in lines if line.startswith(label)]
|
||||
process[key] = [int(v) for v in selected[0].split()[1:]] if len(selected) == 1 else None
|
||||
except FileNotFoundError: process['exited_during_read'] = True
|
||||
probe = Path('/etc/systemd/system/rmm-helper-cloud.service.d/96-cloud-startup-retry-probe.conf')
|
||||
endpoint = owned(Path('/etc/systemd/system/rmm-agent-remote.service.d/30-cloud-root.conf'),0,4096)
|
||||
files = {'agent':Path('/usr/local/libexec/rmm-agent-remote'),
|
||||
'helper':Path('/usr/local/libexec/rmm-privileged-helper'),
|
||||
'provider':Path('/usr/lib/rmm-apt-transaction/apt_transaction.py')}
|
||||
return dict(units=units,agent_process=process,
|
||||
general_unit_sha256=hashlib.sha256(owned(Path('/etc/systemd/system/rmm-helper-cloud.service'),0,16384)).hexdigest(),
|
||||
general_allow_reboot=general.get('allow_reboot'),
|
||||
general_config_sha256=hashlib.sha256(config_raw).hexdigest(),
|
||||
general_pin_sha256=hashlib.sha256(pin_raw).hexdigest(),public_key_cache_match=match,
|
||||
startup_probe_override_exists=probe.exists() or probe.is_symlink(),
|
||||
rollback_timer=service('rmm-general-startup-rollback-'+retry+'.timer'),
|
||||
artifact_sha256={name:hashlib.sha256(owned(path,0,32*1024*1024)).hexdigest() for name,path in files.items()},
|
||||
apt_endpoint_present=b'RMM_APT_HELPER_' in endpoint)
|
||||
|
||||
|
||||
def server(payload):
|
||||
import stat
|
||||
candidate = str(uuid.UUID(payload['candidate_id']))
|
||||
root = Path('/opt/rust-discovery'); folder = root/'devstate'/('source-upgrade-'+candidate)
|
||||
def metadata(path):
|
||||
require(not path.is_symlink(), 'candidate metadata symlink')
|
||||
s = path.stat()
|
||||
require(stat.S_ISREG(s.st_mode) and s.st_uid == 0 and s.st_size <= 1024*1024
|
||||
and not s.st_mode&0o077, 'private bounded candidate metadata')
|
||||
return json.loads(path.read_bytes())
|
||||
record = metadata(folder/'source.json'); built = metadata(folder/'build-result.json')
|
||||
require(record['id'] == candidate and record['image_tag'] == 'rust-discovery-server:upgrade-'+candidate,
|
||||
'fixed qualified candidate image tag')
|
||||
def run(args):return subprocess.run(args,capture_output=True,check=True,timeout=10).stdout
|
||||
def image(tag):return run(['docker','image','inspect',tag,'--format','{{.Id}}']).decode().strip()
|
||||
compose = ['docker','compose','--env-file',str(root/'devstate/.env'),'-f',str(root/'deploy/server.compose.yaml')]
|
||||
# Select fields inside Docker's formatter: no whole inspect/Config.Env and
|
||||
# no State.Health log output is ever read into the result.
|
||||
state_fields = ('Status','Running','Paused','Restarting','OOMKilled','Dead','Pid','ExitCode','StartedAt','FinishedAt')
|
||||
state = '{'+','.join('"'+key+'":{{json .State.'+key+'}}' for key in state_fields)+'}'
|
||||
template = ('{"State":'+state+',"Image":{{json .Image}},"User":{{json .Config.User}},'
|
||||
'"PortBindings":{{json .HostConfig.PortBindings}},'
|
||||
'"HealthStatus":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}')
|
||||
flags = ('RMM_PROFILE','RMM_DEV_AUTH','RMM_DEV_DELIVERY_FAULTS','RMM_DEV_TRANSPORT_FAULTS',
|
||||
'RMM_DEV_DELIVERY_FAULT_FILE','RMM_DEV_TRANSPORT_FAULT_FILE')
|
||||
roles = {}
|
||||
for role in ('api','gateway','postgres'):
|
||||
ids = run(compose+['ps','--all','-q',role]).decode().split()
|
||||
require(len(ids) <= 1 and all(len(value) == 64 and all(c in '0123456789abcdef' for c in value) for value in ids), 'one bounded exact role container')
|
||||
if not ids: roles[role] = dict(container_id=None); continue
|
||||
cid = ids[0]; value = json.loads(run(['docker','inspect','--format',template,cid]))
|
||||
value['container_id'] = cid
|
||||
if role in ('api','gateway'):
|
||||
selected = {}
|
||||
for key in flags:
|
||||
answer = subprocess.run(['docker','exec',cid,'printenv',key],capture_output=True,timeout=10)
|
||||
require(answer.returncode in (0,1) and len(answer.stdout) <= 1024, 'explicit bounded nonsecret flag read')
|
||||
selected[key] = answer.stdout.decode().strip() if answer.returncode == 0 else None
|
||||
value['flags'] = selected
|
||||
roles[role] = value
|
||||
return dict(candidate_id=candidate,candidate_build_exit_code=built['exit_code'],
|
||||
candidate_built_image_id=built.get('image_id'),candidate_tag_image_id=image(record['image_tag']),
|
||||
runtime_tag_image_id=image('rust-discovery-server:local'),roles=roles)
|
||||
Reference in New Issue
Block a user