Files
rust-discovery/deploy/windows/runtime_feature_upgrade.py
2026-11-14 03:53:48 +01:00

258 lines
21 KiB
Python

#!/usr/bin/env python3
"""New passive IPC-client runtime qualification/upgrade; historical V1 stays intact.
No pin/helper service is installed here. Native unit tests, actual runtime change
and later real LocalService-to-SYSTEM IPC are distinct acceptance steps.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import uuid
import runtime_upgrade as previous
import client_bootstrap as cb
import transport_acceptance as base
import auto_transport_acceptance as auto
require=cb.contract.require
write=previous.write
digest=previous.digest
STATE=cb.EVIDENCE/'runtime-feature-upgrade'
CURRENT=cb.EVIDENCE/'current-runtime.json'
TARGET=previous.TARGET
IMAGE=previous.IMAGE
OLD='f64e81155a4d86089a9dd7e6ec42ffba50b939442386c3187a28769a4de1aa0b'
ORIGINAL='31e360e50b177b4d5729d24e436d4f25aeed7b1a7bd35f2a129841ba1abffd37'
def replace_once(source, old, new):
require(source.count(old)==1,'exact historical source transform, no fuzzy replacement')
return source.replace(old,new,1)
def upgrade_source():
source=replace_once(previous.UPGRADE,"$p.old_sha256 -cne '"+ORIGINAL+"'","$p.old_sha256 -cne '"+OLD+"'")
source=replace_once(source,r"$folder='C:\ProgramData\RustDiscoveryBootstrap\runtime-'+$p.id",r"$folder='C:\ProgramData\RustDiscoveryBootstrap\runtime-feature-'+$p.id")
source=replace_once(source,r"$active='C:\ProgramData\RustDiscoveryBootstrap\runtime-active-intent.json'",r"$active='C:\ProgramData\RustDiscoveryBootstrap\runtime-feature-active-intent.json'")
source=replace_once(source,"function Snapshot([string]$expected) {",r'''function Snapshot([string]$expected) {
if ((Test-Path -LiteralPath 'C:\ProgramData\RustDiscoverySystem') -or (Test-Path -LiteralPath 'C:\Program Files\RustDiscovery\rmm-helper-windows.exe') -or $null -ne (Get-Service -Name 'RmmDiscoverySystemHelper' -ErrorAction SilentlyContinue)) {throw 'Complete passive feature upgrade before any helper/pin installation'}''')
return source
UPGRADE=upgrade_source()
PREPARE=base.PREPARE_SCRIPT.replace('TransportAcceptance.ps1','RuntimeFeatureUpgrade.ps1')
LAUNCH=base.LAUNCH_SCRIPT.replace('TransportAcceptance.ps1','RuntimeFeatureUpgrade.ps1').replace('RmmTransportPayload','RmmRuntimePayload')
def source_files():
files=previous.source_files()
# Clippy --all-targets also compiles native_acl and EVERY example. Pin
# those, all nested modules and each local crate's optional build script.
for crate in ('agent-windows','agent-core','domain','protocol'):
root=cb.ROOT/'crates'/crate
for path in root.rglob('*'):
if path.is_file() and path.suffix in ('.rs','.c','.h','.idl','.toml'):
relative=str(path.relative_to(cb.ROOT));files[relative]=digest(path)
for path in ('deploy/windows/runtime_upgrade.py','deploy/windows/runtime_feature_upgrade.py'):
files[path]=digest(cb.ROOT/path)
return dict(sorted(files.items()))
def stable(manifest,current_sources=True):
require(manifest['schema_version']==1 and manifest['architecture']==TARGET and manifest['default_features'] is True and manifest['system_helper_feature'] is True and manifest['system_helper_client_feature'] is True,'explicit optional IPC-client feature profile')
if current_sources:
require(source_files()==manifest['source_files'],'feature build sources unchanged')
require(re.search(r'^pub const ADAPTER_QUALIFIED: bool = false;$',(cb.ROOT/'crates/agent-windows/src/clock.rs').read_text(),re.M) is not None,'production clock gate must remain closed')
require(manifest['checks']['production_clock_qualified'] is False,'compiled production clock stayed unqualified')
for item in manifest['artifacts'].values():require(digest(item['path'])==item['sha256'],'actual retained linked artifact hash')
def build():
os.umask(0o077);STATE.mkdir(mode=0o700,parents=True,exist_ok=True)
folder=STATE/('build-'+str(uuid.uuid4()));folder.mkdir(mode=0o700)
before=source_files();write(folder,'source-before.json',before)
require(re.search(r'^pub const ADAPTER_QUALIFIED: bool = false;$',(cb.ROOT/'crates/agent-windows/src/clock.rs').read_text(),re.M) is not None,'production clock gate must remain closed')
target=cb.ROOT/'devstate/windows-runtime-feature/target';target.mkdir(mode=0o700,parents=True,exist_ok=True)
script='cargo clippy --locked --offline --target '+TARGET+' -p rmm-agent-windows --features system-helper --all-targets -- -D warnings && cargo build --locked --offline --target '+TARGET+' -p rmm-agent-windows --features system-helper --bin rmm-agent-windows --example runtime_state --release && cargo test --locked --offline --target '+TARGET+' -p rmm-agent-windows --features system-helper --lib --bin rmm-agent-windows --no-run --message-format=json'
raw=previous.command(['docker','run','--rm','--network','none','--read-only','--tmpfs','/tmp:rw,size=512m','-v',str(cb.ROOT)+':/src:ro','-v',str(target)+':/target','-e','CARGO_TARGET_DIR=/target','--entrypoint','/bin/sh',IMAGE,'-c',script])
require(source_files()==before,'no source drift during feature build')
units={}
for line in raw.decode().splitlines():
try:item=json.loads(line)
except ValueError:continue
if item.get('reason')=='compiler-artifact' and 'rmm-agent-windows' in item.get('package_id',''):
require('system-helper' in item.get('features',[]),'actual compiled feature profile')
if item.get('profile',{}).get('test') and item.get('executable'):
kinds=item['target']['kind'];key='lib' if 'lib' in kinds else 'bin' if 'bin' in kinds else None
if key:units[key]=target/Path(item['executable']).relative_to('/target')
require(set(units)=={'lib','bin'},'new feature lib and binary unit links')
artifacts={}
for key,path,name,release in [('agent',target/TARGET/'release/rmm-agent-windows.exe','rmm-agent-windows.exe',True),('probe',target/TARGET/'release/examples/runtime_state.exe','RuntimeStateProbe.exe',True),('lib',units['lib'],'windows-tests-lib.exe',False),('bin',units['bin'],'windows-tests-bin.exe',False)]:
out=folder/name;shutil.copyfile(path,out);out.chmod(0o600);original=digest(path)
if key in ('lib','bin'):
previous.command(['docker','run','--rm','--network','none','--read-only','-v',str(folder.resolve())+':/artifacts','--entrypoint','x86_64-w64-mingw32-strip',IMAGE,'--strip-debug','/artifacts/'+name])
require(digest(path)==original,'retain original unit link')
data=out.read_bytes();offset=int.from_bytes(data[60:64],'little')
require(data[:2]==b'MZ' and data[offset:offset+6]==b'PE\0\0\x64\x86','actual Windows AMD64 artifact')
artifacts[key]=dict(path=str(out.resolve()),name=name,sha256=digest(out),bytes=len(data),release_build=release,original_link_sha256=original)
require(source_files()==before,'source unchanged through artifact preservation')
manifest=dict(schema_version=1,architecture=TARGET,default_features=True,system_helper_feature=True,system_helper_client_feature=True,source_files=before,artifacts=artifacts,checks=dict(cross_clippy_D_warnings=True,release_build=True,windows_unit_link=True,production_clock_qualified=False),native_vm_tested=False)
write(folder,'manifest.json',manifest)
print(json.dumps(dict(manifest=str((folder/'manifest.json').resolve()),artifacts={k:v['sha256'] for k,v in artifacts.items()},native_vm_tested=False)))
def qualify(manifest_path):
manifest=cb.contract.protected_json(manifest_path);stable(manifest)
os.umask(0o077);STATE.mkdir(mode=0o700,parents=True,exist_ok=True)
folder=STATE/('qualification-'+str(uuid.uuid4()));folder.mkdir(mode=0o700)
write(folder,'intent.json',dict(manifest_path=str(manifest_path.resolve()),manifest_sha256=digest(manifest_path),source_files=manifest['source_files'],test_runner_sha256=hashlib.sha256(previous.TESTS.encode()).hexdigest(),system_helper_client_feature=True))
before=auto.scopes();flags=auto.status();write(folder,'server-before.json',dict(counts=before,flags=flags))
reports={}
for key,count in [('lib',21),('bin',6)]:
item=manifest['artifacts'][key];previous.upload(item)
report=cb.powershell(previous.TESTS,dict(name=item['name'],sha256=item['sha256']),timeout=150)
write(folder,key+'-native.json',report)
names=re.findall(r'^(.+): test\r?$',report['listed']['stdout'],re.M)
summary=re.search(r'test result: ok\. (\d+) passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;',report['result']['stdout'])
require(report['listed']['exit_code']==report['result']['exit_code']==0 and summary is not None and len(names)==int(summary.group(1))==count,'all exact 21+6 new native units, none skipped or filtered')
reports[key]=dict(artifact_sha256=item['sha256'],passed=len(names),test_names=names,native_context='bootstrap_administrator_unit_tests')
stable(manifest)
after=auto.scopes();after_flags=auto.status();require(before==after and flags==after_flags and auto.flags_disabled(flags),'native units have no jobs/policy/fault effect')
write(folder,'result.json',dict(verified=True,completed=True,native_unit_tests=reports,manifest_path=str(manifest_path.resolve()),manifest_sha256=digest(manifest_path),source_files=manifest['source_files'],server_counts=after,server_flags=flags,no_jobs=True,clock_adapter_qualified=False,system_helper_client_feature=True,local_service_runtime_test=False,native_pipe_peer_test=False))
print(json.dumps(dict(verified=True,qualification=str((folder/'result.json').resolve()),native_tests={k:v['passed'] for k,v in reports.items()},native_pipe_peer_test=False)))
def qualified(manifest_path,qualification_path,current_sources=True):
manifest=cb.contract.protected_json(manifest_path);stable(manifest,current_sources)
proof=cb.contract.protected_json(qualification_path)
require(proof.get('verified') is True and proof.get('completed') is True and proof.get('system_helper_client_feature') is True and proof['manifest_path']==str(manifest_path.resolve()) and proof['manifest_sha256']==digest(manifest_path) and proof['source_files']==manifest['source_files'] and proof.get('no_jobs') is True and proof.get('clock_adapter_qualified') is False,'new source-bound native feature qualification')
for key,count in [('lib',21),('bin',6)]:
require(proof['native_unit_tests'][key]['passed']==count and proof['native_unit_tests'][key]['artifact_sha256']==manifest['artifacts'][key]['sha256'],'exact new compiled native unit association')
return manifest,proof
def prior_runtime():
index=cb.contract.protected_json(CURRENT)
require(index.get('type')=='rmm.windows.current-runtime' and index.get('binary_sha256')==OLD and index.get('agent_id')==base.AGENT and index.get('asset_id')==base.ASSET and index.get('transport')=='auto' and index.get('windows_jobs_enabled') is False and index.get('clock_adapter_qualified') is False and not index.get('system_helper_client_feature',False),'accepted original f64 passive runtime')
path=Path(index['evidence']['path']).resolve()
require(path.is_relative_to(cb.EVIDENCE.resolve()) and digest(path)==index['evidence']['sha256'],'exact accepted old runtime evidence')
proof=cb.contract.protected_json(path)
require(proof.get('verified') is True and proof.get('completed') is True and proof.get('restored') is False and proof.get('no_jobs') is True and proof['native']['binary_sha256']==OLD,'actual old passive runtime, not a historical transport artifact')
return index,proof
def stage_source():
raw=UPGRADE.encode();sha=hashlib.sha256(raw).hexdigest();value=cb.powershell(PREPARE,dict(sha256=sha))
if not value['already_uploaded']:
folder=STATE/'source';folder.mkdir(mode=0o700,parents=True,exist_ok=True);path=folder/(sha+'.ps1')
if not path.exists():
fd=os.open(path,os.O_CREAT|os.O_EXCL|os.O_WRONLY|os.O_NOFOLLOW,0o600)
with os.fdopen(fd,'wb') as stream:stream.write(raw);stream.flush();os.fsync(stream.fileno())
require(digest(path)==sha,'exact feature source bytes')
prefix=cb.contract.protected_json(cb.ROOT/'credentials/windows-dev-access.json')['ssh_prefix']
args=['sftp']+[part for part in prefix[1:-1] if part!='-T']+['-b','-',prefix[-1]]
target='/C:/ProgramData/RustDiscoveryBootstrap/'+sha+'/RuntimeFeatureUpgrade.ps1'
result=subprocess.run(args,input=('put '+json.dumps(str(path.resolve()))+' '+json.dumps(target)+'\n').encode(),capture_output=True,timeout=60)
require(result.returncode==0 and cb.powershell(PREPARE,dict(sha256=sha))['already_uploaded'],'exact staged feature source')
return sha
def remote(intent,action):
require(intent['source_sha256']==hashlib.sha256(UPGRADE.encode()).hexdigest(),'immutable exact feature source')
return cb.powershell(LAUNCH,dict(sha256=intent['source_sha256'],payload=dict(intent['remote_payload'],action=action)),timeout=100,remote_signed=True)
def upgrade(manifest_path,qualification_path):
manifest,proof=qualified(manifest_path,qualification_path);index,old=prior_runtime()
os.umask(0o077);STATE.mkdir(mode=0o700,parents=True,exist_ok=True);ident=str(uuid.uuid4());folder=STATE/ident;folder.mkdir(mode=0o700)
for key in ('agent','probe'):previous.upload(manifest['artifacts'][key])
source=stage_source()
payload=dict(id=ident,old_sha256=OLD,new_sha256=manifest['artifacts']['agent']['sha256'],probe_sha256=manifest['artifacts']['probe']['sha256'],config_sha256=old['native']['config_sha256'])
intent=dict(id=ident,type='passive_feature_runtime_upgrade',started_at=cb.contract.now().isoformat(),source_sha256=source,manifest_path=str(manifest_path.resolve()),manifest_sha256=digest(manifest_path),qualification_path=str(qualification_path.resolve()),qualification_sha256=digest(qualification_path),remote_payload=payload,previous_runtime_index=index,previous_runtime_index_sha256=digest(CURRENT),no_jobs=True,no_helper_install=True,no_reboot=True)
write(folder,'intent.json',intent)
before=dict(native=remote(intent,'state'),central=base.observe(ident),counts=auto.scopes(),server=auto.status())
require(before['counts']==proof['server_counts'] and before['server']==proof['server_flags'] and auto.flags_disabled(before['server']),'unchanged jobs/policies since native units')
require(before['native']['service']['state']=='Running' and before['native']['journal']['pending_inventory_events']==0,'settled retained actual original runtime')
for field in ('binary_sha256','binary_acl_sha256','config_sha256','config_acl_sha256','public_hashes','identity_acls','private_key_metadata'):
require(before['native'][field]==old['native'][field],'accepted old runtime identity/config/ACL continuity')
require(digest(CURRENT)==intent['previous_runtime_index_sha256'],'current runtime index unchanged before mutation')
write(folder,'before.json',before);write(folder,'enter-intent.json',dict(id=ident,stop_swap_start_once=True,source_sha256=source))
try:write(folder,'entered.json',remote(intent,'enter'))
except Exception as error:
write(folder,'enter-reply-incomplete.json',dict(error_type=type(error).__name__,continuation='finish/readback only; never repeat enter or start'));raise
print('Passive feature runtime transition '+ident,flush=True);finish(ident)
def load_case(ident):
require(str(uuid.UUID(ident))==ident,'canonical existing feature case')
folder=STATE/ident;intent=cb.contract.protected_json(folder/'intent.json')
require(intent['id']==ident and intent['type']=='passive_feature_runtime_upgrade' and digest(intent['manifest_path'])==intent['manifest_sha256'] and digest(intent['qualification_path'])==intent['qualification_sha256'],'same original immutable feature case and qualification')
manifest,_=qualified(Path(intent['manifest_path']),Path(intent['qualification_path']),current_sources=False)
return folder,intent,manifest
def publish(folder,result_name,intent,binary,restored):
result=folder/result_name
value=dict(schema_version=1,type='rmm.windows.current-runtime',case_id=folder.name,agent_id=base.AGENT,asset_id=base.ASSET,binary_sha256=binary,transport='auto',evidence=dict(path=str(result.resolve()),sha256=digest(result)),windows_jobs_enabled=False,clock_adapter_qualified=False,system_helper_client_feature=not restored)
actual=cb.contract.protected_json(CURRENT)
if actual==value:return
require(digest(CURRENT)==intent['previous_runtime_index_sha256'] or actual.get('case_id')==folder.name,'CAS current runtime: never overwrite another transition')
write(folder,'previous-index-'+str(uuid.uuid4())+'.json',actual)
temporary=CURRENT.with_name('.feature-runtime-'+str(uuid.uuid4())+'.json');write(temporary.parent,temporary.name,value);os.replace(temporary,CURRENT)
def finish(ident):
folder,intent,manifest=load_case(ident);restored=(folder/'rollback-intent.json').exists()
before=cb.contract.protected_json(folder/'before.json');observed=remote(intent,'finish');write(folder,'finish-readback-'+str(uuid.uuid4())+'.json',observed)
expected=OLD if restored else manifest['artifacts']['agent']['sha256']
require(observed.get('completed') is True and observed.get('read_only') is True and observed.get('restored')==restored and observed['native']['binary_sha256']==expected,'actual selected runtime after fresh readback, never historical success replay')
started=cb.contract.protected_json(folder/'rollback-intent.json')['started_at'] if restored else intent['started_at']
online=base.wait_mode(ident,'wss',cb.contract.timestamp(started),before['central']['device'])
after=remote(intent,'finish');write(folder,'after-readback-'+str(uuid.uuid4())+'.json',after)
require(after.get('restored')==restored and after['native']['binary_sha256']==expected,'same actual selected runtime after native heartbeat wait')
native=after['native'];require(native['journal']['pending_inventory_events']==0 and native['journal']['sequence']>before['native']['journal']['sequence'],'two actual WSS heartbeats/sequence advance and concrete original telemetry ACK')
device=base.read_device();denied=base.foreign_denied()
require(cb.contract.timestamp(device['observed_at'])>=cb.contract.timestamp(intent['started_at']),'actual fresh stored inventory of retained identity')
counts=auto.scopes();flags=auto.status();require(counts==before['counts'] and flags==before['server'] and auto.flags_disabled(flags),'Windows/Linux job counts and server policies/hooks unchanged')
result_name='rollback-completed.json' if restored else 'result.json';path=folder/result_name
value=dict(verified=True,completed=True,restored=restored,id=ident,native=native,online=online,stored_inventory=device,foreign_denied=denied,manifest_path=intent['manifest_path'],manifest_sha256=intent['manifest_sha256'],qualification_sha256=intent['qualification_sha256'],counts=counts,server=flags,no_jobs=True,no_reboot=True,no_clock_change=True,windows_jobs_enabled=False,clock_adapter_qualified=False,system_helper_client_feature=not restored,native_pipe_peer_test=False,configuration_unchanged=True,private_key_read=False,transport='auto',preferred_wire='wss')
if path.exists():require(cb.contract.protected_json(path).get('verified') is True,'retain original completed result')
else:write(folder,result_name,value)
publish(folder,result_name,intent,expected,restored)
print(json.dumps(dict(verified=True,id=ident,system_helper_client_feature=not restored,native_pipe_peer_test=False,windows_jobs_enabled=False)))
def rollback(ident):
folder,intent,_=load_case(ident);before=cb.contract.protected_json(folder/'before.json')
require(auto.scopes()==before['counts'] and auto.status()==before['server'],'no intervening jobs/policy changes before explicit rollback')
write(folder,'rollback-intent.json',dict(id=ident,old_binary_sha256=OLD,started_at=cb.contract.now().isoformat(),start_once=True,no_jobs=True))
try:value=remote(intent,'rollback')
except Exception as error:
write(folder,'rollback-reply-incomplete.json',dict(error_type=type(error).__name__,readback_required=True,no_repeated_start=True));raise
write(folder,'rollback-result.json',value)
require(value.get('restored') is True and value['native']['binary_sha256']==OLD,'actual original f64 binary restored')
finish(ident)
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('phase',choices=['build-local','qualify-native','upgrade-passive','finish','rollback'])
parser.add_argument('--manifest',type=Path);parser.add_argument('--qualification',type=Path);parser.add_argument('--id')
args=parser.parse_args()
if args.phase=='build-local':require(not(args.manifest or args.qualification or args.id),'fresh immutable feature build');build()
elif args.phase=='qualify-native':require(args.manifest and not(args.qualification or args.id),'one new feature manifest');qualify(args.manifest)
elif args.phase=='upgrade-passive':require(args.manifest and args.qualification and not args.id,'same new native feature qualification');upgrade(args.manifest,args.qualification)
else:require(args.id and not(args.manifest or args.qualification),'same original feature case only');{'finish':finish,'rollback':rollback}[args.phase](args.id)
if __name__=='__main__':
try:main()
except Exception as error:
print('WINDOWS_FEATURE_RUNTIME_INCOMPLETE:'+type(error).__name__)
raise SystemExit(2) from None