Files
rust-discovery/deploy/windows/system_delegation_recovery.py
T
2026-11-14 11:38:42 +01:00

194 lines
17 KiB
Python

#!/usr/bin/env python3
"""One reviewed recovery of the unchanged failed6d2 joint upgrade; no retries.
Uses explicit File.Replace backup paths, preserving the original failed source,
attempt, four backups, journals and intent. Readback never swaps or starts.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import uuid
import client_bootstrap as cb
import system_delegation_upgrade as u
CASE = '6d2df065-0877-41c4-983c-2c9fdb6e5437'
ORIGINAL_SOURCE = 'f8dc1c8fbc064e253b1fbe130fb628afef58cf45c2bafb6837d6716125b42d38'
STATE = cb.EVIDENCE / 'system-delegation-recovery'
require = cb.contract.require
PREFIX = u.SCRIPT.split('\nCheckControl $boot $true\n', 1)[0]
SCRIPT = PREFIX + r'''
$r=$global:RmmDelegationRecoveryPayload
if(!$r-or[guid]::ParseExact($r.recovery_id,'D').ToString()-cne$r.recovery_id-or$r.phase-notin@('prepare','enter','readback')-or$p.id-cne'6d2df065-0877-41c4-983c-2c9fdb6e5437'-or$r.script_sha256-notmatch'^[a-f0-9]{64}$'){throw 'Only the original failed6d recovery'}
CheckControl $boot $true
$ownScript=Join-Path (Join-Path $boot $r.script_sha256) 'SystemDelegationRecovery.ps1'
if($PSCommandPath-cne$ownScript-or(Hash $PSCommandPath)-cne$r.script_sha256){throw 'Exact actually executing recovery source'}
CheckControl $ownScript $false
$intent=ReadIntent
$recovery=Join-Path $boot ('system-delegation-recovery-'+$r.recovery_id)
$recoveryActive=Join-Path $boot 'system-delegation-6d2-recovery-once.json'
function ReadService($role){$name=$agentName;if($role-ceq'helper'){$name=$helperName};$s=Get-CimInstance Win32_Service -Filter ("Name='"+$name+"'") -OperationTimeoutSec 10;if(!$s){throw 'Fixed existing service'};return @{state=$s.State;pid=$s.ProcessId;account=$s.StartName;mode=$s.StartMode;path=$s.PathName;exit_code=$s.ExitCode}}
function MaybeHash($path){if(!(Test-Path -LiteralPath $path)){return $null};return Hash $path}
function RecoveryReadback{
$original=[ordered]@{};foreach($name in @('agent-stop-attempt','agent-stopped','helper-stop-attempt','helper-stopped','frozen','agent-backup','helper-backup','settings-backup','pin-backup','agent-replace-attempt','helper-replace-attempt','settings-replace-attempt','pin-replace-attempt','agent-replaced','helper-replaced','settings-replaced','pin-replaced','swapped','helper-start-attempt','helper-started','agent-start-attempt','agent-started','entered')){$original[$name]=Test-Path -LiteralPath (Join-Path $folder ($name+'.json'))}
$own=[ordered]@{};foreach($name in @('intent','prepared','enter-attempt','agent-replace-attempt','agent-replaced','helper-replace-attempt','helper-replaced','settings-replace-attempt','settings-replaced','pin-replace-attempt','pin-replaced','swapped','helper-start-intent','agent-start-intent','entered')){$own[$name]=Test-Path -LiteralPath (Join-Path $recovery ($name+'.json'))}
return @{read_only=$true;original_id=$p.id;recovery_id=$r.recovery_id;hashes=@{agent=(MaybeHash $agentBinary);helper=(MaybeHash $helperBinary);settings=(MaybeHash $settings);pin=(MaybeHash $pin)};service=(ReadService 'agent');helper_service=(ReadService 'helper');original_phases=$original;recovery_phases=$own;recovery_active_exists=(Test-Path -LiteralPath $recoveryActive);no_repeated_start=$true}
}
if($r.phase-ceq'readback'){RecoveryReadback|ConvertTo-Json -Depth 20 -Compress;exit 0}
Add-Type -TypeDefinition @'
using System;
using System.Runtime.InteropServices;
using Microsoft.Win32.SafeHandles;
public static class RmmDelegationRecoveryFileId {
[StructLayout(LayoutKind.Sequential)] public struct Id {
public ulong VolumeSerialNumber;
[MarshalAs(UnmanagedType.ByValArray, SizeConst=16)] public byte[] FileId;
}
[DllImport("kernel32.dll", SetLastError=true)]
public static extern bool GetFileInformationByHandleEx(SafeFileHandle handle, int kind, out Id info, uint length);
}
'@
function FileIdentity($path){
Check $path $false;$file=[IO.File]::Open($path,[IO.FileMode]::Open,[IO.FileAccess]::Read,[IO.FileShare]::Read)
try{$id=New-Object RmmDelegationRecoveryFileId+Id;$size=[Runtime.InteropServices.Marshal]::SizeOf([type][RmmDelegationRecoveryFileId+Id]);if($size-ne24-or![RmmDelegationRecoveryFileId]::GetFileInformationByHandleEx($file.SafeFileHandle,18,[ref]$id,[uint32]$size)){throw 'Exact native file identity'}
return [ordered]@{volume_serial=$id.VolumeSerialNumber.ToString([Globalization.CultureInfo]::InvariantCulture);file_id=([BitConverter]::ToString($id.FileId)).Replace('-','').ToLowerInvariant();bytes=$file.Length;acl_sha256=(AclHash $path);sha256=(Hash $path)}
}finally{$file.Dispose()}
}
function CheckOriginalStopped{
foreach($name in @('intent','agent-stop-attempt','agent-stopped','helper-stop-attempt','helper-stopped','frozen','agent-backup','helper-backup','settings-backup','pin-backup','agent-replace-attempt')){CheckControl (Join-Path $folder ($name+'.json')) $false}
foreach($name in @('helper-replace-attempt','settings-replace-attempt','pin-replace-attempt','agent-replaced','helper-replaced','settings-replaced','pin-replaced','swapped','helper-start-attempt','helper-started','agent-start-attempt','agent-started','entered')){if(Test-Path -LiteralPath (Join-Path $folder ($name+'.json'))){throw 'Original case progressed; no recovery replay'}}
$frozen=Get-Content -LiteralPath (Join-Path $folder 'frozen.json') -Raw|ConvertFrom-Json
$current=State $false
if($current.service.state-cne'Stopped'-or$current.helper_service.state-cne'Stopped'){throw 'Both own original services must remain stopped'}
Preserved $frozen $current $true
$checks=@(@('agent',$p.old_agent,$p.new_agent),@('helper',$p.old_helper,$p.new_helper),@('settings',$p.old_policy,$p.new_policy),@('pin',$p.old_pin,$p.new_pin))
foreach($item in $checks){$backup=Join-Path $folder ($item[0]+'-original');CheckControl $backup $false;if((Hash $backup)-cne$item[1]){throw 'Original exact backup'};$receipt=Get-Content -LiteralPath (Join-Path $folder ($item[0]+'-backup.json')) -Raw|ConvertFrom-Json;if($receipt.sha256-cne$item[1]){throw 'Original backup receipt'}}
$attempt=Get-Content -LiteralPath (Join-Path $folder 'agent-replace-attempt.json') -Raw|ConvertFrom-Json
if($attempt.old_sha256-cne$p.old_agent-or$attempt.new_sha256-cne$p.new_agent){throw 'Original unchanged failed agent attempt'}
if((Hash (Join-Path $folder 'agent-candidate'))-cne$p.new_agent){throw 'Retain exact original agent candidate'}
foreach($label in @('helper','settings','pin')){if(Test-Path -LiteralPath (Join-Path $folder ($label+'-candidate'))){throw 'No other old candidate expected'}}
if((Hash (Join-Path (Join-Path $boot $p.new_agent) 'rmm-agent-windows.exe'))-cne$p.new_agent-or(Hash (Join-Path (Join-Path $boot $p.new_helper) 'rmm-helper-windows.exe'))-cne$p.new_helper-or(Hash (Join-Path $folder 'new-helper.json'))-cne$p.new_policy-or(Hash (Join-Path $folder 'new-agent-pin.json'))-cne$p.new_pin){throw 'All four complete new sources'}
$current['recovery_file_identities']=[ordered]@{agent=(FileIdentity $agentBinary);helper=(FileIdentity $helperBinary);settings=(FileIdentity $settings);pin=(FileIdentity $pin);agent_candidate=(FileIdentity (Join-Path $folder 'agent-candidate'))}
return $current
}
if($r.phase-ceq'prepare'){
if((Test-Path -LiteralPath $recovery)-or(Test-Path -LiteralPath $recoveryActive)){throw 'Existing recovery: readback only'}
$before=CheckOriginalStopped
New-Item -ItemType Directory -Path $recovery|Out-Null;CheckControl $recovery $true
WriteNew $recoveryActive @{original_id=$p.id;recovery_id=$r.recovery_id;script_sha256=$r.script_sha256;create_only=$true}
WriteNew (Join-Path $recovery 'intent.json') @{original_id=$p.id;recovery_id=$r.recovery_id;script_sha256=$r.script_sha256;original_payload=$p;before=$before}
WriteNew (Join-Path $recovery 'prepared.json') @{prepared=$true;no_service_changes=$true;original_failed_attempt_retained=$true}
@{prepared=$true;original_id=$p.id;recovery_id=$r.recovery_id;both_stopped=$true;preserved=$true}|ConvertTo-Json -Compress;exit 0
}
CheckControl $recovery $true;foreach($name in @('intent.json','prepared.json')){CheckControl (Join-Path $recovery $name) $false};CheckControl $recoveryActive $false
$saved=Get-Content -LiteralPath (Join-Path $recovery 'intent.json') -Raw|ConvertFrom-Json;$currentActive=Get-Content -LiteralPath $recoveryActive -Raw|ConvertFrom-Json
if($saved.original_id-cne$p.id-or$saved.recovery_id-cne$r.recovery_id-or$saved.script_sha256-cne$r.script_sha256-or!(JsonEqual $saved.original_payload $p)-or$currentActive.original_id-cne$p.id-or$currentActive.recovery_id-cne$r.recovery_id-or$currentActive.script_sha256-cne$r.script_sha256){throw 'Exact original recovery admission'}
if(Test-Path -LiteralPath (Join-Path $recovery 'enter-attempt.json')){throw 'Recovery already attempted: readback only'}
$before=CheckOriginalStopped;Preserved $saved.before $before $true
if(!(JsonEqual $saved.before.recovery_file_identities $before.recovery_file_identities)){throw 'Original destination/candidate file IDs or ACL/bytes changed'}
WriteNew (Join-Path $recovery 'enter-attempt.json') @{enter_once=$true;recovery_id=$r.recovery_id;script_sha256=$r.script_sha256}
function RecoverSwap($label,$destination,$source,$oldHash,$newHash,[bool]$reuse){
if((Service 'agent').state-cne'Stopped'-or(Service 'helper').state-cne'Stopped'){throw 'No component start during four-file recovery'}
$candidate=Join-Path $folder ($label+'-candidate');$nativeBackup=Join-Path $recovery ($label+'-native-original')
$oldReceipt=Get-Content -LiteralPath (Join-Path $folder ($label+'-backup.json')) -Raw|ConvertFrom-Json;$sddl=$oldReceipt.original_sddl
if((Hash $destination)-cne$oldHash-or(Hash $source)-cne$newHash-or(Hash (Join-Path $folder ($label+'-original')))-cne$oldHash-or(AclHash $destination)-cne(BytesHash ([Text.Encoding]::UTF8.GetBytes($sddl)))-or(Test-Path -LiteralPath $nativeBackup)){throw 'Exact old destination/backup/ACL and new source CAS'}
if($reuse){if((Hash $candidate)-cne$newHash){throw 'Retain original exact candidate'}}else{if(Test-Path -LiteralPath $candidate){throw 'Candidate exists; no overwrite'};[IO.File]::Copy($source,$candidate,$false);FlushCopy $candidate}
if((Hash $candidate)-cne$newHash){throw 'Complete candidate'}
$destinationIdentity=FileIdentity $destination;$candidateIdentity=FileIdentity $candidate
if(!(JsonEqual $destinationIdentity $before.recovery_file_identities.$label)-or($reuse-and!(JsonEqual $candidateIdentity $before.recovery_file_identities.agent_candidate))){throw 'Fresh native CAS identities/ACL/bytes'}
WriteNew (Join-Path $recovery ($label+'-replace-attempt.json')) @{old_sha256=$oldHash;new_sha256=$newHash;native_backup=$nativeBackup;recovery_id=$r.recovery_id;source_sha256=$r.script_sha256;destination_identity=$destinationIdentity;candidate_identity=$candidateIdentity}
[IO.File]::Replace($candidate,$destination,$nativeBackup)
PrivateBackupAcl $nativeBackup
$acl=New-Object Security.AccessControl.FileSecurity;$acl.SetSecurityDescriptorSddlForm($sddl);Set-Acl -LiteralPath $destination -AclObject $acl;FlushCopy $destination
if((Hash $destination)-cne$newHash-or(Hash $nativeBackup)-cne$oldHash-or(AclHash $destination)-cne(BytesHash ([Text.Encoding]::UTF8.GetBytes($sddl)))){throw 'Actual exact replacement and explicit original backup'}
$receipt=@{old_sha256=$oldHash;new_sha256=$newHash;backup_sha256=(Hash (Join-Path $folder ($label+'-original')));acl_sha256=(AclHash $destination);native_backup_sha256=(Hash $nativeBackup);recovery_id=$r.recovery_id;source_sha256=$r.script_sha256}
WriteNew (Join-Path $recovery ($label+'-replaced.json')) $receipt
WriteNew (Join-Path $folder ($label+'-replaced.json')) $receipt
}
try{
RecoverSwap 'agent' $agentBinary (Join-Path (Join-Path $boot $p.new_agent) 'rmm-agent-windows.exe') $p.old_agent $p.new_agent $true
RecoverSwap 'helper' $helperBinary (Join-Path (Join-Path $boot $p.new_helper) 'rmm-helper-windows.exe') $p.old_helper $p.new_helper $false
RecoverSwap 'settings' $settings (Join-Path $folder 'new-helper.json') $p.old_policy $p.new_policy $false
RecoverSwap 'pin' $pin (Join-Path $folder 'new-agent-pin.json') $p.old_pin $p.new_pin $false
$swapped=State $true;Preserved $before $swapped $true
if($swapped.service.state-cne'Stopped'-or$swapped.helper_service.state-cne'Stopped'){throw 'All four new files checked before first start'}
$swapped['recovery_id']=$r.recovery_id;$swapped['recovery_source_sha256']=$r.script_sha256
WriteNew (Join-Path $recovery 'swapped.json') $swapped;WriteNew (Join-Path $folder 'swapped.json') $swapped
WriteNew (Join-Path $recovery 'helper-start-intent.json') @{recovery_id=$r.recovery_id;source_sha256=$r.script_sha256};StartOnce 'helper'
WriteNew (Join-Path $recovery 'agent-start-intent.json') @{recovery_id=$r.recovery_id;source_sha256=$r.script_sha256};StartOnce 'agent'
$after=State $true;Preserved $intent.before $after $false;Preserved $before $after $false
$after['recovery_id']=$r.recovery_id;$after['recovery_source_sha256']=$r.script_sha256
WriteNew (Join-Path $recovery 'entered.json') $after;WriteNew (Join-Path $folder 'entered.json') $after
@{entered=$true;original_id=$p.id;recovery_id=$r.recovery_id;no_jobs=$true;no_reboot=$true;no_clock_change=$true}|ConvertTo-Json -Compress
}catch{WriteNew (Join-Path $recovery ('incomplete-'+[guid]::NewGuid().ToString()+'.json')) @{error_type=$_.Exception.GetType().FullName;line=$_.InvocationInfo.ScriptLineNumber;readback_only=$true;no_retry=$true};throw}
'''
PREPARE = u.transport.PREPARE_SCRIPT.replace('TransportAcceptance.ps1', 'SystemDelegationRecovery.ps1')
LAUNCH = u.transport.LAUNCH_SCRIPT.replace('TransportAcceptance.ps1', 'SystemDelegationRecovery.ps1').replace('RmmTransportPayload', 'RmmDelegationRecoveryPayload')
# Prefix reads the immutable original payload; the recovery data is separate.
LAUNCH = LAUNCH.replace('& $script', '$global:RmmDelegationUpgradePayload=$p.payload.original_payload\n& $script')
def load(recovery_id):
require(str(uuid.UUID(recovery_id)) == recovery_id, 'canonical recovery UUID')
folder = STATE / recovery_id; intent = cb.contract.protected_json(folder / 'intent.json')
require(u.sha(u.__file__) == ORIGINAL_SOURCE and intent['runner_sha256'] == u.sha(__file__) and intent['script_sha256'] == u.raw_sha(SCRIPT.encode()), 'original and new source remain immutable')
_, original = u.load_case(CASE)
require(intent['original_intent_sha256'] == u.sha(u.STATE / CASE / 'intent.json') and intent['original_payload'] == original['payload'], 'unchanged original case')
return folder, intent
def remote(intent, phase):
original_action = 'readback' if phase == 'readback' else 'enter'
payload = dict(recovery_id=intent['recovery_id'], phase=phase, script_sha256=intent['script_sha256'], original_payload=dict(intent['original_payload'], action=original_action))
return cb.powershell(LAUNCH, dict(sha256=intent['script_sha256'], payload=payload), timeout=180, remote_signed=True)
def prepare():
require(u.sha(u.__file__) == ORIGINAL_SOURCE, 'original failed runner must not change')
_, original = u.load_case(CASE)
original_folder = u.STATE / CASE
require((original_folder / 'enter-attempt.json').is_file() and (original_folder / 'reply-incomplete.json').is_file(), 'actual original failed entry retained')
u.original_helper.private_dir(STATE)
recovery_id = str(uuid.uuid4()); folder = STATE / recovery_id; folder.mkdir(mode=0o700)
intent = dict(schema_version=1, type='rmm.windows.passive-joint-recovery', recovery_id=recovery_id, original_id=CASE, original_payload=original['payload'], original_intent_sha256=u.sha(original_folder / 'intent.json'), runner_sha256=u.sha(__file__), script_sha256=u.raw_sha(SCRIPT.encode()))
u.save(folder / 'intent.json', intent)
source = folder / 'SystemDelegationRecovery.ps1'
fd = os.open(source, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, 'wb') as output:
output.write(SCRIPT.encode()); output.flush(); os.fsync(output.fileno())
u.original_helper.transfer(source, 'SystemDelegationRecovery.ps1', intent['script_sha256'], PREPARE)
result = remote(intent, 'prepare'); u.save(folder / 'prepared.json', result)
require(result.get('prepared') is True and result.get('both_stopped') is True and result.get('preserved') is True, 'actual preserved recovery admission')
print(json.dumps({'original_id': CASE, 'recovery_id': recovery_id, 'prepared': True, 'no_start': True}))
def run(recovery_id, phase):
folder, intent = load(recovery_id)
if phase == 'enter':
require(cb.contract.protected_json(folder / 'prepared.json')['prepared'] is True, 'actual prepared admission')
u.save(folder / 'enter-attempt.json', {'once': True, 'lost_reply_readback_only': True})
try:
result = remote(intent, phase)
except Exception as error:
u.save(folder / ('reply-incomplete-' + str(uuid.uuid4()) + '.json'), {'error_type': type(error).__name__, 'readback_only': True})
raise
u.save(folder / ('entered.json' if phase == 'enter' else 'readback-' + str(uuid.uuid4()) + '.json'), result)
print(json.dumps({'original_id': CASE, 'recovery_id': recovery_id, 'entered': result.get('entered', False), 'read_only': result.get('read_only', False), 'original_phases': result.get('original_phases'), 'recovery_phases': result.get('recovery_phases')}))
def main():
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('phase', choices=['prepare', 'enter', 'readback']); parser.add_argument('--id'); args = parser.parse_args()
os.umask(0o077)
if args.phase == 'prepare':
require(args.id is None, 'create a new recovery admission'); prepare()
else:
require(args.id is not None, 'exact existing recovery UUID required'); run(args.id, args.phase)
if __name__ == '__main__':
main()