194 lines
17 KiB
Python
194 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""One reviewed recovery of the unchanged failed6d2 joint upgrade; no retries.
|
|
|
|
Uses explicit File.Replace backup paths, preserving the original failed source,
|
|
attempt, four backups, journals and intent. Readback never swaps or starts.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import uuid
|
|
|
|
import client_bootstrap as cb
|
|
import system_delegation_upgrade as u
|
|
|
|
CASE = '6d2df065-0877-41c4-983c-2c9fdb6e5437'
|
|
ORIGINAL_SOURCE = 'f8dc1c8fbc064e253b1fbe130fb628afef58cf45c2bafb6837d6716125b42d38'
|
|
STATE = cb.EVIDENCE / 'system-delegation-recovery'
|
|
require = cb.contract.require
|
|
PREFIX = u.SCRIPT.split('\nCheckControl $boot $true\n', 1)[0]
|
|
|
|
SCRIPT = PREFIX + r'''
|
|
$r=$global:RmmDelegationRecoveryPayload
|
|
if(!$r-or[guid]::ParseExact($r.recovery_id,'D').ToString()-cne$r.recovery_id-or$r.phase-notin@('prepare','enter','readback')-or$p.id-cne'6d2df065-0877-41c4-983c-2c9fdb6e5437'-or$r.script_sha256-notmatch'^[a-f0-9]{64}$'){throw 'Only the original failed6d recovery'}
|
|
CheckControl $boot $true
|
|
$ownScript=Join-Path (Join-Path $boot $r.script_sha256) 'SystemDelegationRecovery.ps1'
|
|
if($PSCommandPath-cne$ownScript-or(Hash $PSCommandPath)-cne$r.script_sha256){throw 'Exact actually executing recovery source'}
|
|
CheckControl $ownScript $false
|
|
$intent=ReadIntent
|
|
$recovery=Join-Path $boot ('system-delegation-recovery-'+$r.recovery_id)
|
|
$recoveryActive=Join-Path $boot 'system-delegation-6d2-recovery-once.json'
|
|
function ReadService($role){$name=$agentName;if($role-ceq'helper'){$name=$helperName};$s=Get-CimInstance Win32_Service -Filter ("Name='"+$name+"'") -OperationTimeoutSec 10;if(!$s){throw 'Fixed existing service'};return @{state=$s.State;pid=$s.ProcessId;account=$s.StartName;mode=$s.StartMode;path=$s.PathName;exit_code=$s.ExitCode}}
|
|
function MaybeHash($path){if(!(Test-Path -LiteralPath $path)){return $null};return Hash $path}
|
|
function RecoveryReadback{
|
|
$original=[ordered]@{};foreach($name in @('agent-stop-attempt','agent-stopped','helper-stop-attempt','helper-stopped','frozen','agent-backup','helper-backup','settings-backup','pin-backup','agent-replace-attempt','helper-replace-attempt','settings-replace-attempt','pin-replace-attempt','agent-replaced','helper-replaced','settings-replaced','pin-replaced','swapped','helper-start-attempt','helper-started','agent-start-attempt','agent-started','entered')){$original[$name]=Test-Path -LiteralPath (Join-Path $folder ($name+'.json'))}
|
|
$own=[ordered]@{};foreach($name in @('intent','prepared','enter-attempt','agent-replace-attempt','agent-replaced','helper-replace-attempt','helper-replaced','settings-replace-attempt','settings-replaced','pin-replace-attempt','pin-replaced','swapped','helper-start-intent','agent-start-intent','entered')){$own[$name]=Test-Path -LiteralPath (Join-Path $recovery ($name+'.json'))}
|
|
return @{read_only=$true;original_id=$p.id;recovery_id=$r.recovery_id;hashes=@{agent=(MaybeHash $agentBinary);helper=(MaybeHash $helperBinary);settings=(MaybeHash $settings);pin=(MaybeHash $pin)};service=(ReadService 'agent');helper_service=(ReadService 'helper');original_phases=$original;recovery_phases=$own;recovery_active_exists=(Test-Path -LiteralPath $recoveryActive);no_repeated_start=$true}
|
|
}
|
|
if($r.phase-ceq'readback'){RecoveryReadback|ConvertTo-Json -Depth 20 -Compress;exit 0}
|
|
Add-Type -TypeDefinition @'
|
|
using System;
|
|
using System.Runtime.InteropServices;
|
|
using Microsoft.Win32.SafeHandles;
|
|
public static class RmmDelegationRecoveryFileId {
|
|
[StructLayout(LayoutKind.Sequential)] public struct Id {
|
|
public ulong VolumeSerialNumber;
|
|
[MarshalAs(UnmanagedType.ByValArray, SizeConst=16)] public byte[] FileId;
|
|
}
|
|
[DllImport("kernel32.dll", SetLastError=true)]
|
|
public static extern bool GetFileInformationByHandleEx(SafeFileHandle handle, int kind, out Id info, uint length);
|
|
}
|
|
'@
|
|
function FileIdentity($path){
|
|
Check $path $false;$file=[IO.File]::Open($path,[IO.FileMode]::Open,[IO.FileAccess]::Read,[IO.FileShare]::Read)
|
|
try{$id=New-Object RmmDelegationRecoveryFileId+Id;$size=[Runtime.InteropServices.Marshal]::SizeOf([type][RmmDelegationRecoveryFileId+Id]);if($size-ne24-or![RmmDelegationRecoveryFileId]::GetFileInformationByHandleEx($file.SafeFileHandle,18,[ref]$id,[uint32]$size)){throw 'Exact native file identity'}
|
|
return [ordered]@{volume_serial=$id.VolumeSerialNumber.ToString([Globalization.CultureInfo]::InvariantCulture);file_id=([BitConverter]::ToString($id.FileId)).Replace('-','').ToLowerInvariant();bytes=$file.Length;acl_sha256=(AclHash $path);sha256=(Hash $path)}
|
|
}finally{$file.Dispose()}
|
|
}
|
|
function CheckOriginalStopped{
|
|
foreach($name in @('intent','agent-stop-attempt','agent-stopped','helper-stop-attempt','helper-stopped','frozen','agent-backup','helper-backup','settings-backup','pin-backup','agent-replace-attempt')){CheckControl (Join-Path $folder ($name+'.json')) $false}
|
|
foreach($name in @('helper-replace-attempt','settings-replace-attempt','pin-replace-attempt','agent-replaced','helper-replaced','settings-replaced','pin-replaced','swapped','helper-start-attempt','helper-started','agent-start-attempt','agent-started','entered')){if(Test-Path -LiteralPath (Join-Path $folder ($name+'.json'))){throw 'Original case progressed; no recovery replay'}}
|
|
$frozen=Get-Content -LiteralPath (Join-Path $folder 'frozen.json') -Raw|ConvertFrom-Json
|
|
$current=State $false
|
|
if($current.service.state-cne'Stopped'-or$current.helper_service.state-cne'Stopped'){throw 'Both own original services must remain stopped'}
|
|
Preserved $frozen $current $true
|
|
$checks=@(@('agent',$p.old_agent,$p.new_agent),@('helper',$p.old_helper,$p.new_helper),@('settings',$p.old_policy,$p.new_policy),@('pin',$p.old_pin,$p.new_pin))
|
|
foreach($item in $checks){$backup=Join-Path $folder ($item[0]+'-original');CheckControl $backup $false;if((Hash $backup)-cne$item[1]){throw 'Original exact backup'};$receipt=Get-Content -LiteralPath (Join-Path $folder ($item[0]+'-backup.json')) -Raw|ConvertFrom-Json;if($receipt.sha256-cne$item[1]){throw 'Original backup receipt'}}
|
|
$attempt=Get-Content -LiteralPath (Join-Path $folder 'agent-replace-attempt.json') -Raw|ConvertFrom-Json
|
|
if($attempt.old_sha256-cne$p.old_agent-or$attempt.new_sha256-cne$p.new_agent){throw 'Original unchanged failed agent attempt'}
|
|
if((Hash (Join-Path $folder 'agent-candidate'))-cne$p.new_agent){throw 'Retain exact original agent candidate'}
|
|
foreach($label in @('helper','settings','pin')){if(Test-Path -LiteralPath (Join-Path $folder ($label+'-candidate'))){throw 'No other old candidate expected'}}
|
|
if((Hash (Join-Path (Join-Path $boot $p.new_agent) 'rmm-agent-windows.exe'))-cne$p.new_agent-or(Hash (Join-Path (Join-Path $boot $p.new_helper) 'rmm-helper-windows.exe'))-cne$p.new_helper-or(Hash (Join-Path $folder 'new-helper.json'))-cne$p.new_policy-or(Hash (Join-Path $folder 'new-agent-pin.json'))-cne$p.new_pin){throw 'All four complete new sources'}
|
|
$current['recovery_file_identities']=[ordered]@{agent=(FileIdentity $agentBinary);helper=(FileIdentity $helperBinary);settings=(FileIdentity $settings);pin=(FileIdentity $pin);agent_candidate=(FileIdentity (Join-Path $folder 'agent-candidate'))}
|
|
return $current
|
|
}
|
|
if($r.phase-ceq'prepare'){
|
|
if((Test-Path -LiteralPath $recovery)-or(Test-Path -LiteralPath $recoveryActive)){throw 'Existing recovery: readback only'}
|
|
$before=CheckOriginalStopped
|
|
New-Item -ItemType Directory -Path $recovery|Out-Null;CheckControl $recovery $true
|
|
WriteNew $recoveryActive @{original_id=$p.id;recovery_id=$r.recovery_id;script_sha256=$r.script_sha256;create_only=$true}
|
|
WriteNew (Join-Path $recovery 'intent.json') @{original_id=$p.id;recovery_id=$r.recovery_id;script_sha256=$r.script_sha256;original_payload=$p;before=$before}
|
|
WriteNew (Join-Path $recovery 'prepared.json') @{prepared=$true;no_service_changes=$true;original_failed_attempt_retained=$true}
|
|
@{prepared=$true;original_id=$p.id;recovery_id=$r.recovery_id;both_stopped=$true;preserved=$true}|ConvertTo-Json -Compress;exit 0
|
|
}
|
|
CheckControl $recovery $true;foreach($name in @('intent.json','prepared.json')){CheckControl (Join-Path $recovery $name) $false};CheckControl $recoveryActive $false
|
|
$saved=Get-Content -LiteralPath (Join-Path $recovery 'intent.json') -Raw|ConvertFrom-Json;$currentActive=Get-Content -LiteralPath $recoveryActive -Raw|ConvertFrom-Json
|
|
if($saved.original_id-cne$p.id-or$saved.recovery_id-cne$r.recovery_id-or$saved.script_sha256-cne$r.script_sha256-or!(JsonEqual $saved.original_payload $p)-or$currentActive.original_id-cne$p.id-or$currentActive.recovery_id-cne$r.recovery_id-or$currentActive.script_sha256-cne$r.script_sha256){throw 'Exact original recovery admission'}
|
|
if(Test-Path -LiteralPath (Join-Path $recovery 'enter-attempt.json')){throw 'Recovery already attempted: readback only'}
|
|
$before=CheckOriginalStopped;Preserved $saved.before $before $true
|
|
if(!(JsonEqual $saved.before.recovery_file_identities $before.recovery_file_identities)){throw 'Original destination/candidate file IDs or ACL/bytes changed'}
|
|
WriteNew (Join-Path $recovery 'enter-attempt.json') @{enter_once=$true;recovery_id=$r.recovery_id;script_sha256=$r.script_sha256}
|
|
function RecoverSwap($label,$destination,$source,$oldHash,$newHash,[bool]$reuse){
|
|
if((Service 'agent').state-cne'Stopped'-or(Service 'helper').state-cne'Stopped'){throw 'No component start during four-file recovery'}
|
|
$candidate=Join-Path $folder ($label+'-candidate');$nativeBackup=Join-Path $recovery ($label+'-native-original')
|
|
$oldReceipt=Get-Content -LiteralPath (Join-Path $folder ($label+'-backup.json')) -Raw|ConvertFrom-Json;$sddl=$oldReceipt.original_sddl
|
|
if((Hash $destination)-cne$oldHash-or(Hash $source)-cne$newHash-or(Hash (Join-Path $folder ($label+'-original')))-cne$oldHash-or(AclHash $destination)-cne(BytesHash ([Text.Encoding]::UTF8.GetBytes($sddl)))-or(Test-Path -LiteralPath $nativeBackup)){throw 'Exact old destination/backup/ACL and new source CAS'}
|
|
if($reuse){if((Hash $candidate)-cne$newHash){throw 'Retain original exact candidate'}}else{if(Test-Path -LiteralPath $candidate){throw 'Candidate exists; no overwrite'};[IO.File]::Copy($source,$candidate,$false);FlushCopy $candidate}
|
|
if((Hash $candidate)-cne$newHash){throw 'Complete candidate'}
|
|
$destinationIdentity=FileIdentity $destination;$candidateIdentity=FileIdentity $candidate
|
|
if(!(JsonEqual $destinationIdentity $before.recovery_file_identities.$label)-or($reuse-and!(JsonEqual $candidateIdentity $before.recovery_file_identities.agent_candidate))){throw 'Fresh native CAS identities/ACL/bytes'}
|
|
WriteNew (Join-Path $recovery ($label+'-replace-attempt.json')) @{old_sha256=$oldHash;new_sha256=$newHash;native_backup=$nativeBackup;recovery_id=$r.recovery_id;source_sha256=$r.script_sha256;destination_identity=$destinationIdentity;candidate_identity=$candidateIdentity}
|
|
[IO.File]::Replace($candidate,$destination,$nativeBackup)
|
|
PrivateBackupAcl $nativeBackup
|
|
$acl=New-Object Security.AccessControl.FileSecurity;$acl.SetSecurityDescriptorSddlForm($sddl);Set-Acl -LiteralPath $destination -AclObject $acl;FlushCopy $destination
|
|
if((Hash $destination)-cne$newHash-or(Hash $nativeBackup)-cne$oldHash-or(AclHash $destination)-cne(BytesHash ([Text.Encoding]::UTF8.GetBytes($sddl)))){throw 'Actual exact replacement and explicit original backup'}
|
|
$receipt=@{old_sha256=$oldHash;new_sha256=$newHash;backup_sha256=(Hash (Join-Path $folder ($label+'-original')));acl_sha256=(AclHash $destination);native_backup_sha256=(Hash $nativeBackup);recovery_id=$r.recovery_id;source_sha256=$r.script_sha256}
|
|
WriteNew (Join-Path $recovery ($label+'-replaced.json')) $receipt
|
|
WriteNew (Join-Path $folder ($label+'-replaced.json')) $receipt
|
|
}
|
|
try{
|
|
RecoverSwap 'agent' $agentBinary (Join-Path (Join-Path $boot $p.new_agent) 'rmm-agent-windows.exe') $p.old_agent $p.new_agent $true
|
|
RecoverSwap 'helper' $helperBinary (Join-Path (Join-Path $boot $p.new_helper) 'rmm-helper-windows.exe') $p.old_helper $p.new_helper $false
|
|
RecoverSwap 'settings' $settings (Join-Path $folder 'new-helper.json') $p.old_policy $p.new_policy $false
|
|
RecoverSwap 'pin' $pin (Join-Path $folder 'new-agent-pin.json') $p.old_pin $p.new_pin $false
|
|
$swapped=State $true;Preserved $before $swapped $true
|
|
if($swapped.service.state-cne'Stopped'-or$swapped.helper_service.state-cne'Stopped'){throw 'All four new files checked before first start'}
|
|
$swapped['recovery_id']=$r.recovery_id;$swapped['recovery_source_sha256']=$r.script_sha256
|
|
WriteNew (Join-Path $recovery 'swapped.json') $swapped;WriteNew (Join-Path $folder 'swapped.json') $swapped
|
|
WriteNew (Join-Path $recovery 'helper-start-intent.json') @{recovery_id=$r.recovery_id;source_sha256=$r.script_sha256};StartOnce 'helper'
|
|
WriteNew (Join-Path $recovery 'agent-start-intent.json') @{recovery_id=$r.recovery_id;source_sha256=$r.script_sha256};StartOnce 'agent'
|
|
$after=State $true;Preserved $intent.before $after $false;Preserved $before $after $false
|
|
$after['recovery_id']=$r.recovery_id;$after['recovery_source_sha256']=$r.script_sha256
|
|
WriteNew (Join-Path $recovery 'entered.json') $after;WriteNew (Join-Path $folder 'entered.json') $after
|
|
@{entered=$true;original_id=$p.id;recovery_id=$r.recovery_id;no_jobs=$true;no_reboot=$true;no_clock_change=$true}|ConvertTo-Json -Compress
|
|
}catch{WriteNew (Join-Path $recovery ('incomplete-'+[guid]::NewGuid().ToString()+'.json')) @{error_type=$_.Exception.GetType().FullName;line=$_.InvocationInfo.ScriptLineNumber;readback_only=$true;no_retry=$true};throw}
|
|
'''
|
|
|
|
PREPARE = u.transport.PREPARE_SCRIPT.replace('TransportAcceptance.ps1', 'SystemDelegationRecovery.ps1')
|
|
LAUNCH = u.transport.LAUNCH_SCRIPT.replace('TransportAcceptance.ps1', 'SystemDelegationRecovery.ps1').replace('RmmTransportPayload', 'RmmDelegationRecoveryPayload')
|
|
# Prefix reads the immutable original payload; the recovery data is separate.
|
|
LAUNCH = LAUNCH.replace('& $script', '$global:RmmDelegationUpgradePayload=$p.payload.original_payload\n& $script')
|
|
|
|
|
|
def load(recovery_id):
|
|
require(str(uuid.UUID(recovery_id)) == recovery_id, 'canonical recovery UUID')
|
|
folder = STATE / recovery_id; intent = cb.contract.protected_json(folder / 'intent.json')
|
|
require(u.sha(u.__file__) == ORIGINAL_SOURCE and intent['runner_sha256'] == u.sha(__file__) and intent['script_sha256'] == u.raw_sha(SCRIPT.encode()), 'original and new source remain immutable')
|
|
_, original = u.load_case(CASE)
|
|
require(intent['original_intent_sha256'] == u.sha(u.STATE / CASE / 'intent.json') and intent['original_payload'] == original['payload'], 'unchanged original case')
|
|
return folder, intent
|
|
|
|
|
|
def remote(intent, phase):
|
|
original_action = 'readback' if phase == 'readback' else 'enter'
|
|
payload = dict(recovery_id=intent['recovery_id'], phase=phase, script_sha256=intent['script_sha256'], original_payload=dict(intent['original_payload'], action=original_action))
|
|
return cb.powershell(LAUNCH, dict(sha256=intent['script_sha256'], payload=payload), timeout=180, remote_signed=True)
|
|
|
|
|
|
def prepare():
|
|
require(u.sha(u.__file__) == ORIGINAL_SOURCE, 'original failed runner must not change')
|
|
_, original = u.load_case(CASE)
|
|
original_folder = u.STATE / CASE
|
|
require((original_folder / 'enter-attempt.json').is_file() and (original_folder / 'reply-incomplete.json').is_file(), 'actual original failed entry retained')
|
|
u.original_helper.private_dir(STATE)
|
|
recovery_id = str(uuid.uuid4()); folder = STATE / recovery_id; folder.mkdir(mode=0o700)
|
|
intent = dict(schema_version=1, type='rmm.windows.passive-joint-recovery', recovery_id=recovery_id, original_id=CASE, original_payload=original['payload'], original_intent_sha256=u.sha(original_folder / 'intent.json'), runner_sha256=u.sha(__file__), script_sha256=u.raw_sha(SCRIPT.encode()))
|
|
u.save(folder / 'intent.json', intent)
|
|
source = folder / 'SystemDelegationRecovery.ps1'
|
|
fd = os.open(source, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, 'wb') as output:
|
|
output.write(SCRIPT.encode()); output.flush(); os.fsync(output.fileno())
|
|
u.original_helper.transfer(source, 'SystemDelegationRecovery.ps1', intent['script_sha256'], PREPARE)
|
|
result = remote(intent, 'prepare'); u.save(folder / 'prepared.json', result)
|
|
require(result.get('prepared') is True and result.get('both_stopped') is True and result.get('preserved') is True, 'actual preserved recovery admission')
|
|
print(json.dumps({'original_id': CASE, 'recovery_id': recovery_id, 'prepared': True, 'no_start': True}))
|
|
|
|
|
|
def run(recovery_id, phase):
|
|
folder, intent = load(recovery_id)
|
|
if phase == 'enter':
|
|
require(cb.contract.protected_json(folder / 'prepared.json')['prepared'] is True, 'actual prepared admission')
|
|
u.save(folder / 'enter-attempt.json', {'once': True, 'lost_reply_readback_only': True})
|
|
try:
|
|
result = remote(intent, phase)
|
|
except Exception as error:
|
|
u.save(folder / ('reply-incomplete-' + str(uuid.uuid4()) + '.json'), {'error_type': type(error).__name__, 'readback_only': True})
|
|
raise
|
|
u.save(folder / ('entered.json' if phase == 'enter' else 'readback-' + str(uuid.uuid4()) + '.json'), result)
|
|
print(json.dumps({'original_id': CASE, 'recovery_id': recovery_id, 'entered': result.get('entered', False), 'read_only': result.get('read_only', False), 'original_phases': result.get('original_phases'), 'recovery_phases': result.get('recovery_phases')}))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__); parser.add_argument('phase', choices=['prepare', 'enter', 'readback']); parser.add_argument('--id'); args = parser.parse_args()
|
|
os.umask(0o077)
|
|
if args.phase == 'prepare':
|
|
require(args.id is None, 'create a new recovery admission'); prepare()
|
|
else:
|
|
require(args.id is not None, 'exact existing recovery UUID required'); run(args.id, args.phase)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|