154 lines
8.9 KiB
Python
154 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Local-only, separate MIDL readiness crossbuild. Never accesses a VM.
|
|
|
|
Preserves all existing runners, product sources, generated outputs and evidence.
|
|
Each invocation creates a fresh source snapshot and records failed checks too.
|
|
"""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
PACKAGE = Path('deploy/windows/clock-midl-diagnostic')
|
|
IMAGE = 'sha256:1b2fd985d2c1fcc3545ace602405dc68fa899201bd8a73fa319ff96377203fa9'
|
|
CODEGEN = ROOT / 'devstate/windows-clock-midl/native-cases/149bafe2-3469-4580-b133-988812bcc3db'
|
|
READBACK = 'readback-059c15ef-0f31-41aa-ad1d-ea6b774280d7.json'
|
|
READBACK_SHA = '0dc2373825f7f8f224cda04ecd5c521dd3c37a6e712bc3ed83bc4fa6d0d89c8c'
|
|
PINNED = {
|
|
'w32time.h': 'db5dbe5b335eaa58a6512d4abfda44e7273f3aa8a43077dcb0719e4b97ee2bde',
|
|
'w32time_c.c': 'd806adb431f16724c8afdea0c0751588a4bc6f82b6f0b8391d4d189d6a6f4a38',
|
|
}
|
|
|
|
|
|
def sha(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise RuntimeError(message)
|
|
|
|
|
|
def write_new(path, raw):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open('xb') as out:
|
|
out.write(raw)
|
|
out.flush()
|
|
os.fsync(out.fileno())
|
|
|
|
|
|
def save(path, value):
|
|
write_new(path, (json.dumps(value, indent=2) + '\n').encode())
|
|
|
|
|
|
def compiler_inputs():
|
|
paths = [ROOT / name for name in ('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml')]
|
|
paths += [p for p in (ROOT / 'crates').rglob('*') if p.is_file() and p.suffix in ('.rs', '.toml', '.c', '.idl')]
|
|
paths += [p for p in (ROOT / 'proto').rglob('*') if p.is_file()]
|
|
paths += [ROOT / PACKAGE / name for name in ('Cargo.toml', 'build.rs')]
|
|
paths += [Path(__file__).resolve()]
|
|
require(all(not p.is_symlink() for p in paths), 'compiler input symlink')
|
|
return {str(p.relative_to(ROOT)): sha(p.read_bytes()) for p in sorted(paths)}
|
|
|
|
|
|
def external_packages(lock):
|
|
# Only compare Cargo-generated fixed string fields; never execute TOML.
|
|
result = set()
|
|
for block in lock.split('[[package]]')[1:]:
|
|
fields = dict(re.findall(r'^(name|version|source|checksum) = "([^"\n]+)"$', block, re.M))
|
|
if 'source' in fields:
|
|
require(set(fields) == {'name', 'version', 'source', 'checksum'}, 'unrecognized external lock entry')
|
|
result.add(tuple(fields[key] for key in ('name', 'version', 'source', 'checksum')))
|
|
return result
|
|
|
|
|
|
def main():
|
|
os.umask(0o077)
|
|
before = compiler_inputs()
|
|
receipt_raw = (CODEGEN / READBACK).read_bytes()
|
|
require(sha(receipt_raw) == READBACK_SHA, 'original native readback hash changed')
|
|
receipt = json.loads(receipt_raw)
|
|
result = receipt['result']
|
|
require(receipt['readback_only'] is True and result['complete'] is True and result['no_rpc'] is True
|
|
and result['run']['exit_code'] == 0 and result['run']['all_own_children_exited'] is True
|
|
and result['clock_qualified'] is False, 'native codegen readback incomplete')
|
|
ident = str(uuid.uuid4())
|
|
case = ROOT / 'devstate/windows-clock-midl/diagnostic-builds' / ident
|
|
case.mkdir(parents=True, mode=0o700)
|
|
source = case / 'source'
|
|
source.mkdir(mode=0o700)
|
|
for name, digest in before.items():
|
|
raw = (ROOT / name).read_bytes()
|
|
require(sha(raw) == digest, 'source changed during snapshot')
|
|
write_new(source / name, raw)
|
|
for name, digest in PINNED.items():
|
|
raw = (CODEGEN / ('native-' + name)).read_bytes()
|
|
original = next(item for item in receipt['files'] if item['name'] == name)
|
|
require(sha(raw) == digest == original['sha256'] and len(raw) == original['bytes']
|
|
and base64.b64decode(original['base64'], validate=True) == raw, 'actual Microsoft output differs')
|
|
write_new(source / PACKAGE / 'generated' / name, raw)
|
|
write_new(source / PACKAGE / 'Cargo.lock', (ROOT / 'Cargo.lock').read_bytes())
|
|
save(case / 'intent.json', dict(id=ident, image=IMAGE, native_case=str(CODEGEN), native_readback_sha256=READBACK_SHA,
|
|
source_files=before, microsoft_generated=PINNED, no_vm=True, no_jobs=True,
|
|
production_clock_qualified=False))
|
|
print('Retained local MIDL diagnostic build ' + ident, flush=True)
|
|
checks = {}
|
|
|
|
def run(name, args):
|
|
command = ['docker', 'run', '--rm', '--network', 'none', '--read-only', '--tmpfs', '/tmp:rw,size=512m',
|
|
'-v', str(source) + ':/src:ro', '-v', str(case / 'target') + ':/target',
|
|
'-v', str(case) + ':/evidence', '-e', 'CARGO_TARGET_DIR=/target',
|
|
'--entrypoint', args[0], IMAGE] + args[1:]
|
|
with (case / (name + '.log')).open('xb') as output:
|
|
completed = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT, timeout=600)
|
|
checks[name] = dict(exit_code=completed.returncode, command=command, log_sha256=sha((case / (name + '.log')).read_bytes()))
|
|
save(case / (name + '.check.json'), checks[name])
|
|
require(completed.returncode == 0, 'local check failed: ' + name)
|
|
print(name + ' PASS', flush=True)
|
|
|
|
manifest = '/src/' + str(PACKAGE / 'Cargo.toml')
|
|
# Resolve only the new diagnostic leaf against the existing locked versions.
|
|
# Snapshot manifest directory is temporarily a private writable mount for
|
|
# this sole metadata step; the source worktree is never writable in Docker.
|
|
writable = case / 'lock-resolution'
|
|
shutil.copytree(source, writable)
|
|
metadata_command = ['docker', 'run', '--rm', '--network', 'none', '--read-only', '--tmpfs', '/tmp:rw,size=128m',
|
|
'-v', str(writable) + ':/src', '--entrypoint', 'cargo', IMAGE, 'metadata',
|
|
'--manifest-path', manifest, '--offline', '--format-version', '1']
|
|
with (case / 'lock-resolution.log').open('xb') as output:
|
|
lock_result = subprocess.run(metadata_command, stdout=output, stderr=subprocess.STDOUT, timeout=120)
|
|
require(lock_result.returncode == 0, 'diagnostic lock resolution failed')
|
|
resolved = (writable / PACKAGE / 'Cargo.lock').read_bytes()
|
|
require(external_packages(resolved.decode()).issubset(external_packages((ROOT / 'Cargo.lock').read_text())), 'new external dependency version')
|
|
# Replace only our freshly copied diagnostic lock BEFORE build admission;
|
|
# this is never an edit to generated NDR/header data or an OUT_DIR patch.
|
|
(source / PACKAGE / 'Cargo.lock').write_bytes(resolved)
|
|
save(case / 'resolved-lock.json', dict(sha256=sha(resolved), no_new_external_versions=True))
|
|
run('fmt', ['cargo', 'fmt', '--manifest-path', manifest, '--', '--check'])
|
|
run('clippy-windows', ['cargo', 'clippy', '--manifest-path', manifest, '--locked', '--offline', '--target', 'x86_64-pc-windows-gnu', '--all-targets', '--', '-D', 'warnings'])
|
|
run('linux-clock-tests', ['cargo', 'test', '--manifest-path', manifest, '--locked', '--offline', '--bin', 'clock-midl-readiness', '--', '--test-threads=1'])
|
|
require('test result: ok. 5 passed; 0 failed; 0 ignored;' in (case / 'linux-clock-tests.log').read_text(), 'actual five clock tests required')
|
|
run('windows-test-link', ['cargo', 'build', '--manifest-path', manifest, '--locked', '--offline', '--target', 'x86_64-pc-windows-gnu', '--tests'])
|
|
run('release-link', ['cargo', 'build', '--manifest-path', manifest, '--locked', '--offline', '--target', 'x86_64-pc-windows-gnu', '--release', '--bin', 'clock-midl-readiness'])
|
|
run('artifact', ['sh', '-c', 'cp /target/x86_64-pc-windows-gnu/release/clock-midl-readiness.exe /evidence/WindowsClockReadiness.linked.exe && cp /evidence/WindowsClockReadiness.linked.exe /evidence/WindowsClockReadiness.exe && x86_64-w64-mingw32-strip /evidence/WindowsClockReadiness.exe && x86_64-w64-mingw32-objdump -p /evidence/WindowsClockReadiness.exe'])
|
|
require(compiler_inputs() == before, 'source inputs changed during diagnostic build')
|
|
executable = case / 'WindowsClockReadiness.exe'
|
|
qualification = dict(id=ident, source_files=before, source_before_after_identical=True, image=IMAGE,
|
|
native_codegen_case=CODEGEN.name, native_readback_sha256=READBACK_SHA,
|
|
generated_inputs=PINNED, checks=checks, artifact=dict(path=str(executable), sha256=sha(executable.read_bytes()), bytes=executable.stat().st_size),
|
|
linked_binary_sha256=sha((case / 'WindowsClockReadiness.linked.exe').read_bytes()),
|
|
linux_clock_test_count=5, native_windows_execution=False, native_cloud_linux_execution=False,
|
|
production_clock_qualified=False, no_vm=True, no_jobs=True)
|
|
save(case / 'qualification.json', qualification)
|
|
print(json.dumps(dict(id=ident, artifact=qualification['artifact'], qualification_sha256=sha((case / 'qualification.json').read_bytes()), no_vm=True, clock_qualified=False)))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|