212 lines
14 KiB
Python
212 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Freeze and build the Linux compatible baseline and version-only successor.
|
|
|
|
Offline read-only Docker source mounts, independent local cache, no credentials,
|
|
VM/server command, job, daemon, signing-key generation or deployment. Windows
|
|
sources are retained as provenance only, never qualified by this Linux run.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import tarfile
|
|
import time
|
|
import uuid
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
STATE=ROOT/'devstate/linux-updater-qualification'
|
|
BUILDER='sha256:9a76ad3744ebfdddae96ca10001ce8291c1b332fb54c8df5eea0e8e999ac6fc6'
|
|
PACKAGES=['rmm-agent','rmm-agent-core','rmm-domain','rmm-privileged-helper','rmm-updater-linux','rmm-updater-worker','rmm-updater-handoff','rmm-update-history']
|
|
UNIT_TARGETS=[('agent','rmm-agent',['--bin','rmm-agent']),('helper','rmm-privileged-helper',['--lib']),
|
|
('core','rmm-agent-core',['--lib']),('helper-reconcile','rmm-agent-core',['--test','helper_reconcile']),
|
|
('domain','rmm-domain',['--lib']),('updater-linux','rmm-updater-linux',['--lib']),
|
|
('worker','rmm-updater-worker',['--lib']),('handoff','rmm-updater-handoff',['--test','handoff_contract'])]
|
|
SAFE_FILTERS={
|
|
'updater-linux':['services::status_tests::accepts_actual_seven_property_shape_and_denies_missing_or_extra_properties'],
|
|
'helper':['update_admission::tests::typed_handoff_ack_is_not_an_outcome_or_a_new_start_lease'],
|
|
'core':['update_health_sync::tests::same_floor_does_not_admit_a_one_nanosecond_early_or_late_apply'],
|
|
'helper-reconcile':['update_handoff_consumption_start_and_stop_intent_are_one_transaction',
|
|
'update_inflight_negative_control_keeps_every_start_closed_and_owner_loss_is_terminal',
|
|
'native_stop_intent_is_original_head_bound_create_once_and_transactional',
|
|
'restarted_update_can_only_schedule_control_from_its_verified_original_not_unknown'],
|
|
}
|
|
|
|
def require(ok,code):
|
|
if not ok:raise RuntimeError(code)
|
|
|
|
def sha(raw):return hashlib.sha256(raw).hexdigest()
|
|
def encode(value):return json.dumps(value,sort_keys=True,indent=2).encode()+b'\n'
|
|
def write(path,raw):
|
|
path=Path(path);path.parent.mkdir(mode=0o700,parents=True,exist_ok=True)
|
|
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
|
with os.fdopen(fd,'wb') as stream:stream.write(raw);stream.flush();os.fsync(stream.fileno())
|
|
|
|
def public_files():
|
|
names=subprocess.run(['git','ls-files','--cached','--others','--exclude-standard','-z'],cwd=ROOT,capture_output=True,check=True).stdout.split(b'\0')
|
|
prefixes=('crates/','proto/','migrations/','deploy/','tools/','tests/')
|
|
fixed={'Cargo.toml','Cargo.lock','rust-toolchain.toml','Rust_RMM_Implementierungsspezifikation.md','AGENTS.md'}
|
|
result=[]
|
|
for raw in names:
|
|
if not raw:continue
|
|
name=os.fsdecode(raw);path=Path(name)
|
|
if name not in fixed and not name.startswith(prefixes):continue
|
|
require(not path.is_absolute() and '..' not in path.parts,'source relative path')
|
|
if '__pycache__' in path.parts or path.suffix in ('.pyc','.pyo') or path.name=='.env':continue
|
|
full=ROOT/path
|
|
require(not full.is_symlink() and full.is_file(),'public regular source only')
|
|
meta=full.stat();require(meta.st_size<=16*1024*1024,'bounded compiler/public fixture')
|
|
result.append(name)
|
|
require({'Cargo.toml','Cargo.lock','crates/agent/src/main.rs','crates/privileged-helper/src/main.rs','crates/updater-worker/src/main.rs'}<=set(result),'complete Linux entrypoints')
|
|
return sorted(set(result))
|
|
|
|
def capture(root,names):
|
|
result={}
|
|
for name in names:
|
|
file=root/name;before=file.stat();raw=file.read_bytes();after=file.stat()
|
|
require((before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns)==
|
|
(after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns),'source changed while reading')
|
|
result[name]=dict(bytes=len(raw),sha256=sha(raw))
|
|
return result
|
|
|
|
def readonly(root):
|
|
for path in root.rglob('*'):
|
|
path.chmod(0o555 if path.is_dir() else 0o444)
|
|
root.chmod(0o555)
|
|
|
|
def freeze(folder):
|
|
names=public_files();before=capture(ROOT,names);source0=folder/'source0'
|
|
source0.mkdir(mode=0o700)
|
|
for name in names:write(source0/name,(ROOT/name).read_bytes())
|
|
require(capture(source0,names)==before and public_files()==names and capture(ROOT,names)==before,'whole source input snapshot drift; preserve failed case, do not build')
|
|
require(re.search(r'(?ms)^\[workspace\.package\]\s*version = "0\.1\.0"$',(source0/'Cargo.toml').read_text()),'exact baseline version0.1.0')
|
|
readonly(source0);write(folder/'source0-inputs.json',encode(before))
|
|
source1=folder/'source1';shutil.copytree(source0,source1)
|
|
cargo=source1/'Cargo.toml';lock=source1/'Cargo.lock';cargo.chmod(0o600);lock.chmod(0o600)
|
|
original=cargo.read_text();require(original.count('version = "0.1.0"')==1,'one workspace version field')
|
|
cargo.write_text(original.replace('version = "0.1.0"','version = "0.1.1"',1))
|
|
match=re.search(r'(?ms)^members\s*=\s*(\[.*?\])',original)
|
|
require(match is not None,'literal workspace member list')
|
|
members=json.loads(match[1]);internal=[]
|
|
require(all(isinstance(member,str) and member.startswith('crates/') and '..' not in Path(member).parts for member in members),'closed workspace member paths')
|
|
for member in members:
|
|
package=(source1/member/'Cargo.toml').read_text()
|
|
if re.search(r'(?m)^version\.workspace\s*=\s*true$',package):
|
|
name=re.search(r'(?m)^name\s*=\s*"([a-z0-9-]+)"$',package)
|
|
require(name is not None,'literal inherited package name');internal.append(name[1])
|
|
text=lock.read_text()
|
|
for name in internal:
|
|
pattern='name = "'+name+'"\nversion = "0.1.0"'
|
|
require(text.count(pattern)==1,'one inherited internal lock package')
|
|
text=text.replace(pattern,'name = "'+name+'"\nversion = "0.1.1"',1)
|
|
lock.write_text(text)
|
|
# External crate packages and the excluded independent oidc-auth version
|
|
# remain exact; only inherited workspace package versions change.
|
|
inverse=text
|
|
for name in internal:inverse=inverse.replace('name = "'+name+'"\nversion = "0.1.1"','name = "'+name+'"\nversion = "0.1.0"',1)
|
|
require(inverse==(source0/'Cargo.lock').read_text(),'no change outside exact inherited internal version lines')
|
|
cargo.chmod(0o444);lock.chmod(0o444);readonly(source1)
|
|
after=capture(source1,names)
|
|
require({name for name in names if before[name]!=after[name]}=={'Cargo.toml','Cargo.lock'},'successor only differs in version/lock metadata')
|
|
write(folder/'source1-inputs.json',encode(after))
|
|
return names,before,after,internal
|
|
|
|
def run(folder,source,cache,label,args):
|
|
logfile=folder/'logs'/(label+'.log');logfile.parent.mkdir(mode=0o700,exist_ok=True)
|
|
command=['docker','run','--rm','--network','none','--read-only','--tmpfs','/tmp:rw,size=512m',
|
|
'--tmpfs','/run:rw,size=128m','--tmpfs','/var/lib:rw,size=128m',
|
|
'-v',str(source)+':/src:ro','-v',str(cache)+':/target','-e','CARGO_TARGET_DIR=/target',
|
|
'-w','/src','--entrypoint','/bin/sh',BUILDER,'-c','mkdir -p /tmp/opencode\nexec '+__import__('shlex').join(args)]
|
|
write(folder/'logs'/(label+'-command.json'),encode(command))
|
|
started=time.time();print(json.dumps(dict(stage=label,started=True)),flush=True)
|
|
with logfile.open('xb') as stream:
|
|
result=subprocess.run(command,stdout=stream,stderr=subprocess.STDOUT,timeout=1800)
|
|
logfile.chmod(0o600)
|
|
result_record=dict(command=command,exit_code=result.returncode,elapsed_seconds=round(time.time()-started,3),log_sha256=sha(logfile.read_bytes()))
|
|
write(folder/'logs'/(label+'-result.json'),encode(result_record))
|
|
require(result.returncode==0,'build/check failed: '+label+'; original log preserved')
|
|
return logfile.read_bytes()
|
|
|
|
def copy_artifact(path,target):
|
|
raw=path.read_bytes();require(raw[:6]==b'\x7fELF\x02\x01' and raw[18:20]==b'\x3e\x00','native Linux x86-64 ELF')
|
|
write(target,raw);target.chmod(0o500)
|
|
return dict(path=str(target),sha256=sha(raw),bytes=len(raw),platform='x86_64-unknown-linux-gnu')
|
|
|
|
def cargo_packages():
|
|
return [part for package in PACKAGES for part in ('-p',package)]
|
|
|
|
def build():
|
|
os.umask(0o077);ident=str(uuid.uuid4());folder=STATE/ident;folder.mkdir(mode=0o700,parents=True)
|
|
write(folder/'entered.json',encode(dict(schema_version=1,case_id=ident,root=str(ROOT),builder=BUILDER,
|
|
git_head=subprocess.run(['git','rev-parse','HEAD'],cwd=ROOT,capture_output=True,check=True,text=True).stdout.strip(),
|
|
no_target_mutations=True,windows_not_qualified=True)))
|
|
names,input0,input1,internal=freeze(folder)
|
|
cache=folder/'cache';cache.mkdir(mode=0o700)
|
|
results={};units={}
|
|
for version,inputs in (('source0',input0),('source1',input1)):
|
|
source=folder/version
|
|
run(folder,source,cache,version+'-metadata',['cargo','metadata','--locked','--offline','--no-deps','--format-version=1'])
|
|
run(folder,source,cache,version+'-clippy',['cargo','clippy','--locked','--offline',*cargo_packages(),'--all-targets','--','-D','warnings'])
|
|
packages=['-p','rmm-agent','-p','rmm-privileged-helper']+(['-p','rmm-updater-worker'] if version=='source0' else [])
|
|
run(folder,source,cache,version+'-release',['cargo','build','--locked','--offline','--release',*packages,'--bins'])
|
|
output=folder/'artifacts'/version;output.mkdir(mode=0o700,parents=True)
|
|
results[version]={}
|
|
for label,binary in [('agent','rmm-agent'),('helper','rmm-privileged-helper')]+([('updater','rmm-agent-updater')] if version=='source0' else []):
|
|
results[version][label]=copy_artifact(cache/'release'/binary,output/binary)
|
|
require(capture(source,names)==inputs,'frozen source changed during build')
|
|
if version=='source1':continue
|
|
for label,package,selector in UNIT_TARGETS:
|
|
raw=run(folder,source,cache,'units-'+label,['cargo','test','--locked','--offline','--no-run','--message-format=json','-p',package,*selector])
|
|
found=[]
|
|
for line in raw.splitlines():
|
|
try:record=json.loads(line)
|
|
except ValueError:continue
|
|
if record.get('reason')=='compiler-artifact' and record.get('profile',{}).get('test') and record.get('executable'):
|
|
found.append(record['executable'])
|
|
require(len(found)==1,'one linked unit executable '+label)
|
|
relative=Path(found[0]).relative_to('/target')
|
|
artifact=copy_artifact(cache/relative,output/(label+'-tests'))
|
|
listing=run(folder,source,cache,'list-'+label,[str(Path('/target')/relative),'--list','--format','terse']).decode()
|
|
tests=[line[:-6] for line in listing.splitlines() if line.endswith(': test')]
|
|
require(tests and len(set(tests))==len(tests),'complete unique native test names')
|
|
allowed=SAFE_FILTERS.get(label,[])
|
|
if label=='handoff':allowed=tests
|
|
require(set(allowed)<=set(tests),'exact safe filter exists')
|
|
artifact.update(all_tests=tests,allowed_native_tests=allowed,
|
|
other_tests_not_authorized_by_this_pack=True,
|
|
safety='Pure handoff/codec or own temporary portable SQLite fixture only; no product service/database/keys.' if allowed else 'Linked only; no VM execution authorized.')
|
|
units[label]=artifact
|
|
for index,name in enumerate(allowed):
|
|
run(folder,source,cache,'focused-'+label+'-'+str(index),[str(Path('/target')/relative),'--exact',name,'--nocapture'])
|
|
require(capture(source,names)==inputs,'frozen source changed during unit qualification')
|
|
require(results['source0']['agent']['sha256']!=results['source1']['agent']['sha256']
|
|
and results['source0']['helper']['sha256']!=results['source1']['helper']['sha256'],'real new codepair bytes required')
|
|
manifest=dict(schema_version=1,case_id=ident,profile='linux_compatible_updater_buildpack_v1',
|
|
builder=BUILDER,network='none',source_mount='read_only',cache=str(cache),
|
|
source0_version='0.1.0',source1_version='0.1.1',changed_source1_files=['Cargo.toml','Cargo.lock'],
|
|
inherited_packages=internal,external_dependencies_unchanged=True,
|
|
source0_inputs_sha256=sha((folder/'source0-inputs.json').read_bytes()),
|
|
source1_inputs_sha256=sha((folder/'source1-inputs.json').read_bytes()),
|
|
source_files_count=len(names),artifacts=results,units=units,
|
|
stable_updater=results['source0']['updater'],checks=dict(linux_source0_clippy=True,linux_source1_clippy=True,
|
|
release_links=True,exact_test_lists=True,focused_local_contracts=True,frozen_inputs_after_build=True),
|
|
windows_copied_only_not_qualified=True,native_vm_tested=False,
|
|
bootstrap_installed=False,apply_executed=False,keys_generated=False,approvals_performed=False)
|
|
write(folder/'manifest.json',encode(manifest))
|
|
for version in ('source0','source1'):
|
|
archive=folder/(version+'.tar.gz')
|
|
with tarfile.open(archive,'w:gz') as output:
|
|
for name in names:output.add(folder/version/name,arcname=name,recursive=False)
|
|
archive.chmod(0o400)
|
|
write(folder/'archive-hashes.json',encode({version:sha((folder/(version+'.tar.gz')).read_bytes()) for version in ('source0','source1')}))
|
|
write(folder/'completed.json',encode(dict(manifest_sha256=sha((folder/'manifest.json').read_bytes()),native_vm_tested=False)))
|
|
print(json.dumps(dict(completed=True,manifest=str(folder/'manifest.json'),manifest_sha256=sha((folder/'manifest.json').read_bytes()),artifacts=results)),flush=True)
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=['build'])
|
|
parser.parse_args();build()
|
|
if __name__=='__main__':main()
|