Qualify tenant-scoped bootstrap rotation and fresh Windows scripting

This commit is contained in:
2026-11-17 16:19:08 +01:00
parent 81fcd9c8f8
commit fbd24b6d75
26 changed files with 877 additions and 118 deletions
+90 -3
View File
@@ -2,6 +2,7 @@
"""Prepare an Ed25519 agent release on the signing host and publish through the catalog API."""
import argparse
import base64
import fcntl
import getpass
import hashlib
import http.client
@@ -50,6 +51,22 @@ def fingerprint(path):
return digest.hexdigest(), size
def verify_native_image(path, target, size):
with path.open('rb') as binary:
header = binary.read(64)
check(len(header) == 64, 'truncated binary header: ' + path.name)
if target == LINUX:
check(header[:6] == b'\x7fELF\x02\x01' and header[16:18] in (b'\x02\x00', b'\x03\x00')
and header[18:20] == b'\x3e\x00', 'Linux x86-64 ELF image required: ' + path.name)
else:
pe = int.from_bytes(header[60:64], 'little')
check(header[:2] == b'MZ' and 64 <= pe <= size - 26, 'Windows PE image required: ' + path.name)
binary.seek(pe)
coff = binary.read(26)
check(coff[:6] == b'PE\0\0\x64\x86' and coff[24:26] == b'\x0b\x02',
'Windows x86-64 PE32+ image required: ' + path.name)
def binaries(folder, receipt):
record = json.loads(receipt.read_text(encoding='utf-8-sig'))
if 'target' in record and 'sha256' in record:
@@ -65,6 +82,7 @@ def binaries(folder, receipt):
for name in FILES[target]:
path = folder / name
digest, size = fingerprint(path)
verify_native_image(path, target, size)
check(digest == hashes[name], 'file differs from build receipt: ' + name)
result.append({'artifact_id': str(uuid.uuid5(uuid.NAMESPACE_OID, target + '/' + name + '/' + digest)),
'sha256': digest, 'bytes': size})
@@ -108,11 +126,22 @@ def https(value):
def prepare(args):
provision_folder(args.state)
provision_folder(args.state/'private')
with (args.state/'private'/'signing.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
prepare_locked(args)
def prepare_locked(args):
https(args.gateway)
check(0 < args.sequence < 2**63 and re.fullmatch('[A-Za-z0-9.+_-]{1,64}', args.version), 'invalid release version/sequence')
minimum = getattr(args, 'minimum_updater_version', 1)
check(minimum in (1,2), 'unknown minimum updater version')
check(0 < args.days <= 3650, 'invalid validity in days')
platform, blobs = binaries(args.artifacts, args.build_receipt)
label = {LINUX:'linux', WINDOWS:'windows', WINDOWS_GNU:'windows-gnu'}[platform]
check(minimum == 1 or platform == WINDOWS_GNU, 'version 2 currently applies to the GNU Windows worker')
provision_folder(args.state)
config, private = args.state / 'config', args.state / 'private'
provision_folder(config, args.service_uid)
@@ -146,13 +175,15 @@ def prepare(args):
prepared = args.state / 'prepared' / label / str(args.sequence)
provision_folder(prepared.parent)
if prepared.exists():
check(json.loads((prepared / 'inputs.json').read_bytes()) == {'version': args.version, 'blobs': blobs}, 'sequence already bound')
prior = json.loads((prepared / 'inputs.json').read_bytes())
check(prior.get('version') == args.version and prior.get('blobs') == blobs
and prior.get('minimum_updater_version', 1) == minimum, 'sequence already bound')
print(prepared / 'envelope.json')
return
now = int(time.time())
manifest = {'schema_version': 1, 'release_id': str(uuid.uuid4()), 'release_sequence': args.sequence,
'version': args.version, 'platform': platform, 'published_at': now, 'expires_at': now + args.days * 86400,
'protocol_major': 1, 'state_schema': 1, 'minimum_updater_version': 1,
'protocol_major': 1, 'state_schema': 1, 'minimum_updater_version': minimum,
'artifacts': [dict(role=role, **blob) for role, blob in zip(('agent', 'helper'), blobs[:2])]}
raw = encoded(manifest)
with tempfile.TemporaryDirectory(dir=prepared.parent) as tmpdir:
@@ -168,7 +199,7 @@ def prepare(args):
output.mkdir()
env = {'schema_version': 1, 'key_id': hashlib.sha256(pub).hexdigest(), 'payload_bytes': b64(raw), 'signature': b64(sig)}
save_once(output / 'envelope.json', encoded(env))
save_once(output / 'inputs.json', encoded({'version': args.version, 'blobs': blobs}))
save_once(output / 'inputs.json', encoded({'version': args.version, 'blobs': blobs, 'minimum_updater_version': minimum}))
output.rename(prepared)
print(prepared / 'envelope.json')
@@ -197,12 +228,59 @@ class Publisher:
conn.close()
def rotate_pin(state, label, tenant, expected, new_pin):
tenant = str(uuid.UUID(tenant))
check(re.fullmatch('[0-9a-f]{64}', expected), 'expected predecessor SHA-256 required')
directory = state/'config'
default = directory/(label + '-bootstrap.json')
path = default.with_suffix('.' + tenant + '.json')
# The already completed upload attests the API read and stored the object.
# Independently reopen the actual tenant object before changing the pin.
sha, size = fingerprint(state/'objects'/tenant/new_pin['sha256'])
check(new_pin == {'sha256':sha,'bytes':size}, 'uploaded stable updater differs')
with (state/'private'/'bootstrap-rotation.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
if path.exists():
pins = json.loads(path.read_bytes())
else:
baseline = json.loads(default.read_bytes())
pins = {'current':{'sha256':baseline['updater_sha256'],'bytes':baseline['updater_bytes']},'previous':[]}
now = int(time.time())
if pins['current'] == new_pin:
prior = pins['previous'][-1]['pin']['sha256'] if pins['previous'] else None
check(expected in (prior, new_pin['sha256']), 'bootstrap predecessor changed')
return
check(pins['current']['sha256'] == expected, 'bootstrap predecessor changed')
previous = [item for item in pins['previous'] if item['expires_at'] > now
and item['pin'] not in (new_pin,pins['current'])]
previous.append({'pin':pins['current'],'expires_at':now+900})
raw = encoded({'current':new_pin,'previous':previous})
check(len(raw) <= 32768, 'too many concurrent bootstrap rotations')
with tempfile.NamedTemporaryFile(dir=directory,delete=False) as f:
try:
f.write(raw)
os.fchmod(f.fileno(),0o600)
os.fchown(f.fileno(),default.stat().st_uid,default.stat().st_gid)
f.flush()
os.fsync(f.fileno())
os.replace(f.name,path)
finally:
Path(f.name).unlink(missing_ok=True)
fd = os.open(directory,os.O_DIRECTORY)
try:
os.fsync(fd)
finally:
os.close(fd)
def publish(args):
platform, blobs = binaries(args.artifacts, args.build_receipt)
env = json.loads(args.envelope.read_bytes())
manifest = json.loads(base64.urlsafe_b64decode(env['payload_bytes'] + '==='))
check(manifest['platform'] == platform and manifest['artifacts'] == [dict(role=role, **blob)
for role, blob in zip(('agent', 'helper'), blobs[:2])], 'input files differ from release envelope')
check(bool(args.rotate_bootstrap_from) == bool(args.bootstrap_state), 'rotation needs previous SHA and local signing state')
check(not (args.rotate_bootstrap_from and args.bootstrap_worker), 'a rotation uses the new receipt-bound stable worker')
client = Publisher(args.origin, args.ca)
if args.dev_login_file:
path = args.dev_login_file
@@ -222,6 +300,7 @@ def publish(args):
if args.bootstrap_worker:
paths[2] = args.bootstrap_worker
sha, size = fingerprint(paths[2])
verify_native_image(paths[2], platform, size)
name = FILES[platform][2]
blobs[2] = {'artifact_id': str(uuid.uuid5(uuid.NAMESPACE_OID, platform + '/' + name + '/' + sha)),
'sha256': sha, 'bytes': size}
@@ -231,6 +310,11 @@ def publish(args):
result = client.request('POST', path + '/agent-update-artifacts?' + query, data,
{'Content-Type': 'application/octet-stream', 'Content-Length': str(blob['bytes'])})
check(result['sha256'] == blob['sha256'] and result['size_bytes'] == blob['bytes'], 'artifact receipt differs')
if args.rotate_bootstrap_from:
stable = blobs[2]
label = {LINUX:'linux', WINDOWS:'windows', WINDOWS_GNU:'windows-gnu'}[platform]
rotate_pin(args.bootstrap_state,label,tenant,args.rotate_bootstrap_from,
{'sha256':stable['sha256'],'bytes':stable['bytes']})
result = client.request('POST', path + '/agent-releases', encoded(env), {'Content-Type': 'application/json'})
check(result['release_envelope'] == env, 'published envelope changed')
print(json.dumps({'release_id': result['release_id'], 'version': manifest['version'], 'platform': platform,
@@ -253,9 +337,12 @@ def main():
cmd.add_argument('--sequence', type=int, required=True)
cmd.add_argument('--version', required=True)
cmd.add_argument('--days', type=int, default=90)
cmd.add_argument('--minimum-updater-version',type=int,default=1)
cmd.add_argument('--service-uid', type=int, default=os.getuid())
else:
cmd.add_argument('--origin', required=True)
cmd.add_argument('--bootstrap-state',type=Path,help='Local signing host state for an explicit stable updater rotation')
cmd.add_argument('--rotate-bootstrap-from',help='Compare-and-swap the tenant pin from this SHA-256 after verified upload')
cmd.add_argument('--bootstrap-worker', type=Path, help='Pinned initial stable updater when publishing subsequent code pairs')
cmd.add_argument('--ca', type=Path)
cmd.add_argument('--tenant', required=True)
+16 -3
View File
@@ -20,6 +20,7 @@ def on_server(options):
from pathlib import Path
import shutil
import subprocess
import uuid
import tempfile
os.umask(0o077)
@@ -49,7 +50,7 @@ def on_server(options):
raise ValueError('sequence already contains different build: ' + name)
inputs = ['--artifacts', str(build), '--build-receipt', str(build / options['receipt'])]
subprocess.run(['python3', '-I', str(tool), 'prepare', *inputs, '--state', str(state), '--gateway', options['gateway'],
'--service-uid','10001','--sequence',str(options['sequence']),'--version',options['version']],check=True)
'--service-uid','10001','--minimum-updater-version',str(options['minimum']),'--sequence',str(options['sequence']),'--version',options['version']],check=True)
if options['label'].startswith('windows'):
import re
import uuid
@@ -91,10 +92,20 @@ def on_server(options):
with tempfile.NamedTemporaryFile(mode='w', dir=state) as auth:
json.dump({'username':env['RMM_DEV_USERNAME'],'password':env['RMM_DEV_PASSWORD']},auth)
auth.flush()
label=options['label']
tenant=str(uuid.UUID(options['tenant']))
pin=(state/'config'/(label+'-bootstrap.json')).with_suffix('.'+tenant+'.json')
if options['rotate']:
selection=['--rotate-bootstrap-from',options['rotate'],'--bootstrap-state',str(state)]
elif pin.is_file():
sha=json.loads(pin.read_bytes())['current']['sha256']
selection=['--bootstrap-worker',str(state/'objects'/tenant/sha)]
else:
selection=['--bootstrap-worker',str(state/'bootstrap-workers'/label)]
subprocess.run(['python3','-I',str(tool),'publish',*inputs,'--tenant',options['tenant'],
'--envelope',str(state/'prepared'/options['label']/str(options['sequence'])/'envelope.json'),
'--origin','https://localhost:18443','--ca','/opt/rust-discovery/devstate/api-pki/ca.pem',
'--dev-login-file',auth.name,'--bootstrap-worker',str(state/'bootstrap-workers'/options['label'])],check=True)
'--dev-login-file',auth.name,*selection],check=True)
def main():
import inspect
@@ -103,6 +114,8 @@ def main():
parser.add_argument('--build-receipt',type=Path,required=True)
parser.add_argument('--sequence',type=int,required=True)
parser.add_argument('--version',required=True)
parser.add_argument('--minimum-updater-version',type=int,default=1)
parser.add_argument('--rotate-bootstrap-from',help='Rotate only this tenant from the given previous worker SHA-256')
parser.add_argument('--gateway',default='https://discovery.ease-technologies.de')
parser.add_argument('--tenant',default='11111111-1111-4111-8111-111111111111')
parser.add_argument('--prepare-only',action='store_true')
@@ -114,7 +127,7 @@ def main():
raise ValueError('build receipt and binary names must differ')
params={'files':{f.name:hashlib.sha256(f.read_bytes()).hexdigest() for f in files},'tool':hashlib.sha256(tool.read_bytes()).hexdigest(),
'receipt':args.build_receipt.name,'sequence':args.sequence,'version':args.version,'tenant':args.tenant,
'gateway':args.gateway,'prepare_only':args.prepare_only,'label':{agent_release.LINUX:'linux',
'minimum':args.minimum_updater_version,'rotate':args.rotate_bootstrap_from,'gateway':args.gateway,'prepare_only':args.prepare_only,'label':{agent_release.LINUX:'linux',
agent_release.WINDOWS:'windows',agent_release.WINDOWS_GNU:'windows-gnu'}[platform]}
subprocess.run([sys.executable,str(ROOT/'tools/dev_stage.py'),'server',*map(str,files),str(tool)],check=True)
command=inspect.getsource(on_server)+'\non_server('+repr(params)+')\n'
+59 -26
View File
@@ -7,7 +7,6 @@ import json
from pathlib import Path
import re
import subprocess
import sys
def sha(data):
@@ -29,6 +28,39 @@ def write_atomic(path, data, mode=0o644):
tmp.unlink(missing_ok=True)
def pack_web(dist):
if not (dist / 'index.html').is_file():
raise ValueError('A built WebUI directory with index.html is required')
files = {}
for file in dist.rglob('*'):
if file.is_symlink():
raise ValueError('Web build contains a symlink')
if file.is_file():
files[file.relative_to(dist).as_posix()] = base64.b64encode(file.read_bytes()).decode()
return files
def stage_web(dist, files):
if 'index.html' not in files:
raise ValueError('Web build has no index.html')
for name, encoded in files.items():
path = dist / name
if Path(name).is_absolute() or '..' in Path(name).parts or path.resolve() != path:
raise ValueError('Invalid WebUI asset path')
if name != 'index.html':
write_atomic(path, base64.b64decode(encoded, validate=True))
return base64.b64decode(files['index.html'], validate=True)
def publish_index(root, dist, index):
write_atomic(dist / 'index.html', index)
actual = subprocess.check_output(['curl','--noproxy','*','--max-time','10','--cacert',
str(root/'devstate/api-pki/ca.pem'),'--silent','--fail','https://localhost:18443/'])
if actual != index:
raise RuntimeError('Served index differs')
return sha(actual)
def web_files(revision):
"""Build the exact Git revision using the local locked frontend dependencies."""
import tempfile
@@ -40,16 +72,14 @@ def web_files(revision):
subprocess.run(['tar','-xf',archive,'-C',work],check=True)
(work/'web/node_modules').symlink_to(root/'web/node_modules',target_is_directory=True)
subprocess.run(['npm','run','build'],cwd=work/'web',check=True)
dist = work/'web/dist'
return {str(file.relative_to(dist)):base64.b64encode(file.read_bytes()).decode()
for file in dist.rglob('*') if file.is_file()}
return pack_web(work / 'web/dist')
def activate(payload):
import fcntl
import time
revision = payload['revision']
if not re.fullmatch('[0-9a-f]{40}', revision):
if not payload['web_only'] and not re.fullmatch('[0-9a-f]{40}', revision):
raise ValueError('Invalid Git revision')
def run(*command):
return subprocess.check_output(list(command),text=True).strip()
@@ -58,10 +88,19 @@ def activate(payload):
root = Path('/opt/rust-discovery')
work = root/'devstate/mvp-deploy'
work.mkdir(exist_ok=True,mode=0o700)
role_dir = work/revision
role_dir.mkdir(exist_ok=True,mode=0o700)
with (work/'deployment.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
active = inspect('api')
dist = next(Path(m['Source']) for m in active['Mounts'] if m['Destination'] == '/web')
if dist != root / 'web/dist' or active['Config']['Labels']['com.docker.compose.project'] != 'rust-discovery-server':
raise RuntimeError('Unexpected development WebUI deployment')
index = stage_web(dist, payload['web'])
if payload['web_only']:
print(json.dumps(dict(revision=revision, web_index_sha256=publish_index(root, dist, index),
api_container=active['Id'], web_only=True)))
return
role_dir = work / revision
role_dir.mkdir(exist_ok=True, mode=0o700)
matches = list((Path.home()/'rmm-dev-build/server-arm64/artifacts'/revision).glob('*/manifest.json'))
if len(matches) != 1:
raise RuntimeError('Expected one ARM64 build manifest for this revision')
@@ -70,7 +109,6 @@ def activate(payload):
if (manifest['revision'] != revision or manifest['target'] != 'aarch64-unknown-linux-gnu'
or sha(binary) != manifest['sha256']['rmm-server'] or sha(matches[0].read_bytes()) != matches[0].parent.name):
raise RuntimeError('Server build digest mismatch')
active = inspect('api')
if inspect('gateway')['Image'] != active['Image']:
raise RuntimeError('API and Gateway have different base images')
labels = active['Config']['Labels']
@@ -111,13 +149,6 @@ def activate(payload):
raise RuntimeError('Dev admin/gateway boundary changed')
if config['postgres'].get('ports'):
raise RuntimeError('Unexpected public database')
dist=next(Path(m['Source']) for m in active['Mounts'] if m['Destination']=='/web')
assets={name:base64.b64decode(data,validate=True) for name,data in payload['web'].items()}
if 'index.html' not in assets or any(Path(n).is_absolute() or '..' in Path(n).parts for n in assets):
raise ValueError('Invalid WebUI asset set')
for name,data in assets.items():
if name!='index.html':
write_atomic(dist/name,data)
run(*compose,'run','--no-deps','-T','--rm','migrate')
sql = ['docker','exec','rust-discovery-server-postgres-1','psql','-X','-qAt','-v','ON_ERROR_STOP=1','-U','postgres','-d','rmm','-c']
recorded = dict(line.split('|') for line in run(*sql,"SELECT version,encode(checksum,'hex') FROM _sqlx_migrations WHERE success").splitlines())
@@ -136,12 +167,7 @@ def activate(payload):
live=run('docker','exec','rust-discovery-server-'+role+'-1','sha256sum','/usr/local/bin/rmm-server').split()[0]
if live!=sha(binary):
raise RuntimeError(role+' running digest differs')
write_atomic(dist/'index.html',assets['index.html'])
index=subprocess.check_output(['curl','--noproxy','*','--cacert',str(root/'devstate/api-pki/ca.pem'),
'--silent','--fail','https://localhost:18443/'])
if index!=assets['index.html']:
raise RuntimeError('Served index differs')
print(json.dumps(dict(revision=revision,server_sha256=sha(binary),web_index_sha256=sha(index),
print(json.dumps(dict(revision=revision,server_sha256=sha(binary),web_index_sha256=publish_index(root, dist, index),
image=inspect('api')['Image'],api_health='ok',gateway_health='ok')))
@@ -150,13 +176,20 @@ def main():
import inspect
import shlex
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--ref',required=True)
parser.add_argument('--ref', help='Exact Git revision for the server and WebUI')
parser.add_argument('--web-only', action='store_true', help='Only activate built web assets')
parser.add_argument('--web-dist', type=Path, help='Use an existing WebUI build with --web-only')
parser.add_argument('--overlay',action='append',default=[],help='Deploy and use this compose overlay from the selected Git revision')
args=parser.parse_args()
revision=subprocess.check_output(['git','rev-parse','--verify','--end-of-options',args.ref+'^{commit}'],text=True).strip()
web=web_files(revision)
if args.web_dist and (not args.web_only or args.ref) or args.web_only and args.overlay:
parser.error('Choose --web-only with either --web-dist or --ref and no compose overlay')
if not args.web_dist and not args.ref:
parser.error('--ref or --web-only --web-dist is required')
revision = None if args.web_dist else subprocess.check_output(
['git','rev-parse','--verify','--end-of-options',args.ref+'^{commit}'],text=True).strip()
web = pack_web(args.web_dist.resolve(strict=True)) if args.web_dist else web_files(revision)
source=Path(inspect.getfile(activate)).read_text().rsplit('\nif __name__ ==', 1)[0]
names=subprocess.check_output(['git','ls-tree','--name-only','-r',revision,'migrations'],text=True).splitlines()
names=subprocess.check_output(['git','ls-tree','--name-only','-r',revision,'migrations'],text=True).splitlines() if not args.web_only else []
migrations={str(int(Path(n).name.split('_',1)[0])):hashlib.sha384(subprocess.check_output(['git','show',revision+':'+n])).hexdigest()
for n in names if n.endswith('.sql')}
overlays = {}
@@ -165,7 +198,7 @@ def main():
parser.error('Expected a server overlay filename from deploy/')
raw = subprocess.check_output(['git', 'show', revision + ':deploy/' + name])
overlays[name] = base64.b64encode(raw).decode()
payload=json.dumps(dict(revision=revision,web=web,overlays=overlays,migrations=migrations))
payload=json.dumps(dict(revision=revision,web=web,web_only=args.web_only,overlays=overlays,migrations=migrations))
cmd=source+'\nactivate(json.loads('+repr(payload)+'))\n'
subprocess.run(ssh_prefix('server')+[shlex.join(['python3','-I','-'])],input=cmd,text=True,check=True)