Integrate signed agent catalog, tenant enrollment and HTTPS installers

This commit is contained in:
2026-11-17 15:56:18 +01:00
parent 689f326460
commit 7fd17272d5
70 changed files with 12982 additions and 327 deletions
+257
View File
@@ -0,0 +1,257 @@
#!/usr/bin/env python3
"""Prepare an Ed25519 agent release on the signing host and publish through the catalog API."""
import argparse
import base64
import datetime
import getpass
import hashlib
import http.client
from http.cookies import SimpleCookie
import json
import os
from pathlib import Path
import re
import shutil
import ssl
import subprocess
import tempfile
import time
from urllib.parse import urlencode, urlsplit
import uuid
LINUX = 'x86_64-unknown-linux-gnu'
WINDOWS = 'x86_64-pc-windows-msvc'
FILES = {LINUX: ('rmm-agent', 'rmm-privileged-helper', 'rmm-agent-updater'),
WINDOWS: ('rmm-agent-windows.exe', 'rmm-helper-windows.exe', 'rmm-updater-windows.exe')}
WINDOWS_GNU = 'x86_64-pc-windows-gnu'
FILES[WINDOWS_GNU] = FILES[WINDOWS]
LIMIT = 256 * 1024 * 1024
PREFIX = b'rmm.agent-release.v1\0'
def check(test, message):
if not test:
raise ValueError(message)
def encoded(value):
return json.dumps(value, separators=(',', ':'), ensure_ascii=False).encode()
def b64(data):
return base64.urlsafe_b64encode(data).rstrip(b'=').decode()
def fingerprint(path):
digest = hashlib.sha256()
size = path.stat().st_size
check(path.is_file() and 0 < size <= LIMIT, 'invalid release binary size')
with path.open('rb') as f:
for block in iter(lambda: f.read(1024 * 1024), b''):
digest.update(block)
return digest.hexdigest(), size
def binaries(folder, receipt):
record = json.loads(receipt.read_text(encoding='utf-8-sig'))
if 'target' in record and 'sha256' in record:
target, hashes = record['target'], record['sha256']
else:
check(record.get('completed') is True and record['stage'] == 'built_not_deployed', 'completed MSVC receipt required')
target = WINDOWS
entries = record['artifacts']
check(all(entries[name[:-4]]['build_target'] == target for name in FILES[target]), 'incorrect build target')
hashes = {name: entries[name[:-4]]['sha256'] for name in FILES[target]}
check(target in FILES, 'unknown release target')
result = []
for name in FILES[target]:
path = folder / name
digest, size = fingerprint(path)
check(digest == hashes[name], 'file differs from build receipt: ' + name)
result.append({'artifact_id': str(uuid.uuid5(uuid.NAMESPACE_OID, target + '/' + name + '/' + digest)),
'sha256': digest, 'bytes': size})
return target, result
def openssl(*args):
return subprocess.check_output(['openssl', *map(str, args)], stderr=subprocess.DEVNULL, timeout=20)
def save_once(path, raw, uid=None):
if path.exists():
check(path.read_bytes() == raw, 'existing configuration differs: ' + path.name)
return
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as f:
try:
f.write(raw)
os.fchmod(f.fileno(), 0o600)
if uid is not None:
os.fchown(f.fileno(), uid, uid)
f.flush()
os.fsync(f.fileno())
os.link(f.name, path)
finally:
os.unlink(f.name)
def provision_folder(path, uid=None):
path.mkdir(mode=0o700, parents=True, exist_ok=True)
check(path.is_dir() and not path.is_symlink(), 'release state directory required')
if uid is not None and path.stat().st_uid != uid:
os.chown(path, uid, uid)
os.chmod(path, 0o700)
def https(value):
url = urlsplit(value)
check(url.scheme == 'https' and url.hostname and not url.username and not url.password
and url.path in ('', '/') and not url.query and not url.fragment, 'HTTPS origin required')
return url
def prepare(args):
https(args.gateway)
check(0 < args.sequence < 2**63 and re.fullmatch('[A-Za-z0-9.+_-]{1,64}', args.version), 'invalid release version/sequence')
check(0 < args.days <= 3650, 'invalid validity in days')
platform, blobs = binaries(args.artifacts, args.build_receipt)
label = {LINUX:'linux', WINDOWS:'windows', WINDOWS_GNU:'windows-gnu'}[platform]
provision_folder(args.state)
config, private = args.state / 'config', args.state / 'private'
provision_folder(config, args.service_uid)
provision_folder(args.state / 'objects', args.service_uid)
provision_folder(private)
key = private / (label + '.pem')
if not key.exists():
save_once(key, openssl('genpkey', '-algorithm', 'ED25519'))
check(not key.is_symlink() and key.stat().st_mode & 0o077 == 0, 'private release key permissions')
public = openssl('pkey', '-in', key, '-pubout', '-outform', 'DER')
check(len(public) == 44 and public[:12] == bytes.fromhex('302a300506032b6570032100'), 'Ed25519 key required')
pub = public[12:]
ring = {'schema_version': 1, 'generation': 1, 'keys': [{'public_key_hex': pub.hex(), 'platform': platform,
'first_sequence': 1, 'last_sequence': 2**63-1, 'revoked': False}]}
save_once(config / (label + '-keyring.json'), encoded(ring), args.service_uid)
boot = {'gateway_origin': args.gateway.rstrip('/'), 'updater_sha256': blobs[2]['sha256'], 'updater_bytes': blobs[2]['bytes']}
boot_path = config / (label + '-bootstrap.json')
# The stable bootstrap worker is provisioned separately from replaceable code.
if not boot_path.exists():
save_once(boot_path, encoded(boot), args.service_uid)
existing = json.loads(boot_path.read_bytes())
check(existing['gateway_origin'] == boot['gateway_origin'], 'existing gateway origin differs')
prepared = args.state / 'prepared' / label / str(args.sequence)
provision_folder(prepared.parent)
if prepared.exists():
check(json.loads((prepared / 'inputs.json').read_bytes()) == {'version': args.version, 'blobs': blobs}, 'sequence already bound')
print(prepared / 'envelope.json')
return
now = int(time.time())
manifest = {'schema_version': 1, 'release_id': str(uuid.uuid4()), 'release_sequence': args.sequence,
'version': args.version, 'platform': platform, 'published_at': now, 'expires_at': now + args.days * 86400,
'protocol_major': 1, 'state_schema': 1, 'minimum_updater_version': 1,
'artifacts': [dict(role=role, **blob) for role, blob in zip(('agent', 'helper'), blobs[:2])]}
raw = encoded(manifest)
with tempfile.TemporaryDirectory(dir=prepared.parent) as tmpdir:
work = Path(tmpdir)
msg = work / 'message'
msg.write_bytes(PREFIX + raw)
sig = openssl('pkeyutl', '-sign', '-rawin', '-inkey', key, '-in', msg)
(work / 'public.der').write_bytes(public)
(work / 'signature').write_bytes(sig)
openssl('pkeyutl', '-verify', '-rawin', '-pubin', '-keyform', 'DER', '-inkey', work / 'public.der',
'-in', msg, '-sigfile', work / 'signature')
output = work / 'release'
output.mkdir()
env = {'schema_version': 1, 'key_id': hashlib.sha256(pub).hexdigest(), 'payload_bytes': b64(raw), 'signature': b64(sig)}
save_once(output / 'envelope.json', encoded(env))
save_once(output / 'inputs.json', encoded({'version': args.version, 'blobs': blobs}))
output.rename(prepared)
print(prepared / 'envelope.json')
class Publisher:
def __init__(self, address, ca):
self.origin = address.rstrip('/')
self.url = https(self.origin)
self.ctx = ssl.create_default_context(cafile=ca)
self.headers = {'Origin': self.origin}
def request(self, method, route, body=None, headers=None):
conn = http.client.HTTPSConnection(self.url.hostname, self.url.port or 443, context=self.ctx, timeout=180)
try:
conn.request(method, route, body=body, headers={**self.headers, **(headers or {})})
reply = conn.getresponse()
data = reply.read(1048577)
check(200 <= reply.status < 300, f'{method} {route.split("?")[0]} returned HTTP {reply.status}')
check(len(data) <= 1048576, 'response exceeds limit')
if reply.getheader('Set-Cookie'):
cookie = SimpleCookie(reply.getheader('Set-Cookie'))
token = cookie['__Host-rmm_session'].value
self.headers['Cookie'] = '__Host-rmm_session=' + token
return json.loads(data) if data else None
finally:
conn.close()
def publish(args):
platform, blobs = binaries(args.artifacts, args.build_receipt)
env = json.loads(args.envelope.read_bytes())
manifest = json.loads(base64.urlsafe_b64decode(env['payload_bytes'] + '==='))
check(manifest['platform'] == platform and manifest['artifacts'] == [dict(role=role, **blob)
for role, blob in zip(('agent', 'helper'), blobs[:2])], 'input files differ from release envelope')
client = Publisher(args.origin, args.ca)
if args.dev_login_file:
path = args.dev_login_file
check(path.stat().st_mode & 0o077 == 0, 'dev login file requires private permissions')
credentials = json.loads(path.read_bytes())
login = {k: credentials[k] for k in ('username','password')}
else:
login = {'username': args.dev_username, 'password': getpass.getpass('Development WebUI password: ')}
client.request('POST', '/auth/dev/login', encoded(login), {'Content-Type': 'application/json'})
session = client.request('GET', '/auth/session')
client.headers['X-CSRF-Token'] = session['csrf_token']
try:
tenant = str(uuid.UUID(args.tenant))
check(session['tenant_id'] == tenant and 'catalog:write' in session['permissions'], 'tenant catalog write permission required')
path = '/api/v1/tenants/' + tenant
for name, blob in zip(FILES[platform], blobs):
query = urlencode(dict(artifact_id=blob['artifact_id'], sha256=blob['sha256'], size_bytes=blob['bytes']))
with (args.artifacts / name).open('rb') as data:
result = client.request('POST', path + '/agent-update-artifacts?' + query, data,
{'Content-Type': 'application/octet-stream', 'Content-Length': str(blob['bytes'])})
check(result['sha256'] == blob['sha256'] and result['size_bytes'] == blob['bytes'], 'artifact receipt differs')
result = client.request('POST', path + '/agent-releases', encoded(env), {'Content-Type': 'application/json'})
check(result['release_envelope'] == env, 'published envelope changed')
print(json.dumps({'release_id': result['release_id'], 'version': manifest['version'], 'platform': platform,
'manifest_sha256': result['manifest_sha256']}))
finally:
client.request('POST', '/auth/logout', b'{}', {'Content-Type':'application/json'})
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(required=True)
for name, fn in (('prepare', prepare), ('publish', publish)):
cmd = sub.add_parser(name)
cmd.set_defaults(run=fn)
cmd.add_argument('--artifacts', type=Path, required=True)
cmd.add_argument('--build-receipt', type=Path, required=True)
if name == 'prepare':
cmd.add_argument('--state', type=Path, required=True)
cmd.add_argument('--gateway', required=True)
cmd.add_argument('--sequence', type=int, required=True)
cmd.add_argument('--version', required=True)
cmd.add_argument('--days', type=int, default=90)
cmd.add_argument('--service-uid', type=int, default=os.getuid())
else:
cmd.add_argument('--origin', required=True)
cmd.add_argument('--ca', type=Path)
cmd.add_argument('--tenant', required=True)
cmd.add_argument('--envelope', type=Path, required=True)
auth = cmd.add_mutually_exclusive_group(required=True)
auth.add_argument('--dev-username')
auth.add_argument('--dev-login-file', type=Path)
args = parser.parse_args()
args.run(args)
if __name__ == '__main__':
main()
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Stage a build on the development server, sign there, and publish over the loopback HTTPS management API."""
import argparse
import hashlib
import json
from pathlib import Path
import shlex
import subprocess
import sys
import agent_release
from dev_stage import ssh_prefix
ROOT = Path(__file__).resolve().parents[1]
def on_server(options):
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
os.umask(0o077)
state = Path('/opt/rust-discovery/devstate/mvp-releases')
stage = Path.home() / 'rmm-dev-stage'
tool = stage / options['tool'] / 'agent_release.py'
if hashlib.sha256(tool.read_bytes()).hexdigest() != options['tool']:
raise ValueError('staged release tool differs')
build = state / 'builds' / options['label'] / str(options['sequence'])
build.mkdir(parents=True, exist_ok=True)
for name, digest in options['files'].items():
source = stage / digest / name
dest = build / name
if hashlib.sha256(source.read_bytes()).hexdigest() != digest:
raise ValueError('staged build differs: ' + name)
if not dest.exists():
with tempfile.NamedTemporaryFile(dir=build, delete=False) as tmp:
with source.open('rb') as f:
shutil.copyfileobj(f, tmp)
tmp.flush()
os.fsync(tmp.fileno())
try:
os.link(tmp.name, dest)
finally:
os.unlink(tmp.name)
if hashlib.sha256(dest.read_bytes()).hexdigest() != digest:
raise ValueError('sequence already contains different build: ' + name)
inputs = ['--artifacts', str(build), '--build-receipt', str(build / options['receipt'])]
subprocess.run(['python3', '-I', str(tool), 'prepare', *inputs, '--state', str(state), '--gateway', options['gateway'],
'--service-uid','10001','--sequence',str(options['sequence']),'--version',options['version']],check=True)
if options['prepare_only']:
return
# Use the ordinary development catalog permission and CSRF-protected API.
# Credentials are read within the dev server and never returned over SSH.
cid = subprocess.check_output(['docker','ps','-q','--filter','label=com.docker.compose.project=rust-discovery-server',
'--filter','label=com.docker.compose.service=api']).decode().strip().splitlines()
if len(cid) != 1:
raise ValueError('expected one running development API')
env = dict(x.partition('=')[::2] for x in json.loads(subprocess.check_output(['docker','inspect',cid[0]]))[0]['Config']['Env'])
if env.get('RMM_PROFILE') != 'development' or env.get('RMM_DEV_AUTH') != 'true':
raise ValueError('explicit dev API required')
with tempfile.NamedTemporaryFile(mode='w', dir=state) as auth:
json.dump({'username':env['RMM_DEV_USERNAME'],'password':env['RMM_DEV_PASSWORD']},auth)
auth.flush()
subprocess.run(['python3','-I',str(tool),'publish',*inputs,'--tenant',options['tenant'],
'--envelope',str(state/'prepared'/options['label']/str(options['sequence'])/'envelope.json'),
'--origin','https://localhost:18443','--ca','/opt/rust-discovery/devstate/api-pki/ca.pem',
'--dev-login-file',auth.name],check=True)
def main():
import inspect
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--artifacts',type=Path,required=True)
parser.add_argument('--build-receipt',type=Path,required=True)
parser.add_argument('--sequence',type=int,required=True)
parser.add_argument('--version',required=True)
parser.add_argument('--gateway',default='https://discovery.ease-technologies.de')
parser.add_argument('--tenant',default='11111111-1111-4111-8111-111111111111')
parser.add_argument('--prepare-only',action='store_true')
args=parser.parse_args()
platform, _ = agent_release.binaries(args.artifacts,args.build_receipt)
files=[*(args.artifacts/name for name in agent_release.FILES[platform]),args.build_receipt]
tool=Path(agent_release.__file__).resolve()
if args.build_receipt.name in agent_release.FILES[platform]:
raise ValueError('build receipt and binary names must differ')
params={'files':{f.name:hashlib.sha256(f.read_bytes()).hexdigest() for f in files},'tool':hashlib.sha256(tool.read_bytes()).hexdigest(),
'receipt':args.build_receipt.name,'sequence':args.sequence,'version':args.version,'tenant':args.tenant,
'gateway':args.gateway,'prepare_only':args.prepare_only,'label':{agent_release.LINUX:'linux',
agent_release.WINDOWS:'windows',agent_release.WINDOWS_GNU:'windows-gnu'}[platform]}
subprocess.run([sys.executable,str(ROOT/'tools/dev_stage.py'),'server',*map(str,files),str(tool)],check=True)
command=inspect.getsource(on_server)+'\non_server('+repr(params)+')\n'
subprocess.run([*ssh_prefix('server'),shlex.join(['python3','-I','-'])],input=command,text=True,check=True)
if __name__=='__main__':
main()