Files
rust-discovery/tests/release_bootstrap_gateway.py
T

296 lines
17 KiB
Python

"""Exercise release publication and fresh enrollment on disposable loopback TLS/PostgreSQL."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
import hashlib
import http.client
from http.cookies import SimpleCookie
import json
import os
from pathlib import Path
import secrets
import shutil
import socket
import ssl
import subprocess
import sys
import tempfile
import time
from types import SimpleNamespace
from urllib.parse import urlencode
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tools'))
import agent_release as release
import dev_customer_starters as starters
TENANT = '11111111-1111-4111-8111-111111111111'
def run(*cmd, data=None):
return subprocess.run(cmd, input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True).stdout
def test(binary, directory, stack):
db = 'rmm-bootstrap-' + uuid.uuid4().hex[:12]
password = secrets.token_hex(24)
run('docker','run','-d','--name',db,'-e','POSTGRES_PASSWORD=' + password,'-p','127.0.0.1::5432','postgres:17.6-bookworm')
stack.callback(run,'docker','rm','-f',db)
for _ in range(80):
if b'init process complete' in run('docker','logs',db) and subprocess.run(
['docker','exec',db,'pg_isready','-U','postgres'],stdout=subprocess.DEVNULL).returncode == 0:
break
time.sleep(.25)
port = run('docker','port',db,'5432/tcp').decode().strip().split(':')[-1]
def sql(role, database, script):
return run('docker','exec','-i',db,'psql','-XAtq','-v','ON_ERROR_STOP=1','-U',role,'-d',database,data=script.encode()).decode().strip()
sql('postgres','postgres', f"""CREATE ROLE rmm_migrator LOGIN NOSUPERUSER NOBYPASSRLS PASSWORD '{password}';
CREATE ROLE rmm_app LOGIN NOSUPERUSER NOBYPASSRLS PASSWORD '{password}';
CREATE DATABASE rmm OWNER rmm_migrator;""")
sql('postgres','rmm','ALTER SCHEMA public OWNER TO rmm_migrator; REVOKE CREATE ON SCHEMA public FROM PUBLIC;')
for migration in sorted((ROOT/'migrations').glob('*.sql')):
sql('rmm_migrator','rmm',migration.read_text())
assert sql('rmm_migrator','rmm',"SELECT NOT rolsuper AND NOT rolbypassrls FROM pg_roles WHERE rolname=current_user") == 't'
def permissions():
return sql('rmm_app','rmm',f"""BEGIN; SET LOCAL app.tenant_id='{TENANT}';
SELECT ('devices:enroll'=ANY(permissions))::int FROM memberships WHERE principal='simulated-testprincipal-a'; COMMIT;""")
assert permissions() == '1', 'migration 0025 must grant devices:enroll through tenant RLS'
print('Migrations and enrollment permission verified with NOSUPERUSER NOBYPASSRLS migration owner')
artifacts = directory/'bin'
artifacts.mkdir()
files = ('/bin/true','/bin/false','/usr/bin/env')
state = directory/'release'
def make_release(sequence):
for name, system_file in zip(release.FILES[release.LINUX], files):
shutil.copyfile(system_file,artifacts/name)
helper=artifacts/release.FILES[release.LINUX][1]
with helper.open('ab') as f: f.write(bytes([sequence]))
hashes={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in artifacts.iterdir()}
receipt=directory/'receipt.json'
receipt.write_text(json.dumps({'sha256':hashes,'target':release.LINUX}))
release.prepare(SimpleNamespace(artifacts=artifacts,build_receipt=receipt,service_uid=os.getuid(),state=state,
sequence=sequence,version='test.'+str(sequence),gateway='https://localhost',days=1))
env=json.loads((state/'prepared/linux'/str(sequence)/'envelope.json').read_bytes())
return env, json.loads(base64.urlsafe_b64decode(env['payload_bytes']+'==='))
a, ma = make_release(1)
tls=directory/'tls'
tls.mkdir()
run('openssl','req','-x509','-newkey','ec','-pkeyopt','ec_paramgen_curve:prime256v1','-nodes','-days','1',
'-subj','/CN=Test CA','-addext','basicConstraints=critical,CA:TRUE','-keyout',str(tls/'ca.key'),'-out',str(tls/'ca.pem'))
def csr(name):
run('openssl','req','-new','-newkey','ec','-pkeyopt','ec_paramgen_curve:prime256v1','-nodes','-subj','/CN='+name,
'-keyout',str(tls/(name+'.key')),'-out',str(tls/(name+'.csr')))
csr('server')
extensions=tls/'san'
extensions.write_text('subjectAltName=DNS:localhost\nextendedKeyUsage=serverAuth\n')
run('openssl','x509','-req','-in',str(tls/'server.csr'),'-CA',str(tls/'ca.pem'),'-CAkey',str(tls/'ca.key'),
'-CAcreateserial','-days','1','-extfile',str(extensions),'-out',str(tls/'server.pem'))
csr('device')
with socket.socket() as s:
s.bind(('127.0.0.1',0)); http_port=s.getsockname()[1]
origin='https://localhost:'+str(http_port)
env=dict(os.environ,DATABASE_URL=f'postgres://rmm_app:{password}@127.0.0.1:{port}/rmm',
RMM_PROFILE='development',RMM_DEV_AUTH='true',RMM_DEV_USERNAME='acceptance',RMM_DEV_PASSWORD=secrets.token_hex(24),
RMM_VIEWER_A=secrets.token_hex(24),RMM_VIEWER_B=secrets.token_hex(24),RMM_SERVER_ROLE='all',
RMM_LISTEN_ADDR=f'127.0.0.1:{http_port}',RMM_UI_ORIGIN=origin,RMM_TLS_DIR=str(tls),
RMM_JOB_KEY_FILE=str(directory/'job-keys/key.pem'),RMM_AGENT_UPDATE_ARTIFACT_DIR=str(state/'objects'),
RMM_AGENT_UPDATE_LINUX_KEYRING_FILE=str(state/'config/linux-keyring.json'),
RMM_AGENT_BOOTSTRAP_LINUX_FILE=str(state/'config/linux-bootstrap.json'),RMM_BOOTSTRAP_ORIGIN=origin)
log=stack.enter_context((directory/'server.log').open('w+'))
server=subprocess.Popen([binary],cwd=directory,env=env,stdout=log,stderr=log)
def stop_server():
if server.poll() is None:
server.terminate()
server.wait(timeout=15)
stack.callback(stop_server)
ctx=ssl.create_default_context(cafile=str(tls/'ca.pem'))
def ask(method,path,body=None,headers=None,status=200):
conn=http.client.HTTPSConnection('localhost',http_port,context=ctx,timeout=10)
try:
headers=dict(headers or {})
if isinstance(body,dict):
body=json.dumps(body).encode();headers['Content-Type']='application/json'
conn.request(method,path,body=body,headers=headers)
response=conn.getresponse();raw=response.read()
expected=(status,) if isinstance(status,int) else status
assert response.status in expected, (method,path.split('?')[0],response.status,expected)
return response.headers, json.loads(raw) if response.getheader('content-type','').startswith('application/json') else raw
finally:
conn.close()
for _ in range(70):
if server.poll() is not None:
log.seek(0); raise RuntimeError(log.read())
try:
ask('GET','/healthz')
break
except OSError:
time.sleep(.1)
hdr,_=ask('POST','/auth/dev/login',{'username':'acceptance','password':env['RMM_DEV_PASSWORD']},{'Origin':origin})
cookie=SimpleCookie(hdr['set-cookie'])['__Host-rmm_session'].value
auth={'Cookie':'__Host-rmm_session='+cookie,'Origin':origin}
_,session=ask('GET','/auth/session',headers=auth)
auth['X-CSRF-Token']=session['csrf_token']
path='/api/v1/tenants/'+TENANT
def upload(blob, data, status=200):
query=urlencode({'artifact_id':blob['artifact_id'],'sha256':blob['sha256'],'size_bytes':blob['bytes']})
ask('POST',path+'/agent-update-artifacts?'+query,data,{**auth,'Content-Type':'application/octet-stream'},status)
def stable_blob(data):
return {'artifact_id':str(uuid.uuid4()),'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data)}
original_worker=(artifacts/'rmm-agent-updater').read_bytes()
original=stable_blob(original_worker)
# Model a previously published source catalog. Both customers must be able to
# reference exactly the same signed IDs, and each needs its own verified objects.
tenant_b = '22222222-2222-4222-8222-222222222222'
b_store = state/'objects'/tenant_b
b_store.mkdir(mode=0o700)
for name, blob in zip(release.FILES[release.LINUX], [*ma['artifacts'], original]):
release.save_once(b_store/blob['sha256'], (artifacts/name).read_bytes())
agent, helper = ma['artifacts']
env_raw = release.encoded(a)
sha = lambda data: hashlib.sha256(data).hexdigest()
def scoped(tenant, statement):
return sql('rmm_app','rmm', f"BEGIN; SET LOCAL app.tenant_id='{tenant}'; {statement}; COMMIT;")
for blob in ma['artifacts']:
scoped(tenant_b, f"""INSERT INTO agent_update_artifacts(tenant_id,artifact_id,sha256,size_bytes)
VALUES('{tenant_b}','{blob["artifact_id"]}','{blob["sha256"]}',{blob["bytes"]})""")
scoped(tenant_b, f"""INSERT INTO agent_update_releases(tenant_id,release_id,platform,release_sequence,
manifest_sha256,envelope_sha256,envelope,agent_artifact_id,agent_sha256,agent_bytes,
helper_artifact_id,helper_sha256,helper_bytes) VALUES('{tenant_b}','{ma["release_id"]}',
'{release.LINUX}',1,'{sha(release.encoded(ma))}','{sha(env_raw)}',decode('{env_raw.hex()}','hex'),
'{agent["artifact_id"]}','{agent["sha256"]}',{agent["bytes"]},
'{helper["artifact_id"]}','{helper["sha256"]}',{helper["bytes"]})""")
# This source itself uses a starter worker, with a different pin from the global default.
b_worker = original_worker + b'tenant B starter'
b_desc = stable_blob(b_worker)
release.save_once(b_store/b_desc['sha256'], b_worker)
scoped(tenant_b, f"""INSERT INTO customer_starter_workers VALUES('{tenant_b}','{release.LINUX}',
'{b_desc["sha256"]}',{b_desc["bytes"]})""")
starters.publish(tenant_b,release.LINUX,ma['release_id'],root=state,database_container=db,service_uid=os.getuid())
dist = state/'config/customer-starters'
packaged = json.loads((dist/(release.LINUX+'.json')).read_bytes())
assert packaged['worker']['sha256'] == b_desc['sha256']
assert scoped(TENANT, 'SELECT count(*) FROM agent_update_releases') == '0'
# A partial distribution must leave the catalog empty and is repairable by republishing.
(dist/helper['sha256']).unlink()
ask('POST',path+'/enrollment-tokens',{},auth,status=409)
assert scoped(TENANT, 'SELECT count(*) FROM agent_update_artifacts') == '0'
assert scoped(TENANT, 'SELECT count(*) FROM customer_starter_workers') == '0'
starters.publish(tenant_b,release.LINUX,ma['release_id'],root=state,database_container=db,service_uid=os.getuid())
with ThreadPoolExecutor(max_workers=2) as workers:
issues = [workers.submit(ask,'POST',path+'/enrollment-tokens',{},auth) for _ in range(2)]
first, retry = [request.result()[1] for request in issues]
assert first['release_platforms'] == retry['release_platforms'] == [release.LINUX]
assert first['token'] != retry['token']
for customer in (TENANT, tenant_b):
assert scoped(customer, 'SELECT release_id FROM agent_update_releases') == ma['release_id']
assert scoped(customer, 'SELECT count(*) FROM agent_update_artifacts') == '2'
first_auth = {'Authorization':'Enrollment '+first['token']}
current_path = '/agent/v1/bootstrap/current?platform='+release.LINUX
metadata = ask('GET',current_path,headers=first_auth)[1]
assert metadata['release_envelope'] == a and metadata['tenant_id'] == TENANT
assert ask('GET','/agent/v1/bootstrap/updater?platform='+release.LINUX,headers=first_auth)[1] == b_worker
print('Empty customer receives source signed IDs, separate verified objects and its protected worker; incomplete seed can retry')
upload(original,original_worker)
release.save_once(state/'config'/('linux-bootstrap.'+TENANT+'.json'),
release.encoded({'current':{'sha256':original['sha256'],'bytes':original['bytes']},'previous':[]}))
def send_pair(manifest):
for name,blob in zip(release.FILES[release.LINUX],manifest['artifacts']):
upload(blob,(artifacts/name).read_bytes())
send_pair(ma)
ask('POST',path+'/agent-releases',a,auth)
ask('POST',path+'/agent-releases',a,auth)
b,mb=make_release(2)
upload(mb['artifacts'][0],(artifacts/'rmm-agent').read_bytes())
ask('POST',path+'/agent-releases',b,auth,404)
partial=mb['artifacts'][1]
upload(partial,(artifacts/'rmm-privileged-helper').read_bytes()[:-1],400)
ask('POST',path+'/agent-releases',b,auth,404)
h,token=ask('POST',path+'/enrollment-tokens',{},auth)
assert 'no-store' in h['Cache-Control']
bootstrap='/agent/v1/bootstrap/current?platform='+release.LINUX
credential={'Authorization':'Enrollment '+token['token']}
_,active=ask('GET',bootstrap,headers=credential)
assert active['manifest']['release_id']==ma['release_id'] and active['job_signing_public_key_hex']
assert active['tenant_id']==TENANT
ask('GET',bootstrap,headers={'Authorization':'Enrollment '+'0'*64},status=401)
blob=ma['artifacts'][0]
download=f"/agent/v1/bootstrap/releases/{ma['release_id']}/artifacts/{blob['artifact_id']}?platform={release.LINUX}"
_,raw=ask('GET',download,headers=credential)
assert hashlib.sha256(raw).hexdigest()==blob['sha256']
def updater(digest=None, status=200):
return ask('GET','/agent/v1/bootstrap/updater?platform='+release.LINUX+('&sha256='+digest if digest else ''),
headers=credential,status=status)[1]
assert updater()==original_worker
replacement_worker=original_worker+b'test-new-worker'
new=stable_blob(replacement_worker)
pin={'sha256':new['sha256'],'bytes':new['bytes']}
try:
release.rotate_pin(state,'linux',TENANT,original['sha256'],pin)
raise AssertionError('missing upload changed pointer')
except FileNotFoundError:
pass
assert updater()==original_worker
worker_build = directory/'worker-only'
worker_build.mkdir()
worker_name = release.FILES[release.LINUX][2]
(worker_build/worker_name).write_bytes(replacement_worker)
worker_receipt = worker_build/'manifest.json'
worker_receipt.write_text(json.dumps({'target':release.LINUX,'sha256':{worker_name:new['sha256']}}))
login_file = directory/'catalog-login.json'
with open(login_file, 'x', opener=lambda path, flags: os.open(path, flags, 0o600)) as login:
json.dump({'username':env['RMM_DEV_USERNAME'],'password':env['RMM_DEV_PASSWORD']},login)
def rotate(predecessor):
release.publish(SimpleNamespace(artifacts=worker_build,build_receipt=worker_receipt,rotate_worker_only=True,
envelope=None,bootstrap_worker=None,rotate_bootstrap_from=predecessor,bootstrap_state=state,origin=origin,
ca=tls/'ca.pem',dev_login_file=login_file,tenant=TENANT))
published_before = ask('GET',path+'/agent-releases',headers=auth)[1]['items']
assert [item['release_envelope'] for item in published_before] == [a]
try:
rotate('0'*64)
raise AssertionError('unexpected predecessor accepted')
except ValueError:
pass
assert updater()==original_worker
rotate(original['sha256'])
rotate(original['sha256'])
assert ask('GET',path+'/agent-releases',headers=auth)[1]['items'] == published_before
assert updater()==replacement_worker and updater(original['sha256'])==original_worker
assert ask('GET',bootstrap,headers=credential)[1]['updater']==pin
updater('0'*64,404)
assert json.loads((state/'config/linux-bootstrap.json').read_bytes())['updater_sha256']==original['sha256']
# B keeps its independent worker when A has a per-customer rotation.
token_b=secrets.token_hex(32)
sql('rmm_app','rmm',f"BEGIN; SET LOCAL app.tenant_id='{tenant_b}'; SELECT issue_enrollment_token('{hashlib.sha256(token_b.encode()).hexdigest()}'); COMMIT;")
b_auth={'Authorization':'Enrollment '+token_b}
assert ask('GET','/agent/v1/bootstrap/updater?platform='+release.LINUX,headers=b_auth)[1]==b_worker
ask('GET','/agent/v1/bootstrap/updater?platform='+release.LINUX+'&sha256='+new['sha256'],headers=b_auth,status=404)
print('worker-only publication preserves the signed release, tenant isolation and in-flight downloads; stale predecessor rejected')
install={'token':token['token'],'csr_pem':(tls/'device.csr').read_text()}
_,enrolled=ask('POST','/agent/v1/enroll',install)
assert enrolled['certificate_pem']
ask('POST','/agent/v1/enroll',install,status=401)
ask('GET',bootstrap,headers=credential,status=401)
ask('GET',download,headers=credential,status=401)
ask('GET','/agent/v1/releases/current?platform='+release.LINUX,status=401)
cert = tls/'issued-device.pem'
cert.write_text(enrolled['certificate_pem'])
ctx.load_cert_chain(str(cert),str(tls/'device.key'))
_,trust=ask('GET','/agent/v1/releases/device-trust')
assert trust['job_signing_public_key_hex'] == active['job_signing_public_key_hex']
upload(partial,(artifacts/'rmm-privileged-helper').read_bytes())
ask('POST',path+'/agent-releases',b,auth)
print('A stays current after missing/truncated B; B publishes only after complete retry')
print('Unknown token, second enrollment, post-consumption metadata/object fetch and absent mTLS rejected')
if __name__ == '__main__':
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--server-binary',required=True,type=Path)
args=parser.parse_args()
with ExitStack() as cleanup:
folder=Path(cleanup.enter_context(tempfile.TemporaryDirectory()))
test(args.server_binary.resolve(),folder,cleanup)