feat: deploy cloud-only RMM and verify audited APT diagnostics
Rust development slice / linux (push) Failing after 7s
Rust development slice / linux (push) Failing after 7s
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create separate clients for the dedicated server; never execute RMM jobs.
|
||||
|
||||
Old VM identities, helpers, services and journals remain untouched. Each new
|
||||
installation is create-only; failures retain state for inspection, not reenroll.
|
||||
Server secrets stay on the server. Only a single-use token and public trust
|
||||
material are transferred over independently verified SSH.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pwd
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import vm_access_check as access
|
||||
|
||||
ROOT = access.ROOT
|
||||
TENANT = '11111111-1111-4111-8111-111111111111'
|
||||
DOMAIN = 'discovery.ease-technologies.de'
|
||||
UNIT = 'rmm-agent-remote.service'
|
||||
|
||||
|
||||
def require(value, message):
|
||||
if not value:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def execute(args, **kwargs):
|
||||
result = subprocess.run(args, capture_output=True, timeout=40, **kwargs)
|
||||
require(result.returncode == 0, 'command failed; private diagnostics withheld')
|
||||
return result.stdout
|
||||
|
||||
|
||||
def rpc(config, function, payload):
|
||||
cfg = access.load_config(ROOT/'credentials'/config)
|
||||
ssh = ['ssh', '-F', '/dev/null', '-T', '-p', cfg.get('port', '22'),
|
||||
'-o', 'StrictHostKeyChecking=yes', '-o', 'BatchMode=yes',
|
||||
'-o', 'IdentitiesOnly=yes', '-o', 'ForwardAgent=no', '-o', 'ConnectTimeout=10',
|
||||
'-i', str(Path(cfg['ssh_key_path']).expanduser()), cfg['username']+'@'+cfg['address'], 'python3 -']
|
||||
code = ('import os,json,subprocess,base64,hashlib,ssl,http.client,socket,uuid,pwd\nfrom pathlib import Path\n' +
|
||||
inspect.getsource(require) + '\n' + inspect.getsource(function) +
|
||||
'\ntry:\n print(json.dumps('+function.__name__+'('+repr(payload)+')))\n'
|
||||
'except Exception as error:\n import traceback\n'
|
||||
' print(json.dumps(dict(error="REMOTE_BOOTSTRAP_INCOMPLETE",error_class=type(error).__name__, '
|
||||
'frames=[dict(function=f.name,line=f.lineno) for f in traceback.extract_tb(error.__traceback__)])))\n')
|
||||
result = json.loads(execute(ssh, input=code.encode()))
|
||||
if 'error' in result:
|
||||
# Never expose exception messages, locals, commands or remote stderr.
|
||||
raise RuntimeError('remote bootstrap incomplete: '+json.dumps({k:result.get(k) for k in ('error_class','frames')}))
|
||||
return result
|
||||
|
||||
|
||||
def server(payload):
|
||||
root = Path('/opt/rust-discovery')
|
||||
command = ['docker', 'compose', '--env-file', str(root/'devstate/.env'),
|
||||
'-f', str(root/'deploy/server.compose.yaml'), 'exec', '-T', 'postgres',
|
||||
'psql', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1', '-U', 'postgres', '-d', 'rmm']
|
||||
if payload['action'] in ('enroll', 'material'):
|
||||
raw = None
|
||||
if payload['action'] == 'enroll':
|
||||
raw = __import__('secrets').token_hex(32)
|
||||
digest = hashlib.sha256(raw.encode()).hexdigest()
|
||||
query = ("INSERT INTO enrollment_tokens(token_hash,tenant_id,expires_at) VALUES('"+digest+
|
||||
"','11111111-1111-4111-8111-111111111111',now()+interval '15 minutes');")
|
||||
subprocess.run(command, input=query.encode(), capture_output=True, check=True)
|
||||
# This is a public signing-key endpoint, accessed over verified TLS with
|
||||
# a private loopback reader. No private central key leaves the server.
|
||||
class HTTPS(http.client.HTTPSConnection):
|
||||
def connect(self):
|
||||
self.sock = self._context.wrap_socket(socket.create_connection(('127.0.0.1', 18443), timeout=10),
|
||||
server_hostname=self.host)
|
||||
client = HTTPS('rmm-dev.internal', 18443, context=ssl.create_default_context(cafile=str(root/'devstate/pki/ca.pem')))
|
||||
client.request('GET', '/api/v1/job-signing-key', headers={'Authorization': 'Bearer '+
|
||||
(root/'devstate/operator-a.token').read_text().strip()})
|
||||
response = client.getresponse(); require(response.status == 200, 'signing key endpoint rejected')
|
||||
key = json.loads(response.read()); client.close()
|
||||
return dict(token=raw, ca=base64.b64encode((root/'devstate/pki/ca.pem').read_bytes()).decode(), key=key)
|
||||
agent = str(uuid.UUID(payload['agent_id']))
|
||||
query = ("SELECT json_build_object('agent_id',agent_id,'asset_id',asset_id,'enrollment_generation',enrollment_generation,"
|
||||
"'assignment_version',assignment_version,'sync_ready',sync_ready,'restore_blocked',restore_blocked,"
|
||||
"'last_contact_at',last_contact_at,'connection_generation',connection_generation,'transport',transport,'inventory_received',"
|
||||
"EXISTS(SELECT 1 FROM observations o WHERE o.tenant_id=a.tenant_id AND o.agent_id=a.agent_id)) "
|
||||
"FROM agent_installations a WHERE tenant_id='11111111-1111-4111-8111-111111111111' AND agent_id='"+agent+"';")
|
||||
result = subprocess.run(command, input=query.encode(), capture_output=True, check=True).stdout.strip()
|
||||
return json.loads(result) if result else {}
|
||||
|
||||
|
||||
def vm(payload):
|
||||
unit = 'rmm-agent-remote.service'
|
||||
state = Path('/var/lib/rmm-agent-remote')
|
||||
config = Path('/etc/rmm-remote')
|
||||
binary = Path('/usr/local/libexec/rmm-agent-remote')
|
||||
account = pwd.getpwnam('rmm-agent')
|
||||
require(os.geteuid() == 0 and account.pw_uid > 0, 'unprivileged agent account required')
|
||||
if payload['action'] == 'install':
|
||||
for name in ('rmm-agent.service', 'rmm-helper-dev-test.service', 'rmm-apt-helper.service'):
|
||||
result = subprocess.run(['systemctl', 'show', name, '-p', 'ActiveState', '--value'], capture_output=True, check=True)
|
||||
require(result.stdout.strip() == b'inactive', 'old installation is not idle')
|
||||
target = Path('/etc/systemd/system')/unit
|
||||
for path in (state, config, binary, target):
|
||||
require(not path.exists() and not path.is_symlink(), 'existing installation; never reset')
|
||||
data = base64.b64decode(payload['binary'], validate=True)
|
||||
require(hashlib.sha256(data).hexdigest() == payload['binary_sha256'] and data[:6] == b'\x7fELF\x02\x01'
|
||||
and data[18:20] == b'\x3e\x00', 'Linux x86-64 artifact required')
|
||||
os.umask(0o077)
|
||||
state.mkdir(mode=0o700); os.chown(state, account.pw_uid, account.pw_gid)
|
||||
config.mkdir(mode=0o755); config.chmod(0o755)
|
||||
for path, content, mode, owner in (
|
||||
(binary, data, 0o755, 0), (config/'ca.pem', base64.b64decode(payload['ca']), 0o644, 0),
|
||||
(state/'enroll.token', payload['token'].encode(), 0o600, account.pw_uid),
|
||||
(config/'agent.env', payload['environment'].encode(), 0o600, 0),
|
||||
(target, payload['unit'].encode(), 0o644, 0)):
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, mode)
|
||||
with os.fdopen(fd, 'wb') as stream:
|
||||
stream.write(content); stream.flush(); os.fsync(stream.fileno())
|
||||
os.chown(path, owner, account.pw_gid if owner else 0)
|
||||
path.chmod(mode)
|
||||
subprocess.run(['/lib64/ld-linux-x86-64.so.2', '--list', str(binary)], capture_output=True, check=True)
|
||||
subprocess.run(['systemd-analyze', 'verify', str(target)], capture_output=True, check=True)
|
||||
subprocess.run(['systemctl', 'daemon-reload'], capture_output=True, check=True)
|
||||
return dict(installed=True, uid=account.pw_uid)
|
||||
if payload['action'] in ('start', 'stop', 'enable'):
|
||||
subprocess.run(['systemctl', payload['action'], unit], capture_output=True, check=True)
|
||||
return {}
|
||||
if payload['action'] == 'identity':
|
||||
path = state/'identity.json'
|
||||
if not path.exists():
|
||||
return {}
|
||||
identity = json.loads(path.read_text())
|
||||
return {key: identity[key] for key in ('agent_id', 'asset_id')}
|
||||
require(payload['action'] == 'bind', 'unknown bootstrap action')
|
||||
require(subprocess.run(['systemctl', 'is-active', unit], capture_output=True).stdout.strip() == b'inactive', 'stop owned client before binding')
|
||||
path = state/'binding.json'
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, 'w') as stream:
|
||||
json.dump(payload['binding'], stream)
|
||||
os.chown(path, account.pw_uid, account.pw_gid)
|
||||
env = config/'agent.env'
|
||||
require(env.read_text() == payload['old_environment'], 'bootstrap configuration changed')
|
||||
env.write_text(payload['environment']); env.chmod(0o600)
|
||||
(state/'enroll.token').unlink()
|
||||
return dict(bound=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--provision-vm', action='store_true')
|
||||
group.add_argument('--resume-vm', action='store_true', help='Resume own retained bootstrap; no enrollment/token reset')
|
||||
args = parser.parse_args()
|
||||
target = 'vm'
|
||||
def client(payload):
|
||||
return rpc('linux-vm.conf', vm, payload)
|
||||
os.umask(0o077)
|
||||
record = ROOT/('devstate/remote-'+target+'.json')
|
||||
require(record.exists() if args.resume_vm else not record.exists(), 'bootstrap record state does not match requested mode')
|
||||
artifact = (ROOT/'devstate/rmm-agent-remote').read_bytes()
|
||||
material = rpc('server.conf', server, dict(action='material' if args.resume_vm else 'enroll'))
|
||||
cfg = access.load_config(ROOT/'credentials/server.conf')
|
||||
endpoint = cfg['address'] + ':443'
|
||||
environment = ('RMM_SERVER_NAME='+DOMAIN+'\nRMM_CONNECT_ADDR='+endpoint+'\n'
|
||||
'RMM_ENROLLMENT_TOKEN_FILE=/var/lib/rmm-agent-remote/enroll.token\n')
|
||||
unit = (ROOT/'deploy/linux/rmm-agent-remote.service').read_text()
|
||||
if args.resume_vm:
|
||||
report = json.loads(record.read_text())
|
||||
require(report['stage'] in ('installed','passive_observed') and report['domain'] == DOMAIN
|
||||
and report['binary_sha256'] == hashlib.sha256(artifact).hexdigest()
|
||||
and report['ca_sha256'] == hashlib.sha256(base64.b64decode(material['ca'])).hexdigest(),
|
||||
'retained bootstrap or trust binding differs; no reset')
|
||||
else:
|
||||
installed = client(dict(action='install', binary=base64.b64encode(artifact).decode(),
|
||||
binary_sha256=hashlib.sha256(artifact).hexdigest(), unit=unit, environment=environment, **material))
|
||||
report = dict(stage='installed', installed=installed, domain=DOMAIN, unit=UNIT,
|
||||
binary_sha256=hashlib.sha256(artifact).hexdigest(), ca_sha256=hashlib.sha256(base64.b64decode(material['ca'])).hexdigest())
|
||||
def save():
|
||||
record.write_text(json.dumps(report, indent=2)); record.chmod(0o600)
|
||||
save()
|
||||
started = False
|
||||
try:
|
||||
started = True; client(dict(action='start'))
|
||||
end = time.monotonic()+100
|
||||
identity = {}
|
||||
while time.monotonic() < end:
|
||||
identity = client(dict(action='identity'))
|
||||
if identity:
|
||||
observed = rpc('server.conf', server, dict(action='observe', agent_id=identity['agent_id']))
|
||||
if observed.get('inventory_received') and observed.get('last_contact_at'):
|
||||
break
|
||||
time.sleep(2)
|
||||
require(identity and observed.get('inventory_received'), 'native enrollment/inventory not observed')
|
||||
report.update(identity=identity, passive_observation=observed, stage='passive_observed'); save()
|
||||
client(dict(action='stop')); started = False
|
||||
binding = {key: observed[key] for key in ('agent_id','asset_id','enrollment_generation','assignment_version')}
|
||||
binding.update(tenant_id=TENANT, public_key_hex=material['key']['public_key_hex'])
|
||||
bound_environment = ('RMM_SERVER_NAME='+DOMAIN+'\nRMM_CONNECT_ADDR='+endpoint+'\n'
|
||||
'RMM_JOB_BINDING_FILE=/var/lib/rmm-agent-remote/binding.json\nRMM_JOB_SIGNING_KEY_BOOTSTRAP=1\n')
|
||||
client(dict(action='bind', binding=binding, old_environment=environment, environment=bound_environment))
|
||||
started = True; client(dict(action='start'))
|
||||
end = time.monotonic()+100
|
||||
while time.monotonic() < end:
|
||||
observed = rpc('server.conf', server, dict(action='observe', agent_id=identity['agent_id']))
|
||||
if observed.get('sync_ready') and not observed.get('restore_blocked'):
|
||||
break
|
||||
time.sleep(2)
|
||||
require(observed.get('sync_ready') and not observed.get('restore_blocked'), 'native signed sync gate not ready')
|
||||
client(dict(action='enable'))
|
||||
report.update(stage='signed_sync_ready', sync_observation=observed, jobs_executed=False); save()
|
||||
print('PASS '+target+' direct domain TLS/mTLS, passive inventory and signed Sync; no RMM job executed')
|
||||
# The user requested a client against the remote server. Leave this new
|
||||
# unprivileged client running; old installation remains stopped/intact.
|
||||
started = False
|
||||
finally:
|
||||
if started:
|
||||
client(dict(action='stop'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception:
|
||||
print('CLIENT_BOOTSTRAP_INCOMPLETE; preserve identities and inspect private state')
|
||||
raise SystemExit(2) from None
|
||||
Reference in New Issue
Block a user