Build local product stack with Vault-managed secrets (#42)

dev/bootstrap.py

@@ -0,0 +1,105 @@
+"""Idempotent provisioning for the isolated local Compose project."""
+import json
+import os
+import subprocess
+import time
+
+from agents import config
+from vault import STATE, write
+
+ROOT = STATE.parents[1]
+COMPOSE = ['docker', 'compose', '--env-file', str(STATE / 'images.env')]
+
+
+def compose(*args, capture=False, input=None):
+    result = subprocess.run(COMPOSE + list(args), cwd=ROOT, input=input, text=True,
+                            capture_output=capture)
+    if result.returncode:
+        # Bootstrap stdin and child errors may contain credentials. Never echo them.
+        raise RuntimeError('Compose command failed: ' + ' '.join(args[:3]))
+    return result.stdout if capture else None
+
+
+def mongo(script, password):
+    auth = 'const admin = db.getSiblingDB("admin"); admin.auth("local-admin", ' + json.dumps(password) + ');\n'
+    return compose('exec', '-T', 'mongo', 'mongosh', '--quiet', '--file', '/dev/stdin', capture=True, input=auth + script)
+
+
+def provision_mongo(vault):
+    secret = vault.ensure('bootstrap/mongo', lambda: {'password': vault.random(), 'keyfile': vault.random(384), 'manager_password': vault.random()})
+    write(STATE / 'mongo' / 'password', secret['password'])
+    write(STATE / 'mongo' / 'keyfile', secret['keyfile'])
+    compose('up', '-d', 'mongo')
+    for attempt in range(90):
+        try:
+            mongo('try { rs.status(); } catch(e) { if(e.code === 94) rs.initiate({_id:"rs0",members:[{_id:0,host:"mongo:27017"}]}); else throw e; }', secret['password'])
+            if 'true' in mongo('print(db.hello().isWritablePrimary)', secret['password']):
+                break
+        except RuntimeError:
+            pass
+        time.sleep(2)
+    else:
+        raise RuntimeError('Mongo replica set did not become primary')
+    if vault.call('database/config/mongo', missing=True) is None:
+        mongo('if (!admin.getUser("vault-manager")) admin.createUser({user:"vault-manager",pwd:' + json.dumps(secret['manager_password']) + ',roles:[{role:"userAdminAnyDatabase",db:"admin"}]});', secret['password'])
+        vault.call('database/config/mongo', {'plugin_name': 'mongodb-database-plugin',
+            'allowed_roles': ['bff', 'notifications'],
+            'connection_url': 'mongodb://{{username}}:{{password}}@mongo:27017/admin?replicaSet=rs0',
+            'username': 'vault-manager', 'password': secret['manager_password']})
+        vault.call('database/rotate-root/mongo', {})
+    for role, database in [('bff', 'sxchnge'), ('notifications', 'sxchnge_notifications')]:
+        vault.call('database/roles/' + role, {'db_name': 'mongo',
+            'creation_statements': json.dumps({'db': 'admin', 'roles': [{'role': 'readWrite', 'db': database}]}),
+            'default_ttl': '1h', 'max_ttl': '24h'})
+    # Public browser client: an identifier, never a client secret in the web bundle.
+    mongo('db.getSiblingDB("sxchnge").clients.updateOne({clientId:"sxchnge-local-web"}, {$setOnInsert: {clientName:"Local browser",allowedCIDRs:["0.0.0.0/0","::/0"],allowedScopes:["account"],redirectURIs:["https://localhost:5126"],grantTypes:["password","refresh_token"],tokenEndpointAuthMethod:"none",isRevoked:false}}, {upsert:true});', secret['password'])
+
+
+def provision_agents(vault):
+    paths = {'bff': ['bff', 'caller', 'tls'],
+             'notifications': ['caller', 'tls', 'notifications-tls', 'nats', 'email'],
+             'nats': ['nats']}
+    for service, secrets in paths.items():
+        directory = STATE / 'agents' / service
+        directory.mkdir(parents=True, exist_ok=True, mode=0o755)
+        os.chmod(directory, 0o755)
+        policy = ''.join('path "kv/data/' + p + '" { capabilities = ["read"] }\n' for p in secrets)
+        if service != 'nats':
+            policy += 'path "database/creds/' + service + '" { capabilities = ["read"] }\n'
+        vault.call('sys/policies/acl/' + service, {'policy': policy})
+        vault.call('auth/approle/role/' + service, {'token_policies': [service],
+            'token_period': '1h', 'secret_id_ttl': '0', 'secret_id_num_uses': 0})
+        role_id = vault.call('auth/approle/role/' + service + '/role-id')['data']['role_id']
+        write(directory / 'role-id', role_id, 0o444)
+        if not (directory / 'secret-id').exists():
+            secret_id = vault.call('auth/approle/role/' + service + '/secret-id', {})['data']['secret_id']
+            write(directory / 'secret-id', secret_id, 0o444)
+        write(directory / 'config.hcl', config(service), 0o444)
+
+
+def up(vault):
+    if not (STATE / 'images.env').exists():
+        raise RuntimeError('Run make build before make up')
+    (STATE / 'vault').mkdir(parents=True, exist_ok=True, mode=0o755)
+    compose('up', '-d', 'vault')
+    vault.initialize()
+    vault.mounts()
+    vault.ensure('nats', lambda: {'token': vault.random()})
+    vault.ensure('caller', lambda: {'token': vault.random()})
+    vault.ensure('bff', lambda: {'jwt': vault.random()})
+    vault.ensure('email', lambda: {'driver': 'resend', 'api_key': '',
+        'sender_address': 'notifications@localhost.test', 'sender_name': 'SXCHNGE local'})
+    vault.certificates()
+    provision_mongo(vault)
+    provision_agents(vault)
+    compose('up', '-d', '--wait', '--wait-timeout', '120', 'nats')
+    email = vault.kv('email')
+    key, sender = email.get('api_key'), email.get('sender_address')
+    if (not isinstance(key, str) or not key.strip()
+            or not isinstance(sender, str) or not sender.strip()
+            or sender == 'notifications@localhost.test'):
+        raise RuntimeError('Vault kv/email needs a Resend api_key and verified sender_address. Run python3 dev/local.py email, then make up.')
+    compose('up', '-d', '--wait', '--wait-timeout', '180')
+    compose('exec', '-T', 'frontend', 'nginx', '-s', 'reload', capture=True)
+    print('Local stack ready: https://localhost:5126; Vault: http://127.0.0.1:18200')
+    print('Local CA: .local/dev/ca.pem. Email driver: ' + vault.kv('email')['driver'])