Build local product stack with Vault-managed secrets (#42)

dev/agents.py

@@ -0,0 +1,43 @@
+"""Build per-service Vault Agent configs (no secret values in configs)."""
+import json
+from pathlib import Path
+
+BASE = (Path(__file__).parent / 'vault' / 'agent.hcl').read_text()
+
+
+def kv(path, field):
+    return '{{ with secret "kv/data/' + path + '" }}{{ .Data.data.' + field + ' }}{{ end }}'
+
+
+def config(service):
+    env = {}
+    if service == 'nats':
+        env['NATS_TOKEN'] = kv('nats', 'token')
+    else:
+        database = 'sxchnge' if service == 'bff' else 'sxchnge_notifications'
+        env['SXCHNGE_MONGO_URI'] = ('{{ with secret "database/creds/' + service + '" }}'
+            'mongodb://{{ .Data.username | urlquery }}:{{ .Data.password | urlquery }}'
+            '@mongo:27017/' + database + '?authSource=admin&replicaSet=rs0{{ end }}')
+        env['TLS_CA'] = kv('tls', 'ca')
+        if service == 'bff':
+            env['SXCHNGE_JWT_SECRET'] = kv('bff', 'jwt')
+            env['SXCHNGE_NOTIFICATIONS_TOKEN'] = kv('caller', 'token')
+        else:
+            env['SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN'] = kv('caller', 'token')
+            env['SXCHNGE_NATS_URL'] = ('{{ with secret "kv/data/nats" }}'
+                'nats://{{ .Data.data.token }}@nats:4222{{ end }}')
+            env['TLS_CERT'] = kv('notifications-tls', 'certificate')
+            env['TLS_KEY'] = kv('notifications-tls', 'private_key')
+            env['EMAIL_API_KEY'] = kv('email', 'api_key')
+            # JSON is valid YAML and escapes arbitrary provider configuration.
+            env['EMAIL_CONFIG'] = ('{{ with secret "kv/data/email" }}'
+                'email:\n  driver: {{ .Data.data.driver | toJSON }}\n'
+                '  settings:\n    api_key_env: EMAIL_API_KEY\n'
+                '    sender_address: {{ .Data.data.sender_address | toJSON }}\n'
+                '    sender_name: {{ .Data.data.sender_name | toJSON }}\n{{ end }}')
+    result = BASE
+    for name, template in env.items():
+        result += '\nenv_template ' + json.dumps(name) + ' {\n  contents = ' + json.dumps(template) + '\n  error_on_missing_key = true\n}\n'
+    command = '/usr/local/bin/start-nats' if service == 'nats' else '/usr/local/bin/start-service'
+    result += '\nexec {\n  command = [' + json.dumps(command) + ']\n  restart_on_secret_changes = "always"\n  restart_stop_signal = "SIGTERM"\n}\n'
+    return result

dev/bootstrap.py

@@ -0,0 +1,105 @@
+"""Idempotent provisioning for the isolated local Compose project."""
+import json
+import os
+import subprocess
+import time
+
+from agents import config
+from vault import STATE, write
+
+ROOT = STATE.parents[1]
+COMPOSE = ['docker', 'compose', '--env-file', str(STATE / 'images.env')]
+
+
+def compose(*args, capture=False, input=None):
+    result = subprocess.run(COMPOSE + list(args), cwd=ROOT, input=input, text=True,
+                            capture_output=capture)
+    if result.returncode:
+        # Bootstrap stdin and child errors may contain credentials. Never echo them.
+        raise RuntimeError('Compose command failed: ' + ' '.join(args[:3]))
+    return result.stdout if capture else None
+
+
+def mongo(script, password):
+    auth = 'const admin = db.getSiblingDB("admin"); admin.auth("local-admin", ' + json.dumps(password) + ');\n'
+    return compose('exec', '-T', 'mongo', 'mongosh', '--quiet', '--file', '/dev/stdin', capture=True, input=auth + script)
+
+
+def provision_mongo(vault):
+    secret = vault.ensure('bootstrap/mongo', lambda: {'password': vault.random(), 'keyfile': vault.random(384), 'manager_password': vault.random()})
+    write(STATE / 'mongo' / 'password', secret['password'])
+    write(STATE / 'mongo' / 'keyfile', secret['keyfile'])
+    compose('up', '-d', 'mongo')
+    for attempt in range(90):
+        try:
+            mongo('try { rs.status(); } catch(e) { if(e.code === 94) rs.initiate({_id:"rs0",members:[{_id:0,host:"mongo:27017"}]}); else throw e; }', secret['password'])
+            if 'true' in mongo('print(db.hello().isWritablePrimary)', secret['password']):
+                break
+        except RuntimeError:
+            pass
+        time.sleep(2)
+    else:
+        raise RuntimeError('Mongo replica set did not become primary')
+    if vault.call('database/config/mongo', missing=True) is None:
+        mongo('if (!admin.getUser("vault-manager")) admin.createUser({user:"vault-manager",pwd:' + json.dumps(secret['manager_password']) + ',roles:[{role:"userAdminAnyDatabase",db:"admin"}]});', secret['password'])
+        vault.call('database/config/mongo', {'plugin_name': 'mongodb-database-plugin',
+            'allowed_roles': ['bff', 'notifications'],
+            'connection_url': 'mongodb://{{username}}:{{password}}@mongo:27017/admin?replicaSet=rs0',
+            'username': 'vault-manager', 'password': secret['manager_password']})
+        vault.call('database/rotate-root/mongo', {})
+    for role, database in [('bff', 'sxchnge'), ('notifications', 'sxchnge_notifications')]:
+        vault.call('database/roles/' + role, {'db_name': 'mongo',
+            'creation_statements': json.dumps({'db': 'admin', 'roles': [{'role': 'readWrite', 'db': database}]}),
+            'default_ttl': '1h', 'max_ttl': '24h'})
+    # Public browser client: an identifier, never a client secret in the web bundle.
+    mongo('db.getSiblingDB("sxchnge").clients.updateOne({clientId:"sxchnge-local-web"}, {$setOnInsert: {clientName:"Local browser",allowedCIDRs:["0.0.0.0/0","::/0"],allowedScopes:["account"],redirectURIs:["https://localhost:5126"],grantTypes:["password","refresh_token"],tokenEndpointAuthMethod:"none",isRevoked:false}}, {upsert:true});', secret['password'])
+
+
+def provision_agents(vault):
+    paths = {'bff': ['bff', 'caller', 'tls'],
+             'notifications': ['caller', 'tls', 'notifications-tls', 'nats', 'email'],
+             'nats': ['nats']}
+    for service, secrets in paths.items():
+        directory = STATE / 'agents' / service
+        directory.mkdir(parents=True, exist_ok=True, mode=0o755)
+        os.chmod(directory, 0o755)
+        policy = ''.join('path "kv/data/' + p + '" { capabilities = ["read"] }\n' for p in secrets)
+        if service != 'nats':
+            policy += 'path "database/creds/' + service + '" { capabilities = ["read"] }\n'
+        vault.call('sys/policies/acl/' + service, {'policy': policy})
+        vault.call('auth/approle/role/' + service, {'token_policies': [service],
+            'token_period': '1h', 'secret_id_ttl': '0', 'secret_id_num_uses': 0})
+        role_id = vault.call('auth/approle/role/' + service + '/role-id')['data']['role_id']
+        write(directory / 'role-id', role_id, 0o444)
+        if not (directory / 'secret-id').exists():
+            secret_id = vault.call('auth/approle/role/' + service + '/secret-id', {})['data']['secret_id']
+            write(directory / 'secret-id', secret_id, 0o444)
+        write(directory / 'config.hcl', config(service), 0o444)
+
+
+def up(vault):
+    if not (STATE / 'images.env').exists():
+        raise RuntimeError('Run make build before make up')
+    (STATE / 'vault').mkdir(parents=True, exist_ok=True, mode=0o755)
+    compose('up', '-d', 'vault')
+    vault.initialize()
+    vault.mounts()
+    vault.ensure('nats', lambda: {'token': vault.random()})
+    vault.ensure('caller', lambda: {'token': vault.random()})
+    vault.ensure('bff', lambda: {'jwt': vault.random()})
+    vault.ensure('email', lambda: {'driver': 'resend', 'api_key': '',
+        'sender_address': 'notifications@localhost.test', 'sender_name': 'SXCHNGE local'})
+    vault.certificates()
+    provision_mongo(vault)
+    provision_agents(vault)
+    compose('up', '-d', '--wait', '--wait-timeout', '120', 'nats')
+    email = vault.kv('email')
+    key, sender = email.get('api_key'), email.get('sender_address')
+    if (not isinstance(key, str) or not key.strip()
+            or not isinstance(sender, str) or not sender.strip()
+            or sender == 'notifications@localhost.test'):
+        raise RuntimeError('Vault kv/email needs a Resend api_key and verified sender_address. Run python3 dev/local.py email, then make up.')
+    compose('up', '-d', '--wait', '--wait-timeout', '180')
+    compose('exec', '-T', 'frontend', 'nginx', '-s', 'reload', capture=True)
+    print('Local stack ready: https://localhost:5126; Vault: http://127.0.0.1:18200')
+    print('Local CA: .local/dev/ca.pem. Email driver: ' + vault.kv('email')['driver'])

dev/config/bff.yml

@@ -0,0 +1,39 @@
+# SXCHNGE-owned Mongo database and JWT key. Never point these at Sendico.
+http_server:
+  listen_address: 0.0.0.0:8082
+  read_header_timeout: 5
+  shutdown_timeout: 10
+database:
+  driver: mongodb
+  settings:
+    uri_env: SXCHNGE_MONGO_URI
+    database: sxchnge
+api:
+  middleware:
+    domain_env: SXCHNGE_DOMAIN
+    api_endpoint_env: SXCHNGE_API_ENDPOINT
+    api_protocol_env: SXCHNGE_API_PROTOCOL
+    signature:
+      algorithm: HS256
+      secret_key_env: SXCHNGE_JWT_SECRET
+    CORS:
+      allowed_origins: ["https://localhost:5126"]
+      allowed_methods: ["GET", "POST", "PATCH", "DELETE", "OPTIONS"]
+      allowed_headers: ["Content-Type", "Authorization", "X-Auth-Transport", "X-CSRF-Protection"]
+      allow_credentials: true
+      max_age: 300
+    token:
+      expiration_hours:
+        account: 1
+        refresh: 720
+      length: 32
+    password:
+      token_length: 32
+      check:
+        min_length: 12
+  notifications:
+    address: notifications:8083
+    token_env: SXCHNGE_NOTIFICATIONS_TOKEN
+    ca_file: /run/sxchnge/ca.pem
+    server_name: notifications
+    activation_url: https://localhost:5126/verify-email

dev/config/nginx.conf

@@ -0,0 +1,17 @@
+server {
+    listen 8443 ssl;
+    ssl_certificate /etc/tls/server.crt;
+    ssl_certificate_key /etc/tls/server.key;
+    ssl_protocols TLSv1.2 TLSv1.3;
+    root /usr/share/nginx/html;
+    index index.html;
+    location /api/ {
+        resolver 127.0.0.11 valid=10s ipv6=off;
+        set $bff http://bff:8082;
+        proxy_pass $bff;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-Proto https;
+        proxy_set_header X-Forwarded-For $remote_addr;
+    }
+    location / { try_files $uri $uri/ /index.html; }
+}

dev/config/notifications.yml

@@ -0,0 +1,27 @@
+listen: "0.0.0.0:8083"
+tls_cert: "/run/sxchnge/server.crt"
+tls_key: "/run/sxchnge/server.key"
+database:
+  driver: mongodb
+  settings:
+    uri_env: SXCHNGE_MONGO_URI
+    database: sxchnge_notifications
+messaging:
+  driver: NATS
+  settings:
+    url_env: SXCHNGE_NATS_URL
+    broker_name: sxchnge-notifications
+    max_reconnects: -1
+    reconnect_wait: 2
+templates: "/etc/sxchnge/templates.json"
+callers:
+  product: "SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN"
+# Use replicas: 3 and environment: production for a three-node NATS cluster.
+# mode: ops validates resources without modifying them; app provisions them.
+queue:
+  stream:
+    replicas: 1
+    max_age_hours: 48
+  provisioning:
+    mode: app
+    environment: non-production

dev/images.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Resolve stable upstream releases on every build; pin their registry digests."""
+import concurrent.futures
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import urllib.request
+
+ROOT = Path(__file__).resolve().parents[1]
+STATE = ROOT / '.local' / 'dev'
+
+
+def get(url):
+    with urllib.request.urlopen(url, timeout=45) as response:
+        return json.load(response)
+
+
+def image(repo, alias):
+    url = f'https://hub.docker.com/v2/repositories/{repo}/tags'
+    selected = get(f'{url}/{alias}')
+    digest = selected['digest']
+    if not re.fullmatch(r'sha256:[a-f0-9]{64}', digest):
+        raise ValueError('Registry returned an invalid digest')
+    # Find the exact numeric tag represented by the maintained stable alias.
+    candidates = get(f'{url}?page_size=100')['results']
+    names = [x['name'] for x in candidates if x.get('digest') == digest
+             and re.fullmatch(r'\d+\.\d+(?:\.\d+)?(?:-alpine)?', x['name'])]
+    if not names:
+        raise ValueError(f'No stable version tag for {repo}:{alias}')
+    tag = max(names, key=lambda x: tuple(map(int, x.split('-')[0].split('.'))))
+    return f'{repo}:{tag}@{digest}'
+
+
+def flutter():
+    commit = get('https://api.github.com/repos/flutter/flutter/commits/stable')['sha']
+    tags = subprocess.check_output(['git', 'ls-remote', '--tags',
+                                    'https://github.com/flutter/flutter.git'], text=True)
+    names = [line.split('refs/tags/')[1] for line in tags.splitlines()
+             if line.startswith(commit + '\trefs/tags/')
+             and re.fullmatch(r'\d+\.\d+\.\d+', line.split('refs/tags/')[1])]
+    if not names:
+        raise ValueError('Flutter stable does not point at a stable release tag')
+    return max(names, key=lambda x: tuple(map(int, x.split('.')))), commit
+
+
+def main():
+    go = get('https://go.dev/dl/?mode=json')[0]
+    if not go['stable'] or not re.fullmatch(r'go\d+\.\d+\.\d+', go['version']):
+        raise ValueError('Go release is not stable')
+    specs = {'VAULT_IMAGE': ('hashicorp/vault', 'latest'),
+             'MONGO_IMAGE': ('library/mongo', 'latest'),
+             'NATS_IMAGE': ('library/nats', 'latest'),
+             'NGINX_IMAGE': ('library/nginx', 'stable-alpine'),
+             'GO_IMAGE': ('library/golang', go['version'][2:] + '-alpine'),
+             'UBUNTU_IMAGE': ('library/ubuntu', 'latest')}
+    with concurrent.futures.ThreadPoolExecutor() as pool:
+        futures = {key: pool.submit(image, *spec) for key, spec in specs.items()}
+        refs = {key: future.result() for key, future in futures.items()}
+    refs['FLUTTER_VERSION'], refs['FLUTTER_COMMIT'] = flutter()
+    STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
+    target = STATE / 'images.env'
+    temporary = target.with_suffix('.tmp')
+    temporary.write_text(''.join(f'{key}={value}\n' for key, value in refs.items()))
+    os.replace(temporary, target)
+    for key, value in refs.items():
+        print(f'{key}={value}')
+
+
+if __name__ == '__main__':
+    main()

dev/images/backend.Dockerfile

@@ -0,0 +1,21 @@
+ARG GO_IMAGE
+ARG VAULT_IMAGE
+FROM ${GO_IMAGE} AS build
+RUN apk add --no-cache protobuf protobuf-dev make
+WORKDIR /src
+COPY Makefile ./
+COPY backend ./backend
+RUN make generate-notifications
+ARG MODULE
+WORKDIR /src/backend/${MODULE}
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -o /out/service .
+
+FROM ${VAULT_IMAGE}
+USER root
+COPY --from=build /out/service /usr/local/bin/service
+COPY dev/start-service.sh /usr/local/bin/start-service
+COPY dev/config /etc/sxchnge
+COPY backend/notifications/templates.example.json /etc/sxchnge/templates.json
+RUN mkdir -p /run/sxchnge && chown 100:1000 /run/sxchnge
+USER 100:1000
+ENTRYPOINT ["vault", "agent", "-config=/etc/agent/config.hcl"]

dev/images/frontend.Dockerfile

@@ -0,0 +1,17 @@
+ARG UBUNTU_IMAGE
+ARG NGINX_IMAGE
+FROM ${UBUNTU_IMAGE} AS build
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl git unzip xz-utils libglu1-mesa && rm -rf /var/lib/apt/lists/*
+ARG FLUTTER_VERSION
+ARG FLUTTER_COMMIT
+RUN git clone --depth 1 --branch "${FLUTTER_VERSION}" https://github.com/flutter/flutter.git /opt/flutter && test "$(git -C /opt/flutter rev-parse HEAD)" = "${FLUTTER_COMMIT}"
+ENV PATH="/opt/flutter/bin:${PATH}"
+RUN flutter config --no-analytics && flutter precache --web
+WORKDIR /src
+COPY frontend ./frontend
+WORKDIR /src/frontend/app
+ARG FRONTEND_URL=https://localhost:5126
+RUN flutter pub get && flutter gen-l10n && flutter build web --no-wasm-dry-run --dart-define=SXCHNGE_API_URL=${FRONTEND_URL}/api/v1 --dart-define=SXCHNGE_CLIENT_ID=sxchnge-local-web
+FROM ${NGINX_IMAGE}
+COPY dev/config/nginx.conf /etc/nginx/conf.d/default.conf
+COPY --from=build /src/frontend/app/build/web /usr/share/nginx/html

dev/images/nats.Dockerfile

@@ -0,0 +1,10 @@
+ARG NATS_IMAGE
+ARG VAULT_IMAGE
+FROM ${NATS_IMAGE} AS nats
+FROM ${VAULT_IMAGE}
+USER root
+COPY --from=nats /nats-server /usr/local/bin/nats-server
+COPY dev/start-nats.sh /usr/local/bin/start-nats
+RUN mkdir -p /data /run/sxchnge && chown -R 100:1000 /data /run/sxchnge
+USER 100:1000
+ENTRYPOINT ["vault", "agent", "-config=/etc/agent/config.hcl"]

dev/local.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Local stack lifecycle. Requires Python 3 and Docker Compose."""
+import sys
+from bootstrap import up
+from vault import Vault
+
+
+def main():
+    vault = Vault()
+    if len(sys.argv) == 2 and sys.argv[1] == 'up':
+        import fcntl
+        from vault import STATE
+        STATE.mkdir(parents=True, exist_ok=True, mode=0o700)
+        with (STATE / 'bootstrap.lock').open('w') as lock:
+            try:
+                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError:
+                raise RuntimeError('Another local bootstrap is running') from None
+            up(vault)
+    elif len(sys.argv) == 2 and sys.argv[1] == 'verify':
+        from verify import verify
+        vault.existing()
+        verify(vault)
+    elif len(sys.argv) == 2 and sys.argv[1] == 'verify-infra':
+        from verify import infrastructure
+        vault.existing()
+        infrastructure(vault)
+    elif len(sys.argv) == 2 and sys.argv[1] == 'email':
+        import getpass
+        vault.existing()
+        address = input('Verified Resend sender address: ').strip()
+        key = getpass.getpass('Resend API key (hidden): ').strip()
+        if not key or '@' not in address:
+            raise RuntimeError('A key and sender address are required')
+        vault.put('email', {'driver': 'resend', 'api_key': key,
+                           'sender_address': address, 'sender_name': 'SXCHNGE'})
+        print('Email configuration saved in Vault. Run make up.')
+    else:
+        raise RuntimeError('Usage: python3 dev/local.py up|verify|verify-infra|email')
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except (RuntimeError, OSError) as error:
+        print('Local development setup failed: ' + str(error), file=sys.stderr)
+        sys.exit(1)

dev/start-mongo.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+cp /run/mongo/keyfile /tmp/mongo-keyfile
+chown mongodb:mongodb /tmp/mongo-keyfile
+chmod 400 /tmp/mongo-keyfile
+exec docker-entrypoint.sh "$@"

dev/start-nats.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+umask 077
+# Generated by Vault with hexadecimal encoding; never interpolates user input.
+case "$NATS_TOKEN" in ''|*[!0-9a-f]*) exit 1;; esac
+cat > /run/sxchnge/nats.conf <<CONFIG
+port: 4222
+http_port: 8222
+authorization { token: "$NATS_TOKEN" }
+jetstream { store_dir: "/data/jetstream" }
+CONFIG
+unset NATS_TOKEN
+exec nats-server -c /run/sxchnge/nats.conf

dev/start-service.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+umask 077
+printf '%s\n' "$TLS_CA" > /run/sxchnge/ca.pem
+unset TLS_CA
+if [ "$SXCHNGE_SERVICE" = notifications ]; then
+    printf '%s\n' "$TLS_CERT" > /run/sxchnge/server.crt
+    printf '%s\n' "$TLS_KEY" > /run/sxchnge/server.key
+    printf '%s\n' "$EMAIL_CONFIG" > /run/sxchnge/email.yml
+    unset TLS_CERT TLS_KEY EMAIL_CONFIG
+    cat /etc/sxchnge/notifications.yml /run/sxchnge/email.yml > /run/sxchnge/config.yml
+    exec service -config.file=/run/sxchnge/config.yml
+fi
+exec service -config.file=/etc/sxchnge/bff.yml

dev/start-vault.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+# Local-only automatic unseal. The root token is never mounted in this container.
+set -eu
+export VAULT_ADDR=http://127.0.0.1:8200
+vault server -config=/etc/vault/server.hcl &
+pid=$!
+trap 'kill -TERM "$pid"; wait "$pid"' TERM INT
+until [ -s /bootstrap/unseal ]; do
+    kill -0 "$pid" || exit 1
+    sleep 1
+done
+until vault operator unseal "$(cat /bootstrap/unseal)" >/dev/null 2>&1; do
+    kill -0 "$pid" || exit 1
+    sleep 1
+done
+wait "$pid"

dev/test_local.py

@@ -0,0 +1,62 @@
+"""Focused regression tests for release selection and bootstrap failure paths."""
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import bootstrap
+import images
+from vault import Vault
+
+
+class Releases(unittest.TestCase):
+    def test_selects_exact_stable_digest_not_prerelease(self):
+        digest = 'sha256:' + 'a' * 64
+        versions = [{'name': name, 'digest': digest} for name in ['latest', '2', '2.1', '2.1.0', '2.2.0-rc1']]
+        with patch.object(images, 'get', side_effect=[{'digest': digest}, {'results': versions}]):
+            self.assertEqual(images.image('hashicorp/vault', 'latest'), 'hashicorp/vault:2.1.0@' + digest)
+
+    def test_missing_stable_mapping_fails(self):
+        digest = 'sha256:' + 'a' * 64
+        with patch.object(images, 'get', side_effect=[{'digest': digest}, {'results': [{'name': '3.0.0-rc1', 'digest': digest}]}]):
+            with self.assertRaises(ValueError):
+                images.image('hashicorp/vault', 'latest')
+
+    def test_upstream_failure_does_not_replace_existing_pins(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            state = Path(temporary)
+            target = state / 'images.env'
+            target.write_text('previous build')
+            with patch.object(images, 'STATE', state), patch.object(images, 'get', side_effect=OSError('offline')):
+                with self.assertRaises(OSError):
+                    images.main()
+            self.assertEqual(target.read_text(), 'previous build')
+
+
+class Bootstrap(unittest.TestCase):
+    def test_missing_resend_credentials_stops_before_product_start(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            state = Path(temporary)
+            (state / 'images.env').touch()
+            from unittest.mock import Mock
+            vault = Mock()
+            vault.kv.return_value = {'driver': 'resend', 'api_key': '', 'sender_address': 'notifications@localhost.test'}
+            with patch.object(bootstrap, 'STATE', state), patch.object(bootstrap, 'compose') as compose, \
+                 patch.object(bootstrap, 'provision_mongo'), patch.object(bootstrap, 'provision_agents'):
+                with self.assertRaisesRegex(RuntimeError, 'Resend api_key'):
+                    bootstrap.up(vault)
+            self.assertEqual(compose.call_args_list[-1].args[-1], 'nats')
+            email_seed = next(c for c in vault.ensure.call_args_list if c.args[0] == 'email').args[1]()
+            self.assertEqual(email_seed['driver'], 'resend')
+            self.assertEqual(email_seed['api_key'], '')
+
+    def test_initialized_vault_without_local_keys_fails(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            with patch('vault.STATE', Path(temporary)), patch.object(Vault, 'call', return_value={'initialized': True}):
+                with self.assertRaisesRegex(RuntimeError, 'Missing .local'):
+                    Vault().initialize()
+
+
+
+if __name__ == '__main__':
+    unittest.main()

dev/vault.py

@@ -0,0 +1,117 @@
+"""Local Vault bootstrap API; response bodies with secrets never reach logs."""
+import json
+import os
+from pathlib import Path
+import time
+import urllib.error
+import urllib.request
+
+STATE = Path(__file__).resolve().parents[1] / '.local' / 'dev'
+URL = 'http://127.0.0.1:18200/v1/'
+
+
+def write(path, value, mode=0o600):
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    tmp = path.with_name(path.name + '.tmp')
+    fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
+    with os.fdopen(fd, 'w') as out:
+        out.write(value)
+    os.chmod(tmp, mode)
+    os.replace(tmp, path)
+
+
+class Vault:
+    def __init__(self):
+        self.token = None
+
+    def call(self, path, data=None, method=None, missing=False):
+        headers = {'Content-Type': 'application/json'}
+        if self.token:
+            headers['X-Vault-Token'] = self.token
+        req = urllib.request.Request(URL + path, headers=headers,
+            data=json.dumps(data).encode() if data is not None else None, method=method)
+        try:
+            with urllib.request.urlopen(req, timeout=30) as response:
+                body = response.read()
+                return json.loads(body) if body else {}
+        except urllib.error.HTTPError as error:
+            if error.code == 404 and missing:
+                return None
+            raise RuntimeError(f'Vault {path} returned HTTP {error.code}') from None
+
+    def initialize(self):
+        for attempt in range(90):
+            try:
+                initialized = self.call('sys/init')['initialized']
+                break
+            except (urllib.error.URLError, ConnectionError):
+                time.sleep(1)
+        else:
+            raise RuntimeError('Vault did not start within 90 seconds')
+        initfile = STATE / 'admin' / 'init.json'
+        if not initialized:
+            if initfile.exists():
+                raise RuntimeError('Vault data is missing but bootstrap material exists; restore the matching volume or explicitly reset both')
+            init = self.call('sys/init', {'secret_shares': 1, 'secret_threshold': 1})
+            write(initfile, json.dumps(init))
+        if not initfile.exists():
+            raise RuntimeError('Missing .local/dev/admin/init.json for initialized Vault; restore the matching bootstrap material')
+        init = json.loads(initfile.read_text())
+        self.token = init['root_token']
+        write(STATE / 'vault' / 'unseal', init['keys_base64'][0], 0o444)
+        for attempt in range(60):
+            if not self.call('sys/seal-status')['sealed']:
+                return
+            time.sleep(1)
+        raise RuntimeError('Vault did not unseal within 60 seconds')
+
+    def existing(self):
+        self.token = json.loads((STATE / 'admin' / 'init.json').read_text())['root_token']
+
+    def kv(self, path):
+        result = self.call('kv/data/' + path, missing=True)
+        return result['data']['data'] if result else None
+
+    def put(self, path, data):
+        self.call('kv/data/' + path, {'data': data})
+
+    def ensure(self, path, factory):
+        value = self.kv(path)
+        if value is None:
+            value = factory()
+            self.put(path, value)
+        return value
+
+    def random(self, length=32):
+        return self.call('sys/tools/random', {'bytes': length, 'format': 'hex'})['data']['random_bytes']
+
+    def mounts(self):
+        mounts = self.call('sys/mounts')['data']
+        for name, kind in [('kv', 'kv'), ('database', 'database'), ('pki', 'pki')]:
+            if name + '/' not in mounts:
+                settings = {'type': kind}
+                if kind == 'kv':
+                    settings['options'] = {'version': '2'}
+                self.call('sys/mounts/' + name, settings)
+        auth = self.call('sys/auth')['data']
+        if 'approle/' not in auth:
+            self.call('sys/auth/approle', {'type': 'approle'})
+
+    def certificates(self):
+        self.call('sys/mounts/pki/tune', {'max_lease_ttl': '87600h'})
+        ca = self.kv('tls')
+        if ca is None:
+            result = self.call('pki/root/generate/internal', {'common_name': 'SXCHNGE local development CA', 'ttl': '87600h'})['data']
+            ca = {'ca': result['certificate']}
+            self.put('tls', ca)
+        self.call('pki/roles/local', {'allowed_domains': ['localhost', 'notifications'],
+            'allow_bare_domains': True, 'allow_ip_sans': True, 'max_ttl': '720h'})
+        for name, host in [('notifications-tls', 'notifications'), ('frontend-tls', 'localhost')]:
+            cert = self.kv(name)
+            if cert is None or cert['expiration'] < time.time() + 7 * 86400:
+                cert = self.call('pki/issue/local', {'common_name': host, 'ip_sans': '127.0.0.1', 'ttl': '720h'})['data']
+                self.put(name, cert)
+            if name == 'frontend-tls':
+                write(STATE / 'frontend' / 'server.crt', cert['certificate'])
+                write(STATE / 'frontend' / 'server.key', cert['private_key'])
+        write(STATE / 'ca.pem', ca['ca'], 0o644)

dev/vault/agent.hcl

@@ -0,0 +1,14 @@
+vault { address = "http://vault:8200" }
+auto_auth {
+  method "approle" {
+    config = {
+      role_id_file_path = "/etc/agent/role-id"
+      secret_id_file_path = "/etc/agent/secret-id"
+      remove_secret_id_file_after_reading = false
+    }
+  }
+}
+template_config {
+  static_secret_render_interval = "10s"
+  exit_on_retry_failure = true
+}

dev/vault/server.hcl

@@ -0,0 +1,10 @@
+ui = true
+disable_mlock = true
+storage "file" {
+  path = "/vault/file"
+}
+listener "tcp" {
+  address = "0.0.0.0:8200"
+  tls_disable = true
+}
+api_addr = "http://vault:8200"

dev/verify.py

@@ -0,0 +1,72 @@
+"""Integration checks against this project's running stack; never print secrets."""
+import json
+import ssl
+import urllib.request
+
+from bootstrap import compose, mongo
+from vault import STATE
+
+
+def require(condition, message):
+    if not condition:
+        raise RuntimeError(message)
+
+
+def database_access(vault):
+    for role, database, forbidden in [('bff', 'sxchnge', 'sxchnge_notifications'),
+                                      ('notifications', 'sxchnge_notifications', 'sxchnge')]:
+        lease = vault.call('database/creds/' + role)
+        credentials = lease['data']
+        auth = 'db.getSiblingDB("admin").auth(' + json.dumps(credentials['username']) + ',' + json.dumps(credentials['password']) + ');'
+        try:
+            result = compose('exec', '-T', 'mongo', 'mongosh', '--quiet', '--file', '/dev/stdin', capture=True,
+                input=auth + 'const own=db.getSiblingDB(' + json.dumps(database) + '); own.local_smoke.insertOne({_id:"vault-check"}); own.local_smoke.deleteOne({_id:"vault-check"});\n'
+                'let denied=false; try { db.getSiblingDB(' + json.dumps(forbidden) + ').runCommand({find:"local_smoke",filter:{}}); } catch(e) { denied=e.code===13; } if(!denied) throw new Error("cross-database access allowed"); print("scope-ok");')
+            require('scope-ok' in result, 'Mongo service scope check failed')
+            renewed = vault.call('sys/leases/renew', {'lease_id': lease['lease_id'], 'increment': '1h'})
+            require(renewed['lease_duration'] > 0, 'Mongo lease renewal failed')
+        finally:
+            vault.call('sys/leases/revoke', {'lease_id': lease['lease_id']})
+        admin = vault.kv('bootstrap/mongo')['password']
+        result = mongo('print(admin.getUser(' + json.dumps(credentials['username']) + ')===null)', admin)
+        require('true' in result, 'Revoked Mongo user still exists')
+    print('PASS: database-plugin issuance, separate database permissions, lease renewal and revocation')
+
+
+def policies(vault):
+    for service in ['bff', 'notifications', 'nats']:
+        root = STATE / 'agents' / service
+        login = vault.call('auth/approle/login', {'role_id': (root / 'role-id').read_text(),
+                                                'secret_id': (root / 'secret-id').read_text()})
+        token = login['auth']['client_token']
+        try:
+            result = vault.call('sys/capabilities', {'token': token, 'paths': ['kv/data/bootstrap/mongo', 'sys/init']})['data']
+            require(all(result[path] == ['deny'] for path in ['kv/data/bootstrap/mongo', 'sys/init']), 'Service can access bootstrap credentials')
+        finally:
+            vault.call('auth/token/revoke', {'token': token})
+    print('PASS: service identities cannot read bootstrap credentials')
+
+
+def infrastructure(vault):
+    status = compose('ps', '--format', 'json', capture=True)
+    rows = [json.loads(line) for line in status.splitlines() if line.strip()]
+    infra = {r['Service']: r for r in rows if r['Service'] in {'vault', 'mongo', 'nats'}}
+    require(set(infra) == {'vault', 'mongo', 'nats'}, 'Expected Vault, Mongo and NATS')
+    require(all(r['Health'] == 'healthy' for r in infra.values()), 'Infrastructure is unhealthy')
+    print('PASS: Vault, Mongo replica set and NATS JetStream are healthy')
+    policies(vault)
+    database_access(vault)
+
+
+def verify(vault):
+    status = compose('ps', '--format', 'json', capture=True)
+    rows = [json.loads(line) for line in status.splitlines() if line.strip()]
+    require({r['Service'] for r in rows} == {'vault', 'mongo', 'nats', 'notifications', 'bff', 'frontend'}, 'Expected six running services')
+    require(all(r['Health'] == 'healthy' for r in rows), 'Some services are unhealthy')
+    context = ssl.create_default_context(cafile=str(STATE / 'ca.pem'))
+    for path in ['/', '/api/v1/health']:
+        with urllib.request.urlopen('https://localhost:5126' + path, context=context, timeout=10) as response:
+            require(response.status == 200, 'Frontend or BFF health failed')
+    print('PASS: all six services healthy; frontend and proxied BFF use verified TLS')
+    infrastructure(vault)
+    print('Resend credentials are configured; no live email sent by this check')