Keep Mongo operations online across Vault credential rotation

dev/agents.py

@@ -14,10 +14,6 @@ def config(service):
     if service == 'nats':
         env['NATS_TOKEN'] = kv('nats', 'token')
     else:
-        database = 'sxchnge' if service == 'bff' else 'sxchnge_notifications'
-        env['SXCHNGE_MONGO_URI'] = ('{{ with secret "database/creds/' + service + '" }}'
-            'mongodb://{{ .Data.username | urlquery }}:{{ .Data.password | urlquery }}'
-            '@mongo:27017/' + database + '?authSource=admin&replicaSet=rs0{{ end }}')
         env['TLS_CA'] = kv('tls', 'ca')
         if service == 'bff':
             env['SXCHNGE_JWT_SECRET'] = kv('bff', 'jwt')

dev/config/bff.yml

@@ -6,7 +6,12 @@ http_server:
 database:
   driver: mongodb
   settings:
-    uri_env: SXCHNGE_MONGO_URI
+    uri: mongodb://mongo:27017/sxchnge?authSource=admin&replicaSet=rs0
+    vault:
+      address: http://vault:8200
+      role: bff
+      role_id_file: /etc/agent/role-id
+      secret_id_file: /etc/agent/secret-id
     database: sxchnge
 api:
   middleware:

dev/config/notifications.yml

@@ -4,7 +4,12 @@ tls_key: "/run/sxchnge/server.key"
 database:
   driver: mongodb
   settings:
-    uri_env: SXCHNGE_MONGO_URI
+    uri: mongodb://mongo:27017/sxchnge_notifications?authSource=admin&replicaSet=rs0
+    vault:
+      address: http://vault:8200
+      role: notifications
+      role_id_file: /etc/agent/role-id
+      secret_id_file: /etc/agent/secret-id
     database: sxchnge_notifications
 messaging:
   driver: NATS

dev/rotation.py

@@ -0,0 +1,92 @@
+"""Exercise real Vault database rotation under BFF HTTP transaction traffic.
+
+Run after make build/bootstrap. No Resend credential or live email is needed.
+Temporarily shortens the BFF database role TTL, restores it on every exit, and
+preserves whether BFF/frontend were running before this check.
+"""
+import concurrent.futures
+import json
+import ssl
+import threading
+import time
+import urllib.error
+import urllib.request
+import uuid
+
+from bootstrap import compose
+from vault import STATE, Vault
+
+
+def verify_rotation():
+    vault = Vault()
+    vault.existing()
+    role = 'database/roles/bff'
+    original = vault.call(role)['data']
+    running = {json.loads(line)['Service'] for line in compose('ps', '--format', 'json', capture=True).splitlines()}
+    stop = threading.Event()
+    successes = []
+    failures = []
+    lock = threading.Lock()
+    context = ssl.create_default_context(cafile=str(STATE / 'ca.pem'))
+
+    def requests():
+        while not stop.is_set():
+            # Exactly 43 characters invokes the activation transaction and token
+            # lookup, then returns the expected invalid-token response. No user
+            # or notification is created and the test never sends an email.
+            token = uuid.uuid4().hex + uuid.uuid4().hex[:11]
+            request = urllib.request.Request('https://localhost:5126/api/v1/accounts/verification/confirm',
+                data=json.dumps({'token': token}).encode(), headers={
+                    'Content-Type': 'application/json', 'Origin': 'https://localhost:5126',
+                    'X-Auth-Transport': 'cookie', 'X-CSRF-Protection': '1'})
+            try:
+                with urllib.request.urlopen(request, context=context, timeout=5) as response:
+                    code = response.status
+                    body = response.read()
+            except urllib.error.HTTPError as error:
+                code = error.code
+                body = error.read()
+            except Exception:
+                code = 0
+                body = b''
+            with lock:
+                (successes if code == 400 and b'Verification is invalid or already used' in body else failures).append(code)
+            stop.wait(0.03)
+
+    try:
+        vault.call(role, dict(original, default_ttl='12s', max_ttl='24s'))
+        compose('up', '-d', '--no-deps', '--force-recreate', '--wait', '--wait-timeout', '90', 'bff', 'frontend')
+        process = compose('exec', '-T', 'bff', 'pidof', 'service', capture=True).strip()
+        with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
+            workers = [pool.submit(requests) for _ in range(4)]
+            try:
+                deadline = time.monotonic() + 40
+                while time.monotonic() < deadline:
+                    time.sleep(1)
+                    current = compose('exec', '-T', 'bff', 'pidof', 'service', capture=True).strip()
+                    if current != process:
+                        raise RuntimeError('BFF process restarted during credential rotation')
+                    logs = compose('logs', '--no-color', 'bff', capture=True)
+                    if logs.count('Database pool replaced without restarting service') >= 3:
+                        break
+                else:
+                    raise RuntimeError('Three database pool rotations were not observed')
+            finally:
+                stop.set()
+                for worker in workers:
+                    worker.result()
+        if failures or len(successes) < 100:
+            raise RuntimeError(f'Rotation traffic check: {len(successes)} expected responses, {len(failures)} failures')
+        print(f'PASS: three real Vault rotations, unchanged BFF PID, {len(successes)} transaction-backed HTTP requests, zero failures')
+    finally:
+        stop.set()
+        vault.call(role, original)
+        newly_started = [name for name in ['frontend', 'bff'] if name not in running]
+        if newly_started:
+            compose('stop', *newly_started)
+            compose('rm', '-f', *newly_started)
+        # Existing services stay online; their next refresh picks up normal TTLs.
+
+
+if __name__ == '__main__':
+    verify_rotation()

dev/test_local.py

@@ -4,6 +4,7 @@ import tempfile
 import unittest
 from unittest.mock import patch
 
+import agents
 import bootstrap
 import images
 from vault import Vault
@@ -33,6 +34,14 @@ class Releases(unittest.TestCase):
             self.assertEqual(target.read_text(), 'previous build')
 
 
+class AgentContract(unittest.TestCase):
+    def test_database_lease_does_not_trigger_process_restart(self):
+        for service in ['bff', 'notifications']:
+            config = agents.config(service)
+            self.assertNotIn('database/creds/', config)
+            self.assertNotIn('SXCHNGE_MONGO_URI', config)
+
+
 class Bootstrap(unittest.TestCase):
     def test_missing_resend_credentials_stops_before_product_start(self):
         with tempfile.TemporaryDirectory() as temporary: