Keep Mongo operations online across Vault credential rotation

dev/rotation.py

@@ -0,0 +1,92 @@
+"""Exercise real Vault database rotation under BFF HTTP transaction traffic.
+
+Run after make build/bootstrap. No Resend credential or live email is needed.
+Temporarily shortens the BFF database role TTL, restores it on every exit, and
+preserves whether BFF/frontend were running before this check.
+"""
+import concurrent.futures
+import json
+import ssl
+import threading
+import time
+import urllib.error
+import urllib.request
+import uuid
+
+from bootstrap import compose
+from vault import STATE, Vault
+
+
+def verify_rotation():
+    vault = Vault()
+    vault.existing()
+    role = 'database/roles/bff'
+    original = vault.call(role)['data']
+    running = {json.loads(line)['Service'] for line in compose('ps', '--format', 'json', capture=True).splitlines()}
+    stop = threading.Event()
+    successes = []
+    failures = []
+    lock = threading.Lock()
+    context = ssl.create_default_context(cafile=str(STATE / 'ca.pem'))
+
+    def requests():
+        while not stop.is_set():
+            # Exactly 43 characters invokes the activation transaction and token
+            # lookup, then returns the expected invalid-token response. No user
+            # or notification is created and the test never sends an email.
+            token = uuid.uuid4().hex + uuid.uuid4().hex[:11]
+            request = urllib.request.Request('https://localhost:5126/api/v1/accounts/verification/confirm',
+                data=json.dumps({'token': token}).encode(), headers={
+                    'Content-Type': 'application/json', 'Origin': 'https://localhost:5126',
+                    'X-Auth-Transport': 'cookie', 'X-CSRF-Protection': '1'})
+            try:
+                with urllib.request.urlopen(request, context=context, timeout=5) as response:
+                    code = response.status
+                    body = response.read()
+            except urllib.error.HTTPError as error:
+                code = error.code
+                body = error.read()
+            except Exception:
+                code = 0
+                body = b''
+            with lock:
+                (successes if code == 400 and b'Verification is invalid or already used' in body else failures).append(code)
+            stop.wait(0.03)
+
+    try:
+        vault.call(role, dict(original, default_ttl='12s', max_ttl='24s'))
+        compose('up', '-d', '--no-deps', '--force-recreate', '--wait', '--wait-timeout', '90', 'bff', 'frontend')
+        process = compose('exec', '-T', 'bff', 'pidof', 'service', capture=True).strip()
+        with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:
+            workers = [pool.submit(requests) for _ in range(4)]
+            try:
+                deadline = time.monotonic() + 40
+                while time.monotonic() < deadline:
+                    time.sleep(1)
+                    current = compose('exec', '-T', 'bff', 'pidof', 'service', capture=True).strip()
+                    if current != process:
+                        raise RuntimeError('BFF process restarted during credential rotation')
+                    logs = compose('logs', '--no-color', 'bff', capture=True)
+                    if logs.count('Database pool replaced without restarting service') >= 3:
+                        break
+                else:
+                    raise RuntimeError('Three database pool rotations were not observed')
+            finally:
+                stop.set()
+                for worker in workers:
+                    worker.result()
+        if failures or len(successes) < 100:
+            raise RuntimeError(f'Rotation traffic check: {len(successes)} expected responses, {len(failures)} failures')
+        print(f'PASS: three real Vault rotations, unchanged BFF PID, {len(successes)} transaction-backed HTTP requests, zero failures')
+    finally:
+        stop.set()
+        vault.call(role, original)
+        newly_started = [name for name in ['frontend', 'bff'] if name not in running]
+        if newly_started:
+            compose('stop', *newly_started)
+            compose('rm', '-f', *newly_started)
+        # Existing services stay online; their next refresh picks up normal TTLs.
+
+
+if __name__ == '__main__':
+    verify_rotation()