"""Exercise real Vault database rotation under BFF HTTP transaction traffic. Run after make build/bootstrap. No Resend credential or live email is needed. Temporarily shortens the BFF database role TTL, restores it on every exit, and preserves whether BFF/frontend were running before this check. """ import concurrent.futures import json import ssl import threading import time import urllib.error import urllib.request import uuid from bootstrap import compose from vault import STATE, Vault def verify_rotation(): vault = Vault() vault.existing() role = 'database/roles/bff' original = vault.call(role)['data'] running = {json.loads(line)['Service'] for line in compose('ps', '--format', 'json', capture=True).splitlines()} stop = threading.Event() successes = [] failures = [] lock = threading.Lock() context = ssl.create_default_context(cafile=str(STATE / 'ca.pem')) def requests(): while not stop.is_set(): # Exactly 43 characters invokes the activation transaction and token # lookup, then returns the expected invalid-token response. No user # or notification is created and the test never sends an email. token = uuid.uuid4().hex + uuid.uuid4().hex[:11] request = urllib.request.Request('https://localhost:5126/api/v1/accounts/verification/confirm', data=json.dumps({'token': token}).encode(), headers={ 'Content-Type': 'application/json', 'Origin': 'https://localhost:5126', 'X-Auth-Transport': 'cookie', 'X-CSRF-Protection': '1'}) try: with urllib.request.urlopen(request, context=context, timeout=5) as response: code = response.status body = response.read() except urllib.error.HTTPError as error: code = error.code body = error.read() except Exception: code = 0 body = b'' with lock: (successes if code == 400 and b'Verification is invalid or already used' in body else failures).append(code) stop.wait(0.03) try: vault.call(role, dict(original, default_ttl='12s', max_ttl='24s')) compose('up', '-d', '--no-deps', '--force-recreate', '--wait', '--wait-timeout', '90', 'bff', 'frontend') process = compose('exec', '-T', 'bff', 'pidof', 'service', capture=True).strip() with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: workers = [pool.submit(requests) for _ in range(4)] try: deadline = time.monotonic() + 40 while time.monotonic() < deadline: time.sleep(1) current = compose('exec', '-T', 'bff', 'pidof', 'service', capture=True).strip() if current != process: raise RuntimeError('BFF process restarted during credential rotation') logs = compose('logs', '--no-color', 'bff', capture=True) if logs.count('Database pool replaced without restarting service') >= 3: break else: raise RuntimeError('Three database pool rotations were not observed') finally: stop.set() for worker in workers: worker.result() if failures or len(successes) < 100: raise RuntimeError(f'Rotation traffic check: {len(successes)} expected responses, {len(failures)} failures') print(f'PASS: three real Vault rotations, unchanged BFF PID, {len(successes)} transaction-backed HTTP requests, zero failures') finally: stop.set() vault.call(role, original) newly_started = [name for name in ['frontend', 'bff'] if name not in running] if newly_started: compose('stop', *newly_started) compose('rm', '-f', *newly_started) # Existing services stay online; their next refresh picks up normal TTLs. if __name__ == '__main__': verify_rotation()