Organize local infrastructure and Vault secrets by service ownership

infra/local/scripts/bootstrap.sh

@@ -0,0 +1,128 @@
+initialize() {
+ attempt=0
+ until initialized=$(vault sys/init | jq -er '.initialized|tostring'); do
+  attempt=$((attempt+1)); [ "$attempt" -lt 90 ] || fail 'Vault did not start'; sleep 1
+ done
+ if [ "$initialized" = false ]; then
+  [ ! -e "$STATE/admin/init.json" ] || fail 'Vault data missing but bootstrap material exists; restore matching volume or reset both'
+  printf '{"secret_shares":1,"secret_threshold":1}' | vault sys/init POST | write "$STATE/admin/init.json"
+ fi
+ existing
+ jq -r '.keys_base64[0]' "$STATE/admin/init.json" | write "$STATE/vault/unseal" 444
+ attempt=0
+ until vault sys/seal-status | jq -e '.sealed==false' >/dev/null; do
+  attempt=$((attempt+1)); [ "$attempt" -lt 60 ] || fail 'Vault did not unseal'; sleep 1
+ done
+}
+mounts() {
+ vault sys/mounts > "$WORK/mounts"
+ for name in kv database; do
+  if ! jq -e --arg key "$name/" 'has($key)' < "$WORK/mounts" >/dev/null; then
+   if ! jq -e --arg key "$name/" '.data|has($key)' < "$WORK/mounts" >/dev/null; then
+    jq -n --arg name "$name" '{type:$name} + (if $name=="kv" then {options:{version:"2"}} else {} end)' | vault "sys/mounts/$name" POST >/dev/null
+   fi
+  fi
+ done
+ if ! vault sys/auth | jq -e '.data|has("approle/")' >/dev/null; then
+  printf '{"type":"approle"}' | vault sys/auth/approle POST >/dev/null
+ fi
+}
+provision_mongo() {
+ if [ "$(kv bootstrap/mongo)" = null ]; then
+  random > "$WORK/password"; random 384 > "$WORK/keyfile"; random > "$WORK/manager"
+  jq -n --rawfile password "$WORK/password" --rawfile keyfile "$WORK/keyfile" --rawfile manager "$WORK/manager" '{password:($password|rtrimstr("\n")),keyfile:($keyfile|rtrimstr("\n")),manager_password:($manager|rtrimstr("\n"))}' | put bootstrap/mongo
+ fi
+ kv bootstrap/mongo > "$WORK/mongo-secret"
+ jq -r '.password' "$WORK/mongo-secret" | write "$STATE/mongo/password"
+ jq -r '.keyfile' "$WORK/mongo-secret" | write "$STATE/mongo/keyfile"
+ compose up -d mongo
+ admin_auth > "$WORK/admin.js"
+ attempt=0
+ while :; do
+  if { cat "$WORK/admin.js"; printf 'try {rs.status()} catch(e) {if(e.code===94) rs.initiate({_id:"rs0",members:[{_id:0,host:"mongo:27017"}]}); else throw e;} print(db.hello().isWritablePrimary);'; } | mongo > "$WORK/primary" && grep -q true "$WORK/primary"; then break; fi
+  attempt=$((attempt+1)); [ "$attempt" -lt 90 ] || fail 'Mongo replica set did not become primary'; sleep 2
+ done
+ if [ "$(vault database/config/mongo GET yes)" = null ]; then
+  { cat "$WORK/admin.js"; jq -r '"if(!admin.getUser(\"vault-manager\")) admin.createUser({user:\"vault-manager\",pwd:"+(.manager_password|tojson)+",roles:[{role:\"userAdminAnyDatabase\",db:\"admin\"}]});"' "$WORK/mongo-secret"; } | mongo >/dev/null || fail 'Mongo manager provisioning failed'
+  jq '{plugin_name:"mongodb-database-plugin",allowed_roles:["bff","notifications"],connection_url:"mongodb://{{username}}:{{password}}@mongo:27017/admin?replicaSet=rs0",username:"vault-manager",password:.manager_password}' "$WORK/mongo-secret" | vault database/config/mongo POST >/dev/null
+  printf '{}' | vault database/rotate-root/mongo POST >/dev/null
+ fi
+ for role in bff notifications; do
+  database=sxchnge; [ "$role" != notifications ] || database=sxchnge_notifications
+  jq -n --arg database "$database" '{db_name:"mongo",creation_statements:({db:"admin",roles:[{role:"readWrite",db:$database}]}|tojson),default_ttl:"1h",max_ttl:"24h"}' | vault "database/roles/$role" POST >/dev/null
+ done
+ { cat "$WORK/admin.js"; cat infra/local/mongo/mongo-client.js; } | mongo >/dev/null || fail 'Browser client provisioning failed'
+}
+provision_agents() {
+ for service in bff notifications nats; do
+  directory="$STATE/agents/$service"
+  mkdir -p "$directory"; chmod 755 "$directory"
+  case "$service" in
+   bff) paths='services/bff/jwt services/notifications/callers/bff';;
+   notifications) paths='services/notifications/callers/bff infrastructure/nats/auth services/notifications/email/resend';;
+   nats) paths=infrastructure/nats/auth;;
+  esac
+  : > "$WORK/policy"
+  for path in $paths; do printf 'path "kv/data/%s" { capabilities = ["read"] }\n' "$path" >> "$WORK/policy"; done
+  if [ "$service" != nats ]; then printf 'path "database/creds/%s" { capabilities = ["read"] }\n' "$service" >> "$WORK/policy"; fi
+  jq -n --rawfile policy "$WORK/policy" '{policy:$policy}' | vault "sys/policies/acl/$service" POST >/dev/null
+  jq -n --arg service "$service" '{token_policies:[$service],token_period:"1h",secret_id_ttl:"0",secret_id_num_uses:0}' | vault "auth/approle/role/$service" POST >/dev/null
+  vault "auth/approle/role/$service/role-id" | jq -r '.data.role_id' | write "$directory/role-id" 444
+  if [ ! -f "$directory/secret-id" ]; then
+   printf '{}' | vault "auth/approle/role/$service/secret-id" POST | jq -r '.data.secret_id' | write "$directory/secret-id" 444
+  fi
+  if ! cmp -s "infra/local/vault/$service.hcl" "$directory/config.hcl"; then
+   cat "infra/local/vault/$service.hcl" | write "$directory/config.hcl" 444
+   if compose ps --status running --services "$service" | grep -qx "$service"; then
+    compose up -d --no-deps --force-recreate --wait --wait-timeout 120 "$service"
+   fi
+  fi
+ done
+}
+up() {
+ [ -f "$STATE/images.env" ] || fail 'Run make build before make up'
+ exec 9> "$STATE/bootstrap.lock"
+ flock -n 9 || fail 'Another local bootstrap is running'
+ mkdir -p "$STATE/vault"; chmod 755 "$STATE/vault"
+ compose up -d vault
+ # On first bootstrap Compose creates the network after this tool container starts.
+ if ! docker inspect "$HOSTNAME" --format '{{json .NetworkSettings.Networks}}' | jq -e 'has("sxchnge-local_default")' >/dev/null; then
+  docker network connect sxchnge-local_default "$HOSTNAME"
+ fi
+ initialize
+ mounts
+ migrate_secrets
+ for path in infrastructure/nats/auth services/notifications/callers/bff services/bff/jwt; do
+  if [ "$(kv "$path")" = null ]; then
+   field=token; [ "$path" != services/bff/jwt ] || field=jwt
+   random | jq -Rs --arg field "$field" '{($field):rtrimstr("\n")}' | put "$path"
+  fi
+ done
+ printf '{"driver":"resend","api_key":"","sender_address":"notifications@localhost.test","sender_name":"SXCHNGE local"}' | ensure services/notifications/email/resend
+ provision_mongo
+ provision_agents
+ compose up -d --wait --wait-timeout 120 nats
+ retire_secrets
+ kv services/notifications/email/resend | email_valid || fail 'Vault kv/services/notifications/email/resend needs a Resend api_key and verified sender_address. Run make email, then make up.'
+ compose up -d --wait --wait-timeout 180
+ export_ca
+ echo 'Local stack ready: https://localhost:5126; Vault: http://127.0.0.1:18200'
+}
+email() {
+ existing
+ printf 'Verified Resend sender address: ' >&2
+ IFS= read -r address
+ printf 'Resend API key (hidden): ' >&2
+ tty_state=$(stty -g)
+ trap 'stty "$tty_state"; cleanup' EXIT
+ stty -echo
+ IFS= read -r key
+ stty "$tty_state"
+ printf '\n' >&2
+ [ -n "$key" ] || fail 'A key is required'
+ case "$address" in *@*) ;; *) fail 'A sender email address is required';; esac
+ printf '%s' "$key" > "$WORK/email-key"
+ printf '%s' "$address" > "$WORK/email-address"
+ jq -n --rawfile key "$WORK/email-key" --rawfile address "$WORK/email-address" '{driver:"resend",api_key:$key,sender_address:$address,sender_name:"SXCHNGE"}' | put services/notifications/email/resend
+ echo 'Email configuration saved in Vault. Run make up.'
+}

infra/local/scripts/common.sh

@@ -0,0 +1,63 @@
+# Shared container-side primitives. Secret JSON travels through stdin or private files.
+set -eu
+set -o pipefail
+umask 077
+STATE="$PWD/.local/dev"
+TOOL_STATE=$STATE
+mkdir -p "$STATE"
+WORK=$(mktemp -d)
+cleanup() {
+ rm -rf "$WORK"
+ chown -R "${LOCAL_UID:-0}:${LOCAL_GID:-0}" "$TOOL_STATE"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+fail() { echo "$*" >&2; exit 1; }
+compose() { docker compose --project-directory . -f infra/local/compose.yml --env-file "$STATE/images.env" "$@"; }
+write() {
+ target=$1; mode=${2:-600}
+ mkdir -p "$(dirname "$target")"
+ cat > "$target.tmp"
+ chmod "$mode" "$target.tmp"
+ mv "$target.tmp" "$target"
+}
+# Token is never placed in curl's arguments, errors or response logs.
+vault() (
+ path=$1; method=${2:-GET}; missing=${3:-no}
+ header=$(mktemp "$WORK/header.XXXXXX")
+ response=$(mktemp "$WORK/response.XXXXXX")
+ trap 'rm -f "$header" "$response"' EXIT
+ printf 'Content-Type: application/json\n' > "$header"
+ if [ -n "${VAULT_TOKEN:-}" ]; then printf 'X-Vault-Token: %s\n' "$VAULT_TOKEN" >> "$header"; fi
+ if [ "$method" = GET ]; then
+  code=$(curl -s --connect-timeout 5 --max-time 30 -H "@$header" -o "$response" -w '%{http_code}' "http://vault:8200/v1/$path") || exit 1
+ else
+  code=$(curl -s --connect-timeout 5 --max-time 30 -H "@$header" -X "$method" --data-binary @- -o "$response" -w '%{http_code}' "http://vault:8200/v1/$path") || exit 1
+ fi
+ case "$code" in
+  200|204) if [ -s "$response" ]; then cat "$response"; else printf '{}'; fi;;
+  404) if [ "$missing" = yes ]; then printf 'null'; else echo "Vault $path returned HTTP $code" >&2; exit 1; fi;;
+  *) echo "Vault $path returned HTTP $code" >&2; exit 1;;
+ esac
+)
+existing() {
+ [ -f "$STATE/admin/init.json" ] || fail 'Missing .local/dev/admin/init.json; restore matching bootstrap material'
+ VAULT_TOKEN=$(jq -er '.root_token' "$STATE/admin/init.json")
+ export VAULT_TOKEN
+}
+kv() { vault "kv/data/$1" GET yes | jq 'if . == null then null else .data.data end'; }
+put() { jq '{data:.}' | vault "kv/data/$1" POST >/dev/null; }
+random() { printf '{"bytes":%s,"format":"hex"}' "${1:-32}" | vault sys/tools/random POST | jq -er '.data.random_bytes'; }
+ensure() (
+ path=$1
+ value=$(kv "$path")
+ if [ "$value" = null ]; then put "$path"; fi
+)
+# Never forward mongosh errors: authentication/command errors may include secrets.
+mongo() { compose exec -T mongo mongosh --quiet --file /dev/stdin > "$WORK/mongo-result" 2> "$WORK/mongo-error" || return 1; cat "$WORK/mongo-result"; }
+admin_auth() { kv bootstrap/mongo | jq -r '"const admin=db.getSiblingDB(\"admin\"); admin.auth(\"local-admin\","+(.password|tojson)+");"'; }
+email_valid() { jq -e '(.api_key|type=="string") and (.api_key|test("\\S")) and (.sender_address|type=="string") and (.sender_address|test("\\S")) and .sender_address!="notifications@localhost.test"' >/dev/null; }
+export_ca() {
+ compose exec -T frontend cat /data/caddy/pki/authorities/local/root.crt | write "$STATE/ca.pem" 644
+}

infra/local/scripts/dispatch.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+set -o pipefail
+. infra/local/scripts/common.sh
+. infra/local/scripts/secrets.sh
+case "${1:-}" in
+ images) . infra/local/scripts/images.sh; images;;
+ up) . infra/local/scripts/bootstrap.sh; up;;
+ email) . infra/local/scripts/bootstrap.sh; email;;
+ verify|verify-infra) . infra/local/scripts/verify.sh; existing; infrastructure; if [ "$1" = verify ]; then product; fi;;
+ rotation) . infra/local/scripts/verify.sh; . infra/local/scripts/rotation.sh; existing; rotation;;
+ test) sh infra/local/scripts/test.sh;;
+ *) fail 'Unknown local tooling command';;
+esac

infra/local/scripts/images.sh

@@ -0,0 +1,24 @@
+# Resolve maintained stable aliases to exact numeric tags and digests.
+get() { curl -fsS --connect-timeout 10 --max-time 60 "$1"; }
+image() (
+ repo=$1; alias=$2
+ digest=$(get "https://hub.docker.com/v2/repositories/$repo/tags/$alias" | jq -er '.digest | select(test("^sha256:[a-f0-9]{64}$"))') || exit 1
+ tag=$(get "https://hub.docker.com/v2/repositories/$repo/tags?page_size=100" | jq -er --arg digest "$digest" '
+ [.results[] | select(.digest==$digest) | .name | select(test("^[0-9]+\\.[0-9]+(\\.[0-9]+)?(-alpine)?$"))]
+ | sort_by(split("-")[0] | split(".") | map(tonumber)) | last // error("No stable numeric tag matches digest")') || exit 1
+ printf '%s:%s@%s\n' "$repo" "$tag" "$digest"
+)
+images() (
+ go=$(get 'https://go.dev/dl/?mode=json' | jq -er '.[0] | select(.stable) | .version | select(test("^go[0-9]+\\.[0-9]+\\.[0-9]+$"))') || exit 1
+ for spec in VAULT_IMAGE:hashicorp/vault:latest MONGO_IMAGE:library/mongo:latest NATS_IMAGE:library/nats:latest CADDY_IMAGE:library/caddy:alpine "GO_IMAGE:library/golang:${go#go}-alpine" UBUNTU_IMAGE:library/ubuntu:latest; do
+  key=${spec%%:*}; rest=${spec#*:}; repo=${rest%:*}; alias=${rest##*:}
+  ref=$(image "$repo" "$alias") || exit 1
+  printf '%s=%s\n' "$key" "$ref" >> "$WORK/images.env"
+ done
+ commit=$(get https://api.github.com/repos/flutter/flutter/commits/stable | jq -er '.sha | select(test("^[a-f0-9]{40}$"))') || exit 1
+ git ls-remote --tags https://github.com/flutter/flutter.git > "$WORK/flutter-tags" || exit 1
+ tag=$(jq -Rrs --arg commit "$commit" '[split("\n")[] | split("\t") | select(.[0]==$commit) | .[1] | sub("^refs/tags/";"") | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))] | sort_by(split(".")|map(tonumber)) | last // error("Flutter stable has no stable release tag")' "$WORK/flutter-tags") || exit 1
+ printf 'FLUTTER_VERSION=%s\nFLUTTER_COMMIT=%s\n' "$tag" "$commit" >> "$WORK/images.env"
+ cat "$WORK/images.env" | write "$STATE/images.env"
+ cat "$STATE/images.env"
+)

infra/local/scripts/rotation.sh

@@ -0,0 +1,66 @@
+# Temporarily deploy BFF/frontend and shorten the role lifetime. Always restore it.
+rotation() (
+ exec 9> "$STATE/bootstrap.lock"
+ flock -n 9 || fail 'Another local bootstrap or rotation check is running'
+ vault database/roles/bff | jq '.data' > "$WORK/original-role"
+ compose ps --services --status running > "$WORK/running"
+ workers=''
+ restore() {
+  touch "$WORK/stop"
+  for pid in $workers; do wait "$pid" || true; done
+  restore_failed=0
+  vault database/roles/bff POST < "$WORK/original-role" >/dev/null || { echo 'ERROR: restore BFF role TTL manually' >&2; restore_failed=1; }
+  for service in frontend bff; do
+   if ! grep -qx "$service" "$WORK/running"; then compose stop "$service"; compose rm -f "$service"; fi
+  done
+  return "$restore_failed"
+ }
+ trap restore EXIT
+ trap 'exit 130' INT
+ trap 'exit 143' TERM
+ jq '.+{default_ttl:"12s",max_ttl:"24s"}' "$WORK/original-role" | vault database/roles/bff POST >/dev/null
+ compose up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend
+ export_ca
+ https -f https://localhost:5126/ > "$WORK/index"
+ grep -q flutter_bootstrap "$WORK/index" || fail 'Frontend bundle is not served'
+ https -f https://localhost:5126/settings > "$WORK/route"
+ cmp -s "$WORK/index" "$WORK/route" || fail 'Frontend route fallback failed'
+ https -f https://localhost:5126/api/v1/health >/dev/null
+ process=$(compose exec -T bff pidof service)
+ for worker in 1 2 3 4; do
+  (
+   while [ ! -f "$WORK/stop" ]; do
+    # 43 characters passes syntax validation, executes a Mongo transaction/read,
+    # and returns invalid-token without creating users or sending email.
+    token=$(openssl rand -hex 22 | cut -c1-43)
+    code=$(printf '{"token":"%s"}' "$token" | https -o "$WORK/body-$worker" -w '%{http_code}' \
+     -H 'Content-Type: application/json' -H 'Origin: https://localhost:5126' \
+     -H 'X-Auth-Transport: cookie' -H 'X-CSRF-Protection: 1' --data-binary @- \
+     https://localhost:5126/api/v1/accounts/verification/confirm) || code=0
+    if [ "$code" = 400 ] && grep -q 'Verification is invalid or already used' "$WORK/body-$worker"; then
+     echo ok >> "$WORK/results-$worker"
+    else echo fail >> "$WORK/results-$worker"; fi
+    sleep 0.03
+   done
+  ) &
+  workers="$workers $!"
+ done
+ count=0; attempt=0
+ while [ "$attempt" -lt 40 ]; do
+  sleep 1
+  [ "$(compose exec -T bff pidof service)" = "$process" ] || fail 'BFF restarted during rotation'
+  compose logs --no-color bff > "$WORK/logs"
+  count=$(grep -c 'Database pool replaced without restarting service' "$WORK/logs" || true)
+  [ "$count" -lt 3 ] || break
+  attempt=$((attempt+1))
+ done
+ touch "$WORK/stop"
+ for pid in $workers; do wait "$pid"; done
+ workers=''
+ [ "$count" -ge 3 ] || fail 'Three pool rotations were not observed'
+ cat "$WORK"/results-* > "$WORK/results"
+ ! grep -q fail "$WORK/results" || fail 'Unexpected HTTP responses during rotation'
+ count=$(wc -l < "$WORK/results")
+ [ "$count" -ge 100 ] || fail 'Insufficient rotation traffic'
+ echo "PASS: three real Vault rotations, unchanged BFF PID, $count transaction-backed HTTP requests, zero failures"
+)

infra/local/scripts/secrets.sh

@@ -0,0 +1,56 @@
+# Canonical paths follow ownership. Shared callers live under the receiving service.
+secret_moves() {
+ cat <<'PATHS'
+bff services/bff/jwt
+caller services/notifications/callers/bff
+email services/notifications/email/resend
+nats infrastructure/nats/auth
+PATHS
+}
+# Archive readable version history privately before removing an obsolete KV path.
+archive_secret() (
+ old=$1
+ metadata=$(vault "kv/metadata/$old" GET yes)
+ [ "$metadata" != null ] || exit 0
+ directory="$STATE/migrations/organized-v1/$old"
+ mkdir -p "$directory"
+ printf '%s' "$metadata" | write "$directory/metadata.json"
+ printf '%s' "$metadata" | jq -r '.data.versions|to_entries[]|select(.value.destroyed==false and .value.deletion_time=="")|.key' > "$WORK/versions"
+ while IFS= read -r version; do
+  vault "kv/data/$old?version=$version" | write "$directory/version-$version.json"
+ done < "$WORK/versions"
+)
+migrate_secrets() (
+ secret_moves > "$WORK/moves"
+ while read -r old new; do
+  kv "$old" > "$WORK/old"
+  [ "$(cat "$WORK/old")" != null ] || continue
+  archive_secret "$old"
+  kv "$new" > "$WORK/new"
+  if [ "$(cat "$WORK/new")" = null ]; then
+   # CAS=0 prevents overwriting a destination created concurrently.
+   jq '{data:.,options:{cas:0}}' "$WORK/old" | vault "kv/data/$new" POST >/dev/null
+  fi
+  kv "$new" > "$WORK/new"
+  jq -e -s '.[0]==.[1]' "$WORK/old" "$WORK/new" >/dev/null || fail "Conflicting Vault values at $old and $new; no source removed"
+ done < "$WORK/moves"
+)
+retire_secrets() (
+ # Call only after templates, policies and deployed images use canonical paths.
+ secret_moves > "$WORK/moves"
+ while read -r old new; do
+  kv "$old" > "$WORK/old"
+  [ "$(cat "$WORK/old")" != null ] || continue
+  kv "$new" > "$WORK/new"
+  jq -e -s '.[0]==.[1]' "$WORK/old" "$WORK/new" >/dev/null || fail "Vault migration verification failed for $old"
+  archive_secret "$old"
+  printf '{}' | vault "kv/metadata/$old" DELETE >/dev/null
+ done < "$WORK/moves"
+ for old in tls frontend-tls notifications-tls; do
+  archive_secret "$old"
+  if [ "$(vault "kv/metadata/$old" GET yes)" != null ]; then printf '{}' | vault "kv/metadata/$old" DELETE >/dev/null; fi
+ done
+ # The obsolete PKI mount is deliberately not disabled: its private signing key
+ # and history are retained until the owner explicitly chooses to destroy them.
+ echo 'PASS: Vault secrets migrated and verified; obsolete KV entries retired'
+)

infra/local/scripts/start-service.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+umask 077
+if [ "$SXCHNGE_SERVICE" = notifications ]; then
+    printf '%s\n' "$EMAIL_CONFIG" > /run/sxchnge/email.yml
+    unset EMAIL_CONFIG
+    cat /etc/sxchnge/notifications.yml /run/sxchnge/email.yml > /run/sxchnge/config.yml
+    exec service -config.file=/run/sxchnge/config.yml
+fi
+exec service -config.file=/etc/sxchnge/bff.yml

infra/local/scripts/test.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+set -eu
+set -o pipefail
+# Shell syntax and static analysis cover the actual container shell scripts.
+shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh
+shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh
+. infra/local/scripts/common.sh
+. infra/local/scripts/images.sh
+. infra/local/scripts/bootstrap.sh
+# Offline fixtures exercise stable selection, reject prereleases, and preserve pins.
+get() {
+ case "$1" in
+  */tags/latest) printf '{"digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}';;
+  *page_size*) cat <<'JSON'
+{"results":[{"name":"2.1.0","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},{"name":"2.2.0-rc1","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}]}
+JSON
+  ;;
+  *) return 1;;
+ esac
+}
+[ "$(image hashicorp/vault latest)" = 'hashicorp/vault:2.1.0@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' ] || fail 'Stable release selection failed'
+get() { printf '{"digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","results":[{"name":"3.0.0-rc1","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}]}'; }
+if image hashicorp/vault latest >/dev/null 2>&1; then fail 'Prerelease accepted'; fi
+# Isolate state from the developer stack during failure tests.
+STATE="$WORK/state"; mkdir "$STATE"; printf 'previous build' > "$STATE/images.env"
+get() { return 1; }
+if images >/dev/null 2>&1; then fail 'Offline resolution succeeded'; fi
+[ "$(cat "$STATE/images.env")" = 'previous build' ] || fail 'Failed resolution replaced pins'
+if printf '{"api_key":"","sender_address":"notifications@localhost.test"}' | email_valid; then fail 'Empty email accepted'; fi
+printf '{"api_key":"fixture-only","sender_address":"sender@example.test"}' | email_valid
+# Initialized Vault without matching local keys must fail without initializing it again.
+vault() { printf '{"initialized":true}'; }
+if (initialize) > "$WORK/initialize-error" 2>&1; then fail 'Missing local keys accepted'; fi
+grep -q 'Missing .local' "$WORK/initialize-error"
+for service in bff notifications; do
+ if grep -Eq 'database/creds/|SXCHNGE_MONGO_URI' "infra/local/vault/$service.hcl"; then fail 'Database credentials trigger Agent restart'; fi
+done
+echo 'PASS: shell analysis, stable selection, failed-resolution preservation, email validation, missing-key protection and Agent contract'
+if command -v python >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1; then fail 'Tool image unexpectedly contains Python'; fi
+# Migration never overwrites an independently provisioned destination.
+. infra/local/scripts/secrets.sh
+secret_moves() { echo 'email services/notifications/email/resend'; }
+archive_secret() { :; }
+kv() {
+ case "$1" in
+  email) printf '{"api_key":"existing-fixture"}';;
+  *) cat "$WORK/destination";;
+ esac
+}
+vault() {
+ [ "$1" = kv/data/services/notifications/email/resend ] || fail 'Unexpected migration mutation'
+ jq -e '.options.cas==0' >/dev/null
+ printf '{"api_key":"existing-fixture"}' > "$WORK/destination"
+}
+printf 'null' > "$WORK/destination"
+migrate_secrets
+migrate_secrets
+printf '{"api_key":"different-fixture"}' > "$WORK/destination"
+if (migrate_secrets) >/dev/null 2>&1; then fail 'Migration overwrote conflicting destination'; fi
+jq -e '.api_key=="different-fixture"' "$WORK/destination" >/dev/null
+echo 'PASS: secret migration is repeatable and refuses conflicting destination values'

infra/local/scripts/verify.sh

@@ -0,0 +1,60 @@
+policies() (
+ root_token=$VAULT_TOKEN
+ identity=''
+ trap 'if [ -n "$identity" ]; then printf "{}" | VAULT_TOKEN=$identity vault auth/token/revoke-self POST >/dev/null; fi' EXIT
+ for service in bff notifications nats; do
+  directory="$STATE/agents/$service"
+  login=$(jq -n --rawfile role "$directory/role-id" --rawfile secret "$directory/secret-id" '{role_id:($role|rtrimstr("\n")),secret_id:($secret|rtrimstr("\n"))}' | vault auth/approle/login POST)
+  identity=$(printf '%s' "$login" | jq -er '.auth.client_token')
+  printf '%s' "$login" | jq '{token:.auth.client_token,paths:["kv/data/bootstrap/mongo","sys/init","kv/data/bff","kv/data/caller","kv/data/email","kv/data/nats","kv/data/tls"]}' | vault sys/capabilities POST | jq -e '.data | all(.[]; .==["deny"])' >/dev/null || fail 'Service can access bootstrap credentials'
+  VAULT_TOKEN=$(printf '%s' "$login" | jq -er '.auth.client_token')
+  printf '{}' | vault auth/token/revoke-self POST >/dev/null
+  VAULT_TOKEN=$root_token
+  identity=''
+ done
+ echo 'PASS: service identities cannot read bootstrap credentials'
+)
+database_access() (
+ for role in bff notifications; do
+  database=sxchnge; forbidden=sxchnge_notifications
+  if [ "$role" = notifications ]; then database=sxchnge_notifications; forbidden=sxchnge; fi
+  vault "database/creds/$role" > "$WORK/lease"
+  trap 'jq "{lease_id:.lease_id}" "$WORK/lease" | vault sys/leases/revoke POST >/dev/null' EXIT
+  {
+   jq -r '"db.getSiblingDB(\"admin\").auth("+(.data.username|tojson)+","+(.data.password|tojson)+");"' "$WORK/lease"
+   printf 'const database="%s", forbidden="%s";\n' "$database" "$forbidden"
+   cat infra/local/mongo/mongo-check.js
+  } | mongo > "$WORK/check" || fail 'Mongo access/transaction check failed'
+  grep -q scope-ok "$WORK/check" || fail 'Mongo check incomplete'
+  jq '{lease_id:.lease_id,increment:"1h"}' "$WORK/lease" | vault sys/leases/renew POST | jq -e '.lease_duration>0' >/dev/null || fail 'Mongo renewal failed'
+  jq '{lease_id:.lease_id}' "$WORK/lease" | vault sys/leases/revoke POST >/dev/null
+  trap - EXIT
+  { admin_auth; jq -r '"print(admin.getUser("+(.data.username|tojson)+")===null);"' "$WORK/lease"; } | mongo > "$WORK/revoked" || fail 'Mongo revocation check failed'
+  grep -q true "$WORK/revoked" || fail 'Revoked Mongo user still exists'
+ done
+ echo 'PASS: Vault database issuance, isolated access, transaction commit/rollback, renewal and revocation'
+)
+infrastructure() {
+ compose ps --format json | jq -se '[.[] | select(.Service=="vault" or .Service=="mongo" or .Service=="nats")] | length==3 and all(.[];.Health=="healthy")' >/dev/null || fail 'Infrastructure is not healthy'
+ echo 'PASS: Vault, Mongo replica set and NATS JetStream are healthy'
+ layout
+ policies
+ database_access
+}
+# Resolve through Docker DNS while verifying the certificate for localhost.
+https() { curl -sS --max-time 5 --cacert "$STATE/ca.pem" --connect-to localhost:5126:frontend:8443 "$@"; }
+product() {
+ compose ps --format json | jq -se 'length==6 and all(.[];.Health=="healthy")' >/dev/null || fail 'Expected six healthy services'
+ https -f https://localhost:5126/ >/dev/null
+ https -f https://localhost:5126/api/v1/health >/dev/null
+ echo 'PASS: six healthy services, frontend and proxied BFF use verified TLS; no email sent'
+}
+layout() {
+ for old in bff caller email nats tls frontend-tls notifications-tls; do
+  [ "$(vault "kv/metadata/$old" GET yes)" = null ] || fail "Obsolete KV entry remains: $old"
+ done
+ for path in services/bff/jwt services/notifications/callers/bff services/notifications/email/resend infrastructure/nats/auth bootstrap/mongo; do
+  vault "kv/metadata/$path" | jq -e '.data.current_version>0' >/dev/null || fail "Missing canonical KV entry: $path"
+ done
+ echo 'PASS: canonical Vault hierarchy present; obsolete flat and TLS entries absent'
+}