protuberanets
f436b2528c
Two nodes on exch and one on infra-gamma, plain Compose per server, ports on the private addresses only. Nodes unseal through the transit seal on the platform Vault; the transit token reaches only the server process. make prod-images resolves the latest stable Vault release once and pins its digest in ignored .local/prod/images.env, so every node runs the same build. infra/prod/README.md records the layout, the first rollout, node replacement and recovery when the platform Vault is lost (#74).
159 lines
9.2 KiB
Bash
159 lines
9.2 KiB
Bash
#!/bin/sh
|
|
set -eu
|
|
set -o pipefail
|
|
# Shell syntax and static analysis cover the actual container shell scripts.
|
|
shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh
|
|
shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh infra/prod/vault/start.sh
|
|
shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh infra/local/sendico/network.sh
|
|
. infra/local/scripts/common.sh
|
|
. infra/local/scripts/images.sh
|
|
. infra/local/scripts/bootstrap.sh
|
|
# A transient DNS failure must reset the recovery counter, while sustained loss
|
|
# must terminate only the owning session process so Docker can reattach it.
|
|
(
|
|
. infra/local/sendico/network.sh
|
|
network_calls=0; network_recovered=0
|
|
sleep() { :; }
|
|
timeout() { [ "$1" = 5 ] || exit 1; shift; "$@"; }
|
|
getent() {
|
|
[ "$*" = 'hosts vault' ] || exit 1
|
|
network_calls=$((network_calls+1))
|
|
[ "$network_calls" = 3 ]
|
|
}
|
|
kill() {
|
|
[ "$*" = '-TERM 1234' ] && [ "$network_calls" = 6 ] || exit 1
|
|
network_recovered=1
|
|
}
|
|
watch_network 1234 2> "$WORK/network-recovery"
|
|
[ "$network_recovered" = 1 ] || exit 1
|
|
grep -q 'reattach BFF network' "$WORK/network-recovery"
|
|
for invalid_parent in '' 0 00 invalid; do
|
|
if watch_network "$invalid_parent"; then exit 1; fi
|
|
done
|
|
)
|
|
echo 'PASS: session network recovery tolerates transient DNS failures and targets its parent after sustained loss'
|
|
(
|
|
STATE="$WORK/recreate-state"; mkdir -p "$STATE/sendico"
|
|
compose() { printf '%s\n' "$*" > "$WORK/recreate-command"; }
|
|
recreate_services 90 bff frontend
|
|
grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend' "$WORK/recreate-command"
|
|
touch "$STATE/sendico/settings.env"
|
|
recreate_services 90 bff frontend
|
|
grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend sendico' "$WORK/recreate-command"
|
|
recreate_services 120 bff sendico
|
|
grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 120 bff sendico' "$WORK/recreate-command"
|
|
recreate_services 120 notifications
|
|
grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 120 notifications' "$WORK/recreate-command"
|
|
)
|
|
echo 'PASS: BFF recreation pairs its configured session exactly once and preserves unrelated service scope'
|
|
# Offline fixtures exercise stable selection, reject prereleases, and preserve pins.
|
|
get() {
|
|
case "$1" in
|
|
*/tags/latest) printf '{"digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}';;
|
|
*page_size*) cat <<'JSON'
|
|
{"results":[{"name":"2.1.0","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},{"name":"2.2.0-rc1","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}]}
|
|
JSON
|
|
;;
|
|
*) return 1;;
|
|
esac
|
|
}
|
|
[ "$(image hashicorp/vault latest)" = 'hashicorp/vault:2.1.0@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' ] || fail 'Stable release selection failed'
|
|
get() { printf '{"digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","results":[{"name":"3.0.0-rc1","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}]}'; }
|
|
if image hashicorp/vault latest >/dev/null 2>&1; then fail 'Prerelease accepted'; fi
|
|
# Isolate state from the developer stack during failure tests.
|
|
STATE="$WORK/state"; mkdir "$STATE"; printf 'previous build' > "$STATE/images.env"
|
|
get() { return 1; }
|
|
if images >/dev/null 2>&1; then fail 'Offline resolution succeeded'; fi
|
|
[ "$(cat "$STATE/images.env")" = 'previous build' ] || fail 'Failed resolution replaced pins'
|
|
if printf '{"api_key":"","sender_address":"notifications@localhost.test"}' | email_valid; then fail 'Empty email accepted'; fi
|
|
printf '{"api_key":"fixture-only","sender_address":"sender@example.test"}' | email_valid
|
|
printf '{"password":"fixture-only"}' | smtp_valid
|
|
for invalid in '{"password":""}' '{"password":"bad\nvalue"}' '{"password":"bad\u0000value"}' '{"password":"fixture","username":"not-a-secret"}'; do
|
|
if printf '%s' "$invalid" | smtp_valid; then fail 'Invalid SMTP secret accepted'; fi
|
|
done
|
|
# Initialized Vault without matching local keys must fail without initializing it again.
|
|
vault() { printf '{"initialized":true}'; }
|
|
if (initialize) > "$WORK/initialize-error" 2>&1; then fail 'Missing local keys accepted'; fi
|
|
grep -q 'Missing .local' "$WORK/initialize-error"
|
|
for service in bff notifications; do
|
|
if grep -Eq 'database/creds/|SXCHNGE_MONGO_URI' "infra/local/vault/$service.hcl"; then fail 'Database credentials trigger Agent restart'; fi
|
|
done
|
|
echo 'PASS: shell analysis, stable selection, failed-resolution preservation, email validation, missing-key protection and Agent contract'
|
|
if command -v python >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1; then fail 'Tool image unexpectedly contains Python'; fi
|
|
# Migration never overwrites an independently provisioned destination.
|
|
. infra/local/scripts/secrets.sh
|
|
secret_moves() { echo 'email services/notifications/email/resend'; }
|
|
archive_secret() { :; }
|
|
kv() {
|
|
case "$1" in
|
|
email) printf '{"api_key":"existing-fixture"}';;
|
|
*) cat "$WORK/destination";;
|
|
esac
|
|
}
|
|
vault() {
|
|
[ "$1" = kv/data/services/notifications/email/resend ] || fail 'Unexpected migration mutation'
|
|
jq -e '.options.cas==0' >/dev/null
|
|
printf '{"api_key":"existing-fixture"}' > "$WORK/destination"
|
|
}
|
|
printf 'null' > "$WORK/destination"
|
|
migrate_secrets
|
|
migrate_secrets
|
|
printf '{"api_key":"different-fixture"}' > "$WORK/destination"
|
|
if (migrate_secrets) >/dev/null 2>&1; then fail 'Migration overwrote conflicting destination'; fi
|
|
jq -e '.api_key=="different-fixture"' "$WORK/destination" >/dev/null
|
|
echo 'PASS: secret migration is repeatable and refuses conflicting destination values'
|
|
# Removing non-secret fields retains the key and checks the current KV version.
|
|
vault() {
|
|
if [ "${2:-GET}" = GET ]; then cat "$WORK/email-fixture"; else
|
|
jq -e '.options.cas==42 and .data=={api_key:"preserved-fixture-key"}' > /dev/null || return 1
|
|
printf '{"data":{"metadata":{"version":43},"data":{"api_key":"preserved-fixture-key"}}}' > "$WORK/email-fixture"
|
|
fi
|
|
}
|
|
printf '{"data":{"metadata":{"version":42},"data":{"api_key":"preserved-fixture-key","driver":"resend","sender_address":"sender@example.test","sender_name":"Fixture"}}}' > "$WORK/email-fixture"
|
|
normalize_email_secret
|
|
normalize_email_secret
|
|
jq -e '.data.data=={api_key:"preserved-fixture-key"}' "$WORK/email-fixture" >/dev/null
|
|
printf '{"data":{"metadata":{"version":43},"data":{"api_key":"preserved-fixture-key","unexpected":"must-not-drop"}}}' > "$WORK/email-fixture"
|
|
if (normalize_email_secret) >/dev/null 2>&1; then fail 'Unknown secret field was silently dropped'; fi
|
|
if grep -q EMAIL_CONFIG infra/local/vault/notifications.hcl infra/local/scripts/start-service.sh; then fail 'Email configuration is still rendered from Vault'; fi
|
|
echo 'PASS: email migration preserves API key, removes configuration fields and rejects unknown fields'
|
|
# Caller retirement is repeatable and must not remove NATS authentication.
|
|
secret_moves() { :; }
|
|
printf '{"caller":true,"services/notifications/callers/bff":true,"infrastructure/nats/auth":true}' > "$WORK/retirement"
|
|
vault() {
|
|
path=${1#kv/metadata/}
|
|
case "${2:-GET}" in
|
|
GET) jq -c --arg path "$path" '.[$path] // null' "$WORK/retirement";;
|
|
DELETE)
|
|
case "$path" in caller|services/notifications/callers/bff) ;; *) fail 'Unexpected secret retirement';; esac
|
|
jq --arg path "$path" 'del(.[$path])' "$WORK/retirement" > "$WORK/retirement-next"
|
|
mv "$WORK/retirement-next" "$WORK/retirement"
|
|
;;
|
|
*) fail 'Unexpected retirement method';;
|
|
esac
|
|
}
|
|
retire_secrets
|
|
retire_secrets
|
|
jq -e '.=={"infrastructure/nats/auth":true}' "$WORK/retirement" >/dev/null
|
|
echo 'PASS: obsolete caller secrets retired idempotently; NATS authentication preserved'
|
|
# Local integration configuration keeps credentials out of runtime settings and
|
|
# rejects malformed operator inputs before mutating Vault or service identities.
|
|
. infra/local/scripts/sendico.sh
|
|
STATE="$WORK/sendico-state"; mkdir "$STATE"
|
|
existing() { :; }
|
|
put() { cat > "$WORK/sendico-credentials"; }
|
|
provision_agents() { touch "$WORK/sendico-agents"; }
|
|
printf '{}' > "$WORK/sendico-invalid"
|
|
if (configure_sendico < "$WORK/sendico-invalid") >/dev/null 2>&1; then fail 'Invalid Sendico configuration accepted'; fi
|
|
[ ! -e "$WORK/sendico-credentials" ] || fail 'Invalid Sendico input mutated credentials'
|
|
cat > "$WORK/sendico-input" <<'JSON'
|
|
{"organization_ref":"000000000000000000000001","source_account_ref":"000000000000000000000002","client_id":"fixture.client","device_id":"fixture-device","login":"fixture@example.test","password":"fixture-password","client_secret":"fixture-secret"}
|
|
JSON
|
|
configure_sendico < "$WORK/sendico-input" >/dev/null
|
|
jq -e 'keys==["client_secret","login","password"]' "$WORK/sendico-credentials" >/dev/null
|
|
[ -f "$WORK/sendico-agents" ] || fail 'Sendico identity was not provisioned'
|
|
if grep -Eq 'fixture-secret|fixture-password|fixture@example' "$STATE/sendico/settings.env" "$STATE/sendico/runtime.yml"; then fail 'Credential leaked into runtime configuration'; fi
|
|
grep -q 'source_account_ref: 000000000000000000000002' "$STATE/sendico/runtime.yml"
|
|
grep -q 'api_url: http://127.0.0.1:18081/api/v1' "$STATE/sendico/runtime.yml"
|
|
echo 'PASS: Sendico operator input validation, secret separation and local runtime configuration'
|