From 13cc9c302ef6cb5a15653bbbbc574bb72cf779cf Mon Sep 17 00:00:00 2001 From: Stephan D Date: Mon, 19 Oct 2026 12:34:37 +0200 Subject: [PATCH] Recover local Sendico session after BFF network restarts (#57) --- docs/local-sendico.md | 36 ++++++++++++++++++++++++++++ docs/signup-facts.md | 27 +++++++++++++++++++++ infra/local/scripts/bootstrap.sh | 2 +- infra/local/scripts/common.sh | 13 +++++++++++ infra/local/scripts/rotation.sh | 6 +++-- infra/local/scripts/test.sh | 40 +++++++++++++++++++++++++++++++- infra/local/sendico/compose.yml | 3 +++ infra/local/sendico/network.sh | 20 ++++++++++++++++ infra/local/sendico/session.sh | 19 +++++++++++---- 9 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 infra/local/sendico/network.sh diff --git a/docs/local-sendico.md b/docs/local-sendico.md index d1f42b5..a01a8a3 100644 --- a/docs/local-sendico.md +++ b/docs/local-sendico.md @@ -105,6 +105,27 @@ docker compose --project-directory . -f infra/local/compose.yml \ -f infra/local/sendico/compose.yml --env-file .local/dev/images.env restart sendico ``` +A BFF container restart can leave its session worker in the old shared network +namespace. The worker checks Vault DNS every ten seconds, with a five-second +lookup limit. Three consecutive failures log a credential-free recovery reason +and terminate the session process; Docker's restart policy then attaches it to +the BFF's current namespace. A successful lookup resets the counter, so transient +DNS failures do not immediately restart the worker. + +Container recreation changes the BFF container ID as well as its namespace. +Bootstrap and credential-rotation tooling recreate BFF and its configured session +worker together. Compose also marks the session dependency for restart when BFF +is explicitly updated. For a manual forced recreation, select both services: + +```sh +docker compose --project-directory . -f infra/local/compose.yml \ + -f infra/local/sendico/compose.yml --env-file .local/dev/images.env \ + up -d --no-deps --force-recreate --wait bff sendico +``` + +The parent process cleans up its bridge, DNS monitor and renewal delay on exit. +This recovery retains the configured provider URL, account and replay namespace. + `make test-dev` checks seven healthy services, verified browser TLS, the Vault layout and least-privilege identities, Mongo transactions/lease rotation, and authenticated Sendico currency and ledger reads through the same loopback @@ -130,6 +151,21 @@ and the established CA. A subsequent verified TLS check must succeed. ## Acceptance and operator UX review +Recovery follow-up for [task #57](https://git.sendico.io/tech/sxchnge/issues/57) / +[PR #59](https://git.sendico.io/tech/sxchnge/pulls/59), validated on 2026-10-19: +`make test-dev-tooling`, `make test-dev-rotation`, Compose configuration validation, +`sh infra/local/tools.sh verify` and `git diff --check` pass. A BFF-only restart +triggered automatic session restart and restored matching network namespaces; +paired recreation also finished healthy. Rotation observed three real Vault +pool replacements, an unchanged BFF process and 534 transaction-backed requests +with zero failures. The final check verifies all seven SXCHNGE services, browser +HTTPS, Vault/database boundaries and authenticated Sendico currency/ledger reads. +An earlier check failed because Sendico's ledger announcement connection was +closed; restoring Discovery and its service announcers in dependency order +recovered that existing stand. No Sendico source or financial data was changed, +and no email or payment was sent. This follow-up changes shell/Compose tooling; +it does not rerun the earlier full Go/Flutter suite for unchanged product source. + Validated on 2026-10-16: `make build`, `make up`, `make test-dev-tooling`, `make test-dev-infra`, `make test-dev`, and `git diff --check` passed. Both product sites, Sendico Backoffice and both API health endpoints returned HTTP 200; diff --git a/docs/signup-facts.md b/docs/signup-facts.md index 48927cc..7e2adff 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -162,6 +162,33 @@ as passing. The exact response to the reported Safari attempt was not captured, and the frontend's generic unavailable text also covers some HTTP 400 input errors. A fresh user retry is needed to confirm the visible signup outcome. +On October 19 the failing API was directly observed returning HTTP 502. The BFF +had restarted while the session worker retained its old network namespace; +`/proc/self/ns/net` differed between them, and the worker could not resolve +Vault. Sendico's public API itself returned HTTP 200. Restarting only the worker +reattached its namespace and restored SXCHNGE API HTTP 200. This is a confirmed +cause of that 502, separate from the previous process leak and host-load finding. + +The session now monitors bounded DNS failures and exits after three consecutive +failures so Docker can reattach it automatically. Bootstrap and rotation pair +BFF/session recreation, with Compose dependency restart enabled. Offline shell +fixtures cover transient-failure reset, sustained-failure recovery, safe parent +targeting and conditional service pairing. A live BFF-only restart automatically +restarted the session and restored matching namespaces, fresh session readiness +and API HTTP 200. A second live test recreated BFF and its session together and +confirmed healthy services with matching namespaces. No provider bindings, +customer records, credentials or Sendico source files were changed. + +Final October 19 validation passes: `make test-dev-tooling`, Compose config +validation, `make test-dev-rotation` (three real rotations, unchanged BFF process, +534 transaction-backed requests and zero failures), and +`sh infra/local/tools.sh verify`. The last confirms all seven healthy services, +verified frontend/BFF HTTPS, Vault/database checks and authenticated Sendico +currency/ledger reads. An initial ledger-read failure was resolved by restoring +Sendico Discovery and its announcers in their documented dependency order after +closed NATS connections. No email was sent during validation. The recovery fix +and its validation are included in [PR #59](https://git.sendico.io/tech/sxchnge/pulls/59). + With `api.sendico` configured, the product BFF creates the local pending customer and activation token atomically, then registers an independent individual through diff --git a/infra/local/scripts/bootstrap.sh b/infra/local/scripts/bootstrap.sh index 357f8c5..fe304a9 100644 --- a/infra/local/scripts/bootstrap.sh +++ b/infra/local/scripts/bootstrap.sh @@ -81,7 +81,7 @@ provision_agents() { if [ "$service" != sendico ] && ! cmp -s "infra/local/vault/$service.hcl" "$directory/config.hcl"; then cat "infra/local/vault/$service.hcl" | write "$directory/config.hcl" 444 if compose ps --status running --services "$service" | grep -qx "$service"; then - compose up -d --no-deps --force-recreate --wait --wait-timeout 120 "$service" + recreate_services 120 "$service" fi fi done diff --git a/infra/local/scripts/common.sh b/infra/local/scripts/common.sh index 6f98c95..1804546 100644 --- a/infra/local/scripts/common.sh +++ b/infra/local/scripts/common.sh @@ -21,6 +21,19 @@ compose() { docker compose --project-directory . -f infra/local/compose.yml --env-file "$STATE/images.env" "$@" fi } +# A recreated BFF has a new container ID. Recreate its namespace-sharing session +# in the same Compose operation so neither waits on the other's old namespace. +recreate_services() { + recreate_timeout=$1; shift + case " $* " in + *' bff '*) + if [ -f "$STATE/sendico/settings.env" ]; then + case " $* " in *' sendico '*) ;; *) set -- "$@" sendico;; esac + fi + ;; + esac + compose up -d --no-deps --force-recreate --wait --wait-timeout "$recreate_timeout" "$@" +} write() { target=$1; mode=${2:-600} mkdir -p "$(dirname "$target")" diff --git a/infra/local/scripts/rotation.sh b/infra/local/scripts/rotation.sh index 7355773..f9b0203 100644 --- a/infra/local/scripts/rotation.sh +++ b/infra/local/scripts/rotation.sh @@ -10,7 +10,9 @@ rotation() ( for pid in $workers; do wait "$pid" || true; done restore_failed=0 vault database/roles/bff POST < "$WORK/original-role" >/dev/null || { echo 'ERROR: restore BFF role TTL manually' >&2; restore_failed=1; } - for service in frontend bff; do + restore_services='frontend bff' + if [ -f "$STATE/sendico/settings.env" ]; then restore_services='frontend sendico bff'; fi + for service in $restore_services; do if ! grep -qx "$service" "$WORK/running"; then compose stop "$service"; compose rm -f "$service"; fi done return "$restore_failed" @@ -19,7 +21,7 @@ rotation() ( trap 'exit 130' INT trap 'exit 143' TERM jq '.+{default_ttl:"12s",max_ttl:"24s"}' "$WORK/original-role" | vault database/roles/bff POST >/dev/null - compose up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend + recreate_services 90 bff frontend export_ca https -f https://localhost:5126/ > "$WORK/index" grep -q flutter_bootstrap "$WORK/index" || fail 'Frontend bundle is not served' diff --git a/infra/local/scripts/test.sh b/infra/local/scripts/test.sh index 424191d..b026f9d 100644 --- a/infra/local/scripts/test.sh +++ b/infra/local/scripts/test.sh @@ -4,10 +4,48 @@ set -o pipefail # Shell syntax and static analysis cover the actual container shell scripts. shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh -shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh +shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh infra/local/sendico/network.sh . infra/local/scripts/common.sh . infra/local/scripts/images.sh . infra/local/scripts/bootstrap.sh +# A transient DNS failure must reset the recovery counter, while sustained loss +# must terminate only the owning session process so Docker can reattach it. +( + . infra/local/sendico/network.sh + network_calls=0; network_recovered=0 + sleep() { :; } + timeout() { [ "$1" = 5 ] || exit 1; shift; "$@"; } + getent() { + [ "$*" = 'hosts vault' ] || exit 1 + network_calls=$((network_calls+1)) + [ "$network_calls" = 3 ] + } + kill() { + [ "$*" = '-TERM 1234' ] && [ "$network_calls" = 6 ] || exit 1 + network_recovered=1 + } + watch_network 1234 2> "$WORK/network-recovery" + [ "$network_recovered" = 1 ] || exit 1 + grep -q 'reattach BFF network' "$WORK/network-recovery" + for invalid_parent in '' 0 00 invalid; do + if watch_network "$invalid_parent"; then exit 1; fi + done +) +echo 'PASS: session network recovery tolerates transient DNS failures and targets its parent after sustained loss' +( + STATE="$WORK/recreate-state"; mkdir -p "$STATE/sendico" + compose() { printf '%s\n' "$*" > "$WORK/recreate-command"; } + recreate_services 90 bff frontend + grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend' "$WORK/recreate-command" + touch "$STATE/sendico/settings.env" + recreate_services 90 bff frontend + grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend sendico' "$WORK/recreate-command" + recreate_services 120 bff sendico + grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 120 bff sendico' "$WORK/recreate-command" + recreate_services 120 notifications + grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 120 notifications' "$WORK/recreate-command" +) +echo 'PASS: BFF recreation pairs its configured session exactly once and preserves unrelated service scope' # Offline fixtures exercise stable selection, reject prereleases, and preserve pins. get() { case "$1" in diff --git a/infra/local/sendico/compose.yml b/infra/local/sendico/compose.yml index 9f96ad8..a31c562 100644 --- a/infra/local/sendico/compose.yml +++ b/infra/local/sendico/compose.yml @@ -12,10 +12,13 @@ services: restart: unless-stopped # Keep the adapter's loopback-only HTTP rule; forward to the public stand API. network_mode: service:bff + depends_on: + bff: {condition: service_started, restart: true} entrypoint: [/bin/sh, /etc/sendico/session.sh] env_file: ./.local/dev/sendico/settings.env volumes: - ./infra/local/sendico/session.sh:/etc/sendico/session.sh:ro + - ./infra/local/sendico/network.sh:/etc/sendico/network.sh:ro - ./infra/local/sendico/check.sh:/etc/sendico/check.sh:ro - ./.local/dev/agents/sendico:/etc/agent:ro - sendico-session:/run/sendico diff --git a/infra/local/sendico/network.sh b/infra/local/sendico/network.sh new file mode 100644 index 0000000..3143c84 --- /dev/null +++ b/infra/local/sendico/network.sh @@ -0,0 +1,20 @@ +# Restarting the network-owning BFF can strand this sidecar in its old namespace. +# Exiting the session process lets Docker rejoin the BFF's current namespace. +watch_network() { + network_parent=$1 + case "$network_parent" in ''|*[!0-9]*) return 1;; esac + [ "$network_parent" -gt 0 ] || return 1 + network_failures=0 + while sleep 10; do + if timeout 5 getent hosts vault >/dev/null 2>&1; then + network_failures=0 + continue + fi + network_failures=$((network_failures+1)) + if [ "$network_failures" -ge 3 ]; then + echo 'Local Sendico network DNS unavailable; restarting session container to reattach BFF network.' >&2 + kill -TERM "$network_parent" 2>/dev/null || true + return + fi + done +} diff --git a/infra/local/sendico/session.sh b/infra/local/sendico/session.sh index 1dd34ed..995eb87 100644 --- a/infra/local/sendico/session.sh +++ b/infra/local/sendico/session.sh @@ -1,13 +1,24 @@ #!/bin/sh set -eu umask 077 +# shellcheck source=infra/local/sendico/network.sh +. /etc/sendico/network.sh mkdir -p /private /run/sendico chmod 755 /run/sendico cd /private +bridge=''; monitor=''; delay='' +cleanup() { + for child in "$bridge" "$monitor" "$delay"; do + if [ -n "$child" ]; then kill "$child" 2>/dev/null || true; fi + done +} +trap cleanup EXIT +trap 'exit 0' INT TERM socat TCP4-LISTEN:18081,bind=127.0.0.1,reuseaddr,fork TCP4:host.docker.internal:8080 & bridge=$! -trap 'kill "$bridge" 2>/dev/null || true' EXIT -trap 'exit 0' INT TERM +session_pid=$$ +(trap - EXIT INT TERM; watch_network "$session_pid") & +monitor=$! session() { jq -n --rawfile role /etc/agent/role-id --rawfile secret /etc/agent/secret-id \ @@ -36,10 +47,10 @@ session() { while :; do if session; then echo 'Local Sendico session ready; next renewal in 20 minutes.' - sleep 1200 & wait $! + sleep 1200 & delay=$!; wait "$delay" else echo 'Local Sendico session unavailable; check stand health and integration credentials. Retrying in 5 minutes.' >&2 rm -f request.json response.json token vault.header vault.json credentials.json login.json - sleep 300 & wait $! + sleep 300 & delay=$!; wait "$delay" fi done