Recover local Sendico session after BFF network restarts (#57)

infra/local/scripts/bootstrap.sh

@@ -81,7 +81,7 @@ provision_agents() {
   if [ "$service" != sendico ] && ! cmp -s "infra/local/vault/$service.hcl" "$directory/config.hcl"; then
    cat "infra/local/vault/$service.hcl" | write "$directory/config.hcl" 444
    if compose ps --status running --services "$service" | grep -qx "$service"; then
-    compose up -d --no-deps --force-recreate --wait --wait-timeout 120 "$service"
+    recreate_services 120 "$service"
    fi
   fi
  done

infra/local/scripts/common.sh

@@ -21,6 +21,19 @@ compose() {
   docker compose --project-directory . -f infra/local/compose.yml --env-file "$STATE/images.env" "$@"
  fi
 }
+# A recreated BFF has a new container ID. Recreate its namespace-sharing session
+# in the same Compose operation so neither waits on the other's old namespace.
+recreate_services() {
+ recreate_timeout=$1; shift
+ case " $* " in
+  *' bff '*)
+   if [ -f "$STATE/sendico/settings.env" ]; then
+    case " $* " in *' sendico '*) ;; *) set -- "$@" sendico;; esac
+   fi
+   ;;
+ esac
+ compose up -d --no-deps --force-recreate --wait --wait-timeout "$recreate_timeout" "$@"
+}
 write() {
  target=$1; mode=${2:-600}
  mkdir -p "$(dirname "$target")"

infra/local/scripts/rotation.sh

@@ -10,7 +10,9 @@ rotation() (
   for pid in $workers; do wait "$pid" || true; done
   restore_failed=0
   vault database/roles/bff POST < "$WORK/original-role" >/dev/null || { echo 'ERROR: restore BFF role TTL manually' >&2; restore_failed=1; }
-  for service in frontend bff; do
+  restore_services='frontend bff'
+  if [ -f "$STATE/sendico/settings.env" ]; then restore_services='frontend sendico bff'; fi
+  for service in $restore_services; do
    if ! grep -qx "$service" "$WORK/running"; then compose stop "$service"; compose rm -f "$service"; fi
   done
   return "$restore_failed"
@@ -19,7 +21,7 @@ rotation() (
  trap 'exit 130' INT
  trap 'exit 143' TERM
  jq '.+{default_ttl:"12s",max_ttl:"24s"}' "$WORK/original-role" | vault database/roles/bff POST >/dev/null
- compose up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend
+ recreate_services 90 bff frontend
  export_ca
  https -f https://localhost:5126/ > "$WORK/index"
  grep -q flutter_bootstrap "$WORK/index" || fail 'Frontend bundle is not served'

infra/local/scripts/test.sh

@@ -4,10 +4,48 @@ set -o pipefail
 # Shell syntax and static analysis cover the actual container shell scripts.
 shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh
 shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh
-shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh
+shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh infra/local/sendico/network.sh
 . infra/local/scripts/common.sh
 . infra/local/scripts/images.sh
 . infra/local/scripts/bootstrap.sh
+# A transient DNS failure must reset the recovery counter, while sustained loss
+# must terminate only the owning session process so Docker can reattach it.
+(
+ . infra/local/sendico/network.sh
+ network_calls=0; network_recovered=0
+ sleep() { :; }
+ timeout() { [ "$1" = 5 ] || exit 1; shift; "$@"; }
+ getent() {
+  [ "$*" = 'hosts vault' ] || exit 1
+  network_calls=$((network_calls+1))
+  [ "$network_calls" = 3 ]
+ }
+ kill() {
+  [ "$*" = '-TERM 1234' ] && [ "$network_calls" = 6 ] || exit 1
+  network_recovered=1
+ }
+ watch_network 1234 2> "$WORK/network-recovery"
+ [ "$network_recovered" = 1 ] || exit 1
+ grep -q 'reattach BFF network' "$WORK/network-recovery"
+ for invalid_parent in '' 0 00 invalid; do
+  if watch_network "$invalid_parent"; then exit 1; fi
+ done
+)
+echo 'PASS: session network recovery tolerates transient DNS failures and targets its parent after sustained loss'
+(
+ STATE="$WORK/recreate-state"; mkdir -p "$STATE/sendico"
+ compose() { printf '%s\n' "$*" > "$WORK/recreate-command"; }
+ recreate_services 90 bff frontend
+ grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend' "$WORK/recreate-command"
+ touch "$STATE/sendico/settings.env"
+ recreate_services 90 bff frontend
+ grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend sendico' "$WORK/recreate-command"
+ recreate_services 120 bff sendico
+ grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 120 bff sendico' "$WORK/recreate-command"
+ recreate_services 120 notifications
+ grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 120 notifications' "$WORK/recreate-command"
+)
+echo 'PASS: BFF recreation pairs its configured session exactly once and preserves unrelated service scope'
 # Offline fixtures exercise stable selection, reject prereleases, and preserve pins.
 get() {
  case "$1" in

infra/local/sendico/compose.yml

@@ -12,10 +12,13 @@ services:
     restart: unless-stopped
     # Keep the adapter's loopback-only HTTP rule; forward to the public stand API.
     network_mode: service:bff
+    depends_on:
+      bff: {condition: service_started, restart: true}
     entrypoint: [/bin/sh, /etc/sendico/session.sh]
     env_file: ./.local/dev/sendico/settings.env
     volumes:
       - ./infra/local/sendico/session.sh:/etc/sendico/session.sh:ro
+      - ./infra/local/sendico/network.sh:/etc/sendico/network.sh:ro
       - ./infra/local/sendico/check.sh:/etc/sendico/check.sh:ro
       - ./.local/dev/agents/sendico:/etc/agent:ro
       - sendico-session:/run/sendico

infra/local/sendico/network.sh

@@ -0,0 +1,20 @@
+# Restarting the network-owning BFF can strand this sidecar in its old namespace.
+# Exiting the session process lets Docker rejoin the BFF's current namespace.
+watch_network() {
+ network_parent=$1
+ case "$network_parent" in ''|*[!0-9]*) return 1;; esac
+ [ "$network_parent" -gt 0 ] || return 1
+ network_failures=0
+ while sleep 10; do
+  if timeout 5 getent hosts vault >/dev/null 2>&1; then
+   network_failures=0
+   continue
+  fi
+  network_failures=$((network_failures+1))
+  if [ "$network_failures" -ge 3 ]; then
+   echo 'Local Sendico network DNS unavailable; restarting session container to reattach BFF network.' >&2
+   kill -TERM "$network_parent" 2>/dev/null || true
+   return
+  fi
+ done
+}

infra/local/sendico/session.sh

@@ -1,13 +1,24 @@
 #!/bin/sh
 set -eu
 umask 077
+# shellcheck source=infra/local/sendico/network.sh
+. /etc/sendico/network.sh
 mkdir -p /private /run/sendico
 chmod 755 /run/sendico
 cd /private
+bridge=''; monitor=''; delay=''
+cleanup() {
+ for child in "$bridge" "$monitor" "$delay"; do
+  if [ -n "$child" ]; then kill "$child" 2>/dev/null || true; fi
+ done
+}
+trap cleanup EXIT
+trap 'exit 0' INT TERM
 socat TCP4-LISTEN:18081,bind=127.0.0.1,reuseaddr,fork TCP4:host.docker.internal:8080 &
 bridge=$!
-trap 'kill "$bridge" 2>/dev/null || true' EXIT
-trap 'exit 0' INT TERM
+session_pid=$$
+(trap - EXIT INT TERM; watch_network "$session_pid") &
+monitor=$!
 
 session() {
  jq -n --rawfile role /etc/agent/role-id --rawfile secret /etc/agent/secret-id \
@@ -36,10 +47,10 @@ session() {
 while :; do
  if session; then
   echo 'Local Sendico session ready; next renewal in 20 minutes.'
-  sleep 1200 & wait $!
+  sleep 1200 & delay=$!; wait "$delay"
  else
   echo 'Local Sendico session unavailable; check stand health and integration credentials. Retrying in 5 minutes.' >&2
   rm -f request.json response.json token vault.header vault.json credentials.json login.json
-  sleep 300 & wait $!
+  sleep 300 & delay=$!; wait "$delay"
  fi
 done