Organize local infrastructure and Vault secrets by service ownership

AGENTS.md

@@ -169,3 +169,15 @@ an uncommitted working tree. Preserve unrelated changes and avoid mixing them
 into task commits. Record scope, validation, limitations, and PR links in the
 task and durable repository documentation. Merge only when authorized by the
 user. If work is delivered as stacked PRs, document dependencies and merge order.
+
+## Local infrastructure ownership
+
+Keep infrastructure Dockerfiles under `infra/images/` and local Compose, service
+configuration and shell tooling under `infra/local/`. Keep product source under
+`backend/` and `frontend/`. Organize Vault KV paths by owning service under
+`services/<service>/<purpose>`, infrastructure under `infrastructure/<component>/<purpose>`,
+and bootstrap administration under `bootstrap/<component>`. Caller credentials
+belong to the receiving service at `services/<service>/callers/<caller>`.
+Use Caddy for local browser HTTPS and certificate management. Treat the private
+local Docker network as trusted; do not add internal gRPC TLS or Vault-managed
+local TLS certificates without an explicit requirement.

Makefile

@@ -53,16 +53,16 @@ package-notifications-interface: generate-notifications
 	tar -czf dist/notifications-go-v1.tar.gz -C backend/notifications/interface go.mod go.sum notifications README.md
 
 # Local product stack. Releases are resolved afresh, then pinned by digest.
-COMPOSE = docker compose --env-file .local/dev/images.env
+COMPOSE = docker compose --project-directory . -f infra/local/compose.yml --env-file .local/dev/images.env
 .PHONY: build up down logs ps test-dev
 build:
-	sh dev/tools.sh build
-	sh dev/tools.sh images
+	sh infra/local/tools.sh build
+	sh infra/local/tools.sh images
 	$(COMPOSE) pull vault mongo
 	$(COMPOSE) build --pull
 
 up:
-	sh dev/tools.sh up
+	sh infra/local/tools.sh up
 
 down:
 	$(COMPOSE) down
@@ -74,21 +74,21 @@ ps:
 	$(COMPOSE) ps
 
 test-dev:
-	sh dev/tools.sh test
-	sh dev/tools.sh verify
+	sh infra/local/tools.sh test
+	sh infra/local/tools.sh verify
 
 .PHONY: test-dev-infra
 test-dev-infra:
-	sh dev/tools.sh test
-	sh dev/tools.sh verify-infra
+	sh infra/local/tools.sh test
+	sh infra/local/tools.sh verify-infra
 
 .PHONY: test-dev-rotation
 test-dev-rotation:
-	sh dev/tools.sh rotation
+	sh infra/local/tools.sh rotation
 
 .PHONY: email test-dev-tooling
 email:
-	sh dev/tools.sh email
+	sh infra/local/tools.sh email
 
 test-dev-tooling:
-	sh dev/tools.sh test
+	sh infra/local/tools.sh test

README.md

@@ -40,7 +40,7 @@ make build && make up
 
 This builds the product images and starts an isolated Vault, authenticated MongoDB
 replica set and NATS JetStream. First startup intentionally stops after bootstrap
-until you set the Resend API key and verified sender in Vault's `kv/email` secret.
+until you set the Resend API key and verified sender in Vault's `kv/services/notifications/email/resend` secret.
 Then rerun `make up` to start notifications, BFF and the frontend at
 [https://localhost:5126](https://localhost:5126).
 
@@ -72,7 +72,7 @@ repository root:
 
 ```sh
 docker run --rm --pull always -p 127.0.0.1:5123:80 \
-  -v "$PWD/frontend/app/build/web:/usr/share/nginx/html:ro" nginx:stable-alpine
+  -v "$PWD/frontend/app/build/web:/srv:ro" caddy:alpine caddy file-server --root /srv --listen :80
 ```
 
 Open [the local app](http://127.0.0.1:5123/). The language selector supports EN/RU

backend/notifications/README.md

@@ -24,10 +24,11 @@ Requires Go 1.27, protoc, an SXCHNGE MongoDB database and a NATS server with
 JetStream enabled. Copy `config.example.yml` to an untracked `config.yml` here.
 Set `SXCHNGE_MONGO_URI`, `SXCHNGE_NATS_URL`, a unique token of at least 32 bytes for
 each configured caller, and the selected email provider's API key environment
-variable. Configure a verified sender address, TLS certificate/key and template
+variable. Configure a verified sender address and template
 catalog. Paths are relative to the process working directory.
 
-From the root, run `make run-notifications`. The gRPC listener always uses TLS.
+From the root, run `make run-notifications`. The gRPC listener uses plaintext on the private local network. Configure both
+`tls_cert` and `tls_key` only when TLS is explicitly required.
 Use a private authenticated NATS endpoint; `tls://` URLs and credentials in the URL
 are supported by the shared driver. Mongo and messaging both use shared
 `DriverConfig` settings; email settings reject unknown keys.

backend/notifications/app/config.go

@@ -40,9 +40,12 @@ func Load(path string) (Config, error) {
 	if err = decoder.Decode(new(any)); err != io.EOF {
 		return c, merrors.InvalidArgument("unexpected configuration content")
 	}
-	if c.Listen == "" || c.TLSCert == "" || c.TLSKey == "" || c.Database.Driver != db.Mongo || c.Messaging.Driver != messaging.BBNats || c.Templates == "" || len(c.Callers) == 0 {
+	if c.Listen == "" || c.Database.Driver != db.Mongo || c.Messaging.Driver != messaging.BBNats || c.Templates == "" || len(c.Callers) == 0 {
 		return c, merrors.InvalidArgument("missing notification configuration")
 	}
+	if (c.TLSCert == "") != (c.TLSKey == "") {
+		return c, merrors.InvalidArgument("both TLS certificate and key must be configured together")
+	}
 	if err = c.Queue.Validate(); err != nil {
 		return c, err
 	}

backend/notifications/app/config_test.go

@@ -33,3 +33,31 @@ func TestExampleConfigAndSecrets(t *testing.T) {
 		t.Fatal("extra YAML document accepted")
 	}
 }
+
+func TestPlaintextConfigAndIncompleteTLS(t *testing.T) {
+	t.Setenv("SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN", strings.Repeat("a", 32))
+	c, err := Load("../config.example.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	options, err := serverOptions(c)
+	if err != nil || len(options) != 0 {
+		t.Fatal("plain gRPC must not require TLS material")
+	}
+	data, err := os.ReadFile("../config.example.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, partial := range []string{"\ntls_cert: missing.crt\n", "\ntls_key: missing.key\n"} {
+		path := filepath.Join(t.TempDir(), "config.yml")
+		if err := os.WriteFile(path, append(data, []byte(partial)...), 0600); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := Load(path); err == nil {
+			t.Fatal("incomplete TLS configuration accepted")
+		}
+	}
+	if _, err := serverOptions(Config{TLSCert: "missing.crt", TLSKey: "missing.key"}); err == nil {
+		t.Fatal("bad explicit certificate accepted")
+	}
+}

backend/notifications/app/run.go

@@ -33,9 +33,9 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error {
 	if err != nil {
 		return err
 	}
-	cert, err := tls.LoadX509KeyPair(c.TLSCert, c.TLSKey)
+	options, err := serverOptions(c)
 	if err != nil {
-		return merrors.OpaqueWrap(merrors.ErrInvalidArg, err, "invalid gRPC TLS certificate")
+		return err
 	}
 	client, err := db.ConnectMongo(logger, &c.Database)
 	if err != nil {
@@ -67,7 +67,7 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error {
 	for caller, env := range c.Callers {
 		tokens[caller] = os.Getenv(env)
 	}
-	server := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12})), grpc.UnaryInterceptor(transport.Authenticate(tokens)), grpc.MaxRecvMsgSize(128<<10), grpc.MaxConcurrentStreams(64))
+	server := grpc.NewServer(append(options, grpc.UnaryInterceptor(transport.Authenticate(tokens)), grpc.MaxRecvMsgSize(128<<10), grpc.MaxConcurrentStreams(64))...)
 	pb.RegisterNotificationsServiceServer(server, transport.New(svc))
 	listener, err := net.Listen("tcp", c.Listen)
 	if err != nil {
@@ -103,3 +103,15 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error {
 	logger.Info("Notifications service stopped")
 	return nil
 }
+
+// An omitted certificate pair uses plaintext gRPC on the private service network.
+func serverOptions(c Config) ([]grpc.ServerOption, error) {
+	if c.TLSCert == "" && c.TLSKey == "" {
+		return nil, nil
+	}
+	cert, err := tls.LoadX509KeyPair(c.TLSCert, c.TLSKey)
+	if err != nil {
+		return nil, merrors.OpaqueWrap(merrors.ErrInvalidArg, err, "invalid gRPC TLS certificate")
+	}
+	return []grpc.ServerOption{grpc.Creds(credentials.NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}))}, nil
+}

backend/notifications/config.example.yml

@@ -1,6 +1,4 @@
 listen: "127.0.0.1:8083"
-tls_cert: "server.crt"
-tls_key: "server.key"
 database:
   driver: mongodb
   settings:

backend/product/config.example.yml

@@ -32,8 +32,6 @@ api:
       check:
         min_length: 12
   notifications:
-    address: localhost:8443
+    address: localhost:8083
     token_env: SXCHNGE_NOTIFICATIONS_TOKEN
-    ca_file: ../notifications/tls/ca.pem
-    server_name: localhost
     activation_url: http://localhost:5126/verify-email

backend/product/notifications/client.go

@@ -12,6 +12,7 @@ import (
 	merrors "git.sendico.io/tech/sxchnge/shared/errors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
@@ -57,7 +58,11 @@ func Open(c Config) (*Client, error) {
 		}
 		tlsConfig.RootCAs = pool
 	}
-	conn, err := grpc.NewClient(c.Address, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
+	var transport credentials.TransportCredentials = insecure.NewCredentials()
+	if c.CAFile != "" || c.ServerName != "" {
+		transport = credentials.NewTLS(tlsConfig)
+	}
+	conn, err := grpc.NewClient(c.Address, grpc.WithTransportCredentials(transport))
 	if err != nil {
 		return nil, merrors.InternalWrap(err, "create notification client")
 	}

backend/product/notifications/client_test.go

@@ -88,3 +88,19 @@ func TestOpenUsesVerifiedTLSAndCallerAuthentication(t *testing.T) {
 	require.NoError(t, client.Send(context.Background(), Message{ExpiresAt: time.Now().Add(time.Minute)}))
 	require.Equal(t, "Bearer test-service-token-with-at-least-32-bytes", <-rpc.auth)
 }
+
+func TestOpenUsesPlaintextAndCallerTokenOnPrivateNetwork(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	server := grpc.NewServer()
+	rpc := &tlsRPC{auth: make(chan string, 1)}
+	pb.RegisterNotificationsServiceServer(server, rpc)
+	go func() { _ = server.Serve(listener) }()
+	t.Cleanup(server.Stop)
+	t.Setenv("TEST_PLAIN_CALLER_TOKEN", "test-private-service-token-at-least-32-bytes")
+	client, err := Open(Config{Address: listener.Addr().String(), TokenEnv: "TEST_PLAIN_CALLER_TOKEN"})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = client.Close() })
+	require.NoError(t, client.Send(context.Background(), Message{ExpiresAt: time.Now().Add(time.Minute)}))
+	require.Equal(t, "Bearer test-private-service-token-at-least-32-bytes", <-rpc.auth)
+}

dev/config/nginx.conf

@@ -1,17 +0,0 @@
-server {
-    listen 8443 ssl;
-    ssl_certificate /etc/tls/server.crt;
-    ssl_certificate_key /etc/tls/server.key;
-    ssl_protocols TLSv1.2 TLSv1.3;
-    root /usr/share/nginx/html;
-    index index.html;
-    location /api/ {
-        resolver 127.0.0.11 valid=10s ipv6=off;
-        set $bff http://bff:8082;
-        proxy_pass $bff;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto https;
-        proxy_set_header X-Forwarded-For $remote_addr;
-    }
-    location / { try_files $uri $uri/ /index.html; }
-}

dev/dispatch.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -eu
-set -o pipefail
-. dev/common.sh
-case "${1:-}" in
- images) . dev/images.sh; images;;
- up) . dev/bootstrap.sh; up;;
- email) . dev/bootstrap.sh; email;;
- verify|verify-infra) . dev/verify.sh; existing; infrastructure; if [ "$1" = verify ]; then product; fi;;
- rotation) . dev/verify.sh; . dev/rotation.sh; existing; rotation;;
- test) sh dev/test.sh;;
- *) fail 'Unknown local tooling command';;
-esac

dev/vault/notifications.hcl

@@ -1,55 +0,0 @@
-vault { address = "http://vault:8200" }
-auto_auth {
-  method "approle" {
-    config = {
-      role_id_file_path = "/etc/agent/role-id"
-      secret_id_file_path = "/etc/agent/secret-id"
-      remove_secret_id_file_after_reading = false
-    }
-  }
-}
-template_config {
-  static_secret_render_interval = "10s"
-  exit_on_retry_failure = true
-}
-
-env_template "TLS_CA" {
-  contents = "{{ with secret \"kv/data/tls\" }}{{ .Data.data.ca }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN" {
-  contents = "{{ with secret \"kv/data/caller\" }}{{ .Data.data.token }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "SXCHNGE_NATS_URL" {
-  contents = "{{ with secret \"kv/data/nats\" }}nats://{{ .Data.data.token }}@nats:4222{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "TLS_CERT" {
-  contents = "{{ with secret \"kv/data/notifications-tls\" }}{{ .Data.data.certificate }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "TLS_KEY" {
-  contents = "{{ with secret \"kv/data/notifications-tls\" }}{{ .Data.data.private_key }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "EMAIL_API_KEY" {
-  contents = "{{ with secret \"kv/data/email\" }}{{ .Data.data.api_key }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "EMAIL_CONFIG" {
-  contents = "{{ with secret \"kv/data/email\" }}email:\n  driver: {{ .Data.data.driver | toJSON }}\n  settings:\n    api_key_env: EMAIL_API_KEY\n    sender_address: {{ .Data.data.sender_address | toJSON }}\n    sender_name: {{ .Data.data.sender_name | toJSON }}\n{{ end }}"
-  error_on_missing_key = true
-}
-
-exec {
-  command = ["/usr/local/bin/start-service"]
-  restart_on_secret_changes = "always"
-  restart_stop_signal = "SIGTERM"
-}

docs/implementation.md

@@ -705,3 +705,41 @@ were restored and temporary BFF/frontend containers removed. `make up` exits at
 the intentional missing-Resend check, with infrastructure healthy. Backend and
 frontend code did not change in this revision; their earlier test/lint results
 and supplementary legacy test limitations above still apply. No email was sent.
+
+### Organized infrastructure and Vault ownership
+
+This revision supersedes the earlier local Nginx/Vault-certificate arrangement.
+Local Compose, runtime configuration, startup scripts and verification now live
+under `infra/local/`, grouped by service and responsibility. Dockerfiles live in
+`infra/images/`; the root Make targets remain the entry point. Product source
+stays in backend/frontend. Generated state remains in ignored `.local/dev/`.
+
+Vault KV now contains `services/bff/jwt`,
+`services/notifications/callers/bff`, `services/notifications/email/resend`,
+`infrastructure/nats/auth` and `bootstrap/mongo`. Dynamic database credentials
+remain in the database engine. Migration copies with CAS protection, compares
+existing values, archives readable old versions and metadata privately, and then
+removes old flat/TLS KV entries. Conflicting destinations stop migration without
+removing sources. Policies and Agent templates reference only the new paths.
+The unused existing PKI mount is retained to avoid destroying its signing key;
+new bootstrap neither creates nor uses it.
+
+Caddy replaces Nginx, serving Flutter and proxying `/api` to BFF. Caddy owns browser
+certificates and its CA in persistent volumes; only its public CA is exported to
+`.local/dev/ca.pem`. Existing browser trust must be updated to that new CA. Local
+BFF-to-notifications uses plaintext authenticated gRPC. Optional explicitly
+configured TLS remains available in the reusable client/server; incomplete TLS
+configuration is rejected. No local service consumes TLS secrets from Vault.
+
+Validation passed: `make build`, `make test`, `make lint-backend`,
+`make test-dev-tooling`, `make test-dev-infra`, and Compose/diff checks. Tests cover
+plaintext caller-token transport, optional verified TLS, invalid certificate
+configuration and secret migration conflicts. The existing Vault was migrated
+and canonical paths/absence of old entries verified. An isolated fresh project
+passed transaction/lease checks and repeated bootstrap; disposable volumes and
+state were removed. The final rotation check also verified Caddy static serving,
+SPA fallback and API health, then three pool replacements with unchanged BFF PID
+and 1,083 expected HTTP responses, zero failures. Role TTLs were restored and
+temporary BFF/frontend containers removed. `make up` intentionally stops at the
+missing Resend credential check; no real email, optional security scan or merge
+was performed. Earlier supplementary legacy test limitations remain recorded.

docs/local-development.md

@@ -3,219 +3,163 @@
 Task [#42](https://git.sendico.io/tech/sxchnge/issues/42),
 [PR #43](https://git.sendico.io/tech/sxchnge/pulls/43), branch `dev-env-42`.
 
-## Build and bootstrap
+## Build and start
 
 Requires Make, a POSIX shell and a local Docker engine with Compose v2 and
-BuildKit. Docker builds Go, protobuf bindings and Flutter web; host Go, protoc
-and Flutter installations are not required for `make build && make up`. The
-tooling container includes Docker CLI, curl, jq, Git and OpenSSL. No host Python,
-jq, curl or Git installation is needed for these Make targets. It mounts
-the local Docker socket and workspace to orchestrate Compose; a remote Docker
-daemon is not supported. The first Flutter SDK download can
-take several minutes. Both ARM64 and AMD64 build from the same source definitions.
+BuildKit. No host Python, Go, Flutter, jq or curl is required. Make runs bootstrap
+tooling in Docker with the local Docker socket and workspace mounted. Remote
+Docker engines are unsupported.
 
 ```sh
 make build && make up
 ```
 
-`make build` resolves the current stable upstream releases, pulls infrastructure
-and builds NATS packaging, notifications, BFF and frontend images. Vault and Mongo
-use upstream images directly. Resolution excludes prereleases and selects Nginx's
-stable channel. The exact versions and manifest digests are saved in the ignored
-`.local/dev/images.env`; Flutter's stable release is pinned by Git commit. Build
-fails if upstream metadata cannot be resolved. Run `make build` again to refresh
-versions. Review database release upgrade requirements before updating a populated
-local volume across major versions; the tooling does not perform data migrations.
+The build resolves the latest stable releases and pins their digests in ignored
+`.local/dev/images.env`. Prereleases are excluded, resolution failures stop the
+build, and previous pins are retained on failure. Flutter is pinned to the stable
+release commit. Tooling uses a freshly pulled stable Docker CLI digest. Generated
+protobuf bindings and Flutter output are built locally/in Docker and never tracked.
 
-`make up` starts persistent Vault, initializes/unseals it, provisions secrets and
-AppRoles, initializes an authenticated single-member MongoDB replica set, configures
-Vault's MongoDB database plugin, and starts authenticated NATS JetStream. It then
-starts notifications, BFF and the frontend when the required email settings exist.
+Compose is at `infra/local/compose.yml`; Make supplies the repository root as its
+project directory. Dockerfiles are in `infra/images/`. Runtime and startup files
+are grouped by service under `infra/local/`. The stack consists of Vault, Mongo,
+NATS, notifications, BFF and Caddy serving the frontend.
 
-**On the first run, `make up` intentionally exits with an error after infrastructure
-bootstrap because Resend credentials have not been supplied.** No placeholder API
-key, mock delivery mode or embedded credential is installed. The three infrastructure
-services remain running so you can configure Vault, then rerun `make up`.
+On first startup, bootstrap initializes/unseals persistent Vault, provisions
+AppRoles and secrets, initializes Mongo replica set `rs0`, configures Vault's Mongo
+plugin, and starts authenticated NATS JetStream. Product startup fails explicitly
+until Resend credentials exist. Infrastructure remains available for provisioning.
 
-## Configure Resend in Vault
+## Vault layout
 
-The default email driver is `resend`. In the Vault UI at
-[http://127.0.0.1:18200](http://127.0.0.1:18200), edit the KV v2 secret **`kv/email`**
-(API path `kv/data/email`):
+Paths below are inside the KV v2 mount `kv`; the API inserts `data/` after `kv/`.
+
+```text
+kv/
+├── services/
+│   ├── bff/jwt                         # jwt
+│   └── notifications/
+│       ├── callers/bff                 # token; read by BFF and notifications
+│       └── email/resend                # driver, api_key, sender_address, sender_name
+├── infrastructure/
+│   └── nats/auth                       # token; read by NATS and notifications
+└── bootstrap/
+    └── mongo                           # local administrator and replica-set bootstrap material
+```
+
+Every secret has an owning service or infrastructure component. Service policies
+permit only the exact paths they consume. Applications cannot read bootstrap
+credentials. Mongo application users are dynamic secrets under
+`database/creds/bff` and `database/creds/notifications`, not KV entries. They have
+`readWrite` only on their respective `sxchnge` / `sxchnge_notifications` databases.
+Vault uses `mongodb-database-plugin` and its own `vault-manager` account, whose
+password is rotated immediately after initial configuration.
+
+There are no frontend certificates, notification certificates or generic `tls`
+entries in KV. Browser certificates belong to Caddy; internal gRPC is plaintext
+on the private Docker network and retains caller-token authentication.
+
+## Resend
+
+Open [Vault](http://127.0.0.1:18200) and edit
+`kv/services/notifications/email/resend`:
 
 | Field | Value |
 | --- | --- |
 | `driver` | `resend` |
 | `api_key` | Your Resend API key |
-| `sender_address` | Your verified Resend sender email |
-| `sender_name` | `SXCHNGE` or your desired display name |
+| `sender_address` | Your verified sender address |
+| `sender_name` | Your display name |
 
-Local administrative login material is in the private, ignored
-`.local/dev/admin/init.json` (`root_token`). Keep that file private. Product
-containers never mount it. You can also provision the same secret with a hidden
-terminal prompt, without placing the API key in shell history:
+Local administrator login material is in private `.local/dev/admin/init.json`.
+Product containers cannot access it. Alternatively use the hidden terminal prompt:
 
 ```sh
 make email
 make up
 ```
 
-The startup check rejects an empty key or the bootstrap sender placeholder. It
-does not validate the key against Resend or prove domain verification; an invalid
-key fails at delivery. Vault Agent fetches the key and email settings and injects
-them into the notifications process. There is no `.env` secret fallback. Changing
-`kv/email` restarts the notification process automatically within about 10 seconds.
-Bootstrapping again preserves your existing email secret.
+No placeholder key or delivery fallback is installed. Startup validates presence;
+it does not send an email or validate domain ownership with Resend. Updating email
+settings causes Agent to restart notifications. Existing values survive bootstrap.
 
-## Open the product
+## Browser HTTPS and internal connections
 
-After providing Resend credentials:
+Caddy serves Flutter files and proxies `/api/*` to BFF over HTTP. Open
+[https://localhost:5126](https://localhost:5126). HTTPS preserves Secure/HttpOnly
+cookies and same-origin API access. Caddy uses `tls internal` and manages its local
+CA, certificates and renewal in the `caddy-data` volume. The `caddy-config` volume
+stores Caddy runtime configuration. Neither volume nor Caddy has Vault access.
 
-- Frontend and proxied API: [https://localhost:5126](https://localhost:5126).
-- BFF health: [https://localhost:5126/api/v1/health](https://localhost:5126/api/v1/health).
-- Vault UI/API: [http://127.0.0.1:18200](http://127.0.0.1:18200).
-
-Trust `.local/dev/ca.pem` in your local browser/OS certificate store to use HTTPS
-without a certificate error. Trust installation is a separate local action, not
-performed by the bootstrap. For a command-line check without changing trust:
+After frontend starts, Make exports only the public Caddy CA to `.local/dev/ca.pem`.
+Trust it in your browser/OS to avoid certificate errors. This is a new CA when
+migrating from Vault-issued browser certificates; the old CA will not trust it.
+Trust installation is not automated. A CLI check can use:
 
 ```sh
 curl --cacert .local/dev/ca.pem https://localhost:5126/api/v1/health
 ```
 
-HTTPS keeps the existing Secure/HttpOnly refresh cookie contract. Nginx serves
-Flutter and proxies `/api/` to the BFF on the same origin. The bundle selects the
-existing backend auth adapter with the public client ID `sxchnge-local-web`.
-Bootstrap inserts this public browser client if absent; it has no client secret.
-No frontend widget, service or provider architecture changes are needed.
+BFF calls notifications over plaintext gRPC with its caller token. Mongo and NATS
+also stay within the private Compose network; no internal service publishes a host
+port. Only frontend and Vault bind host loopback ports. TLS remains optional in
+the reusable notification client/server for deployments that explicitly configure
+it; local runtime configuration contains no certificate settings.
 
-The rest of the app retains its existing capabilities and limitations: wallet,
-KYC and settings fixtures are not turned into production integrations by this
-container setup. Full email signup/login requires a working Resend account.
+Caddy behavior follows its [internal TLS](https://caddyserver.com/docs/caddyfile/directives/tls)
+and [automatic HTTPS](https://caddyserver.com/docs/automatic-https) documentation.
 
-## Secret and connection lifecycle
+## Mongo transactions and credential rotation
 
-| Consumer | Vault access |
-| --- | --- |
-| BFF | `database/creds/bff`, JWT key, notification caller token and CA |
-| Notifications | `database/creds/notifications`, NATS token, caller token, TLS certificate/key and email settings |
-| NATS | NATS authentication token only |
-| Bootstrap | Infrastructure administration, PKI and initial identities |
+Mongo is an authenticated single-node replica set (`rs0`), supporting multi-document
+transactions. It has no failover or redundant data copy. A Mongo restart interrupts
+access even though application credential replacement does not.
 
-The database engine is configured at `database/config/mongo` with
-`mongodb-database-plugin`. It uses its own `vault-manager` Mongo user, whose
-password is rotated by Vault immediately after configuration. The manager has
-`userAdminAnyDatabase`; applications never receive that credential. The independent
-local Mongo administrator and replica-set key come from Vault's random generator
-and are retained in `kv/bootstrap/mongo` for local provisioning.
+The shared database wrapper authenticates with each service's AppRole and obtains
+an independent token/lease for each pool. At half the shorter token/lease lifetime,
+it verifies a replacement pool before publishing it. Active operations, cursors
+and full transactions retain the old pool until they finish, then it closes and
+its token is revoked. Nested stores and commit retries keep the same pool.
 
-The BFF role has `readWrite` only on `sxchnge`; notifications has `readWrite` only
-on `sxchnge_notifications`. Mongo URIs come from dynamic Vault credentials, not
-static application users. Leases default to one hour with a 24-hour maximum.
-The shared database wrapper obtains credentials directly from Vault using the
-service's AppRole. At half the effective lifetime (the shorter of the database
-lease and owning token TTL), it authenticates independently and prepares a new
-Mongo pool. Only a successful primary ping permits publication. New operations
-use that pool; active operations, cursors and complete transactions, including
-nested stores and commit retries, retain the old pool until they finish. The old
-pool then closes and its dedicated token is revoked. Mongo credential replacement
-never restarts BFF or notifications. Each generation has a separate token so
-cleanup cannot revoke the replacement or Vault Agent's identity.
+Failed refreshes retain valid credentials and retry with bounded backoff. After
+expiry, database operations fail closed until automatic recovery. Administrative
+revocation, infrastructure outages and operations longer than their lease can
+still interrupt work. Repository consumers use `source.Source` operation scopes;
+`source.Fixed` adapts externally owned static/test connections.
 
-Refresh failures keep the current valid pool and retry with bounded backoff.
-Operations are bounded by credential expiry. If Vault remains unavailable beyond
-that deadline, new database operations fail closed; refresh continues and access
-recovers automatically without restarting the service. Administrative revocation,
-operations exceeding their lease and infrastructure failures cannot be made
-uninterrupted by pool rotation. Shutdown rejects new scopes and drains existing
-work. Failed cleanup relies on Vault expiration as a backstop.
+## Migration and persistence
 
-Repositories and transaction factories share `source.Source` rather than retaining
-concrete Mongo collections. Low-level consumers must acquire a scope, use its
-returned context and release it after all work completes; never retain its client
-or context beyond release. `source.Fixed` adapts externally owned test/static
-connections. Database credentials are no longer Agent environment templates;
-Agent continues to fetch non-database secrets and may restart children when those
-secrets change.
+Bootstrap migrates the old flat `bff`, `caller`, `email` and `nats` paths into the
+hierarchy above. It copies with CAS protection, compares values, and refuses to
+overwrite conflicting destinations. After consumers use the new paths, it removes
+old KV metadata and obsolete `tls`, `frontend-tls`, `notifications-tls` entries.
+Readable old version history and metadata are archived privately in
+`.local/dev/migrations/organized-v1/` before removal. That archive is administrative
+recovery material; applications do not mount it. Old deleted/destroyed versions
+are not recreated. The old PKI engine is not used or provisioned by this stack;
+an existing PKI mount is retained to avoid destroying its non-exportable signing
+key and history as part of a KV reorganization.
 
-Each agent authenticates using its own AppRole with a renewable periodic token.
-Local SecretIDs are reusable for container restarts; each container mounts only
-its own identity directory under `.local/dev/agents/`. Revoking an AppRole SecretID
-requires removing its matching local `secret-id` file and rerunning `make up` to
-issue a replacement. Revoking an active token/lease terminates its credentials;
-The database wrapper retries with a fresh identity; Agent independently retries
-its non-database secrets when Vault is reachable.
+Vault, Mongo, JetStream and Caddy data use project-scoped named volumes. Private
+bootstrap material lives in `.local/dev/`. `make down` preserves both. Vault's local
+unseal file allows automatic unsealing on restart. This local convenience and
+loopback HTTP Vault endpoint are for development. Keep bootstrap files paired
+with their matching volumes; do not discard one side independently.
 
-The notification gRPC connection uses a Vault-issued certificate for
-`notifications` and the BFF validates the Vault CA and hostname. Frontend HTTPS
-certificates also come from Vault PKI. Leaf certificates last 30 days; `make up`
-refreshes them when less than seven days remain and reloads Nginx. Run it regularly
-for a development stack kept online for weeks. Agents restart when their TLS KV
-secret changes. Application secrets are never put into the frontend bundle.
-
-Vault, Mongo and JetStream data use project-scoped named Docker volumes. Local
-Vault unseal material, administrator login, per-service bootstrap identities and
-frontend TLS material are under ignored `.local/dev/`, outside image build contexts.
-The local Vault automatically unseals using its local key on container restart.
-These convenience credentials and loopback HTTP Vault access are for this local
-machine; this is not a production Vault deployment. Do not share `.local/dev/`.
-Keep it paired with its matching Docker volumes for recovery.
-
-Mongo and NATS expose no host ports. The `sxchnge-local` network, volumes and
-container names are independent of existing Sendico stacks. Only the frontend
-and Vault bind host loopback ports. No Sendico database, message bus or secrets
-are consumed.
-
-## Manage and verify
+## Verification
 
 ```sh
 make ps
 make logs
-make down             # preserves named volumes and local bootstrap material
-make up               # idempotent provisioning and readiness checks
-make test-dev-tooling # offline shell and failure-path checks
-make test-dev-infra   # works before Resend provisioning
-make test-dev-rotation # real Vault rotations under BFF HTTP traffic
-make test-dev         # requires all six services running
+make down
+make test-dev-tooling   # shell analysis, failure paths, migration conflict checks
+make test-dev-infra     # policy isolation, real Mongo transactions and lease lifecycle
+make test-dev-rotation  # temporarily deploy BFF/frontend and exercise short leases
+make test-dev           # requires all six services; verified frontend/API HTTPS
 ```
 
-The infrastructure check exercises AppRole access restrictions, real Vault
-MongoDB credential issuance, per-database access, lease renewal and user deletion
-on revocation. The complete check also verifies service health and frontend/API
-HTTPS with the generated CA. These checks never send a real email.
-`test-dev-rotation` temporarily sets a 12-second BFF lease, deploys the built BFF
-and frontend, and requires three pool replacements with unchanged process ID and
-zero unexpected HTTP responses. Its invalid-token requests exercise Mongo-backed
-transactions without sending email. The check restores the role TTL and removes
-services it started; it recreates BFF/frontend at the beginning, so run it as a
-development validation command, not against active work you need to preserve.
-Race-enabled backend tests additionally hold a transaction and cursor across a
-replacement and exercise failed candidate validation, outage recovery, rollback,
-expiry rejection and graceful shutdown.
-
-Backend regression tests use fresh `mongo:latest` and `nats:latest` official
-stable images, pulled before `make test-backend`. Normal validation remains
-`make test` and `make lint-backend` and requires the host SDKs described in the
-repository guides.
-
-Deleting Docker volumes removes local data; `make down` deliberately keeps them.
-If starting from scratch, stop the stack and explicitly remove this project's
-three volumes **and** its matching `.local/dev/` bootstrap state together. Never
-reuse old unseal material with a newly initialized Vault volume, or retain Mongo
-users while discarding the Vault data that manages their leases.
-
-## Sources and checked releases
-
-On 2026-09-12 the resolver selected Vault 2.1.0, MongoDB 8.3.9, NATS 2.14.6,
-Go 1.27.1, Nginx stable 1.30.4, Ubuntu 26.04 and Flutter stable 3.47.4. These are
-verification records, not fixed future defaults. Release selection follows the
-[Vault releases](https://releases.hashicorp.com/vault/),
-[MongoDB stable release notes](https://www.mongodb.com/docs/manual/release-notes/),
-[NATS releases](https://github.com/nats-io/nats-server/releases/latest),
-[Nginx downloads](https://nginx.org/en/download.html),
-[Go releases](https://go.dev/dl/) and
-[Flutter stable channel](https://github.com/flutter/flutter/tree/stable).
-The implementation follows HashiCorp's
-[MongoDB database plugin](https://developer.hashicorp.com/vault/docs/secrets/databases/mongodb)
-and [Agent process supervisor](https://developer.hashicorp.com/vault/docs/agent-and-proxy/agent/process-supervisor)
-contracts.
+Rotation validation temporarily shortens the BFF role TTL, recreates BFF/frontend,
+and requires three replacements with unchanged BFF PID and zero unexpected
+transaction-backed HTTP responses. It restores TTLs and removes services it
+started. No verification command sends email. Backend validation remains
+`make test` and `make lint-backend`, using the host SDKs documented in module guides.

infra/README.md

@@ -0,0 +1,20 @@
+# Infrastructure
+
+Run `make build && make up` from the repository root. Local tooling needs Make,
+POSIX shell and Docker Compose/BuildKit; all other tools run in Docker.
+
+- `images/`: reusable Docker build definitions for product images and tooling.
+- `local/compose.yml`: isolated local stack and persistent volumes.
+- `local/bff/`, `local/notifications/`: application runtime configuration.
+- `local/frontend/`: Caddy configuration for Flutter files, browser HTTPS and API proxying.
+- `local/vault/`: server startup/configuration and per-service Agent templates.
+- `local/mongo/`, `local/nats/`: infrastructure startup and Mongo provisioning checks.
+- `local/scripts/`: bootstrap, migration, release resolution and verification.
+- `local/tools.sh`: host entry point used by Make.
+
+Local generated pins, identities and administrator material remain under ignored
+`.local/dev/`; they are never build inputs. Application source remains under
+`backend/` and `frontend/`.
+
+See [local development](../docs/local-development.md) for Vault paths, commands,
+Caddy trust, persistence and migration details.

infra/images/backend.Dockerfile

@@ -13,8 +13,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/g
 FROM ${VAULT_IMAGE}
 USER root
 COPY --from=build /out/service /usr/local/bin/service
-COPY dev/start-service.sh /usr/local/bin/start-service
-COPY dev/config /etc/sxchnge
+COPY infra/local/scripts/start-service.sh /usr/local/bin/start-service
+COPY infra/local/bff/runtime.yml /etc/sxchnge/bff.yml
+COPY infra/local/notifications/runtime.yml /etc/sxchnge/notifications.yml
 COPY backend/notifications/templates.example.json /etc/sxchnge/templates.json
 RUN mkdir -p /run/sxchnge && chown 100:1000 /run/sxchnge
 USER 100:1000

infra/images/frontend.Dockerfile

@@ -1,5 +1,5 @@
 ARG UBUNTU_IMAGE
-ARG NGINX_IMAGE
+ARG CADDY_IMAGE
 FROM ${UBUNTU_IMAGE} AS build
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl git unzip xz-utils libglu1-mesa && rm -rf /var/lib/apt/lists/*
 ARG FLUTTER_VERSION
@@ -12,6 +12,6 @@ COPY frontend ./frontend
 WORKDIR /src/frontend/app
 ARG FRONTEND_URL=https://localhost:5126
 RUN flutter pub get && flutter gen-l10n && flutter build web --no-wasm-dry-run --dart-define=SXCHNGE_API_URL=${FRONTEND_URL}/api/v1 --dart-define=SXCHNGE_CLIENT_ID=sxchnge-local-web
-FROM ${NGINX_IMAGE}
-COPY dev/config/nginx.conf /etc/nginx/conf.d/default.conf
-COPY --from=build /src/frontend/app/build/web /usr/share/nginx/html
+FROM ${CADDY_IMAGE}
+COPY infra/local/frontend/Caddyfile /etc/caddy/Caddyfile
+COPY --from=build /src/frontend/app/build/web /srv

infra/images/nats.Dockerfile

@@ -4,7 +4,7 @@ FROM ${NATS_IMAGE} AS nats
 FROM ${VAULT_IMAGE}
 USER root
 COPY --from=nats /nats-server /usr/local/bin/nats-server
-COPY dev/start-nats.sh /usr/local/bin/start-nats
+COPY infra/local/nats/start.sh /usr/local/bin/start-nats
 RUN mkdir -p /data /run/sxchnge && chown -R 100:1000 /data /run/sxchnge
 USER 100:1000
 ENTRYPOINT ["vault", "agent", "-config=/etc/agent/config.hcl"]

infra/local/bff/runtime.yml

@@ -39,6 +39,4 @@ api:
   notifications:
     address: notifications:8083
     token_env: SXCHNGE_NOTIFICATIONS_TOKEN
-    ca_file: /run/sxchnge/ca.pem
-    server_name: notifications
     activation_url: https://localhost:5126/verify-email

infra/local/compose.yml

@@ -9,7 +9,7 @@ x-agent: &agent
 
 x-backend-build: &backend-build
   context: .
-  dockerfile: dev/images/backend.Dockerfile
+  dockerfile: infra/images/backend.Dockerfile
 
 services:
   vault:
@@ -20,8 +20,8 @@ services:
     ports: ['127.0.0.1:18200:8200']
     volumes:
       - vault-data:/vault/file
-      - ./dev/vault/server.hcl:/etc/vault/server.hcl:ro
-      - ./dev/start-vault.sh:/etc/vault/start.sh:ro
+      - ./infra/local/vault/server.hcl:/etc/vault/server.hcl:ro
+      - ./infra/local/vault/start.sh:/etc/vault/start.sh:ro
       - ./.local/dev/vault:/bootstrap:ro
     healthcheck:
       test: [CMD-SHELL, 'VAULT_ADDR=http://127.0.0.1:8200 vault status >/dev/null']
@@ -38,7 +38,7 @@ services:
       MONGO_INITDB_ROOT_USERNAME: local-admin
       MONGO_INITDB_ROOT_PASSWORD_FILE: /run/mongo/password
     volumes:
-      - ./dev/start-mongo.sh:/etc/mongo/start.sh:ro
+      - ./infra/local/mongo/start.sh:/etc/mongo/start.sh:ro
       - mongo-data:/data/db
       - ./.local/dev/mongo:/run/mongo:ro
     healthcheck:
@@ -52,7 +52,7 @@ services:
     image: sxchnge-local/nats:dev
     build:
       context: .
-      dockerfile: dev/images/nats.Dockerfile
+      dockerfile: infra/images/nats.Dockerfile
       args:
         NATS_IMAGE: ${NATS_IMAGE:?Run make build first}
         VAULT_IMAGE: ${VAULT_IMAGE:?Run make build first}
@@ -121,20 +121,21 @@ services:
     image: sxchnge-local/frontend:dev
     build:
       context: .
-      dockerfile: dev/images/frontend.Dockerfile
+      dockerfile: infra/images/frontend.Dockerfile
       args:
         UBUNTU_IMAGE: ${UBUNTU_IMAGE:?Run make build first}
-        NGINX_IMAGE: ${NGINX_IMAGE:?Run make build first}
+        CADDY_IMAGE: ${CADDY_IMAGE:?Run make build first}
         FLUTTER_VERSION: ${FLUTTER_VERSION:?Run make build first}
         FLUTTER_COMMIT: ${FLUTTER_COMMIT:?Run make build first}
     restart: unless-stopped
     ports: ['127.0.0.1:5126:8443']
     volumes:
-      - ./.local/dev/frontend:/etc/tls:ro
+      - caddy-data:/data
+      - caddy-config:/config
     depends_on:
       bff: {condition: service_healthy}
     healthcheck:
-      test: [CMD, wget, -q, --no-check-certificate, -O, /dev/null, 'https://127.0.0.1:8443/']
+      test: [CMD, wget, -q, --no-check-certificate, -O, /dev/null, 'https://localhost:8443/']
       interval: 3s
       timeout: 3s
       retries: 40
@@ -143,3 +144,5 @@ volumes:
   vault-data:
   mongo-data:
   nats-data:
+  caddy-data:
+  caddy-config:

infra/local/frontend/Caddyfile

@@ -0,0 +1,16 @@
+{
+	skip_install_trust
+	auto_https disable_redirects
+}
+
+https://localhost:8443 {
+	tls internal
+	root * /srv
+	handle /api/* {
+		reverse_proxy bff:8082
+	}
+	handle {
+		try_files {path} /index.html
+		file_server
+	}
+}

infra/local/notifications/runtime.yml

@@ -1,6 +1,4 @@
 listen: "0.0.0.0:8083"
-tls_cert: "/run/sxchnge/server.crt"
-tls_key: "/run/sxchnge/server.key"
 database:
   driver: mongodb
   settings:

infra/local/scripts/bootstrap.sh

@@ -16,7 +16,7 @@ initialize() {
 }
 mounts() {
  vault sys/mounts > "$WORK/mounts"
- for name in kv database pki; do
+ for name in kv database; do
   if ! jq -e --arg key "$name/" 'has($key)' < "$WORK/mounts" >/dev/null; then
    if ! jq -e --arg key "$name/" '.data|has($key)' < "$WORK/mounts" >/dev/null; then
     jq -n --arg name "$name" '{type:$name} + (if $name=="kv" then {options:{version:"2"}} else {} end)' | vault "sys/mounts/$name" POST >/dev/null
@@ -27,27 +27,6 @@ mounts() {
   printf '{"type":"approle"}' | vault sys/auth/approle POST >/dev/null
  fi
 }
-certificates() {
- printf '{"max_lease_ttl":"87600h"}' | vault sys/mounts/pki/tune POST >/dev/null
- ca=$(kv tls)
- if [ "$ca" = null ]; then
-  printf '{"common_name":"SXCHNGE local development CA","ttl":"87600h"}' | vault pki/root/generate/internal POST | jq '{ca:.data.certificate}' | put tls
- fi
- printf '{"allowed_domains":["localhost","notifications"],"allow_bare_domains":true,"allow_ip_sans":true,"max_ttl":"720h"}' | vault pki/roles/local POST >/dev/null
- for name in notifications-tls frontend-tls; do
-  host=notifications; [ "$name" != frontend-tls ] || host=localhost
-  kv "$name" > "$WORK/cert"
-  if ! jq -e '.!=null and .expiration>(now+604800)' "$WORK/cert" >/dev/null; then
-   jq -n --arg host "$host" '{common_name:$host,ip_sans:"127.0.0.1",ttl:"720h"}' | vault pki/issue/local POST | jq '.data' > "$WORK/cert"
-   put "$name" < "$WORK/cert"
-  fi
-  if [ "$name" = frontend-tls ]; then
-   jq -r '.certificate' "$WORK/cert" | write "$STATE/frontend/server.crt"
-   jq -r '.private_key' "$WORK/cert" | write "$STATE/frontend/server.key"
-  fi
- done
- kv tls | jq -r '.ca' | write "$STATE/ca.pem" 644
-}
 provision_mongo() {
  if [ "$(kv bootstrap/mongo)" = null ]; then
   random > "$WORK/password"; random 384 > "$WORK/keyfile"; random > "$WORK/manager"
@@ -72,16 +51,16 @@ provision_mongo() {
   database=sxchnge; [ "$role" != notifications ] || database=sxchnge_notifications
   jq -n --arg database "$database" '{db_name:"mongo",creation_statements:({db:"admin",roles:[{role:"readWrite",db:$database}]}|tojson),default_ttl:"1h",max_ttl:"24h"}' | vault "database/roles/$role" POST >/dev/null
  done
- { cat "$WORK/admin.js"; cat dev/mongo-client.js; } | mongo >/dev/null || fail 'Browser client provisioning failed'
+ { cat "$WORK/admin.js"; cat infra/local/mongo/mongo-client.js; } | mongo >/dev/null || fail 'Browser client provisioning failed'
 }
 provision_agents() {
  for service in bff notifications nats; do
   directory="$STATE/agents/$service"
   mkdir -p "$directory"; chmod 755 "$directory"
   case "$service" in
-   bff) paths='bff caller tls';;
-   notifications) paths='caller tls notifications-tls nats email';;
-   nats) paths=nats;;
+   bff) paths='services/bff/jwt services/notifications/callers/bff';;
+   notifications) paths='services/notifications/callers/bff infrastructure/nats/auth services/notifications/email/resend';;
+   nats) paths=infrastructure/nats/auth;;
   esac
   : > "$WORK/policy"
   for path in $paths; do printf 'path "kv/data/%s" { capabilities = ["read"] }\n' "$path" >> "$WORK/policy"; done
@@ -92,7 +71,12 @@ provision_agents() {
   if [ ! -f "$directory/secret-id" ]; then
    printf '{}' | vault "auth/approle/role/$service/secret-id" POST | jq -r '.data.secret_id' | write "$directory/secret-id" 444
   fi
-  cat "dev/vault/$service.hcl" | write "$directory/config.hcl" 444
+  if ! cmp -s "infra/local/vault/$service.hcl" "$directory/config.hcl"; then
+   cat "infra/local/vault/$service.hcl" | write "$directory/config.hcl" 444
+   if compose ps --status running --services "$service" | grep -qx "$service"; then
+    compose up -d --no-deps --force-recreate --wait --wait-timeout 120 "$service"
+   fi
+  fi
  done
 }
 up() {
@@ -107,20 +91,21 @@ up() {
  fi
  initialize
  mounts
- for path in nats caller bff; do
+ migrate_secrets
+ for path in infrastructure/nats/auth services/notifications/callers/bff services/bff/jwt; do
   if [ "$(kv "$path")" = null ]; then
-   field=token; [ "$path" != bff ] || field=jwt
+   field=token; [ "$path" != services/bff/jwt ] || field=jwt
    random | jq -Rs --arg field "$field" '{($field):rtrimstr("\n")}' | put "$path"
   fi
  done
- printf '{"driver":"resend","api_key":"","sender_address":"notifications@localhost.test","sender_name":"SXCHNGE local"}' | ensure email
- certificates
+ printf '{"driver":"resend","api_key":"","sender_address":"notifications@localhost.test","sender_name":"SXCHNGE local"}' | ensure services/notifications/email/resend
  provision_mongo
  provision_agents
  compose up -d --wait --wait-timeout 120 nats
- kv email | email_valid || fail 'Vault kv/email needs a Resend api_key and verified sender_address. Run make email, then make up.'
+ retire_secrets
+ kv services/notifications/email/resend | email_valid || fail 'Vault kv/services/notifications/email/resend needs a Resend api_key and verified sender_address. Run make email, then make up.'
  compose up -d --wait --wait-timeout 180
- compose exec -T frontend nginx -s reload
+ export_ca
  echo 'Local stack ready: https://localhost:5126; Vault: http://127.0.0.1:18200'
 }
 email() {
@@ -138,6 +123,6 @@ email() {
  case "$address" in *@*) ;; *) fail 'A sender email address is required';; esac
  printf '%s' "$key" > "$WORK/email-key"
  printf '%s' "$address" > "$WORK/email-address"
- jq -n --rawfile key "$WORK/email-key" --rawfile address "$WORK/email-address" '{driver:"resend",api_key:$key,sender_address:$address,sender_name:"SXCHNGE"}' | put email
+ jq -n --rawfile key "$WORK/email-key" --rawfile address "$WORK/email-address" '{driver:"resend",api_key:$key,sender_address:$address,sender_name:"SXCHNGE"}' | put services/notifications/email/resend
  echo 'Email configuration saved in Vault. Run make up.'
 }

infra/local/scripts/common.sh

@@ -14,7 +14,7 @@ trap cleanup EXIT
 trap 'exit 130' INT
 trap 'exit 143' TERM
 fail() { echo "$*" >&2; exit 1; }
-compose() { docker compose --env-file "$STATE/images.env" "$@"; }
+compose() { docker compose --project-directory . -f infra/local/compose.yml --env-file "$STATE/images.env" "$@"; }
 write() {
  target=$1; mode=${2:-600}
  mkdir -p "$(dirname "$target")"
@@ -58,3 +58,6 @@ ensure() (
 mongo() { compose exec -T mongo mongosh --quiet --file /dev/stdin > "$WORK/mongo-result" 2> "$WORK/mongo-error" || return 1; cat "$WORK/mongo-result"; }
 admin_auth() { kv bootstrap/mongo | jq -r '"const admin=db.getSiblingDB(\"admin\"); admin.auth(\"local-admin\","+(.password|tojson)+");"'; }
 email_valid() { jq -e '(.api_key|type=="string") and (.api_key|test("\\S")) and (.sender_address|type=="string") and (.sender_address|test("\\S")) and .sender_address!="notifications@localhost.test"' >/dev/null; }
+export_ca() {
+ compose exec -T frontend cat /data/caddy/pki/authorities/local/root.crt | write "$STATE/ca.pem" 644
+}

infra/local/scripts/dispatch.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+set -o pipefail
+. infra/local/scripts/common.sh
+. infra/local/scripts/secrets.sh
+case "${1:-}" in
+ images) . infra/local/scripts/images.sh; images;;
+ up) . infra/local/scripts/bootstrap.sh; up;;
+ email) . infra/local/scripts/bootstrap.sh; email;;
+ verify|verify-infra) . infra/local/scripts/verify.sh; existing; infrastructure; if [ "$1" = verify ]; then product; fi;;
+ rotation) . infra/local/scripts/verify.sh; . infra/local/scripts/rotation.sh; existing; rotation;;
+ test) sh infra/local/scripts/test.sh;;
+ *) fail 'Unknown local tooling command';;
+esac

infra/local/scripts/images.sh

@@ -10,7 +10,7 @@ image() (
 )
 images() (
  go=$(get 'https://go.dev/dl/?mode=json' | jq -er '.[0] | select(.stable) | .version | select(test("^go[0-9]+\\.[0-9]+\\.[0-9]+$"))') || exit 1
- for spec in VAULT_IMAGE:hashicorp/vault:latest MONGO_IMAGE:library/mongo:latest NATS_IMAGE:library/nats:latest NGINX_IMAGE:library/nginx:stable-alpine "GO_IMAGE:library/golang:${go#go}-alpine" UBUNTU_IMAGE:library/ubuntu:latest; do
+ for spec in VAULT_IMAGE:hashicorp/vault:latest MONGO_IMAGE:library/mongo:latest NATS_IMAGE:library/nats:latest CADDY_IMAGE:library/caddy:alpine "GO_IMAGE:library/golang:${go#go}-alpine" UBUNTU_IMAGE:library/ubuntu:latest; do
   key=${spec%%:*}; rest=${spec#*:}; repo=${rest%:*}; alias=${rest##*:}
   ref=$(image "$repo" "$alias") || exit 1
   printf '%s=%s\n' "$key" "$ref" >> "$WORK/images.env"

infra/local/scripts/rotation.sh

@@ -20,6 +20,12 @@ rotation() (
  trap 'exit 143' TERM
  jq '.+{default_ttl:"12s",max_ttl:"24s"}' "$WORK/original-role" | vault database/roles/bff POST >/dev/null
  compose up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend
+ export_ca
+ https -f https://localhost:5126/ > "$WORK/index"
+ grep -q flutter_bootstrap "$WORK/index" || fail 'Frontend bundle is not served'
+ https -f https://localhost:5126/settings > "$WORK/route"
+ cmp -s "$WORK/index" "$WORK/route" || fail 'Frontend route fallback failed'
+ https -f https://localhost:5126/api/v1/health >/dev/null
  process=$(compose exec -T bff pidof service)
  for worker in 1 2 3 4; do
   (

infra/local/scripts/secrets.sh

@@ -0,0 +1,56 @@
+# Canonical paths follow ownership. Shared callers live under the receiving service.
+secret_moves() {
+ cat <<'PATHS'
+bff services/bff/jwt
+caller services/notifications/callers/bff
+email services/notifications/email/resend
+nats infrastructure/nats/auth
+PATHS
+}
+# Archive readable version history privately before removing an obsolete KV path.
+archive_secret() (
+ old=$1
+ metadata=$(vault "kv/metadata/$old" GET yes)
+ [ "$metadata" != null ] || exit 0
+ directory="$STATE/migrations/organized-v1/$old"
+ mkdir -p "$directory"
+ printf '%s' "$metadata" | write "$directory/metadata.json"
+ printf '%s' "$metadata" | jq -r '.data.versions|to_entries[]|select(.value.destroyed==false and .value.deletion_time=="")|.key' > "$WORK/versions"
+ while IFS= read -r version; do
+  vault "kv/data/$old?version=$version" | write "$directory/version-$version.json"
+ done < "$WORK/versions"
+)
+migrate_secrets() (
+ secret_moves > "$WORK/moves"
+ while read -r old new; do
+  kv "$old" > "$WORK/old"
+  [ "$(cat "$WORK/old")" != null ] || continue
+  archive_secret "$old"
+  kv "$new" > "$WORK/new"
+  if [ "$(cat "$WORK/new")" = null ]; then
+   # CAS=0 prevents overwriting a destination created concurrently.
+   jq '{data:.,options:{cas:0}}' "$WORK/old" | vault "kv/data/$new" POST >/dev/null
+  fi
+  kv "$new" > "$WORK/new"
+  jq -e -s '.[0]==.[1]' "$WORK/old" "$WORK/new" >/dev/null || fail "Conflicting Vault values at $old and $new; no source removed"
+ done < "$WORK/moves"
+)
+retire_secrets() (
+ # Call only after templates, policies and deployed images use canonical paths.
+ secret_moves > "$WORK/moves"
+ while read -r old new; do
+  kv "$old" > "$WORK/old"
+  [ "$(cat "$WORK/old")" != null ] || continue
+  kv "$new" > "$WORK/new"
+  jq -e -s '.[0]==.[1]' "$WORK/old" "$WORK/new" >/dev/null || fail "Vault migration verification failed for $old"
+  archive_secret "$old"
+  printf '{}' | vault "kv/metadata/$old" DELETE >/dev/null
+ done < "$WORK/moves"
+ for old in tls frontend-tls notifications-tls; do
+  archive_secret "$old"
+  if [ "$(vault "kv/metadata/$old" GET yes)" != null ]; then printf '{}' | vault "kv/metadata/$old" DELETE >/dev/null; fi
+ done
+ # The obsolete PKI mount is deliberately not disabled: its private signing key
+ # and history are retained until the owner explicitly chooses to destroy them.
+ echo 'PASS: Vault secrets migrated and verified; obsolete KV entries retired'
+)

infra/local/scripts/start-service.sh

@@ -1,13 +1,9 @@
 #!/bin/sh
 set -eu
 umask 077
-printf '%s\n' "$TLS_CA" > /run/sxchnge/ca.pem
-unset TLS_CA
 if [ "$SXCHNGE_SERVICE" = notifications ]; then
-    printf '%s\n' "$TLS_CERT" > /run/sxchnge/server.crt
-    printf '%s\n' "$TLS_KEY" > /run/sxchnge/server.key
     printf '%s\n' "$EMAIL_CONFIG" > /run/sxchnge/email.yml
-    unset TLS_CERT TLS_KEY EMAIL_CONFIG
+    unset EMAIL_CONFIG
     cat /etc/sxchnge/notifications.yml /run/sxchnge/email.yml > /run/sxchnge/config.yml
     exec service -config.file=/run/sxchnge/config.yml
 fi

infra/local/scripts/test.sh

@@ -2,11 +2,11 @@
 set -eu
 set -o pipefail
 # Shell syntax and static analysis cover the actual container shell scripts.
-shellcheck -s ash -x -S warning dev/dispatch.sh dev/test.sh
-shellcheck -x -S warning dev/tools.sh dev/start-*.sh
-. dev/common.sh
-. dev/images.sh
-. dev/bootstrap.sh
+shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh
+shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh
+. infra/local/scripts/common.sh
+. infra/local/scripts/images.sh
+. infra/local/scripts/bootstrap.sh
 # Offline fixtures exercise stable selection, reject prereleases, and preserve pins.
 get() {
  case "$1" in
@@ -33,7 +33,29 @@ vault() { printf '{"initialized":true}'; }
 if (initialize) > "$WORK/initialize-error" 2>&1; then fail 'Missing local keys accepted'; fi
 grep -q 'Missing .local' "$WORK/initialize-error"
 for service in bff notifications; do
- if grep -Eq 'database/creds/|SXCHNGE_MONGO_URI' "dev/vault/$service.hcl"; then fail 'Database credentials trigger Agent restart'; fi
+ if grep -Eq 'database/creds/|SXCHNGE_MONGO_URI' "infra/local/vault/$service.hcl"; then fail 'Database credentials trigger Agent restart'; fi
 done
 echo 'PASS: shell analysis, stable selection, failed-resolution preservation, email validation, missing-key protection and Agent contract'
 if command -v python >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1; then fail 'Tool image unexpectedly contains Python'; fi
+# Migration never overwrites an independently provisioned destination.
+. infra/local/scripts/secrets.sh
+secret_moves() { echo 'email services/notifications/email/resend'; }
+archive_secret() { :; }
+kv() {
+ case "$1" in
+  email) printf '{"api_key":"existing-fixture"}';;
+  *) cat "$WORK/destination";;
+ esac
+}
+vault() {
+ [ "$1" = kv/data/services/notifications/email/resend ] || fail 'Unexpected migration mutation'
+ jq -e '.options.cas==0' >/dev/null
+ printf '{"api_key":"existing-fixture"}' > "$WORK/destination"
+}
+printf 'null' > "$WORK/destination"
+migrate_secrets
+migrate_secrets
+printf '{"api_key":"different-fixture"}' > "$WORK/destination"
+if (migrate_secrets) >/dev/null 2>&1; then fail 'Migration overwrote conflicting destination'; fi
+jq -e '.api_key=="different-fixture"' "$WORK/destination" >/dev/null
+echo 'PASS: secret migration is repeatable and refuses conflicting destination values'

infra/local/scripts/verify.sh

@@ -6,7 +6,7 @@ policies() (
   directory="$STATE/agents/$service"
   login=$(jq -n --rawfile role "$directory/role-id" --rawfile secret "$directory/secret-id" '{role_id:($role|rtrimstr("\n")),secret_id:($secret|rtrimstr("\n"))}' | vault auth/approle/login POST)
   identity=$(printf '%s' "$login" | jq -er '.auth.client_token')
-  printf '%s' "$login" | jq '{token:.auth.client_token,paths:["kv/data/bootstrap/mongo","sys/init"]}' | vault sys/capabilities POST | jq -e '.data | all(.[]; .==["deny"])' >/dev/null || fail 'Service can access bootstrap credentials'
+  printf '%s' "$login" | jq '{token:.auth.client_token,paths:["kv/data/bootstrap/mongo","sys/init","kv/data/bff","kv/data/caller","kv/data/email","kv/data/nats","kv/data/tls"]}' | vault sys/capabilities POST | jq -e '.data | all(.[]; .==["deny"])' >/dev/null || fail 'Service can access bootstrap credentials'
   VAULT_TOKEN=$(printf '%s' "$login" | jq -er '.auth.client_token')
   printf '{}' | vault auth/token/revoke-self POST >/dev/null
   VAULT_TOKEN=$root_token
@@ -23,7 +23,7 @@ database_access() (
   {
    jq -r '"db.getSiblingDB(\"admin\").auth("+(.data.username|tojson)+","+(.data.password|tojson)+");"' "$WORK/lease"
    printf 'const database="%s", forbidden="%s";\n' "$database" "$forbidden"
-   cat dev/mongo-check.js
+   cat infra/local/mongo/mongo-check.js
   } | mongo > "$WORK/check" || fail 'Mongo access/transaction check failed'
   grep -q scope-ok "$WORK/check" || fail 'Mongo check incomplete'
   jq '{lease_id:.lease_id,increment:"1h"}' "$WORK/lease" | vault sys/leases/renew POST | jq -e '.lease_duration>0' >/dev/null || fail 'Mongo renewal failed'
@@ -37,6 +37,7 @@ database_access() (
 infrastructure() {
  compose ps --format json | jq -se '[.[] | select(.Service=="vault" or .Service=="mongo" or .Service=="nats")] | length==3 and all(.[];.Health=="healthy")' >/dev/null || fail 'Infrastructure is not healthy'
  echo 'PASS: Vault, Mongo replica set and NATS JetStream are healthy'
+ layout
  policies
  database_access
 }
@@ -48,3 +49,12 @@ product() {
  https -f https://localhost:5126/api/v1/health >/dev/null
  echo 'PASS: six healthy services, frontend and proxied BFF use verified TLS; no email sent'
 }
+layout() {
+ for old in bff caller email nats tls frontend-tls notifications-tls; do
+  [ "$(vault "kv/metadata/$old" GET yes)" = null ] || fail "Obsolete KV entry remains: $old"
+ done
+ for path in services/bff/jwt services/notifications/callers/bff services/notifications/email/resend infrastructure/nats/auth bootstrap/mongo; do
+  vault "kv/metadata/$path" | jq -e '.data.current_version>0' >/dev/null || fail "Missing canonical KV entry: $path"
+ done
+ echo 'PASS: canonical Vault hierarchy present; obsolete flat and TLS entries absent'
+}

infra/local/tools.sh

@@ -1,13 +1,13 @@
 #!/bin/sh
 # Host entry point: only POSIX shell and Docker are required.
 set -eu
-cd "$(dirname "$0")/.."
+cd "$(dirname "$0")/../.."
 case "${1:-}" in
  build)
   docker pull docker:cli
   ref=$(docker image inspect docker:cli --format '{{index .RepoDigests 0}}')
   case "$ref" in docker@sha256:*) ;; *) echo 'Cannot pin stable Docker CLI image' >&2; exit 1;; esac
-  docker build --pull --build-arg "DOCKER_IMAGE=$ref" -f dev/images/tools.Dockerfile -t sxchnge-local/tools:dev dev/images
+  docker build --pull --build-arg "DOCKER_IMAGE=$ref" -f infra/images/tools.Dockerfile -t sxchnge-local/tools:dev infra/images
   exit
   ;;
 esac
@@ -22,4 +22,4 @@ if [ "${1:-}" = email ]; then terminal_args=-it; fi
 # shellcheck disable=SC2086
 exec docker run --rm $terminal_args $network_args -e "LOCAL_UID=$(id -u)" -e "LOCAL_GID=$(id -g)" \
  -v "$socket:/var/run/docker.sock" -v "$PWD:$PWD" -w "$PWD" \
- sxchnge-local/tools:dev sh dev/dispatch.sh "$@"
+ sxchnge-local/tools:dev sh infra/local/scripts/dispatch.sh "$@"

infra/local/vault/bff.hcl

@@ -13,18 +13,14 @@ template_config {
   exit_on_retry_failure = true
 }
 
-env_template "TLS_CA" {
-  contents = "{{ with secret \"kv/data/tls\" }}{{ .Data.data.ca }}{{ end }}"
-  error_on_missing_key = true
-}
 
 env_template "SXCHNGE_JWT_SECRET" {
-  contents = "{{ with secret \"kv/data/bff\" }}{{ .Data.data.jwt }}{{ end }}"
+  contents = "{{ with secret \"kv/data/services/bff/jwt\" }}{{ .Data.data.jwt }}{{ end }}"
   error_on_missing_key = true
 }
 
 env_template "SXCHNGE_NOTIFICATIONS_TOKEN" {
-  contents = "{{ with secret \"kv/data/caller\" }}{{ .Data.data.token }}{{ end }}"
+  contents = "{{ with secret \"kv/data/services/notifications/callers/bff\" }}{{ .Data.data.token }}{{ end }}"
   error_on_missing_key = true
 }
 

infra/local/vault/nats.hcl

@@ -14,7 +14,7 @@ template_config {
 }
 
 env_template "NATS_TOKEN" {
-  contents = "{{ with secret \"kv/data/nats\" }}{{ .Data.data.token }}{{ end }}"
+  contents = "{{ with secret \"kv/data/infrastructure/nats/auth\" }}{{ .Data.data.token }}{{ end }}"
   error_on_missing_key = true
 }
 

infra/local/vault/notifications.hcl

@@ -0,0 +1,43 @@
+vault { address = "http://vault:8200" }
+auto_auth {
+  method "approle" {
+    config = {
+      role_id_file_path = "/etc/agent/role-id"
+      secret_id_file_path = "/etc/agent/secret-id"
+      remove_secret_id_file_after_reading = false
+    }
+  }
+}
+template_config {
+  static_secret_render_interval = "10s"
+  exit_on_retry_failure = true
+}
+
+
+env_template "SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN" {
+  contents = "{{ with secret \"kv/data/services/notifications/callers/bff\" }}{{ .Data.data.token }}{{ end }}"
+  error_on_missing_key = true
+}
+
+env_template "SXCHNGE_NATS_URL" {
+  contents = "{{ with secret \"kv/data/infrastructure/nats/auth\" }}nats://{{ .Data.data.token }}@nats:4222{{ end }}"
+  error_on_missing_key = true
+}
+
+
+
+env_template "EMAIL_API_KEY" {
+  contents = "{{ with secret \"kv/data/services/notifications/email/resend\" }}{{ .Data.data.api_key }}{{ end }}"
+  error_on_missing_key = true
+}
+
+env_template "EMAIL_CONFIG" {
+  contents = "{{ with secret \"kv/data/services/notifications/email/resend\" }}email:\n  driver: {{ .Data.data.driver | toJSON }}\n  settings:\n    api_key_env: EMAIL_API_KEY\n    sender_address: {{ .Data.data.sender_address | toJSON }}\n    sender_name: {{ .Data.data.sender_name | toJSON }}\n{{ end }}"
+  error_on_missing_key = true
+}
+
+exec {
+  command = ["/usr/local/bin/start-service"]
+  restart_on_secret_changes = "always"
+  restart_stop_signal = "SIGTERM"
+}