Organize local infrastructure and Vault secrets by service ownership

AGENTS.md

@@ -169,3 +169,15 @@ an uncommitted working tree. Preserve unrelated changes and avoid mixing them
 into task commits. Record scope, validation, limitations, and PR links in the
 task and durable repository documentation. Merge only when authorized by the
 user. If work is delivered as stacked PRs, document dependencies and merge order.
+
+## Local infrastructure ownership
+
+Keep infrastructure Dockerfiles under `infra/images/` and local Compose, service
+configuration and shell tooling under `infra/local/`. Keep product source under
+`backend/` and `frontend/`. Organize Vault KV paths by owning service under
+`services/<service>/<purpose>`, infrastructure under `infrastructure/<component>/<purpose>`,
+and bootstrap administration under `bootstrap/<component>`. Caller credentials
+belong to the receiving service at `services/<service>/callers/<caller>`.
+Use Caddy for local browser HTTPS and certificate management. Treat the private
+local Docker network as trusted; do not add internal gRPC TLS or Vault-managed
+local TLS certificates without an explicit requirement.

Makefile

@@ -53,16 +53,16 @@ package-notifications-interface: generate-notifications
 	tar -czf dist/notifications-go-v1.tar.gz -C backend/notifications/interface go.mod go.sum notifications README.md
 
 # Local product stack. Releases are resolved afresh, then pinned by digest.
-COMPOSE = docker compose --env-file .local/dev/images.env
+COMPOSE = docker compose --project-directory . -f infra/local/compose.yml --env-file .local/dev/images.env
 .PHONY: build up down logs ps test-dev
 build:
-	sh dev/tools.sh build
+	sh infra/local/tools.sh build
-	sh dev/tools.sh images
+	sh infra/local/tools.sh images
 	$(COMPOSE) pull vault mongo
 	$(COMPOSE) build --pull
 
 up:
-	sh dev/tools.sh up
+	sh infra/local/tools.sh up
 
 down:
 	$(COMPOSE) down
@@ -74,21 +74,21 @@ ps:
 	$(COMPOSE) ps
 
 test-dev:
-	sh dev/tools.sh test
+	sh infra/local/tools.sh test
-	sh dev/tools.sh verify
+	sh infra/local/tools.sh verify
 
 .PHONY: test-dev-infra
 test-dev-infra:
-	sh dev/tools.sh test
+	sh infra/local/tools.sh test
-	sh dev/tools.sh verify-infra
+	sh infra/local/tools.sh verify-infra
 
 .PHONY: test-dev-rotation
 test-dev-rotation:
-	sh dev/tools.sh rotation
+	sh infra/local/tools.sh rotation
 
 .PHONY: email test-dev-tooling
 email:
-	sh dev/tools.sh email
+	sh infra/local/tools.sh email
 
 test-dev-tooling:
-	sh dev/tools.sh test
+	sh infra/local/tools.sh test

README.md

@@ -40,7 +40,7 @@ make build && make up
 
 This builds the product images and starts an isolated Vault, authenticated MongoDB
 replica set and NATS JetStream. First startup intentionally stops after bootstrap
-until you set the Resend API key and verified sender in Vault's `kv/email` secret.
+until you set the Resend API key and verified sender in Vault's `kv/services/notifications/email/resend` secret.
 Then rerun `make up` to start notifications, BFF and the frontend at
 [https://localhost:5126](https://localhost:5126).
 
@@ -72,7 +72,7 @@ repository root:
 
 ```sh
 docker run --rm --pull always -p 127.0.0.1:5123:80 \
-  -v "$PWD/frontend/app/build/web:/usr/share/nginx/html:ro" nginx:stable-alpine
+  -v "$PWD/frontend/app/build/web:/srv:ro" caddy:alpine caddy file-server --root /srv --listen :80
 ```
 
 Open [the local app](http://127.0.0.1:5123/). The language selector supports EN/RU

backend/notifications/README.md

@@ -24,10 +24,11 @@ Requires Go 1.27, protoc, an SXCHNGE MongoDB database and a NATS server with
 JetStream enabled. Copy `config.example.yml` to an untracked `config.yml` here.
 Set `SXCHNGE_MONGO_URI`, `SXCHNGE_NATS_URL`, a unique token of at least 32 bytes for
 each configured caller, and the selected email provider's API key environment
-variable. Configure a verified sender address, TLS certificate/key and template
+variable. Configure a verified sender address and template
 catalog. Paths are relative to the process working directory.
 
-From the root, run `make run-notifications`. The gRPC listener always uses TLS.
+From the root, run `make run-notifications`. The gRPC listener uses plaintext on the private local network. Configure both
+`tls_cert` and `tls_key` only when TLS is explicitly required.
 Use a private authenticated NATS endpoint; `tls://` URLs and credentials in the URL
 are supported by the shared driver. Mongo and messaging both use shared
 `DriverConfig` settings; email settings reject unknown keys.

backend/notifications/app/config.go

@@ -40,9 +40,12 @@ func Load(path string) (Config, error) {
 	if err = decoder.Decode(new(any)); err != io.EOF {
 		return c, merrors.InvalidArgument("unexpected configuration content")
 	}
-	if c.Listen == "" || c.TLSCert == "" || c.TLSKey == "" || c.Database.Driver != db.Mongo || c.Messaging.Driver != messaging.BBNats || c.Templates == "" || len(c.Callers) == 0 {
+	if c.Listen == "" || c.Database.Driver != db.Mongo || c.Messaging.Driver != messaging.BBNats || c.Templates == "" || len(c.Callers) == 0 {
 		return c, merrors.InvalidArgument("missing notification configuration")
 	}
+	if (c.TLSCert == "") != (c.TLSKey == "") {
+		return c, merrors.InvalidArgument("both TLS certificate and key must be configured together")
+	}
 	if err = c.Queue.Validate(); err != nil {
 		return c, err
 	}

backend/notifications/app/config_test.go

@@ -33,3 +33,31 @@ func TestExampleConfigAndSecrets(t *testing.T) {
 		t.Fatal("extra YAML document accepted")
 	}
 }
+
+func TestPlaintextConfigAndIncompleteTLS(t *testing.T) {
+	t.Setenv("SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN", strings.Repeat("a", 32))
+	c, err := Load("../config.example.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	options, err := serverOptions(c)
+	if err != nil || len(options) != 0 {
+		t.Fatal("plain gRPC must not require TLS material")
+	}
+	data, err := os.ReadFile("../config.example.yml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, partial := range []string{"\ntls_cert: missing.crt\n", "\ntls_key: missing.key\n"} {
+		path := filepath.Join(t.TempDir(), "config.yml")
+		if err := os.WriteFile(path, append(data, []byte(partial)...), 0600); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := Load(path); err == nil {
+			t.Fatal("incomplete TLS configuration accepted")
+		}
+	}
+	if _, err := serverOptions(Config{TLSCert: "missing.crt", TLSKey: "missing.key"}); err == nil {
+		t.Fatal("bad explicit certificate accepted")
+	}
+}

backend/notifications/app/run.go

@@ -33,9 +33,9 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error {
 	if err != nil {
 		return err
 	}
-	cert, err := tls.LoadX509KeyPair(c.TLSCert, c.TLSKey)
+	options, err := serverOptions(c)
 	if err != nil {
-		return merrors.OpaqueWrap(merrors.ErrInvalidArg, err, "invalid gRPC TLS certificate")
+		return err
 	}
 	client, err := db.ConnectMongo(logger, &c.Database)
 	if err != nil {
@@ -67,7 +67,7 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error {
 	for caller, env := range c.Callers {
 		tokens[caller] = os.Getenv(env)
 	}
-	server := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12})), grpc.UnaryInterceptor(transport.Authenticate(tokens)), grpc.MaxRecvMsgSize(128<<10), grpc.MaxConcurrentStreams(64))
+	server := grpc.NewServer(append(options, grpc.UnaryInterceptor(transport.Authenticate(tokens)), grpc.MaxRecvMsgSize(128<<10), grpc.MaxConcurrentStreams(64))...)
 	pb.RegisterNotificationsServiceServer(server, transport.New(svc))
 	listener, err := net.Listen("tcp", c.Listen)
 	if err != nil {
@@ -103,3 +103,15 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error {
 	logger.Info("Notifications service stopped")
 	return nil
 }
+
+// An omitted certificate pair uses plaintext gRPC on the private service network.
+func serverOptions(c Config) ([]grpc.ServerOption, error) {
+	if c.TLSCert == "" && c.TLSKey == "" {
+		return nil, nil
+	}
+	cert, err := tls.LoadX509KeyPair(c.TLSCert, c.TLSKey)
+	if err != nil {
+		return nil, merrors.OpaqueWrap(merrors.ErrInvalidArg, err, "invalid gRPC TLS certificate")
+	}
+	return []grpc.ServerOption{grpc.Creds(credentials.NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}))}, nil
+}

backend/notifications/config.example.yml

@@ -1,6 +1,4 @@
 listen: "127.0.0.1:8083"
-tls_cert: "server.crt"
-tls_key: "server.key"
 database:
   driver: mongodb
   settings:

backend/product/config.example.yml

@@ -32,8 +32,6 @@ api:
       check:
         min_length: 12
   notifications:
-    address: localhost:8443
+    address: localhost:8083
     token_env: SXCHNGE_NOTIFICATIONS_TOKEN
-    ca_file: ../notifications/tls/ca.pem
-    server_name: localhost
     activation_url: http://localhost:5126/verify-email

backend/product/notifications/client.go

@@ -12,6 +12,7 @@ import (
 	merrors "git.sendico.io/tech/sxchnge/shared/errors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
@@ -57,7 +58,11 @@ func Open(c Config) (*Client, error) {
 		}
 		tlsConfig.RootCAs = pool
 	}
-	conn, err := grpc.NewClient(c.Address, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
+	var transport credentials.TransportCredentials = insecure.NewCredentials()
+	if c.CAFile != "" || c.ServerName != "" {
+		transport = credentials.NewTLS(tlsConfig)
+	}
+	conn, err := grpc.NewClient(c.Address, grpc.WithTransportCredentials(transport))
 	if err != nil {
 		return nil, merrors.InternalWrap(err, "create notification client")
 	}

backend/product/notifications/client_test.go

@@ -88,3 +88,19 @@ func TestOpenUsesVerifiedTLSAndCallerAuthentication(t *testing.T) {
 	require.NoError(t, client.Send(context.Background(), Message{ExpiresAt: time.Now().Add(time.Minute)}))
 	require.Equal(t, "Bearer test-service-token-with-at-least-32-bytes", <-rpc.auth)
 }
+
+func TestOpenUsesPlaintextAndCallerTokenOnPrivateNetwork(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	server := grpc.NewServer()
+	rpc := &tlsRPC{auth: make(chan string, 1)}
+	pb.RegisterNotificationsServiceServer(server, rpc)
+	go func() { _ = server.Serve(listener) }()
+	t.Cleanup(server.Stop)
+	t.Setenv("TEST_PLAIN_CALLER_TOKEN", "test-private-service-token-at-least-32-bytes")
+	client, err := Open(Config{Address: listener.Addr().String(), TokenEnv: "TEST_PLAIN_CALLER_TOKEN"})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = client.Close() })
+	require.NoError(t, client.Send(context.Background(), Message{ExpiresAt: time.Now().Add(time.Minute)}))
+	require.Equal(t, "Bearer test-private-service-token-at-least-32-bytes", <-rpc.auth)
+}

dev/config/nginx.conf

@@ -1,17 +0,0 @@
-server {
-    listen 8443 ssl;
-    ssl_certificate /etc/tls/server.crt;
-    ssl_certificate_key /etc/tls/server.key;
-    ssl_protocols TLSv1.2 TLSv1.3;
-    root /usr/share/nginx/html;
-    index index.html;
-    location /api/ {
-        resolver 127.0.0.11 valid=10s ipv6=off;
-        set $bff http://bff:8082;
-        proxy_pass $bff;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto https;
-        proxy_set_header X-Forwarded-For $remote_addr;
-    }
-    location / { try_files $uri $uri/ /index.html; }
-}

dev/dispatch.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -eu
-set -o pipefail
-. dev/common.sh
-case "${1:-}" in
- images) . dev/images.sh; images;;
- up) . dev/bootstrap.sh; up;;
- email) . dev/bootstrap.sh; email;;
- verify|verify-infra) . dev/verify.sh; existing; infrastructure; if [ "$1" = verify ]; then product; fi;;
- rotation) . dev/verify.sh; . dev/rotation.sh; existing; rotation;;
- test) sh dev/test.sh;;
- *) fail 'Unknown local tooling command';;
-esac

dev/vault/notifications.hcl

@@ -1,55 +0,0 @@
-vault { address = "http://vault:8200" }
-auto_auth {
-  method "approle" {
-    config = {
-      role_id_file_path = "/etc/agent/role-id"
-      secret_id_file_path = "/etc/agent/secret-id"
-      remove_secret_id_file_after_reading = false
-    }
-  }
-}
-template_config {
-  static_secret_render_interval = "10s"
-  exit_on_retry_failure = true
-}
-
-env_template "TLS_CA" {
-  contents = "{{ with secret \"kv/data/tls\" }}{{ .Data.data.ca }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN" {
-  contents = "{{ with secret \"kv/data/caller\" }}{{ .Data.data.token }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "SXCHNGE_NATS_URL" {
-  contents = "{{ with secret \"kv/data/nats\" }}nats://{{ .Data.data.token }}@nats:4222{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "TLS_CERT" {
-  contents = "{{ with secret \"kv/data/notifications-tls\" }}{{ .Data.data.certificate }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "TLS_KEY" {
-  contents = "{{ with secret \"kv/data/notifications-tls\" }}{{ .Data.data.private_key }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "EMAIL_API_KEY" {
-  contents = "{{ with secret \"kv/data/email\" }}{{ .Data.data.api_key }}{{ end }}"
-  error_on_missing_key = true
-}
-
-env_template "EMAIL_CONFIG" {
-  contents = "{{ with secret \"kv/data/email\" }}email:\n  driver: {{ .Data.data.driver | toJSON }}\n  settings:\n    api_key_env: EMAIL_API_KEY\n    sender_address: {{ .Data.data.sender_address | toJSON }}\n    sender_name: {{ .Data.data.sender_name | toJSON }}\n{{ end }}"
-  error_on_missing_key = true
-}
-
-exec {
-  command = ["/usr/local/bin/start-service"]
-  restart_on_secret_changes = "always"
-  restart_stop_signal = "SIGTERM"
-}

docs/implementation.md

@@ -705,3 +705,41 @@ were restored and temporary BFF/frontend containers removed. `make up` exits at
 the intentional missing-Resend check, with infrastructure healthy. Backend and
 frontend code did not change in this revision; their earlier test/lint results
 and supplementary legacy test limitations above still apply. No email was sent.
+
+### Organized infrastructure and Vault ownership
+
+This revision supersedes the earlier local Nginx/Vault-certificate arrangement.
+Local Compose, runtime configuration, startup scripts and verification now live
+under `infra/local/`, grouped by service and responsibility. Dockerfiles live in
+`infra/images/`; the root Make targets remain the entry point. Product source
+stays in backend/frontend. Generated state remains in ignored `.local/dev/`.
+
+Vault KV now contains `services/bff/jwt`,
+`services/notifications/callers/bff`, `services/notifications/email/resend`,
+`infrastructure/nats/auth` and `bootstrap/mongo`. Dynamic database credentials
+remain in the database engine. Migration copies with CAS protection, compares
+existing values, archives readable old versions and metadata privately, and then
+removes old flat/TLS KV entries. Conflicting destinations stop migration without
+removing sources. Policies and Agent templates reference only the new paths.
+The unused existing PKI mount is retained to avoid destroying its signing key;
+new bootstrap neither creates nor uses it.
+
+Caddy replaces Nginx, serving Flutter and proxying `/api` to BFF. Caddy owns browser
+certificates and its CA in persistent volumes; only its public CA is exported to
+`.local/dev/ca.pem`. Existing browser trust must be updated to that new CA. Local
+BFF-to-notifications uses plaintext authenticated gRPC. Optional explicitly
+configured TLS remains available in the reusable client/server; incomplete TLS
+configuration is rejected. No local service consumes TLS secrets from Vault.
+
+Validation passed: `make build`, `make test`, `make lint-backend`,
+`make test-dev-tooling`, `make test-dev-infra`, and Compose/diff checks. Tests cover
+plaintext caller-token transport, optional verified TLS, invalid certificate
+configuration and secret migration conflicts. The existing Vault was migrated
+and canonical paths/absence of old entries verified. An isolated fresh project
+passed transaction/lease checks and repeated bootstrap; disposable volumes and
+state were removed. The final rotation check also verified Caddy static serving,
+SPA fallback and API health, then three pool replacements with unchanged BFF PID
+and 1,083 expected HTTP responses, zero failures. Role TTLs were restored and
+temporary BFF/frontend containers removed. `make up` intentionally stops at the
+missing Resend credential check; no real email, optional security scan or merge
+was performed. Earlier supplementary legacy test limitations remain recorded.

docs/local-development.md

@@ -3,219 +3,163 @@
 Task [#42](https://git.sendico.io/tech/sxchnge/issues/42),
 [PR #43](https://git.sendico.io/tech/sxchnge/pulls/43), branch `dev-env-42`.
 
-## Build and bootstrap
+## Build and start
 
 Requires Make, a POSIX shell and a local Docker engine with Compose v2 and
-BuildKit. Docker builds Go, protobuf bindings and Flutter web; host Go, protoc
+BuildKit. No host Python, Go, Flutter, jq or curl is required. Make runs bootstrap
-and Flutter installations are not required for `make build && make up`. The
+tooling in Docker with the local Docker socket and workspace mounted. Remote
-tooling container includes Docker CLI, curl, jq, Git and OpenSSL. No host Python,
+Docker engines are unsupported.
-jq, curl or Git installation is needed for these Make targets. It mounts
-the local Docker socket and workspace to orchestrate Compose; a remote Docker
-daemon is not supported. The first Flutter SDK download can
-take several minutes. Both ARM64 and AMD64 build from the same source definitions.
 
 ```sh
 make build && make up
 ```
 
-`make build` resolves the current stable upstream releases, pulls infrastructure
+The build resolves the latest stable releases and pins their digests in ignored
-and builds NATS packaging, notifications, BFF and frontend images. Vault and Mongo
+`.local/dev/images.env`. Prereleases are excluded, resolution failures stop the
-use upstream images directly. Resolution excludes prereleases and selects Nginx's
+build, and previous pins are retained on failure. Flutter is pinned to the stable
-stable channel. The exact versions and manifest digests are saved in the ignored
+release commit. Tooling uses a freshly pulled stable Docker CLI digest. Generated
-`.local/dev/images.env`; Flutter's stable release is pinned by Git commit. Build
+protobuf bindings and Flutter output are built locally/in Docker and never tracked.
-fails if upstream metadata cannot be resolved. Run `make build` again to refresh
-versions. Review database release upgrade requirements before updating a populated
-local volume across major versions; the tooling does not perform data migrations.
 
-`make up` starts persistent Vault, initializes/unseals it, provisions secrets and
+Compose is at `infra/local/compose.yml`; Make supplies the repository root as its
-AppRoles, initializes an authenticated single-member MongoDB replica set, configures
+project directory. Dockerfiles are in `infra/images/`. Runtime and startup files
-Vault's MongoDB database plugin, and starts authenticated NATS JetStream. It then
+are grouped by service under `infra/local/`. The stack consists of Vault, Mongo,
-starts notifications, BFF and the frontend when the required email settings exist.
+NATS, notifications, BFF and Caddy serving the frontend.
 
-**On the first run, `make up` intentionally exits with an error after infrastructure
+On first startup, bootstrap initializes/unseals persistent Vault, provisions
-bootstrap because Resend credentials have not been supplied.** No placeholder API
+AppRoles and secrets, initializes Mongo replica set `rs0`, configures Vault's Mongo
-key, mock delivery mode or embedded credential is installed. The three infrastructure
+plugin, and starts authenticated NATS JetStream. Product startup fails explicitly
-services remain running so you can configure Vault, then rerun `make up`.
+until Resend credentials exist. Infrastructure remains available for provisioning.
 
-## Configure Resend in Vault
+## Vault layout
 
-The default email driver is `resend`. In the Vault UI at
+Paths below are inside the KV v2 mount `kv`; the API inserts `data/` after `kv/`.
-[http://127.0.0.1:18200](http://127.0.0.1:18200), edit the KV v2 secret **`kv/email`**
+
-(API path `kv/data/email`):
+```text
+kv/
+├── services/
+│   ├── bff/jwt                         # jwt
+│   └── notifications/
+│       ├── callers/bff                 # token; read by BFF and notifications
+│       └── email/resend                # driver, api_key, sender_address, sender_name
+├── infrastructure/
+│   └── nats/auth                       # token; read by NATS and notifications
+└── bootstrap/
+    └── mongo                           # local administrator and replica-set bootstrap material
+```
+
+Every secret has an owning service or infrastructure component. Service policies
+permit only the exact paths they consume. Applications cannot read bootstrap
+credentials. Mongo application users are dynamic secrets under
+`database/creds/bff` and `database/creds/notifications`, not KV entries. They have
+`readWrite` only on their respective `sxchnge` / `sxchnge_notifications` databases.
+Vault uses `mongodb-database-plugin` and its own `vault-manager` account, whose
+password is rotated immediately after initial configuration.
+
+There are no frontend certificates, notification certificates or generic `tls`
+entries in KV. Browser certificates belong to Caddy; internal gRPC is plaintext
+on the private Docker network and retains caller-token authentication.
+
+## Resend
+
+Open [Vault](http://127.0.0.1:18200) and edit
+`kv/services/notifications/email/resend`:
 
 | Field | Value |
 | --- | --- |
 | `driver` | `resend` |
 | `api_key` | Your Resend API key |
-| `sender_address` | Your verified Resend sender email |
+| `sender_address` | Your verified sender address |
-| `sender_name` | `SXCHNGE` or your desired display name |
+| `sender_name` | Your display name |
 
-Local administrative login material is in the private, ignored
+Local administrator login material is in private `.local/dev/admin/init.json`.
-`.local/dev/admin/init.json` (`root_token`). Keep that file private. Product
+Product containers cannot access it. Alternatively use the hidden terminal prompt:
-containers never mount it. You can also provision the same secret with a hidden
-terminal prompt, without placing the API key in shell history:
 
 ```sh
 make email
 make up
 ```
 
-The startup check rejects an empty key or the bootstrap sender placeholder. It
+No placeholder key or delivery fallback is installed. Startup validates presence;
-does not validate the key against Resend or prove domain verification; an invalid
+it does not send an email or validate domain ownership with Resend. Updating email
-key fails at delivery. Vault Agent fetches the key and email settings and injects
+settings causes Agent to restart notifications. Existing values survive bootstrap.
-them into the notifications process. There is no `.env` secret fallback. Changing
-`kv/email` restarts the notification process automatically within about 10 seconds.
-Bootstrapping again preserves your existing email secret.
 
-## Open the product
+## Browser HTTPS and internal connections
 
-After providing Resend credentials:
+Caddy serves Flutter files and proxies `/api/*` to BFF over HTTP. Open
+[https://localhost:5126](https://localhost:5126). HTTPS preserves Secure/HttpOnly
+cookies and same-origin API access. Caddy uses `tls internal` and manages its local
+CA, certificates and renewal in the `caddy-data` volume. The `caddy-config` volume
+stores Caddy runtime configuration. Neither volume nor Caddy has Vault access.
 
-- Frontend and proxied API: [https://localhost:5126](https://localhost:5126).
+After frontend starts, Make exports only the public Caddy CA to `.local/dev/ca.pem`.
-- BFF health: [https://localhost:5126/api/v1/health](https://localhost:5126/api/v1/health).
+Trust it in your browser/OS to avoid certificate errors. This is a new CA when
-- Vault UI/API: [http://127.0.0.1:18200](http://127.0.0.1:18200).
+migrating from Vault-issued browser certificates; the old CA will not trust it.
-
+Trust installation is not automated. A CLI check can use:
-Trust `.local/dev/ca.pem` in your local browser/OS certificate store to use HTTPS
-without a certificate error. Trust installation is a separate local action, not
-performed by the bootstrap. For a command-line check without changing trust:
 
 ```sh
 curl --cacert .local/dev/ca.pem https://localhost:5126/api/v1/health
 ```
 
-HTTPS keeps the existing Secure/HttpOnly refresh cookie contract. Nginx serves
+BFF calls notifications over plaintext gRPC with its caller token. Mongo and NATS
-Flutter and proxies `/api/` to the BFF on the same origin. The bundle selects the
+also stay within the private Compose network; no internal service publishes a host
-existing backend auth adapter with the public client ID `sxchnge-local-web`.
+port. Only frontend and Vault bind host loopback ports. TLS remains optional in
-Bootstrap inserts this public browser client if absent; it has no client secret.
+the reusable notification client/server for deployments that explicitly configure
-No frontend widget, service or provider architecture changes are needed.
+it; local runtime configuration contains no certificate settings.
 
-The rest of the app retains its existing capabilities and limitations: wallet,
+Caddy behavior follows its [internal TLS](https://caddyserver.com/docs/caddyfile/directives/tls)
-KYC and settings fixtures are not turned into production integrations by this
+and [automatic HTTPS](https://caddyserver.com/docs/automatic-https) documentation.
-container setup. Full email signup/login requires a working Resend account.
 
-## Secret and connection lifecycle
+## Mongo transactions and credential rotation
 
-| Consumer | Vault access |
+Mongo is an authenticated single-node replica set (`rs0`), supporting multi-document
-| --- | --- |
+transactions. It has no failover or redundant data copy. A Mongo restart interrupts
-| BFF | `database/creds/bff`, JWT key, notification caller token and CA |
+access even though application credential replacement does not.
-| Notifications | `database/creds/notifications`, NATS token, caller token, TLS certificate/key and email settings |
-| NATS | NATS authentication token only |
-| Bootstrap | Infrastructure administration, PKI and initial identities |
 
-The database engine is configured at `database/config/mongo` with
+The shared database wrapper authenticates with each service's AppRole and obtains
-`mongodb-database-plugin`. It uses its own `vault-manager` Mongo user, whose
+an independent token/lease for each pool. At half the shorter token/lease lifetime,
-password is rotated by Vault immediately after configuration. The manager has
+it verifies a replacement pool before publishing it. Active operations, cursors
-`userAdminAnyDatabase`; applications never receive that credential. The independent
+and full transactions retain the old pool until they finish, then it closes and
-local Mongo administrator and replica-set key come from Vault's random generator
+its token is revoked. Nested stores and commit retries keep the same pool.
-and are retained in `kv/bootstrap/mongo` for local provisioning.
 
-The BFF role has `readWrite` only on `sxchnge`; notifications has `readWrite` only
+Failed refreshes retain valid credentials and retry with bounded backoff. After
-on `sxchnge_notifications`. Mongo URIs come from dynamic Vault credentials, not
+expiry, database operations fail closed until automatic recovery. Administrative
-static application users. Leases default to one hour with a 24-hour maximum.
+revocation, infrastructure outages and operations longer than their lease can
-The shared database wrapper obtains credentials directly from Vault using the
+still interrupt work. Repository consumers use `source.Source` operation scopes;
-service's AppRole. At half the effective lifetime (the shorter of the database
+`source.Fixed` adapts externally owned static/test connections.
-lease and owning token TTL), it authenticates independently and prepares a new
-Mongo pool. Only a successful primary ping permits publication. New operations
-use that pool; active operations, cursors and complete transactions, including
-nested stores and commit retries, retain the old pool until they finish. The old
-pool then closes and its dedicated token is revoked. Mongo credential replacement
-never restarts BFF or notifications. Each generation has a separate token so
-cleanup cannot revoke the replacement or Vault Agent's identity.
 
-Refresh failures keep the current valid pool and retry with bounded backoff.
+## Migration and persistence
-Operations are bounded by credential expiry. If Vault remains unavailable beyond
-that deadline, new database operations fail closed; refresh continues and access
-recovers automatically without restarting the service. Administrative revocation,
-operations exceeding their lease and infrastructure failures cannot be made
-uninterrupted by pool rotation. Shutdown rejects new scopes and drains existing
-work. Failed cleanup relies on Vault expiration as a backstop.
 
-Repositories and transaction factories share `source.Source` rather than retaining
+Bootstrap migrates the old flat `bff`, `caller`, `email` and `nats` paths into the
-concrete Mongo collections. Low-level consumers must acquire a scope, use its
+hierarchy above. It copies with CAS protection, compares values, and refuses to
-returned context and release it after all work completes; never retain its client
+overwrite conflicting destinations. After consumers use the new paths, it removes
-or context beyond release. `source.Fixed` adapts externally owned test/static
+old KV metadata and obsolete `tls`, `frontend-tls`, `notifications-tls` entries.
-connections. Database credentials are no longer Agent environment templates;
+Readable old version history and metadata are archived privately in
-Agent continues to fetch non-database secrets and may restart children when those
+`.local/dev/migrations/organized-v1/` before removal. That archive is administrative
-secrets change.
+recovery material; applications do not mount it. Old deleted/destroyed versions
+are not recreated. The old PKI engine is not used or provisioned by this stack;
+an existing PKI mount is retained to avoid destroying its non-exportable signing
+key and history as part of a KV reorganization.
 
-Each agent authenticates using its own AppRole with a renewable periodic token.
+Vault, Mongo, JetStream and Caddy data use project-scoped named volumes. Private
-Local SecretIDs are reusable for container restarts; each container mounts only
+bootstrap material lives in `.local/dev/`. `make down` preserves both. Vault's local
-its own identity directory under `.local/dev/agents/`. Revoking an AppRole SecretID
+unseal file allows automatic unsealing on restart. This local convenience and
-requires removing its matching local `secret-id` file and rerunning `make up` to
+loopback HTTP Vault endpoint are for development. Keep bootstrap files paired
-issue a replacement. Revoking an active token/lease terminates its credentials;
+with their matching volumes; do not discard one side independently.
-The database wrapper retries with a fresh identity; Agent independently retries
-its non-database secrets when Vault is reachable.
 
-The notification gRPC connection uses a Vault-issued certificate for
+## Verification
-`notifications` and the BFF validates the Vault CA and hostname. Frontend HTTPS
-certificates also come from Vault PKI. Leaf certificates last 30 days; `make up`
-refreshes them when less than seven days remain and reloads Nginx. Run it regularly
-for a development stack kept online for weeks. Agents restart when their TLS KV
-secret changes. Application secrets are never put into the frontend bundle.
-
-Vault, Mongo and JetStream data use project-scoped named Docker volumes. Local
-Vault unseal material, administrator login, per-service bootstrap identities and
-frontend TLS material are under ignored `.local/dev/`, outside image build contexts.
-The local Vault automatically unseals using its local key on container restart.
-These convenience credentials and loopback HTTP Vault access are for this local
-machine; this is not a production Vault deployment. Do not share `.local/dev/`.
-Keep it paired with its matching Docker volumes for recovery.
-
-Mongo and NATS expose no host ports. The `sxchnge-local` network, volumes and
-container names are independent of existing Sendico stacks. Only the frontend
-and Vault bind host loopback ports. No Sendico database, message bus or secrets
-are consumed.
-
-## Manage and verify
 
 ```sh
 make ps
 make logs
-make down             # preserves named volumes and local bootstrap material
+make down
-make up               # idempotent provisioning and readiness checks
+make test-dev-tooling   # shell analysis, failure paths, migration conflict checks
-make test-dev-tooling # offline shell and failure-path checks
+make test-dev-infra     # policy isolation, real Mongo transactions and lease lifecycle
-make test-dev-infra   # works before Resend provisioning
+make test-dev-rotation  # temporarily deploy BFF/frontend and exercise short leases
-make test-dev-rotation # real Vault rotations under BFF HTTP traffic
+make test-dev           # requires all six services; verified frontend/API HTTPS
-make test-dev         # requires all six services running
 ```
 
-The infrastructure check exercises AppRole access restrictions, real Vault
+Rotation validation temporarily shortens the BFF role TTL, recreates BFF/frontend,
-MongoDB credential issuance, per-database access, lease renewal and user deletion
+and requires three replacements with unchanged BFF PID and zero unexpected
-on revocation. The complete check also verifies service health and frontend/API
+transaction-backed HTTP responses. It restores TTLs and removes services it
-HTTPS with the generated CA. These checks never send a real email.
+started. No verification command sends email. Backend validation remains
-`test-dev-rotation` temporarily sets a 12-second BFF lease, deploys the built BFF
+`make test` and `make lint-backend`, using the host SDKs documented in module guides.
-and frontend, and requires three pool replacements with unchanged process ID and
-zero unexpected HTTP responses. Its invalid-token requests exercise Mongo-backed
-transactions without sending email. The check restores the role TTL and removes
-services it started; it recreates BFF/frontend at the beginning, so run it as a
-development validation command, not against active work you need to preserve.
-Race-enabled backend tests additionally hold a transaction and cursor across a
-replacement and exercise failed candidate validation, outage recovery, rollback,
-expiry rejection and graceful shutdown.
-
-Backend regression tests use fresh `mongo:latest` and `nats:latest` official
-stable images, pulled before `make test-backend`. Normal validation remains
-`make test` and `make lint-backend` and requires the host SDKs described in the
-repository guides.
-
-Deleting Docker volumes removes local data; `make down` deliberately keeps them.
-If starting from scratch, stop the stack and explicitly remove this project's
-three volumes **and** its matching `.local/dev/` bootstrap state together. Never
-reuse old unseal material with a newly initialized Vault volume, or retain Mongo
-users while discarding the Vault data that manages their leases.
-
-## Sources and checked releases
-
-On 2026-09-12 the resolver selected Vault 2.1.0, MongoDB 8.3.9, NATS 2.14.6,
-Go 1.27.1, Nginx stable 1.30.4, Ubuntu 26.04 and Flutter stable 3.47.4. These are
-verification records, not fixed future defaults. Release selection follows the
-[Vault releases](https://releases.hashicorp.com/vault/),
-[MongoDB stable release notes](https://www.mongodb.com/docs/manual/release-notes/),
-[NATS releases](https://github.com/nats-io/nats-server/releases/latest),
-[Nginx downloads](https://nginx.org/en/download.html),
-[Go releases](https://go.dev/dl/) and
-[Flutter stable channel](https://github.com/flutter/flutter/tree/stable).
-The implementation follows HashiCorp's
-[MongoDB database plugin](https://developer.hashicorp.com/vault/docs/secrets/databases/mongodb)
-and [Agent process supervisor](https://developer.hashicorp.com/vault/docs/agent-and-proxy/agent/process-supervisor)
-contracts.

infra/README.md

@@ -0,0 +1,20 @@
+# Infrastructure
+
+Run `make build && make up` from the repository root. Local tooling needs Make,
+POSIX shell and Docker Compose/BuildKit; all other tools run in Docker.
+
+- `images/`: reusable Docker build definitions for product images and tooling.
+- `local/compose.yml`: isolated local stack and persistent volumes.
+- `local/bff/`, `local/notifications/`: application runtime configuration.
+- `local/frontend/`: Caddy configuration for Flutter files, browser HTTPS and API proxying.
+- `local/vault/`: server startup/configuration and per-service Agent templates.
+- `local/mongo/`, `local/nats/`: infrastructure startup and Mongo provisioning checks.
+- `local/scripts/`: bootstrap, migration, release resolution and verification.
+- `local/tools.sh`: host entry point used by Make.
+
+Local generated pins, identities and administrator material remain under ignored
+`.local/dev/`; they are never build inputs. Application source remains under
+`backend/` and `frontend/`.
+
+See [local development](../docs/local-development.md) for Vault paths, commands,
+Caddy trust, persistence and migration details.

infra/images/backend.Dockerfile

@@ -13,8 +13,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/g
 FROM ${VAULT_IMAGE}
 USER root
 COPY --from=build /out/service /usr/local/bin/service
-COPY dev/start-service.sh /usr/local/bin/start-service
+COPY infra/local/scripts/start-service.sh /usr/local/bin/start-service
-COPY dev/config /etc/sxchnge
+COPY infra/local/bff/runtime.yml /etc/sxchnge/bff.yml
+COPY infra/local/notifications/runtime.yml /etc/sxchnge/notifications.yml
 COPY backend/notifications/templates.example.json /etc/sxchnge/templates.json
 RUN mkdir -p /run/sxchnge && chown 100:1000 /run/sxchnge
 USER 100:1000

infra/images/frontend.Dockerfile

@@ -1,5 +1,5 @@
 ARG UBUNTU_IMAGE
-ARG NGINX_IMAGE
+ARG CADDY_IMAGE
 FROM ${UBUNTU_IMAGE} AS build
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl git unzip xz-utils libglu1-mesa && rm -rf /var/lib/apt/lists/*
 ARG FLUTTER_VERSION
@@ -12,6 +12,6 @@ COPY frontend ./frontend
 WORKDIR /src/frontend/app
 ARG FRONTEND_URL=https://localhost:5126
 RUN flutter pub get && flutter gen-l10n && flutter build web --no-wasm-dry-run --dart-define=SXCHNGE_API_URL=${FRONTEND_URL}/api/v1 --dart-define=SXCHNGE_CLIENT_ID=sxchnge-local-web
-FROM ${NGINX_IMAGE}
+FROM ${CADDY_IMAGE}
-COPY dev/config/nginx.conf /etc/nginx/conf.d/default.conf
+COPY infra/local/frontend/Caddyfile /etc/caddy/Caddyfile
-COPY --from=build /src/frontend/app/build/web /usr/share/nginx/html
+COPY --from=build /src/frontend/app/build/web /srv

infra/images/nats.Dockerfile

@@ -4,7 +4,7 @@ FROM ${NATS_IMAGE} AS nats
 FROM ${VAULT_IMAGE}
 USER root
 COPY --from=nats /nats-server /usr/local/bin/nats-server
-COPY dev/start-nats.sh /usr/local/bin/start-nats
+COPY infra/local/nats/start.sh /usr/local/bin/start-nats
 RUN mkdir -p /data /run/sxchnge && chown -R 100:1000 /data /run/sxchnge
 USER 100:1000
 ENTRYPOINT ["vault", "agent", "-config=/etc/agent/config.hcl"]

infra/local/bff/runtime.yml

@@ -39,6 +39,4 @@ api:
   notifications:
     address: notifications:8083
     token_env: SXCHNGE_NOTIFICATIONS_TOKEN
-    ca_file: /run/sxchnge/ca.pem
-    server_name: notifications
     activation_url: https://localhost:5126/verify-email

infra/local/compose.yml

@@ -9,7 +9,7 @@ x-agent: &agent
 
 x-backend-build: &backend-build
   context: .
-  dockerfile: dev/images/backend.Dockerfile
+  dockerfile: infra/images/backend.Dockerfile
 
 services:
   vault:
@@ -20,8 +20,8 @@ services:
     ports: ['127.0.0.1:18200:8200']
     volumes:
       - vault-data:/vault/file
-      - ./dev/vault/server.hcl:/etc/vault/server.hcl:ro
+      - ./infra/local/vault/server.hcl:/etc/vault/server.hcl:ro
-      - ./dev/start-vault.sh:/etc/vault/start.sh:ro
+      - ./infra/local/vault/start.sh:/etc/vault/start.sh:ro
       - ./.local/dev/vault:/bootstrap:ro
     healthcheck:
       test: [CMD-SHELL, 'VAULT_ADDR=http://127.0.0.1:8200 vault status >/dev/null']
@@ -38,7 +38,7 @@ services:
       MONGO_INITDB_ROOT_USERNAME: local-admin
       MONGO_INITDB_ROOT_PASSWORD_FILE: /run/mongo/password
     volumes:
-      - ./dev/start-mongo.sh:/etc/mongo/start.sh:ro
+      - ./infra/local/mongo/start.sh:/etc/mongo/start.sh:ro
       - mongo-data:/data/db
       - ./.local/dev/mongo:/run/mongo:ro
     healthcheck:
@@ -52,7 +52,7 @@ services:
     image: sxchnge-local/nats:dev
     build:
       context: .
-      dockerfile: dev/images/nats.Dockerfile
+      dockerfile: infra/images/nats.Dockerfile
       args:
         NATS_IMAGE: ${NATS_IMAGE:?Run make build first}
         VAULT_IMAGE: ${VAULT_IMAGE:?Run make build first}
@@ -121,20 +121,21 @@ services:
     image: sxchnge-local/frontend:dev
     build:
       context: .
-      dockerfile: dev/images/frontend.Dockerfile
+      dockerfile: infra/images/frontend.Dockerfile
       args:
         UBUNTU_IMAGE: ${UBUNTU_IMAGE:?Run make build first}
-        NGINX_IMAGE: ${NGINX_IMAGE:?Run make build first}
+        CADDY_IMAGE: ${CADDY_IMAGE:?Run make build first}
         FLUTTER_VERSION: ${FLUTTER_VERSION:?Run make build first}
         FLUTTER_COMMIT: ${FLUTTER_COMMIT:?Run make build first}
     restart: unless-stopped
     ports: ['127.0.0.1:5126:8443']
     volumes:
-      - ./.local/dev/frontend:/etc/tls:ro
+      - caddy-data:/data
+      - caddy-config:/config
     depends_on:
       bff: {condition: service_healthy}
     healthcheck:
-      test: [CMD, wget, -q, --no-check-certificate, -O, /dev/null, 'https://127.0.0.1:8443/']
+      test: [CMD, wget, -q, --no-check-certificate, -O, /dev/null, 'https://localhost:8443/']
       interval: 3s
       timeout: 3s
       retries: 40
@@ -143,3 +144,5 @@ volumes:
   vault-data:
   mongo-data:
   nats-data:
+  caddy-data:
+  caddy-config:

infra/local/frontend/Caddyfile

@@ -0,0 +1,16 @@
+{
+	skip_install_trust
+	auto_https disable_redirects
+}
+
+https://localhost:8443 {
+	tls internal
+	root * /srv
+	handle /api/* {
+		reverse_proxy bff:8082
+	}
+	handle {
+		try_files {path} /index.html
+		file_server
+	}
+}

infra/local/notifications/runtime.yml

@@ -1,6 +1,4 @@
 listen: "0.0.0.0:8083"
-tls_cert: "/run/sxchnge/server.crt"
-tls_key: "/run/sxchnge/server.key"
 database:
   driver: mongodb
   settings:

infra/local/scripts/bootstrap.sh

@@ -16,7 +16,7 @@ initialize() {
 }
 mounts() {
  vault sys/mounts > "$WORK/mounts"
- for name in kv database pki; do
+ for name in kv database; do
   if ! jq -e --arg key "$name/" 'has($key)' < "$WORK/mounts" >/dev/null; then
    if ! jq -e --arg key "$name/" '.data|has($key)' < "$WORK/mounts" >/dev/null; then
     jq -n --arg name "$name" '{type:$name} + (if $name=="kv" then {options:{version:"2"}} else {} end)' | vault "sys/mounts/$name" POST >/dev/null
@@ -27,27 +27,6 @@ mounts() {
   printf '{"type":"approle"}' | vault sys/auth/approle POST >/dev/null
  fi
 }
-certificates() {
- printf '{"max_lease_ttl":"87600h"}' | vault sys/mounts/pki/tune POST >/dev/null
- ca=$(kv tls)
- if [ "$ca" = null ]; then
-  printf '{"common_name":"SXCHNGE local development CA","ttl":"87600h"}' | vault pki/root/generate/internal POST | jq '{ca:.data.certificate}' | put tls
- fi
- printf '{"allowed_domains":["localhost","notifications"],"allow_bare_domains":true,"allow_ip_sans":true,"max_ttl":"720h"}' | vault pki/roles/local POST >/dev/null
- for name in notifications-tls frontend-tls; do
-  host=notifications; [ "$name" != frontend-tls ] || host=localhost
-  kv "$name" > "$WORK/cert"
-  if ! jq -e '.!=null and .expiration>(now+604800)' "$WORK/cert" >/dev/null; then
-   jq -n --arg host "$host" '{common_name:$host,ip_sans:"127.0.0.1",ttl:"720h"}' | vault pki/issue/local POST | jq '.data' > "$WORK/cert"
-   put "$name" < "$WORK/cert"
-  fi
-  if [ "$name" = frontend-tls ]; then
-   jq -r '.certificate' "$WORK/cert" | write "$STATE/frontend/server.crt"
-   jq -r '.private_key' "$WORK/cert" | write "$STATE/frontend/server.key"
-  fi
- done
- kv tls | jq -r '.ca' | write "$STATE/ca.pem" 644
-}
 provision_mongo() {
  if [ "$(kv bootstrap/mongo)" = null ]; then
   random > "$WORK/password"; random 384 > "$WORK/keyfile"; random > "$WORK/manager"
@@ -72,16 +51,16 @@ provision_mongo() {
   database=sxchnge; [ "$role" != notifications ] || database=sxchnge_notifications
   jq -n --arg database "$database" '{db_name:"mongo",creation_statements:({db:"admin",roles:[{role:"readWrite",db:$database}]}|tojson),default_ttl:"1h",max_ttl:"24h"}' | vault "database/roles/$role" POST >/dev/null
  done
- { cat "$WORK/admin.js"; cat dev/mongo-client.js; } | mongo >/dev/null || fail 'Browser client provisioning failed'
+ { cat "$WORK/admin.js"; cat infra/local/mongo/mongo-client.js; } | mongo >/dev/null || fail 'Browser client provisioning failed'
 }
 provision_agents() {
  for service in bff notifications nats; do
   directory="$STATE/agents/$service"
   mkdir -p "$directory"; chmod 755 "$directory"
   case "$service" in
-   bff) paths='bff caller tls';;
+   bff) paths='services/bff/jwt services/notifications/callers/bff';;
-   notifications) paths='caller tls notifications-tls nats email';;
+   notifications) paths='services/notifications/callers/bff infrastructure/nats/auth services/notifications/email/resend';;
-   nats) paths=nats;;
+   nats) paths=infrastructure/nats/auth;;
   esac
   : > "$WORK/policy"
   for path in $paths; do printf 'path "kv/data/%s" { capabilities = ["read"] }\n' "$path" >> "$WORK/policy"; done
@@ -92,7 +71,12 @@ provision_agents() {
   if [ ! -f "$directory/secret-id" ]; then
    printf '{}' | vault "auth/approle/role/$service/secret-id" POST | jq -r '.data.secret_id' | write "$directory/secret-id" 444
   fi
-  cat "dev/vault/$service.hcl" | write "$directory/config.hcl" 444
+  if ! cmp -s "infra/local/vault/$service.hcl" "$directory/config.hcl"; then
+   cat "infra/local/vault/$service.hcl" | write "$directory/config.hcl" 444
+   if compose ps --status running --services "$service" | grep -qx "$service"; then
+    compose up -d --no-deps --force-recreate --wait --wait-timeout 120 "$service"
+   fi
+  fi
  done
 }
 up() {
@@ -107,20 +91,21 @@ up() {
  fi
  initialize
  mounts
- for path in nats caller bff; do
+ migrate_secrets
+ for path in infrastructure/nats/auth services/notifications/callers/bff services/bff/jwt; do
   if [ "$(kv "$path")" = null ]; then
-   field=token; [ "$path" != bff ] || field=jwt
+   field=token; [ "$path" != services/bff/jwt ] || field=jwt
    random | jq -Rs --arg field "$field" '{($field):rtrimstr("\n")}' | put "$path"
   fi
  done
- printf '{"driver":"resend","api_key":"","sender_address":"notifications@localhost.test","sender_name":"SXCHNGE local"}' | ensure email
+ printf '{"driver":"resend","api_key":"","sender_address":"notifications@localhost.test","sender_name":"SXCHNGE local"}' | ensure services/notifications/email/resend
- certificates
  provision_mongo
  provision_agents
  compose up -d --wait --wait-timeout 120 nats
- kv email | email_valid || fail 'Vault kv/email needs a Resend api_key and verified sender_address. Run make email, then make up.'
+ retire_secrets
+ kv services/notifications/email/resend | email_valid || fail 'Vault kv/services/notifications/email/resend needs a Resend api_key and verified sender_address. Run make email, then make up.'
  compose up -d --wait --wait-timeout 180
- compose exec -T frontend nginx -s reload
+ export_ca
  echo 'Local stack ready: https://localhost:5126; Vault: http://127.0.0.1:18200'
 }
 email() {
@@ -138,6 +123,6 @@ email() {
  case "$address" in *@*) ;; *) fail 'A sender email address is required';; esac
  printf '%s' "$key" > "$WORK/email-key"
  printf '%s' "$address" > "$WORK/email-address"
- jq -n --rawfile key "$WORK/email-key" --rawfile address "$WORK/email-address" '{driver:"resend",api_key:$key,sender_address:$address,sender_name:"SXCHNGE"}' | put email
+ jq -n --rawfile key "$WORK/email-key" --rawfile address "$WORK/email-address" '{driver:"resend",api_key:$key,sender_address:$address,sender_name:"SXCHNGE"}' | put services/notifications/email/resend
  echo 'Email configuration saved in Vault. Run make up.'
 }

infra/local/scripts/common.sh

@@ -14,7 +14,7 @@ trap cleanup EXIT
 trap 'exit 130' INT
 trap 'exit 143' TERM
 fail() { echo "$*" >&2; exit 1; }
-compose() { docker compose --env-file "$STATE/images.env" "$@"; }
+compose() { docker compose --project-directory . -f infra/local/compose.yml --env-file "$STATE/images.env" "$@"; }
 write() {
  target=$1; mode=${2:-600}
  mkdir -p "$(dirname "$target")"
@@ -58,3 +58,6 @@ ensure() (
 mongo() { compose exec -T mongo mongosh --quiet --file /dev/stdin > "$WORK/mongo-result" 2> "$WORK/mongo-error" || return 1; cat "$WORK/mongo-result"; }
 admin_auth() { kv bootstrap/mongo | jq -r '"const admin=db.getSiblingDB(\"admin\"); admin.auth(\"local-admin\","+(.password|tojson)+");"'; }
 email_valid() { jq -e '(.api_key|type=="string") and (.api_key|test("\\S")) and (.sender_address|type=="string") and (.sender_address|test("\\S")) and .sender_address!="notifications@localhost.test"' >/dev/null; }
+export_ca() {
+ compose exec -T frontend cat /data/caddy/pki/authorities/local/root.crt | write "$STATE/ca.pem" 644
+}

infra/local/scripts/dispatch.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+set -o pipefail
+. infra/local/scripts/common.sh
+. infra/local/scripts/secrets.sh
+case "${1:-}" in
+ images) . infra/local/scripts/images.sh; images;;
+ up) . infra/local/scripts/bootstrap.sh; up;;
+ email) . infra/local/scripts/bootstrap.sh; email;;
+ verify|verify-infra) . infra/local/scripts/verify.sh; existing; infrastructure; if [ "$1" = verify ]; then product; fi;;
+ rotation) . infra/local/scripts/verify.sh; . infra/local/scripts/rotation.sh; existing; rotation;;
+ test) sh infra/local/scripts/test.sh;;
+ *) fail 'Unknown local tooling command';;
+esac

infra/local/scripts/images.sh

@@ -10,7 +10,7 @@ image() (
 )
 images() (
  go=$(get 'https://go.dev/dl/?mode=json' | jq -er '.[0] | select(.stable) | .version | select(test("^go[0-9]+\\.[0-9]+\\.[0-9]+$"))') || exit 1
- for spec in VAULT_IMAGE:hashicorp/vault:latest MONGO_IMAGE:library/mongo:latest NATS_IMAGE:library/nats:latest NGINX_IMAGE:library/nginx:stable-alpine "GO_IMAGE:library/golang:${go#go}-alpine" UBUNTU_IMAGE:library/ubuntu:latest; do
+ for spec in VAULT_IMAGE:hashicorp/vault:latest MONGO_IMAGE:library/mongo:latest NATS_IMAGE:library/nats:latest CADDY_IMAGE:library/caddy:alpine "GO_IMAGE:library/golang:${go#go}-alpine" UBUNTU_IMAGE:library/ubuntu:latest; do
   key=${spec%%:*}; rest=${spec#*:}; repo=${rest%:*}; alias=${rest##*:}
   ref=$(image "$repo" "$alias") || exit 1
   printf '%s=%s\n' "$key" "$ref" >> "$WORK/images.env"

infra/local/scripts/rotation.sh

@@ -20,6 +20,12 @@ rotation() (
  trap 'exit 143' TERM
  jq '.+{default_ttl:"12s",max_ttl:"24s"}' "$WORK/original-role" | vault database/roles/bff POST >/dev/null
  compose up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend
+ export_ca
+ https -f https://localhost:5126/ > "$WORK/index"
+ grep -q flutter_bootstrap "$WORK/index" || fail 'Frontend bundle is not served'
+ https -f https://localhost:5126/settings > "$WORK/route"
+ cmp -s "$WORK/index" "$WORK/route" || fail 'Frontend route fallback failed'
+ https -f https://localhost:5126/api/v1/health >/dev/null
  process=$(compose exec -T bff pidof service)
  for worker in 1 2 3 4; do
   (

infra/local/scripts/secrets.sh

@@ -0,0 +1,56 @@
+# Canonical paths follow ownership. Shared callers live under the receiving service.
+secret_moves() {
+ cat <<'PATHS'
+bff services/bff/jwt
+caller services/notifications/callers/bff
+email services/notifications/email/resend
+nats infrastructure/nats/auth
+PATHS
+}
+# Archive readable version history privately before removing an obsolete KV path.
+archive_secret() (
+ old=$1
+ metadata=$(vault "kv/metadata/$old" GET yes)
+ [ "$metadata" != null ] || exit 0
+ directory="$STATE/migrations/organized-v1/$old"
+ mkdir -p "$directory"
+ printf '%s' "$metadata" | write "$directory/metadata.json"
+ printf '%s' "$metadata" | jq -r '.data.versions|to_entries[]|select(.value.destroyed==false and .value.deletion_time=="")|.key' > "$WORK/versions"
+ while IFS= read -r version; do
+  vault "kv/data/$old?version=$version" | write "$directory/version-$version.json"
+ done < "$WORK/versions"
+)
+migrate_secrets() (
+ secret_moves > "$WORK/moves"
+ while read -r old new; do
+  kv "$old" > "$WORK/old"
+  [ "$(cat "$WORK/old")" != null ] || continue
+  archive_secret "$old"
+  kv "$new" > "$WORK/new"
+  if [ "$(cat "$WORK/new")" = null ]; then
+   # CAS=0 prevents overwriting a destination created concurrently.
+   jq '{data:.,options:{cas:0}}' "$WORK/old" | vault "kv/data/$new" POST >/dev/null
+  fi
+  kv "$new" > "$WORK/new"
+  jq -e -s '.[0]==.[1]' "$WORK/old" "$WORK/new" >/dev/null || fail "Conflicting Vault values at $old and $new; no source removed"
+ done < "$WORK/moves"
+)
+retire_secrets() (
+ # Call only after templates, policies and deployed images use canonical paths.
+ secret_moves > "$WORK/moves"
+ while read -r old new; do
+  kv "$old" > "$WORK/old"
+  [ "$(cat "$WORK/old")" != null ] || continue
+  kv "$new" > "$WORK/new"
+  jq -e -s '.[0]==.[1]' "$WORK/old" "$WORK/new" >/dev/null || fail "Vault migration verification failed for $old"
+  archive_secret "$old"
+  printf '{}' | vault "kv/metadata/$old" DELETE >/dev/null
+ done < "$WORK/moves"
+ for old in tls frontend-tls notifications-tls; do
+  archive_secret "$old"
+  if [ "$(vault "kv/metadata/$old" GET yes)" != null ]; then printf '{}' | vault "kv/metadata/$old" DELETE >/dev/null; fi
+ done
+ # The obsolete PKI mount is deliberately not disabled: its private signing key
+ # and history are retained until the owner explicitly chooses to destroy them.
+ echo 'PASS: Vault secrets migrated and verified; obsolete KV entries retired'
+)

infra/local/scripts/start-service.sh

@@ -1,13 +1,9 @@
 #!/bin/sh
 set -eu
 umask 077
-printf '%s\n' "$TLS_CA" > /run/sxchnge/ca.pem
-unset TLS_CA
 if [ "$SXCHNGE_SERVICE" = notifications ]; then
-    printf '%s\n' "$TLS_CERT" > /run/sxchnge/server.crt
-    printf '%s\n' "$TLS_KEY" > /run/sxchnge/server.key
     printf '%s\n' "$EMAIL_CONFIG" > /run/sxchnge/email.yml
-    unset TLS_CERT TLS_KEY EMAIL_CONFIG
+    unset EMAIL_CONFIG
     cat /etc/sxchnge/notifications.yml /run/sxchnge/email.yml > /run/sxchnge/config.yml
     exec service -config.file=/run/sxchnge/config.yml
 fi

infra/local/scripts/test.sh

@@ -2,11 +2,11 @@
 set -eu
 set -o pipefail
 # Shell syntax and static analysis cover the actual container shell scripts.
-shellcheck -s ash -x -S warning dev/dispatch.sh dev/test.sh
+shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh
-shellcheck -x -S warning dev/tools.sh dev/start-*.sh
+shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh
-. dev/common.sh
+. infra/local/scripts/common.sh
-. dev/images.sh
+. infra/local/scripts/images.sh
-. dev/bootstrap.sh
+. infra/local/scripts/bootstrap.sh
 # Offline fixtures exercise stable selection, reject prereleases, and preserve pins.
 get() {
  case "$1" in
@@ -33,7 +33,29 @@ vault() { printf '{"initialized":true}'; }
 if (initialize) > "$WORK/initialize-error" 2>&1; then fail 'Missing local keys accepted'; fi
 grep -q 'Missing .local' "$WORK/initialize-error"
 for service in bff notifications; do
- if grep -Eq 'database/creds/|SXCHNGE_MONGO_URI' "dev/vault/$service.hcl"; then fail 'Database credentials trigger Agent restart'; fi
+ if grep -Eq 'database/creds/|SXCHNGE_MONGO_URI' "infra/local/vault/$service.hcl"; then fail 'Database credentials trigger Agent restart'; fi
 done
 echo 'PASS: shell analysis, stable selection, failed-resolution preservation, email validation, missing-key protection and Agent contract'
 if command -v python >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1; then fail 'Tool image unexpectedly contains Python'; fi
+# Migration never overwrites an independently provisioned destination.
+. infra/local/scripts/secrets.sh
+secret_moves() { echo 'email services/notifications/email/resend'; }
+archive_secret() { :; }
+kv() {
+ case "$1" in
+  email) printf '{"api_key":"existing-fixture"}';;
+  *) cat "$WORK/destination";;
+ esac
+}
+vault() {
+ [ "$1" = kv/data/services/notifications/email/resend ] || fail 'Unexpected migration mutation'
+ jq -e '.options.cas==0' >/dev/null
+ printf '{"api_key":"existing-fixture"}' > "$WORK/destination"
+}
+printf 'null' > "$WORK/destination"
+migrate_secrets
+migrate_secrets
+printf '{"api_key":"different-fixture"}' > "$WORK/destination"
+if (migrate_secrets) >/dev/null 2>&1; then fail 'Migration overwrote conflicting destination'; fi
+jq -e '.api_key=="different-fixture"' "$WORK/destination" >/dev/null
+echo 'PASS: secret migration is repeatable and refuses conflicting destination values'

infra/local/scripts/verify.sh

@@ -6,7 +6,7 @@ policies() (
   directory="$STATE/agents/$service"
   login=$(jq -n --rawfile role "$directory/role-id" --rawfile secret "$directory/secret-id" '{role_id:($role|rtrimstr("\n")),secret_id:($secret|rtrimstr("\n"))}' | vault auth/approle/login POST)
   identity=$(printf '%s' "$login" | jq -er '.auth.client_token')
-  printf '%s' "$login" | jq '{token:.auth.client_token,paths:["kv/data/bootstrap/mongo","sys/init"]}' | vault sys/capabilities POST | jq -e '.data | all(.[]; .==["deny"])' >/dev/null || fail 'Service can access bootstrap credentials'
+  printf '%s' "$login" | jq '{token:.auth.client_token,paths:["kv/data/bootstrap/mongo","sys/init","kv/data/bff","kv/data/caller","kv/data/email","kv/data/nats","kv/data/tls"]}' | vault sys/capabilities POST | jq -e '.data | all(.[]; .==["deny"])' >/dev/null || fail 'Service can access bootstrap credentials'
   VAULT_TOKEN=$(printf '%s' "$login" | jq -er '.auth.client_token')
   printf '{}' | vault auth/token/revoke-self POST >/dev/null
   VAULT_TOKEN=$root_token
@@ -23,7 +23,7 @@ database_access() (
   {
    jq -r '"db.getSiblingDB(\"admin\").auth("+(.data.username|tojson)+","+(.data.password|tojson)+");"' "$WORK/lease"
    printf 'const database="%s", forbidden="%s";\n' "$database" "$forbidden"
-   cat dev/mongo-check.js
+   cat infra/local/mongo/mongo-check.js
   } | mongo > "$WORK/check" || fail 'Mongo access/transaction check failed'
   grep -q scope-ok "$WORK/check" || fail 'Mongo check incomplete'
   jq '{lease_id:.lease_id,increment:"1h"}' "$WORK/lease" | vault sys/leases/renew POST | jq -e '.lease_duration>0' >/dev/null || fail 'Mongo renewal failed'
@@ -37,6 +37,7 @@ database_access() (
 infrastructure() {
  compose ps --format json | jq -se '[.[] | select(.Service=="vault" or .Service=="mongo" or .Service=="nats")] | length==3 and all(.[];.Health=="healthy")' >/dev/null || fail 'Infrastructure is not healthy'
  echo 'PASS: Vault, Mongo replica set and NATS JetStream are healthy'
+ layout
  policies
  database_access
 }
@@ -48,3 +49,12 @@ product() {
  https -f https://localhost:5126/api/v1/health >/dev/null
  echo 'PASS: six healthy services, frontend and proxied BFF use verified TLS; no email sent'
 }
+layout() {
+ for old in bff caller email nats tls frontend-tls notifications-tls; do
+  [ "$(vault "kv/metadata/$old" GET yes)" = null ] || fail "Obsolete KV entry remains: $old"
+ done
+ for path in services/bff/jwt services/notifications/callers/bff services/notifications/email/resend infrastructure/nats/auth bootstrap/mongo; do
+  vault "kv/metadata/$path" | jq -e '.data.current_version>0' >/dev/null || fail "Missing canonical KV entry: $path"
+ done
+ echo 'PASS: canonical Vault hierarchy present; obsolete flat and TLS entries absent'
+}

infra/local/tools.sh

@@ -1,13 +1,13 @@
 #!/bin/sh
 # Host entry point: only POSIX shell and Docker are required.
 set -eu
-cd "$(dirname "$0")/.."
+cd "$(dirname "$0")/../.."
 case "${1:-}" in
  build)
   docker pull docker:cli
   ref=$(docker image inspect docker:cli --format '{{index .RepoDigests 0}}')
   case "$ref" in docker@sha256:*) ;; *) echo 'Cannot pin stable Docker CLI image' >&2; exit 1;; esac
-  docker build --pull --build-arg "DOCKER_IMAGE=$ref" -f dev/images/tools.Dockerfile -t sxchnge-local/tools:dev dev/images
+  docker build --pull --build-arg "DOCKER_IMAGE=$ref" -f infra/images/tools.Dockerfile -t sxchnge-local/tools:dev infra/images
   exit
   ;;
 esac
@@ -22,4 +22,4 @@ if [ "${1:-}" = email ]; then terminal_args=-it; fi
 # shellcheck disable=SC2086
 exec docker run --rm $terminal_args $network_args -e "LOCAL_UID=$(id -u)" -e "LOCAL_GID=$(id -g)" \
  -v "$socket:/var/run/docker.sock" -v "$PWD:$PWD" -w "$PWD" \
- sxchnge-local/tools:dev sh dev/dispatch.sh "$@"
+ sxchnge-local/tools:dev sh infra/local/scripts/dispatch.sh "$@"

infra/local/vault/bff.hcl

@@ -13,18 +13,14 @@ template_config {
   exit_on_retry_failure = true
 }
 
-env_template "TLS_CA" {
-  contents = "{{ with secret \"kv/data/tls\" }}{{ .Data.data.ca }}{{ end }}"
-  error_on_missing_key = true
-}
 
 env_template "SXCHNGE_JWT_SECRET" {
-  contents = "{{ with secret \"kv/data/bff\" }}{{ .Data.data.jwt }}{{ end }}"
+  contents = "{{ with secret \"kv/data/services/bff/jwt\" }}{{ .Data.data.jwt }}{{ end }}"
   error_on_missing_key = true
 }
 
 env_template "SXCHNGE_NOTIFICATIONS_TOKEN" {
-  contents = "{{ with secret \"kv/data/caller\" }}{{ .Data.data.token }}{{ end }}"
+  contents = "{{ with secret \"kv/data/services/notifications/callers/bff\" }}{{ .Data.data.token }}{{ end }}"
   error_on_missing_key = true
 }
 

infra/local/vault/nats.hcl

@@ -14,7 +14,7 @@ template_config {
 }
 
 env_template "NATS_TOKEN" {
-  contents = "{{ with secret \"kv/data/nats\" }}{{ .Data.data.token }}{{ end }}"
+  contents = "{{ with secret \"kv/data/infrastructure/nats/auth\" }}{{ .Data.data.token }}{{ end }}"
   error_on_missing_key = true
 }
 

infra/local/vault/notifications.hcl

@@ -0,0 +1,43 @@
+vault { address = "http://vault:8200" }
+auto_auth {
+  method "approle" {
+    config = {
+      role_id_file_path = "/etc/agent/role-id"
+      secret_id_file_path = "/etc/agent/secret-id"
+      remove_secret_id_file_after_reading = false
+    }
+  }
+}
+template_config {
+  static_secret_render_interval = "10s"
+  exit_on_retry_failure = true
+}
+
+
+env_template "SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN" {
+  contents = "{{ with secret \"kv/data/services/notifications/callers/bff\" }}{{ .Data.data.token }}{{ end }}"
+  error_on_missing_key = true
+}
+
+env_template "SXCHNGE_NATS_URL" {
+  contents = "{{ with secret \"kv/data/infrastructure/nats/auth\" }}nats://{{ .Data.data.token }}@nats:4222{{ end }}"
+  error_on_missing_key = true
+}
+
+
+
+env_template "EMAIL_API_KEY" {
+  contents = "{{ with secret \"kv/data/services/notifications/email/resend\" }}{{ .Data.data.api_key }}{{ end }}"
+  error_on_missing_key = true
+}
+
+env_template "EMAIL_CONFIG" {
+  contents = "{{ with secret \"kv/data/services/notifications/email/resend\" }}email:\n  driver: {{ .Data.data.driver | toJSON }}\n  settings:\n    api_key_env: EMAIL_API_KEY\n    sender_address: {{ .Data.data.sender_address | toJSON }}\n    sender_name: {{ .Data.data.sender_name | toJSON }}\n{{ end }}"
+  error_on_missing_key = true
+}
+
+exec {
+  command = ["/usr/local/bin/start-service"]
+  restart_on_secret_changes = "always"
+  restart_stop_signal = "SIGTERM"
+}