diff --git a/AGENTS.md b/AGENTS.md index fd3976c..8fc574f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -169,3 +169,15 @@ an uncommitted working tree. Preserve unrelated changes and avoid mixing them into task commits. Record scope, validation, limitations, and PR links in the task and durable repository documentation. Merge only when authorized by the user. If work is delivered as stacked PRs, document dependencies and merge order. + +## Local infrastructure ownership + +Keep infrastructure Dockerfiles under `infra/images/` and local Compose, service +configuration and shell tooling under `infra/local/`. Keep product source under +`backend/` and `frontend/`. Organize Vault KV paths by owning service under +`services//`, infrastructure under `infrastructure//`, +and bootstrap administration under `bootstrap/`. Caller credentials +belong to the receiving service at `services//callers/`. +Use Caddy for local browser HTTPS and certificate management. Treat the private +local Docker network as trusted; do not add internal gRPC TLS or Vault-managed +local TLS certificates without an explicit requirement. diff --git a/Makefile b/Makefile index f0d88a2..756e50f 100644 --- a/Makefile +++ b/Makefile @@ -53,16 +53,16 @@ package-notifications-interface: generate-notifications tar -czf dist/notifications-go-v1.tar.gz -C backend/notifications/interface go.mod go.sum notifications README.md # Local product stack. Releases are resolved afresh, then pinned by digest. -COMPOSE = docker compose --env-file .local/dev/images.env +COMPOSE = docker compose --project-directory . -f infra/local/compose.yml --env-file .local/dev/images.env .PHONY: build up down logs ps test-dev build: - sh dev/tools.sh build - sh dev/tools.sh images + sh infra/local/tools.sh build + sh infra/local/tools.sh images $(COMPOSE) pull vault mongo $(COMPOSE) build --pull up: - sh dev/tools.sh up + sh infra/local/tools.sh up down: $(COMPOSE) down @@ -74,21 +74,21 @@ ps: $(COMPOSE) ps test-dev: - sh dev/tools.sh test - sh dev/tools.sh verify + sh infra/local/tools.sh test + sh infra/local/tools.sh verify .PHONY: test-dev-infra test-dev-infra: - sh dev/tools.sh test - sh dev/tools.sh verify-infra + sh infra/local/tools.sh test + sh infra/local/tools.sh verify-infra .PHONY: test-dev-rotation test-dev-rotation: - sh dev/tools.sh rotation + sh infra/local/tools.sh rotation .PHONY: email test-dev-tooling email: - sh dev/tools.sh email + sh infra/local/tools.sh email test-dev-tooling: - sh dev/tools.sh test + sh infra/local/tools.sh test diff --git a/README.md b/README.md index 1468cbc..054aa99 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ make build && make up This builds the product images and starts an isolated Vault, authenticated MongoDB replica set and NATS JetStream. First startup intentionally stops after bootstrap -until you set the Resend API key and verified sender in Vault's `kv/email` secret. +until you set the Resend API key and verified sender in Vault's `kv/services/notifications/email/resend` secret. Then rerun `make up` to start notifications, BFF and the frontend at [https://localhost:5126](https://localhost:5126). @@ -72,7 +72,7 @@ repository root: ```sh docker run --rm --pull always -p 127.0.0.1:5123:80 \ - -v "$PWD/frontend/app/build/web:/usr/share/nginx/html:ro" nginx:stable-alpine + -v "$PWD/frontend/app/build/web:/srv:ro" caddy:alpine caddy file-server --root /srv --listen :80 ``` Open [the local app](http://127.0.0.1:5123/). The language selector supports EN/RU diff --git a/backend/notifications/README.md b/backend/notifications/README.md index 3cffe8d..6cb0f45 100644 --- a/backend/notifications/README.md +++ b/backend/notifications/README.md @@ -24,10 +24,11 @@ Requires Go 1.27, protoc, an SXCHNGE MongoDB database and a NATS server with JetStream enabled. Copy `config.example.yml` to an untracked `config.yml` here. Set `SXCHNGE_MONGO_URI`, `SXCHNGE_NATS_URL`, a unique token of at least 32 bytes for each configured caller, and the selected email provider's API key environment -variable. Configure a verified sender address, TLS certificate/key and template +variable. Configure a verified sender address and template catalog. Paths are relative to the process working directory. -From the root, run `make run-notifications`. The gRPC listener always uses TLS. +From the root, run `make run-notifications`. The gRPC listener uses plaintext on the private local network. Configure both +`tls_cert` and `tls_key` only when TLS is explicitly required. Use a private authenticated NATS endpoint; `tls://` URLs and credentials in the URL are supported by the shared driver. Mongo and messaging both use shared `DriverConfig` settings; email settings reject unknown keys. diff --git a/backend/notifications/app/config.go b/backend/notifications/app/config.go index 8359077..e4f5dff 100644 --- a/backend/notifications/app/config.go +++ b/backend/notifications/app/config.go @@ -40,9 +40,12 @@ func Load(path string) (Config, error) { if err = decoder.Decode(new(any)); err != io.EOF { return c, merrors.InvalidArgument("unexpected configuration content") } - if c.Listen == "" || c.TLSCert == "" || c.TLSKey == "" || c.Database.Driver != db.Mongo || c.Messaging.Driver != messaging.BBNats || c.Templates == "" || len(c.Callers) == 0 { + if c.Listen == "" || c.Database.Driver != db.Mongo || c.Messaging.Driver != messaging.BBNats || c.Templates == "" || len(c.Callers) == 0 { return c, merrors.InvalidArgument("missing notification configuration") } + if (c.TLSCert == "") != (c.TLSKey == "") { + return c, merrors.InvalidArgument("both TLS certificate and key must be configured together") + } if err = c.Queue.Validate(); err != nil { return c, err } diff --git a/backend/notifications/app/config_test.go b/backend/notifications/app/config_test.go index 6a424be..5643ee7 100644 --- a/backend/notifications/app/config_test.go +++ b/backend/notifications/app/config_test.go @@ -33,3 +33,31 @@ func TestExampleConfigAndSecrets(t *testing.T) { t.Fatal("extra YAML document accepted") } } + +func TestPlaintextConfigAndIncompleteTLS(t *testing.T) { + t.Setenv("SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN", strings.Repeat("a", 32)) + c, err := Load("../config.example.yml") + if err != nil { + t.Fatal(err) + } + options, err := serverOptions(c) + if err != nil || len(options) != 0 { + t.Fatal("plain gRPC must not require TLS material") + } + data, err := os.ReadFile("../config.example.yml") + if err != nil { + t.Fatal(err) + } + for _, partial := range []string{"\ntls_cert: missing.crt\n", "\ntls_key: missing.key\n"} { + path := filepath.Join(t.TempDir(), "config.yml") + if err := os.WriteFile(path, append(data, []byte(partial)...), 0600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Fatal("incomplete TLS configuration accepted") + } + } + if _, err := serverOptions(Config{TLSCert: "missing.crt", TLSKey: "missing.key"}); err == nil { + t.Fatal("bad explicit certificate accepted") + } +} diff --git a/backend/notifications/app/run.go b/backend/notifications/app/run.go index da24ff1..08b8110 100644 --- a/backend/notifications/app/run.go +++ b/backend/notifications/app/run.go @@ -33,9 +33,9 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error { if err != nil { return err } - cert, err := tls.LoadX509KeyPair(c.TLSCert, c.TLSKey) + options, err := serverOptions(c) if err != nil { - return merrors.OpaqueWrap(merrors.ErrInvalidArg, err, "invalid gRPC TLS certificate") + return err } client, err := db.ConnectMongo(logger, &c.Database) if err != nil { @@ -67,7 +67,7 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error { for caller, env := range c.Callers { tokens[caller] = os.Getenv(env) } - server := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12})), grpc.UnaryInterceptor(transport.Authenticate(tokens)), grpc.MaxRecvMsgSize(128<<10), grpc.MaxConcurrentStreams(64)) + server := grpc.NewServer(append(options, grpc.UnaryInterceptor(transport.Authenticate(tokens)), grpc.MaxRecvMsgSize(128<<10), grpc.MaxConcurrentStreams(64))...) pb.RegisterNotificationsServiceServer(server, transport.New(svc)) listener, err := net.Listen("tcp", c.Listen) if err != nil { @@ -103,3 +103,15 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error { logger.Info("Notifications service stopped") return nil } + +// An omitted certificate pair uses plaintext gRPC on the private service network. +func serverOptions(c Config) ([]grpc.ServerOption, error) { + if c.TLSCert == "" && c.TLSKey == "" { + return nil, nil + } + cert, err := tls.LoadX509KeyPair(c.TLSCert, c.TLSKey) + if err != nil { + return nil, merrors.OpaqueWrap(merrors.ErrInvalidArg, err, "invalid gRPC TLS certificate") + } + return []grpc.ServerOption{grpc.Creds(credentials.NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}))}, nil +} diff --git a/backend/notifications/config.example.yml b/backend/notifications/config.example.yml index 373e531..02343a3 100644 --- a/backend/notifications/config.example.yml +++ b/backend/notifications/config.example.yml @@ -1,6 +1,4 @@ listen: "127.0.0.1:8083" -tls_cert: "server.crt" -tls_key: "server.key" database: driver: mongodb settings: diff --git a/backend/product/config.example.yml b/backend/product/config.example.yml index 3ca2ecb..2a49aef 100644 --- a/backend/product/config.example.yml +++ b/backend/product/config.example.yml @@ -32,8 +32,6 @@ api: check: min_length: 12 notifications: - address: localhost:8443 + address: localhost:8083 token_env: SXCHNGE_NOTIFICATIONS_TOKEN - ca_file: ../notifications/tls/ca.pem - server_name: localhost activation_url: http://localhost:5126/verify-email diff --git a/backend/product/notifications/client.go b/backend/product/notifications/client.go index a53159b..9174f79 100644 --- a/backend/product/notifications/client.go +++ b/backend/product/notifications/client.go @@ -12,6 +12,7 @@ import ( merrors "git.sendico.io/tech/sxchnge/shared/errors" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -57,7 +58,11 @@ func Open(c Config) (*Client, error) { } tlsConfig.RootCAs = pool } - conn, err := grpc.NewClient(c.Address, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + var transport credentials.TransportCredentials = insecure.NewCredentials() + if c.CAFile != "" || c.ServerName != "" { + transport = credentials.NewTLS(tlsConfig) + } + conn, err := grpc.NewClient(c.Address, grpc.WithTransportCredentials(transport)) if err != nil { return nil, merrors.InternalWrap(err, "create notification client") } diff --git a/backend/product/notifications/client_test.go b/backend/product/notifications/client_test.go index 7b45734..bc7f5fb 100644 --- a/backend/product/notifications/client_test.go +++ b/backend/product/notifications/client_test.go @@ -88,3 +88,19 @@ func TestOpenUsesVerifiedTLSAndCallerAuthentication(t *testing.T) { require.NoError(t, client.Send(context.Background(), Message{ExpiresAt: time.Now().Add(time.Minute)})) require.Equal(t, "Bearer test-service-token-with-at-least-32-bytes", <-rpc.auth) } + +func TestOpenUsesPlaintextAndCallerTokenOnPrivateNetwork(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := grpc.NewServer() + rpc := &tlsRPC{auth: make(chan string, 1)} + pb.RegisterNotificationsServiceServer(server, rpc) + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + t.Setenv("TEST_PLAIN_CALLER_TOKEN", "test-private-service-token-at-least-32-bytes") + client, err := Open(Config{Address: listener.Addr().String(), TokenEnv: "TEST_PLAIN_CALLER_TOKEN"}) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Send(context.Background(), Message{ExpiresAt: time.Now().Add(time.Minute)})) + require.Equal(t, "Bearer test-private-service-token-at-least-32-bytes", <-rpc.auth) +} diff --git a/dev/config/nginx.conf b/dev/config/nginx.conf deleted file mode 100644 index 463a631..0000000 --- a/dev/config/nginx.conf +++ /dev/null @@ -1,17 +0,0 @@ -server { - listen 8443 ssl; - ssl_certificate /etc/tls/server.crt; - ssl_certificate_key /etc/tls/server.key; - ssl_protocols TLSv1.2 TLSv1.3; - root /usr/share/nginx/html; - index index.html; - location /api/ { - resolver 127.0.0.11 valid=10s ipv6=off; - set $bff http://bff:8082; - proxy_pass $bff; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-For $remote_addr; - } - location / { try_files $uri $uri/ /index.html; } -} diff --git a/dev/dispatch.sh b/dev/dispatch.sh deleted file mode 100644 index eba2e24..0000000 --- a/dev/dispatch.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -eu -set -o pipefail -. dev/common.sh -case "${1:-}" in - images) . dev/images.sh; images;; - up) . dev/bootstrap.sh; up;; - email) . dev/bootstrap.sh; email;; - verify|verify-infra) . dev/verify.sh; existing; infrastructure; if [ "$1" = verify ]; then product; fi;; - rotation) . dev/verify.sh; . dev/rotation.sh; existing; rotation;; - test) sh dev/test.sh;; - *) fail 'Unknown local tooling command';; -esac diff --git a/dev/vault/notifications.hcl b/dev/vault/notifications.hcl deleted file mode 100644 index 63c99ee..0000000 --- a/dev/vault/notifications.hcl +++ /dev/null @@ -1,55 +0,0 @@ -vault { address = "http://vault:8200" } -auto_auth { - method "approle" { - config = { - role_id_file_path = "/etc/agent/role-id" - secret_id_file_path = "/etc/agent/secret-id" - remove_secret_id_file_after_reading = false - } - } -} -template_config { - static_secret_render_interval = "10s" - exit_on_retry_failure = true -} - -env_template "TLS_CA" { - contents = "{{ with secret \"kv/data/tls\" }}{{ .Data.data.ca }}{{ end }}" - error_on_missing_key = true -} - -env_template "SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN" { - contents = "{{ with secret \"kv/data/caller\" }}{{ .Data.data.token }}{{ end }}" - error_on_missing_key = true -} - -env_template "SXCHNGE_NATS_URL" { - contents = "{{ with secret \"kv/data/nats\" }}nats://{{ .Data.data.token }}@nats:4222{{ end }}" - error_on_missing_key = true -} - -env_template "TLS_CERT" { - contents = "{{ with secret \"kv/data/notifications-tls\" }}{{ .Data.data.certificate }}{{ end }}" - error_on_missing_key = true -} - -env_template "TLS_KEY" { - contents = "{{ with secret \"kv/data/notifications-tls\" }}{{ .Data.data.private_key }}{{ end }}" - error_on_missing_key = true -} - -env_template "EMAIL_API_KEY" { - contents = "{{ with secret \"kv/data/email\" }}{{ .Data.data.api_key }}{{ end }}" - error_on_missing_key = true -} - -env_template "EMAIL_CONFIG" { - contents = "{{ with secret \"kv/data/email\" }}email:\n driver: {{ .Data.data.driver | toJSON }}\n settings:\n api_key_env: EMAIL_API_KEY\n sender_address: {{ .Data.data.sender_address | toJSON }}\n sender_name: {{ .Data.data.sender_name | toJSON }}\n{{ end }}" - error_on_missing_key = true -} - -exec { - command = ["/usr/local/bin/start-service"] - restart_on_secret_changes = "always" - restart_stop_signal = "SIGTERM" -} diff --git a/docs/implementation.md b/docs/implementation.md index f2e3b6f..349e31d 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -705,3 +705,41 @@ were restored and temporary BFF/frontend containers removed. `make up` exits at the intentional missing-Resend check, with infrastructure healthy. Backend and frontend code did not change in this revision; their earlier test/lint results and supplementary legacy test limitations above still apply. No email was sent. + +### Organized infrastructure and Vault ownership + +This revision supersedes the earlier local Nginx/Vault-certificate arrangement. +Local Compose, runtime configuration, startup scripts and verification now live +under `infra/local/`, grouped by service and responsibility. Dockerfiles live in +`infra/images/`; the root Make targets remain the entry point. Product source +stays in backend/frontend. Generated state remains in ignored `.local/dev/`. + +Vault KV now contains `services/bff/jwt`, +`services/notifications/callers/bff`, `services/notifications/email/resend`, +`infrastructure/nats/auth` and `bootstrap/mongo`. Dynamic database credentials +remain in the database engine. Migration copies with CAS protection, compares +existing values, archives readable old versions and metadata privately, and then +removes old flat/TLS KV entries. Conflicting destinations stop migration without +removing sources. Policies and Agent templates reference only the new paths. +The unused existing PKI mount is retained to avoid destroying its signing key; +new bootstrap neither creates nor uses it. + +Caddy replaces Nginx, serving Flutter and proxying `/api` to BFF. Caddy owns browser +certificates and its CA in persistent volumes; only its public CA is exported to +`.local/dev/ca.pem`. Existing browser trust must be updated to that new CA. Local +BFF-to-notifications uses plaintext authenticated gRPC. Optional explicitly +configured TLS remains available in the reusable client/server; incomplete TLS +configuration is rejected. No local service consumes TLS secrets from Vault. + +Validation passed: `make build`, `make test`, `make lint-backend`, +`make test-dev-tooling`, `make test-dev-infra`, and Compose/diff checks. Tests cover +plaintext caller-token transport, optional verified TLS, invalid certificate +configuration and secret migration conflicts. The existing Vault was migrated +and canonical paths/absence of old entries verified. An isolated fresh project +passed transaction/lease checks and repeated bootstrap; disposable volumes and +state were removed. The final rotation check also verified Caddy static serving, +SPA fallback and API health, then three pool replacements with unchanged BFF PID +and 1,083 expected HTTP responses, zero failures. Role TTLs were restored and +temporary BFF/frontend containers removed. `make up` intentionally stops at the +missing Resend credential check; no real email, optional security scan or merge +was performed. Earlier supplementary legacy test limitations remain recorded. diff --git a/docs/local-development.md b/docs/local-development.md index 298b9c2..5639c9d 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -3,219 +3,163 @@ Task [#42](https://git.sendico.io/tech/sxchnge/issues/42), [PR #43](https://git.sendico.io/tech/sxchnge/pulls/43), branch `dev-env-42`. -## Build and bootstrap +## Build and start Requires Make, a POSIX shell and a local Docker engine with Compose v2 and -BuildKit. Docker builds Go, protobuf bindings and Flutter web; host Go, protoc -and Flutter installations are not required for `make build && make up`. The -tooling container includes Docker CLI, curl, jq, Git and OpenSSL. No host Python, -jq, curl or Git installation is needed for these Make targets. It mounts -the local Docker socket and workspace to orchestrate Compose; a remote Docker -daemon is not supported. The first Flutter SDK download can -take several minutes. Both ARM64 and AMD64 build from the same source definitions. +BuildKit. No host Python, Go, Flutter, jq or curl is required. Make runs bootstrap +tooling in Docker with the local Docker socket and workspace mounted. Remote +Docker engines are unsupported. ```sh make build && make up ``` -`make build` resolves the current stable upstream releases, pulls infrastructure -and builds NATS packaging, notifications, BFF and frontend images. Vault and Mongo -use upstream images directly. Resolution excludes prereleases and selects Nginx's -stable channel. The exact versions and manifest digests are saved in the ignored -`.local/dev/images.env`; Flutter's stable release is pinned by Git commit. Build -fails if upstream metadata cannot be resolved. Run `make build` again to refresh -versions. Review database release upgrade requirements before updating a populated -local volume across major versions; the tooling does not perform data migrations. +The build resolves the latest stable releases and pins their digests in ignored +`.local/dev/images.env`. Prereleases are excluded, resolution failures stop the +build, and previous pins are retained on failure. Flutter is pinned to the stable +release commit. Tooling uses a freshly pulled stable Docker CLI digest. Generated +protobuf bindings and Flutter output are built locally/in Docker and never tracked. -`make up` starts persistent Vault, initializes/unseals it, provisions secrets and -AppRoles, initializes an authenticated single-member MongoDB replica set, configures -Vault's MongoDB database plugin, and starts authenticated NATS JetStream. It then -starts notifications, BFF and the frontend when the required email settings exist. +Compose is at `infra/local/compose.yml`; Make supplies the repository root as its +project directory. Dockerfiles are in `infra/images/`. Runtime and startup files +are grouped by service under `infra/local/`. The stack consists of Vault, Mongo, +NATS, notifications, BFF and Caddy serving the frontend. -**On the first run, `make up` intentionally exits with an error after infrastructure -bootstrap because Resend credentials have not been supplied.** No placeholder API -key, mock delivery mode or embedded credential is installed. The three infrastructure -services remain running so you can configure Vault, then rerun `make up`. +On first startup, bootstrap initializes/unseals persistent Vault, provisions +AppRoles and secrets, initializes Mongo replica set `rs0`, configures Vault's Mongo +plugin, and starts authenticated NATS JetStream. Product startup fails explicitly +until Resend credentials exist. Infrastructure remains available for provisioning. -## Configure Resend in Vault +## Vault layout -The default email driver is `resend`. In the Vault UI at -[http://127.0.0.1:18200](http://127.0.0.1:18200), edit the KV v2 secret **`kv/email`** -(API path `kv/data/email`): +Paths below are inside the KV v2 mount `kv`; the API inserts `data/` after `kv/`. + +```text +kv/ +├── services/ +│ ├── bff/jwt # jwt +│ └── notifications/ +│ ├── callers/bff # token; read by BFF and notifications +│ └── email/resend # driver, api_key, sender_address, sender_name +├── infrastructure/ +│ └── nats/auth # token; read by NATS and notifications +└── bootstrap/ + └── mongo # local administrator and replica-set bootstrap material +``` + +Every secret has an owning service or infrastructure component. Service policies +permit only the exact paths they consume. Applications cannot read bootstrap +credentials. Mongo application users are dynamic secrets under +`database/creds/bff` and `database/creds/notifications`, not KV entries. They have +`readWrite` only on their respective `sxchnge` / `sxchnge_notifications` databases. +Vault uses `mongodb-database-plugin` and its own `vault-manager` account, whose +password is rotated immediately after initial configuration. + +There are no frontend certificates, notification certificates or generic `tls` +entries in KV. Browser certificates belong to Caddy; internal gRPC is plaintext +on the private Docker network and retains caller-token authentication. + +## Resend + +Open [Vault](http://127.0.0.1:18200) and edit +`kv/services/notifications/email/resend`: | Field | Value | | --- | --- | | `driver` | `resend` | | `api_key` | Your Resend API key | -| `sender_address` | Your verified Resend sender email | -| `sender_name` | `SXCHNGE` or your desired display name | +| `sender_address` | Your verified sender address | +| `sender_name` | Your display name | -Local administrative login material is in the private, ignored -`.local/dev/admin/init.json` (`root_token`). Keep that file private. Product -containers never mount it. You can also provision the same secret with a hidden -terminal prompt, without placing the API key in shell history: +Local administrator login material is in private `.local/dev/admin/init.json`. +Product containers cannot access it. Alternatively use the hidden terminal prompt: ```sh make email make up ``` -The startup check rejects an empty key or the bootstrap sender placeholder. It -does not validate the key against Resend or prove domain verification; an invalid -key fails at delivery. Vault Agent fetches the key and email settings and injects -them into the notifications process. There is no `.env` secret fallback. Changing -`kv/email` restarts the notification process automatically within about 10 seconds. -Bootstrapping again preserves your existing email secret. +No placeholder key or delivery fallback is installed. Startup validates presence; +it does not send an email or validate domain ownership with Resend. Updating email +settings causes Agent to restart notifications. Existing values survive bootstrap. -## Open the product +## Browser HTTPS and internal connections -After providing Resend credentials: +Caddy serves Flutter files and proxies `/api/*` to BFF over HTTP. Open +[https://localhost:5126](https://localhost:5126). HTTPS preserves Secure/HttpOnly +cookies and same-origin API access. Caddy uses `tls internal` and manages its local +CA, certificates and renewal in the `caddy-data` volume. The `caddy-config` volume +stores Caddy runtime configuration. Neither volume nor Caddy has Vault access. -- Frontend and proxied API: [https://localhost:5126](https://localhost:5126). -- BFF health: [https://localhost:5126/api/v1/health](https://localhost:5126/api/v1/health). -- Vault UI/API: [http://127.0.0.1:18200](http://127.0.0.1:18200). - -Trust `.local/dev/ca.pem` in your local browser/OS certificate store to use HTTPS -without a certificate error. Trust installation is a separate local action, not -performed by the bootstrap. For a command-line check without changing trust: +After frontend starts, Make exports only the public Caddy CA to `.local/dev/ca.pem`. +Trust it in your browser/OS to avoid certificate errors. This is a new CA when +migrating from Vault-issued browser certificates; the old CA will not trust it. +Trust installation is not automated. A CLI check can use: ```sh curl --cacert .local/dev/ca.pem https://localhost:5126/api/v1/health ``` -HTTPS keeps the existing Secure/HttpOnly refresh cookie contract. Nginx serves -Flutter and proxies `/api/` to the BFF on the same origin. The bundle selects the -existing backend auth adapter with the public client ID `sxchnge-local-web`. -Bootstrap inserts this public browser client if absent; it has no client secret. -No frontend widget, service or provider architecture changes are needed. +BFF calls notifications over plaintext gRPC with its caller token. Mongo and NATS +also stay within the private Compose network; no internal service publishes a host +port. Only frontend and Vault bind host loopback ports. TLS remains optional in +the reusable notification client/server for deployments that explicitly configure +it; local runtime configuration contains no certificate settings. -The rest of the app retains its existing capabilities and limitations: wallet, -KYC and settings fixtures are not turned into production integrations by this -container setup. Full email signup/login requires a working Resend account. +Caddy behavior follows its [internal TLS](https://caddyserver.com/docs/caddyfile/directives/tls) +and [automatic HTTPS](https://caddyserver.com/docs/automatic-https) documentation. -## Secret and connection lifecycle +## Mongo transactions and credential rotation -| Consumer | Vault access | -| --- | --- | -| BFF | `database/creds/bff`, JWT key, notification caller token and CA | -| Notifications | `database/creds/notifications`, NATS token, caller token, TLS certificate/key and email settings | -| NATS | NATS authentication token only | -| Bootstrap | Infrastructure administration, PKI and initial identities | +Mongo is an authenticated single-node replica set (`rs0`), supporting multi-document +transactions. It has no failover or redundant data copy. A Mongo restart interrupts +access even though application credential replacement does not. -The database engine is configured at `database/config/mongo` with -`mongodb-database-plugin`. It uses its own `vault-manager` Mongo user, whose -password is rotated by Vault immediately after configuration. The manager has -`userAdminAnyDatabase`; applications never receive that credential. The independent -local Mongo administrator and replica-set key come from Vault's random generator -and are retained in `kv/bootstrap/mongo` for local provisioning. +The shared database wrapper authenticates with each service's AppRole and obtains +an independent token/lease for each pool. At half the shorter token/lease lifetime, +it verifies a replacement pool before publishing it. Active operations, cursors +and full transactions retain the old pool until they finish, then it closes and +its token is revoked. Nested stores and commit retries keep the same pool. -The BFF role has `readWrite` only on `sxchnge`; notifications has `readWrite` only -on `sxchnge_notifications`. Mongo URIs come from dynamic Vault credentials, not -static application users. Leases default to one hour with a 24-hour maximum. -The shared database wrapper obtains credentials directly from Vault using the -service's AppRole. At half the effective lifetime (the shorter of the database -lease and owning token TTL), it authenticates independently and prepares a new -Mongo pool. Only a successful primary ping permits publication. New operations -use that pool; active operations, cursors and complete transactions, including -nested stores and commit retries, retain the old pool until they finish. The old -pool then closes and its dedicated token is revoked. Mongo credential replacement -never restarts BFF or notifications. Each generation has a separate token so -cleanup cannot revoke the replacement or Vault Agent's identity. +Failed refreshes retain valid credentials and retry with bounded backoff. After +expiry, database operations fail closed until automatic recovery. Administrative +revocation, infrastructure outages and operations longer than their lease can +still interrupt work. Repository consumers use `source.Source` operation scopes; +`source.Fixed` adapts externally owned static/test connections. -Refresh failures keep the current valid pool and retry with bounded backoff. -Operations are bounded by credential expiry. If Vault remains unavailable beyond -that deadline, new database operations fail closed; refresh continues and access -recovers automatically without restarting the service. Administrative revocation, -operations exceeding their lease and infrastructure failures cannot be made -uninterrupted by pool rotation. Shutdown rejects new scopes and drains existing -work. Failed cleanup relies on Vault expiration as a backstop. +## Migration and persistence -Repositories and transaction factories share `source.Source` rather than retaining -concrete Mongo collections. Low-level consumers must acquire a scope, use its -returned context and release it after all work completes; never retain its client -or context beyond release. `source.Fixed` adapts externally owned test/static -connections. Database credentials are no longer Agent environment templates; -Agent continues to fetch non-database secrets and may restart children when those -secrets change. +Bootstrap migrates the old flat `bff`, `caller`, `email` and `nats` paths into the +hierarchy above. It copies with CAS protection, compares values, and refuses to +overwrite conflicting destinations. After consumers use the new paths, it removes +old KV metadata and obsolete `tls`, `frontend-tls`, `notifications-tls` entries. +Readable old version history and metadata are archived privately in +`.local/dev/migrations/organized-v1/` before removal. That archive is administrative +recovery material; applications do not mount it. Old deleted/destroyed versions +are not recreated. The old PKI engine is not used or provisioned by this stack; +an existing PKI mount is retained to avoid destroying its non-exportable signing +key and history as part of a KV reorganization. -Each agent authenticates using its own AppRole with a renewable periodic token. -Local SecretIDs are reusable for container restarts; each container mounts only -its own identity directory under `.local/dev/agents/`. Revoking an AppRole SecretID -requires removing its matching local `secret-id` file and rerunning `make up` to -issue a replacement. Revoking an active token/lease terminates its credentials; -The database wrapper retries with a fresh identity; Agent independently retries -its non-database secrets when Vault is reachable. +Vault, Mongo, JetStream and Caddy data use project-scoped named volumes. Private +bootstrap material lives in `.local/dev/`. `make down` preserves both. Vault's local +unseal file allows automatic unsealing on restart. This local convenience and +loopback HTTP Vault endpoint are for development. Keep bootstrap files paired +with their matching volumes; do not discard one side independently. -The notification gRPC connection uses a Vault-issued certificate for -`notifications` and the BFF validates the Vault CA and hostname. Frontend HTTPS -certificates also come from Vault PKI. Leaf certificates last 30 days; `make up` -refreshes them when less than seven days remain and reloads Nginx. Run it regularly -for a development stack kept online for weeks. Agents restart when their TLS KV -secret changes. Application secrets are never put into the frontend bundle. - -Vault, Mongo and JetStream data use project-scoped named Docker volumes. Local -Vault unseal material, administrator login, per-service bootstrap identities and -frontend TLS material are under ignored `.local/dev/`, outside image build contexts. -The local Vault automatically unseals using its local key on container restart. -These convenience credentials and loopback HTTP Vault access are for this local -machine; this is not a production Vault deployment. Do not share `.local/dev/`. -Keep it paired with its matching Docker volumes for recovery. - -Mongo and NATS expose no host ports. The `sxchnge-local` network, volumes and -container names are independent of existing Sendico stacks. Only the frontend -and Vault bind host loopback ports. No Sendico database, message bus or secrets -are consumed. - -## Manage and verify +## Verification ```sh make ps make logs -make down # preserves named volumes and local bootstrap material -make up # idempotent provisioning and readiness checks -make test-dev-tooling # offline shell and failure-path checks -make test-dev-infra # works before Resend provisioning -make test-dev-rotation # real Vault rotations under BFF HTTP traffic -make test-dev # requires all six services running +make down +make test-dev-tooling # shell analysis, failure paths, migration conflict checks +make test-dev-infra # policy isolation, real Mongo transactions and lease lifecycle +make test-dev-rotation # temporarily deploy BFF/frontend and exercise short leases +make test-dev # requires all six services; verified frontend/API HTTPS ``` -The infrastructure check exercises AppRole access restrictions, real Vault -MongoDB credential issuance, per-database access, lease renewal and user deletion -on revocation. The complete check also verifies service health and frontend/API -HTTPS with the generated CA. These checks never send a real email. -`test-dev-rotation` temporarily sets a 12-second BFF lease, deploys the built BFF -and frontend, and requires three pool replacements with unchanged process ID and -zero unexpected HTTP responses. Its invalid-token requests exercise Mongo-backed -transactions without sending email. The check restores the role TTL and removes -services it started; it recreates BFF/frontend at the beginning, so run it as a -development validation command, not against active work you need to preserve. -Race-enabled backend tests additionally hold a transaction and cursor across a -replacement and exercise failed candidate validation, outage recovery, rollback, -expiry rejection and graceful shutdown. - -Backend regression tests use fresh `mongo:latest` and `nats:latest` official -stable images, pulled before `make test-backend`. Normal validation remains -`make test` and `make lint-backend` and requires the host SDKs described in the -repository guides. - -Deleting Docker volumes removes local data; `make down` deliberately keeps them. -If starting from scratch, stop the stack and explicitly remove this project's -three volumes **and** its matching `.local/dev/` bootstrap state together. Never -reuse old unseal material with a newly initialized Vault volume, or retain Mongo -users while discarding the Vault data that manages their leases. - -## Sources and checked releases - -On 2026-09-12 the resolver selected Vault 2.1.0, MongoDB 8.3.9, NATS 2.14.6, -Go 1.27.1, Nginx stable 1.30.4, Ubuntu 26.04 and Flutter stable 3.47.4. These are -verification records, not fixed future defaults. Release selection follows the -[Vault releases](https://releases.hashicorp.com/vault/), -[MongoDB stable release notes](https://www.mongodb.com/docs/manual/release-notes/), -[NATS releases](https://github.com/nats-io/nats-server/releases/latest), -[Nginx downloads](https://nginx.org/en/download.html), -[Go releases](https://go.dev/dl/) and -[Flutter stable channel](https://github.com/flutter/flutter/tree/stable). -The implementation follows HashiCorp's -[MongoDB database plugin](https://developer.hashicorp.com/vault/docs/secrets/databases/mongodb) -and [Agent process supervisor](https://developer.hashicorp.com/vault/docs/agent-and-proxy/agent/process-supervisor) -contracts. +Rotation validation temporarily shortens the BFF role TTL, recreates BFF/frontend, +and requires three replacements with unchanged BFF PID and zero unexpected +transaction-backed HTTP responses. It restores TTLs and removes services it +started. No verification command sends email. Backend validation remains +`make test` and `make lint-backend`, using the host SDKs documented in module guides. diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..3289ec1 --- /dev/null +++ b/infra/README.md @@ -0,0 +1,20 @@ +# Infrastructure + +Run `make build && make up` from the repository root. Local tooling needs Make, +POSIX shell and Docker Compose/BuildKit; all other tools run in Docker. + +- `images/`: reusable Docker build definitions for product images and tooling. +- `local/compose.yml`: isolated local stack and persistent volumes. +- `local/bff/`, `local/notifications/`: application runtime configuration. +- `local/frontend/`: Caddy configuration for Flutter files, browser HTTPS and API proxying. +- `local/vault/`: server startup/configuration and per-service Agent templates. +- `local/mongo/`, `local/nats/`: infrastructure startup and Mongo provisioning checks. +- `local/scripts/`: bootstrap, migration, release resolution and verification. +- `local/tools.sh`: host entry point used by Make. + +Local generated pins, identities and administrator material remain under ignored +`.local/dev/`; they are never build inputs. Application source remains under +`backend/` and `frontend/`. + +See [local development](../docs/local-development.md) for Vault paths, commands, +Caddy trust, persistence and migration details. diff --git a/dev/images/backend.Dockerfile b/infra/images/backend.Dockerfile similarity index 77% rename from dev/images/backend.Dockerfile rename to infra/images/backend.Dockerfile index dd3a7d3..836195b 100644 --- a/dev/images/backend.Dockerfile +++ b/infra/images/backend.Dockerfile @@ -13,8 +13,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/g FROM ${VAULT_IMAGE} USER root COPY --from=build /out/service /usr/local/bin/service -COPY dev/start-service.sh /usr/local/bin/start-service -COPY dev/config /etc/sxchnge +COPY infra/local/scripts/start-service.sh /usr/local/bin/start-service +COPY infra/local/bff/runtime.yml /etc/sxchnge/bff.yml +COPY infra/local/notifications/runtime.yml /etc/sxchnge/notifications.yml COPY backend/notifications/templates.example.json /etc/sxchnge/templates.json RUN mkdir -p /run/sxchnge && chown 100:1000 /run/sxchnge USER 100:1000 diff --git a/dev/images/frontend.Dockerfile b/infra/images/frontend.Dockerfile similarity index 83% rename from dev/images/frontend.Dockerfile rename to infra/images/frontend.Dockerfile index 51c5b2c..c0f7eb7 100644 --- a/dev/images/frontend.Dockerfile +++ b/infra/images/frontend.Dockerfile @@ -1,5 +1,5 @@ ARG UBUNTU_IMAGE -ARG NGINX_IMAGE +ARG CADDY_IMAGE FROM ${UBUNTU_IMAGE} AS build RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl git unzip xz-utils libglu1-mesa && rm -rf /var/lib/apt/lists/* ARG FLUTTER_VERSION @@ -12,6 +12,6 @@ COPY frontend ./frontend WORKDIR /src/frontend/app ARG FRONTEND_URL=https://localhost:5126 RUN flutter pub get && flutter gen-l10n && flutter build web --no-wasm-dry-run --dart-define=SXCHNGE_API_URL=${FRONTEND_URL}/api/v1 --dart-define=SXCHNGE_CLIENT_ID=sxchnge-local-web -FROM ${NGINX_IMAGE} -COPY dev/config/nginx.conf /etc/nginx/conf.d/default.conf -COPY --from=build /src/frontend/app/build/web /usr/share/nginx/html +FROM ${CADDY_IMAGE} +COPY infra/local/frontend/Caddyfile /etc/caddy/Caddyfile +COPY --from=build /src/frontend/app/build/web /srv diff --git a/dev/images/nats.Dockerfile b/infra/images/nats.Dockerfile similarity index 83% rename from dev/images/nats.Dockerfile rename to infra/images/nats.Dockerfile index 9fe15c9..3b5b1dc 100644 --- a/dev/images/nats.Dockerfile +++ b/infra/images/nats.Dockerfile @@ -4,7 +4,7 @@ FROM ${NATS_IMAGE} AS nats FROM ${VAULT_IMAGE} USER root COPY --from=nats /nats-server /usr/local/bin/nats-server -COPY dev/start-nats.sh /usr/local/bin/start-nats +COPY infra/local/nats/start.sh /usr/local/bin/start-nats RUN mkdir -p /data /run/sxchnge && chown -R 100:1000 /data /run/sxchnge USER 100:1000 ENTRYPOINT ["vault", "agent", "-config=/etc/agent/config.hcl"] diff --git a/dev/images/tools.Dockerfile b/infra/images/tools.Dockerfile similarity index 100% rename from dev/images/tools.Dockerfile rename to infra/images/tools.Dockerfile diff --git a/dev/config/bff.yml b/infra/local/bff/runtime.yml similarity index 95% rename from dev/config/bff.yml rename to infra/local/bff/runtime.yml index ca914db..e399832 100644 --- a/dev/config/bff.yml +++ b/infra/local/bff/runtime.yml @@ -39,6 +39,4 @@ api: notifications: address: notifications:8083 token_env: SXCHNGE_NOTIFICATIONS_TOKEN - ca_file: /run/sxchnge/ca.pem - server_name: notifications activation_url: https://localhost:5126/verify-email diff --git a/compose.yml b/infra/local/compose.yml similarity index 88% rename from compose.yml rename to infra/local/compose.yml index 169d6fa..a7bb6d3 100644 --- a/compose.yml +++ b/infra/local/compose.yml @@ -9,7 +9,7 @@ x-agent: &agent x-backend-build: &backend-build context: . - dockerfile: dev/images/backend.Dockerfile + dockerfile: infra/images/backend.Dockerfile services: vault: @@ -20,8 +20,8 @@ services: ports: ['127.0.0.1:18200:8200'] volumes: - vault-data:/vault/file - - ./dev/vault/server.hcl:/etc/vault/server.hcl:ro - - ./dev/start-vault.sh:/etc/vault/start.sh:ro + - ./infra/local/vault/server.hcl:/etc/vault/server.hcl:ro + - ./infra/local/vault/start.sh:/etc/vault/start.sh:ro - ./.local/dev/vault:/bootstrap:ro healthcheck: test: [CMD-SHELL, 'VAULT_ADDR=http://127.0.0.1:8200 vault status >/dev/null'] @@ -38,7 +38,7 @@ services: MONGO_INITDB_ROOT_USERNAME: local-admin MONGO_INITDB_ROOT_PASSWORD_FILE: /run/mongo/password volumes: - - ./dev/start-mongo.sh:/etc/mongo/start.sh:ro + - ./infra/local/mongo/start.sh:/etc/mongo/start.sh:ro - mongo-data:/data/db - ./.local/dev/mongo:/run/mongo:ro healthcheck: @@ -52,7 +52,7 @@ services: image: sxchnge-local/nats:dev build: context: . - dockerfile: dev/images/nats.Dockerfile + dockerfile: infra/images/nats.Dockerfile args: NATS_IMAGE: ${NATS_IMAGE:?Run make build first} VAULT_IMAGE: ${VAULT_IMAGE:?Run make build first} @@ -121,20 +121,21 @@ services: image: sxchnge-local/frontend:dev build: context: . - dockerfile: dev/images/frontend.Dockerfile + dockerfile: infra/images/frontend.Dockerfile args: UBUNTU_IMAGE: ${UBUNTU_IMAGE:?Run make build first} - NGINX_IMAGE: ${NGINX_IMAGE:?Run make build first} + CADDY_IMAGE: ${CADDY_IMAGE:?Run make build first} FLUTTER_VERSION: ${FLUTTER_VERSION:?Run make build first} FLUTTER_COMMIT: ${FLUTTER_COMMIT:?Run make build first} restart: unless-stopped ports: ['127.0.0.1:5126:8443'] volumes: - - ./.local/dev/frontend:/etc/tls:ro + - caddy-data:/data + - caddy-config:/config depends_on: bff: {condition: service_healthy} healthcheck: - test: [CMD, wget, -q, --no-check-certificate, -O, /dev/null, 'https://127.0.0.1:8443/'] + test: [CMD, wget, -q, --no-check-certificate, -O, /dev/null, 'https://localhost:8443/'] interval: 3s timeout: 3s retries: 40 @@ -143,3 +144,5 @@ volumes: vault-data: mongo-data: nats-data: + caddy-data: + caddy-config: diff --git a/infra/local/frontend/Caddyfile b/infra/local/frontend/Caddyfile new file mode 100644 index 0000000..8f39041 --- /dev/null +++ b/infra/local/frontend/Caddyfile @@ -0,0 +1,16 @@ +{ + skip_install_trust + auto_https disable_redirects +} + +https://localhost:8443 { + tls internal + root * /srv + handle /api/* { + reverse_proxy bff:8082 + } + handle { + try_files {path} /index.html + file_server + } +} diff --git a/dev/mongo-check.js b/infra/local/mongo/mongo-check.js similarity index 100% rename from dev/mongo-check.js rename to infra/local/mongo/mongo-check.js diff --git a/dev/mongo-client.js b/infra/local/mongo/mongo-client.js similarity index 100% rename from dev/mongo-client.js rename to infra/local/mongo/mongo-client.js diff --git a/dev/start-mongo.sh b/infra/local/mongo/start.sh similarity index 100% rename from dev/start-mongo.sh rename to infra/local/mongo/start.sh diff --git a/dev/start-nats.sh b/infra/local/nats/start.sh similarity index 100% rename from dev/start-nats.sh rename to infra/local/nats/start.sh diff --git a/dev/config/notifications.yml b/infra/local/notifications/runtime.yml similarity index 92% rename from dev/config/notifications.yml rename to infra/local/notifications/runtime.yml index a5a3882..7e4b060 100644 --- a/dev/config/notifications.yml +++ b/infra/local/notifications/runtime.yml @@ -1,6 +1,4 @@ listen: "0.0.0.0:8083" -tls_cert: "/run/sxchnge/server.crt" -tls_key: "/run/sxchnge/server.key" database: driver: mongodb settings: diff --git a/dev/bootstrap.sh b/infra/local/scripts/bootstrap.sh similarity index 77% rename from dev/bootstrap.sh rename to infra/local/scripts/bootstrap.sh index cd6da49..33e3585 100644 --- a/dev/bootstrap.sh +++ b/infra/local/scripts/bootstrap.sh @@ -16,7 +16,7 @@ initialize() { } mounts() { vault sys/mounts > "$WORK/mounts" - for name in kv database pki; do + for name in kv database; do if ! jq -e --arg key "$name/" 'has($key)' < "$WORK/mounts" >/dev/null; then if ! jq -e --arg key "$name/" '.data|has($key)' < "$WORK/mounts" >/dev/null; then jq -n --arg name "$name" '{type:$name} + (if $name=="kv" then {options:{version:"2"}} else {} end)' | vault "sys/mounts/$name" POST >/dev/null @@ -27,27 +27,6 @@ mounts() { printf '{"type":"approle"}' | vault sys/auth/approle POST >/dev/null fi } -certificates() { - printf '{"max_lease_ttl":"87600h"}' | vault sys/mounts/pki/tune POST >/dev/null - ca=$(kv tls) - if [ "$ca" = null ]; then - printf '{"common_name":"SXCHNGE local development CA","ttl":"87600h"}' | vault pki/root/generate/internal POST | jq '{ca:.data.certificate}' | put tls - fi - printf '{"allowed_domains":["localhost","notifications"],"allow_bare_domains":true,"allow_ip_sans":true,"max_ttl":"720h"}' | vault pki/roles/local POST >/dev/null - for name in notifications-tls frontend-tls; do - host=notifications; [ "$name" != frontend-tls ] || host=localhost - kv "$name" > "$WORK/cert" - if ! jq -e '.!=null and .expiration>(now+604800)' "$WORK/cert" >/dev/null; then - jq -n --arg host "$host" '{common_name:$host,ip_sans:"127.0.0.1",ttl:"720h"}' | vault pki/issue/local POST | jq '.data' > "$WORK/cert" - put "$name" < "$WORK/cert" - fi - if [ "$name" = frontend-tls ]; then - jq -r '.certificate' "$WORK/cert" | write "$STATE/frontend/server.crt" - jq -r '.private_key' "$WORK/cert" | write "$STATE/frontend/server.key" - fi - done - kv tls | jq -r '.ca' | write "$STATE/ca.pem" 644 -} provision_mongo() { if [ "$(kv bootstrap/mongo)" = null ]; then random > "$WORK/password"; random 384 > "$WORK/keyfile"; random > "$WORK/manager" @@ -72,16 +51,16 @@ provision_mongo() { database=sxchnge; [ "$role" != notifications ] || database=sxchnge_notifications jq -n --arg database "$database" '{db_name:"mongo",creation_statements:({db:"admin",roles:[{role:"readWrite",db:$database}]}|tojson),default_ttl:"1h",max_ttl:"24h"}' | vault "database/roles/$role" POST >/dev/null done - { cat "$WORK/admin.js"; cat dev/mongo-client.js; } | mongo >/dev/null || fail 'Browser client provisioning failed' + { cat "$WORK/admin.js"; cat infra/local/mongo/mongo-client.js; } | mongo >/dev/null || fail 'Browser client provisioning failed' } provision_agents() { for service in bff notifications nats; do directory="$STATE/agents/$service" mkdir -p "$directory"; chmod 755 "$directory" case "$service" in - bff) paths='bff caller tls';; - notifications) paths='caller tls notifications-tls nats email';; - nats) paths=nats;; + bff) paths='services/bff/jwt services/notifications/callers/bff';; + notifications) paths='services/notifications/callers/bff infrastructure/nats/auth services/notifications/email/resend';; + nats) paths=infrastructure/nats/auth;; esac : > "$WORK/policy" for path in $paths; do printf 'path "kv/data/%s" { capabilities = ["read"] }\n' "$path" >> "$WORK/policy"; done @@ -92,7 +71,12 @@ provision_agents() { if [ ! -f "$directory/secret-id" ]; then printf '{}' | vault "auth/approle/role/$service/secret-id" POST | jq -r '.data.secret_id' | write "$directory/secret-id" 444 fi - cat "dev/vault/$service.hcl" | write "$directory/config.hcl" 444 + if ! cmp -s "infra/local/vault/$service.hcl" "$directory/config.hcl"; then + cat "infra/local/vault/$service.hcl" | write "$directory/config.hcl" 444 + if compose ps --status running --services "$service" | grep -qx "$service"; then + compose up -d --no-deps --force-recreate --wait --wait-timeout 120 "$service" + fi + fi done } up() { @@ -107,20 +91,21 @@ up() { fi initialize mounts - for path in nats caller bff; do + migrate_secrets + for path in infrastructure/nats/auth services/notifications/callers/bff services/bff/jwt; do if [ "$(kv "$path")" = null ]; then - field=token; [ "$path" != bff ] || field=jwt + field=token; [ "$path" != services/bff/jwt ] || field=jwt random | jq -Rs --arg field "$field" '{($field):rtrimstr("\n")}' | put "$path" fi done - printf '{"driver":"resend","api_key":"","sender_address":"notifications@localhost.test","sender_name":"SXCHNGE local"}' | ensure email - certificates + printf '{"driver":"resend","api_key":"","sender_address":"notifications@localhost.test","sender_name":"SXCHNGE local"}' | ensure services/notifications/email/resend provision_mongo provision_agents compose up -d --wait --wait-timeout 120 nats - kv email | email_valid || fail 'Vault kv/email needs a Resend api_key and verified sender_address. Run make email, then make up.' + retire_secrets + kv services/notifications/email/resend | email_valid || fail 'Vault kv/services/notifications/email/resend needs a Resend api_key and verified sender_address. Run make email, then make up.' compose up -d --wait --wait-timeout 180 - compose exec -T frontend nginx -s reload + export_ca echo 'Local stack ready: https://localhost:5126; Vault: http://127.0.0.1:18200' } email() { @@ -138,6 +123,6 @@ email() { case "$address" in *@*) ;; *) fail 'A sender email address is required';; esac printf '%s' "$key" > "$WORK/email-key" printf '%s' "$address" > "$WORK/email-address" - jq -n --rawfile key "$WORK/email-key" --rawfile address "$WORK/email-address" '{driver:"resend",api_key:$key,sender_address:$address,sender_name:"SXCHNGE"}' | put email + jq -n --rawfile key "$WORK/email-key" --rawfile address "$WORK/email-address" '{driver:"resend",api_key:$key,sender_address:$address,sender_name:"SXCHNGE"}' | put services/notifications/email/resend echo 'Email configuration saved in Vault. Run make up.' } diff --git a/dev/common.sh b/infra/local/scripts/common.sh similarity index 92% rename from dev/common.sh rename to infra/local/scripts/common.sh index 45ef441..522bfb0 100644 --- a/dev/common.sh +++ b/infra/local/scripts/common.sh @@ -14,7 +14,7 @@ trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM fail() { echo "$*" >&2; exit 1; } -compose() { docker compose --env-file "$STATE/images.env" "$@"; } +compose() { docker compose --project-directory . -f infra/local/compose.yml --env-file "$STATE/images.env" "$@"; } write() { target=$1; mode=${2:-600} mkdir -p "$(dirname "$target")" @@ -58,3 +58,6 @@ ensure() ( mongo() { compose exec -T mongo mongosh --quiet --file /dev/stdin > "$WORK/mongo-result" 2> "$WORK/mongo-error" || return 1; cat "$WORK/mongo-result"; } admin_auth() { kv bootstrap/mongo | jq -r '"const admin=db.getSiblingDB(\"admin\"); admin.auth(\"local-admin\","+(.password|tojson)+");"'; } email_valid() { jq -e '(.api_key|type=="string") and (.api_key|test("\\S")) and (.sender_address|type=="string") and (.sender_address|test("\\S")) and .sender_address!="notifications@localhost.test"' >/dev/null; } +export_ca() { + compose exec -T frontend cat /data/caddy/pki/authorities/local/root.crt | write "$STATE/ca.pem" 644 +} diff --git a/infra/local/scripts/dispatch.sh b/infra/local/scripts/dispatch.sh new file mode 100644 index 0000000..1adbd42 --- /dev/null +++ b/infra/local/scripts/dispatch.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +set -o pipefail +. infra/local/scripts/common.sh +. infra/local/scripts/secrets.sh +case "${1:-}" in + images) . infra/local/scripts/images.sh; images;; + up) . infra/local/scripts/bootstrap.sh; up;; + email) . infra/local/scripts/bootstrap.sh; email;; + verify|verify-infra) . infra/local/scripts/verify.sh; existing; infrastructure; if [ "$1" = verify ]; then product; fi;; + rotation) . infra/local/scripts/verify.sh; . infra/local/scripts/rotation.sh; existing; rotation;; + test) sh infra/local/scripts/test.sh;; + *) fail 'Unknown local tooling command';; +esac diff --git a/dev/images.sh b/infra/local/scripts/images.sh similarity index 91% rename from dev/images.sh rename to infra/local/scripts/images.sh index 4238fbb..7137117 100644 --- a/dev/images.sh +++ b/infra/local/scripts/images.sh @@ -10,7 +10,7 @@ image() ( ) images() ( go=$(get 'https://go.dev/dl/?mode=json' | jq -er '.[0] | select(.stable) | .version | select(test("^go[0-9]+\\.[0-9]+\\.[0-9]+$"))') || exit 1 - for spec in VAULT_IMAGE:hashicorp/vault:latest MONGO_IMAGE:library/mongo:latest NATS_IMAGE:library/nats:latest NGINX_IMAGE:library/nginx:stable-alpine "GO_IMAGE:library/golang:${go#go}-alpine" UBUNTU_IMAGE:library/ubuntu:latest; do + for spec in VAULT_IMAGE:hashicorp/vault:latest MONGO_IMAGE:library/mongo:latest NATS_IMAGE:library/nats:latest CADDY_IMAGE:library/caddy:alpine "GO_IMAGE:library/golang:${go#go}-alpine" UBUNTU_IMAGE:library/ubuntu:latest; do key=${spec%%:*}; rest=${spec#*:}; repo=${rest%:*}; alias=${rest##*:} ref=$(image "$repo" "$alias") || exit 1 printf '%s=%s\n' "$key" "$ref" >> "$WORK/images.env" diff --git a/dev/rotation.sh b/infra/local/scripts/rotation.sh similarity index 89% rename from dev/rotation.sh rename to infra/local/scripts/rotation.sh index 9f7ce2f..7355773 100644 --- a/dev/rotation.sh +++ b/infra/local/scripts/rotation.sh @@ -20,6 +20,12 @@ rotation() ( trap 'exit 143' TERM jq '.+{default_ttl:"12s",max_ttl:"24s"}' "$WORK/original-role" | vault database/roles/bff POST >/dev/null compose up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend + export_ca + https -f https://localhost:5126/ > "$WORK/index" + grep -q flutter_bootstrap "$WORK/index" || fail 'Frontend bundle is not served' + https -f https://localhost:5126/settings > "$WORK/route" + cmp -s "$WORK/index" "$WORK/route" || fail 'Frontend route fallback failed' + https -f https://localhost:5126/api/v1/health >/dev/null process=$(compose exec -T bff pidof service) for worker in 1 2 3 4; do ( diff --git a/infra/local/scripts/secrets.sh b/infra/local/scripts/secrets.sh new file mode 100644 index 0000000..264a1d2 --- /dev/null +++ b/infra/local/scripts/secrets.sh @@ -0,0 +1,56 @@ +# Canonical paths follow ownership. Shared callers live under the receiving service. +secret_moves() { + cat <<'PATHS' +bff services/bff/jwt +caller services/notifications/callers/bff +email services/notifications/email/resend +nats infrastructure/nats/auth +PATHS +} +# Archive readable version history privately before removing an obsolete KV path. +archive_secret() ( + old=$1 + metadata=$(vault "kv/metadata/$old" GET yes) + [ "$metadata" != null ] || exit 0 + directory="$STATE/migrations/organized-v1/$old" + mkdir -p "$directory" + printf '%s' "$metadata" | write "$directory/metadata.json" + printf '%s' "$metadata" | jq -r '.data.versions|to_entries[]|select(.value.destroyed==false and .value.deletion_time=="")|.key' > "$WORK/versions" + while IFS= read -r version; do + vault "kv/data/$old?version=$version" | write "$directory/version-$version.json" + done < "$WORK/versions" +) +migrate_secrets() ( + secret_moves > "$WORK/moves" + while read -r old new; do + kv "$old" > "$WORK/old" + [ "$(cat "$WORK/old")" != null ] || continue + archive_secret "$old" + kv "$new" > "$WORK/new" + if [ "$(cat "$WORK/new")" = null ]; then + # CAS=0 prevents overwriting a destination created concurrently. + jq '{data:.,options:{cas:0}}' "$WORK/old" | vault "kv/data/$new" POST >/dev/null + fi + kv "$new" > "$WORK/new" + jq -e -s '.[0]==.[1]' "$WORK/old" "$WORK/new" >/dev/null || fail "Conflicting Vault values at $old and $new; no source removed" + done < "$WORK/moves" +) +retire_secrets() ( + # Call only after templates, policies and deployed images use canonical paths. + secret_moves > "$WORK/moves" + while read -r old new; do + kv "$old" > "$WORK/old" + [ "$(cat "$WORK/old")" != null ] || continue + kv "$new" > "$WORK/new" + jq -e -s '.[0]==.[1]' "$WORK/old" "$WORK/new" >/dev/null || fail "Vault migration verification failed for $old" + archive_secret "$old" + printf '{}' | vault "kv/metadata/$old" DELETE >/dev/null + done < "$WORK/moves" + for old in tls frontend-tls notifications-tls; do + archive_secret "$old" + if [ "$(vault "kv/metadata/$old" GET yes)" != null ]; then printf '{}' | vault "kv/metadata/$old" DELETE >/dev/null; fi + done + # The obsolete PKI mount is deliberately not disabled: its private signing key + # and history are retained until the owner explicitly chooses to destroy them. + echo 'PASS: Vault secrets migrated and verified; obsolete KV entries retired' +) diff --git a/dev/start-service.sh b/infra/local/scripts/start-service.sh similarity index 60% rename from dev/start-service.sh rename to infra/local/scripts/start-service.sh index 62bf205..dec163b 100755 --- a/dev/start-service.sh +++ b/infra/local/scripts/start-service.sh @@ -1,13 +1,9 @@ #!/bin/sh set -eu umask 077 -printf '%s\n' "$TLS_CA" > /run/sxchnge/ca.pem -unset TLS_CA if [ "$SXCHNGE_SERVICE" = notifications ]; then - printf '%s\n' "$TLS_CERT" > /run/sxchnge/server.crt - printf '%s\n' "$TLS_KEY" > /run/sxchnge/server.key printf '%s\n' "$EMAIL_CONFIG" > /run/sxchnge/email.yml - unset TLS_CERT TLS_KEY EMAIL_CONFIG + unset EMAIL_CONFIG cat /etc/sxchnge/notifications.yml /run/sxchnge/email.yml > /run/sxchnge/config.yml exec service -config.file=/run/sxchnge/config.yml fi diff --git a/dev/test.sh b/infra/local/scripts/test.sh similarity index 61% rename from dev/test.sh rename to infra/local/scripts/test.sh index c0307ed..7fb8a92 100644 --- a/dev/test.sh +++ b/infra/local/scripts/test.sh @@ -2,11 +2,11 @@ set -eu set -o pipefail # Shell syntax and static analysis cover the actual container shell scripts. -shellcheck -s ash -x -S warning dev/dispatch.sh dev/test.sh -shellcheck -x -S warning dev/tools.sh dev/start-*.sh -. dev/common.sh -. dev/images.sh -. dev/bootstrap.sh +shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh +shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh +. infra/local/scripts/common.sh +. infra/local/scripts/images.sh +. infra/local/scripts/bootstrap.sh # Offline fixtures exercise stable selection, reject prereleases, and preserve pins. get() { case "$1" in @@ -33,7 +33,29 @@ vault() { printf '{"initialized":true}'; } if (initialize) > "$WORK/initialize-error" 2>&1; then fail 'Missing local keys accepted'; fi grep -q 'Missing .local' "$WORK/initialize-error" for service in bff notifications; do - if grep -Eq 'database/creds/|SXCHNGE_MONGO_URI' "dev/vault/$service.hcl"; then fail 'Database credentials trigger Agent restart'; fi + if grep -Eq 'database/creds/|SXCHNGE_MONGO_URI' "infra/local/vault/$service.hcl"; then fail 'Database credentials trigger Agent restart'; fi done echo 'PASS: shell analysis, stable selection, failed-resolution preservation, email validation, missing-key protection and Agent contract' if command -v python >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1; then fail 'Tool image unexpectedly contains Python'; fi +# Migration never overwrites an independently provisioned destination. +. infra/local/scripts/secrets.sh +secret_moves() { echo 'email services/notifications/email/resend'; } +archive_secret() { :; } +kv() { + case "$1" in + email) printf '{"api_key":"existing-fixture"}';; + *) cat "$WORK/destination";; + esac +} +vault() { + [ "$1" = kv/data/services/notifications/email/resend ] || fail 'Unexpected migration mutation' + jq -e '.options.cas==0' >/dev/null + printf '{"api_key":"existing-fixture"}' > "$WORK/destination" +} +printf 'null' > "$WORK/destination" +migrate_secrets +migrate_secrets +printf '{"api_key":"different-fixture"}' > "$WORK/destination" +if (migrate_secrets) >/dev/null 2>&1; then fail 'Migration overwrote conflicting destination'; fi +jq -e '.api_key=="different-fixture"' "$WORK/destination" >/dev/null +echo 'PASS: secret migration is repeatable and refuses conflicting destination values' diff --git a/dev/verify.sh b/infra/local/scripts/verify.sh similarity index 78% rename from dev/verify.sh rename to infra/local/scripts/verify.sh index a0d636f..c378101 100644 --- a/dev/verify.sh +++ b/infra/local/scripts/verify.sh @@ -6,7 +6,7 @@ policies() ( directory="$STATE/agents/$service" login=$(jq -n --rawfile role "$directory/role-id" --rawfile secret "$directory/secret-id" '{role_id:($role|rtrimstr("\n")),secret_id:($secret|rtrimstr("\n"))}' | vault auth/approle/login POST) identity=$(printf '%s' "$login" | jq -er '.auth.client_token') - printf '%s' "$login" | jq '{token:.auth.client_token,paths:["kv/data/bootstrap/mongo","sys/init"]}' | vault sys/capabilities POST | jq -e '.data | all(.[]; .==["deny"])' >/dev/null || fail 'Service can access bootstrap credentials' + printf '%s' "$login" | jq '{token:.auth.client_token,paths:["kv/data/bootstrap/mongo","sys/init","kv/data/bff","kv/data/caller","kv/data/email","kv/data/nats","kv/data/tls"]}' | vault sys/capabilities POST | jq -e '.data | all(.[]; .==["deny"])' >/dev/null || fail 'Service can access bootstrap credentials' VAULT_TOKEN=$(printf '%s' "$login" | jq -er '.auth.client_token') printf '{}' | vault auth/token/revoke-self POST >/dev/null VAULT_TOKEN=$root_token @@ -23,7 +23,7 @@ database_access() ( { jq -r '"db.getSiblingDB(\"admin\").auth("+(.data.username|tojson)+","+(.data.password|tojson)+");"' "$WORK/lease" printf 'const database="%s", forbidden="%s";\n' "$database" "$forbidden" - cat dev/mongo-check.js + cat infra/local/mongo/mongo-check.js } | mongo > "$WORK/check" || fail 'Mongo access/transaction check failed' grep -q scope-ok "$WORK/check" || fail 'Mongo check incomplete' jq '{lease_id:.lease_id,increment:"1h"}' "$WORK/lease" | vault sys/leases/renew POST | jq -e '.lease_duration>0' >/dev/null || fail 'Mongo renewal failed' @@ -37,6 +37,7 @@ database_access() ( infrastructure() { compose ps --format json | jq -se '[.[] | select(.Service=="vault" or .Service=="mongo" or .Service=="nats")] | length==3 and all(.[];.Health=="healthy")' >/dev/null || fail 'Infrastructure is not healthy' echo 'PASS: Vault, Mongo replica set and NATS JetStream are healthy' + layout policies database_access } @@ -48,3 +49,12 @@ product() { https -f https://localhost:5126/api/v1/health >/dev/null echo 'PASS: six healthy services, frontend and proxied BFF use verified TLS; no email sent' } +layout() { + for old in bff caller email nats tls frontend-tls notifications-tls; do + [ "$(vault "kv/metadata/$old" GET yes)" = null ] || fail "Obsolete KV entry remains: $old" + done + for path in services/bff/jwt services/notifications/callers/bff services/notifications/email/resend infrastructure/nats/auth bootstrap/mongo; do + vault "kv/metadata/$path" | jq -e '.data.current_version>0' >/dev/null || fail "Missing canonical KV entry: $path" + done + echo 'PASS: canonical Vault hierarchy present; obsolete flat and TLS entries absent' +} diff --git a/dev/tools.sh b/infra/local/tools.sh similarity index 83% rename from dev/tools.sh rename to infra/local/tools.sh index ff74fc1..7fe4ed2 100644 --- a/dev/tools.sh +++ b/infra/local/tools.sh @@ -1,13 +1,13 @@ #!/bin/sh # Host entry point: only POSIX shell and Docker are required. set -eu -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../.." case "${1:-}" in build) docker pull docker:cli ref=$(docker image inspect docker:cli --format '{{index .RepoDigests 0}}') case "$ref" in docker@sha256:*) ;; *) echo 'Cannot pin stable Docker CLI image' >&2; exit 1;; esac - docker build --pull --build-arg "DOCKER_IMAGE=$ref" -f dev/images/tools.Dockerfile -t sxchnge-local/tools:dev dev/images + docker build --pull --build-arg "DOCKER_IMAGE=$ref" -f infra/images/tools.Dockerfile -t sxchnge-local/tools:dev infra/images exit ;; esac @@ -22,4 +22,4 @@ if [ "${1:-}" = email ]; then terminal_args=-it; fi # shellcheck disable=SC2086 exec docker run --rm $terminal_args $network_args -e "LOCAL_UID=$(id -u)" -e "LOCAL_GID=$(id -g)" \ -v "$socket:/var/run/docker.sock" -v "$PWD:$PWD" -w "$PWD" \ - sxchnge-local/tools:dev sh dev/dispatch.sh "$@" + sxchnge-local/tools:dev sh infra/local/scripts/dispatch.sh "$@" diff --git a/dev/vault/bff.hcl b/infra/local/vault/bff.hcl similarity index 67% rename from dev/vault/bff.hcl rename to infra/local/vault/bff.hcl index 37e62ef..1b1eebe 100644 --- a/dev/vault/bff.hcl +++ b/infra/local/vault/bff.hcl @@ -13,18 +13,14 @@ template_config { exit_on_retry_failure = true } -env_template "TLS_CA" { - contents = "{{ with secret \"kv/data/tls\" }}{{ .Data.data.ca }}{{ end }}" - error_on_missing_key = true -} env_template "SXCHNGE_JWT_SECRET" { - contents = "{{ with secret \"kv/data/bff\" }}{{ .Data.data.jwt }}{{ end }}" + contents = "{{ with secret \"kv/data/services/bff/jwt\" }}{{ .Data.data.jwt }}{{ end }}" error_on_missing_key = true } env_template "SXCHNGE_NOTIFICATIONS_TOKEN" { - contents = "{{ with secret \"kv/data/caller\" }}{{ .Data.data.token }}{{ end }}" + contents = "{{ with secret \"kv/data/services/notifications/callers/bff\" }}{{ .Data.data.token }}{{ end }}" error_on_missing_key = true } diff --git a/dev/vault/nats.hcl b/infra/local/vault/nats.hcl similarity index 83% rename from dev/vault/nats.hcl rename to infra/local/vault/nats.hcl index 2c2aea3..4e712a4 100644 --- a/dev/vault/nats.hcl +++ b/infra/local/vault/nats.hcl @@ -14,7 +14,7 @@ template_config { } env_template "NATS_TOKEN" { - contents = "{{ with secret \"kv/data/nats\" }}{{ .Data.data.token }}{{ end }}" + contents = "{{ with secret \"kv/data/infrastructure/nats/auth\" }}{{ .Data.data.token }}{{ end }}" error_on_missing_key = true } diff --git a/infra/local/vault/notifications.hcl b/infra/local/vault/notifications.hcl new file mode 100644 index 0000000..bb1a912 --- /dev/null +++ b/infra/local/vault/notifications.hcl @@ -0,0 +1,43 @@ +vault { address = "http://vault:8200" } +auto_auth { + method "approle" { + config = { + role_id_file_path = "/etc/agent/role-id" + secret_id_file_path = "/etc/agent/secret-id" + remove_secret_id_file_after_reading = false + } + } +} +template_config { + static_secret_render_interval = "10s" + exit_on_retry_failure = true +} + + +env_template "SXCHNGE_NOTIFICATIONS_PRODUCT_TOKEN" { + contents = "{{ with secret \"kv/data/services/notifications/callers/bff\" }}{{ .Data.data.token }}{{ end }}" + error_on_missing_key = true +} + +env_template "SXCHNGE_NATS_URL" { + contents = "{{ with secret \"kv/data/infrastructure/nats/auth\" }}nats://{{ .Data.data.token }}@nats:4222{{ end }}" + error_on_missing_key = true +} + + + +env_template "EMAIL_API_KEY" { + contents = "{{ with secret \"kv/data/services/notifications/email/resend\" }}{{ .Data.data.api_key }}{{ end }}" + error_on_missing_key = true +} + +env_template "EMAIL_CONFIG" { + contents = "{{ with secret \"kv/data/services/notifications/email/resend\" }}email:\n driver: {{ .Data.data.driver | toJSON }}\n settings:\n api_key_env: EMAIL_API_KEY\n sender_address: {{ .Data.data.sender_address | toJSON }}\n sender_name: {{ .Data.data.sender_name | toJSON }}\n{{ end }}" + error_on_missing_key = true +} + +exec { + command = ["/usr/local/bin/start-service"] + restart_on_secret_changes = "always" + restart_stop_signal = "SIGTERM" +} diff --git a/dev/vault/server.hcl b/infra/local/vault/server.hcl similarity index 100% rename from dev/vault/server.hcl rename to infra/local/vault/server.hcl diff --git a/dev/start-vault.sh b/infra/local/vault/start.sh similarity index 100% rename from dev/start-vault.sh rename to infra/local/vault/start.sh