From f436b2528cb46de1b433e715a65fc7a8c7284448 Mon Sep 17 00:00:00 2001 From: protuberanets Date: Thu, 29 Oct 2026 20:00:33 +0300 Subject: [PATCH 1/3] Run the production Vault as three raft nodes on exch and infra-gamma Two nodes on exch and one on infra-gamma, plain Compose per server, ports on the private addresses only. Nodes unseal through the transit seal on the platform Vault; the transit token reaches only the server process. make prod-images resolves the latest stable Vault release once and pins its digest in ignored .local/prod/images.env, so every node runs the same build. infra/prod/README.md records the layout, the first rollout, node replacement and recovery when the platform Vault is lost (#74). --- Makefile | 6 ++ infra/local/scripts/dispatch.sh | 1 + infra/local/scripts/images.sh | 7 +++ infra/local/scripts/test.sh | 2 +- infra/prod/README.md | 93 +++++++++++++++++++++++++++++ infra/prod/vault/config/vault-1.hcl | 30 ++++++++++ infra/prod/vault/config/vault-2.hcl | 30 ++++++++++ infra/prod/vault/config/vault-3.hcl | 30 ++++++++++ infra/prod/vault/exch.yml | 59 ++++++++++++++++++ infra/prod/vault/gamma.yml | 34 +++++++++++ infra/prod/vault/start.sh | 8 +++ 11 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 infra/prod/README.md create mode 100644 infra/prod/vault/config/vault-1.hcl create mode 100644 infra/prod/vault/config/vault-2.hcl create mode 100644 infra/prod/vault/config/vault-3.hcl create mode 100644 infra/prod/vault/exch.yml create mode 100644 infra/prod/vault/gamma.yml create mode 100644 infra/prod/vault/start.sh diff --git a/Makefile b/Makefile index f5aa3fb..5a23a63 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,12 @@ test-dev: sh infra/local/tools.sh test sh infra/local/tools.sh verify +# Production pins (infra/prod/README.md): resolved once, copied to every node. +.PHONY: prod-images +prod-images: + sh infra/local/tools.sh build + sh infra/local/tools.sh prod-images + .PHONY: test-dev-infra test-dev-infra: sh infra/local/tools.sh test diff --git a/infra/local/scripts/dispatch.sh b/infra/local/scripts/dispatch.sh index 32d20cb..4f202b7 100644 --- a/infra/local/scripts/dispatch.sh +++ b/infra/local/scripts/dispatch.sh @@ -5,6 +5,7 @@ set -o pipefail . infra/local/scripts/secrets.sh case "${1:-}" in images) . infra/local/scripts/images.sh; images;; + prod-images) TOOL_STATE="$PWD/.local"; . infra/local/scripts/images.sh; prod_images;; up) . infra/local/scripts/bootstrap.sh; up;; email) . infra/local/scripts/bootstrap.sh; email;; sendico) . infra/local/scripts/bootstrap.sh; . infra/local/scripts/sendico.sh; configure_sendico;; diff --git a/infra/local/scripts/images.sh b/infra/local/scripts/images.sh index 7137117..85fd348 100644 --- a/infra/local/scripts/images.sh +++ b/infra/local/scripts/images.sh @@ -22,3 +22,10 @@ images() ( cat "$WORK/images.env" | write "$STATE/images.env" cat "$STATE/images.env" ) +# Production pins: one resolution for every node of a cluster, so all of them +# run the same build. Written outside .local/dev; copied to the servers. +prod_images() ( + ref=$(image hashicorp/vault latest) || exit 1 + printf 'VAULT_IMAGE=%s\n' "$ref" | write "$PWD/.local/prod/images.env" + cat "$PWD/.local/prod/images.env" +) diff --git a/infra/local/scripts/test.sh b/infra/local/scripts/test.sh index b026f9d..bca7001 100644 --- a/infra/local/scripts/test.sh +++ b/infra/local/scripts/test.sh @@ -3,7 +3,7 @@ set -eu set -o pipefail # Shell syntax and static analysis cover the actual container shell scripts. shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh -shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh +shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh infra/prod/vault/start.sh shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh infra/local/sendico/network.sh . infra/local/scripts/common.sh . infra/local/scripts/images.sh diff --git a/infra/prod/README.md b/infra/prod/README.md new file mode 100644 index 0000000..013880c --- /dev/null +++ b/infra/prod/README.md @@ -0,0 +1,93 @@ +# Production + +Production runs on two servers of the shared private network: `exch` +(192.168.10.191), the SXCHNGE server, and `infra-gamma` (192.168.10.98), which +keeps the third copy of the data. Each component runs two nodes on `exch` and +one on `infra-gamma`: losing `infra-gamma` keeps the majority, losing one node on +`exch` keeps two of three. Every server runs plain Docker Compose; files are +deployed to `/srv/sxchnge/` on the server. + +Ports are published on the private addresses only. Traffic between the servers +is not wrapped in TLS: the network is private and limited by security groups, +and every component authenticates its peers. + +## Vault + +[#74](https://git.sendico.io/tech/sxchnge/issues/74). Three raft nodes: + +| Node | Server | API | Cluster | Compose | +| --- | --- | --- | --- | --- | +| `sxchnge-vault-1` | exch | 192.168.10.191:8200 | :8201 | `vault/exch.yml` | +| `sxchnge-vault-2` | exch | 192.168.10.191:8210 | :8211 | `vault/exch.yml` | +| `sxchnge-vault-3` | infra-gamma | 192.168.10.98:8200 | :8201 | `vault/gamma.yml` | + +Containers on `exch` join the Docker network `sxchnge` and use +`http://vault:8200`; the alias belongs to both local nodes. A standby node +forwards requests to the active one, so the leader may be on any server. + +Nodes unseal themselves with the `transit` seal on the platform Vault +(`https://vault.sendico.io`, key `transit/keys/sxchnge-vault-unseal`). Each server +has its own periodic token in `secrets/transit-token` (root, mode 600, not in Git) +that may only encrypt and decrypt with that key. If the platform Vault is +unreachable while a node starts, the node stays sealed until it is reachable +again; running nodes are not affected. `vault operator init` returns recovery +keys, not unseal keys: they authorize operations such as generating a new root +token but cannot unseal a node. Only the transit key can. Its backup +(`transit/backup/sxchnge-vault-unseal`), the recovery keys and the root token are +kept outside the servers by their holders; see "Platform Vault lost". + +`make prod-images` resolves the latest stable Vault release and pins its digest in +ignored `.local/prod/images.env`. All three nodes get the same file: a cluster +runs one build, and an upgrade is a separate rollout of a new file, one node at a +time. + +### First rollout + +1. On the platform Vault, with an administrator token: + + ```sh + vault secrets list | grep -q '^transit/' || vault secrets enable transit + vault write transit/keys/sxchnge-vault-unseal exportable=true allow_plaintext_backup=true + vault read -field=backup transit/backup/sxchnge-vault-unseal # store with the holders, then clear the terminal + vault policy write sxchnge-vault-unseal - <<'EOF' + path "transit/encrypt/sxchnge-vault-unseal" { capabilities = ["update"] } + path "transit/decrypt/sxchnge-vault-unseal" { capabilities = ["update"] } + EOF + ``` + + Create one token per server and write it straight into that server's + `/srv/sxchnge/vault/secrets/transit-token`, never to a terminal or a log: + + ```sh + vault token create -orphan -period=768h -policy=sxchnge-vault-unseal \ + -display-name=sxchnge-vault- -field=token + ``` + +2. Copy `vault/` and `.local/prod/images.env` to `/srv/sxchnge/vault` on both + servers. +3. Start the nodes: on `exch` + `docker compose --env-file images.env -f exch.yml up -d`, on `infra-gamma` + the same with `gamma.yml`. +4. Initialize once, on `exch`: + `docker exec -it sxchnge-vault-vault-1-1 vault operator init -recovery-shares=5 -recovery-threshold=3`. + The output is shown once; store the recovery keys and the root token with + their holders. Revoke the root token after the bootstrap; a new one is + generated with the recovery keys when needed. +5. Check: `vault operator raft list-peers` lists three voters; `docker restart` + of each node in turn unseals it without a person and the raft committed index + matches on all three. + +### Replacing a node + +Stop the node, remove it with `vault operator raft remove-peer `, +delete its volume and start it again: it joins through `retry_join` and copies the +data from the leader. A node keeps its node id and addresses; moving it to +another server is a change of its config file and of every `retry_join` list. + +### Platform Vault lost + +Running nodes keep working. To start a node without the platform Vault, start +any temporary Vault, enable `transit`, load the stored key backup with +`vault write transit/restore/sxchnge-vault-unseal backup=`, issue a token +with the policy above, and point `address` in the node configs at that Vault. +Without the key backup the data cannot be unsealed again. diff --git a/infra/prod/vault/config/vault-1.hcl b/infra/prod/vault/config/vault-1.hcl new file mode 100644 index 0000000..740aa71 --- /dev/null +++ b/infra/prod/vault/config/vault-1.hcl @@ -0,0 +1,30 @@ +# SXCHNGE production Vault, node 1. Only api_addr, cluster_addr and node_id +# differ between the three node files. +ui = false +disable_mlock = false +api_addr = "http://192.168.10.191:8200" +cluster_addr = "https://192.168.10.191:8201" + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + # Plain HTTP on the private network; raft and request forwarding on the + # cluster port are encrypted by Vault itself. + tls_disable = true +} + +storage "raft" { + path = "/vault/file" + node_id = "sxchnge-vault-1" + retry_join { leader_api_addr = "http://192.168.10.191:8200" } + retry_join { leader_api_addr = "http://192.168.10.191:8210" } + retry_join { leader_api_addr = "http://192.168.10.98:8200" } +} + +# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by +# start.sh from the secret file; recovery keys stay with people. +seal "transit" { + address = "https://vault.sendico.io" + mount_path = "transit/" + key_name = "sxchnge-vault-unseal" +} diff --git a/infra/prod/vault/config/vault-2.hcl b/infra/prod/vault/config/vault-2.hcl new file mode 100644 index 0000000..ba2f8b5 --- /dev/null +++ b/infra/prod/vault/config/vault-2.hcl @@ -0,0 +1,30 @@ +# SXCHNGE production Vault, node 2. Only api_addr, cluster_addr and node_id +# differ between the three node files. +ui = false +disable_mlock = false +api_addr = "http://192.168.10.191:8210" +cluster_addr = "https://192.168.10.191:8211" + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + # Plain HTTP on the private network; raft and request forwarding on the + # cluster port are encrypted by Vault itself. + tls_disable = true +} + +storage "raft" { + path = "/vault/file" + node_id = "sxchnge-vault-2" + retry_join { leader_api_addr = "http://192.168.10.191:8200" } + retry_join { leader_api_addr = "http://192.168.10.191:8210" } + retry_join { leader_api_addr = "http://192.168.10.98:8200" } +} + +# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by +# start.sh from the secret file; recovery keys stay with people. +seal "transit" { + address = "https://vault.sendico.io" + mount_path = "transit/" + key_name = "sxchnge-vault-unseal" +} diff --git a/infra/prod/vault/config/vault-3.hcl b/infra/prod/vault/config/vault-3.hcl new file mode 100644 index 0000000..7b5f2ae --- /dev/null +++ b/infra/prod/vault/config/vault-3.hcl @@ -0,0 +1,30 @@ +# SXCHNGE production Vault, node 3. Only api_addr, cluster_addr and node_id +# differ between the three node files. +ui = false +disable_mlock = false +api_addr = "http://192.168.10.98:8200" +cluster_addr = "https://192.168.10.98:8201" + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + # Plain HTTP on the private network; raft and request forwarding on the + # cluster port are encrypted by Vault itself. + tls_disable = true +} + +storage "raft" { + path = "/vault/file" + node_id = "sxchnge-vault-3" + retry_join { leader_api_addr = "http://192.168.10.191:8200" } + retry_join { leader_api_addr = "http://192.168.10.191:8210" } + retry_join { leader_api_addr = "http://192.168.10.98:8200" } +} + +# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by +# start.sh from the secret file; recovery keys stay with people. +seal "transit" { + address = "https://vault.sendico.io" + mount_path = "transit/" + key_name = "sxchnge-vault-unseal" +} diff --git a/infra/prod/vault/exch.yml b/infra/prod/vault/exch.yml new file mode 100644 index 0000000..4f24159 --- /dev/null +++ b/infra/prod/vault/exch.yml @@ -0,0 +1,59 @@ +# SXCHNGE production Vault on exch: nodes 1 and 2 of three (#74). Node 3 runs +# on infra-gamma (gamma.yml). Ports are published on the private address only. +name: sxchnge-vault + +x-vault: &vault + image: ${VAULT_IMAGE:?Run make prod-images and copy images.env} + restart: unless-stopped + cap_add: [IPC_LOCK] + ulimits: {memlock: {soft: -1, hard: -1}} + environment: + VAULT_ADDR: http://127.0.0.1:8200 + SKIP_SETCAP: 'true' + entrypoint: [/bin/sh, /etc/vault/start.sh] + secrets: [transit-token] + healthcheck: + test: [CMD-SHELL, 'vault status >/dev/null'] + interval: 10s + timeout: 5s + retries: 6 + +services: + vault-1: + <<: *vault + ports: + - 192.168.10.191:8200:8200 + - 192.168.10.191:8201:8201 + volumes: + - vault-1-data:/vault/file + - ./config/vault-1.hcl:/vault/config/vault.hcl:ro + - ./start.sh:/etc/vault/start.sh:ro + networks: + sxchnge: + aliases: [vault] + + vault-2: + <<: *vault + ports: + - 192.168.10.191:8210:8200 + - 192.168.10.191:8211:8201 + volumes: + - vault-2-data:/vault/file + - ./config/vault-2.hcl:/vault/config/vault.hcl:ro + - ./start.sh:/etc/vault/start.sh:ro + networks: + sxchnge: + aliases: [vault] + +secrets: + transit-token: + file: ./secrets/transit-token + +volumes: + vault-1-data: + vault-2-data: + +# Applications on exch join this network and use http://vault:8200. +networks: + sxchnge: + name: sxchnge diff --git a/infra/prod/vault/gamma.yml b/infra/prod/vault/gamma.yml new file mode 100644 index 0000000..031e45e --- /dev/null +++ b/infra/prod/vault/gamma.yml @@ -0,0 +1,34 @@ +# SXCHNGE production Vault on infra-gamma: node 3 of three (#74). It keeps a +# copy of the data off exch; nodes 1 and 2 run on exch (exch.yml). +name: sxchnge-vault + +services: + vault-3: + image: ${VAULT_IMAGE:?Run make prod-images and copy images.env} + restart: unless-stopped + cap_add: [IPC_LOCK] + ulimits: {memlock: {soft: -1, hard: -1}} + environment: + VAULT_ADDR: http://127.0.0.1:8200 + SKIP_SETCAP: 'true' + entrypoint: [/bin/sh, /etc/vault/start.sh] + secrets: [transit-token] + healthcheck: + test: [CMD-SHELL, 'vault status >/dev/null'] + interval: 10s + timeout: 5s + retries: 6 + ports: + - 192.168.10.98:8200:8200 + - 192.168.10.98:8201:8201 + volumes: + - vault-3-data:/vault/file + - ./config/vault-3.hcl:/vault/config/vault.hcl:ro + - ./start.sh:/etc/vault/start.sh:ro + +secrets: + transit-token: + file: ./secrets/transit-token + +volumes: + vault-3-data: diff --git a/infra/prod/vault/start.sh b/infra/prod/vault/start.sh new file mode 100644 index 0000000..d565233 --- /dev/null +++ b/infra/prod/vault/start.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Hands the transit seal token to the Vault server process only: it is read +# from the secret file here, so it is not part of the container environment. +set -eu +VAULT_TOKEN=$(cat /run/secrets/transit-token) +[ -n "$VAULT_TOKEN" ] || { echo 'Empty transit seal token' >&2; exit 1; } +export VAULT_TOKEN +exec docker-entrypoint.sh vault server -config=/vault/config/vault.hcl -- 2.54.0 From 2f74d3627e6ac4ce25fa8e2070e4a84971d8c486 Mon Sep 17 00:00:00 2001 From: protuberanets Date: Thu, 29 Oct 2026 20:07:22 +0300 Subject: [PATCH 2/3] Match the platform Vault on UI, mlock and the TLS flag The UI stays reachable on the private addresses only. Integrated raft storage runs with mlock disabled, as on the platform Vault and as Vault recommends for raft. --- infra/prod/vault/config/vault-1.hcl | 6 +++--- infra/prod/vault/config/vault-2.hcl | 6 +++--- infra/prod/vault/config/vault-3.hcl | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/infra/prod/vault/config/vault-1.hcl b/infra/prod/vault/config/vault-1.hcl index 740aa71..367fe96 100644 --- a/infra/prod/vault/config/vault-1.hcl +++ b/infra/prod/vault/config/vault-1.hcl @@ -1,7 +1,7 @@ # SXCHNGE production Vault, node 1. Only api_addr, cluster_addr and node_id # differ between the three node files. -ui = false -disable_mlock = false +ui = true +disable_mlock = true api_addr = "http://192.168.10.191:8200" cluster_addr = "https://192.168.10.191:8201" @@ -10,7 +10,7 @@ listener "tcp" { cluster_address = "0.0.0.0:8201" # Plain HTTP on the private network; raft and request forwarding on the # cluster port are encrypted by Vault itself. - tls_disable = true + tls_disable = 1 } storage "raft" { diff --git a/infra/prod/vault/config/vault-2.hcl b/infra/prod/vault/config/vault-2.hcl index ba2f8b5..6456d38 100644 --- a/infra/prod/vault/config/vault-2.hcl +++ b/infra/prod/vault/config/vault-2.hcl @@ -1,7 +1,7 @@ # SXCHNGE production Vault, node 2. Only api_addr, cluster_addr and node_id # differ between the three node files. -ui = false -disable_mlock = false +ui = true +disable_mlock = true api_addr = "http://192.168.10.191:8210" cluster_addr = "https://192.168.10.191:8211" @@ -10,7 +10,7 @@ listener "tcp" { cluster_address = "0.0.0.0:8201" # Plain HTTP on the private network; raft and request forwarding on the # cluster port are encrypted by Vault itself. - tls_disable = true + tls_disable = 1 } storage "raft" { diff --git a/infra/prod/vault/config/vault-3.hcl b/infra/prod/vault/config/vault-3.hcl index 7b5f2ae..f95cdab 100644 --- a/infra/prod/vault/config/vault-3.hcl +++ b/infra/prod/vault/config/vault-3.hcl @@ -1,7 +1,7 @@ # SXCHNGE production Vault, node 3. Only api_addr, cluster_addr and node_id # differ between the three node files. -ui = false -disable_mlock = false +ui = true +disable_mlock = true api_addr = "http://192.168.10.98:8200" cluster_addr = "https://192.168.10.98:8201" @@ -10,7 +10,7 @@ listener "tcp" { cluster_address = "0.0.0.0:8201" # Plain HTTP on the private network; raft and request forwarding on the # cluster port are encrypted by Vault itself. - tls_disable = true + tls_disable = 1 } storage "raft" { -- 2.54.0 From 1b9a89d9c4b7f907f3df479e4b6b88edc570d03c Mon Sep 17 00:00:00 2001 From: protuberanets Date: Thu, 29 Oct 2026 21:07:59 +0300 Subject: [PATCH 3/3] Keep the production Vault private and reach its UI through SSH --- infra/prod/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/prod/README.md b/infra/prod/README.md index 013880c..f6cb080 100644 --- a/infra/prod/README.md +++ b/infra/prod/README.md @@ -25,6 +25,10 @@ Containers on `exch` join the Docker network `sxchnge` and use `http://vault:8200`; the alias belongs to both local nodes. A standby node forwards requests to the active one, so the leader may be on any server. +Vault is not published to the internet. Its UI is opened through an SSH tunnel +from an administrator's machine, for example +`ssh -N -L 8200:192.168.10.191:8200 exch` and then `http://localhost:8200/ui`. + Nodes unseal themselves with the `transit` seal on the platform Vault (`https://vault.sendico.io`, key `transit/keys/sxchnge-vault-unseal`). Each server has its own periodic token in `secrets/transit-token` (root, mode 600, not in Git) -- 2.54.0