From f436b2528cb46de1b433e715a65fc7a8c7284448 Mon Sep 17 00:00:00 2001 From: protuberanets Date: Thu, 29 Oct 2026 20:00:33 +0300 Subject: [PATCH] Run the production Vault as three raft nodes on exch and infra-gamma Two nodes on exch and one on infra-gamma, plain Compose per server, ports on the private addresses only. Nodes unseal through the transit seal on the platform Vault; the transit token reaches only the server process. make prod-images resolves the latest stable Vault release once and pins its digest in ignored .local/prod/images.env, so every node runs the same build. infra/prod/README.md records the layout, the first rollout, node replacement and recovery when the platform Vault is lost (#74). --- Makefile | 6 ++ infra/local/scripts/dispatch.sh | 1 + infra/local/scripts/images.sh | 7 +++ infra/local/scripts/test.sh | 2 +- infra/prod/README.md | 93 +++++++++++++++++++++++++++++ infra/prod/vault/config/vault-1.hcl | 30 ++++++++++ infra/prod/vault/config/vault-2.hcl | 30 ++++++++++ infra/prod/vault/config/vault-3.hcl | 30 ++++++++++ infra/prod/vault/exch.yml | 59 ++++++++++++++++++ infra/prod/vault/gamma.yml | 34 +++++++++++ infra/prod/vault/start.sh | 8 +++ 11 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 infra/prod/README.md create mode 100644 infra/prod/vault/config/vault-1.hcl create mode 100644 infra/prod/vault/config/vault-2.hcl create mode 100644 infra/prod/vault/config/vault-3.hcl create mode 100644 infra/prod/vault/exch.yml create mode 100644 infra/prod/vault/gamma.yml create mode 100644 infra/prod/vault/start.sh diff --git a/Makefile b/Makefile index f5aa3fb..5a23a63 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,12 @@ test-dev: sh infra/local/tools.sh test sh infra/local/tools.sh verify +# Production pins (infra/prod/README.md): resolved once, copied to every node. +.PHONY: prod-images +prod-images: + sh infra/local/tools.sh build + sh infra/local/tools.sh prod-images + .PHONY: test-dev-infra test-dev-infra: sh infra/local/tools.sh test diff --git a/infra/local/scripts/dispatch.sh b/infra/local/scripts/dispatch.sh index 32d20cb..4f202b7 100644 --- a/infra/local/scripts/dispatch.sh +++ b/infra/local/scripts/dispatch.sh @@ -5,6 +5,7 @@ set -o pipefail . infra/local/scripts/secrets.sh case "${1:-}" in images) . infra/local/scripts/images.sh; images;; + prod-images) TOOL_STATE="$PWD/.local"; . infra/local/scripts/images.sh; prod_images;; up) . infra/local/scripts/bootstrap.sh; up;; email) . infra/local/scripts/bootstrap.sh; email;; sendico) . infra/local/scripts/bootstrap.sh; . infra/local/scripts/sendico.sh; configure_sendico;; diff --git a/infra/local/scripts/images.sh b/infra/local/scripts/images.sh index 7137117..85fd348 100644 --- a/infra/local/scripts/images.sh +++ b/infra/local/scripts/images.sh @@ -22,3 +22,10 @@ images() ( cat "$WORK/images.env" | write "$STATE/images.env" cat "$STATE/images.env" ) +# Production pins: one resolution for every node of a cluster, so all of them +# run the same build. Written outside .local/dev; copied to the servers. +prod_images() ( + ref=$(image hashicorp/vault latest) || exit 1 + printf 'VAULT_IMAGE=%s\n' "$ref" | write "$PWD/.local/prod/images.env" + cat "$PWD/.local/prod/images.env" +) diff --git a/infra/local/scripts/test.sh b/infra/local/scripts/test.sh index b026f9d..bca7001 100644 --- a/infra/local/scripts/test.sh +++ b/infra/local/scripts/test.sh @@ -3,7 +3,7 @@ set -eu set -o pipefail # Shell syntax and static analysis cover the actual container shell scripts. shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh -shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh +shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh infra/prod/vault/start.sh shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh infra/local/sendico/network.sh . infra/local/scripts/common.sh . infra/local/scripts/images.sh diff --git a/infra/prod/README.md b/infra/prod/README.md new file mode 100644 index 0000000..013880c --- /dev/null +++ b/infra/prod/README.md @@ -0,0 +1,93 @@ +# Production + +Production runs on two servers of the shared private network: `exch` +(192.168.10.191), the SXCHNGE server, and `infra-gamma` (192.168.10.98), which +keeps the third copy of the data. Each component runs two nodes on `exch` and +one on `infra-gamma`: losing `infra-gamma` keeps the majority, losing one node on +`exch` keeps two of three. Every server runs plain Docker Compose; files are +deployed to `/srv/sxchnge/` on the server. + +Ports are published on the private addresses only. Traffic between the servers +is not wrapped in TLS: the network is private and limited by security groups, +and every component authenticates its peers. + +## Vault + +[#74](https://git.sendico.io/tech/sxchnge/issues/74). Three raft nodes: + +| Node | Server | API | Cluster | Compose | +| --- | --- | --- | --- | --- | +| `sxchnge-vault-1` | exch | 192.168.10.191:8200 | :8201 | `vault/exch.yml` | +| `sxchnge-vault-2` | exch | 192.168.10.191:8210 | :8211 | `vault/exch.yml` | +| `sxchnge-vault-3` | infra-gamma | 192.168.10.98:8200 | :8201 | `vault/gamma.yml` | + +Containers on `exch` join the Docker network `sxchnge` and use +`http://vault:8200`; the alias belongs to both local nodes. A standby node +forwards requests to the active one, so the leader may be on any server. + +Nodes unseal themselves with the `transit` seal on the platform Vault +(`https://vault.sendico.io`, key `transit/keys/sxchnge-vault-unseal`). Each server +has its own periodic token in `secrets/transit-token` (root, mode 600, not in Git) +that may only encrypt and decrypt with that key. If the platform Vault is +unreachable while a node starts, the node stays sealed until it is reachable +again; running nodes are not affected. `vault operator init` returns recovery +keys, not unseal keys: they authorize operations such as generating a new root +token but cannot unseal a node. Only the transit key can. Its backup +(`transit/backup/sxchnge-vault-unseal`), the recovery keys and the root token are +kept outside the servers by their holders; see "Platform Vault lost". + +`make prod-images` resolves the latest stable Vault release and pins its digest in +ignored `.local/prod/images.env`. All three nodes get the same file: a cluster +runs one build, and an upgrade is a separate rollout of a new file, one node at a +time. + +### First rollout + +1. On the platform Vault, with an administrator token: + + ```sh + vault secrets list | grep -q '^transit/' || vault secrets enable transit + vault write transit/keys/sxchnge-vault-unseal exportable=true allow_plaintext_backup=true + vault read -field=backup transit/backup/sxchnge-vault-unseal # store with the holders, then clear the terminal + vault policy write sxchnge-vault-unseal - <<'EOF' + path "transit/encrypt/sxchnge-vault-unseal" { capabilities = ["update"] } + path "transit/decrypt/sxchnge-vault-unseal" { capabilities = ["update"] } + EOF + ``` + + Create one token per server and write it straight into that server's + `/srv/sxchnge/vault/secrets/transit-token`, never to a terminal or a log: + + ```sh + vault token create -orphan -period=768h -policy=sxchnge-vault-unseal \ + -display-name=sxchnge-vault- -field=token + ``` + +2. Copy `vault/` and `.local/prod/images.env` to `/srv/sxchnge/vault` on both + servers. +3. Start the nodes: on `exch` + `docker compose --env-file images.env -f exch.yml up -d`, on `infra-gamma` + the same with `gamma.yml`. +4. Initialize once, on `exch`: + `docker exec -it sxchnge-vault-vault-1-1 vault operator init -recovery-shares=5 -recovery-threshold=3`. + The output is shown once; store the recovery keys and the root token with + their holders. Revoke the root token after the bootstrap; a new one is + generated with the recovery keys when needed. +5. Check: `vault operator raft list-peers` lists three voters; `docker restart` + of each node in turn unseals it without a person and the raft committed index + matches on all three. + +### Replacing a node + +Stop the node, remove it with `vault operator raft remove-peer `, +delete its volume and start it again: it joins through `retry_join` and copies the +data from the leader. A node keeps its node id and addresses; moving it to +another server is a change of its config file and of every `retry_join` list. + +### Platform Vault lost + +Running nodes keep working. To start a node without the platform Vault, start +any temporary Vault, enable `transit`, load the stored key backup with +`vault write transit/restore/sxchnge-vault-unseal backup=`, issue a token +with the policy above, and point `address` in the node configs at that Vault. +Without the key backup the data cannot be unsealed again. diff --git a/infra/prod/vault/config/vault-1.hcl b/infra/prod/vault/config/vault-1.hcl new file mode 100644 index 0000000..740aa71 --- /dev/null +++ b/infra/prod/vault/config/vault-1.hcl @@ -0,0 +1,30 @@ +# SXCHNGE production Vault, node 1. Only api_addr, cluster_addr and node_id +# differ between the three node files. +ui = false +disable_mlock = false +api_addr = "http://192.168.10.191:8200" +cluster_addr = "https://192.168.10.191:8201" + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + # Plain HTTP on the private network; raft and request forwarding on the + # cluster port are encrypted by Vault itself. + tls_disable = true +} + +storage "raft" { + path = "/vault/file" + node_id = "sxchnge-vault-1" + retry_join { leader_api_addr = "http://192.168.10.191:8200" } + retry_join { leader_api_addr = "http://192.168.10.191:8210" } + retry_join { leader_api_addr = "http://192.168.10.98:8200" } +} + +# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by +# start.sh from the secret file; recovery keys stay with people. +seal "transit" { + address = "https://vault.sendico.io" + mount_path = "transit/" + key_name = "sxchnge-vault-unseal" +} diff --git a/infra/prod/vault/config/vault-2.hcl b/infra/prod/vault/config/vault-2.hcl new file mode 100644 index 0000000..ba2f8b5 --- /dev/null +++ b/infra/prod/vault/config/vault-2.hcl @@ -0,0 +1,30 @@ +# SXCHNGE production Vault, node 2. Only api_addr, cluster_addr and node_id +# differ between the three node files. +ui = false +disable_mlock = false +api_addr = "http://192.168.10.191:8210" +cluster_addr = "https://192.168.10.191:8211" + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + # Plain HTTP on the private network; raft and request forwarding on the + # cluster port are encrypted by Vault itself. + tls_disable = true +} + +storage "raft" { + path = "/vault/file" + node_id = "sxchnge-vault-2" + retry_join { leader_api_addr = "http://192.168.10.191:8200" } + retry_join { leader_api_addr = "http://192.168.10.191:8210" } + retry_join { leader_api_addr = "http://192.168.10.98:8200" } +} + +# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by +# start.sh from the secret file; recovery keys stay with people. +seal "transit" { + address = "https://vault.sendico.io" + mount_path = "transit/" + key_name = "sxchnge-vault-unseal" +} diff --git a/infra/prod/vault/config/vault-3.hcl b/infra/prod/vault/config/vault-3.hcl new file mode 100644 index 0000000..7b5f2ae --- /dev/null +++ b/infra/prod/vault/config/vault-3.hcl @@ -0,0 +1,30 @@ +# SXCHNGE production Vault, node 3. Only api_addr, cluster_addr and node_id +# differ between the three node files. +ui = false +disable_mlock = false +api_addr = "http://192.168.10.98:8200" +cluster_addr = "https://192.168.10.98:8201" + +listener "tcp" { + address = "0.0.0.0:8200" + cluster_address = "0.0.0.0:8201" + # Plain HTTP on the private network; raft and request forwarding on the + # cluster port are encrypted by Vault itself. + tls_disable = true +} + +storage "raft" { + path = "/vault/file" + node_id = "sxchnge-vault-3" + retry_join { leader_api_addr = "http://192.168.10.191:8200" } + retry_join { leader_api_addr = "http://192.168.10.191:8210" } + retry_join { leader_api_addr = "http://192.168.10.98:8200" } +} + +# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by +# start.sh from the secret file; recovery keys stay with people. +seal "transit" { + address = "https://vault.sendico.io" + mount_path = "transit/" + key_name = "sxchnge-vault-unseal" +} diff --git a/infra/prod/vault/exch.yml b/infra/prod/vault/exch.yml new file mode 100644 index 0000000..4f24159 --- /dev/null +++ b/infra/prod/vault/exch.yml @@ -0,0 +1,59 @@ +# SXCHNGE production Vault on exch: nodes 1 and 2 of three (#74). Node 3 runs +# on infra-gamma (gamma.yml). Ports are published on the private address only. +name: sxchnge-vault + +x-vault: &vault + image: ${VAULT_IMAGE:?Run make prod-images and copy images.env} + restart: unless-stopped + cap_add: [IPC_LOCK] + ulimits: {memlock: {soft: -1, hard: -1}} + environment: + VAULT_ADDR: http://127.0.0.1:8200 + SKIP_SETCAP: 'true' + entrypoint: [/bin/sh, /etc/vault/start.sh] + secrets: [transit-token] + healthcheck: + test: [CMD-SHELL, 'vault status >/dev/null'] + interval: 10s + timeout: 5s + retries: 6 + +services: + vault-1: + <<: *vault + ports: + - 192.168.10.191:8200:8200 + - 192.168.10.191:8201:8201 + volumes: + - vault-1-data:/vault/file + - ./config/vault-1.hcl:/vault/config/vault.hcl:ro + - ./start.sh:/etc/vault/start.sh:ro + networks: + sxchnge: + aliases: [vault] + + vault-2: + <<: *vault + ports: + - 192.168.10.191:8210:8200 + - 192.168.10.191:8211:8201 + volumes: + - vault-2-data:/vault/file + - ./config/vault-2.hcl:/vault/config/vault.hcl:ro + - ./start.sh:/etc/vault/start.sh:ro + networks: + sxchnge: + aliases: [vault] + +secrets: + transit-token: + file: ./secrets/transit-token + +volumes: + vault-1-data: + vault-2-data: + +# Applications on exch join this network and use http://vault:8200. +networks: + sxchnge: + name: sxchnge diff --git a/infra/prod/vault/gamma.yml b/infra/prod/vault/gamma.yml new file mode 100644 index 0000000..031e45e --- /dev/null +++ b/infra/prod/vault/gamma.yml @@ -0,0 +1,34 @@ +# SXCHNGE production Vault on infra-gamma: node 3 of three (#74). It keeps a +# copy of the data off exch; nodes 1 and 2 run on exch (exch.yml). +name: sxchnge-vault + +services: + vault-3: + image: ${VAULT_IMAGE:?Run make prod-images and copy images.env} + restart: unless-stopped + cap_add: [IPC_LOCK] + ulimits: {memlock: {soft: -1, hard: -1}} + environment: + VAULT_ADDR: http://127.0.0.1:8200 + SKIP_SETCAP: 'true' + entrypoint: [/bin/sh, /etc/vault/start.sh] + secrets: [transit-token] + healthcheck: + test: [CMD-SHELL, 'vault status >/dev/null'] + interval: 10s + timeout: 5s + retries: 6 + ports: + - 192.168.10.98:8200:8200 + - 192.168.10.98:8201:8201 + volumes: + - vault-3-data:/vault/file + - ./config/vault-3.hcl:/vault/config/vault.hcl:ro + - ./start.sh:/etc/vault/start.sh:ro + +secrets: + transit-token: + file: ./secrets/transit-token + +volumes: + vault-3-data: diff --git a/infra/prod/vault/start.sh b/infra/prod/vault/start.sh new file mode 100644 index 0000000..d565233 --- /dev/null +++ b/infra/prod/vault/start.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Hands the transit seal token to the Vault server process only: it is read +# from the secret file here, so it is not part of the container environment. +set -eu +VAULT_TOKEN=$(cat /run/secrets/transit-token) +[ -n "$VAULT_TOKEN" ] || { echo 'Empty transit seal token' >&2; exit 1; } +export VAULT_TOKEN +exec docker-entrypoint.sh vault server -config=/vault/config/vault.hcl