11 changed files with 299 additions and 1 deletions
@@ -80,6 +80,12 @@ test-dev:
 	sh infra/local/tools.sh test
 	sh infra/local/tools.sh verify

+# Production pins (infra/prod/README.md): resolved once, copied to every node.
+.PHONY: prod-images
+prod-images:
+	sh infra/local/tools.sh build
+	sh infra/local/tools.sh prod-images
+
 .PHONY: test-dev-infra
 test-dev-infra:
 	sh infra/local/tools.sh test
@@ -5,6 +5,7 @@ set -o pipefail
 . infra/local/scripts/secrets.sh
 case "${1:-}" in
 images) . infra/local/scripts/images.sh; images;;
+ prod-images) TOOL_STATE="$PWD/.local"; . infra/local/scripts/images.sh; prod_images;;
 up) . infra/local/scripts/bootstrap.sh; up;;
 email) . infra/local/scripts/bootstrap.sh; email;;
 sendico) . infra/local/scripts/bootstrap.sh; . infra/local/scripts/sendico.sh; configure_sendico;;
@@ -22,3 +22,10 @@ images() (
 cat "$WORK/images.env" | write "$STATE/images.env"
 cat "$STATE/images.env"
 )
+# Production pins: one resolution for every node of a cluster, so all of them
+# run the same build. Written outside .local/dev; copied to the servers.
+prod_images() (
+ ref=$(image hashicorp/vault latest) || exit 1
+ printf 'VAULT_IMAGE=%s\n' "$ref" | write "$PWD/.local/prod/images.env"
+ cat "$PWD/.local/prod/images.env"
+)
@@ -3,7 +3,7 @@ set -eu
 set -o pipefail
 # Shell syntax and static analysis cover the actual container shell scripts.
 shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh
-shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh
+shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh infra/prod/vault/start.sh
 shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh infra/local/sendico/network.sh
 . infra/local/scripts/common.sh
 . infra/local/scripts/images.sh
@@ -0,0 +1,93 @@
+# Production
+
+Production runs on two servers of the shared private network: `exch`
+(192.168.10.191), the SXCHNGE server, and `infra-gamma` (192.168.10.98), which
+keeps the third copy of the data. Each component runs two nodes on `exch` and
+one on `infra-gamma`: losing `infra-gamma` keeps the majority, losing one node on
+`exch` keeps two of three. Every server runs plain Docker Compose; files are
+deployed to `/srv/sxchnge/<component>` on the server.
+
+Ports are published on the private addresses only. Traffic between the servers
+is not wrapped in TLS: the network is private and limited by security groups,
+and every component authenticates its peers.
+
+## Vault
+
+[#74](https://git.sendico.io/tech/sxchnge/issues/74). Three raft nodes:
+
+| Node | Server | API | Cluster | Compose |
+| --- | --- | --- | --- | --- |
+| `sxchnge-vault-1` | exch | 192.168.10.191:8200 | :8201 | `vault/exch.yml` |
+| `sxchnge-vault-2` | exch | 192.168.10.191:8210 | :8211 | `vault/exch.yml` |
+| `sxchnge-vault-3` | infra-gamma | 192.168.10.98:8200 | :8201 | `vault/gamma.yml` |
+
+Containers on `exch` join the Docker network `sxchnge` and use
+`http://vault:8200`; the alias belongs to both local nodes. A standby node
+forwards requests to the active one, so the leader may be on any server.
+
+Nodes unseal themselves with the `transit` seal on the platform Vault
+(`https://vault.sendico.io`, key `transit/keys/sxchnge-vault-unseal`). Each server
+has its own periodic token in `secrets/transit-token` (root, mode 600, not in Git)
+that may only encrypt and decrypt with that key. If the platform Vault is
+unreachable while a node starts, the node stays sealed until it is reachable
+again; running nodes are not affected. `vault operator init` returns recovery
+keys, not unseal keys: they authorize operations such as generating a new root
+token but cannot unseal a node. Only the transit key can. Its backup
+(`transit/backup/sxchnge-vault-unseal`), the recovery keys and the root token are
+kept outside the servers by their holders; see "Platform Vault lost".
+
+`make prod-images` resolves the latest stable Vault release and pins its digest in
+ignored `.local/prod/images.env`. All three nodes get the same file: a cluster
+runs one build, and an upgrade is a separate rollout of a new file, one node at a
+time.
+
+### First rollout
+
+1. On the platform Vault, with an administrator token:
+
+   ```sh
+   vault secrets list | grep -q '^transit/' || vault secrets enable transit
+   vault write transit/keys/sxchnge-vault-unseal exportable=true allow_plaintext_backup=true
+   vault read -field=backup transit/backup/sxchnge-vault-unseal   # store with the holders, then clear the terminal
+   vault policy write sxchnge-vault-unseal - <<'EOF'
+   path "transit/encrypt/sxchnge-vault-unseal" { capabilities = ["update"] }
+   path "transit/decrypt/sxchnge-vault-unseal" { capabilities = ["update"] }
+   EOF
+   ```
+
+   Create one token per server and write it straight into that server's
+   `/srv/sxchnge/vault/secrets/transit-token`, never to a terminal or a log:
+
+   ```sh
+   vault token create -orphan -period=768h -policy=sxchnge-vault-unseal \
+     -display-name=sxchnge-vault-<server> -field=token
+   ```
+
+2. Copy `vault/` and `.local/prod/images.env` to `/srv/sxchnge/vault` on both
+   servers.
+3. Start the nodes: on `exch`
+   `docker compose --env-file images.env -f exch.yml up -d`, on `infra-gamma`
+   the same with `gamma.yml`.
+4. Initialize once, on `exch`:
+   `docker exec -it sxchnge-vault-vault-1-1 vault operator init -recovery-shares=5 -recovery-threshold=3`.
+   The output is shown once; store the recovery keys and the root token with
+   their holders. Revoke the root token after the bootstrap; a new one is
+   generated with the recovery keys when needed.
+5. Check: `vault operator raft list-peers` lists three voters; `docker restart`
+   of each node in turn unseals it without a person and the raft committed index
+   matches on all three.
+
+### Replacing a node
+
+Stop the node, remove it with `vault operator raft remove-peer <node id>`,
+delete its volume and start it again: it joins through `retry_join` and copies the
+data from the leader. A node keeps its node id and addresses; moving it to
+another server is a change of its config file and of every `retry_join` list.
+
+### Platform Vault lost
+
+Running nodes keep working. To start a node without the platform Vault, start
+any temporary Vault, enable `transit`, load the stored key backup with
+`vault write transit/restore/sxchnge-vault-unseal backup=<backup>`, issue a token
+with the policy above, and point `address` in the node configs at that Vault.
+Without the key backup the data cannot be unsealed again.
@@ -0,0 +1,30 @@
+# SXCHNGE production Vault, node 1. Only api_addr, cluster_addr and node_id
+# differ between the three node files.
+ui            = false
+disable_mlock = false
+api_addr      = "http://192.168.10.191:8200"
+cluster_addr  = "https://192.168.10.191:8201"
+
+listener "tcp" {
+  address         = "0.0.0.0:8200"
+  cluster_address = "0.0.0.0:8201"
+  # Plain HTTP on the private network; raft and request forwarding on the
+  # cluster port are encrypted by Vault itself.
+  tls_disable = true
+}
+
+storage "raft" {
+  path    = "/vault/file"
+  node_id = "sxchnge-vault-1"
+  retry_join { leader_api_addr = "http://192.168.10.191:8200" }
+  retry_join { leader_api_addr = "http://192.168.10.191:8210" }
+  retry_join { leader_api_addr = "http://192.168.10.98:8200" }
+}
+
+# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by
+# start.sh from the secret file; recovery keys stay with people.
+seal "transit" {
+  address    = "https://vault.sendico.io"
+  mount_path = "transit/"
+  key_name   = "sxchnge-vault-unseal"
+}
@@ -0,0 +1,30 @@
+# SXCHNGE production Vault, node 2. Only api_addr, cluster_addr and node_id
+# differ between the three node files.
+ui            = false
+disable_mlock = false
+api_addr      = "http://192.168.10.191:8210"
+cluster_addr  = "https://192.168.10.191:8211"
+
+listener "tcp" {
+  address         = "0.0.0.0:8200"
+  cluster_address = "0.0.0.0:8201"
+  # Plain HTTP on the private network; raft and request forwarding on the
+  # cluster port are encrypted by Vault itself.
+  tls_disable = true
+}
+
+storage "raft" {
+  path    = "/vault/file"
+  node_id = "sxchnge-vault-2"
+  retry_join { leader_api_addr = "http://192.168.10.191:8200" }
+  retry_join { leader_api_addr = "http://192.168.10.191:8210" }
+  retry_join { leader_api_addr = "http://192.168.10.98:8200" }
+}
+
+# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by
+# start.sh from the secret file; recovery keys stay with people.
+seal "transit" {
+  address    = "https://vault.sendico.io"
+  mount_path = "transit/"
+  key_name   = "sxchnge-vault-unseal"
+}
@@ -0,0 +1,30 @@
+# SXCHNGE production Vault, node 3. Only api_addr, cluster_addr and node_id
+# differ between the three node files.
+ui            = false
+disable_mlock = false
+api_addr      = "http://192.168.10.98:8200"
+cluster_addr  = "https://192.168.10.98:8201"
+
+listener "tcp" {
+  address         = "0.0.0.0:8200"
+  cluster_address = "0.0.0.0:8201"
+  # Plain HTTP on the private network; raft and request forwarding on the
+  # cluster port are encrypted by Vault itself.
+  tls_disable = true
+}
+
+storage "raft" {
+  path    = "/vault/file"
+  node_id = "sxchnge-vault-3"
+  retry_join { leader_api_addr = "http://192.168.10.191:8200" }
+  retry_join { leader_api_addr = "http://192.168.10.191:8210" }
+  retry_join { leader_api_addr = "http://192.168.10.98:8200" }
+}
+
+# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by
+# start.sh from the secret file; recovery keys stay with people.
+seal "transit" {
+  address    = "https://vault.sendico.io"
+  mount_path = "transit/"
+  key_name   = "sxchnge-vault-unseal"
+}
@@ -0,0 +1,59 @@
+# SXCHNGE production Vault on exch: nodes 1 and 2 of three (#74). Node 3 runs
+# on infra-gamma (gamma.yml). Ports are published on the private address only.
+name: sxchnge-vault
+
+x-vault: &vault
+  image: ${VAULT_IMAGE:?Run make prod-images and copy images.env}
+  restart: unless-stopped
+  cap_add: [IPC_LOCK]
+  ulimits: {memlock: {soft: -1, hard: -1}}
+  environment:
+    VAULT_ADDR: http://127.0.0.1:8200
+    SKIP_SETCAP: 'true'
+  entrypoint: [/bin/sh, /etc/vault/start.sh]
+  secrets: [transit-token]
+  healthcheck:
+    test: [CMD-SHELL, 'vault status >/dev/null']
+    interval: 10s
+    timeout: 5s
+    retries: 6
+
+services:
+  vault-1:
+    <<: *vault
+    ports:
+      - 192.168.10.191:8200:8200
+      - 192.168.10.191:8201:8201
+    volumes:
+      - vault-1-data:/vault/file
+      - ./config/vault-1.hcl:/vault/config/vault.hcl:ro
+      - ./start.sh:/etc/vault/start.sh:ro
+    networks:
+      sxchnge:
+        aliases: [vault]
+
+  vault-2:
+    <<: *vault
+    ports:
+      - 192.168.10.191:8210:8200
+      - 192.168.10.191:8211:8201
+    volumes:
+      - vault-2-data:/vault/file
+      - ./config/vault-2.hcl:/vault/config/vault.hcl:ro
+      - ./start.sh:/etc/vault/start.sh:ro
+    networks:
+      sxchnge:
+        aliases: [vault]
+
+secrets:
+  transit-token:
+    file: ./secrets/transit-token
+
+volumes:
+  vault-1-data:
+  vault-2-data:
+
+# Applications on exch join this network and use http://vault:8200.
+networks:
+  sxchnge:
+    name: sxchnge
@@ -0,0 +1,34 @@
+# SXCHNGE production Vault on infra-gamma: node 3 of three (#74). It keeps a
+# copy of the data off exch; nodes 1 and 2 run on exch (exch.yml).
+name: sxchnge-vault
+
+services:
+  vault-3:
+    image: ${VAULT_IMAGE:?Run make prod-images and copy images.env}
+    restart: unless-stopped
+    cap_add: [IPC_LOCK]
+    ulimits: {memlock: {soft: -1, hard: -1}}
+    environment:
+      VAULT_ADDR: http://127.0.0.1:8200
+      SKIP_SETCAP: 'true'
+    entrypoint: [/bin/sh, /etc/vault/start.sh]
+    secrets: [transit-token]
+    healthcheck:
+      test: [CMD-SHELL, 'vault status >/dev/null']
+      interval: 10s
+      timeout: 5s
+      retries: 6
+    ports:
+      - 192.168.10.98:8200:8200
+      - 192.168.10.98:8201:8201
+    volumes:
+      - vault-3-data:/vault/file
+      - ./config/vault-3.hcl:/vault/config/vault.hcl:ro
+      - ./start.sh:/etc/vault/start.sh:ro
+
+secrets:
+  transit-token:
+    file: ./secrets/transit-token
+
+volumes:
+  vault-3-data:
@@ -0,0 +1,8 @@
+#!/bin/sh
+# Hands the transit seal token to the Vault server process only: it is read
+# from the secret file here, so it is not part of the container environment.
+set -eu
+VAULT_TOKEN=$(cat /run/secrets/transit-token)
+[ -n "$VAULT_TOKEN" ] || { echo 'Empty transit seal token' >&2; exit 1; }
+export VAULT_TOKEN
+exec docker-entrypoint.sh vault server -config=/vault/config/vault.hcl