Merge pull request 'docs: require schema code generation at build time' (#77) from codex/docs-build-time-generation into main

AGENTS.md

@@ -13,9 +13,9 @@
 - Never commit autogenerated code to the repository. This applies to backend,
   frontend, shared packages, and every other module, including generated
   protobuf bindings, API clients, serializers, and framework-generated code.
-- Commit the source definitions, generator configuration, and reproducible
+- Commit the schema source definitions, generator configuration, and reproducible
-  generation commands instead. Generate code locally or during build and CI
+  generation commands instead. Generate code from the schema source at build
-  workflows before any dependent validation or compilation steps.
+  time, before any dependent validation or compilation steps, including in CI.
 - Keep generated outputs ignored by Git. Do not force-add ignored generated
   files or copy generated code into tracked source files.
 - Change the source definitions or generator inputs rather than editing

Makefile

@@ -80,12 +80,6 @@ test-dev:
 	sh infra/local/tools.sh test
 	sh infra/local/tools.sh verify
 
-# Production pins (infra/prod/README.md): resolved once, copied to every node.
-.PHONY: prod-images
-prod-images:
-	sh infra/local/tools.sh build
-	sh infra/local/tools.sh prod-images
-
 .PHONY: test-dev-infra
 test-dev-infra:
 	sh infra/local/tools.sh test

infra/local/scripts/dispatch.sh

@@ -5,7 +5,6 @@ set -o pipefail
 . infra/local/scripts/secrets.sh
 case "${1:-}" in
  images) . infra/local/scripts/images.sh; images;;
- prod-images) TOOL_STATE="$PWD/.local"; . infra/local/scripts/images.sh; prod_images;;
  up) . infra/local/scripts/bootstrap.sh; up;;
  email) . infra/local/scripts/bootstrap.sh; email;;
  sendico) . infra/local/scripts/bootstrap.sh; . infra/local/scripts/sendico.sh; configure_sendico;;

infra/local/scripts/images.sh

@@ -22,10 +22,3 @@ images() (
  cat "$WORK/images.env" | write "$STATE/images.env"
  cat "$STATE/images.env"
 )
-# Production pins: one resolution for every node of a cluster, so all of them
-# run the same build. Written outside .local/dev; copied to the servers.
-prod_images() (
- ref=$(image hashicorp/vault latest) || exit 1
- printf 'VAULT_IMAGE=%s\n' "$ref" | write "$PWD/.local/prod/images.env"
- cat "$PWD/.local/prod/images.env"
-)

infra/local/scripts/test.sh

@@ -3,7 +3,7 @@ set -eu
 set -o pipefail
 # Shell syntax and static analysis cover the actual container shell scripts.
 shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh
-shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh infra/prod/vault/start.sh
+shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh
 shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh infra/local/sendico/network.sh
 . infra/local/scripts/common.sh
 . infra/local/scripts/images.sh

infra/prod/README.md

@@ -1,97 +0,0 @@
-# Production
-
-Production runs on two servers of the shared private network: `exch`
-(192.168.10.191), the SXCHNGE server, and `infra-gamma` (192.168.10.98), which
-keeps the third copy of the data. Each component runs two nodes on `exch` and
-one on `infra-gamma`: losing `infra-gamma` keeps the majority, losing one node on
-`exch` keeps two of three. Every server runs plain Docker Compose; files are
-deployed to `/srv/sxchnge/<component>` on the server.
-
-Ports are published on the private addresses only. Traffic between the servers
-is not wrapped in TLS: the network is private and limited by security groups,
-and every component authenticates its peers.
-
-## Vault
-
-[#74](https://git.sendico.io/tech/sxchnge/issues/74). Three raft nodes:
-
-| Node | Server | API | Cluster | Compose |
-| --- | --- | --- | --- | --- |
-| `sxchnge-vault-1` | exch | 192.168.10.191:8200 | :8201 | `vault/exch.yml` |
-| `sxchnge-vault-2` | exch | 192.168.10.191:8210 | :8211 | `vault/exch.yml` |
-| `sxchnge-vault-3` | infra-gamma | 192.168.10.98:8200 | :8201 | `vault/gamma.yml` |
-
-Containers on `exch` join the Docker network `sxchnge` and use
-`http://vault:8200`; the alias belongs to both local nodes. A standby node
-forwards requests to the active one, so the leader may be on any server.
-
-Vault is not published to the internet. Its UI is opened through an SSH tunnel
-from an administrator's machine, for example
-`ssh -N -L 8200:192.168.10.191:8200 exch` and then `http://localhost:8200/ui`.
-
-Nodes unseal themselves with the `transit` seal on the platform Vault
-(`https://vault.sendico.io`, key `transit/keys/sxchnge-vault-unseal`). Each server
-has its own periodic token in `secrets/transit-token` (root, mode 600, not in Git)
-that may only encrypt and decrypt with that key. If the platform Vault is
-unreachable while a node starts, the node stays sealed until it is reachable
-again; running nodes are not affected. `vault operator init` returns recovery
-keys, not unseal keys: they authorize operations such as generating a new root
-token but cannot unseal a node. Only the transit key can. Its backup
-(`transit/backup/sxchnge-vault-unseal`), the recovery keys and the root token are
-kept outside the servers by their holders; see "Platform Vault lost".
-
-`make prod-images` resolves the latest stable Vault release and pins its digest in
-ignored `.local/prod/images.env`. All three nodes get the same file: a cluster
-runs one build, and an upgrade is a separate rollout of a new file, one node at a
-time.
-
-### First rollout
-
-1. On the platform Vault, with an administrator token:
-
-   ```sh
-   vault secrets list | grep -q '^transit/' || vault secrets enable transit
-   vault write transit/keys/sxchnge-vault-unseal exportable=true allow_plaintext_backup=true
-   vault read -field=backup transit/backup/sxchnge-vault-unseal   # store with the holders, then clear the terminal
-   vault policy write sxchnge-vault-unseal - <<'EOF'
-   path "transit/encrypt/sxchnge-vault-unseal" { capabilities = ["update"] }
-   path "transit/decrypt/sxchnge-vault-unseal" { capabilities = ["update"] }
-   EOF
-   ```
-
-   Create one token per server and write it straight into that server's
-   `/srv/sxchnge/vault/secrets/transit-token`, never to a terminal or a log:
-
-   ```sh
-   vault token create -orphan -period=768h -policy=sxchnge-vault-unseal \
-     -display-name=sxchnge-vault-<server> -field=token
-   ```
-
-2. Copy `vault/` and `.local/prod/images.env` to `/srv/sxchnge/vault` on both
-   servers.
-3. Start the nodes: on `exch`
-   `docker compose --env-file images.env -f exch.yml up -d`, on `infra-gamma`
-   the same with `gamma.yml`.
-4. Initialize once, on `exch`:
-   `docker exec -it sxchnge-vault-vault-1-1 vault operator init -recovery-shares=5 -recovery-threshold=3`.
-   The output is shown once; store the recovery keys and the root token with
-   their holders. Revoke the root token after the bootstrap; a new one is
-   generated with the recovery keys when needed.
-5. Check: `vault operator raft list-peers` lists three voters; `docker restart`
-   of each node in turn unseals it without a person and the raft committed index
-   matches on all three.
-
-### Replacing a node
-
-Stop the node, remove it with `vault operator raft remove-peer <node id>`,
-delete its volume and start it again: it joins through `retry_join` and copies the
-data from the leader. A node keeps its node id and addresses; moving it to
-another server is a change of its config file and of every `retry_join` list.
-
-### Platform Vault lost
-
-Running nodes keep working. To start a node without the platform Vault, start
-any temporary Vault, enable `transit`, load the stored key backup with
-`vault write transit/restore/sxchnge-vault-unseal backup=<backup>`, issue a token
-with the policy above, and point `address` in the node configs at that Vault.
-Without the key backup the data cannot be unsealed again.

infra/prod/vault/config/vault-1.hcl

@@ -1,30 +0,0 @@
-# SXCHNGE production Vault, node 1. Only api_addr, cluster_addr and node_id
-# differ between the three node files.
-ui            = true
-disable_mlock = true
-api_addr      = "http://192.168.10.191:8200"
-cluster_addr  = "https://192.168.10.191:8201"
-
-listener "tcp" {
-  address         = "0.0.0.0:8200"
-  cluster_address = "0.0.0.0:8201"
-  # Plain HTTP on the private network; raft and request forwarding on the
-  # cluster port are encrypted by Vault itself.
-  tls_disable     = 1
-}
-
-storage "raft" {
-  path    = "/vault/file"
-  node_id = "sxchnge-vault-1"
-  retry_join { leader_api_addr = "http://192.168.10.191:8200" }
-  retry_join { leader_api_addr = "http://192.168.10.191:8210" }
-  retry_join { leader_api_addr = "http://192.168.10.98:8200" }
-}
-
-# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by
-# start.sh from the secret file; recovery keys stay with people.
-seal "transit" {
-  address    = "https://vault.sendico.io"
-  mount_path = "transit/"
-  key_name   = "sxchnge-vault-unseal"
-}

infra/prod/vault/config/vault-2.hcl

@@ -1,30 +0,0 @@
-# SXCHNGE production Vault, node 2. Only api_addr, cluster_addr and node_id
-# differ between the three node files.
-ui            = true
-disable_mlock = true
-api_addr      = "http://192.168.10.191:8210"
-cluster_addr  = "https://192.168.10.191:8211"
-
-listener "tcp" {
-  address         = "0.0.0.0:8200"
-  cluster_address = "0.0.0.0:8201"
-  # Plain HTTP on the private network; raft and request forwarding on the
-  # cluster port are encrypted by Vault itself.
-  tls_disable     = 1
-}
-
-storage "raft" {
-  path    = "/vault/file"
-  node_id = "sxchnge-vault-2"
-  retry_join { leader_api_addr = "http://192.168.10.191:8200" }
-  retry_join { leader_api_addr = "http://192.168.10.191:8210" }
-  retry_join { leader_api_addr = "http://192.168.10.98:8200" }
-}
-
-# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by
-# start.sh from the secret file; recovery keys stay with people.
-seal "transit" {
-  address    = "https://vault.sendico.io"
-  mount_path = "transit/"
-  key_name   = "sxchnge-vault-unseal"
-}

infra/prod/vault/config/vault-3.hcl

@@ -1,30 +0,0 @@
-# SXCHNGE production Vault, node 3. Only api_addr, cluster_addr and node_id
-# differ between the three node files.
-ui            = true
-disable_mlock = true
-api_addr      = "http://192.168.10.98:8200"
-cluster_addr  = "https://192.168.10.98:8201"
-
-listener "tcp" {
-  address         = "0.0.0.0:8200"
-  cluster_address = "0.0.0.0:8201"
-  # Plain HTTP on the private network; raft and request forwarding on the
-  # cluster port are encrypted by Vault itself.
-  tls_disable     = 1
-}
-
-storage "raft" {
-  path    = "/vault/file"
-  node_id = "sxchnge-vault-3"
-  retry_join { leader_api_addr = "http://192.168.10.191:8200" }
-  retry_join { leader_api_addr = "http://192.168.10.191:8210" }
-  retry_join { leader_api_addr = "http://192.168.10.98:8200" }
-}
-
-# Auto-unseal with the platform Vault. The token comes from VAULT_TOKEN, set by
-# start.sh from the secret file; recovery keys stay with people.
-seal "transit" {
-  address    = "https://vault.sendico.io"
-  mount_path = "transit/"
-  key_name   = "sxchnge-vault-unseal"
-}

infra/prod/vault/exch.yml

@@ -1,59 +0,0 @@
-# SXCHNGE production Vault on exch: nodes 1 and 2 of three (#74). Node 3 runs
-# on infra-gamma (gamma.yml). Ports are published on the private address only.
-name: sxchnge-vault
-
-x-vault: &vault
-  image: ${VAULT_IMAGE:?Run make prod-images and copy images.env}
-  restart: unless-stopped
-  cap_add: [IPC_LOCK]
-  ulimits: {memlock: {soft: -1, hard: -1}}
-  environment:
-    VAULT_ADDR: http://127.0.0.1:8200
-    SKIP_SETCAP: 'true'
-  entrypoint: [/bin/sh, /etc/vault/start.sh]
-  secrets: [transit-token]
-  healthcheck:
-    test: [CMD-SHELL, 'vault status >/dev/null']
-    interval: 10s
-    timeout: 5s
-    retries: 6
-
-services:
-  vault-1:
-    <<: *vault
-    ports:
-      - 192.168.10.191:8200:8200
-      - 192.168.10.191:8201:8201
-    volumes:
-      - vault-1-data:/vault/file
-      - ./config/vault-1.hcl:/vault/config/vault.hcl:ro
-      - ./start.sh:/etc/vault/start.sh:ro
-    networks:
-      sxchnge:
-        aliases: [vault]
-
-  vault-2:
-    <<: *vault
-    ports:
-      - 192.168.10.191:8210:8200
-      - 192.168.10.191:8211:8201
-    volumes:
-      - vault-2-data:/vault/file
-      - ./config/vault-2.hcl:/vault/config/vault.hcl:ro
-      - ./start.sh:/etc/vault/start.sh:ro
-    networks:
-      sxchnge:
-        aliases: [vault]
-
-secrets:
-  transit-token:
-    file: ./secrets/transit-token
-
-volumes:
-  vault-1-data:
-  vault-2-data:
-
-# Applications on exch join this network and use http://vault:8200.
-networks:
-  sxchnge:
-    name: sxchnge

infra/prod/vault/gamma.yml

@@ -1,34 +0,0 @@
-# SXCHNGE production Vault on infra-gamma: node 3 of three (#74). It keeps a
-# copy of the data off exch; nodes 1 and 2 run on exch (exch.yml).
-name: sxchnge-vault
-
-services:
-  vault-3:
-    image: ${VAULT_IMAGE:?Run make prod-images and copy images.env}
-    restart: unless-stopped
-    cap_add: [IPC_LOCK]
-    ulimits: {memlock: {soft: -1, hard: -1}}
-    environment:
-      VAULT_ADDR: http://127.0.0.1:8200
-      SKIP_SETCAP: 'true'
-    entrypoint: [/bin/sh, /etc/vault/start.sh]
-    secrets: [transit-token]
-    healthcheck:
-      test: [CMD-SHELL, 'vault status >/dev/null']
-      interval: 10s
-      timeout: 5s
-      retries: 6
-    ports:
-      - 192.168.10.98:8200:8200
-      - 192.168.10.98:8201:8201
-    volumes:
-      - vault-3-data:/vault/file
-      - ./config/vault-3.hcl:/vault/config/vault.hcl:ro
-      - ./start.sh:/etc/vault/start.sh:ro
-
-secrets:
-  transit-token:
-    file: ./secrets/transit-token
-
-volumes:
-  vault-3-data:

infra/prod/vault/start.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-# Hands the transit seal token to the Vault server process only: it is read
-# from the secret file here, so it is not part of the container environment.
-set -eu
-VAULT_TOKEN=$(cat /run/secrets/transit-token)
-[ -n "$VAULT_TOKEN" ] || { echo 'Empty transit seal token' >&2; exit 1; }
-export VAULT_TOKEN
-exec docker-entrypoint.sh vault server -config=/vault/config/vault.hcl