first db deployment script

2025-11-07 00:59:08 +01:00
parent 2abe1a601d
commit 68707d5c62
16 changed files with 576 additions and 0 deletions
--- a/ci/prod/.env.runtime
+++ b/ci/prod/.env.runtime
@@ -0,0 +1,16 @@
+REGISTRY_URL=registry.sendico.io
+
+VAULT_ADDR=https://vault.sendico.io
+
+MONGO_PORT=27017
+MONGO_REPLICA_SET=sendico-rs
+MONGO_AUTH_SOURCE=admin
+
+PBM_S3_ENDPOINT=https://s3.sendico.io
+PBM_S3_REGION=eu-central-1
+PBM_S3_BUCKET=backup
+
+SSH_HOST=178.57.67.248
+SSH_USER=cloud
+REMOTE_BASE=/srv/sendico
+DB_DIR=db
--- a/ci/prod/compose/backup/config.yml
+++ b/ci/prod/compose/backup/config.yml
--- a/ci/prod/compose/db.yml
+++ b/ci/prod/compose/db.yml
@@ -0,0 +1,193 @@
+# Compose v2
+
+x-common-env: &common-env
+  env_file:
+    - ../env/.env.runtime
+    - ../env/.env.lock.db
+
+volumes:
+  mongo1_data: {}
+  mongo2_data: {}
+  mongo3_data: {}
+  vault_secrets:
+    driver: local
+    driver_opts: { type: tmpfs, device: tmpfs, o: size=32m,uid=999,gid=999,mode=0750 }
+  pbm_cfg:
+    driver: local
+    driver_opts: { type: tmpfs, device: tmpfs, o: size=16m,uid=0,gid=0,mode=0750 }
+
+services:
+  vault-agent-sendico:
+    <<: *common-env
+    image: hashicorp/vault:latest
+    container_name: vault-agent-sendico
+    restart: unless-stopped
+    cap_add: ["IPC_LOCK"]
+    environment:
+      VAULT_ADDR: ${VAULT_ADDR}
+    volumes:
+      - ./vault/agent.hcl:/etc/vault/agent.hcl:ro
+      - ./vault/templates:/etc/vault/templates:ro
+      - /opt/sendico/vault/sendico-db/role_id:/vault/role_id:ro
+      - /opt/sendico/vault/sendico-db/secret_id:/vault/secret_id:ro
+      - vault_secrets:/vault/secrets:rw
+      - pbm_cfg:/etc/backup:rw
+    command: sh -lc 'vault agent -config=/etc/vault/agent.hcl'
+    healthcheck:
+      test: ["CMD-SHELL","test -s /vault/secrets/MONGO_INITDB_ROOT_USERNAME -a -s /vault/secrets/MONGO_INITDB_ROOT_PASSWORD -a -s /vault/secrets/mongo.kf -a -s /etc/backup/pbm.env -a -s /etc/backup/.u -a -s /etc/backup/.p"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+      start_period: 5s
+
+  sendico_db1:
+    <<: *common-env
+    image: ${REGISTRY_URL}/mirror/mongo:${MONGO_TAG}@${MONGO_DIGEST}
+    container_name: sendico_db1
+    restart: unless-stopped
+    depends_on: { vault-agent-sendico: { condition: service_healthy } }
+    entrypoint: ["/usr/local/bin/mongo-entrypoint-wrapper.sh"]
+    command: >
+      mongod --replSet ${MONGO_REPLICA_SET} --bind_ip_all --auth
+             --keyFile /vault/secrets/mongo.kf --port ${MONGO_PORT}
+    volumes:
+      - mongo1_data:/data/db
+      - vault_secrets:/vault/secrets:ro
+      - ./ops/mongo-entrypoint.sh:/usr/local/bin/mongo-entrypoint-wrapper.sh:ro
+    healthcheck:
+      test: ["CMD-SHELL","mongosh --quiet --host localhost --port ${MONGO_PORT} --eval 'db.runCommand({ ping: 1 }).ok' || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    ports: [ "0.0.0.0:${MONGO_PORT}:${MONGO_PORT}" ]
+
+  sendico_db2:
+    <<: *common-env
+    image: ${REGISTRY_URL}/mirror/mongo:${MONGO_TAG}@${MONGO_DIGEST}
+    container_name: sendico_db2
+    restart: unless-stopped
+    depends_on: { vault-agent-sendico: { condition: service_healthy } }
+    entrypoint: ["/usr/local/bin/mongo-entrypoint-wrapper.sh"]
+    command: >
+      mongod --replSet ${MONGO_REPLICA_SET} --bind_ip_all --auth
+             --keyFile /vault/secrets/mongo.kf --port ${MONGO_PORT}
+    volumes:
+      - mongo2_data:/data/db
+      - vault_secrets:/vault/secrets:ro
+      - ./ops/mongo-entrypoint.sh:/usr/local/bin/mongo-entrypoint-wrapper.sh:ro
+    healthcheck:
+      test: ["CMD-SHELL","mongosh --quiet --host localhost --port ${MONGO_PORT} --eval 'db.runCommand({ ping: 1 }).ok' || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+
+  sendico_db3:
+    <<: *common-env
+    image: ${REGISTRY_URL}/mirror/mongo:${MONGO_TAG}@${MONGO_DIGEST}
+    container_name: sendico_db3
+    restart: unless-stopped
+    depends_on: { vault-agent-sendico: { condition: service_healthy } }
+    entrypoint: ["/usr/local/bin/mongo-entrypoint-wrapper.sh"]
+    command: >
+      mongod --replSet ${MONGO_REPLICA_SET} --bind_ip_all --auth
+             --keyFile /vault/secrets/mongo.kf --port ${MONGO_PORT}
+    volumes:
+      - mongo3_data:/data/db
+      - vault_secrets:/vault/secrets:ro
+      - ./ops/mongo-entrypoint.sh:/usr/local/bin/mongo-entrypoint-wrapper.sh:ro
+    healthcheck:
+      test: ["CMD-SHELL","mongosh --quiet --host localhost --port ${MONGO_PORT} --eval 'db.runCommand({ ping: 1 }).ok' || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+
+  mongo_setup:
+    <<: *common-env
+    image: ${REGISTRY_URL}/mirror/mongo:${MONGO_TAG}@${MONGO_DIGEST}
+    depends_on:
+      sendico_db1: { condition: service_healthy }
+      sendico_db2: { condition: service_healthy }
+      sendico_db3: { condition: service_healthy }
+    volumes:
+      - vault_secrets:/vault/secrets:ro
+    entrypoint: |
+      bash -c '
+        u=$(cat /vault/secrets/MONGO_INITDB_ROOT_USERNAME)
+        p=$(cat /vault/secrets/MONGO_INITDB_ROOT_PASSWORD)
+        until mongosh --quiet --host sendico_db1 --port ${MONGO_PORT} --eval "db.adminCommand({ ping: 1 })"; do
+          echo "waiting for MongoDB…"; sleep 2;
+        done
+        mongosh --host sendico_db1 --port ${MONGO_PORT} -u "$u" -p "$p" --authenticationDatabase admin <<EOJS
+          try { rs.status() } catch (e) {
+            rs.initiate({
+              _id: "${MONGO_REPLICA_SET}",
+              members: [
+                { _id: 0, host: "sendico_db1:${MONGO_PORT}", priority: 2 },
+                { _id: 1, host: "sendico_db2:${MONGO_PORT}", priority: 1 },
+                { _id: 2, host: "sendico_db3:${MONGO_PORT}", priority: 1 }
+              ]
+            })
+          }
+        EOJS
+      '
+    restart: "no"
+
+  pbm-agent-1:
+    <<: *common-env
+    image: percona/percona-backup-mongodb:latest
+    container_name: pbm-agent-1
+    restart: unless-stopped
+    depends_on:
+      sendico_db1: { condition: service_healthy }
+      vault-agent-sendico: { condition: service_healthy }
+    volumes:
+      - pbm_cfg:/etc/backup:ro
+    command: |
+      sh -lc '
+        . /etc/backup/pbm.env
+        U=$(cat /etc/backup/.u) ; P=$(cat /etc/backup/.p)
+        export AWS_EC2_METADATA_DISABLED=true
+        export PBM_MONGODB_URI="mongodb://${U}:${P}@sendico_db1:${MONGO_PORT}/?authSource=${MONGO_AUTH_SOURCE}&replicaSet=${MONGO_REPLICA_SET}"
+        exec pbm-agent --config=/etc/backup/pbm-config.yaml
+      '
+
+  pbm-agent-2:
+    <<: *common-env
+    image: percona/percona-backup-mongodb:latest
+    container_name: pbm-agent-2
+    restart: unless-stopped
+    depends_on:
+      sendico_db2: { condition: service_healthy }
+      vault-agent-sendico: { condition: service_healthy }
+    volumes:
+      - pbm_cfg:/etc/backup:ro
+    command: |
+      sh -lc '
+        . /etc/backup/pbm.env
+        U=$(cat /etc/backup/.u) ; P=$(cat /etc/backup/.p)
+        export AWS_EC2_METADATA_DISABLED=true
+        export PBM_MONGODB_URI="mongodb://${U}:${P}@sendico_db2:${MONGO_PORT}/?authSource=${MONGO_AUTH_SOURCE}&replicaSet=${MONGO_REPLICA_SET}"
+        exec pbm-agent --config=/etc/backup/pbm-config.yaml
+      '
+
+  pbm-agent-3:
+    <<: *common-env
+    image: percona/percona-backup-mongodb:latest
+    container_name: pbm-agent-3
+    restart: unless-stopped
+    depends_on:
+      sendico_db3: { condition: service_healthy }
+      vault-agent-sendico: { condition: service_healthy }
+    volumes:
+      - pbm_cfg:/etc/backup:ro
+    command: |
+      sh -lc '
+        . /etc/backup/pbm.env
+        U=$(cat /etc/backup/.u) ; P=$(cat /etc/backup/.p)
+        export AWS_EC2_METADATA_DISABLED=true
+        export PBM_MONGODB_URI="mongodb://${U}:${P}@sendico_db3:${MONGO_PORT}/?authSource=${MONGO_AUTH_SOURCE}&replicaSet=${MONGO_REPLICA_SET}"
+        exec pbm-agent --config=/etc/backup/pbm-config.yaml
+      '
--- a/ci/prod/compose/ops/mongo-entrypoint.sh
+++ b/ci/prod/compose/ops/mongo-entrypoint.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+wait_for_file() {
+  local path="$1" name="$2" retries="${3:-30}" interval="${4:-2}"
+  for _ in $(seq 1 "$retries"); do
+    if [ -s "$path" ]; then
+      return 0
+    fi
+    sleep "$interval"
+  done
+  echo "missing required secret: ${name} (${path})" >&2
+  exit 1
+}
+
+wait_for_file /vault/secrets/MONGO_INITDB_ROOT_USERNAME "root username"
+wait_for_file /vault/secrets/MONGO_INITDB_ROOT_PASSWORD "root password"
+wait_for_file /vault/secrets/mongo.kf "replica set keyFile"
+
+export MONGO_INITDB_ROOT_USERNAME="$(cat /vault/secrets/MONGO_INITDB_ROOT_USERNAME)"
+export MONGO_INITDB_ROOT_PASSWORD="$(cat /vault/secrets/MONGO_INITDB_ROOT_PASSWORD)"
+chown 999:999 /vault/secrets/mongo.kf
+chmod 0400 /vault/secrets/mongo.kf
+
+exec /usr/local/bin/docker-entrypoint.sh "$@"
--- a/ci/prod/compose/vault/agent.hcl
+++ b/ci/prod/compose/vault/agent.hcl
@@ -0,0 +1,50 @@
+# Vault Agent for DB stack. AppRole creds are files on the host.
+pid_file = "/tmp/vault-agent.pid"
+
+auto_auth {
+  method "approle" {
+    mount_path = "auth/approle"
+    config = {
+      role_id_file_path   = "/vault/role_id"
+      secret_id_file_path = "/vault/secret_id"
+    }
+  }
+  sink "file" { config = { path = "/vault/token" } }
+}
+
+vault { address = "{{ env `VAULT_ADDR` }}" }
+
+# Mongo root credentials
+template {
+  source      = "/etc/vault/templates/mongo/user.ctmpl"
+  destination = "/vault/secrets/MONGO_INITDB_ROOT_USERNAME"
+}
+template {
+  source      = "/etc/vault/templates/mongo/pass.ctmpl"
+  destination = "/vault/secrets/MONGO_INITDB_ROOT_PASSWORD"
+}
+
+# Replica set keyFile (strict perms)
+template {
+  source      = "/etc/vault/templates/mongo/keyfile.ctmpl"
+  destination = "/vault/secrets/mongo.kf"
+  command     = "sh -lc 'chown 999:999 /vault/secrets/mongo.kf && chmod 0400 /vault/secrets/mongo.kf'"
+}
+
+# PBM: backup user/pass + S3 creds env
+template {
+  source      = "/etc/vault/templates/backup/user.ctmpl"
+  destination = "/etc/backup/.u"
+}
+template {
+  source      = "/etc/vault/templates/backup/pass.ctmpl"
+  destination = "/etc/backup/.p"
+}
+template {
+  source      = "/etc/vault/templates/pbm/env.ctmpl"
+  destination = "/etc/backup/pbm.env"
+}
+template {
+  source      = "/etc/vault/templates/pbm/config.ctmpl"
+  destination = "/etc/backup/pbm-config.yaml"
+}
--- a/ci/prod/compose/vault/templates/backup/pass.ctmpl
+++ b/ci/prod/compose/vault/templates/backup/pass.ctmpl
@@ -0,0 +1,3 @@
+{{ with secret "kv/data/ops/db/backup" -}}
+{{ .Data.data.user }}
+{{- end }}
--- a/ci/prod/compose/vault/templates/backup/user.ctmpl
+++ b/ci/prod/compose/vault/templates/backup/user.ctmpl
@@ -0,0 +1,3 @@
+{{ with secret "kv/data/ops/db/backup" -}}
+{{ .Data.data.user }}
+{{- end }}
--- a/ci/prod/compose/vault/templates/mongo/keyfile.ctmpl
+++ b/ci/prod/compose/vault/templates/mongo/keyfile.ctmpl
@@ -0,0 +1,3 @@
+{{ with secret "kv/data/sendico/db" -}}
+{{ .Data.data.key }}
+{{- end }}
--- a/ci/prod/compose/vault/templates/mongo/pass.ctmpl
+++ b/ci/prod/compose/vault/templates/mongo/pass.ctmpl
@@ -0,0 +1,3 @@
+{{ with secret "kv/data/sendico/db" -}}
+{{ .Data.data.password }}
+{{- end }}
--- a/ci/prod/compose/vault/templates/mongo/user.ctmpl
+++ b/ci/prod/compose/vault/templates/mongo/user.ctmpl
@@ -0,0 +1,3 @@
+{{ with secret "kv/data/sendico/db" -}}
+{{ .Data.data.user }}
+{{- end }}
--- a/ci/prod/compose/vault/templates/pbm/config.ctmpl
+++ b/ci/prod/compose/vault/templates/pbm/config.ctmpl
@@ -0,0 +1,16 @@
+# Rendered by Vault Agent; contains no secrets.
+storage:
+  type: s3
+  s3:
+    endpointUrl: "{{ env "PBM_S3_ENDPOINT" }}"
+    region: "{{ env "PBM_S3_REGION" }}"
+    bucket: "{{ env "PBM_S3_BUCKET" }}"
+    forcePathStyle: true
+
+pitr:
+  enabled: true
+  oplogSpanMin: 10
+  compression: "s2"
+
+backup:
+  compression: "s2"
--- a/ci/prod/compose/vault/templates/pbm/env.ctmpl
+++ b/ci/prod/compose/vault/templates/pbm/env.ctmpl
@@ -0,0 +1,3 @@
+# Rendered by Vault Agent. Contains only secrets.
+AWS_ACCESS_KEY_ID={{ with secret "kv/data/s3/backup" -}}{{ .Data.data.access_key_id }}{{- end }}
+AWS_SECRET_ACCESS_KEY={{ with secret "kv/data/s3/backup" -}}{{ .Data.data.secret_access_key }}{{- end }}
--- a/ci/vlt
+++ b/ci/vlt
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Minimal Vault helper for CI steps (AppRole login + KVv2 reads).
+# Requires: curl, sed. Uses VAULT_ADDR, VAULT_ROLE_ID, VAULT_SECRET_ID from env.
+set -euo pipefail
+
+: "${VAULT_ADDR:?missing VAULT_ADDR}"
+VAULT_TOKEN_FILE="${VAULT_TOKEN_FILE:-.vault_token}"
+
+log(){ printf '[vlt] %s\n' "$*" >&2; }
+
+login() {
+  : "${VAULT_ROLE_ID:?missing VAULT_ROLE_ID}"
+  : "${VAULT_SECRET_ID:?missing VAULT_SECRET_ID}"
+  log "login approle"
+  resp="$(curl -sfS -X POST -H 'Content-Type: application/json' \
+    -d "{\"role_id\":\"${VAULT_ROLE_ID}\",\"secret_id\":\"${VAULT_SECRET_ID}\"}" \
+    "${VAULT_ADDR%/}/v1/auth/approle/login")"
+  token="$(printf '%s' "$resp" | sed -n 's/.*"client_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')"
+  [ -n "$token" ] || { echo "login failed" >&2; exit 1; }
+  printf '%s' "$token" > "$VAULT_TOKEN_FILE"
+}
+
+ensure_token() {
+  if [ -s "$VAULT_TOKEN_FILE" ]; then
+    VAULT_TOKEN="$(cat "$VAULT_TOKEN_FILE")"
+  else
+    login
+    VAULT_TOKEN="$(cat "$VAULT_TOKEN_FILE")"
+  fi
+}
+
+# kv_get <mount> <path> <field>  (KV v2)
+kv_get() {
+  mount="$1"; path="$2"; field="$3"
+  ensure_token
+  url="${VAULT_ADDR%/}/v1/${mount}/data/${path}"
+  resp="$(curl -sfS -H "X-Vault-Token: ${VAULT_TOKEN}" "$url")"
+  raw="$(printf '%s' "$resp" | sed -n "s/.*\"${field}\"[[:space:]]*:[[:space:]]*\"\([^\"]*\)\".*/\1/p")"
+  [ -n "$raw" ] || { echo "field not found: ${mount}/${path}:${field}" >&2; exit 2; }
+  printf '%s' "$raw" | sed -e 's/\\n/\n/g' -e 's/\\t/\t/g' -e 's/\\"/"/g' -e 's/\\\\/\\/g'
+}
+
+# kv_to_file <mount> <path> <field> <dest> [mode]
+kv_to_file() {
+  mount="$1"; path="$2"; field="$3"; dest="$4"; mode="${5:-600}"
+  tmp="$(mktemp)"
+  kv_get "$mount" "$path" "$field" > "$tmp"
+  install -m "$mode" "$tmp" "$dest"
+  rm -f "$tmp"
+  log "wrote $dest"
+}
+
+case "${1:-}" in
+  login) shift; login "$@";;
+  kv_get) shift; kv_get "$@";;
+  kv_to_file) shift; kv_to_file "$@";;
+  *) echo "usage: vlt {login|kv_get|kv_to_file} ..." >&2; exit 64;;
+esac