+ fx

2025-11-07 20:13:41 +01:00
parent 62a6631b9a
commit ec3019c462
11 changed files with 496 additions and 4 deletions
--- a/.woodpecker/db.yml
+++ b/.woodpecker/db.yml
@@ -18,7 +18,7 @@ steps:
    depends_on: [ version ]
    environment:
      # CI's own AppRole creds for accessing Vault to fetch the SSH key (existing names)
-      VAULT_ADDR: https://vault.sendico.io
+      VAULT_ADDR: { from_secret: VAULT_ADDR }
      VAULT_ROLE_ID:   { from_secret: VAULT_APP_ROLE }
      VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID }
    commands:
--- a/.woodpecker/fx_ingestor.yml
+++ b/.woodpecker/fx_ingestor.yml
@@ -0,0 +1,100 @@
+when:
+  - event: push
+    branch: main
+
+steps:
+  - name: version
+    image: alpine:latest
+    commands:
+      - set -euo pipefail 2>/dev/null || set -eu
+      - apk add --no-cache git
+      - GIT_REV="$(git rev-parse --short HEAD)"
+      - BUILD_BRANCH="$(git rev-parse --abbrev-ref HEAD)"
+      - APP_V="$(cat version)"
+      - BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+      - BUILD_USER="${WOODPECKER_MACHINE:-woodpecker}"
+      - printf "GIT_REV=%s\nBUILD_BRANCH=%s\nAPP_V=%s\nBUILD_DATE=%s\nBUILD_USER=%s\n" \
+          "$GIT_REV" "$BUILD_BRANCH" "$APP_V" "$BUILD_DATE" "$BUILD_USER" | tee .env.version
+
+  - name: secrets
+    image: alpine:latest
+    depends_on: [ version ]
+    environment:
+      VAULT_ADDR: { from_secret: VAULT_ADDR }
+      VAULT_ROLE_ID:   { from_secret: VAULT_APP_ROLE }
+      VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID }
+    commands:
+      - set -euo pipefail
+      - apk add --no-cache bash coreutils openssh-keygen curl sed python3
+      - mkdir -p secrets
+      - ./ci/vlt kv_to_file kv ops/deploy/ssh_key private_b64 secrets/SSH_KEY.b64 600
+      - base64 -d secrets/SSH_KEY.b64 > secrets/SSH_KEY
+      - chmod 600 secrets/SSH_KEY
+      - ssh-keygen -y -f secrets/SSH_KEY >/dev/null
+      - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER
+      - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD
+
+  - name: build-image
+    image: gcr.io/kaniko-project/executor:debug
+    depends_on: [ secrets ]
+    commands:
+      - set -euo pipefail 2>/dev/null || set -eu
+      - sed -i 's/\r$//' ./ci/prod/.env.runtime
+      - sed -i 's/\r$//' ./.env.version
+      - set -a
+      - . ./ci/prod/.env.runtime
+      - . ./.env.version
+      - set +a
+      - FX_GO_VERSION="${FX_GO_VERSION:-1.22}"
+      - ": \"${REGISTRY_URL:?missing REGISTRY_URL}\""
+      - ": \"${APP_V:?missing APP_V}\""
+      - REGISTRY_HOST="${REGISTRY_URL#http://}"
+      - REGISTRY_HOST="${REGISTRY_HOST#https://}"
+      - REGISTRY_USER="$(cat secrets/REGISTRY_USER)"
+      - REGISTRY_PASSWORD="$(cat secrets/REGISTRY_PASSWORD)"
+      - ": \"${REGISTRY_USER:?missing registry user}\""
+      - ": \"${REGISTRY_PASSWORD:?missing registry password}\""
+      - mkdir -p /kaniko/.docker
+      - AUTH_B64="$(printf '%s:%s' "$REGISTRY_USER" "$REGISTRY_PASSWORD" | base64 | tr -d '\n')"
+      - |
+        cat <<EOF >/kaniko/.docker/config.json
+        {
+          "auths": {
+            "https://${REGISTRY_HOST}": { "auth": "${AUTH_B64}" }
+          }
+        }
+        EOF
+      - |
+        /kaniko/executor \
+          --context "${PWD}" \
+          --dockerfile ci/prod/compose/fx_ingestor.dockerfile \
+          --destination "${REGISTRY_URL}/fx/ingestor:${APP_V}" \
+          --build-arg APP_VERSION="${APP_V}" \
+          --build-arg GIT_REV="${GIT_REV}" \
+          --build-arg BUILD_BRANCH="${BUILD_BRANCH}" \
+          --build-arg BUILD_DATE="${BUILD_DATE}" \
+          --build-arg BUILD_USER="${BUILD_USER}" \
+          --build-arg GO_VERSION="${FX_GO_VERSION}" \
+          --single-snapshot
+
+  - name: deploy
+    image: alpine:latest
+    depends_on: [ secrets, build-image ]
+    environment:
+      VAULT_ADDR: { from_secret: VAULT_ADDR }
+      VAULT_ROLE_ID:   { from_secret: VAULT_APP_ROLE }
+      VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID }
+    commands:
+      - set -euo pipefail
+      - apk add --no-cache bash openssh-client rsync coreutils curl sed python3
+      - mkdir -p /root/.ssh
+      - install -m 600 secrets/SSH_KEY /root/.ssh/id_rsa
+      - sed -i 's/\r$//' ./ci/prod/.env.runtime
+      - set -a
+      - . ./ci/prod/.env.runtime
+      - . ./.env.version
+      - set +a
+      - export FX_MONGO_USER="$(./ci/vlt kv_get kv sendico/db user)"
+      - export FX_MONGO_PASSWORD="$(./ci/vlt kv_get kv sendico/db password)"
+      - bash ci/prod/scripts/bootstrap/network.sh
+      - bash ci/prod/scripts/deploy/fx.sh ingestor
--- a/.woodpecker/fx_oracle.yml
+++ b/.woodpecker/fx_oracle.yml
@@ -0,0 +1,103 @@
+when:
+  - event: push
+    branch: main
+
+steps:
+  - name: version
+    image: alpine:latest
+    commands:
+      - set -euo pipefail 2>/dev/null || set -eu
+      - apk add --no-cache git
+      - GIT_REV="$(git rev-parse --short HEAD)"
+      - BUILD_BRANCH="$(git rev-parse --abbrev-ref HEAD)"
+      - APP_V="$(cat version)"
+      - BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+      - BUILD_USER="${WOODPECKER_MACHINE:-woodpecker}"
+      - printf "GIT_REV=%s\nBUILD_BRANCH=%s\nAPP_V=%s\nBUILD_DATE=%s\nBUILD_USER=%s\n" \
+          "$GIT_REV" "$BUILD_BRANCH" "$APP_V" "$BUILD_DATE" "$BUILD_USER" | tee .env.version
+
+  - name: secrets
+    image: alpine:latest
+    depends_on: [ version ]
+    environment:
+      VAULT_ADDR: { from_secret: VAULT_ADDR }
+      VAULT_ROLE_ID:   { from_secret: VAULT_APP_ROLE }
+      VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID }
+    commands:
+      - set -euo pipefail
+      - apk add --no-cache bash coreutils openssh-keygen curl sed python3
+      - mkdir -p secrets
+      - ./ci/vlt kv_to_file kv ops/deploy/ssh_key private_b64 secrets/SSH_KEY.b64 600
+      - base64 -d secrets/SSH_KEY.b64 > secrets/SSH_KEY
+      - chmod 600 secrets/SSH_KEY
+      - ssh-keygen -y -f secrets/SSH_KEY >/dev/null
+      - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER
+      - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD
+
+  - name: build-image
+    image: gcr.io/kaniko-project/executor:debug
+    depends_on: [ secrets ]
+    commands:
+      - set -euo pipefail 2>/dev/null || set -eu
+      - sed -i 's/\r$//' ./ci/prod/.env.runtime
+      - sed -i 's/\r$//' ./.env.version
+      - set -a
+      - . ./ci/prod/.env.runtime
+      - . ./.env.version
+      - set +a
+      - FX_GO_VERSION="${FX_GO_VERSION:-1.22}"
+      - ": \"${REGISTRY_URL:?missing REGISTRY_URL}\""
+      - ": \"${APP_V:?missing APP_V}\""
+      - REGISTRY_HOST="${REGISTRY_URL#http://}"
+      - REGISTRY_HOST="${REGISTRY_HOST#https://}"
+      - REGISTRY_USER="$(cat secrets/REGISTRY_USER)"
+      - REGISTRY_PASSWORD="$(cat secrets/REGISTRY_PASSWORD)"
+      - ": \"${REGISTRY_USER:?missing registry user}\""
+      - ": \"${REGISTRY_PASSWORD:?missing registry password}\""
+      - mkdir -p /kaniko/.docker
+      - AUTH_B64="$(printf '%s:%s' "$REGISTRY_USER" "$REGISTRY_PASSWORD" | base64 | tr -d '\n')"
+      - |
+        cat <<EOF >/kaniko/.docker/config.json
+        {
+          "auths": {
+            "https://${REGISTRY_HOST}": { "auth": "${AUTH_B64}" }
+          }
+        }
+        EOF
+      - |
+        /kaniko/executor \
+          --context "${PWD}" \
+          --dockerfile ci/prod/compose/fx_oracle.dockerfile \
+          --destination "${REGISTRY_URL}/fx/oracle:${APP_V}" \
+          --build-arg APP_VERSION="${APP_V}" \
+          --build-arg GIT_REV="${GIT_REV}" \
+          --build-arg BUILD_BRANCH="${BUILD_BRANCH}" \
+          --build-arg BUILD_DATE="${BUILD_DATE}" \
+          --build-arg BUILD_USER="${BUILD_USER}" \
+          --build-arg GO_VERSION="${FX_GO_VERSION}" \
+          --single-snapshot
+
+  - name: deploy
+    image: alpine:latest
+    depends_on: [ secrets, build-image ]
+    environment:
+      VAULT_ADDR: { from_secret: VAULT_ADDR }
+      VAULT_ROLE_ID:   { from_secret: VAULT_APP_ROLE }
+      VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID }
+    commands:
+      - set -euo pipefail
+      - apk add --no-cache bash openssh-client rsync coreutils curl sed python3
+      - mkdir -p /root/.ssh
+      - install -m 600 secrets/SSH_KEY /root/.ssh/id_rsa
+      - sed -i 's/\r$//' ./ci/prod/.env.runtime
+      - set -a
+      - . ./ci/prod/.env.runtime
+      - . ./.env.version
+      - set +a
+      - export FX_MONGO_USER="$(./ci/vlt kv_get kv sendico/db user)"
+      - export FX_MONGO_PASSWORD="$(./ci/vlt kv_get kv sendico/db password)"
+      - export NATS_USER="$(./ci/vlt kv_get kv sendico/nats user)"
+      - export NATS_PASSWORD="$(./ci/vlt kv_get kv sendico/nats password)"
+      - export FX_NATS_URL="nats://${NATS_USER}:${NATS_PASSWORD}@${NATS_HOST}:${NATS_PORT}"
+      - bash ci/prod/scripts/bootstrap/network.sh
+      - bash ci/prod/scripts/deploy/fx.sh oracle
--- a/.woodpecker/nats.yml
+++ b/.woodpecker/nats.yml
@@ -17,7 +17,7 @@ steps:
    image: alpine:latest
    depends_on: [ version ]
    environment:
-      VAULT_ADDR: https://vault.sendico.io
+      VAULT_ADDR: { from_secret: VAULT_ADDR }
      VAULT_ROLE_ID:   { from_secret: VAULT_APP_ROLE }
      VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID }
    commands:
@@ -33,7 +33,7 @@ steps:
    image: alpine:latest
    depends_on: [ secrets ]
    environment:
-      VAULT_ADDR: https://vault.sendico.io
+      VAULT_ADDR: { from_secret: VAULT_ADDR }
      VAULT_ROLE_ID:   { from_secret: VAULT_APP_ROLE }
      VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID }
    commands:
--- a/ci/prod/.env.runtime
+++ b/ci/prod/.env.runtime
@@ -24,3 +24,26 @@ NATS_PORT=4222
 NATS_MONITORING_PORT=8222
 NATS_PROMETHEUS_PORT=7777
 NATS_COMPOSE_PROJECT=sendico-nats
+
+# FX deployments
+FX_GO_VERSION=latest
+
+# Shared Mongo settings for FX services
+FX_MONGO_HOST=sendico_db1
+FX_MONGO_PORT=27017
+FX_MONGO_DATABASE=fx
+FX_MONGO_AUTH_SOURCE=admin
+FX_MONGO_REPLICA_SET=sendico-rs
+
+# FX ingestor stack
+FX_INGESTOR_DIR=fx_ingestor
+FX_INGESTOR_COMPOSE_PROJECT=sendico-fx-ingestor
+FX_INGESTOR_SERVICE_NAME=sendico_fx_ingestor
+FX_INGESTOR_METRICS_PORT=9102
+
+# FX oracle stack
+FX_ORACLE_DIR=fx_oracle
+FX_ORACLE_COMPOSE_PROJECT=sendico-fx-oracle
+FX_ORACLE_SERVICE_NAME=sendico_fx_oracle
+FX_ORACLE_GRPC_PORT=50051
+FX_ORACLE_METRICS_PORT=9400
--- a/ci/prod/compose/fx_ingestor.dockerfile
+++ b/ci/prod/compose/fx_ingestor.dockerfile
@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1.7
+
+ARG TARGETOS=linux
+ARG TARGETARCH=amd64
+
+FROM golang:alpine AS build
+ARG APP_VERSION=dev
+ARG GIT_REV=unknown
+ARG BUILD_BRANCH=unknown
+ARG BUILD_DATE=unknown
+ARG BUILD_USER=ci
+WORKDIR /src
+COPY . .
+WORKDIR /src/api/fx/ingestor
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
+    go build -trimpath -ldflags "\
+      -s -w \
+      -X github.com/tech/sendico/fx/ingestor/internal/appversion.Version=${APP_VERSION} \
+      -X github.com/tech/sendico/fx/ingestor/internal/appversion.Revision=${GIT_REV} \
+      -X github.com/tech/sendico/fx/ingestor/internal/appversion.Branch=${BUILD_BRANCH} \
+      -X github.com/tech/sendico/fx/ingestor/internal/appversion.BuildUser=${BUILD_USER} \
+      -X github.com/tech/sendico/fx/ingestor/internal/appversion.BuildDate=${BUILD_DATE}" \
+    -o /out/fx-ingestor .
+
+FROM alpine:latest AS runtime
+RUN apk add --no-cache ca-certificates tzdata wget
+WORKDIR /app
+COPY api/fx/ingestor/config.yml /app/config.yml
+COPY --from=build /out/fx-ingestor /app/fx-ingestor
+EXPOSE 9102
+ENTRYPOINT ["/app/fx-ingestor"]
+CMD ["--config.file", "/app/config.yml"]
--- a/ci/prod/compose/fx_ingestor.yml
+++ b/ci/prod/compose/fx_ingestor.yml
@@ -0,0 +1,39 @@
+# Compose v2 - FX ingestor
+
+x-common-env: &common-env
+  env_file:
+    - ../env/.env.runtime
+    - ../env/.env.version
+
+networks:
+  sendico-net:
+    external: true
+    name: sendico-net
+
+services:
+  sendico_fx_ingestor:
+    <<: *common-env
+    container_name: sendico-fx-ingestor
+    restart: unless-stopped
+    image: ${REGISTRY_URL}/fx/ingestor:${APP_V}
+    pull_policy: always
+    environment:
+      FX_MONGO_HOST: ${FX_MONGO_HOST}
+      FX_MONGO_PORT: ${FX_MONGO_PORT}
+      FX_MONGO_DATABASE: ${FX_MONGO_DATABASE}
+      FX_MONGO_USER: ${FX_MONGO_USER}
+      FX_MONGO_PASSWORD: ${FX_MONGO_PASSWORD}
+      FX_MONGO_AUTH_SOURCE: ${FX_MONGO_AUTH_SOURCE}
+      FX_MONGO_REPLICA_SET: ${FX_MONGO_REPLICA_SET}
+      FX_INGESTOR_METRICS_PORT: ${FX_INGESTOR_METRICS_PORT}
+    command: ["--config.file", "/app/config.yml"]
+    ports:
+      - "0.0.0.0:${FX_INGESTOR_METRICS_PORT}:${FX_INGESTOR_METRICS_PORT}"
+    healthcheck:
+      test: ["CMD-SHELL","wget -qO- http://localhost:${FX_INGESTOR_METRICS_PORT}/health | grep -q '\"status\":\"ok\"'"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s
+    networks:
+      - sendico-net
--- a/ci/prod/compose/fx_oracle.dockerfile
+++ b/ci/prod/compose/fx_oracle.dockerfile
@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1.7
+
+ARG TARGETOS=linux
+ARG TARGETARCH=amd64
+
+FROM golang:alpine AS build
+ARG APP_VERSION=dev
+ARG GIT_REV=unknown
+ARG BUILD_BRANCH=unknown
+ARG BUILD_DATE=unknown
+ARG BUILD_USER=ci
+WORKDIR /src
+COPY . .
+WORKDIR /src/api/fx/oracle
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
+    go build -trimpath -ldflags "\
+      -s -w \
+      -X github.com/tech/sendico/fx/oracle/internal/appversion.Version=${APP_VERSION} \
+      -X github.com/tech/sendico/fx/oracle/internal/appversion.Revision=${GIT_REV} \
+      -X github.com/tech/sendico/fx/oracle/internal/appversion.Branch=${BUILD_BRANCH} \
+      -X github.com/tech/sendico/fx/oracle/internal/appversion.BuildUser=${BUILD_USER} \
+      -X github.com/tech/sendico/fx/oracle/internal/appversion.BuildDate=${BUILD_DATE}" \
+    -o /out/fx-oracle .
+
+FROM alpine:latest AS runtime
+RUN apk add --no-cache ca-certificates tzdata wget
+WORKDIR /app
+COPY api/fx/oracle/config.yml /app/config.yml
+COPY --from=build /out/fx-oracle /app/fx-oracle
+EXPOSE 50051 9400
+ENTRYPOINT ["/app/fx-oracle"]
+CMD ["--config.file", "/app/config.yml"]
--- a/ci/prod/compose/fx_oracle.yml
+++ b/ci/prod/compose/fx_oracle.yml
@@ -0,0 +1,46 @@
+# Compose v2 - FX oracle
+
+x-common-env: &common-env
+  env_file:
+    - ../env/.env.runtime
+    - ../env/.env.version
+
+networks:
+  sendico-net:
+    external: true
+    name: sendico-net
+
+services:
+  sendico_fx_oracle:
+    <<: *common-env
+    container_name: sendico-fx-oracle
+    restart: unless-stopped
+    image: ${REGISTRY_URL}/fx/oracle:${APP_V}
+    pull_policy: always
+    environment:
+      FX_MONGO_HOST: ${FX_MONGO_HOST}
+      FX_MONGO_PORT: ${FX_MONGO_PORT}
+      FX_MONGO_DATABASE: ${FX_MONGO_DATABASE}
+      FX_MONGO_USER: ${FX_MONGO_USER}
+      FX_MONGO_PASSWORD: ${FX_MONGO_PASSWORD}
+      FX_MONGO_AUTH_SOURCE: ${FX_MONGO_AUTH_SOURCE}
+      FX_MONGO_REPLICA_SET: ${FX_MONGO_REPLICA_SET}
+      FX_ORACLE_GRPC_PORT: ${FX_ORACLE_GRPC_PORT}
+      FX_ORACLE_METRICS_PORT: ${FX_ORACLE_METRICS_PORT}
+      NATS_URL: ${FX_NATS_URL}
+      NATS_HOST: ${NATS_HOST}
+      NATS_PORT: ${NATS_PORT}
+      NATS_USER: ${NATS_USER}
+      NATS_PASSWORD: ${NATS_PASSWORD}
+    command: ["--config.file", "/app/config.yml"]
+    ports:
+      - "0.0.0.0:${FX_ORACLE_GRPC_PORT}:50051"
+      - "0.0.0.0:${FX_ORACLE_METRICS_PORT}:${FX_ORACLE_METRICS_PORT}"
+    healthcheck:
+      test: ["CMD-SHELL","wget -qO- http://localhost:${FX_ORACLE_METRICS_PORT}/health | grep -q '\"status\":\"ok\"'"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s
+    networks:
+      - sendico-net
--- a/ci/prod/compose/nats.dockerfile
+++ b/ci/prod/compose/nats.dockerfile
@@ -1,4 +1,4 @@
-FROM docker.io/library/busybox:1.36.1-musl AS busybox
+FROM docker.io/library/busybox:stable-musl AS busybox

 FROM docker.io/library/nats:latest
 COPY --from=busybox /bin/busybox /bin/busybox
--- a/ci/prod/scripts/deploy/fx.sh
+++ b/ci/prod/scripts/deploy/fx.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+set -euo pipefail
+[[ "${DEBUG_DEPLOY:-0}" = "1" ]] && set -x
+trap 'echo "[deploy-fx] error at line $LINENO" >&2' ERR
+
+if [[ $# -ne 1 ]]; then
+  echo "usage: $(basename "$0") <ingestor|oracle>" >&2
+  exit 64
+fi
+
+SERVICE="$1"
+
+: "${REMOTE_BASE:?missing REMOTE_BASE}"
+: "${SSH_USER:?missing SSH_USER}"
+: "${SSH_HOST:?missing SSH_HOST}"
+
+REMOTE_TARGET="${SSH_USER}@${SSH_HOST}"
+
+case "$SERVICE" in
+  ingestor)
+    : "${FX_INGESTOR_DIR:?missing FX_INGESTOR_DIR}"
+    REMOTE_DIR="${REMOTE_BASE%/}/${FX_INGESTOR_DIR}"
+    COMPOSE_PROJECT="${FX_INGESTOR_COMPOSE_PROJECT:-sendico-fx-ingestor}"
+    COMPOSE_FILE="fx_ingestor.yml"
+    SERVICE_NAMES="${FX_INGESTOR_SERVICE_NAME:-sendico_fx_ingestor}"
+    REQUIRED_SECRETS=(FX_MONGO_USER FX_MONGO_PASSWORD)
+    ;;
+  oracle)
+    : "${FX_ORACLE_DIR:?missing FX_ORACLE_DIR}"
+    REMOTE_DIR="${REMOTE_BASE%/}/${FX_ORACLE_DIR}"
+    COMPOSE_PROJECT="${FX_ORACLE_COMPOSE_PROJECT:-sendico-fx-oracle}"
+    COMPOSE_FILE="fx_oracle.yml"
+    SERVICE_NAMES="${FX_ORACLE_SERVICE_NAME:-sendico_fx_oracle}"
+    REQUIRED_SECRETS=(FX_MONGO_USER FX_MONGO_PASSWORD NATS_USER NATS_PASSWORD FX_NATS_URL)
+    ;;
+  *)
+    echo "unknown service: ${SERVICE}" >&2
+    exit 64
+    ;;
+esac
+
+for var in "${REQUIRED_SECRETS[@]}"; do
+  if [[ -z "${!var:-}" ]]; then
+    echo "missing required secret env: ${var}" >&2
+    exit 65
+  fi
+done
+
+if [[ ! -s .env.version ]]; then
+  echo ".env.version is missing; run version step first" >&2
+  exit 66
+fi
+
+SSH_OPTS=(
+  -i /root/.ssh/id_rsa
+  -o StrictHostKeyChecking=no
+  -o UserKnownHostsFile=/dev/null
+  -o LogLevel=ERROR
+  -q
+)
+if [[ "${DEBUG_DEPLOY:-0}" = "1" ]]; then
+  SSH_OPTS=("${SSH_OPTS[@]/-q/}" -vv)
+fi
+
+RSYNC_FLAGS=(-az --delete)
+[[ "${DEBUG_DEPLOY:-0}" = "1" ]] && RSYNC_FLAGS=(-avz --delete)
+
+ssh "${SSH_OPTS[@]}" "$REMOTE_TARGET" "mkdir -p ${REMOTE_DIR}/{compose,env}"
+
+rsync "${RSYNC_FLAGS[@]}" -e "ssh ${SSH_OPTS[*]}" ci/prod/compose/ "$REMOTE_TARGET:${REMOTE_DIR}/compose/"
+rsync "${RSYNC_FLAGS[@]}" -e "ssh ${SSH_OPTS[*]}" ci/prod/.env.runtime "$REMOTE_TARGET:${REMOTE_DIR}/env/.env.runtime"
+rsync "${RSYNC_FLAGS[@]}" -e "ssh ${SSH_OPTS[*]}" .env.version         "$REMOTE_TARGET:${REMOTE_DIR}/env/.env.version"
+
+SERVICES_LINE="${SERVICE_NAMES}"
+
+ssh "${SSH_OPTS[@]}" "$REMOTE_TARGET" \
+  REMOTE_DIR="$REMOTE_DIR" \
+  COMPOSE_FILE="$COMPOSE_FILE" \
+  COMPOSE_PROJECT="$COMPOSE_PROJECT" \
+  SERVICES_LINE="$SERVICES_LINE" \
+  FX_MONGO_USER="$FX_MONGO_USER" \
+  FX_MONGO_PASSWORD="$FX_MONGO_PASSWORD" \
+  FX_NATS_URL="${FX_NATS_URL:-}" \
+  NATS_USER="${NATS_USER:-}" \
+  NATS_PASSWORD="${NATS_PASSWORD:-}" \
+  bash -s <<'EOSSH'
+set -euo pipefail
+cd "${REMOTE_DIR}/compose"
+set -a
+. ../env/.env.runtime
+. ../env/.env.version
+set +a
+export FX_MONGO_USER FX_MONGO_PASSWORD FX_NATS_URL NATS_USER NATS_PASSWORD
+COMPOSE_PROJECT_NAME="$COMPOSE_PROJECT"
+export COMPOSE_PROJECT_NAME
+read -r -a SERVICES <<<"${SERVICES_LINE}"
+
+pull_cmd=(docker compose -f "$COMPOSE_FILE" pull)
+up_cmd=(docker compose -f "$COMPOSE_FILE" up -d --remove-orphans)
+ps_cmd=(docker compose -f "$COMPOSE_FILE" ps)
+if [[ "${#SERVICES[@]}" -gt 0 ]]; then
+  pull_cmd+=("${SERVICES[@]}")
+  up_cmd+=("${SERVICES[@]}")
+  ps_cmd+=("${SERVICES[@]}")
+fi
+
+"${pull_cmd[@]}"
+"${up_cmd[@]}"
+"${ps_cmd[@]}"
+
+date -Is > .last_deploy
+logger -t "deploy-${COMPOSE_PROJECT}" "${COMPOSE_PROJECT} deployed at $(date -Is) in ${REMOTE_DIR}"
+EOSSH