diff --git a/.woodpecker/bff.yml b/.woodpecker/bff.yml index c6d70040..645bf746 100644 --- a/.woodpecker/bff.yml +++ b/.woodpecker/bff.yml @@ -81,8 +81,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/billing_documents.yml b/.woodpecker/billing_documents.yml index 9f32de62..87de0729 100644 --- a/.woodpecker/billing_documents.yml +++ b/.woodpecker/billing_documents.yml @@ -76,8 +76,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/billing_fees.yml b/.woodpecker/billing_fees.yml index f00e75e4..a70032f6 100644 --- a/.woodpecker/billing_fees.yml +++ b/.woodpecker/billing_fees.yml @@ -76,8 +76,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/callbacks.yml b/.woodpecker/callbacks.yml index 77bb4062..56ad24f6 100644 --- a/.woodpecker/callbacks.yml +++ b/.woodpecker/callbacks.yml @@ -77,8 +77,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/discovery.yml b/.woodpecker/discovery.yml index 5bba3184..cc6345c9 100644 --- a/.woodpecker/discovery.yml +++ b/.woodpecker/discovery.yml @@ -75,8 +75,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/frontend.yml b/.woodpecker/frontend.yml index 620b93b2..3da55745 100644 --- a/.woodpecker/frontend.yml +++ b/.woodpecker/frontend.yml @@ -49,8 +49,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: frontend-tests image: ghcr.io/cirruslabs/flutter:stable diff --git a/.woodpecker/fx_ingestor.yml b/.woodpecker/fx_ingestor.yml index b690628a..777d50fb 100644 --- a/.woodpecker/fx_ingestor.yml +++ b/.woodpecker/fx_ingestor.yml @@ -81,8 +81,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/fx_oracle.yml b/.woodpecker/fx_oracle.yml index b4f38644..ff3d43ca 100644 --- a/.woodpecker/fx_oracle.yml +++ b/.woodpecker/fx_oracle.yml @@ -82,8 +82,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/gateway_aurora.yml b/.woodpecker/gateway_aurora.yml index 91eda710..392d7ab1 100644 --- a/.woodpecker/gateway_aurora.yml +++ b/.woodpecker/gateway_aurora.yml @@ -76,8 +76,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/gateway_chain.yml b/.woodpecker/gateway_chain.yml index 6c0d1ecf..7a52d803 100644 --- a/.woodpecker/gateway_chain.yml +++ b/.woodpecker/gateway_chain.yml @@ -80,8 +80,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/gateway_chsettle.yml b/.woodpecker/gateway_chsettle.yml index 1a2d9b48..1edea366 100644 --- a/.woodpecker/gateway_chsettle.yml +++ b/.woodpecker/gateway_chsettle.yml @@ -76,8 +76,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/gateway_mntx.yml b/.woodpecker/gateway_mntx.yml index 66a3afc2..fdb4b2a3 100644 --- a/.woodpecker/gateway_mntx.yml +++ b/.woodpecker/gateway_mntx.yml @@ -69,8 +69,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/gateway_tgsettle.yml b/.woodpecker/gateway_tgsettle.yml index d2fb59b5..d674deb4 100644 --- a/.woodpecker/gateway_tgsettle.yml +++ b/.woodpecker/gateway_tgsettle.yml @@ -67,8 +67,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/gateway_tron.yml b/.woodpecker/gateway_tron.yml index c3030164..db052121 100644 --- a/.woodpecker/gateway_tron.yml +++ b/.woodpecker/gateway_tron.yml @@ -80,8 +80,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/ledger.yml b/.woodpecker/ledger.yml index 5bff7f5a..3a35ac42 100644 --- a/.woodpecker/ledger.yml +++ b/.woodpecker/ledger.yml @@ -76,8 +76,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/nats.yml b/.woodpecker/nats.yml index 6417d3a6..93a214cc 100644 --- a/.woodpecker/nats.yml +++ b/.woodpecker/nats.yml @@ -43,7 +43,7 @@ steps: - install -m 600 secrets/SSH_KEY /root/.ssh/id_rsa - . ./ci/scripts/common/runtime_env.sh - load_runtime_env_bundle "$(resolve_runtime_env_name)" - - export NATS_USER="$(./ci/vlt kv_get kv sendico/nats user)" - - export NATS_PASSWORD="$(./ci/vlt kv_get kv sendico/nats password)" + - export NATS_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv sendico/nats user 2>/dev/null || CI_VAULT_SOURCE=external ./ci/vlt kv_get kv sendico/nats user)" + - export NATS_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv sendico/nats password 2>/dev/null || CI_VAULT_SOURCE=external ./ci/vlt kv_get kv sendico/nats password)" - bash ci/prod/scripts/bootstrap/network.sh - bash ci/prod/scripts/deploy/nats.sh diff --git a/.woodpecker/notification.yml b/.woodpecker/notification.yml index 1ba1ac29..6ba9400a 100644 --- a/.woodpecker/notification.yml +++ b/.woodpecker/notification.yml @@ -79,8 +79,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/payments_methods.yml b/.woodpecker/payments_methods.yml index a4e574b5..1a87d659 100644 --- a/.woodpecker/payments_methods.yml +++ b/.woodpecker/payments_methods.yml @@ -77,8 +77,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/payments_orchestrator.yml b/.woodpecker/payments_orchestrator.yml index 60ec494a..6fd68320 100644 --- a/.woodpecker/payments_orchestrator.yml +++ b/.woodpecker/payments_orchestrator.yml @@ -77,8 +77,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/payments_quotation.yml b/.woodpecker/payments_quotation.yml index dc04dbf6..439be3d3 100644 --- a/.woodpecker/payments_quotation.yml +++ b/.woodpecker/payments_quotation.yml @@ -77,8 +77,7 @@ steps: - set -euo pipefail - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - sh ci/scripts/common/fetch_deploy_ssh_key.sh secrets/SSH_KEY - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD + - sh ci/scripts/common/fetch_registry_creds.sh - name: build-image image: gcr.io/kaniko-project/executor:debug diff --git a/.woodpecker/vault.yml b/.woodpecker/vault.yml index 81135bed..9bac812f 100644 --- a/.woodpecker/vault.yml +++ b/.woodpecker/vault.yml @@ -31,6 +31,10 @@ steps: - name: deploy image: alpine:latest depends_on: [ secrets ] + environment: + VAULT_ADDR: { from_secret: VAULT_ADDR } + VAULT_ROLE_ID: { from_secret: VAULT_APP_ROLE } + VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID } commands: - set -euo pipefail - apk add --no-cache bash openssh-client rsync coreutils curl sed python3 diff --git a/README.md b/README.md index a423744b..c244f67a 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,11 @@ First-time dev bootstrap: # after infra is green, merge normal app changes to main ``` +Dev secret source: + +- After the dev Vault bootstrap runs, dev build/deploy secrets are read from the dev Vault on the dev host. +- The dev SSH deploy key remains an external bootstrap secret because CI needs it before it can reach the dev host. + Recommended release preparation: ```bash diff --git a/ci/prod/scripts/deploy/db.sh b/ci/prod/scripts/deploy/db.sh index 39b745ee..01552ab2 100755 --- a/ci/prod/scripts/deploy/db.sh +++ b/ci/prod/scripts/deploy/db.sh @@ -9,9 +9,11 @@ trap 'echo "[deploy-db] error at line $LINENO" >&2' ERR : "${DB_DIR:?missing DB_DIR}" : "${SSH_USER:?missing SSH_USER}" : "${SSH_HOST:?missing SSH_HOST}" -# Pass-through AppRole creds for Vault Agent (provided by Woodpecker secrets with existing names) -: "${VAULT_ROLE_ID:?missing VAULT_ROLE_ID}" -: "${VAULT_SECRET_ID:?missing VAULT_SECRET_ID}" +if [[ "${CI_RUNTIME_ENV_NAME:-prod}" != "devserver" ]]; then + # Pass-through AppRole creds for the prod Vault Agent. + : "${VAULT_ROLE_ID:?missing VAULT_ROLE_ID}" + : "${VAULT_SECRET_ID:?missing VAULT_SECRET_ID}" +fi REMOTE_DIR="${REMOTE_BASE%/}/${DB_DIR}" REMOTE_TARGET="${SSH_USER}@${SSH_HOST}" @@ -49,19 +51,45 @@ rsync "${RSYNC_FLAGS[@]}" -e "ssh ${SSH_OPTS[*]}" "${RUNTIME_ENV_FILE}" "$REMOTE ssh "${SSH_OPTS[@]}" "$REMOTE_TARGET" \ REMOTE_DIR="$REMOTE_DIR" \ COMPOSE_FILE="$COMPOSE_FILE" \ - VAULT_ROLE_ID="$VAULT_ROLE_ID" \ - VAULT_SECRET_ID="$VAULT_SECRET_ID" \ + VAULT_ROLE_ID="${VAULT_ROLE_ID:-}" \ + VAULT_SECRET_ID="${VAULT_SECRET_ID:-}" \ bash -s <<'EOSSH' set -euo pipefail cd "${REMOTE_DIR}/compose" -set -a; . ../env/.env.runtime; set +a +load_kv_file() { + local file="$1" + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|\#*) continue ;; + esac + if printf '%s' "$line" | grep -Eq '^[[:alpha:]_][[:alnum:]_]*='; then + local key="${line%%=*}" + local value="${line#*=}" + key="$(printf '%s' "$key" | tr -d '[:space:]')" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + if [[ -n "$key" ]]; then + export "$key=$value" + fi + fi + done <"$file" +} +set -a +. ../env/.env.runtime +if [[ -f ../env/vault.env ]]; then + load_kv_file ../env/vault.env +fi +set +a COMPOSE_PROJECT_NAME="${DB_COMPOSE_PROJECT:-sendico-db}" export COMPOSE_PROJECT_NAME -# Run with ephemeral AppRole env (scoped only to these commands) -VAULT_ROLE_ID="${VAULT_ROLE_ID}" VAULT_SECRET_ID="${VAULT_SECRET_ID}" docker compose -f "${COMPOSE_FILE}" pull --quiet 2>/dev/null || \ -VAULT_ROLE_ID="${VAULT_ROLE_ID}" VAULT_SECRET_ID="${VAULT_SECRET_ID}" docker compose -f "${COMPOSE_FILE}" pull +: "${VAULT_ADDR:?missing VAULT_ADDR}" +: "${VAULT_ROLE_ID:?missing VAULT_ROLE_ID}" +: "${VAULT_SECRET_ID:?missing VAULT_SECRET_ID}" -VAULT_ROLE_ID="${VAULT_ROLE_ID}" VAULT_SECRET_ID="${VAULT_SECRET_ID}" docker compose -f "${COMPOSE_FILE}" up -d --remove-orphans +VAULT_ROLE_ID="${VAULT_ROLE_ID}" VAULT_SECRET_ID="${VAULT_SECRET_ID}" VAULT_ADDR="${VAULT_ADDR}" docker compose -f "${COMPOSE_FILE}" pull --quiet 2>/dev/null || \ +VAULT_ROLE_ID="${VAULT_ROLE_ID}" VAULT_SECRET_ID="${VAULT_SECRET_ID}" VAULT_ADDR="${VAULT_ADDR}" docker compose -f "${COMPOSE_FILE}" pull + +VAULT_ROLE_ID="${VAULT_ROLE_ID}" VAULT_SECRET_ID="${VAULT_SECRET_ID}" VAULT_ADDR="${VAULT_ADDR}" docker compose -f "${COMPOSE_FILE}" up -d --remove-orphans docker compose -f "${COMPOSE_FILE}" ps date -Is > .last_deploy diff --git a/ci/prod/scripts/deploy/vault.sh b/ci/prod/scripts/deploy/vault.sh index d34141cd..a8c43b7e 100644 --- a/ci/prod/scripts/deploy/vault.sh +++ b/ci/prod/scripts/deploy/vault.sh @@ -13,6 +13,7 @@ REMOTE_DIR="${REMOTE_BASE%/}/${VAULT_DIR}" REMOTE_TARGET="${SSH_USER}@${SSH_HOST}" RUNTIME_ENV_FILE="${RUNTIME_ENV_FILE:-ci/prod/.env.runtime}" COMPOSE_FILE="vault.yml" +SEED_ENV_FILE="${DEV_VAULT_SEED_FILE:-}" SSH_OPTS=( -i /root/.ssh/id_rsa @@ -35,6 +36,9 @@ ssh "${SSH_OPTS[@]}" "$REMOTE_TARGET" "mkdir -p ${REMOTE_DIR}/{compose,env}" rsync "${RSYNC_FLAGS[@]}" -e "ssh ${SSH_OPTS[*]}" ci/prod/compose/ "$REMOTE_TARGET:${REMOTE_DIR}/compose/" rsync "${RSYNC_FLAGS[@]}" -e "ssh ${SSH_OPTS[*]}" "${RUNTIME_ENV_FILE}" "$REMOTE_TARGET:${REMOTE_DIR}/env/.env.runtime" +if [[ -n "${SEED_ENV_FILE}" && -f "${SEED_ENV_FILE}" ]]; then + rsync "${RSYNC_FLAGS[@]}" -e "ssh ${SSH_OPTS[*]}" "${SEED_ENV_FILE}" "$REMOTE_TARGET:${REMOTE_DIR}/env/dev-vault-seed.env" +fi ssh "${SSH_OPTS[@]}" "$REMOTE_TARGET" \ REMOTE_BASE="$REMOTE_BASE" \ @@ -52,6 +56,21 @@ set +a COMPOSE_PROJECT_NAME="$COMPOSE_PROJECT" export COMPOSE_PROJECT_NAME +if base64 -d >/dev/null 2>&1 <<<'AA=='; then + BASE64_DECODE_FLAG='-d' +else + BASE64_DECODE_FLAG='--decode' +fi + +decode_b64() { + val="$1" + if [[ -z "$val" ]]; then + printf '' + return + fi + printf '%s' "$val" | base64 "${BASE64_DECODE_FLAG}" +} + docker compose -f "$COMPOSE_FILE" pull --quiet 2>/dev/null || docker compose -f "$COMPOSE_FILE" pull docker compose -f "$COMPOSE_FILE" up -d --remove-orphans @@ -97,6 +116,63 @@ fi docker exec dev-vault sh -lc "export VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN='${ROOT_TOKEN}'; vault auth list -format=json | grep -q '\"approle/\"' || vault auth enable approle >/dev/null" docker exec dev-vault sh -lc "export VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN='${ROOT_TOKEN}'; vault secrets list -format=json | grep -q '\"kv/\"' || vault secrets enable -path=kv kv-v2 >/dev/null" +if [[ -f ../env/dev-vault-seed.env ]]; then + set -a + . ../env/dev-vault-seed.env + set +a + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv registry \ + user="$(decode_b64 "${REGISTRY_USER_B64:-}")" \ + password="$(decode_b64 "${REGISTRY_PASSWORD_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/db \ + user="$(decode_b64 "${SENDICO_DB_USER_B64:-}")" \ + password="$(decode_b64 "${SENDICO_DB_PASSWORD_B64:-}")" \ + key="$(decode_b64 "${SENDICO_DB_KEY_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/nats \ + user="$(decode_b64 "${SENDICO_NATS_USER_B64:-}")" \ + password="$(decode_b64 "${SENDICO_NATS_PASSWORD_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/api/endpoint \ + secret="$(decode_b64 "${SENDICO_API_ENDPOINT_SECRET_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/notification/mail \ + user="$(decode_b64 "${NOTIFICATION_MAIL_USER_B64:-}")" \ + password="$(decode_b64 "${NOTIFICATION_MAIL_PASSWORD_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/notification/telegram \ + bot_token="$(decode_b64 "${NOTIFICATION_TELEGRAM_BOT_TOKEN_B64:-}")" \ + chat_id="$(decode_b64 "${NOTIFICATION_TELEGRAM_CHAT_ID_B64:-}")" \ + thread_id="$(decode_b64 "${NOTIFICATION_TELEGRAM_THREAD_ID_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/gateway/chain \ + arbitrum_rpc_url="$(decode_b64 "${CHAIN_GATEWAY_RPC_URL_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/gateway/chain/wallet \ + private_key="$(decode_b64 "${CHAIN_GATEWAY_WALLET_PRIVATE_KEY_B64:-}")" \ + address="$(decode_b64 "${CHAIN_GATEWAY_WALLET_ADDRESS_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/gateway/tron \ + rpc_url="$(decode_b64 "${TRON_GATEWAY_RPC_URL_B64:-}")" \ + grpc_url="$(decode_b64 "${TRON_GATEWAY_GRPC_URL_B64:-}")" \ + grpc_token="$(decode_b64 "${TRON_GATEWAY_GRPC_TOKEN_B64:-}")" >/dev/null + + docker exec -e VAULT_ADDR=http://127.0.0.1:8200 -e VAULT_TOKEN="${ROOT_TOKEN}" dev-vault \ + vault kv put -mount=kv sendico/gateway/tron/wallet \ + private_key="$(decode_b64 "${TRON_GATEWAY_WALLET_PRIVATE_KEY_B64:-}")" \ + address="$(decode_b64 "${TRON_GATEWAY_WALLET_ADDRESS_B64:-}")" >/dev/null +fi + docker exec -i dev-vault sh -lc "export VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN='${ROOT_TOKEN}'; vault policy write sendico-dev-apps -" <<'EOF' path "kv/data/*" { capabilities = ["create", "read", "update", "delete", "list"] @@ -124,12 +200,14 @@ write_vault_env() { local env_dir="${REMOTE_BASE%/}/${service_dir}/env" mkdir -p "$env_dir" cat >"${env_dir}/vault.env" <&2 exit 1 diff --git a/ci/scripts/billing_documents/deploy.sh b/ci/scripts/billing_documents/deploy.sh index bb175906..e02dd438 100755 --- a/ci/scripts/billing_documents/deploy.sh +++ b/ci/scripts/billing_documents/deploy.sh @@ -40,8 +40,8 @@ load_runtime_env_bundle "${DOCUMENTS_ENV_NAME}" DOCUMENTS_MONGO_SECRET_PATH="${DOCUMENTS_MONGO_SECRET_PATH:?missing DOCUMENTS_MONGO_SECRET_PATH}" -export DOCUMENTS_MONGO_USER="$(./ci/vlt kv_get kv "${DOCUMENTS_MONGO_SECRET_PATH}" user)" -export DOCUMENTS_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${DOCUMENTS_MONGO_SECRET_PATH}" password)" +export DOCUMENTS_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${DOCUMENTS_MONGO_SECRET_PATH}" user)" +export DOCUMENTS_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${DOCUMENTS_MONGO_SECRET_PATH}" password)" load_nats_env diff --git a/ci/scripts/billing_fees/deploy.sh b/ci/scripts/billing_fees/deploy.sh index 79813875..0bbc04d9 100644 --- a/ci/scripts/billing_fees/deploy.sh +++ b/ci/scripts/billing_fees/deploy.sh @@ -40,8 +40,8 @@ load_runtime_env_bundle "${FEES_ENV_NAME}" FEES_MONGO_SECRET_PATH="${FEES_MONGO_SECRET_PATH:?missing FEES_MONGO_SECRET_PATH}" -export FEES_MONGO_USER="$(./ci/vlt kv_get kv "${FEES_MONGO_SECRET_PATH}" user)" -export FEES_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${FEES_MONGO_SECRET_PATH}" password)" +export FEES_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${FEES_MONGO_SECRET_PATH}" user)" +export FEES_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${FEES_MONGO_SECRET_PATH}" password)" load_nats_env diff --git a/ci/scripts/callbacks/deploy.sh b/ci/scripts/callbacks/deploy.sh index d2408b56..dfed6982 100755 --- a/ci/scripts/callbacks/deploy.sh +++ b/ci/scripts/callbacks/deploy.sh @@ -41,11 +41,11 @@ load_runtime_env_bundle "${CALLBACKS_ENV_NAME}" CALLBACKS_MONGO_SECRET_PATH="${CALLBACKS_MONGO_SECRET_PATH:?missing CALLBACKS_MONGO_SECRET_PATH}" CALLBACKS_VAULT_SECRET_PATH="${CALLBACKS_VAULT_SECRET_PATH:?missing CALLBACKS_VAULT_SECRET_PATH}" -export CALLBACKS_MONGO_USER="$(./ci/vlt kv_get kv "${CALLBACKS_MONGO_SECRET_PATH}" user)" -export CALLBACKS_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${CALLBACKS_MONGO_SECRET_PATH}" password)" +export CALLBACKS_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CALLBACKS_MONGO_SECRET_PATH}" user)" +export CALLBACKS_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CALLBACKS_MONGO_SECRET_PATH}" password)" if [ "${CI_RUNTIME_ENV_NAME:-prod}" != "devserver" ]; then - export CALLBACKS_VAULT_ROLE_ID="$(./ci/vlt kv_get kv "${CALLBACKS_VAULT_SECRET_PATH}" role_id)" - export CALLBACKS_VAULT_SECRET_ID="$(./ci/vlt kv_get kv "${CALLBACKS_VAULT_SECRET_PATH}" secret_id)" + export CALLBACKS_VAULT_ROLE_ID="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CALLBACKS_VAULT_SECRET_PATH}" role_id)" + export CALLBACKS_VAULT_SECRET_ID="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CALLBACKS_VAULT_SECRET_PATH}" secret_id)" if [ -z "${CALLBACKS_VAULT_ROLE_ID}" ] || [ -z "${CALLBACKS_VAULT_SECRET_ID}" ]; then echo "[callbacks-deploy] vault approle creds are empty for path ${CALLBACKS_VAULT_SECRET_PATH}" >&2 exit 1 diff --git a/ci/scripts/chain_gateway/deploy.sh b/ci/scripts/chain_gateway/deploy.sh index c58356cf..f901cace 100755 --- a/ci/scripts/chain_gateway/deploy.sh +++ b/ci/scripts/chain_gateway/deploy.sh @@ -43,17 +43,17 @@ CHAIN_GATEWAY_RPC_SECRET_PATH="${CHAIN_GATEWAY_RPC_SECRET_PATH:?missing CHAIN_GA CHAIN_GATEWAY_WALLET_SECRET_PATH="${CHAIN_GATEWAY_WALLET_SECRET_PATH:?missing CHAIN_GATEWAY_WALLET_SECRET_PATH}" CHAIN_GATEWAY_VAULT_SECRET_PATH="${CHAIN_GATEWAY_VAULT_SECRET_PATH:?missing CHAIN_GATEWAY_VAULT_SECRET_PATH}" -export CHAIN_GATEWAY_MONGO_USER="$(./ci/vlt kv_get kv "${CHAIN_GATEWAY_MONGO_SECRET_PATH}" user)" -export CHAIN_GATEWAY_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${CHAIN_GATEWAY_MONGO_SECRET_PATH}" password)" +export CHAIN_GATEWAY_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHAIN_GATEWAY_MONGO_SECRET_PATH}" user)" +export CHAIN_GATEWAY_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHAIN_GATEWAY_MONGO_SECRET_PATH}" password)" -export CHAIN_GATEWAY_RPC_URL="$(./ci/vlt kv_get kv "${CHAIN_GATEWAY_RPC_SECRET_PATH}" arbitrum_rpc_url)" +export CHAIN_GATEWAY_RPC_URL="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHAIN_GATEWAY_RPC_SECRET_PATH}" arbitrum_rpc_url)" -export CHAIN_GATEWAY_SERVICE_WALLET_KEY="$(./ci/vlt kv_get kv "${CHAIN_GATEWAY_WALLET_SECRET_PATH}" private_key)" -export CHAIN_GATEWAY_SERVICE_WALLET_ADDRESS="$(./ci/vlt kv_get kv "${CHAIN_GATEWAY_WALLET_SECRET_PATH}" address || true)" +export CHAIN_GATEWAY_SERVICE_WALLET_KEY="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHAIN_GATEWAY_WALLET_SECRET_PATH}" private_key)" +export CHAIN_GATEWAY_SERVICE_WALLET_ADDRESS="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHAIN_GATEWAY_WALLET_SECRET_PATH}" address || true)" if [ "${CI_RUNTIME_ENV_NAME:-prod}" != "devserver" ]; then - export CHAIN_GATEWAY_VAULT_ROLE_ID="$(./ci/vlt kv_get kv "${CHAIN_GATEWAY_VAULT_SECRET_PATH}" role_id)" - export CHAIN_GATEWAY_VAULT_SECRET_ID="$(./ci/vlt kv_get kv "${CHAIN_GATEWAY_VAULT_SECRET_PATH}" secret_id)" + export CHAIN_GATEWAY_VAULT_ROLE_ID="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHAIN_GATEWAY_VAULT_SECRET_PATH}" role_id)" + export CHAIN_GATEWAY_VAULT_SECRET_ID="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHAIN_GATEWAY_VAULT_SECRET_PATH}" secret_id)" if [ -z "${CHAIN_GATEWAY_VAULT_ROLE_ID}" ] || [ -z "${CHAIN_GATEWAY_VAULT_SECRET_ID}" ]; then echo "[chain-gateway-deploy] vault approle creds are empty for path ${CHAIN_GATEWAY_VAULT_SECRET_PATH}" >&2 exit 1 diff --git a/ci/scripts/chsettle/deploy.sh b/ci/scripts/chsettle/deploy.sh index c4b2b441..c3689b88 100644 --- a/ci/scripts/chsettle/deploy.sh +++ b/ci/scripts/chsettle/deploy.sh @@ -40,8 +40,8 @@ load_runtime_env_bundle "${CHSETTLE_GATEWAY_ENV_NAME}" CHSETTLE_GATEWAY_MONGO_SECRET_PATH="${CHSETTLE_GATEWAY_MONGO_SECRET_PATH:?missing CHSETTLE_GATEWAY_MONGO_SECRET_PATH}" CHSETTLE_GATEWAY_NATS_SECRET_PATH="${CHSETTLE_GATEWAY_NATS_SECRET_PATH:-sendico/nats}" -export CHSETTLE_GATEWAY_MONGO_USER="$(./ci/vlt kv_get kv "${CHSETTLE_GATEWAY_MONGO_SECRET_PATH}" user)" -export CHSETTLE_GATEWAY_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${CHSETTLE_GATEWAY_MONGO_SECRET_PATH}" password)" +export CHSETTLE_GATEWAY_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHSETTLE_GATEWAY_MONGO_SECRET_PATH}" user)" +export CHSETTLE_GATEWAY_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${CHSETTLE_GATEWAY_MONGO_SECRET_PATH}" password)" NATS_SECRET_PATH="${CHSETTLE_GATEWAY_NATS_SECRET_PATH}" load_nats_env diff --git a/ci/scripts/common/fetch_registry_creds.sh b/ci/scripts/common/fetch_registry_creds.sh new file mode 100644 index 00000000..40ebd450 --- /dev/null +++ b/ci/scripts/common/fetch_registry_creds.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "${REPO_ROOT}" + +mkdir -p secrets +sh ci/scripts/common/runtime_kv_get.sh kv_get kv registry user > secrets/REGISTRY_USER +sh ci/scripts/common/runtime_kv_get.sh kv_get kv registry password > secrets/REGISTRY_PASSWORD diff --git a/ci/scripts/common/nats_env.sh b/ci/scripts/common/nats_env.sh index 75ca0fb0..d0d687a5 100644 --- a/ci/scripts/common/nats_env.sh +++ b/ci/scripts/common/nats_env.sh @@ -4,8 +4,8 @@ load_nats_env() { : "${NATS_PORT:?missing NATS_PORT}" nats_secret_path="${NATS_SECRET_PATH:-sendico/nats}" - export NATS_USER="$(./ci/vlt kv_get kv "${nats_secret_path}" user)" - export NATS_PASSWORD="$(./ci/vlt kv_get kv "${nats_secret_path}" password)" + export NATS_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${nats_secret_path}" user)" + export NATS_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${nats_secret_path}" password)" nats_url_var="${NATS_URL_VAR:-NATS_URL}" nats_url_scheme="${NATS_URL_SCHEME:-nats}" diff --git a/ci/scripts/common/runtime_kv_get.sh b/ci/scripts/common/runtime_kv_get.sh new file mode 100644 index 00000000..1f574b62 --- /dev/null +++ b/ci/scripts/common/runtime_kv_get.sh @@ -0,0 +1,131 @@ +#!/bin/sh +set -eu + +if ! set -o pipefail 2>/dev/null; then + : +fi + +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "${REPO_ROOT}" + +usage() { + echo "usage: runtime_kv_get.sh kv_get " >&2 + exit 64 +} + +[ "${1:-}" = "kv_get" ] || usage +[ $# -eq 4 ] || usage + +MOUNT="$2" +SECRET_PATH="$3" +FIELD="$4" + +. ci/scripts/common/runtime_env.sh + +runtime_env_name="${CI_TARGET_ENV:-${CI_RUNTIME_ENV_NAME:-$(resolve_runtime_env_name)}}" +vault_source="${CI_VAULT_SOURCE:-runtime}" + +if [ "${vault_source}" = "external" ] || [ "${runtime_env_name}" != "devserver" ]; then + exec ./ci/vlt kv_get "${MOUNT}" "${SECRET_PATH}" "${FIELD}" +fi + +runtime_file="$(resolve_runtime_env_file "${runtime_env_name}")" +cleanup_runtime_file=0 +case "${runtime_file}" in + ./.runtime.*.merged.*) + cleanup_runtime_file=1 + ;; +esac + +cleanup() { + if [ "${cleanup_runtime_file}" -eq 1 ]; then + rm -f "${runtime_file}" + fi +} +trap cleanup EXIT INT TERM + +normalize_env_file "${runtime_file}" +load_env_file "${runtime_file}" + +: "${SSH_USER:?missing SSH_USER}" +: "${SSH_HOST:?missing SSH_HOST}" +: "${REMOTE_BASE:?missing REMOTE_BASE}" +: "${VAULT_DIR:?missing VAULT_DIR}" + +SSH_KEY_FILE="${SSH_KEY_FILE:-}" +if [ -z "${SSH_KEY_FILE}" ] || [ ! -f "${SSH_KEY_FILE}" ]; then + for candidate in /root/.ssh/id_rsa secrets/SSH_KEY; do + if [ -f "${candidate}" ]; then + SSH_KEY_FILE="${candidate}" + break + fi + done +fi + +if [ -z "${SSH_KEY_FILE}" ] || [ ! -f "${SSH_KEY_FILE}" ]; then + echo "[runtime-kv-get] ssh key not found; expected /root/.ssh/id_rsa or secrets/SSH_KEY" >&2 + exit 65 +fi + +b64enc() { + printf '%s' "$1" | base64 | tr -d '\n' +} + +MOUNT_B64="$(b64enc "${MOUNT}")" +SECRET_PATH_B64="$(b64enc "${SECRET_PATH}")" +FIELD_B64="$(b64enc "${FIELD}")" +REMOTE_TARGET="${SSH_USER}@${SSH_HOST}" + +SSH_OPTS=" + -i ${SSH_KEY_FILE} + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o PreferredAuthentications=publickey + -o ConnectTimeout=10 +" + +ssh ${SSH_OPTS} "${REMOTE_TARGET}" \ + REMOTE_BASE="${REMOTE_BASE}" \ + VAULT_DIR="${VAULT_DIR}" \ + MOUNT_B64="${MOUNT_B64}" \ + SECRET_PATH_B64="${SECRET_PATH_B64}" \ + FIELD_B64="${FIELD_B64}" \ + sh -s <<'EOSSH' +set -eu + +if printf 'AA==' | base64 -d >/dev/null 2>&1; then + BASE64_DECODE_FLAG='-d' +else + BASE64_DECODE_FLAG='--decode' +fi + +decode_b64() { + printf '%s' "$1" | base64 "${BASE64_DECODE_FLAG}" +} + +MOUNT="$(decode_b64 "${MOUNT_B64}")" +SECRET_PATH="$(decode_b64 "${SECRET_PATH_B64}")" +FIELD="$(decode_b64 "${FIELD_B64}")" +INIT_FILE="${REMOTE_BASE%/}/${VAULT_DIR}/env/vault-init.json" + +if [ ! -s "${INIT_FILE}" ]; then + echo "[runtime-kv-get] dev vault init file not found: ${INIT_FILE}" >&2 + exit 66 +fi + +INIT_JSON_COMPACT="$(tr -d '\r\n\t ' <"${INIT_FILE}")" +ROOT_TOKEN="$(printf '%s' "${INIT_JSON_COMPACT}" | sed -n 's/.*"root_token":"\([^"]*\)".*/\1/p')" + +if [ -z "${ROOT_TOKEN}" ]; then + echo "[runtime-kv-get] failed to extract dev vault root token from ${INIT_FILE}" >&2 + exit 67 +fi + +docker exec \ + -e VAULT_ADDR=http://127.0.0.1:8200 \ + -e VAULT_TOKEN="${ROOT_TOKEN}" \ + dev-vault \ + vault kv get -mount="${MOUNT}" -field="${FIELD}" "${SECRET_PATH}" +EOSSH diff --git a/ci/scripts/fx/deploy.sh b/ci/scripts/fx/deploy.sh index 4ed123b4..e128efc5 100755 --- a/ci/scripts/fx/deploy.sh +++ b/ci/scripts/fx/deploy.sh @@ -49,8 +49,8 @@ if [ -z "${FX_NEEDS_NATS}" ]; then esac fi -export FX_MONGO_USER="$(./ci/vlt kv_get kv "${FX_MONGO_SECRET_PATH}" user)" -export FX_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${FX_MONGO_SECRET_PATH}" password)" +export FX_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${FX_MONGO_SECRET_PATH}" user)" +export FX_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${FX_MONGO_SECRET_PATH}" password)" if [ "${FX_NEEDS_NATS}" = "true" ]; then NATS_URL_VAR=FX_NATS_URL load_nats_env diff --git a/ci/scripts/ledger/deploy.sh b/ci/scripts/ledger/deploy.sh index f7453349..d832aecf 100755 --- a/ci/scripts/ledger/deploy.sh +++ b/ci/scripts/ledger/deploy.sh @@ -40,8 +40,8 @@ load_runtime_env_bundle "${LEDGER_ENV_NAME}" LEDGER_MONGO_SECRET_PATH="${LEDGER_MONGO_SECRET_PATH:?missing LEDGER_MONGO_SECRET_PATH}" -export LEDGER_MONGO_USER="$(./ci/vlt kv_get kv "${LEDGER_MONGO_SECRET_PATH}" user)" -export LEDGER_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${LEDGER_MONGO_SECRET_PATH}" password)" +export LEDGER_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${LEDGER_MONGO_SECRET_PATH}" user)" +export LEDGER_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${LEDGER_MONGO_SECRET_PATH}" password)" load_nats_env diff --git a/ci/scripts/notification/deploy.sh b/ci/scripts/notification/deploy.sh index bb8788cd..8b156092 100755 --- a/ci/scripts/notification/deploy.sh +++ b/ci/scripts/notification/deploy.sh @@ -43,18 +43,18 @@ NOTIFICATION_MAIL_SECRET_PATH="${NOTIFICATION_MAIL_SECRET_PATH:?missing NOTIFICA NOTIFICATION_API_SECRET_PATH="${NOTIFICATION_API_SECRET_PATH:?missing NOTIFICATION_API_SECRET_PATH}" NOTIFICATION_TELEGRAM_SECRET_PATH="${NOTIFICATION_TELEGRAM_SECRET_PATH:?missing NOTIFICATION_TELEGRAM_SECRET_PATH}" -export MONGO_USER="$(./ci/vlt kv_get kv "${NOTIFICATION_MONGO_SECRET_PATH}" user)" -export MONGO_PASSWORD="$(./ci/vlt kv_get kv "${NOTIFICATION_MONGO_SECRET_PATH}" password)" +export MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${NOTIFICATION_MONGO_SECRET_PATH}" user)" +export MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${NOTIFICATION_MONGO_SECRET_PATH}" password)" -export MAIL_USER="$(./ci/vlt kv_get kv "${NOTIFICATION_MAIL_SECRET_PATH}" user)" -export MAIL_SECRET="$(./ci/vlt kv_get kv "${NOTIFICATION_MAIL_SECRET_PATH}" password)" +export MAIL_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${NOTIFICATION_MAIL_SECRET_PATH}" user)" +export MAIL_SECRET="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${NOTIFICATION_MAIL_SECRET_PATH}" password)" -export API_ENDPOINT_SECRET="$(./ci/vlt kv_get kv "${NOTIFICATION_API_SECRET_PATH}" secret)" +export API_ENDPOINT_SECRET="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${NOTIFICATION_API_SECRET_PATH}" secret)" -export TELEGRAM_BOT_TOKEN="$(./ci/vlt kv_get kv "${NOTIFICATION_TELEGRAM_SECRET_PATH}" bot_token)" -export TELEGRAM_CHAT_ID="$(./ci/vlt kv_get kv "${NOTIFICATION_TELEGRAM_SECRET_PATH}" chat_id)" +export TELEGRAM_BOT_TOKEN="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${NOTIFICATION_TELEGRAM_SECRET_PATH}" bot_token)" +export TELEGRAM_CHAT_ID="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${NOTIFICATION_TELEGRAM_SECRET_PATH}" chat_id)" TELEGRAM_THREAD_ID="" -if TELEGRAM_THREAD_ID_VALUE="$(./ci/vlt kv_get kv "${NOTIFICATION_TELEGRAM_SECRET_PATH}" thread_id 2>/dev/null)"; then +if TELEGRAM_THREAD_ID_VALUE="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${NOTIFICATION_TELEGRAM_SECRET_PATH}" thread_id 2>/dev/null)"; then TELEGRAM_THREAD_ID="$TELEGRAM_THREAD_ID_VALUE" fi export TELEGRAM_THREAD_ID diff --git a/ci/scripts/payments_methods/deploy.sh b/ci/scripts/payments_methods/deploy.sh index 72b9758a..3b2cbe60 100755 --- a/ci/scripts/payments_methods/deploy.sh +++ b/ci/scripts/payments_methods/deploy.sh @@ -40,8 +40,8 @@ load_runtime_env_bundle "${PAYMENTS_METHODS_ENV_NAME}" PAYMENTS_METHODS_MONGO_SECRET_PATH="${PAYMENTS_METHODS_MONGO_SECRET_PATH:?missing PAYMENTS_METHODS_MONGO_SECRET_PATH}" -export PAYMENTS_MONGO_USER="$(./ci/vlt kv_get kv "${PAYMENTS_METHODS_MONGO_SECRET_PATH}" user)" -export PAYMENTS_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${PAYMENTS_METHODS_MONGO_SECRET_PATH}" password)" +export PAYMENTS_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${PAYMENTS_METHODS_MONGO_SECRET_PATH}" user)" +export PAYMENTS_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${PAYMENTS_METHODS_MONGO_SECRET_PATH}" password)" load_nats_env diff --git a/ci/scripts/payments_orchestrator/deploy.sh b/ci/scripts/payments_orchestrator/deploy.sh index 2fcf0461..0f2ab7ef 100755 --- a/ci/scripts/payments_orchestrator/deploy.sh +++ b/ci/scripts/payments_orchestrator/deploy.sh @@ -40,8 +40,8 @@ load_runtime_env_bundle "${PAYMENTS_ENV_NAME}" PAYMENTS_MONGO_SECRET_PATH="${PAYMENTS_MONGO_SECRET_PATH:?missing PAYMENTS_MONGO_SECRET_PATH}" -export PAYMENTS_MONGO_USER="$(./ci/vlt kv_get kv "${PAYMENTS_MONGO_SECRET_PATH}" user)" -export PAYMENTS_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${PAYMENTS_MONGO_SECRET_PATH}" password)" +export PAYMENTS_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${PAYMENTS_MONGO_SECRET_PATH}" user)" +export PAYMENTS_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${PAYMENTS_MONGO_SECRET_PATH}" password)" load_nats_env diff --git a/ci/scripts/payments_quotation/deploy.sh b/ci/scripts/payments_quotation/deploy.sh index 78bd3344..063242e9 100755 --- a/ci/scripts/payments_quotation/deploy.sh +++ b/ci/scripts/payments_quotation/deploy.sh @@ -40,8 +40,8 @@ load_runtime_env_bundle "${PAYMENTS_QUOTATION_ENV_NAME}" PAYMENTS_QUOTATION_MONGO_SECRET_PATH="${PAYMENTS_QUOTATION_MONGO_SECRET_PATH:?missing PAYMENTS_QUOTATION_MONGO_SECRET_PATH}" -export PAYMENTS_MONGO_USER="$(./ci/vlt kv_get kv "${PAYMENTS_QUOTATION_MONGO_SECRET_PATH}" user)" -export PAYMENTS_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${PAYMENTS_QUOTATION_MONGO_SECRET_PATH}" password)" +export PAYMENTS_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${PAYMENTS_QUOTATION_MONGO_SECRET_PATH}" user)" +export PAYMENTS_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${PAYMENTS_QUOTATION_MONGO_SECRET_PATH}" password)" load_nats_env diff --git a/ci/scripts/tgsettle/deploy.sh b/ci/scripts/tgsettle/deploy.sh index 3e0e3851..6b5e1359 100755 --- a/ci/scripts/tgsettle/deploy.sh +++ b/ci/scripts/tgsettle/deploy.sh @@ -40,8 +40,8 @@ load_runtime_env_bundle "${TGSETTLE_GATEWAY_ENV_NAME}" TGSETTLE_GATEWAY_MONGO_SECRET_PATH="${TGSETTLE_GATEWAY_MONGO_SECRET_PATH:?missing TGSETTLE_GATEWAY_MONGO_SECRET_PATH}" -export TGSETTLE_GATEWAY_MONGO_USER="$(./ci/vlt kv_get kv "${TGSETTLE_GATEWAY_MONGO_SECRET_PATH}" user)" -export TGSETTLE_GATEWAY_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${TGSETTLE_GATEWAY_MONGO_SECRET_PATH}" password)" +export TGSETTLE_GATEWAY_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TGSETTLE_GATEWAY_MONGO_SECRET_PATH}" user)" +export TGSETTLE_GATEWAY_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TGSETTLE_GATEWAY_MONGO_SECRET_PATH}" password)" load_nats_env diff --git a/ci/scripts/tron_gateway/deploy.sh b/ci/scripts/tron_gateway/deploy.sh index b7503037..e8aca24b 100755 --- a/ci/scripts/tron_gateway/deploy.sh +++ b/ci/scripts/tron_gateway/deploy.sh @@ -43,19 +43,19 @@ TRON_GATEWAY_RPC_SECRET_PATH="${TRON_GATEWAY_RPC_SECRET_PATH:?missing TRON_GATEW TRON_GATEWAY_WALLET_SECRET_PATH="${TRON_GATEWAY_WALLET_SECRET_PATH:?missing TRON_GATEWAY_WALLET_SECRET_PATH}" TRON_GATEWAY_VAULT_SECRET_PATH="${TRON_GATEWAY_VAULT_SECRET_PATH:?missing TRON_GATEWAY_VAULT_SECRET_PATH}" -export TRON_GATEWAY_MONGO_USER="$(./ci/vlt kv_get kv "${TRON_GATEWAY_MONGO_SECRET_PATH}" user)" -export TRON_GATEWAY_MONGO_PASSWORD="$(./ci/vlt kv_get kv "${TRON_GATEWAY_MONGO_SECRET_PATH}" password)" +export TRON_GATEWAY_MONGO_USER="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_MONGO_SECRET_PATH}" user)" +export TRON_GATEWAY_MONGO_PASSWORD="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_MONGO_SECRET_PATH}" password)" -export TRON_GATEWAY_RPC_URL="$(./ci/vlt kv_get kv "${TRON_GATEWAY_RPC_SECRET_PATH}" rpc_url)" -export TRON_GATEWAY_GRPC_URL="$(./ci/vlt kv_get kv "${TRON_GATEWAY_RPC_SECRET_PATH}" grpc_url || true)" -export TRON_GATEWAY_GRPC_TOKEN="$(./ci/vlt kv_get kv "${TRON_GATEWAY_RPC_SECRET_PATH}" grpc_token || true)" +export TRON_GATEWAY_RPC_URL="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_RPC_SECRET_PATH}" rpc_url)" +export TRON_GATEWAY_GRPC_URL="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_RPC_SECRET_PATH}" grpc_url || true)" +export TRON_GATEWAY_GRPC_TOKEN="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_RPC_SECRET_PATH}" grpc_token || true)" -export TRON_GATEWAY_SERVICE_WALLET_KEY="$(./ci/vlt kv_get kv "${TRON_GATEWAY_WALLET_SECRET_PATH}" private_key)" -export TRON_GATEWAY_SERVICE_WALLET_ADDRESS="$(./ci/vlt kv_get kv "${TRON_GATEWAY_WALLET_SECRET_PATH}" address || true)" +export TRON_GATEWAY_SERVICE_WALLET_KEY="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_WALLET_SECRET_PATH}" private_key)" +export TRON_GATEWAY_SERVICE_WALLET_ADDRESS="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_WALLET_SECRET_PATH}" address || true)" if [ "${CI_RUNTIME_ENV_NAME:-prod}" != "devserver" ]; then - export TRON_GATEWAY_VAULT_ROLE_ID="$(./ci/vlt kv_get kv "${TRON_GATEWAY_VAULT_SECRET_PATH}" role_id)" - export TRON_GATEWAY_VAULT_SECRET_ID="$(./ci/vlt kv_get kv "${TRON_GATEWAY_VAULT_SECRET_PATH}" secret_id)" + export TRON_GATEWAY_VAULT_ROLE_ID="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_VAULT_SECRET_PATH}" role_id)" + export TRON_GATEWAY_VAULT_SECRET_ID="$(sh ci/scripts/common/runtime_kv_get.sh kv_get kv "${TRON_GATEWAY_VAULT_SECRET_PATH}" secret_id)" if [ -z "${TRON_GATEWAY_VAULT_ROLE_ID}" ] || [ -z "${TRON_GATEWAY_VAULT_SECRET_ID}" ]; then echo "[tron-gateway-deploy] vault approle creds are empty for path ${TRON_GATEWAY_VAULT_SECRET_PATH}" >&2 exit 1 diff --git a/ci/scripts/vault/deploy.sh b/ci/scripts/vault/deploy.sh index fdb4b247..cf613e68 100644 --- a/ci/scripts/vault/deploy.sh +++ b/ci/scripts/vault/deploy.sh @@ -35,4 +35,53 @@ load_env_file() { VAULT_ENV_NAME="${VAULT_ENV:-$(resolve_runtime_env_name)}" load_runtime_env_bundle "${VAULT_ENV_NAME}" +SEED_FILE=".dev-vault-seed.env" + +cleanup() { + rm -f "${SEED_FILE}" +} +trap cleanup EXIT INT TERM + +seed_field() { + var_name="$1" + secret_path="$2" + field_name="$3" + optional="${4:-0}" + + if [ "${optional}" = "1" ]; then + value="$(CI_VAULT_SOURCE=external ./ci/vlt kv_get kv "${secret_path}" "${field_name}" 2>/dev/null || true)" + else + value="$(CI_VAULT_SOURCE=external ./ci/vlt kv_get kv "${secret_path}" "${field_name}")" + fi + + printf '%s=%s\n' "${var_name}" "$(printf '%s' "${value}" | base64 | tr -d '\n')" >> "${SEED_FILE}" +} + +: > "${SEED_FILE}" +chmod 600 "${SEED_FILE}" + +seed_field REGISTRY_USER_B64 registry user +seed_field REGISTRY_PASSWORD_B64 registry password +seed_field SENDICO_DB_USER_B64 sendico/db user +seed_field SENDICO_DB_PASSWORD_B64 sendico/db password +seed_field SENDICO_DB_KEY_B64 sendico/db key +seed_field SENDICO_NATS_USER_B64 sendico/nats user +seed_field SENDICO_NATS_PASSWORD_B64 sendico/nats password +seed_field SENDICO_API_ENDPOINT_SECRET_B64 sendico/api/endpoint secret +seed_field NOTIFICATION_MAIL_USER_B64 sendico/notification/mail user +seed_field NOTIFICATION_MAIL_PASSWORD_B64 sendico/notification/mail password +seed_field NOTIFICATION_TELEGRAM_BOT_TOKEN_B64 sendico/notification/telegram bot_token +seed_field NOTIFICATION_TELEGRAM_CHAT_ID_B64 sendico/notification/telegram chat_id +seed_field NOTIFICATION_TELEGRAM_THREAD_ID_B64 sendico/notification/telegram thread_id 1 +seed_field CHAIN_GATEWAY_RPC_URL_B64 sendico/gateway/chain arbitrum_rpc_url +seed_field CHAIN_GATEWAY_WALLET_PRIVATE_KEY_B64 sendico/gateway/chain/wallet private_key +seed_field CHAIN_GATEWAY_WALLET_ADDRESS_B64 sendico/gateway/chain/wallet address 1 +seed_field TRON_GATEWAY_RPC_URL_B64 sendico/gateway/tron rpc_url +seed_field TRON_GATEWAY_GRPC_URL_B64 sendico/gateway/tron grpc_url 1 +seed_field TRON_GATEWAY_GRPC_TOKEN_B64 sendico/gateway/tron grpc_token 1 +seed_field TRON_GATEWAY_WALLET_PRIVATE_KEY_B64 sendico/gateway/tron/wallet private_key +seed_field TRON_GATEWAY_WALLET_ADDRESS_B64 sendico/gateway/tron/wallet address 1 + +export DEV_VAULT_SEED_FILE="${SEED_FILE}" + bash ci/prod/scripts/deploy/vault.sh