diff --git a/.woodpecker/fx_ingestor.yml b/.woodpecker/fx.yml similarity index 70% rename from .woodpecker/fx_ingestor.yml rename to .woodpecker/fx.yml index f24b48a..a58f892 100644 --- a/.woodpecker/fx_ingestor.yml +++ b/.woodpecker/fx.yml @@ -1,3 +1,18 @@ +matrix: + include: + - FX_PIPELINE: fx_ingestor + FX_IMAGE_PATH: fx/ingestor + FX_DOCKERFILE: ci/prod/compose/fx_ingestor.dockerfile + FX_DEPLOY_TARGET: ingestor + FX_MONGO_SECRET_PATH: sendico/db + FX_NEEDS_NATS: "false" + - FX_PIPELINE: fx_oracle + FX_IMAGE_PATH: fx/oracle + FX_DOCKERFILE: ci/prod/compose/fx_oracle.dockerfile + FX_DEPLOY_TARGET: oracle + FX_MONGO_SECRET_PATH: sendico/db + FX_NEEDS_NATS: "true" + when: - event: push branch: main @@ -39,29 +54,7 @@ steps: depends_on: [ secrets ] commands: - set -euo pipefail 2>/dev/null || set -eu - - | - if [ ! -s ./.env.version ] || ! grep -q '=' ./.env.version 2>/dev/null; then - echo "[build-image] .env.version missing or malformed, regenerating defaults" >&2 - if [ -n "${WOODPECKER_COMMIT:-}" ]; then - GIT_REV="$(printf '%s' "${WOODPECKER_COMMIT}" | cut -c1-7)" - elif command -v git >/dev/null 2>&1; then - GIT_REV="$(git rev-parse --short HEAD 2>/dev/null || echo dev)" - else - GIT_REV="dev" - fi - if [ -n "${WOODPECKER_BRANCH:-}" ]; then - BUILD_BRANCH="${WOODPECKER_BRANCH}" - elif command -v git >/dev/null 2>&1; then - BUILD_BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo local)" - else - BUILD_BRANCH="local" - fi - APP_V="$(cat version 2>/dev/null || echo dev)" - BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" - BUILD_USER="${WOODPECKER_MACHINE:-woodpecker}" - printf "GIT_REV=%s\nBUILD_BRANCH=%s\nAPP_V=%s\nBUILD_DATE=%s\nBUILD_USER=%s\n" \ - "$GIT_REV" "$BUILD_BRANCH" "$APP_V" "$BUILD_DATE" "$BUILD_USER" > .env.version - fi + - bash ci/pipelines/fx/ensure_env_version.sh - sed -i 's/\r$//' ./ci/prod/.env.runtime - sed -i 's/\r$//' ./.env.version - set -a @@ -90,8 +83,8 @@ steps: - | /kaniko/executor \ --context "${PWD}" \ - --dockerfile ci/prod/compose/fx_ingestor.dockerfile \ - --destination "${REGISTRY_URL}/fx/ingestor:${APP_V}" \ + --dockerfile "${FX_DOCKERFILE}" \ + --destination "${REGISTRY_URL}/${FX_IMAGE_PATH}:${APP_V}" \ --build-arg APP_VERSION="${APP_V}" \ --build-arg GIT_REV="${GIT_REV}" \ --build-arg BUILD_BRANCH="${BUILD_BRANCH}" \ @@ -117,7 +110,13 @@ steps: - . ./ci/prod/.env.runtime - . ./.env.version - set +a - - export FX_MONGO_USER="$(./ci/vlt kv_get kv sendico/db user)" - - export FX_MONGO_PASSWORD="$(./ci/vlt kv_get kv sendico/db password)" + - export FX_MONGO_USER="$(./ci/vlt kv_get kv ${FX_MONGO_SECRET_PATH} user)" + - export FX_MONGO_PASSWORD="$(./ci/vlt kv_get kv ${FX_MONGO_SECRET_PATH} password)" + - | + if [ "${FX_NEEDS_NATS}" = "true" ]; then + export NATS_USER="$(./ci/vlt kv_get kv sendico/nats user)" + export NATS_PASSWORD="$(./ci/vlt kv_get kv sendico/nats password)" + export FX_NATS_URL="nats://${NATS_USER}:${NATS_PASSWORD}@${NATS_HOST}:${NATS_PORT}" + fi - bash ci/prod/scripts/bootstrap/network.sh - - bash ci/prod/scripts/deploy/fx.sh ingestor + - bash ci/prod/scripts/deploy/fx.sh "${FX_DEPLOY_TARGET}" diff --git a/.woodpecker/fx_oracle.yml b/.woodpecker/fx_oracle.yml deleted file mode 100644 index 136346b..0000000 --- a/.woodpecker/fx_oracle.yml +++ /dev/null @@ -1,126 +0,0 @@ -when: - - event: push - branch: main - -steps: - - name: version - image: alpine:latest - commands: - - set -euo pipefail 2>/dev/null || set -eu - - apk add --no-cache git - - GIT_REV="$(git rev-parse --short HEAD)" - - BUILD_BRANCH="$(git rev-parse --abbrev-ref HEAD)" - - APP_V="$(cat version)" - - BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - BUILD_USER="${WOODPECKER_MACHINE:-woodpecker}" - - printf "GIT_REV=%s\nBUILD_BRANCH=%s\nAPP_V=%s\nBUILD_DATE=%s\nBUILD_USER=%s\n" \ - "$GIT_REV" "$BUILD_BRANCH" "$APP_V" "$BUILD_DATE" "$BUILD_USER" | tee .env.version - - - name: secrets - image: alpine:latest - depends_on: [ version ] - environment: - VAULT_ADDR: { from_secret: VAULT_ADDR } - VAULT_ROLE_ID: { from_secret: VAULT_APP_ROLE } - VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID } - commands: - - set -euo pipefail - - apk add --no-cache bash coreutils openssh-keygen curl sed python3 - - mkdir -p secrets - - ./ci/vlt kv_to_file kv ops/deploy/ssh_key private_b64 secrets/SSH_KEY.b64 600 - - base64 -d secrets/SSH_KEY.b64 > secrets/SSH_KEY - - chmod 600 secrets/SSH_KEY - - ssh-keygen -y -f secrets/SSH_KEY >/dev/null - - ./ci/vlt kv_get kv registry user > secrets/REGISTRY_USER - - ./ci/vlt kv_get kv registry password > secrets/REGISTRY_PASSWORD - - - name: build-image - image: gcr.io/kaniko-project/executor:debug - depends_on: [ secrets ] - commands: - - set -euo pipefail 2>/dev/null || set -eu - - | - if [ ! -s ./.env.version ] || ! grep -q '=' ./.env.version 2>/dev/null; then - echo "[build-image] .env.version missing or malformed, regenerating defaults" >&2 - if [ -n "${WOODPECKER_COMMIT:-}" ]; then - GIT_REV="$(printf '%s' "${WOODPECKER_COMMIT}" | cut -c1-7)" - elif command -v git >/dev/null 2>&1; then - GIT_REV="$(git rev-parse --short HEAD 2>/dev/null || echo dev)" - else - GIT_REV="dev" - fi - if [ -n "${WOODPECKER_BRANCH:-}" ]; then - BUILD_BRANCH="${WOODPECKER_BRANCH}" - elif command -v git >/dev/null 2>&1; then - BUILD_BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo local)" - else - BUILD_BRANCH="local" - fi - APP_V="$(cat version 2>/dev/null || echo dev)" - BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" - BUILD_USER="${WOODPECKER_MACHINE:-woodpecker}" - printf "GIT_REV=%s\nBUILD_BRANCH=%s\nAPP_V=%s\nBUILD_DATE=%s\nBUILD_USER=%s\n" \ - "$GIT_REV" "$BUILD_BRANCH" "$APP_V" "$BUILD_DATE" "$BUILD_USER" > .env.version - fi - - sed -i 's/\r$//' ./ci/prod/.env.runtime - - sed -i 's/\r$//' ./.env.version - - set -a - - . ./ci/prod/.env.runtime - - . ./.env.version - - set +a - - FX_GO_VERSION="${FX_GO_VERSION:-1.22}" - - ": \"${REGISTRY_URL:?missing REGISTRY_URL}\"" - - ": \"${APP_V:?missing APP_V}\"" - - REGISTRY_HOST="${REGISTRY_URL#http://}" - - REGISTRY_HOST="${REGISTRY_HOST#https://}" - - REGISTRY_USER="$(cat secrets/REGISTRY_USER)" - - REGISTRY_PASSWORD="$(cat secrets/REGISTRY_PASSWORD)" - - ": \"${REGISTRY_USER:?missing registry user}\"" - - ": \"${REGISTRY_PASSWORD:?missing registry password}\"" - - mkdir -p /kaniko/.docker - - AUTH_B64="$(printf '%s:%s' "$REGISTRY_USER" "$REGISTRY_PASSWORD" | base64 | tr -d '\n')" - - | - cat </kaniko/.docker/config.json - { - "auths": { - "https://${REGISTRY_HOST}": { "auth": "${AUTH_B64}" } - } - } - EOF - - | - /kaniko/executor \ - --context "${PWD}" \ - --dockerfile ci/prod/compose/fx_oracle.dockerfile \ - --destination "${REGISTRY_URL}/fx/oracle:${APP_V}" \ - --build-arg APP_VERSION="${APP_V}" \ - --build-arg GIT_REV="${GIT_REV}" \ - --build-arg BUILD_BRANCH="${BUILD_BRANCH}" \ - --build-arg BUILD_DATE="${BUILD_DATE}" \ - --build-arg BUILD_USER="${BUILD_USER}" \ - --build-arg GO_VERSION="${FX_GO_VERSION}" \ - --single-snapshot - - - name: deploy - image: alpine:latest - depends_on: [ secrets, build-image ] - environment: - VAULT_ADDR: { from_secret: VAULT_ADDR } - VAULT_ROLE_ID: { from_secret: VAULT_APP_ROLE } - VAULT_SECRET_ID: { from_secret: VAULT_SECRET_ID } - commands: - - set -euo pipefail - - apk add --no-cache bash openssh-client rsync coreutils curl sed python3 - - mkdir -p /root/.ssh - - install -m 600 secrets/SSH_KEY /root/.ssh/id_rsa - - sed -i 's/\r$//' ./ci/prod/.env.runtime - - set -a - - . ./ci/prod/.env.runtime - - . ./.env.version - - set +a - - export FX_MONGO_USER="$(./ci/vlt kv_get kv sendico/db user)" - - export FX_MONGO_PASSWORD="$(./ci/vlt kv_get kv sendico/db password)" - - export NATS_USER="$(./ci/vlt kv_get kv sendico/nats user)" - - export NATS_PASSWORD="$(./ci/vlt kv_get kv sendico/nats password)" - - export FX_NATS_URL="nats://${NATS_USER}:${NATS_PASSWORD}@${NATS_HOST}:${NATS_PORT}" - - bash ci/prod/scripts/bootstrap/network.sh - - bash ci/prod/scripts/deploy/fx.sh oracle diff --git a/ci/pipelines/fx/ensure_env_version.sh b/ci/pipelines/fx/ensure_env_version.sh new file mode 100755 index 0000000..bad125d --- /dev/null +++ b/ci/pipelines/fx/ensure_env_version.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +regen_env_version() { + echo "[fx-pipeline] regenerating .env.version defaults" >&2 + local git_rev branch app_v build_date build_user + if [[ -n "${WOODPECKER_COMMIT:-}" ]]; then + git_rev="$(printf '%s' "${WOODPECKER_COMMIT}" | cut -c1-7)" + elif command -v git >/dev/null 2>&1; then + git_rev="$(git rev-parse --short HEAD 2>/dev/null || echo dev)" + else + git_rev="dev" + fi + + if [[ -n "${WOODPECKER_BRANCH:-}" ]]; then + branch="${WOODPECKER_BRANCH}" + elif command -v git >/dev/null 2>&1; then + branch="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo local)" + else + branch="local" + fi + + if [[ -f version ]]; then + app_v="$(cat version 2>/dev/null || echo dev)" + else + app_v="dev" + fi + + build_date="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)" + build_user="${WOODPECKER_MACHINE:-woodpecker}" + + printf "GIT_REV=%s\nBUILD_BRANCH=%s\nAPP_V=%s\nBUILD_DATE=%s\nBUILD_USER=%s\n" \ + "$git_rev" "$branch" "$app_v" "$build_date" "$build_user" > .env.version +} + +ensure_env_version() { + if [[ ! -s ./.env.version ]]; then + regen_env_version + return + fi + + local tmp + tmp="$(mktemp)" + if ! grep -E '^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*=' ./.env.version >"$tmp"; then + rm -f "$tmp" + regen_env_version + return + fi + + if [[ ! -s "$tmp" ]]; then + rm -f "$tmp" + regen_env_version + return + fi + + mv "$tmp" ./.env.version +} + +ensure_env_version