From 72271cfc9ac767738f295c4b6d4c01293dfb7359 Mon Sep 17 00:00:00 2001 From: Stephan D Date: Mon, 24 Nov 2025 19:10:07 +0100 Subject: [PATCH] migration to replicaset connection --- api/pkg/db/internal/mongo/db.go | 138 ++++++++++++++++++---- api/pkg/db/internal/mongo/mongo.go | 47 ++++++-- ci/prod/.env.runtime | 7 ++ ci/prod/compose/bff.yml | 6 + ci/prod/compose/billing_fees.yml | 6 + ci/prod/compose/chain_gateway.yml | 6 + ci/prod/compose/fx_ingestor.yml | 6 + ci/prod/compose/fx_oracle.yml | 6 + ci/prod/compose/ledger.yml | 6 + ci/prod/compose/notification.yml | 6 + ci/prod/compose/payments_orchestrator.yml | 6 + 11 files changed, 204 insertions(+), 36 deletions(-) diff --git a/api/pkg/db/internal/mongo/db.go b/api/pkg/db/internal/mongo/db.go index 02cc9ff..e73eb39 100755 --- a/api/pkg/db/internal/mongo/db.go +++ b/api/pkg/db/internal/mongo/db.go @@ -2,7 +2,10 @@ package mongo import ( "context" + "fmt" + "net" "os" + "strings" "github.com/mitchellh/mapstructure" "github.com/tech/sendico/pkg/auth" @@ -28,33 +31,40 @@ import ( "github.com/tech/sendico/pkg/mservice" mutil "github.com/tech/sendico/pkg/mutil/config" "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/readpref" "go.uber.org/zap" ) // Config represents configuration type Config struct { - Port *string `mapstructure:"port"` - PortEnv *string `mapstructure:"port_env"` - User *string `mapstructure:"user"` - UserEnv *string `mapstructure:"user_env"` - PasswordEnv string `mapstructure:"password_env"` - Database *string `mapstructure:"database"` - DatabaseEnv *string `mapstructure:"database_env"` - Host *string `mapstructure:"host"` - HostEnv *string `mapstructure:"host_env"` - AuthSource *string `mapstructure:"auth_source,omitempty"` - AuthSourceEnv *string `mapstructure:"auth_source_env,omitempty"` - AuthMechanism *string `mapstructure:"auth_mechanism,omitempty"` - AuthMechanismEnv *string `mapstructure:"auth_mechanism_env,omitempty"` - ReplicaSet *string `mapstructure:"replica_set,omitempty"` - ReplicaSetEnv *string `mapstructure:"replica_set_env,omitempty"` - Enforcer *auth.Config `mapstructure:"enforcer"` + Port *string `mapstructure:"port"` + PortEnv *string `mapstructure:"port_env"` + User *string `mapstructure:"user"` + UserEnv *string `mapstructure:"user_env"` + PasswordEnv string `mapstructure:"password_env"` + Database *string `mapstructure:"database"` + DatabaseEnv *string `mapstructure:"database_env"` + Host *string `mapstructure:"host"` + HostEnv *string `mapstructure:"host_env"` + Hosts []string `mapstructure:"hosts,omitempty"` + HostsEnvPrefix *string `mapstructure:"hosts_env_prefix,omitempty"` + HostsEnvPrefixEnv *string `mapstructure:"hosts_env_prefix_env,omitempty"` + PortsEnvPrefix *string `mapstructure:"ports_env_prefix,omitempty"` + PortsEnvPrefixEnv *string `mapstructure:"ports_env_prefix_env,omitempty"` + URI *string `mapstructure:"uri,omitempty"` + URIEnv *string `mapstructure:"uri_env,omitempty"` + AuthSource *string `mapstructure:"auth_source,omitempty"` + AuthSourceEnv *string `mapstructure:"auth_source_env,omitempty"` + AuthMechanism *string `mapstructure:"auth_mechanism,omitempty"` + AuthMechanismEnv *string `mapstructure:"auth_mechanism_env,omitempty"` + ReplicaSet *string `mapstructure:"replica_set,omitempty"` + ReplicaSetEnv *string `mapstructure:"replica_set_env,omitempty"` + Enforcer *auth.Config `mapstructure:"enforcer"` } type DBSettings struct { Host string + Hosts []string Port string User string Password string @@ -62,6 +72,7 @@ type DBSettings struct { AuthSource string AuthMechanism string ReplicaSet string + URI string } func newProtectedDB[T any]( @@ -87,6 +98,19 @@ func Config2DBSettings(logger mlogger.Logger, config *Config) *DBSettings { p.AuthSource = mutil.GetConfigValue(logger, "auth_source", "auth_source_env", config.AuthSource, config.AuthSourceEnv) p.AuthMechanism = mutil.GetConfigValue(logger, "auth_mechanism", "auth_mechanism_env", config.AuthMechanism, config.AuthMechanismEnv) p.ReplicaSet = mutil.GetConfigValue(logger, "replica_set", "replica_set_env", config.ReplicaSet, config.ReplicaSetEnv) + p.URI = mutil.GetConfigValue(logger, "uri", "uri_env", config.URI, config.URIEnv) + + hostPrefix := mutil.GetConfigValue(logger, "hosts_env_prefix", "hosts_env_prefix_env", config.HostsEnvPrefix, config.HostsEnvPrefixEnv) + portPrefix := mutil.GetConfigValue(logger, "ports_env_prefix", "ports_env_prefix_env", config.PortsEnvPrefix, config.PortsEnvPrefixEnv) + + if hostPrefix == "" && p.ReplicaSet != "" { + hostPrefix = "MONGO_HOSTS_" + } + if portPrefix == "" && p.ReplicaSet != "" { + portPrefix = "MONGO_PORTS_" + } + + p.Hosts = collectReplicaHosts(config.Hosts, p.ReplicaSet, p.Port, hostPrefix, portPrefix) return p } @@ -101,21 +125,19 @@ func decodeConfig(logger mlogger.Logger, settings model.SettingsT) (*Config, *DB } func dialMongo(logger mlogger.Logger, dbSettings *DBSettings) (*mongo.Client, error) { - cred := options.Credential{ - AuthMechanism: dbSettings.AuthMechanism, - AuthSource: dbSettings.AuthSource, - Username: dbSettings.User, - Password: dbSettings.Password, - } - dbURI := buildURI(dbSettings) + opts := buildOptions(dbSettings) - client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(dbURI).SetAuth(cred)) + client, err := mongo.Connect(context.Background(), opts) if err != nil { logger.Error("Unable to connect to database", zap.Error(err)) return nil, err } - logger.Info("Connected successfully", zap.String("uri", dbURI)) + if dbSettings.URI != "" { + logger.Info("Connected successfully", zap.Bool("uri_provided", true)) + } else { + logger.Info("Connected successfully", zap.Strings("hosts", opts.Hosts), zap.String("replica_set", dbSettings.ReplicaSet)) + } if err := client.Ping(context.Background(), readpref.Primary()); err != nil { logger.Error("Unable to ping database", zap.Error(err)) @@ -199,6 +221,70 @@ func (db *DB) TransactionFactory() transaction.Factory { return transactionimp.CreateFactory(db.client) } +func collectReplicaHosts(configuredHosts []string, replicaSet, defaultPort, hostPrefix, portPrefix string) []string { + normalize := func(host, port string) string { + host = strings.TrimSpace(host) + port = strings.TrimSpace(port) + if host == "" { + return "" + } + // If host already has a port, keep it; otherwise apply provided/default port. + if _, _, err := net.SplitHostPort(host); err == nil { + return host + } + if port != "" { + return net.JoinHostPort(host, port) + } + if defaultPort != "" { + return net.JoinHostPort(host, defaultPort) + } + return host + } + + appendHost := func(list []string, host, port string) []string { + if normalized := normalize(host, port); normalized != "" { + return append(list, normalized) + } + return list + } + + var hosts []string + for _, h := range configuredHosts { + hosts = appendHost(hosts, h, "") + } + + if replicaSet == "" || hostPrefix == "" { + return hosts + } + + index := 0 + for { + hostEnv := os.Getenv(fmt.Sprintf("%s%d", hostPrefix, index)) + portEnv := "" + if portPrefix != "" { + portEnv = os.Getenv(fmt.Sprintf("%s%d", portPrefix, index)) + } + + if hostEnv == "" && index == 0 { + hostEnv = os.Getenv(fmt.Sprintf("%s%d", hostPrefix, 1)) + if portPrefix != "" { + portEnv = os.Getenv(fmt.Sprintf("%s%d", portPrefix, 1)) + } + if hostEnv == "" { + break + } + index = 1 + } else if hostEnv == "" { + break + } + + hosts = appendHost(hosts, hostEnv, portEnv) + index++ + } + + return hosts +} + func (db *DB) Permissions() auth.Provider { return db } diff --git a/api/pkg/db/internal/mongo/mongo.go b/api/pkg/db/internal/mongo/mongo.go index fdce94d..60d3c1c 100644 --- a/api/pkg/db/internal/mongo/mongo.go +++ b/api/pkg/db/internal/mongo/mongo.go @@ -1,22 +1,49 @@ package mongo import ( - "net/url" + "net" + "strings" + + "go.mongodb.org/mongo-driver/mongo/options" ) -func buildURI(s *DBSettings) string { - u := &url.URL{ - Scheme: "mongodb", - Host: s.Host, - Path: "/" + url.PathEscape(s.Database), // /my%20db +func buildOptions(s *DBSettings) *options.ClientOptions { + opts := options.Client() + + if s.URI != "" { + return opts.ApplyURI(s.URI) + } + + hosts := make([]string, 0, len(s.Hosts)+1) + for _, h := range s.Hosts { + if trimmed := strings.TrimSpace(h); trimmed != "" { + hosts = append(hosts, trimmed) + } + } + + if len(hosts) == 0 && s.Host != "" { + host := s.Host + if _, _, err := net.SplitHostPort(host); err != nil && s.Port != "" { + host = net.JoinHostPort(host, s.Port) + } + hosts = append(hosts, host) + } + + if len(hosts) > 0 { + opts.SetHosts(hosts) } - q := url.Values{} if s.ReplicaSet != "" { - q.Set("replicaSet", s.ReplicaSet) + opts.SetReplicaSet(s.ReplicaSet) } - u.RawQuery = q.Encode() + cred := options.Credential{ + AuthMechanism: s.AuthMechanism, + AuthSource: s.AuthSource, + Username: s.User, + Password: s.Password, + } + opts.SetAuth(cred) - return u.String() + return opts } diff --git a/ci/prod/.env.runtime b/ci/prod/.env.runtime index d67f10c..5d0b478 100644 --- a/ci/prod/.env.runtime +++ b/ci/prod/.env.runtime @@ -8,6 +8,13 @@ MONGO_REPLICA_SET=sendico-rs MONGO_AUTH_SOURCE=admin MONGO_DATABASE=sendico MONGO_ARCH=linux/arm64 +MONGO_HOSTS_0=sendico_db1 +MONGO_PORTS_0=27017 +MONGO_HOSTS_1=sendico_db2 +MONGO_PORTS_1=27017 +MONGO_HOSTS_2=sendico_db3 +MONGO_PORTS_2=27017 + PERMISSION_MODEL=/app/env/permissions_model.conf PERMISSION_COLLECTION=permissions PERMISSION_TIMEOUT=5 diff --git a/ci/prod/compose/bff.yml b/ci/prod/compose/bff.yml index 550c6b0..9387d20 100644 --- a/ci/prod/compose/bff.yml +++ b/ci/prod/compose/bff.yml @@ -37,6 +37,12 @@ services: MONGO_PASSWORD: ${MONGO_PASSWORD} MONGO_AUTH_SOURCE: ${MONGO_AUTH_SOURCE} MONGO_REPLICA_SET: ${MONGO_REPLICA_SET} + MONGO_HOSTS_0: ${MONGO_HOSTS_0} + MONGO_PORTS_0: ${MONGO_PORTS_0} + MONGO_HOSTS_1: ${MONGO_HOSTS_1} + MONGO_PORTS_1: ${MONGO_PORTS_1} + MONGO_HOSTS_2: ${MONGO_HOSTS_2} + MONGO_PORTS_2: ${MONGO_PORTS_2} PERMISSION_MODEL: ${PERMISSION_MODEL} PERMISSION_COLLECTION: ${PERMISSION_COLLECTION} PERMISSION_TIMEOUT: ${PERMISSION_TIMEOUT} diff --git a/ci/prod/compose/billing_fees.yml b/ci/prod/compose/billing_fees.yml index ed6ea85..215cd30 100644 --- a/ci/prod/compose/billing_fees.yml +++ b/ci/prod/compose/billing_fees.yml @@ -25,6 +25,12 @@ services: FEES_MONGO_PASSWORD: ${FEES_MONGO_PASSWORD} FEES_MONGO_AUTH_SOURCE: ${FEES_MONGO_AUTH_SOURCE} FEES_MONGO_REPLICA_SET: ${FEES_MONGO_REPLICA_SET} + MONGO_HOSTS_0: ${MONGO_HOSTS_0} + MONGO_PORTS_0: ${MONGO_PORTS_0} + MONGO_HOSTS_1: ${MONGO_HOSTS_1} + MONGO_PORTS_1: ${MONGO_PORTS_1} + MONGO_HOSTS_2: ${MONGO_HOSTS_2} + MONGO_PORTS_2: ${MONGO_PORTS_2} FEES_GRPC_PORT: ${FEES_GRPC_PORT} FEES_METRICS_PORT: ${FEES_METRICS_PORT} NATS_URL: ${NATS_URL} diff --git a/ci/prod/compose/chain_gateway.yml b/ci/prod/compose/chain_gateway.yml index 4a3a770..045ddae 100644 --- a/ci/prod/compose/chain_gateway.yml +++ b/ci/prod/compose/chain_gateway.yml @@ -33,6 +33,12 @@ services: CHAIN_GATEWAY_MONGO_PASSWORD: ${CHAIN_GATEWAY_MONGO_PASSWORD} CHAIN_GATEWAY_MONGO_AUTH_SOURCE: ${CHAIN_GATEWAY_MONGO_AUTH_SOURCE} CHAIN_GATEWAY_MONGO_REPLICA_SET: ${CHAIN_GATEWAY_MONGO_REPLICA_SET} + MONGO_HOSTS_0: ${MONGO_HOSTS_0} + MONGO_PORTS_0: ${MONGO_PORTS_0} + MONGO_HOSTS_1: ${MONGO_HOSTS_1} + MONGO_PORTS_1: ${MONGO_PORTS_1} + MONGO_HOSTS_2: ${MONGO_HOSTS_2} + MONGO_PORTS_2: ${MONGO_PORTS_2} NATS_URL: ${NATS_URL} NATS_HOST: ${NATS_HOST} NATS_PORT: ${NATS_PORT} diff --git a/ci/prod/compose/fx_ingestor.yml b/ci/prod/compose/fx_ingestor.yml index 0414eb8..68625c9 100644 --- a/ci/prod/compose/fx_ingestor.yml +++ b/ci/prod/compose/fx_ingestor.yml @@ -25,6 +25,12 @@ services: FX_MONGO_PASSWORD: ${FX_MONGO_PASSWORD} FX_MONGO_AUTH_SOURCE: ${FX_MONGO_AUTH_SOURCE} FX_MONGO_REPLICA_SET: ${FX_MONGO_REPLICA_SET} + MONGO_HOSTS_0: ${MONGO_HOSTS_0} + MONGO_PORTS_0: ${MONGO_PORTS_0} + MONGO_HOSTS_1: ${MONGO_HOSTS_1} + MONGO_PORTS_1: ${MONGO_PORTS_1} + MONGO_HOSTS_2: ${MONGO_HOSTS_2} + MONGO_PORTS_2: ${MONGO_PORTS_2} FX_INGESTOR_METRICS_PORT: ${FX_INGESTOR_METRICS_PORT} command: ["--config.file", "/app/config.yml"] ports: diff --git a/ci/prod/compose/fx_oracle.yml b/ci/prod/compose/fx_oracle.yml index 88d468c..b777ed7 100644 --- a/ci/prod/compose/fx_oracle.yml +++ b/ci/prod/compose/fx_oracle.yml @@ -25,6 +25,12 @@ services: FX_MONGO_PASSWORD: ${FX_MONGO_PASSWORD} FX_MONGO_AUTH_SOURCE: ${FX_MONGO_AUTH_SOURCE} FX_MONGO_REPLICA_SET: ${FX_MONGO_REPLICA_SET} + MONGO_HOSTS_0: ${MONGO_HOSTS_0} + MONGO_PORTS_0: ${MONGO_PORTS_0} + MONGO_HOSTS_1: ${MONGO_HOSTS_1} + MONGO_PORTS_1: ${MONGO_PORTS_1} + MONGO_HOSTS_2: ${MONGO_HOSTS_2} + MONGO_PORTS_2: ${MONGO_PORTS_2} FX_ORACLE_GRPC_PORT: ${FX_ORACLE_GRPC_PORT} FX_ORACLE_METRICS_PORT: ${FX_ORACLE_METRICS_PORT} NATS_URL: ${FX_NATS_URL} diff --git a/ci/prod/compose/ledger.yml b/ci/prod/compose/ledger.yml index 8ea049c..c32d6e4 100644 --- a/ci/prod/compose/ledger.yml +++ b/ci/prod/compose/ledger.yml @@ -25,6 +25,12 @@ services: LEDGER_MONGO_PASSWORD: ${LEDGER_MONGO_PASSWORD} LEDGER_MONGO_AUTH_SOURCE: ${LEDGER_MONGO_AUTH_SOURCE} LEDGER_MONGO_REPLICA_SET: ${LEDGER_MONGO_REPLICA_SET} + MONGO_HOSTS_0: ${MONGO_HOSTS_0} + MONGO_PORTS_0: ${MONGO_PORTS_0} + MONGO_HOSTS_1: ${MONGO_HOSTS_1} + MONGO_PORTS_1: ${MONGO_PORTS_1} + MONGO_HOSTS_2: ${MONGO_HOSTS_2} + MONGO_PORTS_2: ${MONGO_PORTS_2} LEDGER_GRPC_PORT: ${LEDGER_GRPC_PORT} LEDGER_METRICS_PORT: ${LEDGER_METRICS_PORT} NATS_URL: ${NATS_URL} diff --git a/ci/prod/compose/notification.yml b/ci/prod/compose/notification.yml index 2ad532b..3ca2723 100644 --- a/ci/prod/compose/notification.yml +++ b/ci/prod/compose/notification.yml @@ -41,6 +41,12 @@ services: MONGO_PASSWORD: ${MONGO_PASSWORD} MONGO_AUTH_SOURCE: ${MONGO_AUTH_SOURCE} MONGO_REPLICA_SET: ${MONGO_REPLICA_SET} + MONGO_HOSTS_0: ${MONGO_HOSTS_0} + MONGO_PORTS_0: ${MONGO_PORTS_0} + MONGO_HOSTS_1: ${MONGO_HOSTS_1} + MONGO_PORTS_1: ${MONGO_PORTS_1} + MONGO_HOSTS_2: ${MONGO_HOSTS_2} + MONGO_PORTS_2: ${MONGO_PORTS_2} PERMISSION_MODEL: ${PERMISSION_MODEL} PERMISSION_COLLECTION: ${PERMISSION_COLLECTION} PERMISSION_TIMEOUT: ${PERMISSION_TIMEOUT} diff --git a/ci/prod/compose/payments_orchestrator.yml b/ci/prod/compose/payments_orchestrator.yml index 6547f06..ab4de19 100644 --- a/ci/prod/compose/payments_orchestrator.yml +++ b/ci/prod/compose/payments_orchestrator.yml @@ -25,6 +25,12 @@ services: PAYMENTS_MONGO_PASSWORD: ${PAYMENTS_MONGO_PASSWORD} PAYMENTS_MONGO_AUTH_SOURCE: ${PAYMENTS_MONGO_AUTH_SOURCE} PAYMENTS_MONGO_REPLICA_SET: ${PAYMENTS_MONGO_REPLICA_SET} + MONGO_HOSTS_0: ${MONGO_HOSTS_0} + MONGO_PORTS_0: ${MONGO_PORTS_0} + MONGO_HOSTS_1: ${MONGO_HOSTS_1} + MONGO_PORTS_1: ${MONGO_PORTS_1} + MONGO_HOSTS_2: ${MONGO_HOSTS_2} + MONGO_PORTS_2: ${MONGO_PORTS_2} NATS_URL: ${NATS_URL} NATS_HOST: ${NATS_HOST} NATS_PORT: ${NATS_PORT}