tech/sxchnge
1
0 Fork 0
Code Issues 6 Pull Requests 1 Actions Packages Projects 1 Releases Wiki Activity

Keep Mongo operations online across Vault credential rotation

Browse Source
This commit is contained in:
Stephan D committed 2026-09-12 22:19:36 +02:00
1 parent 80d6917aa7
commit 23f7b78c3b
30 files changed
+1290 -202
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/shared/database/internal/mongo/db.go
+64 -62
View File
@@ -3,15 +3,18 @@ package mongo
import (
"context"
"fmt"
sharedmodel "git.sendico.io/tech/sxchnge/shared/model"
"net"
"os"
"strings"
"time"
mutil "git.sendico.io/tech/sxchnge/shared/config"
"git.sendico.io/tech/sxchnge/shared/database/internal/transaction"
"git.sendico.io/tech/sxchnge/shared/database/source"
"git.sendico.io/tech/sxchnge/shared/database/transaction"
dbvault "git.sendico.io/tech/sxchnge/shared/database/vault"
"git.sendico.io/tech/sxchnge/shared/logging"
sharedmodel "git.sendico.io/tech/sxchnge/shared/model"
"github.com/go-viper/mapstructure/v2"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/readpref"
@@ -20,28 +23,29 @@ import (
// Config represents configuration
type Config struct {
Port *string `mapstructure:"port"`
PortEnv *string `mapstructure:"port_env"`
User *string `mapstructure:"user"`
UserEnv *string `mapstructure:"user_env"`
PasswordEnv string `mapstructure:"password_env"`
Database *string `mapstructure:"database"`
DatabaseEnv *string `mapstructure:"database_env"`
Host *string `mapstructure:"host"`
HostEnv *string `mapstructure:"host_env"`
Hosts []string `mapstructure:"hosts,omitempty"`
HostsEnvPrefix *string `mapstructure:"hosts_env_prefix,omitempty"`
HostsEnvPrefixEnv *string `mapstructure:"hosts_env_prefix_env,omitempty"`
PortsEnvPrefix *string `mapstructure:"ports_env_prefix,omitempty"`
PortsEnvPrefixEnv *string `mapstructure:"ports_env_prefix_env,omitempty"`
URI *string `mapstructure:"uri,omitempty"`
URIEnv *string `mapstructure:"uri_env,omitempty"`
AuthSource *string `mapstructure:"auth_source,omitempty"`
AuthSourceEnv *string `mapstructure:"auth_source_env,omitempty"`
AuthMechanism *string `mapstructure:"auth_mechanism,omitempty"`
AuthMechanismEnv *string `mapstructure:"auth_mechanism_env,omitempty"`
ReplicaSet *string `mapstructure:"replica_set,omitempty"`
ReplicaSetEnv *string `mapstructure:"replica_set_env,omitempty"`
Vault *dbvault.Config `mapstructure:"vault"`
Port *string `mapstructure:"port"`
PortEnv *string `mapstructure:"port_env"`
User *string `mapstructure:"user"`
UserEnv *string `mapstructure:"user_env"`
PasswordEnv string `mapstructure:"password_env"`
Database *string `mapstructure:"database"`
DatabaseEnv *string `mapstructure:"database_env"`
Host *string `mapstructure:"host"`
HostEnv *string `mapstructure:"host_env"`
Hosts []string `mapstructure:"hosts,omitempty"`
HostsEnvPrefix *string `mapstructure:"hosts_env_prefix,omitempty"`
HostsEnvPrefixEnv *string `mapstructure:"hosts_env_prefix_env,omitempty"`
PortsEnvPrefix *string `mapstructure:"ports_env_prefix,omitempty"`
PortsEnvPrefixEnv *string `mapstructure:"ports_env_prefix_env,omitempty"`
URI *string `mapstructure:"uri,omitempty"`
URIEnv *string `mapstructure:"uri_env,omitempty"`
AuthSource *string `mapstructure:"auth_source,omitempty"`
AuthSourceEnv *string `mapstructure:"auth_source_env,omitempty"`
AuthMechanism *string `mapstructure:"auth_mechanism,omitempty"`
AuthMechanismEnv *string `mapstructure:"auth_mechanism_env,omitempty"`
ReplicaSet *string `mapstructure:"replica_set,omitempty"`
ReplicaSetEnv *string `mapstructure:"replica_set_env,omitempty"`
}
type DBSettings struct {
@@ -119,37 +123,42 @@ func dialMongo(logger mlogger.Logger, dbSettings *DBSettings) (*mongo.Client, er
return client, nil
}
func ConnectClient(logger mlogger.Logger, settings sharedmodel.SettingsT) (*mongo.Client, *Config, *DBSettings, error) {
config, dbSettings, err := decodeConfig(logger, settings)
if err != nil {
return nil, nil, nil, err
}
client, err := dialMongo(logger, dbSettings)
if err != nil {
return nil, nil, nil, err
}
return client, config, dbSettings, nil
}
// DB represents the structure of the database
type DB struct {
logger mlogger.Logger
config *DBSettings
client *mongo.Client
}
func (db *DB) db() *mongo.Database {
return db.client.Database(db.config.Database)
}
func (db *DB) MongoDatabase() *mongo.Database {
return db.db()
logger mlogger.Logger
database source.Source
close func(context.Context) error
}
func (db *DB) MongoDatabase() source.Source { return db.database }
func (db *DB) TransactionFactory() transaction.Factory {
return transactionimp.CreateFactory(db.client)
return transactionimp.CreateSourceFactory(db.database)
}
// OpenSource owns either a fixed connection or a rotating Vault-backed source.
func OpenSource(logger mlogger.Logger, settings sharedmodel.SettingsT) (source.Source, func(context.Context) error, error) {
config, dbSettings, err := decodeConfig(logger, settings)
if err != nil {
return nil, nil, err
}
if config.Vault != nil {
issuer, err := dbvault.New(*config.Vault, dbSettings.URI)
if err != nil {
return nil, nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
manager, err := source.New(ctx, logger.Named("credentials"), dbSettings.Database, issuer)
if err != nil {
return nil, nil, err
}
return manager, manager.Close, nil
}
client, err := dialMongo(logger, dbSettings)
if err != nil {
return nil, nil, err
}
return source.Fixed(client.Database(dbSettings.Database)), client.Disconnect, nil
}
func collectReplicaHosts(configuredHosts []string, replicaSet, defaultPort, hostPrefix, portPrefix string) []string {
@@ -217,24 +226,17 @@ func collectReplicaHosts(configuredHosts []string, replicaSet, defaultPort, host
}
func (db *DB) CloseConnection() {
if err := db.client.Disconnect(context.Background()); err != nil {
db.logger.Warn("Failed to close connection", zap.Error(err))
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
if err := db.close(ctx); err != nil {
db.logger.Warn("Failed to drain database connection", zap.Error(err))
}
db.logger.Info("Database connection closed")
}
// NewConnection creates a new database connection
func NewConnection(logger mlogger.Logger, settings sharedmodel.SettingsT) (*DB, error) {
client, _, dbSettings, err := ConnectClient(logger, settings)
database, close, err := OpenSource(logger, settings)
if err != nil {
return nil, err
}
db := &DB{
logger: logger.Named("db"),
config: dbSettings,
client: client,
}
return db, nil
return &DB{logger: logger, database: database, close: close}, nil
}
backend/shared/database/internal/repository/index.go
+26 -7
View File
@@ -12,7 +12,14 @@ import (
)
func (r *MongoRepository) CreateIndex(def *ri.Definition) error {
if r.collection == nil {
ctx := context.Background()
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
if collection == nil {
return merrors.NoData("data collection is not set")
}
if len(def.Keys) == 0 {
@@ -49,28 +56,40 @@ func (r *MongoRepository) CreateIndex(def *ri.Definition) error {
opts.SetSparse(def.Sparse)
}
_, err := r.collection.Indexes().CreateOne(
context.Background(),
_, err = collection.Indexes().CreateOne(
ctx,
mongo.IndexModel{Keys: keys, Options: opts},
)
return err
}
func (r *MongoRepository) DropIndex(ctx context.Context, name string) error {
if r.collection == nil {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
if collection == nil {
return merrors.NoData("data collection is not set")
}
if name == "" {
return merrors.InvalidArgument("index name is required", "index.name")
}
return r.collection.Indexes().DropOne(ctx, name)
return collection.Indexes().DropOne(ctx, name)
}
func (r *MongoRepository) ListIndexDefinitions(ctx context.Context) ([]*ri.Definition, error) {
if r.collection == nil {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return nil, err
}
defer release()
if collection == nil {
return nil, merrors.NoData("data collection is not set")
}
cursor, err := r.collection.Indexes().List(ctx)
cursor, err := collection.Indexes().List(ctx)
if err != nil {
return nil, err
}
backend/shared/database/internal/repository/repository.go
+122 -21
View File
@@ -8,6 +8,7 @@ import (
"git.sendico.io/tech/sxchnge/shared/database/record"
"git.sendico.io/tech/sxchnge/shared/database/repository/builder"
rd "git.sendico.io/tech/sxchnge/shared/database/repository/decoder"
"git.sendico.io/tech/sxchnge/shared/database/source"
"git.sendico.io/tech/sxchnge/shared/database/transaction"
"git.sendico.io/tech/sxchnge/shared/errors"
"go.mongodb.org/mongo-driver/v2/bson"
@@ -17,7 +18,7 @@ import (
type MongoRepository struct {
collectionName string
collection *mongo.Collection
source source.Source
}
func idFilter(id bson.ObjectID) bson.D {
@@ -27,10 +28,23 @@ func idFilter(id bson.ObjectID) bson.D {
}
func NewMongoRepository(db *mongo.Database, collection string) *MongoRepository {
return &MongoRepository{
collectionName: collection,
collection: db.Collection(collection),
return NewRepository(source.Fixed(db), collection)
}
func NewRepository(database source.Source, collection string) *MongoRepository {
return &MongoRepository{collectionName: collection, source: database}
}
func (r *MongoRepository) acquire(ctx context.Context) (*mongo.Collection, context.Context, func(), error) {
if r.source == nil {
return nil, ctx, nil, merrors.FailedPrecondition("database source is not configured")
}
database, scoped, release, err := r.source.Acquire(ctx)
if err != nil {
return nil, ctx, nil, err
}
return database.Collection(r.collectionName), scoped, release, nil
}
func (r *MongoRepository) Collection() string {
@@ -48,8 +62,14 @@ func prepareForWrite(obj storable.Storable) bson.ObjectID {
}
func (r *MongoRepository) Insert(ctx context.Context, obj storable.Storable, getFilter builder.Query) error {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
prepareForWrite(obj)
_, err := r.collection.InsertOne(ctx, obj)
_, err = collection.InsertOne(ctx, obj)
if mongo.IsDuplicateKeyError(err) {
if getFilter != nil {
if err = r.FindOneByFilter(ctx, getFilter, obj); err != nil {
@@ -62,6 +82,12 @@ func (r *MongoRepository) Insert(ctx context.Context, obj storable.Storable, get
}
func (r *MongoRepository) InsertMany(ctx context.Context, objects []storable.Storable) error {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
if len(objects) == 0 {
return nil
}
@@ -72,12 +98,18 @@ func (r *MongoRepository) InsertMany(ctx context.Context, objects []storable.Sto
docs[i] = obj
}
_, err := r.collection.InsertMany(ctx, docs)
_, err = collection.InsertMany(ctx, docs)
return err
}
func (r *MongoRepository) findOneByFilterImp(ctx context.Context, filter bson.D, errMessage string, result storable.Storable) error {
err := r.collection.FindOne(ctx, filter).Decode(result)
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
err = collection.FindOne(ctx, filter).Decode(result)
if errors.Is(err, mongo.ErrNoDocuments) {
return merrors.NoData(errMessage)
}
@@ -94,7 +126,13 @@ func (r *MongoRepository) Get(ctx context.Context, id bson.ObjectID, result stor
type QueryFunc func(ctx context.Context, collection *mongo.Collection) (*mongo.Cursor, error)
func (r *MongoRepository) executeQuery(ctx context.Context, queryFunc QueryFunc, decoder rd.DecodingFunc) error {
cursor, err := queryFunc(ctx, r.collection)
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
cursor, err := queryFunc(ctx, collection)
if errors.Is(err, mongo.ErrNoDocuments) {
return merrors.NoData("no_items_in_array")
}
@@ -111,7 +149,7 @@ func (r *MongoRepository) executeQuery(ctx context.Context, queryFunc QueryFunc,
}
}
return nil
return cursor.Err()
}
func (r *MongoRepository) Aggregate(ctx context.Context, pipeline builder.Pipeline, decoder rd.DecodingFunc) error {
@@ -129,8 +167,14 @@ func (r *MongoRepository) FindManyByFilter(ctx context.Context, query builder.Qu
}
func (r *MongoRepository) FindAllByFilter(ctx context.Context, query builder.Query, result any) error {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
if query == nil {
cursor, err := r.collection.Find(ctx, bson.D{})
cursor, err := collection.Find(ctx, bson.D{})
if err != nil {
return err
}
@@ -139,7 +183,7 @@ func (r *MongoRepository) FindAllByFilter(ctx context.Context, query builder.Que
}()
return cursor.All(ctx, result)
}
cursor, err := r.collection.Find(ctx, query.BuildQuery(), query.BuildOptions())
cursor, err := collection.Find(ctx, query.BuildQuery(), query.BuildOptions())
if err != nil {
return err
}
@@ -154,18 +198,30 @@ func (r *MongoRepository) FindOneByFilter(ctx context.Context, query builder.Que
}
func (r *MongoRepository) Update(ctx context.Context, obj storable.Storable) error {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
obj.Update()
return r.collection.FindOneAndReplace(ctx, idFilter(*obj.GetID()), obj).Err()
return collection.FindOneAndReplace(ctx, idFilter(*obj.GetID()), obj).Err()
}
func (r *MongoRepository) ReplaceOneByFilter(ctx context.Context, query builder.Query, obj storable.Storable) (bool, error) {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return false, err
}
defer release()
if query == nil {
return false, merrors.InvalidArgument("filter is required", "filter")
}
if obj == nil {
return false, merrors.InvalidArgument("object is required", "obj")
}
result, err := r.collection.ReplaceOne(ctx, query.BuildQuery(), obj)
result, err := collection.ReplaceOne(ctx, query.BuildQuery(), obj)
if mongo.IsDuplicateKeyError(err) {
return false, merrors.DataConflict("duplicate_key")
}
@@ -176,21 +232,39 @@ func (r *MongoRepository) ReplaceOneByFilter(ctx context.Context, query builder.
}
func (r *MongoRepository) Upsert(ctx context.Context, obj storable.Storable) error {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
id := prepareForWrite(obj)
_, err := r.collection.ReplaceOne(ctx, idFilter(id), obj, options.Replace().SetUpsert(true))
_, err = collection.ReplaceOne(ctx, idFilter(id), obj, options.Replace().SetUpsert(true))
return err
}
func (r *MongoRepository) Patch(ctx context.Context, id bson.ObjectID, patch builder.Patch) error {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
if id.IsZero() {
return merrors.InvalidArgument("zero id provided while patching", "id")
}
_, err := r.collection.UpdateByID(ctx, id, patch.Build())
_, err = collection.UpdateByID(ctx, id, patch.Build())
return err
}
func (r *MongoRepository) PatchMany(ctx context.Context, query builder.Query, patch builder.Patch) (int, error) {
result, err := r.collection.UpdateMany(ctx, query.BuildQuery(), patch.Build())
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return 0, err
}
defer release()
result, err := collection.UpdateMany(ctx, query.BuildQuery(), patch.Build())
if err != nil {
return 0, err
}
@@ -198,10 +272,16 @@ func (r *MongoRepository) PatchMany(ctx context.Context, query builder.Query, pa
}
func (r *MongoRepository) ListIDs(ctx context.Context, query builder.Query) ([]bson.ObjectID, error) {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return nil, err
}
defer release()
filter := query.BuildQuery()
findOptions := query.BuildOptions().SetProjection(bson.M{storable.IDField: 1})
cursor, err := r.collection.Find(ctx, filter, findOptions)
cursor, err := collection.Find(ctx, filter, findOptions)
if err != nil {
return nil, err
}
@@ -227,24 +307,45 @@ func (r *MongoRepository) ListIDs(ctx context.Context, query builder.Query) ([]b
}
func (r *MongoRepository) Delete(ctx context.Context, id bson.ObjectID) error {
_, err := r.collection.DeleteOne(ctx, idFilter(id))
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
_, err = collection.DeleteOne(ctx, idFilter(id))
return err
}
func (r *MongoRepository) DeleteMany(ctx context.Context, query builder.Query) error {
_, err := r.collection.DeleteMany(ctx, query.BuildQuery())
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
_, err = collection.DeleteMany(ctx, query.BuildQuery())
return err
}
func (r *MongoRepository) Name() string {
return r.collection.Name()
return r.collectionName
}
func (r *MongoRepository) WithTransaction(ctx context.Context, cb transaction.Callback) (any, error) {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return nil, err
}
defer release()
if cb == nil {
return nil, merrors.InvalidArgument("transaction callback is required", "callback")
}
session, err := r.collection.Database().Client().StartSession()
if transaction.ActiveMongoTransaction(ctx) {
return cb(ctx)
}
session, err := collection.Database().Client().StartSession()
if err != nil {
return nil, err
}
backend/shared/database/internal/repository/repository_filter.go
+21 -3
View File
@@ -12,8 +12,14 @@ import (
)
func (r *MongoRepository) FindFirstByFilter(ctx context.Context, query builder.Query, result storable.Storable) error {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return err
}
defer release()
opts := query.BuildOptions().SetLimit(1)
cursor, err := r.collection.Find(ctx, query.BuildQuery(), opts)
cursor, err := collection.Find(ctx, query.BuildQuery(), opts)
if err != nil {
return err
}
@@ -30,10 +36,22 @@ func (r *MongoRepository) FindFirstByFilter(ctx context.Context, query builder.Q
}
func (r *MongoRepository) CountByFilter(ctx context.Context, query builder.Query) (int64, error) {
return r.collection.CountDocuments(ctx, query.BuildQuery())
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return 0, err
}
defer release()
return collection.CountDocuments(ctx, query.BuildQuery())
}
func (r *MongoRepository) PatchOneByFilter(ctx context.Context, query builder.Query, patch builder.Patch, result storable.Storable) (bool, error) {
collection, ctx, release, err := r.acquire(ctx)
if err != nil {
return false, err
}
defer release()
if query == nil {
return false, merrors.InvalidArgument("filter is required", "filter")
}
@@ -43,7 +61,7 @@ func (r *MongoRepository) PatchOneByFilter(ctx context.Context, query builder.Qu
if result == nil {
return false, merrors.InvalidArgument("result is required", "result")
}
err := r.collection.FindOneAndUpdate(
err = collection.FindOneAndUpdate(
ctx,
query.BuildQuery(),
patch.Build(),
backend/shared/database/internal/transaction/factory.go
+6 -9
View File
@@ -1,18 +1,15 @@
package transactionimp
import (
"git.sendico.io/tech/sxchnge/shared/database/source"
"git.sendico.io/tech/sxchnge/shared/database/transaction"
"go.mongodb.org/mongo-driver/v2/mongo"
)
type MongoTransactionFactory struct {
client *mongo.Client
}
type MongoTransactionFactory struct{ database source.Source }
func (mtf *MongoTransactionFactory) CreateTransaction() transaction.Transaction {
return Create(mtf.client)
func (f *MongoTransactionFactory) CreateTransaction() transaction.Transaction {
return &MongoTransaction{database: f.database}
}
func CreateFactory(client *mongo.Client) transaction.Factory {
return &MongoTransactionFactory{client: client}
func CreateSourceFactory(database source.Source) transaction.Factory {
return &MongoTransactionFactory{database: database}
}
backend/shared/database/internal/transaction/transaction.go
+14 -17
View File
@@ -3,32 +3,29 @@ package transactionimp
import (
"context"
"git.sendico.io/tech/sxchnge/shared/database/source"
"git.sendico.io/tech/sxchnge/shared/database/transaction"
"go.mongodb.org/mongo-driver/v2/mongo"
merrors "git.sendico.io/tech/sxchnge/shared/errors"
)
type MongoTransaction struct {
client *mongo.Client
}
type MongoTransaction struct{ database source.Source }
func (mt *MongoTransaction) Execute(ctx context.Context, cb transaction.Callback) (any, error) {
// Reuse an outer customer workflow transaction when a token store joins it.
func (t *MongoTransaction) Execute(ctx context.Context, cb transaction.Callback) (any, error) {
if cb == nil {
return nil, merrors.InvalidArgument("transaction callback is required")
}
database, ctx, release, err := t.database.Acquire(ctx)
if err != nil {
return nil, err
}
defer release()
if transaction.ActiveMongoTransaction(ctx) {
return cb(ctx)
}
session, err := mt.client.StartSession()
session, err := database.Client().StartSession()
if err != nil {
return nil, err
}
defer session.EndSession(ctx)
callback := func(c context.Context) (any, error) {
return cb(c)
}
return session.WithTransaction(ctx, callback)
}
func Create(client *mongo.Client) *MongoTransaction {
return &MongoTransaction{client: client}
return session.WithTransaction(ctx, cb)
}