Keep Mongo operations online across Vault credential rotation
This commit is contained in:
Stephan D committed
1 parent
80d6917aa7
commit
23f7b78c3b
30 files changed
+1290
-202
No files matched your search
@@ -0,0 +1,275 @@
|
||||
package db_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
db "git.sendico.io/tech/sxchnge/shared/database"
|
||||
dependencies "git.sendico.io/tech/sxchnge/shared/database/dependencies"
|
||||
storable "git.sendico.io/tech/sxchnge/shared/database/record"
|
||||
"git.sendico.io/tech/sxchnge/shared/database/repository"
|
||||
merrors "git.sendico.io/tech/sxchnge/shared/errors"
|
||||
"git.sendico.io/tech/sxchnge/shared/model"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/testcontainers/testcontainers-go/modules/mongodb"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type rotationRecord struct {
|
||||
storable.Base `bson:",inline"`
|
||||
Value string `bson:"value"`
|
||||
}
|
||||
|
||||
func (*rotationRecord) Collection() string { return "rotation" }
|
||||
|
||||
type leaseServer struct {
|
||||
admin *mongo.Database
|
||||
issued atomic.Int64
|
||||
revoked sync.Map
|
||||
outage atomic.Bool
|
||||
badPassword atomic.Bool
|
||||
}
|
||||
|
||||
func (s *leaseServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if s.outage.Load() {
|
||||
w.WriteHeader(503)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch r.URL.Path {
|
||||
case "/v1/auth/approle/login":
|
||||
// A unique token owns each credential in this fake Vault HTTP boundary.
|
||||
token := fmt.Sprintf("test-token-%d", s.issued.Load()+1)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"auth": map[string]any{"client_token": token, "lease_duration": 60}})
|
||||
case "/v1/database/creds/bff":
|
||||
n := s.issued.Add(1)
|
||||
user := fmt.Sprintf("lease%d", n)
|
||||
err := s.admin.RunCommand(r.Context(), bson.D{{Key: "createUser", Value: user}, {Key: "pwd", Value: "test-password"}, {Key: "roles", Value: bson.A{bson.M{"role": "readWrite", "db": "rotation"}}}}).Err()
|
||||
if err != nil {
|
||||
w.WriteHeader(500)
|
||||
return
|
||||
}
|
||||
password := "test-password"
|
||||
if s.badPassword.Load() {
|
||||
password = "wrong-password"
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"lease_id": user, "lease_duration": 8, "data": map[string]string{"username": user, "password": password}})
|
||||
case "/v1/auth/token/revoke-self":
|
||||
user := "lease" + strings.TrimPrefix(r.Header.Get("X-Vault-Token"), "test-token-")
|
||||
_ = s.admin.RunCommand(r.Context(), bson.D{{Key: "dropUser", Value: user}}).Err()
|
||||
s.revoked.Store(user, true)
|
||||
w.WriteHeader(204)
|
||||
default:
|
||||
w.WriteHeader(404)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRotationPreservesRequestsCursorsAndTransactions(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
container, err := mongodb.Run(ctx, "mongo:latest", mongodb.WithReplicaSet("rs"), mongodb.WithUsername("root"), mongodb.WithPassword("test-admin-password"))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = container.Terminate(context.Background()) })
|
||||
raw, err := container.ConnectionString(ctx)
|
||||
require.NoError(t, err)
|
||||
uri, err := url.Parse(raw)
|
||||
require.NoError(t, err)
|
||||
q := uri.Query()
|
||||
q.Set("directConnection", "true")
|
||||
q.Set("authSource", "admin")
|
||||
uri.RawQuery = q.Encode()
|
||||
admin, err := mongo.Connect(options.Client().ApplyURI(uri.String()))
|
||||
require.NoError(t, err)
|
||||
defer admin.Disconnect(context.Background())
|
||||
fake := &leaseServer{admin: admin.Database("admin")}
|
||||
vault := httptest.NewServer(fake)
|
||||
defer vault.Close()
|
||||
directory := t.TempDir()
|
||||
role := filepath.Join(directory, "role")
|
||||
secret := filepath.Join(directory, "secret")
|
||||
require.NoError(t, os.WriteFile(role, []byte("test-role"), 0600))
|
||||
require.NoError(t, os.WriteFile(secret, []byte("test-secret"), 0600))
|
||||
uri.User = nil
|
||||
uri.Path = "/rotation"
|
||||
factory, err := db.NewConnection(zap.NewNop(), &db.Config{Driver: db.Mongo, Settings: model.SettingsT{"database": "rotation", "uri": uri.String(), "vault": map[string]any{"address": vault.URL, "role": "bff", "role_id_file": role, "secret_id_file": secret}}})
|
||||
require.NoError(t, err)
|
||||
defer factory.CloseConnection()
|
||||
deps, err := dependencies.ResolveDependencies(factory)
|
||||
require.NoError(t, err)
|
||||
repo := repository.CreateRepository(deps.Database, "rotation")
|
||||
require.NoError(t, repo.Insert(ctx, &rotationRecord{Value: "cursor-fixture"}, nil))
|
||||
|
||||
var failures, requests atomic.Int64
|
||||
httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, e := repo.CountByFilter(r.Context(), repository.Query()); e != nil {
|
||||
failures.Add(1)
|
||||
w.WriteHeader(503)
|
||||
return
|
||||
}
|
||||
requests.Add(1)
|
||||
w.WriteHeader(200)
|
||||
}))
|
||||
defer httpServer.Close()
|
||||
loadCtx, stopLoad := context.WithCancel(ctx)
|
||||
var load sync.WaitGroup
|
||||
defer func() { stopLoad(); load.Wait() }()
|
||||
for range 4 {
|
||||
load.Add(1)
|
||||
go func() {
|
||||
defer load.Done()
|
||||
for loadCtx.Err() == nil {
|
||||
res, e := http.Get(httpServer.URL)
|
||||
if e != nil {
|
||||
failures.Add(1)
|
||||
} else {
|
||||
res.Body.Close()
|
||||
if res.StatusCode != 200 {
|
||||
failures.Add(1)
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// A cursor and a transaction must keep generation 1 alive through publication.
|
||||
first, scope, release, err := deps.Database.Acquire(ctx)
|
||||
require.NoError(t, err)
|
||||
firstClient := first.Client()
|
||||
release()
|
||||
_, _, _, err = deps.Database.Acquire(scope)
|
||||
require.Error(t, err, "escaped operation scope must be rejected")
|
||||
txEntered := make(chan struct{})
|
||||
finishTx := make(chan struct{})
|
||||
txDone := make(chan error, 1)
|
||||
var entered sync.Once
|
||||
go func() {
|
||||
_, e := factory.TransactionFactory().CreateTransaction().Execute(ctx, func(tx context.Context) (any, error) {
|
||||
if e := repo.Insert(tx, &rotationRecord{Value: "transaction-before"}, nil); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
entered.Do(func() { close(txEntered) })
|
||||
select {
|
||||
case <-finishTx:
|
||||
case <-tx.Done():
|
||||
return nil, tx.Err()
|
||||
}
|
||||
// Nested transaction must share the same session/generation.
|
||||
return repo.WithTransaction(tx, func(nested context.Context) (any, error) {
|
||||
return nil, repo.Insert(nested, &rotationRecord{Value: "transaction-after"}, nil)
|
||||
})
|
||||
})
|
||||
txDone <- e
|
||||
}()
|
||||
select {
|
||||
case <-txEntered:
|
||||
case err := <-txDone:
|
||||
t.Fatalf("transaction did not enter: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
cursorEntered := make(chan struct{})
|
||||
finishCursor := make(chan struct{})
|
||||
cursorDone := make(chan error, 1)
|
||||
go func() {
|
||||
var once sync.Once
|
||||
cursorDone <- repo.FindManyByFilter(ctx, repository.Query(), func(d *mongo.Cursor) error {
|
||||
once.Do(func() { close(cursorEntered) })
|
||||
select {
|
||||
case <-finishCursor:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case <-cursorEntered:
|
||||
case err := <-cursorDone:
|
||||
t.Fatalf("cursor did not enter: %v", err)
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
require.Eventually(t, func() bool {
|
||||
database, _, free, e := deps.Database.Acquire(ctx)
|
||||
if e != nil {
|
||||
return false
|
||||
}
|
||||
defer free()
|
||||
return database.Client() != firstClient
|
||||
}, 6*time.Second, 20*time.Millisecond)
|
||||
_, revoked := fake.revoked.Load("lease1")
|
||||
require.False(t, revoked, "in-flight work lost its credentials")
|
||||
close(finishTx)
|
||||
require.NoError(t, <-txDone)
|
||||
_, revoked = fake.revoked.Load("lease1")
|
||||
require.False(t, revoked, "cursor must retain old pool")
|
||||
close(finishCursor)
|
||||
require.NoError(t, <-cursorDone)
|
||||
require.Eventually(t, func() bool { _, ok := fake.revoked.Load("lease1"); return ok }, 2*time.Second, 20*time.Millisecond)
|
||||
count, err := repo.CountByFilter(ctx, repository.Filter("value", "transaction-after"))
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, 1, count)
|
||||
|
||||
// Reject a candidate with invalid credentials while retaining the valid pool.
|
||||
fake.badPassword.Store(true)
|
||||
before := fake.issued.Load()
|
||||
require.Eventually(t, func() bool { return fake.issued.Load() > before }, 5*time.Second, 20*time.Millisecond)
|
||||
require.Eventually(t, func() bool { _, ok := fake.revoked.Load(fmt.Sprintf("lease%d", before+1)); return ok }, time.Second, 10*time.Millisecond)
|
||||
require.NoError(t, repo.Insert(ctx, &rotationRecord{Value: "during-bad-refresh"}, nil))
|
||||
fake.badPassword.Store(false)
|
||||
fake.outage.Store(true)
|
||||
require.NoError(t, repo.Insert(ctx, &rotationRecord{Value: "during-vault-outage"}, nil))
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
fake.outage.Store(false)
|
||||
require.Eventually(t, func() bool { return fake.issued.Load() > before+1 }, 3*time.Second, 20*time.Millisecond)
|
||||
stopLoad()
|
||||
load.Wait()
|
||||
require.Greater(t, requests.Load(), int64(100))
|
||||
require.Zero(t, failures.Load(), "HTTP requests failed during rotation")
|
||||
|
||||
// If Vault stays unavailable beyond the lease, fail closed without restarting.
|
||||
fake.outage.Store(true)
|
||||
require.Eventually(t, func() bool { _, e := repo.CountByFilter(ctx, repository.Query()); return e != nil }, 10*time.Second, 50*time.Millisecond)
|
||||
fake.outage.Store(false)
|
||||
require.Eventually(t, func() bool { _, e := repo.CountByFilter(ctx, repository.Query()); return e == nil }, 6*time.Second, 20*time.Millisecond)
|
||||
// Transaction errors still roll back all writes after generation changes.
|
||||
_, err = repo.WithTransaction(ctx, func(tx context.Context) (any, error) {
|
||||
if e := repo.Insert(tx, &rotationRecord{Value: "rollback"}, nil); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return nil, merrors.Internal("injected rollback")
|
||||
})
|
||||
require.Error(t, err)
|
||||
count, err = repo.CountByFilter(ctx, repository.Filter("value", "rollback"))
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, count)
|
||||
held, heldContext, releaseHeld, err := deps.Database.Acquire(ctx)
|
||||
require.NoError(t, err)
|
||||
manager := deps.Database.(interface{ Close(context.Context) error })
|
||||
short, stop := context.WithTimeout(ctx, 30*time.Millisecond)
|
||||
require.ErrorIs(t, manager.Close(short), context.DeadlineExceeded)
|
||||
stop()
|
||||
_, _, _, err = deps.Database.Acquire(ctx)
|
||||
require.Error(t, err)
|
||||
require.NoError(t, held.Client().Ping(heldContext, nil), "shutdown must drain, not kill held work")
|
||||
releaseHeld()
|
||||
closed, stop := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer stop()
|
||||
require.NoError(t, manager.Close(closed))
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user