Implement notifications service with shared messaging and email drivers (#36)
This commit is contained in:
Stephan D committed
1 parent
36277b6c45
commit
5c2d36281b
54 files changed
+3746
-6
No files matched your search
@@ -0,0 +1,209 @@
|
||||
package service_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.sendico.io/tech/sxchnge/notifications/bus"
|
||||
"git.sendico.io/tech/sxchnge/notifications/email"
|
||||
"git.sendico.io/tech/sxchnge/notifications/model"
|
||||
"git.sendico.io/tech/sxchnge/notifications/service"
|
||||
"git.sendico.io/tech/sxchnge/notifications/store"
|
||||
"git.sendico.io/tech/sxchnge/notifications/templates"
|
||||
db "git.sendico.io/tech/sxchnge/shared/database"
|
||||
"git.sendico.io/tech/sxchnge/shared/database/repository"
|
||||
merrors "git.sendico.io/tech/sxchnge/shared/errors"
|
||||
"git.sendico.io/tech/sxchnge/shared/messaging"
|
||||
msgruntime "git.sendico.io/tech/sxchnge/shared/messaging/runtime"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/modules/mongodb"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type sender struct {
|
||||
calls atomic.Int32
|
||||
state model.State
|
||||
}
|
||||
|
||||
func (s *sender) Send(context.Context, string, model.Email) email.Outcome {
|
||||
s.calls.Add(1)
|
||||
return email.Outcome{State: s.state, Code: "test", RetryAfter: time.Millisecond}
|
||||
}
|
||||
func TestDurableDelivery(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
container, err := mongodb.Run(ctx, "mongo:8.0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = container.Terminate(context.Background()) }()
|
||||
uri, err := container.ConnectionString(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := db.ConnectMongo(zap.NewNop(), &db.Config{Driver: db.Mongo, Settings: map[string]any{"uri": uri, "database": "notifications_test"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Disconnect(context.Background())
|
||||
repo, err := store.New(zap.NewNop(), conn.Database())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
catalog, err := templates.New([]templates.Definition{{ID: "test", Version: "1", DefaultLocale: "en", Parameters: []string{"name"}, Locales: map[string]templates.Content{"en": {Subject: "Hello", HTML: "<p>{{.name}}</p>"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
driver := &sender{state: model.Accepted}
|
||||
svc := service.New(repo, catalog, driver, "resend", zap.NewNop())
|
||||
request := model.Request{Key: "duplicate", UserID: "u", TemplateID: "test", TemplateVersion: "1", Parameters: map[string]string{"name": "A & B"}, Address: "user@example.com"}
|
||||
var wg sync.WaitGroup
|
||||
ids := make(chan string, 10)
|
||||
for range 10 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
n, e := svc.Send(ctx, "product", request)
|
||||
if e != nil {
|
||||
t.Error(e)
|
||||
return
|
||||
}
|
||||
ids <- n.ID.Hex()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(ids)
|
||||
id := ""
|
||||
for value := range ids {
|
||||
if id != "" && value != id {
|
||||
t.Error("idempotency race")
|
||||
}
|
||||
id = value
|
||||
}
|
||||
if id == "" {
|
||||
t.Fatal("no notification")
|
||||
}
|
||||
request.Parameters = map[string]string{"name": "Changed"}
|
||||
if _, err = svc.Send(ctx, "product", request); !errors.Is(err, merrors.ErrDataConflict) {
|
||||
t.Fatal("missing conflict", err)
|
||||
}
|
||||
if _, err = svc.Get(ctx, "other", id); !errors.Is(err, merrors.ErrNoData) {
|
||||
t.Fatal("caller isolation failed", err)
|
||||
}
|
||||
stored, err := svc.Get(ctx, "product", id)
|
||||
if err != nil || stored.Email.HTML != "<p>A & B</p>" {
|
||||
t.Fatal(stored, err)
|
||||
}
|
||||
for range 10 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if _, e := svc.Dispatch(ctx, id); e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if driver.calls.Load() != 1 {
|
||||
t.Fatalf("sent %d times", driver.calls.Load())
|
||||
}
|
||||
stored, err = svc.Get(ctx, "product", id)
|
||||
if err != nil || stored.State != model.Accepted || stored.Email.Address != "" {
|
||||
t.Fatal("completion/scrubbing failed", stored, err)
|
||||
}
|
||||
// A stale sending lease is never automatically resent.
|
||||
request.Key = "stale"
|
||||
stale, err := svc.Send(ctx, "product", request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := time.Now().Add(-2 * time.Minute)
|
||||
if err = repo.DBImp.Patch(ctx, stale.ID, repository.Patch().Set(repository.Field("next_attempt_at"), old)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := repo.Claim(ctx, stale.ID.Hex(), old)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = repo.Recover(ctx, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed.State = model.Accepted
|
||||
if err = repo.Finish(ctx, claimed); !errors.Is(err, merrors.ErrDataConflict) {
|
||||
t.Fatal("stale worker overwrote recovery", err)
|
||||
}
|
||||
stale, err = svc.Get(ctx, "product", stale.ID.Hex())
|
||||
if err != nil || stale.State != model.Unknown {
|
||||
t.Fatal(stale, err)
|
||||
}
|
||||
// Retry state persists, and a duplicate dispatch cannot bypass the due time.
|
||||
request.Key = "retry"
|
||||
retry, err := svc.Send(ctx, "product", request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retryDriver := &sender{state: model.RetryWait}
|
||||
retrySvc := service.New(repo, catalog, retryDriver, "resend", zap.NewNop())
|
||||
if _, err = retrySvc.Dispatch(ctx, retry.ID.Hex()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = retrySvc.Dispatch(ctx, retry.ID.Hex()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retry, err = svc.Get(ctx, "product", retry.ID.Hex())
|
||||
if err != nil || retry.State != model.RetryWait || retryDriver.calls.Load() != 1 {
|
||||
t.Fatal(retry, err)
|
||||
}
|
||||
// Verify the real JetStream publisher/consumer path with an isolated broker.
|
||||
ncContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ContainerRequest: testcontainers.ContainerRequest{Image: "nats:2.11-alpine", ExposedPorts: []string{"4222/tcp"}, Cmd: []string{"-js"}, WaitingFor: wait.ForListeningPort("4222/tcp")}, Started: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer ncContainer.Terminate(context.Background())
|
||||
endpoint, err := ncContainer.Endpoint(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("TEST_NOTIFICATIONS_NATS", "nats://"+endpoint)
|
||||
rt, err := msgruntime.New(zap.NewNop(), &messaging.Config{Driver: messaging.BBNats, Settings: map[string]any{"url_env": "TEST_NOTIFICATIONS_NATS", "broker_name": "test", "max_reconnects": -1, "reconnect_wait": 1}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rt.Close()
|
||||
broker, err := bus.New(ctx, rt.Durable, rt.DurableAdmin, repo, svc, zap.NewNop())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer broker.Close()
|
||||
request.Key = "jetstream"
|
||||
job, err := svc.Send(ctx, "product", request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = broker.PublishDue(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workerCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan struct{})
|
||||
go func() { broker.RunWorker(workerCtx); close(done) }()
|
||||
defer func() { stop(); <-done }()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
job, err = svc.Get(ctx, "product", job.ID.Hex())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.State == model.Accepted {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("JetStream job did not complete", job.State)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.sendico.io/tech/sxchnge/notifications/email"
|
||||
"git.sendico.io/tech/sxchnge/notifications/model"
|
||||
"git.sendico.io/tech/sxchnge/notifications/templates"
|
||||
"git.sendico.io/tech/sxchnge/shared/database/record"
|
||||
merrors "git.sendico.io/tech/sxchnge/shared/errors"
|
||||
mlogger "git.sendico.io/tech/sxchnge/shared/logging"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
Find(context.Context, string, string) (*model.Notification, error)
|
||||
Get(context.Context, string, string) (*model.Notification, error)
|
||||
Create(context.Context, *model.Notification) (*model.Notification, error)
|
||||
Claim(context.Context, string, time.Time) (*model.Notification, error)
|
||||
Finish(context.Context, *model.Notification) error
|
||||
Recover(context.Context, time.Time) error
|
||||
}
|
||||
type Service struct {
|
||||
repo Repository
|
||||
catalog *templates.Catalog
|
||||
sender email.Sender
|
||||
driver string
|
||||
logger mlogger.Logger
|
||||
}
|
||||
|
||||
func New(repo Repository, catalog *templates.Catalog, sender email.Sender, driver string, logger mlogger.Logger) *Service {
|
||||
return &Service{repo, catalog, sender, driver, mlogger.Named(logger, "notifications")}
|
||||
}
|
||||
func (s *Service) Send(ctx context.Context, caller string, r model.Request) (*model.Notification, error) {
|
||||
if caller == "" {
|
||||
return nil, merrors.Unauthorized("missing caller")
|
||||
}
|
||||
if r.Key == "" || len(r.Key) > 128 || r.UserID == "" || len(r.UserID) > 128 || !email.ValidAddress(r.Address) || len(r.TemplateID) > 128 || len(r.TemplateVersion) > 64 || len(r.Locale) > 64 || strings.ContainsRune(r.TemplateID+r.TemplateVersion, 0) {
|
||||
return nil, merrors.InvalidArgument("invalid notification request")
|
||||
}
|
||||
if len(r.Parameters) > 64 {
|
||||
return nil, merrors.InvalidArgument("too many parameters")
|
||||
}
|
||||
for k, v := range r.Parameters {
|
||||
if k == "" || len(k) > 64 || len(v) > 4096 {
|
||||
return nil, merrors.InvalidArgument("invalid template parameter")
|
||||
}
|
||||
}
|
||||
payload, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return nil, merrors.InvalidArgument("invalid request")
|
||||
}
|
||||
if len(payload) > 64<<10 {
|
||||
return nil, merrors.InvalidArgument("request too large")
|
||||
}
|
||||
sum := sha256.Sum256(payload)
|
||||
fingerprint := hex.EncodeToString(sum[:])
|
||||
existing, err := s.repo.Find(ctx, caller, r.Key)
|
||||
if err == nil {
|
||||
return same(existing, fingerprint)
|
||||
}
|
||||
if !errors.Is(err, merrors.ErrNoData) {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
expiry := r.ExpiresAt
|
||||
if expiry.IsZero() {
|
||||
expiry = now.Add(24 * time.Hour)
|
||||
}
|
||||
if !expiry.After(now) || expiry.After(now.Add(24*time.Hour)) {
|
||||
return nil, merrors.InvalidArgument("expiry must be within the next 24 hours")
|
||||
}
|
||||
rendered, err := s.catalog.Render(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n := &model.Notification{Base: storable.Base{ID: bson.NewObjectID(), CreatedAt: now, UpdatedAt: now}, Caller: caller, Key: r.Key, Fingerprint: fingerprint, UserID: r.UserID, TemplateID: r.TemplateID, TemplateVersion: r.TemplateVersion, Email: rendered, Driver: s.driver, State: model.Queued, ExpiresAt: expiry, NextAttemptAt: now, PurgeAt: now.Add(7 * 24 * time.Hour)}
|
||||
stored, err := s.repo.Create(ctx, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := same(stored, fingerprint)
|
||||
if err == nil {
|
||||
s.logger.Info("Notification accepted", zap.String("notification_id", stored.ID.Hex()), zap.String("caller", caller))
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
func same(n *model.Notification, fingerprint string) (*model.Notification, error) {
|
||||
if n.Fingerprint != fingerprint {
|
||||
return nil, merrors.DataConflict("idempotency key already used with a different request")
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
func (s *Service) Get(ctx context.Context, caller, id string) (*model.Notification, error) {
|
||||
if caller == "" {
|
||||
return nil, merrors.Unauthorized("missing caller")
|
||||
}
|
||||
if _, err := bson.ObjectIDFromHex(id); err != nil {
|
||||
return nil, merrors.InvalidArgument("invalid notification ID")
|
||||
}
|
||||
return s.repo.Get(ctx, caller, id)
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.sendico.io/tech/sxchnge/notifications/model"
|
||||
merrors "git.sendico.io/tech/sxchnge/shared/errors"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Dispatch consumes an opaque job ID from JetStream. Duplicate messages are safe.
|
||||
func (s *Service) Dispatch(ctx context.Context, id string) (bool, error) {
|
||||
now := time.Now().UTC()
|
||||
if err := s.repo.Recover(ctx, now); err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := s.repo.Claim(ctx, id, now)
|
||||
if errors.Is(err, merrors.ErrNoData) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
switch {
|
||||
case !n.ExpiresAt.After(now):
|
||||
n.State = model.Expired
|
||||
n.FailureCode = "expired"
|
||||
case n.Driver != s.driver:
|
||||
n.State = model.Failed
|
||||
n.FailureCode = "driver_changed"
|
||||
default:
|
||||
deadline := now.Add(20 * time.Second)
|
||||
if n.ExpiresAt.Before(deadline) {
|
||||
deadline = n.ExpiresAt
|
||||
}
|
||||
sendCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||
outcome := s.sender.Send(sendCtx, n.ID.Hex(), n.Email)
|
||||
cancel()
|
||||
n.State = outcome.State
|
||||
n.FailureCode = outcome.Code
|
||||
if n.State == model.RetryWait {
|
||||
delay := time.Duration(1<<min(n.Attempts, 8)) * time.Second
|
||||
if outcome.RetryAfter > delay {
|
||||
delay = outcome.RetryAfter
|
||||
}
|
||||
n.NextAttemptAt = time.Now().UTC().Add(delay)
|
||||
if n.Attempts >= 5 {
|
||||
n.State = model.Failed
|
||||
n.FailureCode = "retry_exhausted"
|
||||
} else if !n.NextAttemptAt.Before(n.ExpiresAt) {
|
||||
n.State = model.Expired
|
||||
n.FailureCode = "expired"
|
||||
}
|
||||
} else if n.State != model.Accepted && n.State != model.Failed && n.State != model.Unknown {
|
||||
n.State = model.Unknown
|
||||
n.FailureCode = "invalid_driver_outcome"
|
||||
}
|
||||
}
|
||||
n.UpdatedAt = time.Now().UTC()
|
||||
// Persist the result even when shutdown canceled the provider request.
|
||||
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
if err = s.repo.Finish(finishCtx, n); err != nil {
|
||||
return true, err
|
||||
}
|
||||
fields := []zap.Field{zap.String("notification_id", n.ID.Hex()), zap.String("driver", n.Driver), zap.String("state", string(n.State)), zap.Uint32("attempts", n.Attempts), zap.String("failure_code", n.FailureCode)}
|
||||
if n.State == model.Accepted {
|
||||
s.logger.Info("Email accepted by provider", fields...)
|
||||
} else {
|
||||
s.logger.Warn("Email delivery attempt finished", fields...)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.sendico.io/tech/sxchnge/notifications/email"
|
||||
"git.sendico.io/tech/sxchnge/notifications/model"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type workerRepo struct {
|
||||
Repository
|
||||
job model.Notification
|
||||
saved *model.Notification
|
||||
}
|
||||
|
||||
func (r *workerRepo) Recover(context.Context, time.Time) error { return nil }
|
||||
func (r *workerRepo) Claim(context.Context, string, time.Time) (*model.Notification, error) {
|
||||
n := r.job
|
||||
return &n, nil
|
||||
}
|
||||
func (r *workerRepo) Finish(_ context.Context, n *model.Notification) error { r.saved = n; return nil }
|
||||
|
||||
type workerSender struct {
|
||||
calls int
|
||||
outcome email.Outcome
|
||||
}
|
||||
|
||||
func (s *workerSender) Send(context.Context, string, model.Email) email.Outcome {
|
||||
s.calls++
|
||||
return s.outcome
|
||||
}
|
||||
func TestWorkerTerminalAndRetryOutcomes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
expiry time.Time
|
||||
driver string
|
||||
attempts uint32
|
||||
outcome model.State
|
||||
want model.State
|
||||
calls int
|
||||
}{
|
||||
{"expired", time.Now().Add(-time.Minute), "resend", 1, model.Accepted, model.Expired, 0},
|
||||
{"driver changed", time.Now().Add(time.Hour), "sendgrid", 1, model.Accepted, model.Failed, 0},
|
||||
{"retry exhausted", time.Now().Add(time.Hour), "resend", 5, model.RetryWait, model.Failed, 1},
|
||||
{"retry expiry", time.Now().Add(time.Second), "resend", 1, model.RetryWait, model.Expired, 1},
|
||||
{"ambiguous", time.Now().Add(time.Hour), "resend", 1, model.Unknown, model.Unknown, 1},
|
||||
{"invalid driver outcome", time.Now().Add(time.Hour), "resend", 1, model.Queued, model.Unknown, 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repo := &workerRepo{job: model.Notification{Driver: tc.driver, ExpiresAt: tc.expiry, Attempts: tc.attempts}}
|
||||
sender := &workerSender{outcome: email.Outcome{State: tc.outcome}}
|
||||
svc := New(repo, nil, sender, "resend", zap.NewNop())
|
||||
if _, err := svc.Dispatch(context.Background(), "id"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if repo.saved == nil || repo.saved.State != tc.want || sender.calls != tc.calls {
|
||||
t.Fatal(repo.saved, sender.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user