Complete notification expiry and configurable durable delivery (#36)

backend/notifications/store/mongo.go

@@ -14,6 +14,7 @@ import (
 	mlogger "git.sendico.io/tech/sxchnge/shared/logging"
 	"go.mongodb.org/mongo-driver/v2/bson"
 	"go.mongodb.org/mongo-driver/v2/mongo"
+	"go.uber.org/zap"
 )
 
 type Store struct {
@@ -26,6 +27,7 @@ func New(logger mlogger.Logger, db *mongo.Database) (*Store, error) {
 	for _, index := range []*ri.Definition{
 		{Name: "unique_caller_key", Keys: []ri.Key{{Field: "caller", Sort: ri.Asc}, {Field: "key", Sort: ri.Asc}}, Unique: true},
 		{Name: "pending_dispatch", Keys: []ri.Key{{Field: "state", Sort: ri.Asc}, {Field: "next_attempt_at", Sort: ri.Asc}}},
+		{Name: "pending_expiry", Keys: []ri.Key{{Field: "state", Sort: ri.Asc}, {Field: "expires_at", Sort: ri.Asc}}},
 		{Name: "expired_lease", Keys: []ri.Key{{Field: "state", Sort: ri.Asc}, {Field: "lease_until", Sort: ri.Asc}}},
 		{Name: "purge_notifications", Keys: []ri.Key{{Field: "purge_at", Sort: ri.Asc}}, TTL: &ttl},
 	} {
@@ -69,11 +71,24 @@ func pending(now time.Time) builder.Query {
 	return repository.Query().And(repository.Query().In(repository.Field("state"), model.Queued, model.RetryWait), repository.Query().Comparison(repository.Field("next_attempt_at"), builder.Lte, now))
 }
 
+// Expire pending messages independently of bus availability, then fence lost workers.
 // A lost worker may already have sent its message. Do not automatically replay it.
 func (s *Store) Recover(ctx context.Context, now time.Time) error {
+	expired := repository.Query().And(repository.Query().In(repository.Field("state"), model.Queued, model.RetryWait), repository.Query().Comparison(repository.Field("expires_at"), builder.Lte, now))
+	expirePatch := repository.Patch().Set(repository.Field("state"), model.Expired).Set(repository.Field("failure_code"), "expired").Set(repository.Field("updatedAt"), now).Set(repository.Field("email"), model.Email{})
+	count, err := s.PatchMany(ctx, expired, expirePatch)
+	if err != nil {
+		return wrap(err)
+	}
+	if count > 0 {
+		s.Logger.Info("Pending notifications expired", zap.Int("count", count))
+	}
 	query := repository.Query().And(repository.Filter("state", model.Sending), repository.Query().Comparison(repository.Field("lease_until"), builder.Lte, now))
 	patch := repository.Patch().Set(repository.Field("state"), model.Unknown).Set(repository.Field("failure_code"), "worker_outcome_unknown").Set(repository.Field("updatedAt"), now).Set(repository.Field("email"), model.Email{})
-	_, err := s.PatchMany(ctx, query, patch)
+	count, err = s.PatchMany(ctx, query, patch)
+	if err == nil && count > 0 {
+		s.Logger.Warn("Notification workers lost their leases", zap.Int("count", count))
+	}
 	return wrap(err)
 }
 func (s *Store) Claim(ctx context.Context, id string, now time.Time) (*model.Notification, error) {