From abca15f047b3fd4dcde8ca3ecf5a7fca94b01d21 Mon Sep 17 00:00:00 2001 From: Stephan D Date: Sat, 17 Oct 2026 07:30:33 +0200 Subject: [PATCH 01/10] Fix signup registration receipts and activation worker lifetime (#57) --- backend/notifications/app/run.go | 4 +- backend/notifications/bus/dispatch.go | 6 +- backend/notifications/bus/dispatch_test.go | 45 ++++++++++++++ backend/product/providers/sendico/client.go | 8 +-- .../product/providers/sendico/client_test.go | 37 ++++++++++- docs/signup-facts.md | 61 +++++++++++++++++-- 6 files changed, 148 insertions(+), 13 deletions(-) diff --git a/backend/notifications/app/run.go b/backend/notifications/app/run.go index 118b0bb..a4c0034 100644 --- a/backend/notifications/app/run.go +++ b/backend/notifications/app/run.go @@ -45,8 +45,6 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error { defer cancel() _ = client.Disconnect(cleanup) }() - startup, cancel := context.WithTimeout(ctx, 20*time.Second) - defer cancel() repo, err := store.New(logger, client.Database()) if err != nil { return err @@ -57,7 +55,7 @@ func Run(ctx context.Context, c Config, logger mlogger.Logger) error { } defer rt.Close() svc := service.New(repo, catalog, sender, string(c.Email.Driver), logger) - broker, err := bus.New(startup, rt.Durable, rt.DurableAdmin, repo, svc, logger, c.Queue) + broker, err := bus.New(ctx, rt.Durable, rt.DurableAdmin, repo, svc, logger, c.Queue) if err != nil { return err } diff --git a/backend/notifications/bus/dispatch.go b/backend/notifications/bus/dispatch.go index 37482c0..f62613a 100644 --- a/backend/notifications/bus/dispatch.go +++ b/backend/notifications/bus/dispatch.go @@ -35,9 +35,13 @@ func New(ctx context.Context, transport reliable.Transport, admin reliable.Admin } settings := reliable.DurableSettings{Enabled: true, Stream: config.Stream, Provisioning: config.Provisioning} consumer := reliable.DurableConsumerSettings{Durable: "email", Subject: subject, AckWaitSeconds: 60, MaxDeliver: -1, DeliverPolicy: reliable.DurableDeliverAll} - if err := admin.EnsureConsumerResources(ctx, settings, consumer); err != nil { + startup, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + if err := admin.EnsureConsumerResources(startup, settings, consumer); err != nil { return nil, merrors.OpaqueWrap(merrors.ErrServiceUnavailable, err, "configure notification messaging") } + // NATS retains this context and unsubscribes when it ends. The worker must + // live until service shutdown, independently of the provisioning deadline. subscription, err := transport.OpenConsumer(ctx, settings, consumer) if err != nil { return nil, merrors.OpaqueWrap(merrors.ErrServiceUnavailable, err, "subscribe notification worker") diff --git a/backend/notifications/bus/dispatch_test.go b/backend/notifications/bus/dispatch_test.go index 418b2fc..d4d65ae 100644 --- a/backend/notifications/bus/dispatch_test.go +++ b/backend/notifications/bus/dispatch_test.go @@ -10,6 +10,7 @@ import ( merrors "git.sendico.io/tech/sxchnge/shared/errors" "git.sendico.io/tech/sxchnge/shared/messaging/reliable" "go.mongodb.org/mongo-driver/v2/bson" + "go.uber.org/zap" ) type outbox struct{ jobs []model.Notification } @@ -23,6 +24,50 @@ type publisher struct { records []reliable.Record } +type consumerTransport struct { + reliable.Transport + reliable.Subscription + ctx context.Context +} + +func (t *consumerTransport) OpenConsumer(ctx context.Context, _ reliable.DurableSettings, _ reliable.DurableConsumerSettings) (reliable.Subscription, error) { + t.ctx = ctx + return t.Subscription, nil +} + +type consumerAdmin struct { + reliable.Admin + ctx context.Context +} + +func (a *consumerAdmin) EnsureConsumerResources(ctx context.Context, _ reliable.DurableSettings, _ reliable.DurableConsumerSettings) error { + a.ctx = ctx + return nil +} + +func TestSubscriptionOutlivesProvisioning(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + transport := &consumerTransport{} + admin := &consumerAdmin{} + if _, err := New(ctx, transport, admin, &outbox{}, nil, zap.NewNop(), DefaultConfig()); err != nil { + t.Fatal(err) + } + if admin.ctx.Err() != context.Canceled { + t.Fatal("provisioning context was not released") + } + if _, ok := admin.ctx.Deadline(); !ok { + t.Fatal("provisioning has no deadline") + } + if err := transport.ctx.Err(); err != nil { + t.Fatalf("subscription ended with provisioning: %v", err) + } + cancel() + if transport.ctx.Err() != context.Canceled { + t.Fatal("subscription does not follow service shutdown") + } +} + func (p *publisher) Publish(_ context.Context, r reliable.Record) error { if p.fail { return merrors.ServiceUnavailable("offline") diff --git a/backend/product/providers/sendico/client.go b/backend/product/providers/sendico/client.go index 0b15617..5d79d9e 100644 --- a/backend/product/providers/sendico/client.go +++ b/backend/product/providers/sendico/client.go @@ -12,7 +12,7 @@ import ( "go.mongodb.org/mongo-driver/v2/bson" ) -func (c *Client) post(ctx context.Context, resource, requestRef string, payload any) (receiptDTO, error) { +func (c *Client) post(ctx context.Context, resource, requestRef string, payload any, expectedStatus int) (receiptDTO, error) { var receipt receiptDTO token, err := c.token() if err != nil { @@ -38,7 +38,7 @@ func (c *Client) post(ctx context.Context, resource, requestRef string, payload } defer func() { _ = response.Body.Close() }() // Never decode or expose upstream error bodies, which can contain assertions. - if response.StatusCode != http.StatusOK { + if response.StatusCode != expectedStatus { return receipt, merrors.ServiceUnavailable("Sendico request was not accepted") } decoder := json.NewDecoder(io.LimitReader(response.Body, 16385)) @@ -54,14 +54,14 @@ func (c *Client) post(ctx context.Context, resource, requestRef string, payload } func (c *Client) Register(ctx context.Context, requestRef string) (string, error) { - dto, err := c.post(ctx, "individuals", requestRef, nil) + dto, err := c.post(ctx, "individuals", requestRef, nil, http.StatusCreated) if err != nil { return "", err } return mapSubject(dto) } func (c *Client) Submit(ctx context.Context, subject, source string, facts identity.Facts) (identity.Receipt, error) { - dto, err := c.post(ctx, "identity_facts", bson.NewObjectID().Hex(), mapFacts(subject, source, facts)) + dto, err := c.post(ctx, "identity_facts", bson.NewObjectID().Hex(), mapFacts(subject, source, facts), http.StatusOK) if err != nil { return identity.Receipt{}, err } diff --git a/backend/product/providers/sendico/client_test.go b/backend/product/providers/sendico/client_test.go index c466418..c776db9 100644 --- a/backend/product/providers/sendico/client_test.go +++ b/backend/product/providers/sendico/client_test.go @@ -48,6 +48,7 @@ func TestSignupWireContractAndCredentialRotation(t *testing.T) { if strings.Contains(r.URL.Path, "/individuals/") { require.Equal(t, "registration-request", r.Header.Get("X-Request-ID")) require.Empty(t, body) + w.WriteHeader(http.StatusCreated) _, _ = fmt.Fprintf(w, `{"status":"success","data":{"subject_ref":%q,"registered_at":"2026-10-07T10:30:00.123Z"}}`, subject) } else { require.JSONEq(t, `{"subject_ref":"`+subject+`","source_submission_ref":"signup-1","facts":{"full_name":"Ada Lovelace","email":"ada@example.com","external_customer_ref":"local-user"}}`, string(body)) @@ -95,6 +96,36 @@ func TestRejectInvalidConfiguration(t *testing.T) { require.Error(t, err) } +func TestSignupReceiptsRequireOperationStatus(t *testing.T) { + for _, status := range []int{200, 201, 202, 204, 206, 300, 403, 409, 503} { + t.Run(http.StatusText(status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + _, _ = fmt.Fprintf(w, `{"status":"success","data":{"subject_ref":%q,"registered_at":"2026-10-07T10:30:00Z","submission_ref":%q,"accepted_at":"2026-10-07T10:30:00Z"}}`, subject, submission) + })) + defer server.Close() + client, err := Open(config(t, server.URL)) + require.NoError(t, err) + ref, err := client.Register(context.Background(), "registration-request") + if status == http.StatusCreated { + require.NoError(t, err) + require.Equal(t, subject, ref) + } else { + require.Error(t, err) + require.Empty(t, ref) + } + receipt, err := client.Submit(context.Background(), subject, "key", identity.Facts{}) + if status == http.StatusOK { + require.NoError(t, err) + require.Equal(t, submission, receipt.SubmissionRef) + } else { + require.Error(t, err) + require.Empty(t, receipt) + } + }) + } +} + func TestUpstreamFailuresNeverProduceReceiptsOrLeakBodies(t *testing.T) { for _, tc := range []struct { name string @@ -113,7 +144,11 @@ func TestUpstreamFailuresNeverProduceReceiptsOrLeakBodies(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(tc.status) + status := tc.status + if status == http.StatusOK && strings.Contains(r.URL.Path, "/individuals/") { + status = http.StatusCreated + } + w.WriteHeader(status) _, _ = io.WriteString(w, tc.body) })) defer server.Close() diff --git a/docs/signup-facts.md b/docs/signup-facts.md index bac4c8e..5e0ebe3 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -5,6 +5,58 @@ Task [#46](https://git.sendico.io/tech/sxchnge/issues/46), branch customer creation, runtime, shared database source and Flutter error feedback. Delivery: [PR #47](https://git.sendico.io/tech/sxchnge/pulls/47), targeting `main`. +## Registration response correction + +Follow-up [#57](https://git.sendico.io/tech/sxchnge/issues/57), branch +`identity-facts-57`, corrects the registration success status to HTTP 201 Created. +Sendico's individual creation handler returns 201; the original adapter and test +fixture incorrectly required 200. A successful registration therefore left a +pending customer with an unresolved claim and `signup_attention_required`. +The adapter now checks the exact status per operation before validating receipts: +201 for registration and 200 for Facts intake. Other statuses, including other +2xx responses, remain failures. Malformed receipts and uncertain registrations +still require the existing recovery rules. + +On 2026-10-17, the affected local signup was reconciled using its registration +correlation in Sendico logs and an authenticated organization-scoped individual +GET. A conditional majority write bound that confirmed individual to the existing +local claim, preserving the saved facts, namespace and registration request. +The rebuilt BFF resumed the public activation-resend flow: Facts intake returned +a persisted receipt and the endpoint returned 202 after activation-email queue +acceptance. No replacement individual was registered. The account remains +pending until the customer confirms the email; queue acceptance does not prove +inbox delivery. + +Recovery also exposed a notification worker lifetime issue: its NATS subscription +retained the 20-second startup context and was automatically unsubscribed after +that deadline. Resource provisioning now has its own bounded context, while the +subscription follows the service lifetime and closes at shutdown. A regression +test checks that releasing provisioning leaves the subscription context active, +and service cancellation still ends it. + +The rebuilt worker processed the queued activation message. The live Resend +provider rejected it with `provider_authentication_failed` (HTTP 401 or 403), +leaving delivery in `failed` after one attempt. The configured API key/sender +authorization must be corrected before sending another activation email. Rotate +the credential securely with `make email`, wait for Vault Agent to refresh it, +restart notifications so it reloads its environment, then use activation resend +with a new idempotency key. Do not replay the failed notification as a success or +mark the customer active on their behalf. + +Validation passed: `GOFLAGS=-p=2 make test` (all backend race tests and canonical +frontend suites), `make lint-backend`, focused provider/identity race tests and +notification bus/app race tests. The HTTP 201 regression first failed against the +old adapter and passed after correction. An earlier full run hit a database +credential-rotation timing failure at `rotation_test.go:249` during the Docker +build; the isolated race-test retry and both subsequent full runs passed. +Local BFF and notifications images were rebuilt with fresh upstream manifests +and resolved stable digest pins; the BFF, session sidecar and notifications +services are healthy. NATS shows the worker still waiting for deliveries beyond +the startup deadline, with no pending or unacknowledged jobs. Live email delivery +is blocked by provider authorization as described above. No security scans were +requested or run. Unrelated notifications runtime settings are excluded from +the task commits. + ## Behavior and boundaries With `api.sendico` configured, the product BFF creates the local pending customer @@ -18,7 +70,7 @@ backfilled or blocked at login. The adapter consumes these existing Sendico contracts: -- `POST /individuals/{organization_ref}`, empty body, HTTP 200 registration +- `POST /individuals/{organization_ref}`, empty body, HTTP 201 registration receipt containing `subject_ref` and `registered_at`. - `POST /identity_facts/{organization_ref}`, HTTP 200 immutable intake receipt containing `submission_ref`, `subject_ref` and `accepted_at`. @@ -73,9 +125,10 @@ the adapter rereads it on every request. Startup rejects missing, expired or mismatched credentials and rejects Sendico configuration without notifications. HTTPS is required except for loopback HTTP test/development endpoints. Redirects -are rejected and each HTTP call has a four-second timeout. Non-200 responses, -malformed/truncated receipts, invalid identifiers, wrong subjects and unsuccessful -envelopes never count as acceptance. Upstream error bodies are not surfaced. +are rejected and each HTTP call has a four-second timeout. Unexpected response +statuses (registration requires 201; Facts intake requires 200), malformed/truncated +receipts, invalid identifiers, wrong subjects and unsuccessful envelopes never +count as acceptance. Upstream error bodies are not surfaced. Omitting `api.sendico` keeps existing local-only behavior. Enabling it also gates activation for previously pending customers; use activation resend to submit -- 2.54.0 From c1d28e705e8bc4244ab52bf0fd21c24b010d992c Mon Sep 17 00:00:00 2001 From: Stephan D Date: Sat, 17 Oct 2026 07:30:50 +0200 Subject: [PATCH 02/10] Record signup fix delivery and recovery limitation --- docs/signup-facts.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/signup-facts.md b/docs/signup-facts.md index 5e0ebe3..5618ff1 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -8,7 +8,9 @@ Delivery: [PR #47](https://git.sendico.io/tech/sxchnge/pulls/47), targeting `mai ## Registration response correction Follow-up [#57](https://git.sendico.io/tech/sxchnge/issues/57), branch -`identity-facts-57`, corrects the registration success status to HTTP 201 Created. +`identity-facts-57`, delivered in [PR #59](https://git.sendico.io/tech/sxchnge/pulls/59) +against `main` (implementation `abca15f`), corrects the registration success status +to HTTP 201 Created. The PR remains unmerged. Sendico's individual creation handler returns 201; the original adapter and test fixture incorrectly required 200. A successful registration therefore left a pending customer with an unresolved claim and `signup_attention_required`. -- 2.54.0 From 3e4e4960376687ae71f42d516bca8c897e7be802 Mon Sep 17 00:00:00 2001 From: Stephan D Date: Sat, 17 Oct 2026 11:39:05 +0200 Subject: [PATCH 03/10] Log redacted email provider errors and diagnose sender configuration (#57) --- backend/notifications/README.md | 24 ++++- backend/notifications/email/driver.go | 4 + backend/notifications/email/failure.go | 95 ++++++++++++++++++++ backend/notifications/email/failure_test.go | 90 +++++++++++++++++++ backend/notifications/email/http.go | 25 ++++-- backend/notifications/service/worker.go | 20 ++++- backend/notifications/service/worker_test.go | 47 +++++++++- docs/signup-facts.md | 45 ++++++++-- 8 files changed, 327 insertions(+), 23 deletions(-) create mode 100644 backend/notifications/email/failure.go create mode 100644 backend/notifications/email/failure_test.go diff --git a/backend/notifications/README.md b/backend/notifications/README.md index 645c9d2..fb0bb56 100644 --- a/backend/notifications/README.md +++ b/backend/notifications/README.md @@ -80,8 +80,28 @@ remaining status and request fingerprints expire through a seven-day TTL. - Callers authorize recipients and preferences. No direct customer-database access, bulk campaigns, provider failover, SMS, push or Telegram delivery is included. -Structured logs report notification IDs, driver, state and attempt count; message -content, template values, addresses and provider response bodies are excluded. +Structured logs report notification IDs, driver, state and attempt count. Provider +HTTP outcomes include `provider_status`; failures also include `provider_error` +and `provider_message` when available. The adapter reads at most 16 KiB plus an +overflow byte and extracts only the Resend `name`/`message` or first SendGrid +`errors[].message`. Diagnostic messages are bounded to 512 Unicode characters +plus an ellipsis. Credentials, idempotency keys, outgoing request string values, +email addresses, URLs, long token-like strings and numeric verification/account +values embedded in the request are redacted; controls and formatting characters +are normalized. Malformed, incomplete and oversized error +responses produce a diagnostic fallback instead of raw content. Successful +response bodies are not logged. Diagnostic fields are transient: they are neither +persisted in notifications nor exposed in gRPC responses. The existing bounded +failure codes and delivery/retry decisions remain unchanged. Transport failures +include a sanitized cause. Provider outcome logging happens before the result +write so a storage failure cannot discard the only provider diagnostic. + +Message content, template values, addresses and raw provider bodies must remain +excluded from logs. Provider text is sanitized external diagnostic data, never +an instruction. Use `provider_status`, `provider_error` and `provider_message` +together: the generic `provider_authentication_failed` category combines HTTP +401 and 403, including sender/domain authorization failures. A sending-only +Resend key is valid for sending, but cannot read the provider's log API. Monitor pending age, UNKNOWN/FAILED states and repeated outbox/worker warnings. ## Validate and distribute diff --git a/backend/notifications/email/driver.go b/backend/notifications/email/driver.go index a9e1f65..1686df7 100644 --- a/backend/notifications/email/driver.go +++ b/backend/notifications/email/driver.go @@ -33,6 +33,10 @@ type Outcome struct { State model.State Code string RetryAfter time.Duration + // Provider diagnostics are sanitized operational data, not public failures. + HTTPStatus int + ProviderError string + ProviderMessage string } type Sender interface { Send(context.Context, string, model.Email) Outcome diff --git a/backend/notifications/email/failure.go b/backend/notifications/email/failure.go new file mode 100644 index 0000000..2df22cd --- /dev/null +++ b/backend/notifications/email/failure.go @@ -0,0 +1,95 @@ +package email + +import ( + "encoding/json" + "regexp" + "sort" + "strings" + "unicode" +) + +const maxErrorBody = 16 << 10 +const maxErrorMessage = 512 + +var ( + errorName = regexp.MustCompile(`^[a-zA-Z0-9_.-]{1,64}$`) + addresses = regexp.MustCompile(`[^\s<>"'(),;:]+@[^\s<>"'(),;:]+`) + urls = regexp.MustCompile(`(?i)https?://[^\s<>"']+`) + tokens = regexp.MustCompile(`[a-zA-Z0-9_+/-]{32,}={0,2}`) + numbers = regexp.MustCompile(`[0-9]{4,}`) +) + +// Only the documented human-readable error fields cross the adapter boundary. +// Full bodies, headers and request data must never reach logs or public failures. +func providerFailure(body, request []byte, secrets ...string) (string, string) { + var dto struct { + Name string `json:"name"` + Message string `json:"message"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` + } + if err := json.Unmarshal(body, &dto); err != nil { + return "", "Provider error details unavailable: invalid JSON response" + } + if dto.Message == "" && len(dto.Errors) > 0 { + dto.Message = dto.Errors[0].Message + } + // Providers can echo any scalar string from their request DTOs. Strip those + // values before truncation, in addition to credentials and embedded tokens. + var payload any + if json.Unmarshal(request, &payload) == nil { + for _, value := range requestStrings(payload) { + secrets = append(secrets, value) + // Rendered login/verification codes and numeric account references + // can be echoed as fragments of an otherwise longer email body. + secrets = append(secrets, numbers.FindAllString(value, -1)...) + } + } + sort.Slice(secrets, func(i, j int) bool { return len(secrets[i]) > len(secrets[j]) }) + name := safeErrorText(dto.Name, secrets) + if !errorName.MatchString(name) { + name = "" + } + message := safeErrorText(dto.Message, secrets) + if message == "" { + message = "Provider did not include an error message" + } + return name, message +} + +func requestStrings(value any) []string { + var values []string + switch value := value.(type) { + case string: + values = append(values, value) + case []any: + for _, item := range value { + values = append(values, requestStrings(item)...) + } + case map[string]any: + for _, item := range value { + values = append(values, requestStrings(item)...) + } + } + return values +} + +func safeErrorText(text string, secrets []string) string { + for _, secret := range secrets { + if secret != "" { + text = strings.ReplaceAll(text, secret, "[redacted]") + } + } + text = addresses.ReplaceAllString(text, "[redacted-address]") + text = urls.ReplaceAllString(text, "[redacted-url]") + text = tokens.ReplaceAllString(text, "[redacted-token]") + text = strings.Join(strings.FieldsFunc(text, func(r rune) bool { + return unicode.IsSpace(r) || unicode.IsControl(r) || unicode.Is(unicode.Cf, r) + }), " ") + runes := []rune(text) + if len(runes) > maxErrorMessage { + text = string(runes[:maxErrorMessage]) + "…" + } + return text +} diff --git a/backend/notifications/email/failure_test.go b/backend/notifications/email/failure_test.go new file mode 100644 index 0000000..69899b7 --- /dev/null +++ b/backend/notifications/email/failure_test.go @@ -0,0 +1,90 @@ +package email + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "unicode/utf8" + + "git.sendico.io/tech/sxchnge/notifications/model" +) + +func TestProviderErrorDiagnostics(t *testing.T) { + for _, tc := range []struct { + name, body, code, message string + }{ + {"resend", `{"name":"validation_error","message":"The sxchnge.com domain is not verified"}`, "validation_error", "The sxchnge.com domain is not verified"}, + {"sendgrid", `{"errors":[{"message":"The sender identity is not verified","field":"from","help":"private URL"}]}`, "", "The sender identity is not verified"}, + {"no message", `{"unrecognized":"private response"}`, "", "Provider did not include an error message"}, + {"malformed", `{"message":"private response"`, "", "Provider error details unavailable: invalid JSON response"}, + {"wrong type", `{"message":{"secret":"private response"}}`, "", "Provider error details unavailable: invalid JSON response"}, + {"trailing", `{"message":"private response"} {}`, "", "Provider error details unavailable: invalid JSON response"}, + {"oversized", `{"message":"` + strings.Repeat("x", maxErrorBody) + `"}`, "", "Provider error details unavailable: incomplete or oversized response"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(w, tc.body) + })) + defer server.Close() + c := client{http: server.Client(), endpoint: server.URL} + out := c.post(context.Background(), "", struct{}{}) + if out.State != model.Failed || out.Code != "provider_authentication_failed" || out.HTTPStatus != 403 || out.ProviderError != tc.code || out.ProviderMessage != tc.message { + t.Fatalf("unexpected provider outcome: %+v", out) + } + }) + } +} + +func TestProviderDiagnosticsRedactRequestDataBeforeTruncation(t *testing.T) { + key := "re_credential-secret" + request := []byte(`{"from":"Sender ","to":["recipient@example.com"],"personalizations":[{"dynamic_template_data":{"secret":"short-private-value"}}],"subject":"Private subject","text":"Private text","html":"

Private HTML

","content":[{"value":"Your verification code is 123456; account 9876543210"}]}`) + token := strings.Repeat("a", 43) + message := "Credential " + key + "; recipient@example.com; Private subject; Private text;

Private HTML

; short-private-value; https://example.com/verify?token=" + token + "; " + token + "; 123456; 9876543210; job-secret\n\r\x00\u202eforged-log" + body, err := json.Marshal(map[string]string{"name": "validation_error", "message": message}) + if err != nil { + t.Fatal(err) + } + name, result := providerFailure(body, request, key, "job-secret") + if name != "validation_error" || !strings.Contains(result, "forged-log") || !strings.Contains(result, "[redacted]") { + t.Fatal("lost useful diagnostic", name, result) + } + for _, forbidden := range []string{key, "recipient@example.com", "Private subject", "Private text", "Private HTML", "short-private-value", "https://", token, "123456", "9876543210", "job-secret", "\n", "\r", "\x00", "\u202e"} { + if strings.Contains(result, forbidden) { + t.Fatalf("diagnostic contains sensitive value or control %q", forbidden) + } + } + body, err = json.Marshal(map[string]string{"name": "validation_error", "message": strings.Repeat("ą", maxErrorMessage+20) + key}) + if err != nil { + t.Fatal(err) + } + _, result = providerFailure(body, request, key) + if !utf8.ValidString(result) || utf8.RuneCountInString(result) != maxErrorMessage+1 || !strings.HasSuffix(result, "…") || strings.Contains(result, "re_") { + t.Fatal("invalid or unbounded diagnostic") + } +} + +func TestSuccessfulResponsesDoNotExposeProviderContent(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"name":"private","message":"private provider response"}`) + })) + defer server.Close() + c := client{http: server.Client(), endpoint: server.URL} + if out := c.post(context.Background(), "", struct{}{}); out.State != model.Accepted || out.HTTPStatus != 200 || out.ProviderMessage != "" || out.ProviderError != "" { + t.Fatal("success leaked provider content", out) + } +} + +func TestTransportFailureHasSanitizedDiagnostic(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + c := client{http: http.DefaultClient, key: "credential-secret", endpoint: "https://example.com/emails?credential=credential-secret"} + out := c.post(ctx, "job-secret", struct{}{}) + if out.State != model.Unknown || !strings.Contains(out.ProviderMessage, "context canceled") || strings.Contains(out.ProviderMessage, "credential-secret") || strings.Contains(out.ProviderMessage, "https://") { + t.Fatal("transport cause missing or unsafe", out) + } +} diff --git a/backend/notifications/email/http.go b/backend/notifications/email/http.go index ddeca31..5cf8549 100644 --- a/backend/notifications/email/http.go +++ b/backend/notifications/email/http.go @@ -28,14 +28,15 @@ func (c *client) post(ctx context.Context, key string, payload any) Outcome { } resp, err := c.http.Do(req) if err != nil { - return Outcome{State: model.Unknown, Code: "provider_outcome_unknown"} + return Outcome{State: model.Unknown, Code: "provider_outcome_unknown", ProviderMessage: safeErrorText(err.Error(), []string{c.key, key})} } defer resp.Body.Close() - // Discard a bounded response without logging provider-controlled content. - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + // Never pass raw provider bodies through the outcome or log them directly. + details, readErr := io.ReadAll(io.LimitReader(resp.Body, maxErrorBody+1)) + outcome := Outcome{HTTPStatus: resp.StatusCode} switch { case resp.StatusCode >= 200 && resp.StatusCode < 300: - return Outcome{State: model.Accepted} + outcome.State = model.Accepted case resp.StatusCode == 429: delay := time.Minute if n, e := strconv.Atoi(resp.Header.Get("Retry-After")); e == nil && n > 0 && n <= 86400 { @@ -43,12 +44,20 @@ func (c *client) post(ctx context.Context, key string, payload any) Outcome { } else if date, e := http.ParseTime(resp.Header.Get("Retry-After")); e == nil && time.Until(date) > 0 { delay = time.Until(date) } - return Outcome{State: model.RetryWait, Code: "provider_rate_limited", RetryAfter: delay} + outcome.State, outcome.Code, outcome.RetryAfter = model.RetryWait, "provider_rate_limited", delay case resp.StatusCode >= 500 || resp.StatusCode == 408: - return Outcome{State: model.Unknown, Code: "provider_outcome_unknown"} + outcome.State, outcome.Code = model.Unknown, "provider_outcome_unknown" case resp.StatusCode == 401 || resp.StatusCode == 403: - return Outcome{State: model.Failed, Code: "provider_authentication_failed"} + outcome.State, outcome.Code = model.Failed, "provider_authentication_failed" default: - return Outcome{State: model.Failed, Code: "provider_rejected"} + outcome.State, outcome.Code = model.Failed, "provider_rejected" } + if outcome.State != model.Accepted { + if readErr != nil || len(details) > maxErrorBody { + outcome.ProviderMessage = "Provider error details unavailable: incomplete or oversized response" + } else { + outcome.ProviderError, outcome.ProviderMessage = providerFailure(details, body, c.key, key) + } + } + return outcome } diff --git a/backend/notifications/service/worker.go b/backend/notifications/service/worker.go index 44fadd9..cfb68a6 100644 --- a/backend/notifications/service/worker.go +++ b/backend/notifications/service/worker.go @@ -5,6 +5,7 @@ import ( "errors" "time" + "git.sendico.io/tech/sxchnge/notifications/email" "git.sendico.io/tech/sxchnge/notifications/model" merrors "git.sendico.io/tech/sxchnge/shared/errors" "go.uber.org/zap" @@ -23,6 +24,7 @@ func (s *Service) Dispatch(ctx context.Context, id string) (bool, error) { if err != nil { return false, err } + var outcome email.Outcome switch { case !n.ExpiresAt.After(now): n.State = model.Expired @@ -36,7 +38,7 @@ func (s *Service) Dispatch(ctx context.Context, id string) (bool, error) { deadline = n.ExpiresAt } sendCtx, cancel := context.WithDeadline(ctx, deadline) - outcome := s.sender.Send(sendCtx, n.ID.Hex(), n.Email) + outcome = s.sender.Send(sendCtx, n.ID.Hex(), n.Email) cancel() n.State = outcome.State n.FailureCode = outcome.Code @@ -62,14 +64,24 @@ func (s *Service) Dispatch(ctx context.Context, id string) (bool, error) { // Persist the result even when shutdown canceled the provider request. finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() - if err = s.repo.Finish(finishCtx, n); err != nil { - return true, err - } fields := []zap.Field{zap.String("notification_id", n.ID.Hex()), zap.String("driver", n.Driver), zap.String("state", string(n.State)), zap.Uint32("attempts", n.Attempts), zap.String("failure_code", n.FailureCode)} + if outcome.HTTPStatus != 0 { + fields = append(fields, zap.Int("provider_status", outcome.HTTPStatus)) + } + if outcome.ProviderError != "" { + fields = append(fields, zap.String("provider_error", outcome.ProviderError)) + } + if outcome.ProviderMessage != "" { + fields = append(fields, zap.String("provider_message", outcome.ProviderMessage)) + } + // Keep the provider outcome observable even if its local result write fails. if n.State == model.Accepted { s.logger.Info("Email accepted by provider", fields...) } else { s.logger.Warn("Email delivery attempt finished", fields...) } + if err = s.repo.Finish(finishCtx, n); err != nil { + return true, err + } return true, nil } diff --git a/backend/notifications/service/worker_test.go b/backend/notifications/service/worker_test.go index 2c8e084..d9b0649 100644 --- a/backend/notifications/service/worker_test.go +++ b/backend/notifications/service/worker_test.go @@ -2,18 +2,23 @@ package service import ( "context" + "errors" + "fmt" "testing" "time" "git.sendico.io/tech/sxchnge/notifications/email" "git.sendico.io/tech/sxchnge/notifications/model" + merrors "git.sendico.io/tech/sxchnge/shared/errors" "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) type workerRepo struct { Repository - job model.Notification - saved *model.Notification + job model.Notification + saved *model.Notification + finishErr error } func (r *workerRepo) Recover(context.Context, time.Time) error { return nil } @@ -21,7 +26,10 @@ func (r *workerRepo) Claim(context.Context, string, time.Time) (*model.Notificat n := r.job return &n, nil } -func (r *workerRepo) Finish(_ context.Context, n *model.Notification) error { r.saved = n; return nil } +func (r *workerRepo) Finish(_ context.Context, n *model.Notification) error { + r.saved = n + return r.finishErr +} type workerSender struct { calls int @@ -62,3 +70,36 @@ func TestWorkerTerminalAndRetryOutcomes(t *testing.T) { }) } } + +func TestWorkerLogsProviderFailureOnce(t *testing.T) { + for _, failPersistence := range []bool{false, true} { + t.Run(fmt.Sprint(failPersistence), func(t *testing.T) { + core, logs := observer.New(zap.WarnLevel) + repo := &workerRepo{job: model.Notification{Driver: "resend", ExpiresAt: time.Now().Add(time.Hour), Attempts: 1, Email: model.Email{Address: "private@example.com", Subject: "Private subject", Text: "Private text"}}} + if failPersistence { + repo.finishErr = merrors.ServiceUnavailable("storage offline") + } + sender := &workerSender{outcome: email.Outcome{State: model.Failed, Code: "provider_authentication_failed", HTTPStatus: 403, ProviderError: "validation_error", ProviderMessage: "The sxchnge.com domain is not verified"}} + svc := New(repo, nil, sender, "resend", zap.New(core)) + if _, err := svc.Dispatch(context.Background(), "id"); !errors.Is(err, repo.finishErr) { + t.Fatal("unexpected persistence error", err) + } + entries := logs.All() + if len(entries) != 1 || entries[0].Message != "Email delivery attempt finished" { + t.Fatal("expected one provider outcome log", entries) + } + fields := entries[0].ContextMap() + if fields["provider_status"] != int64(403) || fields["provider_error"] != "validation_error" || fields["provider_message"] != "The sxchnge.com domain is not verified" { + t.Fatal("provider diagnostic missing", fields) + } + if repo.saved.FailureCode != "provider_authentication_failed" { + t.Fatal("public failure code changed", repo.saved.FailureCode) + } + for _, field := range []string{"address", "email", "subject", "text", "html", "parameters"} { + if _, ok := fields[field]; ok { + t.Fatalf("private notification field %s logged", field) + } + } + }) + } +} diff --git a/docs/signup-facts.md b/docs/signup-facts.md index 5618ff1..a0636ce 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -38,12 +38,12 @@ and service cancellation still ends it. The rebuilt worker processed the queued activation message. The live Resend provider rejected it with `provider_authentication_failed` (HTTP 401 or 403), -leaving delivery in `failed` after one attempt. The configured API key/sender -authorization must be corrected before sending another activation email. Rotate -the credential securely with `make email`, wait for Vault Agent to refresh it, -restart notifications so it reloads its environment, then use activation resend -with a new idempotency key. Do not replay the failed notification as a success or -mark the customer active on their behalf. +leaving delivery in `failed` after one attempt. Later diagnostics below identify +the unverified sender domain as the actual blocker. Correct domain authorization +before activation resend with a new idempotency key. Credential rotation is not +required by that error; if a separate rotation is needed, use `make email` and +allow Vault Agent to refresh/restart the notification process. Do not replay the +failed notification as a success or mark the customer active on their behalf. Validation passed: `GOFLAGS=-p=2 make test` (all backend race tests and canonical frontend suites), `make lint-backend`, focused provider/identity race tests and @@ -59,6 +59,39 @@ is blocked by provider authorization as described above. No security scans were requested or run. Unrelated notifications runtime settings are excluded from the task commits. +### Provider diagnostics follow-up + +The same task/PR adds transient provider diagnostics to the worker's existing +structured outcome log: `provider_status`, `provider_error` and +`provider_message`. The adapter extracts bounded Resend/SendGrid error fields and +redacts credentials, request values, addresses, URLs and token/code fragments. +Transport errors have a sanitized cause; invalid or oversized responses have a +safe fallback. Diagnostics remain outside stored notification models and public +gRPC responses. The outcome is logged even when its result write fails. See +the [notification logging contract](../backend/notifications/README.md). + +Runtime inspection confirmed `email.driver: resend`, an API key in Vault, and +sender `noreply@sxchnge.com`. A live diagnostic notification used Resend's +documented `delivered+sxchnge-config@resend.dev` test mailbox, without retrying +the customer's email. The rebuilt worker logged HTTP 403, +`provider_error: validation_error`, and the sanitized message: +`The sxchnge.com domain is not verified. Please, add and verify your domain on [redacted-url]`. +The precise blocker is sender-domain verification, not missing local Resend +configuration. A sending-only key is appropriate for sending; its inability to +read Resend's logs API does not explain the original send failure. Verify the +configured domain in Resend (and its required DNS records), or explicitly choose +an already verified sender domain. No sender/domain/DNS setting was changed by +this follow-up; the customer's account remains pending email confirmation. + +Follow-up validation passed: `GOFLAGS=-p=2 make test`, `make lint-backend`, and +`go -C backend/notifications test -race ./email ./service -count=1`. Coverage +includes Resend/SendGrid messages, malformed/trailing/oversized response bodies, +credential/content/code redaction, Unicode-safe truncation, sanitized transport +errors, and exactly one provider outcome log even when persistence fails. The +notifications image was rebuilt from fresh stable digest pins and deployed +healthy; the live test verified both the diagnostic log and the unchanged generic +gRPC failure response. No security scan or real customer email retry was run. + ## Behavior and boundaries With `api.sendico` configured, the product BFF creates the local pending customer -- 2.54.0 From b36eb72c23e0c8978980a94848dcdcb6bde50458 Mon Sep 17 00:00:00 2001 From: Stephan D Date: Sat, 17 Oct 2026 20:51:28 +0200 Subject: [PATCH 04/10] Use Sendico SMTP mailbox for local notifications (#57) --- backend/notifications/PROVENANCE.md | 6 + backend/notifications/README.md | 33 +++- backend/notifications/config.example.yml | 2 +- backend/notifications/email/driver.go | 5 + backend/notifications/email/failure.go | 6 +- backend/notifications/email/mime.go | 49 +++++ backend/notifications/email/smtp.go | 122 +++++++++++++ backend/notifications/email/smtp_test.go | 223 +++++++++++++++++++++++ backend/notifications/service/worker.go | 3 + docs/local-development.md | 43 +++-- docs/signup-facts.md | 33 ++++ infra/images/backend.Dockerfile | 2 +- infra/local/notifications/runtime.yml | 10 +- infra/local/scripts/bootstrap.sh | 14 +- infra/local/scripts/common.sh | 1 + infra/local/scripts/test.sh | 4 + infra/local/scripts/verify.sh | 4 +- infra/local/vault/notifications.hcl | 4 +- 18 files changed, 528 insertions(+), 36 deletions(-) create mode 100644 backend/notifications/email/mime.go create mode 100644 backend/notifications/email/smtp.go create mode 100644 backend/notifications/email/smtp_test.go diff --git a/backend/notifications/PROVENANCE.md b/backend/notifications/PROVENANCE.md index b8f4902..1a563a7 100644 --- a/backend/notifications/PROVENANCE.md +++ b/backend/notifications/PROVENANCE.md @@ -11,6 +11,12 @@ classifies delivery outcomes and never logs provider response bodies. Sendico's builders, dynamic-template dependency, notification events and unused producer were removed. Resend is a new driver using the same contract. +SMTP is an owned adapter using Go's standard `net/smtp`, verified `crypto/tls` +connections and standard MIME encoders. Its implicit-TLS connection configuration +matches Sendico's existing mail client, without importing its modules or copying +its keepalive/reconnect behavior. Local mailbox credentials are provisioned into +SXCHNGE Vault explicitly; the application does not read Sendico files or secrets. + Shared messaging was selected from that same Sendico revision's `api/pkg/messaging`: `config.go`, `capability/errors.go`, `reliable/transport.go`, `reliable/durable_settings.go`, `reliable/durable_settings_behavior.go`, diff --git a/backend/notifications/README.md b/backend/notifications/README.md index fb0bb56..1b56290 100644 --- a/backend/notifications/README.md +++ b/backend/notifications/README.md @@ -12,12 +12,35 @@ pending-work state atomically in one document through the shared DB wrapper. The outbox dispatcher publishes opaque IDs using shared messaging's durable transport, adapted from Sendico. A durable NATS worker claims the notification with an atomic conditional patch and saves the outcome before acknowledging. -`DriverConfig` selects the email driver: `sendgrid` or `resend`. +`DriverConfig` selects the email driver: `sendgrid`, `resend` or `smtp`. Email adapters only map rendered email models to provider wire payloads. SendGrid uses adapted Sendico SDK construction; Resend uses its email HTTP API. Both support subject, text and HTML, bounded timeouts and safe error classification. +The owned SMTP adapter uses authenticated implicit TLS (normally port 465), +verified server certificates and context-bounded connections. It sends MIME +multipart/alternative with quoted-printable UTF-8 text/HTML and encoded subjects. +It uses Go's standard SMTP client and does not import Sendico modules or share a +mutable connection between workers. Configure it with: + +```yaml +email: + driver: smtp + settings: + host: mail.example.com + port: 465 + username: notifications@example.com + password_env: SMTP_PASSWORD + sender_address: notifications@example.com + sender_name: SXCHNGE + network_timeout: 10 +``` + +`network_timeout` must be 1–20 seconds; the worker's expiry/shutdown context also +bounds every operation. Only the password belongs in Vault. There is no plaintext +fallback or certificate-verification bypass. + ## Run Requires Go 1.27, protoc, an SXCHNGE MongoDB database and a NATS server with @@ -75,6 +98,12 @@ remaining status and request fingerprints expire through a seven-day TTL. - Resend receives the notification ID as its provider idempotency key. Its window is 24 hours. SendGrid has no equivalent guarantee; exactly-once external email delivery is not promised. +- SMTP acceptance requires the final DATA 250 reply. A later QUIT failure cannot + undo acceptance. Explicit SMTP 4xx replies become retryable + `provider_temporarily_unavailable`; 5xx replies are terminal (authentication + failures retain `provider_authentication_failed`). Transport/TLS errors and a + lost final DATA reply become UNKNOWN. No immediate reconnect/resend occurs. + A stable Message-ID is included for correlation, not an exactly-once promise. - Changing the configured driver fails queued jobs pinned to the old driver with `driver_changed`. Drain queued jobs before changing sender credentials/config. - Callers authorize recipients and preferences. No direct customer-database access, @@ -102,6 +131,8 @@ an instruction. Use `provider_status`, `provider_error` and `provider_message` together: the generic `provider_authentication_failed` category combines HTTP 401 and 403, including sender/domain authorization failures. A sending-only Resend key is valid for sending, but cannot read the provider's log API. +SMTP outcomes use `provider_smtp_status` for protocol reply codes and the same +sanitized provider error/message fields, including password/auth-payload redaction. Monitor pending age, UNKNOWN/FAILED states and repeated outbox/worker warnings. ## Validate and distribute diff --git a/backend/notifications/config.example.yml b/backend/notifications/config.example.yml index ff763e2..a1d7194 100644 --- a/backend/notifications/config.example.yml +++ b/backend/notifications/config.example.yml @@ -13,7 +13,7 @@ messaging: reconnect_wait: 2 templates: "templates.example.json" email: - driver: resend # sendgrid or resend + driver: resend # sendgrid or resend; see README for authenticated SMTP settings settings: api_key_env: "RESEND_API_KEY" # SENDGRID_API_KEY when driver is sendgrid sender_address: "notifications@example.com" diff --git a/backend/notifications/email/driver.go b/backend/notifications/email/driver.go index 1686df7..aeb832b 100644 --- a/backend/notifications/email/driver.go +++ b/backend/notifications/email/driver.go @@ -19,6 +19,7 @@ type Driver string const ( SendGrid Driver = "sendgrid" Resend Driver = "resend" + SMTP Driver = "smtp" ) type Config = sharedmodel.DriverConfig[Driver] @@ -35,6 +36,7 @@ type Outcome struct { RetryAfter time.Duration // Provider diagnostics are sanitized operational data, not public failures. HTTPStatus int + SMTPStatus int ProviderError string ProviderMessage string } @@ -50,6 +52,9 @@ type client struct { } func New(config Config) (Sender, error) { + if config.Driver == SMTP { + return newSMTP(config) + } var s Settings decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &s, ErrorUnused: true, TagName: "mapstructure"}) if err != nil { diff --git a/backend/notifications/email/failure.go b/backend/notifications/email/failure.go index 2df22cd..7b1430e 100644 --- a/backend/notifications/email/failure.go +++ b/backend/notifications/email/failure.go @@ -46,7 +46,6 @@ func providerFailure(body, request []byte, secrets ...string) (string, string) { secrets = append(secrets, numbers.FindAllString(value, -1)...) } } - sort.Slice(secrets, func(i, j int) bool { return len(secrets[i]) > len(secrets[j]) }) name := safeErrorText(dto.Name, secrets) if !errorName.MatchString(name) { name = "" @@ -76,11 +75,14 @@ func requestStrings(value any) []string { } func safeErrorText(text string, secrets []string) string { + sort.Slice(secrets, func(i, j int) bool { return len(secrets[i]) > len(secrets[j]) }) + var replacements []string for _, secret := range secrets { if secret != "" { - text = strings.ReplaceAll(text, secret, "[redacted]") + replacements = append(replacements, secret, "[redacted]") } } + text = strings.NewReplacer(replacements...).Replace(text) text = addresses.ReplaceAllString(text, "[redacted-address]") text = urls.ReplaceAllString(text, "[redacted-url]") text = tokens.ReplaceAllString(text, "[redacted-token]") diff --git a/backend/notifications/email/mime.go b/backend/notifications/email/mime.go new file mode 100644 index 0000000..021078e --- /dev/null +++ b/backend/notifications/email/mime.go @@ -0,0 +1,49 @@ +package email + +import ( + "bytes" + "crypto/sha256" + "fmt" + "mime" + "mime/multipart" + "mime/quotedprintable" + "net/mail" + "net/textproto" + "strings" + "time" + + "git.sendico.io/tech/sxchnge/notifications/model" + merrors "git.sendico.io/tech/sxchnge/shared/errors" +) + +func smtpMessage(s smtpSettings, key string, email model.Email) ([]byte, error) { + if !ValidAddress(email.Address) || strings.ContainsAny(email.Subject, "\r\n") || (email.Text == "" && email.HTML == "") { + return nil, merrors.InvalidArgument("invalid email content") + } + var body bytes.Buffer + parts := multipart.NewWriter(&body) + for _, part := range []struct{ kind, value string }{{"text/plain", email.Text}, {"text/html", email.HTML}} { + if part.value == "" { + continue + } + writer, err := parts.CreatePart(textproto.MIMEHeader{"Content-Type": {part.kind + "; charset=utf-8"}, "Content-Transfer-Encoding": {"quoted-printable"}}) + if err != nil { + return nil, merrors.InternalWrap(err, "create email MIME part") + } + encoded := quotedprintable.NewWriter(writer) + if _, err = encoded.Write([]byte(part.value)); err != nil { + return nil, merrors.InternalWrap(err, "encode email MIME part") + } + if err = encoded.Close(); err != nil { + return nil, merrors.InternalWrap(err, "finish email MIME part") + } + } + if err := parts.Close(); err != nil { + return nil, merrors.InternalWrap(err, "finish email MIME body") + } + from := (&mail.Address{Name: s.SenderName, Address: s.SenderAddress}).String() + to := (&mail.Address{Address: email.Address}).String() + domain := s.SenderAddress[strings.LastIndex(s.SenderAddress, "@")+1:] + headers := fmt.Sprintf("From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\nMessage-ID: <%x@%s>\r\nMIME-Version: 1.0\r\nContent-Type: multipart/alternative; boundary=%q\r\n\r\n", from, to, mime.QEncoding.Encode("utf-8", email.Subject), time.Now().UTC().Format(time.RFC1123Z), sha256.Sum256([]byte(key)), domain, parts.Boundary()) + return append([]byte(headers), body.Bytes()...), nil +} diff --git a/backend/notifications/email/smtp.go b/backend/notifications/email/smtp.go new file mode 100644 index 0000000..51bcb2e --- /dev/null +++ b/backend/notifications/email/smtp.go @@ -0,0 +1,122 @@ +package email + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/base64" + "errors" + "io" + "net" + "net/smtp" + "net/textproto" + "os" + "strconv" + "strings" + "time" + + "git.sendico.io/tech/sxchnge/notifications/model" + merrors "git.sendico.io/tech/sxchnge/shared/errors" + "github.com/go-viper/mapstructure/v2" +) + +type smtpSettings struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + PasswordEnv string `mapstructure:"password_env"` + SenderAddress string `mapstructure:"sender_address"` + SenderName string `mapstructure:"sender_name"` + NetworkTimeout int `mapstructure:"network_timeout"` +} + +type smtpSender struct { + settings smtpSettings + password string + tls *tls.Config +} + +func newSMTP(config Config) (Sender, error) { + var s smtpSettings + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{Result: &s, ErrorUnused: true, TagName: "mapstructure"}) + if err != nil { + return nil, merrors.InternalWrap(err, "create SMTP settings decoder") + } + if decoder.Decode(config.Settings) != nil || s.Host == "" || strings.ContainsAny(s.Host, " /@\r\n\t") || s.Port < 1 || s.Port > 65535 || !ValidAddress(s.Username) || !ValidAddress(s.SenderAddress) || strings.ContainsAny(s.SenderName, "\r\n") || s.NetworkTimeout < 1 || s.NetworkTimeout > 20 { + return nil, merrors.InvalidArgument("invalid SMTP settings") + } + password := os.Getenv(s.PasswordEnv) + if s.PasswordEnv == "" || password == "" || strings.ContainsAny(password, "\r\n\x00") { + return nil, merrors.FailedPrecondition("SMTP password environment variable is missing or invalid") + } + return &smtpSender{settings: s, password: password, tls: &tls.Config{ServerName: s.Host, MinVersion: tls.VersionTLS12}}, nil +} + +// SMTP uses a fresh authenticated TLS connection per attempt. No automatic +// resend follows an uncertain result, since SMTP has no idempotency guarantee. +func (s *smtpSender) Send(ctx context.Context, key string, email model.Email) Outcome { + body, err := smtpMessage(s.settings, key, email) + if err != nil { + return Outcome{State: model.Failed, Code: "invalid_payload"} + } + ctx, cancel := context.WithTimeout(ctx, time.Duration(s.settings.NetworkTimeout)*time.Second) + defer cancel() + dialer := tls.Dialer{NetDialer: &net.Dialer{}, Config: s.tls} + conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(s.settings.Host, strconv.Itoa(s.settings.Port))) + if err != nil { + return s.failure(err, key, email) + } + defer conn.Close() + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + defer stop() + deadline, _ := ctx.Deadline() + if err = conn.SetDeadline(deadline); err != nil { + return s.failure(err, key, email) + } + client, err := smtp.NewClient(conn, s.settings.Host) + if err == nil { + err = client.Auth(smtp.PlainAuth("", s.settings.Username, s.password, s.settings.Host)) + } + if err == nil { + err = client.Mail(s.settings.SenderAddress) + } + if err == nil { + err = client.Rcpt(email.Address) + } + if err != nil { + return s.failure(err, key, email) + } + writer, err := client.Data() + if err != nil { + return s.failure(err, key, email) + } + if _, err = io.Copy(writer, bytes.NewReader(body)); err == nil { + err = writer.Close() + } + if err != nil { + return s.failure(err, key, email) + } + // DATA's final 250 is acceptance. A later QUIT error cannot undo it. + _ = client.Quit() + return Outcome{State: model.Accepted, SMTPStatus: 250} +} + +func (s *smtpSender) failure(err error, key string, email model.Email) Outcome { + secrets := []string{s.password, s.settings.Username, key, email.Address, email.Subject, email.HTML, email.Text} + secrets = append(secrets, base64.StdEncoding.EncodeToString([]byte("\x00"+s.settings.Username+"\x00"+s.password))) + secrets = append(secrets, numbers.FindAllString(email.Text+" "+email.HTML, -1)...) + out := Outcome{State: model.Unknown, Code: "provider_outcome_unknown", ProviderError: "smtp", ProviderMessage: safeErrorText(err.Error(), secrets)} + var reply *textproto.Error + if errors.As(err, &reply) { + out.SMTPStatus = reply.Code + if reply.Code >= 400 && reply.Code < 500 { + out.State, out.Code, out.RetryAfter = model.RetryWait, "provider_temporarily_unavailable", time.Minute + } else if reply.Code >= 500 && reply.Code < 600 { + out.State, out.Code = model.Failed, "provider_rejected" + if reply.Code == 530 || reply.Code == 534 || reply.Code == 535 || reply.Code == 538 { + out.Code = "provider_authentication_failed" + } + } + } + return out +} diff --git a/backend/notifications/email/smtp_test.go b/backend/notifications/email/smtp_test.go new file mode 100644 index 0000000..a454bf2 --- /dev/null +++ b/backend/notifications/email/smtp_test.go @@ -0,0 +1,223 @@ +package email + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "io" + "mime" + "mime/multipart" + "net" + "net/http/httptest" + "net/mail" + "net/textproto" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "git.sendico.io/tech/sxchnge/notifications/model" +) + +type smtpFixture struct { + settings smtpSettings + root *x509.CertPool + body chan []byte + auth atomic.Bool +} + +func smtpServer(t *testing.T, reject string, code int) *smtpFixture { + t.Helper() + certServer := httptest.NewTLSServer(nil) + cert := certServer.TLS.Certificates[0] + root := x509.NewCertPool() + root.AddCert(certServer.Certificate()) + certServer.Close() + listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + host, port, _ := net.SplitHostPort(listener.Addr().String()) + n, _ := strconv.Atoi(port) + f := &smtpFixture{settings: smtpSettings{Host: host, Port: n, Username: "sender@example.com", SenderAddress: "sender@example.com", SenderName: "SXCHNGE", NetworkTimeout: 2}, root: root, body: make(chan []byte, 1)} + go func() { + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + if reject == "greeting" { + _ = conn.(*tls.Conn).Handshake() + _, _ = io.Copy(io.Discard, conn) + return + } + c := textproto.NewConn(conn) + if c.PrintfLine("220 localhost ESMTP") != nil { + return + } + for { + line, err := c.ReadLine() + if err != nil { + return + } + command := strings.SplitN(line, " ", 2)[0] + if command == reject { + _ = c.PrintfLine("%d rejected recipient user@example.com credential test-password", code) + continue + } + switch command { + case "EHLO": + _ = c.PrintfLine("250-localhost\r\n250 AUTH PLAIN") + case "AUTH": + want := "AUTH PLAIN " + base64.StdEncoding.EncodeToString([]byte("\x00sender@example.com\x00test-password")) + f.auth.Store(line == want) + _ = c.PrintfLine("235 authenticated") + case "MAIL", "RCPT": + _ = c.PrintfLine("250 OK") + case "DATA": + _ = c.PrintfLine("354 send message") + body, err := c.ReadDotBytes() + if err != nil { + return + } + f.body <- body + if reject == "final" { + _ = c.PrintfLine("%d final rejection", code) + } else if reject == "disconnect" { + return + } else { + _ = c.PrintfLine("250 queued") + } + case "QUIT": + // A lost QUIT reply must not turn accepted DATA into UNKNOWN. + return + default: + _ = c.PrintfLine("500 unexpected command") + } + } + }() + return f +} + +func (f *smtpFixture) sender() *smtpSender { + return &smtpSender{settings: f.settings, password: "test-password", tls: &tls.Config{ServerName: f.settings.Host, RootCAs: f.root, MinVersion: tls.VersionTLS12}} +} + +func TestSMTPAcceptanceAndMIME(t *testing.T) { + f := smtpServer(t, "", 0) + sender := f.sender() + email := model.Email{Address: "user@example.com", Subject: "Подтвердите почту", Text: "Confirm code 123456\n.line", HTML: "

Confirm code 123456

"} + out := sender.Send(context.Background(), "notification-id", email) + if out.State != model.Accepted || out.SMTPStatus != 250 || !f.auth.Load() { + t.Fatal("SMTP acceptance/authentication failed", out) + } + msg, err := mail.ReadMessage(strings.NewReader(string(<-f.body))) + if err != nil { + t.Fatal(err) + } + subject, err := new(mime.WordDecoder).DecodeHeader(msg.Header.Get("Subject")) + if err != nil || subject != email.Subject { + t.Fatal("wrong MIME subject", subject, err) + } + if msg.Header.Get("From") != "\"SXCHNGE\" " || !strings.HasSuffix(msg.Header.Get("Message-ID"), "@example.com>") { + t.Fatal("invalid sender/message identifier") + } + kind, params, err := mime.ParseMediaType(msg.Header.Get("Content-Type")) + if err != nil || kind != "multipart/alternative" { + t.Fatal("wrong MIME type", kind, err) + } + parts := multipart.NewReader(msg.Body, params["boundary"]) + for _, want := range []string{email.Text, email.HTML} { + part, err := parts.NextPart() + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(part) + if err != nil || strings.ReplaceAll(string(body), "\r\n", "\n") != want { + t.Fatal("incorrect MIME content", err) + } + } +} + +func TestSMTPFailureSafety(t *testing.T) { + for _, tc := range []struct { + stage string + code int + state model.State + failure string + }{ + {"AUTH", 535, model.Failed, "provider_authentication_failed"}, + {"RCPT", 550, model.Failed, "provider_rejected"}, + {"final", 451, model.RetryWait, "provider_temporarily_unavailable"}, + {"disconnect", 0, model.Unknown, "provider_outcome_unknown"}, + } { + t.Run(fmt.Sprintf("%s-%d", tc.stage, tc.code), func(t *testing.T) { + f := smtpServer(t, tc.stage, tc.code) + out := f.sender().Send(context.Background(), "id", model.Email{Address: "user@example.com", Subject: "Subject", Text: "Body"}) + if out.State != tc.state || out.Code != tc.failure || out.SMTPStatus != tc.code { + t.Fatal("incorrect SMTP classification", out) + } + if strings.Contains(out.ProviderMessage, "test-password") || strings.Contains(out.ProviderMessage, "user@example.com") { + t.Fatal("SMTP rejection leaked private data") + } + if tc.code == 451 && out.RetryAfter <= 0 { + t.Fatal("temporary rejection did not schedule retry") + } + }) + } +} + +func TestSMTPCertificateAndCancellation(t *testing.T) { + t.Run("untrusted certificate", func(t *testing.T) { + f := smtpServer(t, "", 0) + sender := f.sender() + sender.tls.RootCAs = x509.NewCertPool() + if out := sender.Send(context.Background(), "id", model.Email{Address: "user@example.com", Text: "Body"}); out.State != model.Unknown || f.auth.Load() { + t.Fatal("untrusted TLS accepted", out) + } + }) + t.Run("cancel blocked greeting", func(t *testing.T) { + f := smtpServer(t, "greeting", 0) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + started := time.Now() + if out := f.sender().Send(ctx, "id", model.Email{Address: "user@example.com", Text: "Body"}); out.State != model.Unknown || time.Since(started) > time.Second { + t.Fatal("SMTP ignored cancellation", out) + } + }) +} + +func TestSMTPConfigAndHeaderValidation(t *testing.T) { + t.Setenv("TEST_SMTP_PASSWORD", "test-password") + valid := map[string]any{"host": "mail.example.com", "port": 465, "username": "sender@example.com", "password_env": "TEST_SMTP_PASSWORD", "sender_address": "sender@example.com", "network_timeout": 10} + if _, err := New(Config{Driver: SMTP, Settings: valid}); err != nil { + t.Fatal(err) + } + for field, value := range map[string]any{"port": 0, "host": "https://bad", "username": "bad", "network_timeout": 0, "password_env": "MISSING_SMTP_PASSWORD", "typo": true} { + settings := map[string]any{} + for key, item := range valid { + settings[key] = item + } + settings[field] = value + if _, err := New(Config{Driver: SMTP, Settings: settings}); err == nil { + t.Fatalf("invalid setting %s accepted", field) + } + } + if _, err := smtpMessage(smtpSettings{}, "id", model.Email{Address: "user@example.com", Subject: "Subject\r\nBcc: stolen@example.com", Text: "Body"}); err == nil { + t.Fatal("header injection accepted") + } +} + +func TestSMTPAuthenticationDiagnosticRedactsEncodedCredential(t *testing.T) { + f := &smtpSender{settings: smtpSettings{Username: "sender@example.com"}, password: "a"} + auth := base64.StdEncoding.EncodeToString([]byte("\x00sender@example.com\x00a")) + out := f.failure(&textproto.Error{Code: 535, Msg: "Rejected AUTH " + auth}, "id", model.Email{}) + if out.SMTPStatus != 535 || strings.Contains(out.ProviderMessage, auth) || !strings.Contains(out.ProviderMessage, "AUTH [redacted]") { + t.Fatalf("encoded SMTP credential was not fully redacted: %q", out.ProviderMessage) + } +} diff --git a/backend/notifications/service/worker.go b/backend/notifications/service/worker.go index cfb68a6..1fa96f4 100644 --- a/backend/notifications/service/worker.go +++ b/backend/notifications/service/worker.go @@ -68,6 +68,9 @@ func (s *Service) Dispatch(ctx context.Context, id string) (bool, error) { if outcome.HTTPStatus != 0 { fields = append(fields, zap.Int("provider_status", outcome.HTTPStatus)) } + if outcome.SMTPStatus != 0 { + fields = append(fields, zap.Int("provider_smtp_status", outcome.SMTPStatus)) + } if outcome.ProviderError != "" { fields = append(fields, zap.String("provider_error", outcome.ProviderError)) } diff --git a/docs/local-development.md b/docs/local-development.md index eb57cb4..5cef411 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -30,7 +30,7 @@ NATS, notifications, BFF and Caddy serving the frontend. On first startup, bootstrap initializes/unseals persistent Vault, provisions AppRoles and secrets, initializes Mongo replica set `rs0`, configures Vault's Mongo plugin, and starts authenticated NATS JetStream. Product startup fails explicitly -until Resend credentials exist. Infrastructure remains available for provisioning. +until the SMTP mailbox password exists. Infrastructure remains available for provisioning. After all services pass their startup health checks and the public Caddy CA is exported, `make up` prints a Sendico-style completion message with the frontend, @@ -56,7 +56,7 @@ kv/ ├── services/ │ ├── bff/jwt # jwt │ └── notifications/ -│ └── email/resend # api_key only +│ └── email/smtp # password only └── infrastructure/ ├── nats/auth # token; read by NATS and notifications └── mongo/bootstrap # local administrator and replica-set bootstrap material @@ -74,29 +74,35 @@ There are no frontend certificates, notification certificates or generic `tls` entries in KV. Browser certificates belong to Caddy; internal gRPC is plaintext on the trusted private Docker network without caller authentication. -## Resend +## Local SMTP mail Open [Vault](http://127.0.0.1:18200) and edit -`kv/services/notifications/email/resend`: +`kv/services/notifications/email/smtp`: | Field | Value | | --- | --- | -| `api_key` | Your Resend API key | +| `password` | Password for the configured SMTP mailbox | The driver and sender settings belong in `infra/local/notifications/runtime.yml`: ```yaml email: - driver: resend + driver: smtp settings: - api_key_env: EMAIL_API_KEY - sender_address: "your-verified-sender@example.com" + host: mail.sendico.io + port: 465 + username: no-reply@sendico.io + password_env: SMTP_PASSWORD + network_timeout: 10 + sender_address: no-reply@sendico.io sender_name: "SXCHNGE" ``` -Set your verified address in that file and rebuild. It is intentionally empty by -default, so notifications cannot start with an unconfigured sender. Vault Agent -injects only the API key; it does not generate email configuration. +The local stack uses Sendico's existing `no-reply@sendico.io` mailbox with +authenticated implicit TLS on port 465 and certificate verification enabled. +Vault Agent injects only the password; host, username and sender are non-secret +runtime settings. Credentials are provisioned into SXCHNGE's own Vault; the +notification service does not read Sendico configuration or secrets at runtime. Local administrator login material is in private `.local/dev/admin/init.json`. Product containers cannot access it. Alternatively use the hidden terminal prompt: @@ -106,12 +112,15 @@ make email make build && make up ``` -No placeholder key or delivery fallback is installed. Startup validates presence; -it does not send an email or validate domain ownership with Resend. Updating the -API key causes Agent to restart notifications. Driver/sender changes require an -image rebuild. Existing API keys survive bootstrap. Migration removes legacy -driver/sender fields from the current Vault secret with a version-checked write; -the preceding version is preserved in Vault history and the private archive. +No placeholder password or delivery fallback is installed. Startup validates +configuration and secret presence; the SMTP connection is opened on demand. +Updating the password causes Agent to restart notifications. Driver/sender +changes require an image rebuild. Old Resend secrets remain in Vault history +but are not used or readable by the notifications AppRole. HTTP providers remain +available for other deployments; switching one requires matching Agent/policy +and runtime configuration. SMTP has no provider idempotency guarantee: explicit +4xx rejections retry, 5xx rejections fail, and ambiguous transport outcomes require +operator reconciliation before another send. ## Browser HTTPS and internal connections diff --git a/docs/signup-facts.md b/docs/signup-facts.md index a0636ce..5e95fed 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -94,6 +94,39 @@ gRPC failure response. No security scan or real customer email retry was run. ## Behavior and boundaries +### Local mail client + +The user subsequently selected Sendico's existing mailbox instead of Resend for +local notifications. The authoritative Sendico configuration resolves to +`no-reply@sendico.io`, `mail.sendico.io:465`, authenticated implicit TLS and a +10-second network timeout. SXCHNGE's owned `smtp` adapter consumes rendered email +models through the existing sender contract and queue; it does not import or +access Sendico business modules, databases or messaging. The password was copied +once, without printing it, from the running Sendico notification environment into +`services/notifications/email/smtp` in SXCHNGE Vault. Only `password` is stored +there; host, port, username, display name and sender remain in runtime YAML. + +The SMTP adapter verifies certificates, authenticates before submission, sends +UTF-8 MIME alternatives, and requires the final DATA 250 reply for acceptance. +SMTP 4xx replies retry through the existing worker policy; 5xx replies fail; +ambiguous transport outcomes remain UNKNOWN and are not immediately resent. +Message-ID is correlation only, not a provider idempotency guarantee. Existing +failed Resend notifications are not replayed or rewritten. The unverified +`sxchnge.com` Resend domain is no longer the selected local delivery path. + +Live verified-TLS SMTP authentication and NOOP succeeded with reply 250, without +submitting email. SMTP adapter race tests and backend lint pass. Local tooling +fixtures pass, including password-only Vault validation. Broader validation is +currently blocked by Docker: the service integration test timed out starting its +Testcontainers reaper; both `GOFLAGS=-p=2 make test` and its `-p=1` retry fail in +the existing credential-rotation setup while copying files into a disposable +MongoDB container (90-second context timeout). No checks were disabled. Fresh +stable image pins resolved on retry; two image builds hit a Go proxy TLS +handshake timeout during protobuf generator installation. The generator stage +now shares the existing BuildKit Go module/build caches, preserving its pinned +tool versions and generation command; the image rebuild is in progress. + + With `api.sendico` configured, the product BFF creates the local pending customer and activation token atomically, then registers an independent individual through Sendico's public HTTP API and submits the sign-up assertions. It persists the diff --git a/infra/images/backend.Dockerfile b/infra/images/backend.Dockerfile index 836195b..01e94d0 100644 --- a/infra/images/backend.Dockerfile +++ b/infra/images/backend.Dockerfile @@ -5,7 +5,7 @@ RUN apk add --no-cache protobuf protobuf-dev make WORKDIR /src COPY Makefile ./ COPY backend ./backend -RUN make generate-notifications +RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod make generate-notifications ARG MODULE WORKDIR /src/backend/${MODULE} RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/go/pkg/mod CGO_ENABLED=0 go build -trimpath -o /out/service . diff --git a/infra/local/notifications/runtime.yml b/infra/local/notifications/runtime.yml index 0e45830..e550c14 100644 --- a/infra/local/notifications/runtime.yml +++ b/infra/local/notifications/runtime.yml @@ -18,10 +18,14 @@ messaging: reconnect_wait: 2 templates: "/etc/sxchnge/templates.json" email: - driver: resend + driver: smtp settings: - api_key_env: EMAIL_API_KEY - sender_address: "" # Set your verified Resend sender here, then run make build. + host: mail.sendico.io + port: 465 + username: no-reply@sendico.io + password_env: SMTP_PASSWORD + network_timeout: 10 + sender_address: no-reply@sendico.io sender_name: "SXCHNGE" # Use replicas: 3 and environment: production for a three-node NATS cluster. # mode: ops validates resources without modifying them; app provisions them. diff --git a/infra/local/scripts/bootstrap.sh b/infra/local/scripts/bootstrap.sh index 9e07d40..357f8c5 100644 --- a/infra/local/scripts/bootstrap.sh +++ b/infra/local/scripts/bootstrap.sh @@ -61,7 +61,7 @@ provision_agents() { mkdir -p "$directory"; chmod 755 "$directory" case "$service" in bff) paths='services/bff/jwt';; - notifications) paths='infrastructure/nats/auth services/notifications/email/resend';; + notifications) paths='infrastructure/nats/auth services/notifications/email/smtp';; nats) paths=infrastructure/nats/auth;; sendico) paths=services/bff/sendico;; esac @@ -105,13 +105,12 @@ up() { random | jq -Rs --arg field "$field" '{($field):rtrimstr("\n")}' | put "$path" fi done - printf '{"api_key":""}' | ensure services/notifications/email/resend provision_mongo provision_agents compose up -d --wait --wait-timeout 120 nats retire_secrets normalize_email_secret - kv services/notifications/email/resend | email_valid || fail 'Vault kv/services/notifications/email/resend needs api_key. Run make email; configure the verified sender in infra/local/notifications/runtime.yml, then make build && make up.' + kv services/notifications/email/smtp | smtp_valid || fail 'Vault kv/services/notifications/email/smtp needs the mailbox password. Run make email.' compose up -d --wait --wait-timeout 180 export_ca cat <<'MESSAGE' @@ -134,7 +133,7 @@ MESSAGE } email() { existing - printf 'Resend API key (hidden): ' >&2 + printf 'SMTP mailbox password (hidden): ' >&2 tty_state=$(stty -g) trap 'stty "$tty_state"; cleanup' EXIT stty -echo @@ -143,7 +142,8 @@ email() { printf '\n' >&2 [ -n "$key" ] || fail 'A key is required' printf '%s' "$key" > "$WORK/email-key" - normalize_email_secret - jq -n --rawfile key "$WORK/email-key" '{api_key:$key}' | put services/notifications/email/resend - echo 'API key saved in Vault. Set the verified sender in infra/local/notifications/runtime.yml, then make build && make up.' + jq -n --rawfile password "$WORK/email-key" '{password:$password}' > "$WORK/smtp-secret" + smtp_valid < "$WORK/smtp-secret" || fail 'Invalid SMTP mailbox password' + put services/notifications/email/smtp < "$WORK/smtp-secret" + echo 'SMTP password saved in Vault. The notifications Agent reloads it automatically.' } diff --git a/infra/local/scripts/common.sh b/infra/local/scripts/common.sh index c719756..6f98c95 100644 --- a/infra/local/scripts/common.sh +++ b/infra/local/scripts/common.sh @@ -64,6 +64,7 @@ ensure() ( mongo() { compose exec -T mongo mongosh --quiet --file /dev/stdin > "$WORK/mongo-result" 2> "$WORK/mongo-error" || return 1; cat "$WORK/mongo-result"; } admin_auth() { kv infrastructure/mongo/bootstrap | jq -r '"const admin=db.getSiblingDB(\"admin\"); admin.auth(\"local-admin\","+(.password|tojson)+");"'; } email_valid() { jq -e '(.api_key|type=="string") and (.api_key|test("\\S"))' >/dev/null; } +smtp_valid() { jq -e 'keys==["password"] and (.password|type=="string" and length>0 and (explode|all(.>0 and .!=10 and .!=13)))' >/dev/null; } export_ca() { compose exec -T frontend cat /data/caddy/pki/authorities/local/root.crt | write "$STATE/ca.pem" 644 } diff --git a/infra/local/scripts/test.sh b/infra/local/scripts/test.sh index 0601b48..424191d 100644 --- a/infra/local/scripts/test.sh +++ b/infra/local/scripts/test.sh @@ -29,6 +29,10 @@ if images >/dev/null 2>&1; then fail 'Offline resolution succeeded'; fi [ "$(cat "$STATE/images.env")" = 'previous build' ] || fail 'Failed resolution replaced pins' if printf '{"api_key":"","sender_address":"notifications@localhost.test"}' | email_valid; then fail 'Empty email accepted'; fi printf '{"api_key":"fixture-only","sender_address":"sender@example.test"}' | email_valid +printf '{"password":"fixture-only"}' | smtp_valid +for invalid in '{"password":""}' '{"password":"bad\nvalue"}' '{"password":"bad\u0000value"}' '{"password":"fixture","username":"not-a-secret"}'; do + if printf '%s' "$invalid" | smtp_valid; then fail 'Invalid SMTP secret accepted'; fi +done # Initialized Vault without matching local keys must fail without initializing it again. vault() { printf '{"initialized":true}'; } if (initialize) > "$WORK/initialize-error" 2>&1; then fail 'Missing local keys accepted'; fi diff --git a/infra/local/scripts/verify.sh b/infra/local/scripts/verify.sh index ebb43f0..34f5c54 100644 --- a/infra/local/scripts/verify.sh +++ b/infra/local/scripts/verify.sh @@ -61,11 +61,11 @@ layout() { if [ -f "$STATE/sendico/settings.env" ]; then kv services/bff/sendico | jq -e 'keys==["client_secret","login","password"]' >/dev/null || fail 'Sendico secret contains non-secret configuration' fi - kv services/notifications/email/resend | jq -e 'keys==["api_key"]' >/dev/null || fail 'Non-secret email configuration remains in Vault' + kv services/notifications/email/smtp | jq -e 'keys==["password"]' >/dev/null || fail 'SMTP secret contains non-secret configuration' for old in bootstrap/mongo bff caller services/notifications/callers/bff email nats tls frontend-tls notifications-tls; do [ "$(vault "kv/metadata/$old" GET yes)" = null ] || fail "Obsolete KV entry remains: $old" done - for path in services/bff/jwt services/notifications/email/resend infrastructure/nats/auth infrastructure/mongo/bootstrap; do + for path in services/bff/jwt services/notifications/email/smtp infrastructure/nats/auth infrastructure/mongo/bootstrap; do vault "kv/metadata/$path" | jq -e '.data.current_version>0' >/dev/null || fail "Missing canonical KV entry: $path" done echo 'PASS: canonical Vault hierarchy present; obsolete flat and TLS entries absent' diff --git a/infra/local/vault/notifications.hcl b/infra/local/vault/notifications.hcl index 7e332ff..fd4c967 100644 --- a/infra/local/vault/notifications.hcl +++ b/infra/local/vault/notifications.hcl @@ -22,8 +22,8 @@ env_template "SXCHNGE_NATS_URL" { -env_template "EMAIL_API_KEY" { - contents = "{{ with secret \"kv/data/services/notifications/email/resend\" }}{{ .Data.data.api_key }}{{ end }}" +env_template "SMTP_PASSWORD" { + contents = "{{ with secret \"kv/data/services/notifications/email/smtp\" }}{{ .Data.data.password }}{{ end }}" error_on_missing_key = true } -- 2.54.0 From 69fba8df703e496573aaa27927f797e064bbe53b Mon Sep 17 00:00:00 2001 From: Stephan D Date: Sat, 17 Oct 2026 21:17:39 +0200 Subject: [PATCH 05/10] Record SMTP deployment blocker and remaining restart (#57) --- docs/signup-facts.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/signup-facts.md b/docs/signup-facts.md index 5e95fed..7c41392 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -124,7 +124,23 @@ MongoDB container (90-second context timeout). No checks were disabled. Fresh stable image pins resolved on retry; two image builds hit a Go proxy TLS handshake timeout during protobuf generator installation. The generator stage now shares the existing BuildKit Go module/build caches, preserving its pinned -tool versions and generation command; the image rebuild is in progress. +tool versions and generation command. Cache seeding also stalled during file +extraction; it was canceled after copying the pinned module archives. A normal +build retry remained blocked before starting a compiler process. A fallback +using the same freshly pulled, digest-pinned Go image, canonical Make generation +command and existing module cache stalled during ordinary container startup, +before package installation or compilation produced output. + +Deployment is not complete: the running notifications container still uses the +previous image. The SMTP Vault password and source configuration are prepared, +but its Agent policy/template have not been activated against that old image. +Vault responds over HTTP, while Vault/Mongo Docker health checks time out and +the local Sendico session is unavailable. Restarting Docker Desktop was requested +for recovery because it also interrupts the separate Sendico stack; it has not +been authorized or performed. Rebuild, activate the matching Agent configuration, +restart notifications and verify the live `email_driver: smtp` startup log after +Docker is responsive. Do not treat the successful SMTP login as a deployed +notification delivery check. With `api.sendico` configured, the product BFF creates the local pending customer -- 2.54.0 From 3aeea9aa776442955c80ad640322c371d6625cf4 Mon Sep 17 00:00:00 2001 From: Stephan D Date: Sat, 17 Oct 2026 23:38:10 +0200 Subject: [PATCH 06/10] Record healthy SMTP deployment and passing validation (#57) --- docs/signup-facts.md | 48 ++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/docs/signup-facts.md b/docs/signup-facts.md index 7c41392..cf847bd 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -114,33 +114,29 @@ Message-ID is correlation only, not a provider idempotency guarantee. Existing failed Resend notifications are not replayed or rewritten. The unverified `sxchnge.com` Resend domain is no longer the selected local delivery path. -Live verified-TLS SMTP authentication and NOOP succeeded with reply 250, without -submitting email. SMTP adapter race tests and backend lint pass. Local tooling -fixtures pass, including password-only Vault validation. Broader validation is -currently blocked by Docker: the service integration test timed out starting its -Testcontainers reaper; both `GOFLAGS=-p=2 make test` and its `-p=1` retry fail in -the existing credential-rotation setup while copying files into a disposable -MongoDB container (90-second context timeout). No checks were disabled. Fresh -stable image pins resolved on retry; two image builds hit a Go proxy TLS -handshake timeout during protobuf generator installation. The generator stage -now shares the existing BuildKit Go module/build caches, preserving its pinned -tool versions and generation command. Cache seeding also stalled during file -extraction; it was canceled after copying the pinned module archives. A normal -build retry remained blocked before starting a compiler process. A fallback -using the same freshly pulled, digest-pinned Go image, canonical Make generation -command and existing module cache stalled during ordinary container startup, -before package installation or compilation produced output. +Live verified-TLS SMTP authentication and NOOP succeeded with reply 250 using +the password from SXCHNGE Vault, without submitting email. SMTP adapter race +tests, `make lint-backend`, `make test-dev-tooling` and `make test-dev-infra` +pass, including canonical Vault layout, service permission isolation, database +credential issuance/renewal/revocation and transactions. The final +`GOFLAGS=-p=2 make test` passes all backend race tests and frontend suites. -Deployment is not complete: the running notifications container still uses the -previous image. The SMTP Vault password and source configuration are prepared, -but its Agent policy/template have not been activated against that old image. -Vault responds over HTTP, while Vault/Mongo Docker health checks time out and -the local Sendico session is unavailable. Restarting Docker Desktop was requested -for recovery because it also interrupts the separate Sendico stack; it has not -been authorized or performed. Rebuild, activate the matching Agent configuration, -restart notifications and verify the live `email_driver: smtp` startup log after -Docker is responsive. Do not treat the successful SMTP login as a deployed -notification delivery check. +Earlier attempts hit Docker archive-copy/reaper timeouts, Go proxy TLS failures +and stalled cache extraction/container startup. The user authorized restarting +Docker Desktop. After recovery, the normal Compose `build --pull notifications` +succeeded with the resolved stable digest pins; no checks were disabled. The +generator stage shares the existing BuildKit Go module/build caches and retains +the pinned tools and canonical Make generation command. A discarded fallback +compiler attempt also failed to access its checksum-state directory; it was not +used for the deployed artifact. + +The matching Vault Agent policy/template were applied, and notifications was +recreated healthy. Its running image matches the rebuilt image +`sha256:d4cbc0cb9005e9cef9381abdf2a13f672104d160536ee6d77ac73a909c0a8638`, +and its startup log confirms `email_driver: smtp`. No customer email was sent or +failed Resend job replayed; SMTP authentication is not a recipient-delivery +check. Sendico's Vault sealed during the Docker restart and was unsealed with +its existing local key, without reinitialization or exposing credentials. With `api.sendico` configured, the product BFF creates the local pending customer -- 2.54.0 From 4844fb5bc950fff0aee1880f093f5b4d817a682b Mon Sep 17 00:00:00 2001 From: Stephan D Date: Sat, 17 Oct 2026 23:41:41 +0200 Subject: [PATCH 07/10] Record verified local API recovery after Docker restart (#57) --- docs/signup-facts.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/signup-facts.md b/docs/signup-facts.md index cf847bd..1897e5c 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -137,6 +137,12 @@ and its startup log confirms `email_driver: smtp`. No customer email was sent or failed Resend job replayed; SMTP authentication is not a recipient-delivery check. Sendico's Vault sealed during the Docker restart and was unsealed with its existing local key, without reinitialization or exposing credentials. +Its affected services were restarted in dependency order. The local Sendico +session sidecar shares the BFF network namespace, so it was restarted after the +BFF to restore Docker DNS and the loopback bridge. Final +`sh infra/local/tools.sh verify` passes: seven healthy services, verified HTTPS +for frontend/proxied BFF, authenticated Sendico currency/ledger reads and the +Vault/database checks. No Sendico repository files were changed. With `api.sendico` configured, the product BFF creates the local pending customer -- 2.54.0 From 021bd043243d980d27582ced36c13a7834bc7771 Mon Sep 17 00:00:00 2001 From: Stephan D Date: Sun, 18 Oct 2026 13:54:30 +0200 Subject: [PATCH 08/10] Reap orphaned frontend HTTPS probe processes (#57) --- docs/local-development.md | 7 +++++++ docs/signup-facts.md | 18 ++++++++++++++++++ infra/local/compose.yml | 2 ++ 3 files changed, 27 insertions(+) diff --git a/docs/local-development.md b/docs/local-development.md index 5cef411..9a45573 100644 --- a/docs/local-development.md +++ b/docs/local-development.md @@ -27,6 +27,13 @@ project directory. Dockerfiles are in `infra/images/`. Runtime and startup files are grouped by service under `infra/local/`. The stack consists of Vault, Mongo, NATS, notifications, BFF and Caddy serving the frontend. +The frontend container uses Compose `init: true` to reap orphaned `ssl_client` +children created by BusyBox's HTTPS `wget` health probe. Keep this enabled: +Caddy alone does not reap those children when it runs as PID 1, and repeated +probes can accumulate thousands of zombie processes. Recreate the frontend +container when applying this setting; restarting an old container does not +change its init configuration. + On first startup, bootstrap initializes/unseals persistent Vault, provisions AppRoles and secrets, initializes Mongo replica set `rs0`, configures Vault's Mongo plugin, and starts authenticated NATS JetStream. Product startup fails explicitly diff --git a/docs/signup-facts.md b/docs/signup-facts.md index 1897e5c..48927cc 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -144,6 +144,24 @@ BFF to restore Docker DNS and the loopback bridge. Final for frontend/proxied BFF, authenticated Sendico currency/ledger reads and the Vault/database checks. No Sendico repository files were changed. +On October 18, the frontend had 15,897 processes, including 15,888 unreaped +`ssl_client` zombie children of Caddy PID 1. The three-second HTTPS health +probe used BusyBox `wget`, which left those children behind. This confirmed a +recurring local infrastructure leak, independently of SMTP authentication. +Compose now enables `init: true` for frontend, and only that container was +recreated. PID 1 is now `docker-init`; 40 consecutive HTTPS probes completed +with zero zombie processes and a bounded count of 15 processes afterward. +Compose configuration validation and `make test-dev-tooling` pass. The API +health endpoint returns HTTP 200; a diagnostic request using the existing +account and a generated incorrect password returned HTTP 400 `email_in_use` +without creating an account or sending email. + +The contemporaneous full local verification attempt is blocked by MongoDB's +five-second health-check timeout under continuing host load; it is not recorded +as passing. The exact response to the reported Safari attempt was not captured, +and the frontend's generic unavailable text also covers some HTTP 400 input +errors. A fresh user retry is needed to confirm the visible signup outcome. + With `api.sendico` configured, the product BFF creates the local pending customer and activation token atomically, then registers an independent individual through diff --git a/infra/local/compose.yml b/infra/local/compose.yml index a7bb6d3..4377f84 100644 --- a/infra/local/compose.yml +++ b/infra/local/compose.yml @@ -119,6 +119,8 @@ services: frontend: image: sxchnge-local/frontend:dev + # Reap orphaned ssl_client children from BusyBox wget HTTPS health probes. + init: true build: context: . dockerfile: infra/images/frontend.Dockerfile -- 2.54.0 From 13cc9c302ef6cb5a15653bbbbc574bb72cf779cf Mon Sep 17 00:00:00 2001 From: Stephan D Date: Mon, 19 Oct 2026 12:34:37 +0200 Subject: [PATCH 09/10] Recover local Sendico session after BFF network restarts (#57) --- docs/local-sendico.md | 36 ++++++++++++++++++++++++++++ docs/signup-facts.md | 27 +++++++++++++++++++++ infra/local/scripts/bootstrap.sh | 2 +- infra/local/scripts/common.sh | 13 +++++++++++ infra/local/scripts/rotation.sh | 6 +++-- infra/local/scripts/test.sh | 40 +++++++++++++++++++++++++++++++- infra/local/sendico/compose.yml | 3 +++ infra/local/sendico/network.sh | 20 ++++++++++++++++ infra/local/sendico/session.sh | 19 +++++++++++---- 9 files changed, 158 insertions(+), 8 deletions(-) create mode 100644 infra/local/sendico/network.sh diff --git a/docs/local-sendico.md b/docs/local-sendico.md index d1f42b5..a01a8a3 100644 --- a/docs/local-sendico.md +++ b/docs/local-sendico.md @@ -105,6 +105,27 @@ docker compose --project-directory . -f infra/local/compose.yml \ -f infra/local/sendico/compose.yml --env-file .local/dev/images.env restart sendico ``` +A BFF container restart can leave its session worker in the old shared network +namespace. The worker checks Vault DNS every ten seconds, with a five-second +lookup limit. Three consecutive failures log a credential-free recovery reason +and terminate the session process; Docker's restart policy then attaches it to +the BFF's current namespace. A successful lookup resets the counter, so transient +DNS failures do not immediately restart the worker. + +Container recreation changes the BFF container ID as well as its namespace. +Bootstrap and credential-rotation tooling recreate BFF and its configured session +worker together. Compose also marks the session dependency for restart when BFF +is explicitly updated. For a manual forced recreation, select both services: + +```sh +docker compose --project-directory . -f infra/local/compose.yml \ + -f infra/local/sendico/compose.yml --env-file .local/dev/images.env \ + up -d --no-deps --force-recreate --wait bff sendico +``` + +The parent process cleans up its bridge, DNS monitor and renewal delay on exit. +This recovery retains the configured provider URL, account and replay namespace. + `make test-dev` checks seven healthy services, verified browser TLS, the Vault layout and least-privilege identities, Mongo transactions/lease rotation, and authenticated Sendico currency and ledger reads through the same loopback @@ -130,6 +151,21 @@ and the established CA. A subsequent verified TLS check must succeed. ## Acceptance and operator UX review +Recovery follow-up for [task #57](https://git.sendico.io/tech/sxchnge/issues/57) / +[PR #59](https://git.sendico.io/tech/sxchnge/pulls/59), validated on 2026-10-19: +`make test-dev-tooling`, `make test-dev-rotation`, Compose configuration validation, +`sh infra/local/tools.sh verify` and `git diff --check` pass. A BFF-only restart +triggered automatic session restart and restored matching network namespaces; +paired recreation also finished healthy. Rotation observed three real Vault +pool replacements, an unchanged BFF process and 534 transaction-backed requests +with zero failures. The final check verifies all seven SXCHNGE services, browser +HTTPS, Vault/database boundaries and authenticated Sendico currency/ledger reads. +An earlier check failed because Sendico's ledger announcement connection was +closed; restoring Discovery and its service announcers in dependency order +recovered that existing stand. No Sendico source or financial data was changed, +and no email or payment was sent. This follow-up changes shell/Compose tooling; +it does not rerun the earlier full Go/Flutter suite for unchanged product source. + Validated on 2026-10-16: `make build`, `make up`, `make test-dev-tooling`, `make test-dev-infra`, `make test-dev`, and `git diff --check` passed. Both product sites, Sendico Backoffice and both API health endpoints returned HTTP 200; diff --git a/docs/signup-facts.md b/docs/signup-facts.md index 48927cc..7e2adff 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -162,6 +162,33 @@ as passing. The exact response to the reported Safari attempt was not captured, and the frontend's generic unavailable text also covers some HTTP 400 input errors. A fresh user retry is needed to confirm the visible signup outcome. +On October 19 the failing API was directly observed returning HTTP 502. The BFF +had restarted while the session worker retained its old network namespace; +`/proc/self/ns/net` differed between them, and the worker could not resolve +Vault. Sendico's public API itself returned HTTP 200. Restarting only the worker +reattached its namespace and restored SXCHNGE API HTTP 200. This is a confirmed +cause of that 502, separate from the previous process leak and host-load finding. + +The session now monitors bounded DNS failures and exits after three consecutive +failures so Docker can reattach it automatically. Bootstrap and rotation pair +BFF/session recreation, with Compose dependency restart enabled. Offline shell +fixtures cover transient-failure reset, sustained-failure recovery, safe parent +targeting and conditional service pairing. A live BFF-only restart automatically +restarted the session and restored matching namespaces, fresh session readiness +and API HTTP 200. A second live test recreated BFF and its session together and +confirmed healthy services with matching namespaces. No provider bindings, +customer records, credentials or Sendico source files were changed. + +Final October 19 validation passes: `make test-dev-tooling`, Compose config +validation, `make test-dev-rotation` (three real rotations, unchanged BFF process, +534 transaction-backed requests and zero failures), and +`sh infra/local/tools.sh verify`. The last confirms all seven healthy services, +verified frontend/BFF HTTPS, Vault/database checks and authenticated Sendico +currency/ledger reads. An initial ledger-read failure was resolved by restoring +Sendico Discovery and its announcers in their documented dependency order after +closed NATS connections. No email was sent during validation. The recovery fix +and its validation are included in [PR #59](https://git.sendico.io/tech/sxchnge/pulls/59). + With `api.sendico` configured, the product BFF creates the local pending customer and activation token atomically, then registers an independent individual through diff --git a/infra/local/scripts/bootstrap.sh b/infra/local/scripts/bootstrap.sh index 357f8c5..fe304a9 100644 --- a/infra/local/scripts/bootstrap.sh +++ b/infra/local/scripts/bootstrap.sh @@ -81,7 +81,7 @@ provision_agents() { if [ "$service" != sendico ] && ! cmp -s "infra/local/vault/$service.hcl" "$directory/config.hcl"; then cat "infra/local/vault/$service.hcl" | write "$directory/config.hcl" 444 if compose ps --status running --services "$service" | grep -qx "$service"; then - compose up -d --no-deps --force-recreate --wait --wait-timeout 120 "$service" + recreate_services 120 "$service" fi fi done diff --git a/infra/local/scripts/common.sh b/infra/local/scripts/common.sh index 6f98c95..1804546 100644 --- a/infra/local/scripts/common.sh +++ b/infra/local/scripts/common.sh @@ -21,6 +21,19 @@ compose() { docker compose --project-directory . -f infra/local/compose.yml --env-file "$STATE/images.env" "$@" fi } +# A recreated BFF has a new container ID. Recreate its namespace-sharing session +# in the same Compose operation so neither waits on the other's old namespace. +recreate_services() { + recreate_timeout=$1; shift + case " $* " in + *' bff '*) + if [ -f "$STATE/sendico/settings.env" ]; then + case " $* " in *' sendico '*) ;; *) set -- "$@" sendico;; esac + fi + ;; + esac + compose up -d --no-deps --force-recreate --wait --wait-timeout "$recreate_timeout" "$@" +} write() { target=$1; mode=${2:-600} mkdir -p "$(dirname "$target")" diff --git a/infra/local/scripts/rotation.sh b/infra/local/scripts/rotation.sh index 7355773..f9b0203 100644 --- a/infra/local/scripts/rotation.sh +++ b/infra/local/scripts/rotation.sh @@ -10,7 +10,9 @@ rotation() ( for pid in $workers; do wait "$pid" || true; done restore_failed=0 vault database/roles/bff POST < "$WORK/original-role" >/dev/null || { echo 'ERROR: restore BFF role TTL manually' >&2; restore_failed=1; } - for service in frontend bff; do + restore_services='frontend bff' + if [ -f "$STATE/sendico/settings.env" ]; then restore_services='frontend sendico bff'; fi + for service in $restore_services; do if ! grep -qx "$service" "$WORK/running"; then compose stop "$service"; compose rm -f "$service"; fi done return "$restore_failed" @@ -19,7 +21,7 @@ rotation() ( trap 'exit 130' INT trap 'exit 143' TERM jq '.+{default_ttl:"12s",max_ttl:"24s"}' "$WORK/original-role" | vault database/roles/bff POST >/dev/null - compose up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend + recreate_services 90 bff frontend export_ca https -f https://localhost:5126/ > "$WORK/index" grep -q flutter_bootstrap "$WORK/index" || fail 'Frontend bundle is not served' diff --git a/infra/local/scripts/test.sh b/infra/local/scripts/test.sh index 424191d..b026f9d 100644 --- a/infra/local/scripts/test.sh +++ b/infra/local/scripts/test.sh @@ -4,10 +4,48 @@ set -o pipefail # Shell syntax and static analysis cover the actual container shell scripts. shellcheck -s ash -x -S warning infra/local/scripts/dispatch.sh infra/local/scripts/test.sh shellcheck -x -S warning infra/local/tools.sh infra/local/scripts/start-service.sh infra/local/mongo/start.sh infra/local/nats/start.sh infra/local/vault/start.sh -shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh +shellcheck -s ash -x -S warning infra/local/sendico/session.sh infra/local/sendico/check.sh infra/local/sendico/network.sh . infra/local/scripts/common.sh . infra/local/scripts/images.sh . infra/local/scripts/bootstrap.sh +# A transient DNS failure must reset the recovery counter, while sustained loss +# must terminate only the owning session process so Docker can reattach it. +( + . infra/local/sendico/network.sh + network_calls=0; network_recovered=0 + sleep() { :; } + timeout() { [ "$1" = 5 ] || exit 1; shift; "$@"; } + getent() { + [ "$*" = 'hosts vault' ] || exit 1 + network_calls=$((network_calls+1)) + [ "$network_calls" = 3 ] + } + kill() { + [ "$*" = '-TERM 1234' ] && [ "$network_calls" = 6 ] || exit 1 + network_recovered=1 + } + watch_network 1234 2> "$WORK/network-recovery" + [ "$network_recovered" = 1 ] || exit 1 + grep -q 'reattach BFF network' "$WORK/network-recovery" + for invalid_parent in '' 0 00 invalid; do + if watch_network "$invalid_parent"; then exit 1; fi + done +) +echo 'PASS: session network recovery tolerates transient DNS failures and targets its parent after sustained loss' +( + STATE="$WORK/recreate-state"; mkdir -p "$STATE/sendico" + compose() { printf '%s\n' "$*" > "$WORK/recreate-command"; } + recreate_services 90 bff frontend + grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend' "$WORK/recreate-command" + touch "$STATE/sendico/settings.env" + recreate_services 90 bff frontend + grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 90 bff frontend sendico' "$WORK/recreate-command" + recreate_services 120 bff sendico + grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 120 bff sendico' "$WORK/recreate-command" + recreate_services 120 notifications + grep -qx 'up -d --no-deps --force-recreate --wait --wait-timeout 120 notifications' "$WORK/recreate-command" +) +echo 'PASS: BFF recreation pairs its configured session exactly once and preserves unrelated service scope' # Offline fixtures exercise stable selection, reject prereleases, and preserve pins. get() { case "$1" in diff --git a/infra/local/sendico/compose.yml b/infra/local/sendico/compose.yml index 9f96ad8..a31c562 100644 --- a/infra/local/sendico/compose.yml +++ b/infra/local/sendico/compose.yml @@ -12,10 +12,13 @@ services: restart: unless-stopped # Keep the adapter's loopback-only HTTP rule; forward to the public stand API. network_mode: service:bff + depends_on: + bff: {condition: service_started, restart: true} entrypoint: [/bin/sh, /etc/sendico/session.sh] env_file: ./.local/dev/sendico/settings.env volumes: - ./infra/local/sendico/session.sh:/etc/sendico/session.sh:ro + - ./infra/local/sendico/network.sh:/etc/sendico/network.sh:ro - ./infra/local/sendico/check.sh:/etc/sendico/check.sh:ro - ./.local/dev/agents/sendico:/etc/agent:ro - sendico-session:/run/sendico diff --git a/infra/local/sendico/network.sh b/infra/local/sendico/network.sh new file mode 100644 index 0000000..3143c84 --- /dev/null +++ b/infra/local/sendico/network.sh @@ -0,0 +1,20 @@ +# Restarting the network-owning BFF can strand this sidecar in its old namespace. +# Exiting the session process lets Docker rejoin the BFF's current namespace. +watch_network() { + network_parent=$1 + case "$network_parent" in ''|*[!0-9]*) return 1;; esac + [ "$network_parent" -gt 0 ] || return 1 + network_failures=0 + while sleep 10; do + if timeout 5 getent hosts vault >/dev/null 2>&1; then + network_failures=0 + continue + fi + network_failures=$((network_failures+1)) + if [ "$network_failures" -ge 3 ]; then + echo 'Local Sendico network DNS unavailable; restarting session container to reattach BFF network.' >&2 + kill -TERM "$network_parent" 2>/dev/null || true + return + fi + done +} diff --git a/infra/local/sendico/session.sh b/infra/local/sendico/session.sh index 1dd34ed..995eb87 100644 --- a/infra/local/sendico/session.sh +++ b/infra/local/sendico/session.sh @@ -1,13 +1,24 @@ #!/bin/sh set -eu umask 077 +# shellcheck source=infra/local/sendico/network.sh +. /etc/sendico/network.sh mkdir -p /private /run/sendico chmod 755 /run/sendico cd /private +bridge=''; monitor=''; delay='' +cleanup() { + for child in "$bridge" "$monitor" "$delay"; do + if [ -n "$child" ]; then kill "$child" 2>/dev/null || true; fi + done +} +trap cleanup EXIT +trap 'exit 0' INT TERM socat TCP4-LISTEN:18081,bind=127.0.0.1,reuseaddr,fork TCP4:host.docker.internal:8080 & bridge=$! -trap 'kill "$bridge" 2>/dev/null || true' EXIT -trap 'exit 0' INT TERM +session_pid=$$ +(trap - EXIT INT TERM; watch_network "$session_pid") & +monitor=$! session() { jq -n --rawfile role /etc/agent/role-id --rawfile secret /etc/agent/secret-id \ @@ -36,10 +47,10 @@ session() { while :; do if session; then echo 'Local Sendico session ready; next renewal in 20 minutes.' - sleep 1200 & wait $! + sleep 1200 & delay=$!; wait "$delay" else echo 'Local Sendico session unavailable; check stand health and integration credentials. Retrying in 5 minutes.' >&2 rm -f request.json response.json token vault.header vault.json credentials.json login.json - sleep 300 & wait $! + sleep 300 & delay=$!; wait "$delay" fi done -- 2.54.0 From 4c4b6393219c28c254e820621b2764621453439b Mon Sep 17 00:00:00 2001 From: Stephan D Date: Mon, 19 Oct 2026 12:41:26 +0200 Subject: [PATCH 10/10] Record private local signup request outcomes (#57) --- docs/signup-facts.md | 28 ++++++++++++++++++++++++++++ infra/local/frontend/Caddyfile | 15 +++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/docs/signup-facts.md b/docs/signup-facts.md index 7e2adff..ae86783 100644 --- a/docs/signup-facts.md +++ b/docs/signup-facts.md @@ -269,6 +269,34 @@ adapter fingerprint and blocks existing bindings for explicit reconciliation. Token rotation for the same account retains that fingerprint. Plan such changes as migrations; never reset a binding just to bypass the check. +## Local signup request diagnosis + +The October 19 follow-up screenshot still shows the generic unavailable message +after the network recovery. That screenshot alone does not establish a 503: +the frontend also maps some rejected inputs and browser transport failures to +that message. At the follow-up check the health endpoint returned 200, and BFF +signup counters contained only the operator's two deliberately invalid probes. +The user's current browser failure has not yet been reproduced or resolved. + +Local Caddy now logs requests to exactly `/api/v1/accounts/signup` under +`http.log.access.signup`, including HTTP status and duration. The complete +request object, response headers and user identifier are removed before log +encoding, so paths/queries, cookies, authorization, addresses and user-supplied +headers cannot enter these access records. Bodies are never logged. Other +requests, including health probes and email confirmation URLs, are skipped. +Inspect with: + +```sh +docker logs --since 5m sxchnge-local-frontend-1 2>&1 | + jq -Rrc 'fromjson? | select(.logger=="http.log.access.signup")' +``` + +Validation: Caddy configuration validation and reload passed; a deliberately +invalid signup returned 400 and produced the expected status/duration record. +Sentinel query, cookie and custom-header values were absent from its log. +The frontend was rebuilt with freshly resolved stable image pins. No real +signup or customer email was submitted during this diagnostic work. + ## Persistence, retries and recovery The local `signup_facts` collection uses the customer ID as `_id`, so its unique diff --git a/infra/local/frontend/Caddyfile b/infra/local/frontend/Caddyfile index 8f39041..31173c3 100644 --- a/infra/local/frontend/Caddyfile +++ b/infra/local/frontend/Caddyfile @@ -5,6 +5,21 @@ https://localhost:8443 { tls internal + # Record signup status/timing even when the API rejects a request before + # account creation. Drop the entire request and response headers so URLs, + # cookies, addresses and browser-provided values cannot enter access logs. + log signup { + format filter { + wrap json + fields { + request delete + resp_headers delete + user_id delete + } + } + } + @otherRequests not path /api/v1/accounts/signup + log_skip @otherRequests root * /srv handle /api/* { reverse_proxy bff:8082 -- 2.54.0