service backend
This commit is contained in:
Stephan D
84
api/fx/ingestor/internal/ingestor/metrics.go
Normal file
84
api/fx/ingestor/internal/ingestor/metrics.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package ingestor
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/tech/sendico/fx/ingestor/internal/config"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
type serviceMetrics struct {
|
||||
pollDuration *prometheus.HistogramVec
|
||||
pollTotal *prometheus.CounterVec
|
||||
pairDuration *prometheus.HistogramVec
|
||||
pairTotal *prometheus.CounterVec
|
||||
pairLastUpdate *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
var (
|
||||
metricsOnce sync.Once
|
||||
globalMetricsRef *serviceMetrics
|
||||
)
|
||||
|
||||
func getServiceMetrics() *serviceMetrics {
|
||||
metricsOnce.Do(func() {
|
||||
reg := prometheus.DefaultRegisterer
|
||||
globalMetricsRef = &serviceMetrics{
|
||||
pollDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "fx_ingestor_poll_duration_seconds",
|
||||
Help: "Duration of a polling cycle.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"result"}),
|
||||
pollTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "fx_ingestor_poll_total",
|
||||
Help: "Total polling cycles executed.",
|
||||
}, []string{"result"}),
|
||||
pairDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "fx_ingestor_pair_duration_seconds",
|
||||
Help: "Duration of individual pair ingestion.",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"source", "provider", "symbol", "result"}),
|
||||
pairTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "fx_ingestor_pair_total",
|
||||
Help: "Total ingestion attempts per pair.",
|
||||
}, []string{"source", "provider", "symbol", "result"}),
|
||||
pairLastUpdate: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "fx_ingestor_pair_last_success_unix",
|
||||
Help: "Unix timestamp of the last successful ingestion per pair.",
|
||||
}, []string{"source", "provider", "symbol"}),
|
||||
}
|
||||
})
|
||||
return globalMetricsRef
|
||||
}
|
||||
|
||||
func (m *serviceMetrics) observePoll(duration time.Duration, err error) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
result := labelForError(err)
|
||||
m.pollDuration.WithLabelValues(result).Observe(duration.Seconds())
|
||||
m.pollTotal.WithLabelValues(result).Inc()
|
||||
}
|
||||
|
||||
func (m *serviceMetrics) observePair(pair config.Pair, duration time.Duration, err error) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
result := labelForError(err)
|
||||
labels := []string{pair.Source.String(), pair.Provider, pair.Symbol, result}
|
||||
m.pairDuration.WithLabelValues(labels...).Observe(duration.Seconds())
|
||||
m.pairTotal.WithLabelValues(labels...).Inc()
|
||||
if err == nil {
|
||||
m.pairLastUpdate.WithLabelValues(pair.Source.String(), pair.Provider, pair.Symbol).
|
||||
Set(float64(time.Now().Unix()))
|
||||
}
|
||||
}
|
||||
|
||||
func labelForError(err error) string {
|
||||
if err != nil {
|
||||
return "error"
|
||||
}
|
||||
return "success"
|
||||
}
|
||||
207
api/fx/ingestor/internal/ingestor/service.go
Normal file
207
api/fx/ingestor/internal/ingestor/service.go
Normal file
@@ -0,0 +1,207 @@
|
||||
package ingestor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/big"
|
||||
"time"
|
||||
|
||||
"github.com/tech/sendico/fx/ingestor/internal/config"
|
||||
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
|
||||
"github.com/tech/sendico/fx/ingestor/internal/market"
|
||||
mmodel "github.com/tech/sendico/fx/ingestor/internal/model"
|
||||
"github.com/tech/sendico/fx/storage"
|
||||
"github.com/tech/sendico/fx/storage/model"
|
||||
"github.com/tech/sendico/pkg/mlogger"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
logger mlogger.Logger
|
||||
cfg *config.Config
|
||||
rates storage.RatesStore
|
||||
pairs []config.Pair
|
||||
connectors map[mmodel.Driver]mmodel.Connector
|
||||
metrics *serviceMetrics
|
||||
}
|
||||
|
||||
func New(logger mlogger.Logger, cfg *config.Config, repo storage.Repository) (*Service, error) {
|
||||
if logger == nil {
|
||||
return nil, fmerrors.New("ingestor: nil logger")
|
||||
}
|
||||
if cfg == nil {
|
||||
return nil, fmerrors.New("ingestor: nil config")
|
||||
}
|
||||
if repo == nil {
|
||||
return nil, fmerrors.New("ingestor: nil repository")
|
||||
}
|
||||
|
||||
connectors, err := market.BuildConnectors(logger, cfg.Market.Sources)
|
||||
if err != nil {
|
||||
return nil, fmerrors.Wrap("build connectors", err)
|
||||
}
|
||||
|
||||
return &Service{
|
||||
logger: logger.Named("ingestor"),
|
||||
cfg: cfg,
|
||||
rates: repo.Rates(),
|
||||
pairs: cfg.Pairs(),
|
||||
connectors: connectors,
|
||||
metrics: getServiceMetrics(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) Run(ctx context.Context) error {
|
||||
interval := s.cfg.PollInterval()
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
s.logger.Info("FX ingestion service started", zap.Duration("poll_interval", interval), zap.Int("pairs", len(s.pairs)))
|
||||
|
||||
if err := s.executePoll(ctx); err != nil {
|
||||
s.logger.Warn("Initial poll completed with errors", zap.Error(err))
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.logger.Info("Context cancelled, stopping ingestor")
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if err := s.executePoll(ctx); err != nil {
|
||||
s.logger.Warn("Polling cycle completed with errors", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) executePoll(ctx context.Context) error {
|
||||
start := time.Now()
|
||||
err := s.pollOnce(ctx)
|
||||
if s.metrics != nil {
|
||||
s.metrics.observePoll(time.Since(start), err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) pollOnce(ctx context.Context) error {
|
||||
var firstErr error
|
||||
for _, pair := range s.pairs {
|
||||
start := time.Now()
|
||||
err := s.upsertPair(ctx, pair)
|
||||
elapsed := time.Since(start)
|
||||
if s.metrics != nil {
|
||||
s.metrics.observePair(pair, elapsed, err)
|
||||
}
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
s.logger.Warn("Failed to ingest pair",
|
||||
zap.String("symbol", pair.Symbol),
|
||||
zap.String("source", pair.Source.String()),
|
||||
zap.Duration("elapsed", elapsed),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (s *Service) upsertPair(ctx context.Context, pair config.Pair) error {
|
||||
connector, ok := s.connectors[pair.Source]
|
||||
if !ok {
|
||||
return fmerrors.Wrap("connector not configured for source "+pair.Source.String(), nil)
|
||||
}
|
||||
|
||||
ticker, err := connector.FetchTicker(ctx, pair.Symbol)
|
||||
if err != nil {
|
||||
return fmerrors.Wrap("fetch ticker", err)
|
||||
}
|
||||
|
||||
bid, err := parseDecimal(ticker.BidPrice)
|
||||
if err != nil {
|
||||
return fmerrors.Wrap("parse bid price", err)
|
||||
}
|
||||
ask, err := parseDecimal(ticker.AskPrice)
|
||||
if err != nil {
|
||||
return fmerrors.Wrap("parse ask price", err)
|
||||
}
|
||||
|
||||
if pair.Invert {
|
||||
bid, ask = invertPrices(bid, ask)
|
||||
}
|
||||
|
||||
if ask.Cmp(bid) < 0 {
|
||||
// Ensure bid <= ask to keep downstream logic predictable.
|
||||
bid, ask = ask, bid
|
||||
}
|
||||
|
||||
mid := new(big.Rat).Add(bid, ask)
|
||||
mid.Quo(mid, big.NewRat(2, 1))
|
||||
|
||||
spread := big.NewRat(0, 1)
|
||||
if mid.Sign() != 0 {
|
||||
spread.Sub(ask, bid)
|
||||
if spread.Sign() < 0 {
|
||||
spread.Neg(spread)
|
||||
}
|
||||
spread.Quo(spread, mid)
|
||||
spread.Mul(spread, big.NewRat(10000, 1)) // basis points
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
asOf := now
|
||||
snapshot := &model.RateSnapshot{
|
||||
RateRef: market.BuildRateReference(pair.Provider, pair.Symbol, now),
|
||||
Pair: model.CurrencyPair{Base: pair.Base, Quote: pair.Quote},
|
||||
Provider: pair.Provider,
|
||||
Mid: formatDecimal(mid),
|
||||
Bid: formatDecimal(bid),
|
||||
Ask: formatDecimal(ask),
|
||||
SpreadBps: formatDecimal(spread),
|
||||
AsOfUnixMs: now.UnixMilli(),
|
||||
AsOf: &asOf,
|
||||
Source: ticker.Provider,
|
||||
ProviderRef: ticker.Symbol,
|
||||
}
|
||||
|
||||
if err := s.rates.UpsertSnapshot(ctx, snapshot); err != nil {
|
||||
return fmerrors.Wrap("upsert snapshot", err)
|
||||
}
|
||||
|
||||
s.logger.Debug("Snapshot ingested",
|
||||
zap.String("pair", pair.Base+"/"+pair.Quote),
|
||||
zap.String("provider", pair.Provider),
|
||||
zap.String("bid", snapshot.Bid),
|
||||
zap.String("ask", snapshot.Ask),
|
||||
zap.String("mid", snapshot.Mid),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseDecimal(value string) (*big.Rat, error) {
|
||||
r := new(big.Rat)
|
||||
if _, ok := r.SetString(value); !ok {
|
||||
return nil, fmerrors.NewDecimal(value)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func invertPrices(bid, ask *big.Rat) (*big.Rat, *big.Rat) {
|
||||
if bid.Sign() == 0 || ask.Sign() == 0 {
|
||||
return bid, ask
|
||||
}
|
||||
one := big.NewRat(1, 1)
|
||||
invBid := new(big.Rat).Quo(one, ask) // invert ask to get bid
|
||||
invAsk := new(big.Rat).Quo(one, bid) // invert bid to get ask
|
||||
return invBid, invAsk
|
||||
}
|
||||
|
||||
func formatDecimal(r *big.Rat) string {
|
||||
if r == nil {
|
||||
return "0"
|
||||
}
|
||||
// Format with 8 decimal places, trimming trailing zeros.
|
||||
return r.FloatString(8)
|
||||
}
|
||||
237
api/fx/ingestor/internal/ingestor/service_test.go
Normal file
237
api/fx/ingestor/internal/ingestor/service_test.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package ingestor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/tech/sendico/fx/ingestor/internal/config"
|
||||
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
|
||||
mmarket "github.com/tech/sendico/fx/ingestor/internal/model"
|
||||
"github.com/tech/sendico/fx/storage"
|
||||
"github.com/tech/sendico/fx/storage/model"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func TestParseDecimal(t *testing.T) {
|
||||
got, err := parseDecimal("123.456")
|
||||
if err != nil {
|
||||
t.Fatalf("parseDecimal returned error: %v", err)
|
||||
}
|
||||
if got.String() != "15432/125" { // 123.456 expressed as a rational
|
||||
t.Fatalf("unexpected rational value: %s", got.String())
|
||||
}
|
||||
|
||||
if _, err := parseDecimal("not-a-number"); err == nil {
|
||||
t.Fatalf("parseDecimal should fail on invalid decimal string")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvertPrices(t *testing.T) {
|
||||
bid, err := parseDecimal("2")
|
||||
if err != nil {
|
||||
t.Fatalf("parseDecimal: %v", err)
|
||||
}
|
||||
ask, err := parseDecimal("4")
|
||||
if err != nil {
|
||||
t.Fatalf("parseDecimal: %v", err)
|
||||
}
|
||||
|
||||
invBid, invAsk := invertPrices(bid, ask)
|
||||
if diff := cmp.Diff("0.5", invAsk.FloatString(1)); diff != "" {
|
||||
t.Fatalf("unexpected inverted ask (-want +got):\n%s", diff)
|
||||
}
|
||||
if diff := cmp.Diff("0.25", invBid.FloatString(2)); diff != "" {
|
||||
t.Fatalf("unexpected inverted bid (-want +got):\n%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceUpsertPairStoresSnapshot(t *testing.T) {
|
||||
store := &ratesStoreStub{}
|
||||
svc := testService(store, map[mmarket.Driver]mmarket.Connector{
|
||||
mmarket.DriverBinance: &connectorStub{
|
||||
id: mmarket.DriverBinance,
|
||||
ticker: &mmarket.Ticker{
|
||||
Symbol: "EURUSDT",
|
||||
BidPrice: "1.0000",
|
||||
AskPrice: "1.2000",
|
||||
Provider: "binance",
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
pair := config.Pair{
|
||||
PairConfig: config.PairConfig{
|
||||
Base: "USDT",
|
||||
Quote: "EUR",
|
||||
Symbol: "EURUSDT",
|
||||
Provider: "binance",
|
||||
},
|
||||
Source: mmarket.DriverBinance,
|
||||
}
|
||||
|
||||
if err := svc.upsertPair(context.Background(), pair); err != nil {
|
||||
t.Fatalf("upsertPair returned error: %v", err)
|
||||
}
|
||||
if len(store.snapshots) != 1 {
|
||||
t.Fatalf("expected 1 snapshot stored, got %d", len(store.snapshots))
|
||||
}
|
||||
snap := store.snapshots[0]
|
||||
if snap.Pair.Base != "USDT" || snap.Pair.Quote != "EUR" {
|
||||
t.Fatalf("unexpected currency pair stored: %+v", snap.Pair)
|
||||
}
|
||||
if snap.Provider != "binance" {
|
||||
t.Fatalf("unexpected provider: %s", snap.Provider)
|
||||
}
|
||||
if snap.Bid != "1.00000000" || snap.Ask != "1.20000000" {
|
||||
t.Fatalf("unexpected bid/ask: %s / %s", snap.Bid, snap.Ask)
|
||||
}
|
||||
if snap.Mid != "1.10000000" {
|
||||
t.Fatalf("unexpected mid price: %s", snap.Mid)
|
||||
}
|
||||
if snap.SpreadBps != "1818.18181818" {
|
||||
t.Fatalf("unexpected spread bps: %s", snap.SpreadBps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceUpsertPairInvertsPrices(t *testing.T) {
|
||||
store := &ratesStoreStub{}
|
||||
svc := testService(store, map[mmarket.Driver]mmarket.Connector{
|
||||
mmarket.DriverCoinGecko: &connectorStub{
|
||||
id: mmarket.DriverCoinGecko,
|
||||
ticker: &mmarket.Ticker{
|
||||
Symbol: "RUBUSDT",
|
||||
BidPrice: "2",
|
||||
AskPrice: "4",
|
||||
Provider: "coingecko",
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
pair := config.Pair{
|
||||
PairConfig: config.PairConfig{
|
||||
Base: "RUB",
|
||||
Quote: "USDT",
|
||||
Symbol: "RUBUSDT",
|
||||
Provider: "coingecko",
|
||||
Invert: true,
|
||||
},
|
||||
Source: mmarket.DriverCoinGecko,
|
||||
}
|
||||
|
||||
if err := svc.upsertPair(context.Background(), pair); err != nil {
|
||||
t.Fatalf("upsertPair returned error: %v", err)
|
||||
}
|
||||
|
||||
snap := store.snapshots[0]
|
||||
if snap.Bid != "0.25000000" || snap.Ask != "0.50000000" {
|
||||
t.Fatalf("unexpected inverted bid/ask: %s / %s", snap.Bid, snap.Ask)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServicePollOnceReturnsFirstError(t *testing.T) {
|
||||
errFetch := fmerrors.New("fetch failed")
|
||||
connectorSuccess := &connectorStub{
|
||||
id: mmarket.DriverBinance,
|
||||
ticker: &mmarket.Ticker{
|
||||
Symbol: "EURUSDT",
|
||||
BidPrice: "1",
|
||||
AskPrice: "1",
|
||||
Provider: "binance",
|
||||
},
|
||||
}
|
||||
connectorFail := &connectorStub{
|
||||
id: mmarket.DriverCoinGecko,
|
||||
err: errFetch,
|
||||
}
|
||||
|
||||
store := &ratesStoreStub{}
|
||||
svc := testService(store, map[mmarket.Driver]mmarket.Connector{
|
||||
mmarket.DriverBinance: connectorSuccess,
|
||||
mmarket.DriverCoinGecko: connectorFail,
|
||||
})
|
||||
svc.pairs = []config.Pair{
|
||||
{PairConfig: config.PairConfig{Base: "USDT", Quote: "EUR", Symbol: "EURUSDT"}, Source: mmarket.DriverBinance},
|
||||
{PairConfig: config.PairConfig{Base: "USDT", Quote: "RUB", Symbol: "RUBUSDT"}, Source: mmarket.DriverCoinGecko},
|
||||
}
|
||||
|
||||
err := svc.pollOnce(context.Background())
|
||||
if err == nil {
|
||||
t.Fatalf("pollOnce expected to return error")
|
||||
}
|
||||
if !errors.Is(err, errFetch) {
|
||||
t.Fatalf("pollOnce returned unexpected error: %v", err)
|
||||
}
|
||||
if connectorSuccess.calls != 1 {
|
||||
t.Fatalf("expected success connector called once, got %d", connectorSuccess.calls)
|
||||
}
|
||||
if connectorFail.calls != 1 {
|
||||
t.Fatalf("expected failing connector called once, got %d", connectorFail.calls)
|
||||
}
|
||||
if len(store.snapshots) != 1 {
|
||||
t.Fatalf("expected snapshot stored only for successful pair, got %d", len(store.snapshots))
|
||||
}
|
||||
}
|
||||
|
||||
// -- test helpers -----------------------------------------------------------------
|
||||
|
||||
type ratesStoreStub struct {
|
||||
snapshots []*model.RateSnapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *ratesStoreStub) UpsertSnapshot(_ context.Context, snapshot *model.RateSnapshot) error {
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
}
|
||||
cp := *snapshot
|
||||
r.snapshots = append(r.snapshots, &cp)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ratesStoreStub) LatestSnapshot(context.Context, model.CurrencyPair, string) (*model.RateSnapshot, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type repositoryStub struct {
|
||||
rates storage.RatesStore
|
||||
}
|
||||
|
||||
func (r *repositoryStub) Ping(context.Context) error { return nil }
|
||||
func (r *repositoryStub) Rates() storage.RatesStore { return r.rates }
|
||||
func (r *repositoryStub) Quotes() storage.QuotesStore { return nil }
|
||||
func (r *repositoryStub) Pairs() storage.PairStore { return nil }
|
||||
func (r *repositoryStub) Currencies() storage.CurrencyStore { return nil }
|
||||
|
||||
type connectorStub struct {
|
||||
id mmarket.Driver
|
||||
ticker *mmarket.Ticker
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (c *connectorStub) ID() mmarket.Driver {
|
||||
return c.id
|
||||
}
|
||||
|
||||
func (c *connectorStub) FetchTicker(_ context.Context, symbol string) (*mmarket.Ticker, error) {
|
||||
c.calls++
|
||||
if c.ticker != nil {
|
||||
cp := *c.ticker
|
||||
cp.Symbol = symbol
|
||||
return &cp, c.err
|
||||
}
|
||||
return nil, c.err
|
||||
}
|
||||
|
||||
func testService(store storage.RatesStore, connectors map[mmarket.Driver]mmarket.Connector) *Service {
|
||||
return &Service{
|
||||
logger: zap.NewNop(),
|
||||
cfg: &config.Config{},
|
||||
rates: store,
|
||||
connectors: connectors,
|
||||
pairs: nil,
|
||||
metrics: nil,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user