service backend
All checks were successful
ci/woodpecker/push/db Pipeline was successful
ci/woodpecker/push/nats Pipeline was successful

This commit is contained in:
Stephan D
2025-11-07 18:35:26 +01:00
parent 20e8f9acc4
commit 62a6631b9a
537 changed files with 48453 additions and 0 deletions

BIN
api/fx/ingestor/.DS_Store vendored Normal file

Binary file not shown.

32
api/fx/ingestor/.air.toml Normal file
View File

@@ -0,0 +1,32 @@
# Config file for Air in TOML format
root = "./../.."
tmp_dir = "tmp"
[build]
cmd = "go build -o app -ldflags \"-X 'github.com/tech/sendico/fx/ingestor/internal/appversion.BuildUser=$(whoami)' -X 'github.com/tech/sendico/fx/ingestor/internal/appversion.Version=$APP_V' -X 'github.com/tech/sendico/fx/ingestor/internal/appversion.Branch=$BUILD_BRANCH' -X 'github.com/tech/sendico/fx/ingestor/internal/appversion.Revision=$GIT_REV' -X 'github.com/tech/sendico/fx/ingestor/internal/appversion.BuildDate=$(date)'\""
bin = "./app"
full_bin = "./app --debug --config.file=config.yml"
include_ext = ["go", "yaml", "yml"]
exclude_dir = ["fx/ingestor/tmp", "pkg/.git", "fx/ingestor/env"]
exclude_regex = ["_test\\.go"]
exclude_unchanged = true
follow_symlink = true
log = "air.log"
delay = 0
stop_on_error = true
send_interrupt = true
kill_delay = 500
args_bin = []
[log]
time = false
[color]
main = "magenta"
watcher = "cyan"
build = "yellow"
runner = "green"
[misc]
clean_on_exit = true

3
api/fx/ingestor/.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
internal/generated
.gocache
app

View File

@@ -0,0 +1,43 @@
poll_interval_seconds: 30
market:
sources:
- driver: BINANCE
settings:
base_url: "https://api.binance.com"
- driver: COINGECKO
settings:
base_url: "https://api.coingecko.com/api/v3"
pairs:
BINANCE:
- base: "USDT"
quote: "EUR"
symbol: "EURUSDT"
invert: true
- base: "UAH"
quote: "USDT"
symbol: "USDTUAH"
invert: true
- base: "USDC"
quote: "EUR"
symbol: "EURUSDC"
invert: true
COINGECKO:
- base: "USDT"
quote: "RUB"
symbol: "tether:rub"
metrics:
enabled: true
address: ":9102"
database:
driver: mongodb
settings:
host_env: FX_MONGO_HOST
port_env: FX_MONGO_PORT
database_env: FX_MONGO_DATABASE
user_env: FX_MONGO_USER
password_env: FX_MONGO_PASSWORD
auth_source_env: FX_MONGO_AUTH_SOURCE
replica_set_env: FX_MONGO_REPLICA_SET

1
api/fx/ingestor/env/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
.env.api

55
api/fx/ingestor/go.mod Normal file
View File

@@ -0,0 +1,55 @@
module github.com/tech/sendico/fx/ingestor
go 1.25.3
replace github.com/tech/sendico/pkg => ../../pkg
replace github.com/tech/sendico/fx/storage => ../storage
require (
github.com/go-chi/chi/v5 v5.2.3
github.com/google/go-cmp v0.7.0
github.com/prometheus/client_golang v1.23.2
github.com/tech/sendico/fx/storage v0.0.0
github.com/tech/sendico/pkg v0.1.0
go.uber.org/zap v1.27.0
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect
github.com/casbin/casbin/v2 v2.132.0 // indirect
github.com/casbin/govaluate v1.10.0 // indirect
github.com/casbin/mongodb-adapter/v3 v3.7.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.18.1 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/nats.go v1.47.0 // indirect
github.com/nats-io/nkeys v0.4.11 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.2 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.mongodb.org/mongo-driver v1.17.6 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/crypto v0.43.0 // indirect
golang.org/x/net v0.46.0 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/sys v0.37.0 // indirect
golang.org/x/text v0.30.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect
google.golang.org/grpc v1.76.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
)

225
api/fx/ingestor/go.sum Normal file
View File

@@ -0,0 +1,225 @@
dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE=
github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/casbin/casbin/v2 v2.132.0 h1:73hGmOszGSL3hTVquwkAi98XLl3gPJ+BxB6D7G9Fxtk=
github.com/casbin/casbin/v2 v2.132.0/go.mod h1:FmcfntdXLTcYXv/hxgNntcRPqAbwOG9xsism0yXT+18=
github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
github.com/casbin/govaluate v1.10.0 h1:ffGw51/hYH3w3rZcxO/KcaUIDOLP84w7nsidMVgaDG0=
github.com/casbin/govaluate v1.10.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
github.com/casbin/mongodb-adapter/v3 v3.7.0 h1:w9c3bea1BGK4eZTAmk17JkY52yv/xSZDSHKji8q+z6E=
github.com/casbin/mongodb-adapter/v3 v3.7.0/go.mod h1:F1mu4ojoJVE/8VhIMxMedhjfwRDdIXgANYs6Sd0MgVA=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI=
github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54 h1:mFWunSatvkQQDhpdyuFAYwyAan3hzCuma+Pz8sqvOfg=
github.com/lufia/plan9stats v0.0.0-20250827001030-24949be3fa54/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo=
github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM=
github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8=
github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw=
github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8=
github.com/testcontainers/testcontainers-go/modules/mongodb v0.33.0 h1:iXVA84s5hKMS5gn01GWOYHE3ymy/2b+0YkpFeTxB2XY=
github.com/testcontainers/testcontainers-go/modules/mongodb v0.33.0/go.mod h1:R6tMjTojRiaoo89fh/hf7tOmfzohdqSU17R9DwSVSog=
github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4=
github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4=
github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso=
github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 h1:tRPGkdGHuewF4UisLzzHHr1spKw92qLM98nIzxbC0wY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,27 @@
package appversion
import (
"github.com/tech/sendico/pkg/version"
vf "github.com/tech/sendico/pkg/version/factory"
)
// Build information. Populated at build-time.
var (
Version string
Revision string
Branch string
BuildUser string
BuildDate string
)
func Create() version.Printer {
vi := version.Info{
Program: "MeetX Connectica FX Ingestor Service",
Revision: Revision,
Branch: Branch,
BuildUser: BuildUser,
BuildDate: BuildDate,
Version: Version,
}
return vf.Create(&vi)
}

View File

@@ -0,0 +1,147 @@
package config
import (
"os"
"strings"
"time"
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
mmodel "github.com/tech/sendico/fx/ingestor/internal/model"
"github.com/tech/sendico/pkg/db"
"gopkg.in/yaml.v3"
)
const defaultPollInterval = 30 * time.Second
type Config struct {
PollIntervalSeconds int `yaml:"poll_interval_seconds"`
Market MarketConfig `yaml:"market"`
Database *db.Config `yaml:"database"`
Metrics *MetricsConfig `yaml:"metrics"`
pairs []Pair
pairsBySource map[mmodel.Driver][]PairConfig
}
func Load(path string) (*Config, error) {
if path == "" {
return nil, fmerrors.New("config: path is empty")
}
data, err := os.ReadFile(path)
if err != nil {
return nil, fmerrors.Wrap("config: failed to read file", err)
}
cfg := &Config{}
if err := yaml.Unmarshal(data, cfg); err != nil {
return nil, fmerrors.Wrap("config: failed to parse yaml", err)
}
if len(cfg.Market.Sources) == 0 {
return nil, fmerrors.New("config: no market sources configured")
}
sourceSet := make(map[mmodel.Driver]struct{}, len(cfg.Market.Sources))
for idx := range cfg.Market.Sources {
src := &cfg.Market.Sources[idx]
if src.Driver.IsEmpty() {
return nil, fmerrors.New("config: market source driver is empty")
}
sourceSet[src.Driver] = struct{}{}
}
if len(cfg.Market.Pairs) == 0 {
return nil, fmerrors.New("config: no pairs configured")
}
normalizedPairs := make(map[string][]PairConfig, len(cfg.Market.Pairs))
pairsBySource := make(map[mmodel.Driver][]PairConfig, len(cfg.Market.Pairs))
var flattened []Pair
for rawSource, pairList := range cfg.Market.Pairs {
driver := mmodel.Driver(rawSource)
if driver.IsEmpty() {
return nil, fmerrors.New("config: pair source is empty")
}
if _, ok := sourceSet[driver]; !ok {
return nil, fmerrors.New("config: pair references unknown source: " + driver.String())
}
processed := make([]PairConfig, len(pairList))
for idx := range pairList {
pair := pairList[idx]
pair.Base = strings.ToUpper(strings.TrimSpace(pair.Base))
pair.Quote = strings.ToUpper(strings.TrimSpace(pair.Quote))
pair.Symbol = strings.TrimSpace(pair.Symbol)
if pair.Base == "" || pair.Quote == "" || pair.Symbol == "" {
return nil, fmerrors.New("config: pair entries must define base, quote, and symbol")
}
if strings.TrimSpace(pair.Provider) == "" {
pair.Provider = strings.ToLower(driver.String())
}
processed[idx] = pair
flattened = append(flattened, Pair{
PairConfig: pair,
Source: driver,
})
}
pairsBySource[driver] = processed
normalizedPairs[driver.String()] = processed
}
cfg.Market.Pairs = normalizedPairs
cfg.pairsBySource = pairsBySource
cfg.pairs = flattened
if cfg.Database == nil {
return nil, fmerrors.New("config: database configuration is required")
}
if cfg.Metrics != nil && cfg.Metrics.Enabled {
cfg.Metrics.Address = strings.TrimSpace(cfg.Metrics.Address)
if cfg.Metrics.Address == "" {
cfg.Metrics.Address = ":9102"
}
}
return cfg, nil
}
func (c *Config) PollInterval() time.Duration {
if c == nil {
return defaultPollInterval
}
if c.PollIntervalSeconds <= 0 {
return defaultPollInterval
}
return time.Duration(c.PollIntervalSeconds) * time.Second
}
func (c *Config) Pairs() []Pair {
if c == nil {
return nil
}
out := make([]Pair, len(c.pairs))
copy(out, c.pairs)
return out
}
func (c *Config) PairsBySource() map[mmodel.Driver][]PairConfig {
if c == nil {
return nil
}
out := make(map[mmodel.Driver][]PairConfig, len(c.pairsBySource))
for driver, pairs := range c.pairsBySource {
cp := make([]PairConfig, len(pairs))
copy(cp, pairs)
out[driver] = cp
}
return out
}
func (c *Config) MetricsConfig() *MetricsConfig {
if c == nil || c.Metrics == nil {
return nil
}
cp := *c.Metrics
return &cp
}

View File

@@ -0,0 +1,24 @@
package config
import (
mmodel "github.com/tech/sendico/fx/ingestor/internal/model"
pmodel "github.com/tech/sendico/pkg/model"
)
type PairConfig struct {
Base string `yaml:"base"`
Quote string `yaml:"quote"`
Symbol string `yaml:"symbol"`
Provider string `yaml:"provider"`
Invert bool `yaml:"invert"`
}
type Pair struct {
PairConfig `yaml:",inline"`
Source mmodel.Driver `yaml:"-"`
}
type MarketConfig struct {
Sources []pmodel.DriverConfig[mmodel.Driver] `yaml:"sources"`
Pairs map[string][]PairConfig `yaml:"pairs"`
}

View File

@@ -0,0 +1,6 @@
package config
type MetricsConfig struct {
Enabled bool `yaml:"enabled"`
Address string `yaml:"address"`
}

View File

@@ -0,0 +1,35 @@
package fmerrors
type Error struct {
message string
cause error
}
func (e *Error) Error() string {
if e == nil {
return ""
}
if e.cause == nil {
return e.message
}
return e.message + ": " + e.cause.Error()
}
func (e *Error) Unwrap() error {
if e == nil {
return nil
}
return e.cause
}
func New(message string) error {
return &Error{message: message}
}
func Wrap(message string, cause error) error {
return &Error{message: message, cause: cause}
}
func NewDecimal(value string) error {
return &Error{message: "invalid decimal \"" + value + "\""}
}

View File

@@ -0,0 +1,84 @@
package ingestor
import (
"sync"
"time"
"github.com/tech/sendico/fx/ingestor/internal/config"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
type serviceMetrics struct {
pollDuration *prometheus.HistogramVec
pollTotal *prometheus.CounterVec
pairDuration *prometheus.HistogramVec
pairTotal *prometheus.CounterVec
pairLastUpdate *prometheus.GaugeVec
}
var (
metricsOnce sync.Once
globalMetricsRef *serviceMetrics
)
func getServiceMetrics() *serviceMetrics {
metricsOnce.Do(func() {
reg := prometheus.DefaultRegisterer
globalMetricsRef = &serviceMetrics{
pollDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
Name: "fx_ingestor_poll_duration_seconds",
Help: "Duration of a polling cycle.",
Buckets: prometheus.DefBuckets,
}, []string{"result"}),
pollTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
Name: "fx_ingestor_poll_total",
Help: "Total polling cycles executed.",
}, []string{"result"}),
pairDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
Name: "fx_ingestor_pair_duration_seconds",
Help: "Duration of individual pair ingestion.",
Buckets: prometheus.DefBuckets,
}, []string{"source", "provider", "symbol", "result"}),
pairTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
Name: "fx_ingestor_pair_total",
Help: "Total ingestion attempts per pair.",
}, []string{"source", "provider", "symbol", "result"}),
pairLastUpdate: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
Name: "fx_ingestor_pair_last_success_unix",
Help: "Unix timestamp of the last successful ingestion per pair.",
}, []string{"source", "provider", "symbol"}),
}
})
return globalMetricsRef
}
func (m *serviceMetrics) observePoll(duration time.Duration, err error) {
if m == nil {
return
}
result := labelForError(err)
m.pollDuration.WithLabelValues(result).Observe(duration.Seconds())
m.pollTotal.WithLabelValues(result).Inc()
}
func (m *serviceMetrics) observePair(pair config.Pair, duration time.Duration, err error) {
if m == nil {
return
}
result := labelForError(err)
labels := []string{pair.Source.String(), pair.Provider, pair.Symbol, result}
m.pairDuration.WithLabelValues(labels...).Observe(duration.Seconds())
m.pairTotal.WithLabelValues(labels...).Inc()
if err == nil {
m.pairLastUpdate.WithLabelValues(pair.Source.String(), pair.Provider, pair.Symbol).
Set(float64(time.Now().Unix()))
}
}
func labelForError(err error) string {
if err != nil {
return "error"
}
return "success"
}

View File

@@ -0,0 +1,207 @@
package ingestor
import (
"context"
"math/big"
"time"
"github.com/tech/sendico/fx/ingestor/internal/config"
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
"github.com/tech/sendico/fx/ingestor/internal/market"
mmodel "github.com/tech/sendico/fx/ingestor/internal/model"
"github.com/tech/sendico/fx/storage"
"github.com/tech/sendico/fx/storage/model"
"github.com/tech/sendico/pkg/mlogger"
"go.uber.org/zap"
)
type Service struct {
logger mlogger.Logger
cfg *config.Config
rates storage.RatesStore
pairs []config.Pair
connectors map[mmodel.Driver]mmodel.Connector
metrics *serviceMetrics
}
func New(logger mlogger.Logger, cfg *config.Config, repo storage.Repository) (*Service, error) {
if logger == nil {
return nil, fmerrors.New("ingestor: nil logger")
}
if cfg == nil {
return nil, fmerrors.New("ingestor: nil config")
}
if repo == nil {
return nil, fmerrors.New("ingestor: nil repository")
}
connectors, err := market.BuildConnectors(logger, cfg.Market.Sources)
if err != nil {
return nil, fmerrors.Wrap("build connectors", err)
}
return &Service{
logger: logger.Named("ingestor"),
cfg: cfg,
rates: repo.Rates(),
pairs: cfg.Pairs(),
connectors: connectors,
metrics: getServiceMetrics(),
}, nil
}
func (s *Service) Run(ctx context.Context) error {
interval := s.cfg.PollInterval()
ticker := time.NewTicker(interval)
defer ticker.Stop()
s.logger.Info("FX ingestion service started", zap.Duration("poll_interval", interval), zap.Int("pairs", len(s.pairs)))
if err := s.executePoll(ctx); err != nil {
s.logger.Warn("Initial poll completed with errors", zap.Error(err))
}
for {
select {
case <-ctx.Done():
s.logger.Info("Context cancelled, stopping ingestor")
return ctx.Err()
case <-ticker.C:
if err := s.executePoll(ctx); err != nil {
s.logger.Warn("Polling cycle completed with errors", zap.Error(err))
}
}
}
}
func (s *Service) executePoll(ctx context.Context) error {
start := time.Now()
err := s.pollOnce(ctx)
if s.metrics != nil {
s.metrics.observePoll(time.Since(start), err)
}
return err
}
func (s *Service) pollOnce(ctx context.Context) error {
var firstErr error
for _, pair := range s.pairs {
start := time.Now()
err := s.upsertPair(ctx, pair)
elapsed := time.Since(start)
if s.metrics != nil {
s.metrics.observePair(pair, elapsed, err)
}
if err != nil {
if firstErr == nil {
firstErr = err
}
s.logger.Warn("Failed to ingest pair",
zap.String("symbol", pair.Symbol),
zap.String("source", pair.Source.String()),
zap.Duration("elapsed", elapsed),
zap.Error(err),
)
}
}
return firstErr
}
func (s *Service) upsertPair(ctx context.Context, pair config.Pair) error {
connector, ok := s.connectors[pair.Source]
if !ok {
return fmerrors.Wrap("connector not configured for source "+pair.Source.String(), nil)
}
ticker, err := connector.FetchTicker(ctx, pair.Symbol)
if err != nil {
return fmerrors.Wrap("fetch ticker", err)
}
bid, err := parseDecimal(ticker.BidPrice)
if err != nil {
return fmerrors.Wrap("parse bid price", err)
}
ask, err := parseDecimal(ticker.AskPrice)
if err != nil {
return fmerrors.Wrap("parse ask price", err)
}
if pair.Invert {
bid, ask = invertPrices(bid, ask)
}
if ask.Cmp(bid) < 0 {
// Ensure bid <= ask to keep downstream logic predictable.
bid, ask = ask, bid
}
mid := new(big.Rat).Add(bid, ask)
mid.Quo(mid, big.NewRat(2, 1))
spread := big.NewRat(0, 1)
if mid.Sign() != 0 {
spread.Sub(ask, bid)
if spread.Sign() < 0 {
spread.Neg(spread)
}
spread.Quo(spread, mid)
spread.Mul(spread, big.NewRat(10000, 1)) // basis points
}
now := time.Now().UTC()
asOf := now
snapshot := &model.RateSnapshot{
RateRef: market.BuildRateReference(pair.Provider, pair.Symbol, now),
Pair: model.CurrencyPair{Base: pair.Base, Quote: pair.Quote},
Provider: pair.Provider,
Mid: formatDecimal(mid),
Bid: formatDecimal(bid),
Ask: formatDecimal(ask),
SpreadBps: formatDecimal(spread),
AsOfUnixMs: now.UnixMilli(),
AsOf: &asOf,
Source: ticker.Provider,
ProviderRef: ticker.Symbol,
}
if err := s.rates.UpsertSnapshot(ctx, snapshot); err != nil {
return fmerrors.Wrap("upsert snapshot", err)
}
s.logger.Debug("Snapshot ingested",
zap.String("pair", pair.Base+"/"+pair.Quote),
zap.String("provider", pair.Provider),
zap.String("bid", snapshot.Bid),
zap.String("ask", snapshot.Ask),
zap.String("mid", snapshot.Mid),
)
return nil
}
func parseDecimal(value string) (*big.Rat, error) {
r := new(big.Rat)
if _, ok := r.SetString(value); !ok {
return nil, fmerrors.NewDecimal(value)
}
return r, nil
}
func invertPrices(bid, ask *big.Rat) (*big.Rat, *big.Rat) {
if bid.Sign() == 0 || ask.Sign() == 0 {
return bid, ask
}
one := big.NewRat(1, 1)
invBid := new(big.Rat).Quo(one, ask) // invert ask to get bid
invAsk := new(big.Rat).Quo(one, bid) // invert bid to get ask
return invBid, invAsk
}
func formatDecimal(r *big.Rat) string {
if r == nil {
return "0"
}
// Format with 8 decimal places, trimming trailing zeros.
return r.FloatString(8)
}

View File

@@ -0,0 +1,237 @@
package ingestor
import (
"context"
"errors"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/tech/sendico/fx/ingestor/internal/config"
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
mmarket "github.com/tech/sendico/fx/ingestor/internal/model"
"github.com/tech/sendico/fx/storage"
"github.com/tech/sendico/fx/storage/model"
"go.uber.org/zap"
)
func TestParseDecimal(t *testing.T) {
got, err := parseDecimal("123.456")
if err != nil {
t.Fatalf("parseDecimal returned error: %v", err)
}
if got.String() != "15432/125" { // 123.456 expressed as a rational
t.Fatalf("unexpected rational value: %s", got.String())
}
if _, err := parseDecimal("not-a-number"); err == nil {
t.Fatalf("parseDecimal should fail on invalid decimal string")
}
}
func TestInvertPrices(t *testing.T) {
bid, err := parseDecimal("2")
if err != nil {
t.Fatalf("parseDecimal: %v", err)
}
ask, err := parseDecimal("4")
if err != nil {
t.Fatalf("parseDecimal: %v", err)
}
invBid, invAsk := invertPrices(bid, ask)
if diff := cmp.Diff("0.5", invAsk.FloatString(1)); diff != "" {
t.Fatalf("unexpected inverted ask (-want +got):\n%s", diff)
}
if diff := cmp.Diff("0.25", invBid.FloatString(2)); diff != "" {
t.Fatalf("unexpected inverted bid (-want +got):\n%s", diff)
}
}
func TestServiceUpsertPairStoresSnapshot(t *testing.T) {
store := &ratesStoreStub{}
svc := testService(store, map[mmarket.Driver]mmarket.Connector{
mmarket.DriverBinance: &connectorStub{
id: mmarket.DriverBinance,
ticker: &mmarket.Ticker{
Symbol: "EURUSDT",
BidPrice: "1.0000",
AskPrice: "1.2000",
Provider: "binance",
},
},
})
pair := config.Pair{
PairConfig: config.PairConfig{
Base: "USDT",
Quote: "EUR",
Symbol: "EURUSDT",
Provider: "binance",
},
Source: mmarket.DriverBinance,
}
if err := svc.upsertPair(context.Background(), pair); err != nil {
t.Fatalf("upsertPair returned error: %v", err)
}
if len(store.snapshots) != 1 {
t.Fatalf("expected 1 snapshot stored, got %d", len(store.snapshots))
}
snap := store.snapshots[0]
if snap.Pair.Base != "USDT" || snap.Pair.Quote != "EUR" {
t.Fatalf("unexpected currency pair stored: %+v", snap.Pair)
}
if snap.Provider != "binance" {
t.Fatalf("unexpected provider: %s", snap.Provider)
}
if snap.Bid != "1.00000000" || snap.Ask != "1.20000000" {
t.Fatalf("unexpected bid/ask: %s / %s", snap.Bid, snap.Ask)
}
if snap.Mid != "1.10000000" {
t.Fatalf("unexpected mid price: %s", snap.Mid)
}
if snap.SpreadBps != "1818.18181818" {
t.Fatalf("unexpected spread bps: %s", snap.SpreadBps)
}
}
func TestServiceUpsertPairInvertsPrices(t *testing.T) {
store := &ratesStoreStub{}
svc := testService(store, map[mmarket.Driver]mmarket.Connector{
mmarket.DriverCoinGecko: &connectorStub{
id: mmarket.DriverCoinGecko,
ticker: &mmarket.Ticker{
Symbol: "RUBUSDT",
BidPrice: "2",
AskPrice: "4",
Provider: "coingecko",
},
},
})
pair := config.Pair{
PairConfig: config.PairConfig{
Base: "RUB",
Quote: "USDT",
Symbol: "RUBUSDT",
Provider: "coingecko",
Invert: true,
},
Source: mmarket.DriverCoinGecko,
}
if err := svc.upsertPair(context.Background(), pair); err != nil {
t.Fatalf("upsertPair returned error: %v", err)
}
snap := store.snapshots[0]
if snap.Bid != "0.25000000" || snap.Ask != "0.50000000" {
t.Fatalf("unexpected inverted bid/ask: %s / %s", snap.Bid, snap.Ask)
}
}
func TestServicePollOnceReturnsFirstError(t *testing.T) {
errFetch := fmerrors.New("fetch failed")
connectorSuccess := &connectorStub{
id: mmarket.DriverBinance,
ticker: &mmarket.Ticker{
Symbol: "EURUSDT",
BidPrice: "1",
AskPrice: "1",
Provider: "binance",
},
}
connectorFail := &connectorStub{
id: mmarket.DriverCoinGecko,
err: errFetch,
}
store := &ratesStoreStub{}
svc := testService(store, map[mmarket.Driver]mmarket.Connector{
mmarket.DriverBinance: connectorSuccess,
mmarket.DriverCoinGecko: connectorFail,
})
svc.pairs = []config.Pair{
{PairConfig: config.PairConfig{Base: "USDT", Quote: "EUR", Symbol: "EURUSDT"}, Source: mmarket.DriverBinance},
{PairConfig: config.PairConfig{Base: "USDT", Quote: "RUB", Symbol: "RUBUSDT"}, Source: mmarket.DriverCoinGecko},
}
err := svc.pollOnce(context.Background())
if err == nil {
t.Fatalf("pollOnce expected to return error")
}
if !errors.Is(err, errFetch) {
t.Fatalf("pollOnce returned unexpected error: %v", err)
}
if connectorSuccess.calls != 1 {
t.Fatalf("expected success connector called once, got %d", connectorSuccess.calls)
}
if connectorFail.calls != 1 {
t.Fatalf("expected failing connector called once, got %d", connectorFail.calls)
}
if len(store.snapshots) != 1 {
t.Fatalf("expected snapshot stored only for successful pair, got %d", len(store.snapshots))
}
}
// -- test helpers -----------------------------------------------------------------
type ratesStoreStub struct {
snapshots []*model.RateSnapshot
err error
}
func (r *ratesStoreStub) UpsertSnapshot(_ context.Context, snapshot *model.RateSnapshot) error {
if r.err != nil {
return r.err
}
cp := *snapshot
r.snapshots = append(r.snapshots, &cp)
return nil
}
func (r *ratesStoreStub) LatestSnapshot(context.Context, model.CurrencyPair, string) (*model.RateSnapshot, error) {
return nil, nil
}
type repositoryStub struct {
rates storage.RatesStore
}
func (r *repositoryStub) Ping(context.Context) error { return nil }
func (r *repositoryStub) Rates() storage.RatesStore { return r.rates }
func (r *repositoryStub) Quotes() storage.QuotesStore { return nil }
func (r *repositoryStub) Pairs() storage.PairStore { return nil }
func (r *repositoryStub) Currencies() storage.CurrencyStore { return nil }
type connectorStub struct {
id mmarket.Driver
ticker *mmarket.Ticker
err error
calls int
}
func (c *connectorStub) ID() mmarket.Driver {
return c.id
}
func (c *connectorStub) FetchTicker(_ context.Context, symbol string) (*mmarket.Ticker, error) {
c.calls++
if c.ticker != nil {
cp := *c.ticker
cp.Symbol = symbol
return &cp, c.err
}
return nil, c.err
}
func testService(store storage.RatesStore, connectors map[mmarket.Driver]mmarket.Connector) *Service {
return &Service{
logger: zap.NewNop(),
cfg: &config.Config{},
rates: store,
connectors: connectors,
pairs: nil,
metrics: nil,
}
}

View File

@@ -0,0 +1,139 @@
package binance
import (
"context"
"encoding/json"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
"github.com/tech/sendico/fx/ingestor/internal/market/common"
mmodel "github.com/tech/sendico/fx/ingestor/internal/model"
"github.com/tech/sendico/pkg/mlogger"
"github.com/tech/sendico/pkg/model"
"go.uber.org/zap"
)
type binanceConnector struct {
id mmodel.Driver
provider string
client *http.Client
base string
logger mlogger.Logger
}
const defaultBinanceBaseURL = "https://api.binance.com"
const (
defaultDialTimeoutSeconds = 5 * time.Second
defaultDialKeepAliveSeconds = 30 * time.Second
defaultTLSHandshakeTimeoutSeconds = 5 * time.Second
defaultResponseHeaderTimeoutSeconds = 10 * time.Second
defaultRequestTimeoutSeconds = 10 * time.Second
)
func NewConnector(logger mlogger.Logger, settings model.SettingsT) (mmodel.Connector, error) {
baseURL := defaultBinanceBaseURL
provider := strings.ToLower(mmodel.DriverBinance.String())
dialTimeout := defaultDialTimeoutSeconds
dialKeepAlive := defaultDialKeepAliveSeconds
tlsHandshakeTimeout := defaultTLSHandshakeTimeoutSeconds
responseHeaderTimeout := defaultResponseHeaderTimeoutSeconds
requestTimeout := defaultRequestTimeoutSeconds
if settings != nil {
if value, ok := settings["base_url"].(string); ok && strings.TrimSpace(value) != "" {
baseURL = strings.TrimSpace(value)
}
if value, ok := settings["provider"].(string); ok && strings.TrimSpace(value) != "" {
provider = strings.TrimSpace(value)
}
dialTimeout = common.DurationSetting(settings, "dial_timeout_seconds", dialTimeout)
dialKeepAlive = common.DurationSetting(settings, "dial_keep_alive_seconds", dialKeepAlive)
tlsHandshakeTimeout = common.DurationSetting(settings, "tls_handshake_timeout_seconds", tlsHandshakeTimeout)
responseHeaderTimeout = common.DurationSetting(settings, "response_header_timeout_seconds", responseHeaderTimeout)
requestTimeout = common.DurationSetting(settings, "request_timeout_seconds", requestTimeout)
}
parsed, err := url.Parse(baseURL)
if err != nil {
return nil, fmerrors.Wrap("binance: invalid base url", err)
}
transport := &http.Transport{
DialContext: (&net.Dialer{Timeout: dialTimeout, KeepAlive: dialKeepAlive}).DialContext,
TLSHandshakeTimeout: tlsHandshakeTimeout,
ResponseHeaderTimeout: responseHeaderTimeout,
}
connector := &binanceConnector{
id: mmodel.DriverBinance,
provider: provider,
client: &http.Client{
Timeout: requestTimeout,
Transport: transport,
},
base: parsed.String(),
logger: logger.Named("binance"),
}
return connector, nil
}
func (c *binanceConnector) ID() mmodel.Driver {
return c.id
}
func (c *binanceConnector) FetchTicker(ctx context.Context, symbol string) (*mmodel.Ticker, error) {
if strings.TrimSpace(symbol) == "" {
return nil, fmerrors.New("binance: symbol is empty")
}
endpoint, err := url.Parse(c.base)
if err != nil {
return nil, fmerrors.Wrap("binance: parse base url", err)
}
endpoint.Path = "/api/v3/ticker/bookTicker"
query := endpoint.Query()
query.Set("symbol", strings.ToUpper(strings.TrimSpace(symbol)))
endpoint.RawQuery = query.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
if err != nil {
return nil, fmerrors.Wrap("binance: build request", err)
}
resp, err := c.client.Do(req)
if err != nil {
c.logger.Warn("Binance request failed", zap.String("symbol", symbol), zap.Error(err))
return nil, fmerrors.Wrap("binance: request failed", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
c.logger.Warn("Binance returned non-OK status", zap.String("symbol", symbol), zap.Int("status", resp.StatusCode))
return nil, fmerrors.New("binance: unexpected status " + strconv.Itoa(resp.StatusCode))
}
var payload struct {
Symbol string `json:"symbol"`
BidPrice string `json:"bidPrice"`
AskPrice string `json:"askPrice"`
}
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
c.logger.Warn("Binance decode failed", zap.String("symbol", symbol), zap.Error(err))
return nil, fmerrors.Wrap("binance: decode response", err)
}
return &mmodel.Ticker{
Symbol: payload.Symbol,
BidPrice: payload.BidPrice,
AskPrice: payload.AskPrice,
Provider: c.provider,
Timestamp: time.Now().UnixMilli(),
}, nil
}

View File

@@ -0,0 +1,222 @@
package coingecko
import (
"context"
"encoding/json"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
"github.com/tech/sendico/fx/ingestor/internal/market/common"
mmodel "github.com/tech/sendico/fx/ingestor/internal/model"
"github.com/tech/sendico/pkg/mlogger"
"github.com/tech/sendico/pkg/model"
"go.uber.org/zap"
)
type coingeckoConnector struct {
id mmodel.Driver
provider string
client *http.Client
base string
logger mlogger.Logger
}
const defaultCoinGeckoBaseURL = "https://api.coingecko.com/api/v3"
const (
defaultDialTimeoutSeconds = 5 * time.Second
defaultDialKeepAliveSeconds = 30 * time.Second
defaultTLSHandshakeTimeoutSeconds = 5 * time.Second
defaultResponseHeaderTimeoutSeconds = 10 * time.Second
defaultRequestTimeoutSeconds = 10 * time.Second
)
func NewConnector(logger mlogger.Logger, settings model.SettingsT) (mmodel.Connector, error) {
baseURL := defaultCoinGeckoBaseURL
provider := strings.ToLower(mmodel.DriverCoinGecko.String())
dialTimeout := defaultDialTimeoutSeconds
dialKeepAlive := defaultDialKeepAliveSeconds
tlsHandshakeTimeout := defaultTLSHandshakeTimeoutSeconds
responseHeaderTimeout := defaultResponseHeaderTimeoutSeconds
requestTimeout := defaultRequestTimeoutSeconds
if settings != nil {
if value, ok := settings["base_url"].(string); ok && strings.TrimSpace(value) != "" {
baseURL = strings.TrimSpace(value)
}
if value, ok := settings["provider"].(string); ok && strings.TrimSpace(value) != "" {
provider = strings.TrimSpace(value)
}
dialTimeout = common.DurationSetting(settings, "dial_timeout_seconds", dialTimeout)
dialKeepAlive = common.DurationSetting(settings, "dial_keep_alive_seconds", dialKeepAlive)
tlsHandshakeTimeout = common.DurationSetting(settings, "tls_handshake_timeout_seconds", tlsHandshakeTimeout)
responseHeaderTimeout = common.DurationSetting(settings, "response_header_timeout_seconds", responseHeaderTimeout)
requestTimeout = common.DurationSetting(settings, "request_timeout_seconds", requestTimeout)
}
parsed, err := url.Parse(baseURL)
if err != nil {
return nil, fmerrors.Wrap("coingecko: invalid base url", err)
}
transport := &http.Transport{
DialContext: (&net.Dialer{Timeout: dialTimeout, KeepAlive: dialKeepAlive}).DialContext,
TLSHandshakeTimeout: tlsHandshakeTimeout,
ResponseHeaderTimeout: responseHeaderTimeout,
}
connector := &coingeckoConnector{
id: mmodel.DriverCoinGecko,
provider: provider,
client: &http.Client{
Timeout: requestTimeout,
Transport: transport,
},
base: strings.TrimRight(parsed.String(), "/"),
logger: logger.Named("coingecko"),
}
return connector, nil
}
func (c *coingeckoConnector) ID() mmodel.Driver {
return c.id
}
func (c *coingeckoConnector) FetchTicker(ctx context.Context, symbol string) (*mmodel.Ticker, error) {
coinID, vsCurrency, err := parseSymbol(symbol)
if err != nil {
return nil, err
}
endpoint, err := url.Parse(c.base)
if err != nil {
return nil, fmerrors.Wrap("coingecko: parse base url", err)
}
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/simple/price"
query := endpoint.Query()
query.Set("ids", coinID)
query.Set("vs_currencies", vsCurrency)
query.Set("include_last_updated_at", "true")
endpoint.RawQuery = query.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
if err != nil {
return nil, fmerrors.Wrap("coingecko: build request", err)
}
resp, err := c.client.Do(req)
if err != nil {
c.logger.Warn("CoinGecko request failed", zap.String("symbol", symbol), zap.Error(err))
return nil, fmerrors.Wrap("coingecko: request failed", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
c.logger.Warn("CoinGecko returned non-OK status", zap.String("symbol", symbol), zap.Int("status", resp.StatusCode))
return nil, fmerrors.New("coingecko: unexpected status " + strconv.Itoa(resp.StatusCode))
}
decoder := json.NewDecoder(resp.Body)
decoder.UseNumber()
var payload map[string]map[string]interface{}
if err := decoder.Decode(&payload); err != nil {
c.logger.Warn("CoinGecko decode failed", zap.String("symbol", symbol), zap.Error(err))
return nil, fmerrors.Wrap("coingecko: decode response", err)
}
coinData, ok := payload[coinID]
if !ok {
return nil, fmerrors.New("coingecko: coin id not found in response")
}
priceValue, ok := coinData[vsCurrency]
if !ok {
return nil, fmerrors.New("coingecko: vs currency not found in response")
}
price, ok := toFloat(priceValue)
if !ok || price <= 0 {
return nil, fmerrors.New("coingecko: invalid price value in response")
}
priceStr := strconv.FormatFloat(price, 'f', -1, 64)
timestamp := time.Now().UnixMilli()
if tsValue, ok := coinData["last_updated_at"]; ok {
if tsFloat, ok := toFloat(tsValue); ok && tsFloat > 0 {
tsMillis := int64(tsFloat * 1000)
if tsMillis > 0 {
timestamp = tsMillis
}
}
}
refSymbol := coinID + "_" + vsCurrency
return &mmodel.Ticker{
Symbol: refSymbol,
BidPrice: priceStr,
AskPrice: priceStr,
Provider: c.provider,
Timestamp: timestamp,
}, nil
}
func parseSymbol(symbol string) (string, string, error) {
trimmed := strings.TrimSpace(symbol)
if trimmed == "" {
return "", "", fmerrors.New("coingecko: symbol is empty")
}
parts := strings.FieldsFunc(strings.ToLower(trimmed), func(r rune) bool {
switch r {
case ':', '/', '-', '_':
return true
}
return false
})
if len(parts) != 2 {
return "", "", fmerrors.New("coingecko: symbol must be <coin_id>/<vs_currency>")
}
coinID := strings.TrimSpace(parts[0])
vsCurrency := strings.TrimSpace(parts[1])
if coinID == "" || vsCurrency == "" {
return "", "", fmerrors.New("coingecko: symbol contains empty segments")
}
return coinID, vsCurrency, nil
}
func toFloat(value interface{}) (float64, bool) {
switch v := value.(type) {
case json.Number:
f, err := v.Float64()
if err != nil {
return 0, false
}
return f, true
case float64:
return v, true
case float32:
return float64(v), true
case int:
return float64(v), true
case int64:
return float64(v), true
case uint64:
return float64(v), true
case string:
if parsed, err := strconv.ParseFloat(v, 64); err == nil {
return parsed, true
}
}
return 0, false
}

View File

@@ -0,0 +1,46 @@
package common
import (
"strconv"
"time"
"github.com/tech/sendico/pkg/model"
)
// DurationSetting reads a positive duration override from settings or returns def when the value is missing or invalid.
func DurationSetting(settings model.SettingsT, key string, def time.Duration) time.Duration {
if settings == nil {
return def
}
value, ok := settings[key]
if !ok {
return def
}
switch v := value.(type) {
case time.Duration:
if v > 0 {
return v
}
case int:
if v > 0 {
return time.Duration(v) * time.Second
}
case int64:
if v > 0 {
return time.Duration(v) * time.Second
}
case float64:
if v > 0 {
return time.Duration(v * float64(time.Second))
}
case string:
if parsed, err := time.ParseDuration(v); err == nil && parsed > 0 {
return parsed
}
if seconds, err := strconv.ParseFloat(v, 64); err == nil && seconds > 0 {
return time.Duration(seconds * float64(time.Second))
}
}
return def
}

View File

@@ -0,0 +1,55 @@
package market
import (
"strconv"
"strings"
"time"
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
"github.com/tech/sendico/fx/ingestor/internal/market/binance"
"github.com/tech/sendico/fx/ingestor/internal/market/coingecko"
mmodel "github.com/tech/sendico/fx/ingestor/internal/model"
"github.com/tech/sendico/pkg/mlogger"
"github.com/tech/sendico/pkg/model"
)
type ConnectorFactory func(logger mlogger.Logger, settings model.SettingsT) (mmodel.Connector, error)
func BuildConnectors(logger mlogger.Logger, configs []model.DriverConfig[mmodel.Driver]) (map[mmodel.Driver]mmodel.Connector, error) {
connectors := make(map[mmodel.Driver]mmodel.Connector, len(configs))
for _, cfg := range configs {
driver := mmodel.NormalizeDriver(cfg.Driver)
if driver.IsEmpty() {
return nil, fmerrors.New("market: connector driver is empty")
}
var (
conn mmodel.Connector
err error
)
switch driver {
case mmodel.DriverBinance:
conn, err = binance.NewConnector(logger, cfg.Settings)
case mmodel.DriverCoinGecko:
conn, err = coingecko.NewConnector(logger, cfg.Settings)
default:
err = fmerrors.New("market: unsupported driver " + driver.String())
}
if err != nil {
return nil, fmerrors.Wrap("market: build connector "+driver.String(), err)
}
connectors[driver] = conn
}
return connectors, nil
}
func BuildRateReference(provider, symbol string, now time.Time) string {
if strings.TrimSpace(provider) == "" {
provider = "unknown"
}
return provider + ":" + symbol + ":" + strconv.FormatInt(now.UnixMilli(), 10)
}

View File

@@ -0,0 +1,134 @@
package metrics
import (
"context"
"errors"
"net/http"
"strings"
"time"
"github.com/go-chi/chi/v5"
"github.com/tech/sendico/fx/ingestor/internal/config"
"github.com/tech/sendico/fx/ingestor/internal/fmerrors"
"github.com/tech/sendico/pkg/api/routers"
"github.com/tech/sendico/pkg/api/routers/health"
"github.com/tech/sendico/pkg/mlogger"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
)
const (
defaultAddress = ":9102"
readHeaderTimeout = 5 * time.Second
defaultShutdownWindow = 5 * time.Second
)
type Server interface {
SetStatus(health.ServiceStatus)
Close(context.Context)
}
func NewServer(logger mlogger.Logger, cfg *config.MetricsConfig) (Server, error) {
if logger == nil {
return nil, fmerrors.New("metrics: logger is nil")
}
if cfg == nil || !cfg.Enabled {
logger.Debug("Metrics disabled; using noop server")
return noopServer{}, nil
}
address := strings.TrimSpace(cfg.Address)
if address == "" {
address = defaultAddress
}
metricsLogger := logger.Named("metrics")
router := chi.NewRouter()
router.Handle("/metrics", promhttp.Handler())
var healthRouter routers.Health
if hr, err := routers.NewHealthRouter(metricsLogger, router, ""); err != nil {
metricsLogger.Warn("Failed to initialise health router", zap.Error(err))
} else {
hr.SetStatus(health.SSStarting)
healthRouter = hr
}
httpServer := &http.Server{
Addr: address,
Handler: router,
ReadHeaderTimeout: readHeaderTimeout,
}
ms := &httpServerWrapper{
logger: metricsLogger,
server: httpServer,
health: healthRouter,
timeout: defaultShutdownWindow,
}
go func() {
metricsLogger.Info("Prometheus endpoint listening", zap.String("address", address))
if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
metricsLogger.Error("Prometheus endpoint stopped unexpectedly", zap.Error(err))
if healthRouter != nil {
healthRouter.SetStatus(health.SSTerminating)
}
}
}()
return ms, nil
}
type httpServerWrapper struct {
logger mlogger.Logger
server *http.Server
health routers.Health
timeout time.Duration
}
func (s *httpServerWrapper) SetStatus(status health.ServiceStatus) {
if s == nil || s.health == nil {
return
}
s.logger.Debug("Updating metrics health status", zap.String("status", string(status)))
s.health.SetStatus(status)
}
func (s *httpServerWrapper) Close(ctx context.Context) {
if s == nil {
return
}
if s.health != nil {
s.health.SetStatus(health.SSTerminating)
s.health.Finish()
s.health = nil
}
if s.server == nil {
return
}
shutdownCtx := ctx
if shutdownCtx == nil {
shutdownCtx = context.Background()
}
if s.timeout > 0 {
var cancel context.CancelFunc
shutdownCtx, cancel = context.WithTimeout(shutdownCtx, s.timeout)
defer cancel()
}
if err := s.server.Shutdown(shutdownCtx); err != nil && !errors.Is(err, http.ErrServerClosed) {
s.logger.Warn("Failed to stop metrics server", zap.Error(err))
} else {
s.logger.Info("Metrics server stopped")
}
}
type noopServer struct{}
func (noopServer) SetStatus(health.ServiceStatus) {}
func (noopServer) Close(context.Context) {}

View File

@@ -0,0 +1,30 @@
package model
import (
"context"
"strings"
)
type Driver string
const (
DriverBinance Driver = "BINANCE"
DriverCoinGecko Driver = "COINGECKO"
)
func (d Driver) String() string {
return string(d)
}
func (d Driver) IsEmpty() bool {
return strings.TrimSpace(string(d)) == ""
}
func NormalizeDriver(d Driver) Driver {
return Driver(strings.ToUpper(strings.TrimSpace(string(d))))
}
type Connector interface {
ID() Driver
FetchTicker(ctx context.Context, symbol string) (*Ticker, error)
}

View File

@@ -0,0 +1,9 @@
package model
type Ticker struct {
Symbol string
BidPrice string
AskPrice string
Provider string
Timestamp int64
}

View File

@@ -0,0 +1,14 @@
package signalctx
import (
"context"
"os"
"os/signal"
)
func WithSignals(parent context.Context, sig ...os.Signal) (context.Context, context.CancelFunc) {
if parent == nil {
parent = context.Background()
}
return signal.NotifyContext(parent, sig...)
}

55
api/fx/ingestor/main.go Normal file
View File

@@ -0,0 +1,55 @@
package main
import (
"context"
"errors"
"flag"
"fmt"
"os"
"syscall"
"github.com/tech/sendico/fx/ingestor/internal/app"
"github.com/tech/sendico/fx/ingestor/internal/appversion"
"github.com/tech/sendico/fx/ingestor/internal/signalctx"
lf "github.com/tech/sendico/pkg/mlogger/factory"
"go.uber.org/zap"
)
var (
configFile = flag.String("config.file", app.DefaultConfigPath, "Path to the configuration file.")
debugFlag = flag.Bool("debug", false, "Enable debug logging.")
versionFlag = flag.Bool("version", false, "Show version information.")
)
func main() {
flag.Parse()
logger := lf.NewLogger(*debugFlag).Named("fx_ingestor")
defer logger.Sync()
av := appversion.Create()
if *versionFlag {
fmt.Fprintln(os.Stdout, av.Print())
return
}
logger.Info(fmt.Sprintf("Starting %s", av.Program()), zap.String("version", av.Info()))
ctx, cancel := signalctx.WithSignals(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()
application, err := app.New(logger, *configFile)
if err != nil {
logger.Fatal("Failed to initialise application", zap.Error(err))
}
if err := application.Run(ctx); err != nil {
if errors.Is(err, context.Canceled) {
logger.Info("FX ingestor stopped")
return
}
logger.Fatal("Ingestor terminated with error", zap.Error(err))
}
logger.Info("FX ingestor stopped")
}