Skip to content

Commit

Permalink
Use Badger4 as datastore
Browse files Browse the repository at this point in the history 
Badger4 with optimize settings for gateways, including writethrough Blockservice/Store.

Badger4 is designed to work with SSDs (Value log requires fast random lookup
speed). I think we should try to put the datastore on SSDs and a very large
http cache on spinning disk (if available).

TODO: figure out GC (later).
  • Loading branch information
@hsanjuan
hsanjuan committed Oct 9, 2023
1 parent dad528b commit 11af995
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 54 deletions.
8 changes: 6 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ module github.com/ipfs/rainbow
go 1.20

require (
github.com/dgraph-io/badger/v4 v4.2.0
github.com/ipfs/boxo v0.13.1
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-ds-flatfs v0.5.1
github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981
github.com/ipfs/go-ds-leveldb v0.5.0
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-metrics-interface v0.0.1
Expand All @@ -32,7 +33,6 @@ require (
require (
github.com/Jorropo/jsync v1.0.1 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
Expand All @@ -45,6 +45,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
Expand All @@ -58,9 +59,12 @@ require (
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v1.12.1 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
github.com/google/uuid v1.3.0 // indirect
Expand Down
22 changes: 12 additions & 10 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
Expand Down Expand Up @@ -118,6 +116,11 @@ github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5il
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs=
github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
Expand Down Expand Up @@ -177,10 +180,12 @@ github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5/go.mod h1:xEhNfoBDX1hz
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
Expand Down Expand Up @@ -214,6 +219,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw=
github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
Expand Down Expand Up @@ -302,8 +309,8 @@ github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8
github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk=
github.com/ipfs/go-ds-flatfs v0.5.1 h1:ZCIO/kQOS/PSh3vcF1H6a8fkRGS7pOfwfPdx4n/KJH4=
github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4=
github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981 h1:GOKV62VnjerKwO7mwOyeoArzlaVrDLyoC/YPNtxxGwg=
github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981/go.mod h1:LUU2FbhNdmhAbJmMeoahVRbe4GsduAODSJHWJJh2Vo4=
github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo=
github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
Expand All @@ -330,10 +337,8 @@ github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf
github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk=
github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8=
github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo=
github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g=
github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY=
github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI=
Expand Down Expand Up @@ -531,7 +536,6 @@ github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/
github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg=
github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
Expand Down Expand Up @@ -732,7 +736,6 @@ go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLk
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
Expand All @@ -742,13 +745,11 @@ go.uber.org/fx v1.20.0 h1:ZMC/pnRvhsthOZh9MZjMq5U8Or3mA9zBSPaLnzs3ihQ=
go.uber.org/fx v1.20.0/go.mod h1:qCUj0btiR3/JnanEr1TYEePfSw6o/4qYJscgvzQ5Ub0=
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
Expand Down Expand Up @@ -913,6 +914,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
Expand Down
26 changes: 15 additions & 11 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,11 @@ func main() {
Value: 3000,
Usage: "libp2p connection manager 'high' water mark",
},
&cli.IntFlag{
Name: "inmem-block-cache",
Value: 1 << 30,
Usage: "Size of the in-memory block cache. 0 to disable (disables compression too)",
},

Check warning on line 63 in main.go

View check run for this annotation

Codecov / codecov/patch

main.go#L59-L63 

Added lines #L59 - L63 were not covered by tests
&cli.DurationFlag{
Name: "connmgr-grace",
Value: time.Minute,
Expand All @@ -81,17 +86,16 @@ func main() {
cdns := newCachedDNS(dnsCacheRefreshInterval)
defer cdns.Close()

gnd, err := Setup(cctx.Context, &Config{
ConnMgrLow: cctx.Int("connmgr-low"),
ConnMgrHi: cctx.Int("connmgr-hi"),
ConnMgrGrace: cctx.Duration("connmgr-grace"),
Blockstore: filepath.Join(ddir, "blockstore"),
Datastore: filepath.Join(ddir, "datastore"),
Libp2pKeyFile: filepath.Join(ddir, "libp2p.key"),
RoutingV1: cctx.String("routing"),
KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC),
DHTSharedHost: cctx.Bool("dht-fallback-shared-host"),
DNSCache: cdns,
gnd, err := Setup(cctx.Context, Config{
ConnMgrLow: cctx.Int("connmgr-low"),
ConnMgrHi: cctx.Int("connmgr-hi"),
ConnMgrGrace: cctx.Duration("connmgr-grace"),
InMemBlockCache: cctx.Int64("inmem-block-cache"),
Libp2pKeyFile: filepath.Join(ddir, "libp2p.key"),
RoutingV1: cctx.String("routing"),
KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC),
DHTSharedHost: cctx.Bool("dht-fallback-shared-host"),
DNSCache: cdns,

Check warning on line 98 in main.go

View check run for this annotation

Codecov / codecov/patch

main.go#L89-L98 

Added lines #L89 - L98 were not covered by tests
})
if err != nil {
return err
Expand Down
104 changes: 73 additions & 31 deletions setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"os"
"time"

badger "github.com/dgraph-io/badger/v4"
options "github.com/dgraph-io/badger/v4/options"
bsclient "github.com/ipfs/boxo/bitswap/client"
bsnet "github.com/ipfs/boxo/bitswap/network"
"github.com/ipfs/boxo/blockservice"
Expand All @@ -19,7 +21,7 @@ import (
httpcontentrouter "github.com/ipfs/boxo/routing/http/contentrouter"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
flatfs "github.com/ipfs/go-ds-flatfs"
badger4 "github.com/ipfs/go-ds-badger4"
levelds "github.com/ipfs/go-ds-leveldb"
metri "github.com/ipfs/go-metrics-interface"
mprome "github.com/ipfs/go-metrics-prometheus"
Expand Down Expand Up @@ -66,29 +68,27 @@ type Config struct {
ListenAddrs []string
AnnounceAddrs []string

Blockstore string

Libp2pKeyFile string

Datastore string

ConnMgrLow int
ConnMgrHi int
ConnMgrGrace time.Duration

InMemBlockCache int64

RoutingV1 string
KuboRPCURLs []string
DHTSharedHost bool
DNSCache *cachedDNS
}

func Setup(ctx context.Context, cfg *Config) (*Node, error) {
func Setup(ctx context.Context, cfg Config) (*Node, error) {

Check warning on line 85 in setup.go

View check run for this annotation

Codecov / codecov/patch

setup.go#L85 

Added line #L85 was not covered by tests
peerkey, err := loadOrInitPeerKey(cfg.Libp2pKeyFile)
if err != nil {
return nil, err
}

ds, err := levelds.NewDatastore(cfg.Datastore, nil)
ds, err := setupDatastore(cfg)

Check warning on line 91 in setup.go

View check run for this annotation

Codecov / codecov/patch

setup.go#L91 

Added line #L91 was not covered by tests
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -124,10 +124,14 @@ func Setup(ctx context.Context, cfg *Config) (*Node, error) {
}))
}

blkst, err := loadBlockstore("flatfs", cfg.Blockstore)
if err != nil {
return nil, err
}
blkst := blockstore.NewBlockstore(ds,
blockstore.NoPrefix(),
// Every Has() for every written block is a transaction with a
// seek onto LSM. If not in memory it will be a pain.
// We opt to write every block Put into the blockstore.
// See also comment in blockservice.
blockstore.WriteThrough(),
)

Check warning on line 134 in setup.go

View check run for this annotation

Codecov / codecov/patch

setup.go#L127-L134 

Added lines #L127 - L134 were not covered by tests
blkst = blockstore.NewIdStore(blkst)

bsctx := metri.CtxScope(ctx, "rainbow")
Expand Down Expand Up @@ -256,7 +260,15 @@ func Setup(ctx context.Context, cfg *Config) (*Node, error) {
bswap := bsclient.New(bsctx, bn, blkst)
bn.Start(bswap)

bsrv := blockservice.New(blkst, bswap)
bsrv := blockservice.New(blkst, bswap,
// if we are doing things right, our bitswap wantlists should
// not have blocks that we already have (see
// https://github.com/ipfs/boxo/blob/e0d4b3e9b91e9904066a10278e366c9a6d9645c7/blockservice/blockservice.go#L272). Thus
// we should not be writing many blocks that we already
// have. Thus, no point in checking whether we have a block
// before writing new blocks.
blockservice.WriteThrough(),
)

Check warning on line 271 in setup.go

View check run for this annotation

Codecov / codecov/patch

setup.go#L263-L271 

Added lines #L263 - L271 were not covered by tests

dns, err := gateway.NewDNSResolver(nil)
if err != nil {
Expand All @@ -280,6 +292,55 @@ func Setup(ctx context.Context, cfg *Config) (*Node, error) {
}, nil
}

func setupDatastore(cfg Config) (datastore.Batching, error) {
badgerOpts := badger.DefaultOptions("")
badgerOpts.CompactL0OnClose = false
// ValueThreshold: defaults to 1MB! For us that means everything goes
// into the LSM tree and that means more stuff in memory. We only
// put very small things on the LSM tree by default (i.e. a single
// CID).
badgerOpts.ValueThreshold = 256

// BlockCacheSize: instead of using blockstore, we cache things
// here. This only makes sense if using compression, according to
// docs.
badgerOpts.BlockCacheSize = cfg.InMemBlockCache // default 1 GiB.

// Compression: default. Trades reading less from disk for using more
// CPU. Given gateways are usually IO bound, I think we can make this
// trade.
if badgerOpts.BlockCacheSize == 0 {
badgerOpts.Compression = options.None
} else {
badgerOpts.Compression = options.Snappy
}

Check warning on line 316 in setup.go

View check run for this annotation

Codecov / codecov/patch

setup.go#L295-L316 

Added lines #L295 - L316 were not covered by tests

// If we write something twice, we do it with the same values so
// *shrugh*.
badgerOpts.DetectConflicts = false

// MemTableSize: Defaults to 64MiB which seems an ok amount to flush
// to disk from time to time.
badgerOpts.MemTableSize = 64 << 20
// NumMemtables: more means more memory, faster writes, but more to
// commit to disk if they get full. Default is 5.
badgerOpts.NumMemtables = 5

// IndexCacheSize: 0 means all in memory (default). All means indexes,
// bloom filters etc. Usually not huge amount of memory usage from
// this.
badgerOpts.IndexCacheSize = 0

opts := badger4.Options{
GcDiscardRatio: 0.3,
GcInterval: 20 * time.Minute,
GcSleep: 10 * time.Second,
Options: badgerOpts,
}

return badger4.NewDatastore("badger4", &opts)

Check warning on line 341 in setup.go

View check run for this annotation

Codecov / codecov/patch

setup.go#L320-L341 

Added lines #L320 - L341 were not covered by tests
}

type bundledDHT struct {
standard *dht.IpfsDHT
fullRT *fullrt.FullRT
Expand Down Expand Up @@ -349,25 +410,6 @@ func delegatedHTTPContentRouter(endpoint string, rv1Opts ...routingv1client.Opti
}, nil
}

func loadBlockstore(spec string, path string) (blockstore.Blockstore, error) {
switch spec {
case "flatfs":
sf, err := flatfs.ParseShardFunc("/repo/flatfs/shard/v1/next-to-last/3")
if err != nil {
return nil, err
}

ds, err := flatfs.CreateOrOpen(path, sf, false)
if err != nil {
return nil, err
}

return blockstore.NewBlockstore(ds, blockstore.NoPrefix()), nil
default:
return nil, fmt.Errorf("unsupported blockstore type: %s", spec)
}
}

func loadOrInitPeerKey(kf string) (crypto.PrivKey, error) {
data, err := os.ReadFile(kf)
if err != nil {
Expand Down

0 comments on commit 11af995

Please sign in to comment.