From d797601bafec650b278901b51134a4441dc752c7 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 9 Aug 2023 11:47:51 +0200 Subject: [PATCH 1/5] feat!: IPNS routing based on ?format=ipns-record or /routing/v1 --- blockstore_proxy.go | 3 +- docs/environment-variables.md | 21 ++++++--- handlers.go | 4 +- main.go | 16 +++++-- routing.go | 87 ++++++++++++++--------------------- 5 files changed, 64 insertions(+), 67 deletions(-) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index deab4fb..1d13979 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -25,8 +25,7 @@ import ( const ( EnvProxyGateway = "PROXY_GATEWAY_URL" - DefaultProxyGateway = "http://127.0.0.1:8080" - DefaultKuboRPC = "http://127.0.0.1:5001" + DefaultKuboRPC = "http://127.0.0.1:5001" ) type proxyBlockStore struct { diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 321987c..5a59c53 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -29,9 +29,7 @@ Default: see `DefaultKuboRPC` Single URL or a comma separated list of RPC endpoints that provide `/api/v0` from Kubo. - -We use this as temporary solution for IPNS Record routing until [IPIP-351](https://github.com/ipfs/specs/pull/351) ships with Kubo 0.19, -and we also redirect some legacy `/api/v0` commands that need to be handled on `ipfs.io`. +This is used to redirect legacy `/api/v0` commands that need to be handled on `ipfs.io`. ### `BLOCK_CACHE_SIZE` @@ -59,9 +57,20 @@ in the near feature. ### `PROXY_GATEWAY_URL` -Single URL or a comma separated list of Gateway endpoints that support `?format=block|car` -responses. This is used by default with `http://127.0.0.1:8080` unless `STRN_ORCHESTRATOR_URL` -is set. +Single URL or a comma separated list of Gateway endpoints that support `?format=block|car|ipns-record` +responses. Either this variable of `STRN_ORCHESTRATOR_URL` must be set. + +If you're gateway does not support `?format=ipns-record`, you can use `IPNS_RECORD_GATEWAY` +to override the gateway address from which to retrieve IPNS Records from. + +### `IPNS_RECORD_GATEWAY` + +Single URL or a comma separated list of Gateway endpoints that support `?format=ipns-record`. +This is used for IPNS Record routing. If not set, the value of `PROXY_GATEWAY_URL` will be +used. + +`IPNS_RECORD_GATEWAY` also supports [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) +for IPNS Record routing. To use it, the provided URL must end with `/routing/v1`. ## Saturn Backend diff --git a/handlers.go b/handlers.go index fb844dd..b80edb0 100644 --- a/handlers.go +++ b/handlers.go @@ -68,9 +68,9 @@ func withRequestLogger(next http.Handler) http.Handler { }) } -func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphBackend bool) (*http.Server, error) { +func makeGatewayHandler(bs bstore.Blockstore, kuboRPC, gatewayURLs []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphBackend bool) (*http.Server, error) { // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway. - routing := newProxyRouting(kuboRPC, cdns) + routing := newProxyRouting(gatewayURLs, cdns) // Sets up a cache to store blocks in cacheBlockStore, err := lib.NewCacheBlockStore(blockCacheSize) diff --git a/main.go b/main.go index 58b35f3..32906a5 100644 --- a/main.go +++ b/main.go @@ -31,10 +31,11 @@ func main() { } const ( - EnvKuboRPC = "KUBO_RPC_URL" - EnvBlockCacheSize = "BLOCK_CACHE_SIZE" - EnvGraphBackend = "GRAPH_BACKEND" - RequestIDHeader = "X-Bfid" + EnvKuboRPC = "KUBO_RPC_URL" + EnvIPNSRecordGateway = "IPNS_RECORD_GATEWAY" + EnvBlockCacheSize = "BLOCK_CACHE_SIZE" + EnvGraphBackend = "GRAPH_BACKEND" + RequestIDHeader = "X-Bfid" ) func init() { @@ -107,7 +108,12 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, log.Fatalf("Unable to start. bifrost-gateway requires either PROXY_GATEWAY_URL or STRN_ORCHESTRATOR_URL to be set.\n\nRead docs at https://github.com/ipfs/bifrost-gateway/blob/main/docs/environment-variables.md\n\n") } - gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize, cdns, useGraphBackend) + ipnsProxyGateway := getEnvs(EnvIPNSRecordGateway, "") + if len(ipnsProxyGateway) == 0 { + ipnsProxyGateway = proxyGateway + } + + gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, ipnsProxyGateway, gatewayPort, blockCacheSize, cdns, useGraphBackend) if err != nil { return err } diff --git a/routing.go b/routing.go index 286c5dd..cc6e016 100644 --- a/routing.go +++ b/routing.go @@ -1,11 +1,7 @@ package main import ( - "bytes" "context" - "encoding/base64" - "encoding/json" - "errors" "fmt" "io" "math/rand" @@ -19,20 +15,20 @@ import ( ) type proxyRouting struct { - kuboRPC []string - httpClient *http.Client - rand *rand.Rand + gatewayURLs []string + httpClient *http.Client + rand *rand.Rand } -func newProxyRouting(kuboRPC []string, cdns *cachedDNS) routing.ValueStore { +func newProxyRouting(gatewayURLs []string, cdns *cachedDNS) routing.ValueStore { s := rand.NewSource(time.Now().Unix()) rand := rand.New(s) return &proxyRouting{ - kuboRPC: kuboRPC, + gatewayURLs: gatewayURLs, httpClient: &http.Client{ Transport: otelhttp.NewTransport(&customTransport{ - // Roundtripper with increased defaults than http.Transport such that retrieving + // RoundTripper with increased defaults than http.Transport such that retrieving // multiple lookups concurrently is fast. RoundTripper: &http.Transport{ MaxIdleConns: 1000, @@ -53,7 +49,16 @@ func (ps *proxyRouting) PutValue(context.Context, string, []byte, ...routing.Opt } func (ps *proxyRouting) GetValue(ctx context.Context, k string, opts ...routing.Option) ([]byte, error) { - return ps.fetch(ctx, k) + if !strings.HasPrefix(k, "/ipns/") { + return nil, routing.ErrNotSupported + } + + name, err := ipns.NameFromRoutingKey([]byte(k)) + if err != nil { + return nil, err + } + + return ps.fetch(ctx, name) } func (ps *proxyRouting) SearchValue(ctx context.Context, k string, opts ...routing.Option) (<-chan []byte, error) { @@ -61,10 +66,15 @@ func (ps *proxyRouting) SearchValue(ctx context.Context, k string, opts ...routi return nil, routing.ErrNotSupported } + name, err := ipns.NameFromRoutingKey([]byte(k)) + if err != nil { + return nil, err + } + ch := make(chan []byte) go func() { - v, err := ps.fetch(ctx, k) + v, err := ps.fetch(ctx, name) if err != nil { close(ch) } else { @@ -76,24 +86,18 @@ func (ps *proxyRouting) SearchValue(ctx context.Context, k string, opts ...routi return ch, nil } -func (ps *proxyRouting) fetch(ctx context.Context, key string) (rb []byte, err error) { - name, err := ipns.NameFromRoutingKey([]byte(key)) +func (ps *proxyRouting) fetch(ctx context.Context, name ipns.Name) ([]byte, error) { + urlStr := fmt.Sprintf("%s/ipns/%s", ps.getRandomGatewayURL(), name.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil) if err != nil { return nil, err } + req.Header.Set("Accept", "application/vnd.ipfs.ipns-record") - key = "/ipns/" + name.String() - - urlStr := fmt.Sprintf("%s/api/v0/dht/get?arg=%s", ps.getRandomKuboURL(), key) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, urlStr, nil) - if err != nil { - return nil, err - } - - goLog.Debugw("routing proxy fetch", "key", key, "from", req.URL.String()) + goLog.Debugw("routing proxy fetch", "key", name.String(), "from", req.URL.String()) defer func() { if err != nil { - goLog.Debugw("routing proxy fetch error", "key", key, "from", req.URL.String(), "error", err.Error()) + goLog.Debugw("routing proxy fetch error", "key", name.String(), "from", req.URL.String(), "error", err.Error()) } }() @@ -103,47 +107,26 @@ func (ps *proxyRouting) fetch(ctx context.Context, key string) (rb []byte, err e } defer resp.Body.Close() - // Read at most 10 KiB (max size of IPNS record). - rb, err = io.ReadAll(io.LimitReader(resp.Body, 10240)) if err != nil { return nil, err } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("routing/get RPC returned unexpected status: %s, body: %q", resp.Status, string(rb)) - } - - parts := bytes.Split(bytes.TrimSpace(rb), []byte("\n")) - var b64 string - - for _, part := range parts { - var evt routing.QueryEvent - err = json.Unmarshal(part, &evt) - if err != nil { - return nil, fmt.Errorf("routing/get RPC response cannot be parsed: %w", err) - } - - if evt.Type == routing.Value { - b64 = evt.Extra - break - } - } - - if b64 == "" { - return nil, errors.New("routing/get RPC returned no value") + return nil, fmt.Errorf("unexpected status from remote gateway: %s", resp.Status) } - rb, err = base64.StdEncoding.DecodeString(b64) + rb, err := io.ReadAll(io.LimitReader(resp.Body, int64(ipns.MaxRecordSize))) if err != nil { return nil, err } - entry, err := ipns.UnmarshalRecord(rb) + rec, err := ipns.UnmarshalRecord(rb) if err != nil { return nil, err } - err = ipns.ValidateWithName(entry, name) + err = ipns.ValidateWithName(rec, name) if err != nil { return nil, err } @@ -151,6 +134,6 @@ func (ps *proxyRouting) fetch(ctx context.Context, key string) (rb []byte, err e return rb, nil } -func (ps *proxyRouting) getRandomKuboURL() string { - return ps.kuboRPC[ps.rand.Intn(len(ps.kuboRPC))] +func (ps *proxyRouting) getRandomGatewayURL() string { + return ps.gatewayURLs[ps.rand.Intn(len(ps.gatewayURLs))] } From 9aafa0d605ceab09144d84601d623a7969c4c9ac Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 10 Aug 2023 09:40:24 +0200 Subject: [PATCH 2/5] chore: apply feedback suggestions Co-authored-by: Marcin Rataj --- docs/environment-variables.md | 8 ++++---- routing.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 5a59c53..3664fa2 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -58,19 +58,19 @@ in the near feature. ### `PROXY_GATEWAY_URL` Single URL or a comma separated list of Gateway endpoints that support `?format=block|car|ipns-record` -responses. Either this variable of `STRN_ORCHESTRATOR_URL` must be set. +responses. Either this variable or `STRN_ORCHESTRATOR_URL` must be set. -If you're gateway does not support `?format=ipns-record`, you can use `IPNS_RECORD_GATEWAY` +If this gateway does not support `application/vnd.ipfs.ipns-record`, you can use `IPNS_RECORD_GATEWAY` to override the gateway address from which to retrieve IPNS Records from. ### `IPNS_RECORD_GATEWAY` -Single URL or a comma separated list of Gateway endpoints that support `?format=ipns-record`. +Single URL or a comma separated list of Gateway endpoints that support requests for `application/vnd.ipfs.ipns-record`. This is used for IPNS Record routing. If not set, the value of `PROXY_GATEWAY_URL` will be used. `IPNS_RECORD_GATEWAY` also supports [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) -for IPNS Record routing. To use it, the provided URL must end with `/routing/v1`. +for IPNS Record routing ([IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/)). To use it, the provided URL must end with `/routing/v1`. ## Saturn Backend diff --git a/routing.go b/routing.go index cc6e016..b906125 100644 --- a/routing.go +++ b/routing.go @@ -135,5 +135,5 @@ func (ps *proxyRouting) fetch(ctx context.Context, name ipns.Name) ([]byte, erro } func (ps *proxyRouting) getRandomGatewayURL() string { - return ps.gatewayURLs[ps.rand.Intn(len(ps.gatewayURLs))] + return strings.TrimSuffix(ps.gatewayURLs[ps.rand.Intn(len(ps.gatewayURLs))], "/") } From e909627b52932ef1d5e62e99a39e8b7def09ae8f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 10 Aug 2023 11:55:10 +0200 Subject: [PATCH 3/5] refactor!: remove DefaultKuboRPC --- blockstore_proxy.go | 2 -- docs/environment-variables.md | 3 +-- handlers.go | 11 +++++++---- main.go | 7 +++++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index 1d13979..8e78c84 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -24,8 +24,6 @@ import ( const ( EnvProxyGateway = "PROXY_GATEWAY_URL" - - DefaultKuboRPC = "http://127.0.0.1:5001" ) type proxyBlockStore struct { diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 3664fa2..09ee5a6 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -26,10 +26,9 @@ ### `KUBO_RPC_URL` -Default: see `DefaultKuboRPC` - Single URL or a comma separated list of RPC endpoints that provide `/api/v0` from Kubo. This is used to redirect legacy `/api/v0` commands that need to be handled on `ipfs.io`. +If this is not set, the redirects are not set up. ### `BLOCK_CACHE_SIZE` diff --git a/handlers.go b/handlers.go index b80edb0..92f62a1 100644 --- a/handlers.go +++ b/handlers.go @@ -157,10 +157,13 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC, gatewayURLs []string, por mux := http.NewServeMux() mux.Handle("/ipfs/", ipfsHandler) mux.Handle("/ipns/", ipnsHandler) - // TODO: below is legacy which we want to remove, measuring this separately - // allows us to decide when is the time to do it. - legacyKuboRpcHandler := withHTTPMetrics(newKuboRPCHandler(kuboRPC), "legacyKuboRpc") - mux.Handle("/api/v0/", legacyKuboRpcHandler) + + if len(kuboRPC) != 0 { + // TODO: below is legacy which we want to remove, measuring this separately + // allows us to decide when is the time to do it. + legacyKuboRpcHandler := withHTTPMetrics(newKuboRPCHandler(kuboRPC), "legacyKuboRpc") + mux.Handle("/api/v0/", legacyKuboRpcHandler) + } // Construct the HTTP handler for the gateway. handler := withConnect(mux) diff --git a/main.go b/main.go index 32906a5..884d9af 100644 --- a/main.go +++ b/main.go @@ -58,7 +58,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, // Get env variables. saturnOrchestrator := getEnv(EnvSaturnOrchestrator, "") proxyGateway := getEnvs(EnvProxyGateway, "") - kuboRPC := getEnvs(EnvKuboRPC, DefaultKuboRPC) + kuboRPC := getEnvs(EnvKuboRPC, "") blockCacheSize, err := getEnvInt(EnvBlockCacheSize, lib.DefaultCacheBlockStoreSize) if err != nil { @@ -134,7 +134,10 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, log.Printf("%s: %d", EnvBlockCacheSize, blockCacheSize) log.Printf("%s: %t", EnvGraphBackend, useGraphBackend) - log.Printf("Legacy RPC at /api/v0 (%s) provided by %s", EnvKuboRPC, strings.Join(kuboRPC, " ")) + if len(kuboRPC) != 0 { + log.Printf("Legacy RPC at /api/v0 (%s) provided by %s", EnvKuboRPC, strings.Join(kuboRPC, " ")) + } + log.Printf("Path gateway listening on http://127.0.0.1:%d", gatewayPort) log.Printf(" Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", gatewayPort) log.Printf("Subdomain gateway configured on dweb.link and http://localhost:%d", gatewayPort) From 2bdce7a132015e2dea63cd9c98ce3ddbc1c3d082 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 15 Aug 2023 12:51:42 +0200 Subject: [PATCH 4/5] feat: use Kubo RPC if routing v1 not set --- docs/environment-variables.md | 5 +- handlers.go | 13 +++- main.go | 4 - routing.go | 141 ++++++++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 8 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 09ee5a6..2b0014d 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -65,12 +65,13 @@ to override the gateway address from which to retrieve IPNS Records from. ### `IPNS_RECORD_GATEWAY` Single URL or a comma separated list of Gateway endpoints that support requests for `application/vnd.ipfs.ipns-record`. -This is used for IPNS Record routing. If not set, the value of `PROXY_GATEWAY_URL` will be -used. +This is used for IPNS Record routing. `IPNS_RECORD_GATEWAY` also supports [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) for IPNS Record routing ([IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/)). To use it, the provided URL must end with `/routing/v1`. +If not set, the IPNS records will be fetched from `KUBO_RPC_URL`. + ## Saturn Backend ### `STRN_ORCHESTRATOR_URL` diff --git a/handlers.go b/handlers.go index 92f62a1..a09a3ec 100644 --- a/handlers.go +++ b/handlers.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "math/rand" "net/http" @@ -12,6 +13,7 @@ import ( _ "net/http/pprof" "github.com/ipfs/bifrost-gateway/lib" + "github.com/libp2p/go-libp2p/core/routing" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "github.com/filecoin-saturn/caboose" @@ -69,8 +71,15 @@ func withRequestLogger(next http.Handler) http.Handler { } func makeGatewayHandler(bs bstore.Blockstore, kuboRPC, gatewayURLs []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphBackend bool) (*http.Server, error) { - // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway. - routing := newProxyRouting(gatewayURLs, cdns) + // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway or kubo RPC. + var routing routing.ValueStore + if len(gatewayURLs) != 0 { + routing = newProxyRouting(gatewayURLs, cdns) + } else if len(kuboRPC) != 0 { + routing = newRPCProxyRouting(kuboRPC, cdns) + } else { + return nil, errors.New("kubo rpc or gateway urls must be provided in order to delegate routing") + } // Sets up a cache to store blocks in cacheBlockStore, err := lib.NewCacheBlockStore(blockCacheSize) diff --git a/main.go b/main.go index 884d9af..4292199 100644 --- a/main.go +++ b/main.go @@ -109,10 +109,6 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, } ipnsProxyGateway := getEnvs(EnvIPNSRecordGateway, "") - if len(ipnsProxyGateway) == 0 { - ipnsProxyGateway = proxyGateway - } - gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, ipnsProxyGateway, gatewayPort, blockCacheSize, cdns, useGraphBackend) if err != nil { return err diff --git a/routing.go b/routing.go index b906125..9b1b216 100644 --- a/routing.go +++ b/routing.go @@ -1,7 +1,11 @@ package main import ( + "bytes" "context" + "encoding/base64" + "encoding/json" + "errors" "fmt" "io" "math/rand" @@ -14,6 +18,143 @@ import ( "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) +type rpcProxyRouting struct { + kuboRPC []string + httpClient *http.Client + rand *rand.Rand +} + +func newRPCProxyRouting(kuboRPC []string, cdns *cachedDNS) routing.ValueStore { + s := rand.NewSource(time.Now().Unix()) + rand := rand.New(s) + + return &rpcProxyRouting{ + kuboRPC: kuboRPC, + httpClient: &http.Client{ + Transport: otelhttp.NewTransport(&customTransport{ + // Roundtripper with increased defaults than http.Transport such that retrieving + // multiple lookups concurrently is fast. + RoundTripper: &http.Transport{ + MaxIdleConns: 1000, + MaxConnsPerHost: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + DialContext: cdns.dialWithCachedDNS, + ForceAttemptHTTP2: true, + }, + }), + }, + rand: rand, + } +} + +func (ps *rpcProxyRouting) PutValue(context.Context, string, []byte, ...routing.Option) error { + return routing.ErrNotSupported +} + +func (ps *rpcProxyRouting) GetValue(ctx context.Context, k string, opts ...routing.Option) ([]byte, error) { + return ps.fetch(ctx, k) +} + +func (ps *rpcProxyRouting) SearchValue(ctx context.Context, k string, opts ...routing.Option) (<-chan []byte, error) { + if !strings.HasPrefix(k, "/ipns/") { + return nil, routing.ErrNotSupported + } + + ch := make(chan []byte) + + go func() { + v, err := ps.fetch(ctx, k) + if err != nil { + close(ch) + } else { + ch <- v + close(ch) + } + }() + + return ch, nil +} + +func (ps *rpcProxyRouting) fetch(ctx context.Context, key string) (rb []byte, err error) { + name, err := ipns.NameFromRoutingKey([]byte(key)) + if err != nil { + return nil, err + } + + key = "/ipns/" + name.String() + + urlStr := fmt.Sprintf("%s/api/v0/dht/get?arg=%s", ps.getRandomKuboURL(), key) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, urlStr, nil) + if err != nil { + return nil, err + } + + goLog.Debugw("routing proxy fetch", "key", key, "from", req.URL.String()) + defer func() { + if err != nil { + goLog.Debugw("routing proxy fetch error", "key", key, "from", req.URL.String(), "error", err.Error()) + } + }() + + resp, err := ps.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Read at most 10 KiB (max size of IPNS record). + rb, err = io.ReadAll(io.LimitReader(resp.Body, 10240)) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("routing/get RPC returned unexpected status: %s, body: %q", resp.Status, string(rb)) + } + + parts := bytes.Split(bytes.TrimSpace(rb), []byte("\n")) + var b64 string + + for _, part := range parts { + var evt routing.QueryEvent + err = json.Unmarshal(part, &evt) + if err != nil { + return nil, fmt.Errorf("routing/get RPC response cannot be parsed: %w", err) + } + + if evt.Type == routing.Value { + b64 = evt.Extra + break + } + } + + if b64 == "" { + return nil, errors.New("routing/get RPC returned no value") + } + + rb, err = base64.StdEncoding.DecodeString(b64) + if err != nil { + return nil, err + } + + entry, err := ipns.UnmarshalRecord(rb) + if err != nil { + return nil, err + } + + err = ipns.ValidateWithName(entry, name) + if err != nil { + return nil, err + } + + return rb, nil +} + +func (ps *rpcProxyRouting) getRandomKuboURL() string { + return ps.kuboRPC[ps.rand.Intn(len(ps.kuboRPC))] +} + type proxyRouting struct { gatewayURLs []string httpClient *http.Client From dbec1e9dec0680ebdc6d9d82c51d0e7586b1c840 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 5 Oct 2023 00:56:14 +0200 Subject: [PATCH 5/5] refactor: IPNS_RECORD_GATEWAY_URL Ensures IPNS_RECORD_GATEWAY_URL is prefered, then we fallback to PROXY_GATEWAY_URL, and finally KUBO_RPC_URL --- docs/environment-variables.md | 6 +++--- handlers.go | 8 ++++---- main.go | 13 ++++++++++--- routing.go | 12 ++++++------ 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 2b0014d..da7b5eb 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -59,15 +59,15 @@ in the near feature. Single URL or a comma separated list of Gateway endpoints that support `?format=block|car|ipns-record` responses. Either this variable or `STRN_ORCHESTRATOR_URL` must be set. -If this gateway does not support `application/vnd.ipfs.ipns-record`, you can use `IPNS_RECORD_GATEWAY` +If this gateway does not support `application/vnd.ipfs.ipns-record`, you can use `IPNS_RECORD_GATEWAY_URL` to override the gateway address from which to retrieve IPNS Records from. -### `IPNS_RECORD_GATEWAY` +### `IPNS_RECORD_GATEWAY_URL` Single URL or a comma separated list of Gateway endpoints that support requests for `application/vnd.ipfs.ipns-record`. This is used for IPNS Record routing. -`IPNS_RECORD_GATEWAY` also supports [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) +`IPNS_RECORD_GATEWAY_URL` also supports [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) for IPNS Record routing ([IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/)). To use it, the provided URL must end with `/routing/v1`. If not set, the IPNS records will be fetched from `KUBO_RPC_URL`. diff --git a/handlers.go b/handlers.go index a09a3ec..d64a8dd 100644 --- a/handlers.go +++ b/handlers.go @@ -70,15 +70,15 @@ func withRequestLogger(next http.Handler) http.Handler { }) } -func makeGatewayHandler(bs bstore.Blockstore, kuboRPC, gatewayURLs []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphBackend bool) (*http.Server, error) { +func makeGatewayHandler(bs bstore.Blockstore, kuboRPC, ipnsRecordGateways []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphBackend bool) (*http.Server, error) { // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway or kubo RPC. var routing routing.ValueStore - if len(gatewayURLs) != 0 { - routing = newProxyRouting(gatewayURLs, cdns) + if len(ipnsRecordGateways) != 0 { + routing = newProxyRouting(ipnsRecordGateways, cdns) } else if len(kuboRPC) != 0 { routing = newRPCProxyRouting(kuboRPC, cdns) } else { - return nil, errors.New("kubo rpc or gateway urls must be provided in order to delegate routing") + return nil, errors.New("either KUBO_RPC_URL, IPNS_RECORD_GATEWAY_URL or PROXY_GATEWAY_URL with support for application/vnd.ipfs.ipns-record must be provided in order to delegate IPNS routing") } // Sets up a cache to store blocks in diff --git a/main.go b/main.go index 4292199..ffd27db 100644 --- a/main.go +++ b/main.go @@ -32,7 +32,7 @@ func main() { const ( EnvKuboRPC = "KUBO_RPC_URL" - EnvIPNSRecordGateway = "IPNS_RECORD_GATEWAY" + EnvIPNSRecordGateway = "IPNS_RECORD_GATEWAY_URL" EnvBlockCacheSize = "BLOCK_CACHE_SIZE" EnvGraphBackend = "GRAPH_BACKEND" RequestIDHeader = "X-Bfid" @@ -108,8 +108,15 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, log.Fatalf("Unable to start. bifrost-gateway requires either PROXY_GATEWAY_URL or STRN_ORCHESTRATOR_URL to be set.\n\nRead docs at https://github.com/ipfs/bifrost-gateway/blob/main/docs/environment-variables.md\n\n") } - ipnsProxyGateway := getEnvs(EnvIPNSRecordGateway, "") - gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, ipnsProxyGateway, gatewayPort, blockCacheSize, cdns, useGraphBackend) + // Prefer IPNS_RECORD_GATEWAY_URL when an explicit URL for IPNS routing is set + ipnsRecordGateway := getEnvs(EnvIPNSRecordGateway, "") + if len(ipnsRecordGateway) == 0 { + // Fallback to PROXY_GATEWAY_URL, assuming it is modern + // enough to support application/vnd.ipfs.ipns-record responses + ipnsRecordGateway = proxyGateway + } + + gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, ipnsRecordGateway, gatewayPort, blockCacheSize, cdns, useGraphBackend) if err != nil { return err } diff --git a/routing.go b/routing.go index 9b1b216..17d9f88 100644 --- a/routing.go +++ b/routing.go @@ -156,17 +156,17 @@ func (ps *rpcProxyRouting) getRandomKuboURL() string { } type proxyRouting struct { - gatewayURLs []string - httpClient *http.Client - rand *rand.Rand + ipnsRecordGateways []string + httpClient *http.Client + rand *rand.Rand } -func newProxyRouting(gatewayURLs []string, cdns *cachedDNS) routing.ValueStore { +func newProxyRouting(ipnsRecordGateways []string, cdns *cachedDNS) routing.ValueStore { s := rand.NewSource(time.Now().Unix()) rand := rand.New(s) return &proxyRouting{ - gatewayURLs: gatewayURLs, + ipnsRecordGateways: ipnsRecordGateways, httpClient: &http.Client{ Transport: otelhttp.NewTransport(&customTransport{ // RoundTripper with increased defaults than http.Transport such that retrieving @@ -276,5 +276,5 @@ func (ps *proxyRouting) fetch(ctx context.Context, name ipns.Name) ([]byte, erro } func (ps *proxyRouting) getRandomGatewayURL() string { - return strings.TrimSuffix(ps.gatewayURLs[ps.rand.Intn(len(ps.gatewayURLs))], "/") + return strings.TrimSuffix(ps.ipnsRecordGateways[ps.rand.Intn(len(ps.ipnsRecordGateways))], "/") }