feat!: IPNS routing based on ?format=ipns-record or /routing/v1

blockstore_proxy.go

@@ -25,8 +25,7 @@ import (
 const (
 	EnvProxyGateway = "PROXY_GATEWAY_URL"
 
-	DefaultProxyGateway = "http://127.0.0.1:8080"
-	DefaultKuboRPC      = "http://127.0.0.1:5001"
+	DefaultKuboRPC = "http://127.0.0.1:5001"
 )
 
 type proxyBlockStore struct {

docs/environment-variables.md

@@ -29,9 +29,7 @@
 Default: see `DefaultKuboRPC`
 
 Single URL or a comma separated list of RPC endpoints that provide `/api/v0` from Kubo.
-
-We use this as temporary solution for IPNS Record routing until [IPIP-351](https://github.com/ipfs/specs/pull/351) ships with Kubo 0.19,
-and we also redirect some legacy `/api/v0` commands that need to be handled on `ipfs.io`.
+This is used to redirect legacy `/api/v0` commands that need to be handled on `ipfs.io`.
 
 ### `BLOCK_CACHE_SIZE`
 
@@ -59,9 +57,20 @@ in the near feature.
 
 ### `PROXY_GATEWAY_URL`
 
-Single URL or a comma separated list of Gateway endpoints that support `?format=block|car`
-responses. This is used by default with `http://127.0.0.1:8080` unless `STRN_ORCHESTRATOR_URL`
-is set.
+Single URL or a comma separated list of Gateway endpoints that support `?format=block|car|ipns-record`
+responses. Either this variable of `STRN_ORCHESTRATOR_URL` must be set.
+
+If you're gateway does not support `?format=ipns-record`, you can use `IPNS_RECORD_GATEWAY`
+to override the gateway address from which to retrieve IPNS Records from.
+
+### `IPNS_RECORD_GATEWAY`
+
+Single URL or a comma separated list of Gateway endpoints that support `?format=ipns-record`.
+This is used for IPNS Record routing. If not set, the value of `PROXY_GATEWAY_URL` will be
+used.
+
+`IPNS_RECORD_GATEWAY` also supports [Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/)
+for IPNS Record routing. To use it, the provided URL must end with `/routing/v1`.
 
 ## Saturn Backend
 

handlers.go

@@ -68,9 +68,9 @@ func withRequestLogger(next http.Handler) http.Handler {
 	})
 }
 
-func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphBackend bool) (*http.Server, error) {
+func makeGatewayHandler(bs bstore.Blockstore, kuboRPC, gatewayURLs []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphBackend bool) (*http.Server, error) {
 	// Sets up the routing system, which will proxy the IPNS routing requests to the given gateway.
-	routing := newProxyRouting(kuboRPC, cdns)
+	routing := newProxyRouting(gatewayURLs, cdns)
 
 	// Sets up a cache to store blocks in
 	cacheBlockStore, err := lib.NewCacheBlockStore(blockCacheSize)

main.go

@@ -31,10 +31,11 @@ func main() {
 }
 
 const (
-	EnvKuboRPC        = "KUBO_RPC_URL"
-	EnvBlockCacheSize = "BLOCK_CACHE_SIZE"
-	EnvGraphBackend   = "GRAPH_BACKEND"
-	RequestIDHeader   = "X-Bfid"
+	EnvKuboRPC           = "KUBO_RPC_URL"
+	EnvIPNSRecordGateway = "IPNS_RECORD_GATEWAY"
+	EnvBlockCacheSize    = "BLOCK_CACHE_SIZE"
+	EnvGraphBackend      = "GRAPH_BACKEND"
+	RequestIDHeader      = "X-Bfid"
 )
 
 func init() {
@@ -107,7 +108,12 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`,
 			log.Fatalf("Unable to start. bifrost-gateway requires either PROXY_GATEWAY_URL or STRN_ORCHESTRATOR_URL to be set.\n\nRead docs at https://github.com/ipfs/bifrost-gateway/blob/main/docs/environment-variables.md\n\n")
 		}
 
-		gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize, cdns, useGraphBackend)
+		ipnsProxyGateway := getEnvs(EnvIPNSRecordGateway, "")
+		if len(ipnsProxyGateway) == 0 {
+			ipnsProxyGateway = proxyGateway
+		}
+
+		gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, ipnsProxyGateway, gatewayPort, blockCacheSize, cdns, useGraphBackend)
 		if err != nil {
 			return err
 		}

routing.go

@@ -1,11 +1,7 @@
 package main
 
 import (
-	"bytes"
 	"context"
-	"encoding/base64"
-	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -19,20 +15,20 @@ import (
 )
 
 type proxyRouting struct {
-	kuboRPC    []string
-	httpClient *http.Client
-	rand       *rand.Rand
+	gatewayURLs []string
+	httpClient  *http.Client
+	rand        *rand.Rand
 }
 
-func newProxyRouting(kuboRPC []string, cdns *cachedDNS) routing.ValueStore {
+func newProxyRouting(gatewayURLs []string, cdns *cachedDNS) routing.ValueStore {
 	s := rand.NewSource(time.Now().Unix())
 	rand := rand.New(s)
 
 	return &proxyRouting{
-		kuboRPC: kuboRPC,
+		gatewayURLs: gatewayURLs,
 		httpClient: &http.Client{
 			Transport: otelhttp.NewTransport(&customTransport{
-				// Roundtripper with increased defaults than http.Transport such that retrieving
+				// RoundTripper with increased defaults than http.Transport such that retrieving
 				// multiple lookups concurrently is fast.
 				RoundTripper: &http.Transport{
 					MaxIdleConns:        1000,
@@ -53,18 +49,32 @@ func (ps *proxyRouting) PutValue(context.Context, string, []byte, ...routing.Opt
 }
 
 func (ps *proxyRouting) GetValue(ctx context.Context, k string, opts ...routing.Option) ([]byte, error) {
-	return ps.fetch(ctx, k)
+	if !strings.HasPrefix(k, "/ipns/") {
+		return nil, routing.ErrNotSupported
+	}
+
+	name, err := ipns.NameFromRoutingKey([]byte(k))
+	if err != nil {
+		return nil, err
+	}
+
+	return ps.fetch(ctx, name)
 }
 
 func (ps *proxyRouting) SearchValue(ctx context.Context, k string, opts ...routing.Option) (<-chan []byte, error) {
 	if !strings.HasPrefix(k, "/ipns/") {
 		return nil, routing.ErrNotSupported
 	}
 
+	name, err := ipns.NameFromRoutingKey([]byte(k))
+	if err != nil {
+		return nil, err
+	}
+
 	ch := make(chan []byte)
 
 	go func() {
-		v, err := ps.fetch(ctx, k)
+		v, err := ps.fetch(ctx, name)
 		if err != nil {
 			close(ch)
 		} else {
@@ -76,24 +86,18 @@ func (ps *proxyRouting) SearchValue(ctx context.Context, k string, opts ...routi
 	return ch, nil
 }
 
-func (ps *proxyRouting) fetch(ctx context.Context, key string) (rb []byte, err error) {
-	name, err := ipns.NameFromRoutingKey([]byte(key))
+func (ps *proxyRouting) fetch(ctx context.Context, name ipns.Name) ([]byte, error) {
+	urlStr := fmt.Sprintf("%s/ipns/%s", ps.getRandomGatewayURL(), name.String())
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
 	if err != nil {
 		return nil, err
 	}
+	req.Header.Set("Accept", "application/vnd.ipfs.ipns-record")
 
-	key = "/ipns/" + name.String()
-
-	urlStr := fmt.Sprintf("%s/api/v0/dht/get?arg=%s", ps.getRandomKuboURL(), key)
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, urlStr, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	goLog.Debugw("routing proxy fetch", "key", key, "from", req.URL.String())
+	goLog.Debugw("routing proxy fetch", "key", name.String(), "from", req.URL.String())
 	defer func() {
 		if err != nil {
-			goLog.Debugw("routing proxy fetch error", "key", key, "from", req.URL.String(), "error", err.Error())
+			goLog.Debugw("routing proxy fetch error", "key", name.String(), "from", req.URL.String(), "error", err.Error())
 		}
 	}()
 
@@ -103,54 +107,33 @@ func (ps *proxyRouting) fetch(ctx context.Context, key string) (rb []byte, err e
 	}
 	defer resp.Body.Close()
 
-	// Read at most 10 KiB (max size of IPNS record).
-	rb, err = io.ReadAll(io.LimitReader(resp.Body, 10240))
 	if err != nil {
 		return nil, err
 	}
+	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("routing/get RPC returned unexpected status: %s, body: %q", resp.Status, string(rb))
-	}
-
-	parts := bytes.Split(bytes.TrimSpace(rb), []byte("\n"))
-	var b64 string
-
-	for _, part := range parts {
-		var evt routing.QueryEvent
-		err = json.Unmarshal(part, &evt)
-		if err != nil {
-			return nil, fmt.Errorf("routing/get RPC response cannot be parsed: %w", err)
-		}
-
-		if evt.Type == routing.Value {
-			b64 = evt.Extra
-			break
-		}
-	}
-
-	if b64 == "" {
-		return nil, errors.New("routing/get RPC returned no value")
+		return nil, fmt.Errorf("unexpected status from remote gateway: %s", resp.Status)
 	}
 
-	rb, err = base64.StdEncoding.DecodeString(b64)
+	rb, err := io.ReadAll(io.LimitReader(resp.Body, int64(ipns.MaxRecordSize)))
 	if err != nil {
 		return nil, err
 	}
 
-	entry, err := ipns.UnmarshalRecord(rb)
+	rec, err := ipns.UnmarshalRecord(rb)
 	if err != nil {
 		return nil, err
 	}
 
-	err = ipns.ValidateWithName(entry, name)
+	err = ipns.ValidateWithName(rec, name)
 	if err != nil {
 		return nil, err
 	}
 
 	return rb, nil
 }
 
-func (ps *proxyRouting) getRandomKuboURL() string {
-	return ps.kuboRPC[ps.rand.Intn(len(ps.kuboRPC))]
+func (ps *proxyRouting) getRandomGatewayURL() string {
+	return ps.gatewayURLs[ps.rand.Intn(len(ps.gatewayURLs))]
 }