feat: wildcard support for public gateways

Add support for one or more wildcards in the hostname definition of a public gateway. This is useful for example to support easily multiples environment. Wildcarded hostname are set in the config as for example "*.domain.tld".
ipfs · Jul 15, 2020 · f2beb61 · f2beb61
1 parent b98f797
commit f2beb61
Show file tree

Hide file tree

Showing 4 changed files with 213 additions and 60 deletions.
diff --git a/core/corehttp/hostname.go b/core/corehttp/hostname.go
@@ -6,6 +6,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strings"
 
 	cid "github.com/ipfs/go-cid"
@@ -24,17 +25,17 @@ import (
 
 var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/", "/version"}
 
-var pathGatewaySpec = config.GatewaySpec{
+var pathGatewaySpec = &config.GatewaySpec{
 	Paths:         defaultPaths,
 	UseSubdomains: false,
 }
 
-var subdomainGatewaySpec = config.GatewaySpec{
+var subdomainGatewaySpec = &config.GatewaySpec{
 	Paths:         defaultPaths,
 	UseSubdomains: true,
 }
 
-var defaultKnownGateways = map[string]config.GatewaySpec{
+var defaultKnownGateways = map[string]*config.GatewaySpec{
 	"localhost":       subdomainGatewaySpec,
 	"ipfs.io":         pathGatewaySpec,
 	"gateway.ipfs.io": pathGatewaySpec,
@@ -58,22 +59,8 @@ func HostnameOption() ServeOption {
 		if err != nil {
 			return nil, err
 		}
-		knownGateways := make(
-			map[string]config.GatewaySpec,
-			len(defaultKnownGateways)+len(cfg.Gateway.PublicGateways),
-		)
-		for hostname, gw := range defaultKnownGateways {
-			knownGateways[hostname] = gw
-		}
-		for hostname, gw := range cfg.Gateway.PublicGateways {
-			if gw == nil {
-				// Allows the user to remove gateways but _also_
-				// allows us to continuously update the list.
-				delete(knownGateways, hostname)
-			} else {
-				knownGateways[hostname] = *gw
-			}
-		}
+
+		knownGateways := prepareKnownGateways(cfg.Gateway.PublicGateways)
 
 		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 			// Unfortunately, many (well, ipfs.io) gateways use
@@ -233,22 +220,85 @@ func HostnameOption() ServeOption {
 	}
 }
 
+type gatewayHosts struct {
+	exact    map[string]*config.GatewaySpec
+	wildcard []wildcardHost
+}
+
+type wildcardHost struct {
+	re   *regexp.Regexp
+	spec *config.GatewaySpec
+}
+
+func prepareKnownGateways(publicGateways map[string]*config.GatewaySpec) gatewayHosts {
+	var hosts gatewayHosts
+
+	if len(publicGateways) == 0 {
+		hosts.exact = make(
+			map[string]*config.GatewaySpec,
+			len(defaultKnownGateways),
+		)
+		for hostname, gw := range defaultKnownGateways {
+			hosts.exact[hostname] = gw
+		}
+		return hosts
+	}
+
+	hosts.exact = make(map[string]*config.GatewaySpec, len(publicGateways))
+
+	for hostname, gw := range publicGateways {
+		if gw == nil {
+			continue
+		}
+		if strings.Contains(hostname, "*") {
+			// from *.domain.tld, construct a regexp that match any direct subdomain
+			// of .domain.tld.
+			//
+			// Regexp will be in the form of ^[^.]+\.domain.tld(?::\d+)?$
+
+			escaped := strings.ReplaceAll(hostname, ".", `\.`)
+			regexed := strings.ReplaceAll(escaped, "*", "[^.]+")
+
+			re, err := regexp.Compile(fmt.Sprintf(`^%s(?::\d+)?$`, regexed))
+			if err != nil {
+				log.Warn("invalid wildcard gateway hostname \"%s\"", hostname)
+			}
+
+			hosts.wildcard = append(hosts.wildcard, wildcardHost{re: re, spec: gw})
+		} else {
+			hosts.exact[hostname] = gw
+		}
+	}
+
+	return hosts
+}
+
 // isKnownHostname checks Gateway.PublicGateways and returns matching
 // GatewaySpec with gracefull fallback to version without port
-func isKnownHostname(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, ok bool) {
+func isKnownHostname(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, ok bool) {
 	// Try hostname (host+optional port - value from Host header as-is)
-	if gw, ok := knownGateways[hostname]; ok {
+	if gw, ok := knownGateways.exact[hostname]; ok {
+		return gw, ok
+	}
+	// Also test without port
+	if gw, ok = knownGateways.exact[stripPort(hostname)]; ok {
 		return gw, ok
 	}
-	// Fallback to hostname without port
-	gw, ok = knownGateways[stripPort(hostname)]
+
+	// Wildcard support. Test both with and without port.
+	for _, host := range knownGateways.wildcard {
+		if host.re.MatchString(hostname) {
+			return host.spec, true
+		}
+	}
+
 	return gw, ok
 }
 
 // Parses Host header and looks for a known subdomain gateway host.
 // If found, returns GatewaySpec and subdomain components.
 // Note: hostname is host + optional port
-func knownSubdomainDetails(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, knownHostname, ns, rootID string, ok bool) {
+func knownSubdomainDetails(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, knownHostname, ns, rootID string, ok bool) {
 	labels := strings.Split(hostname, ".")
 	// Look for FQDN of a known gateway hostname.
 	// Example: given "dist.ipfs.io.ipns.dweb.link":

diff --git a/core/corehttp/hostname_test.go b/core/corehttp/hostname_test.go
@@ -32,6 +32,7 @@ func TestToSubdomainURL(t *testing.T) {
 		{"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil},
 		// PeerID: ed25519+identity multihash → CIDv1Base36
 		{"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil},
+		{"sub.localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.sub.localhost/", nil},
 	} {
 		url, err := toSubdomainURL(test.hostname, test.path, r)
 		if url != test.url || !equalError(err, test.err) {
@@ -104,60 +105,73 @@ func TestDNSPrefix(t *testing.T) {
 }
 
 func TestKnownSubdomainDetails(t *testing.T) {
-	gwSpec := config.GatewaySpec{
-		UseSubdomains: true,
-	}
-	knownGateways := map[string]config.GatewaySpec{
-		"localhost":               gwSpec,
-		"dweb.link":               gwSpec,
-		"dweb.ipfs.pvt.k12.ma.us": gwSpec, // note the sneaky ".ipfs." ;-)
-	}
+	gwLocalhost := &config.GatewaySpec{}
+	gwDweb := &config.GatewaySpec{}
+	gwLong := &config.GatewaySpec{}
+	gwWildcard1 := &config.GatewaySpec{}
+	gwWildcard2 := &config.GatewaySpec{}
+
+	knownGateways := prepareKnownGateways(map[string]*config.GatewaySpec{
+		"localhost":               gwLocalhost,
+		"dweb.link":               gwDweb,
+		"dweb.ipfs.pvt.k12.ma.us": gwLong, // note the sneaky ".ipfs." ;-)
+		"*.wildcard1.tld":         gwWildcard1,
+		"*.*.wildcard2.tld":       gwWildcard2,
+	})
 
 	for _, test := range []struct {
 		// in:
 		hostHeader string
 		// out:
+		gw       *config.GatewaySpec
 		hostname string
 		ns       string
 		rootID   string
 		ok       bool
 	}{
 		// no subdomain
-		{"127.0.0.1:8080", "", "", "", false},
-		{"[::1]:8080", "", "", "", false},
-		{"hey.look.example.com", "", "", "", false},
-		{"dweb.link", "", "", "", false},
+		{"127.0.0.1:8080", nil, "", "", "", false},
+		{"[::1]:8080", nil, "", "", "", false},
+		{"hey.look.example.com", nil, "", "", "", false},
+		{"dweb.link", nil, "", "", "", false},
 		// malformed Host header
-		{".....dweb.link", "", "", "", false},
-		{"link", "", "", "", false},
-		{"8080:dweb.link", "", "", "", false},
-		{" ", "", "", "", false},
-		{"", "", "", "", false},
+		{".....dweb.link", nil, "", "", "", false},
+		{"link", nil, "", "", "", false},
+		{"8080:dweb.link", nil, "", "", "", false},
+		{" ", nil, "", "", "", false},
+		{"", nil, "", "", "", false},
 		// unknown gateway host
-		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", "", "", "", false},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", nil, "", "", "", false},
 		// cid in subdomain, known gateway
-		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
-		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", gwLocalhost, "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
 		// capture everything before .ipfs.
-		{"foo.bar.boo-buzz.ipfs.dweb.link", "dweb.link", "ipfs", "foo.bar.boo-buzz", true},
+		{"foo.bar.boo-buzz.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "foo.bar.boo-buzz", true},
 		// ipns
-		{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
-		{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
+		{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
+		{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
 		// edge case check: public gateway under long TLD (see: https://publicsuffix.org)
-		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
-		{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
+		{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
 		// dnslink in subdomain
-		{"en.wikipedia-on-ipfs.org.ipns.localhost:8080", "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true},
-		{"en.wikipedia-on-ipfs.org.ipns.localhost", "localhost", "ipns", "en.wikipedia-on-ipfs.org", true},
-		{"dist.ipfs.io.ipns.localhost:8080", "localhost:8080", "ipns", "dist.ipfs.io", true},
-		{"en.wikipedia-on-ipfs.org.ipns.dweb.link", "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true},
+		{"en.wikipedia-on-ipfs.org.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true},
+		{"en.wikipedia-on-ipfs.org.ipns.localhost", gwLocalhost, "localhost", "ipns", "en.wikipedia-on-ipfs.org", true},
+		{"dist.ipfs.io.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "dist.ipfs.io", true},
+		{"en.wikipedia-on-ipfs.org.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true},
 		// edge case check: public gateway under long TLD (see: https://publicsuffix.org)
-		{"foo.dweb.ipfs.pvt.k12.ma.us", "", "", "", false},
-		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
-		{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
+		{"foo.dweb.ipfs.pvt.k12.ma.us", nil, "", "", "", false},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
+		{"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true},
 		// other namespaces
-		{"api.localhost", "", "", "", false},
-		{"peerid.p2p.localhost", "localhost", "p2p", "peerid", true},
+		{"api.localhost", nil, "", "", "", false},
+		{"peerid.p2p.localhost", gwLocalhost, "localhost", "p2p", "peerid", true},
+		// wildcards
+		{"wildcard1.tld", nil, "", "", "", false},
+		{".wildcard1.tld", nil, "", "", "", false},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.wildcard1.tld", nil, "", "", "", false},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub.wildcard1.tld", gwWildcard1, "sub.wildcard1.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard1.tld", nil, "", "", "", false},
+		{"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard2.tld", gwWildcard2, "sub1.sub2.wildcard2.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true},
 	} {
 		gw, hostname, ns, rootID, ok := knownSubdomainDetails(test.hostHeader, knownGateways)
 		if ok != test.ok {
@@ -172,8 +186,8 @@ func TestKnownSubdomainDetails(t *testing.T) {
 		if hostname != test.hostname {
 			t.Errorf("knownSubdomainDetails(%s): hostname is '%s', expected '%s'", test.hostHeader, hostname, test.hostname)
 		}
-		if ok && gw.UseSubdomains != gwSpec.UseSubdomains {
-			t.Errorf("knownSubdomainDetails(%s): gw is  %+v, expected %+v", test.hostHeader, gw, gwSpec)
+		if gw != test.gw {
+			t.Errorf("knownSubdomainDetails(%s): gw is  %+v, expected %+v", test.hostHeader, gw, test.gw)
 		}
 	}
 

diff --git a/docs/config.md b/docs/config.md
@@ -586,6 +586,12 @@ Type: `array[string]`
 
 `PublicGateways` is a dictionary for defining gateway behavior on specified hostnames.
 
+Hostnames can optionally be defined with one or more wildcards.
+
+Examples:
+- `*.example.com` will match requests to `http://foo.example.com/ipfs/*` or `http://{cid}.ipfs.bar.example.com/*`.
+- `foo-*.example.com` will match requests to `http://foo-bar.example.com/ipfs/*` or `http://{cid}.ipfs.foo-xyz.example.com/*`.
+
 #### `Gateway.PublicGateways: Paths`
 
 Array of paths that should be exposed on the hostname.

diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh
@@ -766,6 +766,89 @@ test_expect_success "request for http://fake.domain.com/ipfs/{CID} with X-Forwar
   test_should_contain \"Location: https://$CIDv1.ipfs.example.com/\" response
 "
 
+## ============================================================================
+## Test support for wildcards in gateway config
+## ============================================================================
+
+# set explicit subdomain gateway config for the hostnames
+ipfs config --json Gateway.PublicGateways '{
+  "*.example1.com": {
+    "UseSubdomains": true,
+    "Paths": ["/ipfs"]
+  },
+  "*.*.example2.com": {
+    "UseSubdomains": true,
+    "Paths": ["/ipfs"]
+  },
+  "foo.*.example3.com": {
+    "UseSubdomains": true,
+    "Paths": ["/ipfs"]
+  },
+  "foo.bar-*-boo.example4.com": {
+    "UseSubdomains": true,
+    "Paths": ["/ipfs"]
+  }
+}' || exit 1
+# restart daemon to apply config changes
+test_kill_ipfs_daemon
+test_launch_ipfs_daemon --offline
+
+# *.example1.com
+
+test_hostname_gateway_response_should_contain \
+  "request for foo.example1.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.example1.com" \
+  "foo.example1.com" \
+  "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \
+  "Location: http://$CIDv1.ipfs.foo.example1.com/"
+
+test_hostname_gateway_response_should_contain \
+  "request for {CID}.ipfs.foo.example1.com should return expected payload" \
+  "${CIDv1}.ipfs.foo.example1.com" \
+  "http://127.0.0.1:$GWAY_PORT/" \
+  "$CID_VAL"
+
+# *.*.example2.com
+
+test_hostname_gateway_response_should_contain \
+  "request for foo.bar.example2.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar.example2.com" \
+  "foo.bar.example2.com" \
+  "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \
+  "Location: http://$CIDv1.ipfs.foo.bar.example2.com/"
+
+test_hostname_gateway_response_should_contain \
+  "request for {CID}.ipfs.foo.bar.example2.com should return expected payload" \
+  "${CIDv1}.ipfs.foo.bar.example2.com" \
+  "http://127.0.0.1:$GWAY_PORT/" \
+  "$CID_VAL"
+
+# foo.*.example3.com
+
+test_hostname_gateway_response_should_contain \
+  "request for foo.bar.example3.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar.example3.com" \
+  "foo.bar.example3.com" \
+  "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \
+  "Location: http://$CIDv1.ipfs.foo.bar.example3.com/"
+
+test_hostname_gateway_response_should_contain \
+  "request for {CID}.ipfs.foo.bar.example3.com should return expected payload" \
+  "${CIDv1}.ipfs.foo.bar.example3.com" \
+  "http://127.0.0.1:$GWAY_PORT/" \
+  "$CID_VAL"
+
+# foo.bar-*-boo.example4.com
+
+test_hostname_gateway_response_should_contain \
+  "request for foo.bar-dev-boo.example4.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar-dev-boo.example4.com" \
+  "foo.bar-dev-boo.example4.com" \
+  "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \
+  "Location: http://$CIDv1.ipfs.foo.bar-dev-boo.example4.com/"
+
+test_hostname_gateway_response_should_contain \
+  "request for {CID}.ipfs.foo.bar-dev-boo.example4.com should return expected payload" \
+  "${CIDv1}.ipfs.foo.bar-dev-boo.example4.com" \
+  "http://127.0.0.1:$GWAY_PORT/" \
+  "$CID_VAL"
+
 # =============================================================================
 # ensure we end with empty Gateway.PublicGateways
 ipfs config --json Gateway.PublicGateways '{}'