feat: support subdomains in isIPFS.url(url) (#32)

* feat: support DNSLink subdomains

This change adds support for DNSLink subdomains on localhost gateway
(ipfs/kubo#6096)

Example: en.wikipedia-on-ipfs.org.ipfs.localhost:8080

BREAKING CHANGE: `isIPFS.subdomain` now returns true for <domain.tld>.ipns.localhost
BREAKING CHANGE: `isIPFS.subdomainPattern` changed

* test: support peer multiaddr with /p2p/

Context: libp2p/libp2p#79

* fix: explicitly ignore URL param and hash

.url and .path now return true when validating:
https://ipfs.io/ipfs/<CID>?filename=name.png#foo

* refactor: simplify dnslinkSubdomain

License: MIT
Signed-off-by: Marcin Rataj <[email protected]>

* fix: url() check should include subdomain()

When .url was created we only had path gateways.  When .subdomain was
added, we did not update .url to test for subdomain gateways, which in
the long run will confuse people and feels like a bug.

Let's fix this: .url() will now check for both subdomain and path gateways

#32 (comment)

BREAKING CHANGE: .url(url) now returns true if .subdomain(url) is true

* refactor: merge DNSLink check into ipnsSubdomain()

This makes subdomain checks follow what path gateway checks do, removing
confusion.

In both cases (IPNS and DNSLink) user needs to perform online record
check, so this is just a handy way of detecting potential matches.

* docs: update examples
* refactor: switch to iso-url
* refactor: lint-package-json
* chore: update deps

License: MIT
Signed-off-by: Marcin Rataj <[email protected]>

.gitignore

@@ -38,3 +38,4 @@ node_modules
 
 dist
 lib
+docs

README.md

@@ -23,7 +23,7 @@ $ npm install --save is-ipfs
 The code published to npm that gets loaded on require is in fact an ES5 transpiled version with the right shims added. This means that you can require it and use with your favorite bundler without having to adjust asset management process.
 
 ```js
-var isIPFS = require('is-ipfs')
+const isIPFS = require('is-ipfs')
 ```
 
 
@@ -53,18 +53,23 @@ isIPFS.base32cid('bafybeie5gq4jxvzmsym6hjlwxej4rwdoxt7wadqvmmwbqi7r27fclha2va')
 isIPFS.base32cid('QmYjtig7VJQ6XsnUjqqJvj7QaMcCAwtrgNdahSiFofrE7o') // false
 
 isIPFS.url('https://ipfs.io/ipfs/QmYjtig7VJQ6XsnUjqqJvj7QaMcCAwtrgNdahSiFofrE7o') // true
+isIPFS.url('https://ipfs.io/ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR?filename=guardian.jpg') // true
 isIPFS.url('https://ipfs.io/ipns/github.com') // true
+isIPFS.url('https://bafybeie5gq4jxvzmsym6hjlwxej4rwdoxt7wadqvmmwbqi7r27fclha2va.ipfs.dweb.link') // true
+isIPFS.url('http://en.wikipedia-on-ipfs.org.ipfs.localhost:8080') // true
 isIPFS.url('https://github.com/ipfs/js-ipfs/blob/master/README.md') // false
 isIPFS.url('https://google.com') // false
 
 isIPFS.path('/ipfs/QmYjtig7VJQ6XsnUjqqJvj7QaMcCAwtrgNdahSiFofrE7o') // true
+isIPFS.path('/ipfs/QmbcBPAwCDxRMB1Qe7CRQmxdrTSkxKwM9y6rZw2FjGtbsb/?weird-filename=test.jpg') // true
 isIPFS.path('/ipns/github.com') // true
 isIPFS.path('/ipfs/js-ipfs/blob/master/README.md') // false
 
 isIPFS.urlOrPath('https://ipfs.io/ipfs/QmYjtig7VJQ6XsnUjqqJvj7QaMcCAwtrgNdahSiFofrE7o') // true
 isIPFS.urlOrPath('https://ipfs.io/ipns/github.com') // true
 isIPFS.urlOrPath('/ipfs/QmYjtig7VJQ6XsnUjqqJvj7QaMcCAwtrgNdahSiFofrE7o') // true
 isIPFS.urlOrPath('/ipns/github.com') // true
+isIPFS.urlOrPath('https://bafybeie5gq4jxvzmsym6hjlwxej4rwdoxt7wadqvmmwbqi7r27fclha2va.ipfs.dweb.link') // true
 isIPFS.urlOrPath('https://google.com') // false
 
 isIPFS.ipfsUrl('https://ipfs.io/ipfs/QmYjtig7VJQ6XsnUjqqJvj7QaMcCAwtrgNdahSiFofrE7o') // true
@@ -96,17 +101,19 @@ isIPFS.ipfsSubdomain('http://bafybeie5gq4jxvzmsym6hjlwxej4rwdoxt7wadqvmmwbqi7r27
 isIPFS.ipnsSubdomain('http://bafybeiabc2xofh6tdi6vutusorpumwcikw3hf3st4ecjugo6j52f6xwc6q.ipns.dweb.link') // true
 isIPFS.ipnsSubdomain('http://bafybeiabc2xofh6tdi6vutusorpumwcikw3hf3st4ecjugo6j52f6xwc6q.dweb.link') // false
 isIPFS.ipnsSubdomain('http://QmcNioXSC1bfJj1dcFErhUfyjFzoX2HodkRccsFFVJJvg8.ipns.dweb.link') // false
-isIPFS.ipnsSubdomain('http://foo-bar.ipns.dweb.link') // false (not a PeerID)
+isIPFS.ipnsSubdomain('http://en.wikipedia-on-ipfs.org.ipns.localhost:8080') // true (assuming DNSLink)
+isIPFS.ipnsSubdomain('http://hostname-without-tld.ipns.dweb.link') // false (missing TLD)
 
 isIPFS.multiaddr('/ip4/127.0.0.1/udp/1234') // true
 isIPFS.multiaddr('/ip4/127.0.0.1/udp/1234/http') // true
 isIPFS.multiaddr('/ip6/::1/udp/1234') // true
 isIPFS.multiaddr('ip6/::1/udp/1234') // false
 isIPFS.multiaddr('/yoloinvalid/::1/udp/1234') // false
 
-isIPFS.peerMultiaddr('/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4') // true
-isIPFS.peerMultiaddr('/ip4/127.0.0.1/tcp/1234/ws/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj') // true
-isIPFS.peerMultiaddr('/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4/p2p-circuit/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj') // true
+isIPFS.peerMultiaddr('/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4') // true
+isIPFS.peerMultiaddr('/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4') // true (legacy notation)
+isIPFS.peerMultiaddr('/ip4/127.0.0.1/tcp/1234/ws/p2p/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj') // true
+isIPFS.peerMultiaddr('/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4/p2p-circuit/p2p/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj') // true
 isIPFS.peerMultiaddr('/ip4/127.0.0.1/udp/1234') // false
 ```
 
@@ -115,10 +122,9 @@ isIPFS.peerMultiaddr('/ip4/127.0.0.1/udp/1234') // false
 A suite of util methods that provides efficient validation.
 
 Detection of IPFS Paths and identifiers in URLs is a two-stage process:
-1.  `urlPattern`/`pathPattern`/`subdomainPattern` regex is applied to quickly identify potential candidates
+1.  `pathPattern`/`pathGatewayPattern`/`subdomainGatewayPattern` regex is applied to quickly identify potential candidates
 2.  proper CID validation is applied to remove false-positives
 
-
 ## Content Identifiers
 
 ### `isIPFS.multihash(hash)`
@@ -178,15 +184,25 @@ Validated subdomain convention: `cidv1b32.ip(f|n)s.domain.tld`
 
 ### `isIPFS.subdomain(url)`
 
-Returns `true` if the provided string includes a valid IPFS or IPNS subdomain or `false` otherwise.
+Returns `true` if the provided `url` string includes a valid IPFS, looks like IPNS/DNSLink subdomain or `false` otherwise.
 
 ### `isIPFS.ipfsSubdomain(url)`
 
-Returns `true` if the provided string includes a valid IPFS subdomain or `false` otherwise.
+Returns `true` if the provided `url` string includes a valid IPFS subdomain (case-insensitive CIDv1) or `false` otherwise.
 
 ### `isIPFS.ipnsSubdomain(url)`
 
-Returns `true` if the provided string includes a valid IPNS subdomain or `false` otherwise.
+Returns `true` if the provided `url` string looks like a valid IPNS subdomain
+(CIDv1 with `libp2p-key` multicodec or something that looks like a FQDN, for example `en.wikipedia-on-ipfs.org.ipns.localhost:8080`) or `false`
+otherwise.
+
+**Note:** `ipnsSubdomain` method works in offline mode: it does not perform
+actual IPNS record lookup over DHT or other content routing method. It may
+return false-positives:
+
+- To ensure IPNS record  exists, make a call to `/api/v0/name/resolve?arg=<ipnsid>`
+- To ensure DNSLink exists, make a call to `/api/v0/dns?arg=<fqdn>`
+
 
 ## Multiaddrs
 

package.json

@@ -1,55 +1,67 @@
 {
   "name": "is-ipfs",
   "version": "0.6.3",
-  "description": "A set of utilities to help identify IPFS resources",
+  "description": "A set of utilities to help identify IPFS resources on the web",
+  "keywords": [
+    "js-ipfs",
+    "ipns",
+    "gateway",
+    "dnslink",
+    "ipfs"
+  ],
+  "homepage": "https://github.com/ipfs/is-ipfs",
+  "bugs": {
+    "url": "https://github.com/ipfs/is-ipfs/issues"
+  },
+  "license": "MIT",
+  "author": "Francisco Dias <[email protected]> (http://franciscodias.net/)",
   "leadMaintainer": "Marcin Rataj <[email protected]>",
+  "files": [
+    "src",
+    "dist"
+  ],
   "main": "src/index.js",
   "browser": {
     "fs": false
   },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ipfs/is-ipfs.git"
+  },
   "scripts": {
     "test:node": "aegir test --target node",
     "test:browser": "aegir test --target browser",
     "test": "aegir test",
-    "lint": "aegir lint",
+    "lint": "aegir lint && aegir lint-package-json",
     "release": "aegir release",
     "release-minor": "aegir release --type minor",
     "release-major": "aegir release --type major",
     "build": "aegir build",
     "coverage": "aegir coverage",
     "coverage-publish": "aegir coverage --upload"
   },
-  "pre-commit": [
-    "test",
-    "lint"
-  ],
-  "keywords": [
-    "js-ipfs",
-    "ipfs"
-  ],
-  "author": "Francisco Dias <[email protected]> (http://franciscodias.net/)",
-  "license": "MIT",
   "dependencies": {
     "bs58": "^4.0.1",
-    "cids": "~0.7.0",
-    "mafmt": "^7.0.0",
-    "multiaddr": "^7.2.1",
-    "multibase": "~0.6.0",
-    "multihashes": "~0.4.13"
+    "cids": "~0.8.0",
+    "iso-url": "~0.4.7",
+    "mafmt": "^7.1.0",
+    "multiaddr": "^7.4.3",
+    "multibase": "~0.7.0",
+    "multihashes": "~0.4.19"
   },
   "devDependencies": {
-    "aegir": "^20.5.0",
+    "aegir": "^21.4.5",
     "chai": "^4.2.0",
     "pre-commit": "^1.2.2"
   },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/ipfs/is-ipfs.git"
-  },
-  "bugs": {
-    "url": "https://github.com/ipfs/is-ipfs/issues"
+  "engines": {
+    "node": ">=10.0.0",
+    "npm": ">=6.0.0"
   },
-  "homepage": "https://github.com/ipfs/is-ipfs",
+  "pre-commit": [
+    "test",
+    "lint"
+  ],
   "contributors": [
     "Alan Shaw <[email protected]>",
     "David Dias <[email protected]>",

src/index.js

@@ -6,15 +6,20 @@ const multibase = require('multibase')
 const Multiaddr = require('multiaddr')
 const mafmt = require('mafmt')
 const CID = require('cids')
+const { URL } = require('iso-url')
 
-const urlPattern = /^https?:\/\/[^/]+\/(ip(f|n)s)\/((\w+).*)/
-const pathPattern = /^\/(ip(f|n)s)\/((\w+).*)/
+const pathGatewayPattern = /^https?:\/\/[^/]+\/(ip[fn]s)\/([^/?#]+)/
+const pathPattern = /^\/(ip[fn]s)\/([^/?#]+)/
 const defaultProtocolMatch = 1
-const defaultHashMath = 4
+const defaultHashMath = 2
 
-const fqdnPattern = /^https?:\/\/([^/]+)\.(ip(?:f|n)s)\.[^/]+/
-const fqdnHashMatch = 1
-const fqdnProtocolMatch = 2
+// CID, libp2p-key or DNSLink
+const subdomainGatewayPattern = /^https?:\/\/([^/]+)\.(ip[fn]s)\.[^/?]+/
+const subdomainIdMatch = 1
+const subdomainProtocolMatch = 2
+
+// Fully qualified domain name (FQDN) that has an explicit .tld suffix
+const fqdnWithTld = /^(([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])$/
 
 function isMultihash (hash) {
   const formatted = convertToString(hash)
@@ -76,7 +81,7 @@ function isIpfs (input, pattern, protocolMatch = defaultProtocolMatch, hashMatch
 
   let hash = match[hashMatch]
 
-  if (hash && pattern === fqdnPattern) {
+  if (hash && pattern === subdomainGatewayPattern) {
     // when doing checks for subdomain context
     // ensure hash is case-insensitive
     // (browsers force-lowercase authority compotent anyway)
@@ -100,13 +105,24 @@ function isIpns (input, pattern, protocolMatch = defaultProtocolMatch, hashMatch
     return false
   }
 
-  if (hashMatch && pattern === fqdnPattern) {
-    let hash = match[hashMatch]
+  let ipnsId = match[hashMatch]
+
+  if (ipnsId && pattern === subdomainGatewayPattern) {
     // when doing checks for subdomain context
-    // ensure hash is case-insensitive
+    // ensure ipnsId is case-insensitive
     // (browsers force-lowercase authority compotent anyway)
-    hash = hash.toLowerCase()
-    return isCID(hash)
+    ipnsId = ipnsId.toLowerCase()
+    // Check if it is cidv1
+    if (isCID(ipnsId)) return true
+    // Check if it looks like FQDN
+    try {
+      // URL implementation in web browsers forces lowercase of the hostname
+      const { hostname } = new URL(`http://${ipnsId}`) // eslint-disable-line no-new
+      // Check if potential FQDN has an explicit TLD
+      return fqdnWithTld.test(hostname)
+    } catch (e) {
+      return false
+    }
   }
 
   return true
@@ -128,27 +144,34 @@ function convertToString (input) {
   return false
 }
 
-const ipfsSubdomain = (url) => isIpfs(url, fqdnPattern, fqdnProtocolMatch, fqdnHashMatch)
-const ipnsSubdomain = (url) => isIpns(url, fqdnPattern, fqdnProtocolMatch, fqdnHashMatch)
+const ipfsSubdomain = (url) => isIpfs(url, subdomainGatewayPattern, subdomainProtocolMatch, subdomainIdMatch)
+const ipnsSubdomain = (url) => isIpns(url, subdomainGatewayPattern, subdomainProtocolMatch, subdomainIdMatch)
+const subdomain = (url) => ipfsSubdomain(url) || ipnsSubdomain(url)
+
+const ipfsUrl = (url) => isIpfs(url, pathGatewayPattern) || ipfsSubdomain(url)
+const ipnsUrl = (url) => isIpns(url, pathGatewayPattern) || ipnsSubdomain(url)
+const url = (url) => ipfsUrl(url) || ipnsUrl(url) || subdomain(url)
+
+const path = (path) => isIpfs(path, pathPattern) || isIpns(path, pathPattern)
 
 module.exports = {
   multihash: isMultihash,
   multiaddr: isMultiaddr,
   peerMultiaddr: isPeerMultiaddr,
   cid: isCID,
   base32cid: (cid) => (isMultibase(cid) === 'base32' && isCID(cid)),
-  ipfsSubdomain: ipfsSubdomain,
-  ipnsSubdomain: ipnsSubdomain,
-  subdomain: (url) => (ipfsSubdomain(url) || ipnsSubdomain(url)),
-  subdomainPattern: fqdnPattern,
-  ipfsUrl: (url) => isIpfs(url, urlPattern),
-  ipnsUrl: (url) => isIpns(url, urlPattern),
-  url: (url) => (isIpfs(url, urlPattern) || isIpns(url, urlPattern)),
-  urlPattern: urlPattern,
+  ipfsSubdomain,
+  ipnsSubdomain,
+  subdomain,
+  subdomainGatewayPattern,
+  ipfsUrl,
+  ipnsUrl,
+  url,
+  pathGatewayPattern: pathGatewayPattern,
   ipfsPath: (path) => isIpfs(path, pathPattern),
   ipnsPath: (path) => isIpns(path, pathPattern),
-  path: (path) => (isIpfs(path, pathPattern) || isIpns(path, pathPattern)),
-  pathPattern: pathPattern,
-  urlOrPath: (x) => (isIpfs(x, urlPattern) || isIpns(x, urlPattern) || isIpfs(x, pathPattern) || isIpns(x, pathPattern)),
+  path,
+  pathPattern,
+  urlOrPath: (x) => url(x) || path(x),
   cidPath: path => isString(path) && !isCID(path) && isIpfs(`/ipfs/${path}`, pathPattern)
 }

test/test-multiaddr.spec.js

@@ -67,19 +67,27 @@ describe('ipfs peerMultiaddr', () => {
   // https://github.com/multiformats/js-mafmt/blob/v6.0.6/test/index.spec.js#L137
   const goodCircuit = [
     '/p2p-circuit',
-    '/p2p-circuit/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj',
+    '/p2p-circuit/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj', // /ipfs/ is legacy notation replaced with /p2p/
+    '/p2p-circuit/p2p/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj',
     '/p2p-circuit/ip4/127.0.0.1/tcp/20008/ws/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj',
+    '/p2p-circuit/ip4/127.0.0.1/tcp/20008/ws/p2p/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj',
     '/p2p-circuit/ip4/1.2.3.4/tcp/3456/ws/p2p-webrtc-star/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4',
+    '/p2p-circuit/ip4/1.2.3.4/tcp/3456/ws/p2p-webrtc-star/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4',
+    '/p2p-circuit/ip4/1.2.3.4/tcp/3456/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4',
     '/p2p-circuit/ip4/1.2.3.4/tcp/3456/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4',
     '/p2p-circuit/ip4/127.0.0.1/tcp/4002/ipfs/QmddWMcQX6orJGHpETYMyPgXrCXCtYANMFVDCvhKoDwLqA',
     '/p2p-circuit/ipfs/QmddWMcQX6orJGHpETYMyPgXrCXCtYANMFVDCvhKoDwLqA',
+    '/p2p-circuit/p2p/QmddWMcQX6orJGHpETYMyPgXrCXCtYANMFVDCvhKoDwLqA',
     '/p2p-circuit/ip4/127.0.0.1/tcp/20008/ws/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj/' +
     'p2p-circuit/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj'
   ]
   // https://github.com/multiformats/js-mafmt/blob/v6.0.6/test/index.spec.js#L157
   const validPeerMultiaddrs = [
     '/ip4/127.0.0.1/tcp/20008/ws/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj',
+    '/ip4/127.0.0.1/tcp/20008/ws/p2p/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj',
+    '/ip4/127.0.0.1/tcp/20008/ws/p2p/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5', // ed25519+identity multihash
     '/ip4/1.2.3.4/tcp/3456/ws/p2p-webrtc-star/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4',
+    '/ip4/1.2.3.4/tcp/3456/ws/p2p-webrtc-star/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4',
     '/ip4/1.2.3.4/tcp/3456/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4',
     '/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4/p2p-circuit',
     '/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSoooo4/p2p-circuit/ipfs/QmUjNmr8TgJCn1Ao7DvMy4cjoZU15b9bwSCBLE3vwXiwgj'

test/test-path.spec.js

@@ -7,13 +7,13 @@ const expect = require('chai').expect
 
 describe('ipfs path', () => {
   it('isIPFS.ipfsPath should match an ipfs path', (done) => {
-    const actual = isIPFS.ipfsPath('/ipfs/QmYHNYAaYK5hm3ZhZFx5W9H6xydKDGimjdgJMrMSdnctEm')
+    const actual = isIPFS.ipfsPath('/ipfs/QmYHNYAaYK5hm3ZhZFx5W9H6xydKDGimjdgJMrMSdnctEm?arg=val#hash')
     expect(actual).to.equal(true)
     done()
   })
 
   it('isIPFS.ipfsPath should match a complex ipfs path', (done) => {
-    const actual = isIPFS.ipfsPath('/ipfs/QmeWz9YZEeNFXQhHg4PnR5ZiNr5isttgi5n1tc1eD5EfGU/content/index.html')
+    const actual = isIPFS.ipfsPath('/ipfs/QmeWz9YZEeNFXQhHg4PnR5ZiNr5isttgi5n1tc1eD5EfGU/content/index.html?arg=val#hash')
     expect(actual).to.equal(true)
     done()
   })