From 02fde38fb45c5c1e120e8228a53280029ac296a4 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 8 Aug 2024 19:41:04 +0200 Subject: [PATCH 1/3] feat(daemon): improve startup stdout It no longer makes sense to list all listener and announced multiaddrs when the node starts. Modern libp2p node will have 3-4 transports sharign the same UDP port, and also have a very long list of webtransport and webrtc direct certhashes inlined in the address. This makes startup output hard to parse, and what is worse, there is no PeerID, making printing these addresses not very useful without calling 'ipfs id' anyway. This change removes annouced and discovered multiaddrs from the stdout and replaces them with prompt to run "ipfs id". This allows user to learn about public address obtained via relays or UPnP. Second change is to replace list of listeners in multiaddr form with simpler one, that deduplicates listeners and provides node operatior with useful information about opened ports in 'host:port (tcp,udp)' format, which is way more useful during debuging connectivity issues related to port forwarding etc. --- cmd/ipfs/kubo/daemon.go | 73 ++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 5dec799dd20..54aae249994 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -10,8 +10,10 @@ import ( "net/http" _ "net/http/pprof" "os" + "regexp" "runtime" "sort" + "strings" "sync" "time" @@ -89,7 +91,7 @@ running, calls to 'ipfs' commands will be sent over the network to the daemon. `, LongDescription: ` -The daemon will start listening on ports on the network, which are +The Kubo daemon will start listening on ports on the network, which are documented in (and can be modified through) 'ipfs config Addresses'. For example, to change the 'Gateway' port: @@ -109,11 +111,11 @@ other computers in the network, use 0.0.0.0 as the ip address: Be careful if you expose the RPC API. It is a security risk, as anyone could control your node remotely. If you need to control the node remotely, make sure to protect the port as you would other services or database -(firewall, authenticated proxy, etc). +(firewall, authenticated proxy, etc), or at least set API.Authorizations. HTTP Headers -ipfs supports passing arbitrary headers to the RPC API and Gateway. You can +Kubo supports passing arbitrary headers to the RPC API and Gateway. You can do this by setting headers on the API.HTTPHeaders and Gateway.HTTPHeaders keys: @@ -124,7 +126,7 @@ Note that the value of the keys is an _array_ of strings. This is because headers can have more than one value, and it is convenient to pass through to other libraries. -CORS Headers (for API) +CORS Headers (for RPC API) You can setup CORS headers the same way: @@ -141,7 +143,7 @@ second signal. IPFS_PATH environment variable -ipfs uses a repository in the local file system. By default, the repo is +Kubo uses a repository in the local file system. By default, the repo is located at ~/.ipfs. To change the repo location, set the $IPFS_PATH environment variable: @@ -149,7 +151,7 @@ environment variable: DEPRECATION NOTICE -Previously, ipfs used an environment variable as seen below: +Previously, Kubo used an environment variable as seen below: export API_ORIGIN="http://localhost:8888/" @@ -160,14 +162,14 @@ Headers. }, Options: []cmds.Option{ - cmds.BoolOption(initOptionKwd, "Initialize ipfs with default settings if not already initialized"), + cmds.BoolOption(initOptionKwd, "Initialize Kubo with default settings if not already initialized"), cmds.StringOption(initConfigOptionKwd, "Path to existing configuration file to be loaded during --init"), cmds.StringOption(initProfileOptionKwd, "Configuration profiles to apply for --init. See ipfs init --help for more"), cmds.StringOption(routingOptionKwd, "Overrides the routing option").WithDefault(routingOptionDefaultKwd), cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem using FUSE (experimental)"), cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."), cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."), - cmds.BoolOption(unrestrictedAPIAccessKwd, "Allow API access to unlisted hashes"), + cmds.BoolOption(unrestrictedAPIAccessKwd, "Allow RPC API access to unlisted hashes"), cmds.BoolOption(unencryptTransportKwd, "Disable transport encryption (for debugging protocols)"), cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection"), cmds.BoolOption(adjustFDLimitKwd, "Check and raise file descriptor limits if needed").WithDefault(true), @@ -373,6 +375,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return err } + fmt.Printf("PeerID: %s\n", cfg.Identity.PeerID) + if !psSet { pubsub = cfg.Pubsub.Enabled.WithDefault(false) } @@ -463,7 +467,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)") } - printSwarmAddrs(node) + printLibp2pPorts(node) if node.PrivateKey.Type() == p2pcrypto.RSA { fmt.Print(` @@ -563,7 +567,7 @@ take effect. // Add ipfs version info to prometheus metrics ipfsInfoMetric := promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "ipfs_info", - Help: "IPFS version information.", + Help: "Kubo IPFS version information.", }, []string{"version", "commit"}) // Setting to 1 lets us multiply it with other stats to add the version labels @@ -776,8 +780,8 @@ func rewriteMaddrToUseLocalhostIfItsAny(maddr ma.Multiaddr) ma.Multiaddr { } } -// printSwarmAddrs prints the addresses of the host. -func printSwarmAddrs(node *core.IpfsNode) { +// printLibp2pPorts prints which ports are opened to facilitate swarm connectivity. +func printLibp2pPorts(node *core.IpfsNode) { if !node.IsOnline { fmt.Println("Swarm not listening, running in offline mode.") return @@ -787,24 +791,39 @@ func printSwarmAddrs(node *core.IpfsNode) { if err != nil { log.Errorf("failed to read listening addresses: %s", err) } - lisAddrs := make([]string, len(ifaceAddrs)) - for i, addr := range ifaceAddrs { - lisAddrs[i] = addr.String() - } - sort.Strings(lisAddrs) - for _, addr := range lisAddrs { - fmt.Printf("Swarm listening on %s\n", addr) - } - nodePhostAddrs := node.PeerHost.Addrs() - addrs := make([]string, len(nodePhostAddrs)) - for i, addr := range nodePhostAddrs { - addrs[i] = addr.String() + // Multiple libp2p transports can use same port. + // Deduplicate all listeners and collect unique IP:port (udp|tcp) combinations + // which is useful information for operator deploying Kubo in TCP/IP infra. + addrMap := make(map[string]map[string]struct{}) + re := regexp.MustCompile(`^/(?:ip[46]|dns(?:[46])?)/([^/]+)/(tcp|udp)/(\d+)(/.*)?$`) + for _, addr := range ifaceAddrs { + matches := re.FindStringSubmatch(addr.String()) + if matches != nil { + hostname := matches[1] + protocol := strings.ToUpper(matches[2]) + port := matches[3] + var host string + if matches[0][:4] == "/ip6" { + host = fmt.Sprintf("[%s]:%s", hostname, port) + } else { + host = fmt.Sprintf("%s:%s", hostname, port) + } + if _, ok := addrMap[host]; !ok { + addrMap[host] = make(map[string]struct{}) + } + addrMap[host][protocol] = struct{}{} + } } - sort.Strings(addrs) - for _, addr := range addrs { - fmt.Printf("Swarm announcing %s\n", addr) + for host, protocolsSet := range addrMap { + protocols := make([]string, 0, len(protocolsSet)) + for protocol := range protocolsSet { + protocols = append(protocols, protocol) + } + sort.Strings(protocols) + fmt.Printf("Swarm listening on %s (%s)\n", host, strings.Join(protocols, ",")) } + fmt.Printf("Run 'ipfs id' to inspect announced and discovered multiaddrs of this node.\n") } // serveHTTPGateway collects options, creates listener, prints status message and starts serving requests. From 222a403fd360f9032d8db01eac4c20084a777e1d Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 8 Aug 2024 21:14:50 +0200 Subject: [PATCH 2/3] docs: ipfs daemon stdout cleanup --- docs/changelogs/v0.30.md | 29 +++++++++++++++++++++++++++++ test/sharness/t0060-daemon.sh | 21 ++++++++++----------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/docs/changelogs/v0.30.md b/docs/changelogs/v0.30.md index dde03cd2594..06f8633b188 100644 --- a/docs/changelogs/v0.30.md +++ b/docs/changelogs/v0.30.md @@ -10,6 +10,7 @@ - [AutoNAT V2 Service Introduced Alongside V1](#autonat-v2-service-introduced-alongside-v1) - [Automated `ipfs version check`](#automated-ipfs-version-check) - [Version Suffix Configuration](#version-suffix-configuration) + - [Cleaned Up `ipfs daemon` Startup Log](#cleaned-up-ipfs-daemon-startup-log) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -48,6 +49,34 @@ Defining the optional agent version suffix is now simpler. The [`Version.AgentSu > [!NOTE] > Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at https://stats.ipfs.network +> + +#### Cleaned Up `ipfs daemon` Startup Log + +The `ipfs daemon` startup output has been streamlined to enhance clarity and usability: + +```console +$ ipfs daemon +Initializing daemon... +Kubo version: 0.30.0 +Repo version: 16 +System version: amd64/linux +Golang version: go1.22.5 +PeerID: 12D3KooWQ73s1CQsm4jWwQvdCAtc5w8LatyQt7QLQARk5xdhK9CE +Swarm listening on 127.0.0.1:4001 (TCP,UDP) +Swarm listening on 192.0.2.10:4001 (TCP,UDP) +Swarm listening on [::1]:4001 (TCP,UDP) +Swarm listening on [2001:0db8::10]:4001 (TCP,UDP) +Run 'ipfs id' to inspect announced and discovered multiaddrs of this node. +RPC API server listening on /ip4/127.0.0.1/tcp/5001 +WebUI: http://127.0.0.1:5001/webui +Gateway server listening on /ip4/127.0.0.1/tcp/8080 +Daemon is ready +``` + +The previous lengthy listing of all listener and announced multiaddrs has been removed due to its complexity, especially with modern libp2p nodes sharing multiple transports and long lists of `/webtransport` and `/webrtc-direct` certhashes. +The output now features a simplified list of swarm listeners, displayed in the format `host:port (TCP,UDP)`, which provides essential information for debugging connectivity issues, particularly related to port forwarding. +Announced libp2p addresses are no longer printed on startup, because libp2p may change or augument them based on AutoNAT, relay, and UPnP state. Instead, users are prompted to run `ipfs id` to obtain up-to-date list of listeners and announced multiaddrs in libp2p format. ### ๐Ÿ“ Changelog diff --git a/test/sharness/t0060-daemon.sh b/test/sharness/t0060-daemon.sh index 29474c7ffd0..431ff245ca3 100755 --- a/test/sharness/t0060-daemon.sh +++ b/test/sharness/t0060-daemon.sh @@ -76,17 +76,16 @@ test_expect_success "ipfs gateway works with the correct allowed origin port" ' curl -s -X POST -H "Origin:http://localhost:$GWAY_PORT" -I "http://$GWAY_ADDR/api/v0/version" ' -test_expect_success "ipfs daemon output looks good" ' - STARTFILE="ipfs cat /ipfs/$HASH_WELCOME_DOCS/readme" && - echo "Initializing daemon..." >expected_daemon && - ipfs version --all >> expected_daemon && - sed "s/^/Swarm listening on /" listen_addrs >>expected_daemon && - sed "s/^/Swarm announcing /" local_addrs >>expected_daemon && - echo "RPC API server listening on '$API_MADDR'" >>expected_daemon && - echo "WebUI: http://'$API_ADDR'/webui" >>expected_daemon && - echo "Gateway server listening on '$GWAY_MADDR'" >>expected_daemon && - echo "Daemon is ready" >>expected_daemon && - test_cmp expected_daemon actual_daemon +test_expect_success "ipfs daemon output includes looks good" ' + test_should_contain "Initializing daemon..." actual_daemon && + test_should_contain "$(ipfs version --all)" actual_daemon && + test_should_contain "PeerID: $(ipfs config Identity.PeerID)" actual_daemon && + test_should_contain "Swarm listening on 127.0.0.1:" actual_daemon && + test_should_contain "RPC API server listening on '$API_MADDR'" actual_daemon && + test_should_contain "WebUI: http://'$API_ADDR'/webui" actual_daemon && + test_should_contain "Gateway server listening on '$GWAY_MADDR'" actual_daemon && + test_should_contain "Daemon is ready" actual_daemon && + cat actual_daemon ' test_expect_success ".ipfs/ has been created" ' From 7b2d58ed8f3c858b052be118316c97d914a8e1e2 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 13 Aug 2024 16:23:45 +0200 Subject: [PATCH 3/3] chore: use + as separator subjective, but looks better than , --- cmd/ipfs/kubo/daemon.go | 2 +- docs/changelogs/v0.30.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 54aae249994..1c5b2e9536b 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -821,7 +821,7 @@ func printLibp2pPorts(node *core.IpfsNode) { protocols = append(protocols, protocol) } sort.Strings(protocols) - fmt.Printf("Swarm listening on %s (%s)\n", host, strings.Join(protocols, ",")) + fmt.Printf("Swarm listening on %s (%s)\n", host, strings.Join(protocols, "+")) } fmt.Printf("Run 'ipfs id' to inspect announced and discovered multiaddrs of this node.\n") } diff --git a/docs/changelogs/v0.30.md b/docs/changelogs/v0.30.md index 06f8633b188..314e5a2e08e 100644 --- a/docs/changelogs/v0.30.md +++ b/docs/changelogs/v0.30.md @@ -63,10 +63,10 @@ Repo version: 16 System version: amd64/linux Golang version: go1.22.5 PeerID: 12D3KooWQ73s1CQsm4jWwQvdCAtc5w8LatyQt7QLQARk5xdhK9CE -Swarm listening on 127.0.0.1:4001 (TCP,UDP) -Swarm listening on 192.0.2.10:4001 (TCP,UDP) -Swarm listening on [::1]:4001 (TCP,UDP) -Swarm listening on [2001:0db8::10]:4001 (TCP,UDP) +Swarm listening on 127.0.0.1:4001 (TCP+UDP) +Swarm listening on 192.0.2.10:4001 (TCP+UDP) +Swarm listening on [::1]:4001 (TCP+UDP) +Swarm listening on [2001:0db8::10]:4001 (TCP+UDP) Run 'ipfs id' to inspect announced and discovered multiaddrs of this node. RPC API server listening on /ip4/127.0.0.1/tcp/5001 WebUI: http://127.0.0.1:5001/webui @@ -75,7 +75,7 @@ Daemon is ready ``` The previous lengthy listing of all listener and announced multiaddrs has been removed due to its complexity, especially with modern libp2p nodes sharing multiple transports and long lists of `/webtransport` and `/webrtc-direct` certhashes. -The output now features a simplified list of swarm listeners, displayed in the format `host:port (TCP,UDP)`, which provides essential information for debugging connectivity issues, particularly related to port forwarding. +The output now features a simplified list of swarm listeners, displayed in the format `host:port (TCP+UDP)`, which provides essential information for debugging connectivity issues, particularly related to port forwarding. Announced libp2p addresses are no longer printed on startup, because libp2p may change or augument them based on AutoNAT, relay, and UPnP state. Instead, users are prompted to run `ipfs id` to obtain up-to-date list of listeners and announced multiaddrs in libp2p format. ### ๐Ÿ“ Changelog