From 1e8e65db29fb533dc5701a6ceed040a5009839e5 Mon Sep 17 00:00:00 2001 From: ian Date: Mon, 16 Sep 2024 13:31:14 +0100 Subject: [PATCH 1/2] Allow adding explicit announce addresses (e.g. an external NAT address) --- src/main/java/org/peergos/EmbeddedIpfs.java | 17 ++++++++++++----- .../org/peergos/protocol/IdentifyBuilder.java | 4 ++-- src/test/java/org/peergos/APIServiceTest.java | 4 ++-- .../java/org/peergos/BitswapMirrorTest.java | 2 +- src/test/java/org/peergos/BootstrapTest.java | 2 +- src/test/java/org/peergos/FindPeerTest.java | 2 +- src/test/java/org/peergos/HandlerTest.java | 6 +++--- src/test/java/org/peergos/HttpProxyTest.java | 4 +--- src/test/java/org/peergos/IpnsTest.java | 4 ++-- src/test/java/org/peergos/KademliaTest.java | 8 ++++---- .../java/org/peergos/KuboFindProviderTest.java | 2 +- src/test/java/org/peergos/KuboPingTest.java | 2 +- src/test/java/org/peergos/KuboTest.java | 2 +- src/test/java/org/peergos/PingTest.java | 8 ++++---- 14 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/peergos/EmbeddedIpfs.java b/src/main/java/org/peergos/EmbeddedIpfs.java index 8eebb3bf..321127b7 100644 --- a/src/main/java/org/peergos/EmbeddedIpfs.java +++ b/src/main/java/org/peergos/EmbeddedIpfs.java @@ -52,6 +52,7 @@ public class EmbeddedIpfs { public final Optional p2pHttp; private final List bootstrap; private final Optional blockProvider; + private final List announce; private final List mdns = new ArrayList<>(); public EmbeddedIpfs(Host node, @@ -61,7 +62,8 @@ public EmbeddedIpfs(Host node, Bitswap bitswap, Optional p2pHttp, List bootstrap, - Optional> newBlockProvider) { + Optional> newBlockProvider, + List announce) { this.node = node; this.blockstore = blockstore; this.records = records; @@ -72,6 +74,7 @@ public EmbeddedIpfs(Host node, this.blocks = new BitswapBlockService(node, bitswap, dht); this.blockProvider = newBlockProvider.map(q -> new PeriodicBlockProvider(22 * 3600_000L, () -> blockstore.refs(false).join().stream(), node, dht, q)); + this.announce = announce; } public int maxBlockSize() { @@ -147,7 +150,10 @@ public void start() { }); Runtime.getRuntime().addShutdownHook(shutdownHook); node.start().join(); - IdentifyBuilder.addIdentifyProtocol(node); + IdentifyBuilder.addIdentifyProtocol(node, announce.stream() + .map(MultiAddress::toString) + .map(Multiaddr::new) + .collect(Collectors.toList())); LOG.info("Node started and listening on " + node.listenAddresses()); LOG.info("Bootstrapping IPFS routing table"); if (bootstrap.isEmpty()) @@ -246,8 +252,8 @@ public static EmbeddedIpfs build(RecordStore records, IdentitySection identity, BlockRequestAuthoriser authoriser, Optional handler) { - return build(records, blocks, provideBlocks, swarmAddresses, bootstrap, identity, authoriser, handler, - Optional.empty(), Optional.empty()); + return build(records, blocks, provideBlocks, swarmAddresses, bootstrap, identity, Collections.emptyList(), + authoriser, handler, Optional.empty(), Optional.empty()); } public static EmbeddedIpfs build(RecordStore records, @@ -256,6 +262,7 @@ public static EmbeddedIpfs build(RecordStore records, List swarmAddresses, List bootstrap, IdentitySection identity, + List announce, BlockRequestAuthoriser authoriser, Optional handler, Optional bitswapProtocolId, @@ -291,7 +298,7 @@ public static EmbeddedIpfs build(RecordStore records, Optional> newBlockProvider = provideBlocks ? Optional.of(((ProvidingBlockstore)blockstore).toPublish) : Optional.empty(); - return new EmbeddedIpfs(node, blockstore, records, dht, bitswap, httpHandler, bootstrap, newBlockProvider); + return new EmbeddedIpfs(node, blockstore, records, dht, bitswap, httpHandler, bootstrap, newBlockProvider, announce); } public static Multiaddr[] getAddresses(Host node, Kademlia dht, Multihash targetNodeId) throws ConnectionException { diff --git a/src/main/java/org/peergos/protocol/IdentifyBuilder.java b/src/main/java/org/peergos/protocol/IdentifyBuilder.java index 402120c6..78a887f9 100644 --- a/src/main/java/org/peergos/protocol/IdentifyBuilder.java +++ b/src/main/java/org/peergos/protocol/IdentifyBuilder.java @@ -15,12 +15,12 @@ public class IdentifyBuilder { - public static void addIdentifyProtocol(Host node) { + public static void addIdentifyProtocol(Host node, List announceAddresses) { IdentifyOuterClass.Identify.Builder identifyBuilder = IdentifyOuterClass.Identify.newBuilder() .setProtocolVersion("ipfs/0.1.0") .setAgentVersion("nabu/v0.1.0") .setPublicKey(ByteArrayExtKt.toProtobuf(node.getPrivKey().publicKey().bytes())) - .addAllListenAddrs(node.listenAddresses().stream() + .addAllListenAddrs(Stream.concat(node.listenAddresses().stream(), announceAddresses.stream()) .flatMap(a -> expandWildcardAddresses(a).stream()) .map(Multiaddr::serialize) .map(ByteArrayExtKt::toProtobuf) diff --git a/src/test/java/org/peergos/APIServiceTest.java b/src/test/java/org/peergos/APIServiceTest.java index 9ecd3db8..8f1763ca 100644 --- a/src/test/java/org/peergos/APIServiceTest.java +++ b/src/test/java/org/peergos/APIServiceTest.java @@ -45,7 +45,7 @@ public void runAPIServiceWithFileStorageTest() { @Test public void bulkGetTest() { EmbeddedIpfs ipfs = new EmbeddedIpfs(null, new ProvidingBlockstore(new RamBlockstore()), null, - null, null, Optional.empty(), Collections.emptyList(), Optional.empty()); + null, null, Optional.empty(), Collections.emptyList(), Optional.empty(), Collections.emptyList()); Cid cid1 = ipfs.blockstore.put("Hello".getBytes(), Cid.Codec.Raw).join(); Cid cid2= ipfs.blockstore.put("world!".getBytes(), Cid.Codec.Raw).join(); List wants = new ArrayList<>(); @@ -57,7 +57,7 @@ public void bulkGetTest() { public static void runAPIServiceTest(Blockstore blocks) { EmbeddedIpfs ipfs = new EmbeddedIpfs(null, new ProvidingBlockstore(blocks), null, - null, null, Optional.empty(), Collections.emptyList(), Optional.empty()); + null, null, Optional.empty(), Collections.emptyList(), Optional.empty(), Collections.emptyList()); Cid cid = Cid.decode("zdpuAwfJrGYtiGFDcSV3rDpaUrqCtQZRxMjdC6Eq9PNqLqTGg"); Assert.assertFalse("cid found", ipfs.blockstore.has(cid).join()); String text = "Hello world!"; diff --git a/src/test/java/org/peergos/BitswapMirrorTest.java b/src/test/java/org/peergos/BitswapMirrorTest.java index 7ba9fd36..04d1762f 100644 --- a/src/test/java/org/peergos/BitswapMirrorTest.java +++ b/src/test/java/org/peergos/BitswapMirrorTest.java @@ -26,7 +26,7 @@ public void mirrorTree() throws IOException { new RamProviderStore(1000), new RamRecordStore(), new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true)); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); IPFS kubo = new IPFS("localhost", 5001); Multiaddr kuboAddress = Multiaddr.fromString("/ip4/127.0.0.1/tcp/4001/p2p/" + kubo.id().get("ID")); preloadBlocksToKubo(kubo); diff --git a/src/test/java/org/peergos/BootstrapTest.java b/src/test/java/org/peergos/BootstrapTest.java index 912356b7..770252e6 100644 --- a/src/test/java/org/peergos/BootstrapTest.java +++ b/src/test/java/org/peergos/BootstrapTest.java @@ -40,7 +40,7 @@ public void bootstrap() { new RamProviderStore(1000), new RamRecordStore(), new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true)); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); Multihash node1Id = Multihash.deserialize(node1.getPeerId().getBytes()); try { diff --git a/src/test/java/org/peergos/FindPeerTest.java b/src/test/java/org/peergos/FindPeerTest.java index 18d71d57..4f013878 100644 --- a/src/test/java/org/peergos/FindPeerTest.java +++ b/src/test/java/org/peergos/FindPeerTest.java @@ -21,7 +21,7 @@ public void findLongRunningNode() { new RamProviderStore(1000), new RamRecordStore(), blockstore1, (c, p, a) -> CompletableFuture.completedFuture(true)); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); try { // bootstrap node 1 diff --git a/src/test/java/org/peergos/HandlerTest.java b/src/test/java/org/peergos/HandlerTest.java index 3aed6452..8ad69ba6 100644 --- a/src/test/java/org/peergos/HandlerTest.java +++ b/src/test/java/org/peergos/HandlerTest.java @@ -46,7 +46,7 @@ public void codecTest() { apiServer = HttpServer.create(localAPIAddress, 500); Blockstore blocks = new TypeLimitedBlockstore(new RamBlockstore(), Set.of(Cid.Codec.Raw)); EmbeddedIpfs ipfs = new EmbeddedIpfs(null, new ProvidingBlockstore(blocks), null, null, - new Bitswap(new BitswapEngine(null, null, Bitswap.MAX_MESSAGE_SIZE)), Optional.empty(), Collections.emptyList(), Optional.empty()); + new Bitswap(new BitswapEngine(null, null, Bitswap.MAX_MESSAGE_SIZE)), Optional.empty(), Collections.emptyList(), Optional.empty(), Collections.emptyList()); apiServer.createContext(APIHandler.API_URL, new APIHandler(ipfs)); apiServer.setExecutor(Executors.newFixedThreadPool(50)); apiServer.start(); @@ -100,7 +100,7 @@ public void findBlockProviderTest() throws IOException { apiServer = HttpServer.create(localAPIAddress, 500); EmbeddedIpfs ipfs = new EmbeddedIpfs(node1, new ProvidingBlockstore(new RamBlockstore()), null, dht, - null, Optional.empty(), Collections.emptyList(), Optional.empty()); + null, Optional.empty(), Collections.emptyList(), Optional.empty(), Collections.emptyList()); apiServer.createContext(APIHandler.API_URL, new APIHandler(ipfs)); apiServer.setExecutor(Executors.newFixedThreadPool(50)); apiServer.start(); @@ -126,7 +126,7 @@ public void blockMethodsTest() { apiServer = HttpServer.create(localAPIAddress, 500); EmbeddedIpfs ipfs = new EmbeddedIpfs(null, new ProvidingBlockstore(new RamBlockstore()), null, - null, new Bitswap(new BitswapEngine(null, null, Bitswap.MAX_MESSAGE_SIZE)), Optional.empty(), Collections.emptyList(), Optional.empty()); + null, new Bitswap(new BitswapEngine(null, null, Bitswap.MAX_MESSAGE_SIZE)), Optional.empty(), Collections.emptyList(), Optional.empty(), Collections.emptyList()); apiServer.createContext(APIHandler.API_URL, new APIHandler(ipfs)); apiServer.setExecutor(Executors.newFixedThreadPool(50)); apiServer.start(); diff --git a/src/test/java/org/peergos/HttpProxyTest.java b/src/test/java/org/peergos/HttpProxyTest.java index 046d0d4d..0e3fc26a 100644 --- a/src/test/java/org/peergos/HttpProxyTest.java +++ b/src/test/java/org/peergos/HttpProxyTest.java @@ -12,11 +12,9 @@ import org.peergos.protocol.dht.*; import org.peergos.protocol.http.*; import org.peergos.util.*; -import org.peergos.util.HttpUtil; import java.io.*; import java.net.*; -import java.nio.charset.*; import java.util.*; import java.util.concurrent.*; import java.util.zip.*; @@ -99,7 +97,7 @@ public void p2pProxyClientTest() throws Exception { .addProtocol(new HttpProtocol.Binding(unusedProxyTarget)); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); Kademlia dht = builder1.getWanDht().get(); dht.bootstrapRoutingTable(node1, BootstrapTest.BOOTSTRAP_NODES, a -> true); System.out.println("Bootstrapping node..."); diff --git a/src/test/java/org/peergos/IpnsTest.java b/src/test/java/org/peergos/IpnsTest.java index 72d4d3fa..f32b5358 100644 --- a/src/test/java/org/peergos/IpnsTest.java +++ b/src/test/java/org/peergos/IpnsTest.java @@ -26,7 +26,7 @@ public void publishIPNSRecordToKubo() throws IOException { new RamProviderStore(1000), new RamRecordStore(), blockstore1, (c, p, a) -> CompletableFuture.completedFuture(true)); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); Multihash node1Id = Multihash.deserialize(node1.getPeerId().getBytes()); try { @@ -73,7 +73,7 @@ public void retrieveKuboPublishedIPNS() throws IOException { new RamProviderStore(1000), new RamRecordStore(), blockstore1, (c, p, a) -> CompletableFuture.completedFuture(true)); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); try { IPFS kubo = new IPFS("localhost", 5001); diff --git a/src/test/java/org/peergos/KademliaTest.java b/src/test/java/org/peergos/KademliaTest.java index 6d81b010..50879d52 100644 --- a/src/test/java/org/peergos/KademliaTest.java +++ b/src/test/java/org/peergos/KademliaTest.java @@ -23,13 +23,13 @@ public void findOtherNode() throws Exception { new RamProviderStore(1000), new RamRecordStore(), blockstore1, (c, p, a) -> CompletableFuture.completedFuture(true)); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); HostBuilder builder2 = HostBuilder.create(TestPorts.getPort(), new RamProviderStore(1000), new RamRecordStore(), new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true)); Host node2 = builder2.build(); node2.start().join(); - IdentifyBuilder.addIdentifyProtocol(node2); + IdentifyBuilder.addIdentifyProtocol(node2, Collections.emptyList()); try { // bootstrap node 2 @@ -64,13 +64,13 @@ public void ipnsBenchmark() throws Exception { new RamProviderStore(1000), new RamRecordStore(), blockstore1, (c, p, a) -> CompletableFuture.completedFuture(true)); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); HostBuilder builder2 = HostBuilder.create(TestPorts.getPort(), new RamProviderStore(1000), new RamRecordStore(), new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true)); Host node2 = builder2.build(); node2.start().join(); - IdentifyBuilder.addIdentifyProtocol(node2); + IdentifyBuilder.addIdentifyProtocol(node2, Collections.emptyList()); Cid value = blockstore1.put("Publish me.".getBytes(), Cid.Codec.Raw).join(); diff --git a/src/test/java/org/peergos/KuboFindProviderTest.java b/src/test/java/org/peergos/KuboFindProviderTest.java index 68d3b7f1..eac0d7c7 100644 --- a/src/test/java/org/peergos/KuboFindProviderTest.java +++ b/src/test/java/org/peergos/KuboFindProviderTest.java @@ -24,7 +24,7 @@ public void findProviderOverYamux() throws IOException { .addMuxers(List.of(StreamMuxerProtocol.getYamux())); Host node1 = builder1.build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); try { IPFS kubo = new IPFS("localhost", 5001); Multiaddr address2 = Multiaddr.fromString("/ip4/127.0.0.1/tcp/4001/p2p/" + kubo.id().get("ID")); diff --git a/src/test/java/org/peergos/KuboPingTest.java b/src/test/java/org/peergos/KuboPingTest.java index ff8b00ae..da976b4f 100644 --- a/src/test/java/org/peergos/KuboPingTest.java +++ b/src/test/java/org/peergos/KuboPingTest.java @@ -26,7 +26,7 @@ public void runPingOverYamux() throws IOException { .addMuxers(List.of(StreamMuxerProtocol.getYamux())) .build(); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); try { IPFS kubo = new IPFS("localhost", 5001); Multiaddr address2 = Multiaddr.fromString("/ip4/127.0.0.1/tcp/4001/p2p/" + kubo.id().get("ID")); diff --git a/src/test/java/org/peergos/KuboTest.java b/src/test/java/org/peergos/KuboTest.java index e1041662..36b88b86 100644 --- a/src/test/java/org/peergos/KuboTest.java +++ b/src/test/java/org/peergos/KuboTest.java @@ -21,7 +21,7 @@ public void getBlock() throws IOException { Bitswap bitswap1 = new Bitswap(new BitswapEngine(new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true), Bitswap.MAX_MESSAGE_SIZE)); Host node1 = HostBuilder.build(TestPorts.getPort(), List.of(bitswap1)); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); try { IPFS kubo = new IPFS("localhost", 5001); Multiaddr address2 = Multiaddr.fromString("/ip4/127.0.0.1/tcp/4001/p2p/" + kubo.id().get("ID")); diff --git a/src/test/java/org/peergos/PingTest.java b/src/test/java/org/peergos/PingTest.java index 637a39a4..355dbcb3 100644 --- a/src/test/java/org/peergos/PingTest.java +++ b/src/test/java/org/peergos/PingTest.java @@ -62,8 +62,8 @@ public void runPingEd25519ToRSA() { new Bitswap(new BitswapEngine(new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true), Bitswap.MAX_MESSAGE_SIZE)))); node1.start().join(); node2.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); - IdentifyBuilder.addIdentifyProtocol(node2); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); + IdentifyBuilder.addIdentifyProtocol(node2, Collections.emptyList()); Assert.assertTrue(new Multihash(Multihash.Type.id, node1Keys.publicKey().bytes()).toString().equals(node1.getPeerId().toString())); Assert.assertTrue(new Multihash(Multihash.Type.sha2_256, Hash.sha256(node2Keys.publicKey().bytes())).toString().equals(node2.getPeerId().toString())); @@ -87,9 +87,9 @@ public void replyIdentifyOnNewDial() { Host node1 = HostBuilder.build(TestPorts.getPort(), List.of(new Ping(), new Bitswap(new BitswapEngine(new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true), Bitswap.MAX_MESSAGE_SIZE)))); Host node2 = HostBuilder.build(TestPorts.getPort(), List.of(new Ping(), new Bitswap(new BitswapEngine(new RamBlockstore(), (c, p, a) -> CompletableFuture.completedFuture(true), Bitswap.MAX_MESSAGE_SIZE)))); node1.start().join(); - IdentifyBuilder.addIdentifyProtocol(node1); + IdentifyBuilder.addIdentifyProtocol(node1, Collections.emptyList()); node2.start().join(); - IdentifyBuilder.addIdentifyProtocol(node2); + IdentifyBuilder.addIdentifyProtocol(node2, Collections.emptyList()); try { // ping from 1 to 2 Multiaddr address2 = node2.listenAddresses().get(0); From b0adef5c5452a3e52f4a62a8e97416c00e076a02 Mon Sep 17 00:00:00 2001 From: ian Date: Mon, 16 Sep 2024 13:34:57 +0100 Subject: [PATCH 2/2] Linting --- src/main/java/org/peergos/EmbeddedIpfs.java | 8 +++----- src/test/java/org/peergos/HttpProxyTest.java | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/peergos/EmbeddedIpfs.java b/src/main/java/org/peergos/EmbeddedIpfs.java index 321127b7..c7cc0bf5 100644 --- a/src/main/java/org/peergos/EmbeddedIpfs.java +++ b/src/main/java/org/peergos/EmbeddedIpfs.java @@ -27,8 +27,6 @@ import org.peergos.protocol.ipns.*; import org.peergos.util.Logging; -import java.io.*; -import java.net.*; import java.nio.file.*; import java.sql.Connection; import java.sql.DriverManager; @@ -145,7 +143,7 @@ public void start() { try { this.stop().join(); } catch (Exception ex) { - ex.printStackTrace(); + LOG.info(ex.getMessage()); } }); Runtime.getRuntime().addShutdownHook(shutdownHook); @@ -179,14 +177,14 @@ public void start() { }); mdns.start(); - blockProvider.ifPresent(p -> p.start()); + blockProvider.ifPresent(PeriodicBlockProvider::start); } public CompletableFuture stop() throws Exception { if (records != null) { records.close(); } - blockProvider.ifPresent(b -> b.stop()); + blockProvider.ifPresent(PeriodicBlockProvider::stop); dht.stopBootstrapThread(); for (MDnsDiscovery m : mdns) { m.stop(); diff --git a/src/test/java/org/peergos/HttpProxyTest.java b/src/test/java/org/peergos/HttpProxyTest.java index 0e3fc26a..d470323a 100644 --- a/src/test/java/org/peergos/HttpProxyTest.java +++ b/src/test/java/org/peergos/HttpProxyTest.java @@ -77,7 +77,7 @@ public void p2pProxyRequest() throws IOException { ByteArrayOutputStream bout = new ByteArrayOutputStream(); resp.content().readBytes(bout, resp.headers().getInt("content-length")); - Assert.assertTrue(resp.headers().get(headerName).equals(headerValue)); + Assert.assertEquals(resp.headers().get(headerName), headerValue); resp.release(); byte[] replyBody = bout.toByteArray(); equal(replyBody, httpReply);