diff --git a/.typos.toml b/.typos.toml index 1463f03c4dc..81164ff34df 100644 --- a/.typos.toml +++ b/.typos.toml @@ -33,3 +33,5 @@ Forgetten = "Forgetten" typ = "typ" # Token Generation Event tge = "tge" +# Rust crate name +bimap = "bimap" diff --git a/crates/cliquenet/src/addr.rs b/crates/cliquenet/src/addr.rs index f4202adb9ed..79b99853311 100644 --- a/crates/cliquenet/src/addr.rs +++ b/crates/cliquenet/src/addr.rs @@ -60,6 +60,29 @@ impl NetAddr { pub fn is_ip(&self) -> bool { matches!(self, Self::Inet(..)) } + + /// Whether this address is plausibly publicly routable. Returns `false` for IP literals + /// in non-globally-routable ranges (loopback, unspecified, RFC 1918 private, link-local, + /// broadcast, documentation, IPv6 multicast) and the literal `localhost`. Other hostnames + /// are trusted and return `true`. Approximates the (still unstable) `IpAddr::is_global` + /// using stable predicates; the IPv6 surface is incomplete (`fe80::/10` link-local and + /// `fc00::/7` unique-local addresses are treated as global here). + pub fn is_probably_global(&self) -> bool { + match self { + Self::Inet(IpAddr::V4(v4), _) => { + !(v4.is_loopback() + || v4.is_unspecified() + || v4.is_private() + || v4.is_link_local() + || v4.is_broadcast() + || v4.is_documentation()) + }, + Self::Inet(IpAddr::V6(v6), _) => { + !(v6.is_loopback() || v6.is_unspecified() || v6.is_multicast()) + }, + Self::Name(host, _) => !host.eq_ignore_ascii_case("localhost"), + } + } } impl fmt::Display for NetAddr { @@ -212,4 +235,32 @@ mod tests { fn empty_is_invalid() { assert!("".parse::().is_err()) } + + #[test] + fn test_is_probably_global() { + let cases: &[(&str, bool)] = &[ + ("127.0.0.1:1234", false), + ("0.0.0.0:1234", false), + ("10.0.0.1:1234", false), + ("172.16.5.4:1234", false), + ("192.168.1.1:1234", false), + ("169.254.0.1:1234", false), + ("255.255.255.255:1234", false), + ("192.0.2.1:1234", false), + ("::1:1234", false), + (":::1234", false), + ("ff00::1:1234", false), + ("localhost:1234", false), + ("LOCALHOST:1234", false), + ("8.8.8.8:1234", true), + ("1.1.1.1:1234", true), + ("2606:4700:4700::1111:1234", true), + ("example.com:1234", true), + ("node.internal:1234", true), + ]; + for (s, expected) in cases { + let a: NetAddr = s.parse().unwrap_or_else(|_| panic!("parse {s}")); + assert_eq!(a.is_probably_global(), *expected, "for input {s}"); + } + } } diff --git a/crates/espresso/node/src/lib.rs b/crates/espresso/node/src/lib.rs index a2dfdcdc377..1dc86d24050 100644 --- a/crates/espresso/node/src/lib.rs +++ b/crates/espresso/node/src/lib.rs @@ -140,8 +140,10 @@ pub struct NetworkParams { pub cliquenet_advertise_addr: Option, /// X25519 secret key. pub x25519_secret_key: x25519::SecretKey, - /// The address to send to other Libp2p nodes to contact us - pub libp2p_advertise_address: String, + /// The address to send to other Libp2p nodes to contact us. Required for orchestrator + /// bootstrap; optional otherwise. When set, it is added to the swarm as an external address + /// so peers can reach us behind NAT. + pub libp2p_advertise_address: Option, /// The address to bind to for Libp2p pub libp2p_bind_address: String, /// The (optional) bootstrap node addresses for Libp2p. If supplied, these will @@ -356,16 +358,47 @@ where &network_params.libp2p_bind_address ) })?; - let libp2p_advertise_address = - derive_libp2p_multiaddr(&network_params.libp2p_advertise_address).with_context(|| { - format!( - "Failed to derive Libp2p advertise address of {}", - &network_params.libp2p_advertise_address - ) - })?; + let advertise_multiaddr = network_params + .libp2p_advertise_address + .as_ref() + .map(|addr| { + derive_libp2p_multiaddr(addr) + .with_context(|| format!("Failed to derive Libp2p advertise address of {addr}")) + }) + .transpose()?; + let advertise_is_global = match network_params + .libp2p_advertise_address + .as_deref() + .and_then(|s| s.parse::().ok()) + { + Some(parsed) if !parsed.is_probably_global() => { + tracing::error!( + "Libp2p advertise address {parsed} is probably not publicly routable. This is \ + fine for local testing (demo-native, docker-compose) but is wrong for any real \ + deployment: remote peers will fail to dial us." + ); + false + }, + _ => true, + }; + + // Always pass the configured address to the orchestrator stake table; that path is + // testing-only and demo-native legitimately uses loopback. + let libp2p_announce_addresses: Vec = advertise_multiaddr.iter().cloned().collect(); + + // Only register the advertise address as a libp2p `external_address` when it looks + // publicly routable: announcing local/private values via Identify / Kademlia poisons peer + // routing tables in production. Local tests don't need it since peers find each other via + // `libp2p_bootstrap_nodes`. + let libp2p_external_addresses: Vec = if advertise_is_global { + advertise_multiaddr.iter().cloned().collect() + } else { + Vec::new() + }; info!("Libp2p bind address: {}", libp2p_bind_address); - info!("Libp2p advertise address: {}", libp2p_advertise_address); + info!("Libp2p announce addresses: {:?}", libp2p_announce_addresses); + info!("Libp2p external addresses: {:?}", libp2p_external_addresses); // Orchestrator client let orchestrator_client = OrchestratorClient::new(network_params.orchestrator_url); @@ -441,12 +474,17 @@ where validator_config.p2p_addr = Some(advertise_addr); } + let bootstrap_advertise_addr = libp2p_announce_addresses.first().cloned().context( + "ESPRESSO_NODE_LIBP2P_ADVERTISE_ADDRESS must be set when bootstrapping a libp2p \ + network from the orchestrator", + )?; + let config = get_complete_config( &orchestrator_client, validator_config.clone(), // Register in our Libp2p advertise address and public key so other nodes // can contact us on startup - Some(libp2p_advertise_address), + Some(bootstrap_advertise_addr), Some(libp2p_public_key), ) .await? @@ -726,6 +764,7 @@ where gossip_config, request_response_config, libp2p_bind_address, + libp2p_external_addresses, &validator_config.public_key, // We need the private key so we can derive our Libp2p keypair // (using https://docs.rs/blake3/latest/blake3/fn.derive_key.html) diff --git a/crates/espresso/node/src/options.rs b/crates/espresso/node/src/options.rs index 36424edf4f6..5270d3adc24 100644 --- a/crates/espresso/node/src/options.rs +++ b/crates/espresso/node/src/options.rs @@ -213,12 +213,18 @@ pub struct Options { /// The address we advertise to other nodes as being a Libp2p endpoint. /// Should be supplied in `host:port` form. - #[clap( - long, - env = "ESPRESSO_NODE_LIBP2P_ADVERTISE_ADDRESS", - default_value = "localhost:1769" - )] - pub libp2p_advertise_address: String, + /// + /// Operators should set this to a publicly routable address whenever the bind address + /// is not directly reachable from peers (NAT, K8s NodePort, Docker bridge). It is added + /// to libp2p as an `external_address` so that Identify and Kademlia announce it to the + /// network. Non-globally-routable IP literals (loopback, RFC 1918 private, link-local, + /// etc.) only work for local testing (`demo-native`, `docker-compose`) and are dropped + /// from the libp2p announcement; hostnames are passed through unchanged. + /// + /// Also required when bootstrapping a fresh network from the orchestrator, where it is + /// registered into the stake table so peers can dial us. + #[clap(long, env = "ESPRESSO_NODE_LIBP2P_ADVERTISE_ADDRESS")] + pub libp2p_advertise_address: Option, /// A comma-separated list of Libp2p multiaddresses to use as bootstrap /// nodes. diff --git a/crates/espresso/node/src/run.rs b/crates/espresso/node/src/run.rs index d2ff28d5c7c..06903474c2c 100644 --- a/crates/espresso/node/src/run.rs +++ b/crates/espresso/node/src/run.rs @@ -270,6 +270,11 @@ mod test { .to_string(), "--cliquenet-bind-address", &format!("127.0.0.1:{port2}"), + // Never bound: this test blocks at orchestrator before libp2p starts. Port 0 is a + // placeholder to satisfy the orchestrator-bootstrap requirement on the advertise + // address. + "--libp2p-advertise-address", + "127.0.0.1:0", "--genesis-file", &genesis_file.display().to_string(), ]); diff --git a/crates/hotshot/examples/src/infra.rs b/crates/hotshot/examples/src/infra.rs index bd4403c494e..7d0fadf50ec 100755 --- a/crates/hotshot/examples/src/infra.rs +++ b/crates/hotshot/examples/src/infra.rs @@ -765,6 +765,7 @@ where GossipConfig::default(), RequestResponseConfig::default(), bind_address, + Vec::new(), public_key, private_key, Libp2pMetricsValue::default(), diff --git a/crates/hotshot/hotshot/src/traits/networking/libp2p_network.rs b/crates/hotshot/hotshot/src/traits/networking/libp2p_network.rs index 5266fb0bbe5..b4e4aaea705 100644 --- a/crates/hotshot/hotshot/src/traits/networking/libp2p_network.rs +++ b/crates/hotshot/hotshot/src/traits/networking/libp2p_network.rs @@ -402,6 +402,7 @@ impl Libp2pNetwork { gossip_config: GossipConfig, request_response_config: RequestResponseConfig, bind_address: Multiaddr, + announce_addresses: Vec, pub_key: &T::SignatureKey, priv_key: &::PrivateKey, metrics: Libp2pMetricsValue, @@ -453,7 +454,8 @@ impl Libp2pNetwork { config_builder .keypair(keypair) .replication_factor(replication_factor) - .bind_address(Some(bind_address.clone())); + .bind_address(Some(bind_address.clone())) + .announce_addresses(announce_addresses); // Connect to the provided bootstrap nodes config_builder.to_connect_addrs(HashSet::from_iter(libp2p_config.bootstrap_nodes.clone())); diff --git a/crates/hotshot/libp2p-networking/src/network/node.rs b/crates/hotshot/libp2p-networking/src/network/node.rs index a84b7263c50..11e7585073e 100644 --- a/crates/hotshot/libp2p-networking/src/network/node.rs +++ b/crates/hotshot/libp2p-networking/src/network/node.rs @@ -267,7 +267,7 @@ impl NetworkNode { .set_publication_interval(Some(kademlia_record_republication_interval)) .set_record_ttl(Some(kademlia_ttl)); - // allowing panic here because something is very wrong if this fales + // allowing panic here because something is very wrong if this fails #[allow(clippy::panic)] if let Some(factor) = config.replication_factor { kconfig.set_replication_factor(factor); @@ -341,6 +341,11 @@ impl NetworkNode { } } + for addr in &config.announce_addresses { + info!("Adding announce address {addr}"); + swarm.add_external_address(addr.clone()); + } + Ok(Self { peer_id, swarm, diff --git a/crates/hotshot/libp2p-networking/src/network/node/config.rs b/crates/hotshot/libp2p-networking/src/network/node/config.rs index 76eabc84a2e..d4b08a4ffdb 100644 --- a/crates/hotshot/libp2p-networking/src/network/node/config.rs +++ b/crates/hotshot/libp2p-networking/src/network/node/config.rs @@ -26,6 +26,15 @@ pub struct NetworkNodeConfig { #[builder(default)] pub bind_address: Option, + /// Addresses to announce as external addresses to peers. + /// + /// Each is added via `Swarm::add_external_address` during node setup. Identify will publish + /// them, and Kademlia will record them in our self-routing-table entry. Required when the + /// node is behind NAT, K8s NodePort, Docker bridge, etc., where the bind address is not + /// reachable from peers. + #[builder(default)] + pub announce_addresses: Vec, + /// Replication factor for entries in the DHT #[builder(setter(into, strip_option), default = "DEFAULT_REPLICATION_FACTOR")] pub replication_factor: Option, @@ -68,6 +77,7 @@ impl Clone for NetworkNodeConfig { Self { keypair: self.keypair.clone(), bind_address: self.bind_address.clone(), + announce_addresses: self.announce_addresses.clone(), replication_factor: self.replication_factor, gossip_config: self.gossip_config.clone(), request_response_config: self.request_response_config.clone(), diff --git a/process-compose.yaml b/process-compose.yaml index 634c9227782..c4158a8029f 100644 --- a/process-compose.yaml +++ b/process-compose.yaml @@ -429,6 +429,8 @@ processes: - ESPRESSO_NODE_TONIC_PORT=${ESPRESSO_NODE_4_TONIC_PORT} - ESPRESSO_NODE_STATE_PEERS=http://localhost:${ESPRESSO_NODE_0_API_PORT} - ESPRESSO_NODE_API_PEERS=http://localhost:${ESPRESSO_NODE_0_API_PORT} + - ESPRESSO_NODE_LIBP2P_BIND_ADDRESS=0.0.0.0:${ESPRESSO_DEMO_NODE_LIBP2P_PORT_4} + - ESPRESSO_NODE_LIBP2P_ADVERTISE_ADDRESS=localhost:${ESPRESSO_DEMO_NODE_LIBP2P_PORT_4} - ESPRESSO_NODE_CLIQUENET_BIND_ADDRESS=0.0.0.0:${ESPRESSO_DEMO_NODE_CLIQUENET_PORT_4} - ESPRESSO_NODE_STORAGE_PATH=${ESPRESSO_BASE_STORAGE_PATH}/seq4 - LIGHT_CLIENT_DB_PATH=${ESPRESSO_BASE_STORAGE_PATH}/seq4-lc