diff --git a/.circleci/config.yml b/.circleci/config.yml index e4108ffa91..57bd1b2c09 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -648,6 +648,8 @@ jobs: resource_class: << pipeline.parameters.medium >> steps: - checkout + # Use nightly until we bump the MSRV to 1.89.0 or greater + - install_rust_nightly - setup_environment: cache_key: v4.2.0-rust-1.88.0-cargo-audit-cache - run: diff --git a/.gitignore b/.gitignore index 3a44ec7ba7..c418acd56d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ wasm/Cargo.lock **/build **.history-* validator-* -**.bft-storage-*/ **proptest-regressions/ **package-lock.json **node_modules/ @@ -14,12 +13,15 @@ validator-* **.logs-*/ # Data created when running a node -**.current-proposal-cache-* **.ledger-* **.node-data-* -jwt_secret*.txt -cached_router_peers -cached_gateway_peers + +# Legacy files created by a running snarkOS node +**.current-proposal-cache-* +**cached_gateway_peers +**cached_router_peers +**.bft-storage-*/ +**jwt_secret*.txt # For release upgrde CI .ci/release-snarkos diff --git a/Cargo.lock b/Cargo.lock index 0c232797c2..84e26737af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2472,9 +2472,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.179" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libgit2-sys" @@ -4652,6 +4652,7 @@ dependencies = [ "snarkos-node-bft-storage-service", "snarkos-node-metrics", "snarkos-node-sync", + "snarkos-utilities", "snarkvm", "tokio", "tracing", @@ -4674,7 +4675,6 @@ dependencies = [ name = "snarkos-node-network" version = "4.4.0" dependencies = [ - "aleo-std", "anyhow", "built", "locktick", @@ -4725,7 +4725,6 @@ dependencies = [ name = "snarkos-node-router" version = "4.4.0" dependencies = [ - "aleo-std", "anyhow", "async-trait", "colored 3.0.0", @@ -4748,6 +4747,7 @@ dependencies = [ "snarkos-node-sync-communication-service", "snarkos-node-sync-locators", "snarkos-node-tcp", + "snarkos-utilities", "snarkvm", "time", "tokio", diff --git a/README.md b/README.md index 1a6ed94aaa..dcde4feee4 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,20 @@ APrivateKey1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Note, using the `--raw` flag with the command will sign plaintext messages as bytes rather than [Aleo values](https://developer.aleo.org/guides/aleo/language#data-types-and-values) such as `1u8` or `100field`. +### 5. What is the different between `node-data` and `ledger`? +Ledger contains only public ledger information, while `node-data` contains information specific to the node that created it. The latter should not be shared with untrusted sources, and, for validators, contains data required to participate in consensus. + +### 6. At startup I get an error telling me the nodes still uses the old storage format. What should I do? +The node should have created a new folder for the node data. For example, for mainnet a folder should exist at `~/.aleo/storage/node-data-0`. + +The error message will tell you what data to migrate. Alternatively, you can start the node with `--auto-migrate-node-data` and it will attempt to do this automatically. + +The following is an overview of all files that may be needed to be migrated. +- The JSON Web token secret, located at `~/.aleo/storage/jwt_secrect_{address}.txt`. Move it to `~/.aleo/storage/node-data-0/jwt_secret_{address}.txt`. + Note, if you are running different nodes with different addresses there may be multiple of these. The error message will tell you which one to migrate. +- The router peer cache located at `~/.aleo/storage/ledger-0/cache_router_peers`. Migrate it to `~/.aleo/storage/node-data-0/router-peer-cache`. +- The gateway peer cache located at `~/.aleo/storage/ledger-0/cache_gateway_peers` (only for validators). Migrate it to `~/.aleo/storage/node-data-0/gateway-peer-cache`. +- The latest proposal cache located at `~/.aleo/storage/current-proposal-cache-0` (only for validators). Migrate it to `~/.aleo/storage/node-data-0/current-proposal-cache`. ## 5. Command Line Interface @@ -299,30 +313,145 @@ SUBCOMMANDS: The following are the options for the `snarkos start` command: ``` -USAGE: - snarkos start [OPTIONS] + --network + Specify the network ID of this node [options: 0 = mainnet, 1 = testnet, 2 = canary] + + [default: 0] -OPTIONS: - --network Specify the network ID of this node [default: 3] - - --validator Specify this node as a validator - --prover Specify this node as a prover - --client Specify this node as a client - - --private-key Specify the node's account private key - --private-key-file Specify the path to a file containing the node's account private key - - --node Specify the IP address and port for the node server [default: 0.0.0.0:4130] - --connect Specify the IP address and port of a peer to connect to - - --rest Specify the IP address and port for the REST server [default: 0.0.0.0:3030] - --norest If the flag is set, the node will not initialize the REST server - - --nodisplay If the flag is set, the node will not render the display - --verbosity Specify the verbosity of the node [options: 0, 1, 2, 3] [default: 2] - --logfile Specify the path to the file where logs will be stored [default: /tmp/snarkos.log] - - --dev Enables development mode, specify a unique ID for this node + --prover + Start the node as a prover + + --client + Start the node as a client (default). + + Client are "full nodes", i.e, validate and execute all blocks they receive, but they do not participate in AleoBFT consensus. + + --bootstrap-client + Start the node as a bootstrap client. + + --validator + Start the node as a validator. + + Validators are "full nodes", like clients, but also participate in AleoBFT. + + --noupdater + Disable checking for new versions at startup + + --private-key + Specify the account private key of the node + + --private-key-file + Specify the path to a file containing the account private key of the node + + --node + Set the IP address and port used for P2P communication + + --bft + Set the IP address and port used for BFT communication. This argument is only allowed for validator nodes + + --peers + Specify the IP address and port of the peer(s) to connect to (as a comma-separated list). + + These peers will be set as "trusted", which means the node will not disconnect from them when performing peer rotation. + + Setting peers to "" has the same effect as not setting the flag at all, except when using `--dev`. + + --validators + Specify the IP address and port of the validator(s) to connect to + + --rest + Specify the IP address and port for the REST server + + --rest-rps + Specify the requests per second (RPS) rate limit per IP for the REST server + + [default: 10] + + --jwt-secret + Specify the JWT secret for the REST server (16B, base64-encoded) + + --jwt-timestamp + Specify the JWT creation timestamp; can be any time in the last 10 years + + --norest + If the flag is set, the node will not initialize the REST server + + --nojwt + If the flag is set, the node will not require JWT authentication for the REST server + + --trusted-peers-only + If the flag is set, the node will only connect to trusted peers and validators + + --nodisplay + Write log message to stdout instead of showing a terminal UI. + + This is useful, for example, for running a node as a service instead of in the foreground or to pipe its output into a file. + + --verbosity + Specify the log verbosity of the node. [options: 0 (lowest log level) to 6 (highest level)] + + [default: 1] + + --log-filter + Set a custom log filtering scheme, e.g., "off,snarkos_bft=trace", to show all log messages of snarkos_bft but nothing else + + --logfile + Specify the path to the file where logs will be stored + + [default: /var/folders/6v/1bwnpyjd1r5f9wr_9hq25qsm0000gn/T/snarkos.log] + + --metrics + Enable the metrics exporter + + --metrics-ip + Specify the IP address and port for the metrics exporter + + --ledger-storage + Specify the directory that holds all ledger data, e.g., blocks and transactions. + This flag overrides the default path, even when `--dev` is set. + + The old name for this flag (`--storage`) is DEPRECATED and will eventually be removed. + + --node-data-storage + Specify the directory that holds node-specific data, that is not part of the global ledger. + This flag overrides the default path, even when `--dev` is set. + + That folder may contain sensitive data, such as the JWT secret, and should not be shared with untrusted parties. + For validators, it also contains the latest proposal cache, which is required to participate in consensus. + + --cdn + Enables the node to prefetch initial blocks from a CDN + + --nocdn + If the flag is set, the node will not prefetch from a CDN + + --dev + Enables development mode used to set up test networks. + + The purpose of this flag is to run multiple nodes on the same machine and in the same working directory. + To do this, set the value to a unique ID within the test work. For example if there are four nodes in the network, pass `--dev 0` for the first node, `--dev 1` for the second, and so forth. + + If you do not explicitly set the `--peers` flag, this will also populate the set of trusted peers, so that the network is fully connected. + Additionally, if you do not set the `--rest` or the `--norest` flags, it will also set the REST port to `3030` for the first node, `3031` for the second, and so forth. + + --dev-num-validators + If development mode is enabled, specify the number of genesis validator + + [default: 4] + + --dev-num-clients + If development mode is enabled, specify the number of clients. This is only used by validators to automatically populate their set of trusted peers. + + This option cannot be used while also passing the `--peers` flag. + + --no-dev-txs + If development mode is enabled, specify whether node 0 should generate traffic to drive the network + + --dev-bonded-balances + If development mode is enabled, specify the custom bonded balances as a JSON object + + -h, --help + Print help (see a summary with '-h') ``` ## 6. Development Guide diff --git a/cli/src/commands/clean.rs b/cli/src/commands/clean.rs index a16db9485f..ede52d1746 100644 --- a/cli/src/commands/clean.rs +++ b/cli/src/commands/clean.rs @@ -13,11 +13,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use snarkos_node::bft::helpers::proposal_cache_path; +use crate::helpers::args::parse_node_data_dir; + +use snarkos_utilities::NodeDataDir; + use snarkvm::console::network::{CanaryV0, MainnetV0, Network}; use aleo_std::StorageMode; -use anyhow::{Result, bail}; +use anyhow::{Context, Result}; use clap::Parser; use colored::Colorize; use std::path::PathBuf; @@ -28,19 +31,29 @@ pub struct Clean { /// Specify the network to remove from storage (0 = mainnet, 1 = testnet, 2 = canary) #[clap(default_value_t=MainnetV0::ID, long = "network", value_parser = clap::value_parser!(u16).range((MainnetV0::ID as i64)..=(CanaryV0::ID as i64)))] pub network: u16, + /// Enables development mode, specify the unique ID of the local node to clean. #[clap(long)] pub dev: Option, + /// Specify the path to a directory containing the ledger. Overrides the default path (also for /// dev). #[clap(long = "path")] pub path: Option, + + /// Sets a custom path for the node configuration. + #[clap(long)] + pub node_data_path: Option, } impl Clean { /// Cleans the snarkOS node storage. pub fn parse(self) -> Result { - // Initialize the storage mode. + // Remove the specified node configuration from storage. + let node_data_dir = parse_node_data_dir(&self.node_data_path, self.network, self.dev)?; + println!("{}", Self::remove_node_data(&node_data_dir)?); + + // Remove the specified ledger from storage. let storage_mode = match self.path { Some(path) => StorageMode::Custom(path), None => match self.dev { @@ -48,16 +61,23 @@ impl Clean { None => StorageMode::Production, }, }; + Self::remove_ledger(self.network, &storage_mode) + } + + /// Removes the specified node configuration from storage. + fn remove_node_data(node_data_dir: &NodeDataDir) -> Result { + // With the new layout, we can remove the entire folder. + let data_path = node_data_dir.path(); + + // Prepare the path string. + let path_string = format!("(in \"{}\")", data_path.display()).dimmed(); - // Remove the current proposal cache file, if it exists. - let proposal_cache_path = proposal_cache_path(self.network, &storage_mode); - if proposal_cache_path.exists() { - if let Err(err) = std::fs::remove_file(&proposal_cache_path) { - bail!("Failed to remove the current proposal cache file at {}: {err}", proposal_cache_path.display()); - } + if data_path.exists() { + std::fs::remove_dir_all(data_path).with_context(|| format!("Failed to remove node data {path_string}"))?; + Ok(format!("✅ Cleaned up node data {path_string}")) + } else { + Ok(format!("✅ No node data was found {path_string}")) } - // Remove the specified ledger from storage. - Self::remove_ledger(self.network, &storage_mode) } /// Removes the specified ledger from storage. @@ -71,14 +91,11 @@ impl Clean { // Check if the path to the ledger exists in storage. if path.exists() { // Remove the ledger files from storage. - match std::fs::remove_dir_all(&path) { - Ok(_) => Ok(format!("✅ Cleaned the snarkOS node storage {path_string}")), - Err(error) => { - bail!("Failed to remove the snarkOS node storage {path_string}\n{}", error.to_string().dimmed()) - } - } + std::fs::remove_dir_all(&path) + .with_context(|| format!("Failed to remove the snarkOS ledger {path_string}"))?; + Ok(format!("✅ Cleaned the snarkOS ledger {path_string}")) } else { - Ok(format!("✅ No snarkOS node storage was found {path_string}")) + Ok(format!("✅ No snarkOS ledger was found {path_string}")) } } } diff --git a/cli/src/commands/start.rs b/cli/src/commands/start.rs index d29027269f..1c6406c881 100644 --- a/cli/src/commands/start.rs +++ b/cli/src/commands/start.rs @@ -13,7 +13,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::helpers::{args::network_id_parser, dev::*}; +use crate::helpers::{ + args::{network_id_parser, parse_node_data_dir}, + dev::*, +}; use snarkos_account::Account; use snarkos_display::Display; @@ -24,7 +27,7 @@ use snarkos_node::{ rest::DEFAULT_REST_PORT, router::DEFAULT_NODE_PORT, }; -use snarkos_utilities::SignalHandler; +use snarkos_utilities::{NodeDataDir, SignalHandler, jwt_secret_file, node_data}; use snarkvm::{ console::{ @@ -42,7 +45,7 @@ use snarkvm::{ utilities::to_bytes_le, }; -use aleo_std::StorageMode; +use aleo_std::{StorageMode, aleo_ledger_dir}; use anyhow::{Context, Result, anyhow, bail, ensure}; use base64::prelude::{BASE64_STANDARD, Engine}; use clap::Parser; @@ -52,9 +55,11 @@ use indexmap::IndexMap; use rand::{Rng, SeedableRng}; use rand_chacha::ChaChaRng; use serde::{Deserialize, Serialize}; + use std::{ + fs, net::{Ipv4Addr, SocketAddr, SocketAddrV4, ToSocketAddrs}, - path::PathBuf, + path::{Path, PathBuf}, sync::{Arc, atomic::AtomicBool}, }; use tokio::{ @@ -232,10 +237,20 @@ pub struct Start { #[clap(long, requires = "metrics")] pub metrics_ip: Option, - /// Specify the path to a directory containing the storage database for the ledger. + /// Specify the directory that holds all ledger data, e.g., blocks and transactions. /// This flag overrides the default path, even when `--dev` is set. - #[clap(long)] - pub storage: Option, + /// + /// The old name for this flag (`--storage`) is DEPRECATED and will eventually be removed. + #[clap(long, verbatim_doc_comment, alias = "storage")] + pub ledger_storage: Option, + + /// Specify the directory that holds node-specific data, that is not part of the global ledger. + /// This flag overrides the default path, even when `--dev` is set. + /// + /// That folder may contain sensitive data, such as the JWT secret, and should not be shared with untrusted parties. + /// For validators, it also contains the latest proposal cache, which is required to participate in consensus. + #[clap(long, verbatim_doc_comment)] + pub node_data_storage: Option, /// Enables the node to prefetch initial blocks from a CDN #[clap(long, conflicts_with = "nocdn")] @@ -273,6 +288,10 @@ pub struct Start { /// If development mode is enabled, specify the custom bonded balances as a JSON object. #[clap(long, group = "dev_flags")] pub dev_bonded_balances: Option, + + /// If the flag is set, the node will attempt to automatically migrate the node data to the new format. + #[clap(long)] + pub auto_migrate_node_data: bool, } impl Start { @@ -485,6 +504,11 @@ impl Start { Ok(()) } + /// Returns the path to where the JWT secret for the node is stored. + fn jwt_secret_path(node_data_dir: &NodeDataDir, address: &Address) -> PathBuf { + node_data_dir.path().join(jwt_secret_file(address)) + } + /// Returns an alternative genesis block if the node is in development mode. /// Otherwise, returns the actual genesis block. fn parse_genesis(&self) -> Result> { @@ -684,7 +708,7 @@ impl Start { }; // Initialize the storage mode. - let storage_mode = match &self.storage { + let storage_mode = match &self.ledger_storage { Some(path) => StorageMode::Custom(path.clone()), None => match self.dev { Some(id) => StorageMode::Development(id), @@ -692,13 +716,42 @@ impl Start { }, }; - // Helper function to store the JWT secret. - let store_jwt_secret = |network: u16, storage_mode: &StorageMode, address: &Address, token: String| -> Result<()> { - let mut jwt_secret_path = aleo_std::aleo_ledger_dir(network, storage_mode); - std::fs::create_dir_all(&jwt_secret_path)?; - jwt_secret_path.push(format!("jwt_secret_{address}.txt")); - Ok(std::fs::write(jwt_secret_path, token)?) - }; + // Users may have unintentionally set a custom path for the ledger, but not for the node data. + // For validators, we make this an errors, so important files like the proposal cache are stored at the location + // exepcted by the node operator. + if self.node_data_storage.is_some() && !matches!(storage_mode, StorageMode::Custom(_)) { + if node_type == NodeType::Validator { + bail!("Custom path set for `--node-data-storage`, but not for `--ledger-storage`.") + } else { + warn!("Custom path set for `--node-data-storage`, but not for `--ledger-storage`. The latter will use the default path."); + } + } else if matches!(storage_mode, StorageMode::Custom(_)) && self.node_data_storage.is_none() { + if node_type == NodeType::Validator { + bail!("Custom path set for `--ledger-storage`, but not for `--node-data-storage`."); + } else { + warn!("Custom path set for `--ledger-storage`, but not for `--node-data-storage`. The latter will use the default path."); + } + } + + // Parse the node data directory. + let node_data_dir = parse_node_data_dir(&self.node_data_storage, N::ID, self.dev).with_context(|| "Failed to setup node configuration directory")?; + + // Make sure the directory exists before continue. + let data_path = node_data_dir.path(); + if !data_path.exists() { + info!("Creating directore for node data storage at {data_path:?}"); + std::fs::create_dir_all(data_path) + .with_context(|| format!("Failed to create directory for node data storage at {data_path:?}"))? + } else if !data_path.is_dir() { + bail!("Node data storage location at {data_path:?} is not a directory"); + } else { + debug!("Using existing directory at {data_path:?} for node data storage"); + } + + // Checks for the old storage format and prints instructions to migrate. + // We perform this check after creating the node data directory, so that migrating the data is easier. + Self::check_for_old_storage_format(&aleo_ledger_dir(N::ID, &storage_mode), &account.address(), &node_data_dir, self.dev, self.auto_migrate_node_data).with_context(|| "Node still uses the old storage format.")?; + // Compute the optional REST server JWT. let jwt_token = if self.nojwt { None @@ -711,14 +764,16 @@ impl Start { // Create the JWT token based on the given secret. let jwt_token = snarkos_node_rest::Claims::new(account.address(), Some(jwt_bytes), self.jwt_timestamp).to_jwt_string()?; // Store the JWT secret to a file. - store_jwt_secret(self.network, &storage_mode, &account.address(), jwt_token.clone())?; + let path = Self::jwt_secret_path(&node_data_dir, &account.address()); + std::fs::write(path, &jwt_token)?; // Return the JWT token for optional printing. Some(jwt_token) } else { // Create a random JWT token. let jwt_token = snarkos_node_rest::Claims::new(account.address(), None, self.jwt_timestamp).to_jwt_string()?; // Store the JWT secret to a file. - store_jwt_secret(self.network, &storage_mode, &account.address(), jwt_token.clone())?; + let path = Self::jwt_secret_path(&node_data_dir, &account.address()); + std::fs::write(path, &jwt_token)? ; // Return the JWT token for optional printing. Some(jwt_token) }; @@ -778,9 +833,9 @@ impl Start { // Initialize the node. let node = match node_type { - NodeType::Validator => Node::new_validator(node_ip, self.bft, rest_ip, self.rest_rps, account, &trusted_peers, &trusted_validators, genesis, cdn, storage_mode, self.trusted_peers_only, dev_txs, self.dev, signal_handler.clone()).await, - NodeType::Prover => Node::new_prover(node_ip, account, &trusted_peers, genesis, storage_mode, self.trusted_peers_only, self.dev, signal_handler.clone()).await, - NodeType::Client => Node::new_client(node_ip, rest_ip, self.rest_rps, account, &trusted_peers, genesis, cdn, storage_mode, self.trusted_peers_only, self.dev, signal_handler.clone()).await, + NodeType::Validator => Node::new_validator(node_ip, self.bft, rest_ip, self.rest_rps, account, &trusted_peers, &trusted_validators, genesis, cdn, storage_mode, node_data_dir, self.trusted_peers_only, dev_txs, self.dev, signal_handler.clone()).await, + NodeType::Prover => Node::new_prover(node_ip, account, &trusted_peers, genesis, node_data_dir, self.trusted_peers_only, self.dev, signal_handler.clone()).await, + NodeType::Client => Node::new_client(node_ip, rest_ip, self.rest_rps, account, &trusted_peers, genesis, cdn, storage_mode, node_data_dir, self.trusted_peers_only, self.dev, signal_handler.clone()).await, NodeType::BootstrapClient => Node::new_bootstrap_client(node_ip, account, *genesis.header(), self.dev).await, }?; @@ -792,6 +847,88 @@ impl Start { Ok(()) } + /// Check if the node is still using the old storage format, + /// in which case we print an error and exit. + /// We detect this by checking if + /// - a peer-cache file exists inside the ledger directory, + /// - a current-proposal-cache file exists at the parent directory of the ledger directory + /// - a jwt_secret_*.txt file exists at the parent directory of the ledger directory + fn check_for_old_storage_format( + ledger_dir: &Path, + address: &Address, + node_data_dir: &NodeDataDir, + dev: Option, + auto_migrate: bool, + ) -> Result<()> { + let ledger_parent_dir = ledger_dir.parent().unwrap(); + + // Determine the old paths used for node configuration files. + let old_router_cache_path = ledger_dir.join(node_data::LEGACY_ROUTER_PEER_CACHE_FILE); + let old_gateway_cache_path = ledger_dir.join(node_data::LEGACY_GATEWAY_PEER_CACHE_FILE); + let old_proposal_cache_path = ledger_dir.join(node_data::legacy_current_proposal_cache_file(N::ID, dev)); + let old_jwt_secret_path = ledger_parent_dir.join(node_data::jwt_secret_file(address)); + + if auto_migrate { + if old_router_cache_path.exists() { + let new_router_cache_path = node_data_dir.router_peer_cache_path(); + info!("Migrating node data file \"{old_router_cache_path:?}\" to \"{new_router_cache_path:?}\""); + fs::rename(old_router_cache_path, new_router_cache_path) + .with_context(|| "Failed to migrate node data file")?; + } + + if old_gateway_cache_path.exists() { + let new_gateway_cache_path = node_data_dir.gateway_peer_cache_path(); + info!("Migrating node data file \"{old_gateway_cache_path:?}\" to \"{new_gateway_cache_path:?}\""); + fs::rename(old_gateway_cache_path, new_gateway_cache_path) + .with_context(|| "Failed to migrate node data file")?; + } + + if old_proposal_cache_path.exists() { + let new_proposal_cache_path = node_data_dir.current_proposal_cache_path(); + info!("Migrating node data file \"{old_proposal_cache_path:?}\" to \"{new_proposal_cache_path:?}\""); + fs::rename(old_proposal_cache_path, new_proposal_cache_path) + .with_context(|| "Failed to migrate node data file")?; + } + + if old_jwt_secret_path.exists() { + let new_jwt_secret_path = node_data_dir.jwt_secret_path(&address); + info!("Migrating node data file \"{old_jwt_secret_path:?}\" to \"{new_jwt_secret_path:?}\""); + fs::rename(old_jwt_secret_path, new_jwt_secret_path) + .with_context(|| "Failed to migrate node data file")?; + } + } else { + if old_router_cache_path.exists() { + let new_router_cache_path = node_data_dir.router_peer_cache_path(); + bail!( + "Please migrate the node data file \"{old_router_cache_path:?}\" to \"{new_router_cache_path:?}\" before restarting, or restart with `--auto-migrate-node-data`." + ); + } + + if old_gateway_cache_path.exists() { + let new_gateway_cache_path = node_data_dir.gateway_peer_cache_path(); + bail!( + "Please migrate the node data file \"{old_gateway_cache_path:?}\" to \"{new_gateway_cache_path:?}\" before restarting, or restart with `--auto-migrate-node-data`." + ); + } + + if old_proposal_cache_path.exists() { + let new_proposal_cache_path = node_data_dir.current_proposal_cache_path(); + bail!( + "Please migrate the node data file \"{old_proposal_cache_path:?}\" to \"{new_proposal_cache_path:?}\" before restarting, or restart with `--auto-migrate-node-data`." + ); + } + + if old_jwt_secret_path.exists() { + let new_jwt_secret_path = node_data_dir.jwt_secret_path(&address); + bail!( + "Please migrate the node data file \"{old_jwt_secret_path:?}\" to \"{new_jwt_secret_path:?}\" before restarting, or restart with `--auto-migrate-node-data`." + ); + } + } + + Ok(()) + } + /// Starts a rayon thread pool and tokio runtime for the node, and returns the tokio `Runtime`. fn runtime() -> Runtime { // Retrieve the number of cores. diff --git a/cli/src/helpers/args.rs b/cli/src/helpers/args.rs index 939c33ca73..ab6ba9e55d 100644 --- a/cli/src/helpers/args.rs +++ b/cli/src/helpers/args.rs @@ -13,18 +13,21 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::helpers::dev::get_development_key; + +use snarkos_utilities::NodeDataDir; + use snarkvm::{ console::network::{CanaryV0, MainnetV0}, prelude::{Network, PrivateKey}, }; +use aleo_std::aleo_dir; use anyhow::{Context, Result, anyhow}; use clap::builder::RangedU64ValueParser; use std::{path::PathBuf, str::FromStr}; use ureq::http::{Uri, uri}; -use crate::helpers::dev::get_development_key; - pub(crate) fn network_id_parser() -> RangedU64ValueParser { RangedU64ValueParser::::new().range((MainnetV0::ID as u64)..=(CanaryV0::ID as u64)) } @@ -74,6 +77,27 @@ pub(crate) fn parse_private_key( Ok(private_key) } +/// Returns the path to the node configuration directory. +pub(crate) fn parse_node_data_dir( + custom_node_data_dir: &Option, + network: u16, + dev: Option, +) -> Result { + if let Some(custom) = custom_node_data_dir { + return Ok(NodeDataDir::new(custom.clone())); + } + + let config = if let Some(dev) = dev { + NodeDataDir::new_development(network, dev) + } else { + // Production mode + let path = aleo_dir().join("storage").join(format!("node-data-{network}")); + NodeDataDir::new(path) + }; + + Ok(config) +} + #[cfg(test)] mod tests { use super::*; diff --git a/node/bft/examples/simple_node.rs b/node/bft/examples/simple_node.rs index 99536287f1..fe4bd4bc27 100644 --- a/node/bft/examples/simple_node.rs +++ b/node/bft/examples/simple_node.rs @@ -29,7 +29,7 @@ use snarkos_node_bft::{ use snarkos_node_bft_ledger_service::TranslucentLedgerService; use snarkos_node_bft_storage_service::BFTMemoryService; use snarkos_node_sync::BlockSync; -use snarkos_utilities::SimpleStoppable; +use snarkos_utilities::{NodeDataDir, SimpleStoppable}; use aleo_std::StorageMode; use snarkvm::{ @@ -140,7 +140,7 @@ pub async fn start_bft( Some(ip) => Some(*ip), None => Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), MEMORY_POOL_PORT + node_id)), }; - let storage_mode = StorageMode::new_test(None); + let node_data_dir = NodeDataDir::new_test(None); // Initialize the trusted validators. let trusted_validators = trusted_validators(node_id, num_nodes, peers); let trusted_peers_only = false; @@ -158,7 +158,7 @@ pub async fn start_bft( ip, &trusted_validators, trusted_peers_only, - storage_mode, + node_data_dir, None, )?; // Run the BFT instance. @@ -194,7 +194,7 @@ pub async fn start_primary( Some(ip) => Some(*ip), None => Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), MEMORY_POOL_PORT + node_id)), }; - let storage_mode = StorageMode::new_test(None); + let node_data_dir = NodeDataDir::new_test(None); // Initialize the trusted validators. let trusted_validators = trusted_validators(node_id, num_nodes, peers); let trusted_peers_only = false; @@ -208,7 +208,7 @@ pub async fn start_primary( ip, &trusted_validators, trusted_peers_only, - storage_mode, + node_data_dir, None, )?; // Run the primary instance. diff --git a/node/bft/src/bft.rs b/node/bft/src/bft.rs index 0ce559dab7..f85a1756a4 100644 --- a/node/bft/src/bft.rs +++ b/node/bft/src/bft.rs @@ -28,9 +28,12 @@ use crate::{ now, }, }; + use snarkos_account::Account; use snarkos_node_bft_ledger_service::LedgerService; use snarkos_node_sync::{BlockSync, Ping}; +use snarkos_utilities::NodeDataDir; + use snarkvm::{ console::account::Address, ledger::{ @@ -43,7 +46,6 @@ use snarkvm::{ utilities::flatten_error, }; -use aleo_std::StorageMode; use anyhow::Context; use colored::Colorize; use indexmap::{IndexMap, IndexSet}; @@ -99,7 +101,7 @@ impl BFT { ip: Option, trusted_validators: &[SocketAddr], trusted_peers_only: bool, - storage_mode: StorageMode, + node_data_dir: NodeDataDir, dev: Option, ) -> Result { Ok(Self { @@ -111,7 +113,7 @@ impl BFT { ip, trusted_validators, trusted_peers_only, - storage_mode, + node_data_dir, dev, )?, dag: Default::default(), @@ -994,6 +996,8 @@ mod tests { use snarkos_node_bft_ledger_service::{LedgerService, MockLedgerService}; use snarkos_node_bft_storage_service::BFTMemoryService; use snarkos_node_sync::BlockSync; + use snarkos_utilities::NodeDataDir; + use snarkvm::{ console::account::{Address, PrivateKey}, ledger::{ @@ -1013,7 +1017,6 @@ mod tests { utilities::TestRng, }; - use aleo_std::StorageMode; use anyhow::Result; use indexmap::{IndexMap, IndexSet}; use std::sync::Arc; @@ -1060,7 +1063,7 @@ mod tests { None, &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), None, ) } diff --git a/node/bft/src/gateway.rs b/node/bft/src/gateway.rs index f9fc7739ab..a313837f57 100644 --- a/node/bft/src/gateway.rs +++ b/node/bft/src/gateway.rs @@ -24,7 +24,6 @@ use crate::{ helpers::{Cache, PrimarySender, Storage, SyncSender, WorkerSender, assign_to_worker}, spawn_blocking, }; -use aleo_std::StorageMode; use snarkos_account::Account; use snarkos_node_bft_events::{ BlockRequest, @@ -61,6 +60,7 @@ use snarkos_node_tcp::{ Tcp, protocols::{Disconnect, Handshake, OnConnect, Reading, Writing}, }; +use snarkos_utilities::NodeDataDir; use snarkvm::{ console::prelude::*, ledger::{ @@ -116,12 +116,6 @@ const CONNECTION_ATTEMPTS_SINCE_SECS: i64 = 10; /// The amount of time an IP address is prohibited from connecting. const IP_BAN_TIME_IN_SECS: u64 = 300; -/// The name of the file containing cached validators. -const VALIDATOR_CACHE_FILENAME: &str = "cached_gateway_peers"; - -/// The name of the file containing the dynamic validator whitelist. -const VALIDATOR_WHITELIST_FILENAME: &str = "dynamic_validator_whitelist"; - /// Part of the Gateway API that deals with networking. /// This is a separate trait to allow for easier testing/mocking. #[async_trait] @@ -169,7 +163,7 @@ pub struct InnerGateway { /// The spawned handles. handles: Mutex>>, /// The storage mode. - storage_mode: StorageMode, + node_data_dir: NodeDataDir, /// If the flag is set, the node will only connect to trusted peers. trusted_peers_only: bool, /// The development mode. @@ -212,7 +206,7 @@ impl Gateway { ip: Option, trusted_validators: &[SocketAddr], trusted_peers_only: bool, - storage_mode: StorageMode, + node_data_dir: NodeDataDir, dev: Option, ) -> Result { // Initialize the gateway IP. @@ -229,7 +223,7 @@ impl Gateway { // Load entries from the validator cache (if present and if we are not in trusted peers only mode). if !trusted_peers_only { - let cached_peers = Self::load_cached_peers(&storage_mode, VALIDATOR_CACHE_FILENAME)?; + let cached_peers = Self::load_cached_peers(&node_data_dir.gateway_peer_cache_path())?; for addr in cached_peers { initial_peers.insert(addr, Peer::new_candidate(addr, false)); } @@ -254,7 +248,7 @@ impl Gateway { worker_senders: Default::default(), sync_sender: Default::default(), handles: Default::default(), - storage_mode, + node_data_dir, trusted_peers_only, dev, }))) @@ -848,7 +842,7 @@ impl Gateway { pub async fn shut_down(&self) { info!("Shutting down the gateway..."); // Save the best peers for future use. - if let Err(e) = self.save_best_peers(&self.storage_mode, VALIDATOR_CACHE_FILENAME, None, true) { + if let Err(e) = self.save_best_peers(&self.node_data_dir.gateway_peer_cache_path(), None, true) { warn!("Failed to persist best validators to disk: {e}"); } // Abort the tasks. @@ -1128,7 +1122,7 @@ impl Gateway { // Update the dynamic validator whitelist. fn update_validator_whitelist(&self) { if let Err(e) = - self.save_best_peers(&self.storage_mode, VALIDATOR_WHITELIST_FILENAME, Some(MAX_VALIDATORS_TO_SEND), false) + self.save_best_peers(&self.node_data_dir.validator_whitelist_path(), Some(MAX_VALIDATORS_TO_SEND), false) { warn!("Couldn't update the validator whitelist: {e}"); } @@ -1683,12 +1677,14 @@ mod prop_tests { gateway::prop_tests::GatewayAddress::{Dev, Prod}, helpers::{Storage, init_primary_channels, init_worker_channels}, }; - use aleo_std::StorageMode; + use snarkos_account::Account; use snarkos_node_bft_ledger_service::MockLedgerService; use snarkos_node_bft_storage_service::BFTMemoryService; use snarkos_node_network::PeerPoolHandling; use snarkos_node_tcp::P2P; + use snarkos_utilities::NodeDataDir; + use snarkvm::{ ledger::{ committee::{ @@ -1759,7 +1755,7 @@ mod prop_tests { address.ip(), &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), address.port(), ) .unwrap() @@ -1812,7 +1808,7 @@ mod prop_tests { dev.ip(), &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), dev.port(), ) .unwrap(); @@ -1837,7 +1833,7 @@ mod prop_tests { dev.ip(), &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), dev.port(), ) .unwrap(); @@ -1872,7 +1868,7 @@ mod prop_tests { dev.ip(), &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), dev.port(), ) .unwrap(); @@ -1946,7 +1942,7 @@ mod prop_tests { dev.ip(), &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), dev.port(), ) .unwrap(); diff --git a/node/bft/src/helpers/proposal_cache.rs b/node/bft/src/helpers/proposal_cache.rs index 6e19f28285..faebadb8d9 100644 --- a/node/bft/src/helpers/proposal_cache.rs +++ b/node/bft/src/helpers/proposal_cache.rs @@ -14,6 +14,7 @@ // limitations under the License. use crate::helpers::{Proposal, SignedProposals}; +use snarkos_utilities::NodeDataDir; use snarkvm::{ console::{account::Address, network::Network, program::SUBDAG_CERTIFICATES_DEPTH}, @@ -21,27 +22,12 @@ use snarkvm::{ prelude::{FromBytes, IoResult, Read, Result, ToBytes, Write, anyhow, bail, error}, }; -use aleo_std::{StorageMode, aleo_ledger_dir}; use indexmap::IndexSet; use std::{fs, path::PathBuf}; /// Returns the path where a proposal cache file may be stored. -pub fn proposal_cache_path(network: u16, storage_mode: &StorageMode) -> PathBuf { - const PROPOSAL_CACHE_FILE_NAME: &str = "current-proposal-cache"; - // Obtain the path to the ledger. - let mut path = aleo_ledger_dir(network, storage_mode); - // Go to the folder right above the ledger when using the default paths, - // otherwise store in the same directory as storage. - if !matches!(storage_mode, StorageMode::Custom(_)) { - path.pop(); - } - // Append the proposal store's file name. - match storage_mode.dev() { - Some(id) => path.push(format!(".{PROPOSAL_CACHE_FILE_NAME}-{network}-{id}")), - None => path.push(format!("{PROPOSAL_CACHE_FILE_NAME}-{network}")), - } - - path +pub fn proposal_cache_path(node_data_dir: &NodeDataDir) -> PathBuf { + node_data_dir.path().join("current-proposal-cache") } /// A helper type for the cache of proposal and signed proposals. @@ -80,14 +66,14 @@ impl ProposalCache { } /// Returns `true` if a proposal cache exists for the given network and `dev`. - pub fn exists(storage_mode: &StorageMode) -> bool { - proposal_cache_path(N::ID, storage_mode).exists() + pub fn exists(node_data_dir: &NodeDataDir) -> bool { + proposal_cache_path(node_data_dir).exists() } /// Load the proposal cache from the file system and ensure that the proposal cache is valid. - pub fn load(expected_signer: Address, storage_mode: &StorageMode) -> Result { + pub fn load(expected_signer: Address, node_data_dir: &NodeDataDir) -> Result { // Construct the proposal cache file system path. - let path = proposal_cache_path(N::ID, storage_mode); + let path = proposal_cache_path(node_data_dir); // Deserialize the proposal cache from the file system. let proposal_cache = match fs::read(&path) { @@ -109,8 +95,8 @@ impl ProposalCache { } /// Store the proposal cache to the file system. - pub fn store(&self, storage_mode: &StorageMode) -> Result<()> { - let path = proposal_cache_path(N::ID, storage_mode); + pub fn store(&self, node_data_dir: &NodeDataDir) -> Result<()> { + let path = proposal_cache_path(node_data_dir); info!("Storing the proposal cache to {}...", path.display()); // Serialize the proposal cache. diff --git a/node/bft/src/primary.rs b/node/bft/src/primary.rs index f04389ae72..762954912b 100644 --- a/node/bft/src/primary.rs +++ b/node/bft/src/primary.rs @@ -41,11 +41,14 @@ use crate::{ }, spawn_blocking, }; + use snarkos_account::Account; use snarkos_node_bft_events::PrimaryPing; use snarkos_node_bft_ledger_service::LedgerService; use snarkos_node_network::PeerPoolHandling; use snarkos_node_sync::{BlockSync, DUMMY_SELF_IP, Ping}; +use snarkos_utilities::NodeDataDir; + use snarkvm::{ console::{ prelude::*, @@ -60,7 +63,6 @@ use snarkvm::{ utilities::flatten_error, }; -use aleo_std::StorageMode; use anyhow::Context; use colored::Colorize; use futures::stream::{FuturesUnordered, StreamExt}; @@ -114,8 +116,8 @@ pub struct Primary { handles: Arc>>>, /// The lock for propose_batch. propose_lock: Arc>, - /// The storage mode of the node. - storage_mode: StorageMode, + /// The node configuration directory. + node_data_dir: NodeDataDir, } impl Primary { @@ -132,7 +134,7 @@ impl Primary { ip: Option, trusted_validators: &[SocketAddr], trusted_peers_only: bool, - storage_mode: StorageMode, + node_data_dir: NodeDataDir, dev: Option, ) -> Result { // Initialize the gateway. @@ -143,7 +145,7 @@ impl Primary { ip, trusted_validators, trusted_peers_only, - storage_mode.clone(), + node_data_dir.clone(), dev, )?; // Initialize the sync module. @@ -162,16 +164,16 @@ impl Primary { signed_proposals: Default::default(), handles: Default::default(), propose_lock: Default::default(), - storage_mode, + node_data_dir, }) } /// Load the proposal cache file and update the Primary state with the stored data. async fn load_proposal_cache(&self) -> Result<()> { // Fetch the signed proposals from the file system if it exists. - match ProposalCache::::exists(&self.storage_mode) { + match ProposalCache::::exists(&self.node_data_dir) { // If the proposal cache exists, then process the proposal cache. - true => match ProposalCache::::load(self.gateway.account().address(), &self.storage_mode) { + true => match ProposalCache::::load(self.gateway.account().address(), &self.node_data_dir) { Ok(proposal_cache) => { // Extract the proposal and signed proposals. let (latest_certificate_round, proposed_batch, signed_proposals, pending_certificates) = @@ -1990,7 +1992,7 @@ impl Primary { let pending_certificates = self.storage.get_pending_certificates(); ProposalCache::new(latest_round, proposal, signed_proposals, pending_certificates) }; - if let Err(err) = proposal_cache.store(&self.storage_mode) { + if let Err(err) = proposal_cache.store(&self.node_data_dir) { error!("{}", flatten_error(err.context("Failed to store the current proposal cache"))); } // Close the gateway. @@ -2049,7 +2051,7 @@ mod tests { let account = accounts[account_index].1.clone(); let block_sync = Arc::new(BlockSync::new(ledger.clone())); let mut primary = - Primary::new(account, storage, ledger, block_sync, None, &[], false, StorageMode::new_test(None), None) + Primary::new(account, storage, ledger, block_sync, None, &[], false, NodeDataDir::new_test(None), None) .unwrap(); // Construct a worker instance. diff --git a/node/bft/src/sync/mod.rs b/node/bft/src/sync/mod.rs index 046af6f3df..189e935cc6 100644 --- a/node/bft/src/sync/mod.rs +++ b/node/bft/src/sync/mod.rs @@ -1065,7 +1065,8 @@ mod tests { use snarkos_account::Account; use snarkos_node_sync::BlockSync; - use snarkos_utilities::SimpleStoppable; + use snarkos_utilities::{NodeDataDir, SimpleStoppable}; + use snarkvm::{ console::{ account::{Address, PrivateKey}, @@ -1287,7 +1288,7 @@ mod tests { None, &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), None, )?; let block_sync = Arc::new(BlockSync::new(syncing_ledger.clone())); diff --git a/node/bft/tests/common/primary.rs b/node/bft/tests/common/primary.rs index 3e65ec77db..77fbdff2a4 100644 --- a/node/bft/tests/common/primary.rs +++ b/node/bft/tests/common/primary.rs @@ -30,7 +30,7 @@ use snarkos_node_bft::{ use snarkos_node_bft_storage_service::BFTMemoryService; use snarkos_node_network::PeerPoolHandling; use snarkos_node_sync::BlockSync; -use snarkos_utilities::SimpleStoppable; +use snarkos_utilities::{NodeDataDir, SimpleStoppable}; use snarkvm::{ console::{ @@ -180,7 +180,7 @@ impl TestNetwork { Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), MEMORY_POOL_PORT + id as u16)), &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), None, ) .unwrap(); @@ -194,7 +194,7 @@ impl TestNetwork { Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), MEMORY_POOL_PORT + id as u16)), &[], false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), None, ) .unwrap(); diff --git a/node/bft/tests/common/utils.rs b/node/bft/tests/common/utils.rs index c9ba1d2aec..cf48a9264d 100644 --- a/node/bft/tests/common/utils.rs +++ b/node/bft/tests/common/utils.rs @@ -14,16 +14,15 @@ // limitations under the License. use crate::common::{CurrentNetwork, TranslucentLedgerService, primary}; -use aleo_std::StorageMode; + use snarkos_account::Account; use snarkos_node_bft::{ Gateway, Worker, helpers::{PrimarySender, Storage}, + storage_service::BFTMemoryService, }; - -use snarkos_node_bft::storage_service::BFTMemoryService; -use snarkos_utilities::SimpleStoppable; +use snarkos_utilities::{NodeDataDir, SimpleStoppable}; use snarkvm::{ console::account::Address, @@ -219,7 +218,7 @@ pub fn sample_gateway( ledger: Arc>>, ) -> Gateway { // Initialize the gateway. - Gateway::new(account, storage, ledger, None, &[], false, StorageMode::new_test(None), None).unwrap() + Gateway::new(account, storage, ledger, None, &[], false, NodeDataDir::new_test(None), None).unwrap() } /// Samples a new worker with the given ledger. diff --git a/node/consensus/Cargo.toml b/node/consensus/Cargo.toml index 7d6b75a66a..0ab4772983 100644 --- a/node/consensus/Cargo.toml +++ b/node/consensus/Cargo.toml @@ -79,6 +79,9 @@ features = [ "persistent" ] [dependencies.snarkos-node-sync] workspace = true +[dependencies.snarkos-utilities] +workspace = true + [dependencies.snarkvm] workspace = true diff --git a/node/consensus/src/lib.rs b/node/consensus/src/lib.rs index c72a112f8b..c34e71d8e4 100644 --- a/node/consensus/src/lib.rs +++ b/node/consensus/src/lib.rs @@ -42,6 +42,7 @@ use snarkos_node_bft::{ use snarkos_node_bft_ledger_service::LedgerService; use snarkos_node_bft_storage_service::BFTPersistentStorage; use snarkos_node_sync::{BlockSync, Ping}; +use snarkos_utilities::NodeDataDir; use snarkvm::{ ledger::{ @@ -124,6 +125,7 @@ impl Consensus { trusted_validators: &[SocketAddr], trusted_peers_only: bool, storage_mode: StorageMode, + node_data_dir: NodeDataDir, ping: Arc>, dev: Option, ) -> Result { @@ -142,7 +144,7 @@ impl Consensus { ip, trusted_validators, trusted_peers_only, - storage_mode, + node_data_dir, dev, )?; // Create a new instance of Consensus. diff --git a/node/network/Cargo.toml b/node/network/Cargo.toml index 619c5f09f5..0652089372 100644 --- a/node/network/Cargo.toml +++ b/node/network/Cargo.toml @@ -20,9 +20,6 @@ edition = "2024" locktick = [ "dep:locktick", "snarkos-node-tcp/locktick" ] test = [ ] -[dependencies.aleo-std] -workspace = true - [dependencies.anyhow] workspace = true diff --git a/node/network/src/peering.rs b/node/network/src/peering.rs index 2abf13f00e..227d268709 100644 --- a/node/network/src/peering.rs +++ b/node/network/src/peering.rs @@ -15,7 +15,6 @@ use crate::{CandidatePeer, ConnectedPeer, ConnectionMode, NodeType, Peer, Resolver, bootstrap_peers}; -use aleo_std::{StorageMode, aleo_ledger_dir}; use snarkos_node_tcp::{P2P, is_bogon_ip, is_unspecified_or_broadcast_ip}; use snarkvm::prelude::{Address, Network}; @@ -33,6 +32,7 @@ use std::{ fs, io::{self, Write}, net::{IpAddr, SocketAddr}, + path::Path, str::FromStr, time::Instant, }; @@ -461,11 +461,8 @@ pub trait PeerPoolHandling: P2P { /// Loads any previously cached peer addresses so they can be introduced as initial /// candidate peers to connect to. - fn load_cached_peers(storage_mode: &StorageMode, filename: &str) -> Result> { - let mut peer_cache_path = aleo_ledger_dir(N::ID, storage_mode); - peer_cache_path.push(filename); - - let peers = match fs::read_to_string(&peer_cache_path) { + fn load_cached_peers(path: &Path) -> Result> { + let peers = match fs::read_to_string(path) { Ok(cached_peers_str) => { let mut cached_peers = Vec::new(); for peer_addr_str in cached_peers_str.lines() { @@ -481,7 +478,7 @@ pub trait PeerPoolHandling: P2P { Vec::new() } Err(error) => { - warn!("{} Couldn't load cached peers at {}: {error}", Self::OWNER, peer_cache_path.display()); + warn!("{} Couldn't load cached peers at {}: {error}", Self::OWNER, path.display()); Vec::new() } }; @@ -491,13 +488,12 @@ pub trait PeerPoolHandling: P2P { /// Preserve the peers who have the greatest known block heights, and the lowest /// number of registered network failures. - fn save_best_peers( - &self, - storage_mode: &StorageMode, - filename: &str, - max_entries: Option, - store_ports: bool, - ) -> Result<()> { + /// + /// # Arguments + /// * `path` - The path to the file to save the peers to. + /// * `max_entries` - The maximum number of peers to save (if there are more, the extra ones are truncated). + /// * `store_ports` - Whether to store the ports of the peers, or just the IP addresses. + fn save_best_peers(&self, path: &Path, max_entries: Option, store_ports: bool) -> Result<()> { // Collect all prospect peers. let mut peers = self.get_peers(); @@ -519,8 +515,6 @@ pub trait PeerPoolHandling: P2P { } // Dump the connected peers to a file. - let mut path = aleo_ledger_dir(N::ID, storage_mode); - path.push(filename); let mut file = fs::File::create(path)?; for peer in peers { writeln!( diff --git a/node/router/Cargo.toml b/node/router/Cargo.toml index a5e91536fc..993d6fa2af 100644 --- a/node/router/Cargo.toml +++ b/node/router/Cargo.toml @@ -28,9 +28,6 @@ metrics = [ "dep:snarkos-node-metrics" ] cuda = [ "snarkvm/cuda", "snarkos-account/cuda" ] serial = [ "snarkos-node-bft-ledger-service/serial" ] -[dependencies.aleo-std] -workspace = true - [dependencies.anyhow] workspace = true @@ -87,6 +84,9 @@ workspace = true [dependencies.snarkos-node-tcp] workspace = true +[dependencies.snarkos-utilities] +workspace = true + [dependencies.snarkvm] workspace = true diff --git a/node/router/src/lib.rs b/node/router/src/lib.rs index 2e6084b0df..3e2dc364df 100644 --- a/node/router/src/lib.rs +++ b/node/router/src/lib.rs @@ -24,6 +24,7 @@ extern crate tracing; extern crate snarkos_node_metrics as metrics; pub use snarkos_node_router_messages as messages; +use snarkos_utilities::NodeDataDir; mod handshake; @@ -62,7 +63,6 @@ use snarkos_node_tcp::{Config, ConnectionSide, Tcp}; use snarkvm::prelude::{Address, Network, PrivateKey, ViewKey}; -use aleo_std::StorageMode; use anyhow::Result; #[cfg(feature = "locktick")] use locktick::parking_lot::{Mutex, RwLock}; @@ -74,9 +74,6 @@ use tokio::task::JoinHandle; /// The default port used by the router. pub const DEFAULT_NODE_PORT: u16 = 4130; -/// The name of the file containing cached peers. -const PEER_CACHE_FILENAME: &str = "cached_router_peers"; - /// The router keeps track of connected and connecting peers. /// The actual network communication happens in Inbound/Outbound, /// which is implemented by Validator, Prover, and Client. @@ -137,7 +134,7 @@ pub struct InnerRouter { /// If the flag is set, the node will only connect to trusted peers. trusted_peers_only: bool, /// The storage mode. - storage_mode: StorageMode, + node_data_dir: NodeDataDir, /// The boolean flag for the development mode. is_dev: bool, } @@ -165,7 +162,7 @@ impl Router { trusted_peers: &[SocketAddr], max_peers: u16, trusted_peers_only: bool, - storage_mode: StorageMode, + node_data_dir: NodeDataDir, is_dev: bool, ) -> Result { // Initialize the TCP stack. @@ -176,7 +173,7 @@ impl Router { // Load entries from the peer cache (if present and if we are not in trusted peers only mode). if !trusted_peers_only { - let cached_peers = Self::load_cached_peers(&storage_mode, PEER_CACHE_FILENAME)?; + let cached_peers = Self::load_cached_peers(&node_data_dir.router_peer_cache_path())?; for addr in cached_peers { initial_peers.insert(addr, Peer::new_candidate(addr, false)); } @@ -197,7 +194,7 @@ impl Router { peer_pool: RwLock::new(initial_peers), handles: Default::default(), trusted_peers_only, - storage_mode, + node_data_dir, is_dev, }))) } @@ -279,7 +276,9 @@ impl Router { pub async fn shut_down(&self) { info!("Shutting down the router..."); // Save the best peers for future use. - if let Err(e) = self.save_best_peers(&self.storage_mode, PEER_CACHE_FILENAME, Some(MAX_PEERS_TO_SEND), true) { + if let Err(e) = + self.save_best_peers(&self.node_data_dir.router_peer_cache_path(), Some(MAX_PEERS_TO_SEND), true) + { warn!("Failed to persist best peers to disk: {e}"); } // Abort the tasks. diff --git a/node/router/tests/common/mod.rs b/node/router/tests/common/mod.rs index 8b601febed..63902cedff 100644 --- a/node/router/tests/common/mod.rs +++ b/node/router/tests/common/mod.rs @@ -22,11 +22,12 @@ use std::{ sync::Arc, }; -use aleo_std::StorageMode; use snarkos_account::Account; use snarkos_node_bft_ledger_service::MockLedgerService; use snarkos_node_network::NodeType; use snarkos_node_router::Router; +use snarkos_utilities::NodeDataDir; + use snarkvm::{ prelude::{FromBytes, MainnetV0 as CurrentNetwork, Network, PrivateKey, block::Block}, utilities::TestRng, @@ -68,7 +69,7 @@ pub async fn client(listening_port: u16, max_peers: u16, rng: &mut TestRng) -> T &[], max_peers, false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), true, ) .await @@ -89,7 +90,7 @@ pub async fn prover(listening_port: u16, max_peers: u16, rng: &mut TestRng) -> T &[], max_peers, false, - StorageMode::new_test(None), + NodeDataDir::new_test(None), true, ) .await @@ -116,7 +117,7 @@ pub async fn validator( trusted_peers, max_peers, trusted_peers_only, - StorageMode::new_test(None), + NodeDataDir::new_test(None), true, ) .await diff --git a/node/src/client/mod.rs b/node/src/client/mod.rs index faa54886b1..ec3832ee7e 100644 --- a/node/src/client/mod.rs +++ b/node/src/client/mod.rs @@ -37,7 +37,7 @@ use snarkos_node_tcp::{ P2P, protocols::{Disconnect, Handshake, OnConnect, Reading}, }; -use snarkos_utilities::{SignalHandler, Stoppable}; +use snarkos_utilities::{NodeDataDir, SignalHandler, Stoppable}; use snarkvm::{ console::network::Network, @@ -143,6 +143,7 @@ impl> Client { genesis: Block, cdn: Option, storage_mode: StorageMode, + node_data_dir: NodeDataDir, trusted_peers_only: bool, dev: Option, signal_handler: Arc, @@ -167,7 +168,7 @@ impl> Client { trusted_peers, Self::MAXIMUM_NUMBER_OF_PEERS as u16, trusted_peers_only, - storage_mode.clone(), + node_data_dir.clone(), dev.is_some(), ) .await?; diff --git a/node/src/node.rs b/node/src/node.rs index 93ffeb6453..dcc1e3297a 100644 --- a/node/src/node.rs +++ b/node/src/node.rs @@ -24,7 +24,7 @@ use crate::{ }; use snarkos_account::Account; -use snarkos_utilities::SignalHandler; +use snarkos_utilities::{NodeDataDir, SignalHandler}; use snarkvm::prelude::{ Address, @@ -71,6 +71,7 @@ impl Node { genesis: Block, cdn: Option, storage_mode: StorageMode, + node_data_dir: NodeDataDir, trusted_peers_only: bool, dev_txs: bool, dev: Option, @@ -88,6 +89,7 @@ impl Node { genesis, cdn, storage_mode, + node_data_dir, trusted_peers_only, dev_txs, dev, @@ -103,7 +105,7 @@ impl Node { account: Account, trusted_peers: &[SocketAddr], genesis: Block, - storage_mode: StorageMode, + node_data_dir: NodeDataDir, trusted_peers_only: bool, dev: Option, signal_handler: Arc, @@ -114,7 +116,7 @@ impl Node { account, trusted_peers, genesis, - storage_mode, + node_data_dir, trusted_peers_only, dev, signal_handler, @@ -133,6 +135,7 @@ impl Node { genesis: Block, cdn: Option, storage_mode: StorageMode, + node_data_dir: NodeDataDir, trusted_peers_only: bool, dev: Option, signal_handler: Arc, @@ -147,6 +150,7 @@ impl Node { genesis, cdn, storage_mode, + node_data_dir, trusted_peers_only, dev, signal_handler, diff --git a/node/src/prover/mod.rs b/node/src/prover/mod.rs index 5cedb19aeb..477dcd669e 100644 --- a/node/src/prover/mod.rs +++ b/node/src/prover/mod.rs @@ -23,7 +23,6 @@ use crate::{ use snarkos_account::Account; use snarkos_node_network::{NodeType, PeerPoolHandling}; - use snarkos_node_router::{ Heartbeat, Inbound, @@ -36,7 +35,7 @@ use snarkos_node_tcp::{ P2P, protocols::{Disconnect, Handshake, OnConnect, Reading}, }; -use snarkos_utilities::{SignalHandler, Stoppable}; +use snarkos_utilities::{NodeDataDir, SignalHandler, Stoppable}; use snarkvm::{ ledger::narwhal::Data, @@ -49,7 +48,6 @@ use snarkvm::{ synthesizer::VM, }; -use aleo_std::StorageMode; use anyhow::Result; use colored::Colorize; use core::{marker::PhantomData, time::Duration}; @@ -104,7 +102,7 @@ impl> Prover { account: Account, trusted_peers: &[SocketAddr], genesis: Block, - storage_mode: StorageMode, + node_data_dir: NodeDataDir, trusted_peers_only: bool, dev: Option, signal_handler: Arc, @@ -121,7 +119,7 @@ impl> Prover { trusted_peers, Self::MAXIMUM_NUMBER_OF_PEERS as u16, trusted_peers_only, - storage_mode, + node_data_dir, dev.is_some(), ) .await?; diff --git a/node/src/validator/mod.rs b/node/src/validator/mod.rs index c3d75d6e9e..fea065f00c 100644 --- a/node/src/validator/mod.rs +++ b/node/src/validator/mod.rs @@ -36,7 +36,7 @@ use snarkos_node_tcp::{ P2P, protocols::{Disconnect, Handshake, OnConnect, Reading}, }; -use snarkos_utilities::SignalHandler; +use snarkos_utilities::{NodeDataDir, SignalHandler}; use snarkvm::prelude::{ Ledger, @@ -88,6 +88,7 @@ impl> Validator { genesis: Block, cdn: Option, storage_mode: StorageMode, + node_data_dir: NodeDataDir, trusted_peers_only: bool, dev_txs: bool, dev: Option, @@ -114,7 +115,7 @@ impl> Validator { trusted_peers, Self::MAXIMUM_NUMBER_OF_PEERS as u16, trusted_peers_only, - storage_mode.clone(), + node_data_dir.clone(), dev.is_some(), ) .await?; @@ -133,6 +134,7 @@ impl> Validator { trusted_validators, trusted_peers_only, storage_mode.clone(), + node_data_dir.clone(), ping.clone(), dev, ) @@ -498,6 +500,7 @@ mod tests { let node = SocketAddr::from_str("0.0.0.0:4130").unwrap(); let rest = SocketAddr::from_str("0.0.0.0:3030").unwrap(); let storage_mode = StorageMode::Development(0); + let node_data_dir = NodeDataDir::new_development(CurrentNetwork::ID, 0); let dev_txs = true; // Initialize an (insecure) fixed RNG. @@ -524,6 +527,7 @@ mod tests { genesis, None, storage_mode, + node_data_dir, false, dev_txs, None, diff --git a/node/tests/common/node.rs b/node/tests/common/node.rs index 2e55778e88..fe419ee8f2 100644 --- a/node/tests/common/node.rs +++ b/node/tests/common/node.rs @@ -17,7 +17,7 @@ use crate::common::test_peer::sample_genesis_block; use snarkos_account::Account; use snarkos_node::{Client, Prover, Validator}; -use snarkos_utilities::SignalHandler; +use snarkos_utilities::{NodeDataDir, SignalHandler}; use snarkvm::prelude::{MainnetV0 as CurrentNetwork, store::helpers::memory::ConsensusMemory}; @@ -34,6 +34,7 @@ pub async fn client() -> Client> sample_genesis_block(), None, // No CDN. StorageMode::new_test(None), + NodeDataDir::new_test(None), false, // Connect to untrusted peers. None, SignalHandler::new(), @@ -48,7 +49,7 @@ pub async fn prover() -> Prover> Account::::from_str("APrivateKey1zkp2oVPTci9kKcUprnbzMwq95Di1MQERpYBhEeqvkrDirK1").unwrap(), &[], sample_genesis_block(), - StorageMode::new_test(None), + NodeDataDir::new_test(None), false, None, SignalHandler::new(), @@ -69,6 +70,7 @@ pub async fn validator() -> Validator(address: &D) -> PathBuf { + PathBuf::from(format!("jwt_secret_{address}.txt")) +} + +/// The old filename of the current proposal cache. +pub fn legacy_current_proposal_cache_file(network: u16, dev: Option) -> PathBuf { + if let Some(dev) = dev { + PathBuf::from(format!(".current-proposal-cache-{network}-{dev}")) + } else { + PathBuf::from(format!("current-proposal-cache-{network}")) + } +} + +/// Tracks information about where the node-specfic configuration files are stored. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeDataDir { + path: PathBuf, +} + +impl NodeDataDir { + /// Initializes the node data directory the given path. + pub fn new(path: PathBuf) -> Self { + Self { path } + } + + /// Initializes the node data directory to a location suitable for unit/integration tests. + pub fn new_test(dev: Option) -> Self { + if let Some(dev) = dev { + Self { path: PathBuf::from(format!(".node-data-test-{dev}")) } + } else { + Self { path: PathBuf::from(".node-data-test") } + } + } + + /// Initializes the node data directory path to the development path for the specified network and node index. + pub fn new_development(network: u16, dev: u16) -> Self { + // Use the current directory as the base path, and fall back to the + // cargo manifest directory if the current directory is not available. + let path = std::env::current_dir() + .unwrap_or(PathBuf::from(env!("CARGO_MANIFEST_DIR"))) + .join(format!(".node-data-{network}-{dev}")); + + Self::new(path) + } + + pub fn path(&self) -> &Path { + &self.path + } + + /// The location to store the previous peer cache. + pub fn router_peer_cache_path(&self) -> PathBuf { + self.path.join(ROUTER_PEER_CACHE_FILE) + } + + pub fn gateway_peer_cache_path(&self) -> PathBuf { + self.path.join(GATEWAY_PEER_CACHE_FILE) + } + + pub fn validator_whitelist_path(&self) -> PathBuf { + self.path.join(VALIDATOR_WHITELIST_FILE) + } + + /// The location to store the current proposal cache. + pub fn current_proposal_cache_path(&self) -> PathBuf { + self.path.join(CURRENT_PROPOSAL_CACHE_FILE) + } + + /// The location to store the JWT secret for a given address. + pub fn jwt_secret_path(&self, address: &D) -> PathBuf { + self.path.join(jwt_secret_file(address)) + } +} diff --git a/utilities/src/signals.rs b/utilities/src/signals.rs index 83809d1b2a..3c5ed63446 100644 --- a/utilities/src/signals.rs +++ b/utilities/src/signals.rs @@ -22,6 +22,7 @@ use std::sync::{ Arc, atomic::{AtomicBool, Ordering}, }; + use tokio::sync::oneshot; use tracing::{debug, error, trace}; @@ -62,7 +63,7 @@ impl Stoppable for SimpleStoppable { /// Helper for signal handling that implements the `Stoppable` trait. /// /// This struct will set itself to "stopped" as soon as the process receives Ctrl+C. -/// It can also be manually stopped (e.g., when the node encounters a fatal error). +/// It can also be manually stopped (e.g., when the node encounters a fatal error) pub struct SignalHandler { /// This sender is used to notify a waiting task that the node has been stopped. /// If this is `None`, the node is in the process of shutting down.