[Feature] Filesystem reorganization

.circleci/config.yml

@@ -648,6 +648,8 @@ jobs:
     resource_class: << pipeline.parameters.medium >>
     steps:
       - checkout
+      # Use nightly until we bump the MSRV to 1.89.0 or greater
+      - install_rust_nightly
       - setup_environment:
           cache_key: v4.2.0-rust-1.88.0-cargo-audit-cache
       - run:

.gitignore

@@ -5,7 +5,6 @@ wasm/Cargo.lock
 **/build
 **.history-*
 validator-*
-**.bft-storage-*/
 **proptest-regressions/
 **package-lock.json
 **node_modules/
@@ -14,12 +13,15 @@ validator-*
 **.logs-*/
 
 # Data created when running a node
-**.current-proposal-cache-*
 **.ledger-*
 **.node-data-*
-jwt_secret*.txt
-cached_router_peers
-cached_gateway_peers
+
+# Legacy files created by a running snarkOS node
+**.current-proposal-cache-*
+**cached_gateway_peers
+**cached_router_peers
+**.bft-storage-*/
+**jwt_secret*.txt
 
 # For release upgrde CI
 .ci/release-snarkos

README.md

@@ -272,6 +272,20 @@ APrivateKey1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
 Note, using the `--raw` flag with the command will sign plaintext messages as bytes rather than [Aleo values](https://developer.aleo.org/guides/aleo/language#data-types-and-values) such as `1u8` or `100field`.
 
+### 5. What is the different between `node-data` and `ledger`?
+Ledger contains only public ledger information, while `node-data` contains information specific to the node that created it. The latter should not be shared with untrusted sources, and, for validators, contains data required to participate in consensus.
+
+### 6. At startup I get an error telling me the nodes still uses the old storage format. What should I do?
+The node should have created a new folder for the node data. For example, for mainnet a folder should exist at `~/.aleo/storage/node-data-0`. 
+
+The error message will tell you what data to migrate. Alternatively, you can start the node with `--auto-migrate-node-data` and it will attempt to do this automatically.
+
+The following is an overview of all files that may be needed to be migrated. 
+- The JSON Web token secret, located at `~/.aleo/storage/jwt_secrect_{address}.txt`. Move it to `~/.aleo/storage/node-data-0/jwt_secret_{address}.txt`.
+  Note, if you are running different nodes with different addresses there may be multiple of these. The error message will tell you which one to migrate. 
+- The router peer cache located at `~/.aleo/storage/ledger-0/cache_router_peers`. Migrate it to `~/.aleo/storage/node-data-0/router-peer-cache`.
+- The gateway peer cache located at `~/.aleo/storage/ledger-0/cache_gateway_peers` (only for validators). Migrate it to `~/.aleo/storage/node-data-0/gateway-peer-cache`.
+- The latest proposal cache located at `~/.aleo/storage/current-proposal-cache-0` (only for validators). Migrate it to `~/.aleo/storage/node-data-0/current-proposal-cache`.
 
 ## 5. Command Line Interface
 
@@ -299,30 +313,145 @@ SUBCOMMANDS:
 
 The following are the options for the `snarkos start` command:
 ```
-USAGE:
-    snarkos start [OPTIONS]
+      --network <NETWORK>
+          Specify the network ID of this node [options: 0 = mainnet, 1 = testnet, 2 = canary]
+
+          [default: 0]
 
-OPTIONS:
-        --network <NETWORK_ID>                  Specify the network ID of this node [default: 3]
-
-        --validator                             Specify this node as a validator
-        --prover                                Specify this node as a prover
-        --client                                Specify this node as a client
-
-        --private-key <PRIVATE_KEY>             Specify the node's account private key
-        --private-key-file <PRIVATE_KEY_FILE>   Specify the path to a file containing the node's account private key
-
-        --node <IP:PORT>                        Specify the IP address and port for the node server [default: 0.0.0.0:4130]
-        --connect <IP:PORT>                     Specify the IP address and port of a peer to connect to
-
-        --rest <REST>                           Specify the IP address and port for the REST server [default: 0.0.0.0:3030]
-        --norest                                If the flag is set, the node will not initialize the REST server
-
-        --nodisplay                             If the flag is set, the node will not render the display
-        --verbosity <VERBOSITY_LEVEL>           Specify the verbosity of the node [options: 0, 1, 2, 3] [default: 2]
-        --logfile <PATH>                        Specify the path to the file where logs will be stored [default: /tmp/snarkos.log]
-
-        --dev <NODE_ID>                         Enables development mode, specify a unique ID for this node
+      --prover
+          Start the node as a prover
+
+      --client
+          Start the node as a client (default).
+
+          Client are "full nodes", i.e, validate and execute all blocks they receive, but they do not participate in AleoBFT consensus.
+
+      --bootstrap-client
+          Start the node as a bootstrap client.
+
+      --validator
+          Start the node as a validator.
+
+          Validators are "full nodes", like clients, but also participate in AleoBFT.
+
+      --noupdater
+          Disable checking for new versions at startup
+
+      --private-key <PRIVATE_KEY>
+          Specify the account private key of the node
+
+      --private-key-file <PRIVATE_KEY_FILE>
+          Specify the path to a file containing the account private key of the node
+
+      --node <NODE>
+          Set the IP address and port used for P2P communication
+
+      --bft <BFT>
+          Set the IP address and port used for BFT communication. This argument is only allowed for validator nodes
+
+      --peers <PEERS>
+          Specify the IP address and port of the peer(s) to connect to (as a comma-separated list).
+
+          These peers will be set as "trusted", which means the node will not disconnect from them when performing peer rotation.
+
+          Setting peers to "" has the same effect as not setting the flag at all, except when using `--dev`.
+
+      --validators <VALIDATORS>
+          Specify the IP address and port of the validator(s) to connect to
+
+      --rest <REST>
+          Specify the IP address and port for the REST server
+
+      --rest-rps <REST_RPS>
+          Specify the requests per second (RPS) rate limit per IP for the REST server
+
+          [default: 10]
+
+      --jwt-secret <JWT_SECRET>
+          Specify the JWT secret for the REST server (16B, base64-encoded)
+
+      --jwt-timestamp <JWT_TIMESTAMP>
+          Specify the JWT creation timestamp; can be any time in the last 10 years
+
+      --norest
+          If the flag is set, the node will not initialize the REST server
+
+      --nojwt
+          If the flag is set, the node will not require JWT authentication for the REST server
+
+      --trusted-peers-only
+          If the flag is set, the node will only connect to trusted peers and validators
+
+      --nodisplay
+          Write log message to stdout instead of showing a terminal UI.
+
+          This is useful, for example, for running a node as a service instead of in the foreground or to pipe its output into a file.
+
+      --verbosity <VERBOSITY>
+          Specify the log verbosity of the node. [options: 0 (lowest log level) to 6 (highest level)]
+
+          [default: 1]
+
+      --log-filter <LOG_FILTER>
+          Set a custom log filtering scheme, e.g., "off,snarkos_bft=trace", to show all log messages of snarkos_bft but nothing else
+
+      --logfile <LOGFILE>
+          Specify the path to the file where logs will be stored
+
+          [default: /var/folders/6v/1bwnpyjd1r5f9wr_9hq25qsm0000gn/T/snarkos.log]
+
+      --metrics
+          Enable the metrics exporter
+
+      --metrics-ip <METRICS_IP>
+          Specify the IP address and port for the metrics exporter
+
+      --ledger-storage <LEDGER_STORAGE>
+          Specify the directory that holds all ledger data, e.g., blocks and transactions.
+          This flag overrides the default path, even when `--dev` is set.
+
+          The old name for this flag (`--storage`) is DEPRECATED and will eventually be removed.
+
+      --node-data-storage <NODE_DATA_STORAGE>
+          Specify the directory that holds node-specific data, that is not part of the global ledger.
+          This flag overrides the default path, even when `--dev` is set.
+
+          That folder may contain sensitive data, such as the JWT secret, and should not be shared with untrusted parties.
+          For validators, it also contains the latest proposal cache, which is required to participate in consensus.
+
+      --cdn <CDN>
+          Enables the node to prefetch initial blocks from a CDN
+
+      --nocdn
+          If the flag is set, the node will not prefetch from a CDN
+
+      --dev <DEV>
+          Enables development mode used to set up test networks.
+
+          The purpose of this flag is to run multiple nodes on the same machine and in the same working directory.
+          To do this, set the value to a unique ID within the test work. For example if there are four nodes in the network, pass `--dev 0` for the first node, `--dev 1` for the second, and so forth.
+
+          If you do not explicitly set the `--peers` flag, this will also populate the set of trusted peers, so that the network is fully connected.
+          Additionally, if you do not set the `--rest` or the `--norest` flags, it will also set the REST port to `3030` for the first node, `3031` for the second, and so forth.
+
+      --dev-num-validators <DEV_NUM_VALIDATORS>
+          If development mode is enabled, specify the number of genesis validator
+
+          [default: 4]
+
+      --dev-num-clients <DEV_NUM_CLIENTS>
+          If development mode is enabled, specify the number of clients. This is only used by validators to automatically populate their set of trusted peers.
+
+          This option cannot be used while also passing the `--peers` flag.
+
+      --no-dev-txs
+          If development mode is enabled, specify whether node 0 should generate traffic to drive the network
+
+      --dev-bonded-balances <DEV_BONDED_BALANCES>
+          If development mode is enabled, specify the custom bonded balances as a JSON object
+
+  -h, --help
+          Print help (see a summary with '-h')
 ```
 
 ## 6. Development Guide

cli/src/commands/clean.rs

@@ -13,11 +13,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use snarkos_node::bft::helpers::proposal_cache_path;
+use crate::helpers::args::parse_node_data_dir;
+
+use snarkos_utilities::NodeDataDir;
+
 use snarkvm::console::network::{CanaryV0, MainnetV0, Network};
 
 use aleo_std::StorageMode;
-use anyhow::{Result, bail};
+use anyhow::{Context, Result};
 use clap::Parser;
 use colored::Colorize;
 use std::path::PathBuf;
@@ -28,36 +31,53 @@ pub struct Clean {
     /// Specify the network to remove from storage (0 = mainnet, 1 = testnet, 2 = canary)
     #[clap(default_value_t=MainnetV0::ID, long = "network", value_parser = clap::value_parser!(u16).range((MainnetV0::ID as i64)..=(CanaryV0::ID as i64)))]
     pub network: u16,
+
     /// Enables development mode, specify the unique ID of the local node to clean.
     #[clap(long)]
     pub dev: Option<u16>,
+
     /// Specify the path to a directory containing the ledger. Overrides the default path (also for
     /// dev).
     #[clap(long = "path")]
     pub path: Option<PathBuf>,
+
+    /// Sets a custom path for the node configuration.
+    #[clap(long)]
+    pub node_data_path: Option<PathBuf>,
 }
 
 impl Clean {
     /// Cleans the snarkOS node storage.
     pub fn parse(self) -> Result<String> {
-        // Initialize the storage mode.
+        // Remove the specified node configuration from storage.
+        let node_data_dir = parse_node_data_dir(&self.node_data_path, self.network, self.dev)?;
+        println!("{}", Self::remove_node_data(&node_data_dir)?);
+
+        // Remove the specified ledger from storage.
         let storage_mode = match self.path {
             Some(path) => StorageMode::Custom(path),
             None => match self.dev {
                 Some(id) => StorageMode::Development(id),
                 None => StorageMode::Production,
             },
         };
+        Self::remove_ledger(self.network, &storage_mode)
+    }
+
+    /// Removes the specified node configuration from storage.
+    fn remove_node_data(node_data_dir: &NodeDataDir) -> Result<String> {
+        // With the new layout, we can remove the entire folder.
+        let data_path = node_data_dir.path();
+
+        // Prepare the path string.
+        let path_string = format!("(in \"{}\")", data_path.display()).dimmed();
 
-        // Remove the current proposal cache file, if it exists.
-        let proposal_cache_path = proposal_cache_path(self.network, &storage_mode);
-        if proposal_cache_path.exists() {
-            if let Err(err) = std::fs::remove_file(&proposal_cache_path) {
-                bail!("Failed to remove the current proposal cache file at {}: {err}", proposal_cache_path.display());
-            }
+        if data_path.exists() {
+            std::fs::remove_dir_all(data_path).with_context(|| format!("Failed to remove node data {path_string}"))?;
+            Ok(format!("✅ Cleaned up node data {path_string}"))
+        } else {
+            Ok(format!("✅ No node data was found {path_string}"))
         }
-        // Remove the specified ledger from storage.
-        Self::remove_ledger(self.network, &storage_mode)
     }
 
     /// Removes the specified ledger from storage.
@@ -71,14 +91,11 @@ impl Clean {
         // Check if the path to the ledger exists in storage.
         if path.exists() {
             // Remove the ledger files from storage.
-            match std::fs::remove_dir_all(&path) {
-                Ok(_) => Ok(format!("✅ Cleaned the snarkOS node storage {path_string}")),
-                Err(error) => {
-                    bail!("Failed to remove the snarkOS node storage {path_string}\n{}", error.to_string().dimmed())
-                }
-            }
+            std::fs::remove_dir_all(&path)
+                .with_context(|| format!("Failed to remove the snarkOS ledger {path_string}"))?;
+            Ok(format!("✅ Cleaned the snarkOS ledger {path_string}"))
         } else {
-            Ok(format!("✅ No snarkOS node storage was found {path_string}"))
+            Ok(format!("✅ No snarkOS ledger was found {path_string}"))
         }
     }
 }