Check # CPUs + memory size on startup

Author: drebelsky

Cargo.lock

@@ -57,6 +57,7 @@
 #include <iostream>
 #include <lib/clara.hpp>
 #include <optional>
+#include <thread>
 
 namespace stellar
 {
@@ -1607,6 +1608,121 @@ runReportLastHistoryCheckpoint(CommandLineArgs const& args)
         });
 }
 
+namespace
+{
+// Before starting the application, we want to check that the host machine meets
+// the minimum system requirements we document (16 GiB RAM, 8 vCPUs). To make
+// the check meaningful, we prevent the node from starting when it fails
+// (instead of, e.g., writing a log message that may go unread). In the case
+// that we aren't able to automatically determine the relevant system
+// information, we allow the operator to set config values with their system
+// information. Note: we intentionally have these as integer values instead of
+// booleans so that there isn't a silent failure if we bump the minimum
+// requirements. If the auto-detected (or operator-provided, in the case of
+// auto-detection failure) system information doesn't meet the minimum
+// requirements, we require the operator to set an additional config value to
+// explicitly acknowledge that they are ignoring the warning.
+
+// We want to make the flag value annoying enough to set that the operators have
+// to make an intentional and continuous decision to ignore the warning. We use
+// the version to make sure that every time they upgrade the package, they have
+// to make a new decision to ignore the warning, and we use the public key to
+// make sure that the value is unique per node. Notably, we don't use something
+// that depends on the current time so that restarts after crashes are handled
+// gracefully (assuming the package wasn't upgraded in between).
+bool
+validateSystemInfo(Config const& cfg)
+{
+    std::string annoyingValue =
+        fmt::format(FMT_STRING("{}-{}"), STELLAR_CORE_VERSION,
+                    KeyUtils::toStrKey(cfg.NODE_SEED.getPublicKey()));
+
+    uint64_t memory = rust_bridge::get_host_total_memory();
+    if (memory == 0)
+    {
+        if (!cfg.SYSCHECK_UNKNOWN_MEMORY_DEFAULT)
+        {
+            LOG_ERROR(DEFAULT_LOG,
+                      "Unable to determine total memory of the host; please "
+                      "ensure that the system has at least 16 GiB of RAM. Once "
+                      "confirmed, set SYSCHECK_UNKNOWN_MEMORY_DEFAULT to the "
+                      "size of RAM in KiB.");
+            return false;
+        }
+
+        LOG_WARNING(DEFAULT_LOG,
+                    "Unable to determine total memory of the host; using "
+                    "SYSCHECK_UNKNOWN_MEMORY_DEFAULT value of {} KiB for "
+                    "checks. Please ensure this is still the correct value.",
+                    cfg.SYSCHECK_UNKNOWN_MEMORY_DEFAULT);
+
+        memory = cfg.SYSCHECK_UNKNOWN_MEMORY_DEFAULT;
+    }
+
+    if (memory < static_cast<uint32_t>(16) * 1024 * 1024)
+    {
+        if (cfg.SYSCHECK_FORCE_IGNORE_MEMORY != annoyingValue)
+        {
+            LOG_ERROR(
+                DEFAULT_LOG,
+                "Host only has {} KiB of RAM; stellar-core may not function "
+                "properly under heavy load; please ensure that the system has "
+                "at least 16 GiB of RAM. To force ignore this warning, set "
+                "SYSCHECK_FORCE_IGNORE_MEMORY to \"{}\". Note that this value "
+                "differs for every node and version.",
+                memory, annoyingValue);
+            return false;
+        }
+        LOG_WARNING(
+            DEFAULT_LOG,
+            "Host only has {} KiB of RAM; the recommended minimum is 16 GiB",
+            memory);
+    }
+
+    unsigned int cpus = std::thread::hardware_concurrency();
+    if (cpus == 0)
+    {
+        if (!cfg.SYSCHECK_UNKNOWN_CPU_DEFAULT)
+        {
+            LOG_ERROR(
+                DEFAULT_LOG,
+                "Unable to determine number of vCPUs of the host; please "
+                "ensure that the system has at least 8 vCPUs. Once confirmed, "
+                "set SYSCHECK_UNKNOWN_CPU_DEFAULT to the number of vCPUs.");
+            return false;
+        }
+
+        LOG_WARNING(DEFAULT_LOG,
+                    "Unable to determine number of vCPUs of the host; using "
+                    "SYSCHECK_UNKNOWN_CPU_DEFAULT value of {} for checks. "
+                    "Please ensure this is still the correct value.",
+                    cfg.SYSCHECK_UNKNOWN_CPU_DEFAULT);
+
+        cpus = cfg.SYSCHECK_UNKNOWN_CPU_DEFAULT;
+    }
+
+    if (cpus < 8)
+    {
+        if (cfg.SYSCHECK_FORCE_IGNORE_CPU != annoyingValue)
+        {
+            LOG_ERROR(DEFAULT_LOG,
+                      "Host only has {} vCPUs; stellar-core may not function "
+                      "properly under heavy load; please ensure that the "
+                      "system has at least 8 vCPUs. To force ignore this "
+                      "warning, set SYSCHECK_FORCE_IGNORE_CPU to \"{}\". Note "
+                      "that value differs for every node and version.",
+                      cpus, annoyingValue);
+            return false;
+        }
+        LOG_WARNING(DEFAULT_LOG,
+                    "Host only has {} vCPUs; the recommended minimum is 8",
+                    cpus);
+    }
+
+    return true;
+}
+} // namespace
+
 int
 run(CommandLineArgs const& args)
 {
@@ -1682,6 +1798,16 @@ run(CommandLineArgs const& args)
                                              "enabled (for testing only)");
                 }
 
+                if (gIsProductionNetwork && cfg.NODE_IS_VALIDATOR &&
+                    !validateSystemInfo(cfg))
+                {
+                    LOG_ERROR(
+                        DEFAULT_LOG,
+                        "Host system does not meet the minimum requirements "
+                        "for running stellar-core. Exiting.");
+                    return 1;
+                }
+
                 // Second, setup the app with the final configuration.
                 clock = std::make_shared<VirtualClock>(clockMode);
                 app = setupApp(cfg, *clock);

src/main/CommandLine.cpp

@@ -1255,6 +1255,18 @@ Config::processConfig(std::shared_ptr<cpptoml::table> t)
                 {"LOG_COLOR", [&]() { LOG_COLOR = readBool(item); }},
                 {"BUCKET_DIR_PATH",
                  [&]() { BUCKET_DIR_PATH = readString(item); }},
+                {"SYSCHECK_UNKNOWN_MEMORY_DEFAULT",
+                 [&]() {
+                     SYSCHECK_UNKNOWN_MEMORY_DEFAULT = readInt<uint32_t>(item);
+                 }},
+                {"SYSCHECK_UNKNOWN_CPU_DEFAULT",
+                 [&]() {
+                     SYSCHECK_UNKNOWN_CPU_DEFAULT = readInt<uint32_t>(item);
+                 }},
+                {"SYSCHECK_FORCE_IGNORE_MEMORY",
+                 [&]() { SYSCHECK_FORCE_IGNORE_MEMORY = readString(item); }},
+                {"SYSCHECK_FORCE_IGNORE_CPU",
+                 [&]() { SYSCHECK_FORCE_IGNORE_CPU = readString(item); }},
                 {"FILTERED_SOROBAN_KEYS_PATH",
                  [&]() {
                      LOG_WARNING(DEFAULT_LOG,

src/main/Config.cpp

@@ -672,6 +672,14 @@ class Config : public std::enable_shared_from_this<Config>
     bool LOG_COLOR;
     std::string BUCKET_DIR_PATH;
 
+    // Config parameters controlling startup system info validation (only
+    // relevant for mainnet validators). See the comment above
+    // validateSystemInfo in CommandLine.cpp for details.
+    uint32_t SYSCHECK_UNKNOWN_MEMORY_DEFAULT{0};
+    uint32_t SYSCHECK_UNKNOWN_CPU_DEFAULT{0};
+    std::string SYSCHECK_FORCE_IGNORE_MEMORY;
+    std::string SYSCHECK_FORCE_IGNORE_CPU;
+
     // Path to Protocol 23 corruption CSV file for testing/recovery
     std::string PATH_TO_PROTOCOL_23_CORRUPTION_FILE;
 

src/main/Config.h

@@ -17,6 +17,7 @@ rustc-simple-version = "=0.1.0"
 # NB: this must match the same rand version used by soroban (but the tooling
 # will complain if it does not match)
 rand = "=0.8.5"
+sys-info = "=0.9.1"
 
 itertools = "=0.10.5"
 backtrace = { version = "=0.3.76", features = [ "cpp_demangle" ] }

src/rust/Cargo.toml

@@ -251,6 +251,10 @@ pub(crate) mod rust_bridge {
         // Check to see if the XDR files used by different rust dependencies match.
         fn check_xdr_version_identities() -> Result<()>;
 
+        // Get the total memory available on the host machine, in kibibytes.
+        // Returns 0 if the value cannot be obtained for any reason.
+        fn get_host_total_memory() -> u64;
+
         // Computes the resource fee given the transaction resource consumption
         // and network configuration.
         fn compute_transaction_resource_fee(

src/rust/src/bridge.rs

@@ -120,3 +120,7 @@ pub(crate) fn check_xdr_version_identities() -> Result<(), Box<dyn std::error::E
     // Add more comparisons between XDR file lists as needed
     Ok(())
 }
+
+pub(crate) fn get_host_total_memory() -> u64 {
+    sys_info::mem_info().map(|mem| mem.total).unwrap_or(0)
+}