feat(clp-s::log_converter): Add max-log-event-size argument; Make default max log event size 512 MiB to match clp-s (fixes #2176). (#2193)

Co-authored-by: Junhao Liao <junhao.liao@yscope.com>
Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com>

Author: 3 people

components/core/src/clp_s/log_converter/CommandLineArguments.cpp

@@ -137,6 +137,12 @@ auto CommandLineArguments::parse_arguments(int argc, char const** argv)
                 "Type of authentication required for network requests (s3 | none). Authentication"
                 " with s3 requires the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment"
                 " variables, and optionally the AWS_SESSION_TOKEN environment variable."
+        )(
+                "max-log-event-size",
+                po::value<size_t>(&m_max_log_event_size)
+                    ->value_name("LOG_EVENT_SIZE")
+                    ->default_value(m_max_log_event_size),
+                "Maximum allowed size (B) for a single log event before conversion fails."
         );
         // clang-format on
 
@@ -188,6 +194,10 @@ auto CommandLineArguments::parse_arguments(int argc, char const** argv)
         }
 
         validate_network_auth(auth, m_network_auth);
+
+        if (m_max_log_event_size <= 0) {
+            throw std::invalid_argument("Max event size must be greater than zero.");
+        }
     } catch (std::exception& e) {
         SPDLOG_ERROR("{}", e.what());
         print_basic_usage();

components/core/src/clp_s/log_converter/CommandLineArguments.hpp

@@ -1,6 +1,7 @@
 #ifndef CLP_S_COMMANDLINEARGUMENTS_HPP
 #define CLP_S_COMMANDLINEARGUMENTS_HPP
 
+#include <cstddef>
 #include <cstdint>
 #include <string>
 #include <string_view>
@@ -34,6 +35,8 @@ class CommandLineArguments {
 
     [[nodiscard]] auto get_output_dir() const -> std::string const& { return m_output_dir; }
 
+    [[nodiscard]] auto get_max_log_event_size() const -> size_t { return m_max_log_event_size; }
+
 private:
     // Methods
     void print_basic_usage() const;
@@ -43,6 +46,7 @@ class CommandLineArguments {
     std::vector<Path> m_input_paths;
     NetworkAuthOption m_network_auth{};
     std::string m_output_dir{"./"};
+    size_t m_max_log_event_size{512ULL * 1024ULL * 1024ULL};  // 512 MiB
 };
 }  // namespace clp_s::log_converter
 

components/core/src/clp_s/log_converter/LogConverter.cpp

@@ -136,7 +136,7 @@ auto LogConverter::grow_buffer_if_full() -> ystdlib::error_handling::Result<void
     }
 
     size_t const new_size{2 * m_buffer.size()};
-    if (new_size > cMaxBufferSize) {
+    if (new_size > m_max_buffer_size) {
         return std::errc::result_out_of_range;
     }
     ystdlib::containers::Array<char> new_buffer(new_size);

components/core/src/clp_s/log_converter/LogConverter.hpp

@@ -17,7 +17,9 @@ namespace clp_s::log_converter {
 class LogConverter {
 public:
     // Constructors
-    LogConverter() : m_buffer(cDefaultBufferSize) {}
+    explicit LogConverter(size_t max_buffer_size)
+            : m_buffer(max_buffer_size < cDefaultBufferSize ? max_buffer_size : cDefaultBufferSize),
+              m_max_buffer_size{max_buffer_size} {}
 
     // Methods
     /**
@@ -38,7 +40,6 @@ class LogConverter {
 private:
     // Constants
     static constexpr size_t cDefaultBufferSize{64ULL * 1024ULL};  // 64 KiB
-    static constexpr size_t cMaxBufferSize{64ULL * 1024ULL * 1024ULL};  // 64 MiB
 
     // Methods
     /**
@@ -68,6 +69,7 @@ class LogConverter {
     ystdlib::containers::Array<char> m_buffer;
     size_t m_num_bytes_buffered{};
     size_t m_parser_offset{};
+    size_t m_max_buffer_size{cDefaultBufferSize};
 };
 }  // namespace clp_s::log_converter
 #endif  // CLP_S_LOG_CONVERTER_LOGCONVERTER_HPP

components/core/src/clp_s/log_converter/log_converter.cpp

@@ -26,7 +26,7 @@ namespace {
 [[nodiscard]] auto convert_files(CommandLineArguments const& command_line_arguments) -> bool;
 
 auto convert_files(CommandLineArguments const& command_line_arguments) -> bool {
-    LogConverter log_converter;
+    LogConverter log_converter{command_line_arguments.get_max_log_event_size()};
 
     std::error_code ec{};
     if (false == std::filesystem::create_directory(command_line_arguments.get_output_dir(), ec)