diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 72c3f96b04..6761324359 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -157,12 +157,18 @@ if(CLP_NEED_BOOST) if(CLP_USE_STATIC_LIBS) set(Boost_USE_STATIC_LIBS ON) endif() - find_package(Boost 1.89 REQUIRED iostreams program_options filesystem regex url) - if(Boost_FOUND) - message(STATUS "Found Boost ${Boost_VERSION}") - else() - message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for Boost") - endif() + find_package( + Boost + 1.81...1.88 + REQUIRED + filesystem + iostreams + program_options + regex + system + url + ) + message(STATUS "Found Boost ${Boost_VERSION}") endif() if(CLP_NEED_CATCH2) @@ -271,14 +277,10 @@ if(CLP_NEED_MONGOCXX) endif() endif() -# Find and setup msgpack +# Find and setup msgpack-cxx if(CLP_NEED_MSGPACKCXX) find_package(msgpack-cxx 7.0.0 REQUIRED) - if(msgpack-cxx_FOUND) - message(STATUS "Found msgpack-cxx ${msgpack-cxx_VERSION}") - else() - message(FATAL_ERROR "Could not find msgpack-cxx") - endif() + message(STATUS "Found msgpack-cxx ${msgpack-cxx_VERSION}") endif() find_package(Threads REQUIRED) @@ -731,7 +733,11 @@ if(CLP_BUILD_TESTING) target_link_libraries(unitTest PRIVATE absl::flat_hash_map - Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url + Boost::filesystem + Boost::iostreams + Boost::program_options + Boost::regex + Boost::url Catch2::Catch2WithMain ${CURL_LIBRARIES} clp_s::search::ast @@ -745,6 +751,7 @@ if(CLP_BUILD_TESTING) LibLZMA::LibLZMA MariaDBClient::MariaDBClient ${MONGOCXX_TARGET} + msgpack-cxx nlohmann_json::nlohmann_json simdjson::simdjson spdlog::spdlog diff --git a/components/core/cmake/Options/options.cmake b/components/core/cmake/Options/options.cmake index 42edf242e1..a66d489fd9 100644 --- a/components/core/cmake/Options/options.cmake +++ b/components/core/cmake/Options/options.cmake @@ -284,6 +284,7 @@ function(set_clp_s_json_constructor_dependencies) CLP_NEED_FMT CLP_NEED_MONGOCXX CLP_NEED_SPDLOG + CLP_NEED_ZSTD ) endfunction() diff --git a/components/core/src/clp/clg/CMakeLists.txt b/components/core/src/clp/clg/CMakeLists.txt index 659ad5c737..d96f240b67 100644 --- a/components/core/src/clp/clg/CMakeLists.txt +++ b/components/core/src/clp/clg/CMakeLists.txt @@ -144,6 +144,7 @@ if(CLP_BUILD_EXECUTABLES) fmt::fmt log_surgeon::log_surgeon MariaDBClient::MariaDBClient + msgpack-cxx nlohmann_json::nlohmann_json spdlog::spdlog ${sqlite_LIBRARY_DEPENDENCIES} diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index b4a1c4108e..c0bf965a61 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -185,6 +185,7 @@ if(CLP_BUILD_EXECUTABLES) ${sqlite_LIBRARY_DEPENDENCIES} LibArchive::LibArchive MariaDBClient::MariaDBClient + msgpack-cxx nlohmann_json::nlohmann_json ${STD_FS_LIBS} clp::string_utils diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 46cbda49d3..840d90c2ee 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -18,7 +18,7 @@ set( ../clp/cli_utils.hpp ../clp/Defs.h ../clp/EncodedVariableInterpreter.cpp - ../clp/EncodedVariableInterpreter.hpp + ../clp/EncodedVariableInterpreter.hpp ../clp/ErrorCode.hpp ../clp/ffi/encoding_methods.cpp ../clp/ffi/encoding_methods.hpp @@ -69,7 +69,7 @@ set( ../clp/ir/parsing.inc ../clp/ir/types.hpp ../clp/LogSurgeonReader.cpp - ../clp/LogSurgeonReader.hpp + ../clp/LogSurgeonReader.hpp ../clp/NetworkReader.cpp ../clp/NetworkReader.hpp ../clp/networking/socket_utils.cpp @@ -77,7 +77,7 @@ set( ../clp/Query.cpp ../clp/Query.hpp ../clp/QueryToken.cpp - ../clp/QueryToken.hpp + ../clp/QueryToken.hpp ../clp/ReaderInterface.cpp ../clp/ReaderInterface.hpp ../clp/ReadOnlyMemoryMappedFile.cpp @@ -115,6 +115,7 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) clp::string_utils log_surgeon::log_surgeon ystdlib::containers + ${zstd_TARGET} PRIVATE Boost::regex ${CURL_LIBRARIES} @@ -124,7 +125,6 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) OpenSSL::Crypto spdlog::spdlog ystdlib::error_handling - ${zstd_TARGET} ) endif() @@ -354,6 +354,7 @@ if(CLP_BUILD_CLP_S_JSONCONSTRUCTOR) clp_s_json_constructor PUBLIC clp_s::archive_reader + ${zstd_TARGET} PRIVATE fmt::fmt ${MONGOCXX_TARGET} diff --git a/components/core/src/clp_s/indexer/CMakeLists.txt b/components/core/src/clp_s/indexer/CMakeLists.txt index dbf8c35006..e63b4021fe 100644 --- a/components/core/src/clp_s/indexer/CMakeLists.txt +++ b/components/core/src/clp_s/indexer/CMakeLists.txt @@ -15,7 +15,7 @@ set( ../../clp/database_utils.hpp ../../clp/Defs.h ../../clp/EncodedVariableInterpreter.cpp - ../../clp/EncodedVariableInterpreter.hpp + ../../clp/EncodedVariableInterpreter.hpp ../../clp/ErrorCode.hpp ../../clp/ffi/encoding_methods.cpp ../../clp/ffi/encoding_methods.hpp @@ -118,6 +118,7 @@ if(CLP_BUILD_EXECUTABLES) clp::string_utils date::date MariaDBClient::MariaDBClient + msgpack-cxx nlohmann_json::nlohmann_json OpenSSL::Crypto simdjson::simdjson diff --git a/components/core/src/reducer/CMakeLists.txt b/components/core/src/reducer/CMakeLists.txt index c4818166d4..ec830624b8 100644 --- a/components/core/src/reducer/CMakeLists.txt +++ b/components/core/src/reducer/CMakeLists.txt @@ -44,6 +44,7 @@ if(CLP_BUILD_EXECUTABLES) target_link_libraries(reducer-server PRIVATE Boost::program_options + Boost::system clp::string_utils fmt::fmt ${MONGOCXX_TARGET} diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-packages-from-source.sh index 4757c5c7b6..a71856c1af 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-packages-from-source.sh @@ -9,8 +9,4 @@ set -u script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir="${script_dir}/.." -# NOTE: The remaining installation scripts depend on boost, so we install it beforehand. -"${lib_install_scripts_dir}/install-boost.sh" 1.89.0 - "${lib_install_scripts_dir}/liblzma.sh" 5.8.1 -"${lib_install_scripts_dir}/msgpack.sh" 7.0.0 diff --git a/components/core/tools/scripts/lib_install/install-boost.sh b/components/core/tools/scripts/lib_install/install-boost.sh deleted file mode 100755 index 2733e9886c..0000000000 --- a/components/core/tools/scripts/lib_install/install-boost.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -# Exit on error -set -e - -cUsage="Usage: ${BASH_SOURCE[0]} " -if [ "$#" -lt 1 ] ; then - echo $cUsage - exit -fi -version=$1 -version_with_underscores=${version//./_} - -echo "Checking for elevated privileges..." -if [ ${EUID:-$(id -u)} -ne 0 ] ; then - sudo echo "Script can elevate privileges." -fi - -# Get number of cpu cores -num_cpus=$(grep -c ^processor /proc/cpuinfo) - -package_name=boost - -# Create temp dir for installation -temp_dir=/tmp/${package_name}-installation -mkdir -p $temp_dir - -cd $temp_dir - -# Download source -tar_filename=boost_${version_with_underscores}.tar.gz -curl -fsSL https://archives.boost.io/release/${version}/source/${tar_filename} -o ${tar_filename} -tar xzf ${tar_filename} -cd boost_${version_with_underscores} - -# Build -./bootstrap.sh --with-libraries=filesystem,iostreams,program_options,regex,system,url -./b2 -j${num_cpus} - -# Install -if [ ${EUID:-$(id -u)} -ne 0 ] ; then - sudo ./b2 install -else - ./b2 install -fi - -# Clean up -rm -rf $temp_dir diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 06b5798e62..78327c4956 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -27,7 +27,6 @@ if ! command -v uv ; then fi brew install \ - boost \ coreutils \ gcc \ java11 \ @@ -35,7 +34,6 @@ brew install \ llvm@16 \ lz4 \ mariadb-connector-c \ - msgpack-cxx \ xz \ zstd diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh index 8cb33f4f4b..dc185d7406 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh @@ -6,9 +6,6 @@ set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir="${script_dir}/.." -# NOTE: The remaining installation scripts depend on boost, so we install it beforehand. -"${lib_install_scripts_dir}/install-boost.sh" 1.89.0 - # NOTE: # 1. libarchive may statically link with LZMA, LZ4, and Zstandard, so we install them beforehand. # 2. The versions of libarchive, LZMA, LZ4, and Zstandard available in manylinux_2_28's package @@ -18,5 +15,3 @@ lib_install_scripts_dir="${script_dir}/.." "${lib_install_scripts_dir}/lz4.sh" 1.10.0 "${lib_install_scripts_dir}/zstandard.sh" 1.5.7 "${lib_install_scripts_dir}/libarchive.sh" 3.8.0 - -"${lib_install_scripts_dir}/msgpack.sh" 7.0.0 diff --git a/components/core/tools/scripts/lib_install/msgpack.sh b/components/core/tools/scripts/lib_install/msgpack.sh deleted file mode 100755 index a89cba0715..0000000000 --- a/components/core/tools/scripts/lib_install/msgpack.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# Dependencies: -# - Boost -# NOTE: Dependencies should be installed outside the script to allow the script to be largely distro-agnostic - -# Exit on any error -set -e - -cUsage="Usage: ${BASH_SOURCE[0]} [ <.deb output directory>]" -if [ "$#" -lt 1 ] ; then - echo $cUsage - exit -fi -version=$1 - -package_name=msgpackc-cxx -temp_dir=/tmp/${package_name}-installation -deb_output_dir=${temp_dir} -if [[ "$#" -gt 1 ]] ; then - deb_output_dir="$(readlink -f "$2")" - if [ ! -d ${deb_output_dir} ] ; then - echo "${deb_output_dir} does not exist or is not a directory" - exit - fi -fi - -# Check if already installed -set +e -dpkg -l ${package_name} | grep ${version} -installed=$? -set -e -if [ $installed -eq 0 ] ; then - # Nothing to do - exit -fi - -echo "Checking for elevated privileges..." -install_cmd_args=() -if [ ${EUID:-$(id -u)} -ne 0 ] ; then - sudo echo "Script can elevate privileges." - install_cmd_args+=("sudo") -fi - -# Download -mkdir -p $temp_dir -cd $temp_dir -extracted_dir=${temp_dir}/msgpack-cxx-${version} -if [ ! -e ${extracted_dir} ] ; then - tar_filename=msgpack-cxx-${version}.tar.gz - if [ ! -e ${tar_filename} ] ; then - curl -fsSL https://github.com/msgpack/msgpack-c/releases/download/cpp-${version}/${tar_filename} -o ${tar_filename} - fi - - tar -xf ${tar_filename} -fi - -# Set up -cd ${extracted_dir} -cmake . - -# Check if checkinstall is installed -set +e -command -v checkinstall -checkinstall_installed=$? -set -e - -# Install -if [ $checkinstall_installed -eq 0 ] ; then - install_cmd_args+=( - checkinstall - --default - --fstrans=no - --nodoc - --pkgname "${package_name}" - --pkgversion "${version}" - --provides "${package_name}" - --pakdir "${deb_output_dir}" - ) -fi -install_cmd_args+=( - cmake - --build . - --target install -) -"${install_cmd_args[@]}" - -# Clean up -rm -rf $temp_dir diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh index bf1575284c..645f95dd4a 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh @@ -6,9 +6,6 @@ set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir="${script_dir}/.." -# NOTE: The remaining installation scripts depend on boost, so we install it beforehand. -"${lib_install_scripts_dir}/install-boost.sh" 1.89.0 - # NOTE: # 1. libarchive may statically link with LZMA, LZ4, and Zstandard, so we install them beforehand. # 2. The versions of libarchive, LZMA, LZ4, and Zstandard available in musllinux_1_2's package @@ -18,5 +15,3 @@ lib_install_scripts_dir="${script_dir}/.." "${lib_install_scripts_dir}/lz4.sh" 1.10.0 "${lib_install_scripts_dir}/zstandard.sh" 1.5.7 "${lib_install_scripts_dir}/libarchive.sh" 3.8.0 - -"${lib_install_scripts_dir}/msgpack.sh" 7.0.0 diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh index f8dfc74465..7b0488beff 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh @@ -9,11 +9,7 @@ set -u script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir=$script_dir/.. -# NOTE: boost must be installed first since the remaining packages depend on it -"$lib_install_scripts_dir"/install-boost.sh 1.89.0 - "$lib_install_scripts_dir"/libarchive.sh 3.5.1 "$lib_install_scripts_dir"/liblzma.sh 5.8.1 "$lib_install_scripts_dir"/lz4.sh 1.10.0 -"$lib_install_scripts_dir"/msgpack.sh 7.0.0 "$lib_install_scripts_dir"/zstandard.sh 1.5.7 diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index a8d36d8c87..272fcc0269 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -37,6 +37,7 @@ The task will download, build, and install (within the build directory) the foll |-----------------------------------------------------------------------|----------------| | [abseil-cpp](https://github.com/abseil/abseil-cpp) | 20250512.0 | | [ANTLR](https://www.antlr.org) | v4.13.2 | +| [Boost](https://github.com/boostorg/boost) | v1.87.0 | | [Catch2](https://github.com/catchorg/Catch2.git) | v3.8.0 | | [date](https://github.com/HowardHinnant/date.git) | v3.0.1 | | [fmt](https://github.com/fmtlib/fmt) | v10.2.1 | @@ -45,6 +46,7 @@ The task will download, build, and install (within the build directory) the foll | [liblzma](https://github.com/tukaani-project/xz) | v5.8.1 | | [lz4](https://github.com/lz4/lz4) | v1.10.0 | | [mongo-cxx-driver](https://github.com/mongodb/mongo-cxx-driver) | r3.10.2 | +| [msgpack-cxx](https://github.com/msgpack/msgpack-c/tree/cpp_master) | v7.0.0 | | [simdjson](https://github.com/simdjson/simdjson) | v3.13.0 | | [spdlog](https://github.com/gabime/spdlog) | v1.14.1 | | [SQLite3](https://www.sqlite.org/download.html) | v3.36.0 | diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index ec97e1e273..8eaf638d52 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -18,6 +18,7 @@ vars: G_DEPS_CORE_CMAKE_SETTINGS_DIR: "{{.G_DEPS_CORE_DIR}}/cmake-settings" # Library names + G_BOOST_LIB_NAME: "Boost" G_FMT_LIB_NAME: "fmt" G_GSL_LIB_NAME: "Microsoft.GSL" @@ -25,6 +26,10 @@ vars: G_ANTLR_VERSION: "4.13.2" G_ANTLR_JAR_FILE: "{{.G_DEPS_CORE_DIR}}/antlr-{{.G_ANTLR_VERSION}}-complete.jar" +# NOTE: For dependencies built using CMake, we set CMP0074 to NEW whenever: +# - a component's minimum required CMake version is less than 3.27 (where CMP0074 defaults to OLD); +# and +# - the component depends on another via `_ROOT`. tasks: default: deps: @@ -63,6 +68,7 @@ tasks: - task: "absl" - task: "antlr-jar" - task: "antlr-runtime" + - task: "boost" - task: "catch2" - task: "date" - task: "fmt" @@ -71,6 +77,7 @@ tasks: - task: "lz4" - task: "microsoft.gsl" - task: "mongocxx" + - task: "msgpack-cxx" - task: "nlohmann_json" - task: "simdjson" - task: "spdlog" @@ -181,6 +188,27 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.INSTALL_PREFIX}}"] + boost: + internal: true + vars: + VERSION: "1.87.0" + run: "once" + cmds: + - task: "yscope-dev-utils:boost:download-and-install" + vars: + CMAKE_SETTINGS_DIR: "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}" + FILE_SHA256: "d6c69e4459eb5d6ec208250291221e7ff4a2affde9af6e49c9303b89c687461f" + TARGETS: + - "filesystem" + - "iostreams" + - "program_options" + - "regex" + - "system" + - "url" + URL: "https://github.com/boostorg/boost/releases/download/boost-{{.VERSION}}/\ + boost-{{.VERSION}}-b2-nodocs.tar.gz" + WORK_DIR: "{{.G_DEPS_CORE_DIR}}" + catch2: internal: true run: "once" @@ -343,6 +371,28 @@ tasks: TARBALL_URL: "https://github.com/mongodb/mongo-cxx-driver/releases/download/{{.VERSION}}\ /mongo-cxx-driver-{{.VERSION}}.tar.gz" + msgpack-cxx: + internal: true + vars: + VERSION: "7.0.0" + run: "once" + deps: + - task: "boost" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-C {{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.G_BOOST_LIB_NAME}}.cmake" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" + - "-DMSGPACK_BUILD_DOCS=OFF" + - "-DMSGPACK_CXX20=ON" + LIB_NAME: "msgpack-cxx" + TARBALL_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" + TARBALL_URL: "https://github.com/msgpack/msgpack-c/releases/download/cpp-{{.VERSION}}\ + /msgpack-cxx-{{.VERSION}}.tar.gz" + nlohmann_json: internal: true run: "once" @@ -386,11 +436,7 @@ tasks: - "-C {{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.G_FMT_LIB_NAME}}.cmake" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - # Set CMP0074 since our minimum required CMake version is less than 3.27, and we - # depend on fmt's settings file, which uses `fmt_ROOT`. - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - - "-DSPDLOG_BUILD_EXAMPLE=OFF" - "-DSPDLOG_BUILD_EXAMPLE_HO=OFF" - "-DSPDLOG_FMT_EXTERNAL=ON"