From a10c8068bdc85c84c26cf1c2c619d943c502144f Mon Sep 17 00:00:00 2001 From: K1ngfish3r <26593485+K1ngfish3r@users.noreply.github.com> Date: Mon, 17 Nov 2025 00:38:35 +0500 Subject: [PATCH] Squashed 'extern/boost/nowide/' changes from 02f40f0b5f..fa328e8ea5 fa328e8ea5 Include 'Merge pull request #201 from boostorg/boost-166-requirement 2479e86b5c Include 'Update links to regression test matrix' d0f7add871 Include 'Increase CMake version range to 3.20 de3bb558eb Include 'Fix readme badges' f0fc265561 Include 'Merge pull request #191 from boostorg/get-env-thread bffe4ad9b0 Include 'Fix documentation of enable_if_path_t' df13d5c1cd Include 'Fix test for coverage' 47df3041de Include 'CMake: Build tests only on make: Nothing to be done for 'test'. (#182) 12651dad48 Include 'Merge pull request #178 from striezel-stash/fix-typos 1810c37169 Include 'Merge pull request #177 from Jackarain/develop a064a3aa0f Include 'Merge pull request #173 from boostorg/fix-warning 2a0543f827 Include 'Add some missing C++11 requirements' 302f6d63ab Include 'CI: Update GHA container jobs 7d2278ee8b Include 'Increase (internal) version to 11.3.0' a2683462ca Include 'CI: Extend coverage collection to C++14-20 ade7bb9d0c Include 'Update docs regarding C++11 usage' ec5a2eb4d3 Include 'Fix potential macro redefinition in test_traits b98cdaa47a Include 'Remove uneccessary cast in test_swap 40666d41d2 Include 'Update readme 37abcc3f71 Include 'CI: Test standalone release tarball' dcda1fec3f Include 'Prepare release 11.2.0' 495bbcc11f Include 'Merge pull request #161 from Flamefire/coverity 3e81b6e08c Include 'Merge pull request #160 from Flamefire/filebuf_perf a58c678d19 Include 'Merge pull request #159 from Flamefire/simplify_putback 1229a7b905 Include 'Merge pull request #155 from Flamefire/fix-flush 69b9a2a86f Include 'Fix MSYS2 builds' 1119935c51 Include 'Fix sign confusion in filebuf sanity check 300c6de72f Include 'Don't force using the custom filebuf (especially on Cygwin) 92943916cd Include 'Fix return value of sync() when fflush fails 960c2016a2 Include 'CI: Remove Visual Studio 2019 CMake test 89ac8cc5f9 Include 'Docu update especially related to filesystem::path 37ed2c0151 Include 'Enhance CTRL+Z iostream test 6dd7118fc0 Include 'Merge pull request #148 from Flamefire/gcc11_ming-w64-compat 03d71d614c Include 'Merge pull request #147 from Flamefire/fix/missing_include fccb720589 Include 'Fix update of standalone branch git-subtree-dir: extern/boost/nowide git-subtree-split: fa328e8ea5da612fb8f018750e1c35a0c6c6c0a1 --- config/check_attribute_init_priority.cpp | 14 + config/check_lfs_support.cpp | 29 + config/check_movable_fstreams.cpp | 17 + .../nowide/cmake/NowideAddWarnings.cmake | 25 +- extern/boost/nowide/include/nowide/args.hpp | 11 +- extern/boost/nowide/include/nowide/config.hpp | 107 ++- .../boost/nowide/include/nowide/convert.hpp | 51 +- extern/boost/nowide/include/nowide/cstdio.hpp | 11 +- .../boost/nowide/include/nowide/cstdlib.hpp | 16 +- .../nowide/include/nowide/detail/convert.hpp | 98 +-- .../nowide/include/nowide/detail/is_path.hpp | 36 + .../nowide/detail/is_string_container.hpp | 93 ++ .../nowide/include/nowide/detail/utf.hpp | 456 +--------- .../boost/nowide/include/nowide/filebuf.hpp | 321 ++++--- .../nowide/include/nowide/filesystem.hpp | 17 +- .../boost/nowide/include/nowide/fstream.hpp | 122 +-- .../boost/nowide/include/nowide/iostream.hpp | 24 +- extern/boost/nowide/include/nowide/quoted.hpp | 107 +++ .../nowide/include/nowide/replacement.hpp | 9 +- .../nowide/include/nowide/stackstring.hpp | 46 +- extern/boost/nowide/include/nowide/stat.hpp | 67 ++ .../nowide/include/nowide/utf/convert.hpp | 106 +++ .../boost/nowide/include/nowide/utf/utf.hpp | 452 ++++++++++ .../nowide/include/nowide/utf8_codecvt.hpp | 234 ++--- .../boost/nowide/include/nowide/windows.hpp | 24 +- extern/boost/nowide/src/cstdio.cpp | 24 +- extern/boost/nowide/src/cstdlib.cpp | 63 +- extern/boost/nowide/src/iostream.cpp | 262 ++---- extern/boost/nowide/test/CMakeLists.txt | 51 +- .../boost/nowide/test/benchmark_fstream.cpp | 94 +- .../nowide/test/check_movable_fstreams.cpp | 7 +- .../nowide/test/cmake_test/CMakeLists.txt | 16 + extern/boost/nowide/test/cmake_test/main.cpp | 18 + .../test/exampleProject/example_main.cpp | 8 +- .../boost/nowide/test/file_test_helpers.cpp | 104 +++ .../boost/nowide/test/file_test_helpers.hpp | 49 ++ extern/boost/nowide/test/test.hpp | 155 +++- extern/boost/nowide/test/test_codecvt.cpp | 212 +++-- extern/boost/nowide/test/test_convert.cpp | 109 ++- extern/boost/nowide/test/test_env.cpp | 35 +- extern/boost/nowide/test/test_filebuf.cpp | 824 ++++++++++++++++++ extern/boost/nowide/test/test_fs.cpp | 125 ++- extern/boost/nowide/test/test_fstream.cpp | 816 +++++++++-------- .../nowide/test/test_fstream_special.cpp | 286 ++++++ extern/boost/nowide/test/test_ifstream.cpp | 211 +++++ extern/boost/nowide/test/test_iostream.cpp | 651 ++++++++++++-- .../test/test_iostream_passthrough.cmake | 28 + extern/boost/nowide/test/test_ofstream.cpp | 215 +++++ extern/boost/nowide/test/test_sets.hpp | 45 +- extern/boost/nowide/test/test_stackstring.cpp | 56 +- extern/boost/nowide/test/test_stat.cpp | 72 ++ extern/boost/nowide/test/test_stdio.cpp | 24 +- extern/boost/nowide/test/test_system.cpp | 86 +- extern/boost/nowide/test/test_traits.cpp | 146 ++++ src/console_buffer.cpp | 148 ++++ src/console_buffer.hpp | 79 ++ src/filebuf.cpp | 78 ++ src/stat.cpp | 50 ++ 58 files changed, 5667 insertions(+), 1973 deletions(-) create mode 100644 config/check_attribute_init_priority.cpp create mode 100644 config/check_lfs_support.cpp create mode 100644 config/check_movable_fstreams.cpp create mode 100644 extern/boost/nowide/include/nowide/detail/is_path.hpp create mode 100644 extern/boost/nowide/include/nowide/detail/is_string_container.hpp create mode 100644 extern/boost/nowide/include/nowide/quoted.hpp create mode 100644 extern/boost/nowide/include/nowide/stat.hpp create mode 100644 extern/boost/nowide/include/nowide/utf/convert.hpp create mode 100644 extern/boost/nowide/include/nowide/utf/utf.hpp create mode 100644 extern/boost/nowide/test/cmake_test/CMakeLists.txt create mode 100644 extern/boost/nowide/test/cmake_test/main.cpp create mode 100644 extern/boost/nowide/test/file_test_helpers.cpp create mode 100644 extern/boost/nowide/test/file_test_helpers.hpp create mode 100644 extern/boost/nowide/test/test_filebuf.cpp create mode 100644 extern/boost/nowide/test/test_fstream_special.cpp create mode 100644 extern/boost/nowide/test/test_ifstream.cpp create mode 100644 extern/boost/nowide/test/test_iostream_passthrough.cmake create mode 100644 extern/boost/nowide/test/test_ofstream.cpp create mode 100644 extern/boost/nowide/test/test_stat.cpp create mode 100644 extern/boost/nowide/test/test_traits.cpp create mode 100644 src/console_buffer.cpp create mode 100644 src/console_buffer.hpp create mode 100644 src/filebuf.cpp create mode 100644 src/stat.cpp diff --git a/config/check_attribute_init_priority.cpp b/config/check_attribute_init_priority.cpp new file mode 100644 index 0000000000..fab03e2428 --- /dev/null +++ b/config/check_attribute_init_priority.cpp @@ -0,0 +1,14 @@ +// Copyright (c) 2021 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +class Foo +{}; + +Foo foo __attribute__((init_priority(101))); + +int main() +{ + return 0; +} diff --git a/config/check_lfs_support.cpp b/config/check_lfs_support.cpp new file mode 100644 index 0000000000..791787f7e5 --- /dev/null +++ b/config/check_lfs_support.cpp @@ -0,0 +1,29 @@ +// +// Copyright (c) 2020 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#define _LARGEFILE_SOURCE +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 64 +#endif + +#include + +void check(FILE* f) +{ +#if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__MINGW32__) + (void)_fseeki64(f, 0, SEEK_CUR); + (void)_ftelli64(f); +#else + // Check that those functions and off_t are available + (void)fseeko(f, off_t(0), SEEK_CUR); + (void)ftello(f); +#endif +} + +int main() +{ + check(nullptr); +} diff --git a/config/check_movable_fstreams.cpp b/config/check_movable_fstreams.cpp new file mode 100644 index 0000000000..ed6be3859b --- /dev/null +++ b/config/check_movable_fstreams.cpp @@ -0,0 +1,17 @@ +// +// Copyright (c) 2020 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include +#include + +/// Check that the stdlib supports swapping and moving fstreams +/// (and by extension all other streams and streambufs) +void check() +{ + std::fstream s1, s2; + s1.swap(s2); + s2 = std::move(s1); +} diff --git a/extern/boost/nowide/cmake/NowideAddWarnings.cmake b/extern/boost/nowide/cmake/NowideAddWarnings.cmake index ab05e9cd01..2ab8b31ec7 100644 --- a/extern/boost/nowide/cmake/NowideAddWarnings.cmake +++ b/extern/boost/nowide/cmake/NowideAddWarnings.cmake @@ -14,13 +14,19 @@ function(nowide_add_warnings target level) if(NOT level IN_LIST allowed_levels) message(FATAL_ERROR "${level} is not a valid warning level (${allowed_levels})") endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(supression -Wno-long-long) + if(MSVC) + set(warn_off /W0) + set(warn_on /W3) + foreach(_lvl IN ITEMS all extra pedantic) + set(warn_${_lvl} /W4) + endforeach() + set(werror /WX) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(warn_off -w) - set(warn_on -Wall ${supression}) - set(warn_all -Wall ${supression}) - set(warn_extra -Wall -Wextra ${supression}) - set(warn_pedantic -Wall -Wextra -pedantic ${supression}) + set(warn_on -Wall) + set(warn_all -Wall) + set(warn_extra -Wall -Wextra) + set(warn_pedantic -Wall -Wextra -pedantic) set(werror -Werror) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") set(warn_off -w0) @@ -28,13 +34,6 @@ function(nowide_add_warnings target level) set(warn_${_lvl} -w1) endforeach() set(werror "") - elseif(MSVC) - set(warn_off /W0) - set(warn_on /W3) - foreach(_lvl IN ITEMS all extra pedantic) - set(warn_${_lvl} /W4) - endforeach() - set(werror /WX) endif() target_compile_options(${target} PRIVATE ${warn_${level}}) if(warningsAsErrors) diff --git a/extern/boost/nowide/include/nowide/args.hpp b/extern/boost/nowide/include/nowide/args.hpp index 29917111c2..baac8d7046 100644 --- a/extern/boost/nowide/include/nowide/args.hpp +++ b/extern/boost/nowide/include/nowide/args.hpp @@ -1,10 +1,9 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_ARGS_HPP_INCLUDED #define NOWIDE_ARGS_HPP_INCLUDED @@ -112,7 +111,7 @@ namespace nowide { } operator bool() const { - return p != NULL; + return p != nullptr; } const wchar_t* operator[](size_t i) const { diff --git a/extern/boost/nowide/include/nowide/config.hpp b/extern/boost/nowide/include/nowide/config.hpp index 6489889e7e..e007b4416e 100644 --- a/extern/boost/nowide/include/nowide/config.hpp +++ b/extern/boost/nowide/include/nowide/config.hpp @@ -1,17 +1,19 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// Copyright (c) 2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2019 - 2022 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_CONFIG_HPP_INCLUDED #define NOWIDE_CONFIG_HPP_INCLUDED +/// @file + #include -#if(defined(__WIN32) || defined(_WIN32) || defined(WIN32)) && !defined(__CYGWIN__) +//! @cond Doxygen_Suppress +#if(defined(_WIN32) || defined(__WIN32__) || defined(WIN32)) && !defined(__CYGWIN__) #define NOWIDE_WINDOWS #endif @@ -19,20 +21,6 @@ #define NOWIDE_MSVC _MSC_VER #endif -#if defined(__MINGW64__) -#define NOWIDE_FTELL64 ftello64 -#define NOWIDE_FSEEK64 fseeko64 -#elif defined(__APPLE__) -#define NOWIDE_FTELL64 ftello -#define NOWIDE_FSEEK64 fseeko -#elif defined(_MSC_VER) -#define NOWIDE_FTELL64 _ftelli64 -#define NOWIDE_FSEEK64 _fseeki64 -#else -#define NOWIDE_FTELL64 ftell -#define NOWIDE_FSEEK64 fseek -#endif - #ifdef __GNUC__ #define NOWIDE_SYMBOL_VISIBLE __attribute__((__visibility__("default"))) #endif @@ -44,11 +32,26 @@ #ifdef NOWIDE_WINDOWS #define NOWIDE_SYMBOL_EXPORT __declspec(dllexport) #define NOWIDE_SYMBOL_IMPORT __declspec(dllimport) +#elif defined(__CYGWIN__) && defined(__GNUC__) && (__GNUC__ >= 4) +#define NOWIDE_SYMBOL_EXPORT __attribute__((__dllexport__)) +#define NOWIDE_SYMBOL_IMPORT __attribute__((__dllimport__)) #else #define NOWIDE_SYMBOL_EXPORT NOWIDE_SYMBOL_VISIBLE #define NOWIDE_SYMBOL_IMPORT #endif +#if defined __GNUC__ +#define NOWIDE_LIKELY(x) __builtin_expect(x, 1) +#define NOWIDE_UNLIKELY(x) __builtin_expect(x, 0) +#else +#if !defined(NOWIDE_LIKELY) +#define NOWIDE_LIKELY(x) x +#endif +#if !defined(NOWIDE_UNLIKELY) +#define NOWIDE_UNLIKELY(x) x +#endif +#endif + #if defined(NOWIDE_DYN_LINK) #ifdef NOWIDE_SOURCE #define NOWIDE_DECL NOWIDE_SYMBOL_EXPORT @@ -59,30 +62,74 @@ #define NOWIDE_DECL #endif // NOWIDE_DYN_LINK -#ifndef NOWIDE_DECL -#define NOWIDE_DECL + +//! @endcond + +/// @def NOWIDE_USE_WCHAR_OVERLOADS +/// @brief Whether to use the wchar_t* overloads in fstream-classes. +/// +/// Enabled by default on Windows and Cygwin as the latter may use wchar_t in filesystem::path. +#ifndef NOWIDE_USE_WCHAR_OVERLOADS +#if defined(NOWIDE_WINDOWS) || defined(__CYGWIN__) || defined(NOWIDE_DOXYGEN) +#define NOWIDE_USE_WCHAR_OVERLOADS 1 +#else +#define NOWIDE_USE_WCHAR_OVERLOADS 0 +#endif #endif -#if defined(NOWIDE_WINDOWS) +/// @def NOWIDE_USE_FILEBUF_REPLACEMENT +/// @brief Define to 1 to use the class from that is used on Windows. +/// +/// - On Windows: No effect, always overwritten to 1 +/// - Others (including Cygwin): Defaults to the value of #NOWIDE_USE_WCHAR_OVERLOADS if not set. +/// +/// When set to 0 nowide::basic_filebuf will be an alias for std::basic_filebuf. +/// +/// Affects nowide::basic_filebuf, +/// nowide::basic_ofstream, nowide::basic_ifstream, nowide::basic_fstream +#if defined(NOWIDE_WINDOWS) || defined(NOWIDE_DOXYGEN) #ifdef NOWIDE_USE_FILEBUF_REPLACEMENT #undef NOWIDE_USE_FILEBUF_REPLACEMENT #endif #define NOWIDE_USE_FILEBUF_REPLACEMENT 1 #elif !defined(NOWIDE_USE_FILEBUF_REPLACEMENT) -#define NOWIDE_USE_FILEBUF_REPLACEMENT 0 +#define NOWIDE_USE_FILEBUF_REPLACEMENT NOWIDE_USE_WCHAR_OVERLOADS #endif +//! @cond Doxygen_Suppress + #if defined(__GNUC__) && __GNUC__ >= 7 #define NOWIDE_FALLTHROUGH __attribute__((fallthrough)) #else #define NOWIDE_FALLTHROUGH #endif -#if !defined(NOWIDE_LIKELY) -#define NOWIDE_LIKELY(x) x -#endif -#if !defined(NOWIDE_UNLIKELY) -#define NOWIDE_UNLIKELY(x) x +// The std::codecvt are deprecated in C++20 +// These macros can suppress this warning +#if defined(_MSC_VER) +#define NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN __pragma(warning(push)) __pragma(warning(disable : 4996)) +#define NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END __pragma(warning(pop)) +#elif(__cplusplus >= 202002L) && defined(__clang__) +#define NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN \ + _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#define NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END _Pragma("clang diagnostic pop") +#elif(__cplusplus >= 202002L) && defined(__GNUC__) +#define NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN \ + _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#define NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END _Pragma("GCC diagnostic pop") +#else +#define NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN +#define NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END #endif +//! @endcond + +/// +/// \brief This namespace includes implementations of the standard library functions and +/// classes such that they accept UTF-8 strings on Windows. +/// On other platforms (i.e. not on Windows) those functions and classes are just aliases +/// of the corresponding ones from the std namespace or behave like them. +/// +namespace nowide {} + #endif diff --git a/extern/boost/nowide/include/nowide/convert.hpp b/extern/boost/nowide/include/nowide/convert.hpp index 1d8867dd82..d008c9325e 100644 --- a/extern/boost/nowide/include/nowide/convert.hpp +++ b/extern/boost/nowide/include/nowide/convert.hpp @@ -1,14 +1,15 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2020 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_CONVERT_HPP_INCLUDED #define NOWIDE_CONVERT_HPP_INCLUDED -#include +#include +#include #include namespace nowide { @@ -22,7 +23,7 @@ namespace nowide { /// inline char* narrow(char* output, size_t output_size, const wchar_t* begin, const wchar_t* end) { - return detail::convert_buffer(output, output_size, begin, end); + return utf::convert_buffer(output, output_size, begin, end); } /// /// Convert NULL terminated wide string (UTF-16/32) to NULL terminated narrow string (UTF-8) @@ -33,7 +34,7 @@ namespace nowide { /// inline char* narrow(char* output, size_t output_size, const wchar_t* source) { - return narrow(output, output_size, source, source + detail::strlen(source)); + return narrow(output, output_size, source, source + utf::strlen(source)); } /// @@ -45,7 +46,7 @@ namespace nowide { /// inline wchar_t* widen(wchar_t* output, size_t output_size, const char* begin, const char* end) { - return detail::convert_buffer(output, output_size, begin, end); + return utf::convert_buffer(output, output_size, begin, end); } /// /// Convert NULL terminated narrow string (UTF-8) to NULL terminated wide string (UTF-16/32) @@ -56,7 +57,7 @@ namespace nowide { /// inline wchar_t* widen(wchar_t* output, size_t output_size, const char* source) { - return widen(output, output_size, source, source + detail::strlen(source)); + return widen(output, output_size, source, source + utf::strlen(source)); } /// @@ -66,9 +67,10 @@ namespace nowide { /// \param count Number of characters to convert /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER /// - inline std::string narrow(const wchar_t* s, size_t count) + template> + inline std::string narrow(const T_Char* s, size_t count) { - return detail::convert_string(s, s + count); + return utf::convert_string(s, s + count); } /// /// Convert wide string (UTF-16/32) to narrow string (UTF-8). @@ -76,9 +78,10 @@ namespace nowide { /// \param s NULL terminated input string /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER /// - inline std::string narrow(const wchar_t* s) + template> + inline std::string narrow(const T_Char* s) { - return narrow(s, detail::strlen(s)); + return narrow(s, utf::strlen(s)); } /// /// Convert wide string (UTF-16/32) to narrow string (UTF-8). @@ -86,9 +89,10 @@ namespace nowide { /// \param s Input string /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER /// - inline std::string narrow(const std::wstring& s) + template> + inline std::string narrow(const StringOrStringView& s) { - return narrow(s.c_str(), s.size()); + return utf::convert_string(s.data(), s.data() + s.size()); } /// @@ -98,9 +102,10 @@ namespace nowide { /// \param count Number of characters to convert /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER /// - inline std::wstring widen(const char* s, size_t count) + template> + inline std::wstring widen(const T_Char* s, size_t count) { - return detail::convert_string(s, s + count); + return utf::convert_string(s, s + count); } /// /// Convert narrow string (UTF-8) to wide string (UTF-16/32). @@ -108,9 +113,10 @@ namespace nowide { /// \param s NULL terminated input string /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER /// - inline std::wstring widen(const char* s) + template> + inline std::wstring widen(const T_Char* s) { - return widen(s, detail::strlen(s)); + return widen(s, utf::strlen(s)); } /// /// Convert narrow string (UTF-8) to wide string (UTF-16/32). @@ -118,9 +124,10 @@ namespace nowide { /// \param s Input string /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER /// - inline std::wstring widen(const std::string& s) + template> + inline std::wstring widen(const StringOrStringView& s) { - return widen(s.c_str(), s.size()); + return utf::convert_string(s.data(), s.data() + s.size()); } } // namespace nowide diff --git a/extern/boost/nowide/include/nowide/cstdio.hpp b/extern/boost/nowide/include/nowide/cstdio.hpp index d5bc5ca7d1..c2dd8e725d 100644 --- a/extern/boost/nowide/include/nowide/cstdio.hpp +++ b/extern/boost/nowide/include/nowide/cstdio.hpp @@ -1,11 +1,10 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// Copyright (c) 2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2020 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_CSTDIO_HPP_INCLUDED #define NOWIDE_CSTDIO_HPP_INCLUDED diff --git a/extern/boost/nowide/include/nowide/cstdlib.hpp b/extern/boost/nowide/include/nowide/cstdlib.hpp index e6d4a57da5..36be587fa4 100644 --- a/extern/boost/nowide/include/nowide/cstdlib.hpp +++ b/extern/boost/nowide/include/nowide/cstdlib.hpp @@ -1,10 +1,9 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_CSTDLIB_HPP_INCLUDED #define NOWIDE_CSTDLIB_HPP_INCLUDED @@ -21,7 +20,12 @@ namespace nowide { /// /// \brief UTF-8 aware getenv. Returns 0 if the variable is not set. /// - /// This function is not thread safe or reenterable as defined by the standard library + /// The string pointed to shall not be modified by the program. + /// This function is thread-safe as long as no other thread modifies the host environment. + /// However subsequent calls to this function might overwrite the string pointed to. + /// + /// Warning: The returned pointer might only be valid for as long as the calling thread is alive. + /// So avoid passing it across thread boundaries. /// NOWIDE_DECL char* getenv(const char* key); diff --git a/extern/boost/nowide/include/nowide/detail/convert.hpp b/extern/boost/nowide/include/nowide/detail/convert.hpp index ad02b8f727..9b150fce5a 100644 --- a/extern/boost/nowide/include/nowide/detail/convert.hpp +++ b/extern/boost/nowide/include/nowide/detail/convert.hpp @@ -1,100 +1,22 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2020 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_DETAIL_CONVERT_HPP_INCLUDED #define NOWIDE_DETAIL_CONVERT_HPP_INCLUDED -#include -#include -#include -#include +#include + +// Legacy compatibility header only. Include instead namespace nowide { - /// \cond INTERNAL namespace detail { - /// - /// Convert a buffer of UTF sequences in the range [source_begin, source_end) - /// from \tparam CharIn to \tparam CharOut to the output \a buffer of size \a buffer_size. - /// - /// \return original buffer containing the NULL terminated string or NULL - /// - /// If there is not enough room in the buffer NULL is returned, and the content of the buffer is undefined. - /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER - /// - template - CharOut* - convert_buffer(CharOut* buffer, size_t buffer_size, const CharIn* source_begin, const CharIn* source_end) - { - CharOut* rv = buffer; - if(buffer_size == 0) - return 0; - buffer_size--; - while(source_begin != source_end) - { - using namespace detail::utf; - code_point c = utf_traits::decode(source_begin, source_end); - if(c == illegal || c == incomplete) - { - c = NOWIDE_REPLACEMENT_CHARACTER; - } - size_t width = utf_traits::width(c); - if(buffer_size < width) - { - rv = NULL; - break; - } - buffer = utf_traits::encode(c, buffer); - buffer_size -= width; - } - *buffer++ = 0; - return rv; - } - - /// - /// Convert the UTF sequences in range [begin, end) from \tparam CharIn to \tparam CharOut - /// and return it as a string - /// - /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER - /// - template - std::basic_string convert_string(const CharIn* begin, const CharIn* end) - { - std::basic_string result; - result.reserve(end - begin); - typedef std::back_insert_iterator > inserter_type; - inserter_type inserter(result); - using namespace detail::utf; - code_point c; - while(begin != end) - { - c = utf_traits::decode(begin, end); - if(c == illegal || c == incomplete) - { - c = NOWIDE_REPLACEMENT_CHARACTER; - } - utf_traits::encode(c, inserter); - } - return result; - } - - /// Return the length of the given string. - /// That is the number of characters until the first NULL character - /// Equivalent to `std::strlen(s)` but can handle wide-strings - template - size_t strlen(const Char* s) - { - const Char* end = s; - while(*end) - end++; - return end - s; - } - + using nowide::utf::convert_buffer; + using nowide::utf::convert_string; + using nowide::utf::strlen; } // namespace detail - /// \endcond } // namespace nowide #endif diff --git a/extern/boost/nowide/include/nowide/detail/is_path.hpp b/extern/boost/nowide/include/nowide/detail/is_path.hpp new file mode 100644 index 0000000000..1e0c523305 --- /dev/null +++ b/extern/boost/nowide/include/nowide/detail/is_path.hpp @@ -0,0 +1,36 @@ +// +// Copyright (c) 2020 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef NOWIDE_DETAIL_IS_PATH_HPP_INCLUDED +#define NOWIDE_DETAIL_IS_PATH_HPP_INCLUDED + +#include + +namespace nowide { + namespace detail { + + /// Trait to heuristically check for a *\::filesystem::path + /// Done by checking for make_preferred and filename member functions with correct signature + template + struct is_path + { + template + struct Check; + template + static std::true_type test(Check*); + template + static std::false_type test(...); + + static constexpr bool value = decltype(test(0))::value; + }; + /// SFINAE trait/alias which resolves to Result if the Path is a *\::filesystem::path + template + using enable_if_path_t = typename std::enable_if::value, Result>::type; + + } // namespace detail +} // namespace nowide + +#endif diff --git a/extern/boost/nowide/include/nowide/detail/is_string_container.hpp b/extern/boost/nowide/include/nowide/detail/is_string_container.hpp new file mode 100644 index 0000000000..013c87de63 --- /dev/null +++ b/extern/boost/nowide/include/nowide/detail/is_string_container.hpp @@ -0,0 +1,93 @@ +// +// Copyright (c) 2020 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef NOWIDE_DETAIL_IS_STRING_CONTAINER_HPP_INCLUDED +#define NOWIDE_DETAIL_IS_STRING_CONTAINER_HPP_INCLUDED + +#include +#include + +namespace nowide { + namespace detail { + template + struct make_void + { + typedef void type; + }; + + template + using void_t = typename make_void::type; + + template + struct is_char_type : std::false_type + {}; + template<> + struct is_char_type : std::true_type + {}; + template<> + struct is_char_type : std::true_type + {}; + template<> + struct is_char_type : std::true_type + {}; + template<> + struct is_char_type : std::true_type + {}; +#ifdef __cpp_char8_t + template<> + struct is_char_type : std::true_type + {}; +#endif + + template + struct is_c_string : std::false_type + {}; + template + struct is_c_string : is_char_type + {}; + + template + using const_data_result = decltype(std::declval().data()); + /// Return the size of the char type returned by the data() member function + template + using get_data_width = + std::integral_constant>::type)>; + template + using size_result = decltype(std::declval().size()); + /// Return true if the data() member function returns a pointer to a type of size 1 + template + using has_narrow_data = std::integral_constant::value == 1)>; + + /// Return true if T is a string container, e.g. std::basic_string, std::basic_string_view + /// Requires a static value `npos`, a member function `size()` returning an integral, + /// and a member function `data()` returning a C string + template + struct is_string_container : std::false_type + {}; + // clang-format off + template + struct is_string_container, const_data_result>> + : std::integral_constant::value + && std::is_integral>::value + && is_c_string>::value + && isNarrow == has_narrow_data::value> + {}; + // clang-format on + template + using requires_narrow_string_container = typename std::enable_if::value>::type; + template + using requires_wide_string_container = typename std::enable_if::value>::type; + + template + using requires_narrow_char = typename std::enable_if::value>::type; + template + using requires_wide_char = typename std::enable_if<(sizeof(T) > 1) && is_char_type::value>::type; + + } // namespace detail +} // namespace nowide + +#endif diff --git a/extern/boost/nowide/include/nowide/detail/utf.hpp b/extern/boost/nowide/include/nowide/detail/utf.hpp index 17b27459b1..09b111f1e8 100644 --- a/extern/boost/nowide/include/nowide/detail/utf.hpp +++ b/extern/boost/nowide/include/nowide/detail/utf.hpp @@ -1,454 +1,20 @@ // -// Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2020 Alexander Grund +// Copyright (c) 2020 Alexander Grund // -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) -// -#ifndef NOWIDE_UTF_HPP_INCLUDED -#define NOWIDE_UTF_HPP_INCLUDED - -#include -#include - -namespace nowide { - namespace detail { - /// - /// \brief Namespace that holds basic operations on UTF encoded sequences - /// - /// All functions defined in this namespace do not require linking with Boost.Nowide library - /// Extracted from Boost.Locale - /// - namespace utf { - - /// - /// \brief The integral type that can hold a Unicode code point - /// - typedef uint32_t code_point; - - /// - /// \brief Special constant that defines illegal code point - /// - static const code_point illegal = 0xFFFFFFFFu; - - /// - /// \brief Special constant that defines incomplete code point - /// - static const code_point incomplete = 0xFFFFFFFEu; - - /// - /// \brief the function checks if \a v is a valid code point - /// - inline bool is_valid_codepoint(code_point v) - { - if(v > 0x10FFFF) - return false; - if(0xD800 <= v && v <= 0xDFFF) // surrogates - return false; - return true; - } - -#ifdef NOWIDE_DOXYGEN - /// - /// \brief UTF Traits class - functions to convert UTF sequences to and from Unicode code points - /// - template - struct utf_traits - { - /// - /// The type of the character - /// - typedef CharType char_type; - /// - /// Read one code point from the range [p,e) and return it. - /// - /// - If the sequence that was read is incomplete sequence returns \ref incomplete, - /// - If illegal sequence detected returns \ref illegal - /// - /// Requirements - /// - /// - Iterator is valid input iterator - /// - /// Postconditions - /// - /// - p points to the last consumed character - /// - template - static code_point decode(Iterator& p, Iterator e); - - /// - /// Maximal width of valid sequence in the code units: - /// - /// - UTF-8 - 4 - /// - UTF-16 - 2 - /// - UTF-32 - 1 - /// - static const int max_width; - /// - /// The width of specific code point in the code units. - /// - /// Requirement: value is a valid Unicode code point - /// Returns value in range [1..max_width] - /// - static int width(code_point value); - - /// - /// Get the size of the trail part of variable length encoded sequence. - /// - /// Returns -1 if C is not valid lead character - /// - static int trail_length(char_type c); - /// - /// Returns true if c is trail code unit, always false for UTF-32 - /// - static bool is_trail(char_type c); - /// - /// Returns true if c is lead code unit, always true of UTF-32 - /// - static bool is_lead(char_type c); - - /// - /// Convert valid Unicode code point \a value to the UTF sequence. - /// - /// Requirements: - /// - /// - \a value is valid code point - /// - \a out is an output iterator should be able to accept at least width(value) units - /// - /// Returns the iterator past the last written code unit. - /// - template - static Iterator encode(code_point value, Iterator out); - /// - /// Decodes valid UTF sequence that is pointed by p into code point. - /// - /// If the sequence is invalid or points to end the behavior is undefined - /// - template - static code_point decode_valid(Iterator& p); - }; - -#else - - template - struct utf_traits; - - template - struct utf_traits - { - typedef CharType char_type; - - static int trail_length(char_type ci) - { - unsigned char c = ci; - if(c < 128) - return 0; - if(NOWIDE_UNLIKELY(c < 194)) - return -1; - if(c < 224) - return 1; - if(c < 240) - return 2; - if(NOWIDE_LIKELY(c <= 244)) - return 3; - return -1; - } - - static const int max_width = 4; - - static int width(code_point value) - { - if(value <= 0x7F) - { - return 1; - } else if(value <= 0x7FF) - { - return 2; - } else if(NOWIDE_LIKELY(value <= 0xFFFF)) - { - return 3; - } else - { - return 4; - } - } - - static bool is_trail(char_type ci) - { - unsigned char c = ci; - return (c & 0xC0) == 0x80; - } - - static bool is_lead(char_type ci) - { - return !is_trail(ci); - } - - template - static code_point decode(Iterator& p, Iterator e) - { - if(NOWIDE_UNLIKELY(p == e)) - return incomplete; - - unsigned char lead = *p++; - - // First byte is fully validated here - int trail_size = trail_length(lead); +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt - if(NOWIDE_UNLIKELY(trail_size < 0)) - return illegal; +#ifndef NOWIDE_DETAIL_UTF_HPP_INCLUDED +#define NOWIDE_DETAIL_UTF_HPP_INCLUDED - // - // OK as only ASCII may be of size = 0 - // also optimize for ASCII text - // - if(trail_size == 0) - return lead; +#include - code_point c = lead & ((1 << (6 - trail_size)) - 1); +// Legacy compatibility header only. Include instead - // Read the rest - unsigned char tmp; - switch(trail_size) - { - case 3: - if(NOWIDE_UNLIKELY(p == e)) - return incomplete; - tmp = *p++; - if(!is_trail(tmp)) - return illegal; - c = (c << 6) | (tmp & 0x3F); - NOWIDE_FALLTHROUGH; - case 2: - if(NOWIDE_UNLIKELY(p == e)) - return incomplete; - tmp = *p++; - if(!is_trail(tmp)) - return illegal; - c = (c << 6) | (tmp & 0x3F); - NOWIDE_FALLTHROUGH; - case 1: - if(NOWIDE_UNLIKELY(p == e)) - return incomplete; - tmp = *p++; - if(!is_trail(tmp)) - return illegal; - c = (c << 6) | (tmp & 0x3F); - } - - // Check code point validity: no surrogates and - // valid range - if(NOWIDE_UNLIKELY(!is_valid_codepoint(c))) - return illegal; - - // make sure it is the most compact representation - if(NOWIDE_UNLIKELY(width(c) != trail_size + 1)) - return illegal; - - return c; - } - - template - static code_point decode_valid(Iterator& p) - { - unsigned char lead = *p++; - if(lead < 192) - return lead; - - int trail_size; - - if(lead < 224) - trail_size = 1; - else if(NOWIDE_LIKELY(lead < 240)) // non-BMP rare - trail_size = 2; - else - trail_size = 3; - - code_point c = lead & ((1 << (6 - trail_size)) - 1); - - switch(trail_size) - { - case 3: c = (c << 6) | (static_cast(*p++) & 0x3F); NOWIDE_FALLTHROUGH; - case 2: c = (c << 6) | (static_cast(*p++) & 0x3F); NOWIDE_FALLTHROUGH; - case 1: c = (c << 6) | (static_cast(*p++) & 0x3F); - } - - return c; - } - - template - static Iterator encode(code_point value, Iterator out) - { - if(value <= 0x7F) - { - *out++ = static_cast(value); - } else if(value <= 0x7FF) - { - *out++ = static_cast((value >> 6) | 0xC0); - *out++ = static_cast((value & 0x3F) | 0x80); - } else if(NOWIDE_LIKELY(value <= 0xFFFF)) - { - *out++ = static_cast((value >> 12) | 0xE0); - *out++ = static_cast(((value >> 6) & 0x3F) | 0x80); - *out++ = static_cast((value & 0x3F) | 0x80); - } else - { - *out++ = static_cast((value >> 18) | 0xF0); - *out++ = static_cast(((value >> 12) & 0x3F) | 0x80); - *out++ = static_cast(((value >> 6) & 0x3F) | 0x80); - *out++ = static_cast((value & 0x3F) | 0x80); - } - return out; - } - }; // utf8 - - template - struct utf_traits - { - typedef CharType char_type; - - // See RFC 2781 - static bool is_first_surrogate(uint16_t x) - { - return 0xD800 <= x && x <= 0xDBFF; - } - static bool is_second_surrogate(uint16_t x) - { - return 0xDC00 <= x && x <= 0xDFFF; - } - static code_point combine_surrogate(uint16_t w1, uint16_t w2) - { - return ((code_point(w1 & 0x3FF) << 10) | (w2 & 0x3FF)) + 0x10000; - } - static int trail_length(char_type c) - { - if(is_first_surrogate(c)) - return 1; - if(is_second_surrogate(c)) - return -1; - return 0; - } - /// - /// Returns true if c is trail code unit, always false for UTF-32 - /// - static bool is_trail(char_type c) - { - return is_second_surrogate(c); - } - /// - /// Returns true if c is lead code unit, always true of UTF-32 - /// - static bool is_lead(char_type c) - { - return !is_second_surrogate(c); - } - - template - static code_point decode(It& current, It last) - { - if(NOWIDE_UNLIKELY(current == last)) - return incomplete; - uint16_t w1 = *current++; - if(NOWIDE_LIKELY(w1 < 0xD800 || 0xDFFF < w1)) - { - return w1; - } - if(w1 > 0xDBFF) - return illegal; - if(current == last) - return incomplete; - uint16_t w2 = *current++; - if(w2 < 0xDC00 || 0xDFFF < w2) - return illegal; - return combine_surrogate(w1, w2); - } - template - static code_point decode_valid(It& current) - { - uint16_t w1 = *current++; - if(NOWIDE_LIKELY(w1 < 0xD800 || 0xDFFF < w1)) - { - return w1; - } - uint16_t w2 = *current++; - return combine_surrogate(w1, w2); - } - - static const int max_width = 2; - static int width(code_point u) - { - return u >= 0x10000 ? 2 : 1; - } - template - static It encode(code_point u, It out) - { - if(NOWIDE_LIKELY(u <= 0xFFFF)) - { - *out++ = static_cast(u); - } else - { - u -= 0x10000; - *out++ = static_cast(0xD800 | (u >> 10)); - *out++ = static_cast(0xDC00 | (u & 0x3FF)); - } - return out; - } - }; // utf16; - - template - struct utf_traits - { - typedef CharType char_type; - static int trail_length(char_type c) - { - if(is_valid_codepoint(c)) - return 0; - return -1; - } - static bool is_trail(char_type /*c*/) - { - return false; - } - static bool is_lead(char_type /*c*/) - { - return true; - } - - template - static code_point decode_valid(It& current) - { - return *current++; - } - - template - static code_point decode(It& current, It last) - { - if(NOWIDE_UNLIKELY(current == last)) - return incomplete; - code_point c = *current++; - if(NOWIDE_UNLIKELY(!is_valid_codepoint(c))) - return illegal; - return c; - } - static const int max_width = 1; - static int width(code_point /*u*/) - { - return 1; - } - template - static It encode(code_point u, It out) - { - *out++ = static_cast(u); - return out; - } - - }; // utf32 - -#endif - - } // namespace utf - } // namespace detail +namespace nowide { + namespace detail { + namespace utf = nowide::utf; + } // namespace detail } // namespace nowide #endif diff --git a/extern/boost/nowide/include/nowide/filebuf.hpp b/extern/boost/nowide/include/nowide/filebuf.hpp index 465f253089..897f062b06 100644 --- a/extern/boost/nowide/include/nowide/filebuf.hpp +++ b/extern/boost/nowide/include/nowide/filebuf.hpp @@ -1,17 +1,17 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// Copyright (c) 2019-2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2019-2020 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_FILEBUF_HPP_INCLUDED #define NOWIDE_FILEBUF_HPP_INCLUDED #include #if NOWIDE_USE_FILEBUF_REPLACEMENT #include +#include #include #include #include @@ -25,29 +25,38 @@ #endif namespace nowide { + namespace detail { + /// Same as std::ftell but potentially with Large File Support + NOWIDE_DECL std::streampos ftell(FILE* file); + /// Same as std::fseek but potentially with Large File Support + NOWIDE_DECL int fseek(FILE* file, std::streamoff offset, int origin); + } // namespace detail + #if !NOWIDE_USE_FILEBUF_REPLACEMENT && !defined(NOWIDE_DOXYGEN) using std::basic_filebuf; using std::filebuf; #else // Windows /// - /// \brief This forward declaration defines the basic_filebuf type. + /// \brief This forward declaration defines the basic_filebuf type + /// which is used when #NOWIDE_USE_FILEBUF_REPLACEMENT is set, e.g. on Windows. /// - /// it is implemented and specialized for CharType = char, it + /// It is implemented and specialized for CharType = char, it /// implements std::filebuf over standard C I/O /// - template > + template> class basic_filebuf; /// /// \brief This is the implementation of std::filebuf + /// which is used when #NOWIDE_USE_FILEBUF_REPLACEMENT is set, e.g. on Windows. /// - /// it is implemented and specialized for CharType = char, it + /// It is implemented and specialized for CharType = char, it /// implements std::filebuf over standard C I/O /// template<> class basic_filebuf : public std::basic_streambuf { - typedef std::char_traits Traits; + using Traits = std::char_traits; public: #ifdef NOWIDE_MSVC @@ -58,11 +67,11 @@ namespace nowide { /// Creates new filebuf /// basic_filebuf() : - buffer_size_(BUFSIZ), buffer_(0), file_(0), owns_buffer_(false), last_char_(), - mode_(std::ios_base::openmode(0)) + file_(nullptr), buffer_(nullptr), buffer_size_(BUFSIZ), owns_buffer_(false), unbuffered_read_(false), + last_char_(), mode_(std::ios_base::openmode(0)) { - setg(0, 0, 0); - setp(0, 0); + setg(nullptr, nullptr, nullptr); + setp(nullptr, nullptr); } #ifdef NOWIDE_MSVC #pragma warning(pop) @@ -75,6 +84,7 @@ namespace nowide { } basic_filebuf& operator=(basic_filebuf&& other) noexcept { + close(); swap(other); return *this; } @@ -82,23 +92,26 @@ namespace nowide { { std::basic_streambuf::swap(rhs); using std::swap; - swap(buffer_size_, rhs.buffer_size_); - swap(buffer_, rhs.buffer_); swap(file_, rhs.file_); + swap(buffer_, rhs.buffer_); + swap(buffer_size_, rhs.buffer_size_); swap(owns_buffer_, rhs.owns_buffer_); + swap(unbuffered_read_, rhs.unbuffered_read_); swap(last_char_[0], rhs.last_char_[0]); swap(mode_, rhs.mode_); + // Fixup last_char references - if(epptr() == rhs.last_char_) - setp(last_char_, last_char_); - if(egptr() == rhs.last_char_) - rhs.setg(last_char_, gptr() == rhs.last_char_ ? last_char_ : last_char_ + 1, last_char_ + 1); - if(rhs.epptr() == last_char_) - setp(rhs.last_char_, rhs.last_char_); - if(rhs.egptr() == rhs.last_char_) + if(pbase() == rhs.last_char_) + setp(last_char_, (pptr() == epptr()) ? last_char_ : last_char_ + 1); + if(eback() == rhs.last_char_) + setg(last_char_, (gptr() == rhs.last_char_) ? last_char_ : last_char_ + 1, last_char_ + 1); + + if(rhs.pbase() == last_char_) + rhs.setp(rhs.last_char_, (rhs.pptr() == rhs.epptr()) ? rhs.last_char_ : rhs.last_char_ + 1); + if(rhs.eback() == last_char_) { rhs.setg(rhs.last_char_, - rhs.gptr() == last_char_ ? rhs.last_char_ : rhs.last_char_ + 1, + (rhs.gptr() == last_char_) ? rhs.last_char_ : rhs.last_char_ + 1, rhs.last_char_ + 1); } } @@ -127,68 +140,59 @@ namespace nowide { basic_filebuf* open(const wchar_t* s, std::ios_base::openmode mode) { if(is_open()) - return NULL; + return nullptr; validate_cvt(this->getloc()); const bool ate = (mode & std::ios_base::ate) != 0; if(ate) mode &= ~std::ios_base::ate; const wchar_t* smode = get_mode(mode); if(!smode) - return 0; + return nullptr; file_ = detail::wfopen(s, smode); if(!file_) - return 0; - if(ate && NOWIDE_FSEEK64(file_, 0, SEEK_END) != 0) + return nullptr; + if(ate && detail::fseek(file_, 0, SEEK_END) != 0) { close(); - return 0; + return nullptr; } mode_ = mode; + set_unbuffered_read(); return this; } + template + detail::enable_if_path_t open(const Path& file_name, std::ios_base::openmode mode) + { + return open(file_name.c_str(), mode); + } /// /// Same as std::filebuf::close() /// basic_filebuf* close() { if(!is_open()) - return NULL; + return nullptr; bool res = sync() == 0; if(std::fclose(file_) != 0) res = false; - file_ = NULL; + file_ = nullptr; mode_ = std::ios_base::openmode(0); if(owns_buffer_) { delete[] buffer_; - buffer_ = NULL; + buffer_ = nullptr; owns_buffer_ = false; } - return res ? this : NULL; + setg(nullptr, nullptr, nullptr); + setp(nullptr, nullptr); + return res ? this : nullptr; } /// /// Same as std::filebuf::is_open() /// bool is_open() const { - return file_ != NULL; - } - - private: - void make_buffer() - { - if(buffer_) - return; - if(buffer_size_ > 0) - { - buffer_ = new char[buffer_size_]; - owns_buffer_ = true; - } - } - void validate_cvt(const std::locale& loc) - { - if(!std::use_facet >(loc).always_noconv()) - throw std::runtime_error("Converting codecvts are not supported"); + return file_ != nullptr; } protected: @@ -197,18 +201,41 @@ namespace nowide { assert(n >= 0); // Maximum compatibility: Discard all local buffers and use user-provided values // Users should call sync() before or better use it before any IO is done or any file is opened - setg(NULL, NULL, NULL); - setp(NULL, NULL); + setg(nullptr, nullptr, nullptr); + setp(nullptr, nullptr); if(owns_buffer_) + { delete[] buffer_; + owns_buffer_ = false; + } buffer_ = s; buffer_size_ = (n >= 0) ? static_cast(n) : 0; + set_unbuffered_read(); return this; } + int sync() override + { + if(!file_) + return 0; + bool result; + if(pptr()) + { + // Only flush if anything was written, otherwise behavior of fflush is undefined. I.e.: + // - Buffered mode: pptr was set to buffer_ and advanced + // - Unbuffered mode: pptr set to last_char_ + const bool has_prev_write = pptr() != buffer_; + result = overflow() != EOF; + if(has_prev_write && std::fflush(file_) != 0) + result = false; + } else + result = stop_reading(); + return result ? 0 : -1; + } + int overflow(int c = EOF) override { - if(!(mode_ & std::ios_base::out)) + if(!(mode_ & (std::ios_base::out | std::ios_base::app))) return EOF; if(!stop_reading()) @@ -218,7 +245,8 @@ namespace nowide { if(n > 0) { if(std::fwrite(pbase(), 1, n, file_) != n) - return -1; + return EOF; + assert(buffer_); setp(buffer_, buffer_ + buffer_size_); if(c != EOF) { @@ -245,29 +273,37 @@ namespace nowide { return Traits::not_eof(c); } - int sync() override + std::streamsize xsputn(const char* s, std::streamsize n) override { - if(!file_) + // Only optimize when writing more than a buffer worth of data + if(n <= static_cast(buffer_size_)) + return std::basic_streambuf::xsputn(s, n); + if(!(mode_ & (std::ios_base::out | std::ios_base::app)) || !stop_reading()) return 0; - bool result; - if(pptr()) + + assert(n >= 0); + // First empty the remaining put area, if any + const char* const base = pbase(); + const size_t num_buffered = pptr() - base; + if(num_buffered != 0) { - result = overflow() != EOF; - // Only flush if anything was written, otherwise behavior of fflush is undefined - if(std::fflush(file_) != 0) - return result = false; - } else - result = stop_reading(); - return result ? 0 : -1; + const auto num_written = std::fwrite(base, 1, num_buffered, file_); + setp(const_cast(base + num_written), epptr()); // i.e. pbump(num_written) + if(num_written != num_buffered) + return 0; // Error writing buffered chars + } + // Then write directly to file + const auto num_written = std::fwrite(s, 1, static_cast(n), file_); + if(num_written > 0u && base != last_char_) + setp(last_char_, last_char_); // Mark as "written" if not done yet + return num_written; } int underflow() override { - if(!(mode_ & std::ios_base::in)) + if(!(mode_ & std::ios_base::in) || !stop_writing()) return EOF; - if(!stop_writing()) - return EOF; - if(buffer_size_ == 0) + if(unbuffered_read_) { const int c = std::fgetc(file_); if(c == EOF) @@ -285,27 +321,50 @@ namespace nowide { return Traits::to_int_type(*gptr()); } + std::streamsize xsgetn(char* s, std::streamsize n) override + { + // Only optimize when reading more than a buffer worth of data + if(n <= static_cast(unbuffered_read_ ? 1u : buffer_size_)) + return std::basic_streambuf::xsgetn(s, n); + if(!(mode_ & std::ios_base::in) || !stop_writing()) + return 0; + assert(n >= 0); + std::streamsize num_copied = 0; + // First empty the remaining get area, if any + const auto num_buffered = egptr() - gptr(); + if(num_buffered != 0) + { + const auto num_read = num_buffered > n ? n : num_buffered; + traits_type::copy(s, gptr(), static_cast(num_read)); + s += num_read; + n -= num_read; + num_copied = num_read; + setg(eback(), gptr() + num_read, egptr()); // i.e. gbump(num_read) + } + // Then read directly from file (loop as number of bytes read may be less than requested) + while(n > 0) + { + const auto num_read = std::fread(s, 1, static_cast(n), file_); + if(num_read == 0) // EOF or error + break; + s += num_read; + n -= num_read; + num_copied += num_read; + } + return num_copied; + } + int pbackfail(int c = EOF) override { - if(!(mode_ & std::ios_base::in)) - return EOF; - if(!stop_writing()) - return EOF; + // For simplicity we only allow putting back into our read buffer + // So putting back more chars than we have read from the buffer will fail if(gptr() > eback()) gbump(-1); - else if(seekoff(-1, std::ios_base::cur) != std::streampos(std::streamoff(-1))) - { - if(underflow() == EOF) - return EOF; - } else + else return EOF; - // Case 1: Caller just wanted space for 1 char - if(c == EOF) - return Traits::not_eof(c); - // Case 2: Caller wants to put back different char - // gptr now points to the (potentially newly read) previous char - if(*gptr() != c) + // Assign the new value if requested + if(c != EOF && *gptr() != Traits::to_char_type(c)) *gptr() = Traits::to_char_type(c); return Traits::not_eof(c); } @@ -330,10 +389,9 @@ namespace nowide { case std::ios_base::end: whence = SEEK_END; break; default: assert(false); return EOF; } - assert(off <= std::numeric_limits::max()); - if(NOWIDE_FSEEK64(file_, static_cast(off), whence) != 0) + if(detail::fseek(file_, off, whence) != 0) return EOF; - return NOWIDE_FTELL64(file_); + return detail::ftell(file_); } std::streampos seekpos(std::streampos pos, std::ios_base::openmode m = std::ios_base::in | std::ios_base::out) override @@ -347,47 +405,70 @@ namespace nowide { } private: - /// Stop reading adjusting the file pointer if necessary - /// Postcondition: gptr() == NULL - bool stop_reading() + void make_buffer() { - if(gptr()) + if(buffer_) + return; + if(buffer_size_ > 0) { - const std::streamsize off = gptr() - egptr(); - setg(0, 0, 0); - assert(off <= std::numeric_limits::max()); - if(off && NOWIDE_FSEEK64(file_, static_cast(off), SEEK_CUR) != 0) - return false; + buffer_ = new char[buffer_size_]; + owns_buffer_ = true; } - return true; + } + + void set_unbuffered_read() + { + // In text mode we cannot use buffering as we are required to know the (file) position of each + // char in the get area and to seek back in case of a sync to "put back" unread chars. + // However std::fseek with non-zero offsets is unsupported for text files and the (file) offset + // to seek back is unknown anyway due to newlines which may got converted. + unbuffered_read_ = !(mode_ & std::ios_base::binary) || buffer_size_ == 0u; + } + + void validate_cvt(const std::locale& loc) + { + if(!std::use_facet>(loc).always_noconv()) + throw std::runtime_error("Converting codecvts are not supported"); + } + + /// Stop reading adjusting the file pointer if necessary + /// Postcondition: gptr() == nullptr + bool stop_reading() + { + if(!gptr()) + return true; + const auto off = gptr() - egptr(); + setg(nullptr, nullptr, nullptr); + if(!off) + return true; +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-constant-out-of-range-compare" +#endif + // coverity[result_independent_of_operands] + if(off < std::numeric_limits::min()) + return false; +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + return detail::fseek(file_, static_cast(off), SEEK_CUR) == 0; } /// Stop writing. If any bytes are to be written, writes them to file - /// Postcondition: pptr() == NULL + /// Postcondition: pptr() == nullptr bool stop_writing() { if(pptr()) { const char* const base = pbase(); const size_t n = pptr() - base; - setp(0, 0); + setp(nullptr, nullptr); if(n && std::fwrite(base, 1, n, file_) != n) return false; } return true; } - void reset(FILE* f = 0) - { - sync(); - if(file_) - { - fclose(file_); - file_ = 0; - } - file_ = f; - } - static const wchar_t* get_mode(std::ios_base::openmode mode) { // @@ -432,13 +513,14 @@ namespace nowide { return L"a+b"; if(mode == (std::ios_base::binary | std::ios_base::in | std::ios_base::app)) return L"a+b"; - return 0; + return nullptr; } - size_t buffer_size_; - char* buffer_; FILE* file_; + char* buffer_; + size_t buffer_size_; bool owns_buffer_; + bool unbuffered_read_; // True to read char by char char last_char_[1]; std::ios::openmode mode_; }; @@ -446,7 +528,14 @@ namespace nowide { /// /// \brief Convenience typedef /// - typedef basic_filebuf filebuf; + using filebuf = basic_filebuf; + + /// Swap the basic_filebuf instances + template + void swap(basic_filebuf& lhs, basic_filebuf& rhs) + { + lhs.swap(rhs); + } #endif // windows diff --git a/extern/boost/nowide/include/nowide/filesystem.hpp b/extern/boost/nowide/include/nowide/filesystem.hpp index a6bec96fb5..c23ee06361 100644 --- a/extern/boost/nowide/include/nowide/filesystem.hpp +++ b/extern/boost/nowide/include/nowide/filesystem.hpp @@ -1,21 +1,20 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_INTEGRATION_FILESYSTEM_HPP_INCLUDED #define NOWIDE_INTEGRATION_FILESYSTEM_HPP_INCLUDED -#if(defined(__GNUC__) && __GNUC__ < 5) -#pragma GCC diagnostic ignored "-Wunused-parameter" -#endif #include #include + namespace nowide { /// - /// Install utf8_codecvt facet into boost::filesystem::path such all char strings are interpreted as utf-8 strings + /// Install utf8_codecvt facet into boost::filesystem::path + /// such that all char strings are interpreted as UTF-8 strings + /// \return The previous imbued path locale. /// inline std::locale nowide_filesystem() { diff --git a/extern/boost/nowide/include/nowide/fstream.hpp b/extern/boost/nowide/include/nowide/fstream.hpp index bb7945a9d3..65d12014d9 100644 --- a/extern/boost/nowide/include/nowide/fstream.hpp +++ b/extern/boost/nowide/include/nowide/fstream.hpp @@ -1,17 +1,18 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_FSTREAM_HPP_INCLUDED #define NOWIDE_FSTREAM_HPP_INCLUDED #include +#include #include #include #include +#include namespace nowide { /// \cond INTERNAL @@ -23,7 +24,7 @@ namespace nowide { static std::ios_base::openmode mode_modifier() { return mode(); } template struct stream_base{ - typedef std::basic_istream type; + using type = std::basic_istream; }; }; struct StreamTypeOut @@ -32,7 +33,7 @@ namespace nowide { static std::ios_base::openmode mode_modifier() { return mode(); } template struct stream_base{ - typedef std::basic_ostream type; + using type = std::basic_ostream; }; }; struct StreamTypeInOut @@ -41,7 +42,7 @@ namespace nowide { static std::ios_base::openmode mode_modifier() { return std::ios_base::openmode(); } template struct stream_base{ - typedef std::basic_iostream type; + using type = std::basic_iostream; }; }; // clang-format on @@ -52,21 +53,24 @@ namespace nowide { /// the correct std::basic_[io]stream class and initializing it /// \tparam T_StreamType One of StreamType* above. /// Class used instead of value, because openmode::operator| may not be constexpr - template + /// \tparam FileBufType Discriminator to force a differing ABI if depending on the contained filebuf + template class fstream_impl; - template - struct enable_if_path; } // namespace detail /// \endcond /// /// \brief Same as std::basic_ifstream but accepts UTF-8 strings under Windows /// - template > + /// Affected by #NOWIDE_USE_FILEBUF_REPLACEMENT and #NOWIDE_USE_WCHAR_OVERLOADS + template> class basic_ifstream : public detail::fstream_impl { - typedef detail::fstream_impl fstream_impl; + using fstream_impl = detail::fstream_impl; public: basic_ifstream() @@ -89,9 +93,8 @@ namespace nowide { } template - explicit basic_ifstream( - const Path& file_name, - typename detail::enable_if_path::type mode = std::ios_base::in) + explicit basic_ifstream(const Path& file_name, + detail::enable_if_path_t mode = std::ios_base::in) { open(file_name, mode); } @@ -114,11 +117,11 @@ namespace nowide { /// /// \brief Same as std::basic_ofstream but accepts UTF-8 strings under Windows /// - - template > + /// Affected by #NOWIDE_USE_FILEBUF_REPLACEMENT and #NOWIDE_USE_WCHAR_OVERLOADS + template> class basic_ofstream : public detail::fstream_impl { - typedef detail::fstream_impl fstream_impl; + using fstream_impl = detail::fstream_impl; public: basic_ofstream() @@ -138,9 +141,8 @@ namespace nowide { open(file_name, mode); } template - explicit basic_ofstream( - const Path& file_name, - typename detail::enable_if_path::type mode = std::ios_base::out) + explicit basic_ofstream(const Path& file_name, + detail::enable_if_path_t mode = std::ios_base::out) { open(file_name, mode); } @@ -168,10 +170,11 @@ namespace nowide { /// /// \brief Same as std::basic_fstream but accepts UTF-8 strings under Windows /// - template > + /// Affected by #NOWIDE_USE_FILEBUF_REPLACEMENT and #NOWIDE_USE_WCHAR_OVERLOADS + template> class basic_fstream : public detail::fstream_impl { - typedef detail::fstream_impl fstream_impl; + using fstream_impl = detail::fstream_impl; public: basic_fstream() @@ -195,8 +198,8 @@ namespace nowide { } template explicit basic_fstream(const Path& file_name, - typename detail::enable_if_path::type mode = - std::ios_base::in | std::ios_base::out) + detail::enable_if_path_t mode = std::ios_base::in + | std::ios_base::out) { open(file_name, mode); } @@ -216,11 +219,7 @@ namespace nowide { return *this; } }; - template - void swap(basic_filebuf& lhs, basic_filebuf& rhs) - { - lhs.swap(rhs); - } + template void swap(basic_ifstream& lhs, basic_ifstream& rhs) { @@ -240,22 +239,22 @@ namespace nowide { /// /// Same as std::filebuf but accepts UTF-8 strings under Windows /// - typedef basic_filebuf filebuf; + using filebuf = basic_filebuf; /// /// Same as std::ifstream but accepts UTF-8 strings under Windows /// and *\::filesystem::path on all systems /// - typedef basic_ifstream ifstream; + using ifstream = basic_ifstream; /// /// Same as std::ofstream but accepts UTF-8 strings under Windows /// and *\::filesystem::path on all systems /// - typedef basic_ofstream ofstream; + using ofstream = basic_ofstream; /// /// Same as std::fstream but accepts UTF-8 strings under Windows /// and *\::filesystem::path on all systems /// - typedef basic_fstream fstream; + using fstream = basic_fstream; // Implementation namespace detail { @@ -266,13 +265,13 @@ namespace nowide { { T buf_; }; - template - class fstream_impl : private buf_holder >, // must be first due to init order + template + class fstream_impl : private buf_holder>, // must be first due to init order public T_StreamType::template stream_base::type { - typedef basic_filebuf internal_buffer_type; - typedef buf_holder base_buf_holder; - typedef typename T_StreamType::template stream_base::type stream_base; + using internal_buffer_type = basic_filebuf; + using base_buf_holder = buf_holder; + using stream_base = typename T_StreamType::template stream_base::type; public: using stream_base::setstate; @@ -287,8 +286,8 @@ namespace nowide { fstream_impl& operator=(const fstream_impl&) = delete; // coverity[exn_spec_violation] - fstream_impl(fstream_impl&& other) noexcept : base_buf_holder(std::move(other)), - stream_base(std::move(other)) + fstream_impl(fstream_impl&& other) noexcept : + base_buf_holder(std::move(other)), stream_base(std::move(other)) { this->set_rdbuf(rdbuf()); } @@ -309,8 +308,8 @@ namespace nowide { open(file_name.c_str(), mode); } template - typename detail::enable_if_path::type open(const Path& file_name, - std::ios_base::openmode mode = T_StreamType::mode()) + detail::enable_if_path_t open(const Path& file_name, + std::ios_base::openmode mode = T_StreamType::mode()) { open(file_name.c_str(), mode); } @@ -352,41 +351,6 @@ namespace nowide { #ifdef NOWIDE_MSVC #pragma warning(pop) #endif - /// Trait to heuristically check for a *\::filesystem::path - /// Done by checking for make_preferred and filename member functions with correct signature - template - struct is_path - { - typedef char one; - struct two - { - char dummy[2]; - }; - - template - struct Check; - template - static one test(Check*); - template - static two test(...); - - enum - { - value = sizeof(test(0)) == sizeof(one) - }; - }; - template - struct enable_if - {}; - template - struct enable_if - { - typedef T type; - }; - /// SFINAE trait which has a member "type = Result" if the Path is a *\::filesystem::path - template - struct enable_if_path : enable_if::value, Result> - {}; } // namespace detail } // namespace nowide diff --git a/extern/boost/nowide/include/nowide/iostream.hpp b/extern/boost/nowide/include/nowide/iostream.hpp index 6d0ea83be3..1a9a821d3b 100644 --- a/extern/boost/nowide/include/nowide/iostream.hpp +++ b/extern/boost/nowide/include/nowide/iostream.hpp @@ -1,11 +1,9 @@ +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2020-2021 Alexander Grund // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// Copyright (c) 2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) -// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_IOSTREAM_HPP_INCLUDED #define NOWIDE_IOSTREAM_HPP_INCLUDED @@ -40,11 +38,19 @@ namespace nowide { class NOWIDE_DECL winconsole_ostream : public std::ostream { public: - winconsole_ostream(int fd, winconsole_ostream* tieStream); + enum class target_stream + { + output, + error, + log, + }; + winconsole_ostream(target_stream target, bool isBuffered, winconsole_ostream* tieStream); ~winconsole_ostream(); private: std::unique_ptr d; + // Ensure the std streams are initialized and alive during the lifetime of this instance + std::ios_base::Init init_; }; class NOWIDE_DECL winconsole_istream : public std::istream @@ -55,6 +61,8 @@ namespace nowide { private: std::unique_ptr d; + // Ensure the std streams are initialized and alive during the lifetime of this instance + std::ios_base::Init init_; }; } // namespace detail diff --git a/extern/boost/nowide/include/nowide/quoted.hpp b/extern/boost/nowide/include/nowide/quoted.hpp new file mode 100644 index 0000000000..b52cecbe4f --- /dev/null +++ b/extern/boost/nowide/include/nowide/quoted.hpp @@ -0,0 +1,107 @@ +// +// Copyright (c) 2023 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef NOWIDE_QUOTED_HPP_INCLUDED +#define NOWIDE_QUOTED_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include +#include + +#if defined(__cpp_lib_quoted_string_io) && __cpp_lib_quoted_string_io >= 201304 + +namespace nowide { + /// \cond INTERNAL + namespace detail { + template + struct quoted; + template + using remove_cvref_t = typename std::remove_cv::type>::type; + + } // namespace detail + /// \endcond + + /// \brief Allows insertion and extraction of `filesystem::path` into/from streams. + /// + /// When used in an expression such as `out << quoted(path)`, where `out` is an output stream, + /// has the effect as-if `out << std::quoted(path.native())` was used. + /// + /// When used in an expression like `in >> quoted(path)`, where `in` is an input stream, + /// has the effect as-if `in >> std::quoted(path.native())` was used if that would be valid. + /// To that effect a temporary string is used, which on success is assigned to `path`. + /// + /// Will automatically convert between the streams `char_type` and `path::value_type` if necessary. + template +#ifdef NOWIDE_DOXYGEN + unspecified_type +#else + detail::enable_if_path_t, detail::quoted> +#endif + quoted(Path& path) + { + return {path}; + } + + /// \cond INTERNAL + // Same but for const-refs and r-values + template + detail::enable_if_path_t, detail::quoted> quoted(const Path& path) + { + return {path}; + } + + namespace detail { + template::value>::type> + std::basic_string maybe_convert_string(const std::basic_string& s) + { + return utf::convert_string(s); + } + template + const std::basic_string& maybe_convert_string(const std::basic_string& s) + { + return s; + } + + template + using requires_non_const = + typename std::enable_if::type>::value>::type; + + template + struct quoted + { + Path value; + template + friend std::basic_ostream& operator<<(std::basic_ostream& out, const quoted& path) + { + return out << std::quoted(maybe_convert_string(path.value.native())); + } + + template> + friend std::basic_istream& operator>>(std::basic_istream& in, const quoted& path) + { + std::basic_string value; + using PlainPath = remove_cvref_t; + if(in >> std::quoted(value)) + path.value = PlainPath(maybe_convert_string(value)); + return in; + } + }; + + } // namespace detail + /// \endcond +} // namespace nowide + +#elif defined(NOWIDE_PRAGMA_MESSAGE) +NOWIDE_PRAGMA_MESSAGE("To use nowide::quoted at least C++14 is required.") +#endif + +#endif diff --git a/extern/boost/nowide/include/nowide/replacement.hpp b/extern/boost/nowide/include/nowide/replacement.hpp index d00db0fff7..755e62f18c 100644 --- a/extern/boost/nowide/include/nowide/replacement.hpp +++ b/extern/boost/nowide/include/nowide/replacement.hpp @@ -1,10 +1,9 @@ // -// Copyright (c) 2018 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2018 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_REPLACEMENT_HPP_INCLUDED #define NOWIDE_REPLACEMENT_HPP_INCLUDED diff --git a/extern/boost/nowide/include/nowide/stackstring.hpp b/extern/boost/nowide/include/nowide/stackstring.hpp index 6aac24ea97..4b6f385875 100644 --- a/extern/boost/nowide/include/nowide/stackstring.hpp +++ b/extern/boost/nowide/include/nowide/stackstring.hpp @@ -1,14 +1,14 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_STACKSTRING_HPP_INCLUDED #define NOWIDE_STACKSTRING_HPP_INCLUDED #include +#include #include #include @@ -24,7 +24,7 @@ namespace nowide { /// Invalid UTF characters are replaced by the substitution character, see #NOWIDE_REPLACEMENT_CHARACTER /// /// If a NULL pointer is passed to the constructor or convert method, NULL will be returned by c_str. - /// Similarily a default constructed stackstring will return NULL on calling c_str. + /// Similarly a default constructed stackstring will return NULL on calling c_str. /// template class basic_stackstring @@ -33,29 +33,29 @@ namespace nowide { /// Size of the stack buffer static const size_t buffer_size = BufferSize; /// Type of the output character (converted to) - typedef CharOut output_char; + using output_char = CharOut; /// Type of the input character (converted from) - typedef CharIn input_char; + using input_char = CharIn; /// Creates a NULL stackstring - basic_stackstring() : data_(NULL) + basic_stackstring() { buffer_[0] = 0; } /// Convert the NULL terminated string input and store in internal buffer /// If input is NULL, nothing will be stored - explicit basic_stackstring(const input_char* input) : data_(NULL) + explicit basic_stackstring(const input_char* input) { convert(input); } /// Convert the sequence [begin, end) and store in internal buffer /// If begin is NULL, nothing will be stored - basic_stackstring(const input_char* begin, const input_char* end) : data_(NULL) + basic_stackstring(const input_char* begin, const input_char* end) { convert(begin, end); } /// Copy construct from other - basic_stackstring(const basic_stackstring& other) : data_(NULL) + basic_stackstring(const basic_stackstring& other) { *this = other; } @@ -72,7 +72,7 @@ namespace nowide { data_ = new output_char[len + 1]; else { - data_ = NULL; + data_ = nullptr; return *this; } std::memcpy(data_, other.data_, sizeof(output_char) * (len + 1)); @@ -90,7 +90,7 @@ namespace nowide { output_char* convert(const input_char* input) { if(input) - return convert(input, input + detail::strlen(input)); + return convert(input, input + utf::strlen(input)); clear(); return get(); } @@ -106,15 +106,15 @@ namespace nowide { // Minimum size required: 1 output char per input char + trailing NULL const size_t min_output_size = input_len + 1; // If there is a chance the converted string fits on stack, try it - if(min_output_size <= buffer_size && detail::convert_buffer(buffer_, buffer_size, begin, end)) + if(min_output_size <= buffer_size && utf::convert_buffer(buffer_, buffer_size, begin, end)) data_ = buffer_; else { // Fallback: Allocate a buffer that is surely large enough on heap // Max size: Every input char is transcoded to the output char with maximum with + trailing NULL - const size_t max_output_size = input_len * detail::utf::utf_traits::max_width + 1; + const size_t max_output_size = input_len * utf::utf_traits::max_width + 1; data_ = new output_char[max_output_size]; - const bool success = detail::convert_buffer(data_, max_output_size, begin, end) == data_; + const bool success = utf::convert_buffer(data_, max_output_size, begin, end) == data_; assert(success); (void)success; } @@ -136,7 +136,7 @@ namespace nowide { { if(!uses_stack_memory()) delete[] data_; - data_ = NULL; + data_ = nullptr; } /// Swap lhs with rhs friend void swap(basic_stackstring& lhs, basic_stackstring& rhs) @@ -184,25 +184,25 @@ namespace nowide { private: output_char buffer_[buffer_size]; - output_char* data_; + output_char* data_ = nullptr; }; // basic_stackstring /// /// Convenience typedef /// - typedef basic_stackstring wstackstring; + using wstackstring = basic_stackstring; /// /// Convenience typedef /// - typedef basic_stackstring stackstring; + using stackstring = basic_stackstring; /// /// Convenience typedef /// - typedef basic_stackstring wshort_stackstring; + using wshort_stackstring = basic_stackstring; /// /// Convenience typedef /// - typedef basic_stackstring short_stackstring; + using short_stackstring = basic_stackstring; } // namespace nowide diff --git a/extern/boost/nowide/include/nowide/stat.hpp b/extern/boost/nowide/include/nowide/stat.hpp new file mode 100644 index 0000000000..43a536340e --- /dev/null +++ b/extern/boost/nowide/include/nowide/stat.hpp @@ -0,0 +1,67 @@ +// +// Copyright (c) 2020 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef NOWIDE_STAT_HPP_INCLUDED +#define NOWIDE_STAT_HPP_INCLUDED + +#include +#include +// Include after sys/types.h +#include + +#if defined(__MINGW32__) && defined(__MSVCRT_VERSION__) && __MSVCRT_VERSION__ < 0x0601 +/// Forward declaration in case MinGW32 is used and __MSVCRT_VERSION__ is defined lower than 6.1 +struct __stat64; +#endif + +namespace nowide { +#if !defined(NOWIDE_WINDOWS) && !defined(NOWIDE_DOXYGEN) + // Note: `using x = struct ::stat` causes a bogus warning in GCC < 11 + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66159 + + typedef struct ::stat stat_t; + typedef struct ::stat posix_stat_t; + + using ::stat; +#else + /// \brief Typedef for the file info structure. + /// Able to hold 64 bit file size and timestamps on Windows and usually also on other 64 Bit systems + /// This allows to write portable code with optional LFS support + typedef struct ::__stat64 stat_t; + /// \brief Typedef for the file info structure used in the POSIX stat call + /// Resolves to `struct _stat` on Windows and `struct stat` otherwise + /// This allows to write portable code using the default stat function + typedef struct ::_stat posix_stat_t; + + /// \cond INTERNAL + namespace detail { + NOWIDE_DECL int stat(const char* path, stat_t* buffer, size_t buffer_size); + NOWIDE_DECL int stat(const char* path, posix_stat_t* buffer, size_t buffer_size); + } // namespace detail + /// \endcond + + /// + /// \brief UTF-8 aware stat function, returns 0 on success + /// + /// Return information about a file from an UTF-8 encoded path + /// + inline int stat(const char* path, stat_t* buffer) + { + return detail::stat(path, buffer, sizeof(*buffer)); + } + /// + /// \brief UTF-8 aware stat function, returns 0 on success + /// + /// Return information about a file from an UTF-8 encoded path + /// + inline int stat(const char* path, posix_stat_t* buffer) + { + return detail::stat(path, buffer, sizeof(*buffer)); + } +#endif +} // namespace nowide + +#endif diff --git a/extern/boost/nowide/include/nowide/utf/convert.hpp b/extern/boost/nowide/include/nowide/utf/convert.hpp new file mode 100644 index 0000000000..21b03274d3 --- /dev/null +++ b/extern/boost/nowide/include/nowide/utf/convert.hpp @@ -0,0 +1,106 @@ +// +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2020 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef NOWIDE_UTF_CONVERT_HPP_INCLUDED +#define NOWIDE_UTF_CONVERT_HPP_INCLUDED + +#include +#include +#include +#include +#include + +namespace nowide { + namespace utf { + + /// Return the length of the given string in code units. + /// That is the number of elements of type Char until the first NULL character. + /// Equivalent to `std::strlen(s)` but can handle wide-strings + template + size_t strlen(const Char* s) + { + const Char* end = s; + while(*end) + end++; + return end - s; + } + + /// Convert a buffer of UTF sequences in the range [source_begin, source_end) + /// from \a CharIn to \a CharOut to the output \a buffer of size \a buffer_size. + /// + /// \return original buffer containing the NULL terminated string or NULL + /// + /// If there is not enough room in the buffer NULL is returned, and the content of the buffer is undefined. + /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER + template + CharOut* + convert_buffer(CharOut* buffer, size_t buffer_size, const CharIn* source_begin, const CharIn* source_end) + { + CharOut* rv = buffer; + if(buffer_size == 0) + return nullptr; + buffer_size--; + while(source_begin != source_end) + { + code_point c = utf_traits::decode(source_begin, source_end); + if(c == illegal || c == incomplete) + { + c = NOWIDE_REPLACEMENT_CHARACTER; + } + size_t width = utf_traits::width(c); + if(buffer_size < width) + { + rv = nullptr; + break; + } + buffer = utf_traits::encode(c, buffer); + buffer_size -= width; + } + *buffer++ = 0; + return rv; + } + + /// Convert the UTF sequences in range [begin, end) from \a CharIn to \a CharOut + /// and return it as a string + /// + /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER + /// \tparam CharOut Output character type + template + std::basic_string convert_string(const CharIn* begin, const CharIn* end) + { + std::basic_string result; + result.reserve(end - begin); + using inserter_type = std::back_insert_iterator>; + inserter_type inserter(result); + code_point c; + while(begin != end) + { + c = utf_traits::decode(begin, end); + if(c == illegal || c == incomplete) + { + c = NOWIDE_REPLACEMENT_CHARACTER; + } + utf_traits::encode(c, inserter); + } + return result; + } + + /// Convert the UTF sequence in the input string from \a CharIn to \a CharOut + /// and return it as a string + /// + /// Any illegal sequences are replaced with the replacement character, see #NOWIDE_REPLACEMENT_CHARACTER + /// \tparam CharOut Output character type + template + std::basic_string convert_string(const std::basic_string& s) + { + return convert_string(s.data(), s.data() + s.size()); + } + + } // namespace utf +} // namespace nowide + +#endif diff --git a/extern/boost/nowide/include/nowide/utf/utf.hpp b/extern/boost/nowide/include/nowide/utf/utf.hpp new file mode 100644 index 0000000000..198f89ca90 --- /dev/null +++ b/extern/boost/nowide/include/nowide/utf/utf.hpp @@ -0,0 +1,452 @@ +// +// Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2020 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef NOWIDE_UTF_HPP_INCLUDED +#define NOWIDE_UTF_HPP_INCLUDED + +#include +#include + +namespace nowide { + /// + /// \brief Namespace that holds basic operations on UTF encoded sequences + /// + /// All functions defined in this namespace do not require linking with Boost.Nowide library. + /// Extracted from Boost.Locale + /// + namespace utf { + + /// + /// \brief The integral type that can hold a Unicode code point + /// + using code_point = uint32_t; + + /// + /// \brief Special constant that defines illegal code point + /// + static const code_point illegal = 0xFFFFFFFFu; + + /// + /// \brief Special constant that defines incomplete code point + /// + static const code_point incomplete = 0xFFFFFFFEu; + + /// + /// \brief the function checks if \a v is a valid code point + /// + inline bool is_valid_codepoint(code_point v) + { + if(v > 0x10FFFF) + return false; + if(0xD800 <= v && v <= 0xDFFF) // surrogates + return false; + return true; + } + +#ifdef NOWIDE_DOXYGEN + /// + /// \brief UTF Traits class - functions to convert UTF sequences to and from Unicode code points + /// + template + struct utf_traits + { + /// + /// The type of the character + /// + using char_type = CharType; + /// + /// Read one code point from the range [p,e) and return it. + /// + /// - If the sequence that was read is incomplete sequence returns \ref incomplete, + /// - If illegal sequence detected returns \ref illegal + /// + /// Requirements + /// + /// - Iterator is valid input iterator + /// + /// Postconditions + /// + /// - p points to the last consumed character + /// + template + static code_point decode(Iterator& p, Iterator e); + + /// + /// Maximal width of valid sequence in the code units: + /// + /// - UTF-8 - 4 + /// - UTF-16 - 2 + /// - UTF-32 - 1 + /// + static const int max_width; + /// + /// The width of specific code point in the code units. + /// + /// Requirement: value is a valid Unicode code point + /// Returns value in range [1..max_width] + /// + static int width(code_point value); + + /// + /// Get the size of the trail part of variable length encoded sequence. + /// + /// Returns -1 if C is not valid lead character + /// + static int trail_length(char_type c); + /// + /// Returns true if c is trail code unit, always false for UTF-32 + /// + static bool is_trail(char_type c); + /// + /// Returns true if c is lead code unit, always true of UTF-32 + /// + static bool is_lead(char_type c); + + /// + /// Convert valid Unicode code point \a value to the UTF sequence. + /// + /// Requirements: + /// + /// - \a value is valid code point + /// - \a out is an output iterator should be able to accept at least width(value) units + /// + /// Returns the iterator past the last written code unit. + /// + template + static Iterator encode(code_point value, Iterator out); + /// + /// Decodes valid UTF sequence that is pointed by p into code point. + /// + /// If the sequence is invalid or points to end the behavior is undefined + /// + template + static code_point decode_valid(Iterator& p); + }; + +#else + + template + struct utf_traits; + + template + struct utf_traits + { + using char_type = CharType; + + static int trail_length(char_type ci) + { + unsigned char c = ci; + if(c < 128) + return 0; + if(NOWIDE_UNLIKELY(c < 194)) + return -1; + if(c < 224) + return 1; + if(c < 240) + return 2; + if(NOWIDE_LIKELY(c <= 244)) + return 3; + return -1; + } + + static const int max_width = 4; + + static int width(code_point value) + { + if(value <= 0x7F) + { + return 1; + } else if(value <= 0x7FF) + { + return 2; + } else if(NOWIDE_LIKELY(value <= 0xFFFF)) + { + return 3; + } else + { + return 4; + } + } + + static bool is_trail(char_type ci) + { + unsigned char c = ci; + return (c & 0xC0) == 0x80; + } + + static bool is_lead(char_type ci) + { + return !is_trail(ci); + } + + template + static code_point decode(Iterator& p, Iterator e) + { + if(NOWIDE_UNLIKELY(p == e)) + return incomplete; + + unsigned char lead = *p++; + + // First byte is fully validated here + int trail_size = trail_length(lead); + + if(NOWIDE_UNLIKELY(trail_size < 0)) + return illegal; + + // OK as only ASCII may be of size = 0 + // also optimize for ASCII text + if(trail_size == 0) + return lead; + + code_point c = lead & ((1 << (6 - trail_size)) - 1); + + // Read the rest + unsigned char tmp; + switch(trail_size) + { + case 3: + if(NOWIDE_UNLIKELY(p == e)) + return incomplete; + tmp = *p++; + if(!is_trail(tmp)) + return illegal; + c = (c << 6) | (tmp & 0x3F); + NOWIDE_FALLTHROUGH; + case 2: + if(NOWIDE_UNLIKELY(p == e)) + return incomplete; + tmp = *p++; + if(!is_trail(tmp)) + return illegal; + c = (c << 6) | (tmp & 0x3F); + NOWIDE_FALLTHROUGH; + case 1: + if(NOWIDE_UNLIKELY(p == e)) + return incomplete; + tmp = *p++; + if(!is_trail(tmp)) + return illegal; + c = (c << 6) | (tmp & 0x3F); + } + + // Check code point validity: + // - no surrogates and valid range + // - most compact representation + if(NOWIDE_UNLIKELY(!is_valid_codepoint(c)) || NOWIDE_UNLIKELY(width(c) != trail_size + 1)) + { + p -= trail_size; + return illegal; + } + + return c; + } + + template + static code_point decode_valid(Iterator& p) + { + unsigned char lead = *p++; + if(lead < 192) + return lead; + + int trail_size; + + if(lead < 224) + trail_size = 1; + else if(NOWIDE_LIKELY(lead < 240)) // non-BMP rare + trail_size = 2; + else + trail_size = 3; + + code_point c = lead & ((1 << (6 - trail_size)) - 1); + + switch(trail_size) + { + case 3: c = (c << 6) | (static_cast(*p++) & 0x3F); NOWIDE_FALLTHROUGH; + case 2: c = (c << 6) | (static_cast(*p++) & 0x3F); NOWIDE_FALLTHROUGH; + case 1: c = (c << 6) | (static_cast(*p++) & 0x3F); + } + + return c; + } + + template + static Iterator encode(code_point value, Iterator out) + { + if(value <= 0x7F) + { + *out++ = static_cast(value); + } else if(value <= 0x7FF) + { + *out++ = static_cast((value >> 6) | 0xC0); + *out++ = static_cast((value & 0x3F) | 0x80); + } else if(NOWIDE_LIKELY(value <= 0xFFFF)) + { + *out++ = static_cast((value >> 12) | 0xE0); + *out++ = static_cast(((value >> 6) & 0x3F) | 0x80); + *out++ = static_cast((value & 0x3F) | 0x80); + } else + { + *out++ = static_cast((value >> 18) | 0xF0); + *out++ = static_cast(((value >> 12) & 0x3F) | 0x80); + *out++ = static_cast(((value >> 6) & 0x3F) | 0x80); + *out++ = static_cast((value & 0x3F) | 0x80); + } + return out; + } + }; // utf8 + + template + struct utf_traits + { + using char_type = CharType; + + // See RFC 2781 + static bool is_single_codepoint(uint16_t x) + { + // Ranges [U+0000, 0+D7FF], [U+E000, U+FFFF] are numerically equal in UTF-16 + return x <= 0xD7FF || x >= 0xE000; + } + static bool is_first_surrogate(uint16_t x) + { + // Range [U+D800, 0+DBFF]: High surrogate + return 0xD800 <= x && x <= 0xDBFF; + } + static bool is_second_surrogate(uint16_t x) + { + // Range [U+DC00, 0+DFFF]: Low surrogate + return 0xDC00 <= x && x <= 0xDFFF; + } + static code_point combine_surrogate(uint16_t w1, uint16_t w2) + { + return ((code_point(w1 & 0x3FF) << 10) | (w2 & 0x3FF)) + 0x10000; + } + static int trail_length(char_type c) + { + if(is_first_surrogate(c)) + return 1; + if(is_second_surrogate(c)) + return -1; + return 0; + } + /// Return true if c is trail code unit, always false for UTF-32 + static bool is_trail(char_type c) + { + return is_second_surrogate(c); + } + /// Return true if c is lead code unit, always true of UTF-32 + static bool is_lead(char_type c) + { + return !is_second_surrogate(c); + } + + template + static code_point decode(It& current, It last) + { + if(NOWIDE_UNLIKELY(current == last)) + return incomplete; + uint16_t w1 = *current++; + if(NOWIDE_LIKELY(is_single_codepoint(w1))) + { + return w1; + } + // Now it's either a high or a low surrogate, the latter is invalid + if(w1 >= 0xDC00) + return illegal; + if(current == last) + return incomplete; + uint16_t w2 = *current++; + if(!is_second_surrogate(w2)) + return illegal; + return combine_surrogate(w1, w2); + } + template + static code_point decode_valid(It& current) + { + uint16_t w1 = *current++; + if(NOWIDE_LIKELY(is_single_codepoint(w1))) + { + return w1; + } + uint16_t w2 = *current++; + return combine_surrogate(w1, w2); + } + + static const int max_width = 2; + static int width(code_point u) // LCOV_EXCL_LINE + { + return u >= 0x10000 ? 2 : 1; + } + template + static It encode(code_point u, It out) + { + if(NOWIDE_LIKELY(u <= 0xFFFF)) + { + *out++ = static_cast(u); + } else + { + u -= 0x10000; + *out++ = static_cast(0xD800 | (u >> 10)); + *out++ = static_cast(0xDC00 | (u & 0x3FF)); + } + return out; + } + }; // utf16; + + template + struct utf_traits + { + using char_type = CharType; + static int trail_length(char_type c) + { + if(is_valid_codepoint(c)) + return 0; + return -1; + } + static bool is_trail(char_type /*c*/) + { + return false; + } + static bool is_lead(char_type /*c*/) + { + return true; + } + + template + static code_point decode_valid(It& current) + { + return *current++; + } + + template + static code_point decode(It& current, It last) + { + if(NOWIDE_UNLIKELY(current == last)) + return incomplete; + code_point c = *current++; + if(NOWIDE_UNLIKELY(!is_valid_codepoint(c))) + return illegal; + return c; + } + static const int max_width = 1; + static int width(code_point /*u*/) + { + return 1; + } + template + static It encode(code_point u, It out) + { + *out++ = static_cast(u); + return out; + } + }; // utf32 + +#endif + + } // namespace utf +} // namespace nowide + +#endif diff --git a/extern/boost/nowide/include/nowide/utf8_codecvt.hpp b/extern/boost/nowide/include/nowide/utf8_codecvt.hpp index b7aa87bc3d..b32918cdc0 100644 --- a/extern/boost/nowide/include/nowide/utf8_codecvt.hpp +++ b/extern/boost/nowide/include/nowide/utf8_codecvt.hpp @@ -1,16 +1,16 @@ // -// Copyright (c) 2015 Artyom Beilis (Tonkikh) -// Copyright (c) 2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2015 Artyom Beilis (Tonkikh) +// Copyright (c) 2020 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_UTF8_CODECVT_HPP_INCLUDED #define NOWIDE_UTF8_CODECVT_HPP_INCLUDED -#include #include +#include +#include #include #include @@ -38,11 +38,6 @@ namespace nowide { } } // namespace detail -#if defined _MSC_VER && _MSC_VER < 1700 -// MSVC do_length is non-standard it counts wide characters instead of narrow and does not change mbstate -#define NOWIDE_DO_LENGTH_MBSTATE_CONST -#endif - /// std::codecvt implementation that converts between UTF-8 and UTF-16 or UTF-32 /// /// @tparam CharSize Determines the encoding: 2 for UTF-16, 4 for UTF-32 @@ -52,6 +47,7 @@ namespace nowide { template class utf8_codecvt; + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN /// Specialization for the UTF-8 <-> UTF-16 variant of the std::codecvt implementation template class NOWIDE_SYMBOL_VISIBLE utf8_codecvt : public std::codecvt @@ -61,9 +57,10 @@ namespace nowide { utf8_codecvt(size_t refs = 0) : std::codecvt(refs) {} + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END protected: - typedef CharType uchar; + using uchar = CharType; std::codecvt_base::result do_unshift(std::mbstate_t& s, char* from, char* /*to*/, char*& next) const override { @@ -85,55 +82,48 @@ namespace nowide { return false; } - int do_length(std::mbstate_t -#ifdef NOWIDE_DO_LENGTH_MBSTATE_CONST - const -#endif - & std_state, - const char* from, - const char* from_end, - size_t max) const override + // LCOV_EXCL_START + int do_length(std::mbstate_t& std_state, const char* from, const char* from_end, size_t max) const override { + // LCOV_EXCL_STOP + using utf16_traits = utf::utf_traits; std::uint16_t state = detail::read_state(std_state); -#ifndef NOWIDE_DO_LENGTH_MBSTATE_CONST const char* save_from = from; -#else - size_t save_max = max; -#endif + if(state && max > 0) + { + max--; + state = 0; + } while(max > 0 && from < from_end) { const char* prev_from = from; - std::uint32_t ch = detail::utf::utf_traits::decode(from, from_end); - if(ch == detail::utf::illegal) + std::uint32_t ch = utf::utf_traits::decode(from, from_end); + if(ch == utf::illegal) { ch = NOWIDE_REPLACEMENT_CHARACTER; - } else if(ch == detail::utf::incomplete) + } else if(ch == utf::incomplete) { from = prev_from; break; } - max--; - if(ch > 0xFFFF) + // If we can't write the char, we have to save the low surrogate in state + if(NOWIDE_LIKELY(static_cast(utf16_traits::width(ch)) <= max)) { - if(state == 0) - { - from = prev_from; - state = 1; - } else - { - state = 0; - } + max -= utf16_traits::width(ch); + } else + { + static_assert(utf16_traits::max_width == 2, "Required for below"); + std::uint16_t tmpOut[2]{}; + utf16_traits::encode(ch, tmpOut); + state = tmpOut[1]; + break; } } -#ifndef NOWIDE_DO_LENGTH_MBSTATE_CONST detail::write_state(std_state, state); return static_cast(from - save_from); -#else - return static_cast(save_max - max); -#endif } - std::codecvt_base::result do_in(std::mbstate_t& std_state, + std::codecvt_base::result do_in(std::mbstate_t& std_state, // LCOV_EXCL_LINE const char* from, const char* from_end, const char*& from_next, @@ -142,58 +132,45 @@ namespace nowide { uchar*& to_next) const override { std::codecvt_base::result r = std::codecvt_base::ok; + using utf16_traits = utf::utf_traits; - // mbstate_t is POD type and should be initialized to 0 (i.a. state = stateT()) - // according to standard. We use it to keep a flag 0/1 for surrogate pair writing - // - // if 0 no code above >0xFFFF observed, of 1 a code above 0xFFFF observed - // and first pair is written, but no input consumed + // mbstate_t is POD type and should be initialized to 0 (i.e. state = stateT()) + // according to standard. + // We use it to store a low surrogate if it was not yet written, else state is 0 std::uint16_t state = detail::read_state(std_state); + // Write low surrogate if present + if(state && to < to_end) + { + *to++ = static_cast(state); + state = 0; + } while(to < to_end && from < from_end) { const char* from_saved = from; - uint32_t ch = detail::utf::utf_traits::decode(from, from_end); + uint32_t ch = utf::utf_traits::decode(from, from_end); - if(ch == detail::utf::illegal) + if(ch == utf::illegal) { ch = NOWIDE_REPLACEMENT_CHARACTER; - } else if(ch == detail::utf::incomplete) + } else if(ch == utf::incomplete) { from = from_saved; r = std::codecvt_base::partial; break; } - // Normal codepoints go directly to stream - if(ch <= 0xFFFF) + // If the encoded char fits, write directly, else safe the low surrogate in state + if(NOWIDE_LIKELY(utf16_traits::width(ch) <= to_end - to)) { - *to++ = static_cast(ch); + to = utf16_traits::encode(ch, to); } else { - // for other codepoints we do following - // - // 1. We can't consume our input as we may find ourself - // in state where all input consumed but not all output written,i.e. only - // 1st pair is written - // 2. We only write first pair and mark this in the state, we also revert back - // the from pointer in order to make sure this codepoint would be read - // once again and then we would consume our input together with writing - // second surrogate pair - ch -= 0x10000; - std::uint16_t vh = static_cast(ch >> 10); - std::uint16_t vl = ch & 0x3FF; - std::uint16_t w1 = vh + 0xD800; - std::uint16_t w2 = vl + 0xDC00; - if(state == 0) - { - from = from_saved; - *to++ = static_cast(w1); - state = 1; - } else - { - *to++ = static_cast(w2); - state = 0; - } + static_assert(utf16_traits::max_width == 2, "Required for below"); + std::uint16_t tmpOut[2]{}; + utf16_traits::encode(ch, tmpOut); + *to++ = static_cast(tmpOut[0]); + state = tmpOut[1]; + break; } } from_next = from; @@ -213,69 +190,53 @@ namespace nowide { char*& to_next) const override { std::codecvt_base::result r = std::codecvt_base::ok; - // mbstate_t is POD type and should be initialized to 0 (i.a. state = stateT()) - // according to standard. We assume that sizeof(mbstate_t) >=2 in order - // to be able to store first observed surrogate pair - // - // State: state!=0 - a first surrogate pair was observed (state = first pair), - // we expect the second one to come and then zero the state - /// + using utf16_traits = utf::utf_traits; + // mbstate_t is POD type and should be initialized to 0 + // (i.e. state = stateT()) according to standard. + // We use it to store the first observed surrogate pair, or 0 if there is none yet std::uint16_t state = detail::read_state(std_state); - while(to < to_end && from < from_end) + for(; to < to_end && from < from_end; ++from) { std::uint32_t ch = 0; if(state != 0) { - // if the state indicates that 1st surrogate pair was written - // we should make sure that the second one that comes is actually - // second surrogate + // We have a high surrogate, so now there should be a low surrogate std::uint16_t w1 = state; std::uint16_t w2 = *from; - // we don't forward from as writing may fail to incomplete or - // partial conversion - if(0xDC00 <= w2 && w2 <= 0xDFFF) + if(NOWIDE_LIKELY(utf16_traits::is_trail(w2))) { - std::uint16_t vh = w1 - 0xD800; - std::uint16_t vl = w2 - 0xDC00; - ch = ((uint32_t(vh) << 10) | vl) + 0x10000; + ch = utf16_traits::combine_surrogate(w1, w2); } else { ch = NOWIDE_REPLACEMENT_CHARACTER; } } else { - ch = *from; - if(0xD800 <= ch && ch <= 0xDBFF) + std::uint16_t w1 = *from; + if(NOWIDE_LIKELY(utf16_traits::is_single_codepoint(w1))) + { + ch = w1; + } else if(NOWIDE_LIKELY(utf16_traits::is_first_surrogate(w1))) { - // if this is a first surrogate pair we put - // it into the state and consume it, note we don't - // go forward as it should be illegal so we increase - // the from pointer manually - state = static_cast(ch); - from++; + // Store into state and continue at next character + state = w1; continue; - } else if(0xDC00 <= ch && ch <= 0xDFFF) + } else { - // if we observe second surrogate pair and - // first only may be expected we should break from the loop with error - // as it is illegal input + // Neither a single codepoint nor a high surrogate so must be low surrogate. + // This is an error -> Replace character ch = NOWIDE_REPLACEMENT_CHARACTER; } } - if(!detail::utf::is_valid_codepoint(ch)) - { - r = std::codecvt_base::error; - break; - } - int len = detail::utf::utf_traits::width(ch); + assert(utf::is_valid_codepoint(ch)); // Any valid UTF16 sequence is a valid codepoint + int len = utf::utf_traits::width(ch); if(to_end - to < len) { r = std::codecvt_base::partial; break; } - to = detail::utf::utf_traits::encode(ch, to); + to = utf::utf_traits::encode(ch, to); state = 0; - from++; } from_next = from; to_next = to; @@ -286,6 +247,7 @@ namespace nowide { } }; + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN /// Specialization for the UTF-8 <-> UTF-32 variant of the std::codecvt implementation template class NOWIDE_SYMBOL_VISIBLE utf8_codecvt : public std::codecvt @@ -293,15 +255,16 @@ namespace nowide { public: utf8_codecvt(size_t refs = 0) : std::codecvt(refs) {} + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END protected: - typedef CharType uchar; + using uchar = CharType; std::codecvt_base::result do_unshift(std::mbstate_t& /*s*/, char* from, char* /*to*/, char*& next) const override { next = from; - return std::codecvt_base::ok; + return std::codecvt_base::noconv; } int do_encoding() const noexcept override { @@ -316,40 +279,25 @@ namespace nowide { return false; } - int do_length(std::mbstate_t -#ifdef NOWIDE_DO_LENGTH_MBSTATE_CONST - const -#endif - & /*state*/, - const char* from, - const char* from_end, - size_t max) const override + int do_length(std::mbstate_t& /*state*/, const char* from, const char* from_end, size_t max) const override { -#ifndef NOWIDE_DO_LENGTH_MBSTATE_CONST const char* start_from = from; -#else - size_t save_max = max; -#endif while(max > 0 && from < from_end) { const char* save_from = from; - std::uint32_t ch = detail::utf::utf_traits::decode(from, from_end); - if(ch == detail::utf::incomplete) + std::uint32_t ch = utf::utf_traits::decode(from, from_end); + if(ch == utf::incomplete) { from = save_from; break; - } else if(ch == detail::utf::illegal) + } else if(ch == utf::illegal) { ch = NOWIDE_REPLACEMENT_CHARACTER; } max--; } -#ifndef NOWIDE_DO_LENGTH_MBSTATE_CONST - return from - start_from; -#else - return save_max - max; -#endif + return static_cast(from - start_from); } std::codecvt_base::result do_in(std::mbstate_t& /*state*/, @@ -366,12 +314,12 @@ namespace nowide { { const char* from_saved = from; - uint32_t ch = detail::utf::utf_traits::decode(from, from_end); + uint32_t ch = utf::utf_traits::decode(from, from_end); - if(ch == detail::utf::illegal) + if(ch == utf::illegal) { ch = NOWIDE_REPLACEMENT_CHARACTER; - } else if(ch == detail::utf::incomplete) + } else if(ch == utf::incomplete) { r = std::codecvt_base::partial; from = from_saved; @@ -399,17 +347,17 @@ namespace nowide { { std::uint32_t ch = 0; ch = *from; - if(!detail::utf::is_valid_codepoint(ch)) + if(!utf::is_valid_codepoint(ch)) { ch = NOWIDE_REPLACEMENT_CHARACTER; } - int len = detail::utf::utf_traits::width(ch); + int len = utf::utf_traits::width(ch); if(to_end - to < len) { r = std::codecvt_base::partial; break; } - to = detail::utf::utf_traits::encode(ch, to); + to = utf::utf_traits::encode(ch, to); from++; } from_next = from; diff --git a/extern/boost/nowide/include/nowide/windows.hpp b/extern/boost/nowide/include/nowide/windows.hpp index 940488a351..3c8a849b9e 100644 --- a/extern/boost/nowide/include/nowide/windows.hpp +++ b/extern/boost/nowide/include/nowide/windows.hpp @@ -1,30 +1,36 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2022 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_WINDOWS_HPP_INCLUDED #define NOWIDE_WINDOWS_HPP_INCLUDED #ifdef NOWIDE_USE_WINDOWS_H #include +// (Usually) included by windows.h +#include #else -// -// These are function prototypes... Allow to avoid including windows.h -// +// When NOWIDE_USE_WINDOWS_H is not defined we declare the function prototypes to avoid including windows.h + extern "C" { +// From windows.h + __declspec(dllimport) wchar_t* __stdcall GetEnvironmentStringsW(void); __declspec(dllimport) int __stdcall FreeEnvironmentStringsW(wchar_t*); __declspec(dllimport) wchar_t* __stdcall GetCommandLineW(void); -__declspec(dllimport) wchar_t** __stdcall CommandLineToArgvW(const wchar_t*, int*); __declspec(dllimport) unsigned long __stdcall GetLastError(); __declspec(dllimport) void* __stdcall LocalFree(void*); __declspec(dllimport) int __stdcall SetEnvironmentVariableW(const wchar_t*, const wchar_t*); __declspec(dllimport) unsigned long __stdcall GetEnvironmentVariableW(const wchar_t*, wchar_t*, unsigned long); + +// From shellapi.h + +__declspec(dllimport) wchar_t** __stdcall CommandLineToArgvW(const wchar_t*, int*); } #endif diff --git a/extern/boost/nowide/src/cstdio.cpp b/extern/boost/nowide/src/cstdio.cpp index 4004ca6e10..a33210a80e 100644 --- a/extern/boost/nowide/src/cstdio.cpp +++ b/extern/boost/nowide/src/cstdio.cpp @@ -1,20 +1,23 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// Copyright (c) 2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2020-2022 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #define NOWIDE_SOURCE #ifdef _MSC_VER +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS -#elif(defined(__MINGW32__) || defined(__CYGWIN__)) && defined(__STRICT_ANSI__) -// Need the _w* functions which are extensions on MinGW/Cygwin +#endif +#elif defined(__MINGW32__) && defined(__STRICT_ANSI__) +// Need the _w* functions which are extensions on MinGW but not on MinGW-w64 +#include <_mingw.h> +#ifndef __MINGW64_VERSION_MAJOR #undef __STRICT_ANSI__ #endif +#endif #include #include @@ -24,6 +27,7 @@ namespace nowide { FILE* wfopen(const wchar_t* filename, const wchar_t* mode) { #ifdef NOWIDE_WINDOWS + // coverity[var_deref_model] return ::_wfopen(filename, mode); #else const stackstring name(filename); @@ -41,6 +45,7 @@ namespace nowide { { const wstackstring wname(file_name); const wshort_stackstring wmode(mode); + // coverity[var_deref_model] return _wfreopen(wname.get(), wmode.get(), stream); } /// @@ -50,7 +55,8 @@ namespace nowide { { const wstackstring wname(file_name); const wshort_stackstring wmode(mode); - return _wfopen(wname.get(), wmode.get()); + // coverity[var_deref_model] + return detail::wfopen(wname.get(), wmode.get()); } /// /// \brief Same as rename but old_name and new_name are UTF-8 strings diff --git a/extern/boost/nowide/src/cstdlib.cpp b/extern/boost/nowide/src/cstdlib.cpp index 2335da1886..1394398caa 100644 --- a/extern/boost/nowide/src/cstdlib.cpp +++ b/extern/boost/nowide/src/cstdlib.cpp @@ -1,24 +1,30 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// Copyright (c) 2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2020-2022 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #define NOWIDE_SOURCE #ifdef _MSC_VER +#ifndef _CRT_SECURE_NO_WARNINGS #define _CRT_SECURE_NO_WARNINGS -#elif(defined(__MINGW32__) || defined(__CYGWIN__)) && defined(__STRICT_ANSI__) -// Need the _w* functions which are extensions on MinGW/Cygwin +#endif +#elif defined(__MINGW32__) && defined(__STRICT_ANSI__) +// Need the _w* functions which are extensions on MinGW but not on MinGW-w64 +#include <_mingw.h> +#ifndef __MINGW64_VERSION_MAJOR #undef __STRICT_ANSI__ #endif +#elif defined(__CYGWIN__) && !defined(_GNU_SOURCE) +// The setenv family of functions is an extension on Cygwin +#define _GNU_SOURCE 1 +#endif #include -#if !defined(NOWIDE_WINDOWS) +#ifndef NOWIDE_WINDOWS namespace nowide { int setenv(const char* key, const char* value, int overwrite) { @@ -40,10 +46,45 @@ namespace nowide { #include #include +namespace { +// thread_local was broken on MinGW for all 32bit compiler releases prior to 11.x, see +// https://sourceforge.net/p/mingw-w64/bugs/527/ +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83562 +// Using a non-trivial destructor causes program termination on thread exit. +#if defined(__MINGW32__) && !defined(__MINGW64__) && !defined(__clang__) && (__GNUC__ < 11) +class stackstring_for_thread +{ + union + { + nowide::stackstring s_; + }; + +public: + stackstring_for_thread() : s_(){}; + // Empty destructor so the union member (using a non-trivial destructor) does not get destroyed. + // This will leak memory if any is allocated by the stackstring for each terminated thread + // but as most values fit into the stack buffer this is rare and still better than a crash. + ~stackstring_for_thread(){}; + void convert(const wchar_t* begin, const wchar_t* end) + { + s_.convert(begin, end); + } + + char* get() + { + return s_.get(); + } +}; +#else +using stackstring_for_thread = nowide::stackstring; +#endif + +} // namespace + namespace nowide { char* getenv(const char* key) { - static stackstring value; + thread_local stackstring_for_thread value; const wshort_stackstring name(key); @@ -63,7 +104,7 @@ namespace nowide { return 0; ptr = &tmp[0]; } - value.convert(ptr); + value.convert(ptr, ptr + n); return value.get(); } diff --git a/extern/boost/nowide/src/iostream.cpp b/extern/boost/nowide/src/iostream.cpp index b17a124c68..83a065785e 100644 --- a/extern/boost/nowide/src/iostream.cpp +++ b/extern/boost/nowide/src/iostream.cpp @@ -1,34 +1,32 @@ +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2020-2021 Alexander Grund // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) -// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #define NOWIDE_SOURCE #include #ifndef NOWIDE_WINDOWS namespace nowide { + // LCOV_EXCL_START /// Avoid empty compilation unit warnings /// \internal NOWIDE_DECL void dummy_exported_function() {} + // LCOV_EXCL_STOP } // namespace nowide #else -#include +#include "console_buffer.hpp" #include -#include #include -#include #ifndef NOMINMAX #define NOMINMAX #endif - #include namespace nowide { @@ -37,220 +35,76 @@ namespace nowide { namespace { bool is_atty_handle(HANDLE h) { - if(h) - { - DWORD dummy; - return GetConsoleMode(h, &dummy) != FALSE; - } - return false; + DWORD dummy; + return h && GetConsoleMode(h, &dummy) != FALSE; } } // namespace - class console_output_buffer : public std::streambuf + class console_output_buffer final : public console_output_buffer_base { + HANDLE handle_; + public: - console_output_buffer(HANDLE h) : handle_(h) + explicit console_output_buffer(HANDLE handle) : handle_(handle) {} protected: - int sync() + bool + do_write(const wchar_t* buffer, std::size_t num_chars_to_write, std::size_t& num_chars_written) override { - return overflow(traits_type::eof()); - } - int overflow(int c) - { - if(!handle_) - return -1; - int n = static_cast(pptr() - pbase()); - int r = 0; - - if(n > 0 && (r = write(pbase(), n)) < 0) - return -1; - if(r < n) - { - std::memmove(pbase(), pbase() + r, n - r); - } - setp(buffer_, buffer_ + buffer_size); - pbump(n - r); - if(c != traits_type::eof()) - sputc(traits_type::to_char_type(c)); - return 0; - } - - private: - typedef detail::utf::utf_traits decoder; - typedef detail::utf::utf_traits encoder; - - int write(const char* p, int n) - { - namespace uf = detail::utf; - const char* b = p; - const char* e = p + n; DWORD size = 0; - if(n > buffer_size) - return -1; - wchar_t* out = wbuffer_; - uf::code_point c; - size_t decoded = 0; - while((c = decoder::decode(p, e)) != uf::incomplete) - { - if(c == uf::illegal) - c = NOWIDE_REPLACEMENT_CHARACTER; - assert(out - wbuffer_ + encoder::width(c) <= static_cast(wbuffer_size)); - out = encoder::encode(c, out); - decoded = p - b; - } - if(!WriteConsoleW(handle_, wbuffer_, static_cast(out - wbuffer_), &size, 0)) - return -1; - return static_cast(decoded); + const bool result = + WriteConsoleW(handle_, buffer, static_cast(num_chars_to_write), &size, 0) != 0; + num_chars_written = size; + return result; } - - static const int buffer_size = 1024; - static const int wbuffer_size = buffer_size * encoder::max_width; - char buffer_[buffer_size]; - wchar_t wbuffer_[wbuffer_size]; - HANDLE handle_; }; - class console_input_buffer : public std::streambuf + class console_input_buffer final : public console_input_buffer_base { + HANDLE handle_; + public: - console_input_buffer(HANDLE h) : handle_(h), wsize_(0), was_newline_(true) + explicit console_input_buffer(HANDLE handle) : handle_(handle) {} protected: - int sync() - { - if(FlushConsoleInputBuffer(handle_) == 0) - return -1; - wsize_ = 0; - was_newline_ = true; - pback_buffer_.clear(); - setg(0, 0, 0); - return 0; - } - int pbackfail(int c) - { - if(c == traits_type::eof()) - return traits_type::eof(); - - if(gptr() != eback()) - { - gbump(-1); - *gptr() = traits_type::to_char_type(c); - return 0; - } - - char* pnext; - if(pback_buffer_.empty()) - { - pback_buffer_.resize(4); - pnext = &pback_buffer_[0] + pback_buffer_.size() - 1u; - } else - { - size_t n = pback_buffer_.size(); - pback_buffer_.resize(n * 2); - std::memcpy(&pback_buffer_[n], &pback_buffer_[0], n); - pnext = &pback_buffer_[0] + n - 1; - } - - char* pFirst = &pback_buffer_[0]; - char* pLast = pFirst + pback_buffer_.size(); - setg(pFirst, pnext, pLast); - *gptr() = traits_type::to_char_type(c); - - return 0; - } - - int underflow() - { - if(!handle_) - return -1; - if(!pback_buffer_.empty()) - pback_buffer_.clear(); - - size_t n = read(); - setg(buffer_, buffer_, buffer_ + n); - if(n == 0) - return traits_type::eof(); - return std::char_traits::to_int_type(*gptr()); - } - - private: - typedef detail::utf::utf_traits decoder; - typedef detail::utf::utf_traits encoder; - - size_t read() + bool do_read(wchar_t* buffer, std::size_t num_chars_to_read, std::size_t& num_chars_read) override { - DWORD read_wchars = 0; - const size_t n = wbuffer_size - wsize_; - if(!ReadConsoleW(handle_, wbuffer_ + wsize_, static_cast(n), &read_wchars, 0)) - return 0; - wsize_ += read_wchars; - char* out = buffer_; - const wchar_t* cur_input_ptr = wbuffer_; - const wchar_t* const end_input_ptr = wbuffer_ + wsize_; - while(cur_input_ptr != end_input_ptr) - { - const wchar_t* const prev_input_ptr = cur_input_ptr; - detail::utf::code_point c = decoder::decode(cur_input_ptr, end_input_ptr); - // If incomplete restore to beginning of incomplete char to use on next buffer - if(c == detail::utf::incomplete) - { - cur_input_ptr = prev_input_ptr; - break; - } - if(c == detail::utf::illegal) - c = NOWIDE_REPLACEMENT_CHARACTER; - assert(out - buffer_ + encoder::width(c) <= static_cast(buffer_size)); - // Skip \r chars as std::cin does - if(c != '\r') - out = encoder::encode(c, out); - } - - wsize_ = end_input_ptr - cur_input_ptr; - if(wsize_ > 0) - std::memmove(wbuffer_, end_input_ptr - wsize_, sizeof(wchar_t) * wsize_); - - // A CTRL+Z at the start of the line should be treated as EOF - if(was_newline_ && out > buffer_ && buffer_[0] == '\x1a') - { - sync(); - return 0; - } - was_newline_ = out == buffer_ || out[-1] == '\n'; - - return out - buffer_; + DWORD size = 0; + const auto to_read_size = static_cast(num_chars_to_read); + const bool result = ReadConsoleW(handle_, buffer, to_read_size, &size, 0) != 0; + num_chars_read = size; + return result; } - - static const size_t wbuffer_size = 1024; - static const size_t buffer_size = wbuffer_size * encoder::max_width; - char buffer_[buffer_size]; - wchar_t wbuffer_[wbuffer_size]; - HANDLE handle_; - size_t wsize_; - std::vector pback_buffer_; - bool was_newline_; }; - winconsole_ostream::winconsole_ostream(int fd, winconsole_ostream* tieStream) : std::ostream(0) + winconsole_ostream::winconsole_ostream(const target_stream target, + const bool isBuffered, + winconsole_ostream* tieStream) : + std::ostream(nullptr) { - HANDLE h = 0; - switch(fd) - { - case 1: h = GetStdHandle(STD_OUTPUT_HANDLE); break; - case 2: h = GetStdHandle(STD_ERROR_HANDLE); break; - } + const HANDLE h = GetStdHandle(target == target_stream::output ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE); + if(is_atty_handle(h)) { d.reset(new console_output_buffer(h)); - std::ostream::rdbuf(d.get()); + rdbuf(d.get()); } else { - std::ostream::rdbuf(fd == 1 ? std::cout.rdbuf() : std::cerr.rdbuf()); + switch(target) + { + case target_stream::error: rdbuf(std::cerr.rdbuf()); break; + case target_stream::log: rdbuf(std::clog.rdbuf()); break; + case target_stream::output: rdbuf(std::cout.rdbuf()); break; + } } + assert(rdbuf()); + if(tieStream) tie(tieStream); + if(!isBuffered) + setf(ios_base::unitbuf); } winconsole_ostream::~winconsole_ostream() { @@ -258,7 +112,7 @@ namespace nowide { { flush(); } catch(...) - {} + {} // LCOV_EXCL_LINE } winconsole_istream::winconsole_istream(winconsole_ostream* tieStream) : std::istream(0) @@ -271,6 +125,7 @@ namespace nowide { } else { std::istream::rdbuf(std::cin.rdbuf()); + assert(rdbuf()); } if(tieStream) tie(tieStream); @@ -281,10 +136,21 @@ namespace nowide { } // namespace detail - detail::winconsole_ostream cout(1, NULL); - detail::winconsole_istream cin(&cout); - detail::winconsole_ostream cerr(2, &cout); - detail::winconsole_ostream clog(2, &cout); +// Make sure those are initialized as early as possible +#ifdef NOWIDE_MSVC +#pragma warning(disable : 4073) +#pragma init_seg(lib) +#endif +#ifdef NOWIDE_HAS_INIT_PRIORITY +#define NOWIDE_INIT_PRIORITY __attribute__((init_priority(101))) +#else +#define NOWIDE_INIT_PRIORITY +#endif + using target_stream = detail::winconsole_ostream::target_stream; + detail::winconsole_ostream cout NOWIDE_INIT_PRIORITY(target_stream::output, true, nullptr); + detail::winconsole_istream cin NOWIDE_INIT_PRIORITY(&cout); + detail::winconsole_ostream cerr NOWIDE_INIT_PRIORITY(target_stream::error, false, &cout); + detail::winconsole_ostream clog NOWIDE_INIT_PRIORITY(target_stream::log, true, nullptr); } // namespace nowide #endif diff --git a/extern/boost/nowide/test/CMakeLists.txt b/extern/boost/nowide/test/CMakeLists.txt index 651cb5d475..a6ca1ce1de 100644 --- a/extern/boost/nowide/test/CMakeLists.txt +++ b/extern/boost/nowide/test/CMakeLists.txt @@ -1,10 +1,22 @@ -# Copyright 2019 - 2020 Alexander Grund +# Copyright 2019 - 2024 Alexander Grund # Distributed under the Nowide Software License, Version 1.0. -# (See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt) +# https://www.boost.org/LICENSE_1_0.txt include(CheckCXXCompilerFlag) check_cxx_compiler_flag(-Wsuggest-override _NOWIDE_SUGGEST_OVERRIDE_SUPPORTED) +add_library(nowide_file_test_helpers STATIC EXCLUDE_FROM_ALL file_test_helpers.cpp) +target_link_libraries(nowide_file_test_helpers PRIVATE nowide::nowide) +target_compile_definitions(nowide_file_test_helpers PRIVATE) + +if(NOT TARGET tests) + add_custom_target(tests) +endif() + + +# In some environments this test (part) may fail, so allow to disable it +option(NOWIDE_DISABLE_CIN_TEST "Disable integration test using console input" OFF) + function(nowide_add_test name) cmake_parse_arguments(PARSE_ARGV 1 ARG "COMPILE_ONLY" "SRC" "LIBRARIES;DEFINITIONS;ARGS") if(NOT ARG_SRC) @@ -12,7 +24,8 @@ function(nowide_add_test name) endif() set(name ${PROJECT_NAME}-${name}) - add_executable(${name} ${ARG_SRC}) + add_executable(${name} EXCLUDE_FROM_ALL ${ARG_SRC}) + add_dependencies(tests ${name}) target_link_libraries(${name} PRIVATE nowide::nowide ${ARG_LIBRARIES}) nowide_add_warnings(${name} pedantic ${NOWIDE_WERROR}) if(_NOWIDE_SUGGEST_OVERRIDE_SUPPORTED) @@ -28,20 +41,38 @@ nowide_add_test(test_codecvt) nowide_add_test(test_convert) nowide_add_test(test_env) nowide_add_test(test_env_win SRC test_env.cpp DEFINITIONS NOWIDE_TEST_INCLUDE_WINDOWS) -nowide_add_test(test_fstream) -nowide_add_test(test_fstream_cxx11) -nowide_add_test(test_iostream) -if(MSVC AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.13) - set_target_properties(${PROJECT_NAME}-test_iostream PROPERTIES VS_DEBUGGER_COMMAND_ARGUMENTS -i) +nowide_add_test(test_filebuf LIBRARIES nowide_file_test_helpers) +nowide_add_test(test_ifstream LIBRARIES nowide_file_test_helpers) +nowide_add_test(test_ofstream LIBRARIES nowide_file_test_helpers) +nowide_add_test(test_fstream LIBRARIES nowide_file_test_helpers) +nowide_add_test(test_fstream_special LIBRARIES nowide_file_test_helpers) +nowide_add_test(test_iostream LIBRARIES nowide_file_test_helpers) +if(NOWIDE_DISABLE_CIN_TEST) + target_compile_definitions(${PROJECT_NAME}-test_iostream PRIVATE NOWIDE_DISABLE_CIN_TEST) endif() +nowide_add_test(test_iostream_interactive COMPILE_ONLY SRC test_iostream.cpp DEFINITIONS NOWIDE_TEST_INTERACTIVE LIBRARIES nowide_file_test_helpers) nowide_add_test(test_stackstring) +nowide_add_test(test_stat) nowide_add_test(test_stdio) nowide_add_test(test_system_n SRC test_system.cpp DEFINITIONS NOWIDE_TEST_USE_NARROW=1) if(WIN32) nowide_add_test(test_system_w SRC test_system.cpp DEFINITIONS NOWIDE_TEST_USE_NARROW=0) + nowide_add_test(test_system_use_windows_h SRC test_system.cpp DEFINITIONS NOWIDE_TEST_USE_NARROW=0 BOOST_USE_WINDOWS_H) + nowide_add_test(test_system_use_windows_h_lean SRC test_system.cpp DEFINITIONS NOWIDE_TEST_USE_NARROW=0 BOOST_USE_WINDOWS_H WIN32_LEAN_AND_MEAN) else() - nowide_add_test(test_internal_fstream SRC test_fstream.cpp DEFINITIONS NOWIDE_USE_FILEBUF_REPLACEMENT=1) - nowide_add_test(test_internal_fstream_cxx11 SRC test_fstream_cxx11.cpp DEFINITIONS NOWIDE_USE_FILEBUF_REPLACEMENT=1) + foreach(test test_filebuf test_ifstream test_ofstream test_fstream test_fstream_special) + nowide_add_test(${test}_internal SRC ${test}.cpp DEFINITIONS NOWIDE_USE_FILEBUF_REPLACEMENT=1 LIBRARIES nowide_file_test_helpers) + endforeach() endif() +nowide_add_test(test_traits) + +# Test that passthrough writes everything from stdin to stdout +# Needs to be done with CMake as the test driver to write any input to stdin and check output +add_test( + NAME ${PROJECT_NAME}-test_iostream_passthrough + COMMAND ${CMAKE_COMMAND} + -D TEST_BINARY=$ + -P ${CMAKE_CURRENT_SOURCE_DIR}/test_iostream_passthrough.cmake +) nowide_add_test(benchmark_fstream COMPILE_ONLY DEFINITIONS NOWIDE_USE_FILEBUF_REPLACEMENT=1) diff --git a/extern/boost/nowide/test/benchmark_fstream.cpp b/extern/boost/nowide/test/benchmark_fstream.cpp index 792f699426..34908bacd2 100644 --- a/extern/boost/nowide/test/benchmark_fstream.cpp +++ b/extern/boost/nowide/test/benchmark_fstream.cpp @@ -1,11 +1,9 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// Copyright (c) 2019 - 2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2019 - 2020 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #define NOWIDE_TEST_NO_MAIN @@ -19,6 +17,7 @@ #include #include #include +#include #include #include @@ -38,9 +37,12 @@ template class io_fstream { public: - explicit io_fstream(const char* file, bool read) + explicit io_fstream(const char* file, bool binary, bool read) { - f_.open(file, read ? std::fstream::in : std::fstream::out | std::fstream::trunc); + auto mode = read ? std::fstream::in : std::fstream::out | std::fstream::trunc; + if(binary) + mode |= std::fstream::binary; + f_.open(file, mode); TEST(f_); } // coverity[exn_spec_violation] @@ -70,12 +72,17 @@ class io_fstream FStream f_; }; +#include + class io_stdio { public: - io_stdio(const char* file, bool read) + io_stdio(const char* file, bool binary, bool read) { - f_ = nw::fopen(file, read ? "r" : "w+"); + const char* mode = read ? "r" : "w+"; + if(binary) + mode = read ? "rb" : "wb+"; + f_ = nw::fopen(file, mode); TEST(f_); } ~io_stdio() @@ -85,11 +92,11 @@ class io_stdio } void write(const char* buf, int size) { - TEST(std::fwrite(buf, 1, size, f_) == static_cast(size)); + TEST_EQ(std::fwrite(buf, 1, size, f_), static_cast(size)); } void read(char* buf, int size) { - TEST(std::fread(buf, 1, size, f_) == static_cast(size)); + TEST_EQ(std::fread(buf, 1, size, f_), static_cast(size)); } void rewind() { @@ -118,22 +125,22 @@ extern "C" void _ReadWriteBarrier(void); #define NOWIDE_READ_WRITE_BARRIER() (void) #endif +using blocksize_to_performance = std::map; + struct perf_data { // Block-size to read/write performance in MB/s - std::map read, write; + blocksize_to_performance read, write; }; -char rand_char() -{ - // coverity[dont_call] - return static_cast(std::rand()); -} - -std::vector get_rand_data(int size) +std::vector get_rand_data(int size, bool binary) { + std::mt19937 rng{std::random_device{}()}; + auto distr = (binary) ? std::uniform_int_distribution(std::numeric_limits::min(), + std::numeric_limits::max()) : + std::uniform_int_distribution(' ', 'z'); std::vector data(size); - std::generate(data.begin(), data.end(), rand_char); + std::generate(data.begin(), data.end(), [&]() { return static_cast(distr(rng)); }); return data; } @@ -141,19 +148,19 @@ static const int MIN_BLOCK_SIZE = 32; static const int MAX_BLOCK_SIZE = 8192; template -perf_data test_io(const char* file) +perf_data test_io(const char* file, bool binary) { namespace chrono = std::chrono; - typedef chrono::high_resolution_clock clock; - typedef chrono::duration milliseconds; + using clock = chrono::high_resolution_clock; + using milliseconds = chrono::duration; perf_data results; // Use vector to force write to memory and avoid possible reordering std::vector start_and_end(2); const int data_size = 64 * 1024 * 1024; for(int block_size = MIN_BLOCK_SIZE / 2; block_size <= MAX_BLOCK_SIZE; block_size *= 2) { - std::vector buf = get_rand_data(block_size); - FStream tmp(file, false); + std::vector buf = get_rand_data(block_size, binary); + FStream tmp(file, binary, false); tmp.rewind(); start_and_end[0] = clock::now(); NOWIDE_READ_WRITE_BARRIER(); @@ -176,8 +183,8 @@ perf_data test_io(const char* file) } for(int block_size = MIN_BLOCK_SIZE; block_size <= MAX_BLOCK_SIZE; block_size *= 2) { - std::vector buf = get_rand_data(block_size); - FStream tmp(file, true); + std::vector buf(block_size); + FStream tmp(file, binary, true); tmp.rewind(); start_and_end[0] = clock::now(); NOWIDE_READ_WRITE_BARRIER(); @@ -193,19 +200,19 @@ perf_data test_io(const char* file) std::cout << " read block size " << std::setw(8) << block_size << " " << std::fixed << std::setprecision(3) << speed << " MB/s" << std::endl; } - TEST(std::remove(file) == 0); + TEST_EQ(std::remove(file), 0); return results; } template -perf_data test_io_driver(const char* file, const char* type) +perf_data test_io_driver(const char* file, const char* type, bool binary) { std::cout << "Testing I/O performance for " << type << std::endl; const int repeats = 5; std::vector results(repeats); for(int i = 0; i < repeats; i++) - results[i] = test_io(file); + results[i] = test_io(file, binary); for(int block_size = MIN_BLOCK_SIZE; block_size <= MAX_BLOCK_SIZE; block_size *= 2) { double read_speed = 0, write_speed = 0; @@ -220,9 +227,9 @@ perf_data test_io_driver(const char* file, const char* type) return results[0]; } -void print_perf_data(const std::map& stdio_data, - const std::map& std_data, - const std::map& nowide_data) +void print_perf_data(const blocksize_to_performance& stdio_data, + const blocksize_to_performance& std_data, + const blocksize_to_performance& nowide_data) { std::cout << "block size" << " stdio " @@ -240,13 +247,20 @@ void print_perf_data(const std::map& stdio_data, void test_perf(const char* file) { - perf_data stdio_data = test_io_driver(file, "stdio"); - perf_data std_data = test_io_driver >(file, "std::fstream"); - perf_data nowide_data = test_io_driver >(file, "nowide::fstream"); - std::cout << "================== Read performance ==================" << std::endl; + perf_data stdio_data = test_io_driver(file, "stdio", true); + perf_data std_data = test_io_driver>(file, "std::fstream", true); + perf_data nowide_data = test_io_driver>(file, "nowide::fstream", true); + perf_data stdio_data_txt = test_io_driver(file, "stdio", false); + perf_data std_data_txt = test_io_driver>(file, "std::fstream", false); + perf_data nowide_data_txt = test_io_driver>(file, "nowide::fstream", false); + std::cout << "================== Read performance (binary) ==================" << std::endl; print_perf_data(stdio_data.read, std_data.read, nowide_data.read); - std::cout << "================== Write performance =================" << std::endl; + std::cout << "================== Write performance (binary) =================" << std::endl; print_perf_data(stdio_data.write, std_data.write, nowide_data.write); + std::cout << "================== Read performance (text) ====================" << std::endl; + print_perf_data(stdio_data_txt.read, std_data_txt.read, nowide_data_txt.read); + std::cout << "================== Write performance (text) ===================" << std::endl; + print_perf_data(stdio_data_txt.write, std_data_txt.write, nowide_data_txt.write); } int main(int argc, char** argv) @@ -263,7 +277,7 @@ int main(int argc, char** argv) try { test_perf(filename.c_str()); - } catch(const std::runtime_error& err) + } catch(const std::exception& err) { std::cerr << "Benchmarking failed: " << err.what() << std::endl; return 1; diff --git a/extern/boost/nowide/test/check_movable_fstreams.cpp b/extern/boost/nowide/test/check_movable_fstreams.cpp index 4970d9d795..ed6be3859b 100644 --- a/extern/boost/nowide/test/check_movable_fstreams.cpp +++ b/extern/boost/nowide/test/check_movable_fstreams.cpp @@ -1,9 +1,8 @@ // -// Copyright (c) 2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at http://www.boost.org/LICENSE.txt) +// Copyright (c) 2020 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #include #include diff --git a/extern/boost/nowide/test/cmake_test/CMakeLists.txt b/extern/boost/nowide/test/cmake_test/CMakeLists.txt new file mode 100644 index 0000000000..69db423b08 --- /dev/null +++ b/extern/boost/nowide/test/cmake_test/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2021 Alexander Grund +# +# Distributed under the Nowide Software License, Version 1.0. +# https://www.boost.org/LICENSE_1_0.txt + +cmake_minimum_required(VERSION 3.5...3.16) + +project(cmake_subdir_test LANGUAGES CXX) + + find_package(nowide 11 REQUIRED) + +add_executable(main main.cpp) +target_link_libraries(main nowide::nowide) + +enable_testing() +add_test(NAME main COMMAND main) diff --git a/extern/boost/nowide/test/cmake_test/main.cpp b/extern/boost/nowide/test/cmake_test/main.cpp new file mode 100644 index 0000000000..59b8fbf8e8 --- /dev/null +++ b/extern/boost/nowide/test/cmake_test/main.cpp @@ -0,0 +1,18 @@ +// +// Copyright (c) 2019-2021 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include +#include + +int main(int argc, char** argv, char** env) +{ + nowide::args _(argc, argv, env); + if(argc < 1) + return 1; + if(nowide::narrow(nowide::widen(argv[0])) != argv[0]) + return 1; + return 0; +} diff --git a/extern/boost/nowide/test/exampleProject/example_main.cpp b/extern/boost/nowide/test/exampleProject/example_main.cpp index 4049c813d3..59b8fbf8e8 100644 --- a/extern/boost/nowide/test/exampleProject/example_main.cpp +++ b/extern/boost/nowide/test/exampleProject/example_main.cpp @@ -1,10 +1,8 @@ // -// Copyright (c) 2019 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2019-2021 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #include #include diff --git a/extern/boost/nowide/test/file_test_helpers.cpp b/extern/boost/nowide/test/file_test_helpers.cpp new file mode 100644 index 0000000000..f3216ce922 --- /dev/null +++ b/extern/boost/nowide/test/file_test_helpers.cpp @@ -0,0 +1,104 @@ +// Copyright (c) 2019-2021 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#define NOWIDE_TEST_NO_MAIN +#include "file_test_helpers.hpp" + +#include +#include "test.hpp" +#include +#include +#include +#include +#include + +namespace nowide { + namespace test { + + void create_file(const std::string& filepath, const std::string& contents, data_type type /*= data_type::text*/) + { + auto* f = fopen(filepath.c_str(), (type == data_type::binary) ? "wb" : "w"); + TEST(f); + TEST(std::fwrite(contents.data(), 1, contents.size(), f) == contents.size()); + std::fclose(f); + } + + bool file_exists(const std::string& filepath) + { + FILE* f = fopen(filepath.c_str(), "r"); + if(f) + { + std::fclose(f); + return true; + } else + return false; + } + + std::string read_file(const std::string& filepath, data_type type /*= data_type::text*/) + { + FILE* f = fopen(filepath.c_str(), (type == data_type::binary) ? "rb" : "r"); + TEST(f); + std::fseek(f, 0, SEEK_END); + const auto file_size_tmp = ftell(f); + TEST(file_size_tmp >= 0); + const auto file_size = static_cast(file_size_tmp); + std::fseek(f, 0, SEEK_SET); + std::string content(file_size, '\0'); + const auto read_bytes = std::fread(&content[0], 1, file_size, f); + if(read_bytes != file_size) + { + TEST(std::feof(f)); + content.resize(read_bytes); + } + std::fclose(f); + return content; + } + + void ensure_not_exists(const char* filepath) + { + if(file_exists(filepath)) + { + remove(filepath); + TEST(!file_exists(filepath)); + } + } + + static std::string get_ascii_chars() + { + const auto first_char = char(0x20); + const auto last_char = char(0x7E); + std::string result(last_char - first_char + 2, '\0'); + result[0] = '\n'; // Include newline + std::iota(result.begin() + 1, result.end(), first_char); + return result; + } + + static std::minstd_rand make_rand_engine() + { + const auto seed = std::random_device{}(); + std::cout << "RNG seed: " << seed << std::endl; + return std::minstd_rand(seed); + } + + std::string create_random_data(size_t num_chars, data_type type) + { + static std::minstd_rand rng = make_rand_engine(); + std::string result(num_chars, '\0'); + if(type == data_type::binary) + { + std::uniform_int_distribution distr(std::numeric_limits::min(), + std::numeric_limits::max()); + std::generate(result.begin(), result.end(), [&]() { return static_cast(distr(rng)); }); + } else + { + static const std::string text_data = get_ascii_chars(); + std::uniform_int_distribution distr(0, text_data.size() - 1u); + std::generate(result.begin(), result.end(), [&]() { return text_data[distr(rng)]; }); + } + return result; + } + + } // namespace test +} // namespace nowide diff --git a/extern/boost/nowide/test/file_test_helpers.hpp b/extern/boost/nowide/test/file_test_helpers.hpp new file mode 100644 index 0000000000..daa7da3a43 --- /dev/null +++ b/extern/boost/nowide/test/file_test_helpers.hpp @@ -0,0 +1,49 @@ +// Copyright (c) 2019-2021 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef NOWIDE_FILE_TEST_HELPERS_HPP_INCLUDED +#define NOWIDE_FILE_TEST_HELPERS_HPP_INCLUDED + +#include + +namespace nowide { + namespace test { + enum class data_type + { + text, + binary + }; + + void create_empty_file(const std::string& filepath); + + void create_file(const std::string& filepath, const std::string& contents, data_type type = data_type::text); + + bool file_exists(const std::string& filepath); + + std::string read_file(const std::string& filepath, data_type type = data_type::text); + + void ensure_not_exists(const char* filepath); + inline void ensure_not_exists(const std::string& filepath) + { + ensure_not_exists(filepath.c_str()); + } + + std::string create_random_data(size_t num_chars, data_type type); + + struct remove_file_at_exit + { + const std::string filepath; + explicit remove_file_at_exit(std::string filepath) : filepath(std::move(filepath)) + {} + ~remove_file_at_exit() noexcept(false) + { + ensure_not_exists(filepath); + } + }; + + } // namespace test +} // namespace nowide + +#endif // NOWIDE_FILE_TEST_HELPERS_HPP_INCLUDED \ No newline at end of file diff --git a/extern/boost/nowide/test/test.hpp b/extern/boost/nowide/test/test.hpp index 618c70fde9..34f71cbde7 100644 --- a/extern/boost/nowide/test/test.hpp +++ b/extern/boost/nowide/test/test.hpp @@ -1,64 +1,113 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// Copyright (c) 2019-2020 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) +// Copyright (c) 2019-2020 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + #ifndef NOWIDE_LIB_TEST_H_INCLUDED #define NOWIDE_LIB_TEST_H_INCLUDED #include +#include #include +#include #include #include +#include #if defined(_MSC_VER) && defined(_CPPLIB_VER) && defined(_DEBUG) #include #endif namespace nowide { - struct test_monitor - { - test_monitor() + namespace test { + struct test_error : std::runtime_error + { + using std::runtime_error::runtime_error; + }; + struct test_monitor { + test_monitor() + { #if defined(_MSC_VER) && (_MSC_VER > 1310) - // disable message boxes on assert(), abort() - ::_set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); + // disable message boxes on assert(), abort() + ::_set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); #endif #if defined(_MSC_VER) && defined(_CPPLIB_VER) && defined(_DEBUG) - // disable message boxes on iterator debugging violations - _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE); - _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); + // disable message boxes on iterator debugging violations + _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE); + _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); #endif + } + std::string context; + }; + inline test_monitor& test_mon() + { + static test_monitor instance; + return instance; + } + struct context + { + std::string oldCtx; + context(std::string ctx) : oldCtx(std::move(test_mon().context)) + { + test_mon().context = std::move(ctx); + } + ~context() + { + test_mon().context = std::move(oldCtx); + } + }; + /// Function called when a test failed to be able set a breakpoint for debugging + inline void test_failed(const char* expr, const char* file, const int line, const char* function) + { + std::ostringstream ss; + ss << expr << " at " << file << ':' << line << " in " << function; + std::string& ctx = test_mon().context; + if(!ctx.empty()) + ss << " context: " << ctx; + throw test_error(ss.str()); } - }; -} // namespace nowide - -inline nowide::test_monitor& test_mon() -{ - static nowide::test_monitor instance; - return instance; -} -/// Function called when a test failed to be able set a breakpoint for debugging -inline void test_failed(const char* expr, const char* file, const int line, const char* function) -{ - std::ostringstream ss; - ss << "Error " << expr << " at " << file << ':' << line << " in " << function; - throw std::runtime_error(ss.str()); -} + template + const T& print_value(const T& value) + { + return value; + } + inline std::string print_value(const std::wstring& value) + { + std::ostringstream ss; + const auto defWidth = ss.width(); + ss << std::hex << std::uppercase << std::setfill('0'); + for(const wchar_t c : value) + { +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wtautological-constant-out-of-range-compare" +#endif + if(c >= (std::numeric_limits::min)() && c <= (std::numeric_limits::max)()) + ss << static_cast(c); + else + ss << "\\" << std::setw(sizeof(wchar_t) * 2) << static_cast(c) << std::setw(defWidth); +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + } + return ss.str(); + } -template -inline void test_equal_impl(const T& lhs, const U& rhs, const char* file, const int line, const char* function) -{ - if(lhs == rhs) - return; - std::ostringstream ss; - ss << "[" << lhs << "!=" << rhs << "]"; - test_failed(ss.str().c_str(), file, line, function); -} + template + void test_equal_impl(const T& lhs, const U& rhs, const char* file, const int line, const char* function) + { + if(lhs == rhs) + return; + std::ostringstream ss; + ss << "[" << print_value(lhs) << "!=" << print_value(rhs) << "]"; + test_failed(ss.str().c_str(), file, line, function); + } + } // namespace test +} // namespace nowide #ifdef _MSC_VER #define DISABLE_CONST_EXPR_DETECTED __pragma(warning(push)) __pragma(warning(disable : 4127)) @@ -71,6 +120,7 @@ inline void test_equal_impl(const T& lhs, const U& rhs, const char* file, const #define TEST(x) \ do \ { \ + using namespace nowide::test; \ test_mon(); \ if(x) \ break; \ @@ -80,12 +130,32 @@ inline void test_equal_impl(const T& lhs, const U& rhs, const char* file, const #define TEST_EQ(lhs, rhs) \ do \ { \ + using namespace nowide::test; \ test_mon(); \ test_equal_impl((lhs), (rhs), __FILE__, __LINE__, __FUNCTION__); \ break; \ DISABLE_CONST_EXPR_DETECTED \ } while(0) DISABLE_CONST_EXPR_DETECTED_POP +#define TEST_THROW(expr, error) \ + do \ + { \ + test_mon(); \ + try \ + { \ + expr; \ + test_failed(#error " not thrown", __FILE__, __LINE__, __FUNCTION__); \ + } catch(const error&) \ + { /* OK */ \ + } \ + break; \ + DISABLE_CONST_EXPR_DETECTED \ + } while(0) DISABLE_CONST_EXPR_DETECTED_POP + +#define TEST_CONTEXT(expr) \ + nowide::test::context _##__COUNTER__( \ + static_cast(std::stringstream{} << expr).str()) + #ifndef NOWIDE_TEST_NO_MAIN // Tests should implement this void test_main(int argc, char** argv, char** env); @@ -94,10 +164,17 @@ int main(int argc, char** argv, char** env) { try { +#ifdef _MSC_VER + std::cout << "MSVC version " << _MSC_VER << std::endl; +#endif test_main(argc, argv, env); + } catch(const nowide::test::test_error& e) + { + std::cerr << "Failed test assertion: " << e.what() << std::endl; + return 1; } catch(const std::exception& e) { - std::cerr << "Failed " << e.what() << std::endl; + std::cerr << "Failed with unexpected exception: " << e.what() << std::endl; return 1; } return 0; diff --git a/extern/boost/nowide/test/test_codecvt.cpp b/extern/boost/nowide/test/test_codecvt.cpp index a2d8cb572f..ffb748b6aa 100644 --- a/extern/boost/nowide/test/test_codecvt.cpp +++ b/extern/boost/nowide/test/test_codecvt.cpp @@ -1,29 +1,124 @@ // -// Copyright (c) 2015 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2015 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #include #include +#include "test.hpp" +#include "test_sets.hpp" #include #include #include #include #include -#include "test.hpp" -#include "test_sets.hpp" +// MSVC has problems with an undefined symbol std::codecvt::id in some versions if the utf char types are used. See +// https://social.msdn.microsoft.com/Forums/vstudio/en-US/8f40dcd8-c67f-4eba-9134-a19b9178e481/vs-2015-rc-linker-stdcodecvt-error?forum=vcgeneral +// Workaround: use int16_t instead of char16_t +#if defined(_MSC_VER) && _MSC_VER >= 1900 && _MSC_VER <= 1916 +#define NOWIDE_REQUIRE_UTF_CHAR_WORKAROUND 1 +#else +#define NOWIDE_REQUIRE_UTF_CHAR_WORKAROUND 0 +#endif static const char* utf8_name = "\xf0\x9d\x92\x9e-\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82-\xE3\x82\x84\xE3\x81\x82.txt"; static const std::wstring wide_name_str = nowide::widen(utf8_name); static const wchar_t* wide_name = wide_name_str.c_str(); -typedef std::codecvt cvt_type; +using cvt_type = std::codecvt; + +#if NOWIDE_REQUIRE_UTF_CHAR_WORKAROUND +using utf16_char_t = int16_t; +using utf32_char_t = int32_t; +#else +using utf16_char_t = char16_t; +using utf32_char_t = char32_t; +#endif + +NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN +using cvt_type16 = std::codecvt; +using cvt_type32 = std::codecvt; +using utf8_utf16_codecvt = nowide::utf8_codecvt; +using utf8_utf32_codecvt = nowide::utf8_codecvt; +NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END + +void test_codecvt_basic() +{ + // UTF-16 + { + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN + std::locale l(std::locale::classic(), new utf8_utf16_codecvt()); + const cvt_type16& cvt = std::use_facet(l); + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END + TEST_EQ(cvt.encoding(), 0); // Characters have a variable width + TEST_EQ(cvt.max_length(), 4); // At most 4 UTF-8 code units are one internal char (one or two UTF-16 code units) + TEST(!cvt.always_noconv()); // Always convert + } + // UTF-32 + { + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN + std::locale l(std::locale::classic(), new utf8_utf32_codecvt()); + const cvt_type32& cvt = std::use_facet(l); + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END + TEST_EQ(cvt.encoding(), 0); // Characters have a variable width + TEST_EQ(cvt.max_length(), 4); // At most 4 UTF-8 code units are one internal char (one UTF-32 code unit) + TEST(!cvt.always_noconv()); // Always convert + } +} + +void test_codecvt_unshift() +{ + char buf[256]; + // UTF-16 + { + const auto name16 = + nowide::utf::convert_string(utf8_name, utf8_name + std::strlen(utf8_name)); + + utf8_utf16_codecvt cvt16; + // Unshift on initial state does nothing + std::mbstate_t mb{}; + char* to_next; + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN + const cvt_type16& cvt = cvt16; + TEST_EQ(cvt.unshift(mb, buf, std::end(buf), to_next), cvt_type16::ok); + TEST(to_next == buf); + const utf16_char_t* from_next; + // Convert into a to small buffer + TEST_EQ(cvt.out(mb, &name16.front(), &name16.back(), from_next, buf, buf + 1, to_next), cvt_type16::partial); + TEST(from_next == &name16[1]); + TEST(to_next == buf); + // Unshift on non-default state is not possible + TEST_EQ(cvt.unshift(mb, buf, std::end(buf), to_next), cvt_type16::error); + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END + } + // UTF-32 + { + const auto name32 = + nowide::utf::convert_string(utf8_name, utf8_name + std::strlen(utf8_name)); + + utf8_utf32_codecvt cvt32; + // Unshift on initial state does nothing + std::mbstate_t mb{}; + char* to_next; + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_BEGIN + const cvt_type32& cvt = cvt32; + TEST_EQ(cvt.unshift(mb, buf, std::end(buf), to_next), cvt_type32::noconv); + TEST(to_next == buf); + const utf32_char_t* from_next; + // Convert into a too small buffer + TEST_EQ(cvt.out(mb, &name32.front(), &name32.back(), from_next, buf, buf + 1, to_next), cvt_type32::partial); + TEST(from_next == &name32.front()); // Noting consumed + TEST(to_next == buf); + TEST(std::mbsinit(&mb) != 0); // State unchanged --> Unshift does nothing + TEST_EQ(cvt.unshift(mb, buf, std::end(buf), to_next), cvt_type32::noconv); + TEST(to_next == buf); + NOWIDE_SUPPRESS_UTF_CODECVT_DEPRECATION_END + } +} void test_codecvt_in_n_m(const cvt_type& cvt, size_t n, size_t m) { @@ -31,17 +126,17 @@ void test_codecvt_in_n_m(const cvt_type& cvt, size_t n, size_t m) size_t wlen = std::wcslen(wide_name); size_t u8len = std::strlen(utf8_name); const char* from = utf8_name; - const char* end = from; + const char* from_end = from; const char* real_end = utf8_name + u8len; const char* from_next = from; - std::mbstate_t mb = std::mbstate_t(); + std::mbstate_t mb{}; while(from_next < real_end) { - if(from == end) + if(from == from_end) { - end = from + n; - if(end > real_end) - end = real_end; + from_end = from + n; + if(from_end > real_end) + from_end = real_end; } wchar_t buf[128]; @@ -50,27 +145,19 @@ void test_codecvt_in_n_m(const cvt_type& cvt, size_t n, size_t m) wchar_t* to_next = to; std::mbstate_t mb2 = mb; - std::codecvt_base::result r = cvt.in(mb, from, end, from_next, to, to_end, to_next); + std::codecvt_base::result r = cvt.in(mb, from, from_end, from_next, to, to_end, to_next); - int count = cvt.length(mb2, from, end, to_end - to); -#ifndef NOWIDE_DO_LENGTH_MBSTATE_CONST - TEST(std::memcmp(&mb, &mb2, sizeof(mb)) == 0); - if(count != from_next - from) - { - std::cout << count << " " << from_next - from << std::endl; - } - TEST(count == from_next - from); -#else - TEST(count == to_next - to); -#endif + int count = cvt.length(mb2, from, from_end, to_end - to); + TEST_EQ(std::memcmp(&mb, &mb2, sizeof(mb)), 0); + TEST_EQ(count, from_next - from); if(r == cvt_type::partial) { - end += n; - if(end > real_end) - end = real_end; + from_end += n; + if(from_end > real_end) + from_end = real_end; } else - TEST(r == cvt_type::ok); + TEST_EQ(r, cvt_type::ok); while(to != to_next) { TEST(*wptr == *to); @@ -90,7 +177,7 @@ void test_codecvt_out_n_m(const cvt_type& cvt, size_t n, size_t m) size_t wlen = std::wcslen(wide_name); size_t u8len = std::strlen(utf8_name); - std::mbstate_t mb = std::mbstate_t(); + std::mbstate_t mb{}; const wchar_t* from_next = wide_name; const wchar_t* real_from_end = wide_name + wlen; @@ -121,17 +208,16 @@ void test_codecvt_out_n_m(const cvt_type& cvt, size_t n, size_t m) { TEST(to_end - to_next < cvt.max_length()); to_end += n; - if(to_end > real_to_end) - to_end = real_to_end; + TEST(to_end <= real_to_end); // Should always be big enough } } else { - TEST(r == cvt_type::ok); + TEST_EQ(r, cvt_type::ok); } while(to != to_next) { - TEST(*nptr == *to); + TEST_EQ(*nptr, *to); nptr++; to++; } @@ -139,7 +225,8 @@ void test_codecvt_out_n_m(const cvt_type& cvt, size_t n, size_t m) } TEST(nptr == utf8_name + u8len); TEST(from_next == real_from_end); - TEST(cvt.unshift(mb, to, to + n, to_next) == cvt_type::ok); + const auto expected = (sizeof(wchar_t) == 2) ? cvt_type::ok : cvt_type::noconv; // UTF-32 is not state-dependent + TEST_EQ(cvt.unshift(mb, to, to + n, to_next), expected); TEST(to_next == to); } @@ -160,10 +247,10 @@ void test_codecvt_conv() { test_codecvt_in_n_m(cvt, i, j); test_codecvt_out_n_m(cvt, i, j); - } catch(...) + } catch(...) // LCOV_EXCL_LINE { - std::cerr << "Wlen=" << j << " Nlen=" << i << std::endl; - throw; + std::cerr << "Wlen=" << j << " Nlen=" << i << std::endl; // LCOV_EXCL_LINE + throw; // LCOV_EXCL_LINE } } } @@ -183,12 +270,12 @@ void test_codecvt_err() wchar_t* const to = buf; wchar_t* const to_end = buf + 4; const char* err_utf = "1\xFF\xFF\xd7\xa9"; - std::mbstate_t mb = std::mbstate_t(); + std::mbstate_t mb{}; const char* from = err_utf; const char* from_end = from + std::strlen(from); const char* from_next = from; wchar_t* to_next = to; - TEST(cvt.in(mb, from, from_end, from_next, to, to_end, to_next) == cvt_type::ok); + TEST_EQ(cvt.in(mb, from, from_end, from_next, to, to_end, to_next), cvt_type::ok); TEST(from_next == from + 5); TEST(to_next == to + 4); TEST(std::wstring(to, to_end) == nowide::widen(err_utf)); @@ -198,15 +285,23 @@ void test_codecvt_err() wchar_t* const to = buf; wchar_t* const to_end = buf + 4; const char* err_utf = "1\xd7"; // 1 valid, 1 incomplete UTF-8 char - std::mbstate_t mb = std::mbstate_t(); + std::mbstate_t mb{}; const char* from = err_utf; const char* from_end = from + std::strlen(from); const char* from_next = from; wchar_t* to_next = to; - TEST(cvt.in(mb, from, from_end, from_next, to, to_end, to_next) == cvt_type::partial); + TEST_EQ(cvt.in(mb, from, from_end, from_next, to, to_end, to_next), cvt_type::partial); TEST(from_next == from + 1); TEST(to_next == to + 1); + // False positive in GCC 13 in MinGW +#if defined(__GNUC__) && __GNUC__ >= 13 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfree-nonheap-object" +#endif TEST(std::wstring(to, to_next) == std::wstring(L"1")); +#if defined(__GNUC__) && __GNUC__ >= 13 +#pragma GCC diagnostic pop +#endif } { char buf[4] = {}; @@ -214,7 +309,7 @@ void test_codecvt_err() char* const to_end = buf + 4; char* to_next = to; const wchar_t* err_utf = L"\xD800"; // Trailing UTF-16 surrogate - std::mbstate_t mb = std::mbstate_t(); + std::mbstate_t mb{}; const wchar_t* from = err_utf; const wchar_t* from_end = from + 1; const wchar_t* from_next = from; @@ -224,17 +319,17 @@ void test_codecvt_err() #endif if(sizeof(wchar_t) == 2) { - TEST(res == cvt_type::partial); + TEST_EQ(res, cvt_type::partial); TEST(from_next == from_end); TEST(to_next == to); - TEST(buf[0] == 0); + TEST_EQ(buf[0], 0); } else { - TEST(res == cvt_type::ok); + TEST_EQ(res, cvt_type::ok); TEST(from_next == from_end); TEST(to_next == to + 3); // surrogate is invalid - TEST(std::string(to, to_next) == nowide::narrow(wreplacement_str)); + TEST_EQ(std::string(to, to_next), nowide::narrow(wreplacement_str)); } } } @@ -245,17 +340,17 @@ void test_codecvt_err() char* to = buf; char* to_end = buf + 32; char* to_next = to; - wchar_t err_buf[3] = {'1', 0xDC9E, 0}; // second surrogate not works both for UTF-16 and 32 + wchar_t err_buf[] = {'1', 0xDC9E, 0}; // second value is invalid for UTF-16 and 32 const wchar_t* err_utf = err_buf; { - std::mbstate_t mb = std::mbstate_t(); + std::mbstate_t mb{}; const wchar_t* from = err_utf; const wchar_t* from_end = from + std::wcslen(from); const wchar_t* from_next = from; - TEST(cvt.out(mb, from, from_end, from_next, to, to_end, to_next) == cvt_type::ok); + TEST_EQ(cvt.out(mb, from, from_end, from_next, to, to_end, to_next), cvt_type::ok); TEST(from_next == from + 2); TEST(to_next == to + 4); - TEST(std::string(to, to_next) == "1" + nowide::narrow(wreplacement_str)); + TEST_EQ(std::string(to, to_next), nowide::narrow(err_buf)); } } } @@ -266,7 +361,7 @@ std::wstring codecvt_to_wide(const std::string& s) const cvt_type& cvt = std::use_facet(l); - std::mbstate_t mb = std::mbstate_t(); + std::mbstate_t mb{}; const char* const from = s.c_str(); const char* const from_end = from + s.size(); const char* from_next = from; @@ -276,13 +371,15 @@ std::wstring codecvt_to_wide(const std::string& s) wchar_t* const to_end = to + buf.size(); wchar_t* to_next = to; + const auto expected_consumed = cvt.length(mb, from, from_end, buf.size()); cvt_type::result res = cvt.in(mb, from, from_end, from_next, to, to_end, to_next); + TEST_EQ(expected_consumed, from_next - from); if(res == cvt_type::partial) { TEST(to_next < to_end); *(to_next++) = NOWIDE_REPLACEMENT_CHARACTER; } else - TEST(res == cvt_type::ok); + TEST_EQ(res, cvt_type::ok); return std::wstring(to, to_next); } @@ -293,7 +390,7 @@ std::string codecvt_to_narrow(const std::wstring& s) const cvt_type& cvt = std::use_facet(l); - std::mbstate_t mb = std::mbstate_t(); + std::mbstate_t mb{}; const wchar_t* const from = s.c_str(); const wchar_t* const from_end = from + s.size(); const wchar_t* from_next = from; @@ -309,7 +406,7 @@ std::string codecvt_to_narrow(const std::wstring& s) TEST(to_next < to_end); return std::string(to, to_next) + nowide::narrow(wreplacement_str); } else - TEST(res == cvt_type::ok); + TEST_EQ(res, cvt_type::ok); return std::string(to, to_next); } @@ -320,8 +417,11 @@ void test_codecvt_subst() run_all(codecvt_to_wide, codecvt_to_narrow); } +// coverity[root_function] void test_main(int, char**, char**) { + test_codecvt_basic(); + test_codecvt_unshift(); test_codecvt_conv(); test_codecvt_err(); test_codecvt_subst(); diff --git a/extern/boost/nowide/test/test_convert.cpp b/extern/boost/nowide/test/test_convert.cpp index 06a55ff6ae..48b90417bf 100644 --- a/extern/boost/nowide/test/test_convert.cpp +++ b/extern/boost/nowide/test/test_convert.cpp @@ -1,16 +1,20 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #include -#include - #include "test.hpp" #include "test_sets.hpp" +#include +#include +#include + +#ifdef __cpp_lib_string_view +#include +#define NOWIDE_TEST_STD_STRINGVIEW +#endif #if defined(NOWIDE_MSVC) && NOWIDE_MSVC < 1700 #pragma warning(disable : 4428) // universal-character-name encountered in source @@ -68,6 +72,60 @@ std::string narrow_raw_string_and_size(const std::wstring& s) return nowide::narrow(s2.c_str(), s.size()); } +std::wstring widen_convert_buffer(const std::string& s) +{ + std::array out; + const auto* result = nowide::utf::convert_buffer(out.data(), out.size(), s.data(), s.data() + s.size()); + TEST(result == out.data()); + return result; +} + +std::string narrow_convert_buffer(const std::wstring& s) +{ + std::array out; + const auto* result = nowide::utf::convert_buffer(out.data(), out.size(), s.data(), s.data() + s.size()); + TEST(result == out.data()); + return result; +} + +#ifdef NOWIDE_TEST_STD_STRINGVIEW +#define ASSERT_RETURN_TYPE_SV(FUNCTION, CHAR_INPUT, OUTPUT) \ + static_assert(std::is_same>())), OUTPUT>::value, \ + "Should be " #OUTPUT); +#else +#define ASSERT_RETURN_TYPE_SV(FUNCTION, CHAR_INPUT, OUTPUT) +#endif +// Check that the functions are callable with various types +#define ASSERT_RETURN_TYPE(FUNCTION, CHAR_INPUT, OUTPUT) \ + ASSERT_RETURN_TYPE_SV(FUNCTION, CHAR_INPUT, OUTPUT) \ + static_assert(std::is_same())), OUTPUT>::value, \ + "Should be " #OUTPUT); \ + static_assert(std::is_same(), size_t{})), OUTPUT>::value, \ + "Should be " #OUTPUT); \ + static_assert(std::is_same>())), OUTPUT>::value, \ + "Should be " #OUTPUT) + +ASSERT_RETURN_TYPE(nowide::widen, char, std::wstring); +#ifdef __cpp_char8_t +ASSERT_RETURN_TYPE(nowide::widen, char8_t, std::wstring); +#endif +ASSERT_RETURN_TYPE(nowide::narrow, wchar_t, std::string); +ASSERT_RETURN_TYPE(nowide::narrow, char16_t, std::string); +ASSERT_RETURN_TYPE(nowide::narrow, char32_t, std::string); + +#ifdef NOWIDE_TEST_STD_STRINGVIEW +std::wstring widen_string_view(const std::string& s) +{ + return nowide::widen(std::string_view(s)); +} + +std::string narrow_string_view(const std::wstring& s) +{ + return nowide::narrow(std::wstring_view(s)); +} +#endif + +// coverity[root_function] void test_main(int, char**, char**) { std::string hello = "\xd7\xa9\xd7\x9c\xd7\x95\xd7\x9d"; @@ -90,6 +148,9 @@ void test_main(int, char**, char**) TEST(buf == whello_3); TEST(nowide::widen(buf, 5, b, b) == buf && buf[0] == 0); TEST(nowide::widen(buf, 5, b, b + 2) == buf && buf[1] == 0 && buf[0] == whello[0]); + + // Raw literals are also possible + TEST(nowide::widen("\xd7\xa9\xd7\x9c\xd7\x95\xd7\x9d") == whello); } std::cout << "- nowide::narrow" << std::endl; { @@ -102,9 +163,35 @@ void test_main(int, char**, char**) TEST(buf[9] == 1); TEST(nowide::narrow(buf, 8, b, e) == 0); TEST(nowide::narrow(buf, 7, b, e - 1) == buf); - TEST(buf == hello.substr(0, 6)); + TEST_EQ(buf, hello.substr(0, 6)); + + // Raw literals are also possible + TEST_EQ(nowide::narrow(L"\u05e9\u05dc\u05d5\u05dd"), hello); + } + + std::cout << "- nowide::utf::convert_buffer" << std::endl; + { + std::array buf; + const char* b = hello.c_str(); + const char* e = b + hello.size(); + using nowide::utf::convert_buffer; + // Buffer to small (need 5 chars) -> nullptr returned + for(size_t len = 0; len <= 4; ++len) + TEST(convert_buffer(buf.data(), len, b, e) == nullptr); + buf.fill(42); + TEST(convert_buffer(buf.data(), buf.size(), b, e) == buf.data()); + TEST(buf[4] == 0); // NULL terminator added + TEST(buf.back() == 42); // Rest untouched + TEST(std::wstring(buf.data()) == whello); } + namespace utf = nowide::utf; + // Decode of empty range yields incomplete + auto helloIt = hello.begin(); + TEST_EQ(utf::utf_traits::decode(helloIt, helloIt), utf::incomplete); + TEST_EQ(utf::utf_traits::decode(helloIt, helloIt), utf::incomplete); + TEST_EQ(utf::utf_traits::decode(helloIt, helloIt), utf::incomplete); + std::cout << "- (output_buffer, buffer_size, input_raw_string)" << std::endl; run_all(widen_buf_ptr, narrow_buf_ptr); std::cout << "- (output_buffer, buffer_size, input_raw_string, string_len)" << std::endl; @@ -115,4 +202,10 @@ void test_main(int, char**, char**) run_all(widen_raw_string_and_size, narrow_raw_string_and_size); std::cout << "- (const std::string&)" << std::endl; run_all(nowide::widen, nowide::narrow); +#ifdef NOWIDE_TEST_STD_STRINGVIEW + std::cout << "- (std::string_view)" << std::endl; + run_all(widen_string_view, narrow_string_view); +#endif + std::cout << "- (utf::convert_buffer)" << std::endl; + run_all(widen_convert_buffer, narrow_convert_buffer); } diff --git a/extern/boost/nowide/test/test_env.cpp b/extern/boost/nowide/test/test_env.cpp index f648d05ada..6d4018d89a 100644 --- a/extern/boost/nowide/test/test_env.cpp +++ b/extern/boost/nowide/test/test_env.cpp @@ -1,43 +1,29 @@ // -// Copyright (c) 2012 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2012 Artyom Beilis (Tonkikh) // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #include +#include "test.hpp" #include #if defined(NOWIDE_TEST_INCLUDE_WINDOWS) && defined(NOWIDE_WINDOWS) #include #endif -#include "test.hpp" - -// "Safe" strcpy version with NULL termination to make MSVC runtime happy -// which warns when using strncpy -template -void strcpy_safe(char (&dest)[size], const char* src) -{ - size_t len = std::strlen(src); - if(len >= size) - len = size - 1u; - std::memcpy(dest, src, len); - dest[len] = 0; -} - +// coverity[root_function] void test_main(int, char**, char**) { std::string example = "\xd7\xa9-\xd0\xbc-\xce\xbd"; - char penv[256] = {0}; - strcpy_safe(penv, ("NOWIDE_TEST2=" + example + "x").c_str()); + std::string envVar = "NOWIDE_TEST2=" + example + "x"; TEST(nowide::setenv("NOWIDE_TEST1", example.c_str(), 1) == 0); TEST(nowide::getenv("NOWIDE_TEST1")); TEST(nowide::getenv("NOWIDE_TEST1") == example); TEST(nowide::setenv("NOWIDE_TEST1", "xx", 0) == 0); TEST(nowide::getenv("NOWIDE_TEST1") == example); + char* penv = const_cast(envVar.c_str()); TEST(nowide::putenv(penv) == 0); TEST(nowide::getenv("NOWIDE_TEST2")); TEST(nowide::getenv("NOWIDE_TEST_INVALID") == 0); @@ -45,11 +31,10 @@ void test_main(int, char**, char**) #ifdef NOWIDE_WINDOWS // Passing a variable without an equals sign (before \0) is an error // But GLIBC has an extension that unsets the env var instead - char penv2[256] = {0}; - const char* sPenv2 = "NOWIDE_TEST1SOMEGARBAGE="; - strcpy_safe(penv2, sPenv2); + std::string envVar2 = "NOWIDE_TEST1SOMEGARBAGE="; // End the string before the equals sign -> Expect fail - penv2[strlen("NOWIDE_TEST1")] = '\0'; + envVar2[strlen("NOWIDE_TEST1")] = '\0'; + char* penv2 = const_cast(envVar2.c_str()); TEST(nowide::putenv(penv2) == -1); TEST(nowide::getenv("NOWIDE_TEST1")); TEST(nowide::getenv("NOWIDE_TEST1") == example); diff --git a/extern/boost/nowide/test/test_filebuf.cpp b/extern/boost/nowide/test/test_filebuf.cpp new file mode 100644 index 0000000000..867692232a --- /dev/null +++ b/extern/boost/nowide/test/test_filebuf.cpp @@ -0,0 +1,824 @@ +// Copyright (c) 2015 Artyom Beilis (Tonkikh) +// Copyright (c) 2019-2021 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include + +#include "file_test_helpers.hpp" +#include "test.hpp" +#include +#include +#include +#include +#include +#include + +namespace nw = nowide; +using namespace nowide::test; + +// Check member types +static_assert(std::is_same::value, "!"); +static_assert(std::is_same::value, "!"); +static_assert(std::is_same::value, "!"); +static_assert(std::is_same::value, "!"); +static_assert(std::is_same::value, "!"); + +using CharTraits = nw::filebuf::traits_type; +const auto eof = CharTraits::eof(); + +constexpr std::ios_base::openmode make_mode(std::ios_base::openmode flags, bool binary) +{ + return binary ? flags | std::ios_base::binary : flags; +} + +template +bool open_with_buffer(T_FileBuf& filebuf, + const std::string& filepath, + std::ios_base::openmode flags, + char* buffer, + size_t size) +{ + const bool bufferSet = filebuf.pubsetbuf(buffer, size) != nullptr; + return filebuf.open(filepath, flags) && (bufferSet || filebuf.pubsetbuf(buffer, size)); +} + +template +bool open_with_buffer(T_FileBuf& filebuf, + const std::string& filepath, + std::ios_base::openmode flags, + char (&buffer)[T_size]) +{ + return open_with_buffer(filebuf, filepath, flags, buffer, T_size); +} + +template +bool open_unbuffered(T_FileBuf& filebuf, const std::string& filepath, std::ios_base::openmode flags) +{ + return open_with_buffer(filebuf, filepath, flags, nullptr, 0); +} + +// Read the given number of chars from the buffer, checking that there are that many +template +bool skip_chars(T_Buf& buf, size_t num_chars) +{ + for(size_t i = 0; i < num_chars; ++i) + { + if(buf.sbumpc() == eof) + return false; // LCOV_EXCL_LINE + } + return true; +} + +void test_open_close(const std::string& filepath) +{ + const std::string filepath2 = filepath + ".2"; + ensure_not_exists(filepath2); + remove_file_at_exit _(filepath); + remove_file_at_exit _2(filepath2); + + { + nw::filebuf buf; + TEST(buf.open(filepath, std::ios_base::out) == &buf); + TEST(buf.is_open()); + + // Opening when already open fails + TEST(buf.open(filepath2, std::ios_base::out) == nullptr); + // Still open + TEST(buf.is_open()); + TEST(buf.close() == &buf); + // Failed opening did not create file + TEST(!file_exists(filepath2)); + + // But it should work now: + TEST(buf.open(filepath2, std::ios_base::out) == &buf); + TEST(buf.close() == &buf); + TEST(file_exists(filepath2)); + } + + const auto file_data = create_random_data(20, data_type::text); + create_file(filepath, file_data); + // Can't write to read-only buf + { + nw::filebuf buf; + TEST(buf.open(filepath, std::ios_base::in)); + TEST_EQ(buf.sputc('a'), eof); + // Even if chars were copied to put area, they cannot be written (in sync) + TEST(buf.sputn("hello", 5) == 0 || buf.pubsync() == -1); + } + TEST_EQ(read_file(filepath), file_data); // File is unchanged + { + nw::filebuf buf; + char buffer[3]; + TEST(open_with_buffer(buf, filepath, std::ios_base::in, buffer)); + // Also doesn't write when using direct I/O due to data>buffersize (at least in Nowide) + TEST(buf.sputn("hello", 5) == 0 || buf.pubsync() == -1); + } + TEST_EQ(read_file(filepath), file_data); // File is unchanged + + // Can't read from write-only buf + { + for(const auto flags : {std::ios_base::out, std::ios_base::app}) + { + create_file(filepath, file_data); + nw::filebuf buf; + TEST(buf.open(filepath, flags)); + TEST_EQ(buf.sgetc(), eof); + TEST_EQ(buf.sbumpc(), eof); + char str[3]; + TEST_EQ(buf.sgetn(str, sizeof(str)), 0); + // Putback is also just for reading + TEST(buf.pubseekoff(0, std::ios_base::end) != std::streampos(-1)); + TEST_EQ(buf.sungetc(), eof); + TEST_EQ(buf.sputbackc('t'), eof); + } + } +} + +void test_pubseekpos(const std::string& filepath) +{ + const std::string data = create_random_data(BUFSIZ * 4, data_type::binary); + create_file(filepath, data, data_type::binary); + nw::filebuf buf; + TEST(buf.open(filepath, std::ios_base::in | std::ios_base::binary) == &buf); + + // Fuzzy test: Seek to a couple random positions + std::minstd_rand rng(std::random_device{}()); + using pos_type = nw::filebuf::pos_type; + const auto eofPos = pos_type(data.size()); + std::uniform_int_distribution distr(0, static_cast(eofPos) - 1); + + const auto getData = [&](pos_type pos) { return CharTraits::to_int_type(data[static_cast(pos)]); }; + + for(int i = 0; i < 100; i++) + { + const pos_type pos = distr(rng); + TEST_EQ(buf.pubseekpos(pos), pos); + TEST_EQ(buf.sgetc(), getData(pos)); + } + // Seek to first and last as corner case tests + TEST_EQ(buf.pubseekpos(0), pos_type(0)); + TEST_EQ(buf.sgetc(), CharTraits::to_int_type(data[0])); + TEST_EQ(buf.pubseekpos(eofPos), eofPos); + TEST_EQ(buf.sgetc(), eof); +} + +void test_pubseekoff(const std::string& filepath) +{ + const std::string data = create_random_data(BUFSIZ * 4, data_type::binary); + create_file(filepath, data, data_type::binary); + nw::filebuf buf; + TEST(buf.open(filepath, std::ios_base::in | std::ios_base::binary) == &buf); + + // Fuzzy test: Seek to a couple random positions + std::minstd_rand rng(std::random_device{}()); + using pos_type = nw::filebuf::pos_type; + using off_type = nw::filebuf::off_type; + const auto eofPos = pos_type(data.size()); + std::uniform_int_distribution distr(0, static_cast(eofPos) - 1); + + const auto getData = [&](pos_type pos) { return CharTraits::to_int_type(data[static_cast(pos)]); }; + // tellg/tellp function as called by basic_[io]fstream + const auto tellg = [&]() { return buf.pubseekoff(0, std::ios_base::cur); }; + + for(int i = 0; i < 100; i++) + { + // beg + pos_type pos = distr(rng); + TEST_EQ(buf.pubseekoff(pos, std::ios_base::beg), pos); + TEST_EQ(tellg(), pos); + TEST_EQ(buf.sgetc(), getData(pos)); + // cur + off_type diff = static_cast(distr(rng)) - pos; + pos += diff; + TEST_EQ(buf.pubseekoff(diff, std::ios_base::cur), pos); + TEST_EQ(tellg(), pos); + TEST_EQ(buf.sgetc(), getData(pos)); + // end + diff = static_cast(distr(rng)) - eofPos; + pos = eofPos + diff; + TEST_EQ(buf.pubseekoff(diff, std::ios_base::end), pos); + TEST_EQ(tellg(), pos); + TEST_EQ(buf.sgetc(), getData(pos)); + } + // Seek to first and last as corner case tests + TEST_EQ(buf.pubseekoff(0, std::ios_base::beg), pos_type(0)); + TEST_EQ(tellg(), pos_type(0)); + TEST_EQ(buf.sgetc(), CharTraits::to_int_type(data[0])); + TEST_EQ(buf.pubseekoff(0, std::ios_base::end), eofPos); + TEST_EQ(tellg(), eofPos); + TEST_EQ(buf.sgetc(), eof); +} + +void test_64_bit_seek(const std::string& filepath) +{ + // Create a value which does not fit into a 32 bit value. + // Use an unsigned intermediate to have the truncation defined to wrap to 0 + using unsigned_off_type = std::make_unsigned::type; + nw::filebuf::off_type offset = static_cast(std::uint64_t(1) << 33u); + +#ifdef NOWIDE_MSVC +#pragma warning(push) +#pragma warning(disable : 4127) +#endif + // if we can't use 64 bit offsets through the API, don't test anything + // coverity[result_independent_of_operands] + if(offset == nw::filebuf::off_type(0)) + { + // coverity[dead_error_line] + return; // LCOV_EXCL_LINE + } +#ifdef NOWIDE_MSVC +#pragma warning(pop) +#endif + + create_file(filepath, "test"); + remove_file_at_exit _(filepath); + + nw::filebuf buf; + TEST(buf.open(filepath, std::ios_base::in | std::ios_base::binary) == &buf); + const std::streampos knownPos = 2; + TEST_EQ(buf.pubseekpos(knownPos), knownPos); // Just to make sure we know where we are + const std::streampos newPos = buf.pubseekoff(offset, std::ios_base::cur); + // On 32 bit mode or when seek beyond EOF is not allowed, the current position should be unchanged + if(newPos == std::streampos(-1)) + TEST_EQ(buf.pubseekoff(0, std::ios_base::cur), knownPos); + else + { +#if !NOWIDE_USE_FILEBUF_REPLACEMENT + // libc++ may truncate the 64 bit value when calling fseek which yields an offset of 0 + if(newPos == knownPos) + offset = 0; // LCOV_EXCL_LINE +#endif + TEST_EQ(newPos, offset + knownPos); + TEST_EQ(buf.pubseekoff(0, std::ios_base::cur), newPos); + } +} + +void test_xsgetn(const std::string& filepath, bool binary) +{ + char buffer[200]{}; + const auto dataType = binary ? data_type::binary : data_type::text; + const std::string data = create_random_data(sizeof(buffer) + 50, dataType); + create_file(filepath, data, dataType); + + for(const bool unbuffered : {false, true}) + { + TEST_CONTEXT((unbuffered ? "unbuffered" : "buffered")); + nw::filebuf buf; + if(unbuffered) + TEST(open_unbuffered(buf, filepath, make_mode(std::ios_base::in, binary))); + else + TEST(open_with_buffer(buf, filepath, make_mode(std::ios_base::in, binary), buffer)); + + std::string strBuf(data.size() * 2, '\0'); + // Reading stops at EOF + TEST_EQ(buf.sgetn(&strBuf[0], strBuf.size()), static_cast(data.size())); + strBuf.resize(data.size()); + TEST_EQ(strBuf, data); + TEST(buf.pubseekpos(0) != std::streampos(-1)); + // Read a bit using regular underflow, then via sgetn and again via underflow + TEST_EQ(buf.sbumpc(), CharTraits::to_int_type(data[0])); + TEST_EQ(buf.sbumpc(), CharTraits::to_int_type(data[1])); + strBuf.clear(); + // Go definitely over a buffer boundary + strBuf.resize(sizeof(buffer)); + TEST_EQ(buf.sgetn(&strBuf[0], strBuf.size()), static_cast(strBuf.size())); + TEST_EQ(strBuf, data.substr(2, strBuf.size())); + TEST_EQ(buf.sbumpc(), CharTraits::to_int_type(data[strBuf.size() + 2])); + TEST_EQ(buf.sbumpc(), CharTraits::to_int_type(data[strBuf.size() + 3])); + } + // Corner cases: + // - sgetn with zero or negative count is a no-op + // - sgetn fails on closed filebuf + for(const bool unbuffered : {false, true}) + { + TEST_CONTEXT((unbuffered ? "unbuffered" : "buffered")); + create_file(filepath, "Hello World"); + nw::filebuf buf; + // Set a buffer just to see if it is written to + if(unbuffered) + TEST(open_unbuffered(buf, filepath, make_mode(std::ios_base::in, binary))); + else + TEST(open_with_buffer(buf, filepath, make_mode(std::ios_base::in, binary), buffer)); + + std::string str = create_random_data(data.size(), data_type::binary); + const auto origStr = str; + buffer[0] = origStr[0]; + + TEST_EQ(buf.sgetn(&str[0], 0), 0); +#if defined(__GNUC__) && __GNUC__ >= 12 + // GCC may not detect that the negative value is checked by xsgetn +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wrestrict" +#endif + // coverity[negative_returns] + TEST_EQ(buf.sgetn(&str[0], -1), 0); + // coverity[negative_returns] + TEST_EQ(buf.sgetn(&str[0], -999), 0); +#if defined(__GNUC__) && __GNUC__ >= 12 +#pragma GCC diagnostic pop +#endif + buf.close(); + // No read when closed + TEST_EQ(buf.sgetn(&str[0], 1), 0); + TEST_EQ(str, origStr); + TEST_EQ(buffer[0], origStr[0]); + TEST_EQ(buf.sgetn(&str[0], str.size()), 0); + TEST_EQ(str, origStr); + TEST_EQ(buffer[0], origStr[0]); + } +} + +void test_xsputn(const std::string& filepath, bool binary) +{ + char buffer[200]{}; + const auto dataType = binary ? data_type::binary : data_type::text; + const std::string data = create_random_data(sizeof(buffer) + 50, dataType); + + for(const bool unbuffered : {false, true}) + { + TEST_CONTEXT((unbuffered ? "unbuffered" : "buffered")); + nw::filebuf buf; + const auto flags = make_mode(std::ios_base::out | std::ios_base::trunc, binary); + if(unbuffered) + TEST(open_unbuffered(buf, filepath, flags)); + else + TEST(open_with_buffer(buf, filepath, flags, buffer)); + + TEST_EQ(buf.sputn(data.data(), data.size()), static_cast(data.size())); + buf.close(); + TEST_EQ(read_file(filepath, dataType), data); + + // Write a bit using regular overflow, then via sputn and back using overflow + TEST(buf.open(filepath, flags)); + TEST_EQ(buf.sputc(data[0]), CharTraits::to_int_type(data[0])); + TEST_EQ(buf.sputc(data[1]), CharTraits::to_int_type(data[1])); + // This is more than 1 buffer size + std::streamsize numBytesToWrite = data.size() - 4; + TEST_EQ(buf.sputn(&data[2], numBytesToWrite), numBytesToWrite); + TEST_EQ(buf.sputc(data[data.size() - 2]), CharTraits::to_int_type(data[data.size() - 2])); + TEST_EQ(buf.sputc(data[data.size() - 1]), CharTraits::to_int_type(data[data.size() - 1])); + buf.close(); + TEST_EQ(read_file(filepath, dataType), data); + } + // Corner cases: + // - sputn with zero or negative count is a no-op + // - sputn fails on closed filebuf + for(const bool unbuffered : {false, true}) + { + TEST_CONTEXT((unbuffered ? "unbuffered" : "buffered")); + nw::filebuf buf; + if(unbuffered) + TEST(open_unbuffered(buf, filepath, make_mode(std::ios_base::out | std::ios_base::trunc, binary))); + else + TEST(open_with_buffer(buf, filepath, make_mode(std::ios_base::out | std::ios_base::trunc, binary), buffer)); + + TEST_EQ(buf.sputn(data.data(), 0), 0); + // coverity[negative_returns] + TEST_EQ(buf.sputn(data.data(), -1), 0); + // coverity[negative_returns] + TEST_EQ(buf.sputn(data.data(), -999), 0); + buf.close(); + // No write when closed + TEST_EQ(buf.sputn(data.data(), 1), 0); + TEST_EQ(buf.sputn(data.data(), data.size()), 0); + TEST_EQ(read_file(filepath), ""); + } +} + +void test_read_write_switch(const std::string& filepath, bool binary) +{ + // Switching between read and write requires a seek or (for W->R) a sync + const std::string data = "1234567890"; + nw::filebuf buf; + TEST(buf.open(filepath, make_mode(std::ios_base::in | std::ios_base::out | std::ios_base::trunc, binary))); + TEST_EQ(buf.sputn(data.data(), data.size()), static_cast(data.size())); + // W->R via seek + buf.pubseekpos(0); + TEST_EQ(buf.sbumpc(), '1'); + // R->W via seek + const auto pos = buf.pubseekoff(0, std::ios_base::cur); + TEST(pos != std::streampos(-1)); + buf.sputc('b'); + // W->R via sync + TEST(buf.pubsync() == 0); + TEST_EQ(buf.sbumpc(), '3'); + // R->W via seek + const auto pos2 = buf.pubseekoff(0, std::ios_base::cur); + buf.sputc('c'); + // Read right back + TEST_EQ(buf.pubseekpos(pos2), pos2); + TEST_EQ(buf.sbumpc(), 'c'); + // R->W + buf.pubseekoff(0, std::ios_base::cur); + buf.sputc('d'); + // Sync & seek + TEST(buf.pubsync() == 0); + TEST(buf.pubseekoff(0, std::ios_base::cur) != std::streampos(-1)); + TEST_EQ(buf.sbumpc(), '6'); + // R->W + buf.pubseekoff(0, std::ios_base::cur); + buf.sputc('e'); + // Seek & sync + TEST(buf.pubseekoff(0, std::ios_base::cur) != std::streampos(-1)); + TEST(buf.pubsync() == 0); + TEST_EQ(buf.sbumpc(), '8'); + + buf.close(); + TEST_EQ(read_file(filepath), "1b3cd6e890"); +} + +void subtest_sync(const std::string& filepath, bool binary, const std::string& data) +{ + nw::filebuf buf; + // Use a small buffer to force filling it up w/o requiring to write lot's of data + char buffer[3]; + buf.pubsetbuf(buffer, sizeof(buffer)); + const auto flags = make_mode(std::ios_base::out | std::ios_base::trunc, binary); + + // Do a series of single-char and multi-char writes with varying size combinations + // Especially test the case of only single-char and only multi-char ops + for(unsigned singleCharOps = 0; singleCharOps <= 3; ++singleCharOps) + { + // Write less than buffer size, 1 or 2 buffers or even more, assuming buffer size of 3 + for(size_t bufSize : {0, 2, 3, 6, 7}) + { + if(singleCharOps + bufSize == 0u) + continue; + TEST(buf.open(filepath, flags)); + for(size_t i = 0; i < data.size();) + { + TEST_CONTEXT("sc:" << singleCharOps << " buf:" << bufSize << " i:" << i); + for(unsigned j = 0; j < singleCharOps && i < data.size(); ++j, ++i) + { + TEST_EQ(buf.sputc(data[i]), CharTraits::to_int_type(data[i])); + } + if(bufSize != 0u) + { + const auto remainSize = std::min(data.size() - i, bufSize); + TEST_EQ(buf.sputn(&data[i], remainSize), static_cast(remainSize)); + i += remainSize; + } + TEST_EQ(buf.pubsync(), 0); + TEST_EQ(read_file(filepath, binary ? data_type::binary : data_type::text), data.substr(0, i)); + } + TEST(buf.close()); + TEST_EQ(read_file(filepath, binary ? data_type::binary : data_type::text), data); + } + } +} + +void subtest_singlechar_positioning(const std::string& filepath, bool binary, const std::string& data) +{ + nw::filebuf buf; + // Use a small buffer to force filling it up w/o requiring to write lot's of data + char buffer[3]{}; + const auto mode = make_mode(std::ios_base::in | std::ios_base::out | std::ios_base::trunc, binary); + TEST(open_with_buffer(buf, filepath, mode, buffer)); + + // Put each char and record its position + std::vector pos(data.size()); + for(unsigned i = 0; i < data.size(); ++i) + { + buf.sputc(data[i]); + pos[i] = buf.pubseekoff(0, std::ios_base::cur); + } + // Go back to start and verify reading yields the same data and positions + buf.pubseekoff(0, std::ios_base::beg); + for(unsigned i = 0; i < data.size(); ++i) + { + TEST_CONTEXT("Position " << i); + TEST_EQ(buf.sbumpc(), CharTraits::to_int_type(data[i])); + TEST_EQ(buf.pubseekoff(0, std::ios_base::cur), pos[i]); + } +} + +void subtest_singlechar_multichar_reads(const std::string& filepath, bool binary, const std::string& data) +{ + create_file(filepath, data, binary ? data_type::binary : data_type::text); + nw::filebuf buf; + // Use a small buffer to force filling it up w/o requiring to write lot's of data + char buffer[3]{}; + TEST(open_with_buffer(buf, filepath, make_mode(std::ios_base::in, binary), buffer)); + + // Do a series of single-char and multi-char reads with varying size combinations + // Especially test the case of only single-char and only multi-char ops + for(unsigned singleCharOps = 0; singleCharOps <= 3; ++singleCharOps) + { + // Read less than buffer size, 1 or 2 buffers or even more, assuming buffer size of 3 + for(size_t bufSize : {0, 2, 3, 6, 7}) + { + if(singleCharOps + bufSize == 0u) + continue; + + std::string outBuf(bufSize, '\0'); + buf.pubseekoff(0, std::ios_base::beg); + for(size_t i = 0; i < data.size();) + { + TEST_CONTEXT("sc:" << singleCharOps << " buf:" << bufSize << " i:" << i); + for(unsigned j = 0; j < singleCharOps && i < data.size(); ++j, ++i) + { + TEST_EQ(buf.sbumpc(), CharTraits::to_int_type(data[i])); + } + if(bufSize == 0u) + continue; + const size_t readSize = std::min(data.size() - i, bufSize); + TEST_EQ(buf.sgetn(&outBuf.front(), bufSize), static_cast(readSize)); + if(readSize < bufSize) + outBuf.resize(readSize); + TEST_EQ(outBuf, data.substr(i, readSize)); + i += bufSize; + } + } + } +} + +void test_sungetc(const std::string& filepath, bool binary) +{ + const std::string data = "012345\n6"; + create_file(filepath, data, binary ? data_type::binary : data_type::text); + + // Use a small buffer to force filling it up w/o requiring to write lot's of data + char buffer[4]; + nw::filebuf buf; + TEST(open_with_buffer(buf, filepath, make_mode(std::ios_base::in, binary), buffer)); + + // Nothing to unget at beginning of file + TEST_EQ(buf.sungetc(), eof); + + // Able to unget first char and get it again + TEST_EQ(buf.sbumpc(), '0'); + TEST(buf.sungetc() != eof); + TEST_EQ(buf.sbumpc(), '0'); + + // Able to unget and reread after filling up new buffer + TEST(skip_chars(buf, sizeof(buffer) - 1u)); // skip remaining chars + TEST_EQ(buf.sbumpc(), '4'); + TEST(buf.sungetc() != eof); + // Ungetting multiple chars may or may not be possible + if(buf.sungetc() != eof) + TEST_EQ(buf.sbumpc(), '3'); + TEST_EQ(buf.sbumpc(), '4'); + + // \n also works + TEST_EQ(buf.sbumpc(), '5'); + TEST_EQ(buf.sbumpc(), '\n'); + TEST(buf.sungetc() != eof); + TEST_EQ(buf.sbumpc(), '\n'); + TEST_EQ(buf.sbumpc(), '6'); + TEST(buf.sungetc() != eof); + if(buf.sungetc() != eof) + TEST_EQ(buf.sbumpc(), '\n'); + TEST_EQ(buf.sbumpc(), '6'); + + // Go back as far as possible + auto idx = data.size(); + while(buf.sungetc() != eof) + { + TEST(idx > 0u); + --idx; + } + TEST(idx < data.size()); // At least 1 + // Get all put back chars + for(; idx < data.size(); ++idx) + TEST_EQ(buf.sbumpc(), data[idx]); +} + +void test_sputbackc(const std::string& filepath, bool binary) +{ + const std::string data = "012345\n6"; + create_file(filepath, data, binary ? data_type::binary : data_type::text); + + // Use a small buffer to force filling it up w/o requiring to write lot's of data + char buffer[4]; + nw::filebuf buf; + TEST(open_with_buffer(buf, filepath, make_mode(std::ios_base::in, binary), buffer)); + TEST(skip_chars(buf, data.size())); + + // Put back same chars explicitly (as many as possible) + auto idx = data.size(); + while(true) + { + auto res = buf.sputbackc((idx > 0u) ? data[idx - 1] : 'z'); + if(res == eof) + break; + TEST(idx > 0u); + TEST_EQ(res, data[idx - 1]); + --idx; + } + TEST(idx < data.size()); // At least 1 + // Get all put back chars + for(; idx < data.size(); ++idx) + TEST_EQ(buf.sbumpc(), data[idx]); + + // Put back different chars (as many as possible) + const std::string data2 = "789\nab\nc"; + TEST_EQ(data.size(), data2.size()); + idx = data2.size(); + while(true) + { + auto res = buf.sputbackc((idx > 0u) ? data2[idx - 1] : 'z'); + if(res == eof) + break; + TEST(idx > 0u); + TEST_EQ(res, data2[idx - 1]); + --idx; +#if !NOWIDE_USE_FILEBUF_REPLACEMENT + break; // Some stdlibs fail on putting back multiple different chars +#endif + } +#if NOWIDE_USE_FILEBUF_REPLACEMENT + // At least 1 when using custom filebuf + // But e.g. libc++ doesn't allow putting back a different char + TEST(idx < data2.size()); +#endif + // Get all put back chars + for(; idx < data2.size(); ++idx) + TEST_EQ(buf.sbumpc(), data2[idx]); +} + +void test_textmode(const std::string& filepath) +{ + // Test input, output and getting the file position works for text files with newlines + const std::string data = []() { + // Some simple test data + std::string result = "1234567890"; + // Line break after every char + result.reserve(result.size() + 27 * 2); + for(char c = 'a'; c <= 'z'; ++c) + (result += c) += '\n'; + // Some continuous line breaks + result.append(4, '\n'); + return result; + }(); + subtest_singlechar_positioning(filepath, false, data); + subtest_singlechar_multichar_reads(filepath, false, data); + subtest_sync(filepath, false, data); +} + +// Almost the same test as test_textmode but uses a binary stream. +// Useful as the buffer handling is very different +void test_binarymode(const std::string& filepath) +{ + const std::string data = "123" + create_random_data(65, data_type::binary); + subtest_singlechar_positioning(filepath, true, data); + subtest_singlechar_multichar_reads(filepath, true, data); + subtest_sync(filepath, true, data); +} + +void test_swap(const std::string& filepath) +{ + const std::string filepath2 = filepath + ".2"; + remove_file_at_exit _(filepath); + remove_file_at_exit _2(filepath2); + + // Note: Make sure to have en uneven number of swaps so the destructor runs on the others data + + // Check: FILE*, buffer, buffer_size + { + nw::filebuf buf1, buf2; + char buffer1[3]{}, buffer2[5]{}; + buf1.pubsetbuf(buffer1, sizeof(buffer1)); + buf2.pubsetbuf(buffer2, sizeof(buffer2)); + TEST(buf1.open(filepath, std::ios_base::out) == &buf1); + buf1.swap(buf2); + TEST(!buf1.is_open()); + TEST(buf2.is_open()); + TEST(buf1.open(filepath2, std::ios_base::out | std::ios_base::binary) == &buf1); + + // Write "FooBar" to filepath and "HelloWorld" to filepath2 + buf1.sputc('H'); + buf1.sputn("ello", 4); + buf2.sputc('F'); + buf2.sputn("oo", 2); + buf2.swap(buf1); + buf1.sputc('B'); + buf1.sputn("ar", 2); + buf2.sputc('W'); + buf2.sputn("orld", 4); + + buf1.close(); + TEST(!buf1.is_open()); + TEST(buf2.is_open()); + buf1.swap(buf2); + TEST(buf1.is_open()); + TEST(!buf2.is_open()); + buf1.close(); + TEST(!buf1.is_open()); + TEST(!buf2.is_open()); + TEST_EQ(read_file(filepath), "FooBar"); + TEST_EQ(read_file(filepath2), "HelloWorld"); + } + // Check: mode, owns_buffer + { + nw::filebuf buf1, buf2; + char buffer[3]{}; + buf1.pubsetbuf(buffer, sizeof(buffer)); + TEST(buf1.open(filepath, std::ios_base::out) == &buf1); + TEST(buf2.open(filepath2, std::ios_base::in) == &buf2); + TEST_EQ(buf1.sputc('B'), 'B'); + TEST_EQ(buf2.sbumpc(), 'H'); + buf1.swap(buf2); + // Trying to read in write mode or other way round should fail + TEST_EQ(buf1.sputc('x'), eof); + TEST_EQ(buf2.sbumpc(), eof); + TEST_EQ(buf1.sbumpc(), 'e'); + TEST_EQ(buf2.sputc('a'), 'a'); + buf2.swap(buf1); + TEST_EQ(buf2.sputc('x'), eof); + TEST_EQ(buf1.sbumpc(), eof); + TEST_EQ(buf2.sbumpc(), 'l'); + TEST_EQ(buf1.sputn("zXYZ", 4), 4); + swap(buf2, buf1); + buf1.close(); + buf2.close(); + TEST_EQ(read_file(filepath), "BazXYZ"); + TEST_EQ(read_file(filepath2), "HelloWorld"); + } + // Check: last_char, gptr, eback + { + nw::filebuf buf1, buf2; + // Need to disable buffering to use last_char, but only for 1 to detect wrong conditions + buf1.pubsetbuf(0, 0); + TEST(buf1.open(filepath, std::ios_base::in) == &buf1); + TEST(buf2.open(filepath2, std::ios_base::in) == &buf2); + // Peek + TEST_EQ(buf1.sgetc(), 'B'); + TEST_EQ(buf2.sgetc(), 'H'); + swap(buf1, buf2); + TEST_EQ(buf2.sgetc(), 'B'); + TEST_EQ(buf1.sgetc(), 'H'); + // Advance + TEST_EQ(buf2.sbumpc(), 'B'); + TEST_EQ(buf1.sbumpc(), 'H'); + TEST_EQ(buf2.sbumpc(), 'a'); + TEST_EQ(buf1.sbumpc(), 'e'); + swap(buf1, buf2); + TEST_EQ(buf1.sbumpc(), 'z'); + TEST_EQ(buf2.sbumpc(), 'l'); + swap(buf1, buf2); + TEST_EQ(buf2.sgetc(), 'X'); + TEST_EQ(buf1.sgetc(), 'l'); + } + // Check: pptr, epptr + { + nw::filebuf buf1, buf2; + // Need to disable buffering to use last_char, but only for 1 to detect wrong conditions + buf1.pubsetbuf(0, 0); + TEST(buf1.open(filepath, std::ios_base::out) == &buf1); + TEST(buf2.open(filepath2, std::ios_base::out) == &buf2); + TEST_EQ(buf1.sputc('1'), '1'); + TEST_EQ(buf2.sputc('a'), 'a'); + swap(buf1, buf2); + // buf1: filepath2, buf2: filepath + TEST_EQ(buf1.sputc('b'), 'b'); + TEST_EQ(buf2.sputc('2'), '2'); + // Sync and check if file was written + TEST_EQ(buf1.pubsync(), 0); + TEST_EQ(read_file(filepath2), "ab"); + TEST_EQ(buf2.pubsync(), 0); + TEST_EQ(read_file(filepath), "12"); + swap(buf1, buf2); + // buf1: filepath, buf2: filepath2 + TEST_EQ(buf1.pubsync(), 0); + TEST_EQ(read_file(filepath), "12"); + TEST_EQ(buf2.pubsync(), 0); + TEST_EQ(read_file(filepath2), "ab"); + TEST_EQ(buf1.sputc('3'), '3'); + TEST_EQ(buf2.sputc('c'), 'c'); + swap(buf1, buf2); + // buf1: filepath2, buf2: filepath + TEST_EQ(buf1.pubsync(), 0); + TEST_EQ(read_file(filepath2), "abc"); + TEST_EQ(buf2.pubsync(), 0); + TEST_EQ(read_file(filepath), "123"); + } +} + +// coverity[root_function] +void test_main(int, char** argv, char**) +{ + const std::string exampleFilename = std::string(argv[0]) + "-\xd7\xa9-\xd0\xbc-\xce\xbd.txt"; + test_open_close(exampleFilename); + test_pubseekpos(exampleFilename); + test_pubseekoff(exampleFilename); + test_64_bit_seek(exampleFilename); + for(const auto binary : {false, true}) + { + std::cout << "Testing " << (binary ? "binary" : "text") << " mode\n"; + remove_file_at_exit _(exampleFilename); + test_xsgetn(exampleFilename, binary); + test_xsputn(exampleFilename, binary); + test_read_write_switch(exampleFilename, binary); + test_sungetc(exampleFilename, binary); + test_sputbackc(exampleFilename, binary); + binary ? test_binarymode(exampleFilename) : test_textmode(exampleFilename); + } +// These tests are only useful for the nowide filebuf and are known to fail for +// std::filebuf due to bugs in libc++ +#if NOWIDE_USE_FILEBUF_REPLACEMENT + test_swap(exampleFilename); +#endif +} diff --git a/extern/boost/nowide/test/test_fs.cpp b/extern/boost/nowide/test/test_fs.cpp index ed73a2c9d0..a10049b2ab 100644 --- a/extern/boost/nowide/test/test_fs.cpp +++ b/extern/boost/nowide/test/test_fs.cpp @@ -1,24 +1,120 @@ // -// Copyright (c) 2015 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) +// Copyright (c) 2015 Artyom Beilis (Tonkikh) +// Copyright (c) 2021 Alexander Grund // +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt +#if defined(__GNUC__) && __GNUC__ >= 7 +#pragma GCC diagnostic ignored "-Wattributes" +#endif #include #include #include #include -#include - +#include #include "test.hpp" -void test_main(int, char**, char**) +#include // Required for feature macro check below +// Conditional include to avoid warning/message +#if defined(__cpp_lib_quoted_string_io) && __cpp_lib_quoted_string_io >= 201304 +#include +#endif + +#include +#include +#if defined(_MSC_VER) +#pragma warning(disable : 4714) // function marked as __forceinline not inlined +#endif +#include + +// Exclude apple as support there is target level specific -.- +#if defined(__cpp_lib_filesystem) && !defined(__APPLE__) +#include +#define NOWIDE_TEST_STD_PATH +#endif +#if defined(__cpp_lib_experimental_filesystem) +#ifndef _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING +#define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING +#endif +#include +#define NOWIDE_TEST_STD_EXPERIMENTAL_PATH +#endif + +template +struct is_istreamable : std::false_type +{}; +using nowide::detail::void_t; +template +struct is_istreamable() >> std::declval())>> : std::true_type +{}; + +template +std::string maybe_narrow(const std::basic_string& s) +{ + return nowide::narrow(s); +} + +const std::string& maybe_narrow(const std::string& s) +{ + return s; +} + +template +void test_fs_path_io(std::string utf8_name) +{ +#if defined(__cpp_lib_quoted_string_io) && __cpp_lib_quoted_string_io >= 201304 + Path path(nowide::utf::convert_string(utf8_name)); + // Get native and UTF-8/narrow name here as the Path ctor may change the string (e.g. slash substitution) + const auto nativeName = path.native(); + utf8_name = maybe_narrow(nativeName); + // Output + std::ostringstream s, sRef; + sRef << std::quoted(utf8_name); + s << nowide::quoted(path); + TEST_EQ(s.str(), sRef.str()); + // const + const Path constPath(path); + s.str(""); + s << nowide::quoted(constPath); + TEST_EQ(s.str(), sRef.str()); + // Rvalue + s.str(""); + s << nowide::quoted(Path(path)); + TEST_EQ(s.str(), sRef.str()); + + // Input + std::istringstream sIn(sRef.str()); + Path pathOut; + static_assert(is_istreamable::value, "!"); + sIn >> nowide::quoted(pathOut); + TEST_EQ(pathOut.native(), nativeName); + // Can't read into a const path + static_assert(!is_istreamable::value, "!"); + // or an Rvalue + static_assert(!is_istreamable::value, "!"); + + // Wide stream + std::wostringstream ws, wsRef; + wsRef << std::quoted(nowide::widen(utf8_name)); + ws << nowide::quoted(path); + TEST_EQ(ws.str(), wsRef.str()); + std::wistringstream wsIn(wsRef.str()); + pathOut.clear(); + wsIn >> nowide::quoted(pathOut); + TEST_EQ(maybe_narrow(pathOut.native()), utf8_name); +#else + (void)utf8_name; // Suppress unused warning + std::cout << "Skipping tests for nowide::quoted" << std::endl; +#endif +} + +// coverity[root_function] +void test_main(int, char** argv, char**) { nowide::nowide_filesystem(); - const std::string prefix = nowide::filesystem::unique_path("nowide-%%%%-%%%%-").string(); + const std::string prefix = argv[0]; const std::string utf8_name = prefix + "\xf0\x9d\x92\x9e-\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82-\xE3\x82\x84\xE3\x81\x82.txt"; @@ -58,4 +154,15 @@ void test_main(int, char**, char**) TEST(test == "Test"); } nowide::filesystem::remove(path); + + std::cout << "Testing nowide::filesystem::path" << std::endl; + test_fs_path_io(utf8_name); +#ifdef NOWIDE_TEST_STD_EXPERIMENTAL_PATH + std::cout << "Testing std::experimental::filesystem::path" << std::endl; + test_fs_path_io(utf8_name); +#endif +#ifdef NOWIDE_TEST_STD_PATH + std::cout << "Testing std::filesystem::path" << std::endl; + test_fs_path_io(utf8_name); +#endif } diff --git a/extern/boost/nowide/test/test_fstream.cpp b/extern/boost/nowide/test/test_fstream.cpp index b290a20272..01254dc76d 100644 --- a/extern/boost/nowide/test/test_fstream.cpp +++ b/extern/boost/nowide/test/test_fstream.cpp @@ -1,475 +1,398 @@ +// Copyright (c) 2015 Artyom Beilis (Tonkikh) +// Copyright (c) 2019-2021 Alexander Grund // -// Copyright (c) 2015 Artyom Beilis (Tonkikh) -// Copyright (c) 2019 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) -// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt #include #include #include +#include "file_test_helpers.hpp" +#include "test.hpp" #include #include +#include #include -#include "test.hpp" - namespace nw = nowide; +using namespace nowide::test; -void make_empty_file(const char* filepath) -{ - nw::ofstream f(filepath, std::ios_base::out | std::ios::trunc); - TEST(f); -} - -bool file_exists(const char* filepath) +class dummyCvtConverting : public std::codecvt { - FILE* f = nw::fopen(filepath, "r"); - if(f) +protected: + bool do_always_noconv() const noexcept override { - std::fclose(f); - return true; - } else return false; -} - -std::string read_file(const char* filepath, bool binary_mode = false) -{ - FILE* f = nw::fopen(filepath, binary_mode ? "rb" : "r"); - TEST(f); - std::string content; - int c; - while((c = std::fgetc(f)) != EOF) - content.push_back(static_cast(c)); - std::fclose(f); - return content; -} + } +}; -void test_with_different_buffer_sizes(const char* filepath) +class dummyCvtNonConverting : public std::codecvt { - /* Important part of the standard for mixing input with output: - However, output shall not be directly followed by input without an intervening call to the fflush function - or to a file positioning function (fseek, fsetpos, or rewind), - and input shall not be directly followed by output without an intervening call to a file positioning function, - unless the input operation encounters end-of-file. - */ - for(int i = -1; i < 16; i++) +protected: + bool do_always_noconv() const noexcept override { - std::cout << "Buffer size = " << i << std::endl; - char buf[16]; - nw::fstream f; - // Different conditions when setbuf might be called: Usually before opening a file is OK - if(i >= 0) - f.rdbuf()->pubsetbuf((i == 0) ? NULL : buf, i); - f.open(filepath, std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary); - TEST(f); - // Add 'abcdefg' - TEST(f.put('a')); - TEST(f.put('b')); - TEST(f.put('c')); - TEST(f.put('d')); - TEST(f.put('e')); - TEST(f.put('f')); - TEST(f.put('g')); - // Read first char - TEST(f.seekg(0)); - TEST(f.get() == 'a'); - TEST(f.gcount() == 1u); - // Skip next char - TEST(f.seekg(1, std::ios::cur)); - TEST(f.get() == 'c'); - TEST(f.gcount() == 1u); - // Go back 1 char - TEST(f.seekg(-1, std::ios::cur)); - TEST(f.get() == 'c'); - TEST(f.gcount() == 1u); - - // Test switching between read->write->read - // case 1) overwrite, flush, read - TEST(f.seekg(1)); - TEST(f.put('B')); - TEST(f.flush()); // Flush when changing out->in - TEST(f.get() == 'c'); - TEST(f.gcount() == 1u); - TEST(f.seekg(1)); - TEST(f.get() == 'B'); - TEST(f.gcount() == 1u); - // case 2) overwrite, seek, read - TEST(f.seekg(2)); - TEST(f.put('C')); - TEST(f.seekg(3)); // Seek when changing out->in - TEST(f.get() == 'd'); - TEST(f.gcount() == 1u); - - // Check that sequence from start equals expected - TEST(f.seekg(0)); - TEST(f.get() == 'a'); - TEST(f.get() == 'B'); - TEST(f.get() == 'C'); - TEST(f.get() == 'd'); - TEST(f.get() == 'e'); - - // Putback after flush is implementation defined - // Boost.Nowide: Works -#if NOWIDE_USE_FILEBUF_REPLACEMENT - TEST(f << std::flush); - TEST(f.putback('e')); - TEST(f.putback('d')); - TEST(f.get() == 'd'); - TEST(f.get() == 'e'); -#endif - // Rest of sequence - TEST(f.get() == 'f'); - TEST(f.get() == 'g'); - TEST(f.get() == EOF); - - // Put back until front of file is reached - f.clear(); - TEST(f.seekg(1)); - TEST(f.get() == 'B'); - TEST(f.putback('B')); - // Putting back multiple chars is not possible on all implementations after a seek/flush -#if NOWIDE_USE_FILEBUF_REPLACEMENT - TEST(f.putback('a')); - TEST(!f.putback('x')); // At beginning of file -> No putback possible - // Get characters that were putback to avoid MSVC bug https://github.com/microsoft/STL/issues/342 - f.clear(); - TEST(f.get() == 'a'); -#endif - TEST(f.get() == 'B'); - f.close(); - TEST(nw::remove(filepath) == 0); + return true; } -} +}; -void test_close(const char* filepath) +std::string get_narrow_name(const std::string& name) { - const std::string filepath2 = std::string(filepath) + ".2"; - // Make sure file does not exist yet - TEST(!file_exists(filepath2.c_str()) || nw::remove(filepath2.c_str()) == 0); - TEST(!file_exists(filepath2.c_str())); - nw::filebuf buf; - TEST(buf.open(filepath, std::ios_base::out) == &buf); - TEST(buf.is_open()); - // Opening when already open fails - TEST(buf.open(filepath2.c_str(), std::ios_base::out) == NULL); - // Still open - TEST(buf.is_open()); - TEST(buf.close() == &buf); - // Failed opening did not create file - TEST(!file_exists(filepath2.c_str())); - // But it should work now: - TEST(buf.open(filepath2.c_str(), std::ios_base::out) == &buf); - TEST(buf.close() == &buf); - TEST(file_exists(filepath2.c_str())); - TEST(nw::remove(filepath) == 0); - TEST(nw::remove(filepath2.c_str()) == 0); + return name; } - -template -void test_flush(const char* filepath) +std::string get_narrow_name(const std::wstring& name) // LCOV_EXCL_LINE { - OFStream fo(filepath, std::ios_base::out | std::ios::trunc); - TEST(fo); - std::string curValue; - for(int repeat = 0; repeat < 2; repeat++) - { - for(size_t len = 1; len <= 1024; len *= 2) - { - char c = static_cast(len % 13 + repeat + 'a'); // semi-random char - std::string input(len, c); - fo << input; - curValue += input; - TEST(fo.flush()); - std::string s; - // Note: Flush on read area is implementation defined, so check whole file instead - IFStream fi(filepath); - TEST(fi >> s); - // coverity[tainted_data] - TEST(s == curValue); - } - } + return nowide::narrow(name); } -void test_ofstream_creates_file(const char* filename) +template +void test_ctor(const T& filename) { - TEST(!file_exists(filename) || nw::remove(filename) == 0); - TEST(!file_exists(filename)); - // Ctor - { - nw::ofstream fo(filename); - TEST(fo); - } - TEST(file_exists(filename)); - TEST(read_file(filename).empty()); - TEST(nw::remove(filename) == 0); - // Open - { - nw::ofstream fo; - fo.open(filename); - TEST(fo); - } - TEST(file_exists(filename)); - TEST(read_file(filename).empty()); - TEST(nw::remove(filename) == 0); -} + const std::string narrow_filename = get_narrow_name(filename); + remove_file_at_exit _(narrow_filename); -// Create filename file with content "test\n" -void test_ofstream_write(const char* filename) -{ - // char* ctor - { - nw::ofstream fo(filename); - TEST(fo << "test" << 2 << std::endl); - } - // char* open - TEST(read_file(filename) == "test2\n"); - TEST(nw::remove(filename) == 0); - { - nw::ofstream fo; - fo.open(filename); - TEST(fo << "test" << 2 << std::endl); - } - TEST(read_file(filename) == "test2\n"); - TEST(nw::remove(filename) == 0); - // string ctor + // Fail on non-existing file { - std::string name = filename; - nw::ofstream fo(name); - TEST(fo << "test" << 2 << std::endl); + ensure_not_exists(narrow_filename); + nw::fstream f(filename); + TEST(!f); } - TEST(read_file(filename) == "test2\n"); - TEST(nw::remove(filename) == 0); - // string open + TEST(!file_exists(narrow_filename)); + + // Create empty file { - nw::ofstream fo; - fo.open(std::string(filename)); - TEST(fo << "test" << 2 << std::endl); + ensure_not_exists(narrow_filename); + nw::fstream f(filename, std::ios::out); + TEST(f); } - TEST(read_file(filename) == "test2\n"); - TEST(nw::remove(filename) == 0); - // Binary mode + TEST(read_file(narrow_filename).empty()); + + // Read+write existing file + create_file(narrow_filename, "Hello"); { - nw::ofstream fo(filename, std::ios::binary); - TEST(fo << "test" << 2 << std::endl); + nw::fstream f(filename); + TEST(f); + std::string tmp; + TEST(f >> tmp); + TEST(f.eof()); + TEST_EQ(tmp, "Hello"); + f.clear(); + TEST(f << "World"); } - TEST(read_file(filename, true) == "test2\n"); - TEST(nw::remove(filename) == 0); - // At end + TEST_EQ(read_file(narrow_filename), "HelloWorld"); + create_file(narrow_filename, "Hello"); { - { - nw::ofstream fo(filename); - TEST(fo << "test" << 2 << std::endl); - } - nw::ofstream fo(filename, std::ios::ate | std::ios::in); - fo << "second" << 2 << std::endl; + nw::fstream f(filename, std::ios::out | std::ios::in); + TEST(f); + std::string tmp; + TEST(f >> tmp); + TEST(f.eof()); + TEST_EQ(tmp, "Hello"); + f.clear(); + TEST(f << "World"); } - TEST(read_file(filename) == "test2\nsecond2\n"); - TEST(nw::remove(filename) == 0); -} + TEST_EQ(read_file(narrow_filename), "HelloWorld"); -void test_ifstream_open_read(const char* filename) -{ - // Create test file + // Readonly existing file + create_file(narrow_filename, "Hello"); { - nw::ofstream fo(filename); - TEST(fo << "test" << std::endl); + nw::fstream f(filename, std::ios::in); + TEST(f); + std::string tmp; + TEST(f >> tmp); + TEST_EQ(tmp, "Hello"); + f.clear(); + TEST(f); + TEST(!(f << "World")); } + TEST_EQ(read_file(narrow_filename), "Hello"); - // char* Ctor + // Write existing file + create_file(narrow_filename, "Hello"); { - nw::ifstream fi(filename); - TEST(fi); + nw::fstream f(filename, std::ios::out); + TEST(f); std::string tmp; - TEST(fi >> tmp); - TEST(tmp == "test"); + TEST(!(f >> tmp)); + f.clear(); + TEST(f << "World"); } - // char* open + TEST_EQ(read_file(narrow_filename), "World"); + // Write existing file with explicit trunc + create_file(narrow_filename, "Hello"); { - nw::ifstream fi; - fi.open(filename); - TEST(fi); + nw::fstream f(filename, std::ios::out | std::ios::trunc); + TEST(f); std::string tmp; - TEST(fi >> tmp); - TEST(tmp == "test"); + TEST(!(f >> tmp)); + f.clear(); + TEST(f << "World"); } - // string ctor + TEST_EQ(read_file(narrow_filename), "World"); + + // append existing file + create_file(narrow_filename, "Hello"); { - std::string name = filename; - nw::ifstream fi(name); - TEST(fi); - std::string tmp; - TEST(fi >> tmp); - TEST(tmp == "test"); + nw::fstream f(filename, std::ios::app); + TEST(f); + TEST(f << "World"); } - // string open + TEST_EQ(read_file(narrow_filename), "HelloWorld"); + create_file(narrow_filename, "Hello"); { - nw::ifstream fi; - fi.open(std::string(filename)); - TEST(fi); - std::string tmp; - TEST(fi >> tmp); - TEST(tmp == "test"); + nw::fstream f(filename, std::ios::out | std::ios::app); + TEST(f); + TEST(f << "World"); } - // Binary mode + TEST_EQ(read_file(narrow_filename), "HelloWorld"); + + // read+write+truncate existing file + create_file(narrow_filename, "Hello"); { - nw::ifstream fi(filename, std::ios::binary); - TEST(fi); + nw::fstream f(filename, std::ios::out | std::ios::in | std::ios::trunc); + TEST(f); std::string tmp; - TEST(fi >> tmp); - TEST(tmp == "test"); + TEST(!(f >> tmp)); + f.clear(); + TEST(f << "World"); + f.seekg(0); + TEST(f >> tmp); + TEST_EQ(tmp, "World"); } - // At end + TEST_EQ(read_file(narrow_filename), "World"); + + // read+write+append existing file + create_file(narrow_filename, "Hello"); { - // Need binary file or position check might be throw off by newline conversion - { - nw::ofstream fo(filename, nw::fstream::binary); - TEST(fo << "test"); - } - nw::ifstream fi(filename, nw::fstream::ate | nw::fstream::binary); - TEST(fi); - TEST(fi.tellg() == std::streampos(4)); - fi.seekg(-2, std::ios_base::cur); + nw::fstream f(filename, std::ios::out | std::ios::in | std::ios::app); + TEST(f); + TEST(f.seekg(0)); // It is not defined where the read position is after opening + TEST_EQ(f.tellg(), std::streampos(0)); std::string tmp; - TEST(fi >> tmp); - TEST(tmp == "st"); + TEST(f >> tmp); + TEST_EQ(tmp, "Hello"); + f.seekg(0); + TEST(f << "World"); } - // Fail on non-existing file - TEST(nw::remove(filename) == 0); + TEST_EQ(read_file(narrow_filename), "HelloWorld"); + create_file(narrow_filename, "Hello"); { - nw::ifstream fi(filename); - TEST(!fi); + nw::fstream f(filename, std::ios::in | std::ios::app); + TEST(f); + std::string tmp; + TEST(f.seekg(0)); // It is not defined where the read position is after opening + TEST_EQ(f.tellg(), std::streampos(0)); + TEST(f >> tmp); + TEST_EQ(tmp, "Hello"); + f.seekg(0); + TEST(f << "World"); } -} + TEST_EQ(read_file(narrow_filename), "HelloWorld"); -void test_fstream(const char* filename) -{ - const std::string sFilename = filename; - TEST(!file_exists(filename) || nw::remove(filename) == 0); - TEST(!file_exists(filename)); - // Fail on non-existing file + // Write at end + create_file(narrow_filename, "Hello"); { - nw::fstream f(filename); - TEST(!f); - nw::fstream f2(sFilename); - TEST(!f2); + nw::fstream f(filename, std::ios::out | std::ios::in | std::ios::ate); + TEST(f); + std::string tmp; + TEST(!(f >> tmp)); + f.clear(); + TEST(f << "World"); + f.seekg(0); + TEST(f >> tmp); + TEST_EQ(tmp, "HelloWorld"); } + TEST_EQ(read_file(narrow_filename), "HelloWorld"); + + // binary append existing file + create_file(narrow_filename, "Hello"); { - nw::fstream f; - f.open(filename); - TEST(!f); - f.open(sFilename); - TEST(!f); + nw::fstream f(filename, std::ios::binary | std::ios::out | std::ios::app); + TEST(f); + TEST(f << "World"); + TEST(f.seekp(0)); + TEST(f.seekg(0)); + TEST(f << "World\n"); } - TEST(!file_exists(filename)); - // Create empty file (Ctor) + TEST_EQ(read_file(narrow_filename, data_type::binary), "HelloWorldWorld\n"); + create_file(narrow_filename, "Hello"); { - nw::fstream f(filename, std::ios::out); + nw::fstream f(filename, std::ios::binary | std::ios::app); TEST(f); + TEST(f << "World"); + TEST(f.seekp(0)); + TEST(f.seekg(0)); + TEST(f << "World\n"); } - TEST(read_file(filename).empty()); - // Char* ctor + TEST_EQ(read_file(narrow_filename, data_type::binary), "HelloWorldWorld\n"); + + // binary out & trunc + create_file(narrow_filename, "Hello"); { - nw::fstream f(filename); + nw::fstream f(filename, std::ios::binary | std::ios::out | std::ios::trunc); TEST(f); - TEST(f << "test"); - std::string tmp; - TEST(f.seekg(0)); - TEST(f >> tmp); - TEST(tmp == "test"); + TEST(f << "Hello\n"); + TEST(f << "World"); } - TEST(read_file(filename) == "test"); - // String ctor + TEST_EQ(read_file(narrow_filename, data_type::binary), "Hello\nWorld"); + + // Binary in&out + create_file(narrow_filename, "Hello"); { - nw::fstream f(sFilename); + nw::fstream f(filename, std::ios::binary | std::ios::out | std::ios::in); TEST(f); - TEST(f << "string_ctor"); std::string tmp; - TEST(f.seekg(0)); TEST(f >> tmp); - TEST(tmp == "string_ctor"); + TEST(f.eof()); + TEST_EQ(tmp, "Hello"); + f.clear(); + TEST(f << "World\n"); } - TEST(read_file(filename) == "string_ctor"); - TEST(nw::remove(filename) == 0); - // Create empty file (open) + TEST_EQ(read_file(narrow_filename, data_type::binary), "HelloWorld\n"); + + // Trunc & binary + create_file(narrow_filename, "Hello"); { - nw::fstream f; - f.open(filename, std::ios::out); + nw::fstream f(filename, std::ios::binary | std::ios::in | std::ios::out | std::ios::trunc); TEST(f); + TEST(f << "test\n"); + std::string tmp(5, '\0'); + TEST(f.seekg(0)); + TEST(f.read(&tmp[0], 5)); + TEST_EQ(tmp, "test\n"); } - TEST(read_file(filename).empty()); - // Open + TEST_EQ(read_file(narrow_filename, data_type::binary), "test\n"); + + // Binary in&out append + create_file(narrow_filename, "Hello"); { - nw::fstream f; - f.open(filename); + nw::fstream f(filename, std::ios::binary | std::ios::in | std::ios::out | std::ios::app); TEST(f); - TEST(f << "test"); + TEST(f.seekg(0)); // It is not defined where the read position is after opening std::string tmp; - TEST(f.seekg(0)); TEST(f >> tmp); - TEST(tmp == "test"); + TEST(f.eof()); + TEST_EQ(tmp, "Hello"); + f.clear(); + f.seekg(0); + f.seekp(0); + TEST(f << "World\n"); } - TEST(read_file(filename) == "test"); - // Ctor existing file + TEST_EQ(read_file(narrow_filename, data_type::binary), "HelloWorld\n"); + create_file(narrow_filename, "Hello"); { - nw::fstream f(filename); + nw::fstream f(filename, std::ios::binary | std::ios::in | std::ios::app); TEST(f); + TEST(f.seekg(0)); // It is not defined where the read position is after opening std::string tmp; TEST(f >> tmp); - TEST(tmp == "test"); TEST(f.eof()); + TEST_EQ(tmp, "Hello"); f.clear(); - TEST(f << "second"); + f.seekg(0); + f.seekp(0); + TEST(f << "World\n"); } - TEST(read_file(filename) == "testsecond"); - // Trunc & binary + TEST_EQ(read_file(narrow_filename, data_type::binary), "HelloWorld\n"); + + // Invalid modes + const std::initializer_list invalid_modes{ + // clang-format off + std::ios::trunc, + std::ios::trunc | std::ios::app, + std::ios::out | std::ios::trunc | std::ios::app, + std::ios::in | std::ios::trunc, + std::ios::in | std::ios::trunc | std::ios::app, + std::ios::out | std::ios::in | std::ios::trunc | std::ios::app + // clang-format on + }; + for(const auto mode : invalid_modes) + { + create_file(narrow_filename, "Hello"); + { + nw::fstream f(filename, mode); + TEST(!f); + } + TEST_EQ(read_file(narrow_filename), "Hello"); + } +} + +void test_imbue() +{ + nowide::fstream f; +#if NOWIDE_USE_FILEBUF_REPLACEMENT + std::locale convLocale(std::locale::classic(), new dummyCvtConverting); + TEST_THROW(f.imbue(convLocale), std::runtime_error); +#endif + std::locale nonconvLocale(std::locale::classic(), new dummyCvtNonConverting); + f.imbue(nonconvLocale); // No exception, do nothing +} + +template +void test_open(const T& filename) +{ + const std::string narrow_filename = get_narrow_name(filename); + remove_file_at_exit _(narrow_filename); + + // Fail on non-existing file { - nw::fstream f(filename, std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary); + ensure_not_exists(narrow_filename); + nw::fstream f; + f.open(filename); + TEST(!f); + } + TEST(!file_exists(narrow_filename)); + + // Create empty file + { + ensure_not_exists(narrow_filename); + nw::fstream f; + f.open(filename, std::ios::out); TEST(f); - TEST(f << "test2"); - std::string tmp; - TEST(f.seekg(0)); - TEST(f >> tmp); - TEST(tmp == "test2"); } - TEST(read_file(filename) == "test2"); - // Reading in write mode fails (existing file!) + TEST(read_file(narrow_filename).empty()); + + // Read+write existing file + create_file(narrow_filename, "Hello"); { - nw::fstream f(filename, std::ios::out); + nw::fstream f; + f.open(filename); std::string tmp; - TEST(!(f >> tmp)); + TEST(f >> tmp); + TEST(f.eof()); + TEST_EQ(tmp, "Hello"); f.clear(); - TEST(f << "foo"); - TEST(f.seekg(0)); - TEST(!(f >> tmp)); + TEST(f << "World"); } - TEST(read_file(filename) == "foo"); - // Writing in read mode fails (existing file!) + TEST_EQ(read_file(narrow_filename), "HelloWorld"); + + // Readonly existing file + create_file(narrow_filename, "Hello"); { - nw::fstream f(filename, std::ios::in); - TEST(!(f << "bar")); - f.clear(); + nw::fstream f; + f.open(filename, std::ios::in); std::string tmp; TEST(f >> tmp); - TEST(tmp == "foo"); + TEST_EQ(tmp, "Hello"); + f.clear(); + TEST(f); + TEST(!(f << "World")); } - TEST(read_file(filename) == "foo"); - TEST(nw::remove(filename) == 0); + TEST_EQ(read_file(narrow_filename), "Hello"); + + // remaining mode cases skipped as they are already tested by the ctor tests } template bool is_open(T& stream) { // There are const and non const versions of is_open, so test both - TEST(stream.is_open() == const_cast(stream).is_open()); + TEST_EQ(stream.is_open(), const_cast(stream).is_open()); return stream.is_open(); } template -void do_test_is_open(const char* filename) +void do_test_is_open(const std::string& filename) { T f; TEST(!is_open(f)); @@ -479,35 +402,164 @@ void do_test_is_open(const char* filename) f.close(); TEST(f); TEST(!is_open(f)); + // Closing again fails + f.close(); + TEST(!f); } -void test_is_open(const char* filename) +/// Test is_open for all 3 fstream classes +void test_is_open(const std::string& filename) { // Note the order: Output before input so file exists do_test_is_open(filename); + remove_file_at_exit _(filename); do_test_is_open(filename); do_test_is_open(filename); - TEST(nw::remove(filename) == 0); } +void test_move_and_swap(const std::string& filename) +{ + const std::string filename2 = filename + ".2"; + create_file(filename2, "Foo Bar"); + remove_file_at_exit _(filename); + remove_file_at_exit _2(filename2); + + // Move construct + { + nw::fstream f_old(filename, std::ios::out); + TEST(f_old << "Hello "); + + nw::fstream f_new(std::move(f_old)); + // old is closed + TEST(!f_old.is_open()); + // It is unclear if the std streams can be reused after move-from +#if NOWIDE_USE_FILEBUF_REPLACEMENT + f_old.open(filename2, std::ios::in); +#else + f_old = nw::fstream(filename2, std::ios::in); +#endif + std::string s; + TEST(f_old); + TEST(f_old >> s); + TEST_EQ(s, "Foo"); + + // new starts where the old was left of + TEST(f_new); + TEST(f_new << "World"); + } + TEST_EQ(read_file(filename), "Hello World"); + TEST_EQ(read_file(filename2), "Foo Bar"); + + // Move assign + { + nw::fstream f_new(filename2); + TEST(f_new << "ReadThis"); + { + nw::fstream f_old(filename, std::ios::out); + TEST(f_old << "Hello "); + + f_new = std::move(f_old); + // old is closed + TEST(!f_old.is_open()); + // It is unclear if the std streams can be reused after move-from +#if NOWIDE_USE_FILEBUF_REPLACEMENT + f_old.open(filename2, std::ios::in); +#else + f_old = nw::fstream(filename2, std::ios::in); +#endif + std::string s; + TEST(f_old >> s); + TEST_EQ(s, "ReadThis"); + } + // new starts where the old was left of + TEST(f_new); + TEST(f_new << "World"); + } + TEST_EQ(read_file(filename), "Hello World"); + TEST_EQ(read_file(filename2), "ReadThis"); + + create_file(filename2, "Foo Bar"); + // Swap + { + nw::fstream f_old(filename, std::ios::out); + TEST(f_old << "Hello "); + + nw::fstream f_new(filename2, std::ios::in); + std::string s; + TEST(f_new >> s); + TEST_EQ(s, "Foo"); + + // After swapping both are valid and where they left + f_new.swap(f_old); + TEST(f_old >> s); + TEST_EQ(s, "Bar"); + TEST(f_new << "World"); + + f_new.close(); + swap(f_new, f_old); + TEST(!f_old.is_open()); + TEST(f_new.is_open()); + } + TEST_EQ(read_file(filename), "Hello World"); + TEST_EQ(read_file(filename2), "Foo Bar"); +} + +void test_flush(const std::string& filepath) +{ + remove_file_at_exit _(filepath); + nw::fstream fo(filepath, std::ios_base::out | std::ios::trunc); + TEST(fo); + std::string curValue; + for(int repeat = 0; repeat < 2; repeat++) + { + for(size_t len = 1; len <= 1024; len *= 2) + { + char c = static_cast(len % 13 + repeat + 'a'); // semi-random char + std::string input(len, c); + fo << input; + curValue += input; + TEST(fo.flush()); + std::string s; + // Note: Flush on read area is implementation defined, so check whole file instead + nw::fstream fi(filepath, std::ios_base::in); + TEST(fi >> s); + // coverity[tainted_data] + TEST_EQ(s, curValue); + } + } + fo.close(); + TEST(fo.flush()); // Should also work on closed stream + TEST(!fo.seekg(0)); // Does not work on closed stream +} + +// coverity[root_function] void test_main(int, char** argv, char**) { const std::string exampleFilename = std::string(argv[0]) + "-\xd7\xa9-\xd0\xbc-\xce\xbd.txt"; - std::cout << "Testing fstream" << std::endl; - test_ofstream_creates_file(exampleFilename.c_str()); - test_ofstream_write(exampleFilename.c_str()); - test_ifstream_open_read(exampleFilename.c_str()); - test_fstream(exampleFilename.c_str()); - test_is_open(exampleFilename.c_str()); - - std::cout << "Complex IO" << std::endl; - test_with_different_buffer_sizes(exampleFilename.c_str()); - - std::cout << "filebuf::close" << std::endl; - test_close(exampleFilename.c_str()); - - std::cout << "Flush - Sanity Check" << std::endl; - test_flush(exampleFilename.c_str()); - std::cout << "Flush - Test" << std::endl; - test_flush(exampleFilename.c_str()); + + std::cout << "Ctor" << std::endl; + test_ctor(exampleFilename.c_str()); + test_ctor(exampleFilename); +#if NOWIDE_USE_WCHAR_OVERLOADS + test_ctor(nowide::widen(exampleFilename).c_str()); +#endif + + std::cout << "Open" << std::endl; + test_open(exampleFilename.c_str()); + test_open(exampleFilename); +#if NOWIDE_USE_WCHAR_OVERLOADS + test_open(nowide::widen(exampleFilename).c_str()); +#endif + + std::cout << "IsOpen" << std::endl; + test_is_open(exampleFilename); + + std::cout << "imbue" << std::endl; + test_imbue(); + + std::cout << "Move and swap" << std::endl; + test_move_and_swap(exampleFilename); + + std::cout << "Flush" << std::endl; + test_flush(exampleFilename); } diff --git a/extern/boost/nowide/test/test_fstream_special.cpp b/extern/boost/nowide/test/test_fstream_special.cpp new file mode 100644 index 0000000000..7f63ec7351 --- /dev/null +++ b/extern/boost/nowide/test/test_fstream_special.cpp @@ -0,0 +1,286 @@ +// Copyright (c) 2015 Artyom Beilis (Tonkikh) +// Copyright (c) 2019-2021 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include + +#include +#include +#include "file_test_helpers.hpp" +#include "test.hpp" +#include +#include +#include + +namespace nw = nowide; + +using namespace nowide::test; + +void test_with_different_buffer_sizes(const char* filepath) +{ + /* Important part of the standard for mixing input with output: + However, output shall not be directly followed by input without an intervening call to the fflush function + or to a file positioning function (fseek, fsetpos, or rewind), + and input shall not be directly followed by output without an intervening call to a file positioning function, + unless the input operation encounters end-of-file. + */ + for(int i = -1; i < 16; i++) + { + remove_file_at_exit _(filepath); + + std::cout << "Buffer size = " << i << std::endl; + char buf[16]; + nw::fstream f; + // Different conditions when setbuf might be called: Usually before opening a file is OK + if(i >= 0) + f.rdbuf()->pubsetbuf((i == 0) ? nullptr : buf, i); + f.open(filepath, std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary); + TEST(f); + + // Add 'abcdefg' + TEST(f.put('a')); + TEST(f.put('b')); + TEST(f.put('c')); + TEST(f.write("defg", 4)); + // Read first char + TEST(f.seekg(0)); + TEST_EQ(f.get(), 'a'); + TEST_EQ(f.gcount(), std::streamsize(1)); + // Skip next char + TEST(f.seekg(1, std::ios::cur)); + TEST_EQ(f.get(), 'c'); + TEST_EQ(f.gcount(), std::streamsize(1)); + // Go back 1 char + TEST(f.seekg(-1, std::ios::cur)); + TEST_EQ(f.get(), 'c'); + TEST_EQ(f.gcount(), std::streamsize(1)); + + // Test switching between read->write->read + // case 1) overwrite, flush, read + TEST(f.seekg(1)); + TEST(f.put('B')); + TEST(f.flush()); // Flush when changing out->in + TEST_EQ(f.get(), 'c'); + TEST_EQ(f.gcount(), std::streamsize(1)); + TEST(f.seekg(1)); + TEST_EQ(f.get(), 'B'); + TEST_EQ(f.gcount(), std::streamsize(1)); + // case 2) overwrite, seek, read + TEST(f.seekg(2)); + TEST(f.put('C')); + TEST(f.seekg(3)); // Seek when changing out->in + TEST_EQ(f.get(), 'd'); + TEST_EQ(f.gcount(), std::streamsize(1)); + + // Check that sequence from start equals expected + TEST(f.seekg(0)); + TEST_EQ(f.get(), 'a'); + TEST_EQ(f.get(), 'B'); + TEST_EQ(f.get(), 'C'); + TEST_EQ(f.get(), 'd'); + TEST_EQ(f.get(), 'e'); + + // Putback after flush is implementation defined + TEST(f << std::flush); + if(f.putback('e')) + { + if(f.putback('d')) + TEST_EQ(f.get(), 'd'); + else + f.clear(); // LCOV_EXCL_LINE + TEST_EQ(f.get(), 'e'); + } else + f.clear(); + TEST(f << std::flush); + if(f.unget()) + TEST_EQ(f.get(), 'e'); + else + f.clear(); + + // Put back different char + TEST(f.seekg(-1, std::ios::cur)); + TEST_EQ(f.get(), 'e'); + TEST(f.putback('x')); + TEST_EQ(f.get(), 'x'); + // Rest of sequence + TEST_EQ(f.get(), 'f'); + TEST_EQ(f.get(), 'g'); + TEST_EQ(f.get(), EOF); + + // Put back until front of file is reached + f.clear(); + TEST(f.seekg(1)); + TEST_EQ(f.get(), 'B'); + TEST(f.putback('B')); + // Putting back multiple chars is not possible on all implementations after a seek/flush +#if NOWIDE_USE_FILEBUF_REPLACEMENT + if(f.putback('a')) + { + // At beginning of file -> No putback possible + TEST(!f.putback('x')); // LCOV_EXCL_LINE + f.clear(); // LCOV_EXCL_LINE + // Get characters that were putback to avoid MSVC bug https://github.com/microsoft/STL/issues/342 + TEST_EQ(f.get(), 'a'); // LCOV_EXCL_LINE + } else + f.clear(); +#endif + TEST_EQ(f.get(), 'B'); + f.close(); + } +} + +void test_switch_to_custom_buffer(const std::string& filename) +{ + // Switching the buffer after file stream was used is not always defined. So only test custom stream +#if NOWIDE_USE_FILEBUF_REPLACEMENT + nw::test::create_file(filename, "HelloWorld"); + nw::ifstream f(filename, std::ios::binary); + std::string s(5, '\0'); + TEST(f.read(&s.front(), s.size())); + TEST_EQ(s, "Hello"); + // Switch buffer + std::string buffer(10, '\0'); + TEST_EQ(f.sync(), 0); + TEST(f.rdbuf()->pubsetbuf(&buffer.front(), buffer.size()) == f.rdbuf()); + TEST(f >> s); + TEST_EQ(s, "World"); + TEST_EQ(s, buffer.c_str()); // same should be in buffer and some trailing NULL bytes +#else + (void)filename; // Suppress unused warning +#endif +} + +// Reproducer for https://github.com/boostorg/nowide/issues/126 +void test_getline_and_tellg(const char* filename) +{ + { + nw::ofstream f(filename); + f << "Line 1" << std::endl; + f << "Line 2" << std::endl; + f << "Line 3" << std::endl; + } + remove_file_at_exit _(filename); + nw::fstream f; + // Open file in text mode, to read + f.open(filename, std::ios_base::in); + TEST(f); + std::string line1, line2, line3; + TEST(getline(f, line1)); + TEST_EQ(line1, "Line 1"); + const auto tg = f.tellg(); // This may cause issues + TEST(tg > 0u); + TEST(getline(f, line2)); + TEST_EQ(line2, "Line 2"); + TEST(getline(f, line3)); + TEST_EQ(line3, "Line 3"); +} + +// Test that a sync after a peek does not swallow newlines +// This can happen because peek reads a char which needs to be "unread" on sync which may loose a converted newline +void test_peek_sync_get(const char* filename) +{ + { + nw::ofstream f(filename); + f << "Line 1" << std::endl; + f << "Line 2" << std::endl; + } + remove_file_at_exit _(filename); + nw::ifstream f(filename); + TEST(f); + while(f) + { + const int curChar = f.peek(); + if(curChar == std::char_traits::eof()) + break; + f.sync(); + TEST_EQ(f.get(), char(curChar)); + } +} + +/// Test swapping at many possible positions within a stream to shake out missed state +void test_swap(const char* filename, const char* filename2) +{ + remove_file_at_exit _(filename); + remove_file_at_exit _2(filename2); + + { + nw::ofstream f(filename); + f << create_random_data(BUFSIZ * 2, data_type::text); + f.close(); + f.open(filename2); + f << create_random_data(BUFSIZ * 3, data_type::text); + } + + nw::ifstream f1(filename); + nw::ifstream f2(filename2); + TEST(f1); + TEST(f2); + unsigned ctr = 0; + while(f1 && f2) + { + const int curChar1 = f1.peek(); + const int curChar2 = f2.peek(); + TEST_CONTEXT("ctr " << ctr << ": c1=" << curChar1 << " c2=" << curChar2); + // Randomly do a no-op seek of either or both streams to flush internal buffer + if(ctr % 10 == 0) + TEST(f1.seekg(f1.tellg())); + else if(ctr % 15 == 0) + TEST(f2.seekg(f2.tellg())); + f1.swap(f2); + TEST_EQ(f1.peek(), curChar2); + TEST_EQ(f2.peek(), curChar1); + if(ctr % 10 == 4) + TEST(f1.seekg(f1.tellg())); + else if(ctr % 15 == 4) + TEST(f2.seekg(f2.tellg())); + TEST_EQ(f1.get(), curChar2); + f1.swap(f2); + TEST_EQ(f1.get(), curChar1); + ++ctr; + } +} + +void testPutback(const char* filename) +{ + nw::test::create_file(filename, "abc"); + // Does work for ifstreams + { + nw::ifstream f(filename); + const int c = f.get(); + TEST(f.putback(static_cast(c))); + TEST_EQ(f.get(), c); + } + // Does work for io fstreams + { + nw::fstream f(filename); + const int c = f.get(); + TEST(f.putback(static_cast(c))); + TEST_EQ(f.get(), c); + } + // Doesn't work for output fstreams + { + nw::fstream f(filename, std::ios::out); + TEST(!f.putback('x')); + } +} + +// coverity[root_function] +void test_main(int, char** argv, char**) +{ + const std::string exampleFilename = std::string(argv[0]) + "-\xd7\xa9-\xd0\xbc-\xce\xbd.txt"; + const std::string exampleFilename2 = std::string(argv[0]) + "-\xd7\xa9-\xd0\xbc-\xce\xbd 2.txt"; + + std::cout << "Putback" << std::endl; + testPutback(exampleFilename.c_str()); + + std::cout << "Complex IO" << std::endl; + test_with_different_buffer_sizes(exampleFilename.c_str()); + test_switch_to_custom_buffer(exampleFilename.c_str()); + + std::cout << "Regression tests" << std::endl; + test_getline_and_tellg(exampleFilename.c_str()); + test_peek_sync_get(exampleFilename.c_str()); + test_swap(exampleFilename.c_str(), exampleFilename2.c_str()); +} diff --git a/extern/boost/nowide/test/test_ifstream.cpp b/extern/boost/nowide/test/test_ifstream.cpp new file mode 100644 index 0000000000..fece2e1ca3 --- /dev/null +++ b/extern/boost/nowide/test/test_ifstream.cpp @@ -0,0 +1,211 @@ +// Copyright (c) 2015 Artyom Beilis (Tonkikh) +// Copyright (c) 2019-2021 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include + +#include "file_test_helpers.hpp" +#include "test.hpp" +#include + +namespace nw = nowide; +using namespace nowide::test; + +template +void test_ctor(const T& filename) +{ + // Fail on non-existing file + ensure_not_exists(filename); + { + nw::ifstream f(filename); + TEST(!f); + } + TEST(!file_exists(filename)); + + create_file(filename, "test"); + + // Default + { + nw::ifstream f(filename); + TEST(f); + std::string tmp; + TEST(f >> tmp); + TEST_EQ(tmp, "test"); + } + TEST_EQ(read_file(filename), "test"); + + // At end + { + nw::ifstream f(filename, std::ios::ate); + TEST(f); + std::string tmp; + TEST(!(f >> tmp)); + TEST(f.eof()); + f.clear(); + f.seekg(0, std::ios::beg); + TEST(f >> tmp); + TEST_EQ(tmp, "test"); + } + TEST_EQ(read_file(filename), "test"); + + create_file(filename, "test\r\n", data_type::binary); + // Binary mode + { + nw::ifstream f(filename, std::ios::binary); + TEST(f); + std::string tmp(6, '\0'); + TEST(f.read(&tmp[0], 6)); + TEST_EQ(tmp, "test\r\n"); + } +} + +template +void test_open(const T& filename) +{ + // Fail on non-existing file + ensure_not_exists(filename); + { + nw::ifstream f; + f.open(filename); + TEST(!f); + } + TEST(!file_exists(filename)); + + create_file(filename, "test"); + + // Default + { + nw::ifstream f; + f.open(filename); + TEST(f); + std::string tmp; + TEST(f >> tmp); + TEST_EQ(tmp, "test"); + } + TEST_EQ(read_file(filename), "test"); + + // At end + { + nw::ifstream f; + f.open(filename, std::ios::ate); + TEST(f); + std::string tmp; + TEST(!(f >> tmp)); + TEST(f.eof()); + f.clear(); + f.seekg(0, std::ios::beg); + TEST(f >> tmp); + TEST_EQ(tmp, "test"); + } + TEST_EQ(read_file(filename), "test"); + + create_file(filename, "test\r\n", data_type::binary); + // Binary mode + { + nw::ifstream f; + f.open(filename, std::ios::binary); + TEST(f); + std::string tmp(6, '\0'); + TEST(f.read(&tmp[0], 6)); + TEST_EQ(tmp, "test\r\n"); + } +} + +void test_move_and_swap(const std::string& filename) +{ + const std::string filename2 = filename + ".2"; + create_file(filename, "Hello\nWorld"); + create_file(filename2, "Foo\nBar"); + remove_file_at_exit _(filename); + remove_file_at_exit _2(filename2); + + // Move construct + { + nw::ifstream f_old(filename); + std::string s; + TEST(f_old >> s && s == "Hello"); + + nw::ifstream f_new(std::move(f_old)); + // old is closed + TEST(!f_old.is_open()); + // It is unclear if the std streams can be reused after move-from +#if NOWIDE_USE_FILEBUF_REPLACEMENT + f_old.open(filename2); +#else + f_old = nw::ifstream(filename2); +#endif + TEST(f_old); + TEST(f_old >> s); + TEST_EQ(s, "Foo"); + TEST(f_old >> s && s == "Bar"); + + // new starts where the old was left of + TEST(f_new); + TEST(f_new >> s); + TEST_EQ(s, "World"); + } + // Move assign + { + nw::ifstream f_new(filename2); + std::string s; + TEST(f_new >> s && s == "Foo"); + { + nw::ifstream f_old(filename); + TEST(f_old >> s && s == "Hello"); + + f_new = std::move(f_old); + // old is closed + TEST(!f_old.is_open()); + // It is unclear if the std streams can be reused after move-from +#if NOWIDE_USE_FILEBUF_REPLACEMENT + f_old.open(filename2); +#else + f_old = nw::ifstream(filename2); +#endif + TEST(f_old); + TEST(f_old >> s); + TEST_EQ(s, "Foo"); + TEST(f_old >> s && s == "Bar"); + } + // new starts where the old was left of + TEST(f_new); + TEST(f_new >> s); + TEST_EQ(s, "World"); + } + // Swap + { + nw::ifstream f_old(filename); + std::string s; + TEST(f_old >> s && s == "Hello"); + + nw::ifstream f_new(filename2); + TEST(f_new >> s && s == "Foo"); + + // After swapping both are valid and where they left + f_new.swap(f_old); + TEST(f_old >> s); + TEST_EQ(s, "Bar"); + + TEST(f_new >> s); + TEST_EQ(s, "World"); + + f_new.close(); + swap(f_new, f_old); + TEST(!f_old.is_open()); + TEST(f_new.is_open()); + } +} + +// coverity[root_function] +void test_main(int, char** argv, char**) +{ + const std::string exampleFilename = std::string(argv[0]) + "-\xd7\xa9-\xd0\xbc-\xce\xbd.txt"; + + test_ctor(exampleFilename.c_str()); + test_ctor(exampleFilename); + test_open(exampleFilename.c_str()); + test_open(exampleFilename); + test_move_and_swap(exampleFilename); +} diff --git a/extern/boost/nowide/test/test_iostream.cpp b/extern/boost/nowide/test/test_iostream.cpp index 12f0b907da..53aa014482 100644 --- a/extern/boost/nowide/test/test_iostream.cpp +++ b/extern/boost/nowide/test/test_iostream.cpp @@ -1,22 +1,118 @@ +// Copyright (c) 2015 Artyom Beilis (Tonkikh) +// Copyright (c) 2020 - 2021 Alexander Grund // -// Copyright (c) 2015 Artyom Beilis (Tonkikh) -// -// Distributed under the Boost Software License, Version 1.0. (See -// accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) -// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt +#ifndef _SCL_SECURE_NO_WARNINGS +// Call to 'std::copy_n' with parameters that may be unsafe +#define _SCL_SECURE_NO_WARNINGS +#endif #include -#include +#include +#include +#include "../src/console_buffer.hpp" +#include "file_test_helpers.hpp" +#include "test.hpp" +#include "test_sets.hpp" +#include +#include #include +#include #include -#include "test.hpp" +namespace nw = nowide; + +const std::string outputString = + // German umlauts (aou with 2 dots), cyrillic small m, greek small nu + "Basic letters: \xc3\xa4-\xc3\xb6-\xc3\xbc-\xd0\xbc-\xce\xbd\n" + "East Asian Letters: \xe5\x92\x8c-\xe5\xb9\xb3\n" + "Non-BMP letter: \xf0\x9d\x84\x9e\n" // musical symbol g clef + "Invalid UTF-8: `\xFF' `\xd7\xFF' `\xe5\xFF\x8c' `\xf0\x9d\x84\xFF' \n" + "\n"; + +const bool usesNowideRdBufIn = nw::cin.rdbuf() != std::cin.rdbuf(); +const bool usesNowideRdBufOut = nw::cout.rdbuf() != std::cout.rdbuf(); + +bool is_buffered(const std::ostream& os) +{ + return (os.flags() & std::ios_base::unitbuf) == 0; +} + +#ifndef NOWIDE_TEST_INTERACTIVE +class mock_output_buffer final : public nw::detail::console_output_buffer_base +{ +public: + std::wstring output; + bool succeed = true; + +protected: + bool do_write(const wchar_t* buffer, std::size_t num_chars_to_write, std::size_t& num_chars_written) override + { + if(succeed) + { + output.insert(output.end(), buffer, buffer + num_chars_to_write); + num_chars_written = num_chars_to_write; + return true; + } else + { + num_chars_written = 0; + return false; + } + } +}; -bool isValidUTF8(const std::string& s) +class mock_input_buffer final : public nw::detail::console_input_buffer_base { - using namespace nowide::detail::utf; +public: + std::queue inputs; + +protected: + bool do_read(wchar_t* buffer, std::size_t num_chars_to_read, std::size_t& num_chars_read) override + { + if(inputs.empty()) + return false; + std::wstring& input = inputs.front(); + num_chars_read = std::min(num_chars_to_read, input.size()); + std::copy_n(input.begin(), num_chars_read, buffer); + input.erase(input.begin(), input.begin() + num_chars_read); + if(input.empty()) + inputs.pop(); + return true; + } +}; + +/// Scoped change of a streams rdbuf +struct scoped_rdbuf_change +{ + std::ios& stream; + std::streambuf* orig_buf; + scoped_rdbuf_change(std::ios& stream, std::streambuf* new_buf) : stream(stream), orig_buf(stream.rdbuf(new_buf)) + {} + ~scoped_rdbuf_change() + { + stream.rdbuf(orig_buf); + } +}; + +// Macros to be used to avoid littering the code with #ifndef checks +/// Install a mock buffer into the given stream, when compiling as non-interactive +#define INSTALL_MOCK_BUF(STREAM, BUF_TYPE) \ + BUF_TYPE mock_buf; \ + scoped_rdbuf_change _(nw::STREAM, &mock_buf) +/// Run the given cmd(s) only when compiling as non-interactive +#define RUN_MOCKED(what) what +#else +#define INSTALL_MOCK_BUF(STREAM, BUF_TYPE) +#define RUN_MOCKED(what) +#endif +/// Assert the given condition/code only when compiling as non-interactive +#define TEST_MOCKED(what) RUN_MOCKED(TEST(what)) + +bool is_valid_UTF8(const std::string& s) +{ + using namespace nowide::utf; for(std::string::const_iterator it = s.begin(); it != s.end();) { code_point c = utf_traits::decode(it, s.end()); @@ -26,81 +122,488 @@ bool isValidUTF8(const std::string& s) return true; } -void test_main(int argc, char** argv, char**) +std::string create_random_one_line_string(std::size_t num_chars) { - const char* example = "Basic letters: \xd7\xa9-\xd0\xbc-\xce\xbd\n" - "East Asian Letters: \xe5\x92\x8c\xe5\xb9\xb3\n" - "Non-BMP letters: \xf0\x9d\x84\x9e\n" - "Invalid UTF-8: `\xFF' `\xd7\xFF' `\xe5\xFF\x8c' `\xf0\x9d\x84\xFF' \n" - "\n"; + std::string result = nw::test::create_random_data(num_chars, nowide::test::data_type::text); + // Make sure it is a single line + std::replace(result.begin(), result.end(), '\n', 'a'); + return result; +} + +void test_is_valid_UTF8() +{ + // Sanity check of the test function + TEST(is_valid_UTF8("")); // Empty string is valid by definition + TEST(is_valid_UTF8(create_random_one_line_string(100))); // ASCII string is valid + TEST(is_valid_UTF8(roundtrip_tests[5].utf8)); // UTF-8 string is valid + TEST(!is_valid_UTF8(invalid_utf8_tests[0].utf8)); // Detect invalid +} +void test_tie_and_buffered() +{ + TEST(nw::cin.tie() == &nw::cout); + TEST(nw::cerr.tie() == &nw::cout); + TEST(nw::clog.tie() == nullptr); + TEST(is_buffered(nw::cout)); + TEST(!is_buffered(nw::cerr)); + TEST(is_buffered(nw::clog)); +} + +void test_putback_and_get() +{ // If we are using the standard rdbuf we can only put back 1 char - if(nowide::cin.rdbuf() == std::cin.rdbuf()) + // This should always work + int maxval = 15000; + for(int i = 0; i < maxval; i++) + { + char c = i % 96 + ' '; + TEST(nw::cin.putback(c)); + int ci = i % 96 + ' '; + TEST_EQ(nw::cin.get(), ci); + } + + INSTALL_MOCK_BUF(cin, mock_input_buffer); + if(usesNowideRdBufIn RUN_MOCKED(|| true)) + { + // Test with a few small values and around power-of-2 values as buffer size doubles. + // Finally test a large value + for(const int num_putback_chars : {1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 1000}) + { + const auto getChar = [&](int i) { return (i + num_putback_chars) % 96 + ' '; }; + for(int i = 0; i < num_putback_chars; i++) + { + const char c = static_cast(getChar(i)); + TEST(nw::cin.putback(c)); + } + for(int i = num_putback_chars - 1; i >= 0; i--) + { + const int c = getChar(i); + TEST_EQ(nw::cin.get(), c); + } + // Check unget (all chars) + for(int i = 0; i < num_putback_chars; i++) + TEST(nw::cin.unget()); + TEST(!nw::cin.unget()); + nw::cin.clear(); + for(int i = num_putback_chars - 1; i >= 0; i--) + { + const int c = getChar(i); + TEST_EQ(nw::cin.get(), c); + } + } +#ifndef NOWIDE_TEST_INTERACTIVE + // Put back 1 char, then get the rest from "real" input + nw::cin.putback('T'); + mock_buf.inputs.push(L"est\r\n"); + std::string test; + TEST(nw::cin >> test); + TEST_EQ(test, "Test"); +#endif + } +} + +void test_cout() +{ + INSTALL_MOCK_BUF(cout, mock_output_buffer); + TEST(nw::cout); + TEST(nw::cout << outputString << std::endl); + TEST_MOCKED(mock_buf.output == nw::widen(outputString + "\n")); +#ifndef NOWIDE_TEST_INTERACTIVE + // Pretend the actual write to console fails + mock_buf.output.clear(); + mock_buf.succeed = false; + TEST(!(nw::cout << "Fail this" << std::endl)); + TEST(mock_buf.output.empty()); +#endif +} + +void test_cout_single_char() +{ + INSTALL_MOCK_BUF(cout, mock_output_buffer); + for(const char s : outputString) + TEST(nw::cout << s << std::flush); + TEST(nw::cout); + TEST_MOCKED(mock_buf.output == nw::widen(outputString)); +} + +void test_cerr() +{ + INSTALL_MOCK_BUF(cerr, mock_output_buffer); + TEST(nw::cerr); + TEST(nw::cerr << outputString << std::endl); + TEST_MOCKED(mock_buf.output == nw::widen(outputString + "\n")); + // Output to cerr is unbuffered and should be flushed for every single output + RUN_MOCKED(mock_buf.output.clear()); + TEST_MOCKED(nw::cerr << "a"); + TEST_MOCKED(mock_buf.output == nw::widen("a")); + TEST_MOCKED(nw::cerr << "Hello World"); + TEST_MOCKED(mock_buf.output == nw::widen("aHello World")); +} + +void test_clog() +{ + if(usesNowideRdBufOut) // Only executed when attached to a real terminal, i.e. not on CI + { + TEST(nw::clog.rdbuf() != std::clog.rdbuf()); // LCOV_EXCL_LINE + // for the std:: streams this is not true for all implementations, so only check when using custom buffers + TEST(nw::clog.rdbuf() != nw::cerr.rdbuf()); // LCOV_EXCL_LINE + } + + TEST(nw::clog.rdbuf() != nw::cin.rdbuf()); + TEST(nw::clog.rdbuf() != nw::cout.rdbuf()); + TEST(nw::clog.rdbuf() != std::cout.rdbuf()); +} + +void test_cerr_single_char() +{ + INSTALL_MOCK_BUF(cerr, mock_output_buffer); + for(const char s : outputString) + TEST(nw::cerr << s << std::flush); + TEST(nw::cerr); + TEST_MOCKED(mock_buf.output == nw::widen(outputString)); +} + +void test_cin() +{ + RUN_MOCKED(const std::wstring lineBreak = L"\r\n"; const std::wstring space = L" "); + INSTALL_MOCK_BUF(cin, mock_input_buffer); + nw::cout << "Input 2 strings, e.g. 'Hello World'" << std::endl; + static_assert(array_size(roundtrip_tests) > 7, "!"); + RUN_MOCKED(mock_buf.inputs.push(roundtrip_tests[6].wide + space + roundtrip_tests[7].wide + lineBreak)); + std::string v1, v2; + nw::cin >> v1 >> v2; + TEST(nw::cin); + TEST(is_valid_UTF8(v1)); + TEST(is_valid_UTF8(v2)); + TEST(nw::cout << "First: " << v1 << std::endl); + TEST(nw::cout << "Second: " << v2 << std::endl); + TEST_MOCKED(v1 == roundtrip_tests[6].utf8); + TEST_MOCKED(v2 == roundtrip_tests[7].utf8); + + // Check sync + nw::cout << "Input 2 strings, e.g. 'Two more'" << std::endl; + RUN_MOCKED(mock_buf.inputs.push(L"First_String\u00F1" + space + L"Second_String_Ignored" + lineBreak)); + TEST(nw::cin >> v1); + nw::cin.sync(); + nw::cout << "The 2nd string should have been ignored. Input 1 more + [ENTER]" << std::endl; + RUN_MOCKED(mock_buf.inputs.push(L"Third_\xDC01_String" + lineBreak)); // Note: Invalid UTF-16 + TEST(nw::cin >> v2); + TEST(!v2.empty()); + nw::cout << "First: " << v1 << std::endl; + nw::cout << "Second: " << v2 << std::endl; + TEST_MOCKED(v1 == "First_String\xc3\xb1"); + TEST_MOCKED(v2 == "Third_\xEF\xBF\xBD_String"); +} + +void test_cin_getline() +{ + INSTALL_MOCK_BUF(cin, mock_input_buffer); + std::string value; + for(int i = 0; i < 10; i++) { - std::cout << "Using std::cin" << std::endl; - int maxval = 15000; - for(int i = 0; i < maxval; i++) + nw::cout << "Input a line of text or simply press ENTER to exit, e.g. 'Hello World to you!'" + << std::endl; + // Add a longish string which eventually exceeds the buffer size of the console_buffer + RUN_MOCKED(const std::string expected = (i == 9) ? "" : create_random_one_line_string(i * 211 + 13)); + // Convert to wstring and push (we can do this as the chars are ASCII) + RUN_MOCKED(mock_buf.inputs.push(std::wstring(expected.begin(), expected.end()) + L"\r\n")); + TEST(std::getline(nw::cin, value)); + if(value.empty()) { - char c = i % 96 + ' '; - TEST(nowide::cin.putback(c)); - int ci = i % 96 + ' '; - TEST(nowide::cin.get() == ci); + TEST_MOCKED(i == 9); + nw::cout << "END\n"; + break; } - } else + TEST_MOCKED(i != 9); + // It should not include the CR + TEST(value.back() != '\r'); + nw::cout << i << ": " << value << std::endl; + TEST_MOCKED(value == expected); + } +#ifndef NOWIDE_TEST_INTERACTIVE + TEST(!std::getline(nw::cin, value)); + TEST(nw::cin.eof()); + TEST(value.empty()); + nw::cin.clear(); + // Check that incomplete chars at the buffer border are handled correctly by creating a very long string + std::wstring input; + constexpr std::size_t buffer_size = 1024; // From console_buffer_base + input.reserve(buffer_size * 4); + input.append(buffer_size - 2, L'a'); // 2 chars before end of buffer + input.append(L"\U0001033C"); // 2 UTF-16 chars --> Buffer full + input.append(buffer_size - 1, L'a'); // Just before end of buffer + input.append(L"\U0001033C"); // over buffer boundary + input.append(L"\U0001033Ca"); // in new buffer and align again + // Fill up with largest code point + for(std::size_t i = 0; i < buffer_size; i++) + input.append(L"\U0010FFFF"); + mock_buf.inputs.push(input + L"\r\n"); + TEST(std::getline(nw::cin, value)); + TEST_EQ(value, nw::narrow(input)); +#endif +} + +void test_ctrl_z_is_eof() +{ + INSTALL_MOCK_BUF(cin, mock_input_buffer); + std::string value; + nw::cout << "Input a line of text and then press CTRL+Z, e.g. 'Hello World!'" << std::endl; + RUN_MOCKED(mock_buf.inputs.push(L"Hello World!\r\n")); + RUN_MOCKED(mock_buf.inputs.push(L"\x1a\r\n")); + RUN_MOCKED(mock_buf.inputs.push(L"Reached after clear()\r\n")); + TEST(std::getline(nw::cin, value)); + // It should not include the CR + TEST(!value.empty()); + TEST(value.back() != '\r'); + nw::cout << "Value: " << value << std::endl; + TEST_MOCKED(value == "Hello World!"); + TEST(!std::getline(nw::cin, value)); + TEST(nw::cin.eof()); + nw::cin.clear(); + nw::cout << "clear() called, input another line, e.g. 'Hi there!'" << std::endl; + TEST(std::getline(nw::cin, value)); + // It should not include the CR + TEST(!value.empty()); + TEST(value.back() != '\r'); + nw::cout << "Value: " << value << std::endl; + TEST_MOCKED(value == "Reached after clear()"); +#ifndef NOWIDE_TEST_INTERACTIVE + // CTRL+Z anywhere else but at the start of a line does not matter + nw::cout << "CTRL+Z Test:"; + for(int i = 1; i <= 1100; i++) { - int maxval = 15000; - for(int i = 0; i < maxval; i++) + nw::cout << '.' << std::flush; // Progress indicator + const std::string expected = create_random_one_line_string(i) + "\x1a"; + mock_buf.inputs.push(std::wstring(expected.begin(), expected.end()) + L"\r\n"); + TEST(std::getline(nw::cin, value)); + TEST_EQ(value, expected); + } + nw::cout << std::endl; +#endif +} + +#ifndef NOWIDE_TEST_INTERACTIVE +#ifdef NOWIDE_WINDOWS +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +/// Test class swapping the original in/out handles for a buffer which +/// can be filled (for stdin) or read (for stdout/stderr) +class RedirectStdio +{ + DWORD handleType; + HANDLE h, oldHandle; + +public: + RedirectStdio(DWORD handleType) : handleType(handleType), oldHandle(GetStdHandle(handleType)) + { + if(handleType == STD_INPUT_HANDLE) + { + h = CreateFile("CONIN$", + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, + OPEN_EXISTING, + 0, + 0); + } else { - char c = i % 96 + ' '; - TEST(nowide::cin.putback(c)); + h = CreateConsoleScreenBuffer(GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, + CONSOLE_TEXTMODE_BUFFER, + nullptr); } - for(int i = maxval - 1; i >= 0; i--) + TEST(h != INVALID_HANDLE_VALUE); + TEST(SetStdHandle(handleType, h)); + if(handleType == STD_INPUT_HANDLE) + TEST(SetConsoleMode(h, ENABLE_PROCESSED_INPUT | ENABLE_LINE_INPUT | ENABLE_EXTENDED_FLAGS)); + else + TEST(SetConsoleActiveScreenBuffer(h)); + } + ~RedirectStdio() + { + SetStdHandle(handleType, oldHandle); + if(handleType != STD_INPUT_HANDLE) + SetConsoleActiveScreenBuffer(oldHandle); + CloseHandle(h); + } + + std::wstring getBufferData() + { + CONSOLE_SCREEN_BUFFER_INFO info; + TEST(GetConsoleScreenBufferInfo(h, &info)); + TEST(info.dwSize.X > 0 && info.dwSize.Y > 0); + std::cout << "Mock console buffer size: " << info.dwSize.X << "x" << info.dwSize.Y << "\n"; + + std::wstring result; + std::vector buffer(info.dwSize.X); + const auto isSpace = [](const wchar_t c) { return c != L' '; }; + for(COORD readPos{}; readPos.Y < info.dwSize.Y; ++readPos.Y) { - int c = i % 96 + ' '; - TEST(nowide::cin.get() == c); + DWORD dwRead, bufferSize = static_cast(buffer.size()); + TEST(ReadConsoleOutputCharacterW(h, buffer.data(), bufferSize, readPos, &dwRead)); + const auto itEnd = std::find_if(buffer.rbegin() + (buffer.size() - dwRead), buffer.rend(), isSpace); + if(itEnd == buffer.rend()) + break; + result.append(buffer.begin(), itEnd.base()); + result.push_back('\n'); } + return result; } - nowide::cout << "Normal I/O:" << std::endl; - nowide::cout << example << std::endl; - nowide::cerr << example << std::endl; - - nowide::cout << "Flushing each character:" << std::endl; - - for(const char* s = example; *s; s++) - { - nowide::cout << *s << std::flush; - TEST(nowide::cout); - } - - TEST(nowide::cout); - TEST(nowide::cerr); - if(argc == 2 && argv[1] == std::string("-i")) - { - nowide::cout << "Input 2 strings" << std::endl; - std::string v1, v2; - nowide::cin >> v1 >> v2; - TEST(nowide::cin); - TEST(isValidUTF8(v1)); - TEST(isValidUTF8(v2)); - nowide::cout << "First: " << v1 << std::endl; - nowide::cout << "Second: " << v2 << std::endl; - TEST(nowide::cout); - - // Check sync - nowide::cout << "Input 2 strings\n"; - nowide::cout.flush(); - TEST(nowide::cin >> v1); - nowide::cin.sync(); - nowide::cout << "First: " << v1 << std::endl; - nowide::cout << "2nd string should be ignored. Input 1 more + [ENTER]" << std::endl; - // And check getline not getting the CR - TEST(std::getline(nowide::cin, v1)); - TEST(!v1.empty() && v1[v1.size() - 1u] != '\r'); - nowide::cout << "Value: " << v1 << std::endl; - - nowide::cout << "Press ENTER to exit"; - nowide::cin.clear(); - nowide::cin.ignore(std::numeric_limits::max(), '\n'); - nowide::cin.get(); + + void setBufferData(const std::wstring& data) + { + std::vector buffer; + buffer.reserve(data.size() * 2 + 2); + for(const auto c : data) + { + INPUT_RECORD ev; + ev.EventType = KEY_EVENT; + ev.Event.KeyEvent.bKeyDown = TRUE; + ev.Event.KeyEvent.dwControlKeyState = 0; + ev.Event.KeyEvent.wRepeatCount = 1; + if(c == '\n') + { + ev.Event.KeyEvent.uChar.UnicodeChar = '\r'; + ev.Event.KeyEvent.wVirtualKeyCode = VK_RETURN; + } else + { + ev.Event.KeyEvent.uChar.UnicodeChar = c; + ev.Event.KeyEvent.wVirtualKeyCode = VkKeyScanW(c); + } + ev.Event.KeyEvent.wVirtualScanCode = + static_cast(MapVirtualKeyW(ev.Event.KeyEvent.wVirtualKeyCode, MAPVK_VK_TO_VSC)); + buffer.push_back(ev); + ev.Event.KeyEvent.bKeyDown = FALSE; + buffer.push_back(ev); + } + DWORD dwWritten; + TEST(WriteConsoleInputW(h, buffer.data(), static_cast(buffer.size()), &dwWritten)); + TEST_EQ(dwWritten, static_cast(buffer.size())); + } +}; + +void test_console() +{ +#ifndef NOWIDE_DISABLE_CIN_TEST + std::cout << "Test cin console: " << std::flush; + { + RedirectStdio stdinHandle(STD_INPUT_HANDLE); + std::cout << "stdin redirected, " << std::flush; + // Recreate to react on redirected streams + decltype(nw::cin) cin(nullptr); + std::cout << "cin recreated " << std::flush; + TEST(cin.rdbuf() != std::cin.rdbuf()); + std::cout << "and validated" << std::endl; + const std::string testStringIn1 = "Hello std in "; + const std::string testStringIn2 = "\xc3\xa4 - \xc3\xb6 - \xc3\xbc - \xd0\xbc - \xce\xbd"; + std::cout << "Setting mock buffer data" << std::endl; + stdinHandle.setBufferData(nw::widen(testStringIn1 + "\n" + testStringIn2 + "\n")); + std::cout << "Done" << std::endl; + std::string line; + TEST(std::getline(cin, line)); + std::cout << "ASCII line read" << std::endl; + TEST_EQ(line, testStringIn1); + TEST(std::getline(cin, line)); + std::cout << "UTF-8 line read" << std::endl; + TEST_EQ(line, testStringIn2); } +#endif + std::cout << "Test cout console" << std::endl; + { + RedirectStdio stdoutHandle(STD_OUTPUT_HANDLE); + using cout_t = decltype(nw::cout); + cout_t cout(cout_t::target_stream::output, true, nullptr); + TEST(cout.rdbuf() != std::cout.rdbuf()); + + const std::string testString = "Hello std out\n\xc3\xa4-\xc3\xb6-\xc3\xbc\n"; + cout << testString << std::flush; + + const auto data = stdoutHandle.getBufferData(); + TEST_EQ(data, nw::widen(testString)); + } + std::cout << "Test cerr console" << std::endl; + { + RedirectStdio stderrHandle(STD_ERROR_HANDLE); + + using cerr_t = decltype(nw::cerr); + cerr_t cerr(cerr_t::target_stream::error, false, nullptr); + TEST(cerr.rdbuf() != std::cerr.rdbuf()); + + const std::string testString = "Hello std err\n\xc3\xa4-\xc3\xb6-\xc3\xbc\n"; + cerr << testString << std::flush; + + const auto data = stderrHandle.getBufferData(); + TEST_EQ(data, nw::widen(testString)); + } + std::cout << "Console tests done" << std::endl; +} + +#else +void test_console() +{} +#endif +#endif + +// coverity[root_function] +void test_main(int argc, char** argv, char**) +{ + if(usesNowideRdBufIn) + nw::cout << "Using Nowide input buffer\n"; + else + nw::cout << "NOT using Nowide input buffer\n"; + if(usesNowideRdBufOut) + nw::cout << "Using Nowide output buffer\n"; // LCOV_EXCL_LINE + else + nw::cout << "NOT using Nowide output buffer\n"; + + const std::string arg = (argc == 1) ? "" : argv[1]; + if(arg == "passthrough") // Read string from cin and write to cout + { + // Check that input and output are not using the nowide filebufs as expected when the consoles are redirected + TEST(!usesNowideRdBufIn); + TEST(!usesNowideRdBufOut); + std::string s; + TEST(std::getline(nw::cin, s)); + TEST(nw::cout << s); + return; + } + +#ifdef NOWIDE_TEST_INTERACTIVE + nw::cout << "Output different chars:" << std::endl; // LCOV_EXCL_LINE + test_cout(); // LCOV_EXCL_LINE + nw::cout << "Same again:" << std::endl; // LCOV_EXCL_LINE + test_cout_single_char(); // LCOV_EXCL_LINE + + nw::cout << "Same 2 outputs but to stderr:" << std::endl; // LCOV_EXCL_LINE + test_cerr(); // LCOV_EXCL_LINE + test_cerr_single_char(); // LCOV_EXCL_LINE + + nw::cout << "Basic cin tests:" << std::endl; // LCOV_EXCL_LINE + test_cin(); // LCOV_EXCL_LINE + + nw::cout << "getline test:" << std::endl; // LCOV_EXCL_LINE + // Clear newline from last test + nw::cin.ignore(std::numeric_limits::max(), '\n'); // LCOV_EXCL_LINE + test_cin_getline(); // LCOV_EXCL_LINE + + nw::cout << "CTRL+Z test:" << std::endl; // LCOV_EXCL_LINE + test_ctrl_z_is_eof(); // LCOV_EXCL_LINE +#else + test_is_valid_UTF8(); + test_tie_and_buffered(); + test_putback_and_get(); + test_cout(); + test_cout_single_char(); + test_cerr(); + test_cerr_single_char(); + test_clog(); + test_cin(); + test_cin_getline(); + test_ctrl_z_is_eof(); + test_console(); +#endif // NOWIDE_TEST_INTERACTIVE } diff --git a/extern/boost/nowide/test/test_iostream_passthrough.cmake b/extern/boost/nowide/test/test_iostream_passthrough.cmake new file mode 100644 index 0000000000..a4d5040f2f --- /dev/null +++ b/extern/boost/nowide/test/test_iostream_passthrough.cmake @@ -0,0 +1,28 @@ +# Copyright 2019 - 2021 Alexander Grund +# Distributed under the Nowide Software License, Version 1.0. +# https://www.boost.org/LICENSE_1_0.txt + +if(NOT DEFINED TEST_BINARY) + if(CMAKE_ARGC GREATER 3) + # cmake(0) -P(1)