From 238553a1c470e4cadc08c720a1ab915ac8d41e13 Mon Sep 17 00:00:00 2001 From: Erik Flodin Date: Fri, 11 Dec 2020 19:30:21 +0100 Subject: [PATCH 01/38] Add missing pragma once (#744) --- src/Depfile.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Depfile.hpp b/src/Depfile.hpp index 770f78964d..7250a4cb32 100644 --- a/src/Depfile.hpp +++ b/src/Depfile.hpp @@ -16,6 +16,8 @@ // this program; if not, write to the Free Software Foundation, Inc., 51 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +#pragma once + class Context; class Hash; From 77ad1f78a14caa1d4283a3b8df54e78f8d67a431 Mon Sep 17 00:00:00 2001 From: Nicholas Hutchinson Date: Sat, 19 Dec 2020 19:52:45 +0000 Subject: [PATCH 02/38] GitHub: Ensure apt update is run for all Linux jobs --- .github/workflows/build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1257a86ad7..5e5ee8ae2e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -85,6 +85,8 @@ jobs: - name: Install dependencies run: | if [ "${{ runner.os }}" = "Linux" ]; then + sudo apt-get update + if [ "${{ matrix.config.os }}" = "ubuntu-20.04" ]; then sudo apt-get install -y ninja-build elfutils libzstd-dev else @@ -100,7 +102,6 @@ jobs: echo "CC=clang-${{ matrix.config.version }}" >> $GITHUB_ENV echo "CXX=clang++-${{ matrix.config.version }}" >> $GITHUB_ENV - sudo apt update sudo apt install -y clang-${{ matrix.config.version }} g++-multilib fi elif [ "${{ runner.os }}" = "macOS" ]; then From e09c9753d10ea779bfa34ab353738c1ff9de70b0 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 20 Dec 2020 19:57:42 +0100 Subject: [PATCH 03/38] Add Util::hard_link utility function --- src/Util.cpp | 33 +++++++++++++++++++++++++++++---- src/Util.hpp | 3 +++ src/system.hpp | 1 - unittest/test_Util.cpp | 25 +++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/Util.cpp b/src/Util.cpp index bac6b7d123..84f1265cf7 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -290,16 +290,17 @@ clone_hard_link_or_copy_file(const Context& ctx, #endif } if (ctx.config.hard_link()) { - unlink(dest.c_str()); LOG("Hard linking {} to {}", source, dest); - int ret = link(source.c_str(), dest.c_str()); - if (ret == 0) { + try { + Util::hard_link(source, dest); if (chmod(dest.c_str(), 0444) != 0) { LOG("Failed to chmod: {}", strerror(errno)); } return; + } catch (const Error& e) { + LOG_RAW(e.what()); + // Fall back to copying. } - LOG("Failed to hard link: {}", strerror(errno)); } LOG("Copying {} to {}", source, dest); @@ -777,6 +778,30 @@ get_path_in_cache(string_view cache_dir, uint8_t level, string_view name) return path; } +void +hard_link(const std::string& oldpath, const std::string& newpath) +{ + // Assumption: newpath may already exist as a left-over file from a previous + // run, but it's only we who can create the file entry now so we don't try to + // handle a race between unlink() and link() below. + unlink(newpath.c_str()); + +#ifndef _WIN32 + if (link(oldpath.c_str(), newpath.c_str()) != 0) { + throw Error( + "failed to link {} to {}: {}", oldpath, newpath, strerror(errno)); + } +#else + if (!CreateHardLink(newpath.c_str(), oldpath.c_str(), nullptr)) { + DWORD error = GetLastError(); + throw Error("failed to link {} to {}: {}", + oldpath, + newpath, + Win32Util::error_message(error)); + } +#endif +} + bool is_absolute_path(string_view path) { diff --git a/src/Util.hpp b/src/Util.hpp index 3fbab45a49..e353f4ce77 100644 --- a/src/Util.hpp +++ b/src/Util.hpp @@ -234,6 +234,9 @@ std::string get_path_in_cache(nonstd::string_view cache_dir, uint8_t level, nonstd::string_view name); +// Hard-link `oldpath` to `newpath`. Throws `Error` on error. +void hard_link(const std::string& oldpath, const std::string& newpath); + // Write bytes in big endian order from an integer value. // // Parameters: diff --git a/src/system.hpp b/src/system.hpp index 79d07effa9..bf4a26a47b 100644 --- a/src/system.hpp +++ b/src/system.hpp @@ -139,7 +139,6 @@ const mode_t S_IWUSR = mode_t(_S_IWRITE); # define NOMINMAX 1 # include # define mkdir(a, b) _mkdir(a) -# define link(src, dst) (CreateHardLink(dst, src, nullptr) ? 0 : -1) # define execv(a, b) win32execute(a, b, 0, -1, -1) # define strncasecmp _strnicmp # define strcasecmp _stricmp diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp index 917c137836..d8c2ddba1a 100644 --- a/unittest/test_Util.cpp +++ b/unittest/test_Util.cpp @@ -443,6 +443,31 @@ TEST_CASE("Util::get_path_in_cache") == "/zz/ccache/A/B/C/D/EF.suffix"); } +TEST_CASE("Util::hard_link") +{ + TestContext test_context; + + SUBCASE("Link file to nonexistent destination") + { + Util::write_file("old", "content"); + CHECK_NOTHROW(Util::hard_link("old", "new")); + CHECK(Util::read_file("new") == "content"); + } + + SUBCASE("Link file to existing destination") + { + Util::write_file("old", "content"); + Util::write_file("new", "other content"); + CHECK_NOTHROW(Util::hard_link("old", "new")); + CHECK(Util::read_file("new") == "content"); + } + + SUBCASE("Link nonexistent file") + { + CHECK_THROWS_AS(Util::hard_link("old", "new"), Error); + } +} + TEST_CASE("Util::int_to_big_endian") { uint8_t bytes[8]; From 4da9f2b474b7ee39cd54ab4261f90936f3c944ec Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Mon, 21 Dec 2020 19:02:19 +0100 Subject: [PATCH 04/38] Add preprocessed file extension to cpp stdout early MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unless when compiling a preprocessed file directly, ccache creates a temporary file to store the output of the preprocessor, registers the file for removal at program exit, renames the file to one with a .i/.ii extension and then registers that file for removal as well. This works by chance in practice as long as mkstemp() returns something with low probability of being reused, but as discussed in #736 it risks failing when mkstemp() doesn’t behave that way. Fix this by creating the new name (with the needed extension) using a hard link so that the original file will outlive the new file, thus blocking another ccache process from creating a file with the same name again. To make the new temporary file outlive the old file, also delete pending temporary files in LIFO order. --- src/Context.cpp | 10 ++++++---- src/ccache.cpp | 13 +++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Context.cpp b/src/Context.cpp index 7706b7d947..a7ce450c52 100644 --- a/src/Context.cpp +++ b/src/Context.cpp @@ -56,9 +56,10 @@ Context::register_pending_tmp_file(const std::string& path) void Context::unlink_pending_tmp_files_signal_safe() { - for (const std::string& path : m_pending_tmp_files) { + for (auto it = m_pending_tmp_files.rbegin(); it != m_pending_tmp_files.rend(); + ++it) { // Don't call Util::unlink_tmp since its log calls aren't signal safe. - unlink(path.c_str()); + unlink(it->c_str()); } // Don't clear m_pending_tmp_files since this method must be signal safe. } @@ -68,8 +69,9 @@ Context::unlink_pending_tmp_files() { SignalHandlerBlocker signal_handler_blocker; - for (const std::string& path : m_pending_tmp_files) { - Util::unlink_tmp(path, Util::UnlinkLog::ignore_failure); + for (auto it = m_pending_tmp_files.rbegin(); it != m_pending_tmp_files.rend(); + ++it) { + Util::unlink_tmp(*it, Util::UnlinkLog::ignore_failure); } m_pending_tmp_files.clear(); } diff --git a/src/ccache.cpp b/src/ccache.cpp index 654af6f878..11d6164498 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -1081,7 +1081,12 @@ get_result_name_from_cpp(Context& ctx, Args& args, Hash& hash) TemporaryFile tmp_stdout( FMT("{}/tmp.cpp_stdout", ctx.config.temporary_dir())); - stdout_path = tmp_stdout.path; + ctx.register_pending_tmp_file(tmp_stdout.path); + + // stdout_path needs the proper cpp_extension for the compiler to do its + // thing correctly. + stdout_path = FMT("{}.{}", tmp_stdout.path, ctx.config.cpp_extension()); + Util::hard_link(tmp_stdout.path, stdout_path); ctx.register_pending_tmp_file(stdout_path); TemporaryFile tmp_stderr( @@ -1131,11 +1136,7 @@ get_result_name_from_cpp(Context& ctx, Args& args, Hash& hash) if (ctx.args_info.direct_i_file) { ctx.i_tmpfile = ctx.args_info.input_file; } else { - // i_tmpfile needs the proper cpp_extension for the compiler to do its - // thing correctly - ctx.i_tmpfile = FMT("{}.{}", stdout_path, ctx.config.cpp_extension()); - Util::rename(stdout_path, ctx.i_tmpfile); - ctx.register_pending_tmp_file(ctx.i_tmpfile); + ctx.i_tmpfile = stdout_path; } if (!ctx.config.run_second_cpp()) { From 55a0e3efd4c762bcb2985ca6dc3afeac61ee68ea Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 27 Dec 2020 16:09:14 +0100 Subject: [PATCH 05/38] Mention GitHub discussions --- CONTRIBUTING.md | 4 ++-- README.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 46315c48b1..b93c70723e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,8 +6,8 @@ Want to contribute to ccache? Awesome! There are several options: -1. Ask a question in the [issue - tracker](https://github.com/ccache/ccache/issues/new/choose). +1. Ask a question in + [discussions](https://github.com/ccache/ccache/issues/discussions). 2. Post your question to the [mailing list](https://lists.samba.org/mailman/listinfo/ccache/). 3. Chat in the [Gitter room](https://gitter.im/ccache/ccache). diff --git a/README.md b/README.md index aa785934b6..57ae2ca3e0 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Contributing to ccache * [Mailing list](https://lists.samba.org/mailman/listinfo/ccache/) * [Chat](https://gitter.im/ccache/ccache) * [Bug report info](https://ccache.dev/bugs.html) +* [Discussions](https://github.com/ccache/ccache/discussions) * [Issue tracker](https://github.com/ccache/ccache/issues) * [Help wanted!](https://github.com/ccache/ccache/labels/help%20wanted) * [Good first issues!](https://github.com/ccache/ccache/labels/good%20first%20issue) From 3eb0551eebf0d79b8c2ccfab9de4ffeed6c0ce2a Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 27 Dec 2020 16:21:37 +0100 Subject: [PATCH 06/38] =?UTF-8?q?Remove=20=E2=80=9CSupport=E2=80=9D=20issu?= =?UTF-8?q?e=20template?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub Discussions is preferred now. --- .github/ISSUE_TEMPLATE/support.md | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/support.md diff --git a/.github/ISSUE_TEMPLATE/support.md b/.github/ISSUE_TEMPLATE/support.md deleted file mode 100644 index 9d10686dad..0000000000 --- a/.github/ISSUE_TEMPLATE/support.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: Question -about: Ask for support or make an enquiry -title: '' -labels: support -assignees: '' - ---- -### Question ### - From 942f8e14a3d6d61ba7e6b4a2e278c68479de9873 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 27 Dec 2020 16:22:22 +0100 Subject: [PATCH 07/38] Rephrase hint about C-style code left in the code base --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b93c70723e..3bf043248b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,9 +50,9 @@ Here are some hints to make the process smoother: ## Code style Ccache was written in C99 until 2019 when it started being converted to C++11. -The conversion is a slow work in progress, which is why there is a lot of -C-style code left. Please refrain from doing large C to C++ conversions; do it -little by little. +The conversion is a slow work in progress, which is why there is some C-style +code left. Please refrain from doing large C to C++ conversions; do it little by +little. Source code formatting is defined by `.clang-format` in the root directory. The format is loosely based on [LLVM's code formatting From da54bff4e2231a9b4619cb3299edda769fa3b2ba Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 27 Dec 2020 16:22:39 +0100 Subject: [PATCH 08/38] Tweak markdown formatting --- CONTRIBUTING.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3bf043248b..cb35bc0bf8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,8 +39,8 @@ Here are some hints to make the process smoother: avoid potentially wasting time on doing something that may need major rework to be accepted, or maybe doesn't end up being accepted at all. * Is your pull request "work in progress", i.e. you don't think that it's ready - for merging yet but you want early comments and CI test results? Then create - a draft pull request as described in [this Github blog + for merging yet but you want early comments and CI test results? Then create a + draft pull request as described in [this Github blog post](https://github.blog/2019-02-14-introducing-draft-pull-requests/). * Please follow the ccache's code style (see the section below). * Consider [A Note About Git Commit @@ -60,14 +60,15 @@ style](https://llvm.org/docs/CodingStandards.html) with some exceptions. It's highly recommended to install [Clang-Format](https://clang.llvm.org/docs/ClangFormat.html) 6.0 or newer and run `make format` to format changes according to ccache's code style. Or even -better: set up your editor to run Clang-Format automatically when saving. If -you don't run Clang-Format then the ccache authors have to do it for you. +better: set up your editor to run Clang-Format automatically when saving. If you +don't run Clang-Format then the ccache authors have to do it for you. Please follow these conventions: * Use `UpperCamelCase` for types (e.g. classes and structs) and namespaces. * Use `UPPER_CASE` names for macros and (non-class )enum values. -* Use `snake_case` for other names (functions, variables, enum class values, etc.). +* Use `snake_case` for other names (functions, variables, enum class values, + etc.). * Use an `m_` prefix for non-public member variables. * Use a `g_` prefix for global mutable variables. * Use a `k_` prefix for global constants. From eead7e9d63992684ce7290b09c05410485323096 Mon Sep 17 00:00:00 2001 From: Nicholas Hutchinson Date: Mon, 28 Dec 2020 14:22:40 +0000 Subject: [PATCH 09/38] Improve TemporaryFile implementation for Windows (#736) On Windows, multiple ccache process could race each other to create, rename and delete temporary files, because they would attempt to generate the same sequence of temporary file names (`tmp.cpp_stdout.iG2Kb7`, `tmp.cpp_stdout.P1kAlM`, `tmp.cpp_stdout.FzP5tM`, ...). This is because ccache used mingw-w64's [implementation of mkstemp][1], which uses `rand()` to generate temporary file names, and ccache was never seeding the thread-local PRNG used by `rand()`. Replace ccache's use of `mkstemp()` on Windows with an implementation based on OpenBSD. This allows us to sidestep mingw-w64's problematic implementation, and allows us to build using MSVC again. (MSVC's C standard library does not provide `mkstemp()`.) Example errors: - Some ccache process is in the process of deleting a temporary file: ccache: error: Failed to create temporary file for C:\Users\someuser/.ccache/tmp/tmp.cpp_stdout.FzP5tM: Access is denied. - Some ccache process has destination file open, so it can't be overwritten: ccache: error: failed to rename C:\Users\someuser/.ccache/tmp/tmp.cpp_stdout.iG2Kb7 to C:\Users\someuser/.ccache/tmp/tmp.cpp_stdout.iG2Kb7.ii: Access is denied. - Source file has been deleted by some other ccache process: ccache: error: failed to rename C:\Users\someuser/.ccache/tmp/tmp.cpp_stdout.P1kAlM to C:\Users\someuser/.ccache/tmp/tmp.cpp_stdout.P1kAlM.ii: The system cannot find the file specified. [1]: https://github.com/mirror/mingw-w64/blob/v8.0.0/mingw-w64-crt/misc/mkstemp.c --- LICENSE.adoc | 24 +++ src/TemporaryFile.cpp | 13 ++ src/third_party/CMakeLists.txt | 4 + src/third_party/win32/mktemp.c | 260 +++++++++++++++++++++++++++++++++ src/third_party/win32/mktemp.h | 18 +++ unittest/CMakeLists.txt | 4 +- unittest/test_bsdmkstemp.cpp | 193 ++++++++++++++++++++++++ 7 files changed, 515 insertions(+), 1 deletion(-) create mode 100644 src/third_party/win32/mktemp.c create mode 100644 src/third_party/win32/mktemp.h create mode 100644 unittest/test_bsdmkstemp.cpp diff --git a/LICENSE.adoc b/LICENSE.adoc index 3c15c57262..a9e5b9da13 100644 --- a/LICENSE.adoc +++ b/LICENSE.adoc @@ -638,6 +638,30 @@ The full license text can be found in LGPL-3.0.txt and at https://www.gnu.org/licenses/lgpl-3.0.html. +src/third_party/win32/mktemp.* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This implementation of `mkstemp()` for Win32 was adapted from + +and has the folowing license text: + +------------------------------------------------------------------------------- +Copyright (c) 1996-1998, 2008 Theo de Raadt +Copyright (c) 1997, 2008-2009 Todd C. Miller + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +------------------------------------------------------------------------------- + src/third_party/xxh* ~~~~~~~~~~~~~~~~~~~~ diff --git a/src/TemporaryFile.cpp b/src/TemporaryFile.cpp index 11fd8ba1be..ca0c8374f1 100644 --- a/src/TemporaryFile.cpp +++ b/src/TemporaryFile.cpp @@ -20,6 +20,10 @@ #include "Util.hpp" +#ifdef _WIN32 +# include "third_party/win32/mktemp.h" +#endif + using nonstd::string_view; namespace { @@ -45,7 +49,16 @@ TemporaryFile::TemporaryFile(string_view path_prefix) : path(std::string(path_prefix) + ".XXXXXX") { Util::ensure_dir_exists(Util::dir_name(path)); +#ifdef _WIN32 + // MSVC lacks mkstemp() and [mingw-w64's implementation][1] is problematic, as + // it can reuse the names of recently-deleted files unless the caller + // remembers to call srand(). + // [1]: + // https://github.com/Alexpux/mingw-w64/blob/d0d7f784833bbb0b2d279310ddc6afb52fe47a46/mingw-w64-crt/misc/mkstemp.c + fd = Fd(bsd_mkstemp(&path[0])); +#else fd = Fd(mkstemp(&path[0])); +#endif if (!fd) { throw Fatal( "Failed to create temporary file for {}: {}", path, strerror(errno)); diff --git a/src/third_party/CMakeLists.txt b/src/third_party/CMakeLists.txt index d40110a61a..977cff60cc 100644 --- a/src/third_party/CMakeLists.txt +++ b/src/third_party/CMakeLists.txt @@ -6,6 +6,10 @@ else() target_compile_definitions(third_party_lib PUBLIC -DSTATIC_GETOPT) endif() +if (WIN32) + target_sources(third_party_lib PRIVATE win32/mktemp.c) +endif () + if(ENABLE_TRACING) target_sources(third_party_lib PRIVATE minitrace.c) endif() diff --git a/src/third_party/win32/mktemp.c b/src/third_party/win32/mktemp.c new file mode 100644 index 0000000000..8963b89061 --- /dev/null +++ b/src/third_party/win32/mktemp.c @@ -0,0 +1,260 @@ +/* $OpenBSD: mktemp.c,v 1.39 2017/11/28 06:55:49 tb Exp $ */ +/* + * Copyright (c) 1996-1998, 2008 Theo de Raadt + * Copyright (c) 1997, 2008-2009 Todd C. Miller + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifdef _WIN32 +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 // _WIN32_WINNT_VISTA +#endif + +#ifndef _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_NONSTDC_NO_DEPRECATE +#endif + +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include + +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX 1 +#define WIN32_NO_STATUS +#include +#undef WIN32_NO_STATUS +#include + +// Work-around wrong calling convention for RtlGenRandom in old mingw-w64 +#define SystemFunction036 __stdcall SystemFunction036 +#include +#undef SystemFunction036 +#endif + +#ifdef _MSC_VER +#define S_IRUSR (_S_IREAD) +#define S_IWUSR (_S_IWRITE) +#endif + +#define MKTEMP_NAME 0 +#define MKTEMP_FILE 1 +#define MKTEMP_DIR 2 + +#define TEMPCHARS "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" +#define NUM_CHARS (sizeof(TEMPCHARS) - 1) +#define MIN_X 6 + +#ifdef _WIN32 +#define MKOTEMP_FLAGS (_O_APPEND|_O_NOINHERIT|_O_BINARY|_O_TEXT| \ + _O_U16TEXT|_O_U8TEXT|_O_WTEXT) +#define MKTEMP_FLAGS_DEFAULT (_O_BINARY) +#else +#define MKOTEMP_FLAGS (O_APPEND|O_CLOEXEC|O_DSYNC|O_RSYNC|O_SYNC) +#define MKTEMP_FLAGS_DEFAULT (0) +#endif + +#ifndef nitems +#define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) +#endif + +#ifdef _WIN32 +static BOOL CALLBACK +lookup_ntdll_function_once( + PINIT_ONCE init_once, PVOID parameter, PVOID *context) +{ + (void)init_once; + *context = (PVOID)GetProcAddress( + GetModuleHandleA("ntdll.dll"), parameter); + return(TRUE); +} + +static NTSTATUS +GetLastNtStatus() +{ + static INIT_ONCE init_once = INIT_ONCE_STATIC_INIT; + typedef NTSTATUS(NTAPI * RtlGetLastNtStatus_t)(void); + RtlGetLastNtStatus_t get_last_nt_status = NULL; + InitOnceExecuteOnce(&init_once, lookup_ntdll_function_once, + "RtlGetLastNtStatus", (LPVOID *)&get_last_nt_status); + return(get_last_nt_status()); +} + +static int +normalize_msvcrt_errno(int ret) +{ + if (ret == -1 && errno == EACCES && _doserrno == ERROR_ACCESS_DENIED) { + /* + * Win32 APIs return ERROR_ACCESS_DENIED for many distinct + * NTSTATUS codes, even when it's arguably inappropriate to do + * so, e.g. if you attempt to open a directory, or open a file + * that's in the "pending delete" state. These are mapped to + * EACCESS in the C runtime. We instead map these to EEXIST. + */ + NTSTATUS nt_err = GetLastNtStatus(); + if (nt_err == STATUS_FILE_IS_A_DIRECTORY || + nt_err == STATUS_DELETE_PENDING) { + errno = EEXIST; + } + } + return(ret); +} + +#define open(...) (normalize_msvcrt_errno(open(__VA_ARGS__))) +#define mkdir(path, mode) (normalize_msvcrt_errno(mkdir(path))) +#define lstat(path, sb) (normalize_msvcrt_errno(stat(path, sb))) + +static void (*_bsd_mkstemp_random_source)(void *buf, size_t n); + +void +bsd_mkstemp_set_random_source(void (*f)(void *buf, size_t n)) +{ + _bsd_mkstemp_random_source = f; +} + +static void +arc4random_buf(void *buf, size_t nbytes) +{ + if (_bsd_mkstemp_random_source != NULL) { + _bsd_mkstemp_random_source(buf, nbytes); + } else { + RtlGenRandom(buf, (ULONG)nbytes); + } +} +#endif + +static int +mktemp_internal(char *path, int slen, int mode, int flags) +{ + char *start, *cp, *ep; + const char tempchars[] = TEMPCHARS; + unsigned int tries; + struct stat sb; + size_t len; + int fd; + + len = strlen(path); + if (len < MIN_X || slen < 0 || (size_t)slen > len - MIN_X) { + errno = EINVAL; + return(-1); + } + ep = path + len - slen; + + for (start = ep; start > path && start[-1] == 'X'; start--) + ; + if (ep - start < MIN_X) { + errno = EINVAL; + return(-1); + } + + if (flags & ~MKOTEMP_FLAGS) { + errno = EINVAL; + return(-1); + } + flags |= O_CREAT|O_EXCL|O_RDWR; + + tries = INT_MAX; + do { + cp = start; + do { + unsigned short rbuf[16]; + unsigned int i; + + /* + * Avoid lots of arc4random() calls by using + * a buffer sized for up to 16 Xs at a time. + */ + arc4random_buf(rbuf, sizeof(rbuf)); + for (i = 0; i < nitems(rbuf) && cp != ep; i++) + *cp++ = tempchars[rbuf[i] % NUM_CHARS]; + } while (cp != ep); + + switch (mode) { + case MKTEMP_NAME: + if (lstat(path, &sb) != 0) + return(errno == ENOENT ? 0 : -1); + break; + case MKTEMP_FILE: + fd = open(path, flags, S_IRUSR|S_IWUSR); + if (fd != -1 || errno != EEXIST) + return(fd); + break; + case MKTEMP_DIR: + if (mkdir(path, S_IRUSR|S_IWUSR|S_IXUSR) == 0) + return(0); + if (errno != EEXIST) + return(-1); + break; + } + } while (--tries); + + errno = EEXIST; + return(-1); +} + +char * +bsd_mktemp(char *path) +{ + if (mktemp_internal(path, 0, MKTEMP_NAME, MKTEMP_FLAGS_DEFAULT) == -1) + return(NULL); + return(path); +} + +int +bsd_mkostemps(char *path, int slen, int flags) +{ + return(mktemp_internal(path, slen, MKTEMP_FILE, flags)); +} + +int +bsd_mkstemp(char *path) +{ + return(mktemp_internal(path, 0, MKTEMP_FILE, MKTEMP_FLAGS_DEFAULT)); +} + +int +bsd_mkostemp(char *path, int flags) +{ + return(mktemp_internal(path, 0, MKTEMP_FILE, flags)); +} + +int +bsd_mkstemps(char *path, int slen) +{ + return(mktemp_internal(path, slen, MKTEMP_FILE, MKTEMP_FLAGS_DEFAULT)); +} + +char * +bsd_mkdtemp(char *path) +{ + int error; + + error = mktemp_internal(path, 0, MKTEMP_DIR, 0); + return(error ? NULL : path); +} diff --git a/src/third_party/win32/mktemp.h b/src/third_party/win32/mktemp.h new file mode 100644 index 0000000000..40e0c167b8 --- /dev/null +++ b/src/third_party/win32/mktemp.h @@ -0,0 +1,18 @@ +#ifndef CCACHE_THIRD_PARTY_WIN32_MKTEMP_H_ +#define CCACHE_THIRD_PARTY_WIN32_MKTEMP_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int bsd_mkstemp(char *); + +// Exposed for testing. +void bsd_mkstemp_set_random_source(void (*)(void *buf, size_t n)); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index c82a226621..b29fd653e5 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -27,7 +27,9 @@ if(INODE_CACHE_SUPPORTED) endif() if(WIN32) - list(APPEND source_files test_Win32Util.cpp) + list(APPEND source_files + test_bsdmkstemp.cpp + test_Win32Util.cpp) endif() add_executable(unittest ${source_files}) diff --git a/unittest/test_bsdmkstemp.cpp b/unittest/test_bsdmkstemp.cpp new file mode 100644 index 0000000000..0237f5c80d --- /dev/null +++ b/unittest/test_bsdmkstemp.cpp @@ -0,0 +1,193 @@ +#include "../src/Fd.hpp" +#include "TestUtil.hpp" + +#include "third_party/doctest.h" +#include "third_party/win32/mktemp.h" + +#include +#include +#include +#include +#include + +using TestUtil::TestContext; + +namespace { + +class ScopedHANDLE +{ +public: + ScopedHANDLE() = default; + + explicit ScopedHANDLE(HANDLE h) : h_(h) + { + } + + ScopedHANDLE(ScopedHANDLE&& other) : ScopedHANDLE(other.release()) + { + } + + ~ScopedHANDLE() + { + if (h_ != INVALID_HANDLE_VALUE) { + CloseHandle(h_); + } + } + + ScopedHANDLE& + operator=(ScopedHANDLE rhs) + { + std::swap(h_, rhs.h_); + return *this; + } + + explicit operator bool() const + { + return h_ != INVALID_HANDLE_VALUE; + } + + HANDLE + get() const + { + return h_; + } + + HANDLE + release() + { + HANDLE h = h_; + h_ = INVALID_HANDLE_VALUE; + return h; + } + +private: + HANDLE h_ = INVALID_HANDLE_VALUE; +}; + +} // namespace + +TEST_SUITE_BEGIN("thirdparty"); + +TEST_CASE("thirdparty::bsd_mkstemp") +{ + TestContext test_context; + + static int rand_iter; + rand_iter = 0; + + bsd_mkstemp_set_random_source([](void* buf, size_t nbytes) { + std::fill_n( + static_cast(buf), nbytes / sizeof(uint16_t), rand_iter); + rand_iter++; + }); + + struct Cleanup + { + ~Cleanup() + { + bsd_mkstemp_set_random_source(nullptr); + } + } cleanup; + + SUBCASE("successful") + { + std::string path("XXXXXX"); + CHECK_MESSAGE(Fd(bsd_mkstemp(&path[0])), "errno=" << errno); + CHECK(path == "AAAAAA"); + } + + SUBCASE("existing_file") + { + CHECK_MESSAGE(ScopedHANDLE(CreateFileA("AAAAAA", + GENERIC_READ | GENERIC_WRITE, + 0, + nullptr, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + nullptr)), + "errno=" << errno); + + std::string path("XXXXXX"); + CHECK_MESSAGE(Fd(bsd_mkstemp(&path[0])), "errno=" << errno); + CHECK(path == "BBBBBB"); + } + + SUBCASE("existing_file_pending_delete") + { + ScopedHANDLE h; + CHECK_MESSAGE( + (h = ScopedHANDLE(CreateFileA("AAAAAA", + GENERIC_READ | GENERIC_WRITE | DELETE, + 0, + nullptr, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + nullptr))), + "errno=" << errno); + + // Mark file as deleted. This puts it into a "pending delete" state that + // will persist until the handle is closed. + FILE_DISPOSITION_INFO info{}; + info.DeleteFile = TRUE; + CHECK_MESSAGE(SetFileInformationByHandle( + h.get(), FileDispositionInfo, &info, sizeof(info)), + "errno=" << errno); + + std::string path("XXXXXX"); + CHECK_MESSAGE(Fd(bsd_mkstemp(&path[0])), "errno=" << errno); + CHECK(path == "BBBBBB"); + } + + SUBCASE("existing_dir") + { + CHECK_MESSAGE(CreateDirectoryA("AAAAAA", nullptr), "errno=" << errno); + + std::string path("XXXXXX"); + CHECK_MESSAGE(Fd(bsd_mkstemp(&path[0])), "errno=" << errno); + CHECK(path == "BBBBBB"); + } + + SUBCASE("permission denied") + { + auto makeACL = [](const char* aclString) { + PSECURITY_DESCRIPTOR desc = nullptr; + ConvertStringSecurityDescriptorToSecurityDescriptorA( + aclString, SDDL_REVISION_1, &desc, nullptr); + return std::shared_ptr( + static_cast(desc), &LocalFree); + }; + + // Create a directory with a contrived ACL that denies creation of new + // files and directories to the "Everybody" (WD) group. + std::shared_ptr desc; + CHECK_MESSAGE((desc = makeACL("D:(D;;DCLCRPCR;;;WD)(A;;FA;;;WD)")), + "errno=" << errno); + + SECURITY_ATTRIBUTES attrs{}; + attrs.nLength = sizeof(attrs); + attrs.lpSecurityDescriptor = desc.get(); + CHECK_MESSAGE(CreateDirectoryA("my_readonly_dir", &attrs), + "errno=" << errno); + + // Sanity check that we cannot write to this directory. (E.g. Wine + // doesn't appear to emulate Windows ACLs properly when run under root.) + bool broken_acls = static_cast(ScopedHANDLE( + CreateFileA("my_readonly_dir/.writable", + GENERIC_WRITE, + 0, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE, + nullptr))); + + if (!broken_acls) { + std::string path("my_readonly_dir/XXXXXX"); + CHECK(!Fd(bsd_mkstemp(&path[0]))); + CHECK(errno == EACCES); + } else { + MESSAGE("ACLs do not appear to function properly on this filesystem"); + } + } +} + +TEST_SUITE_END(); From f46466d101ae2e4c2097e01f88f3292898c8a81f Mon Sep 17 00:00:00 2001 From: Nicholas Hutchinson Date: Mon, 28 Dec 2020 14:25:12 +0000 Subject: [PATCH 10/38] Fix running tests on macOS (#756) Bash tests were not actually being run on the macOS CI agents because the version of sed installed there does not understand the `-r` flag: sed: illegal option -- r usage: sed script [-Ealn] [-i extension] [file ...] sed [-Ealn] [-i extension] [-e script] ... [-f script_file] ... [file ...] - use `sed -E` instead of `sed -r` as the latter isn't supported by BSD sed. - export `SDKROOT` in `test/run`. Otherwise it appears at least some some Apple toolchains (e.g. Xcode 10.3) will pick the _latest_ SDK installed on the host (e.g. `/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk`) instead of using the SDK bundled with the toolchain (e.g. `/Applications/Xcode_10.3.app/.../MacOSX10.14.sdk`). The 10.15 SDK is not compatible with Xcode 10.3: ld: unsupported tapi file type '!tapi-tbd' in YAML file '/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk/usr/lib/libSystem.tbd' for architecture x86_64 clang: error: linker command failed with exit code 1 --- misc/format-files | 2 +- test/run | 3 ++- test/suites/base.bash | 4 ++-- test/suites/cache_levels.bash | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/misc/format-files b/misc/format-files index 1c0405b190..111705ada4 100755 --- a/misc/format-files +++ b/misc/format-files @@ -46,7 +46,7 @@ for file in "$@"; do echo "Error: $file not formatted with Clang-Format" echo 'Run "make format" or apply this diff:' git diff $cf_color --no-index "$file" "$tmp_file" \ - | sed -r -e "s!^---.*!--- a/$file!" \ + | sed -E -e "s!^---.*!--- a/$file!" \ -e "s!^\+\+\+.*!+++ b/$file!" \ -e "/diff --/d" -e "/index /d" \ -e "s/.[0-9]*.clang-format.tmp//" diff --git a/test/run b/test/run index 9623e49df3..9005dc97b5 100755 --- a/test/run +++ b/test/run @@ -485,6 +485,7 @@ if $HOST_OS_APPLE; then echo "Error: xcrun --show-sdk-path failure" exit 1 fi + export SDKROOT SYSROOT="-isysroot `echo \"$SDKROOT\" | sed 's/ /\\ /g'`" else @@ -493,7 +494,7 @@ fi # --------------------------------------- -all_suites="$(sed -rn 's/^addtest\((.*)\)$/\1/p' $(dirname $0)/CMakeLists.txt)" +all_suites="$(sed -En 's/^addtest\((.*)\)$/\1/p' $(dirname $0)/CMakeLists.txt)" for suite in $all_suites; do . $(dirname $0)/suites/$suite.bash diff --git a/test/suites/base.bash b/test/suites/base.bash index 75e2768f9b..30c1c41f41 100644 --- a/test/suites/base.bash +++ b/test/suites/base.bash @@ -788,7 +788,7 @@ EOF chmod +x gcc CCACHE_DEBUG=1 $CCACHE ./gcc -c test1.c - compiler_type=$(sed -rn 's/.*Compiler type: (.*)/\1/p' test1.o.ccache-log) + compiler_type=$(sed -En 's/.*Compiler type: (.*)/\1/p' test1.o.ccache-log) if [ "$compiler_type" != gcc ]; then test_failed "Compiler type $compiler_type != gcc" fi @@ -796,7 +796,7 @@ EOF rm test1.o.ccache-log CCACHE_COMPILERTYPE=clang CCACHE_DEBUG=1 $CCACHE ./gcc -c test1.c - compiler_type=$(sed -rn 's/.*Compiler type: (.*)/\1/p' test1.o.ccache-log) + compiler_type=$(sed -En 's/.*Compiler type: (.*)/\1/p' test1.o.ccache-log) if [ "$compiler_type" != clang ]; then test_failed "Compiler type $compiler_type != clang" fi diff --git a/test/suites/cache_levels.bash b/test/suites/cache_levels.bash index 776508bf8a..ef2e8d5767 100644 --- a/test/suites/cache_levels.bash +++ b/test/suites/cache_levels.bash @@ -11,7 +11,7 @@ expect_on_level() { local expected_level="$2" slashes=$(find $CCACHE_DIR -name "*$type" \ - | sed -r -e 's!.*\.ccache/!!' -e 's![^/]*$!!' -e 's![^/]!!g') + | sed -E -e 's!.*\.ccache/!!' -e 's![^/]*$!!' -e 's![^/]!!g') actual_level=$(echo -n "$slashes" | wc -c) if [ "$actual_level" -ne "$expected_level" ]; then test_failed "$type file on level $actual_level, expected level $expected_level" From 66724a6d01d9259c07f8bf30be8b196be0e9bd15 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Mon, 28 Dec 2020 20:46:06 +0100 Subject: [PATCH 11/38] Add simple unit test of Util::make_relative_path --- src/Util.cpp | 21 +++++++++++++++------ src/Util.hpp | 10 ++++++++-- unittest/test_Util.cpp | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/src/Util.cpp b/src/Util.cpp index 84f1265cf7..caaed55435 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -858,10 +858,12 @@ localtime(optional time) } std::string -make_relative_path(const Context& ctx, string_view path) +make_relative_path(const std::string& base_dir, + const std::string& actual_cwd, + const std::string& apparent_cwd, + nonstd::string_view path) { - if (ctx.config.base_dir().empty() - || !Util::starts_with(path, ctx.config.base_dir())) { + if (base_dir.empty() || !Util::starts_with(path, base_dir)) { return std::string(path); } @@ -894,11 +896,11 @@ make_relative_path(const Context& ctx, string_view path) std::string path_str(path); std::string normalized_path = Util::normalize_absolute_path(path_str); std::vector relpath_candidates = { - Util::get_relative_path(ctx.actual_cwd, normalized_path), + Util::get_relative_path(actual_cwd, normalized_path), }; - if (ctx.apparent_cwd != ctx.actual_cwd) { + if (apparent_cwd != actual_cwd) { relpath_candidates.emplace_back( - Util::get_relative_path(ctx.apparent_cwd, normalized_path)); + Util::get_relative_path(apparent_cwd, normalized_path)); // Move best (= shortest) match first: if (relpath_candidates[0].length() > relpath_candidates[1].length()) { std::swap(relpath_candidates[0], relpath_candidates[1]); @@ -915,6 +917,13 @@ make_relative_path(const Context& ctx, string_view path) return std::string(original_path); } +std::string +make_relative_path(const Context& ctx, string_view path) +{ + return make_relative_path( + ctx.config.base_dir(), ctx.actual_cwd, ctx.apparent_cwd, path); +} + bool matches_dir_prefix_or_file(string_view dir_prefix_or_file, string_view path) { diff --git a/src/Util.hpp b/src/Util.hpp index e353f4ce77..819950b5f3 100644 --- a/src/Util.hpp +++ b/src/Util.hpp @@ -309,8 +309,14 @@ bool is_precompiled_header(nonstd::string_view path); // time of day is used. nonstd::optional localtime(nonstd::optional time = {}); -// Make a relative path from current working directory to `path` if `path` is -// under the base directory. +// Make a relative path from current working directory (either `actual_cwd` or +// `apparent_cwd`) to `path` if `path` is under `base_dir`. +std::string make_relative_path(const std::string& base_dir, + const std::string& actual_cwd, + const std::string& apparent_cwd, + nonstd::string_view path); + +// Like above but with base directory and apparent/actual CWD taken from `ctx`. std::string make_relative_path(const Context& ctx, nonstd::string_view path); // Return whether `path` is equal to `dir_prefix_or_file` or if diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp index d8c2ddba1a..ff4da6c337 100644 --- a/unittest/test_Util.cpp +++ b/unittest/test_Util.cpp @@ -554,6 +554,46 @@ TEST_CASE("Util::is_dir_separator") #endif } +#ifndef _WIN32 +TEST_CASE("Util::make_relative_path") +{ + using Util::make_relative_path; + + const TestContext test_context; + + const std::string cwd = Util::get_actual_cwd(); + const std::string actual_cwd = FMT("{}/d", cwd); + const std::string apparent_cwd = FMT("{}/s", cwd); + + REQUIRE(Util::create_dir("d")); + REQUIRE(symlink("d", "s") == 0); + REQUIRE(chdir("s") == 0); + Util::setenv("PWD", apparent_cwd); + + SUBCASE("No base directory") + { + CHECK(make_relative_path("", "/a", "/a", "/a/x") == "/a/x"); + } + + SUBCASE("Path matches neither actual nor apparent CWD") + { + CHECK(make_relative_path("/", "/a", "/b", "/x") == "/x"); + } + + SUBCASE("Match of actual CWD") + { + CHECK(make_relative_path("/", actual_cwd, apparent_cwd, actual_cwd + "/x") + == "./x"); + } + + SUBCASE("Match of apparent CWD") + { + CHECK(make_relative_path("/", actual_cwd, apparent_cwd, apparent_cwd + "/x") + == "./x"); + } +} +#endif + TEST_CASE("Util::matches_dir_prefix_or_file") { CHECK(!Util::matches_dir_prefix_or_file("", "")); From d609e926050dc0812a4d0b47330a291bd3c22d95 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Tue, 29 Dec 2020 19:06:17 +0100 Subject: [PATCH 12/38] Fix Util::dir_name for Windows paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Util::dir_name does not understand “C:\”-style Windows path so add such knowledge. --- src/Util.cpp | 12 +++++++++++- unittest/test_Util.cpp | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Util.cpp b/src/Util.cpp index caaed55435..2aaaa0a82b 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -413,9 +413,19 @@ dir_name(string_view path) #endif size_t n = path.find_last_of(delim); if (n == std::string::npos) { + // "foo" -> "." return "."; + } else if (n == 0) { + // "/" -> "/" (Windows: or "\\" -> "\\") + return path.substr(0, 1); +#ifdef _WIN32 + } else if (n == 2 && path[1] == ':') { + // Windows: "C:\\foo" -> "C:\\" or "C:/foo" -> "C:/" + return path.substr(0, 3); +#endif } else { - return n == 0 ? "/" : path.substr(0, n); + // "/dir/foo" -> "/dir" (Windows: or "C:\\dir\\foo" -> "C:\\dir") + return path.substr(0, n); } } diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp index ff4da6c337..0d3cae3675 100644 --- a/unittest/test_Util.cpp +++ b/unittest/test_Util.cpp @@ -146,6 +146,16 @@ TEST_CASE("Util::dir_name") CHECK(Util::dir_name("/") == "/"); CHECK(Util::dir_name("/foo") == "/"); CHECK(Util::dir_name("/foo/bar/f.txt") == "/foo/bar"); + +#ifdef _WIN32 + CHECK(Util::dir_name("C:/x/y") == "C:/x"); + CHECK(Util::dir_name("X:/x/y") == "X:/x"); + CHECK(Util::dir_name("C:\\x\\y") == "C:\\x"); + CHECK(Util::dir_name("C:/x") == "C:/"); + CHECK(Util::dir_name("C:\\x") == "C:\\"); + CHECK(Util::dir_name("C:/") == "C:/"); + CHECK(Util::dir_name("C:\\") == "C:\\"); +#endif } TEST_CASE("Util::strip_ansi_csi_seqs") From 76eb7f4c46e876db07bbc8fa850b7d84a480c5ea Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Tue, 29 Dec 2020 19:33:54 +0100 Subject: [PATCH 13/38] Remove obsolete (and now incorrect) fallback replacement of realpath(3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fallback replacement of realpath(3) (from 8e918ccc) uses readlink(3) under the assumption that we’re only interested about symlinks, but that’s no longer the case: we’re using it for normalization purposes as well. Let’s just remove it. If it turns out that there still are non-Windows systems that don’t have realpath(3) and that we care about we’ll figure out something else. --- src/Util.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/Util.cpp b/src/Util.cpp index 2aaaa0a82b..edeba42325 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -1245,15 +1245,7 @@ real_path(const std::string& path, bool return_empty_on_error) resolved = buffer; } #else - // Yes, there are such systems. This replacement relies on the fact that when - // we call x_realpath we only care about symlinks. - { - ssize_t len = readlink(path.c_str(), buffer, buffer_size - 1); - if (len != -1) { - buffer[len] = 0; - resolved = buffer; - } - } +# error No realpath function available #endif return resolved ? resolved : (return_empty_on_error ? "" : path); From 61ce8c44c5b1da0be7a8d10c014f1a85b7967433 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 30 Dec 2020 21:22:26 +0100 Subject: [PATCH 14/38] Only accept -f(no-)color-diagnostics for Clang If a non-Clang compiler gets -f(no-)color-diagnostics then bail out and just execute the compiler. The reason is that we don't include -f(no-)color-diagnostics in the hash so there can be a false cache hit in the following scenario: 1. ccache gcc -c example.c # adds a cache entry 2. ccache gcc -c example.c -fcolor-diagnostics # unexpectedly succeeds Closes: #740. --- src/argprocessing.cpp | 13 +++++++++++++ test/suites/color_diagnostics.bash | 17 ++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/argprocessing.cpp b/src/argprocessing.cpp index 1783f9f032..983ccdb456 100644 --- a/src/argprocessing.cpp +++ b/src/argprocessing.cpp @@ -688,6 +688,19 @@ process_arg(Context& ctx, return nullopt; } + if (config.compiler_type() != CompilerType::clang + && (args[i] == "-fcolor-diagnostics" + || args[i] == "-fno-color-diagnostics")) { + // Special case: If a non-Clang compiler gets -f(no-)color-diagnostics we'll + // bail out and just execute the compiler. The reason is that we don't + // include -f(no-)color-diagnostics in the hash so there can be a false + // cache hit in the following scenario: + // + // 1. ccache gcc -c example.c # adds a cache entry + // 2. ccache gcc -c example.c -fcolor-diagnostics # unexpectedly succeeds + return Statistic::unsupported_compiler_option; + } + if (args[i] == "-fcolor-diagnostics" || args[i] == "-fdiagnostics-color" || args[i] == "-fdiagnostics-color=always") { state.color_diagnostics = ColorDiagnostics::always; diff --git a/test/suites/color_diagnostics.bash b/test/suites/color_diagnostics.bash index 64c7d4f182..4ec99f4022 100644 --- a/test/suites/color_diagnostics.bash +++ b/test/suites/color_diagnostics.bash @@ -113,17 +113,32 @@ color_diagnostics_test() { expect_stat 'cache miss' 1 expect_stat 'cache hit (preprocessed)' 1 - # ------------------------------------------------------------------------- if $COMPILER_TYPE_GCC; then + # --------------------------------------------------------------------- TEST "-fcolor-diagnostics not accepted for GCC" generate_code 1 test.c + + if $CCACHE_COMPILE -fcolor-diagnostics -c test.c >&/dev/null; then + test_failed "-fcolor-diagnostics unexpectedly accepted by GCC" + fi + + # --------------------------------------------------------------------- + TEST "-fcolor-diagnostics not accepted for GCC for cached result" + + generate_code 1 test.c + + if ! $CCACHE_COMPILE -c test.c >&/dev/null; then + test_failed "unknown error compiling" + fi + if $CCACHE_COMPILE -fcolor-diagnostics -c test.c >&/dev/null; then test_failed "-fcolor-diagnostics unexpectedly accepted by GCC" fi fi while read -r case; do + # --------------------------------------------------------------------- TEST "Cache object shared across ${case} (run_second_cpp=$run_second_cpp)" color_diagnostics_generate_code test1.c From 4e878b9297b4d36e25fce48f40a2ad6fbb0508a0 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 3 Jan 2021 11:44:06 +0100 Subject: [PATCH 15/38] Deduce split dwarf filename from object file with zero or multiple dots (cherry picked from commit 5bcba58358ef2e88e1cc910583830c6830f6c712) --- src/argprocessing.cpp | 8 +------- test/suites/split_dwarf.bash | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/argprocessing.cpp b/src/argprocessing.cpp index 983ccdb456..330dc0eac3 100644 --- a/src/argprocessing.cpp +++ b/src/argprocessing.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -1069,12 +1069,6 @@ process_args(Context& ctx) } if (args_info.seen_split_dwarf) { - size_t pos = args_info.output_obj.rfind('.'); - if (pos == std::string::npos || pos == args_info.output_obj.size() - 1) { - LOG_RAW("Badly formed object filename"); - return Statistic::bad_compiler_arguments; - } - args_info.output_dwo = Util::change_extension(args_info.output_obj, ".dwo"); } diff --git a/test/suites/split_dwarf.bash b/test/suites/split_dwarf.bash index d8c38050fb..28a329345c 100644 --- a/test/suites/split_dwarf.bash +++ b/test/suites/split_dwarf.bash @@ -142,4 +142,38 @@ SUITE_split_dwarf() { elif [ ! -f reference.dwo ] && [ -f test.dwo ]; then test_failed ".dwo not missing" fi + + # ------------------------------------------------------------------------- + TEST "Object file without dot" + + $CCACHE_COMPILE -gsplit-dwarf -c test.c -o test + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache hit (preprocessed)' 0 + expect_stat 'cache miss' 1 + expect_exists test.dwo + + rm test.dwo + + $CCACHE_COMPILE -gsplit-dwarf -c test.c -o test + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache hit (preprocessed)' 0 + expect_stat 'cache miss' 1 + expect_exists test.dwo + + # ------------------------------------------------------------------------- + TEST "Object file with two dots" + + $CCACHE_COMPILE -gsplit-dwarf -c test.c -o test.x.y + expect_stat 'cache hit (direct)' 0 + expect_stat 'cache hit (preprocessed)' 0 + expect_stat 'cache miss' 1 + expect_exists test.x.dwo + + rm test.x.dwo + + $CCACHE_COMPILE -gsplit-dwarf -c test.c -o test.x.y + expect_stat 'cache hit (direct)' 1 + expect_stat 'cache hit (preprocessed)' 0 + expect_stat 'cache miss' 1 + expect_exists test.x.dwo } From 56d3ead4b35a2672bef84bb5bd08b4557b1f9580 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 3 Jan 2021 12:55:00 +0100 Subject: [PATCH 16/38] Capitalize log message with inode cache statistics --- src/InodeCache.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/InodeCache.cpp b/src/InodeCache.cpp index 93e15b338c..5e473ec7e8 100644 --- a/src/InodeCache.cpp +++ b/src/InodeCache.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -399,7 +399,7 @@ InodeCache::get(const std::string& path, } else { ++m_sr->misses; } - LOG("accumulated stats for inode cache: hits={}, misses={}, errors={}", + LOG("Accumulated stats for inode cache: hits={}, misses={}, errors={}", m_sr->hits.load(), m_sr->misses.load(), m_sr->errors.load()); From 20080cc3c9a673a7bb5d622070899710e27bbd33 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 3 Jan 2021 13:04:58 +0100 Subject: [PATCH 17/38] Improve log message when manifest entry already exists --- src/Manifest.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Manifest.cpp b/src/Manifest.cpp index 9ee87dcba7..f594ed1d79 100644 --- a/src/Manifest.cpp +++ b/src/Manifest.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors +// Copyright (C) 2009-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -587,6 +587,8 @@ put(const Config& config, } catch (const Error& e) { LOG("Error: {}", e.what()); } + } else { + LOG_RAW("The entry already exists in the manifest, not adding"); } return false; } From 66464308b5faf8ef42ffca2d61c648daa4ac4be7 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 3 Jan 2021 13:31:15 +0100 Subject: [PATCH 18/38] Improve log messages and comments related to retrieving results --- src/ResultRetriever.cpp | 58 ++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/ResultRetriever.cpp b/src/ResultRetriever.cpp index 77e044dd0e..1e45dbb4f9 100644 --- a/src/ResultRetriever.cpp +++ b/src/ResultRetriever.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -40,8 +40,13 @@ ResultRetriever::on_entry_start(uint32_t entry_number, uint64_t file_len, nonstd::optional raw_file) { - std::string dest_path; + LOG("Reading {} entry #{} {} ({} bytes)", + raw_file ? "raw" : "embedded", + entry_number, + Result::file_type_to_string(file_type), + file_len); + std::string dest_path; m_dest_file_type = file_type; switch (file_type) { @@ -50,6 +55,8 @@ ResultRetriever::on_entry_start(uint32_t entry_number, break; case FileType::dependency: + // Dependency file: Open destination file but accumulate data in m_dest_data + // and write it in on_entry_end. if (m_ctx.args_info.generating_dependencies) { dest_path = m_ctx.args_info.output_dep; m_dest_data.reserve(file_len); @@ -57,8 +64,10 @@ ResultRetriever::on_entry_start(uint32_t entry_number, break; case FileType::stderr_output: + // Stderr data: Don't open a destination file. Instead accumulate it in + // m_dest_data and write it in on_entry_end. m_dest_data.reserve(file_len); - return; + break; case FileType::coverage_unmangled: if (m_ctx.args_info.generating_coverage) { @@ -92,33 +101,27 @@ ResultRetriever::on_entry_start(uint32_t entry_number, break; } - if (dest_path.empty()) { - LOG_RAW("Not copying"); + if (file_type == FileType::stderr_output) { + // Written in on_entry_end. + } else if (dest_path.empty()) { + LOG_RAW("Not writing"); } else if (dest_path == "/dev/null") { - LOG_RAW("Not copying to /dev/null"); + LOG_RAW("Not writing to /dev/null"); + } else if (raw_file) { + Util::clone_hard_link_or_copy_file(m_ctx, *raw_file, dest_path, false); + + // Update modification timestamp to save the file from LRU cleanup (and, if + // hard-linked, to make the object file newer than the source file). + Util::update_mtime(*raw_file); } else { - LOG("Retrieving {} file #{} {} ({} bytes)", - raw_file ? "raw" : "embedded", - entry_number, - Result::file_type_to_string(file_type), - file_len); - - if (raw_file) { - Util::clone_hard_link_or_copy_file(m_ctx, *raw_file, dest_path, false); - - // Update modification timestamp to save the file from LRU cleanup (and, - // if hard-linked, to make the object file newer than the source file). - Util::update_mtime(*raw_file); - } else { - LOG("Copying to {}", dest_path); - m_dest_fd = Fd( - open(dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); - if (!m_dest_fd) { - throw Error( - "Failed to open {} for writing: {}", dest_path, strerror(errno)); - } - m_dest_path = dest_path; + LOG("Writing to {}", dest_path); + m_dest_fd = Fd( + open(dest_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0666)); + if (!m_dest_fd) { + throw Error( + "Failed to open {} for writing: {}", dest_path, strerror(errno)); } + m_dest_path = dest_path; } } @@ -144,6 +147,7 @@ void ResultRetriever::on_entry_end() { if (m_dest_file_type == FileType::stderr_output) { + LOG("Writing to file descriptor {}", STDERR_FILENO); Util::send_to_stderr(m_ctx, m_dest_data); } else if (m_dest_file_type == FileType::dependency && !m_dest_path.empty()) { write_dependency_file(); From e09543bbbdea86df6b9ba1dab386bb2636980d30 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 3 Jan 2021 13:39:57 +0100 Subject: [PATCH 19/38] Fix retrieval of object file when destination is /dev/null MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ResultRetriever::on_entry_data assumes that a destination file has been opened if the entry type is not stderr_output, but that’s incorrect since on_entry_start doesn’t open a destination file if it’s /dev/null. An assertion is triggered: ccache: ResultRetriever.cpp:129: virtual void ResultRetriever::on_entry_data(const uint8_t *, size_t): failed assertion: (m_dest_file_type == FileType::stderr_output && !m_dest_fd) || (m_dest_file_type != FileType::stderr_output && m_dest_fd) Fix this by letting on_entry_data handle the “destination file not opened” case and correcting the assert. Fixes #752. --- src/ResultRetriever.cpp | 5 ++--- test/suites/base.bash | 11 +++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/ResultRetriever.cpp b/src/ResultRetriever.cpp index 1e45dbb4f9..957bbee492 100644 --- a/src/ResultRetriever.cpp +++ b/src/ResultRetriever.cpp @@ -128,13 +128,12 @@ ResultRetriever::on_entry_start(uint32_t entry_number, void ResultRetriever::on_entry_data(const uint8_t* data, size_t size) { - ASSERT((m_dest_file_type == FileType::stderr_output && !m_dest_fd) - || (m_dest_file_type != FileType::stderr_output && m_dest_fd)); + ASSERT(!(m_dest_file_type == FileType::stderr_output && m_dest_fd)); if (m_dest_file_type == FileType::stderr_output || (m_dest_file_type == FileType::dependency && !m_dest_path.empty())) { m_dest_data.append(reinterpret_cast(data), size); - } else { + } else if (m_dest_fd) { try { Util::write_fd(*m_dest_fd, data, size); } catch (Error& e) { diff --git a/test/suites/base.bash b/test/suites/base.bash index 30c1c41f41..2131ac92a8 100644 --- a/test/suites/base.bash +++ b/test/suites/base.bash @@ -1085,6 +1085,17 @@ EOF CCACHE_PREFIX=`pwd`/empty-object-prefix $CCACHE_COMPILE -c test_empty_obj.c expect_stat 'compiler produced empty output' 1 + # ------------------------------------------------------------------------- + TEST "Output to /dev/null" + + $CCACHE_COMPILE -c test1.c + expect_stat 'cache hit (preprocessed)' 0 + expect_stat 'cache miss' 1 + + $CCACHE_COMPILE -c test1.c -o /dev/null + expect_stat 'cache hit (preprocessed)' 1 + expect_stat 'cache miss' 1 + # ------------------------------------------------------------------------- TEST "Caching stderr" From ea3216e43342a0577f8d7c29f06576d66b10c249 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 3 Jan 2021 15:33:24 +0100 Subject: [PATCH 20/38] =?UTF-8?q?Don=E2=80=99t=20capture=20=E2=80=9Cthis?= =?UTF-8?q?=E2=80=9D=20implicitly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids an “implicit capture of 'this' via '[=]' is deprecated” warning when building for C++20 and does no harm for earlier versions. --- src/Hash.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Hash.cpp b/src/Hash.cpp index ccc6f7bb31..61cc5a3e73 100644 --- a/src/Hash.cpp +++ b/src/Hash.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -106,7 +106,7 @@ bool Hash::hash_fd(int fd) { return Util::read_fd( - fd, [=](const void* data, size_t size) { hash(data, size); }); + fd, [this](const void* data, size_t size) { hash(data, size); }); } bool From 930e44d9561e76a90ec77ccb84a85985b1c23ef6 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sun, 3 Jan 2021 15:43:50 +0100 Subject: [PATCH 21/38] =?UTF-8?q?Configure=20nonstd::string=5Fview=20to=20?= =?UTF-8?q?don=E2=80=99t=20fall=20back=20to=20std::string=5Fview?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes the code base use nonstd::string regardless of the C++ target version, which avoids some compatibilty issues. This decision can be revisited in the future when C++17 is the lower bar. Closes #749. --- src/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a88efc985e..30a2f92c26 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -52,6 +52,9 @@ if(WIN32) endif() add_library(ccache_lib STATIC ${source_files}) +target_compile_definitions( + ccache_lib PUBLIC -Dnssv_CONFIG_SELECT_STRING_VIEW=nssv_STRING_VIEW_NONSTD +) if(WIN32) target_link_libraries(ccache_lib PRIVATE ws2_32 "psapi") From 4878d8e538d6800f0e8a6ba6269954492191572c Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 18:49:10 +0100 Subject: [PATCH 22/38] Add hint on how to link statically with libzstd As mentioned in #750. --- doc/INSTALL.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/INSTALL.md b/doc/INSTALL.md index 77c2a72342..8a22618dff 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -20,6 +20,8 @@ To build ccache you need: from the Internet and unpack it in the local binary tree. Ccache will then be linked statically to the locally built libzstd. + To link libzstd statically you can use `-DZSTD_LIBRARY=/path/to/libzstd.a`. + Optional: - GNU Bourne Again SHell (bash) for tests. From 8223ed38abd9cf75ab75cb19dc2081c44bf568d0 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 19:23:03 +0100 Subject: [PATCH 23/38] Make Util::make_relative_path able to find matches for canonical path (#760) Scenario: - /tmp is a symlink to /private/tmp. - Both apparent and actual CWD is /private/tmp/dir. - A path (e.g. the object file) on the command line is /tmp/dir/file.o. - Base directory is set up to match /tmp/dir/file.o. Ccache then rewrites /tmp/dir/file.o into ../../private/tmp/dir/file.o, which is correct but not optimal since ./file.o would be a better relative path. Especially on macOS where, for unknown reasons, the kernel sometimes disallows opening a file like ../../private/tmp/dir/file.o for writing. Improve this by letting Util::make_relative_path try to match real_path(path) against the CWDs and choose the best match. Closes #724. --- src/Util.cpp | 38 +++++++++++++++++++++++--------------- unittest/test_Util.cpp | 29 ++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/Util.cpp b/src/Util.cpp index edeba42325..4be232edcb 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -895,28 +895,36 @@ make_relative_path(const std::string& base_dir, // The algorithm for computing relative paths below only works for existing // paths. If the path doesn't exist, find the first ancestor directory that // does exist and assemble the path again afterwards. - string_view original_path = path; - std::string path_suffix; + + std::vector relpath_candidates; + const auto original_path = path; Stat path_stat; while (!(path_stat = Stat::stat(std::string(path)))) { path = Util::dir_name(path); } - path_suffix = std::string(original_path.substr(path.length())); + const auto path_suffix = std::string(original_path.substr(path.length())); + const auto real_path = Util::real_path(std::string(path)); - std::string path_str(path); - std::string normalized_path = Util::normalize_absolute_path(path_str); - std::vector relpath_candidates = { - Util::get_relative_path(actual_cwd, normalized_path), - }; - if (apparent_cwd != actual_cwd) { - relpath_candidates.emplace_back( - Util::get_relative_path(apparent_cwd, normalized_path)); - // Move best (= shortest) match first: - if (relpath_candidates[0].length() > relpath_candidates[1].length()) { - std::swap(relpath_candidates[0], relpath_candidates[1]); + const auto add_relpath_candidates = [&](nonstd::string_view path) { + const std::string normalized_path = Util::normalize_absolute_path(path); + relpath_candidates.push_back( + Util::get_relative_path(actual_cwd, normalized_path)); + if (apparent_cwd != actual_cwd) { + relpath_candidates.emplace_back( + Util::get_relative_path(apparent_cwd, normalized_path)); } + }; + add_relpath_candidates(path); + if (real_path != path) { + add_relpath_candidates(real_path); } + // Find best (i.e. shortest existing) match: + std::sort(relpath_candidates.begin(), + relpath_candidates.end(), + [](const std::string& path1, const std::string& path2) { + return path1.length() < path2.length(); + }); for (const auto& relpath : relpath_candidates) { if (Stat::stat(relpath).same_inode_as(path_stat)) { return relpath + path_suffix; diff --git a/unittest/test_Util.cpp b/unittest/test_Util.cpp index 0d3cae3675..ce3c1115de 100644 --- a/unittest/test_Util.cpp +++ b/unittest/test_Util.cpp @@ -564,7 +564,6 @@ TEST_CASE("Util::is_dir_separator") #endif } -#ifndef _WIN32 TEST_CASE("Util::make_relative_path") { using Util::make_relative_path; @@ -573,11 +572,17 @@ TEST_CASE("Util::make_relative_path") const std::string cwd = Util::get_actual_cwd(); const std::string actual_cwd = FMT("{}/d", cwd); +#ifdef _WIN32 + const std::string apparent_cwd = actual_cwd; +#else const std::string apparent_cwd = FMT("{}/s", cwd); +#endif REQUIRE(Util::create_dir("d")); +#ifndef _WIN32 REQUIRE(symlink("d", "s") == 0); - REQUIRE(chdir("s") == 0); +#endif + REQUIRE(chdir("d") == 0); Util::setenv("PWD", apparent_cwd); SUBCASE("No base directory") @@ -587,22 +592,40 @@ TEST_CASE("Util::make_relative_path") SUBCASE("Path matches neither actual nor apparent CWD") { +#ifdef _WIN32 + CHECK(make_relative_path("C:/", "C:/a", "C:/b", "C:/x") == "C:/x"); +#else CHECK(make_relative_path("/", "/a", "/b", "/x") == "/x"); +#endif } SUBCASE("Match of actual CWD") { +#ifdef _WIN32 + CHECK( + make_relative_path( + actual_cwd.substr(0, 3), actual_cwd, apparent_cwd, actual_cwd + "/x") + == "./x"); +#else CHECK(make_relative_path("/", actual_cwd, apparent_cwd, actual_cwd + "/x") == "./x"); +#endif } +#ifndef _WIN32 SUBCASE("Match of apparent CWD") { CHECK(make_relative_path("/", actual_cwd, apparent_cwd, apparent_cwd + "/x") == "./x"); } -} + + SUBCASE("Match if using resolved (using realpath(3)) path") + { + CHECK(make_relative_path("/", actual_cwd, actual_cwd, apparent_cwd + "/x") + == "./x"); + } #endif +} TEST_CASE("Util::matches_dir_prefix_or_file") { From 64df37ad89af7d668b1125aa0580b3823c04138c Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Mon, 4 Jan 2021 13:55:13 +0100 Subject: [PATCH 24/38] Remove redundant initialization of std::string variables --- src/Config.hpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Config.hpp b/src/Config.hpp index a5945facc7..1d8a8e85e5 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -133,34 +133,34 @@ class Config std::string m_secondary_config_path; bool m_absolute_paths_in_stderr = false; - std::string m_base_dir = ""; + std::string m_base_dir; std::string m_cache_dir; - std::string m_compiler = ""; + std::string m_compiler; std::string m_compiler_check = "mtime"; CompilerType m_compiler_type = CompilerType::auto_guess; bool m_compression = true; int8_t m_compression_level = 0; // Use default level - std::string m_cpp_extension = ""; + std::string m_cpp_extension; bool m_debug = false; bool m_depend_mode = false; bool m_direct_mode = true; bool m_disable = false; - std::string m_extra_files_to_hash = ""; + std::string m_extra_files_to_hash; bool m_file_clone = false; bool m_hard_link = false; bool m_hash_dir = true; - std::string m_ignore_headers_in_manifest = ""; - std::string m_ignore_options = ""; + std::string m_ignore_headers_in_manifest; + std::string m_ignore_options; bool m_inode_cache = false; bool m_keep_comments_cpp = false; double m_limit_multiple = 0.8; - std::string m_log_file = ""; + std::string m_log_file; uint64_t m_max_files = 0; uint64_t m_max_size = 5ULL * 1000 * 1000 * 1000; - std::string m_path = ""; + std::string m_path; bool m_pch_external_checksum = false; - std::string m_prefix_command = ""; - std::string m_prefix_command_cpp = ""; + std::string m_prefix_command; + std::string m_prefix_command_cpp; bool m_read_only = false; bool m_read_only_direct = false; bool m_recache = false; From a181d44d19e6cd35d186eb5c61cd183fcfb3f9e5 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Mon, 4 Jan 2021 14:30:32 +0100 Subject: [PATCH 25/38] Add debug_dir setting for specifying a directory for debug files This makes it possible to store debug files outside a transient build directory. As a bonus, it also allows for getting debug files when the object file is /dev/null. --- doc/MANUAL.adoc | 17 +++++++++++++++ src/Config.cpp | 12 ++++++++++- src/Config.hpp | 8 +++++++ src/ccache.cpp | 46 +++++++++++++++++++++++----------------- unittest/test_Config.cpp | 5 ++++- 5 files changed, 66 insertions(+), 22 deletions(-) diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 062f041c55..1862985617 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -550,6 +550,19 @@ See the http://zstd.net[Zstandard documentation] for more information. _<<_cache_debugging,Cache debugging>>_ for more information. The default is false. +[[config_debug_dir]] *debug_dir* (*CCACHE_DEBUGDIR*):: + + Specifies where to write per-object debug files if the _<>_ is enabled. If set to the empty string, the files will be written + next to the object file. If set to a directory, the debug files will be + written with full absolute paths in that directory, creating it if needed. + The default is the empty string. + + For example, if *debug_dir* is set to `/example`, the current working + directory is `/home/user` and the object file is `build/output.o` then the + debug log will be written to `/example/home/user/build/output.o.ccache-log`. + See also _<<_cache_debugging,Cache debugging>>_. + [[config_depend_mode]] *depend_mode* (*CCACHE_DEPEND* or *CCACHE_NODEPEND*, see _<<_boolean_values,Boolean values>>_ above):: If true, the depend mode will be used. The default is false. See @@ -1243,6 +1256,10 @@ Log for this object file. |============================================================================== +If <> (environment variable *CCACHE_DEBUGDIR*) is +set, the files above will be written to that directory with full absolute paths +instead of next to the object file. + In the direct mode, ccache uses the 160 bit BLAKE3 hash of the *ccache-input-c* + *ccache-input-d* data (where *+* means concatenation), while the *ccache-input-c* + *ccache-input-p* data is used in the preprocessor mode. diff --git a/src/Config.cpp b/src/Config.cpp index a2e09ca1eb..4866294c79 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Joel Rosdahl and other contributors +// Copyright (C) 2019-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -52,6 +52,7 @@ enum class ConfigItem { compression_level, cpp_extension, debug, + debug_dir, depend_mode, direct_mode, disable, @@ -92,6 +93,7 @@ const std::unordered_map k_config_key_table = { {"compression_level", ConfigItem::compression_level}, {"cpp_extension", ConfigItem::cpp_extension}, {"debug", ConfigItem::debug}, + {"debug_dir", ConfigItem::debug_dir}, {"depend_mode", ConfigItem::depend_mode}, {"direct_mode", ConfigItem::direct_mode}, {"disable", ConfigItem::disable}, @@ -133,6 +135,7 @@ const std::unordered_map k_env_variable_table = { {"COMPRESSLEVEL", "compression_level"}, {"CPP2", "run_second_cpp"}, {"DEBUG", "debug"}, + {"DEBUGDIR", "debug_dir"}, {"DEPEND", "depend_mode"}, {"DIR", "cache_dir"}, {"DIRECT", "direct_mode"}, @@ -546,6 +549,9 @@ Config::get_string_value(const std::string& key) const case ConfigItem::debug: return format_bool(m_debug); + case ConfigItem::debug_dir: + return m_debug_dir; + case ConfigItem::depend_mode: return format_bool(m_depend_mode); @@ -756,6 +762,10 @@ Config::set_item(const std::string& key, m_debug = parse_bool(value, env_var_key, negate); break; + case ConfigItem::debug_dir: + m_debug_dir = value; + break; + case ConfigItem::depend_mode: m_depend_mode = parse_bool(value, env_var_key, negate); break; diff --git a/src/Config.hpp b/src/Config.hpp index 1d8a8e85e5..95a1b68a83 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -51,6 +51,7 @@ class Config int8_t compression_level() const; const std::string& cpp_extension() const; bool debug() const; + const std::string& debug_dir() const; bool depend_mode() const; bool direct_mode() const; bool disable() const; @@ -142,6 +143,7 @@ class Config int8_t m_compression_level = 0; // Use default level std::string m_cpp_extension; bool m_debug = false; + std::string m_debug_dir; bool m_depend_mode = false; bool m_direct_mode = true; bool m_disable = false; @@ -243,6 +245,12 @@ Config::debug() const return m_debug; } +inline const std::string& +Config::debug_dir() const +{ + return m_debug_dir; +} + inline bool Config::depend_mode() const { diff --git a/src/ccache.cpp b/src/ccache.cpp index 11d6164498..d18ba0f916 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -1,5 +1,5 @@ // Copyright (C) 2002-2007 Andrew Tridgell -// Copyright (C) 2009-2020 Joel Rosdahl and other contributors +// Copyright (C) 2009-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -232,10 +232,25 @@ clean_up_internal_tempdir(const Config& config) }); } +static std::string +prepare_debug_path(const std::string& debug_dir, + const std::string& output_obj, + string_view suffix) +{ + const std::string prefix = + debug_dir.empty() ? output_obj : debug_dir + Util::real_path(output_obj); + try { + Util::ensure_dir_exists(Util::dir_name(prefix)); + } catch (Error&) { + // Ignore since we can't handle an error in another way in this context. The + // caller takes care of logging when trying to open the path for writing. + } + return FMT("{}.ccache-{}", prefix, suffix); +} + static void init_hash_debug(Context& ctx, Hash& hash, - string_view obj_path, char type, string_view section_name, FILE* debug_text_file) @@ -244,7 +259,8 @@ init_hash_debug(Context& ctx, return; } - std::string path = FMT("{}.ccache-input-{}", obj_path, type); + const auto path = prepare_debug_path( + ctx.config.debug_dir(), ctx.args_info.output_obj, FMT("input-{}", type)); File debug_binary_file(path, "wb"); if (debug_binary_file) { hash.enable_debug(section_name, debug_binary_file.get(), debug_text_file); @@ -2172,8 +2188,8 @@ finalize_at_exit(Context& ctx) // Dump log buffer last to not lose any logs. if (ctx.config.debug() && !ctx.args_info.output_obj.empty()) { - const auto path = FMT("{}.ccache-log", ctx.args_info.output_obj); - Logging::dump_log(path); + Logging::dump_log(prepare_debug_path( + ctx.config.debug_dir(), ctx.args_info.output_obj, "log")); } } @@ -2321,7 +2337,8 @@ do_cache_compilation(Context& ctx, const char* const* argv) MTR_META_THREAD_NAME(ctx.args_info.output_obj.c_str()); if (ctx.config.debug()) { - std::string path = FMT("{}.ccache-input-text", ctx.args_info.output_obj); + const auto path = prepare_debug_path( + ctx.config.debug_dir(), ctx.args_info.output_obj, "input-text"); File debug_text_file(path, "w"); if (debug_text_file) { ctx.hash_debug_files.push_back(std::move(debug_text_file)); @@ -2335,8 +2352,7 @@ do_cache_compilation(Context& ctx, const char* const* argv) : nullptr; Hash common_hash; - init_hash_debug( - ctx, common_hash, ctx.args_info.output_obj, 'c', "COMMON", debug_text_file); + init_hash_debug(ctx, common_hash, 'c', "COMMON", debug_text_file); MTR_BEGIN("hash", "common_hash"); hash_common_info( @@ -2345,12 +2361,7 @@ do_cache_compilation(Context& ctx, const char* const* argv) // Try to find the hash using the manifest. Hash direct_hash = common_hash; - init_hash_debug(ctx, - direct_hash, - ctx.args_info.output_obj, - 'd', - "DIRECT MODE", - debug_text_file); + init_hash_debug(ctx, direct_hash, 'd', "DIRECT MODE", debug_text_file); Args args_to_hash = processed.preprocessor_args; args_to_hash.push_back(processed.extra_args_to_hash); @@ -2394,12 +2405,7 @@ do_cache_compilation(Context& ctx, const char* const* argv) // Find the hash using the preprocessed output. Also updates // ctx.included_files. Hash cpp_hash = common_hash; - init_hash_debug(ctx, - cpp_hash, - ctx.args_info.output_obj, - 'p', - "PREPROCESSOR MODE", - debug_text_file); + init_hash_debug(ctx, cpp_hash, 'p', "PREPROCESSOR MODE", debug_text_file); MTR_BEGIN("hash", "cpp_hash"); result_name = calculate_result_name( diff --git a/unittest/test_Config.cpp b/unittest/test_Config.cpp index 3661c692d5..46e56a33b6 100644 --- a/unittest/test_Config.cpp +++ b/unittest/test_Config.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2011-2020 Joel Rosdahl and other contributors +// Copyright (C) 2011-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -48,6 +48,7 @@ TEST_CASE("Config: default values") CHECK(config.compression_level() == 0); CHECK(config.cpp_extension().empty()); CHECK(!config.debug()); + CHECK(config.debug_dir().empty()); CHECK(!config.depend_mode()); CHECK(config.direct_mode()); CHECK(!config.disable()); @@ -375,6 +376,7 @@ TEST_CASE("Config::visit_items") "compression_level = 8\n" "cpp_extension = ce\n" "debug = false\n" + "debug_dir = /dd\n" "depend_mode = true\n" "direct_mode = false\n" "disable = true\n" @@ -431,6 +433,7 @@ TEST_CASE("Config::visit_items") "(test.conf) compression_level = 8", "(test.conf) cpp_extension = ce", "(test.conf) debug = false", + "(test.conf) debug_dir = /dd", "(test.conf) depend_mode = true", "(test.conf) direct_mode = false", "(test.conf) disable = true", From 69bb827346996a0bb5bc3720ad76ce8543c851bf Mon Sep 17 00:00:00 2001 From: Nicholas Hutchinson Date: Wed, 6 Jan 2021 18:53:16 +0000 Subject: [PATCH 26/38] CI: Add VS2019 build jobs (#757) Add VS2019 build jobs that use clang for the test suite. There are many test failures on Windows, but these are ignored for now. Tweak CMake build scripts: - Fix CI build type handling for MSVC (recognise `/NDEBUG` and not just `-DNDEBUG`) - Fix incorrect warnings-as-errors flag for MSVC - Suppress an additional conversion warning on MSVC --- .github/workflows/build.yaml | 67 ++++++++++++++++++++++++++++++++++-- cmake/CIBuildType.cmake | 4 +-- cmake/DevModeWarnings.cmake | 3 +- 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5e5ee8ae2e..a5cbadd6dd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,6 +7,10 @@ env: CTEST_OUTPUT_ON_FAILURE: ON VERBOSE: 1 +defaults: + run: + shell: bash + jobs: build_and_test: env: @@ -191,6 +195,30 @@ jobs: RUN_TESTS: unittest-in-wine apt_get: elfutils mingw-w64 wine + - name: Windows VS2019 32-bit + os: windows-2019 + msvc_arch: x64_x86 + allow_test_failures: true # For now, don't fail the build on failure + CC: cl + CXX: cl + ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Ninja + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + # -mno-incremental-linker-compatible: reproducible object files + TEST_CC: clang -target i686-pc-windows-msvc -mno-incremental-linker-compatible + + - name: Windows VS2019 64-bit + os: windows-2019 + msvc_arch: x64 + allow_test_failures: true # For now, don't fail the build on failure + CC: cl + CXX: cl + ENABLE_CACHE_CLEANUP_TESTS: 1 + CMAKE_GENERATOR: Ninja + CMAKE_PARAMS: -DCMAKE_BUILD_TYPE=CI -DZSTD_FROM_INTERNET=ON + # -mno-incremental-linker-compatible: reproducible object files + TEST_CC: clang -target x86_64-pc-windows-msvc -mno-incremental-linker-compatible + - name: Clang address & UB sanitizer os: ubuntu-20.04 CC: clang @@ -258,13 +286,38 @@ jobs: if: matrix.config.apt_get != '' run: sudo apt-get update && sudo apt-get install ${{ matrix.config.apt_get }} + - name: Prepare Windows Environment (Visual Studio) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1.5.0 + with: + arch: ${{ matrix.config.msvc_arch }} + + - name: Prepare Windows Environment (Clang) + if: runner.os == 'Windows' + shell: powershell + run: | + $ErrorActionPreference = 'Stop' + + # The test suite currently requires that the compiler specified by the "CC" + # env variable is on a path without spaces. Provide that by creating a + # junction from ~/opt/llvm to the Visual Studio path. + $null = New-Item ` + -Path "${HOME}\opt\llvm" ` + -ItemType Junction ` + -Target "${env:VCINSTALLDIR}\Tools\Llvm\x64" ` + -Force + "Path=${HOME}\opt\llvm\bin;${env:Path}" | ` + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Build and test + id: build-and-test env: ASAN_OPTIONS: ${{ matrix.config.ASAN_OPTIONS }} BUILDDIR: ${{ matrix.config.BUILDDIR }} CC: ${{ matrix.config.CC }} CCACHE_LOC: ${{ matrix.config.CCACHE_LOC }} CFLAGS: ${{ matrix.config.CFLAGS }} + CMAKE_GENERATOR: ${{ matrix.config.CMAKE_GENERATOR }} CMAKE_PARAMS: ${{ matrix.config.CMAKE_PARAMS }} CXX: ${{ matrix.config.CXX }} CXXFLAGS: ${{ matrix.config.CXXFLAGS }} @@ -273,15 +326,23 @@ jobs: LDFLAGS: ${{ matrix.config.LDFLAGS }} RUN_TESTS: ${{ matrix.config.RUN_TESTS }} SPECIAL: ${{ matrix.config.SPECIAL }} - run: ci/build + TEST_CC: ${{ matrix.config.TEST_CC }} + run: | + rc=0 + ci/build || rc=$? + echo "::set-output name=exit_status::$rc" + exit $rc + # Ctest exits with a return code of `8` on test failure. + continue-on-error: ${{ matrix.config.allow_test_failures == true && + steps.build-and-test.outputs.exit_status == 8 }} - name: Collect testdir from failed tests - if: failure() + if: failure() || steps.build-and-test.outcome == 'failure' run: ci/collect-testdir # TODO: in case of build-and-verify-*package the BUILDDIR is set within those scripts. - name: Upload testdir from failed tests - if: failure() + if: failure() || steps.build-and-test.outcome == 'failure' uses: actions/upload-artifact@v2 with: name: ${{ matrix.config.name }} - testdir.tar.xz diff --git a/cmake/CIBuildType.cmake b/cmake/CIBuildType.cmake index 963bc5007b..e72161414d 100644 --- a/cmake/CIBuildType.cmake +++ b/cmake/CIBuildType.cmake @@ -25,7 +25,7 @@ set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel CI." FORCE) -string(REPLACE -DNDEBUG "" CMAKE_CXX_FLAGS_CI ${CMAKE_CXX_FLAGS_CI}) -string(REPLACE -DNDEBUG "" CMAKE_C_FLAGS_CI ${CMAKE_C_FLAGS_CI}) +string(REGEX REPLACE "[/-]DNDEBUG" "" CMAKE_CXX_FLAGS_CI ${CMAKE_CXX_FLAGS_CI}) +string(REGEX REPLACE "[/-]DNDEBUG" "" CMAKE_C_FLAGS_CI ${CMAKE_C_FLAGS_CI}) string(STRIP ${CMAKE_CXX_FLAGS_CI} CMAKE_CXX_FLAGS_CI) string(STRIP ${CMAKE_C_FLAGS_CI} CMAKE_C_FLAGS_CI) diff --git a/cmake/DevModeWarnings.cmake b/cmake/DevModeWarnings.cmake index f578a9bcb7..7ff5411b89 100644 --- a/cmake/DevModeWarnings.cmake +++ b/cmake/DevModeWarnings.cmake @@ -115,7 +115,7 @@ elseif(MSVC) string(REGEX REPLACE "/W[0-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") if(WARNINGS_AS_ERRORS) - list(APPEND CCACHE_COMPILER_WARNINGS /WE) + list(APPEND CCACHE_COMPILER_WARNINGS /WX) endif() list( @@ -126,6 +126,7 @@ elseif(MSVC) /wd5105 # Conversion warnings: /wd4244 + /wd4245 /wd4267 # Assignment in conditional: /wd4706 From a02746cfc325d7508e1b12528fa7962f7098eb48 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 19:54:53 +0100 Subject: [PATCH 27/38] Tweak build.yaml --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a5cbadd6dd..0e096f23d6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -286,21 +286,21 @@ jobs: if: matrix.config.apt_get != '' run: sudo apt-get update && sudo apt-get install ${{ matrix.config.apt_get }} - - name: Prepare Windows Environment (Visual Studio) + - name: Prepare Windows environment (Visual Studio) if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1.5.0 with: arch: ${{ matrix.config.msvc_arch }} - - name: Prepare Windows Environment (Clang) + - name: Prepare Windows environment (Clang) if: runner.os == 'Windows' shell: powershell run: | $ErrorActionPreference = 'Stop' - # The test suite currently requires that the compiler specified by the "CC" - # env variable is on a path without spaces. Provide that by creating a - # junction from ~/opt/llvm to the Visual Studio path. + # The test suite currently requires that the compiler specified by the + # "CC" environment variable is on a path without spaces. Provide that + # by creating a junction from ~/opt/llvm to the Visual Studio path. $null = New-Item ` -Path "${HOME}\opt\llvm" ` -ItemType Junction ` @@ -332,7 +332,7 @@ jobs: ci/build || rc=$? echo "::set-output name=exit_status::$rc" exit $rc - # Ctest exits with a return code of `8` on test failure. + # CTest exits with return code 8 on test failure. continue-on-error: ${{ matrix.config.allow_test_failures == true && steps.build-and-test.outputs.exit_status == 8 }} From d62ce304f1c323cb3ca4e716069c033931cd50a7 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Thu, 31 Dec 2020 19:57:26 +0100 Subject: [PATCH 28/38] Adapt to the ccache code style --- src/TemporaryFile.cpp | 9 ++-- src/third_party/CMakeLists.txt | 4 +- unittest/CMakeLists.txt | 4 +- unittest/test_bsdmkstemp.cpp | 85 ++++++++++++++++++++-------------- 4 files changed, 57 insertions(+), 45 deletions(-) diff --git a/src/TemporaryFile.cpp b/src/TemporaryFile.cpp index ca0c8374f1..feaa5f1823 100644 --- a/src/TemporaryFile.cpp +++ b/src/TemporaryFile.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -50,11 +50,12 @@ TemporaryFile::TemporaryFile(string_view path_prefix) { Util::ensure_dir_exists(Util::dir_name(path)); #ifdef _WIN32 - // MSVC lacks mkstemp() and [mingw-w64's implementation][1] is problematic, as + // MSVC lacks mkstemp() and Mingw-w64's implementation[1] is problematic, as // it can reuse the names of recently-deleted files unless the caller // remembers to call srand(). - // [1]: - // https://github.com/Alexpux/mingw-w64/blob/d0d7f784833bbb0b2d279310ddc6afb52fe47a46/mingw-w64-crt/misc/mkstemp.c + + // [1]: fd = Fd(bsd_mkstemp(&path[0])); #else fd = Fd(mkstemp(&path[0])); diff --git a/src/third_party/CMakeLists.txt b/src/third_party/CMakeLists.txt index 977cff60cc..c361974a44 100644 --- a/src/third_party/CMakeLists.txt +++ b/src/third_party/CMakeLists.txt @@ -6,8 +6,8 @@ else() target_compile_definitions(third_party_lib PUBLIC -DSTATIC_GETOPT) endif() -if (WIN32) - target_sources(third_party_lib PRIVATE win32/mktemp.c) +if(WIN32) + target_sources(third_party_lib PRIVATE win32/mktemp.c) endif () if(ENABLE_TRACING) diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index b29fd653e5..65401041a4 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -27,9 +27,7 @@ if(INODE_CACHE_SUPPORTED) endif() if(WIN32) - list(APPEND source_files - test_bsdmkstemp.cpp - test_Win32Util.cpp) + list(APPEND source_files test_bsdmkstemp.cpp test_Win32Util.cpp) endif() add_executable(unittest ${source_files}) diff --git a/unittest/test_bsdmkstemp.cpp b/unittest/test_bsdmkstemp.cpp index 0237f5c80d..021c73d589 100644 --- a/unittest/test_bsdmkstemp.cpp +++ b/unittest/test_bsdmkstemp.cpp @@ -1,4 +1,23 @@ +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors +// +// See doc/AUTHORS.adoc for a complete list of contributors. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + #include "../src/Fd.hpp" +#include "../src/Finalizer.hpp" #include "TestUtil.hpp" #include "third_party/doctest.h" @@ -19,7 +38,7 @@ class ScopedHANDLE public: ScopedHANDLE() = default; - explicit ScopedHANDLE(HANDLE h) : h_(h) + explicit ScopedHANDLE(HANDLE handle) : m_handle(handle) { } @@ -29,74 +48,68 @@ class ScopedHANDLE ~ScopedHANDLE() { - if (h_ != INVALID_HANDLE_VALUE) { - CloseHandle(h_); + if (m_handle != INVALID_HANDLE_VALUE) { + CloseHandle(m_handle); } } ScopedHANDLE& operator=(ScopedHANDLE rhs) { - std::swap(h_, rhs.h_); + std::swap(m_handle, rhs.m_handle); return *this; } explicit operator bool() const { - return h_ != INVALID_HANDLE_VALUE; + return m_handle != INVALID_HANDLE_VALUE; } HANDLE get() const { - return h_; + return m_handle; } HANDLE release() { - HANDLE h = h_; - h_ = INVALID_HANDLE_VALUE; - return h; + HANDLE handle = m_handle; + m_handle = INVALID_HANDLE_VALUE; + return handle; } private: - HANDLE h_ = INVALID_HANDLE_VALUE; + HANDLE m_handle = INVALID_HANDLE_VALUE; }; } // namespace -TEST_SUITE_BEGIN("thirdparty"); +TEST_SUITE_BEGIN("bsd_mkstemp"); -TEST_CASE("thirdparty::bsd_mkstemp") +TEST_CASE("bsd_mkstemp") { TestContext test_context; - static int rand_iter; + static uint16_t rand_iter; rand_iter = 0; bsd_mkstemp_set_random_source([](void* buf, size_t nbytes) { std::fill_n( static_cast(buf), nbytes / sizeof(uint16_t), rand_iter); - rand_iter++; + ++rand_iter; }); - struct Cleanup - { - ~Cleanup() - { - bsd_mkstemp_set_random_source(nullptr); - } - } cleanup; + Finalizer reset_random_source([] { bsd_mkstemp_set_random_source(nullptr); }); SUBCASE("successful") { - std::string path("XXXXXX"); + std::string path = "XXXXXX"; CHECK_MESSAGE(Fd(bsd_mkstemp(&path[0])), "errno=" << errno); CHECK(path == "AAAAAA"); } - SUBCASE("existing_file") + SUBCASE("existing file") { CHECK_MESSAGE(ScopedHANDLE(CreateFileA("AAAAAA", GENERIC_READ | GENERIC_WRITE, @@ -107,12 +120,12 @@ TEST_CASE("thirdparty::bsd_mkstemp") nullptr)), "errno=" << errno); - std::string path("XXXXXX"); + std::string path = "XXXXXX"; CHECK_MESSAGE(Fd(bsd_mkstemp(&path[0])), "errno=" << errno); CHECK(path == "BBBBBB"); } - SUBCASE("existing_file_pending_delete") + SUBCASE("existing file, pending delete") { ScopedHANDLE h; CHECK_MESSAGE( @@ -133,34 +146,34 @@ TEST_CASE("thirdparty::bsd_mkstemp") h.get(), FileDispositionInfo, &info, sizeof(info)), "errno=" << errno); - std::string path("XXXXXX"); + std::string path = "XXXXXX"; CHECK_MESSAGE(Fd(bsd_mkstemp(&path[0])), "errno=" << errno); CHECK(path == "BBBBBB"); } - SUBCASE("existing_dir") + SUBCASE("existing directory") { CHECK_MESSAGE(CreateDirectoryA("AAAAAA", nullptr), "errno=" << errno); - std::string path("XXXXXX"); + std::string path = "XXXXXX"; CHECK_MESSAGE(Fd(bsd_mkstemp(&path[0])), "errno=" << errno); CHECK(path == "BBBBBB"); } SUBCASE("permission denied") { - auto makeACL = [](const char* aclString) { + auto make_ACL = [](const char* acl_string) { PSECURITY_DESCRIPTOR desc = nullptr; ConvertStringSecurityDescriptorToSecurityDescriptorA( - aclString, SDDL_REVISION_1, &desc, nullptr); + acl_string, SDDL_REVISION_1, &desc, nullptr); return std::shared_ptr( static_cast(desc), &LocalFree); }; - // Create a directory with a contrived ACL that denies creation of new - // files and directories to the "Everybody" (WD) group. + // Create a directory with a contrived ACL that denies creation of new files + // and directories to the "Everybody" (WD) group. std::shared_ptr desc; - CHECK_MESSAGE((desc = makeACL("D:(D;;DCLCRPCR;;;WD)(A;;FA;;;WD)")), + CHECK_MESSAGE((desc = make_ACL("D:(D;;DCLCRPCR;;;WD)(A;;FA;;;WD)")), "errno=" << errno); SECURITY_ATTRIBUTES attrs{}; @@ -169,8 +182,8 @@ TEST_CASE("thirdparty::bsd_mkstemp") CHECK_MESSAGE(CreateDirectoryA("my_readonly_dir", &attrs), "errno=" << errno); - // Sanity check that we cannot write to this directory. (E.g. Wine - // doesn't appear to emulate Windows ACLs properly when run under root.) + // Sanity check that we cannot write to this directory. (E.g. Wine doesn't + // appear to emulate Windows ACLs properly when run under root.) bool broken_acls = static_cast(ScopedHANDLE( CreateFileA("my_readonly_dir/.writable", GENERIC_WRITE, @@ -181,7 +194,7 @@ TEST_CASE("thirdparty::bsd_mkstemp") nullptr))); if (!broken_acls) { - std::string path("my_readonly_dir/XXXXXX"); + std::string path = "my_readonly_dir/XXXXXX"; CHECK(!Fd(bsd_mkstemp(&path[0]))); CHECK(errno == EACCES); } else { From 3a9a7f3b4ac92d77ea3778e9ab07552801c678f9 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 19:59:27 +0100 Subject: [PATCH 29/38] Upgrade to zstd 1.4.8 --- cmake/Findzstd.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Findzstd.cmake b/cmake/Findzstd.cmake index 848348fd5f..0044937b3e 100644 --- a/cmake/Findzstd.cmake +++ b/cmake/Findzstd.cmake @@ -6,7 +6,7 @@ if(ZSTD_FROM_INTERNET) # Although ${zstd_FIND_VERSION} was requested, let's download a newer version. # Note: The directory structure has changed in 1.3.0; we only support 1.3.0 # and newer. - set(zstd_version "1.4.5") + set(zstd_version "1.4.8") set(zstd_url https://github.com/facebook/zstd/archive/v${zstd_version}.tar.gz) set(zstd_dir ${CMAKE_BINARY_DIR}/zstd-${zstd_version}) From 251396f2aae33bcc200d8f5c9a69ce98a66334a9 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 20:01:34 +0100 Subject: [PATCH 30/38] Upgrade to doctest 2.4.4 Version 2.4.4 includes the fix in PR #743 for issue #731. --- LICENSE.adoc | 4 +- src/third_party/doctest.h | 332 ++++++++++++++++++++++++++------------ 2 files changed, 230 insertions(+), 106 deletions(-) diff --git a/LICENSE.adoc b/LICENSE.adoc index a9e5b9da13..fa1f32095c 100644 --- a/LICENSE.adoc +++ b/LICENSE.adoc @@ -421,12 +421,12 @@ src/third_party/doctest.h ~~~~~~~~~~~~~~~~~~~~~~~~~ This is the single header version of https://github.com/onqtam/doctest[doctest] -2.4.0 with the following license: +2.4.4 with the following license: ------------------------------------------------------------------------------- The MIT License (MIT) -Copyright (c) 2016-2019 Viktor Kirilov +Copyright (c) 2016-2020 Viktor Kirilov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/third_party/doctest.h b/src/third_party/doctest.h index b8d66feebc..7712dd6b63 100644 --- a/src/third_party/doctest.h +++ b/src/third_party/doctest.h @@ -48,8 +48,8 @@ #define DOCTEST_VERSION_MAJOR 2 #define DOCTEST_VERSION_MINOR 4 -#define DOCTEST_VERSION_PATCH 1 -#define DOCTEST_VERSION_STR "2.4.1" +#define DOCTEST_VERSION_PATCH 4 +#define DOCTEST_VERSION_STR "2.4.4" #define DOCTEST_VERSION \ (DOCTEST_VERSION_MAJOR * 10000 + DOCTEST_VERSION_MINOR * 100 + DOCTEST_VERSION_PATCH) @@ -747,6 +747,7 @@ struct ContextOptions //!OCLINT too many fields bool gnu_file_line; // if line numbers should be surrounded with :x: and not (x): bool no_path_in_filenames; // if the path to files should be removed from the output bool no_line_numbers; // if source code line numbers should be omitted from the output + bool no_debug_output; // no output in the debug console when a debugger is attached bool no_skipped_summary; // don't print "skipped" in the summary !!! UNDOCUMENTED !!! bool no_time_in_output; // omit any time/timestamps from output !!! UNDOCUMENTED !!! @@ -806,7 +807,7 @@ namespace detail { } // namespace has_insertion_operator_impl template - using has_insertion_operator = has_insertion_operator_impl::check; + using has_insertion_operator = has_insertion_operator_impl::check; DOCTEST_INTERFACE void my_memcpy(void* dest, const void* src, unsigned num); @@ -1035,6 +1036,7 @@ namespace detail { template String stringifyBinaryExpr(const DOCTEST_REF_WRAP(L) lhs, const char* op, const DOCTEST_REF_WRAP(R) rhs) { + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) return toString(lhs) + op + toString(rhs); } @@ -1122,6 +1124,7 @@ namespace detail { #define DOCTEST_COMPARISON_RETURN_TYPE bool #else // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING #define DOCTEST_COMPARISON_RETURN_TYPE typename enable_if::value || can_use_op::value, bool>::type + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) inline bool eq(const char* lhs, const char* rhs) { return String(lhs) == String(rhs); } inline bool ne(const char* lhs, const char* rhs) { return String(lhs) != String(rhs); } inline bool lt(const char* lhs, const char* rhs) { return String(lhs) < String(rhs); } @@ -1541,12 +1544,24 @@ namespace detail { MessageBuilder() = delete; ~MessageBuilder(); + // the preferred way of chaining parameters for stringification template - MessageBuilder& operator<<(const T& in) { + MessageBuilder& operator,(const T& in) { toStream(m_stream, in); return *this; } + // kept here just for backwards-compatibility - the comma operator should be preferred now + template + MessageBuilder& operator<<(const T& in) { return this->operator,(in); } + + // the `,` operator has the lowest operator precedence - if `<<` is used by the user then + // the `,` operator will be called last which is not what we want and thus the `*` operator + // is used first (has higher operator precedence compared to `<<`) so that we guarantee that + // an operator of the MessageBuilder class is called first before the rest of the parameters + template + MessageBuilder& operator*(const T& in) { return this->operator,(in); } + bool log(); void react(); }; @@ -1962,38 +1977,38 @@ int registerReporter(const char* name, int priority, bool isReporter) { DOCTEST_GLOBAL_NO_WARNINGS_END() typedef int DOCTEST_ANONYMOUS(_DOCTEST_ANON_FOR_SEMICOLON_) // for logging -#define DOCTEST_INFO(expression) \ +#define DOCTEST_INFO(...) \ DOCTEST_INFO_IMPL(DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), \ - DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), expression) + DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_), __VA_ARGS__) -#define DOCTEST_INFO_IMPL(lambda_name, mb_name, s_name, expression) \ +#define DOCTEST_INFO_IMPL(lambda_name, mb_name, s_name, ...) \ DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4626) \ auto lambda_name = [&](std::ostream* s_name) { \ doctest::detail::MessageBuilder mb_name(__FILE__, __LINE__, doctest::assertType::is_warn); \ mb_name.m_stream = s_name; \ - mb_name << expression; \ + mb_name * __VA_ARGS__; \ }; \ DOCTEST_MSVC_SUPPRESS_WARNING_POP \ auto DOCTEST_ANONYMOUS(_DOCTEST_CAPTURE_) = doctest::detail::MakeContextScope(lambda_name) -#define DOCTEST_CAPTURE(x) DOCTEST_INFO(#x " := " << x) +#define DOCTEST_CAPTURE(x) DOCTEST_INFO(#x " := ", x) -#define DOCTEST_ADD_AT_IMPL(type, file, line, mb, x) \ +#define DOCTEST_ADD_AT_IMPL(type, file, line, mb, ...) \ do { \ doctest::detail::MessageBuilder mb(file, line, doctest::assertType::type); \ - mb << x; \ + mb * __VA_ARGS__; \ DOCTEST_ASSERT_LOG_AND_REACT(mb); \ } while(false) // clang-format off -#define DOCTEST_ADD_MESSAGE_AT(file, line, x) DOCTEST_ADD_AT_IMPL(is_warn, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), x) -#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, x) DOCTEST_ADD_AT_IMPL(is_check, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), x) -#define DOCTEST_ADD_FAIL_AT(file, line, x) DOCTEST_ADD_AT_IMPL(is_require, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), x) +#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_warn, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__) +#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_check, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__) +#define DOCTEST_ADD_FAIL_AT(file, line, ...) DOCTEST_ADD_AT_IMPL(is_require, file, line, DOCTEST_ANONYMOUS(_DOCTEST_MESSAGE_), __VA_ARGS__) // clang-format on -#define DOCTEST_MESSAGE(x) DOCTEST_ADD_MESSAGE_AT(__FILE__, __LINE__, x) -#define DOCTEST_FAIL_CHECK(x) DOCTEST_ADD_FAIL_CHECK_AT(__FILE__, __LINE__, x) -#define DOCTEST_FAIL(x) DOCTEST_ADD_FAIL_AT(__FILE__, __LINE__, x) +#define DOCTEST_MESSAGE(...) DOCTEST_ADD_MESSAGE_AT(__FILE__, __LINE__, __VA_ARGS__) +#define DOCTEST_FAIL_CHECK(...) DOCTEST_ADD_FAIL_CHECK_AT(__FILE__, __LINE__, __VA_ARGS__) +#define DOCTEST_FAIL(...) DOCTEST_ADD_FAIL_AT(__FILE__, __LINE__, __VA_ARGS__) #define DOCTEST_TO_LVALUE(...) __VA_ARGS__ // Not removed to keep backwards compatibility. @@ -2036,12 +2051,12 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_REQUIRE_FALSE(...) DOCTEST_ASSERT_IMPLEMENT_1(DT_REQUIRE_FALSE, __VA_ARGS__) // clang-format off -#define DOCTEST_WARN_MESSAGE(cond, msg) do { DOCTEST_INFO(msg); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN, cond); } while(false) -#define DOCTEST_CHECK_MESSAGE(cond, msg) do { DOCTEST_INFO(msg); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK, cond); } while(false) -#define DOCTEST_REQUIRE_MESSAGE(cond, msg) do { DOCTEST_INFO(msg); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE, cond); } while(false) -#define DOCTEST_WARN_FALSE_MESSAGE(cond, msg) do { DOCTEST_INFO(msg); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN_FALSE, cond); } while(false) -#define DOCTEST_CHECK_FALSE_MESSAGE(cond, msg) do { DOCTEST_INFO(msg); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK_FALSE, cond); } while(false) -#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, msg) do { DOCTEST_INFO(msg); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE_FALSE, cond); } while(false) +#define DOCTEST_WARN_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN, cond); } while(false) +#define DOCTEST_CHECK_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK, cond); } while(false) +#define DOCTEST_REQUIRE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE, cond); } while(false) +#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_WARN_FALSE, cond); } while(false) +#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_CHECK_FALSE, cond); } while(false) +#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_ASSERT_IMPLEMENT_2(DT_REQUIRE_FALSE, cond); } while(false) // clang-format on #define DOCTEST_ASSERT_THROWS_AS(expr, assert_type, message, ...) \ @@ -2051,8 +2066,8 @@ int registerReporter(const char* name, int priority, bool isReporter) { __LINE__, #expr, #__VA_ARGS__, message); \ try { \ DOCTEST_CAST_TO_VOID(expr) \ - } catch(const doctest::detail::remove_const< \ - doctest::detail::remove_reference<__VA_ARGS__>::type>::type&) { \ + } catch(const typename doctest::detail::remove_const< \ + typename doctest::detail::remove_reference<__VA_ARGS__>::type>::type&) { \ _DOCTEST_RB.translateException(); \ _DOCTEST_RB.m_threw_as = true; \ } catch(...) { _DOCTEST_RB.translateException(); } \ @@ -2103,21 +2118,21 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_CHECK_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_CHECK_NOTHROW, __VA_ARGS__) #define DOCTEST_REQUIRE_NOTHROW(...) DOCTEST_ASSERT_NOTHROW(DT_REQUIRE_NOTHROW, __VA_ARGS__) -#define DOCTEST_WARN_THROWS_MESSAGE(expr, msg) do { DOCTEST_INFO(msg); DOCTEST_WARN_THROWS(expr); } while(false) -#define DOCTEST_CHECK_THROWS_MESSAGE(expr, msg) do { DOCTEST_INFO(msg); DOCTEST_CHECK_THROWS(expr); } while(false) -#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, msg) do { DOCTEST_INFO(msg); DOCTEST_REQUIRE_THROWS(expr); } while(false) -#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, msg) do { DOCTEST_INFO(msg); DOCTEST_WARN_THROWS_AS(expr, ex); } while(false) -#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, msg) do { DOCTEST_INFO(msg); DOCTEST_CHECK_THROWS_AS(expr, ex); } while(false) -#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, msg) do { DOCTEST_INFO(msg); DOCTEST_REQUIRE_THROWS_AS(expr, ex); } while(false) -#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, msg) do { DOCTEST_INFO(msg); DOCTEST_WARN_THROWS_WITH(expr, with); } while(false) -#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, msg) do { DOCTEST_INFO(msg); DOCTEST_CHECK_THROWS_WITH(expr, with); } while(false) -#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, msg) do { DOCTEST_INFO(msg); DOCTEST_REQUIRE_THROWS_WITH(expr, with); } while(false) -#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) do { DOCTEST_INFO(msg); DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex); } while(false) -#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) do { DOCTEST_INFO(msg); DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex); } while(false) -#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) do { DOCTEST_INFO(msg); DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex); } while(false) -#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, msg) do { DOCTEST_INFO(msg); DOCTEST_WARN_NOTHROW(expr); } while(false) -#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, msg) do { DOCTEST_INFO(msg); DOCTEST_CHECK_NOTHROW(expr); } while(false) -#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, msg) do { DOCTEST_INFO(msg); DOCTEST_REQUIRE_NOTHROW(expr); } while(false) +#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS(expr); } while(false) +#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS(expr); } while(false) +#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS(expr); } while(false) +#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_AS(expr, ex); } while(false) +#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_AS(expr, ex); } while(false) +#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_AS(expr, ex); } while(false) +#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH(expr, with); } while(false) +#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH(expr, with); } while(false) +#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH(expr, with); } while(false) +#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_THROWS_WITH_AS(expr, with, ex); } while(false) +#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_THROWS_WITH_AS(expr, with, ex); } while(false) +#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_THROWS_WITH_AS(expr, with, ex); } while(false) +#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_WARN_NOTHROW(expr); } while(false) +#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_CHECK_NOTHROW(expr); } while(false) +#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) do { DOCTEST_INFO(__VA_ARGS__); DOCTEST_REQUIRE_NOTHROW(expr); } while(false) // clang-format on #ifndef DOCTEST_CONFIG_SUPER_FAST_ASSERTS @@ -2230,21 +2245,21 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_CHECK_NOTHROW(...) (static_cast(0)) #define DOCTEST_REQUIRE_NOTHROW(...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, msg) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, msg) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, msg) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, msg) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) (static_cast(0)) -#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, msg) (static_cast(0)) +#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) +#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) +#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) +#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) +#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) +#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) +#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) #else // DOCTEST_CONFIG_NO_EXCEPTIONS_BUT_WITH_ALL_ASSERTS @@ -2335,14 +2350,14 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_REGISTER_REPORTER(name, priority, reporter) #define DOCTEST_REGISTER_LISTENER(name, priority, reporter) -#define DOCTEST_INFO(x) (static_cast(0)) +#define DOCTEST_INFO(...) (static_cast(0)) #define DOCTEST_CAPTURE(x) (static_cast(0)) -#define DOCTEST_ADD_MESSAGE_AT(file, line, x) (static_cast(0)) -#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, x) (static_cast(0)) -#define DOCTEST_ADD_FAIL_AT(file, line, x) (static_cast(0)) -#define DOCTEST_MESSAGE(x) (static_cast(0)) -#define DOCTEST_FAIL_CHECK(x) (static_cast(0)) -#define DOCTEST_FAIL(x) (static_cast(0)) +#define DOCTEST_ADD_MESSAGE_AT(file, line, ...) (static_cast(0)) +#define DOCTEST_ADD_FAIL_CHECK_AT(file, line, ...) (static_cast(0)) +#define DOCTEST_ADD_FAIL_AT(file, line, ...) (static_cast(0)) +#define DOCTEST_MESSAGE(...) (static_cast(0)) +#define DOCTEST_FAIL_CHECK(...) (static_cast(0)) +#define DOCTEST_FAIL(...) (static_cast(0)) #define DOCTEST_WARN(...) (static_cast(0)) #define DOCTEST_CHECK(...) (static_cast(0)) @@ -2351,12 +2366,12 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_CHECK_FALSE(...) (static_cast(0)) #define DOCTEST_REQUIRE_FALSE(...) (static_cast(0)) -#define DOCTEST_WARN_MESSAGE(cond, msg) (static_cast(0)) -#define DOCTEST_CHECK_MESSAGE(cond, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_MESSAGE(cond, msg) (static_cast(0)) -#define DOCTEST_WARN_FALSE_MESSAGE(cond, msg) (static_cast(0)) -#define DOCTEST_CHECK_FALSE_MESSAGE(cond, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, msg) (static_cast(0)) +#define DOCTEST_WARN_MESSAGE(cond, ...) (static_cast(0)) +#define DOCTEST_CHECK_MESSAGE(cond, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_MESSAGE(cond, ...) (static_cast(0)) +#define DOCTEST_WARN_FALSE_MESSAGE(cond, ...) (static_cast(0)) +#define DOCTEST_CHECK_FALSE_MESSAGE(cond, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_FALSE_MESSAGE(cond, ...) (static_cast(0)) #define DOCTEST_WARN_THROWS(...) (static_cast(0)) #define DOCTEST_CHECK_THROWS(...) (static_cast(0)) @@ -2374,21 +2389,21 @@ int registerReporter(const char* name, int priority, bool isReporter) { #define DOCTEST_CHECK_NOTHROW(...) (static_cast(0)) #define DOCTEST_REQUIRE_NOTHROW(...) (static_cast(0)) -#define DOCTEST_WARN_THROWS_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, msg) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, msg) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, msg) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, msg) (static_cast(0)) -#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) (static_cast(0)) -#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, msg) (static_cast(0)) -#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, msg) (static_cast(0)) -#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, msg) (static_cast(0)) +#define DOCTEST_WARN_THROWS_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_CHECK_THROWS_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_THROWS_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_WARN_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) +#define DOCTEST_CHECK_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_THROWS_AS_MESSAGE(expr, ex, ...) (static_cast(0)) +#define DOCTEST_WARN_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) +#define DOCTEST_CHECK_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_THROWS_WITH_MESSAGE(expr, with, ...) (static_cast(0)) +#define DOCTEST_WARN_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) +#define DOCTEST_CHECK_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_THROWS_WITH_AS_MESSAGE(expr, with, ex, ...) (static_cast(0)) +#define DOCTEST_WARN_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_CHECK_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) +#define DOCTEST_REQUIRE_NOTHROW_MESSAGE(expr, ...) (static_cast(0)) #define DOCTEST_WARN_EQ(...) (static_cast(0)) #define DOCTEST_CHECK_EQ(...) (static_cast(0)) @@ -2754,9 +2769,7 @@ DOCTEST_MAKE_STD_HEADERS_CLEAN_FROM_WARNINGS_ON_WALL_BEGIN #include #include #include -#ifdef DOCTEST_CONFIG_POSIX_SIGNALS #include -#endif // DOCTEST_CONFIG_POSIX_SIGNALS #include #include #include @@ -3071,6 +3084,7 @@ String::String() { String::~String() { if(!isOnStack()) delete[] data.ptr; + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) } String::String(const char* in) @@ -3112,6 +3126,7 @@ String& String::operator+=(const String& other) { if(total_size < len) { // append to the current stack space memcpy(buf + my_old_size, other.c_str(), other_size + 1); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) setLast(last - total_size); } else { // alloc new chunk @@ -3153,6 +3168,7 @@ String& String::operator+=(const String& other) { return *this; } +// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) String String::operator+(const String& other) const { return String(*this) += other; } String::String(String&& other) { @@ -3307,6 +3323,7 @@ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference") DOCTEST_GCC_SUPPRESS_WARNING_WITH_PUSH("-Wnull-dereference") // depending on the current options this will remove the path of filenames const char* skipPathFromFilename(const char* file) { +#ifndef DOCTEST_CONFIG_DISABLE if(getContextOptions()->no_path_in_filenames) { auto back = std::strrchr(file, '\\'); auto forward = std::strrchr(file, '/'); @@ -3316,6 +3333,7 @@ const char* skipPathFromFilename(const char* file) { return forward + 1; } } +#endif // DOCTEST_CONFIG_DISABLE return file; } DOCTEST_CLANG_SUPPRESS_WARNING_POP @@ -3334,6 +3352,7 @@ IContextScope::~IContextScope() = default; #ifdef DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING String toString(char* in) { return toString(static_cast(in)); } +// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) String toString(const char* in) { return String("\"") + (in ? in : "{null string}") + "\""; } #endif // DOCTEST_CONFIG_TREAT_CHAR_STAR_AS_STRING String toString(bool in) { return in ? "true" : "false"; } @@ -3406,6 +3425,7 @@ bool operator>(double lhs, const Approx& rhs) { return lhs > rhs.m_value && lhs bool operator>(const Approx& lhs, double rhs) { return lhs.m_value > rhs && lhs != rhs; } String toString(const Approx& in) { + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) return String("Approx( ") + doctest::toString(in.m_value) + " )"; } const ContextOptions* getContextOptions() { return DOCTEST_BRANCH_ON_DISABLED(nullptr, g_cs); } @@ -3698,11 +3718,15 @@ namespace detail { } bool TestCase::operator<(const TestCase& other) const { + // this will be used only to differentiate between test cases - not relevant for sorting if(m_line != other.m_line) return m_line < other.m_line; const int file_cmp = m_file.compare(other.m_file); if(file_cmp != 0) return file_cmp < 0; + const int name_cmp = strcmp(m_name, other.m_name); + if(name_cmp != 0) + return name_cmp < 0; return m_template_id < other.m_template_id; } } // namespace detail @@ -4009,24 +4033,40 @@ namespace { // Windows can easily distinguish between SO and SigSegV, // but SigInt, SigTerm, etc are handled differently. SignalDefs signalDefs[] = { - {EXCEPTION_ILLEGAL_INSTRUCTION, "SIGILL - Illegal instruction signal"}, - {EXCEPTION_STACK_OVERFLOW, "SIGSEGV - Stack overflow"}, - {EXCEPTION_ACCESS_VIOLATION, "SIGSEGV - Segmentation violation signal"}, - {EXCEPTION_INT_DIVIDE_BY_ZERO, "Divide by zero error"}, + {static_cast(EXCEPTION_ILLEGAL_INSTRUCTION), + "SIGILL - Illegal instruction signal"}, + {static_cast(EXCEPTION_STACK_OVERFLOW), "SIGSEGV - Stack overflow"}, + {static_cast(EXCEPTION_ACCESS_VIOLATION), + "SIGSEGV - Segmentation violation signal"}, + {static_cast(EXCEPTION_INT_DIVIDE_BY_ZERO), "Divide by zero error"}, }; struct FatalConditionHandler { static LONG CALLBACK handleException(PEXCEPTION_POINTERS ExceptionInfo) { - for(size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) { - if(ExceptionInfo->ExceptionRecord->ExceptionCode == signalDefs[i].id) { - reportFatal(signalDefs[i].name); - break; + // Multiple threads may enter this filter/handler at once. We want the error message to be printed on the + // console just once no matter how many threads have crashed. + static std::mutex mutex; + static bool execute = true; + { + std::lock_guard lock(mutex); + if(execute) { + bool reported = false; + for(size_t i = 0; i < DOCTEST_COUNTOF(signalDefs); ++i) { + if(ExceptionInfo->ExceptionRecord->ExceptionCode == signalDefs[i].id) { + reportFatal(signalDefs[i].name); + reported = true; + break; + } + } + if(reported == false) + reportFatal("Unhandled SEH exception caught"); + if(isDebuggerActive() && !g_cs->no_breaks) + DOCTEST_BREAK_INTO_DEBUGGER(); } + execute = false; } - // If its not an exception we care about, pass it along. - // This stops us from eating debugger breaks etc. - return EXCEPTION_CONTINUE_SEARCH; + std::exit(EXIT_FAILURE); } FatalConditionHandler() { @@ -4038,6 +4078,51 @@ namespace { previousTop = SetUnhandledExceptionFilter(handleException); // Pass in guarantee size to be filled SetThreadStackGuarantee(&guaranteeSize); + + // On Windows uncaught exceptions from another thread, exceptions from + // destructors, or calls to std::terminate are not a SEH exception + + // The terminal handler gets called when: + // - std::terminate is called FROM THE TEST RUNNER THREAD + // - an exception is thrown from a destructor FROM THE TEST RUNNER THREAD + original_terminate_handler = std::get_terminate(); + std::set_terminate([]() noexcept { + reportFatal("Terminate handler called"); + if(isDebuggerActive() && !g_cs->no_breaks) + DOCTEST_BREAK_INTO_DEBUGGER(); + std::exit(EXIT_FAILURE); // explicitly exit - otherwise the SIGABRT handler may be called as well + }); + + // SIGABRT is raised when: + // - std::terminate is called FROM A DIFFERENT THREAD + // - an exception is thrown from a destructor FROM A DIFFERENT THREAD + // - an uncaught exception is thrown FROM A DIFFERENT THREAD + prev_sigabrt_handler = std::signal(SIGABRT, [](int signal) noexcept { + if(signal == SIGABRT) { + reportFatal("SIGABRT - Abort (abnormal termination) signal"); + if(isDebuggerActive() && !g_cs->no_breaks) + DOCTEST_BREAK_INTO_DEBUGGER(); + std::exit(EXIT_FAILURE); + } + }); + + // The following settings are taken from google test, and more + // specifically from UnitTest::Run() inside of gtest.cc + + // the user does not want to see pop-up dialogs about crashes + prev_error_mode_1 = SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT | + SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); + // This forces the abort message to go to stderr in all circumstances. + prev_error_mode_2 = _set_error_mode(_OUT_TO_STDERR); + // In the debug version, Visual Studio pops up a separate dialog + // offering a choice to debug the aborted program - we want to disable that. + prev_abort_behavior = _set_abort_behavior(0x0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); + // In debug mode, the Windows CRT can crash with an assertion over invalid + // input (e.g. passing an invalid file descriptor). The default handling + // for these assertions is to pop up a dialog and wait for user input. + // Instead ask the CRT to dump such assertions to stderr non-interactively. + prev_report_mode = _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); + prev_report_file = _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); } static void reset() { @@ -4045,7 +4130,13 @@ namespace { // Unregister handler and restore the old guarantee SetUnhandledExceptionFilter(previousTop); SetThreadStackGuarantee(&guaranteeSize); - previousTop = nullptr; + std::set_terminate(original_terminate_handler); + std::signal(SIGABRT, prev_sigabrt_handler); + SetErrorMode(prev_error_mode_1); + _set_error_mode(prev_error_mode_2); + _set_abort_behavior(prev_abort_behavior, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); + _CrtSetReportMode(_CRT_ASSERT, prev_report_mode); + _CrtSetReportFile(_CRT_ASSERT, prev_report_file); isSet = false; } } @@ -4053,11 +4144,25 @@ namespace { ~FatalConditionHandler() { reset(); } private: + static UINT prev_error_mode_1; + static int prev_error_mode_2; + static unsigned int prev_abort_behavior; + static int prev_report_mode; + static _HFILE prev_report_file; + static void (*prev_sigabrt_handler)(int); + static std::terminate_handler original_terminate_handler; static bool isSet; static ULONG guaranteeSize; static LPTOP_LEVEL_EXCEPTION_FILTER previousTop; }; + UINT FatalConditionHandler::prev_error_mode_1; + int FatalConditionHandler::prev_error_mode_2; + unsigned int FatalConditionHandler::prev_abort_behavior; + int FatalConditionHandler::prev_report_mode; + _HFILE FatalConditionHandler::prev_report_file; + void (*FatalConditionHandler::prev_sigabrt_handler)(int); + std::terminate_handler FatalConditionHandler::original_terminate_handler; bool FatalConditionHandler::isSet = false; ULONG FatalConditionHandler::guaranteeSize = 0; LPTOP_LEVEL_EXCEPTION_FILTER FatalConditionHandler::previousTop = nullptr; @@ -4257,6 +4362,7 @@ namespace detail { // ################################################################################### DOCTEST_ASSERT_OUT_OF_TESTS(result.m_decomp); DOCTEST_ASSERT_IN_TESTS(result.m_decomp); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) } MessageBuilder::MessageBuilder(const char* file, int line, assertType::Enum severity) { @@ -4979,7 +5085,6 @@ namespace { } // TODO: - // - log_contexts() // - log_message() // - respond to queries // - honor remaining options @@ -4993,7 +5098,6 @@ namespace { struct JUnitTestCaseData { -DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") // gmtime static std::string getCurrentTimestamp() { // Beware, this is not reentrant because of backward compatibility issues // Also, UTC only, again because of backward compatibility (%z is C++11) @@ -5001,16 +5105,19 @@ DOCTEST_CLANG_SUPPRESS_WARNING_WITH_PUSH("-Wdeprecated-declarations") // gmtime std::time(&rawtime); auto const timeStampSize = sizeof("2017-01-16T17:06:45Z"); - std::tm* timeInfo; - timeInfo = std::gmtime(&rawtime); + std::tm timeInfo; +#ifdef DOCTEST_PLATFORM_WINDOWS + gmtime_s(&timeInfo, &rawtime); +#else // DOCTEST_PLATFORM_WINDOWS + gmtime_r(&rawtime, &timeInfo); +#endif // DOCTEST_PLATFORM_WINDOWS char timeStamp[timeStampSize]; const char* const fmt = "%Y-%m-%dT%H:%M:%SZ"; - std::strftime(timeStamp, timeStampSize, fmt, timeInfo); + std::strftime(timeStamp, timeStampSize, fmt, &timeInfo); return std::string(timeStamp); } -DOCTEST_CLANG_SUPPRESS_WARNING_POP struct JUnitTestMessage { @@ -5175,12 +5282,27 @@ DOCTEST_CLANG_SUPPRESS_WARNING_POP << line(rb.m_line) << (opt.gnu_file_line ? ":" : "):") << std::endl; fulltext_log_assert_to_stream(os, rb); + log_contexts(os); testCaseData.addFailure(rb.m_decomp.c_str(), assertString(rb.m_at), os.str()); } void log_message(const MessageData&) override {} void test_case_skipped(const TestCaseData&) override {} + + void log_contexts(std::ostringstream& s) { + int num_contexts = get_num_active_contexts(); + if(num_contexts) { + auto contexts = get_active_contexts(); + + s << " logged: "; + for(int i = 0; i < num_contexts; ++i) { + s << (i == 0 ? "" : " "); + contexts[i]->stringify(&s); + s << std::endl; + } + } + } }; DOCTEST_REGISTER_REPORTER("junit", 0, JUnitReporter); @@ -5894,6 +6016,7 @@ void Context::parseArgs(int argc, const char* const* argv, bool withDefaults) { DOCTEST_PARSE_AS_BOOL_OR_FLAG("gnu-file-line", "gfl", gnu_file_line, !bool(DOCTEST_MSVC)); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-path-filenames", "npf", no_path_in_filenames, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-line-numbers", "nln", no_line_numbers, false); + DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-debug-output", "ndo", no_debug_output, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-skipped-summary", "nss", no_skipped_summary, false); DOCTEST_PARSE_AS_BOOL_OR_FLAG("no-time-in-output", "ntio", no_time_in_output, false); // clang-format on @@ -5951,6 +6074,7 @@ void Context::clearFilters() { // allows the user to override procedurally the int/bool options from the command line void Context::setOption(const char* option, int value) { setOption(option, toString(value).c_str()); + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) } // allows the user to override procedurally the string options from the command line @@ -6026,7 +6150,7 @@ int Context::run() { p->reporters_currently_used.insert(p->reporters_currently_used.begin(), curr.second(*g_cs)); #ifdef DOCTEST_PLATFORM_WINDOWS - if(isDebuggerActive()) + if(isDebuggerActive() && p->no_debug_output == false) p->reporters_currently_used.push_back(new DebugOutputWindowReporter(*g_cs)); #endif // DOCTEST_PLATFORM_WINDOWS From 1c36557abc4ff44a9740e04102790b354fc99a4f Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 20:05:47 +0100 Subject: [PATCH 31/38] Upgrade to fmt 7.1.3 --- LICENSE.adoc | 2 +- src/third_party/fmt/core.h | 446 +++++-- src/third_party/fmt/format-inl.h | 1894 +++++++++++++++++++++++++----- src/third_party/fmt/format.h | 1157 ++++++++++-------- src/third_party/format.cpp | 34 +- 5 files changed, 2691 insertions(+), 842 deletions(-) diff --git a/LICENSE.adoc b/LICENSE.adoc index fa1f32095c..9ccdb0717e 100644 --- a/LICENSE.adoc +++ b/LICENSE.adoc @@ -451,7 +451,7 @@ SOFTWARE. src/third_party/fmt/*.h and src/third_party/format.cpp ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This is a subset of https://fmt.dev[fmt] 7.0.3 with the following license: +This is a subset of https://fmt.dev[fmt] 7.1.3 with the following license: ------------------------------------------------------------------------------- Formatting library for C++ diff --git a/src/third_party/fmt/core.h b/src/third_party/fmt/core.h index 031bf869e1..0a81e0ccd9 100644 --- a/src/third_party/fmt/core.h +++ b/src/third_party/fmt/core.h @@ -18,7 +18,7 @@ #include // The fmt library version in the form major * 10000 + minor * 100 + patch. -#define FMT_VERSION 70003 +#define FMT_VERSION 70103 #ifdef __clang__ # define FMT_CLANG_VERSION (__clang_major__ * 100 + __clang_minor__) @@ -57,6 +57,7 @@ # define FMT_MSC_VER 0 # define FMT_SUPPRESS_MSC_WARNING(n) #endif + #ifdef __has_feature # define FMT_HAS_FEATURE(x) __has_feature(x) #else @@ -64,7 +65,7 @@ #endif #if defined(__has_include) && !defined(__INTELLISENSE__) && \ - !(FMT_ICC_VERSION && FMT_ICC_VERSION < 1600) + (!FMT_ICC_VERSION || FMT_ICC_VERSION >= 1600) # define FMT_HAS_INCLUDE(x) __has_include(x) #else # define FMT_HAS_INCLUDE(x) 0 @@ -152,7 +153,7 @@ # if FMT_HAS_CPP14_ATTRIBUTE(deprecated) || FMT_MSC_VER >= 1900 # define FMT_DEPRECATED [[deprecated]] # else -# if defined(__GNUC__) || defined(__clang__) +# if (defined(__GNUC__) && !defined(__LCC__)) || defined(__clang__) # define FMT_DEPRECATED __attribute__((deprecated)) # elif FMT_MSC_VER # define FMT_DEPRECATED __declspec(deprecated) @@ -177,9 +178,17 @@ # endif #endif -#ifndef FMT_BEGIN_NAMESPACE +#ifndef FMT_USE_INLINE_NAMESPACES # if FMT_HAS_FEATURE(cxx_inline_namespaces) || FMT_GCC_VERSION >= 404 || \ - FMT_MSC_VER >= 1900 + (FMT_MSC_VER >= 1900 && !_MANAGED) +# define FMT_USE_INLINE_NAMESPACES 1 +# else +# define FMT_USE_INLINE_NAMESPACES 0 +# endif +#endif + +#ifndef FMT_BEGIN_NAMESPACE +# if FMT_USE_INLINE_NAMESPACES # define FMT_INLINE_NAMESPACE inline namespace # define FMT_END_NAMESPACE \ } \ @@ -269,8 +278,7 @@ struct monostate {}; namespace detail { -// A helper function to suppress bogus "conditional expression is constant" -// warnings. +// A helper function to suppress "conditional expression is constant" warnings. template constexpr T const_check(T value) { return value; } FMT_NORETURN FMT_API void assert_fail(const char* file, int line, @@ -299,7 +307,8 @@ template struct std_string_view {}; #ifdef FMT_USE_INT128 // Do nothing. -#elif defined(__SIZEOF_INT128__) && !FMT_NVCC && !(FMT_CLANG_VERSION && FMT_MSC_VER) +#elif defined(__SIZEOF_INT128__) && !FMT_NVCC && \ + !(FMT_CLANG_VERSION && FMT_MSC_VER) # define FMT_USE_INT128 1 using int128_t = __int128_t; using uint128_t = __uint128_t; @@ -506,6 +515,18 @@ template struct char_t_impl::value>> { using type = typename result::value_type; }; +// Reports a compile-time error if S is not a valid format string. +template ::value)> +FMT_INLINE void check_format_string(const S&) { +#ifdef FMT_ENFORCE_COMPILE_STRING + static_assert(is_compile_string::value, + "FMT_ENFORCE_COMPILE_STRING requires all format strings to use " + "FMT_STRING."); +#endif +} +template ::value)> +void check_format_string(S); + struct error_handler { constexpr error_handler() = default; constexpr error_handler(const error_handler&) = default; @@ -545,8 +566,9 @@ class basic_format_parse_context : private ErrorHandler { using iterator = typename basic_string_view::iterator; explicit constexpr basic_format_parse_context( - basic_string_view format_str, ErrorHandler eh = {}) - : ErrorHandler(eh), format_str_(format_str), next_arg_id_(0) {} + basic_string_view format_str, ErrorHandler eh = {}, + int next_arg_id = 0) + : ErrorHandler(eh), format_str_(format_str), next_arg_id_(next_arg_id) {} /** Returns an iterator to the beginning of the format string range being @@ -616,8 +638,24 @@ template using has_formatter = std::is_constructible>; +// Checks whether T is a container with contiguous storage. +template struct is_contiguous : std::false_type {}; +template +struct is_contiguous> : std::true_type {}; + namespace detail { +// Extracts a reference to the container from back_insert_iterator. +template +inline Container& get_container(std::back_insert_iterator it) { + using bi_iterator = std::back_insert_iterator; + struct accessor : bi_iterator { + accessor(bi_iterator iter) : bi_iterator(iter) {} + using bi_iterator::container; + }; + return *accessor(it).container; +} + /** \rst A contiguous memory buffer with an optional growing ability. It is an internal @@ -640,6 +678,8 @@ template class buffer { size_(sz), capacity_(cap) {} + ~buffer() = default; + /** Sets the buffer data and capacity. */ void set(T* buf_data, size_t buf_capacity) FMT_NOEXCEPT { ptr_ = buf_data; @@ -655,7 +695,6 @@ template class buffer { buffer(const buffer&) = delete; void operator=(const buffer&) = delete; - virtual ~buffer() = default; T* begin() FMT_NOEXCEPT { return ptr_; } T* end() FMT_NOEXCEPT { return ptr_ + size_; } @@ -675,24 +714,26 @@ template class buffer { /** Returns a pointer to the buffer data. */ const T* data() const FMT_NOEXCEPT { return ptr_; } - /** - Resizes the buffer. If T is a POD type new elements may not be initialized. - */ - void resize(size_t new_size) { - reserve(new_size); - size_ = new_size; - } - /** Clears this buffer. */ void clear() { size_ = 0; } - /** Reserves space to store at least *capacity* elements. */ - void reserve(size_t new_capacity) { + // Tries resizing the buffer to contain *count* elements. If T is a POD type + // the new elements may not be initialized. + void try_resize(size_t count) { + try_reserve(count); + size_ = count <= capacity_ ? count : capacity_; + } + + // Tries increasing the buffer capacity to *new_capacity*. It can increase the + // capacity by a smaller amount than requested but guarantees there is space + // for at least one additional element either by increasing the capacity or by + // flushing the buffer if it is full. + void try_reserve(size_t new_capacity) { if (new_capacity > capacity_) grow(new_capacity); } void push_back(const T& value) { - reserve(size_ + 1); + try_reserve(size_ + 1); ptr_[size_++] = value; } @@ -705,32 +746,150 @@ template class buffer { } }; -// A container-backed buffer. +struct buffer_traits { + explicit buffer_traits(size_t) {} + size_t count() const { return 0; } + size_t limit(size_t size) { return size; } +}; + +class fixed_buffer_traits { + private: + size_t count_ = 0; + size_t limit_; + + public: + explicit fixed_buffer_traits(size_t limit) : limit_(limit) {} + size_t count() const { return count_; } + size_t limit(size_t size) { + size_t n = limit_ > count_ ? limit_ - count_ : 0; + count_ += size; + return size < n ? size : n; + } +}; + +// A buffer that writes to an output iterator when flushed. +template +class iterator_buffer final : public Traits, public buffer { + private: + OutputIt out_; + enum { buffer_size = 256 }; + T data_[buffer_size]; + + protected: + void grow(size_t) final FMT_OVERRIDE { + if (this->size() == buffer_size) flush(); + } + void flush(); + + public: + explicit iterator_buffer(OutputIt out, size_t n = buffer_size) + : Traits(n), + buffer(data_, 0, buffer_size), + out_(out) {} + ~iterator_buffer() { flush(); } + + OutputIt out() { + flush(); + return out_; + } + size_t count() const { return Traits::count() + this->size(); } +}; + +template class iterator_buffer final : public buffer { + protected: + void grow(size_t) final FMT_OVERRIDE {} + + public: + explicit iterator_buffer(T* out, size_t = 0) : buffer(out, 0, ~size_t()) {} + + T* out() { return &*this->end(); } +}; + +// A buffer that writes to a container with the contiguous storage. template -class container_buffer : public buffer { +class iterator_buffer, + enable_if_t::value, + typename Container::value_type>> + final : public buffer { private: Container& container_; protected: - void grow(size_t capacity) FMT_OVERRIDE { + void grow(size_t capacity) final FMT_OVERRIDE { container_.resize(capacity); this->set(&container_[0], capacity); } public: - explicit container_buffer(Container& c) + explicit iterator_buffer(Container& c) : buffer(c.size()), container_(c) {} + explicit iterator_buffer(std::back_insert_iterator out, size_t = 0) + : iterator_buffer(get_container(out)) {} + std::back_insert_iterator out() { + return std::back_inserter(container_); + } }; -// Extracts a reference to the container from back_insert_iterator. -template -inline Container& get_container(std::back_insert_iterator it) { - using bi_iterator = std::back_insert_iterator; - struct accessor : bi_iterator { - accessor(bi_iterator iter) : bi_iterator(iter) {} - using bi_iterator::container; - }; - return *accessor(it).container; +// A buffer that counts the number of code units written discarding the output. +template class counting_buffer final : public buffer { + private: + enum { buffer_size = 256 }; + T data_[buffer_size]; + size_t count_ = 0; + + protected: + void grow(size_t) final FMT_OVERRIDE { + if (this->size() != buffer_size) return; + count_ += this->size(); + this->clear(); + } + + public: + counting_buffer() : buffer(data_, 0, buffer_size) {} + + size_t count() { return count_ + this->size(); } +}; + +// An output iterator that appends to the buffer. +// It is used to reduce symbol sizes for the common case. +template +class buffer_appender : public std::back_insert_iterator> { + using base = std::back_insert_iterator>; + + public: + explicit buffer_appender(buffer& buf) : base(buf) {} + buffer_appender(base it) : base(it) {} + + buffer_appender& operator++() { + base::operator++(); + return *this; + } + + buffer_appender operator++(int) { + buffer_appender tmp = *this; + ++*this; + return tmp; + } +}; + +// Maps an output iterator into a buffer. +template +iterator_buffer get_buffer(OutputIt); +template buffer& get_buffer(buffer_appender); + +template OutputIt get_buffer_init(OutputIt out) { + return out; +} +template buffer& get_buffer_init(buffer_appender out) { + return get_container(out); +} + +template +auto get_iterator(Buffer& buf) -> decltype(buf.out()) { + return buf.out(); +} +template buffer_appender get_iterator(buffer& buf) { + return buffer_appender(buf); } template @@ -759,7 +918,8 @@ template struct named_arg_info { template struct arg_data { // args_[0].named_args points to named_args_ to avoid bloating format_args. - T args_[1 + (NUM_ARGS != 0 ? NUM_ARGS : 1)]; + // +1 to workaround a bug in gcc 7.5 that causes duplicated-branches warning. + T args_[1 + (NUM_ARGS != 0 ? NUM_ARGS : +1)]; named_arg_info named_args_[NUM_NAMED_ARGS]; template @@ -771,7 +931,8 @@ struct arg_data { template struct arg_data { - T args_[NUM_ARGS != 0 ? NUM_ARGS : 1]; + // +1 to workaround a bug in gcc 7.5 that causes duplicated-branches warning. + T args_[NUM_ARGS != 0 ? NUM_ARGS : +1]; template FMT_INLINE arg_data(const U&... init) : args_{init...} {} @@ -959,6 +1120,8 @@ enum { long_short = sizeof(long) == sizeof(int) }; using long_type = conditional_t; using ulong_type = conditional_t; +struct unformattable {}; + // Maps formatting arguments to core types. template struct arg_mapper { using char_type = typename Context::char_type; @@ -1067,15 +1230,7 @@ template struct arg_mapper { return map(val.value); } - int map(...) { - constexpr bool formattable = sizeof(Context) == 0; - static_assert( - formattable, - "Cannot format argument. To make type T formattable provide a " - "formatter specialization: " - "https://fmt.dev/latest/api.html#formatting-user-defined-types"); - return 0; - } + unformattable map(...) { return {}; } }; // A type constant after applying arg_mapper. @@ -1199,15 +1354,25 @@ FMT_CONSTEXPR_DECL FMT_INLINE auto visit_format_arg( return vis(monostate()); } -// Checks whether T is a container with contiguous storage. -template struct is_contiguous : std::false_type {}; -template -struct is_contiguous> : std::true_type {}; -template -struct is_contiguous> : std::true_type {}; +template struct formattable : std::false_type {}; namespace detail { +// A workaround for gcc 4.8 to make void_t work in a SFINAE context. +template struct void_t_impl { using type = void; }; +template +using void_t = typename detail::void_t_impl::type; + +template +struct is_output_iterator : std::false_type {}; + +template +struct is_output_iterator< + It, T, + void_t::iterator_category, + decltype(*std::declval() = std::declval())>> + : std::true_type {}; + template struct is_back_insert_iterator : std::false_type {}; template @@ -1219,6 +1384,9 @@ struct is_contiguous_back_insert_iterator : std::false_type {}; template struct is_contiguous_back_insert_iterator> : is_contiguous {}; +template +struct is_contiguous_back_insert_iterator> + : std::true_type {}; // A type-erased reference to an std::locale to avoid heavy include. class locale_ref { @@ -1250,13 +1418,24 @@ FMT_CONSTEXPR basic_format_arg make_arg(const T& value) { return arg; } +template int check(unformattable) { + static_assert( + formattable(), + "Cannot format an argument. To make type T formattable provide a " + "formatter specialization: https://fmt.dev/latest/api.html#udt"); + return 0; +} +template inline const U& check(const U& val) { + return val; +} + // The type template parameter is there to avoid an ODR violation when using // a fallback formatter in one translation unit and an implicit conversion in // another (not recommended). template inline value make_arg(const T& val) { - return arg_mapper().map(val); + return check(arg_mapper().map(val)); } template class basic_format_context { template using buffer_context = - basic_format_context>, Char>; + basic_format_context, Char>; using format_context = buffer_context; using wformat_context = buffer_context; -// Workaround a bug in gcc: https://stackoverflow.com/q/62767544/471164. +// Workaround an alias issue: https://stackoverflow.com/q/62767544/471164. #define FMT_BUFFER_CONTEXT(Char) \ - basic_format_context>, Char> + basic_format_context, Char> /** \rst @@ -1414,7 +1593,7 @@ class format_arg_store /** \rst - Constructs an `~fmt::format_arg_store` object that contains references to + Constructs a `~fmt::format_arg_store` object that contains references to arguments and can be implicitly converted to `~fmt::format_args`. `Context` can be omitted in which case it defaults to `~fmt::context`. See `~fmt::arg` for lifetime considerations. @@ -1426,6 +1605,27 @@ inline format_arg_store make_format_args( return {args...}; } +/** + \rst + Constructs a `~fmt::format_arg_store` object that contains references + to arguments and can be implicitly converted to `~fmt::format_args`. + If ``format_str`` is a compile-time string then `make_args_checked` checks + its validity at compile time. + \endrst + */ +template > +inline auto make_args_checked(const S& format_str, + const remove_reference_t&... args) + -> format_arg_store, remove_reference_t...> { + static_assert( + detail::count<( + std::is_base_of>::value && + std::is_reference::value)...>() == 0, + "passing views as lvalues is disallowed"); + detail::check_format_string(format_str); + return {args...}; +} + /** \rst Returns a named argument to be used in a formatting function. It should only @@ -1729,7 +1929,14 @@ template class basic_format_args { } }; -/** An alias to ``basic_format_args``. */ +#ifdef FMT_ARM_ABI_COMPATIBILITY +/** An alias to ``basic_format_args``. */ +// Separate types would result in shorter symbols but break ABI compatibility +// between clang and gcc on ARM (#1919). +using format_args = basic_format_args; +using wformat_args = basic_format_args; +#else +// DEPRECATED! These are kept for ABI compatibility. // It is a separate type rather than an alias to make symbols readable. struct format_args : basic_format_args { template @@ -1738,31 +1945,9 @@ struct format_args : basic_format_args { struct wformat_args : basic_format_args { using basic_format_args::basic_format_args; }; - -namespace detail { - -// Reports a compile-time error if S is not a valid format string. -template ::value)> -FMT_INLINE void check_format_string(const S&) { -#ifdef FMT_ENFORCE_COMPILE_STRING - static_assert(is_compile_string::value, - "FMT_ENFORCE_COMPILE_STRING requires all format strings to use " - "FMT_STRING."); #endif -} -template ::value)> -void check_format_string(S); -template > -inline format_arg_store, remove_reference_t...> -make_args_checked(const S& format_str, - const remove_reference_t&... args) { - static_assert(count<(std::is_base_of>::value && - std::is_reference::value)...>() == 0, - "passing views as lvalues is disallowed"); - check_format_string(format_str); - return {args...}; -} +namespace detail { template ::value)> std::basic_string vformat( @@ -1772,9 +1957,10 @@ std::basic_string vformat( FMT_API std::string vformat(string_view format_str, format_args args); template -typename FMT_BUFFER_CONTEXT(Char)::iterator vformat_to( +void vformat_to( buffer& buf, basic_string_view format_str, - basic_format_args)> args); + basic_format_args)> args, + detail::locale_ref loc = {}); template ::value)> @@ -1789,26 +1975,80 @@ inline void vprint_mojibake(std::FILE*, string_view, format_args) {} /** Formats a string and writes the output to ``out``. */ // GCC 8 and earlier cannot handle std::back_insert_iterator with // vformat_to(...) overload, so SFINAE on iterator type instead. -template < - typename OutputIt, typename S, typename Char = char_t, - FMT_ENABLE_IF(detail::is_contiguous_back_insert_iterator::value)> -OutputIt vformat_to( - OutputIt out, const S& format_str, - basic_format_args>> args) { - auto& c = detail::get_container(out); - detail::container_buffer> buf(c); +template , + bool enable = detail::is_output_iterator::value> +auto vformat_to(OutputIt out, const S& format_str, + basic_format_args>> args) + -> typename std::enable_if::type { + decltype(detail::get_buffer(out)) buf(detail::get_buffer_init(out)); detail::vformat_to(buf, to_string_view(format_str), args); - return out; + return detail::get_iterator(buf); +} + +/** + \rst + Formats arguments, writes the result to the output iterator ``out`` and returns + the iterator past the end of the output range. + + **Example**:: + + std::vector out; + fmt::format_to(std::back_inserter(out), "{}", 42); + \endrst + */ +// We cannot use FMT_ENABLE_IF because of a bug in gcc 8.3. +template >::value> +inline auto format_to(OutputIt out, const S& format_str, Args&&... args) -> + typename std::enable_if::type { + const auto& vargs = fmt::make_args_checked(format_str, args...); + return vformat_to(out, to_string_view(format_str), vargs); +} + +template struct format_to_n_result { + /** Iterator past the end of the output range. */ + OutputIt out; + /** Total (not truncated) output size. */ + size_t size; +}; + +template ::value)> +inline format_to_n_result vformat_to_n( + OutputIt out, size_t n, basic_string_view format_str, + basic_format_args>> args) { + detail::iterator_buffer buf(out, + n); + detail::vformat_to(buf, format_str, args); + return {buf.out(), buf.count()}; +} + +/** + \rst + Formats arguments, writes up to ``n`` characters of the result to the output + iterator ``out`` and returns the total output size and the iterator past the + end of the output range. + \endrst + */ +template >::value> +inline auto format_to_n(OutputIt out, size_t n, const S& format_str, + const Args&... args) -> + typename std::enable_if>::type { + const auto& vargs = fmt::make_args_checked(format_str, args...); + return vformat_to_n(out, n, to_string_view(format_str), vargs); } -template ::value&& detail::is_string::value)> -inline std::back_insert_iterator format_to( - std::back_insert_iterator out, const S& format_str, - Args&&... args) { - return vformat_to(out, to_string_view(format_str), - detail::make_args_checked(format_str, args...)); +/** + Returns the number of characters in the output of + ``format(format_str, args...)``. + */ +template +inline size_t formatted_size(string_view format_str, Args&&... args) { + const auto& vargs = fmt::make_args_checked(format_str, args...); + detail::counting_buffer<> buf; + detail::vformat_to(buf, format_str, vargs); + return buf.count(); } template > @@ -1832,7 +2072,7 @@ FMT_INLINE std::basic_string vformat( // std::basic_string> to reduce the symbol size. template > FMT_INLINE std::basic_string format(const S& format_str, Args&&... args) { - const auto& vargs = detail::make_args_checked(format_str, args...); + const auto& vargs = fmt::make_args_checked(format_str, args...); return detail::vformat(to_string_view(format_str), vargs); } @@ -1852,7 +2092,7 @@ FMT_API void vprint(std::FILE*, string_view, format_args); */ template > inline void print(std::FILE* f, const S& format_str, Args&&... args) { - const auto& vargs = detail::make_args_checked(format_str, args...); + const auto& vargs = fmt::make_args_checked(format_str, args...); return detail::is_unicode() ? vprint(f, to_string_view(format_str), vargs) : detail::vprint_mojibake(f, to_string_view(format_str), vargs); @@ -1871,7 +2111,7 @@ inline void print(std::FILE* f, const S& format_str, Args&&... args) { */ template > inline void print(const S& format_str, Args&&... args) { - const auto& vargs = detail::make_args_checked(format_str, args...); + const auto& vargs = fmt::make_args_checked(format_str, args...); return detail::is_unicode() ? vprint(to_string_view(format_str), vargs) : detail::vprint_mojibake(stdout, to_string_view(format_str), diff --git a/src/third_party/fmt/format-inl.h b/src/third_party/fmt/format-inl.h index d8c9c8a5ee..8f2fe7354a 100644 --- a/src/third_party/fmt/format-inl.h +++ b/src/third_party/fmt/format-inl.h @@ -13,32 +13,19 @@ #include #include #include -#include // for std::memmove +#include // std::memmove #include #include -#include "format.h" -#if !defined(FMT_STATIC_THOUSANDS_SEPARATOR) +#ifndef FMT_STATIC_THOUSANDS_SEPARATOR # include #endif #ifdef _WIN32 -# if !defined(NOMINMAX) && !defined(WIN32_LEAN_AND_MEAN) -# define NOMINMAX -# define WIN32_LEAN_AND_MEAN -# include -# undef WIN32_LEAN_AND_MEAN -# undef NOMINMAX -# else -# include -# endif -# include +# include // _isatty #endif -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable : 4702) // unreachable code -#endif +#include "format.h" // Dummy implementations of strerror_r and strerror_s called if corresponding // system functions are not available. @@ -79,8 +66,8 @@ inline int fmt_snprintf(char* buffer, size_t size, const char* format, ...) { // ERANGE - buffer is not large enough to store the error message // other - failure // Buffer should be at least of size 1. -FMT_FUNC int safe_strerror(int error_code, char*& buffer, - size_t buffer_size) FMT_NOEXCEPT { +inline int safe_strerror(int error_code, char*& buffer, + size_t buffer_size) FMT_NOEXCEPT { FMT_ASSERT(buffer != nullptr && buffer_size != 0, "invalid buffer"); class dispatcher { @@ -145,7 +132,7 @@ FMT_FUNC void format_error_code(detail::buffer& out, int error_code, // Report error code making sure that the output fits into // inline_buffer_size to avoid dynamic memory allocation and potential // bad_alloc. - out.resize(0); + out.try_resize(0); static const char SEP[] = ": "; static const char ERROR_STR[] = "error "; // Subtract 2 to account for terminating null characters in SEP and ERROR_STR. @@ -156,7 +143,7 @@ FMT_FUNC void format_error_code(detail::buffer& out, int error_code, ++error_code_size; } error_code_size += detail::to_unsigned(detail::count_digits(abs_value)); - auto it = std::back_inserter(out); + auto it = buffer_appender(out); if (message.size() <= inline_buffer_size - error_code_size) format_to(it, "{}{}", message, SEP); format_to(it, "{}{}", ERROR_STR, error_code); @@ -173,8 +160,8 @@ FMT_FUNC void report_error(format_func func, int error_code, } // A wrapper around fwrite that throws on error. -FMT_FUNC void fwrite_fully(const void* ptr, size_t size, size_t count, - FILE* stream) { +inline void fwrite_fully(const void* ptr, size_t size, size_t count, + FILE* stream) { size_t written = std::fwrite(ptr, size, count, stream); if (written < count) FMT_THROW(system_error(errno, "cannot write to file")); } @@ -242,26 +229,23 @@ template <> FMT_FUNC int count_digits<4>(detail::fallback_uintptr n) { template const typename basic_data::digit_pair basic_data::digits[] = { - {'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, - {'0', '5'}, {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'}, - {'1', '0'}, {'1', '1'}, {'1', '2'}, {'1', '3'}, {'1', '4'}, - {'1', '5'}, {'1', '6'}, {'1', '7'}, {'1', '8'}, {'1', '9'}, - {'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'}, {'2', '4'}, - {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'}, - {'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, - {'3', '5'}, {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'}, - {'4', '0'}, {'4', '1'}, {'4', '2'}, {'4', '3'}, {'4', '4'}, - {'4', '5'}, {'4', '6'}, {'4', '7'}, {'4', '8'}, {'4', '9'}, - {'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'}, {'5', '4'}, - {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'}, - {'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, - {'6', '5'}, {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'}, - {'7', '0'}, {'7', '1'}, {'7', '2'}, {'7', '3'}, {'7', '4'}, - {'7', '5'}, {'7', '6'}, {'7', '7'}, {'7', '8'}, {'7', '9'}, - {'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'}, {'8', '4'}, - {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'}, - {'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, - {'9', '5'}, {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'}}; + {'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, {'0', '5'}, + {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'}, {'1', '0'}, {'1', '1'}, + {'1', '2'}, {'1', '3'}, {'1', '4'}, {'1', '5'}, {'1', '6'}, {'1', '7'}, + {'1', '8'}, {'1', '9'}, {'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'}, + {'2', '4'}, {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'}, + {'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, {'3', '5'}, + {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'}, {'4', '0'}, {'4', '1'}, + {'4', '2'}, {'4', '3'}, {'4', '4'}, {'4', '5'}, {'4', '6'}, {'4', '7'}, + {'4', '8'}, {'4', '9'}, {'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'}, + {'5', '4'}, {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'}, + {'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, {'6', '5'}, + {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'}, {'7', '0'}, {'7', '1'}, + {'7', '2'}, {'7', '3'}, {'7', '4'}, {'7', '5'}, {'7', '6'}, {'7', '7'}, + {'7', '8'}, {'7', '9'}, {'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'}, + {'8', '4'}, {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'}, + {'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, {'9', '5'}, + {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'}}; template const char basic_data::hex_digits[] = "0123456789abcdef"; @@ -279,16 +263,24 @@ const uint64_t basic_data::powers_of_10_64[] = { template const uint32_t basic_data::zero_or_powers_of_10_32[] = {0, FMT_POWERS_OF_10(1)}; - template const uint64_t basic_data::zero_or_powers_of_10_64[] = { 0, FMT_POWERS_OF_10(1), FMT_POWERS_OF_10(1000000000ULL), 10000000000000000000ULL}; +template +const uint32_t basic_data::zero_or_powers_of_10_32_new[] = { + 0, 0, FMT_POWERS_OF_10(1)}; + +template +const uint64_t basic_data::zero_or_powers_of_10_64_new[] = { + 0, 0, FMT_POWERS_OF_10(1), FMT_POWERS_OF_10(1000000000ULL), + 10000000000000000000ULL}; + // Normalized 64-bit significands of pow(10, k), for k = -348, -340, ..., 340. // These are generated by support/compute-powers.py. template -const uint64_t basic_data::pow10_significands[] = { +const uint64_t basic_data::grisu_pow10_significands[] = { 0xfa8fd5a0081c0288, 0xbaaee17fa23ebf76, 0x8b16fb203055ac76, 0xcf42894a5dce35ea, 0x9a6bb0aa55653b2d, 0xe61acf033d1a45df, 0xab70fe17c79ac6ca, 0xff77b1fcbebcdc4f, 0xbe5691ef416bd60c, @@ -323,7 +315,7 @@ const uint64_t basic_data::pow10_significands[] = { // Binary exponents of pow(10, k), for k = -348, -340, ..., 340, corresponding // to significands above. template -const int16_t basic_data::pow10_exponents[] = { +const int16_t basic_data::grisu_pow10_exponents[] = { -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007, -980, -954, -927, -901, -874, -847, -821, -794, -768, -741, -715, -688, -661, -635, -608, -582, -555, -529, -502, -475, -449, -422, -396, -369, @@ -333,6 +325,744 @@ const int16_t basic_data::pow10_exponents[] = { 534, 561, 588, 614, 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, 907, 933, 960, 986, 1013, 1039, 1066}; +template +const divtest_table_entry basic_data::divtest_table_for_pow5_32[] = + {{0x00000001, 0xffffffff}, {0xcccccccd, 0x33333333}, + {0xc28f5c29, 0x0a3d70a3}, {0x26e978d5, 0x020c49ba}, + {0x3afb7e91, 0x0068db8b}, {0x0bcbe61d, 0x0014f8b5}, + {0x68c26139, 0x000431bd}, {0xae8d46a5, 0x0000d6bf}, + {0x22e90e21, 0x00002af3}, {0x3a2e9c6d, 0x00000897}, + {0x3ed61f49, 0x000001b7}}; + +template +const divtest_table_entry basic_data::divtest_table_for_pow5_64[] = + {{0x0000000000000001, 0xffffffffffffffff}, + {0xcccccccccccccccd, 0x3333333333333333}, + {0x8f5c28f5c28f5c29, 0x0a3d70a3d70a3d70}, + {0x1cac083126e978d5, 0x020c49ba5e353f7c}, + {0xd288ce703afb7e91, 0x0068db8bac710cb2}, + {0x5d4e8fb00bcbe61d, 0x0014f8b588e368f0}, + {0x790fb65668c26139, 0x000431bde82d7b63}, + {0xe5032477ae8d46a5, 0x0000d6bf94d5e57a}, + {0xc767074b22e90e21, 0x00002af31dc46118}, + {0x8e47ce423a2e9c6d, 0x0000089705f4136b}, + {0x4fa7f60d3ed61f49, 0x000001b7cdfd9d7b}, + {0x0fee64690c913975, 0x00000057f5ff85e5}, + {0x3662e0e1cf503eb1, 0x000000119799812d}, + {0xa47a2cf9f6433fbd, 0x0000000384b84d09}, + {0x54186f653140a659, 0x00000000b424dc35}, + {0x7738164770402145, 0x0000000024075f3d}, + {0xe4a4d1417cd9a041, 0x000000000734aca5}, + {0xc75429d9e5c5200d, 0x000000000170ef54}, + {0xc1773b91fac10669, 0x000000000049c977}, + {0x26b172506559ce15, 0x00000000000ec1e4}, + {0xd489e3a9addec2d1, 0x000000000002f394}, + {0x90e860bb892c8d5d, 0x000000000000971d}, + {0x502e79bf1b6f4f79, 0x0000000000001e39}, + {0xdcd618596be30fe5, 0x000000000000060b}}; + +template +const uint64_t basic_data::dragonbox_pow10_significands_64[] = { + 0x81ceb32c4b43fcf5, 0xa2425ff75e14fc32, 0xcad2f7f5359a3b3f, + 0xfd87b5f28300ca0e, 0x9e74d1b791e07e49, 0xc612062576589ddb, + 0xf79687aed3eec552, 0x9abe14cd44753b53, 0xc16d9a0095928a28, + 0xf1c90080baf72cb2, 0x971da05074da7bef, 0xbce5086492111aeb, + 0xec1e4a7db69561a6, 0x9392ee8e921d5d08, 0xb877aa3236a4b44a, + 0xe69594bec44de15c, 0x901d7cf73ab0acda, 0xb424dc35095cd810, + 0xe12e13424bb40e14, 0x8cbccc096f5088cc, 0xafebff0bcb24aaff, + 0xdbe6fecebdedd5bf, 0x89705f4136b4a598, 0xabcc77118461cefd, + 0xd6bf94d5e57a42bd, 0x8637bd05af6c69b6, 0xa7c5ac471b478424, + 0xd1b71758e219652c, 0x83126e978d4fdf3c, 0xa3d70a3d70a3d70b, + 0xcccccccccccccccd, 0x8000000000000000, 0xa000000000000000, + 0xc800000000000000, 0xfa00000000000000, 0x9c40000000000000, + 0xc350000000000000, 0xf424000000000000, 0x9896800000000000, + 0xbebc200000000000, 0xee6b280000000000, 0x9502f90000000000, + 0xba43b74000000000, 0xe8d4a51000000000, 0x9184e72a00000000, + 0xb5e620f480000000, 0xe35fa931a0000000, 0x8e1bc9bf04000000, + 0xb1a2bc2ec5000000, 0xde0b6b3a76400000, 0x8ac7230489e80000, + 0xad78ebc5ac620000, 0xd8d726b7177a8000, 0x878678326eac9000, + 0xa968163f0a57b400, 0xd3c21bcecceda100, 0x84595161401484a0, + 0xa56fa5b99019a5c8, 0xcecb8f27f4200f3a, 0x813f3978f8940984, + 0xa18f07d736b90be5, 0xc9f2c9cd04674ede, 0xfc6f7c4045812296, + 0x9dc5ada82b70b59d, 0xc5371912364ce305, 0xf684df56c3e01bc6, + 0x9a130b963a6c115c, 0xc097ce7bc90715b3, 0xf0bdc21abb48db20, + 0x96769950b50d88f4, 0xbc143fa4e250eb31, 0xeb194f8e1ae525fd, + 0x92efd1b8d0cf37be, 0xb7abc627050305ad, 0xe596b7b0c643c719, + 0x8f7e32ce7bea5c6f, 0xb35dbf821ae4f38b, 0xe0352f62a19e306e}; + +template +const uint128_wrapper basic_data::dragonbox_pow10_significands_128[] = { +#if FMT_USE_FULL_CACHE_DRAGONBOX + {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, + {0x9faacf3df73609b1, 0x77b191618c54e9ad}, + {0xc795830d75038c1d, 0xd59df5b9ef6a2418}, + {0xf97ae3d0d2446f25, 0x4b0573286b44ad1e}, + {0x9becce62836ac577, 0x4ee367f9430aec33}, + {0xc2e801fb244576d5, 0x229c41f793cda740}, + {0xf3a20279ed56d48a, 0x6b43527578c11110}, + {0x9845418c345644d6, 0x830a13896b78aaaa}, + {0xbe5691ef416bd60c, 0x23cc986bc656d554}, + {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa9}, + {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6aa}, + {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc54}, + {0xe858ad248f5c22c9, 0xd1b3400f8f9cff69}, + {0x91376c36d99995be, 0x23100809b9c21fa2}, + {0xb58547448ffffb2d, 0xabd40a0c2832a78b}, + {0xe2e69915b3fff9f9, 0x16c90c8f323f516d}, + {0x8dd01fad907ffc3b, 0xae3da7d97f6792e4}, + {0xb1442798f49ffb4a, 0x99cd11cfdf41779d}, + {0xdd95317f31c7fa1d, 0x40405643d711d584}, + {0x8a7d3eef7f1cfc52, 0x482835ea666b2573}, + {0xad1c8eab5ee43b66, 0xda3243650005eed0}, + {0xd863b256369d4a40, 0x90bed43e40076a83}, + {0x873e4f75e2224e68, 0x5a7744a6e804a292}, + {0xa90de3535aaae202, 0x711515d0a205cb37}, + {0xd3515c2831559a83, 0x0d5a5b44ca873e04}, + {0x8412d9991ed58091, 0xe858790afe9486c3}, + {0xa5178fff668ae0b6, 0x626e974dbe39a873}, + {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, + {0x80fa687f881c7f8e, 0x7ce66634bc9d0b9a}, + {0xa139029f6a239f72, 0x1c1fffc1ebc44e81}, + {0xc987434744ac874e, 0xa327ffb266b56221}, + {0xfbe9141915d7a922, 0x4bf1ff9f0062baa9}, + {0x9d71ac8fada6c9b5, 0x6f773fc3603db4aa}, + {0xc4ce17b399107c22, 0xcb550fb4384d21d4}, + {0xf6019da07f549b2b, 0x7e2a53a146606a49}, + {0x99c102844f94e0fb, 0x2eda7444cbfc426e}, + {0xc0314325637a1939, 0xfa911155fefb5309}, + {0xf03d93eebc589f88, 0x793555ab7eba27cb}, + {0x96267c7535b763b5, 0x4bc1558b2f3458df}, + {0xbbb01b9283253ca2, 0x9eb1aaedfb016f17}, + {0xea9c227723ee8bcb, 0x465e15a979c1cadd}, + {0x92a1958a7675175f, 0x0bfacd89ec191eca}, + {0xb749faed14125d36, 0xcef980ec671f667c}, + {0xe51c79a85916f484, 0x82b7e12780e7401b}, + {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908811}, + {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa16}, + {0xdfbdcece67006ac9, 0x67a791e093e1d49b}, + {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e1}, + {0xaecc49914078536d, 0x58fae9f773886e19}, + {0xda7f5bf590966848, 0xaf39a475506a899f}, + {0x888f99797a5e012d, 0x6d8406c952429604}, + {0xaab37fd7d8f58178, 0xc8e5087ba6d33b84}, + {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a65}, + {0x855c3be0a17fcd26, 0x5cf2eea09a550680}, + {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f}, + {0xd0601d8efc57b08b, 0xf13b94daf124da27}, + {0x823c12795db6ce57, 0x76c53d08d6b70859}, + {0xa2cb1717b52481ed, 0x54768c4b0c64ca6f}, + {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd0a}, + {0xfe5d54150b090b02, 0xd3f93b35435d7c4d}, + {0x9efa548d26e5a6e1, 0xc47bc5014a1a6db0}, + {0xc6b8e9b0709f109a, 0x359ab6419ca1091c}, + {0xf867241c8cc6d4c0, 0xc30163d203c94b63}, + {0x9b407691d7fc44f8, 0x79e0de63425dcf1e}, + {0xc21094364dfb5636, 0x985915fc12f542e5}, + {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939e}, + {0x979cf3ca6cec5b5a, 0xa705992ceecf9c43}, + {0xbd8430bd08277231, 0x50c6ff782a838354}, + {0xece53cec4a314ebd, 0xa4f8bf5635246429}, + {0x940f4613ae5ed136, 0x871b7795e136be9a}, + {0xb913179899f68584, 0x28e2557b59846e40}, + {0xe757dd7ec07426e5, 0x331aeada2fe589d0}, + {0x9096ea6f3848984f, 0x3ff0d2c85def7622}, + {0xb4bca50b065abe63, 0x0fed077a756b53aa}, + {0xe1ebce4dc7f16dfb, 0xd3e8495912c62895}, + {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95d}, + {0xb080392cc4349dec, 0xbd8d794d96aacfb4}, + {0xdca04777f541c567, 0xecf0d7a0fc5583a1}, + {0x89e42caaf9491b60, 0xf41686c49db57245}, + {0xac5d37d5b79b6239, 0x311c2875c522ced6}, + {0xd77485cb25823ac7, 0x7d633293366b828c}, + {0x86a8d39ef77164bc, 0xae5dff9c02033198}, + {0xa8530886b54dbdeb, 0xd9f57f830283fdfd}, + {0xd267caa862a12d66, 0xd072df63c324fd7c}, + {0x8380dea93da4bc60, 0x4247cb9e59f71e6e}, + {0xa46116538d0deb78, 0x52d9be85f074e609}, + {0xcd795be870516656, 0x67902e276c921f8c}, + {0x806bd9714632dff6, 0x00ba1cd8a3db53b7}, + {0xa086cfcd97bf97f3, 0x80e8a40eccd228a5}, + {0xc8a883c0fdaf7df0, 0x6122cd128006b2ce}, + {0xfad2a4b13d1b5d6c, 0x796b805720085f82}, + {0x9cc3a6eec6311a63, 0xcbe3303674053bb1}, + {0xc3f490aa77bd60fc, 0xbedbfc4411068a9d}, + {0xf4f1b4d515acb93b, 0xee92fb5515482d45}, + {0x991711052d8bf3c5, 0x751bdd152d4d1c4b}, + {0xbf5cd54678eef0b6, 0xd262d45a78a0635e}, + {0xef340a98172aace4, 0x86fb897116c87c35}, + {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da1}, + {0xbae0a846d2195712, 0x8974836059cca10a}, + {0xe998d258869facd7, 0x2bd1a438703fc94c}, + {0x91ff83775423cc06, 0x7b6306a34627ddd0}, + {0xb67f6455292cbf08, 0x1a3bc84c17b1d543}, + {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a94}, + {0x8e938662882af53e, 0x547eb47b7282ee9d}, + {0xb23867fb2a35b28d, 0xe99e619a4f23aa44}, + {0xdec681f9f4c31f31, 0x6405fa00e2ec94d5}, + {0x8b3c113c38f9f37e, 0xde83bc408dd3dd05}, + {0xae0b158b4738705e, 0x9624ab50b148d446}, + {0xd98ddaee19068c76, 0x3badd624dd9b0958}, + {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d7}, + {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4d}, + {0xd47487cc8470652b, 0x7647c32000696720}, + {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e074}, + {0xa5fb0a17c777cf09, 0xf468107100525891}, + {0xcf79cc9db955c2cc, 0x7182148d4066eeb5}, + {0x81ac1fe293d599bf, 0xc6f14cd848405531}, + {0xa21727db38cb002f, 0xb8ada00e5a506a7d}, + {0xca9cf1d206fdc03b, 0xa6d90811f0e4851d}, + {0xfd442e4688bd304a, 0x908f4a166d1da664}, + {0x9e4a9cec15763e2e, 0x9a598e4e043287ff}, + {0xc5dd44271ad3cdba, 0x40eff1e1853f29fe}, + {0xf7549530e188c128, 0xd12bee59e68ef47d}, + {0x9a94dd3e8cf578b9, 0x82bb74f8301958cf}, + {0xc13a148e3032d6e7, 0xe36a52363c1faf02}, + {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac2}, + {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0ba}, + {0xbcb2b812db11a5de, 0x7415d448f6b6f0e8}, + {0xebdf661791d60f56, 0x111b495b3464ad22}, + {0x936b9fcebb25c995, 0xcab10dd900beec35}, + {0xb84687c269ef3bfb, 0x3d5d514f40eea743}, + {0xe65829b3046b0afa, 0x0cb4a5a3112a5113}, + {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ac}, + {0xb3f4e093db73a093, 0x59ed216765690f57}, + {0xe0f218b8d25088b8, 0x306869c13ec3532d}, + {0x8c974f7383725573, 0x1e414218c73a13fc}, + {0xafbd2350644eeacf, 0xe5d1929ef90898fb}, + {0xdbac6c247d62a583, 0xdf45f746b74abf3a}, + {0x894bc396ce5da772, 0x6b8bba8c328eb784}, + {0xab9eb47c81f5114f, 0x066ea92f3f326565}, + {0xd686619ba27255a2, 0xc80a537b0efefebe}, + {0x8613fd0145877585, 0xbd06742ce95f5f37}, + {0xa798fc4196e952e7, 0x2c48113823b73705}, + {0xd17f3b51fca3a7a0, 0xf75a15862ca504c6}, + {0x82ef85133de648c4, 0x9a984d73dbe722fc}, + {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebbb}, + {0xcc963fee10b7d1b3, 0x318df905079926a9}, + {0xffbbcfe994e5c61f, 0xfdf17746497f7053}, + {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa634}, + {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc1}, + {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b1}, + {0x9c1661a651213e2d, 0x06bea10ca65c084f}, + {0xc31bfa0fe5698db8, 0x486e494fcff30a63}, + {0xf3e2f893dec3f126, 0x5a89dba3c3efccfb}, + {0x986ddb5c6b3a76b7, 0xf89629465a75e01d}, + {0xbe89523386091465, 0xf6bbb397f1135824}, + {0xee2ba6c0678b597f, 0x746aa07ded582e2d}, + {0x94db483840b717ef, 0xa8c2a44eb4571cdd}, + {0xba121a4650e4ddeb, 0x92f34d62616ce414}, + {0xe896a0d7e51e1566, 0x77b020baf9c81d18}, + {0x915e2486ef32cd60, 0x0ace1474dc1d122f}, + {0xb5b5ada8aaff80b8, 0x0d819992132456bb}, + {0xe3231912d5bf60e6, 0x10e1fff697ed6c6a}, + {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2}, + {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb3}, + {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbdf}, + {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96c}, + {0xad4ab7112eb3929d, 0x86c16c98d2c953c7}, + {0xd89d64d57a607744, 0xe871c7bf077ba8b8}, + {0x87625f056c7c4a8b, 0x11471cd764ad4973}, + {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bd0}, + {0xd389b47879823479, 0x4aff1d108d4ec2c4}, + {0x843610cb4bf160cb, 0xcedf722a585139bb}, + {0xa54394fe1eedb8fe, 0xc2974eb4ee658829}, + {0xce947a3da6a9273e, 0x733d226229feea33}, + {0x811ccc668829b887, 0x0806357d5a3f5260}, + {0xa163ff802a3426a8, 0xca07c2dcb0cf26f8}, + {0xc9bcff6034c13052, 0xfc89b393dd02f0b6}, + {0xfc2c3f3841f17c67, 0xbbac2078d443ace3}, + {0x9d9ba7832936edc0, 0xd54b944b84aa4c0e}, + {0xc5029163f384a931, 0x0a9e795e65d4df12}, + {0xf64335bcf065d37d, 0x4d4617b5ff4a16d6}, + {0x99ea0196163fa42e, 0x504bced1bf8e4e46}, + {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d7}, + {0xf07da27a82c37088, 0x5d767327bb4e5a4d}, + {0x964e858c91ba2655, 0x3a6a07f8d510f870}, + {0xbbe226efb628afea, 0x890489f70a55368c}, + {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842f}, + {0x92c8ae6b464fc96f, 0x3b0b8bc90012929e}, + {0xb77ada0617e3bbcb, 0x09ce6ebb40173745}, + {0xe55990879ddcaabd, 0xcc420a6a101d0516}, + {0x8f57fa54c2a9eab6, 0x9fa946824a12232e}, + {0xb32df8e9f3546564, 0x47939822dc96abfa}, + {0xdff9772470297ebd, 0x59787e2b93bc56f8}, + {0x8bfbea76c619ef36, 0x57eb4edb3c55b65b}, + {0xaefae51477a06b03, 0xede622920b6b23f2}, + {0xdab99e59958885c4, 0xe95fab368e45ecee}, + {0x88b402f7fd75539b, 0x11dbcb0218ebb415}, + {0xaae103b5fcd2a881, 0xd652bdc29f26a11a}, + {0xd59944a37c0752a2, 0x4be76d3346f04960}, + {0x857fcae62d8493a5, 0x6f70a4400c562ddc}, + {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb953}, + {0xd097ad07a71f26b2, 0x7e2000a41346a7a8}, + {0x825ecc24c873782f, 0x8ed400668c0c28c9}, + {0xa2f67f2dfa90563b, 0x728900802f0f32fb}, + {0xcbb41ef979346bca, 0x4f2b40a03ad2ffba}, + {0xfea126b7d78186bc, 0xe2f610c84987bfa9}, + {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7ca}, + {0xc6ede63fa05d3143, 0x91503d1c79720dbc}, + {0xf8a95fcf88747d94, 0x75a44c6397ce912b}, + {0x9b69dbe1b548ce7c, 0xc986afbe3ee11abb}, + {0xc24452da229b021b, 0xfbe85badce996169}, + {0xf2d56790ab41c2a2, 0xfae27299423fb9c4}, + {0x97c560ba6b0919a5, 0xdccd879fc967d41b}, + {0xbdb6b8e905cb600f, 0x5400e987bbc1c921}, + {0xed246723473e3813, 0x290123e9aab23b69}, + {0x9436c0760c86e30b, 0xf9a0b6720aaf6522}, + {0xb94470938fa89bce, 0xf808e40e8d5b3e6a}, + {0xe7958cb87392c2c2, 0xb60b1d1230b20e05}, + {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c3}, + {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af4}, + {0xe2280b6c20dd5232, 0x25c6da63c38de1b1}, + {0x8d590723948a535f, 0x579c487e5a38ad0f}, + {0xb0af48ec79ace837, 0x2d835a9df0c6d852}, + {0xdcdb1b2798182244, 0xf8e431456cf88e66}, + {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b5900}, + {0xac8b2d36eed2dac5, 0xe272467e3d222f40}, + {0xd7adf884aa879177, 0x5b0ed81dcc6abb10}, + {0x86ccbb52ea94baea, 0x98e947129fc2b4ea}, + {0xa87fea27a539e9a5, 0x3f2398d747b36225}, + {0xd29fe4b18e88640e, 0x8eec7f0d19a03aae}, + {0x83a3eeeef9153e89, 0x1953cf68300424ad}, + {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd8}, + {0xcdb02555653131b6, 0x3792f412cb06794e}, + {0x808e17555f3ebf11, 0xe2bbd88bbee40bd1}, + {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec5}, + {0xc8de047564d20a8b, 0xf245825a5a445276}, + {0xfb158592be068d2e, 0xeed6e2f0f0d56713}, + {0x9ced737bb6c4183d, 0x55464dd69685606c}, + {0xc428d05aa4751e4c, 0xaa97e14c3c26b887}, + {0xf53304714d9265df, 0xd53dd99f4b3066a9}, + {0x993fe2c6d07b7fab, 0xe546a8038efe402a}, + {0xbf8fdb78849a5f96, 0xde98520472bdd034}, + {0xef73d256a5c0f77c, 0x963e66858f6d4441}, + {0x95a8637627989aad, 0xdde7001379a44aa9}, + {0xbb127c53b17ec159, 0x5560c018580d5d53}, + {0xe9d71b689dde71af, 0xaab8f01e6e10b4a7}, + {0x9226712162ab070d, 0xcab3961304ca70e9}, + {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d23}, + {0xe45c10c42a2b3b05, 0x8cb89a7db77c506b}, + {0x8eb98a7a9a5b04e3, 0x77f3608e92adb243}, + {0xb267ed1940f1c61c, 0x55f038b237591ed4}, + {0xdf01e85f912e37a3, 0x6b6c46dec52f6689}, + {0x8b61313bbabce2c6, 0x2323ac4b3b3da016}, + {0xae397d8aa96c1b77, 0xabec975e0a0d081b}, + {0xd9c7dced53c72255, 0x96e7bd358c904a22}, + {0x881cea14545c7575, 0x7e50d64177da2e55}, + {0xaa242499697392d2, 0xdde50bd1d5d0b9ea}, + {0xd4ad2dbfc3d07787, 0x955e4ec64b44e865}, + {0x84ec3c97da624ab4, 0xbd5af13bef0b113f}, + {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58f}, + {0xcfb11ead453994ba, 0x67de18eda5814af3}, + {0x81ceb32c4b43fcf4, 0x80eacf948770ced8}, + {0xa2425ff75e14fc31, 0xa1258379a94d028e}, + {0xcad2f7f5359a3b3e, 0x096ee45813a04331}, + {0xfd87b5f28300ca0d, 0x8bca9d6e188853fd}, + {0x9e74d1b791e07e48, 0x775ea264cf55347e}, + {0xc612062576589dda, 0x95364afe032a819e}, + {0xf79687aed3eec551, 0x3a83ddbd83f52205}, + {0x9abe14cd44753b52, 0xc4926a9672793543}, + {0xc16d9a0095928a27, 0x75b7053c0f178294}, + {0xf1c90080baf72cb1, 0x5324c68b12dd6339}, + {0x971da05074da7bee, 0xd3f6fc16ebca5e04}, + {0xbce5086492111aea, 0x88f4bb1ca6bcf585}, + {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6}, + {0x9392ee8e921d5d07, 0x3aff322e62439fd0}, + {0xb877aa3236a4b449, 0x09befeb9fad487c3}, + {0xe69594bec44de15b, 0x4c2ebe687989a9b4}, + {0x901d7cf73ab0acd9, 0x0f9d37014bf60a11}, + {0xb424dc35095cd80f, 0x538484c19ef38c95}, + {0xe12e13424bb40e13, 0x2865a5f206b06fba}, + {0x8cbccc096f5088cb, 0xf93f87b7442e45d4}, + {0xafebff0bcb24aafe, 0xf78f69a51539d749}, + {0xdbe6fecebdedd5be, 0xb573440e5a884d1c}, + {0x89705f4136b4a597, 0x31680a88f8953031}, + {0xabcc77118461cefc, 0xfdc20d2b36ba7c3e}, + {0xd6bf94d5e57a42bc, 0x3d32907604691b4d}, + {0x8637bd05af6c69b5, 0xa63f9a49c2c1b110}, + {0xa7c5ac471b478423, 0x0fcf80dc33721d54}, + {0xd1b71758e219652b, 0xd3c36113404ea4a9}, + {0x83126e978d4fdf3b, 0x645a1cac083126ea}, + {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4}, + {0xcccccccccccccccc, 0xcccccccccccccccd}, + {0x8000000000000000, 0x0000000000000000}, + {0xa000000000000000, 0x0000000000000000}, + {0xc800000000000000, 0x0000000000000000}, + {0xfa00000000000000, 0x0000000000000000}, + {0x9c40000000000000, 0x0000000000000000}, + {0xc350000000000000, 0x0000000000000000}, + {0xf424000000000000, 0x0000000000000000}, + {0x9896800000000000, 0x0000000000000000}, + {0xbebc200000000000, 0x0000000000000000}, + {0xee6b280000000000, 0x0000000000000000}, + {0x9502f90000000000, 0x0000000000000000}, + {0xba43b74000000000, 0x0000000000000000}, + {0xe8d4a51000000000, 0x0000000000000000}, + {0x9184e72a00000000, 0x0000000000000000}, + {0xb5e620f480000000, 0x0000000000000000}, + {0xe35fa931a0000000, 0x0000000000000000}, + {0x8e1bc9bf04000000, 0x0000000000000000}, + {0xb1a2bc2ec5000000, 0x0000000000000000}, + {0xde0b6b3a76400000, 0x0000000000000000}, + {0x8ac7230489e80000, 0x0000000000000000}, + {0xad78ebc5ac620000, 0x0000000000000000}, + {0xd8d726b7177a8000, 0x0000000000000000}, + {0x878678326eac9000, 0x0000000000000000}, + {0xa968163f0a57b400, 0x0000000000000000}, + {0xd3c21bcecceda100, 0x0000000000000000}, + {0x84595161401484a0, 0x0000000000000000}, + {0xa56fa5b99019a5c8, 0x0000000000000000}, + {0xcecb8f27f4200f3a, 0x0000000000000000}, + {0x813f3978f8940984, 0x4000000000000000}, + {0xa18f07d736b90be5, 0x5000000000000000}, + {0xc9f2c9cd04674ede, 0xa400000000000000}, + {0xfc6f7c4045812296, 0x4d00000000000000}, + {0x9dc5ada82b70b59d, 0xf020000000000000}, + {0xc5371912364ce305, 0x6c28000000000000}, + {0xf684df56c3e01bc6, 0xc732000000000000}, + {0x9a130b963a6c115c, 0x3c7f400000000000}, + {0xc097ce7bc90715b3, 0x4b9f100000000000}, + {0xf0bdc21abb48db20, 0x1e86d40000000000}, + {0x96769950b50d88f4, 0x1314448000000000}, + {0xbc143fa4e250eb31, 0x17d955a000000000}, + {0xeb194f8e1ae525fd, 0x5dcfab0800000000}, + {0x92efd1b8d0cf37be, 0x5aa1cae500000000}, + {0xb7abc627050305ad, 0xf14a3d9e40000000}, + {0xe596b7b0c643c719, 0x6d9ccd05d0000000}, + {0x8f7e32ce7bea5c6f, 0xe4820023a2000000}, + {0xb35dbf821ae4f38b, 0xdda2802c8a800000}, + {0xe0352f62a19e306e, 0xd50b2037ad200000}, + {0x8c213d9da502de45, 0x4526f422cc340000}, + {0xaf298d050e4395d6, 0x9670b12b7f410000}, + {0xdaf3f04651d47b4c, 0x3c0cdd765f114000}, + {0x88d8762bf324cd0f, 0xa5880a69fb6ac800}, + {0xab0e93b6efee0053, 0x8eea0d047a457a00}, + {0xd5d238a4abe98068, 0x72a4904598d6d880}, + {0x85a36366eb71f041, 0x47a6da2b7f864750}, + {0xa70c3c40a64e6c51, 0x999090b65f67d924}, + {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d}, + {0x82818f1281ed449f, 0xbff8f10e7a8921a4}, + {0xa321f2d7226895c7, 0xaff72d52192b6a0d}, + {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490}, + {0xfee50b7025c36a08, 0x02f236d04753d5b4}, + {0x9f4f2726179a2245, 0x01d762422c946590}, + {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5}, + {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2}, + {0x9b934c3b330c8577, 0x63cc55f49f88eb2f}, + {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb}, + {0xf316271c7fc3908a, 0x8bef464e3945ef7a}, + {0x97edd871cfda3a56, 0x97758bf0e3cbb5ac}, + {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317}, + {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd}, + {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a}, + {0xb975d6b6ee39e436, 0xb3e2fd538e122b44}, + {0xe7d34c64a9c85d44, 0x60dbbca87196b616}, + {0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd}, + {0xb51d13aea4a488dd, 0x6babab6398bdbe41}, + {0xe264589a4dcdab14, 0xc696963c7eed2dd1}, + {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2}, + {0xb0de65388cc8ada8, 0x3b25a55f43294bcb}, + {0xdd15fe86affad912, 0x49ef0eb713f39ebe}, + {0x8a2dbf142dfcc7ab, 0x6e3569326c784337}, + {0xacb92ed9397bf996, 0x49c2c37f07965404}, + {0xd7e77a8f87daf7fb, 0xdc33745ec97be906}, + {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3}, + {0xa8acd7c0222311bc, 0xc40832ea0d68ce0c}, + {0xd2d80db02aabd62b, 0xf50a3fa490c30190}, + {0x83c7088e1aab65db, 0x792667c6da79e0fa}, + {0xa4b8cab1a1563f52, 0x577001b891185938}, + {0xcde6fd5e09abcf26, 0xed4c0226b55e6f86}, + {0x80b05e5ac60b6178, 0x544f8158315b05b4}, + {0xa0dc75f1778e39d6, 0x696361ae3db1c721}, + {0xc913936dd571c84c, 0x03bc3a19cd1e38e9}, + {0xfb5878494ace3a5f, 0x04ab48a04065c723}, + {0x9d174b2dcec0e47b, 0x62eb0d64283f9c76}, + {0xc45d1df942711d9a, 0x3ba5d0bd324f8394}, + {0xf5746577930d6500, 0xca8f44ec7ee36479}, + {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb}, + {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e}, + {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e}, + {0x95d04aee3b80ece5, 0xbba1f1d158724a12}, + {0xbb445da9ca61281f, 0x2a8a6e45ae8edc97}, + {0xea1575143cf97226, 0xf52d09d71a3293bd}, + {0x924d692ca61be758, 0x593c2626705f9c56}, + {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c}, + {0xe498f455c38b997a, 0x0b6dfb9c0f956447}, + {0x8edf98b59a373fec, 0x4724bd4189bd5eac}, + {0xb2977ee300c50fe7, 0x58edec91ec2cb657}, + {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed}, + {0x8b865b215899f46c, 0xbd79e0d20082ee74}, + {0xae67f1e9aec07187, 0xecd8590680a3aa11}, + {0xda01ee641a708de9, 0xe80e6f4820cc9495}, + {0x884134fe908658b2, 0x3109058d147fdcdd}, + {0xaa51823e34a7eede, 0xbd4b46f0599fd415}, + {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a}, + {0x850fadc09923329e, 0x03e2cf6bc604ddb0}, + {0xa6539930bf6bff45, 0x84db8346b786151c}, + {0xcfe87f7cef46ff16, 0xe612641865679a63}, + {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e}, + {0xa26da3999aef7749, 0xe3be5e330f38f09d}, + {0xcb090c8001ab551c, 0x5cadf5bfd3072cc5}, + {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6}, + {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa}, + {0xc646d63501a1511d, 0xb281e1fd541501b8}, + {0xf7d88bc24209a565, 0x1f225a7ca91a4226}, + {0x9ae757596946075f, 0x3375788de9b06958}, + {0xc1a12d2fc3978937, 0x0052d6b1641c83ae}, + {0xf209787bb47d6b84, 0xc0678c5dbd23a49a}, + {0x9745eb4d50ce6332, 0xf840b7ba963646e0}, + {0xbd176620a501fbff, 0xb650e5a93bc3d898}, + {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe}, + {0x93ba47c980e98cdf, 0xc66f336c36b10137}, + {0xb8a8d9bbe123f017, 0xb80b0047445d4184}, + {0xe6d3102ad96cec1d, 0xa60dc059157491e5}, + {0x9043ea1ac7e41392, 0x87c89837ad68db2f}, + {0xb454e4a179dd1877, 0x29babe4598c311fb}, + {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a}, + {0x8ce2529e2734bb1d, 0x1899e4a65f58660c}, + {0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f}, + {0xdc21a1171d42645d, 0x76707543f4fa1f73}, + {0x899504ae72497eba, 0x6a06494a791c53a8}, + {0xabfa45da0edbde69, 0x0487db9d17636892}, + {0xd6f8d7509292d603, 0x45a9d2845d3c42b6}, + {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b2}, + {0xa7f26836f282b732, 0x8e6cac7768d7141e}, + {0xd1ef0244af2364ff, 0x3207d795430cd926}, + {0x8335616aed761f1f, 0x7f44e6bd49e807b8}, + {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6}, + {0xcd036837130890a1, 0x36dba887c37a8c0f}, + {0x802221226be55a64, 0xc2494954da2c9789}, + {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c}, + {0xc83553c5c8965d3d, 0x6f92829494e5acc7}, + {0xfa42a8b73abbf48c, 0xcb772339ba1f17f9}, + {0x9c69a97284b578d7, 0xff2a760414536efb}, + {0xc38413cf25e2d70d, 0xfef5138519684aba}, + {0xf46518c2ef5b8cd1, 0x7eb258665fc25d69}, + {0x98bf2f79d5993802, 0xef2f773ffbd97a61}, + {0xbeeefb584aff8603, 0xaafb550ffacfd8fa}, + {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38}, + {0x952ab45cfa97a0b2, 0xdd945a747bf26183}, + {0xba756174393d88df, 0x94f971119aeef9e4}, + {0xe912b9d1478ceb17, 0x7a37cd5601aab85d}, + {0x91abb422ccb812ee, 0xac62e055c10ab33a}, + {0xb616a12b7fe617aa, 0x577b986b314d6009}, + {0xe39c49765fdf9d94, 0xed5a7e85fda0b80b}, + {0x8e41ade9fbebc27d, 0x14588f13be847307}, + {0xb1d219647ae6b31c, 0x596eb2d8ae258fc8}, + {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb}, + {0x8aec23d680043bee, 0x25de7bb9480d5854}, + {0xada72ccc20054ae9, 0xaf561aa79a10ae6a}, + {0xd910f7ff28069da4, 0x1b2ba1518094da04}, + {0x87aa9aff79042286, 0x90fb44d2f05d0842}, + {0xa99541bf57452b28, 0x353a1607ac744a53}, + {0xd3fa922f2d1675f2, 0x42889b8997915ce8}, + {0x847c9b5d7c2e09b7, 0x69956135febada11}, + {0xa59bc234db398c25, 0x43fab9837e699095}, + {0xcf02b2c21207ef2e, 0x94f967e45e03f4bb}, + {0x8161afb94b44f57d, 0x1d1be0eebac278f5}, + {0xa1ba1ba79e1632dc, 0x6462d92a69731732}, + {0xca28a291859bbf93, 0x7d7b8f7503cfdcfe}, + {0xfcb2cb35e702af78, 0x5cda735244c3d43e}, + {0x9defbf01b061adab, 0x3a0888136afa64a7}, + {0xc56baec21c7a1916, 0x088aaa1845b8fdd0}, + {0xf6c69a72a3989f5b, 0x8aad549e57273d45}, + {0x9a3c2087a63f6399, 0x36ac54e2f678864b}, + {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd}, + {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5}, + {0x969eb7c47859e743, 0x9f644ae5a4b1b325}, + {0xbc4665b596706114, 0x873d5d9f0dde1fee}, + {0xeb57ff22fc0c7959, 0xa90cb506d155a7ea}, + {0x9316ff75dd87cbd8, 0x09a7f12442d588f2}, + {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb2f}, + {0xe5d3ef282a242e81, 0x8f1668c8a86da5fa}, + {0x8fa475791a569d10, 0xf96e017d694487bc}, + {0xb38d92d760ec4455, 0x37c981dcc395a9ac}, + {0xe070f78d3927556a, 0x85bbe253f47b1417}, + {0x8c469ab843b89562, 0x93956d7478ccec8e}, + {0xaf58416654a6babb, 0x387ac8d1970027b2}, + {0xdb2e51bfe9d0696a, 0x06997b05fcc0319e}, + {0x88fcf317f22241e2, 0x441fece3bdf81f03}, + {0xab3c2fddeeaad25a, 0xd527e81cad7626c3}, + {0xd60b3bd56a5586f1, 0x8a71e223d8d3b074}, + {0x85c7056562757456, 0xf6872d5667844e49}, + {0xa738c6bebb12d16c, 0xb428f8ac016561db}, + {0xd106f86e69d785c7, 0xe13336d701beba52}, + {0x82a45b450226b39c, 0xecc0024661173473}, + {0xa34d721642b06084, 0x27f002d7f95d0190}, + {0xcc20ce9bd35c78a5, 0x31ec038df7b441f4}, + {0xff290242c83396ce, 0x7e67047175a15271}, + {0x9f79a169bd203e41, 0x0f0062c6e984d386}, + {0xc75809c42c684dd1, 0x52c07b78a3e60868}, + {0xf92e0c3537826145, 0xa7709a56ccdf8a82}, + {0x9bbcc7a142b17ccb, 0x88a66076400bb691}, + {0xc2abf989935ddbfe, 0x6acff893d00ea435}, + {0xf356f7ebf83552fe, 0x0583f6b8c4124d43}, + {0x98165af37b2153de, 0xc3727a337a8b704a}, + {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c}, + {0xeda2ee1c7064130c, 0x1162def06f79df73}, + {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8}, + {0xb9a74a0637ce2ee1, 0x6d953e2bd7173692}, + {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437}, + {0x910ab1d4db9914a0, 0x1d9c9892400a22a2}, + {0xb54d5e4a127f59c8, 0x2503beb6d00cab4b}, + {0xe2a0b5dc971f303a, 0x2e44ae64840fd61d}, + {0x8da471a9de737e24, 0x5ceaecfed289e5d2}, + {0xb10d8e1456105dad, 0x7425a83e872c5f47}, + {0xdd50f1996b947518, 0xd12f124e28f77719}, + {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f}, + {0xace73cbfdc0bfb7b, 0x636cc64d1001550b}, + {0xd8210befd30efa5a, 0x3c47f7e05401aa4e}, + {0x8714a775e3e95c78, 0x65acfaec34810a71}, + {0xa8d9d1535ce3b396, 0x7f1839a741a14d0d}, + {0xd31045a8341ca07c, 0x1ede48111209a050}, + {0x83ea2b892091e44d, 0x934aed0aab460432}, + {0xa4e4b66b68b65d60, 0xf81da84d5617853f}, + {0xce1de40642e3f4b9, 0x36251260ab9d668e}, + {0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019}, + {0xa1075a24e4421730, 0xb24cf65b8612f81f}, + {0xc94930ae1d529cfc, 0xdee033f26797b627}, + {0xfb9b7cd9a4a7443c, 0x169840ef017da3b1}, + {0x9d412e0806e88aa5, 0x8e1f289560ee864e}, + {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2}, + {0xf5b5d7ec8acb58a2, 0xae10af696774b1db}, + {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29}, + {0xbff610b0cc6edd3f, 0x17fd090a58d32af3}, + {0xeff394dcff8a948e, 0xddfc4b4cef07f5b0}, + {0x95f83d0a1fb69cd9, 0x4abdaf101564f98e}, + {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1}, + {0xea53df5fd18d5513, 0x84c86189216dc5ed}, + {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4}, + {0xb7118682dbb66a77, 0x3fbc8c33221dc2a1}, + {0xe4d5e82392a40515, 0x0fabaf3feaa5334a}, + {0x8f05b1163ba6832d, 0x29cb4d87f2a7400e}, + {0xb2c71d5bca9023f8, 0x743e20e9ef511012}, + {0xdf78e4b2bd342cf6, 0x914da9246b255416}, + {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e}, + {0xae9672aba3d0c320, 0xa184ac2473b529b1}, + {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e}, + {0x8865899617fb1871, 0x7e2fa67c7a658892}, + {0xaa7eebfb9df9de8d, 0xddbb901b98feeab7}, + {0xd51ea6fa85785631, 0x552a74227f3ea565}, + {0x8533285c936b35de, 0xd53a88958f87275f}, + {0xa67ff273b8460356, 0x8a892abaf368f137}, + {0xd01fef10a657842c, 0x2d2b7569b0432d85}, + {0x8213f56a67f6b29b, 0x9c3b29620e29fc73}, + {0xa298f2c501f45f42, 0x8349f3ba91b47b8f}, + {0xcb3f2f7642717713, 0x241c70a936219a73}, + {0xfe0efb53d30dd4d7, 0xed238cd383aa0110}, + {0x9ec95d1463e8a506, 0xf4363804324a40aa}, + {0xc67bb4597ce2ce48, 0xb143c6053edcd0d5}, + {0xf81aa16fdc1b81da, 0xdd94b7868e94050a}, + {0x9b10a4e5e9913128, 0xca7cf2b4191c8326}, + {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0}, + {0xf24a01a73cf2dccf, 0xbc633b39673c8cec}, + {0x976e41088617ca01, 0xd5be0503e085d813}, + {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18}, + {0xec9c459d51852ba2, 0xddf8e7d60ed1219e}, + {0x93e1ab8252f33b45, 0xcabb90e5c942b503}, + {0xb8da1662e7b00a17, 0x3d6a751f3b936243}, + {0xe7109bfba19c0c9d, 0x0cc512670a783ad4}, + {0x906a617d450187e2, 0x27fb2b80668b24c5}, + {0xb484f9dc9641e9da, 0xb1f9f660802dedf6}, + {0xe1a63853bbd26451, 0x5e7873f8a0396973}, + {0x8d07e33455637eb2, 0xdb0b487b6423e1e8}, + {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62}, + {0xdc5c5301c56b75f7, 0x7641a140cc7810fb}, + {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d}, + {0xac2820d9623bf429, 0x546345fa9fbdcd44}, + {0xd732290fbacaf133, 0xa97c177947ad4095}, + {0x867f59a9d4bed6c0, 0x49ed8eabcccc485d}, + {0xa81f301449ee8c70, 0x5c68f256bfff5a74}, + {0xd226fc195c6a2f8c, 0x73832eec6fff3111}, + {0x83585d8fd9c25db7, 0xc831fd53c5ff7eab}, + {0xa42e74f3d032f525, 0xba3e7ca8b77f5e55}, + {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb}, + {0x80444b5e7aa7cf85, 0x7980d163cf5b81b3}, + {0xa0555e361951c366, 0xd7e105bcc332621f}, + {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7}, + {0xfa856334878fc150, 0xb14f98f6f0feb951}, + {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3}, + {0xc3b8358109e84f07, 0x0a862f80ec4700c8}, + {0xf4a642e14c6262c8, 0xcd27bb612758c0fa}, + {0x98e7e9cccfbd7dbd, 0x8038d51cb897789c}, + {0xbf21e44003acdd2c, 0xe0470a63e6bd56c3}, + {0xeeea5d5004981478, 0x1858ccfce06cac74}, + {0x95527a5202df0ccb, 0x0f37801e0c43ebc8}, + {0xbaa718e68396cffd, 0xd30560258f54e6ba}, + {0xe950df20247c83fd, 0x47c6b82ef32a2069}, + {0x91d28b7416cdd27e, 0x4cdc331d57fa5441}, + {0xb6472e511c81471d, 0xe0133fe4adf8e952}, + {0xe3d8f9e563a198e5, 0x58180fddd97723a6}, + {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648}, + {0xb201833b35d63f73, 0x2cd2cc6551e513da}, + {0xde81e40a034bcf4f, 0xf8077f7ea65e58d1}, + {0x8b112e86420f6191, 0xfb04afaf27faf782}, + {0xadd57a27d29339f6, 0x79c5db9af1f9b563}, + {0xd94ad8b1c7380874, 0x18375281ae7822bc}, + {0x87cec76f1c830548, 0x8f2293910d0b15b5}, + {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb22}, + {0xd433179d9c8cb841, 0x5fa60692a46151eb}, + {0x849feec281d7f328, 0xdbc7c41ba6bcd333}, + {0xa5c7ea73224deff3, 0x12b9b522906c0800}, + {0xcf39e50feae16bef, 0xd768226b34870a00}, + {0x81842f29f2cce375, 0xe6a1158300d46640}, + {0xa1e53af46f801c53, 0x60495ae3c1097fd0}, + {0xca5e89b18b602368, 0x385bb19cb14bdfc4}, + {0xfcf62c1dee382c42, 0x46729e03dd9ed7b5}, + {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d1}, + {0xc5a05277621be293, 0xc7098b7305241885}, + {0xf70867153aa2db38, 0xb8cbee4fc66d1ea7} +#else + {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, + {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, + {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f}, + {0x86a8d39ef77164bc, 0xae5dff9c02033198}, + {0xd98ddaee19068c76, 0x3badd624dd9b0958}, + {0xafbd2350644eeacf, 0xe5d1929ef90898fb}, + {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2}, + {0xe55990879ddcaabd, 0xcc420a6a101d0516}, + {0xb94470938fa89bce, 0xf808e40e8d5b3e6a}, + {0x95a8637627989aad, 0xdde7001379a44aa9}, + {0xf1c90080baf72cb1, 0x5324c68b12dd6339}, + {0xc350000000000000, 0x0000000000000000}, + {0x9dc5ada82b70b59d, 0xf020000000000000}, + {0xfee50b7025c36a08, 0x02f236d04753d5b4}, + {0xcde6fd5e09abcf26, 0xed4c0226b55e6f86}, + {0xa6539930bf6bff45, 0x84db8346b786151c}, + {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b2}, + {0xd910f7ff28069da4, 0x1b2ba1518094da04}, + {0xaf58416654a6babb, 0x387ac8d1970027b2}, + {0x8da471a9de737e24, 0x5ceaecfed289e5d2}, + {0xe4d5e82392a40515, 0x0fabaf3feaa5334a}, + {0xb8da1662e7b00a17, 0x3d6a751f3b936243}, + {0x95527a5202df0ccb, 0x0f37801e0c43ebc8} +#endif +}; + +#if !FMT_USE_FULL_CACHE_DRAGONBOX +template +const uint64_t basic_data::powers_of_5_64[] = { + 0x0000000000000001, 0x0000000000000005, 0x0000000000000019, + 0x000000000000007d, 0x0000000000000271, 0x0000000000000c35, + 0x0000000000003d09, 0x000000000001312d, 0x000000000005f5e1, + 0x00000000001dcd65, 0x00000000009502f9, 0x0000000002e90edd, + 0x000000000e8d4a51, 0x0000000048c27395, 0x000000016bcc41e9, + 0x000000071afd498d, 0x0000002386f26fc1, 0x000000b1a2bc2ec5, + 0x000003782dace9d9, 0x00001158e460913d, 0x000056bc75e2d631, + 0x0001b1ae4d6e2ef5, 0x000878678326eac9, 0x002a5a058fc295ed, + 0x00d3c21bcecceda1, 0x0422ca8b0a00a425, 0x14adf4b7320334b9}; + +template +const uint32_t basic_data::dragonbox_pow10_recovery_errors[] = { + 0x50001400, 0x54044100, 0x54014555, 0x55954415, 0x54115555, 0x00000001, + 0x50000000, 0x00104000, 0x54010004, 0x05004001, 0x55555544, 0x41545555, + 0x54040551, 0x15445545, 0x51555514, 0x10000015, 0x00101100, 0x01100015, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x04450514, 0x45414110, + 0x55555145, 0x50544050, 0x15040155, 0x11054140, 0x50111514, 0x11451454, + 0x00400541, 0x00000000, 0x55555450, 0x10056551, 0x10054011, 0x55551014, + 0x69514555, 0x05151109, 0x00155555}; +#endif + template const char basic_data::foreground_color[] = "\x1b[38;2;"; template @@ -366,6 +1096,10 @@ class fp { private: using significand_type = uint64_t; + template + using is_supported_float = bool_constant; + public: significand_type f; int e; @@ -388,63 +1122,38 @@ class fp { template explicit fp(Double d) { assign(d); } // Assigns d to this and return true iff predecessor is closer than successor. - template - bool assign(Double d) { - // Assume double is in the format [sign][exponent][significand]. - using limits = std::numeric_limits; + template ::value)> + bool assign(Float d) { + // Assume float is in the format [sign][exponent][significand]. + using limits = std::numeric_limits; + const int float_significand_size = limits::digits - 1; const int exponent_size = - bits::value - double_significand_size - 1; // -1 for sign - const uint64_t significand_mask = implicit_bit - 1; + bits::value - float_significand_size - 1; // -1 for sign + const uint64_t float_implicit_bit = 1ULL << float_significand_size; + const uint64_t significand_mask = float_implicit_bit - 1; const uint64_t exponent_mask = (~0ULL >> 1) & ~significand_mask; const int exponent_bias = (1 << exponent_size) - limits::max_exponent - 1; - auto u = bit_cast(d); + constexpr bool is_double = sizeof(Float) == sizeof(uint64_t); + auto u = bit_cast>(d); f = u & significand_mask; int biased_e = - static_cast((u & exponent_mask) >> double_significand_size); + static_cast((u & exponent_mask) >> float_significand_size); // Predecessor is closer if d is a normalized power of 2 (f == 0) other than // the smallest normalized number (biased_e > 1). bool is_predecessor_closer = f == 0 && biased_e > 1; if (biased_e != 0) - f += implicit_bit; + f += float_implicit_bit; else biased_e = 1; // Subnormals use biased exponent 1 (min exponent). - e = biased_e - exponent_bias - double_significand_size; + e = biased_e - exponent_bias - float_significand_size; return is_predecessor_closer; } - template - bool assign(Double) { + template ::value)> + bool assign(Float) { *this = fp(); return false; } - - // Assigns d to this together with computing lower and upper boundaries, - // where a boundary is a value half way between the number and its predecessor - // (lower) or successor (upper). The upper boundary is normalized and lower - // has the same exponent but may be not normalized. - template boundaries assign_with_boundaries(Double d) { - bool is_lower_closer = assign(d); - fp lower = - is_lower_closer ? fp((f << 2) - 1, e - 2) : fp((f << 1) - 1, e - 1); - // 1 in normalize accounts for the exponent shift above. - fp upper = normalize<1>(fp((f << 1) + 1, e - 1)); - lower.f <<= lower.e - upper.e; - return boundaries{lower.f, upper.f}; - } - - template boundaries assign_float_with_boundaries(Double d) { - assign(d); - constexpr int min_normal_e = std::numeric_limits::min_exponent - - std::numeric_limits::digits; - significand_type half_ulp = 1 << (std::numeric_limits::digits - - std::numeric_limits::digits - 1); - if (min_normal_e > e) half_ulp <<= min_normal_e - e; - fp upper = normalize<0>(fp(f + half_ulp, e)); - fp lower = fp( - f - (half_ulp >> ((f == implicit_bit && e > min_normal_e) ? 1 : 0)), e); - lower.f <<= lower.e - upper.e; - return boundaries{lower.f, upper.f}; - } }; // Normalizes the value converted from double and multiplied by (1 << SHIFT). @@ -488,11 +1197,12 @@ inline fp operator*(fp x, fp y) { return {multiply(x.f, y.f), x.e + y.e + 64}; } // Returns a cached power of 10 `c_k = c_k.f * pow(2, c_k.e)` such that its // (binary) exponent satisfies `min_exponent <= c_k.e <= min_exponent + 28`. inline fp get_cached_power(int min_exponent, int& pow10_exponent) { - const int64_t one_over_log2_10 = 0x4d104d42; // round(pow(2, 32) / log2(10)) + const int shift = 32; + const auto significand = static_cast(data::log10_2_significand); int index = static_cast( - ((min_exponent + fp::significand_size - 1) * one_over_log2_10 + - ((int64_t(1) << 32) - 1)) // ceil - >> 32 // arithmetic shift + ((min_exponent + fp::significand_size - 1) * (significand >> shift) + + ((int64_t(1) << shift) - 1)) // ceil + >> 32 // arithmetic shift ); // Decimal exponent of the first (smallest) cached power of 10. const int first_dec_exp = -348; @@ -500,7 +1210,8 @@ inline fp get_cached_power(int min_exponent, int& pow10_exponent) { const int dec_exp_step = 8; index = (index - first_dec_exp - 1) / dec_exp_step + 1; pow10_exponent = first_dec_exp + index * dec_exp_step; - return {data::pow10_significands[index], data::pow10_exponents[index]}; + return {data::grisu_pow10_significands[index], + data::grisu_pow10_exponents[index]}; } // A simple accumulator to hold the sums of terms in bigint::square if uint128_t @@ -559,9 +1270,8 @@ class bigint { FMT_ASSERT(compare(*this, other) >= 0, ""); bigit borrow = 0; int i = other.exp_ - exp_; - for (size_t j = 0, n = other.bigits_.size(); j != n; ++i, ++j) { + for (size_t j = 0, n = other.bigits_.size(); j != n; ++i, ++j) subtract_bigits(i, other.bigits_[j], borrow); - } while (borrow > 0) subtract_bigits(i, 0, borrow); remove_leading_zeros(); } @@ -733,22 +1443,26 @@ class bigint { exp_ *= 2; } + // If this bigint has a bigger exponent than other, adds trailing zero to make + // exponents equal. This simplifies some operations such as subtraction. + void align(const bigint& other) { + int exp_difference = exp_ - other.exp_; + if (exp_difference <= 0) return; + int num_bigits = static_cast(bigits_.size()); + bigits_.resize(to_unsigned(num_bigits + exp_difference)); + for (int i = num_bigits - 1, j = i + exp_difference; i >= 0; --i, --j) + bigits_[j] = bigits_[i]; + std::uninitialized_fill_n(bigits_.data(), exp_difference, 0); + exp_ -= exp_difference; + } + // Divides this bignum by divisor, assigning the remainder to this and // returning the quotient. int divmod_assign(const bigint& divisor) { FMT_ASSERT(this != &divisor, ""); if (compare(*this, divisor) < 0) return 0; - int num_bigits = static_cast(bigits_.size()); FMT_ASSERT(divisor.bigits_[divisor.bigits_.size() - 1u] != 0, ""); - int exp_difference = exp_ - divisor.exp_; - if (exp_difference > 0) { - // Align bigints by adding trailing zeros to simplify subtraction. - bigits_.resize(to_unsigned(num_bigits + exp_difference)); - for (int i = num_bigits - 1, j = i + exp_difference; i >= 0; --i, --j) - bigits_[j] = bigits_[i]; - std::uninitialized_fill_n(bigits_.data(), exp_difference, 0); - exp_ -= exp_difference; - } + align(divisor); int quotient = 0; do { subtract_aligned(divisor); @@ -788,20 +1502,6 @@ enum result { }; } -// A version of count_digits optimized for grisu_gen_digits. -inline int grisu_count_digits(uint32_t n) { - if (n < 10) return 1; - if (n < 100) return 2; - if (n < 1000) return 3; - if (n < 10000) return 4; - if (n < 100000) return 5; - if (n < 1000000) return 6; - if (n < 10000000) return 7; - if (n < 100000000) return 8; - if (n < 1000000000) return 9; - return 10; -} - // Generates output using the Grisu digit-gen algorithm. // error: the size of the region (lower, upper) outside of which numbers // definitely do not round to value (Delta in Grisu3). @@ -817,7 +1517,7 @@ FMT_ALWAYS_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, FMT_ASSERT(integral == value.f >> -one.e, ""); // The fractional part of scaled value (p2 in Grisu) c = value % one. uint64_t fractional = value.f & (one.f - 1); - exp = grisu_count_digits(integral); // kappa in Grisu. + exp = count_digits(integral); // kappa in Grisu. // Divide by 10 to prevent overflow. auto result = handler.on_start(data::powers_of_10_64[exp - 1] << -one.e, value.f / 10, error * 10, exp); @@ -867,8 +1567,7 @@ FMT_ALWAYS_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, FMT_ASSERT(false, "invalid number of digits"); } --exp; - uint64_t remainder = - (static_cast(integral) << -one.e) + fractional; + auto remainder = (static_cast(integral) << -one.e) + fractional; result = handler.on_digit(static_cast('0' + digit), data::powers_of_10_64[exp] << -one.e, remainder, error, exp, true); @@ -878,8 +1577,7 @@ FMT_ALWAYS_INLINE digits::result grisu_gen_digits(fp value, uint64_t error, for (;;) { fractional *= 10; error *= 10; - char digit = - static_cast('0' + static_cast(fractional >> -one.e)); + char digit = static_cast('0' + (fractional >> -one.e)); fractional &= one.f - 1; --exp; result = handler.on_digit(digit, one.f, fractional, error, exp, false); @@ -916,6 +1614,7 @@ struct fixed_handler { uint64_t error, int, bool integral) { FMT_ASSERT(remainder < divisor, ""); buf[size++] = digit; + if (!integral && error >= remainder) return digits::error; if (size < precision) return digits::more; if (!integral) { // Check if error * 2 < divisor with overflow prevention. @@ -935,59 +1634,684 @@ struct fixed_handler { } if (buf[0] > '9') { buf[0] = '1'; - buf[size++] = '0'; + if (fixed) + buf[size++] = '0'; + else + ++exp10; } return digits::done; } }; -// The shortest representation digit handler. -struct grisu_shortest_handler { - char* buf; - int size; - // Distance between scaled value and upper bound (wp_W in Grisu3). - uint64_t diff; +// Implementation of Dragonbox algorithm: https://github.com/jk-jeon/dragonbox. +namespace dragonbox { +// Computes 128-bit result of multiplication of two 64-bit unsigned integers. +FMT_SAFEBUFFERS inline uint128_wrapper umul128(uint64_t x, + uint64_t y) FMT_NOEXCEPT { +#if FMT_USE_INT128 + return static_cast(x) * static_cast(y); +#elif defined(_MSC_VER) && defined(_M_X64) + uint128_wrapper result; + result.low_ = _umul128(x, y, &result.high_); + return result; +#else + const uint64_t mask = (uint64_t(1) << 32) - uint64_t(1); + + uint64_t a = x >> 32; + uint64_t b = x & mask; + uint64_t c = y >> 32; + uint64_t d = y & mask; + + uint64_t ac = a * c; + uint64_t bc = b * c; + uint64_t ad = a * d; + uint64_t bd = b * d; + + uint64_t intermediate = (bd >> 32) + (ad & mask) + (bc & mask); + + return {ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32), + (intermediate << 32) + (bd & mask)}; +#endif +} + +// Computes upper 64 bits of multiplication of two 64-bit unsigned integers. +FMT_SAFEBUFFERS inline uint64_t umul128_upper64(uint64_t x, + uint64_t y) FMT_NOEXCEPT { +#if FMT_USE_INT128 + auto p = static_cast(x) * static_cast(y); + return static_cast(p >> 64); +#elif defined(_MSC_VER) && defined(_M_X64) + return __umulh(x, y); +#else + return umul128(x, y).high(); +#endif +} + +// Computes upper 64 bits of multiplication of a 64-bit unsigned integer and a +// 128-bit unsigned integer. +FMT_SAFEBUFFERS inline uint64_t umul192_upper64(uint64_t x, uint128_wrapper y) + FMT_NOEXCEPT { + uint128_wrapper g0 = umul128(x, y.high()); + g0 += umul128_upper64(x, y.low()); + return g0.high(); +} + +// Computes upper 32 bits of multiplication of a 32-bit unsigned integer and a +// 64-bit unsigned integer. +inline uint32_t umul96_upper32(uint32_t x, uint64_t y) FMT_NOEXCEPT { + return static_cast(umul128_upper64(x, y)); +} + +// Computes middle 64 bits of multiplication of a 64-bit unsigned integer and a +// 128-bit unsigned integer. +FMT_SAFEBUFFERS inline uint64_t umul192_middle64(uint64_t x, uint128_wrapper y) + FMT_NOEXCEPT { + uint64_t g01 = x * y.high(); + uint64_t g10 = umul128_upper64(x, y.low()); + return g01 + g10; +} + +// Computes lower 64 bits of multiplication of a 32-bit unsigned integer and a +// 64-bit unsigned integer. +inline uint64_t umul96_lower64(uint32_t x, uint64_t y) FMT_NOEXCEPT { + return x * y; +} + +// Computes floor(log10(pow(2, e))) for e in [-1700, 1700] using the method from +// https://fmt.dev/papers/Grisu-Exact.pdf#page=5, section 3.4. +inline int floor_log10_pow2(int e) FMT_NOEXCEPT { + FMT_ASSERT(e <= 1700 && e >= -1700, "too large exponent"); + const int shift = 22; + return (e * static_cast(data::log10_2_significand >> (64 - shift))) >> + shift; +} + +// Various fast log computations. +inline int floor_log2_pow10(int e) FMT_NOEXCEPT { + FMT_ASSERT(e <= 1233 && e >= -1233, "too large exponent"); + const uint64_t log2_10_integer_part = 3; + const uint64_t log2_10_fractional_digits = 0x5269e12f346e2bf9; + const int shift_amount = 19; + return (e * static_cast( + (log2_10_integer_part << shift_amount) | + (log2_10_fractional_digits >> (64 - shift_amount)))) >> + shift_amount; +} +inline int floor_log10_pow2_minus_log10_4_over_3(int e) FMT_NOEXCEPT { + FMT_ASSERT(e <= 1700 && e >= -1700, "too large exponent"); + const uint64_t log10_4_over_3_fractional_digits = 0x1ffbfc2bbc780375; + const int shift_amount = 22; + return (e * static_cast(data::log10_2_significand >> + (64 - shift_amount)) - + static_cast(log10_4_over_3_fractional_digits >> + (64 - shift_amount))) >> + shift_amount; +} + +// Returns true iff x is divisible by pow(2, exp). +inline bool divisible_by_power_of_2(uint32_t x, int exp) FMT_NOEXCEPT { + FMT_ASSERT(exp >= 1, ""); + FMT_ASSERT(x != 0, ""); +#ifdef FMT_BUILTIN_CTZ + return FMT_BUILTIN_CTZ(x) >= exp; +#else + return exp < num_bits() && x == ((x >> exp) << exp); +#endif +} +inline bool divisible_by_power_of_2(uint64_t x, int exp) FMT_NOEXCEPT { + FMT_ASSERT(exp >= 1, ""); + FMT_ASSERT(x != 0, ""); +#ifdef FMT_BUILTIN_CTZLL + return FMT_BUILTIN_CTZLL(x) >= exp; +#else + return exp < num_bits() && x == ((x >> exp) << exp); +#endif +} + +// Returns true iff x is divisible by pow(5, exp). +inline bool divisible_by_power_of_5(uint32_t x, int exp) FMT_NOEXCEPT { + FMT_ASSERT(exp <= 10, "too large exponent"); + return x * data::divtest_table_for_pow5_32[exp].mod_inv <= + data::divtest_table_for_pow5_32[exp].max_quotient; +} +inline bool divisible_by_power_of_5(uint64_t x, int exp) FMT_NOEXCEPT { + FMT_ASSERT(exp <= 23, "too large exponent"); + return x * data::divtest_table_for_pow5_64[exp].mod_inv <= + data::divtest_table_for_pow5_64[exp].max_quotient; +} + +// Replaces n by floor(n / pow(5, N)) returning true if and only if n is +// divisible by pow(5, N). +// Precondition: n <= 2 * pow(5, N + 1). +template +bool check_divisibility_and_divide_by_pow5(uint32_t& n) FMT_NOEXCEPT { + static constexpr struct { + uint32_t magic_number; + int bits_for_comparison; + uint32_t threshold; + int shift_amount; + } infos[] = {{0xcccd, 16, 0x3333, 18}, {0xa429, 8, 0x0a, 20}}; + constexpr auto info = infos[N - 1]; + n *= info.magic_number; + const uint32_t comparison_mask = (1u << info.bits_for_comparison) - 1; + bool result = (n & comparison_mask) <= info.threshold; + n >>= info.shift_amount; + return result; +} + +// Computes floor(n / pow(10, N)) for small n and N. +// Precondition: n <= pow(10, N + 1). +template uint32_t small_division_by_pow10(uint32_t n) FMT_NOEXCEPT { + static constexpr struct { + uint32_t magic_number; + int shift_amount; + uint32_t divisor_times_10; + } infos[] = {{0xcccd, 19, 100}, {0xa3d8, 22, 1000}}; + constexpr auto info = infos[N - 1]; + FMT_ASSERT(n <= info.divisor_times_10, "n is too large"); + return n * info.magic_number >> info.shift_amount; +} + +// Computes floor(n / 10^(kappa + 1)) (float) +inline uint32_t divide_by_10_to_kappa_plus_1(uint32_t n) FMT_NOEXCEPT { + return n / float_info::big_divisor; +} +// Computes floor(n / 10^(kappa + 1)) (double) +inline uint64_t divide_by_10_to_kappa_plus_1(uint64_t n) FMT_NOEXCEPT { + return umul128_upper64(n, 0x83126e978d4fdf3c) >> 9; +} + +// Various subroutines using pow10 cache +template struct cache_accessor; + +template <> struct cache_accessor { + using carrier_uint = float_info::carrier_uint; + using cache_entry_type = uint64_t; + + static uint64_t get_cached_power(int k) FMT_NOEXCEPT { + FMT_ASSERT(k >= float_info::min_k && k <= float_info::max_k, + "k is out of range"); + return data::dragonbox_pow10_significands_64[k - float_info::min_k]; + } + + static carrier_uint compute_mul(carrier_uint u, + const cache_entry_type& cache) FMT_NOEXCEPT { + return umul96_upper32(u, cache); + } + + static uint32_t compute_delta(const cache_entry_type& cache, + int beta_minus_1) FMT_NOEXCEPT { + return static_cast(cache >> (64 - 1 - beta_minus_1)); + } + + static bool compute_mul_parity(carrier_uint two_f, + const cache_entry_type& cache, + int beta_minus_1) FMT_NOEXCEPT { + FMT_ASSERT(beta_minus_1 >= 1, ""); + FMT_ASSERT(beta_minus_1 < 64, ""); + + return ((umul96_lower64(two_f, cache) >> (64 - beta_minus_1)) & 1) != 0; + } + + static carrier_uint compute_left_endpoint_for_shorter_interval_case( + const cache_entry_type& cache, int beta_minus_1) FMT_NOEXCEPT { + return static_cast( + (cache - (cache >> (float_info::significand_bits + 2))) >> + (64 - float_info::significand_bits - 1 - beta_minus_1)); + } + + static carrier_uint compute_right_endpoint_for_shorter_interval_case( + const cache_entry_type& cache, int beta_minus_1) FMT_NOEXCEPT { + return static_cast( + (cache + (cache >> (float_info::significand_bits + 1))) >> + (64 - float_info::significand_bits - 1 - beta_minus_1)); + } + + static carrier_uint compute_round_up_for_shorter_interval_case( + const cache_entry_type& cache, int beta_minus_1) FMT_NOEXCEPT { + return (static_cast( + cache >> + (64 - float_info::significand_bits - 2 - beta_minus_1)) + + 1) / + 2; + } +}; + +template <> struct cache_accessor { + using carrier_uint = float_info::carrier_uint; + using cache_entry_type = uint128_wrapper; + + static uint128_wrapper get_cached_power(int k) FMT_NOEXCEPT { + FMT_ASSERT(k >= float_info::min_k && k <= float_info::max_k, + "k is out of range"); + +#if FMT_USE_FULL_CACHE_DRAGONBOX + return data::dragonbox_pow10_significands_128[k - + float_info::min_k]; +#else + static const int compression_ratio = 27; + + // Compute base index. + int cache_index = (k - float_info::min_k) / compression_ratio; + int kb = cache_index * compression_ratio + float_info::min_k; + int offset = k - kb; + + // Get base cache. + uint128_wrapper base_cache = + data::dragonbox_pow10_significands_128[cache_index]; + if (offset == 0) return base_cache; + + // Compute the required amount of bit-shift. + int alpha = floor_log2_pow10(kb + offset) - floor_log2_pow10(kb) - offset; + FMT_ASSERT(alpha > 0 && alpha < 64, "shifting error detected"); + + // Try to recover the real cache. + uint64_t pow5 = data::powers_of_5_64[offset]; + uint128_wrapper recovered_cache = umul128(base_cache.high(), pow5); + uint128_wrapper middle_low = + umul128(base_cache.low() - (kb < 0 ? 1u : 0u), pow5); + + recovered_cache += middle_low.high(); + + uint64_t high_to_middle = recovered_cache.high() << (64 - alpha); + uint64_t middle_to_low = recovered_cache.low() << (64 - alpha); + + recovered_cache = + uint128_wrapper{(recovered_cache.low() >> alpha) | high_to_middle, + ((middle_low.low() >> alpha) | middle_to_low)}; + + if (kb < 0) recovered_cache += 1; + + // Get error. + int error_idx = (k - float_info::min_k) / 16; + uint32_t error = (data::dragonbox_pow10_recovery_errors[error_idx] >> + ((k - float_info::min_k) % 16) * 2) & + 0x3; + + // Add the error back. + FMT_ASSERT(recovered_cache.low() + error >= recovered_cache.low(), ""); + return {recovered_cache.high(), recovered_cache.low() + error}; +#endif + } + + static carrier_uint compute_mul(carrier_uint u, + const cache_entry_type& cache) FMT_NOEXCEPT { + return umul192_upper64(u, cache); + } + + static uint32_t compute_delta(cache_entry_type const& cache, + int beta_minus_1) FMT_NOEXCEPT { + return static_cast(cache.high() >> (64 - 1 - beta_minus_1)); + } + + static bool compute_mul_parity(carrier_uint two_f, + const cache_entry_type& cache, + int beta_minus_1) FMT_NOEXCEPT { + FMT_ASSERT(beta_minus_1 >= 1, ""); + FMT_ASSERT(beta_minus_1 < 64, ""); - digits::result on_start(uint64_t, uint64_t, uint64_t, int&) { - return digits::more; + return ((umul192_middle64(two_f, cache) >> (64 - beta_minus_1)) & 1) != 0; } - // Decrement the generated number approaching value from above. - void round(uint64_t d, uint64_t divisor, uint64_t& remainder, - uint64_t error) { - while ( - remainder < d && error - remainder >= divisor && - (remainder + divisor < d || d - remainder >= remainder + divisor - d)) { - --buf[size - 1]; - remainder += divisor; + static carrier_uint compute_left_endpoint_for_shorter_interval_case( + const cache_entry_type& cache, int beta_minus_1) FMT_NOEXCEPT { + return (cache.high() - + (cache.high() >> (float_info::significand_bits + 2))) >> + (64 - float_info::significand_bits - 1 - beta_minus_1); + } + + static carrier_uint compute_right_endpoint_for_shorter_interval_case( + const cache_entry_type& cache, int beta_minus_1) FMT_NOEXCEPT { + return (cache.high() + + (cache.high() >> (float_info::significand_bits + 1))) >> + (64 - float_info::significand_bits - 1 - beta_minus_1); + } + + static carrier_uint compute_round_up_for_shorter_interval_case( + const cache_entry_type& cache, int beta_minus_1) FMT_NOEXCEPT { + return ((cache.high() >> + (64 - float_info::significand_bits - 2 - beta_minus_1)) + + 1) / + 2; + } +}; + +// Various integer checks +template +bool is_left_endpoint_integer_shorter_interval(int exponent) FMT_NOEXCEPT { + return exponent >= + float_info< + T>::case_shorter_interval_left_endpoint_lower_threshold && + exponent <= + float_info::case_shorter_interval_left_endpoint_upper_threshold; +} +template +bool is_endpoint_integer(typename float_info::carrier_uint two_f, + int exponent, int minus_k) FMT_NOEXCEPT { + if (exponent < float_info::case_fc_pm_half_lower_threshold) return false; + // For k >= 0. + if (exponent <= float_info::case_fc_pm_half_upper_threshold) return true; + // For k < 0. + if (exponent > float_info::divisibility_check_by_5_threshold) return false; + return divisible_by_power_of_5(two_f, minus_k); +} + +template +bool is_center_integer(typename float_info::carrier_uint two_f, int exponent, + int minus_k) FMT_NOEXCEPT { + // Exponent for 5 is negative. + if (exponent > float_info::divisibility_check_by_5_threshold) return false; + if (exponent > float_info::case_fc_upper_threshold) + return divisible_by_power_of_5(two_f, minus_k); + // Both exponents are nonnegative. + if (exponent >= float_info::case_fc_lower_threshold) return true; + // Exponent for 2 is negative. + return divisible_by_power_of_2(two_f, minus_k - exponent + 1); +} + +// Remove trailing zeros from n and return the number of zeros removed (float) +FMT_ALWAYS_INLINE int remove_trailing_zeros(uint32_t& n) FMT_NOEXCEPT { +#ifdef FMT_BUILTIN_CTZ + int t = FMT_BUILTIN_CTZ(n); +#else + int t = ctz(n); +#endif + if (t > float_info::max_trailing_zeros) + t = float_info::max_trailing_zeros; + + const uint32_t mod_inv1 = 0xcccccccd; + const uint32_t max_quotient1 = 0x33333333; + const uint32_t mod_inv2 = 0xc28f5c29; + const uint32_t max_quotient2 = 0x0a3d70a3; + + int s = 0; + for (; s < t - 1; s += 2) { + if (n * mod_inv2 > max_quotient2) break; + n *= mod_inv2; + } + if (s < t && n * mod_inv1 <= max_quotient1) { + n *= mod_inv1; + ++s; + } + n >>= s; + return s; +} + +// Removes trailing zeros and returns the number of zeros removed (double) +FMT_ALWAYS_INLINE int remove_trailing_zeros(uint64_t& n) FMT_NOEXCEPT { +#ifdef FMT_BUILTIN_CTZLL + int t = FMT_BUILTIN_CTZLL(n); +#else + int t = ctzll(n); +#endif + if (t > float_info::max_trailing_zeros) + t = float_info::max_trailing_zeros; + // Divide by 10^8 and reduce to 32-bits + // Since ret_value.significand <= (2^64 - 1) / 1000 < 10^17, + // both of the quotient and the r should fit in 32-bits + + const uint32_t mod_inv1 = 0xcccccccd; + const uint32_t max_quotient1 = 0x33333333; + const uint64_t mod_inv8 = 0xc767074b22e90e21; + const uint64_t max_quotient8 = 0x00002af31dc46118; + + // If the number is divisible by 1'0000'0000, work with the quotient + if (t >= 8) { + auto quotient_candidate = n * mod_inv8; + + if (quotient_candidate <= max_quotient8) { + auto quotient = static_cast(quotient_candidate >> 8); + + int s = 8; + for (; s < t; ++s) { + if (quotient * mod_inv1 > max_quotient1) break; + quotient *= mod_inv1; + } + quotient >>= (s - 8); + n = quotient; + return s; } } - // Implements Grisu's round_weed. - digits::result on_digit(char digit, uint64_t divisor, uint64_t remainder, - uint64_t error, int exp, bool integral) { - buf[size++] = digit; - if (remainder >= error) return digits::more; - uint64_t unit = integral ? 1 : data::powers_of_10_64[-exp]; - uint64_t up = (diff - 1) * unit; // wp_Wup - round(up, divisor, remainder, error); - uint64_t down = (diff + 1) * unit; // wp_Wdown - if (remainder < down && error - remainder >= divisor && - (remainder + divisor < down || - down - remainder > remainder + divisor - down)) { - return digits::error; + // Otherwise, work with the remainder + auto quotient = static_cast(n / 100000000); + auto remainder = static_cast(n - 100000000 * quotient); + + if (t == 0 || remainder * mod_inv1 > max_quotient1) { + return 0; + } + remainder *= mod_inv1; + + if (t == 1 || remainder * mod_inv1 > max_quotient1) { + n = (remainder >> 1) + quotient * 10000000ull; + return 1; + } + remainder *= mod_inv1; + + if (t == 2 || remainder * mod_inv1 > max_quotient1) { + n = (remainder >> 2) + quotient * 1000000ull; + return 2; + } + remainder *= mod_inv1; + + if (t == 3 || remainder * mod_inv1 > max_quotient1) { + n = (remainder >> 3) + quotient * 100000ull; + return 3; + } + remainder *= mod_inv1; + + if (t == 4 || remainder * mod_inv1 > max_quotient1) { + n = (remainder >> 4) + quotient * 10000ull; + return 4; + } + remainder *= mod_inv1; + + if (t == 5 || remainder * mod_inv1 > max_quotient1) { + n = (remainder >> 5) + quotient * 1000ull; + return 5; + } + remainder *= mod_inv1; + + if (t == 6 || remainder * mod_inv1 > max_quotient1) { + n = (remainder >> 6) + quotient * 100ull; + return 6; + } + remainder *= mod_inv1; + + n = (remainder >> 7) + quotient * 10ull; + return 7; +} + +// The main algorithm for shorter interval case +template +FMT_ALWAYS_INLINE FMT_SAFEBUFFERS decimal_fp shorter_interval_case( + int exponent) FMT_NOEXCEPT { + decimal_fp ret_value; + // Compute k and beta + const int minus_k = floor_log10_pow2_minus_log10_4_over_3(exponent); + const int beta_minus_1 = exponent + floor_log2_pow10(-minus_k); + + // Compute xi and zi + using cache_entry_type = typename cache_accessor::cache_entry_type; + const cache_entry_type cache = cache_accessor::get_cached_power(-minus_k); + + auto xi = cache_accessor::compute_left_endpoint_for_shorter_interval_case( + cache, beta_minus_1); + auto zi = cache_accessor::compute_right_endpoint_for_shorter_interval_case( + cache, beta_minus_1); + + // If the left endpoint is not an integer, increase it + if (!is_left_endpoint_integer_shorter_interval(exponent)) ++xi; + + // Try bigger divisor + ret_value.significand = zi / 10; + + // If succeed, remove trailing zeros if necessary and return + if (ret_value.significand * 10 >= xi) { + ret_value.exponent = minus_k + 1; + ret_value.exponent += remove_trailing_zeros(ret_value.significand); + return ret_value; + } + + // Otherwise, compute the round-up of y + ret_value.significand = + cache_accessor::compute_round_up_for_shorter_interval_case( + cache, beta_minus_1); + ret_value.exponent = minus_k; + + // When tie occurs, choose one of them according to the rule + if (exponent >= float_info::shorter_interval_tie_lower_threshold && + exponent <= float_info::shorter_interval_tie_upper_threshold) { + ret_value.significand = ret_value.significand % 2 == 0 + ? ret_value.significand + : ret_value.significand - 1; + } else if (ret_value.significand < xi) { + ++ret_value.significand; + } + return ret_value; +} + +template +FMT_SAFEBUFFERS decimal_fp to_decimal(T x) FMT_NOEXCEPT { + // Step 1: integer promotion & Schubfach multiplier calculation. + + using carrier_uint = typename float_info::carrier_uint; + using cache_entry_type = typename cache_accessor::cache_entry_type; + auto br = bit_cast(x); + + // Extract significand bits and exponent bits. + const carrier_uint significand_mask = + (static_cast(1) << float_info::significand_bits) - 1; + carrier_uint significand = (br & significand_mask); + int exponent = static_cast((br & exponent_mask()) >> + float_info::significand_bits); + + if (exponent != 0) { // Check if normal. + exponent += float_info::exponent_bias - float_info::significand_bits; + + // Shorter interval case; proceed like Schubfach. + if (significand == 0) return shorter_interval_case(exponent); + + significand |= + (static_cast(1) << float_info::significand_bits); + } else { + // Subnormal case; the interval is always regular. + if (significand == 0) return {0, 0}; + exponent = float_info::min_exponent - float_info::significand_bits; + } + + const bool include_left_endpoint = (significand % 2 == 0); + const bool include_right_endpoint = include_left_endpoint; + + // Compute k and beta. + const int minus_k = floor_log10_pow2(exponent) - float_info::kappa; + const cache_entry_type cache = cache_accessor::get_cached_power(-minus_k); + const int beta_minus_1 = exponent + floor_log2_pow10(-minus_k); + + // Compute zi and deltai + // 10^kappa <= deltai < 10^(kappa + 1) + const uint32_t deltai = cache_accessor::compute_delta(cache, beta_minus_1); + const carrier_uint two_fc = significand << 1; + const carrier_uint two_fr = two_fc | 1; + const carrier_uint zi = + cache_accessor::compute_mul(two_fr << beta_minus_1, cache); + + // Step 2: Try larger divisor; remove trailing zeros if necessary + + // Using an upper bound on zi, we might be able to optimize the division + // better than the compiler; we are computing zi / big_divisor here + decimal_fp ret_value; + ret_value.significand = divide_by_10_to_kappa_plus_1(zi); + uint32_t r = static_cast(zi - float_info::big_divisor * + ret_value.significand); + + if (r > deltai) { + goto small_divisor_case_label; + } else if (r < deltai) { + // Exclude the right endpoint if necessary + if (r == 0 && !include_right_endpoint && + is_endpoint_integer(two_fr, exponent, minus_k)) { + --ret_value.significand; + r = float_info::big_divisor; + goto small_divisor_case_label; + } + } else { + // r == deltai; compare fractional parts + // Check conditions in the order different from the paper + // to take advantage of short-circuiting + const carrier_uint two_fl = two_fc - 1; + if ((!include_left_endpoint || + !is_endpoint_integer(two_fl, exponent, minus_k)) && + !cache_accessor::compute_mul_parity(two_fl, cache, beta_minus_1)) { + goto small_divisor_case_label; } - return 2 * unit <= remainder && remainder <= error - 4 * unit - ? digits::done - : digits::error; } -}; + ret_value.exponent = minus_k + float_info::kappa + 1; + + // We may need to remove trailing zeros + ret_value.exponent += remove_trailing_zeros(ret_value.significand); + return ret_value; + + // Step 3: Find the significand with the smaller divisor + +small_divisor_case_label: + ret_value.significand *= 10; + ret_value.exponent = minus_k + float_info::kappa; + + const uint32_t mask = (1u << float_info::kappa) - 1; + auto dist = r - (deltai / 2) + (float_info::small_divisor / 2); + + // Is dist divisible by 2^kappa? + if ((dist & mask) == 0) { + const bool approx_y_parity = + ((dist ^ (float_info::small_divisor / 2)) & 1) != 0; + dist >>= float_info::kappa; + + // Is dist divisible by 5^kappa? + if (check_divisibility_and_divide_by_pow5::kappa>(dist)) { + ret_value.significand += dist; + + // Check z^(f) >= epsilon^(f) + // We have either yi == zi - epsiloni or yi == (zi - epsiloni) - 1, + // where yi == zi - epsiloni if and only if z^(f) >= epsilon^(f) + // Since there are only 2 possibilities, we only need to care about the + // parity. Also, zi and r should have the same parity since the divisor + // is an even number + if (cache_accessor::compute_mul_parity(two_fc, cache, beta_minus_1) != + approx_y_parity) { + --ret_value.significand; + } else { + // If z^(f) >= epsilon^(f), we might have a tie + // when z^(f) == epsilon^(f), or equivalently, when y is an integer + if (is_center_integer(two_fc, exponent, minus_k)) { + ret_value.significand = ret_value.significand % 2 == 0 + ? ret_value.significand + : ret_value.significand - 1; + } + } + } + // Is dist not divisible by 5^kappa? + else { + ret_value.significand += dist; + } + } + // Is dist not divisible by 2^kappa? + else { + // Since we know dist is small, we might be able to optimize the division + // better than the compiler; we are computing dist / small_divisor here + ret_value.significand += + small_division_by_pow10::kappa>(dist); + } + return ret_value; +} +} // namespace dragonbox // Formats value using a variation of the Fixed-Precision Positive // Floating-Point Printout ((FPP)^2) algorithm by Steele & White: // https://fmt.dev/p372-steele.pdf. template -void fallback_format(Double d, buffer& buf, int& exp10) { +void fallback_format(Double d, int num_digits, bool binary32, buffer& buf, + int& exp10) { bigint numerator; // 2 * R in (FPP)^2. bigint denominator; // 2 * S in (FPP)^2. // lower and upper are differences between value and corresponding boundaries. @@ -998,8 +2322,9 @@ void fallback_format(Double d, buffer& buf, int& exp10) { // Shift numerator and denominator by an extra bit or two (if lower boundary // is closer) to make lower and upper integers. This eliminates multiplication // by 2 during later computations. - // TODO: handle float - int shift = value.assign(d) ? 2 : 1; + const bool is_predecessor_closer = + binary32 ? value.assign(static_cast(d)) : value.assign(d); + int shift = is_predecessor_closer ? 2 : 1; uint64_t significand = value.f << shift; if (value.e >= 0) { numerator.assign(significand); @@ -1012,7 +2337,7 @@ void fallback_format(Double d, buffer& buf, int& exp10) { upper = &upper_store; } denominator.assign_pow10(exp10); - denominator <<= 1; + denominator <<= shift; } else if (exp10 < 0) { numerator.assign_pow10(-exp10); lower.assign(numerator); @@ -1034,39 +2359,73 @@ void fallback_format(Double d, buffer& buf, int& exp10) { upper = &upper_store; } } - if (!upper) upper = &lower; // Invariant: value == (numerator / denominator) * pow(10, exp10). - bool even = (value.f & 1) == 0; - int num_digits = 0; - char* data = buf.data(); - for (;;) { - int digit = numerator.divmod_assign(denominator); - bool low = compare(numerator, lower) - even < 0; // numerator <[=] lower. - // numerator + upper >[=] pow10: - bool high = add_compare(numerator, *upper, denominator) + even > 0; - data[num_digits++] = static_cast('0' + digit); - if (low || high) { - if (!low) { - ++data[num_digits - 1]; - } else if (high) { - int result = add_compare(numerator, numerator, denominator); - // Round half to even. - if (result > 0 || (result == 0 && (digit % 2) != 0)) + if (num_digits < 0) { + // Generate the shortest representation. + if (!upper) upper = &lower; + bool even = (value.f & 1) == 0; + num_digits = 0; + char* data = buf.data(); + for (;;) { + int digit = numerator.divmod_assign(denominator); + bool low = compare(numerator, lower) - even < 0; // numerator <[=] lower. + // numerator + upper >[=] pow10: + bool high = add_compare(numerator, *upper, denominator) + even > 0; + data[num_digits++] = static_cast('0' + digit); + if (low || high) { + if (!low) { ++data[num_digits - 1]; + } else if (high) { + int result = add_compare(numerator, numerator, denominator); + // Round half to even. + if (result > 0 || (result == 0 && (digit % 2) != 0)) + ++data[num_digits - 1]; + } + buf.try_resize(to_unsigned(num_digits)); + exp10 -= num_digits - 1; + return; } - buf.resize(to_unsigned(num_digits)); - exp10 -= num_digits - 1; - return; + numerator *= 10; + lower *= 10; + if (upper != &lower) *upper *= 10; } + } + // Generate the given number of digits. + exp10 -= num_digits - 1; + if (num_digits == 0) { + buf.try_resize(1); + denominator *= 10; + buf[0] = add_compare(numerator, numerator, denominator) > 0 ? '1' : '0'; + return; + } + buf.try_resize(to_unsigned(num_digits)); + for (int i = 0; i < num_digits - 1; ++i) { + int digit = numerator.divmod_assign(denominator); + buf[i] = static_cast('0' + digit); numerator *= 10; - lower *= 10; - if (upper != &lower) *upper *= 10; } + int digit = numerator.divmod_assign(denominator); + auto result = add_compare(numerator, numerator, denominator); + if (result > 0 || (result == 0 && (digit % 2) != 0)) { + if (digit == 9) { + const auto overflow = '0' + 10; + buf[num_digits - 1] = overflow; + // Propagate the carry. + for (int i = num_digits - 1; i > 0 && buf[i] == overflow; --i) { + buf[i] = '0'; + ++buf[i - 1]; + } + if (buf[0] == overflow) { + buf[0] = '1'; + ++exp10; + } + return; + } + ++digit; + } + buf[num_digits - 1] = static_cast('0' + digit); } -// Formats value using the Grisu algorithm -// (https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf) -// if T is a IEEE754 binary32 or binary64 and snprintf otherwise. template int format_float(T value, int precision, float_specs specs, buffer& buf) { static_assert(!std::is_same::value, ""); @@ -1078,66 +2437,57 @@ int format_float(T value, int precision, float_specs specs, buffer& buf) { buf.push_back('0'); return 0; } - buf.resize(to_unsigned(precision)); + buf.try_resize(to_unsigned(precision)); std::uninitialized_fill_n(buf.data(), precision, '0'); return -precision; } if (!specs.use_grisu) return snprintf_float(value, precision, specs, buf); + if (precision < 0) { + // Use Dragonbox for the shortest format. + if (specs.binary32) { + auto dec = dragonbox::to_decimal(static_cast(value)); + write(buffer_appender(buf), dec.significand); + return dec.exponent; + } + auto dec = dragonbox::to_decimal(static_cast(value)); + write(buffer_appender(buf), dec.significand); + return dec.exponent; + } + + // Use Grisu + Dragon4 for the given precision: + // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf. int exp = 0; const int min_exp = -60; // alpha in Grisu. int cached_exp10 = 0; // K in Grisu. - if (precision < 0) { - fp fp_value; - auto boundaries = specs.binary32 - ? fp_value.assign_float_with_boundaries(value) - : fp_value.assign_with_boundaries(value); - fp_value = normalize(fp_value); - // Find a cached power of 10 such that multiplying value by it will bring - // the exponent in the range [min_exp, -32]. - const fp cached_pow = get_cached_power( - min_exp - (fp_value.e + fp::significand_size), cached_exp10); - // Multiply value and boundaries by the cached power of 10. - fp_value = fp_value * cached_pow; - boundaries.lower = multiply(boundaries.lower, cached_pow.f); - boundaries.upper = multiply(boundaries.upper, cached_pow.f); - assert(min_exp <= fp_value.e && fp_value.e <= -32); - --boundaries.lower; // \tilde{M}^- - 1 ulp -> M^-_{\downarrow}. - ++boundaries.upper; // \tilde{M}^+ + 1 ulp -> M^+_{\uparrow}. - // Numbers outside of (lower, upper) definitely do not round to value. - grisu_shortest_handler handler{buf.data(), 0, - boundaries.upper - fp_value.f}; - auto result = - grisu_gen_digits(fp(boundaries.upper, fp_value.e), - boundaries.upper - boundaries.lower, exp, handler); - if (result == digits::error) { - exp += handler.size - cached_exp10 - 1; - fallback_format(value, buf, exp); - return exp; - } - buf.resize(to_unsigned(handler.size)); + fp normalized = normalize(fp(value)); + const auto cached_pow = get_cached_power( + min_exp - (normalized.e + fp::significand_size), cached_exp10); + normalized = normalized * cached_pow; + // Limit precision to the maximum possible number of significant digits in an + // IEEE754 double because we don't need to generate zeros. + const int max_double_digits = 767; + if (precision > max_double_digits) precision = max_double_digits; + fixed_handler handler{buf.data(), 0, precision, -cached_exp10, fixed}; + if (grisu_gen_digits(normalized, 1, exp, handler) == digits::error) { + exp += handler.size - cached_exp10 - 1; + fallback_format(value, handler.precision, specs.binary32, buf, exp); } else { - if (precision > 17) return snprintf_float(value, precision, specs, buf); - fp normalized = normalize(fp(value)); - const auto cached_pow = get_cached_power( - min_exp - (normalized.e + fp::significand_size), cached_exp10); - normalized = normalized * cached_pow; - fixed_handler handler{buf.data(), 0, precision, -cached_exp10, fixed}; - if (grisu_gen_digits(normalized, 1, exp, handler) == digits::error) - return snprintf_float(value, precision, specs, buf); - int num_digits = handler.size; - if (!fixed) { - // Remove trailing zeros. - while (num_digits > 0 && buf[num_digits - 1] == '0') { - --num_digits; - ++exp; - } + exp += handler.exp10; + buf.try_resize(to_unsigned(handler.size)); + } + if (!fixed && !specs.showpoint) { + // Remove trailing zeros. + auto num_digits = buf.size(); + while (num_digits > 0 && buf[num_digits - 1] == '0') { + --num_digits; + ++exp; } - buf.resize(to_unsigned(num_digits)); + buf.try_resize(num_digits); } - return exp - cached_exp10; -} + return exp; +} // namespace detail template int snprintf_float(T value, int precision, float_specs specs, @@ -1185,19 +2535,20 @@ int snprintf_float(T value, int precision, float_specs specs, ? snprintf_ptr(begin, capacity, format, precision, value) : snprintf_ptr(begin, capacity, format, value); if (result < 0) { - buf.reserve(buf.capacity() + 1); // The buffer will grow exponentially. + // The buffer will grow exponentially. + buf.try_reserve(buf.capacity() + 1); continue; } auto size = to_unsigned(result); // Size equal to capacity means that the last character was truncated. if (size >= capacity) { - buf.reserve(size + offset + 1); // Add 1 for the terminating '\0'. + buf.try_reserve(size + offset + 1); // Add 1 for the terminating '\0'. continue; } auto is_digit = [](char c) { return c >= '0' && c <= '9'; }; if (specs.format == float_format::fixed) { if (precision == 0) { - buf.resize(size); + buf.try_resize(size); return 0; } // Find and remove the decimal point. @@ -1207,11 +2558,11 @@ int snprintf_float(T value, int precision, float_specs specs, } while (is_digit(*p)); int fraction_size = static_cast(end - p - 1); std::memmove(p, p + 1, to_unsigned(fraction_size)); - buf.resize(size - 1); + buf.try_resize(size - 1); return -fraction_size; } if (specs.format == float_format::hex) { - buf.resize(size + offset); + buf.try_resize(size + offset); return 0; } // Find and parse the exponent. @@ -1237,7 +2588,7 @@ int snprintf_float(T value, int precision, float_specs specs, fraction_size = static_cast(fraction_end - begin - 1); std::memmove(begin + 1, begin + 2, to_unsigned(fraction_size)); } - buf.resize(to_unsigned(fraction_size) + offset + 1); + buf.try_resize(to_unsigned(fraction_size) + offset + 1); return exp - fraction_size; } } @@ -1259,25 +2610,18 @@ int snprintf_float(T value, int precision, float_specs specs, * occurs, this pointer will be a guess that depends on the particular * error, but it will always advance at least one byte. */ -FMT_FUNC const char* utf8_decode(const char* buf, uint32_t* c, int* e) { - static const char lengths[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 2, 2, 2, 2, 3, 3, 4, 0}; +inline const char* utf8_decode(const char* buf, uint32_t* c, int* e) { static const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07}; static const uint32_t mins[] = {4194304, 0, 128, 2048, 65536}; static const int shiftc[] = {0, 18, 12, 6, 0}; static const int shifte[] = {0, 6, 4, 2, 0}; - auto s = reinterpret_cast(buf); - int len = lengths[s[0] >> 3]; - - // Compute the pointer to the next character early so that the next - // iteration can start working on the next character. Neither Clang - // nor GCC figure out this reordering on their own. - const char* next = buf + len + !len; + int len = code_point_length(buf); + const char* next = buf + len; // Assume a four-byte character and load four bytes. Unused bits are // shifted out. + auto s = reinterpret_cast(buf); *c = uint32_t(s[0] & masks[len]) << 18; *c |= uint32_t(s[1] & 0x3f) << 12; *c |= uint32_t(s[2] & 0x3f) << 6; @@ -1296,6 +2640,19 @@ FMT_FUNC const char* utf8_decode(const char* buf, uint32_t* c, int* e) { return next; } + +struct stringifier { + template FMT_INLINE std::string operator()(T value) const { + return to_string(value); + } + std::string operator()(basic_format_arg::handle h) const { + memory_buffer buf; + format_parse_context parse_ctx({}); + format_context format_ctx(buffer_appender(buf), {}, {}); + h.format(parse_ctx, format_ctx); + return to_string(buf); + } +}; } // namespace detail template <> struct formatter { @@ -1363,7 +2720,8 @@ FMT_FUNC void format_system_error(detail::buffer& out, int error_code, int result = detail::safe_strerror(error_code, system_message, buf.size()); if (result == 0) { - format_to(std::back_inserter(out), "{}: {}", message, system_message); + format_to(detail::buffer_appender(out), "{}: {}", message, + system_message); return; } if (result != ERANGE) @@ -1384,20 +2742,6 @@ FMT_FUNC void report_system_error(int error_code, report_error(format_system_error, error_code, message); } -struct stringifier { - template FMT_INLINE std::string operator()(T value) const { - return to_string(value); - } - std::string operator()(basic_format_arg::handle h) const { - memory_buffer buf; - detail::buffer& base = buf; - format_parse_context parse_ctx({}); - format_context format_ctx(std::back_inserter(base), {}, {}); - h.format(parse_ctx, format_ctx); - return to_string(buf); - } -}; - FMT_FUNC std::string detail::vformat(string_view format_str, format_args args) { if (format_str.size() == 2 && equal2(format_str.data(), "{}")) { auto arg = args.get(0); @@ -1409,6 +2753,14 @@ FMT_FUNC std::string detail::vformat(string_view format_str, format_args args) { return to_string(buffer); } +#ifdef _WIN32 +namespace detail { +using dword = conditional_t; +extern "C" __declspec(dllimport) int __stdcall WriteConsoleW( // + void*, const void*, dword, dword*, void*); +} // namespace detail +#endif + FMT_FUNC void vprint(std::FILE* f, string_view format_str, format_args args) { memory_buffer buffer; detail::vformat_to(buffer, format_str, @@ -1417,10 +2769,10 @@ FMT_FUNC void vprint(std::FILE* f, string_view format_str, format_args args) { auto fd = _fileno(f); if (_isatty(fd)) { detail::utf8_to_utf16 u16(string_view(buffer.data(), buffer.size())); - auto written = DWORD(); - if (!WriteConsoleW(reinterpret_cast(_get_osfhandle(fd)), - u16.c_str(), static_cast(u16.size()), &written, - nullptr)) { + auto written = detail::dword(); + if (!detail::WriteConsoleW(reinterpret_cast(_get_osfhandle(fd)), + u16.c_str(), static_cast(u16.size()), + &written, nullptr)) { FMT_THROW(format_error("failed to write to console")); } return; @@ -1446,8 +2798,4 @@ FMT_FUNC void vprint(string_view format_str, format_args args) { FMT_END_NAMESPACE -#ifdef _MSC_VER -# pragma warning(pop) -#endif - #endif // FMT_FORMAT_INL_H_ diff --git a/src/third_party/fmt/format.h b/src/third_party/fmt/format.h index 17509b7b45..1a037b02b7 100644 --- a/src/third_party/fmt/format.h +++ b/src/third_party/fmt/format.h @@ -70,9 +70,11 @@ #endif #if __cplusplus == 201103L || __cplusplus == 201402L -# if defined(__clang__) +# if defined(__INTEL_COMPILER) || defined(__PGI) +# define FMT_FALLTHROUGH +# elif defined(__clang__) # define FMT_FALLTHROUGH [[clang::fallthrough]] -# elif FMT_GCC_VERSION >= 700 && !defined(__PGI) && \ +# elif FMT_GCC_VERSION >= 700 && \ (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 520) # define FMT_FALLTHROUGH [[gnu::fallthrough]] # else @@ -139,12 +141,13 @@ FMT_END_NAMESPACE #endif #ifndef FMT_USE_UDL_TEMPLATE -// EDG frontend based compilers (icc, nvcc, etc) and GCC < 6.4 do not properly -// support UDL templates and GCC >= 9 warns about them. +// EDG frontend based compilers (icc, nvcc, PGI, etc) and GCC < 6.4 do not +// properly support UDL templates and GCC >= 9 warns about them. # if FMT_USE_USER_DEFINED_LITERALS && \ (!defined(__EDG_VERSION__) || __EDG_VERSION__ >= 501) && \ ((FMT_GCC_VERSION >= 604 && __cplusplus >= 201402L) || \ - FMT_CLANG_VERSION >= 304) + FMT_CLANG_VERSION >= 304) && \ + !defined(__PGI) && !defined(__NVCC__) # define FMT_USE_UDL_TEMPLATE 1 # else # define FMT_USE_UDL_TEMPLATE 0 @@ -163,6 +166,14 @@ FMT_END_NAMESPACE # define FMT_USE_LONG_DOUBLE 1 #endif +// Defining FMT_REDUCE_INT_INSTANTIATIONS to 1, will reduce the number of +// int_writer template instances to just one by only using the largest integer +// type. This results in a reduction in binary size but will cause a decrease in +// integer formatting performance. +#if !defined(FMT_REDUCE_INT_INSTANTIATIONS) +# define FMT_REDUCE_INT_INSTANTIATIONS 0 +#endif + // __builtin_clz is broken in clang with Microsoft CodeGen: // https://github.com/fmtlib/fmt/issues/519 #if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_clz)) && !FMT_MSC_VER @@ -171,56 +182,87 @@ FMT_END_NAMESPACE #if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_clzll)) && !FMT_MSC_VER # define FMT_BUILTIN_CLZLL(n) __builtin_clzll(n) #endif +#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_ctz)) +# define FMT_BUILTIN_CTZ(n) __builtin_ctz(n) +#endif +#if (FMT_GCC_VERSION || FMT_HAS_BUILTIN(__builtin_ctzll)) +# define FMT_BUILTIN_CTZLL(n) __builtin_ctzll(n) +#endif + +#if FMT_MSC_VER +# include // _BitScanReverse[64], _BitScanForward[64], _umul128 +#endif // Some compilers masquerade as both MSVC and GCC-likes or otherwise support // __builtin_clz and __builtin_clzll, so only define FMT_BUILTIN_CLZ using the // MSVC intrinsics if the clz and clzll builtins are not available. -#if FMT_MSC_VER && !defined(FMT_BUILTIN_CLZLL) && !defined(_MANAGED) -# include // _BitScanReverse, _BitScanReverse64 - +#if FMT_MSC_VER && !defined(FMT_BUILTIN_CLZLL) && \ + !defined(FMT_BUILTIN_CTZLL) && !defined(_MANAGED) FMT_BEGIN_NAMESPACE namespace detail { // Avoid Clang with Microsoft CodeGen's -Wunknown-pragmas warning. # ifndef __clang__ +# pragma intrinsic(_BitScanForward) # pragma intrinsic(_BitScanReverse) # endif -inline uint32_t clz(uint32_t x) { +# if defined(_WIN64) && !defined(__clang__) +# pragma intrinsic(_BitScanForward64) +# pragma intrinsic(_BitScanReverse64) +# endif + +inline int clz(uint32_t x) { unsigned long r = 0; _BitScanReverse(&r, x); - FMT_ASSERT(x != 0, ""); // Static analysis complains about using uninitialized data // "r", but the only way that can happen is if "x" is 0, // which the callers guarantee to not happen. FMT_SUPPRESS_MSC_WARNING(6102) - return 31 - r; + return 31 ^ static_cast(r); } # define FMT_BUILTIN_CLZ(n) detail::clz(n) -# if defined(_WIN64) && !defined(__clang__) -# pragma intrinsic(_BitScanReverse64) -# endif - -inline uint32_t clzll(uint64_t x) { +inline int clzll(uint64_t x) { unsigned long r = 0; # ifdef _WIN64 _BitScanReverse64(&r, x); # else // Scan the high 32 bits. - if (_BitScanReverse(&r, static_cast(x >> 32))) return 63 - (r + 32); - + if (_BitScanReverse(&r, static_cast(x >> 32))) return 63 ^ (r + 32); // Scan the low 32 bits. _BitScanReverse(&r, static_cast(x)); # endif - FMT_ASSERT(x != 0, ""); - // Static analysis complains about using uninitialized data - // "r", but the only way that can happen is if "x" is 0, - // which the callers guarantee to not happen. - FMT_SUPPRESS_MSC_WARNING(6102) - return 63 - r; + FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. + return 63 ^ static_cast(r); } # define FMT_BUILTIN_CLZLL(n) detail::clzll(n) + +inline int ctz(uint32_t x) { + unsigned long r = 0; + _BitScanForward(&r, x); + FMT_ASSERT(x != 0, ""); + FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. + return static_cast(r); +} +# define FMT_BUILTIN_CTZ(n) detail::ctz(n) + +inline int ctzll(uint64_t x) { + unsigned long r = 0; + FMT_ASSERT(x != 0, ""); + FMT_SUPPRESS_MSC_WARNING(6102) // Suppress a bogus static analysis warning. +# ifdef _WIN64 + _BitScanForward64(&r, x); +# else + // Scan the low 32 bits. + if (_BitScanForward(&r, static_cast(x))) return static_cast(r); + // Scan the high 32 bits. + _BitScanForward(&r, static_cast(x >> 32)); + r += 32; +# endif + return static_cast(r); +} +# define FMT_BUILTIN_CTZLL(n) detail::ctzll(n) } // namespace detail FMT_END_NAMESPACE #endif @@ -298,50 +340,11 @@ FMT_INLINE void assume(bool condition) { #endif } -// A workaround for gcc 4.8 to make void_t work in a SFINAE context. -template struct void_t_impl { using type = void; }; - -template -using void_t = typename detail::void_t_impl::type; - // An approximation of iterator_t for pre-C++20 systems. template using iterator_t = decltype(std::begin(std::declval())); template using sentinel_t = decltype(std::end(std::declval())); -// Detect the iterator category of *any* given type in a SFINAE-friendly way. -// Unfortunately, older implementations of std::iterator_traits are not safe -// for use in a SFINAE-context. -template -struct iterator_category : std::false_type {}; - -template struct iterator_category { - using type = std::random_access_iterator_tag; -}; - -template -struct iterator_category> { - using type = typename It::iterator_category; -}; - -// Detect if *any* given type models the OutputIterator concept. -template class is_output_iterator { - // Check for mutability because all iterator categories derived from - // std::input_iterator_tag *may* also meet the requirements of an - // OutputIterator, thereby falling into the category of 'mutable iterators' - // [iterator.requirements.general] clause 4. The compiler reveals this - // property only at the point of *actually dereferencing* the iterator! - template - static decltype(*(std::declval())) test(std::input_iterator_tag); - template static char& test(std::output_iterator_tag); - template static const char& test(...); - - using type = decltype(test(typename iterator_category::type{})); - - public: - enum { value = !std::is_const>::value }; -}; - // A workaround for std::string not having mutable data() until C++17. template inline Char* get_data(std::basic_string& s) { return &s[0]; @@ -374,10 +377,29 @@ reserve(std::back_insert_iterator it, size_t n) { return make_checked(get_data(c) + size, n); } +template +inline buffer_appender reserve(buffer_appender it, size_t n) { + buffer& buf = get_container(it); + buf.try_reserve(buf.size() + n); + return it; +} + template inline Iterator& reserve(Iterator& it, size_t) { return it; } +template +constexpr T* to_pointer(OutputIt, size_t) { + return nullptr; +} +template T* to_pointer(buffer_appender it, size_t n) { + buffer& buf = get_container(it); + auto size = buf.size(); + if (buf.capacity() < size + n) return nullptr; + buf.try_resize(size + n); + return buf.data() + size; +} + template ::value)> inline std::back_insert_iterator base_iterator( std::back_insert_iterator& it, @@ -415,13 +437,17 @@ class counting_iterator { ++count_; return *this; } - counting_iterator operator++(int) { auto it = *this; ++*this; return it; } + friend counting_iterator operator+(counting_iterator it, difference_type n) { + it.count_ += static_cast(n); + return it; + } + value_type operator*() const { return {}; } }; @@ -555,23 +581,38 @@ OutputIt copy_str(InputIt begin, InputIt end, OutputIt it) { [](char c) { return static_cast(c); }); } -#ifndef FMT_USE_GRISU -# define FMT_USE_GRISU 1 -#endif - -template constexpr bool use_grisu() { - return FMT_USE_GRISU && std::numeric_limits::is_iec559 && - sizeof(T) <= sizeof(double); +template +inline counting_iterator copy_str(InputIt begin, InputIt end, + counting_iterator it) { + return it + (end - begin); } +template +using is_fast_float = bool_constant::is_iec559 && + sizeof(T) <= sizeof(double)>; + +#ifndef FMT_USE_FULL_CACHE_DRAGONBOX +# define FMT_USE_FULL_CACHE_DRAGONBOX 0 +#endif + template template void buffer::append(const U* begin, const U* end) { - size_t new_size = size_ + to_unsigned(end - begin); - reserve(new_size); - std::uninitialized_copy(begin, end, - make_checked(ptr_ + size_, capacity_ - size_)); - size_ = new_size; + do { + auto count = to_unsigned(end - begin); + try_reserve(size_ + count); + auto free_cap = capacity_ - size_; + if (free_cap < count) count = free_cap; + std::uninitialized_copy_n(begin, count, make_checked(ptr_ + size_, count)); + size_ += count; + begin += count; + } while (begin != end); +} + +template +void iterator_buffer::flush() { + out_ = std::copy_n(data_, this->limit(this->size()), out_); + this->clear(); } } // namespace detail @@ -610,7 +651,7 @@ enum { inline_buffer_size = 500 }; */ template > -class basic_memory_buffer : public detail::buffer { +class basic_memory_buffer final : public detail::buffer { private: T store_[SIZE]; @@ -624,7 +665,7 @@ class basic_memory_buffer : public detail::buffer { } protected: - void grow(size_t size) FMT_OVERRIDE; + void grow(size_t size) final FMT_OVERRIDE; public: using value_type = T; @@ -634,7 +675,7 @@ class basic_memory_buffer : public detail::buffer { : alloc_(alloc) { this->set(store_, SIZE); } - ~basic_memory_buffer() FMT_OVERRIDE { deallocate(); } + ~basic_memory_buffer() { deallocate(); } private: // Move data from other to this buffer. @@ -678,6 +719,22 @@ class basic_memory_buffer : public detail::buffer { // Returns a copy of the allocator associated with this buffer. Allocator get_allocator() const { return alloc_; } + + /** + Resizes the buffer to contain *count* elements. If T is a POD type new + elements may not be initialized. + */ + void resize(size_t count) { this->try_resize(count); } + + /** Increases the buffer capacity to *new_capacity*. */ + void reserve(size_t new_capacity) { this->try_reserve(new_capacity); } + + // Directly append data into the buffer + using detail::buffer::append; + template + void append(const ContiguousRange& range) { + append(range.data(), range.data() + range.size()); + } }; template @@ -748,19 +805,81 @@ FMT_CONSTEXPR bool is_supported_floating_point(T) { } // Smallest of uint32_t, uint64_t, uint128_t that is large enough to -// represent all values of T. +// represent all values of an integral type T. template using uint32_or_64_or_128_t = - conditional_t() <= 32, uint32_t, + conditional_t() <= 32 && !FMT_REDUCE_INT_INSTANTIATIONS, + uint32_t, conditional_t() <= 64, uint64_t, uint128_t>>; +// 128-bit integer type used internally +struct FMT_EXTERN_TEMPLATE_API uint128_wrapper { + uint128_wrapper() = default; + +#if FMT_USE_INT128 + uint128_t internal_; + + uint128_wrapper(uint64_t high, uint64_t low) FMT_NOEXCEPT + : internal_{static_cast(low) | + (static_cast(high) << 64)} {} + + uint128_wrapper(uint128_t u) : internal_{u} {} + + uint64_t high() const FMT_NOEXCEPT { return uint64_t(internal_ >> 64); } + uint64_t low() const FMT_NOEXCEPT { return uint64_t(internal_); } + + uint128_wrapper& operator+=(uint64_t n) FMT_NOEXCEPT { + internal_ += n; + return *this; + } +#else + uint64_t high_; + uint64_t low_; + + uint128_wrapper(uint64_t high, uint64_t low) FMT_NOEXCEPT : high_{high}, + low_{low} {} + + uint64_t high() const FMT_NOEXCEPT { return high_; } + uint64_t low() const FMT_NOEXCEPT { return low_; } + + uint128_wrapper& operator+=(uint64_t n) FMT_NOEXCEPT { +# if defined(_MSC_VER) && defined(_M_X64) + unsigned char carry = _addcarry_u64(0, low_, n, &low_); + _addcarry_u64(carry, high_, 0, &high_); + return *this; +# else + uint64_t sum = low_ + n; + high_ += (sum < low_ ? 1 : 0); + low_ = sum; + return *this; +# endif + } +#endif +}; + +// Table entry type for divisibility test used internally +template struct FMT_EXTERN_TEMPLATE_API divtest_table_entry { + T mod_inv; + T max_quotient; +}; + // Static data is placed in this class template for the header-only config. template struct FMT_EXTERN_TEMPLATE_API basic_data { static const uint64_t powers_of_10_64[]; - static const uint32_t zero_or_powers_of_10_32[]; - static const uint64_t zero_or_powers_of_10_64[]; - static const uint64_t pow10_significands[]; - static const int16_t pow10_exponents[]; + static const uint32_t zero_or_powers_of_10_32_new[]; + static const uint64_t zero_or_powers_of_10_64_new[]; + static const uint64_t grisu_pow10_significands[]; + static const int16_t grisu_pow10_exponents[]; + static const divtest_table_entry divtest_table_for_pow5_32[]; + static const divtest_table_entry divtest_table_for_pow5_64[]; + static const uint64_t dragonbox_pow10_significands_64[]; + static const uint128_wrapper dragonbox_pow10_significands_128[]; + // log10(2) = 0x0.4d104d427de7fbcc... + static const uint64_t log10_2_significand = 0x4d104d427de7fbcc; +#if !FMT_USE_FULL_CACHE_DRAGONBOX + static const uint64_t powers_of_5_64[]; + static const uint32_t dragonbox_pow10_recovery_errors[]; +#endif // GCC generates slightly better code for pairs than chars. using digit_pair = char[2]; static const digit_pair digits[]; @@ -772,8 +891,23 @@ template struct FMT_EXTERN_TEMPLATE_API basic_data { static const char signs[]; static const char left_padding_shifts[5]; static const char right_padding_shifts[5]; + + // DEPRECATED! These are for ABI compatibility. + static const uint32_t zero_or_powers_of_10_32[]; + static const uint64_t zero_or_powers_of_10_64[]; }; +// Maps bsr(n) to ceil(log10(pow(2, bsr(n) + 1) - 1)). +// This is a function instead of an array to workaround a bug in GCC10 (#1810). +FMT_INLINE uint16_t bsr2log10(int bsr) { + static constexpr uint16_t data[] = { + 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, + 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10, + 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 15, 15, + 15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 19, 20}; + return data[bsr]; +} + #ifndef FMT_EXPORTED FMT_EXTERN template struct basic_data; #endif @@ -785,10 +919,9 @@ struct data : basic_data<> {}; // Returns the number of decimal digits in n. Leading zeros are not counted // except for n == 0 in which case count_digits returns 1. inline int count_digits(uint64_t n) { - // Based on http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10 - // and the benchmark https://github.com/localvoid/cxx-benchmark-count-digits. - int t = (64 - FMT_BUILTIN_CLZLL(n | 1)) * 1233 >> 12; - return t - (n < data::zero_or_powers_of_10_64[t]) + 1; + // https://github.com/fmtlib/format-benchmark/blob/master/digits10 + auto t = bsr2log10(FMT_BUILTIN_CLZLL(n | 1) ^ 63); + return t - (n < data::zero_or_powers_of_10_64_new[t]); } #else // Fallback version of count_digits used when __builtin_clz is not available. @@ -838,15 +971,24 @@ template <> int count_digits<4>(detail::fallback_uintptr n); #if FMT_GCC_VERSION || FMT_CLANG_VERSION # define FMT_ALWAYS_INLINE inline __attribute__((always_inline)) +#elif FMT_MSC_VER +# define FMT_ALWAYS_INLINE __forceinline #else -# define FMT_ALWAYS_INLINE +# define FMT_ALWAYS_INLINE inline +#endif + +// To suppress unnecessary security cookie checks +#if FMT_MSC_VER && !FMT_CLANG_VERSION +# define FMT_SAFEBUFFERS __declspec(safebuffers) +#else +# define FMT_SAFEBUFFERS #endif #ifdef FMT_BUILTIN_CLZ // Optional version of count_digits for better performance on 32-bit platforms. inline int count_digits(uint32_t n) { - int t = (32 - FMT_BUILTIN_CLZ(n | 1)) * 1233 >> 12; - return t - (n < data::zero_or_powers_of_10_32[t]) + 1; + auto t = bsr2log10(FMT_BUILTIN_CLZ(n | 1) ^ 31); + return t - (n < data::zero_or_powers_of_10_32_new[t]); } #endif @@ -893,7 +1035,7 @@ template void copy2(Char* dst, const char* src) { *dst++ = static_cast(*src++); *dst = static_cast(*src); } -inline void copy2(char* dst, const char* src) { memcpy(dst, src, 2); } +FMT_INLINE void copy2(char* dst, const char* src) { memcpy(dst, src, 2); } template struct format_decimal_result { Iterator begin; @@ -929,11 +1071,10 @@ inline format_decimal_result format_decimal(Char* out, UInt value, template >::value)> inline format_decimal_result format_decimal(Iterator out, UInt value, - int num_digits) { - // Buffer should be large enough to hold all digits (<= digits10 + 1). - enum { max_size = digits10() + 1 }; - Char buffer[2 * max_size]; - auto end = format_decimal(buffer, value, num_digits).end; + int size) { + // Buffer is large enough to hold all digits (digits10 + 1). + Char buffer[digits10() + 1]; + auto end = format_decimal(buffer, value, size).end; return {out, detail::copy_str(buffer, end, out)}; } @@ -975,6 +1116,10 @@ Char* format_uint(Char* buffer, detail::fallback_uintptr n, int num_digits, template inline It format_uint(It out, UInt value, int num_digits, bool upper = false) { + if (auto ptr = to_pointer(out, to_unsigned(num_digits))) { + format_uint(ptr, value, num_digits, upper); + return out; + } // Buffer should be large enough to hold all digits (digits / BASE_BITS + 1). char buffer[num_bits() / BASE_BITS + 1]; format_uint(buffer, value, num_digits, upper); @@ -1000,8 +1145,8 @@ template struct null {}; template struct fill_t { private: enum { max_size = 4 }; - Char data_[max_size]; - unsigned char size_; + Char data_[max_size] = {Char(' '), Char(0), Char(0), Char(0)}; + unsigned char size_ = 1; public: FMT_CONSTEXPR void operator=(basic_string_view s) { @@ -1021,13 +1166,6 @@ template struct fill_t { FMT_CONSTEXPR const Char& operator[](size_t index) const { return data_[index]; } - - static FMT_CONSTEXPR fill_t make() { - auto fill = fill_t(); - fill[0] = Char(' '); - fill.size_ = 1; - return fill; - } }; } // namespace detail @@ -1059,13 +1197,84 @@ template struct basic_format_specs { type(0), align(align::none), sign(sign::none), - alt(false), - fill(detail::fill_t::make()) {} + alt(false) {} }; using format_specs = basic_format_specs; namespace detail { +namespace dragonbox { + +// Type-specific information that Dragonbox uses. +template struct float_info; + +template <> struct float_info { + using carrier_uint = uint32_t; + static const int significand_bits = 23; + static const int exponent_bits = 8; + static const int min_exponent = -126; + static const int max_exponent = 127; + static const int exponent_bias = -127; + static const int decimal_digits = 9; + static const int kappa = 1; + static const int big_divisor = 100; + static const int small_divisor = 10; + static const int min_k = -31; + static const int max_k = 46; + static const int cache_bits = 64; + static const int divisibility_check_by_5_threshold = 39; + static const int case_fc_pm_half_lower_threshold = -1; + static const int case_fc_pm_half_upper_threshold = 6; + static const int case_fc_lower_threshold = -2; + static const int case_fc_upper_threshold = 6; + static const int case_shorter_interval_left_endpoint_lower_threshold = 2; + static const int case_shorter_interval_left_endpoint_upper_threshold = 3; + static const int shorter_interval_tie_lower_threshold = -35; + static const int shorter_interval_tie_upper_threshold = -35; + static const int max_trailing_zeros = 7; +}; + +template <> struct float_info { + using carrier_uint = uint64_t; + static const int significand_bits = 52; + static const int exponent_bits = 11; + static const int min_exponent = -1022; + static const int max_exponent = 1023; + static const int exponent_bias = -1023; + static const int decimal_digits = 17; + static const int kappa = 2; + static const int big_divisor = 1000; + static const int small_divisor = 100; + static const int min_k = -292; + static const int max_k = 326; + static const int cache_bits = 128; + static const int divisibility_check_by_5_threshold = 86; + static const int case_fc_pm_half_lower_threshold = -2; + static const int case_fc_pm_half_upper_threshold = 9; + static const int case_fc_lower_threshold = -4; + static const int case_fc_upper_threshold = 9; + static const int case_shorter_interval_left_endpoint_lower_threshold = 2; + static const int case_shorter_interval_left_endpoint_upper_threshold = 3; + static const int shorter_interval_tie_lower_threshold = -77; + static const int shorter_interval_tie_upper_threshold = -77; + static const int max_trailing_zeros = 16; +}; + +template struct decimal_fp { + using significand_type = typename float_info::carrier_uint; + significand_type significand; + int exponent; +}; + +template FMT_API decimal_fp to_decimal(T x) FMT_NOEXCEPT; +} // namespace dragonbox + +template +constexpr typename dragonbox::float_info::carrier_uint exponent_mask() { + using uint = typename dragonbox::float_info::carrier_uint; + return ((uint(1) << dragonbox::float_info::exponent_bits) - 1) + << dragonbox::float_info::significand_bits; +} // A floating-point presentation format. enum class float_format : unsigned char { @@ -1107,113 +1316,6 @@ template It write_exponent(int exp, It it) { return it; } -template class float_writer { - private: - // The number is given as v = digits_ * pow(10, exp_). - const char* digits_; - int num_digits_; - int exp_; - size_t size_; - float_specs specs_; - Char decimal_point_; - - template It prettify(It it) const { - // pow(10, full_exp - 1) <= v <= pow(10, full_exp). - int full_exp = num_digits_ + exp_; - if (specs_.format == float_format::exp) { - // Insert a decimal point after the first digit and add an exponent. - *it++ = static_cast(*digits_); - int num_zeros = specs_.precision - num_digits_; - if (num_digits_ > 1 || specs_.showpoint) *it++ = decimal_point_; - it = copy_str(digits_ + 1, digits_ + num_digits_, it); - if (num_zeros > 0 && specs_.showpoint) - it = std::fill_n(it, num_zeros, static_cast('0')); - *it++ = static_cast(specs_.upper ? 'E' : 'e'); - return write_exponent(full_exp - 1, it); - } - if (num_digits_ <= full_exp) { - // 1234e7 -> 12340000000[.0+] - it = copy_str(digits_, digits_ + num_digits_, it); - it = std::fill_n(it, full_exp - num_digits_, static_cast('0')); - if (specs_.showpoint || specs_.precision < 0) { - *it++ = decimal_point_; - int num_zeros = specs_.precision - full_exp; - if (num_zeros <= 0) { - if (specs_.format != float_format::fixed) - *it++ = static_cast('0'); - return it; - } -#ifdef FMT_FUZZ - if (num_zeros > 5000) - throw std::runtime_error("fuzz mode - avoiding excessive cpu use"); -#endif - it = std::fill_n(it, num_zeros, static_cast('0')); - } - } else if (full_exp > 0) { - // 1234e-2 -> 12.34[0+] - it = copy_str(digits_, digits_ + full_exp, it); - if (!specs_.showpoint) { - // Remove trailing zeros. - int num_digits = num_digits_; - while (num_digits > full_exp && digits_[num_digits - 1] == '0') - --num_digits; - if (num_digits != full_exp) *it++ = decimal_point_; - return copy_str(digits_ + full_exp, digits_ + num_digits, it); - } - *it++ = decimal_point_; - it = copy_str(digits_ + full_exp, digits_ + num_digits_, it); - if (specs_.precision > num_digits_) { - // Add trailing zeros. - int num_zeros = specs_.precision - num_digits_; - it = std::fill_n(it, num_zeros, static_cast('0')); - } - } else { - // 1234e-6 -> 0.001234 - *it++ = static_cast('0'); - int num_zeros = -full_exp; - int num_digits = num_digits_; - if (num_digits == 0 && specs_.precision >= 0 && - specs_.precision < num_zeros) { - num_zeros = specs_.precision; - } - // Remove trailing zeros. - if (!specs_.showpoint) - while (num_digits > 0 && digits_[num_digits - 1] == '0') --num_digits; - if (num_zeros != 0 || num_digits != 0 || specs_.showpoint) { - *it++ = decimal_point_; - it = std::fill_n(it, num_zeros, static_cast('0')); - it = copy_str(digits_, digits_ + num_digits, it); - } - } - return it; - } - - public: - float_writer(const char* digits, int num_digits, int exp, float_specs specs, - Char decimal_point) - : digits_(digits), - num_digits_(num_digits), - exp_(exp), - specs_(specs), - decimal_point_(decimal_point) { - int full_exp = num_digits + exp - 1; - int precision = specs.precision > 0 ? specs.precision : 16; - if (specs_.format == float_format::general && - !(full_exp >= -4 && full_exp < precision)) { - specs_.format = float_format::exp; - } - size_ = prettify(counting_iterator()).count(); - size_ += specs.sign ? 1 : 0; - } - - size_t size() const { return size_; } - - template It operator()(It it) const { - if (specs_.sign) *it++ = static_cast(data::signs[specs_.sign]); - return prettify(it); - } -}; - template int format_float(T value, int precision, float_specs specs, buffer& buf); @@ -1392,7 +1494,7 @@ template inline OutputIt write_padded(OutputIt out, const basic_format_specs& specs, size_t size, - size_t width, const F& f) { + size_t width, F&& f) { static_assert(align == align::left || align == align::right, ""); unsigned spec_width = to_unsigned(specs.width); size_t padding = spec_width > width ? spec_width - width : 0; @@ -1410,7 +1512,7 @@ template inline OutputIt write_padded(OutputIt out, const basic_format_specs& specs, size_t size, - const F& f) { + F&& f) { return write_padded(out, specs, size, size, f); } @@ -1577,15 +1679,16 @@ template struct int_writer { char digits[40]; format_decimal(digits, abs_value, num_digits); basic_memory_buffer buffer; - size += prefix_size; - buffer.resize(size); + size += static_cast(prefix_size); + const auto usize = to_unsigned(size); + buffer.resize(usize); basic_string_view s(&sep, sep_size); // Index of a decimal digit with the least significant digit having index 0. int digit_index = 0; group = groups.cbegin(); - auto p = buffer.data() + size; - for (int i = num_digits - 1; i >= 0; --i) { - *--p = static_cast(digits[i]); + auto p = buffer.data() + size - 1; + for (int i = num_digits - 1; i > 0; --i) { + *p-- = static_cast(digits[i]); if (*group <= 0 || ++digit_index % *group != 0 || *group == max_value()) continue; @@ -1593,16 +1696,16 @@ template struct int_writer { digit_index = 0; ++group; } - p -= s.size(); std::uninitialized_copy(s.data(), s.data() + s.size(), make_checked(p, s.size())); + p -= s.size(); } - if (prefix_size != 0) p[-1] = static_cast('-'); - using iterator = remove_reference_t; + *p-- = static_cast(*digits); + if (prefix_size != 0) *p = static_cast('-'); auto data = buffer.data(); - out = write_padded(out, specs, size, size, [=](iterator it) { - return copy_str(data, data + size, it); - }); + out = write_padded( + out, specs, usize, usize, + [=](iterator it) { return copy_str(data, data + size, it); }); } void on_chr() { *out++ = static_cast(abs_value); } @@ -1628,6 +1731,168 @@ OutputIt write_nonfinite(OutputIt out, bool isinf, }); } +// A decimal floating-point number significand * pow(10, exp). +struct big_decimal_fp { + const char* significand; + int significand_size; + int exponent; +}; + +inline int get_significand_size(const big_decimal_fp& fp) { + return fp.significand_size; +} +template +inline int get_significand_size(const dragonbox::decimal_fp& fp) { + return count_digits(fp.significand); +} + +template +inline OutputIt write_significand(OutputIt out, const char* significand, + int& significand_size) { + return copy_str(significand, significand + significand_size, out); +} +template +inline OutputIt write_significand(OutputIt out, UInt significand, + int significand_size) { + return format_decimal(out, significand, significand_size).end; +} + +template ::value)> +inline Char* write_significand(Char* out, UInt significand, + int significand_size, int integral_size, + Char decimal_point) { + if (!decimal_point) + return format_decimal(out, significand, significand_size).end; + auto end = format_decimal(out + 1, significand, significand_size).end; + if (integral_size == 1) + out[0] = out[1]; + else + std::copy_n(out + 1, integral_size, out); + out[integral_size] = decimal_point; + return end; +} + +template >::value)> +inline OutputIt write_significand(OutputIt out, UInt significand, + int significand_size, int integral_size, + Char decimal_point) { + // Buffer is large enough to hold digits (digits10 + 1) and a decimal point. + Char buffer[digits10() + 2]; + auto end = write_significand(buffer, significand, significand_size, + integral_size, decimal_point); + return detail::copy_str(buffer, end, out); +} + +template +inline OutputIt write_significand(OutputIt out, const char* significand, + int significand_size, int integral_size, + Char decimal_point) { + out = detail::copy_str(significand, significand + integral_size, out); + if (!decimal_point) return out; + *out++ = decimal_point; + return detail::copy_str(significand + integral_size, + significand + significand_size, out); +} + +template +OutputIt write_float(OutputIt out, const DecimalFP& fp, + const basic_format_specs& specs, float_specs fspecs, + Char decimal_point) { + auto significand = fp.significand; + int significand_size = get_significand_size(fp); + static const Char zero = static_cast('0'); + auto sign = fspecs.sign; + size_t size = to_unsigned(significand_size) + (sign ? 1 : 0); + using iterator = remove_reference_t; + + int output_exp = fp.exponent + significand_size - 1; + auto use_exp_format = [=]() { + if (fspecs.format == float_format::exp) return true; + if (fspecs.format != float_format::general) return false; + // Use the fixed notation if the exponent is in [exp_lower, exp_upper), + // e.g. 0.0001 instead of 1e-04. Otherwise use the exponent notation. + const int exp_lower = -4, exp_upper = 16; + return output_exp < exp_lower || + output_exp >= (fspecs.precision > 0 ? fspecs.precision : exp_upper); + }; + if (use_exp_format()) { + int num_zeros = 0; + if (fspecs.showpoint) { + num_zeros = (std::max)(fspecs.precision - significand_size, 0); + size += to_unsigned(num_zeros); + } else if (significand_size == 1) { + decimal_point = Char(); + } + auto abs_output_exp = output_exp >= 0 ? output_exp : -output_exp; + int exp_digits = 2; + if (abs_output_exp >= 100) exp_digits = abs_output_exp >= 1000 ? 4 : 3; + + size += to_unsigned((decimal_point ? 1 : 0) + 2 + exp_digits); + char exp_char = fspecs.upper ? 'E' : 'e'; + auto write = [=](iterator it) { + if (sign) *it++ = static_cast(data::signs[sign]); + // Insert a decimal point after the first digit and add an exponent. + it = write_significand(it, significand, significand_size, 1, + decimal_point); + if (num_zeros > 0) it = std::fill_n(it, num_zeros, zero); + *it++ = static_cast(exp_char); + return write_exponent(output_exp, it); + }; + return specs.width > 0 ? write_padded(out, specs, size, write) + : base_iterator(out, write(reserve(out, size))); + } + + int exp = fp.exponent + significand_size; + if (fp.exponent >= 0) { + // 1234e5 -> 123400000[.0+] + size += to_unsigned(fp.exponent); + int num_zeros = fspecs.precision - exp; +#ifdef FMT_FUZZ + if (num_zeros > 5000) + throw std::runtime_error("fuzz mode - avoiding excessive cpu use"); +#endif + if (fspecs.showpoint) { + if (num_zeros <= 0 && fspecs.format != float_format::fixed) num_zeros = 1; + if (num_zeros > 0) size += to_unsigned(num_zeros); + } + return write_padded(out, specs, size, [&](iterator it) { + if (sign) *it++ = static_cast(data::signs[sign]); + it = write_significand(it, significand, significand_size); + it = std::fill_n(it, fp.exponent, zero); + if (!fspecs.showpoint) return it; + *it++ = decimal_point; + return num_zeros > 0 ? std::fill_n(it, num_zeros, zero) : it; + }); + } else if (exp > 0) { + // 1234e-2 -> 12.34[0+] + int num_zeros = fspecs.showpoint ? fspecs.precision - significand_size : 0; + size += 1 + to_unsigned(num_zeros > 0 ? num_zeros : 0); + return write_padded(out, specs, size, [&](iterator it) { + if (sign) *it++ = static_cast(data::signs[sign]); + it = write_significand(it, significand, significand_size, exp, + decimal_point); + return num_zeros > 0 ? std::fill_n(it, num_zeros, zero) : it; + }); + } + // 1234e-6 -> 0.001234 + int num_zeros = -exp; + if (significand_size == 0 && fspecs.precision >= 0 && + fspecs.precision < num_zeros) { + num_zeros = fspecs.precision; + } + size += 2 + to_unsigned(num_zeros); + return write_padded(out, specs, size, [&](iterator it) { + if (sign) *it++ = static_cast(data::signs[sign]); + *it++ = zero; + if (num_zeros == 0 && significand_size == 0 && !fspecs.showpoint) return it; + *it++ = decimal_point; + it = std::fill_n(it, num_zeros, zero); + return write_significand(it, significand, significand_size); + }); +} + template ::value)> OutputIt write(OutputIt out, T value, basic_format_specs specs, @@ -1667,39 +1932,45 @@ OutputIt write(OutputIt out, T value, basic_format_specs specs, ++precision; } if (const_check(std::is_same())) fspecs.binary32 = true; - fspecs.use_grisu = use_grisu(); + fspecs.use_grisu = is_fast_float(); int exp = format_float(promote_float(value), precision, fspecs, buffer); fspecs.precision = precision; Char point = fspecs.locale ? decimal_point(loc) : static_cast('.'); - float_writer w(buffer.data(), static_cast(buffer.size()), exp, - fspecs, point); - return write_padded(out, specs, w.size(), w); + auto fp = big_decimal_fp{buffer.data(), static_cast(buffer.size()), exp}; + return write_float(out, fp, specs, fspecs, point); } template ::value)> + FMT_ENABLE_IF(is_fast_float::value)> OutputIt write(OutputIt out, T value) { if (const_check(!is_supported_floating_point(value))) return out; + + using floaty = conditional_t::value, double, T>; + using uint = typename dragonbox::float_info::carrier_uint; + auto bits = bit_cast(value); + auto fspecs = float_specs(); - if (std::signbit(value)) { // value < 0 is false for NaN so use signbit. + auto sign_bit = bits & (uint(1) << (num_bits() - 1)); + if (sign_bit != 0) { fspecs.sign = sign::minus; value = -value; } - auto specs = basic_format_specs(); - if (!std::isfinite(value)) + static const auto specs = basic_format_specs(); + uint mask = exponent_mask(); + if ((bits & mask) == mask) return write_nonfinite(out, std::isinf(value), specs, fspecs); - memory_buffer buffer; - int precision = -1; - if (const_check(std::is_same())) fspecs.binary32 = true; - fspecs.use_grisu = use_grisu(); - int exp = format_float(promote_float(value), precision, fspecs, buffer); - fspecs.precision = precision; - float_writer w(buffer.data(), static_cast(buffer.size()), exp, - fspecs, static_cast('.')); - return base_iterator(out, w(reserve(out, w.size()))); + auto dec = dragonbox::to_decimal(static_cast(value)); + return write_float(out, dec, specs, fspecs, static_cast('.')); +} + +template ::value && + !is_fast_float::value)> +inline OutputIt write(OutputIt out, T value) { + return write(out, value, basic_format_specs()); } template @@ -1752,6 +2023,13 @@ OutputIt write(OutputIt out, basic_string_view value) { return base_iterator(out, it); } +template +buffer_appender write(buffer_appender out, + basic_string_view value) { + get_container(out).append(value.begin(), value.end()); + return out; +} + template ::value && !std::is_same::value && @@ -1762,7 +2040,13 @@ OutputIt write(OutputIt out, T value) { // Don't do -abs_value since it trips unsigned-integer-overflow sanitizer. if (negative) abs_value = ~abs_value + 1; int num_digits = count_digits(abs_value); - auto it = reserve(out, (negative ? 1 : 0) + static_cast(num_digits)); + auto size = (negative ? 1 : 0) + static_cast(num_digits); + auto it = reserve(out, size); + if (auto ptr = to_pointer(it, size)) { + if (negative) *ptr++ = static_cast('-'); + format_decimal(ptr, abs_value, num_digits); + return out; + } if (negative) *it++ = static_cast('-'); it = format_decimal(it, abs_value, num_digits).end; return base_iterator(out, it); @@ -1801,8 +2085,13 @@ auto write(OutputIt out, const T& value) -> typename std::enable_if< mapped_type_constant>::value == type::custom_type, OutputIt>::type { - basic_format_context ctx(out, {}, {}); - return formatter().format(value, ctx); + using context_type = basic_format_context; + using formatter_type = + conditional_t::value, + typename context_type::template formatter_type, + fallback_formatter>; + context_type ctx(out, {}, {}); + return formatter_type().format(value, ctx); } // An argument visitor that formats the argument and writes it via the output @@ -2008,6 +2297,48 @@ class arg_formatter_base { } }; +/** The default argument formatter. */ +template +class arg_formatter : public arg_formatter_base { + private: + using char_type = Char; + using base = arg_formatter_base; + using context_type = basic_format_context; + + context_type& ctx_; + basic_format_parse_context* parse_ctx_; + const Char* ptr_; + + public: + using iterator = typename base::iterator; + using format_specs = typename base::format_specs; + + /** + \rst + Constructs an argument formatter object. + *ctx* is a reference to the formatting context, + *specs* contains format specifier information for standard argument types. + \endrst + */ + explicit arg_formatter( + context_type& ctx, + basic_format_parse_context* parse_ctx = nullptr, + format_specs* specs = nullptr, const Char* ptr = nullptr) + : base(ctx.out(), specs, ctx.locale()), + ctx_(ctx), + parse_ctx_(parse_ctx), + ptr_(ptr) {} + + using base::operator(); + + /** Formats an argument of a user-defined type. */ + iterator operator()(typename basic_format_arg::handle handle) { + if (ptr_) advance_to(*parse_ctx_, ptr_); + handle.format(*parse_ctx_, ctx_); + return ctx_.out(); + } +}; + template FMT_CONSTEXPR bool is_name_start(Char c) { return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || '_' == c; } @@ -2047,12 +2378,11 @@ template class custom_formatter { Context& ctx) : parse_ctx_(parse_ctx), ctx_(ctx) {} - bool operator()(typename basic_format_arg::handle h) const { + void operator()(typename basic_format_arg::handle h) const { h.format(parse_ctx_, ctx_); - return true; } - template bool operator()(T) const { return false; } + template void operator()(T) const {} }; template @@ -2434,12 +2764,30 @@ template struct precision_adapter { }; template -FMT_CONSTEXPR const Char* next_code_point(const Char* begin, const Char* end) { - if (const_check(sizeof(Char) != 1) || (*begin & 0x80) == 0) return begin + 1; - do { - ++begin; - } while (begin != end && (*begin & 0xc0) == 0x80); - return begin; +FMT_CONSTEXPR int code_point_length(const Char* begin) { + if (const_check(sizeof(Char) != 1)) return 1; + constexpr char lengths[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0}; + int len = lengths[static_cast(*begin) >> 3]; + + // Compute the pointer to the next character early so that the next + // iteration can start working on the next character. Neither Clang + // nor GCC figure out this reordering on their own. + return len + !len; +} + +template constexpr bool is_ascii_letter(Char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +// Converts a character to ASCII. Returns a number > 127 on conversion failure. +template ::value)> +constexpr Char to_ascii(Char value) { + return value; +} +template ::value)> +constexpr typename std::underlying_type::type to_ascii(Char value) { + return value; } // Parses fill and alignment. @@ -2448,10 +2796,10 @@ FMT_CONSTEXPR const Char* parse_align(const Char* begin, const Char* end, Handler&& handler) { FMT_ASSERT(begin != end, ""); auto align = align::none; - auto p = next_code_point(begin, end); - if (p == end) p = begin; + auto p = begin + code_point_length(begin); + if (p >= end) p = begin; for (;;) { - switch (static_cast(*p)) { + switch (to_ascii(*p)) { case '<': align = align::left; break; @@ -2530,13 +2878,13 @@ FMT_CONSTEXPR const Char* parse_precision(const Char* begin, const Char* end, template FMT_CONSTEXPR const Char* parse_format_specs(const Char* begin, const Char* end, SpecHandler&& handler) { - if (begin == end || *begin == '}') return begin; + if (begin == end) return begin; begin = parse_align(begin, end, handler); if (begin == end) return begin; // Parse sign. - switch (static_cast(*begin)) { + switch (to_ascii(*begin)) { case '+': handler.on_plus(); ++begin; @@ -2613,7 +2961,7 @@ FMT_CONSTEXPR const Char* parse_replacement_field(const Char* begin, Handler&& handler) { ++begin; if (begin == end) return handler.on_error("invalid format string"), end; - if (static_cast(*begin) == '}') { + if (*begin == '}') { handler.on_replacement_field(handler.on_arg_id(), begin); } else if (*begin == '{') { handler.on_text(begin, begin + 1); @@ -2658,17 +3006,17 @@ FMT_CONSTEXPR_DECL FMT_INLINE void parse_format_string( return; } struct writer { - FMT_CONSTEXPR void operator()(const Char* begin, const Char* end) { - if (begin == end) return; + FMT_CONSTEXPR void operator()(const Char* pbegin, const Char* pend) { + if (pbegin == pend) return; for (;;) { const Char* p = nullptr; - if (!find(begin, end, '}', p)) - return handler_.on_text(begin, end); + if (!find(pbegin, pend, '}', p)) + return handler_.on_text(pbegin, pend); ++p; - if (p == end || *p != '}') + if (p == pend || *p != '}') return handler_.on_error("unmatched '}' in format string"); - handler_.on_text(begin, p); - begin = p + 1; + handler_.on_text(pbegin, p); + pbegin = p + 1; } } Handler& handler_; @@ -2699,13 +3047,12 @@ FMT_CONSTEXPR const typename ParseContext::char_type* parse_format_specs( return f.parse(ctx); } -template +template struct format_handler : detail::error_handler { basic_format_parse_context parse_context; Context context; - format_handler(typename ArgFormatter::iterator out, - basic_string_view str, + format_handler(OutputIt out, basic_string_view str, basic_format_args format_args, detail::locale_ref loc) : parse_context(str), context(out, format_args, loc) {} @@ -2728,26 +3075,33 @@ struct format_handler : detail::error_handler { FMT_INLINE void on_replacement_field(int id, const Char*) { auto arg = get_arg(context, id); context.advance_to(visit_format_arg( - default_arg_formatter{ - context.out(), context.args(), context.locale()}, + default_arg_formatter{context.out(), context.args(), + context.locale()}, arg)); } const Char* on_format_specs(int id, const Char* begin, const Char* end) { - advance_to(parse_context, begin); auto arg = get_arg(context, id); - custom_formatter f(parse_context, context); - if (visit_format_arg(f, arg)) return parse_context.begin(); - basic_format_specs specs; - using parse_context_t = basic_format_parse_context; - specs_checker> handler( - specs_handler(specs, parse_context, context), - arg.type()); - begin = parse_format_specs(begin, end, handler); - if (begin == end || *begin != '}') on_error("missing '}' in format string"); - advance_to(parse_context, begin); - context.advance_to( - visit_format_arg(ArgFormatter(context, &parse_context, &specs), arg)); + if (arg.type() == type::custom_type) { + advance_to(parse_context, begin); + visit_format_arg(custom_formatter(parse_context, context), arg); + return parse_context.begin(); + } + auto specs = basic_format_specs(); + if (begin + 1 < end && begin[1] == '}' && is_ascii_letter(*begin)) { + specs.type = static_cast(*begin++); + } else { + using parse_context_t = basic_format_parse_context; + specs_checker> handler( + specs_handler(specs, parse_context, + context), + arg.type()); + begin = parse_format_specs(begin, end, handler); + if (begin == end || *begin != '}') + on_error("missing '}' in format string"); + } + context.advance_to(visit_format_arg( + arg_formatter(context, &parse_context, &specs), arg)); return begin; } }; @@ -2899,53 +3253,11 @@ FMT_API void format_error_code(buffer& out, int error_code, FMT_API void report_error(format_func func, int error_code, string_view message) FMT_NOEXCEPT; - -/** The default argument formatter. */ -template -class arg_formatter : public arg_formatter_base { - private: - using char_type = Char; - using base = arg_formatter_base; - using context_type = basic_format_context; - - context_type& ctx_; - basic_format_parse_context* parse_ctx_; - const Char* ptr_; - - public: - using iterator = typename base::iterator; - using format_specs = typename base::format_specs; - - /** - \rst - Constructs an argument formatter object. - *ctx* is a reference to the formatting context, - *specs* contains format specifier information for standard argument types. - \endrst - */ - explicit arg_formatter( - context_type& ctx, - basic_format_parse_context* parse_ctx = nullptr, - format_specs* specs = nullptr, const Char* ptr = nullptr) - : base(ctx.out(), specs, ctx.locale()), - ctx_(ctx), - parse_ctx_(parse_ctx), - ptr_(ptr) {} - - using base::operator(); - - /** Formats an argument of a user-defined type. */ - iterator operator()(typename basic_format_arg::handle handle) { - if (ptr_) advance_to(*parse_ctx_, ptr_); - handle.format(*parse_ctx_, ctx_); - return ctx_.out(); - } -}; } // namespace detail template using arg_formatter FMT_DEPRECATED_ALIAS = - detail::arg_formatter; + detail::arg_formatter; /** An error returned by an operating system or a language runtime, @@ -3208,8 +3520,10 @@ struct formatter : formatter, Char> { // using variant = std::variant; // template <> // struct formatter: dynamic_formatter<> { -// void format(buffer &buf, const variant &v, context &ctx) { -// visit([&](const auto &val) { format(buf, val, ctx); }, v); +// auto format(const variant& v, format_context& ctx) { +// return visit([&](const auto& val) { +// return dynamic_formatter<>::format(val, ctx); +// }, v); // } // }; template class dynamic_formatter { @@ -3277,28 +3591,15 @@ FMT_CONSTEXPR void advance_to( ctx.advance_to(ctx.begin() + (p - &*ctx.begin())); } -/** Formats arguments and writes the output to the range. */ -template -typename Context::iterator vformat_to( - typename ArgFormatter::iterator out, basic_string_view format_str, - basic_format_args args, - detail::locale_ref loc = detail::locale_ref()) { - if (format_str.size() == 2 && detail::equal2(format_str.data(), "{}")) { - auto arg = args.get(0); - if (!arg) detail::error_handler().on_error("argument not found"); - using iterator = typename ArgFormatter::iterator; - return visit_format_arg( - detail::default_arg_formatter{out, args, loc}, arg); - } - detail::format_handler h(out, format_str, args, - loc); - detail::parse_format_string(format_str, h); - return h.context.out(); -} +/** + \rst + Converts ``p`` to ``const void*`` for pointer formatting. -// Casts ``p`` to ``const void*`` for pointer formatting. -// Example: -// auto s = format("{}", ptr(p)); + **Example**:: + + auto s = fmt::format("{}", fmt::ptr(p)); + \endrst + */ template inline const void* ptr(const T* p) { return p; } template inline const void* ptr(const std::unique_ptr& p) { return p.get(); @@ -3317,6 +3618,10 @@ class bytes { }; template <> struct formatter { + private: + detail::dynamic_format_specs specs_; + + public: template FMT_CONSTEXPR auto parse(ParseContext& ctx) -> decltype(ctx.begin()) { using handler_type = detail::dynamic_specs_handler; @@ -3335,9 +3640,6 @@ template <> struct formatter { specs_.precision, specs_.precision_ref, ctx); return detail::write_bytes(ctx.out(), b.data_, specs_); } - - private: - detail::dynamic_format_specs specs_; }; template @@ -3402,15 +3704,14 @@ arg_join join(It begin, Sentinel end, wstring_view sep) { \endrst */ template -arg_join, detail::sentinel_t, char> -join(const Range& range, string_view sep) { +arg_join, detail::sentinel_t, char> join( + Range&& range, string_view sep) { return join(std::begin(range), std::end(range), sep); } template -arg_join, detail::sentinel_t, - wchar_t> -join(const Range& range, wstring_view sep) { +arg_join, detail::sentinel_t, wchar_t> join( + Range&& range, wstring_view sep) { return join(std::begin(range), std::end(range), sep); } @@ -3437,7 +3738,7 @@ inline std::string to_string(T value) { // The buffer should be large enough to store the number including the sign or // "false" for bool. constexpr int max_size = detail::digits10() + 2; - char buffer[max_size > 5 ? max_size : 5]; + char buffer[max_size > 5 ? static_cast(max_size) : 5]; char* begin = buffer; return std::string(begin, detail::write(begin, value)); } @@ -3457,18 +3758,30 @@ std::basic_string to_string(const basic_memory_buffer& buf) { } template -typename buffer_context::iterator detail::vformat_to( +void detail::vformat_to( detail::buffer& buf, basic_string_view format_str, - basic_format_args>> args) { - using af = arg_formatter::iterator, Char>; - return vformat_to(std::back_inserter(buf), to_string_view(format_str), - args); + basic_format_args>> args, + detail::locale_ref loc) { + using iterator = typename buffer_context::iterator; + auto out = buffer_appender(buf); + if (format_str.size() == 2 && equal2(format_str.data(), "{}")) { + auto arg = args.get(0); + if (!arg) error_handler().on_error("argument not found"); + visit_format_arg(default_arg_formatter{out, args, loc}, + arg); + return; + } + format_handler> h(out, format_str, args, + loc); + parse_format_string(format_str, h); } #ifndef FMT_HEADER_ONLY -extern template format_context::iterator detail::vformat_to( - detail::buffer&, string_view, basic_format_args); +extern template void detail::vformat_to(detail::buffer&, string_view, + basic_format_args, + detail::locale_ref); namespace detail { + extern template FMT_API std::string grouping_impl(locale_ref loc); extern template FMT_API std::string grouping_impl(locale_ref loc); extern template FMT_API char thousands_sep_impl(locale_ref loc); @@ -3494,7 +3807,7 @@ extern template int snprintf_float(long double value, template , FMT_ENABLE_IF(detail::is_string::value)> -inline typename FMT_BUFFER_CONTEXT(Char)::iterator vformat_to( +inline void vformat_to( detail::buffer& buf, const S& format_str, basic_format_args)> args) { return detail::vformat_to(buf, to_string_view(format_str), args); @@ -3504,10 +3817,9 @@ template ::value, char_t>> inline typename buffer_context::iterator format_to( basic_memory_buffer& buf, const S& format_str, Args&&... args) { - detail::check_format_string(format_str); - using context = buffer_context; - return detail::vformat_to(buf, to_string_view(format_str), - make_format_args(args...)); + const auto& vargs = fmt::make_args_checked(format_str, args...); + detail::vformat_to(buf, to_string_view(format_str), vargs); + return detail::buffer_appender(buf); } template @@ -3516,88 +3828,17 @@ using format_context_t = basic_format_context; template using format_args_t = basic_format_args>; -template < - typename S, typename OutputIt, typename... Args, - FMT_ENABLE_IF(detail::is_output_iterator::value && - !detail::is_contiguous_back_insert_iterator::value)> -inline OutputIt vformat_to( - OutputIt out, const S& format_str, - format_args_t, char_t> args) { - using af = detail::arg_formatter>; - return vformat_to(out, to_string_view(format_str), args); -} - -/** - \rst - Formats arguments, writes the result to the output iterator ``out`` and returns - the iterator past the end of the output range. - - **Example**:: - - std::vector out; - fmt::format_to(std::back_inserter(out), "{}", 42); - \endrst - */ -template ::value && - !detail::is_contiguous_back_insert_iterator::value && - detail::is_string::value)> -inline OutputIt format_to(OutputIt out, const S& format_str, Args&&... args) { - detail::check_format_string(format_str); - using context = format_context_t>; - return vformat_to(out, to_string_view(format_str), - make_format_args(args...)); -} - -template struct format_to_n_result { - /** Iterator past the end of the output range. */ - OutputIt out; - /** Total (not truncated) output size. */ - size_t size; -}; - template -using format_to_n_context = - format_context_t, Char>; +using format_to_n_context FMT_DEPRECATED_ALIAS = buffer_context; template -using format_to_n_args = basic_format_args>; +using format_to_n_args FMT_DEPRECATED_ALIAS = + basic_format_args>; template -inline format_arg_store, Args...> +FMT_DEPRECATED format_arg_store, Args...> make_format_to_n_args(const Args&... args) { - return format_arg_store, Args...>( - args...); -} - -template ::value)> -inline format_to_n_result vformat_to_n( - OutputIt out, size_t n, basic_string_view format_str, - format_to_n_args, type_identity_t> args) { - auto it = vformat_to(detail::truncating_iterator(out, n), - format_str, args); - return {it.base(), it.count()}; -} - -/** - \rst - Formats arguments, writes up to ``n`` characters of the result to the output - iterator ``out`` and returns the total output size and the iterator past the - end of the output range. - \endrst - */ -template ::value&& - detail::is_output_iterator::value)> -inline format_to_n_result format_to_n(OutputIt out, size_t n, - const S& format_str, - const Args&... args) { - detail::check_format_string(format_str); - using context = format_to_n_context>; - return vformat_to_n(out, n, to_string_view(format_str), - make_format_args(args...)); + return format_arg_store, Args...>(args...); } template ::value), int>> @@ -3609,15 +3850,6 @@ std::basic_string detail::vformat( return to_string(buffer); } -/** - Returns the number of characters in the output of - ``format(format_str, args...)``. - */ -template -inline size_t formatted_size(string_view format_str, const Args&... args) { - return format_to(detail::counting_iterator(), format_str, args...).count(); -} - template ::value)> void vprint(std::FILE* f, basic_string_view format_str, wformat_args args) { @@ -3642,8 +3874,7 @@ template class udl_formatter { template std::basic_string operator()(Args&&... args) const { static FMT_CONSTEXPR_DECL Char s[] = {CHARS..., '\0'}; - check_format_string...>(FMT_STRING(s)); - return format(s, std::forward(args)...); + return format(FMT_STRING(s), std::forward(args)...); } }; # else diff --git a/src/third_party/format.cpp b/src/third_party/format.cpp index a64a1f3893..6141d964a7 100644 --- a/src/third_party/format.cpp +++ b/src/third_party/format.cpp @@ -23,6 +23,36 @@ int format_float(char* buf, std::size_t size, const char* format, int precision, return precision < 0 ? snprintf_ptr(buf, size, format, value) : snprintf_ptr(buf, size, format, precision, value); } + +template FMT_API dragonbox::decimal_fp dragonbox::to_decimal(float x) + FMT_NOEXCEPT; +template FMT_API dragonbox::decimal_fp dragonbox::to_decimal(double x) + FMT_NOEXCEPT; + +// DEPRECATED! This function exists for ABI compatibility. +template +typename basic_format_context>, + Char>::iterator +vformat_to(buffer& buf, basic_string_view format_str, + basic_format_args>>, + type_identity_t>> + args) { + using iterator = std::back_insert_iterator>; + using context = basic_format_context< + std::back_insert_iterator>>, + type_identity_t>; + auto out = iterator(buf); + format_handler h(out, format_str, args, {}); + parse_format_string(format_str, h); + return out; +} +template basic_format_context>, + char>::iterator +vformat_to(buffer&, string_view, + basic_format_args>>, + type_identity_t>>); } // namespace detail template struct FMT_INSTANTIATION_DEF_API detail::basic_data; @@ -44,9 +74,9 @@ template FMT_API char detail::decimal_point_impl(locale_ref); template FMT_API void detail::buffer::append(const char*, const char*); -template FMT_API FMT_BUFFER_CONTEXT(char)::iterator detail::vformat_to( +template FMT_API void detail::vformat_to( detail::buffer&, string_view, - basic_format_args); + basic_format_args, detail::locale_ref); template FMT_API int detail::snprintf_float(double, int, detail::float_specs, detail::buffer&); From 5b380e4b043b5ab7be19f9382b11ddc3e9a8c3fd Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 20:13:38 +0100 Subject: [PATCH 32/38] Upgrade to optional-lite 3.4.0 --- LICENSE.adoc | 6 +- src/third_party/nonstd/optional.hpp | 249 +++++++++++++++++++--------- 2 files changed, 172 insertions(+), 83 deletions(-) diff --git a/LICENSE.adoc b/LICENSE.adoc index 9ccdb0717e..045b282cdd 100644 --- a/LICENSE.adoc +++ b/LICENSE.adoc @@ -558,9 +558,9 @@ SOFTWARE. src/third_party/nonstd/optional.hpp ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This alternative implementation of `std::optional` was downloaded from - and has the following license -text: +This is the single header version of +https://github.com/martinmoene/optional-lite[optional-lite] 3.4.0 with the +following license: ------------------------------------------------------------------------------- Copyright (c) 2014-2018 Martin Moene diff --git a/src/third_party/nonstd/optional.hpp b/src/third_party/nonstd/optional.hpp index 33a9b987e0..8b371e5a81 100644 --- a/src/third_party/nonstd/optional.hpp +++ b/src/third_party/nonstd/optional.hpp @@ -12,7 +12,7 @@ #define NONSTD_OPTIONAL_LITE_HPP #define optional_lite_MAJOR 3 -#define optional_lite_MINOR 2 +#define optional_lite_MINOR 4 #define optional_lite_PATCH 0 #define optional_lite_VERSION optional_STRINGIFY(optional_lite_MAJOR) "." optional_STRINGIFY(optional_lite_MINOR) "." optional_STRINGIFY(optional_lite_PATCH) @@ -26,6 +26,20 @@ #define optional_OPTIONAL_NONSTD 1 #define optional_OPTIONAL_STD 2 +// tweak header support: + +#ifdef __has_include +# if __has_include() +# include +# endif +#define optional_HAVE_TWEAK_HEADER 1 +#else +#define optional_HAVE_TWEAK_HEADER 0 +//# pragma message("optional.hpp: Note: Tweak header not supported.") +#endif + +// optional selection and configuration: + #if !defined( optional_CONFIG_SELECT_OPTIONAL ) # define optional_CONFIG_SELECT_OPTIONAL ( optional_HAVE_STD_OPTIONAL ? optional_OPTIONAL_STD : optional_OPTIONAL_NONSTD ) #endif @@ -33,7 +47,10 @@ // Control presence of exception handling (try and auto discover): #ifndef optional_CONFIG_NO_EXCEPTIONS -# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) +# if _MSC_VER +# include // for _HAS_EXCEPTIONS +# endif +# if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || (_HAS_EXCEPTIONS) # define optional_CONFIG_NO_EXCEPTIONS 0 # else # define optional_CONFIG_NO_EXCEPTIONS 1 @@ -227,16 +244,17 @@ namespace nonstd { // Compiler versions: // -// MSVC++ 6.0 _MSC_VER == 1200 (Visual Studio 6.0) -// MSVC++ 7.0 _MSC_VER == 1300 (Visual Studio .NET 2002) -// MSVC++ 7.1 _MSC_VER == 1310 (Visual Studio .NET 2003) -// MSVC++ 8.0 _MSC_VER == 1400 (Visual Studio 2005) -// MSVC++ 9.0 _MSC_VER == 1500 (Visual Studio 2008) -// MSVC++ 10.0 _MSC_VER == 1600 (Visual Studio 2010) -// MSVC++ 11.0 _MSC_VER == 1700 (Visual Studio 2012) -// MSVC++ 12.0 _MSC_VER == 1800 (Visual Studio 2013) -// MSVC++ 14.0 _MSC_VER == 1900 (Visual Studio 2015) -// MSVC++ 14.1 _MSC_VER >= 1910 (Visual Studio 2017) +// MSVC++ 6.0 _MSC_VER == 1200 optional_COMPILER_MSVC_VERSION == 60 (Visual Studio 6.0) +// MSVC++ 7.0 _MSC_VER == 1300 optional_COMPILER_MSVC_VERSION == 70 (Visual Studio .NET 2002) +// MSVC++ 7.1 _MSC_VER == 1310 optional_COMPILER_MSVC_VERSION == 71 (Visual Studio .NET 2003) +// MSVC++ 8.0 _MSC_VER == 1400 optional_COMPILER_MSVC_VERSION == 80 (Visual Studio 2005) +// MSVC++ 9.0 _MSC_VER == 1500 optional_COMPILER_MSVC_VERSION == 90 (Visual Studio 2008) +// MSVC++ 10.0 _MSC_VER == 1600 optional_COMPILER_MSVC_VERSION == 100 (Visual Studio 2010) +// MSVC++ 11.0 _MSC_VER == 1700 optional_COMPILER_MSVC_VERSION == 110 (Visual Studio 2012) +// MSVC++ 12.0 _MSC_VER == 1800 optional_COMPILER_MSVC_VERSION == 120 (Visual Studio 2013) +// MSVC++ 14.0 _MSC_VER == 1900 optional_COMPILER_MSVC_VERSION == 140 (Visual Studio 2015) +// MSVC++ 14.1 _MSC_VER >= 1910 optional_COMPILER_MSVC_VERSION == 141 (Visual Studio 2017) +// MSVC++ 14.2 _MSC_VER >= 1920 optional_COMPILER_MSVC_VERSION == 142 (Visual Studio 2019) #if defined(_MSC_VER ) && !defined(__clang__) # define optional_COMPILER_MSVC_VER (_MSC_VER ) @@ -295,13 +313,26 @@ namespace nonstd { #define optional_CPP14_000 (optional_CPP14_OR_GREATER) #define optional_CPP17_000 (optional_CPP17_OR_GREATER) +// gcc >= 4.9, msvc >= vc14.1 (vs17): +#define optional_CPP11_140_G490 ((optional_CPP11_OR_GREATER_ && optional_COMPILER_GNUC_VERSION >= 490) || (optional_COMPILER_MSVC_VER >= 1910)) + +// clang >= 3.5, msvc >= vc11 (vs12): +#define optional_CPP11_110_C350 ( optional_CPP11_110 && !optional_BETWEEN( optional_COMPILER_CLANG_VERSION, 1, 350 ) ) + +// clang >= 3.5, gcc >= 5.0, msvc >= vc11 (vs12): +#define optional_CPP11_110_C350_G500 \ + ( optional_CPP11_110 && \ + !( optional_BETWEEN( optional_COMPILER_CLANG_VERSION, 1, 350 ) \ + || optional_BETWEEN( optional_COMPILER_GNUC_VERSION , 1, 500 ) ) ) + // Presence of C++11 language features: #define optional_HAVE_CONSTEXPR_11 optional_CPP11_140 #define optional_HAVE_IS_DEFAULT optional_CPP11_140 #define optional_HAVE_NOEXCEPT optional_CPP11_140 #define optional_HAVE_NULLPTR optional_CPP11_100 -#define optional_HAVE_REF_QUALIFIER optional_CPP11_140 +#define optional_HAVE_REF_QUALIFIER optional_CPP11_140_G490 +#define optional_HAVE_INITIALIZER_LIST optional_CPP11_140 // Presence of C++14 language features: @@ -320,6 +351,13 @@ namespace nonstd { #define optional_HAVE_TR1_TYPE_TRAITS (!! optional_COMPILER_GNUC_VERSION ) #define optional_HAVE_TR1_ADD_POINTER (!! optional_COMPILER_GNUC_VERSION ) +#define optional_HAVE_IS_ASSIGNABLE optional_CPP11_110_C350 +#define optional_HAVE_IS_MOVE_CONSTRUCTIBLE optional_CPP11_110_C350 +#define optional_HAVE_IS_NOTHROW_MOVE_ASSIGNABLE optional_CPP11_110_C350 +#define optional_HAVE_IS_NOTHROW_MOVE_CONSTRUCTIBLE optional_CPP11_110_C350 +#define optional_HAVE_IS_TRIVIALLY_COPY_CONSTRUCTIBLE optional_CPP11_110_C350_G500 +#define optional_HAVE_IS_TRIVIALLY_MOVE_CONSTRUCTIBLE optional_CPP11_110_C350_G500 + // C++ feature usage: #if optional_HAVE( CONSTEXPR_11 ) @@ -397,7 +435,7 @@ namespace nonstd { template< bool B = (__VA_ARGS__), typename std::enable_if::type = 0 > #define optional_REQUIRES_T(...) \ - , typename = typename std::enable_if< (__VA_ARGS__), nonstd::optional_lite::detail::enabler >::type + , typename std::enable_if< (__VA_ARGS__), int >::type = 0 #define optional_REQUIRES_R(R, ...) \ typename std::enable_if< (__VA_ARGS__), R>::type @@ -415,6 +453,12 @@ namespace nonstd { namespace optional_lite { namespace std11 { +template< class T, T v > struct integral_constant { enum { value = v }; }; +template< bool B > struct bool_constant : integral_constant{}; + +typedef bool_constant< true > true_type; +typedef bool_constant< false > false_type; + #if optional_CPP11_OR_GREATER using std::move; #else @@ -428,6 +472,42 @@ namespace std11 { template< typename T, typename F > struct conditional { typedef F type; }; #endif // optional_HAVE_CONDITIONAL +#if optional_HAVE( IS_ASSIGNABLE ) + using std::is_assignable; +#else + template< class T, class U > struct is_assignable : std11::true_type{}; +#endif + +#if optional_HAVE( IS_MOVE_CONSTRUCTIBLE ) + using std::is_move_constructible; +#else + template< class T > struct is_move_constructible : std11::true_type{}; +#endif + +#if optional_HAVE( IS_NOTHROW_MOVE_ASSIGNABLE ) + using std::is_nothrow_move_assignable; +#else + template< class T > struct is_nothrow_move_assignable : std11::true_type{}; +#endif + +#if optional_HAVE( IS_NOTHROW_MOVE_CONSTRUCTIBLE ) + using std::is_nothrow_move_constructible; +#else + template< class T > struct is_nothrow_move_constructible : std11::true_type{}; +#endif + +#if optional_HAVE( IS_TRIVIALLY_COPY_CONSTRUCTIBLE ) + using std::is_trivially_copy_constructible; +#else + template< class T > struct is_trivially_copy_constructible : std11::true_type{}; +#endif + +#if optional_HAVE( IS_TRIVIALLY_MOVE_CONSTRUCTIBLE ) + using std::is_trivially_move_constructible; +#else + template< class T > struct is_trivially_move_constructible : std11::true_type{}; +#endif + } // namespace std11 #if optional_CPP11_OR_GREATER @@ -450,10 +530,10 @@ using std::swap; struct is_swappable { template< typename T, typename = decltype( swap( std::declval(), std::declval() ) ) > - static std::true_type test( int /*unused*/ ); + static std11::true_type test( int /*unused*/ ); template< typename > - static std::false_type test(...); + static std11::false_type test(...); }; struct is_nothrow_swappable @@ -467,10 +547,10 @@ struct is_nothrow_swappable } template< typename T > - static auto test( int /*unused*/ ) -> std::integral_constant()>{} + static auto test( int /*unused*/ ) -> std11::integral_constant()>{} template< typename > - static auto test(...) -> std::false_type; + static auto test(...) -> std11::false_type; }; } // namespace detail @@ -508,12 +588,6 @@ class optional; namespace detail { -// for optional_REQUIRES_T - -#if optional_CPP11_OR_GREATER -enum class enabler{}; -#endif - // C++11 emulation: struct nulltype{}; @@ -704,6 +778,12 @@ union storage_t ::new( value_ptr() ) value_type( std::move( v ) ); } + template< class... Args > + storage_t( nonstd_lite_in_place_t(T), Args&&... args ) + { + emplace( std::forward(args)... ); + } + template< class... Args > void emplace( Args&&... args ) { @@ -743,7 +823,7 @@ union storage_t return * value_ptr(); } -#if optional_CPP11_OR_GREATER +#if optional_HAVE( REF_QUALIFIER ) optional_nodiscard value_type const && value() const optional_refref_qual { @@ -861,13 +941,15 @@ class optional {} // 2 - copy-construct - optional_constexpr14 optional( optional const & other #if optional_CPP11_OR_GREATER - optional_REQUIRES_A( - true || std::is_copy_constructible::value - ) + // template< typename U = T + // optional_REQUIRES_T( + // std::is_copy_constructible::value + // || std11::is_trivially_copy_constructible::value + // ) + // > #endif - ) + optional_constexpr14 optional( optional const & other ) : has_value_( other.has_value() ) { if ( other.has_value() ) @@ -879,12 +961,15 @@ class optional #if optional_CPP11_OR_GREATER // 3 (C++11) - move-construct from optional - optional_constexpr14 optional( optional && other - optional_REQUIRES_A( - true || std::is_move_constructible::value + template< typename U = T + optional_REQUIRES_T( + std11::is_move_constructible::value + || std11::is_trivially_move_constructible::value ) - // NOLINTNEXTLINE( performance-noexcept-move-constructor ) - ) noexcept( std::is_nothrow_move_constructible::value ) + > + optional_constexpr14 optional( optional && other ) + // NOLINTNEXTLINE( performance-noexcept-move-constructor ) + noexcept( std11::is_nothrow_move_constructible::value ) : has_value_( other.has_value() ) { if ( other.has_value() ) @@ -894,9 +979,8 @@ class optional } // 4a (C++11) - explicit converting copy-construct from optional - template< typename U > - explicit optional( optional const & other - optional_REQUIRES_A( + template< typename U + optional_REQUIRES_T( std::is_constructible::value && !std::is_constructible & >::value && !std::is_constructible && >::value @@ -908,7 +992,8 @@ class optional && !std::is_convertible< optional const &&, T>::value && !std::is_convertible< U const & , T>::value /*=> explicit */ ) - ) + > + explicit optional( optional const & other ) : has_value_( other.has_value() ) { if ( other.has_value() ) @@ -919,11 +1004,9 @@ class optional #endif // optional_CPP11_OR_GREATER // 4b (C++98 and later) - non-explicit converting copy-construct from optional - template< typename U > - // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) - optional( optional const & other + template< typename U #if optional_CPP11_OR_GREATER - optional_REQUIRES_A( + optional_REQUIRES_T( std::is_constructible::value && !std::is_constructible & >::value && !std::is_constructible && >::value @@ -936,7 +1019,9 @@ class optional && std::is_convertible< U const & , T>::value /*=> non-explicit */ ) #endif // optional_CPP11_OR_GREATER - ) + > + // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) + /*non-explicit*/ optional( optional const & other ) : has_value_( other.has_value() ) { if ( other.has_value() ) @@ -948,9 +1033,8 @@ class optional #if optional_CPP11_OR_GREATER // 5a (C++11) - explicit converting move-construct from optional - template< typename U > - explicit optional( optional && other - optional_REQUIRES_A( + template< typename U + optional_REQUIRES_T( std::is_constructible::value && !std::is_constructible & >::value && !std::is_constructible && >::value @@ -962,6 +1046,8 @@ class optional && !std::is_convertible< optional const &&, T>::value && !std::is_convertible< U &&, T>::value /*=> explicit */ ) + > + explicit optional( optional && other ) : has_value_( other.has_value() ) { @@ -972,10 +1058,8 @@ class optional } // 5a (C++11) - non-explicit converting move-construct from optional - template< typename U > - // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) - optional( optional && other - optional_REQUIRES_A( + template< typename U + optional_REQUIRES_T( std::is_constructible::value && !std::is_constructible & >::value && !std::is_constructible && >::value @@ -987,7 +1071,9 @@ class optional && !std::is_convertible< optional const &&, T>::value && std::is_convertible< U &&, T>::value /*=> non-explicit */ ) - ) + > + // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) + /*non-explicit*/ optional( optional && other ) : has_value_( other.has_value() ) { if ( other.has_value() ) @@ -1019,32 +1105,32 @@ class optional {} // 8a (C++11) - explicit move construct from value - template< typename U = value_type > - optional_constexpr explicit optional( U && value - optional_REQUIRES_A( + template< typename U = T + optional_REQUIRES_T( std::is_constructible::value && !std::is_same::type, nonstd_lite_in_place_t(U)>::value && !std::is_same::type, optional>::value && !std::is_convertible::value /*=> explicit */ ) - ) + > + optional_constexpr explicit optional( U && value ) : has_value_( true ) - , contained( T{ std::forward( value ) } ) + , contained( nonstd_lite_in_place(T), std::forward( value ) ) {} // 8b (C++11) - non-explicit move construct from value - template< typename U = value_type > - // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) - optional_constexpr optional( U && value - optional_REQUIRES_A( + template< typename U = T + optional_REQUIRES_T( std::is_constructible::value && !std::is_same::type, nonstd_lite_in_place_t(U)>::value && !std::is_same::type, optional>::value && std::is_convertible::value /*=> non-explicit */ ) - ) + > + // NOLINTNEXTLINE( google-explicit-constructor, hicpp-explicit-conversions ) + optional_constexpr /*non-explicit*/ optional( U && value ) : has_value_( true ) - , contained( std::forward( value ) ) + , contained( nonstd_lite_in_place(T), std::forward( value ) ) {} #else // optional_CPP11_OR_GREATER @@ -1087,8 +1173,8 @@ class optional ) operator=( optional const & other ) noexcept( - std::is_nothrow_move_assignable::value - && std::is_nothrow_move_constructible::value + std11::is_nothrow_move_assignable::value + && std11::is_nothrow_move_constructible::value ) #else optional & operator=( optional const & other ) @@ -1107,7 +1193,7 @@ class optional optional_REQUIRES_R( optional &, true -// std::is_move_constructible::value +// std11::is_move_constructible::value // && std::is_move_assignable::value ) operator=( optional && other ) noexcept @@ -1124,7 +1210,7 @@ class optional optional_REQUIRES_R( optional &, std::is_constructible::value - && std::is_assignable::value + && std11::is_assignable::value && !std::is_same::type, nonstd_lite_in_place_t(U)>::value && !std::is_same::type, optional>::value && !(std::is_scalar::value && std::is_same::type>::value) @@ -1162,7 +1248,7 @@ class optional optional_REQUIRES_R( optional&, std::is_constructible< T , U const &>::value - && std::is_assignable< T&, U const &>::value + && std11::is_assignable< T&, U const &>::value && !std::is_constructible & >::value && !std::is_constructible && >::value && !std::is_constructible const & >::value @@ -1171,10 +1257,10 @@ class optional && !std::is_convertible< optional && , T>::value && !std::is_convertible< optional const & , T>::value && !std::is_convertible< optional const &&, T>::value - && !std::is_assignable< T&, optional & >::value - && !std::is_assignable< T&, optional && >::value - && !std::is_assignable< T&, optional const & >::value - && !std::is_assignable< T&, optional const && >::value + && !std11::is_assignable< T&, optional & >::value + && !std11::is_assignable< T&, optional && >::value + && !std11::is_assignable< T&, optional const & >::value + && !std11::is_assignable< T&, optional const && >::value ) #else optional& @@ -1192,7 +1278,7 @@ class optional optional_REQUIRES_R( optional&, std::is_constructible< T , U>::value - && std::is_assignable< T&, U>::value + && std11::is_assignable< T&, U>::value && !std::is_constructible & >::value && !std::is_constructible && >::value && !std::is_constructible const & >::value @@ -1201,10 +1287,10 @@ class optional && !std::is_convertible< optional && , T>::value && !std::is_convertible< optional const & , T>::value && !std::is_convertible< optional const &&, T>::value - && !std::is_assignable< T&, optional & >::value - && !std::is_assignable< T&, optional && >::value - && !std::is_assignable< T&, optional const & >::value - && !std::is_assignable< T&, optional const && >::value + && !std11::is_assignable< T&, optional & >::value + && !std11::is_assignable< T&, optional && >::value + && !std11::is_assignable< T&, optional const & >::value + && !std11::is_assignable< T&, optional const && >::value ) operator=( optional && other ) { @@ -1246,7 +1332,7 @@ class optional void swap( optional & other ) #if optional_CPP11_OR_GREATER noexcept( - std::is_nothrow_move_constructible::value + std11::is_nothrow_move_constructible::value && std17::is_nothrow_swappable::value ) #endif @@ -1283,7 +1369,7 @@ class optional contained.value(); } -#if optional_HAVE( REF_QUALIFIER ) && ( !optional_COMPILER_GNUC_VERSION || optional_COMPILER_GNUC_VERSION >= 490 ) +#if optional_HAVE( REF_QUALIFIER ) optional_constexpr value_type const && operator *() const optional_refref_qual { @@ -1612,7 +1698,7 @@ inline optional_constexpr bool operator>=( U const & v, optional const & x ) template< typename T #if optional_CPP11_OR_GREATER optional_REQUIRES_T( - std::is_move_constructible::value + std11::is_move_constructible::value && std17::is_swappable::value ) #endif > @@ -1659,7 +1745,10 @@ optional make_optional( T const & value ) using optional_lite::optional; using optional_lite::nullopt_t; using optional_lite::nullopt; + +#if ! optional_CONFIG_NO_EXCEPTIONS using optional_lite::bad_optional_access; +#endif using optional_lite::make_optional; From 9bc012166d180c4ae0c3042e83ab34b4b21e8926 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 20:23:40 +0100 Subject: [PATCH 33/38] Improve header inclusion for SignalHandler --- src/SignalHandler.cpp | 7 ++++--- src/SignalHandler.hpp | 4 +--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/SignalHandler.cpp b/src/SignalHandler.cpp index 37604c2692..7c6a579044 100644 --- a/src/SignalHandler.cpp +++ b/src/SignalHandler.cpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -18,11 +18,12 @@ #include "SignalHandler.hpp" -#include "assertions.hpp" - #ifndef _WIN32 # include "Context.hpp" +# include "assertions.hpp" + +# include namespace { diff --git a/src/SignalHandler.hpp b/src/SignalHandler.hpp index 50e7b0e234..3ef88479dc 100644 --- a/src/SignalHandler.hpp +++ b/src/SignalHandler.hpp @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Joel Rosdahl and other contributors +// Copyright (C) 2020-2021 Joel Rosdahl and other contributors // // See doc/AUTHORS.adoc for a complete list of contributors. // @@ -20,8 +20,6 @@ #include "system.hpp" -#include "signal.h" - class Context; class SignalHandler From f147414f66447d1ae06f3932ed0aedf571c307d3 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Wed, 6 Jan 2021 21:29:37 +0100 Subject: [PATCH 34/38] Suppress Clang-Tidy warning about including signal.h sigaddset and similar functions are specified by POSIX to be in signal.h and the C++ csignal header only contains a subset of the signal.h declarations. Related to PR #758. --- src/SignalHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SignalHandler.cpp b/src/SignalHandler.cpp index 7c6a579044..8a07cc7e1a 100644 --- a/src/SignalHandler.cpp +++ b/src/SignalHandler.cpp @@ -23,7 +23,7 @@ # include "Context.hpp" # include "assertions.hpp" -# include +# include // NOLINT: sigaddset et al are defined in signal.h namespace { From 94ace27068b07f548e71a217d781ff37a352bdd7 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 6 Jan 2021 21:50:41 +0100 Subject: [PATCH 35/38] Fix scanning of headers with Clang-Tidy (#758) By adding . as an include directory, CMake actually took it literally and files are included from e.g. src/./AtomicFile.h. However in .clang-tidy headers with an additional slash (headers from subdirectory third_party) are excluded from reports. This commit: - gets rid of include via . - fixes some warnings - disables the rest --- src/.clang-tidy | 8 +++++++- src/CMakeLists.txt | 2 +- src/Config.hpp | 2 +- src/NonCopyable.hpp | 8 ++++---- src/argprocessing.hpp | 8 ++++---- src/third_party/CMakeLists.txt | 2 +- unittest/CMakeLists.txt | 2 +- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/.clang-tidy b/src/.clang-tidy index 0f3fa75345..f30529dc0d 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -13,14 +13,17 @@ Checks: '-*, -readability-implicit-bool-conversion, -readability-magic-numbers, -readability-else-after-return, + -readability-named-parameter, -readability-qualified-auto, - -readability-magic-numbers, + -readability-redundant-declaration, performance-*, -performance-unnecessary-value-param, modernize-*, -modernize-avoid-c-arrays, -modernize-pass-by-value, + -modernize-return-braced-init-list, -modernize-use-auto, + -modernize-use-default-member-init, -modernize-use-trailing-return-type, cppcoreguidelines-*, -cppcoreguidelines-pro-bounds-array-to-pointer-decay, @@ -40,6 +43,8 @@ Checks: '-*, -cppcoreguidelines-pro-type-reinterpret-cast, -cppcoreguidelines-pro-type-union-access, -cppcoreguidelines-narrowing-conversions, + -cppcoreguidelines-non-private-member-variables-in-classes, + -cppcoreguidelines-special-member-functions, bugprone-*, -bugprone-signed-char-misuse, -bugprone-branch-clone, @@ -47,6 +52,7 @@ Checks: '-*, cert-*, -cert-err34-c, -cert-dcl50-cpp, + -cert-dcl58-cpp, -cert-err58-cpp, clang-diagnostic-*, clang-analyzer-*, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 30a2f92c26..beefd81f1f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -77,6 +77,6 @@ target_link_libraries( PRIVATE standard_settings standard_warnings ZSTD::ZSTD Threads::Threads third_party_lib) -target_include_directories(ccache_lib PRIVATE ${CMAKE_BINARY_DIR} .) +target_include_directories(ccache_lib PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) add_subdirectory(third_party) diff --git a/src/Config.hpp b/src/Config.hpp index 95a1b68a83..eb574dc8c4 100644 --- a/src/Config.hpp +++ b/src/Config.hpp @@ -84,7 +84,7 @@ class Config void set_cache_dir(const std::string& value); void set_cpp_extension(const std::string& value); void set_compiler(const std::string& value); - void set_compiler_type(CompilerType compiler_type); + void set_compiler_type(CompilerType value); void set_depend_mode(bool value); void set_debug(bool value); void set_direct_mode(bool value); diff --git a/src/NonCopyable.hpp b/src/NonCopyable.hpp index 86004a9342..37fe7e7147 100644 --- a/src/NonCopyable.hpp +++ b/src/NonCopyable.hpp @@ -20,10 +20,10 @@ class NonCopyable { -protected: - NonCopyable() = default; - -private: +public: NonCopyable(const NonCopyable&) = delete; NonCopyable& operator=(const NonCopyable&) = delete; + +protected: + NonCopyable() = default; }; diff --git a/src/argprocessing.hpp b/src/argprocessing.hpp index c040c44e57..1da89d3f37 100644 --- a/src/argprocessing.hpp +++ b/src/argprocessing.hpp @@ -27,10 +27,10 @@ class Context; struct ProcessArgsResult { - ProcessArgsResult(Statistic error); - ProcessArgsResult(const Args& preprocessor_args, - const Args& extra_args_to_hash, - const Args& compiler_args); + ProcessArgsResult(Statistic error_); + ProcessArgsResult(const Args& preprocessor_args_, + const Args& extra_args_to_hash_, + const Args& compiler_args_); // nullopt on success, otherwise the statistics counter that should be // incremented. diff --git a/src/third_party/CMakeLists.txt b/src/third_party/CMakeLists.txt index c361974a44..ed0ff9e848 100644 --- a/src/third_party/CMakeLists.txt +++ b/src/third_party/CMakeLists.txt @@ -40,7 +40,7 @@ endif() # Treat third party headers as system files (no warning for those headers). target_include_directories( third_party_lib - PRIVATE ${CMAKE_BINARY_DIR} . SYSTEM) + PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} SYSTEM) target_link_libraries(third_party_lib PRIVATE standard_settings) target_link_libraries(third_party_lib INTERFACE blake3) diff --git a/unittest/CMakeLists.txt b/unittest/CMakeLists.txt index 65401041a4..48cf058ecb 100644 --- a/unittest/CMakeLists.txt +++ b/unittest/CMakeLists.txt @@ -36,6 +36,6 @@ target_link_libraries( unittest PRIVATE standard_settings standard_warnings ccache_lib third_party_lib) -target_include_directories(unittest PRIVATE ${CMAKE_BINARY_DIR} . ../src) +target_include_directories(unittest PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${ccache_SOURCE_DIR}/src) add_test(NAME unittest COMMAND unittest) From be1ed7749cce3fdb4b3321d4c88749afd2a790a2 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 9 Jan 2021 21:44:30 +0300 Subject: [PATCH 36/38] Ignore SOURCE_DATE_EPOCH under time_macros sloppiness (#755) SOURCE_DATE_EPOCH will be passed from debhelpers, by extracting last entry from d/changelog (or current time if there is entries) And this will not allow to use cache. --- doc/MANUAL.adoc | 3 ++- src/ccache.cpp | 23 +++++++++++++++-------- test/suites/base.bash | 18 ++++++++++++++++++ 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index 1862985617..dbdbd0d7d7 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -839,7 +839,8 @@ still has to do _some_ preprocessing (like macros). hash but not add the system header files to the list of include files. *time_macros*:: Ignore `__DATE__`, `__TIME__` and `__TIMESTAMP__` being present in the - source code. + source code. It will also ignore `SOURCE_DATE_EPOCH` + https://gcc.gnu.org/onlinedocs/gcc/Environment-Variables.html[environment variable] -- + See the discussion under _<<_troubleshooting,Troubleshooting>>_ for more diff --git a/src/ccache.cpp b/src/ccache.cpp index d18ba0f916..6522a8de05 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -1288,17 +1288,24 @@ hash_common_info(const Context& ctx, hash.hash(Util::base_name(args[0])); // Hash variables that may affect the compilation. - const char* always_hash_env_vars[] = { + struct + { + const char* name; + uint32_t sloppiness; + } hash_env_vars[] = { // From : - "COMPILER_PATH", - "GCC_COMPARE_DEBUG", - "GCC_EXEC_PREFIX", - "SOURCE_DATE_EPOCH", + {"COMPILER_PATH", 0}, + {"GCC_COMPARE_DEBUG", 0}, + {"GCC_EXEC_PREFIX", 0}, + {"SOURCE_DATE_EPOCH", SLOPPY_TIME_MACROS}, }; - for (const char* name : always_hash_env_vars) { - const char* value = getenv(name); + for (const auto& env : hash_env_vars) { + if (env.sloppiness && (ctx.config.sloppiness() & env.sloppiness)) { + continue; + } + const char* value = getenv(env.name); if (value) { - hash.hash_delimiter(name); + hash.hash_delimiter(env.name); hash.hash(value); } } diff --git a/test/suites/base.bash b/test/suites/base.bash index 2131ac92a8..d0143009e6 100644 --- a/test/suites/base.bash +++ b/test/suites/base.bash @@ -292,6 +292,24 @@ base_tests() { expect_stat 'cache miss' 1 expect_stat 'files in cache' 1 + # ------------------------------------------------------------------------- + TEST "SOURCE_DATE_EPOCH with time_macros sloppiness" + + CCACHE_SLOPPINESS=time_macros SOURCE_DATE_EPOCH=1 $CCACHE_COMPILE -c test1.c + expect_stat 'cache hit (preprocessed)' 0 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 1 + + CCACHE_SLOPPINESS=time_macros SOURCE_DATE_EPOCH=2 $CCACHE_COMPILE -c test1.c + expect_stat 'cache hit (preprocessed)' 1 + expect_stat 'cache miss' 1 + expect_stat 'files in cache' 1 + + SOURCE_DATE_EPOCH=1 $CCACHE_COMPILE -c test1.c + expect_stat 'cache hit (preprocessed)' 1 + expect_stat 'cache miss' 2 + expect_stat 'files in cache' 2 + # ------------------------------------------------------------------------- TEST "Result file is compressed" From a515303fe6ebcfc6083599b458642e6d850b5d39 Mon Sep 17 00:00:00 2001 From: Joel Rosdahl Date: Sat, 9 Jan 2021 19:52:08 +0100 Subject: [PATCH 37/38] Tweak documentation and code related to SOURCE_DATE_EPOCH --- doc/MANUAL.adoc | 5 +++-- src/ccache.cpp | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/MANUAL.adoc b/doc/MANUAL.adoc index dbdbd0d7d7..591650e6e8 100644 --- a/doc/MANUAL.adoc +++ b/doc/MANUAL.adoc @@ -839,8 +839,9 @@ still has to do _some_ preprocessing (like macros). hash but not add the system header files to the list of include files. *time_macros*:: Ignore `__DATE__`, `__TIME__` and `__TIMESTAMP__` being present in the - source code. It will also ignore `SOURCE_DATE_EPOCH` - https://gcc.gnu.org/onlinedocs/gcc/Environment-Variables.html[environment variable] + source code. The + https://reproducible-builds.org/docs/source-date-epoch/[`SOURCE_DATE_EPOCH` + environment variable] will also be ignored. -- + See the discussion under _<<_troubleshooting,Troubleshooting>>_ for more diff --git a/src/ccache.cpp b/src/ccache.cpp index 6522a8de05..1b430f851a 100644 --- a/src/ccache.cpp +++ b/src/ccache.cpp @@ -1288,10 +1288,10 @@ hash_common_info(const Context& ctx, hash.hash(Util::base_name(args[0])); // Hash variables that may affect the compilation. - struct + const struct { const char* name; - uint32_t sloppiness; + uint32_t sloppiness; // 0 for "always hash" } hash_env_vars[] = { // From : {"COMPILER_PATH", 0}, @@ -1300,7 +1300,7 @@ hash_common_info(const Context& ctx, {"SOURCE_DATE_EPOCH", SLOPPY_TIME_MACROS}, }; for (const auto& env : hash_env_vars) { - if (env.sloppiness && (ctx.config.sloppiness() & env.sloppiness)) { + if (env.sloppiness != 0 && (ctx.config.sloppiness() & env.sloppiness)) { continue; } const char* value = getenv(env.name); From 7ce48a87105f9da9e8f36ba110d316bb9bba94a9 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 11 Dec 2020 09:33:39 +0100 Subject: [PATCH 38/38] Add CodeQL --- .github/workflows/codeql-analysis.yaml | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yaml diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml new file mode 100644 index 0000000000..2ba9e56175 --- /dev/null +++ b/.github/workflows/codeql-analysis.yaml @@ -0,0 +1,49 @@ +# More info: +# https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning + +name: "CodeQL" + +on: + push: + branches: [*] + pull_request: + # The branches below must be a subset of the branches above + branches: [*] + paths-ignore: + - '**/*.adoc' + - '**/*.bash' + - '**/*.md' + schedule: + # Full scan once a week + - cron: '0 14 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-18.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install ninja-build elfutils libzstd1-dev + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: cpp + queries: +security-and-quality + + - name: Build + run: ci/build + env: + RUN_TESTS: none + CMAKE_GENERATOR: Ninja + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1