From c50f94ac04c15b6415e1709812daf2609fc088e4 Mon Sep 17 00:00:00 2001 From: Victor Zverovich Date: Fri, 7 Dec 2012 13:20:50 -0800 Subject: [PATCH] Separate optimized and debug bloat tests. Add Benchmark section to the README. --- CMakeLists.txt | 19 ++++++++- README.rst | 102 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 944d1b3f..3ac59fa7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,18 +46,33 @@ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tinyformat/tinyformat_test.cpp) COMMAND @time -p ./tinyformat_speed_test boost > /dev/null DEPENDS tinyformat_speed_test) - add_custom_target(bloat_test + add_custom_target(bloat_test_debug COMMAND echo running bloat tests... COMMAND echo printf results: COMMAND ./bloat_test.sh COMMAND echo iostreams results: COMMAND ./bloat_test.sh -DUSE_IOSTREAMS COMMAND echo format results: - COMMAND ./bloat_test.sh -DUSE_FORMAT -L.. -lformat + COMMAND ./bloat_test.sh -DUSE_FORMAT COMMAND echo tinyformat results: COMMAND ./bloat_test.sh -DUSE_TINYFORMAT COMMAND echo boost results: COMMAND ./bloat_test.sh -DUSE_BOOST WORKING_DIRECTORY tinyformat DEPENDS format) + + add_custom_target(bloat_test_optimized + COMMAND echo running bloat tests... + COMMAND echo printf results: + COMMAND ./bloat_test.sh -O3 + COMMAND echo iostreams results: + COMMAND ./bloat_test.sh -O3 -DUSE_IOSTREAMS + COMMAND echo format results: + COMMAND ./bloat_test.sh -O3 -DUSE_FORMAT + COMMAND echo tinyformat results: + COMMAND ./bloat_test.sh -O3 -DUSE_TINYFORMAT + COMMAND echo boost results: + COMMAND ./bloat_test.sh -O3 -DUSE_BOOST + WORKING_DIRECTORY tinyformat + DEPENDS format) endif () diff --git a/README.rst b/README.rst index 691b4cdb..f1b04b8f 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,104 @@ format ====== -Small, safe and fast printf-like formatting library for C++ \ No newline at end of file +Small, safe and fast printf-like formatting library for C++ + +Benchmarks +---------- + +Compile time and code bloat +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The script ``bloat_test.sh`` from the `tinyformat +`__ repository tests compile time and +code bloat for nontrivial projects. It generates 100 translation units +and uses ``printf()`` or its alternative five times in each to simulate +a medium sized project. The resulting executable size and compile time +(g++-4.7.2, Ubuntu GNU/Linux 12.10, best of three) is shown in the following +tables. + +**Non-optimized build** + +====================== ================== ========================== +test name total compile time executable size (stripped) +====================== ================== ========================== +libc printf 2.8s 44K (32K) +std::ostream 12.9s 84K (60K) +format 16.0s 152K (128K) +tinyformat 20.6s 240K (200K) +boost::format 76.0s 888K (780K) +====================== ================== ========================== + +**Optimized build (-O3)** + +====================== ================== ========================== +test name total compile time executable size (stripped) +====================== ================== ========================== +libc printf 3.5s 40K (28K) +std::ostream 14.1s 88K (64K) +format 25.1s 552K (536K) +tinyformat 56.3s 200K (164K) +boost::format 169.4s 1.7M (1.6M) +====================== ================== ========================== + +Printf and std::ostream win here which is not surprising considering +that they are included in the standard library. Tinyformat has somewhat +slower compilation times compared to format. Interestingly optimized +executable size is smaller with tinyformat then with format and for +non-optimized build its vice versa. Boost::format has by far the +largest overheads. + +Speed tests +~~~~~~~~~~~ + +The following speed tests results were generated by building +``tinyformat_test.cpp`` on Ubuntu GNU/Linux 12.10 with +``g++-4.7.2 -O3 -DSPEED_TEST -DHAVE_FORMAT``, and taking the best of three +runs. In the test, the format string ``"%0.10f:%04d:%+g:%s:%p:%c:%%\n"`` or +equivalent is filled 2000000 times with output sent to ``/dev/null``; for +further details see the `source +`__. + +============== ======== +test name run time +============== ======== +libc printf 1.26s +std::ostream 2.02s +format 2.20s +tinyformat 2.51s +boost::format 10.40s +============== ======== + +As you can see boost::format is much slower than the alternative methods; this +is confirmed by `other tests `__. +Tinyformat is quite good coming close to iostreams. Unfortunately tinyformat +cannot be faster than the iostreams because it uses them internally. + +Running the tests +~~~~~~~~~~~~~~~~~ + +To run the tests you first need to get the format repository with submodules:: + + $ git clone --recursive git://github.com/vitaut/format.git + +Then go to the format directory and generate Makefiles with CMake:: + + $ cd format + $ cmake . + +Next use the following commands to run the speed test:: + + $ make speed_test + +or the bloat test:: + + $ make bloat_test + +Acknowledgments +--------------- + +The benchmark section of this readme file and the performance tests are taken +from the excellent `tinyformat `__ library +written by Chris Foster. boost::format is acknowledged transitively since +it had some influence on tinyformat. +TODO: SafeFormat, Clang DiagnosticHandler