Eliminate extra copy on floating-point formatting

include/fmt/format-inl.h

@@ -559,131 +559,13 @@ FMT_FUNC int grisu2_gen_digits(char* buf, uint32_t hi, uint64_t lo, int& exp,
 #  define FMT_FALLTHROUGH
 #endif
 
-struct gen_digits_params {
-  int num_digits;
-  bool fixed;
-  bool upper;
-  bool trailing_zeros;
-};
-
-struct prettify_handler {
-  char* data;
-  ptrdiff_t size;
-  buffer& buf;
-
-  explicit prettify_handler(buffer& b, ptrdiff_t n)
-      : data(b.data()), size(n), buf(b) {}
-  ~prettify_handler() {
-    assert(size <= inline_buffer_size);
-    buf.resize(to_unsigned(size));
-  }
-
-  template <typename F> void insert(ptrdiff_t pos, ptrdiff_t n, F f) {
-    std::memmove(data + pos + n, data + pos, to_unsigned(size - pos));
-    f(data + pos);
-    size += n;
-  }
-
-  void insert(ptrdiff_t pos, char c) {
-    std::memmove(data + pos + 1, data + pos, to_unsigned(size - pos));
-    data[pos] = c;
-    ++size;
-  }
-
-  void append(ptrdiff_t n, char c) {
-    std::uninitialized_fill_n(data + size, n, c);
-    size += n;
-  }
-
-  void append(char c) { data[size++] = c; }
-
-  void remove_trailing(char c) {
-    while (data[size - 1] == c) --size;
-  }
-};
-
-// Writes the exponent exp in the form "[+-]d{2,3}" to buffer.
-template <typename Handler> FMT_FUNC void write_exponent(int exp, Handler&& h) {
-  FMT_ASSERT(-1000 < exp && exp < 1000, "exponent out of range");
-  if (exp < 0) {
-    h.append('-');
-    exp = -exp;
-  } else {
-    h.append('+');
-  }
-  if (exp >= 100) {
-    h.append(static_cast<char>('0' + exp / 100));
-    exp %= 100;
-    const char* d = data::DIGITS + exp * 2;
-    h.append(d[0]);
-    h.append(d[1]);
-  } else {
-    const char* d = data::DIGITS + exp * 2;
-    if (d[0] != '0') h.append(d[0]);
-    h.append(d[1]);
-  }
-}
-
-struct fill {
-  size_t n;
-  void operator()(char* buf) const {
-    buf[0] = '0';
-    buf[1] = '.';
-    std::uninitialized_fill_n(buf + 2, n, '0');
-  }
-};
-
-// The number is given as v = f * pow(10, exp), where f has size digits.
-template <typename Handler>
-FMT_FUNC void grisu2_prettify(int size, int exp, Handler&& handler) {
-  // pow(10, full_exp - 1) <= v <= pow(10, full_exp).
-  int full_exp = size + exp;
-  auto params = gen_digits_params();
-  params.fixed = (full_exp - 1) >= -4 && (full_exp - 1) <= 10;
-  if (!params.fixed) {
-    // Insert a decimal point after the first digit and add an exponent.
-    if (size > 1) handler.insert(1, '.');
-    exp += size - 1;
-    if (size < params.num_digits) handler.append(params.num_digits - size, '0');
-    handler.append(params.upper ? 'E' : 'e');
-    write_exponent(exp, handler);
-    return;
-  }
-  params.trailing_zeros = true;
-  const int exp_threshold = 21;
-  if (size <= full_exp && full_exp <= exp_threshold) {
-    // 1234e7 -> 12340000000[.0+]
-    handler.append(full_exp - size, '0');
-    int num_zeros = std::max(params.num_digits - full_exp, 1);
-    if (params.trailing_zeros) {
-      handler.append('.');
-      handler.append(num_zeros, '0');
-    }
-  } else if (full_exp > 0) {
-    // 1234e-2 -> 12.34[0+]
-    handler.insert(full_exp, '.');
-    if (!params.trailing_zeros) {
-      // Remove trailing zeros.
-      handler.remove_trailing('0');
-    } else if (params.num_digits > size) {
-      // Add trailing zeros.
-      ptrdiff_t num_zeros = params.num_digits - size;
-      handler.append(num_zeros, '0');
-    }
-  } else {
-    // 1234e-6 -> 0.001234
-    handler.insert(0, 2 - full_exp, fill{to_unsigned(-full_exp)});
-  }
-}
-
 template <typename Double>
 FMT_FUNC typename std::enable_if<sizeof(Double) == sizeof(uint64_t), bool>::type
-grisu2_format(Double value, buffer& buf, core_format_specs) {
+grisu2_format(Double value, buffer& buf, core_format_specs, int& exp) {
   FMT_ASSERT(value >= 0, "value is negative");
   if (value <= 0) {  // <= instead of == to silence a warning.
-    buf[0] = '0';
+    buf.push_back('0');
-    const int size = 1;
+    exp = 0;
-    grisu2_prettify(size, 0, prettify_handler(buf, size));
     return true;
   }
 
@@ -704,7 +586,7 @@ grisu2_format(Double value, buffer& buf, core_format_specs) {
   // hi (p1 in Grisu) contains the most significant digits of scaled upper.
   // hi = floor(upper / one).
   uint32_t hi = static_cast<uint32_t>(upper.f >> -one.e);
-  int exp = count_digits(hi);  // kappa in Grisu.
+  exp = count_digits(hi);  // kappa in Grisu.
   fp_value.normalize();
   fp scaled_value = fp_value * cached_pow;
   lower = lower * cached_pow;  // \tilde{M}^- in Grisu.
@@ -718,7 +600,8 @@ grisu2_format(Double value, buffer& buf, core_format_specs) {
   int size =
       grisu2_gen_digits(buf.data(), hi, lo, exp, delta, one, diff, max_digits);
   if (size < 0) return false;
-  grisu2_prettify(size, cached_exp + exp, prettify_handler(buf, size));
+  buf.resize(to_unsigned(size));
+  exp += cached_exp;
   return true;
 }
 

include/fmt/format.h

@@ -1135,13 +1135,96 @@ namespace internal {
 // https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf
 template <typename Double>
 FMT_API typename std::enable_if<sizeof(Double) == sizeof(uint64_t), bool>::type
-grisu2_format(Double value, buffer& buf, core_format_specs);
+grisu2_format(Double value, buffer& buf, core_format_specs, int& exp);
 template <typename Double>
 inline typename std::enable_if<sizeof(Double) != sizeof(uint64_t), bool>::type
-grisu2_format(Double, buffer&, core_format_specs) {
+grisu2_format(Double, buffer&, core_format_specs, int&) {
   return false;
 }
 
+struct gen_digits_params {
+  int num_digits;
+  bool fixed;
+  bool upper;
+  bool trailing_zeros;
+};
+
+// Writes the exponent exp in the form "[+-]d{2,3}" to buffer.
+template <typename Char, typename It> It write_exponent(int exp, It it) {
+  FMT_ASSERT(-1000 < exp && exp < 1000, "exponent out of range");
+  if (exp < 0) {
+    *it++ = static_cast<Char>('-');
+    exp = -exp;
+  } else {
+    *it++ = static_cast<Char>('+');
+  }
+  if (exp >= 100) {
+    *it++ = static_cast<Char>(static_cast<char>('0' + exp / 100));
+    exp %= 100;
+    const char* d = data::DIGITS + exp * 2;
+    *it++ = static_cast<Char>(d[0]);
+    *it++ = static_cast<Char>(d[1]);
+  } else {
+    const char* d = data::DIGITS + exp * 2;
+    if (d[0] != '0') *it++ = static_cast<Char>(d[0]);
+    *it++ = static_cast<Char>(d[1]);
+  }
+  return it;
+}
+
+// The number is given as v = digits * pow(10, exp).
+template <typename Char, typename It>
+It grisu2_prettify(const char* digits, int size, int exp, It it) {
+  // pow(10, full_exp - 1) <= v <= pow(10, full_exp).
+  int full_exp = size + exp;
+  auto params = gen_digits_params();
+  params.fixed = (full_exp - 1) >= -4 && (full_exp - 1) <= 10;
+  if (!params.fixed) {
+    // Insert a decimal point after the first digit and add an exponent.
+    *it++ = static_cast<Char>(*digits);
+    if (size > 1) *it++ = static_cast<Char>('.');
+    exp += size - 1;
+    it = copy_str<Char>(digits + 1, digits + size, it);
+    if (size < params.num_digits)
+      it = std::fill_n(it, params.num_digits - size, static_cast<Char>('0'));
+    *it++ = static_cast<Char>(params.upper ? 'E' : 'e');
+    return write_exponent<Char>(exp, it);
+  }
+  params.trailing_zeros = true;
+  const int exp_threshold = 21;
+  if (size <= full_exp && full_exp <= exp_threshold) {
+    // 1234e7 -> 12340000000[.0+]
+    it = copy_str<Char>(digits, digits + size, it);
+    it = std::fill_n(it, full_exp - size, static_cast<Char>('0'));
+    int num_zeros = (std::max)(params.num_digits - full_exp, 1);
+    if (params.trailing_zeros) {
+      *it++ = static_cast<Char>('.');
+      it = std::fill_n(it, num_zeros, static_cast<Char>('0'));
+    }
+  } else if (full_exp > 0) {
+    // 1234e-2 -> 12.34[0+]
+    it = copy_str<Char>(digits, digits + full_exp, it);
+    *it++ = static_cast<Char>('.');
+    it = copy_str<Char>(digits + full_exp, digits + size, it);
+    if (!params.trailing_zeros) {
+      // Remove trailing zeros.
+      // TODO
+      // handler.remove_trailing('0');
+    } else if (params.num_digits > size) {
+      // Add trailing zeros.
+      int num_zeros = params.num_digits - size;
+      it = std::fill_n(it, num_zeros, static_cast<Char>('0'));
+    }
+  } else {
+    // 1234e-6 -> 0.001234
+    *it++ = static_cast<Char>('0');
+    *it++ = static_cast<Char>('.');
+    it = std::fill_n(it, -full_exp, static_cast<Char>('0'));
+    it = copy_str<Char>(digits, digits + size, it);
+  }
+  return it;
+}
+
 template <typename Double>
 void sprintf_format(Double, internal::buffer&, core_format_specs);
 
@@ -2550,7 +2633,6 @@ template <typename Range> class basic_writer {
   };
 
   struct double_writer {
-    size_t n;
     char sign;
     internal::buffer& buffer;
 
@@ -2558,14 +2640,38 @@ template <typename Range> class basic_writer {
     size_t width() const { return size(); }
 
     template <typename It> void operator()(It&& it) {
-      if (sign) {
+      if (sign) *it++ = static_cast<char_type>(sign);
-        *it++ = static_cast<char_type>(sign);
-        --n;
-      }
       it = internal::copy_str<char_type>(buffer.begin(), buffer.end(), it);
     }
   };
 
+  class grisu_writer {
+   private:
+    internal::buffer& digits_;
+    size_t size_;
+    char sign_;
+    int exp_;
+
+   public:
+    grisu_writer(char sign, internal::buffer& digits, int exp)
+        : digits_(digits), sign_(sign), exp_(exp) {
+      int num_digits = static_cast<int>(digits.size());
+      auto it = internal::grisu2_prettify<char>(
+          digits.data(), num_digits, exp, internal::counting_iterator<char>());
+      size_ = it.count();
+    }
+
+    size_t size() const { return size_ + (sign_ ? 1 : 0); }
+    size_t width() const { return size(); }
+
+    template <typename It> void operator()(It&& it) {
+      if (sign_) *it++ = static_cast<char_type>(sign_);
+      int num_digits = static_cast<int>(digits_.size());
+      it = internal::grisu2_prettify<char_type>(digits_.data(), num_digits,
+                                                exp_, it);
+    }
+  };
+
   // Formats a floating-point number (double or long double).
   template <typename T> void write_double(T value, const format_specs& spec);
 
@@ -2731,11 +2837,11 @@ void basic_writer<Range>::write_double(T value, const format_specs& spec) {
     return write_inf_or_nan(handler.upper ? "INF" : "inf");
 
   memory_buffer buffer;
+  int exp = 0;
   bool use_grisu =
       fmt::internal::use_grisu<T>() && !spec.type && !spec.has_precision() &&
-      internal::grisu2_format(static_cast<double>(value), buffer, spec);
+      internal::grisu2_format(static_cast<double>(value), buffer, spec, exp);
   if (!use_grisu) internal::sprintf_format(value, buffer, spec);
-  size_t n = buffer.size();
   align_spec as = spec;
   if (spec.align() == ALIGN_NUMERIC) {
     if (sign) {
@@ -2745,11 +2851,13 @@ void basic_writer<Range>::write_double(T value, const format_specs& spec) {
       if (as.width_) --as.width_;
     }
     as.align_ = ALIGN_RIGHT;
-  } else {
+  } else if (spec.align() == ALIGN_DEFAULT) {
-    if (spec.align() == ALIGN_DEFAULT) as.align_ = ALIGN_RIGHT;
+    as.align_ = ALIGN_RIGHT;
-    if (sign) ++n;
   }
-  write_padded(as, double_writer{n, sign, buffer});
+  if (use_grisu)
+    write_padded(as, grisu_writer{sign, buffer, exp});
+  else
+    write_padded(as, double_writer{sign, buffer});
 }
 
 // Reports a system error without throwing an exception.

src/format.cc

@@ -11,8 +11,8 @@ FMT_BEGIN_NAMESPACE
 template struct internal::basic_data<void>;
 
 // Workaround a bug in MSVC2013 that prevents instantiation of grisu2_format.
-bool (*instantiate_grisu2_format)(double, internal::buffer&,
+bool (*instantiate_grisu2_format)(double, internal::buffer&, core_format_specs,
-                                  core_format_specs) = internal::grisu2_format;
+                                  int&) = internal::grisu2_format;
 
 #ifndef FMT_STATIC_THOUSANDS_SEPARATOR
 template FMT_API internal::locale_ref::locale_ref(const std::locale& loc);

test/format-impl-test.cc

@@ -102,7 +102,8 @@ TEST(FPTest, GetCachedPower) {
 
 TEST(FPTest, Grisu2FormatCompilesWithNonIEEEDouble) {
   fmt::memory_buffer buf;
-  grisu2_format(4.2f, buf, fmt::core_format_specs());
+  int exp = 0;
+  grisu2_format(4.2f, buf, fmt::core_format_specs(), exp);
 }
 
 template <typename T> struct ValueExtractor : fmt::internal::function<T> {

test/grisu-test.cc

@@ -51,5 +51,4 @@ TEST(GrisuTest, Prettify) {
   EXPECT_EQ("12340000000.0", fmt::format("{}", 1234e7));
   EXPECT_EQ("12.34", fmt::format("{}", 1234e-2));
   EXPECT_EQ("0.001234", fmt::format("{}", 1234e-6));
-
 }