From 0012917f6956780ab4bb2e8d9a4f637c7ae3a892 Mon Sep 17 00:00:00 2001
From: Victor Zverovich <victor.zverovich@gmail.com>
Date: Sun, 15 Dec 2019 07:27:09 -0800
Subject: [PATCH] Add a UTF-8 decoder

---
 include/fmt/format-inl.h | 57 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/include/fmt/format-inl.h b/include/fmt/format-inl.h
index 72b30466..25c2f904 100644
--- a/include/fmt/format-inl.h
+++ b/include/fmt/format-inl.h
@@ -1240,6 +1240,63 @@ template <> struct formatter<internal::bigint> {
   }
 };
 
+// A public domain branchless UTF-8 decoder:
+// https://github.com/skeeto/branchless-utf8
+/* Decode the next character, C, from BUF, reporting errors in E.
+ *
+ * Since this is a branchless decoder, four bytes will be read from the
+ * buffer regardless of the actual length of the next character. This
+ * means the buffer _must_ have at least three bytes of zero padding
+ * following the end of the data stream.
+ *
+ * Errors are reported in E, which will be non-zero if the parsed
+ * character was somehow invalid: invalid byte sequence, non-canonical
+ * encoding, or a surrogate half.
+ *
+ * The function returns a pointer to the next character. When an error
+ * occurs, this pointer will be a guess that depends on the particular
+ * error, but it will always advance at least one byte.
+ */
+static void* utf8_decode(void* buf, uint32_t* c, int* e) {
+  static const char lengths[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+                                 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+                                 0, 0, 2, 2, 2, 2, 3, 3, 4, 0};
+  static const int masks[] = {0x00, 0x7f, 0x1f, 0x0f, 0x07};
+  static const uint32_t mins[] = {4194304, 0, 128, 2048, 65536};
+  static const int shiftc[] = {0, 18, 12, 6, 0};
+  static const int shifte[] = {0, 6, 4, 2, 0};
+
+  auto s = reinterpret_cast<unsigned char*>(buf);
+  int len = lengths[s[0] >> 3];
+
+  /* Compute the pointer to the next character early so that the next
+   * iteration can start working on the next character. Neither Clang
+   * nor GCC figure out this reordering on their own.
+   */
+  unsigned char* next = s + len + !len;
+
+  /* Assume a four-byte character and load four bytes. Unused bits are
+   * shifted out.
+   */
+  *c = (uint32_t)(s[0] & masks[len]) << 18;
+  *c |= (uint32_t)(s[1] & 0x3f) << 12;
+  *c |= (uint32_t)(s[2] & 0x3f) << 6;
+  *c |= (uint32_t)(s[3] & 0x3f) << 0;
+  *c >>= shiftc[len];
+
+  /* Accumulate the various error conditions. */
+  *e = (*c < mins[len]) << 6;       // non-canonical encoding
+  *e |= ((*c >> 11) == 0x1b) << 7;  // surrogate half?
+  *e |= (*c > 0x10FFFF) << 8;       // out of range?
+  *e |= (s[1] & 0xc0) >> 2;
+  *e |= (s[2] & 0xc0) >> 4;
+  *e |= (s[3]) >> 6;
+  *e ^= 0x2a;  // top two bits of each tail byte correct?
+  *e >>= shifte[len];
+
+  return next;
+}
+
 #if FMT_USE_WINDOWS_H
 
 FMT_FUNC internal::utf8_to_utf16::utf8_to_utf16(string_view s) {