Improve Unicode handling when writing to an ostream on Windows (#2994)

* Refactor detail::print() by splitting into two functions.

The part with SetConsoleW is a separate function, without fwrite().

* Make Unicode handing when writing to std::ostream more robust.

Calls to print(ostream&) in the special Unicode case on Windows fallback
to writing via ostream::write instead of fwrite().

* Fix Unicode handling when writing to an ostream on GCC on Windows

* Add TODO note about detail::is_utf8()

* Fix warning -Wundef

* Fix for non-Windows OSs

* Fix building as DLL on Windows

* Refactor

* Suppress warning

include/fmt/format-inl.h

@@ -1473,17 +1473,13 @@ FMT_FUNC std::string vformat(string_view fmt, format_args args) {
   return to_string(buffer);
 }
 
-#ifdef _WIN32
 namespace detail {
+#ifdef _WIN32
 using dword = conditional_t<sizeof(long) == 4, unsigned long, unsigned>;
 extern "C" __declspec(dllimport) int __stdcall WriteConsoleW(  //
     void*, const void*, dword, dword*, void*);
-}  // namespace detail
-#endif
 
-namespace detail {
-FMT_FUNC void print(std::FILE* f, string_view text) {
-#ifdef _WIN32
+FMT_FUNC bool write_console(std::FILE* f, string_view text) {
   auto fd = _fileno(f);
   if (_isatty(fd)) {
     detail::utf8_to_utf16 u16(string_view(text.data(), text.size()));
@@ -1491,11 +1487,20 @@ FMT_FUNC void print(std::FILE* f, string_view text) {
     if (detail::WriteConsoleW(reinterpret_cast<void*>(_get_osfhandle(fd)),
                               u16.c_str(), static_cast<uint32_t>(u16.size()),
                               &written, nullptr)) {
-      return;
+      return true;
     }
-    // Fallback to fwrite on failure. It can happen if the output has been
-    // redirected to NUL.
   }
+  // We return false if the file descriptor was not TTY, or it was but
+  // SetConsoleW failed which can happen if the output has been redirected to
+  // NUL. In both cases when we return false, we should attempt to do regular
+  // write via fwrite or std::ostream::write.
+  return false;
+}
+#endif
+
+FMT_FUNC void print(std::FILE* f, string_view text) {
+#ifdef _WIN32
+  if (write_console(f, text)) return;
 #endif
   detail::fwrite_fully(text.data(), 1, text.size(), f);
 }

include/fmt/format.h

@@ -921,6 +921,9 @@ struct is_contiguous<basic_memory_buffer<T, SIZE, Allocator>> : std::true_type {
 };
 
 namespace detail {
+#ifdef _WIN32
+FMT_API bool write_console(std::FILE* f, string_view text);
+#endif
 FMT_API void print(std::FILE*, string_view);
 }
 

include/fmt/ostream.h

@@ -10,6 +10,10 @@
 
 #include <fstream>
 #include <ostream>
+#if defined(_WIN32) && defined(__GLIBCXX__)
+#  include <ext/stdio_filebuf.h>
+#  include <ext/stdio_sync_filebuf.h>
+#endif
 
 #include "format.h"
 
@@ -81,13 +85,28 @@ template class filebuf_access<filebuf_access_tag,
                               decltype(&filebuf_type::_Myfile),
                               &filebuf_type::_Myfile>;
 
-inline bool write(std::filebuf& buf, fmt::string_view data) {
-  FILE* f = get_file(buf);
-  if (!f) return false;
-  print(f, data);
-  return true;
+inline bool write_ostream_unicode(std::ostream& os, fmt::string_view data) {
+#if FMT_MSC_VERSION
+  if (auto* buf = dynamic_cast<std::filebuf*>(os.rdbuf()))
+    if (FILE* f = get_file(*buf)) return write_console(f, data);
+#elif defined(_WIN32) && defined(__GLIBCXX__)
+  auto* rdbuf = os.rdbuf();
+  FILE* c_file;
+  if (auto* fbuf = dynamic_cast<__gnu_cxx::stdio_sync_filebuf<char>*>(rdbuf))
+    c_file = fbuf->file();
+  else if (auto* fbuf = dynamic_cast<__gnu_cxx::stdio_filebuf<char>*>(rdbuf))
+    c_file = fbuf->file();
+  else
+    return false;
+  if (c_file) return write_console(c_file, data);
+#else
+  ignore_unused(os);
+  ignore_unused(data);
+#endif
+  return false;
 }
-inline bool write(std::wfilebuf&, fmt::basic_string_view<wchar_t>) {
+inline bool write_ostream_unicode(std::wostream&,
+                                  fmt::basic_string_view<wchar_t>) {
   return false;
 }
 
@@ -95,10 +114,7 @@ inline bool write(std::wfilebuf&, fmt::basic_string_view<wchar_t>) {
 // It is a separate function rather than a part of vprint to simplify testing.
 template <typename Char>
 void write_buffer(std::basic_ostream<Char>& os, buffer<Char>& buf) {
-  if (const_check(FMT_MSC_VERSION)) {
-    auto filebuf = dynamic_cast<std::basic_filebuf<Char>*>(os.rdbuf());
-    if (filebuf && write(*filebuf, {buf.data(), buf.size()})) return;
-  }
+  if (write_ostream_unicode(os, {buf.data(), buf.size()})) return;
   const Char* buf_data = buf.data();
   using unsigned_streamsize = std::make_unsigned<std::streamsize>::type;
   unsigned_streamsize size = buf.size();