diff --git a/include/fmt/core.h b/include/fmt/core.h
index 9589aa86..b2e4937e 100644
--- a/include/fmt/core.h
+++ b/include/fmt/core.h
@@ -1675,6 +1675,15 @@ template <typename Context> class basic_format_arg {
   auto is_arithmetic() const -> bool {
     return detail::is_arithmetic_type(type_);
   }
+
+  FMT_INLINE auto format_custom(const char_type* parse_begin,
+                                typename Context::parse_context_type& parse_ctx,
+                                Context& ctx) -> bool {
+    if (type_ != detail::type::custom_type) return false;
+    parse_ctx.advance_to(parse_begin);
+    value_.custom.format(value_.custom.value, parse_ctx, ctx);
+    return true;
+  }
 };
 
 /**
diff --git a/include/fmt/format.h b/include/fmt/format.h
index bc0d4762..3df1222e 100644
--- a/include/fmt/format.h
+++ b/include/fmt/format.h
@@ -3801,17 +3801,6 @@ template <typename Char> struct arg_formatter {
   }
 };
 
-template <typename Char> struct custom_formatter {
-  basic_format_parse_context<Char>& parse_ctx;
-  buffer_context<Char>& ctx;
-
-  void operator()(
-      typename basic_format_arg<buffer_context<Char>>::handle h) const {
-    h.format(parse_ctx, ctx);
-  }
-  template <typename T> void operator()(T) const {}
-};
-
 struct width_checker {
   template <typename T, FMT_ENABLE_IF(is_integer<T>::value)>
   FMT_CONSTEXPR auto operator()(T value) -> unsigned long long {
@@ -4408,11 +4397,9 @@ void vformat_to(buffer<Char>& buf, basic_string_view<Char> fmt,
     auto on_format_specs(int id, const Char* begin, const Char* end)
         -> const Char* {
       auto arg = get_arg(context, id);
-      if (arg.type() == type::custom_type) {
-        parse_context.advance_to(begin);
-        visit_format_arg(custom_formatter<Char>{parse_context, context}, arg);
+      // Not using a visitor for custom types gives better codegen.
+      if (arg.format_custom(begin, parse_context, context))
         return parse_context.begin();
-      }
       auto specs = detail::dynamic_format_specs<Char>();
       begin = parse_format_specs(begin, end, specs, parse_context, arg.type());
       detail::handle_dynamic_spec<detail::width_checker>(
@@ -4529,7 +4516,7 @@ formatter<T, Char,
   }
   auto specs = specs_;
   detail::handle_dynamic_spec<detail::width_checker>(specs.width,
-                                                      specs.width_ref, ctx);
+                                                     specs.width_ref, ctx);
   detail::handle_dynamic_spec<detail::precision_checker>(
       specs.precision, specs.precision_ref, ctx);
   return detail::write<Char>(ctx.out(), val, specs, ctx.locale());