diff --git a/gfx/drivers/omap_gfx.c b/gfx/drivers/omap_gfx.c
index 248f1f22ac..a97d30f198 100644
--- a/gfx/drivers/omap_gfx.c
+++ b/gfx/drivers/omap_gfx.c
@@ -1102,13 +1102,11 @@ static bool omap_gfx_read_viewport(void *data, uint8_t *buffer, bool is_idle)
    return true;
 }
 
-static void omap_gfx_set_texture_frame(void *data,
-      const void *frame, bool rgb32,
+static void omap_gfx_set_texture_frame(void *data, const void *frame, bool rgb32,
       unsigned width, unsigned height, float alpha)
 {
    omap_video_t          *vid = (omap_video_t*)data;
-   enum scaler_pix_fmt format = rgb32 ? 
-      SCALER_FMT_ARGB8888 : SCALER_FMT_RGBA4444;
+   enum scaler_pix_fmt format = rgb32 ? SCALER_FMT_ARGB8888 : SCALER_FMT_RGBA4444;
 
    video_frame_scale(
          &vid->menu.scaler,
@@ -1121,8 +1119,6 @@ static void omap_gfx_set_texture_frame(void *data,
          width,
          height,
          width * (rgb32 ? sizeof(uint32_t) : sizeof(uint16_t)));
-   scaler_ctx_scale_direct(&vid->menu.scaler, vid->menu.frame,
-         frame);
 }
 
 static void omap_gfx_set_texture_enable(void *data, bool state, bool full_screen)
diff --git a/gfx/drivers/sdl_gfx.c b/gfx/drivers/sdl_gfx.c
index f90ae63bd4..fb4220752c 100644
--- a/gfx/drivers/sdl_gfx.c
+++ b/gfx/drivers/sdl_gfx.c
@@ -360,8 +360,6 @@ static bool sdl_gfx_frame(void *data, const void *frame, unsigned width,
          width,
          height,
          pitch);
-   scaler_ctx_scale_direct(&vid->scaler, vid->screen->pixels,
-         frame);
 
 #ifdef HAVE_MENU
    menu_driver_frame(video_info);
@@ -487,8 +485,6 @@ static void sdl_set_texture_frame(void *data, const void *frame, bool rgb32,
          height,
          width * (rgb32 ? sizeof(uint32_t) : sizeof(uint16_t))
          );
-   scaler_ctx_scale_direct(&vid->menu.scaler, vid->menu.frame->pixels,
-         frame);
 
    SDL_SetAlpha(vid->menu.frame, SDL_SRCALPHA, 255.0 * alpha);
 }
diff --git a/gfx/drivers_renderchain/gl1_renderchain.c b/gfx/drivers_renderchain/gl1_renderchain.c
index 857e3dfe11..3eb1f887c7 100644
--- a/gfx/drivers_renderchain/gl1_renderchain.c
+++ b/gfx/drivers_renderchain/gl1_renderchain.c
@@ -237,7 +237,6 @@ static void gl1_renderchain_copy_frame(
             width,
             height,
             pitch);
-      scaler_ctx_scale_direct(&gl->scaler, &gl->conv_buffer, frame); 
       data_buf = gl->conv_buffer;
    }
    else
diff --git a/gfx/drivers_renderchain/gl2_renderchain.c b/gfx/drivers_renderchain/gl2_renderchain.c
index ae07df5346..1cb856c4e8 100644
--- a/gfx/drivers_renderchain/gl2_renderchain.c
+++ b/gfx/drivers_renderchain/gl2_renderchain.c
@@ -1340,7 +1340,6 @@ static void gl2_renderchain_copy_frame(
                &gl->scaler,
                gl->conv_buffer,
                frame, width, height, pitch);
-         scaler_ctx_scale_direct(&gl->scaler, gl->conv_buffer, frame);
          glTexSubImage2D(GL_TEXTURE_2D,
                0, 0, 0, width, height, gl->texture_type,
                gl->texture_fmt, gl->conv_buffer);
@@ -1401,7 +1400,6 @@ static void gl2_renderchain_copy_frame(
                width,
                height,
                pitch);
-         scaler_ctx_scale_direct(&gl->scaler, &gl->conv_buffer, frame); 
          data_buf = gl->conv_buffer;
       }
       else
diff --git a/gfx/video_driver.c b/gfx/video_driver.c
index 2a749e4fed..e953666ebd 100644
--- a/gfx/video_driver.c
+++ b/gfx/video_driver.c
@@ -2356,20 +2356,14 @@ void video_driver_frame(const void *data, unsigned width,
          (video_driver_pix_fmt == RETRO_PIXEL_FORMAT_0RGB1555) &&
          (data != RETRO_HW_FRAME_BUFFER_VALID))
    {
-      struct scaler_ctx *scaler = video_driver_scaler_ptr->scaler;
-      void *output              = video_driver_scaler_ptr->scaler_out;
-
-      scaler->in_width          = width;
-      scaler->in_height         = height;
-      scaler->out_width         = width;
-      scaler->out_height        = height;
-      scaler->in_stride         = (int)pitch;
-      scaler->out_stride        = width * sizeof(uint16_t);
-
-      scaler_ctx_scale_direct(scaler, output, data);
-
-      data                      = output;
-      pitch                     = scaler->out_stride;
+      if (video_pixel_frame_scale(
+               video_driver_scaler_ptr->scaler,
+               video_driver_scaler_ptr->scaler_out,
+               data, width, height, pitch))
+      {
+         data                = video_driver_scaler_ptr->scaler_out;
+         pitch               = video_driver_scaler_ptr->scaler->out_stride;
+      }
    }
 
 
@@ -2409,16 +2403,14 @@ void video_driver_frame(const void *data, unsigned width,
                      " ||  Frames: %" PRIu64,
                      (uint64_t)video_driver_frame_count);
             }
-            snprintf(video_driver_window_title,
-                  sizeof(video_driver_window_title),
+            snprintf(video_driver_window_title, sizeof(video_driver_window_title),
                "%s%s%s", title, video_info.fps_text,
                video_info.framecount_show ? frames_text : "");
          }
          else
          {
             if (!string_is_equal(video_driver_window_title, title))
-               strlcpy(video_driver_window_title,
-                     title, sizeof(video_driver_window_title));
+               strlcpy(video_driver_window_title, title, sizeof(video_driver_window_title));
          }
 
          curr_time = new_time;
@@ -2476,8 +2468,7 @@ void video_driver_frame(const void *data, unsigned width,
           || video_driver_record_gpu_buffer
          ) && recording_data
       )
-      recording_dump_frame(data, width, height,
-            pitch, video_info.runloop_is_idle);
+      recording_dump_frame(data, width, height, pitch, video_info.runloop_is_idle);
 
    if (data && video_driver_state_filter &&
          video_driver_frame_filter(data, &video_info, width, height, pitch,
diff --git a/libretro-common/include/gfx/video_frame.h b/libretro-common/include/gfx/video_frame.h
index 8389ea83f6..e5c0540d9c 100644
--- a/libretro-common/include/gfx/video_frame.h
+++ b/libretro-common/include/gfx/video_frame.h
@@ -33,18 +33,15 @@
 
 RETRO_BEGIN_DECLS
 
-static INLINE void scaler_ctx_scale_direct(
-      struct scaler_ctx *scaler,
-      void *output,
-      const void *input)
-{
-   /* Just perform straight pixel conversion if possible. */
-   if (scaler && scaler->unscaled && scaler->direct_pixconv)
-      scaler->direct_pixconv(output, input,
-            scaler->out_width,  scaler->out_height,
-            scaler->out_stride, scaler->in_stride);
-   else
-      scaler_ctx_scale(scaler, output, input);
+#define scaler_ctx_scale_direct(ctx, output, input) \
+{ \
+   if (ctx && ctx->unscaled && ctx->direct_pixconv) \
+      /* Just perform straight pixel conversion. */ \
+      ctx->direct_pixconv(output, input, \
+            ctx->out_width,  ctx->out_height, \
+            ctx->out_stride, ctx->in_stride); \
+   else \
+      scaler_ctx_scale(ctx, output, input); \
 }
 
 static INLINE void video_frame_convert_rgb16_to_rgb32(
@@ -68,6 +65,8 @@ static INLINE void video_frame_convert_rgb16_to_rgb32(
 
    scaler->in_stride  = in_pitch;
    scaler->out_stride = width * sizeof(uint32_t);
+
+   scaler_ctx_scale_direct(scaler, output, input);
 }
 
 static INLINE void video_frame_scale(
@@ -89,10 +88,10 @@ static INLINE void video_frame_scale(
          || pitch  != (unsigned)scaler->in_stride
       )
    {
-      scaler->in_fmt     = format;
-      scaler->in_width   = width;
-      scaler->in_height  = height;
-      scaler->in_stride  = pitch;
+      scaler->in_fmt    = format;
+      scaler->in_width  = width;
+      scaler->in_height = height;
+      scaler->in_stride = pitch;
 
       scaler->out_width  = scaler_width;
       scaler->out_height = scaler_height;
@@ -100,6 +99,8 @@ static INLINE void video_frame_scale(
 
       scaler_ctx_gen_filter(scaler);
    }
+
+   scaler_ctx_scale_direct(scaler, output, input);
 }
 
 static INLINE void video_frame_record_scale(
@@ -132,6 +133,8 @@ static INLINE void video_frame_record_scale(
 
       scaler_ctx_gen_filter(scaler);
    }
+
+   scaler_ctx_scale_direct(scaler, output, input);
 }
 
 static INLINE void video_frame_convert_argb8888_to_abgr8888(
@@ -153,6 +156,8 @@ static INLINE void video_frame_convert_argb8888_to_abgr8888(
 
    scaler->in_stride  = in_pitch;
    scaler->out_stride = width * sizeof(uint32_t);
+
+   scaler_ctx_scale_direct(scaler, output, input);
 }
 
 static INLINE void video_frame_convert_to_bgr24(
@@ -171,6 +176,8 @@ static INLINE void video_frame_convert_to_bgr24(
 
    scaler->in_stride   = in_pitch;
    scaler->out_stride  = width * 3;
+
+   scaler_ctx_scale_direct(scaler, output, input);
 }
 
 static INLINE void video_frame_convert_rgba_to_bgr(
@@ -190,6 +197,24 @@ static INLINE void video_frame_convert_rgba_to_bgr(
    }
 }
 
+static INLINE bool video_pixel_frame_scale(
+      struct scaler_ctx *scaler,
+      void *output, const void *data,
+      unsigned width, unsigned height,
+      size_t pitch)
+{
+   scaler->in_width      = width;
+   scaler->in_height     = height;
+   scaler->out_width     = width;
+   scaler->out_height    = height;
+   scaler->in_stride     = (int)pitch;
+   scaler->out_stride    = width * sizeof(uint16_t);
+
+   scaler_ctx_scale_direct(scaler, output, data);
+
+   return true;
+}
+
 RETRO_END_DECLS
 
 #endif
diff --git a/record/drivers/record_ffmpeg.c b/record/drivers/record_ffmpeg.c
index 67b5af1b2b..9763ceee7b 100644
--- a/record/drivers/record_ffmpeg.c
+++ b/record/drivers/record_ffmpeg.c
@@ -1007,10 +1007,6 @@ static void ffmpeg_scale_input(ffmpeg_t *handle,
             vid->height,
             vid->pitch,
             shrunk);
-      scaler_ctx_scale_direct(
-            &handle->video.scaler,
-            handle->video.conv_frame->data[0],
-            vid->data);
    }
 }