diff --git a/Source/Core/VideoBackends/D3D/VideoBackend.h b/Source/Core/VideoBackends/D3D/VideoBackend.h
index 59491f735d..363c4b5068 100644
--- a/Source/Core/VideoBackends/D3D/VideoBackend.h
+++ b/Source/Core/VideoBackends/D3D/VideoBackend.h
@@ -16,6 +16,7 @@ class VideoBackend : public VideoBackendBase
 
   std::string GetName() const override;
   std::string GetDisplayName() const override;
+  std::string GetConfigName() const override;
 
   void Video_Prepare() override;
   void Video_Cleanup() override;
diff --git a/Source/Core/VideoBackends/D3D/main.cpp b/Source/Core/VideoBackends/D3D/main.cpp
index 8fd9989a92..791eda2a08 100644
--- a/Source/Core/VideoBackends/D3D/main.cpp
+++ b/Source/Core/VideoBackends/D3D/main.cpp
@@ -61,6 +61,11 @@ std::string VideoBackend::GetDisplayName() const
   return "Direct3D 11";
 }
 
+std::string VideoBackend::GetConfigName() const
+{
+  return "gfx_dx11";
+}
+
 void InitBackendInfo()
 {
   HRESULT hr = DX11::D3D::LoadDXGI();
@@ -141,7 +146,7 @@ void InitBackendInfo()
 void VideoBackend::ShowConfig(void* hParent)
 {
   InitBackendInfo();
-  Host_ShowVideoConfig(hParent, GetDisplayName(), "gfx_dx11");
+  Host_ShowVideoConfig(hParent, GetDisplayName(), GetConfigName());
 }
 
 bool VideoBackend::Initialize(void* window_handle)
@@ -149,23 +154,10 @@ bool VideoBackend::Initialize(void* window_handle)
   if (window_handle == nullptr)
     return false;
 
-  InitializeShared();
   InitBackendInfo();
-
-  frameCount = 0;
-
-  if (File::Exists(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini"))
-    g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini");
-  else
-    g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + "gfx_dx11.ini");
-
-  g_Config.GameIniLoad();
-  g_Config.UpdateProjectionHack();
-  g_Config.VerifyValidity();
-  UpdateActiveConfig();
+  InitializeShared();
 
   m_window_handle = window_handle;
-  m_initialized = true;
 
   return true;
 }
@@ -181,55 +173,27 @@ void VideoBackend::Video_Prepare()
   PixelShaderCache::Init();
   GeometryShaderCache::Init();
   D3D::InitUtils();
-
-  // VideoCommon
-  BPInit();
-  Fifo::Init();
-  IndexGenerator::Init();
-  VertexLoaderManager::Init();
-  OpcodeDecoder::Init();
-  VertexShaderManager::Init();
-  PixelShaderManager::Init();
-  GeometryShaderManager::Init();
-  CommandProcessor::Init();
-  PixelEngine::Init();
-  BBox::Init();
-
-  // Tell the host that the window is ready
-  Host_Message(WM_USER_CREATE);
 }
 
 void VideoBackend::Shutdown()
 {
-  m_initialized = false;
-
   // TODO: should be in Video_Cleanup
-  if (g_renderer)
-  {
-    // VideoCommon
-    Fifo::Shutdown();
-    CommandProcessor::Shutdown();
-    GeometryShaderManager::Shutdown();
-    PixelShaderManager::Shutdown();
-    VertexShaderManager::Shutdown();
-    OpcodeDecoder::Shutdown();
-    VertexLoaderManager::Shutdown();
+  D3D::ShutdownUtils();
+  PixelShaderCache::Shutdown();
+  VertexShaderCache::Shutdown();
+  GeometryShaderCache::Shutdown();
+  BBox::Shutdown();
 
-    // internal interfaces
-    D3D::ShutdownUtils();
-    PixelShaderCache::Shutdown();
-    VertexShaderCache::Shutdown();
-    GeometryShaderCache::Shutdown();
-    BBox::Shutdown();
+  g_perf_query.reset();
+  g_vertex_manager.reset();
+  g_texture_cache.reset();
+  g_renderer.reset();
 
-    g_perf_query.reset();
-    g_vertex_manager.reset();
-    g_texture_cache.reset();
-    g_renderer.reset();
-  }
+  ShutdownShared();
 }
 
 void VideoBackend::Video_Cleanup()
 {
+  CleanupShared();
 }
 }
diff --git a/Source/Core/VideoBackends/D3D12/VideoBackend.h b/Source/Core/VideoBackends/D3D12/VideoBackend.h
index 9fbf639238..2efb3fba69 100644
--- a/Source/Core/VideoBackends/D3D12/VideoBackend.h
+++ b/Source/Core/VideoBackends/D3D12/VideoBackend.h
@@ -16,6 +16,7 @@ class VideoBackend : public VideoBackendBase
 
   std::string GetName() const override;
   std::string GetDisplayName() const override;
+  std::string GetConfigName() const override;
 
   void Video_Prepare() override;
   void Video_Cleanup() override;
diff --git a/Source/Core/VideoBackends/D3D12/main.cpp b/Source/Core/VideoBackends/D3D12/main.cpp
index c0be65c7c4..5fc1658c9f 100644
--- a/Source/Core/VideoBackends/D3D12/main.cpp
+++ b/Source/Core/VideoBackends/D3D12/main.cpp
@@ -62,6 +62,11 @@ std::string VideoBackend::GetDisplayName() const
   return "Direct3D 12 (experimental)";
 }
 
+std::string VideoBackend::GetConfigName() const
+{
+  return "gfx_dx12";
+}
+
 void InitBackendInfo()
 {
   HRESULT hr = D3D::LoadDXGI();
@@ -155,7 +160,7 @@ void InitBackendInfo()
 void VideoBackend::ShowConfig(void* hParent)
 {
   InitBackendInfo();
-  Host_ShowVideoConfig(hParent, GetDisplayName(), "gfx_dx12");
+  Host_ShowVideoConfig(hParent, GetDisplayName(), GetConfigName());
 }
 
 bool VideoBackend::Initialize(void* window_handle)
@@ -163,26 +168,13 @@ bool VideoBackend::Initialize(void* window_handle)
   if (window_handle == nullptr)
     return false;
 
-  InitializeShared();
   InitBackendInfo();
-
-  frameCount = 0;
-
-  if (File::Exists(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini"))
-    g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini");
-  else
-    g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + "gfx_dx12.ini");
-
-  g_Config.GameIniLoad();
-  g_Config.UpdateProjectionHack();
-  g_Config.VerifyValidity();
-  UpdateActiveConfig();
+  InitializeShared();
 
   if (FAILED(D3D::Create((HWND)window_handle)))
     return false;
 
   m_window_handle = window_handle;
-  m_initialized = true;
 
   return true;
 }
@@ -200,62 +192,36 @@ void VideoBackend::Video_Prepare()
   StaticShaderCache::Init();
   StateCache::Init();  // PSO cache is populated here, after constituent shaders are loaded.
   D3D::InitUtils();
-
-  // VideoCommon
-  BPInit();
-  Fifo::Init();
-  IndexGenerator::Init();
-  VertexLoaderManager::Init();
-  OpcodeDecoder::Init();
-  VertexShaderManager::Init();
-  PixelShaderManager::Init();
-  GeometryShaderManager::Init();
-  CommandProcessor::Init();
-  PixelEngine::Init();
-  BBox::Init();
-
-  // Tell the host that the window is ready
-  Host_Message(WM_USER_CREATE);
 }
 
 void VideoBackend::Shutdown()
 {
-  m_initialized = true;
-
   // TODO: should be in Video_Cleanup
-  if (g_renderer)
-  {
-    // Immediately stop app from submitting work to GPU, and wait for all submitted work to
-    // complete. D3D12TODO: Check this.
-    D3D::command_list_mgr->ExecuteQueuedWork(true);
 
-    // VideoCommon
-    Fifo::Shutdown();
-    CommandProcessor::Shutdown();
-    GeometryShaderManager::Shutdown();
-    PixelShaderManager::Shutdown();
-    VertexShaderManager::Shutdown();
-    OpcodeDecoder::Shutdown();
-    VertexLoaderManager::Shutdown();
+  // Immediately stop app from submitting work to GPU, and wait for all submitted work to complete.
+  // D3D12TODO: Check this.
+  D3D::command_list_mgr->ExecuteQueuedWork(true);
 
-    // internal interfaces
-    D3D::ShutdownUtils();
-    ShaderCache::Shutdown();
-    ShaderConstantsManager::Shutdown();
-    StaticShaderCache::Shutdown();
-    BBox::Shutdown();
+  // internal interfaces
+  D3D::ShutdownUtils();
+  ShaderCache::Shutdown();
+  ShaderConstantsManager::Shutdown();
+  StaticShaderCache::Shutdown();
+  BBox::Shutdown();
 
-    g_xfb_encoder.reset();
-    g_perf_query.reset();
-    g_vertex_manager.reset();
-    g_texture_cache.reset();
-    g_renderer.reset();
+  g_xfb_encoder.reset();
+  g_perf_query.reset();
+  g_vertex_manager.reset();
+  g_texture_cache.reset();
+  g_renderer.reset();
 
-    D3D::Close();
-  }
+  D3D::Close();
+
+  ShutdownShared();
 }
 
 void VideoBackend::Video_Cleanup()
 {
+  CleanupShared();
 }
 }
diff --git a/Source/Core/VideoBackends/Null/NullBackend.cpp b/Source/Core/VideoBackends/Null/NullBackend.cpp
index 7a1691b9d5..2a1555ed27 100644
--- a/Source/Core/VideoBackends/Null/NullBackend.cpp
+++ b/Source/Core/VideoBackends/Null/NullBackend.cpp
@@ -53,7 +53,7 @@ static void InitBackendInfo()
 void VideoBackend::ShowConfig(void* parent)
 {
   InitBackendInfo();
-  Host_ShowVideoConfig(parent, GetDisplayName(), "gfx_null");
+  Host_ShowVideoConfig(parent, GetDisplayName(), GetConfigName());
 }
 
 bool VideoBackend::Initialize(void* window_handle)
@@ -61,28 +61,6 @@ bool VideoBackend::Initialize(void* window_handle)
   InitializeShared();
   InitBackendInfo();
 
-  // Load Configs
-  g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini");
-  g_Config.GameIniLoad();
-  g_Config.UpdateProjectionHack();
-  g_Config.VerifyValidity();
-  UpdateActiveConfig();
-
-  // Do our OSD callbacks
-  OSD::DoCallbacks(OSD::CallbackType::Initialization);
-
-  // Initialize VideoCommon
-  CommandProcessor::Init();
-  PixelEngine::Init();
-  BPInit();
-  Fifo::Init();
-  OpcodeDecoder::Init();
-  IndexGenerator::Init();
-  VertexShaderManager::Init();
-  PixelShaderManager::Init();
-  VertexLoaderManager::Init();
-  Host_Message(WM_USER_CREATE);
-
   return true;
 }
 
@@ -102,19 +80,12 @@ void VideoBackend::Video_Prepare()
 
 void VideoBackend::Shutdown()
 {
-  // Shutdown VideoCommon
-  Fifo::Shutdown();
-  VertexLoaderManager::Shutdown();
-  VertexShaderManager::Shutdown();
-  PixelShaderManager::Shutdown();
-  OpcodeDecoder::Shutdown();
-
-  // Do our OSD callbacks
-  OSD::DoCallbacks(OSD::CallbackType::Shutdown);
+  ShutdownShared();
 }
 
 void VideoBackend::Video_Cleanup()
 {
+  CleanupShared();
   PixelShaderCache::s_instance.reset();
   VertexShaderCache::s_instance.reset();
   GeometryShaderCache::s_instance.reset();
diff --git a/Source/Core/VideoBackends/Null/VideoBackend.h b/Source/Core/VideoBackends/Null/VideoBackend.h
index e83f4b8279..46b7e39fc4 100644
--- a/Source/Core/VideoBackends/Null/VideoBackend.h
+++ b/Source/Core/VideoBackends/Null/VideoBackend.h
@@ -15,6 +15,7 @@ class VideoBackend : public VideoBackendBase
 
   std::string GetName() const override { return "Null"; }
   std::string GetDisplayName() const override { return "Null"; }
+  std::string GetConfigName() const override { return "gfx_null"; }
   void Video_Prepare() override;
   void Video_Cleanup() override;
 
diff --git a/Source/Core/VideoBackends/OGL/Render.cpp b/Source/Core/VideoBackends/OGL/Render.cpp
index 8805a4bea8..bde616deac 100644
--- a/Source/Core/VideoBackends/OGL/Render.cpp
+++ b/Source/Core/VideoBackends/OGL/Render.cpp
@@ -39,6 +39,7 @@
 #include "VideoCommon/DriverDetails.h"
 #include "VideoCommon/Fifo.h"
 #include "VideoCommon/ImageWrite.h"
+#include "VideoCommon/IndexGenerator.h"
 #include "VideoCommon/OnScreenDisplay.h"
 #include "VideoCommon/PixelEngine.h"
 #include "VideoCommon/PixelShaderManager.h"
@@ -753,6 +754,8 @@ Renderer::Renderer()
       }
     }
   }
+  IndexGenerator::Init();
+
   UpdateActiveConfig();
   ClearEFBCache();
 }
diff --git a/Source/Core/VideoBackends/OGL/VideoBackend.h b/Source/Core/VideoBackends/OGL/VideoBackend.h
index 755ed9fa6f..b658615dd8 100644
--- a/Source/Core/VideoBackends/OGL/VideoBackend.h
+++ b/Source/Core/VideoBackends/OGL/VideoBackend.h
@@ -16,6 +16,7 @@ class VideoBackend : public VideoBackendBase
 
   std::string GetName() const override;
   std::string GetDisplayName() const override;
+  std::string GetConfigName() const override;
 
   void Video_Prepare() override;
   void Video_Cleanup() override;
diff --git a/Source/Core/VideoBackends/OGL/main.cpp b/Source/Core/VideoBackends/OGL/main.cpp
index 48b580ad2d..9f08f62601 100644
--- a/Source/Core/VideoBackends/OGL/main.cpp
+++ b/Source/Core/VideoBackends/OGL/main.cpp
@@ -91,6 +91,11 @@ std::string VideoBackend::GetDisplayName() const
     return "OpenGL";
 }
 
+std::string VideoBackend::GetConfigName() const
+{
+  return "gfx_opengl";
+}
+
 static std::vector<std::string> GetShaders(const std::string& sub_dir = "")
 {
   std::vector<std::string> paths =
@@ -116,6 +121,12 @@ static void InitBackendInfo()
   g_Config.backend_info.bSupportsPostProcessing = true;
   g_Config.backend_info.bSupportsSSAA = true;
 
+  // Overwritten in Render.cpp later
+  g_Config.backend_info.bSupportsDualSourceBlend = true;
+  g_Config.backend_info.bSupportsPrimitiveRestart = true;
+  g_Config.backend_info.bSupportsPaletteConversion = true;
+  g_Config.backend_info.bSupportsClipControl = true;
+
   g_Config.backend_info.Adapters.clear();
 
   // aamodes - 1 is to stay consistent with D3D (means no AA)
@@ -131,35 +142,19 @@ void VideoBackend::ShowConfig(void* parent_handle)
   if (!m_initialized)
     InitBackendInfo();
 
-  Host_ShowVideoConfig(parent_handle, GetDisplayName(), "gfx_opengl");
+  Host_ShowVideoConfig(parent_handle, GetDisplayName(), GetConfigName());
 }
 
 bool VideoBackend::Initialize(void* window_handle)
 {
-  InitializeShared();
   InitBackendInfo();
-
-  frameCount = 0;
-
-  if (File::Exists(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini"))
-    g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini");
-  else
-    g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + "gfx_opengl.ini");
-  g_Config.GameIniLoad();
-  g_Config.UpdateProjectionHack();
-  g_Config.VerifyValidity();
-  UpdateActiveConfig();
+  InitializeShared();
 
   InitInterface();
   GLInterface->SetMode(GLInterfaceMode::MODE_DETECT);
   if (!GLInterface->Create(window_handle))
     return false;
 
-  // Do our OSD callbacks
-  OSD::DoCallbacks(OSD::CallbackType::Initialization);
-
-  m_initialized = true;
-
   return true;
 }
 
@@ -171,65 +166,36 @@ void VideoBackend::Video_Prepare()
 
   g_renderer = std::make_unique<Renderer>();
 
-  CommandProcessor::Init();
-  PixelEngine::Init();
-
-  BPInit();
   g_vertex_manager = std::make_unique<VertexManager>();
   g_perf_query = GetPerfQuery();
-  Fifo::Init();  // must be done before OpcodeDecoder::Init()
-  OpcodeDecoder::Init();
-  IndexGenerator::Init();
-  VertexShaderManager::Init();
-  PixelShaderManager::Init();
-  GeometryShaderManager::Init();
   ProgramShaderCache::Init();
   g_texture_cache = std::make_unique<TextureCache>();
   g_sampler_cache = std::make_unique<SamplerCache>();
   Renderer::Init();
-  VertexLoaderManager::Init();
   TextureConverter::Init();
   BoundingBox::Init();
-
-  // Notify the core that the video backend is ready
-  Host_Message(WM_USER_CREATE);
 }
 
 void VideoBackend::Shutdown()
 {
-  m_initialized = false;
-
-  // Do our OSD callbacks
-  OSD::DoCallbacks(OSD::CallbackType::Shutdown);
-
   GLInterface->Shutdown();
   GLInterface.reset();
+  ShutdownShared();
 }
 
 void VideoBackend::Video_Cleanup()
 {
-  if (!g_renderer)
-    return;
-
-  Fifo::Shutdown();
-
   // The following calls are NOT Thread Safe
   // And need to be called from the video thread
+  CleanupShared();
   Renderer::Shutdown();
   BoundingBox::Shutdown();
   TextureConverter::Shutdown();
-  VertexLoaderManager::Shutdown();
   g_sampler_cache.reset();
   g_texture_cache.reset();
   ProgramShaderCache::Shutdown();
-  VertexShaderManager::Shutdown();
-  PixelShaderManager::Shutdown();
-  GeometryShaderManager::Shutdown();
-
   g_perf_query.reset();
   g_vertex_manager.reset();
-
-  OpcodeDecoder::Shutdown();
   g_renderer.reset();
   GLInterface->ClearCurrent();
 }
diff --git a/Source/Core/VideoBackends/Software/SWmain.cpp b/Source/Core/VideoBackends/Software/SWmain.cpp
index 083e9766e7..f37bc4aa14 100644
--- a/Source/Core/VideoBackends/Software/SWmain.cpp
+++ b/Source/Core/VideoBackends/Software/SWmain.cpp
@@ -134,6 +134,11 @@ std::string VideoSoftware::GetDisplayName() const
   return "Software Renderer";
 }
 
+std::string VideoSoftware::GetConfigName() const
+{
+  return "gfx_software";
+}
+
 static void InitBackendInfo()
 {
   g_Config.backend_info.APIType = API_NONE;
@@ -151,19 +156,13 @@ void VideoSoftware::ShowConfig(void* hParent)
 {
   if (!m_initialized)
     InitBackendInfo();
-  Host_ShowVideoConfig(hParent, GetDisplayName(), "gfx_software");
+  Host_ShowVideoConfig(hParent, GetDisplayName(), GetConfigName());
 }
 
 bool VideoSoftware::Initialize(void* window_handle)
 {
-  InitializeShared();
   InitBackendInfo();
-
-  g_Config.Load((File::GetUserPath(D_CONFIG_IDX) + "gfx_software.ini").c_str());
-  g_Config.GameIniLoad();
-  g_Config.UpdateProjectionHack();
-  g_Config.VerifyValidity();
-  UpdateActiveConfig();
+  InitializeShared();
 
   SWOGLWindow::Init(window_handle);
 
@@ -173,71 +172,41 @@ bool VideoSoftware::Initialize(void* window_handle)
   SWRenderer::Init();
   DebugUtil::Init();
 
-  // Do our OSD callbacks
-  OSD::DoCallbacks(OSD::CallbackType::Initialization);
-
-  m_initialized = true;
-
   return true;
 }
 
 void VideoSoftware::Shutdown()
 {
-  m_initialized = false;
-
-  // Do our OSD callbacks
-  OSD::DoCallbacks(OSD::CallbackType::Shutdown);
-
   SWOGLWindow::Shutdown();
+
+  ShutdownShared();
 }
 
 void VideoSoftware::Video_Cleanup()
 {
-  if (g_renderer)
-  {
-    Fifo::Shutdown();
-    SWRenderer::Shutdown();
-    DebugUtil::Shutdown();
-    // The following calls are NOT Thread Safe
-    // And need to be called from the video thread
-    SWRenderer::Shutdown();
-    VertexLoaderManager::Shutdown();
-    g_framebuffer_manager.reset();
-    g_texture_cache.reset();
-    VertexShaderManager::Shutdown();
-    PixelShaderManager::Shutdown();
-    g_perf_query.reset();
-    g_vertex_manager.reset();
-    OpcodeDecoder::Shutdown();
-    g_renderer.reset();
-  }
+  CleanupShared();
+
+  SWRenderer::Shutdown();
+  DebugUtil::Shutdown();
+  // The following calls are NOT Thread Safe
+  // And need to be called from the video thread
+  SWRenderer::Shutdown();
+  g_framebuffer_manager.reset();
+  g_texture_cache.reset();
+  g_perf_query.reset();
+  g_vertex_manager.reset();
+  g_renderer.reset();
 }
 
 // This is called after Video_Initialize() from the Core
 void VideoSoftware::Video_Prepare()
 {
   g_renderer = std::make_unique<SWRenderer>();
-
-  CommandProcessor::Init();
-  PixelEngine::Init();
-
-  BPInit();
   g_vertex_manager = std::make_unique<SWVertexLoader>();
   g_perf_query = std::make_unique<PerfQuery>();
-  Fifo::Init();  // must be done before OpcodeDecoder::Init()
-  OpcodeDecoder::Init();
-  IndexGenerator::Init();
-  VertexShaderManager::Init();
-  PixelShaderManager::Init();
   g_texture_cache = std::make_unique<TextureCache>();
   SWRenderer::Init();
-  VertexLoaderManager::Init();
   g_framebuffer_manager = std::make_unique<FramebufferManager>();
-
-  // Notify the core that the video backend is ready
-  Host_Message(WM_USER_CREATE);
-
-  INFO_LOG(VIDEO, "Video backend initialized.");
 }
 
 unsigned int VideoSoftware::PeekMessages()
diff --git a/Source/Core/VideoBackends/Software/VideoBackend.h b/Source/Core/VideoBackends/Software/VideoBackend.h
index efb057d968..121961c21c 100644
--- a/Source/Core/VideoBackends/Software/VideoBackend.h
+++ b/Source/Core/VideoBackends/Software/VideoBackend.h
@@ -21,6 +21,7 @@ class VideoSoftware : public VideoBackendBase
 
   std::string GetName() const override;
   std::string GetDisplayName() const override;
+  std::string GetConfigName() const override;
 
   void Video_Prepare() override;
   void Video_Cleanup() override;
diff --git a/Source/Core/VideoCommon/CommandProcessor.cpp b/Source/Core/VideoCommon/CommandProcessor.cpp
index fe53ffe8fe..cd471aae44 100644
--- a/Source/Core/VideoCommon/CommandProcessor.cpp
+++ b/Source/Core/VideoCommon/CommandProcessor.cpp
@@ -453,10 +453,6 @@ void SetCPStatusFromCPU()
   }
 }
 
-void Shutdown()
-{
-}
-
 void SetCpStatusRegister()
 {
   // Here always there is one fifo attached to the GPU
diff --git a/Source/Core/VideoCommon/CommandProcessor.h b/Source/Core/VideoCommon/CommandProcessor.h
index 372ed55c3f..043062630e 100644
--- a/Source/Core/VideoCommon/CommandProcessor.h
+++ b/Source/Core/VideoCommon/CommandProcessor.h
@@ -119,7 +119,6 @@ union UCPClearReg {
 
 // Init
 void Init();
-void Shutdown();
 void DoState(PointerWrap& p);
 
 void RegisterMMIO(MMIO::Mapping* mmio, u32 base);
diff --git a/Source/Core/VideoCommon/GeometryShaderManager.cpp b/Source/Core/VideoCommon/GeometryShaderManager.cpp
index 533bed8ecd..1bb841459d 100644
--- a/Source/Core/VideoCommon/GeometryShaderManager.cpp
+++ b/Source/Core/VideoCommon/GeometryShaderManager.cpp
@@ -30,10 +30,6 @@ void GeometryShaderManager::Init()
   dirty = true;
 }
 
-void GeometryShaderManager::Shutdown()
-{
-}
-
 void GeometryShaderManager::Dirty()
 {
   // This function is called after a savestate is loaded.
diff --git a/Source/Core/VideoCommon/GeometryShaderManager.h b/Source/Core/VideoCommon/GeometryShaderManager.h
index 878c6d72bd..55f3b8ad43 100644
--- a/Source/Core/VideoCommon/GeometryShaderManager.h
+++ b/Source/Core/VideoCommon/GeometryShaderManager.h
@@ -15,7 +15,6 @@ class GeometryShaderManager
 public:
   static void Init();
   static void Dirty();
-  static void Shutdown();
   static void DoState(PointerWrap& p);
 
   static void SetConstants();
diff --git a/Source/Core/VideoCommon/MainBase.cpp b/Source/Core/VideoCommon/MainBase.cpp
index 6b2e3c91c3..a084e0cc09 100644
--- a/Source/Core/VideoCommon/MainBase.cpp
+++ b/Source/Core/VideoCommon/MainBase.cpp
@@ -9,12 +9,22 @@
 #include "Common/Event.h"
 #include "Common/Flag.h"
 #include "Common/Logging/Log.h"
+#include "Core/Host.h"
 #include "VideoCommon/AsyncRequests.h"
 #include "VideoCommon/BPStructs.h"
+#include "VideoCommon/CPMemory.h"
+#include "VideoCommon/CommandProcessor.h"
 #include "VideoCommon/Fifo.h"
+#include "VideoCommon/GeometryShaderManager.h"
+#include "VideoCommon/IndexGenerator.h"
+#include "VideoCommon/OnScreenDisplay.h"
+#include "VideoCommon/OpcodeDecoding.h"
+#include "VideoCommon/PixelEngine.h"
+#include "VideoCommon/PixelShaderManager.h"
 #include "VideoCommon/RenderBase.h"
 #include "VideoCommon/TextureCacheBase.h"
 #include "VideoCommon/VertexLoaderManager.h"
+#include "VideoCommon/VertexShaderManager.h"
 #include "VideoCommon/VideoBackendBase.h"
 #include "VideoCommon/VideoConfig.h"
 #include "VideoCommon/VideoState.h"
@@ -149,11 +159,58 @@ u16 VideoBackendBase::Video_GetBoundingBox(int index)
 
 void VideoBackendBase::InitializeShared()
 {
-  VideoCommon_Init();
+  memset(&g_main_cp_state, 0, sizeof(g_main_cp_state));
+  memset(&g_preprocess_cp_state, 0, sizeof(g_preprocess_cp_state));
+  memset(texMem, 0, TMEM_SIZE);
+
+  // Do our OSD callbacks
+  OSD::DoCallbacks(OSD::CallbackType::Initialization);
+
+  // do not initialize again for the config window
+  m_initialized = true;
 
   s_FifoShuttingDown.Clear();
   memset((void*)&s_beginFieldArgs, 0, sizeof(s_beginFieldArgs));
   m_invalid = false;
+  frameCount = 0;
+
+  CommandProcessor::Init();
+  Fifo::Init();
+  OpcodeDecoder::Init();
+  PixelEngine::Init();
+  BPInit();
+  VertexLoaderManager::Init();
+  IndexGenerator::Init();
+  VertexShaderManager::Init();
+  GeometryShaderManager::Init();
+  PixelShaderManager::Init();
+
+  if (File::Exists(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini"))
+    g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + "GFX.ini");
+  else
+    g_Config.Load(File::GetUserPath(D_CONFIG_IDX) + GetConfigName() + ".ini");
+  g_Config.GameIniLoad();
+  g_Config.UpdateProjectionHack();
+  g_Config.VerifyValidity();
+  UpdateActiveConfig();
+
+  // Notify the core that the video backend is ready
+  Host_Message(WM_USER_CREATE);
+}
+
+void VideoBackendBase::ShutdownShared()
+{
+  // Do our OSD callbacks
+  OSD::DoCallbacks(OSD::CallbackType::Shutdown);
+
+  m_initialized = false;
+
+  Fifo::Shutdown();
+}
+
+void VideoBackendBase::CleanupShared()
+{
+  VertexLoaderManager::Clear();
 }
 
 // Run from the CPU thread
diff --git a/Source/Core/VideoCommon/OpcodeDecoding.cpp b/Source/Core/VideoCommon/OpcodeDecoding.cpp
index 8932cde5fa..0a2d3c4014 100644
--- a/Source/Core/VideoCommon/OpcodeDecoding.cpp
+++ b/Source/Core/VideoCommon/OpcodeDecoding.cpp
@@ -121,10 +121,6 @@ void Init()
   s_bFifoErrorSeen = false;
 }
 
-void Shutdown()
-{
-}
-
 template <bool is_preprocess>
 u8* Run(DataReader src, u32* cycles, bool in_display_list)
 {
diff --git a/Source/Core/VideoCommon/OpcodeDecoding.h b/Source/Core/VideoCommon/OpcodeDecoding.h
index 13a68548d2..df462e0c30 100644
--- a/Source/Core/VideoCommon/OpcodeDecoding.h
+++ b/Source/Core/VideoCommon/OpcodeDecoding.h
@@ -42,7 +42,6 @@ class DataReader;
 namespace OpcodeDecoder
 {
 void Init();
-void Shutdown();
 
 template <bool is_preprocess = false>
 u8* Run(DataReader src, u32* cycles, bool in_display_list);
diff --git a/Source/Core/VideoCommon/PixelShaderManager.cpp b/Source/Core/VideoCommon/PixelShaderManager.cpp
index be1a9cd1f3..b77f5e31a2 100644
--- a/Source/Core/VideoCommon/PixelShaderManager.cpp
+++ b/Source/Core/VideoCommon/PixelShaderManager.cpp
@@ -56,10 +56,6 @@ void PixelShaderManager::Dirty()
   dirty = true;
 }
 
-void PixelShaderManager::Shutdown()
-{
-}
-
 void PixelShaderManager::SetConstants()
 {
   if (s_bFogRangeAdjustChanged)
diff --git a/Source/Core/VideoCommon/PixelShaderManager.h b/Source/Core/VideoCommon/PixelShaderManager.h
index 32493a6eec..fe57fd4655 100644
--- a/Source/Core/VideoCommon/PixelShaderManager.h
+++ b/Source/Core/VideoCommon/PixelShaderManager.h
@@ -15,7 +15,6 @@ class PixelShaderManager
 public:
   static void Init();
   static void Dirty();
-  static void Shutdown();
   static void DoState(PointerWrap& p);
 
   static void SetConstants();  // sets pixel shader constants
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.cpp b/Source/Core/VideoCommon/VertexLoaderManager.cpp
index c8fa5485a9..7975ea1eb7 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.cpp
+++ b/Source/Core/VideoCommon/VertexLoaderManager.cpp
@@ -57,7 +57,7 @@ void Init()
   SETSTAT(stats.numVertexLoaders, 0);
 }
 
-void Shutdown()
+void Clear()
 {
   std::lock_guard<std::mutex> lk(s_vertex_loader_map_lock);
   s_vertex_loader_map.clear();
diff --git a/Source/Core/VideoCommon/VertexLoaderManager.h b/Source/Core/VideoCommon/VertexLoaderManager.h
index 1386df2381..2b273bb80f 100644
--- a/Source/Core/VideoCommon/VertexLoaderManager.h
+++ b/Source/Core/VideoCommon/VertexLoaderManager.h
@@ -20,7 +20,7 @@ using NativeVertexFormatMap =
     std::unordered_map<PortableVertexDeclaration, std::unique_ptr<NativeVertexFormat>>;
 
 void Init();
-void Shutdown();
+void Clear();
 
 void MarkAllDirty();
 
diff --git a/Source/Core/VideoCommon/VertexShaderManager.cpp b/Source/Core/VideoCommon/VertexShaderManager.cpp
index c90195f13e..19fb1f371d 100644
--- a/Source/Core/VideoCommon/VertexShaderManager.cpp
+++ b/Source/Core/VideoCommon/VertexShaderManager.cpp
@@ -216,10 +216,6 @@ void VertexShaderManager::Init()
   dirty = true;
 }
 
-void VertexShaderManager::Shutdown()
-{
-}
-
 void VertexShaderManager::Dirty()
 {
   // This function is called after a savestate is loaded.
diff --git a/Source/Core/VideoCommon/VertexShaderManager.h b/Source/Core/VideoCommon/VertexShaderManager.h
index 1820a7fa71..a3da612b56 100644
--- a/Source/Core/VideoCommon/VertexShaderManager.h
+++ b/Source/Core/VideoCommon/VertexShaderManager.h
@@ -19,7 +19,6 @@ class VertexShaderManager
 public:
   static void Init();
   static void Dirty();
-  static void Shutdown();
   static void DoState(PointerWrap& p);
 
   // constant management
diff --git a/Source/Core/VideoCommon/VideoBackendBase.h b/Source/Core/VideoCommon/VideoBackendBase.h
index bc08469002..a5571c98fc 100644
--- a/Source/Core/VideoCommon/VideoBackendBase.h
+++ b/Source/Core/VideoCommon/VideoBackendBase.h
@@ -73,6 +73,8 @@ public:
 
   virtual std::string GetName() const = 0;
   virtual std::string GetDisplayName() const { return GetName(); }
+  virtual std::string GetConfigName() const = 0;
+
   virtual void ShowConfig(void*) = 0;
 
   virtual void Video_Prepare() = 0;
@@ -98,6 +100,8 @@ public:
 
 protected:
   void InitializeShared();
+  void ShutdownShared();
+  void CleanupShared();
 
   bool m_initialized = false;
   bool m_invalid = false;
diff --git a/Source/Core/VideoCommon/VideoState.cpp b/Source/Core/VideoCommon/VideoState.cpp
index 6caccd5fd9..3eed6b5de0 100644
--- a/Source/Core/VideoCommon/VideoState.cpp
+++ b/Source/Core/VideoCommon/VideoState.cpp
@@ -65,10 +65,3 @@ void VideoCommon_DoState(PointerWrap& p)
 
   // TODO: search for more data that should be saved and add it here
 }
-
-void VideoCommon_Init()
-{
-  memset(&g_main_cp_state, 0, sizeof(g_main_cp_state));
-  memset(&g_preprocess_cp_state, 0, sizeof(g_preprocess_cp_state));
-  memset(texMem, 0, TMEM_SIZE);
-}
diff --git a/Source/Core/VideoCommon/VideoState.h b/Source/Core/VideoCommon/VideoState.h
index 0f15c977f7..ec9b3e584e 100644
--- a/Source/Core/VideoCommon/VideoState.h
+++ b/Source/Core/VideoCommon/VideoState.h
@@ -7,4 +7,3 @@
 class PointerWrap;
 
 void VideoCommon_DoState(PointerWrap& p);
-void VideoCommon_Init();