diff --git a/src/app/CMakeLists.txt b/src/app/CMakeLists.txt
index a05bcb90b..616122439 100644
--- a/src/app/CMakeLists.txt
+++ b/src/app/CMakeLists.txt
@@ -407,6 +407,7 @@ if(ENABLE_UI)
 endif()
 
 add_library(app-lib
+  active_site_handler.cpp
   app.cpp
   check_update.cpp
   cli/app_options.cpp
diff --git a/src/app/active_site_handler.cpp b/src/app/active_site_handler.cpp
new file mode 100644
index 000000000..75646b804
--- /dev/null
+++ b/src/app/active_site_handler.cpp
@@ -0,0 +1,141 @@
+// Aseprite
+// Copyright (C) 2019  Igara Studio S.A.
+//
+// This program is distributed under the terms of
+// the End-User License Agreement for Aseprite.
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "app/active_site_handler.h"
+#include "app/doc.h"
+#include "app/doc_event.h"
+#include "app/site.h"
+#include "doc/layer.h"
+
+namespace app {
+
+ActiveSiteHandler::ActiveSiteHandler()
+{
+}
+
+ActiveSiteHandler::~ActiveSiteHandler()
+{
+}
+
+void ActiveSiteHandler::addDoc(Doc* doc)
+{
+  Data data;
+  if (doc::Layer* layer = doc->sprite()->root()->firstLayer())
+    data.layer = layer->id();
+  else
+    data.layer = doc::NullId;
+  data.frame = 0;
+  m_data.insert(std::make_pair(doc, data));
+  doc->add_observer(this);
+}
+
+void ActiveSiteHandler::removeDoc(Doc* doc)
+{
+  auto it = m_data.find(doc);
+  if (it == m_data.end())
+    return;
+
+  doc->remove_observer(this);
+  m_data.erase(it);
+}
+
+ActiveSiteHandler::Data& ActiveSiteHandler::getData(Doc* doc)
+{
+  auto it = m_data.find(doc);
+  if (it == m_data.end()) {
+    addDoc(doc);
+    it = m_data.find(doc);
+    ASSERT(it != m_data.end());
+  }
+  return it->second;
+}
+
+void ActiveSiteHandler::getActiveSiteForDoc(Doc* doc, Site* site)
+{
+  Data& data = getData(doc);
+  site->document(doc);
+  site->sprite(doc->sprite());
+  site->layer(doc::get<doc::Layer>(data.layer));
+  site->frame(data.frame);
+}
+
+void ActiveSiteHandler::setActiveLayerInDoc(Doc* doc, doc::Layer* layer)
+{
+  Data& data = getData(doc);
+  data.layer = (layer ? layer->id(): 0);
+}
+
+void ActiveSiteHandler::setActiveFrameInDoc(Doc* doc, doc::frame_t frame)
+{
+  Data& data = getData(doc);
+  data.frame = frame;
+}
+
+void ActiveSiteHandler::onAddLayer(DocEvent& ev)
+{
+  Data& data = getData(ev.document());
+  data.layer = ev.layer()->id();
+}
+
+void ActiveSiteHandler::onAddFrame(DocEvent& ev)
+{
+  Data& data = getData(ev.document());
+  data.frame = ev.frame();
+}
+
+// TODO similar to Timeline::onBeforeRemoveLayer()
+void ActiveSiteHandler::onBeforeRemoveLayer(DocEvent& ev)
+{
+  Data& data = getData(ev.document());
+
+  Layer* layer = ev.layer();
+
+  // If the layer that was removed is the selected one
+  ASSERT(layer);
+  if (layer && data.layer == layer->id()) {
+    LayerGroup* parent = layer->parent();
+    Layer* layer_select = nullptr;
+
+    // Select previous layer, or next layer, or the parent (if it is
+    // not the main layer of sprite set).
+    if (layer->getPrevious()) {
+      layer_select = layer->getPrevious();
+    }
+    else if (layer->getNext())
+      layer_select = layer->getNext();
+    else if (parent != layer->sprite()->root())
+      layer_select = parent;
+
+    data.layer = (layer_select ? layer_select->id(): 0);
+  }
+}
+
+// TODO similar to Timeline::onRemoveFrame()
+void ActiveSiteHandler::onRemoveFrame(DocEvent& ev)
+{
+  Data& data = getData(ev.document());
+
+  // Adjust current frame of the data that are in a frame more
+  // advanced that the removed one.
+  if (data.frame > ev.frame()) {
+    --data.frame;
+  }
+  // If the data was in the previous "last frame" (current value of
+  // totalFrames()), we've to adjust it to the new last frame
+  // (lastFrame())
+  else if (data.frame >= ev.sprite()->totalFrames()) {
+    data.frame = ev.sprite()->lastFrame();
+  }
+
+  if (data.frame < ev.frame())
+    --data.frame;
+}
+
+} // namespace app
diff --git a/src/app/active_site_handler.h b/src/app/active_site_handler.h
new file mode 100644
index 000000000..43b2f67af
--- /dev/null
+++ b/src/app/active_site_handler.h
@@ -0,0 +1,61 @@
+// Aseprite
+// Copyright (C) 2019  Igara Studio S.A.
+//
+// This program is distributed under the terms of
+// the End-User License Agreement for Aseprite.
+
+#ifndef APP_ACTIVE_SITE_HANDLER_H_INCLUDED
+#define APP_ACTIVE_SITE_HANDLER_H_INCLUDED
+#pragma once
+
+#include "app/doc_observer.h"
+#include "doc/frame.h"
+#include "doc/object_id.h"
+
+#include <map>
+
+namespace doc {
+  class Layer;
+}
+
+namespace app {
+  class Doc;
+  class Site;
+
+  // Pseudo-DocViews to handle active layer/frame in a non-UI context
+  // per Doc.
+  //
+  // TODO we could move code to handle active frame/layer from
+  //      Timeline to this class.
+  class ActiveSiteHandler : public DocObserver {
+  public:
+    ActiveSiteHandler();
+    virtual ~ActiveSiteHandler();
+
+    void addDoc(Doc* doc);
+    void removeDoc(Doc* doc);
+    void getActiveSiteForDoc(Doc* doc, Site* site);
+    void setActiveLayerInDoc(Doc* doc, doc::Layer* layer);
+    void setActiveFrameInDoc(Doc* doc, doc::frame_t frame);
+
+  private:
+    // DocObserver impl
+    void onAddLayer(DocEvent& ev) override;
+    void onAddFrame(DocEvent& ev) override;
+    void onBeforeRemoveLayer(DocEvent& ev) override;
+    void onRemoveFrame(DocEvent& ev) override;
+
+    // Active data for a document
+    struct Data {
+      doc::ObjectId layer;
+      doc::frame_t frame;
+    };
+
+    Data& getData(Doc* doc);
+
+    std::map<Doc*, Data> m_data;
+  };
+
+} // namespace app
+
+#endif
diff --git a/src/app/context.cpp b/src/app/context.cpp
index 5a7be8608..cce362fcf 100644
--- a/src/app/context.cpp
+++ b/src/app/context.cpp
@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018  Igara Studio S.A.
+// Copyright (C) 2018-2019  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -11,6 +11,7 @@
 
 #include "app/context.h"
 
+#include "app/active_site_handler.h"
 #include "app/app.h"
 #include "app/commands/command.h"
 #include "app/commands/commands.h"
@@ -63,6 +64,16 @@ void Context::setActiveDocument(Doc* document)
   onSetActiveDocument(document);
 }
 
+void Context::setActiveLayer(doc::Layer* layer)
+{
+  onSetActiveLayer(layer);
+}
+
+void Context::setActiveFrame(const doc::frame_t frame)
+{
+  onSetActiveFrame(frame);
+}
+
 bool Context::hasModifiedDocuments() const
 {
   for (auto doc : documents())
@@ -150,23 +161,25 @@ void Context::executeCommand(Command* command, const Params& params)
 void Context::onAddDocument(Doc* doc)
 {
   m_lastSelectedDoc = doc;
+
+  if (m_activeSiteHandler)
+    m_activeSiteHandler->addDoc(doc);
 }
 
 void Context::onRemoveDocument(Doc* doc)
 {
   if (doc == m_lastSelectedDoc)
     m_lastSelectedDoc = nullptr;
+
+  if (m_activeSiteHandler)
+    m_activeSiteHandler->removeDoc(doc);
 }
 
 void Context::onGetActiveSite(Site* site) const
 {
   // Default/dummy site (maybe for batch/command line mode)
-  if (Doc* doc = m_lastSelectedDoc) {
-    site->document(doc);
-    site->sprite(doc->sprite());
-    site->layer(doc->sprite()->root()->firstLayer());
-    site->frame(0);
-  }
+  if (Doc* doc = m_lastSelectedDoc)
+    activeSiteHandler()->getActiveSiteForDoc(doc, site);
 }
 
 void Context::onSetActiveDocument(Doc* doc)
@@ -174,6 +187,21 @@ void Context::onSetActiveDocument(Doc* doc)
   m_lastSelectedDoc = doc;
 }
 
+void Context::onSetActiveLayer(doc::Layer* layer)
+{
+  Doc* newDoc = (layer ? static_cast<Doc*>(layer->sprite()->document()): nullptr);
+  if (newDoc != m_lastSelectedDoc)
+    setActiveDocument(newDoc);
+  if (newDoc)
+    activeSiteHandler()->setActiveLayerInDoc(newDoc, layer);
+}
+
+void Context::onSetActiveFrame(const doc::frame_t frame)
+{
+  if (m_lastSelectedDoc)
+    activeSiteHandler()->setActiveFrameInDoc(m_lastSelectedDoc, frame);
+}
+
 void Context::setTransaction(Transaction* transaction)
 {
   if (transaction) {
@@ -186,4 +214,11 @@ void Context::setTransaction(Transaction* transaction)
   }
 }
 
+ActiveSiteHandler* Context::activeSiteHandler() const
+{
+  if (!m_activeSiteHandler)
+    m_activeSiteHandler.reset(new ActiveSiteHandler);
+  return m_activeSiteHandler.get();
+}
+
 } // namespace app
diff --git a/src/app/context.h b/src/app/context.h
index 7aefb9ae4..a35caf929 100644
--- a/src/app/context.h
+++ b/src/app/context.h
@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2018  Igara Studio S.A.
+// Copyright (C) 2018-2019  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -16,12 +16,19 @@
 #include "app/docs_observer.h"
 #include "base/disable_copying.h"
 #include "base/exception.h"
+#include "doc/frame.h"
 #include "obs/observable.h"
 #include "obs/signal.h"
 
+#include <memory>
 #include <vector>
 
+namespace doc {
+  class Layer;
+}
+
 namespace app {
+  class ActiveSiteHandler;
   class Command;
   class Doc;
   class DocView;
@@ -75,6 +82,8 @@ namespace app {
     Site activeSite() const;
     Doc* activeDocument() const;
     void setActiveDocument(Doc* document);
+    void setActiveLayer(doc::Layer* layer);
+    void setActiveFrame(doc::frame_t frame);
     bool hasModifiedDocuments() const;
     void notifyActiveSiteChanged();
 
@@ -99,14 +108,19 @@ namespace app {
 
     virtual void onGetActiveSite(Site* site) const;
     virtual void onSetActiveDocument(Doc* doc);
+    virtual void onSetActiveLayer(doc::Layer* layer);
+    virtual void onSetActiveFrame(const doc::frame_t frame);
 
     Doc* lastSelectedDoc() { return m_lastSelectedDoc; }
 
   private:
+    ActiveSiteHandler* activeSiteHandler() const;
+
     Docs m_docs;
     ContextFlags m_flags;       // Last updated flags.
     Doc* m_lastSelectedDoc;
     Transaction* m_transaction;
+    mutable std::unique_ptr<ActiveSiteHandler> m_activeSiteHandler;
 
     DISABLE_COPYING(Context);
   };
diff --git a/src/app/script/app_object.cpp b/src/app/script/app_object.cpp
index 947f97b58..fd4355746 100644
--- a/src/app/script/app_object.cpp
+++ b/src/app/script/app_object.cpp
@@ -449,54 +449,26 @@ int App_set_activeSprite(lua_State* L)
 
 int App_set_activeLayer(lua_State* L)
 {
-#ifdef ENABLE_UI
   auto layer = get_docobj<Layer>(L, 2);
   app::Context* ctx = App::instance()->context();
-  if (auto uiCtx = dynamic_cast<UIContext*>(ctx)) {
-    DocView* docView = uiCtx->activeView();
-    if (docView) {
-      Editor* editor = docView->editor();
-      if (editor)
-        editor->setLayer(static_cast<LayerImage*>(layer));
-    }
-  }
-#endif
+  ctx->setActiveLayer(layer);
   return 0;
 }
 
 int App_set_activeFrame(lua_State* L)
 {
-#ifdef ENABLE_UI
   const doc::frame_t frame = get_frame_number_from_arg(L, 2);
   app::Context* ctx = App::instance()->context();
-  if (auto uiCtx = dynamic_cast<UIContext*>(ctx)) {
-    DocView* docView = uiCtx->activeView();
-    if (docView) {
-      Editor* editor = docView->editor();
-      if (editor)
-        editor->setFrame(frame);
-    }
-  }
-#endif
+  ctx->setActiveFrame(frame);
   return 0;
 }
 
 int App_set_activeCel(lua_State* L)
 {
-#ifdef ENABLE_UI
   const auto cel = get_docobj<Cel>(L, 2);
   app::Context* ctx = App::instance()->context();
-  if (auto uiCtx = dynamic_cast<UIContext*>(ctx)) {
-    DocView* docView = uiCtx->activeView();
-    if (docView) {
-      Editor* editor = docView->editor();
-      if (editor) {
-        editor->setLayer(static_cast<LayerImage*>(cel->layer()));
-        editor->setFrame(cel->frame());
-      }
-    }
-  }
-#endif
+  ctx->setActiveLayer(cel->layer());
+  ctx->setActiveFrame(cel->frame());
   return 0;
 }
 
@@ -505,19 +477,10 @@ int App_set_activeImage(lua_State* L)
   const auto cel = get_image_cel_from_arg(L, 2);
   if (!cel)
     return 0;
-#ifdef ENABLE_UI
+
   app::Context* ctx = App::instance()->context();
-  if (auto uiCtx = dynamic_cast<UIContext*>(ctx)) {
-    DocView* docView = uiCtx->activeView();
-    if (docView) {
-      Editor* editor = docView->editor();
-      if (editor) {
-        editor->setLayer(static_cast<LayerImage*>(cel->layer()));
-        editor->setFrame(cel->frame());
-      }
-    }
-  }
-#endif
+  ctx->setActiveLayer(cel->layer());
+  ctx->setActiveFrame(cel->frame());
   return 0;
 }
 
diff --git a/src/app/ui/timeline/timeline.cpp b/src/app/ui/timeline/timeline.cpp
index 541b693eb..88e546a9f 100644
--- a/src/app/ui/timeline/timeline.cpp
+++ b/src/app/ui/timeline/timeline.cpp
@@ -1726,7 +1726,8 @@ void Timeline::onAddLayer(DocEvent& ev)
   invalidate();
 }
 
-void Timeline::onAfterRemoveLayer(DocEvent& ev)
+// TODO similar to ActiveSiteHandler::onBeforeRemoveLayer()
+void Timeline::onBeforeRemoveLayer(DocEvent& ev)
 {
   Sprite* sprite = ev.sprite();
   Layer* layer = ev.layer();
@@ -1763,6 +1764,7 @@ void Timeline::onAddFrame(DocEvent& ev)
   invalidate();
 }
 
+// TODO similar to ActiveSiteHandler::onRemoveFrame()
 void Timeline::onRemoveFrame(DocEvent& ev)
 {
   // Adjust current frame of all editors that are in a frame more
diff --git a/src/app/ui/timeline/timeline.h b/src/app/ui/timeline/timeline.h
index 9ad0c6de1..f014c7646 100644
--- a/src/app/ui/timeline/timeline.h
+++ b/src/app/ui/timeline/timeline.h
@@ -141,7 +141,7 @@ namespace app {
     // DocObserver impl.
     void onGeneralUpdate(DocEvent& ev) override;
     void onAddLayer(DocEvent& ev) override;
-    void onAfterRemoveLayer(DocEvent& ev) override;
+    void onBeforeRemoveLayer(DocEvent& ev) override;
     void onAddFrame(DocEvent& ev) override;
     void onRemoveFrame(DocEvent& ev) override;
     void onSelectionBoundariesChanged(DocEvent& ev) override;
diff --git a/src/app/ui_context.cpp b/src/app/ui_context.cpp
index 9e1802f8a..7655fc1f9 100644
--- a/src/app/ui_context.cpp
+++ b/src/app/ui_context.cpp
@@ -1,4 +1,5 @@
 // Aseprite
+// Copyright (C) 2019  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -141,6 +142,26 @@ void UIContext::onSetActiveDocument(Doc* document)
     notifyActiveSiteChanged();
 }
 
+void UIContext::onSetActiveLayer(doc::Layer* layer)
+{
+  if (DocView* docView = activeView()) {
+    if (Editor* editor = docView->editor())
+      editor->setLayer(layer);
+  }
+  else if (!isUIAvailable())
+    Context::onSetActiveLayer(layer);
+}
+
+void UIContext::onSetActiveFrame(const doc::frame_t frame)
+{
+  if (DocView* docView = activeView()) {
+    if (Editor* editor = docView->editor())
+      editor->setFrame(frame);
+  }
+  else if (!isUIAvailable())
+    Context::onSetActiveFrame(frame);
+}
+
 DocView* UIContext::getFirstDocView(Doc* document) const
 {
   Workspace* workspace = App::instance()->workspace();
diff --git a/src/app/ui_context.h b/src/app/ui_context.h
index 6c86c664a..99b4e1de2 100644
--- a/src/app/ui_context.h
+++ b/src/app/ui_context.h
@@ -1,4 +1,5 @@
 // Aseprite
+// Copyright (C) 2019  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -46,6 +47,8 @@ namespace app {
     void onRemoveDocument(Doc* doc) override;
     void onGetActiveSite(Site* site) const override;
     void onSetActiveDocument(Doc* doc) override;
+    void onSetActiveLayer(doc::Layer* layer) override;
+    void onSetActiveFrame(const doc::frame_t frame) override;
 
   private:
     DocView* m_lastSelectedView;