Add possibility to flatten visible/selected layers (fix #1226)

Discussion:
* https://github.com/aseprite/aseprite/issues/1226
* https://community.aseprite.org/t/wish-flatten-selected-layers/163

data/gui.xml

@@ -741,6 +741,9 @@
         <item command="DuplicateLayer" text="@.layer_duplicate" />
         <item command="MergeDownLayer" text="@.layer_merge_down" />
         <item command="FlattenLayers" text="@.layer_flatten" />
+        <item command="FlattenLayers" text="@.layer_flatten_visible">
+          <param name="visibleOnly" value="true" />
+        </item>
         <separator />
         <item command="NewLayer" text="@.layer_add_reference_layer">
           <param name="reference" value="true" />
@@ -914,6 +917,9 @@
       <item command="DuplicateLayer" text="@main_menu.layer_duplicate" />
       <item command="MergeDownLayer" text="@main_menu.layer_merge_down" />
       <item command="FlattenLayers" text="@main_menu.layer_flatten" />
+      <item command="FlattenLayers" text="@main_menu.layer_flatten_visible">
+        <param name="visibleOnly" value="true" />
+      </item>
     </menu>
 
     <menu id="frame_popup_menu">

data/strings/en.ini

@@ -247,6 +247,7 @@ Eyedropper = Eyedropper
 Fill = Fill Selection with Foreground Color
 FitScreen = Fit on Screen
 FlattenLayers = Flatten Layers
+FlattenLayers_Visible = Flatten Visible Layers
 Flip = Flip {0} {1}
 Flip_Canvas = Canvas
 Flip_Horizontally = Horizontally
@@ -729,6 +730,7 @@ layer_layer_from_background = &Layer from Background
 layer_duplicate = &Duplicate
 layer_merge_down = &Merge Down
 layer_flatten = &Flatten
+layer_flatten_visible = Flatten Vi&sible
 layer_add_reference_layer = Add R&eference Layer
 frame = F&rame
 frame_properties = Frame &Properties...

src/app/CMakeLists.txt

@@ -214,7 +214,6 @@ if(ENABLE_UI)
     commands/cmd_eyedropper.cpp
     commands/cmd_fill_and_stroke.cpp
     commands/cmd_fit_screen.cpp
-    commands/cmd_flatten_layers.cpp
     commands/cmd_flip.cpp
     commands/cmd_frame_properties.cpp
     commands/cmd_frame_tag_properties.cpp
@@ -505,6 +504,7 @@ add_library(app-lib
   commands/cmd_cel_opacity.cpp
   commands/cmd_change_pixel_format.cpp
   commands/cmd_crop.cpp
+  commands/cmd_flatten_layers.cpp
   commands/cmd_layer_from_background.cpp
   commands/cmd_load_palette.cpp
   commands/cmd_new_layer.cpp

src/app/cmd/flatten_layers.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2001-2016  David Capello
+// Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
 // the End-User License Agreement for Aseprite.
@@ -13,12 +13,14 @@
 #include "app/cmd/add_layer.h"
 #include "app/cmd/configure_background.h"
 #include "app/cmd/copy_rect.h"
-#include "app/cmd/remove_layer.h"
+#include "app/cmd/move_layer.h"
 #include "app/cmd/remove_layer.h"
 #include "app/cmd/set_layer_flags.h"
 #include "app/cmd/set_layer_name.h"
 #include "app/cmd/unlink_cel.h"
 #include "app/doc.h"
+#include "app/restore_visible_layers.h"
+#include "doc/algorithm/shrink_bounds.h"
 #include "doc/cel.h"
 #include "doc/layer.h"
 #include "doc/primitives.h"
@@ -28,9 +30,16 @@
 namespace app {
 namespace cmd {
 
-FlattenLayers::FlattenLayers(Sprite* sprite)
+FlattenLayers::FlattenLayers(doc::Sprite* sprite,
+                             const doc::SelectedLayers& layers0)
   : WithSprite(sprite)
 {
+  doc::SelectedLayers layers(layers0);
+  layers.removeChildrenIfParentIsSelected();
+
+  m_layerIds.reserve(layers.size());
+  for (auto layer : layers)
+    m_layerIds.push_back(layer->id());
 }
 
 void FlattenLayers::onExecute()
@@ -38,6 +47,17 @@ void FlattenLayers::onExecute()
   Sprite* sprite = this->sprite();
   auto doc = static_cast<Doc*>(sprite->document());
 
+  // Set of layers to be flattened.
+  bool backgroundIsSel = false;
+  SelectedLayers layers;
+  for (auto layerId : m_layerIds) {
+    doc::Layer* layer = doc::get<doc::Layer>(layerId);
+    ASSERT(layer);
+    layers.insert(layer);
+    if (layer->isBackground())
+      backgroundIsSel = true;
+  }
+
   // Create a temporary image.
   ImageRef image(Image::create(sprite->pixelFormat(),
       sprite->width(),
@@ -47,22 +67,34 @@ void FlattenLayers::onExecute()
   color_t     bgcolor;    // The background color to use for flatLayer.
 
   flatLayer = sprite->backgroundLayer();
-  if (flatLayer && flatLayer->isVisible()) {
+  if (backgroundIsSel && flatLayer && flatLayer->isVisible()) {
     // There exists a visible background layer, so we will flatten onto that.
     bgcolor = doc->bgColor(flatLayer);
   }
   else {
-    // Create a new transparent layer to flatten everything onto.
+    // Create a new transparent layer to flatten everything onto it.
     flatLayer = new LayerImage(sprite);
     ASSERT(flatLayer->isVisible());
     executeAndAdd(new cmd::AddLayer(sprite->root(), flatLayer, nullptr));
     executeAndAdd(new cmd::SetLayerName(flatLayer, "Flattened"));
     bgcolor = sprite->transparentColor();
+
+    LayerList list = layers.toLayerList();
+    if (list.front())
+      executeAndAdd(new cmd::MoveLayer(flatLayer,
+                                       list.front()->parent(),
+                                       list.front()));
   }
 
   render::Render render;
   render.setBgType(render::BgType::NONE);
 
+  {
+    // Show only the layers to be flattened so other layers are hidden
+    // temporarily.
+    RestoreVisibleLayers restore;
+    restore.showSelectedLayers(sprite, layers);
+
     // Copy all frames to the background.
     for (frame_t frame(0); frame<sprite->totalFrames(); ++frame) {
       // Clear the image and render this frame.
@@ -80,22 +112,33 @@ void FlattenLayers::onExecute()
         cel_image = cel->imageRef();
         ASSERT(cel_image);
 
-      executeAndAdd(new cmd::CopyRect(cel_image.get(), image.get(),
+        executeAndAdd(
+          new cmd::CopyRect(cel_image.get(), image.get(),
                             gfx::Clip(0, 0, image->bounds())));
       }
       else {
-      cel_image.reset(Image::createCopy(image.get()));
+        gfx::Rect bounds(image->bounds());
+        if (doc::algorithm::shrink_bounds(
+              image.get(), bounds, image->maskColor())) {
+          cel_image.reset(
+            doc::crop_image(image.get(), bounds, image->maskColor()));
           cel = new Cel(frame, cel_image);
+          cel->setPosition(bounds.origin());
           flatLayer->addCel(cel);
         }
       }
+    }
+  }
 
-  // Delete old layers.
+  // Delete flattened layers.
-  LayerList layers = sprite->root()->layers();
+  for (Layer* layer : layers) {
-  for (Layer* layer : layers)
+    // layer can be == flatLayer when we are flattening on the
-    if (layer != flatLayer)
+    // background layer.
+    if (layer != flatLayer) {
       executeAndAdd(new cmd::RemoveLayer(layer));
     }
+  }
+}
 
 } // namespace cmd
 } // namespace app

src/app/cmd/flatten_layers.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2001-2015  David Capello
+// Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
 // the End-User License Agreement for Aseprite.
@@ -10,18 +10,25 @@
 
 #include "app/cmd/with_sprite.h"
 #include "app/cmd_sequence.h"
+#include "doc/object_id.h"
+#include "doc/selected_layers.h"
+
+#include <vector>
 
 namespace app {
 namespace cmd {
-  using namespace doc;
 
   class FlattenLayers : public CmdSequence
                       , public WithSprite {
   public:
-    FlattenLayers(Sprite* sprite);
+    FlattenLayers(doc::Sprite* sprite,
+                  const doc::SelectedLayers& layers);
 
   protected:
     void onExecute() override;
+
+  private:
+    std::vector<doc::ObjectId> m_layerIds;
   };
 
 } // namespace cmd

src/app/commands/cmd_change_pixel_format.cpp

@@ -29,6 +29,7 @@
 #include "base/bind.h"
 #include "base/thread.h"
 #include "doc/image.h"
+#include "doc/layer.h"
 #include "doc/sprite.h"
 #include "fmt/format.h"
 #include "render/dithering_algorithm.h"
@@ -471,8 +472,12 @@ void ChangePixelFormatCommand::onExecute(Context* context)
       [this, &job, flatten] {
         Sprite* sprite(job.sprite());
 
-        if (flatten)
+        if (flatten) {
-          job.tx()(new cmd::FlattenLayers(sprite));
+          SelectedLayers selLayers;
+          for (auto layer : sprite->root()->layers())
+            selLayers.insert(layer);
+          job.tx()(new cmd::FlattenLayers(sprite, selLayers));
+        }
 
         job.tx()(
           new cmd::SetPixelFormat(

src/app/commands/cmd_flatten_layers.cpp

@@ -11,9 +11,13 @@
 #include "app/cmd/flatten_layers.h"
 #include "app/commands/command.h"
 #include "app/context_access.h"
+#include "app/doc_range.h"
+#include "app/i18n/strings.h"
 #include "app/modules/gui.h"
 #include "app/tx.h"
 #include "app/ui/color_bar.h"
+#include "app/ui/timeline/timeline.h"
+#include "doc/layer.h"
 #include "doc/sprite.h"
 
 namespace app {
@@ -24,13 +28,24 @@ public:
   Command* clone() const override { return new FlattenLayersCommand(*this); }
 
 protected:
+  void onLoadParams(const Params& params) override;
   bool onEnabled(Context* context) override;
   void onExecute(Context* context) override;
+  std::string onGetFriendlyName() const override;
+
+  bool m_visibleOnly;
 };
 
 FlattenLayersCommand::FlattenLayersCommand()
   : Command(CommandId::FlattenLayers(), CmdUIOnlyFlag)
 {
+  m_visibleOnly = false;
+}
+
+void FlattenLayersCommand::onLoadParams(const Params& params)
+{
+  std::string visibleOnly = params.get("visibleOnly");
+  m_visibleOnly = (visibleOnly == "true");
 }
 
 bool FlattenLayersCommand::onEnabled(Context* context)
@@ -44,10 +59,42 @@ void FlattenLayersCommand::onExecute(Context* context)
   Sprite* sprite = writer.sprite();
   {
     Tx tx(writer.context(), "Flatten Layers");
-    tx(new cmd::FlattenLayers(sprite));
+
+    // TODO the range of selected layers should be in app::Site.
+    DocRange range;
+
+    if (m_visibleOnly) {
+      for (auto layer : sprite->root()->layers())
+        if (layer->isVisible())
+          range.selectLayer(layer);
+    }
+    else {
+#ifdef ENABLE_UI
+      if (context->isUIAvailable())
+        range = App::instance()->timeline()->range();
+#endif
+
+      if (!range.enabled()) {
+        for (auto layer : sprite->root()->layers())
+          range.selectLayer(layer);
+      }
+    }
+
+    tx(new cmd::FlattenLayers(sprite, range.selectedLayers()));
     tx.commit();
   }
+
+#ifdef ENABLE_UI
   update_screen_for_document(writer.document());
+#endif
+}
+
+std::string FlattenLayersCommand::onGetFriendlyName() const
+{
+  if (m_visibleOnly)
+    return Strings::commands_FlattenLayers_Visible();
+  else
+    return Strings::commands_FlattenLayers();
 }
 
 Command* CommandFactory::createFlattenLayersCommand()

src/app/commands/commands_list.h

@@ -9,6 +9,7 @@ FOR_EACH_COMMAND(BackgroundFromLayer)
 FOR_EACH_COMMAND(CelOpacity)
 FOR_EACH_COMMAND(ChangePixelFormat)
 FOR_EACH_COMMAND(CropSprite)
+FOR_EACH_COMMAND(FlattenLayers)
 FOR_EACH_COMMAND(LayerFromBackground)
 FOR_EACH_COMMAND(LoadPalette)
 FOR_EACH_COMMAND(NewLayer)
@@ -54,7 +55,6 @@ FOR_EACH_COMMAND(ExportSpriteSheet)
 FOR_EACH_COMMAND(Eyedropper)
 FOR_EACH_COMMAND(Fill)
 FOR_EACH_COMMAND(FitScreen)
-FOR_EACH_COMMAND(FlattenLayers)
 FOR_EACH_COMMAND(Flip)
 FOR_EACH_COMMAND(FrameProperties)
 FOR_EACH_COMMAND(FrameTagProperties)