Move the logic to get the refColor for trimming to app::get_best_refcolor_for_trimming() function

src/app/doc_api.cpp

@@ -49,6 +49,7 @@
 #include "app/doc_undo.h"
 #include "app/pref/preferences.h"
 #include "app/transaction.h"
+#include "app/util/autocrop.h"
 #include "doc/algorithm/flip_image.h"
 #include "doc/algorithm/shrink_bounds.h"
 #include "doc/cel.h"
@@ -316,68 +317,20 @@ void DocApi::trimSprite(Sprite* sprite, const bool byGrid)
 {
   gfx::Rect bounds;
 
-  std::unique_ptr<Image> image_wrap(Image::create(sprite->pixelFormat(),
+  std::unique_ptr<Image> image_wrap(Image::create(sprite->spec()));
-                                                  sprite->width(),
-                                                  sprite->height()));
   Image* image = image_wrap.get();
+
   render::Render render;
 
   for (frame_t frame(0); frame<sprite->totalFrames(); ++frame) {
     render.renderSprite(image, sprite, frame);
     gfx::Rect frameBounds;
+    doc::color_t refColor;
 
-    // Before this fix, we did use the top-left corner pixel by default as "refColor" in
+    if (get_best_refcolor_for_trimming(image, refColor) &&
-    // doc::algorithm::shrink_bounds(image, frameBounds, refColor).
+        doc::algorithm::shrink_bounds(image, frameBounds, refColor)) {
-    // Now, we are looking for one color whole border (i.e: solid color border)
-    // searching on the sprite square contour.
-    //
-    // If the algorithm finds a transparentColor pixel on border frame,
-    // automatically "refColor" turns to transparentColor and it goes directly to
-    // shrink_bounds function.
-    // If the borders don't contain transparentColor, the algorithm keeps checking:
-    // It search for a solid color border (top border first), then it checks
-    // the opposite border (bottom border), then:
-    // 1- If the opposite border color is equal to the first border,
-    //    this color will be the "refColor".
-    // 2- If the color of the opposite border is solid, BUT different to the first
-    //    border we have to trigger a dialog, delegating the choise to the user.
-    // 3- If opposite border contains differents colors, we choose the first border color as
-    //    "refColor".
-    // 4- It repeat analisys to sprite left and right columns pixels.
-    // If no border has solid color, trimSprite do nothing.
-
-    bool skipTrim = false;
-    bool analizeMoreBorders = true;
-    color_t refColor = get_pixel(image, 0, 0);
-    bool firstBorderIsSolidColor = true;
-    bool secondBorderIsSolidColor = true;
-    const color_t probableRefColor1 = refColor;
-    const color_t probableRefColor2 = get_pixel(image, sprite->width() - 1, sprite->height() - 1);
-
-    if (analizeFrameSpritePixels(image, sprite, refColor,
-                                 firstBorderIsSolidColor,
-                                 secondBorderIsSolidColor, true)) {
-      // Here, we know that analizeFrameSpritePixels did not find transparent pixels on top and bottom borders.
-      if (!firstBorderIsSolidColor && secondBorderIsSolidColor)
-        refColor = probableRefColor2;
-      else if (firstBorderIsSolidColor && secondBorderIsSolidColor && probableRefColor1 != probableRefColor2) {
-        // Both border are solids colors but different, so we have to trigger the dialog
-        skipTrim = true;
-        analizeMoreBorders = false;
-      }
-      if (analizeMoreBorders && analizeFrameSpritePixels(image, sprite, refColor,
-                                                         firstBorderIsSolidColor,
-                                                         secondBorderIsSolidColor, false)) {
-        if (!firstBorderIsSolidColor && secondBorderIsSolidColor)
-          refColor = probableRefColor2;
-        else if (firstBorderIsSolidColor && secondBorderIsSolidColor && probableRefColor1 != probableRefColor2)
-          // Both border are solids colors but different, so we have to trigger the dialog
-          skipTrim = true;
-      }
-    }
-
-    if (!skipTrim && doc::algorithm::shrink_bounds(image, frameBounds, refColor))
       bounds = bounds.createUnion(frameBounds);
+    }
 
     // TODO merge this code with the code in DocExporter::captureSamples()
     if (byGrid) {
@@ -399,61 +352,6 @@ void DocApi::trimSprite(Sprite* sprite, const bool byGrid)
     cropSprite(sprite, bounds);
 }
 
-bool DocApi::analizeFrameSpritePixels(Image* image,
-                                      const Sprite* sprite,
-                                      color_t& refColor,
-                                      bool& firstBorderIsSolidColor,
-                                      bool& secondBorderIsSolidColor,
-                                      bool topBottomLookUp) {
-  firstBorderIsSolidColor = true;
-  secondBorderIsSolidColor = true;
-  int spriteW = sprite->width();
-  int spriteH = sprite->height();
-  const color_t probableRefColor1 = get_pixel(image, 0, 0);
-  const color_t probableRefColor2 = get_pixel(image, spriteW - 1, spriteH - 1);
-  if (!topBottomLookUp) {
-    int aux = spriteW;
-    spriteW = spriteH;
-    spriteH = aux;
-  }
-  color_t currentPixel = 0;
-  color_t transparentColor = sprite->transparentColor();
-
-  if (probableRefColor1 == transparentColor || probableRefColor2 == transparentColor) {
-    refColor = transparentColor;
-    return false;
-  }
-
-  for (int i=0; i < spriteW; i++) {
-    if (topBottomLookUp)
-      currentPixel = get_pixel(image, i, 0);
-    else
-      currentPixel = get_pixel(image, 0, i);
-    if (currentPixel != probableRefColor1) {
-      firstBorderIsSolidColor = false;
-      if (currentPixel == transparentColor) {
-        refColor = transparentColor;
-        return false;
-      }
-    }
-  }
-
-  for (int i=0; i < spriteW; i++) {
-    if (topBottomLookUp)
-      currentPixel = get_pixel(image, i, spriteH - 1);
-    else
-      currentPixel = get_pixel(image, spriteH - 1, i);
-    if (currentPixel != probableRefColor2) {
-      secondBorderIsSolidColor = false;
-      if (currentPixel == transparentColor) {
-        refColor = transparentColor;
-        return false;
-      }
-    }
-  }
-  return true;
-}
-
 void DocApi::addFrame(Sprite* sprite, frame_t newFrame)
 {
   copyFrame(sprite, newFrame-1, newFrame,

src/app/doc_api.h

@@ -117,12 +117,6 @@ namespace app {
     void setPalette(Sprite* sprite, frame_t frame, const Palette* newPalette);
 
   private:
-    bool analizeFrameSpritePixels(Image* image,
-                                  const Sprite* sprite,
-                                  color_t& refColor,
-                                  bool& firstBorderIsSolidColor,
-                                  bool& secondBorderIsSolidColor,
-                                  bool topBottomLookUp);
     void cropImageLayer(LayerImage* layer,
                         const gfx::Rect& bounds,
                         const bool trimOutside);

src/app/util/autocrop.cpp

@@ -1,4 +1,5 @@
 // Aseprite
+// Copyright (C) 2019  Igara Studio S.A.
 // Copyright (C) 2001-2015  David Capello
 //
 // This program is distributed under the terms of
@@ -13,10 +14,76 @@
 #include "doc/sprite.h"
 #include "app/util/autocrop.h"
 
+#include <algorithm>
+
 namespace app {
 
 using namespace doc;
 
+namespace {
+
+// We call a "solid border" when a specific color is repeated in every
+// pixel of the image edge.  This will return isBorder1Solid=true if
+// the top (when topBottomLookUp=true) or left edge (when
+// topBottomLookUp=false) is a "solid border". In the case of
+// isBorder2Solid is for the bottom (or right) edge.
+bool analize_if_image_has_solid_borders(const Image* image,
+                                        color_t& refColor,
+                                        bool& isBorder1Solid,
+                                        bool& isBorder2Solid,
+                                        bool topBottomLookUp)
+{
+  isBorder1Solid = true;
+  isBorder2Solid = true;
+
+  int w = image->width();
+  int h = image->height();
+  const color_t probableRefColor1 = get_pixel(image, 0, 0);
+  const color_t probableRefColor2 = get_pixel(image, w-1, h-1);
+  color_t currentPixel;
+  color_t transparentColor = image->maskColor();
+
+  if (probableRefColor1 == transparentColor ||
+      probableRefColor2 == transparentColor) {
+    refColor = transparentColor;
+    return false;
+  }
+
+  if (!topBottomLookUp)
+    std::swap(w, h);
+
+  for (int i=0; i<w; ++i) {
+    if (topBottomLookUp)
+      currentPixel = get_pixel(image, i, 0);
+    else
+      currentPixel = get_pixel(image, 0, i);
+    if (currentPixel != probableRefColor1) {
+      isBorder1Solid = false;
+      if (currentPixel == transparentColor) {
+        refColor = transparentColor;
+        return false;
+      }
+    }
+  }
+
+  for (int i=0; i<w; ++i) {
+    if (topBottomLookUp)
+      currentPixel = get_pixel(image, i, h-1);
+    else
+      currentPixel = get_pixel(image, h-1, i);
+    if (currentPixel != probableRefColor2) {
+      isBorder2Solid = false;
+      if (currentPixel == transparentColor) {
+        refColor = transparentColor;
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+} // anonymous namespace
+
 bool get_shrink_rect(int *x1, int *y1, int *x2, int *y2,
                      Image *image, color_t refpixel)
 {
@@ -107,4 +174,84 @@ bool get_shrink_rect2(int *x1, int *y1, int *x2, int *y2,
 #undef SHRINK_SIDE
 }
 
+// A simple method to trim an image we have used in the past is
+// selecting the top-left corner pixel as the "refColor" in
+// shrink_bounds() algorithm (whatever that color is, transparent
+// or non-transparent).
+//
+// Now we have changed it to other heuristic: we look if there are
+// borders in the sprite with a solid color ("solid border"),
+// i.e. a color repeating in every pixel of that specific side
+// (left, top, right, or bottom).
+//
+// If we find a transparent pixel at the edges of the sprite, we
+// automatically set "refColor" as the transparent color and it
+// goes directly to shrink_bounds() function. Because this mean
+// that we are in a transparent layer and the transparent color is
+// the one that must be trimmed.
+//
+// The other case is when borders don't contain the transparent
+// color, we search for a "solid border" (top border first), then
+// it checks the opposite border (bottom border), then:
+//
+// 1) If the opposite border is equal to the first border,
+//    the color of both borders will be the "refColor".
+// 2) If the color of the opposite border is solid, BUT different to
+//    the first border we will need the user intervention to select a
+//    valid refColor (in this case the function returns false, which
+//    means that we cannot automatically trim the image).
+// 3) If opposite border contains differents colors, we choose the
+//    first border color as "refColor".
+// 4) It repeats the analysis with the left and right edges.
+//
+// If no border has solid color, trimSprite() does nothing.
+bool get_best_refcolor_for_trimming(
+  doc::Image* image,
+  color_t& refColor)
+{
+  const color_t probableRefColor1 = get_pixel(image, 0, 0);
+  const color_t probableRefColor2 = get_pixel(image, image->width()-1, image->height()-1);
+  bool isBorder1Solid = true;
+  bool isBorder2Solid = true;
+
+  refColor = probableRefColor1;
+
+  if (analize_if_image_has_solid_borders(
+        image, refColor,
+        isBorder1Solid,
+        isBorder2Solid,
+        true)) {                // Analize top vs. bottom borders
+    // Here, we know that analize_if_image_has_solid_borders() did not
+    // find transparent pixels on top and bottom borders.
+    if (!isBorder1Solid &&
+        isBorder2Solid) {
+      refColor = probableRefColor2;
+    }
+    else if (isBorder1Solid &&
+             isBorder2Solid &&
+             probableRefColor1 != probableRefColor2) {
+      // Both border are solid but with different colors, so the
+      // decision should be asked to the user.
+      return false;
+    }
+
+    if (analize_if_image_has_solid_borders(
+          image, refColor,
+          isBorder1Solid,
+          isBorder2Solid,
+          false)) {                // Analize left vs. right borders
+      if (!isBorder1Solid &&
+          isBorder2Solid)
+        refColor = probableRefColor2;
+      else if (isBorder1Solid &&
+               isBorder2Solid &&
+               probableRefColor1 != probableRefColor2)
+        // Both border are solid but with different colors, so the
+        // decision should be asked to the user.
+        return false;
+    }
+  }
+  return true;
+}
+
 } // namespace app

src/app/util/autocrop.h

@@ -1,4 +1,5 @@
 // Aseprite
+// Copyright (C) 2019  Igara Studio S.A.
 // Copyright (C) 2001-2015  David Capello
 //
 // This program is distributed under the terms of
@@ -21,6 +22,13 @@ namespace app {
   bool get_shrink_rect2(int* x1, int* y1, int* x2, int* y2,
                         doc::Image* image, doc::Image* regimage);
 
+  // Returns false if a refColor cannot be automatically decided (so
+  // in this case we should ask to the user to select a specific
+  // "refColor" to trim).
+  bool get_best_refcolor_for_trimming(
+    doc::Image* image,
+    doc::color_t& refColor);
+
 } // namespace app
 
 #endif