Remove fillOrphansNodes(), now new colors are added to the octree color arragement

This gives more accuracy in the color picking criteria on new images
pasted into an INDEXED sprite.

Also, added findMaskColor to fix the behavior reported in #3207

Both issues are related when RGBMAP is created. The 'mask color' and
'mask index' must be defined correctly to include/exclude during the
table/octree map color search.

To do: Converting sprite to INDEXED should add 'mask color' to the
palette when color count < 256 and transparent color isn't in the
palette.

src/app/cmd/set_pixel_format.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2019-2020  Igara Studio S.A.
+// Copyright (C) 2019-2022  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -22,6 +22,7 @@
 #include "doc/document.h"
 #include "doc/layer.h"
 #include "doc/palette.h"
+#include "doc/rgbmap.h"
 #include "doc/sprite.h"
 #include "doc/tilesets.h"
 #include "render/quantization.h"
@@ -177,6 +178,12 @@ void SetPixelFormat::setFormat(PixelFormat format)
   Sprite* sprite = this->sprite();
 
   sprite->setPixelFormat(format);
+  if (format == IMAGE_INDEXED) {
+    int maskIndex = sprite->palette(0)->findMaskColor();
+    sprite->setTransparentColor(maskIndex == -1 ? 0 : maskIndex);
+  }
+  else
+    sprite->setTransparentColor(0);
   sprite->incrementVersion();
 
   // Regenerate extras
@@ -201,14 +208,25 @@ void SetPixelFormat::convertImage(doc::Sprite* sprite,
   ASSERT(oldImage);
   ASSERT(oldImage->pixelFormat() != IMAGE_TILEMAP);
 
+  // Making the RGBMap for Image->INDEXDED conversion.
+  // TODO: this is needed only when newImage
+  RgbMap* rgbmap;
+  int newMaskIndex = (isBackground ? -1 : 0);
+  if (m_newFormat == IMAGE_INDEXED) {
+    rgbmap = sprite->rgbMap(frame, sprite->rgbMapForSprite(), mapAlgorithm);
+    if (m_oldFormat == IMAGE_INDEXED)
+      newMaskIndex = sprite->transparentColor();
+    else
+      newMaskIndex = rgbmap->maskIndex();
+  }
   ImageRef newImage(
     render::convert_pixel_format
     (oldImage.get(), nullptr, m_newFormat,
      dithering,
-     sprite->rgbMap(frame, sprite->rgbMapForSprite(), mapAlgorithm),
+     rgbmap,
      sprite->palette(frame),
      isBackground,
-     oldImage->maskColor(),
+     newMaskIndex,
      toGray,
      delegate));
 

src/doc/octree_map.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (c) 2020-2021 Igara Studio S.A.
+// Copyright (c) 2020-2022 Igara Studio S.A.
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -36,64 +36,19 @@ void OctreeNode::addColor(color_t c, int level, OctreeNode* parent,
   (*m_children)[index].addColor(c, level + 1, this, paletteIndex, levelDeep);
 }
 
-void OctreeNode::fillOrphansNodes(const Palette* palette,
+int OctreeNode::mapColor(int  r, int g, int b, int a, int mask_index, const Palette* palette, int level) const
-                                  const color_t upstreamBranchColor,
-                                  const int level)
 {
-  for (int i=0; i<16; i++) {
+  // New behavior: if mapColor do not have an exact rgba match, it must calculate which
-    OctreeNode& child = (*m_children)[i];
+  // color of the current palette is the bestfit and memorize the index in a octree leaf.
-
+  if (level >= 8) {
-    if (child.hasChildren()) {
+    if (m_paletteIndex == -1)
-      child.fillOrphansNodes(
+      m_paletteIndex = palette->findBestfit(r, g, b, a, mask_index);
-        palette,
+    return m_paletteIndex;
-        upstreamBranchColor + hextetToBranchColor(i, level),
-        level + 1);
-    }
-    else if (!(child.isLeaf())) {
-      // Here the node IS NOT a Leaf and HAS NOT children
-      // So, we must assign palette index to the current node
-      // to fill the "map holes" (i.e "death tree branches")
-      // BUT, if the level is low (a few bits to identify a color)
-      // 0, 1, 2, or 3, we need to create branchs/Leaves until
-      // the desired minimum color MSB bits.
-      if (level < MIN_LEVEL_OCTREE_DEEP) {
-        child.fillMostSignificantNodes(level);
-        i--;
-        continue;
-      }
-      int currentBranchColorAdd = hextetToBranchColor(i, level);
-      color_t branchColorMed = upstreamBranchColor |
-                               currentBranchColorAdd |
-                               ((level == 7) ? 0 : (0x01010101 << (6 - level))); // mid color adition
-      int indexMed = palette->findBestfit2(rgba_getr(branchColorMed),
-                                           rgba_getg(branchColorMed),
-                                           rgba_getb(branchColorMed),
-                                           rgba_geta(branchColorMed));
-      child.paletteIndex(indexMed);
-    }
   }
-}
+  int index = getHextet(r, g, b, a, level);
-
+  if (!m_children)
-void OctreeNode::fillMostSignificantNodes(int level)
-{
-  if (level < MIN_LEVEL_OCTREE_DEEP) {
     m_children.reset(new std::array<OctreeNode, 16>());
-    level++;
+  return (*m_children)[index].mapColor(r, g, b, a, mask_index, palette, level + 1);
-    for (int i=0; i<16; i++) {
-      OctreeNode& child = (*m_children)[i];
-
-      child.fillMostSignificantNodes(level);
-    }
-  }
-}
-
-int OctreeNode::mapColor(int r, int g, int b, int a, int level) const
-{
-  OctreeNode& child = (*m_children)[getHextet(rgba(r, g, b, a), level)];
-  if (child.hasChildren())
-    return child.mapColor(r, g, b, a, level+1);
-  else
-    return child.m_paletteIndex;
 }
 
 void OctreeNode::collectLeafNodes(OctreeNodes& leavesVector, int& paletteIndex)
@@ -143,6 +98,14 @@ int OctreeNode::getHextet(color_t c, int level)
          ((c & (0x80000000 >> level)) ? 8 : 0);
 }
 
+int OctreeNode::getHextet(int r, int g, int b, int a, int level)
+{
+  return ((r & (0x80 >> level)) ? 1 : 0) |
+         ((g & (0x80 >> level)) ? 2 : 0) |
+         ((b & (0x80 >> level)) ? 4 : 0) |
+         ((a & (0x80 >> level)) ? 8 : 0);
+}
+
 // static
 color_t OctreeNode::hextetToBranchColor(int hextet, int level)
 {
@@ -261,11 +224,6 @@ bool OctreeMap::makePalette(Palette* palette,
   return true;
 }
 
-void OctreeMap::fillOrphansNodes(const Palette* palette)
-{
-  m_root.fillOrphansNodes(palette, 0, 0);
-}
-
 void OctreeMap::feedWithImage(const Image* image,
                               const bool withAlpha,
                               const color_t maskColor,
@@ -310,13 +268,12 @@ void OctreeMap::feedWithImage(const Image* image,
 
 int OctreeMap::mapColor(color_t rgba) const
 {
-  if (m_root.hasChildren())
+  return m_root.mapColor(rgba_getr(rgba),
-    return m_root.mapColor(rgba_getr(rgba),
+                         rgba_getg(rgba),
-                           rgba_getg(rgba),
+                         rgba_getb(rgba),
-                           rgba_getb(rgba),
+                         rgba_geta(rgba),
-                           rgba_geta(rgba), 0);
+                         m_maskIndex,
-  else
+                         m_palette, 0);
-    return -1;
 }
 
 void OctreeMap::regenerateMap(const Palette* palette, const int maskIndex)
@@ -354,7 +311,6 @@ void OctreeMap::regenerateMap(const Palette* palette, const int maskIndex)
     }
     m_root.addColor(palette->entry(i), 0, &m_root, i, 8);
   }
-  m_root.fillOrphansNodes(palette, 0, 0);
 
   m_palette = palette;
   m_modifications = palette->getModifications();

src/doc/octree_map.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (c) 2020-2021 Igara Studio S.A.
+// Copyright (c) 2020-2022 Igara Studio S.A.
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -93,13 +93,7 @@ public:
   void addColor(color_t c, int level, OctreeNode* parent,
                 int paletteIndex = 0, int levelDeep = 7);
 
-  void fillOrphansNodes(const Palette* palette,
+  int mapColor(int  r, int g, int b, int a, int mask_index, const Palette* palette, int level) const;
-                        const color_t upstreamBranchColor,
-                        const int level);
-
-  void fillMostSignificantNodes(int level);
-
-  int mapColor(int r, int g, int b, int a, int level) const;
 
   void collectLeafNodes(OctreeNodes& leavesVector, int& paletteIndex);
 
@@ -114,11 +108,12 @@ private:
   void paletteIndex(int index) { m_paletteIndex = index; }
 
   static int getHextet(color_t c, int level);
+  static int getHextet(int r, int g, int b, int a, int level);
   static color_t hextetToBranchColor(int hextet, int level);
 
   LeafColor m_leafColor;
-  int m_paletteIndex = 0;
+  mutable int m_paletteIndex = -1;
-  std::unique_ptr<std::array<OctreeNode, 16>> m_children;
+  mutable std::unique_ptr<std::array<OctreeNode, 16>> m_children;
   OctreeNode* m_parent = nullptr;
 };
 
@@ -142,12 +137,20 @@ public:
   // RgbMap impl
   void regenerateMap(const Palette* palette, const int maskIndex) override;
   int mapColor(color_t rgba) const override;
+  int maskIndex() const override { return m_maskIndex; }
+  int mapColor(const int r, const int g,
+               const int b, const int a) const
+  {
+    ASSERT(r >= 0 && r < 256);
+    ASSERT(g >= 0 && g < 256);
+    ASSERT(b >= 0 && b < 256);
+    ASSERT(a >= 0 && a < 256);
+    return mapColor(rgba(r, g, b, a));
+  }
 
   int moodifications() const { return m_modifications; };
 
 private:
-  void fillOrphansNodes(const Palette* palette);
-
   OctreeNode m_root;
   OctreeNodes m_leavesVector;
   const Palette* m_palette = nullptr;

src/doc/palette.cpp

@@ -483,37 +483,14 @@ int Palette::findBestfit(int r, int g, int b, int a, int mask_index) const
   return bestfit;
 }
 
-int Palette::findBestfit2(int r, int g, int b, int a) const
+int Palette::findMaskColor() const
 {
-  ASSERT(r >= 0 && r <= 255);
-  ASSERT(g >= 0 && g <= 255);
-  ASSERT(b >= 0 && b <= 255);
-  ASSERT(a >= 0 && a <= 255);
-
-  int bestfit = 0;
-  int lowest = std::numeric_limits<int>::max();
   int size = m_colors.size();
-
+  for (int i = 0; i < size; ++i) {
-  for (int i=0; i<size; ++i) {
+    if (m_colors[i] == 0)
-    color_t rgb = m_colors[i];
+      return i;
-    int rDiff = r - rgba_getr(rgb);
-    int gDiff = g - rgba_getg(rgb);
-    int bDiff = b - rgba_getb(rgb);
-    int aDiff = a - rgba_geta(rgb);
-
-    // TODO We should have two different ways to calculate the
-    // distance between colors, like "Perceptual" and "Linear", or a
-    // way to configure these coefficients.
-    int diff = rDiff * rDiff * 900  +
-               gDiff * gDiff * 3481 +
-               bDiff * bDiff * 121  +
-               aDiff * aDiff * 900; // there is no scientific reason to choose this value.
-    if (diff < lowest) {
-      lowest = diff;
-      bestfit = i;
-    }
   }
-  return bestfit;
+  return -1;
 }
 
 void Palette::applyRemap(const Remap& remap)

src/doc/palette.h

@@ -101,7 +101,7 @@ namespace doc {
     int findExactMatch(int r, int g, int b, int a, int mask_index) const;
     bool findExactMatch(color_t color) const;
     int findBestfit(int r, int g, int b, int a, int mask_index) const;
-    int findBestfit2(int r, int g, int b, int a) const;
+    int findMaskColor() const;
 
     void applyRemap(const Remap& remap);
 

src/doc/rgbmap.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2020 Igara Studio S.A.
+// Copyright (c) 2020-2022 Igara Studio S.A.
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -25,6 +25,8 @@ namespace doc {
     // Should return the best index in a palette that matches the given RGBA values.
     virtual int mapColor(const color_t rgba) const = 0;
 
+    virtual int maskIndex() const = 0;
+
     int mapColor(const int r,
                  const int g,
                  const int b,

src/doc/rgbmap_rgb5a3.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2020 Igara Studio S.A.
+// Copyright (c) 2020-2022 Igara Studio S.A.
 // Copyright (c) 2001-2016 David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -41,7 +41,7 @@ namespace doc {
       return (v & INVALID) ? generateEntry(i, r, g, b, a): v;
     }
 
-    int maskIndex() const { return m_maskIndex; }
+    int maskIndex() const override { return m_maskIndex; }
 
   private:
     int generateEntry(int i, int r, int g, int b, int a) const;

src/doc/sprite.cpp

@@ -395,9 +395,6 @@ RgbMap* Sprite::rgbMap(const frame_t frame,
                        const RgbMapFor forLayer,
                        RgbMapAlgorithm mapAlgo) const
 {
-  int maskIndex = (forLayer == RgbMapFor::OpaqueLayer ?
-                   -1: transparentColor());
-
   if (!m_rgbMap || m_rgbMapAlgorithm != mapAlgo) {
     m_rgbMapAlgorithm = mapAlgo;
     switch (m_rgbMapAlgorithm) {
@@ -410,7 +407,14 @@ RgbMap* Sprite::rgbMap(const frame_t frame,
         return nullptr;
     }
   }
-
+  int maskIndex;
+  if (forLayer == RgbMapFor::OpaqueLayer)
+    maskIndex = -1;
+  else {
+    maskIndex = palette(frame)->findMaskColor();
+    if (maskIndex == -1)
+      maskIndex = 0;
+  }
   m_rgbMap->regenerateMap(palette(frame), maskIndex);
   return m_rgbMap.get();
 }

src/render/quantization.cpp

@@ -1,5 +1,5 @@
 // Aseprite Render Library
-// Copyright (c) 2019-2021  Igara Studio S.A.
+// Copyright (c) 2019-2022  Igara Studio S.A.
 // Copyright (c) 2001-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -237,7 +237,7 @@ Image* convert_pixel_format(
             a = rgba_geta(c);
 
             if (a == 0)
-              *dst_it = new_mask_color;
+              *dst_it = (new_mask_color == -1? 0 : new_mask_color);
             else if (rgbmap)
               *dst_it = rgbmap->mapColor(c);
             else
@@ -296,7 +296,7 @@ Image* convert_pixel_format(
             c = graya_getv(c);
 
             if (a == 0)
-              *dst_it = new_mask_color;
+              *dst_it = (new_mask_color == -1? 0 : new_mask_color);
             else if (rgbmap)
               *dst_it = rgbmap->mapColor(c, c, c, a);
             else