Add octree quantization algorithm supports alpha channel

Before this commit, Octree wasn't support alpha channel.
Also the automatic quantization algorithm selection was removed because Octree support alpha channel now.

data/strings/en.ini

@@ -1080,9 +1080,9 @@ double_high = Double-high Pixels (1:2)
 
 [rgbmap_algorithm_selector]
 label = RGB to palette index mapping:
-default = Default (chose best algorithm automatically)
+default = Default (Octree)
 rgb5a3 = Table RGB 5 bits + Alpha 3 bits
-octree = Octree without Alpha
+octree = Octree
 
 [open_sequence]
 title = Notice

src/app/commands/cmd_color_quantization.cpp

@@ -54,19 +54,6 @@ public:
         expandWindow(sizeHint());
       });
 
-    m_algoSelector.Change.connect(
-      [this](){
-        switch (algorithm()) {
-          case RgbMapAlgorithm::DEFAULT:
-          case RgbMapAlgorithm::RGB5A3:
-            alphaChannel()->setEnabled(true);
-            break;
-          case RgbMapAlgorithm::OCTREE:
-            alphaChannel()->setSelected(false);
-            alphaChannel()->setEnabled(false);
-            break;
-        }
-      });
   }
 
   doc::RgbMapAlgorithm algorithm() {

src/app/commands/cmd_select_palette.cpp

@@ -138,7 +138,7 @@ void SelectPaletteColorsCommand::onExecute(Context* context)
 
         case IMAGE_RGB:
         case IMAGE_GRAYSCALE:
-          octreemap.feedWithImage(image, image->maskColor(), 8);
+          octreemap.feedWithImage(image, true, image->maskColor(), 8);
           break;
 
         case IMAGE_INDEXED:

src/doc/octree_map.cpp

@@ -13,6 +13,7 @@
 #include "doc/palette.h"
 
 #define MIN_LEVEL_OCTREE_DEEP 3
+#define MIN_ALPHA_THRESHOLD 16
 
 namespace doc {
 
@@ -28,9 +29,9 @@ void OctreeNode::addColor(color_t c, int level, OctreeNode* parent,
     m_paletteIndex = paletteIndex;
     return;
   }
-  int index = getOctet(c, level);
+  int index = getHextet(c, level);
   if (!m_children) {
-    m_children.reset(new std::array<OctreeNode, 8>());
+    m_children.reset(new std::array<OctreeNode, 16>());
   }
   (*m_children)[index].addColor(c, level + 1, this, paletteIndex, levelDeep);
 }
@@ -39,13 +40,13 @@ void OctreeNode::fillOrphansNodes(const Palette* palette,
                                   const color_t upstreamBranchColor,
                                   const int level)
 {
-  for (int i=0; i<8; i++) {
+  for (int i=0; i<16; i++) {
     OctreeNode& child = (*m_children)[i];
 
     if (child.hasChildren()) {
       child.fillOrphansNodes(
         palette,
-        upstreamBranchColor + octetToBranchColor(i, level),
+        upstreamBranchColor + hextetToBranchColor(i, level),
         level + 1);
     }
     else if (!(child.isLeaf())) {
@@ -60,14 +61,14 @@ void OctreeNode::fillOrphansNodes(const Palette* palette,
         i--;
         continue;
       }
-      int currentBranchColorAdd = octetToBranchColor(i, level);
+      int currentBranchColorAdd = hextetToBranchColor(i, level);
-      color_t branchColorMed = rgba_a_mask |
+      color_t branchColorMed = upstreamBranchColor |
-                               upstreamBranchColor |
                                currentBranchColorAdd |
-                               ((level == 7) ? 0 : (0x00010101 << (6 - level))); // mid color adition
+                               ((level == 7) ? 0 : (0x01010101 << (6 - level))); // mid color adition
       int indexMed = palette->findBestfit2(rgba_getr(branchColorMed),
                                            rgba_getg(branchColorMed),
-                                           rgba_getb(branchColorMed));
+                                           rgba_getb(branchColorMed),
+                                           rgba_geta(branchColorMed));
       child.paletteIndex(indexMed);
     }
   }
@@ -76,9 +77,9 @@ void OctreeNode::fillOrphansNodes(const Palette* palette,
 void OctreeNode::fillMostSignificantNodes(int level)
 {
   if (level < MIN_LEVEL_OCTREE_DEEP) {
-    m_children.reset(new std::array<OctreeNode, 8>());
+    m_children.reset(new std::array<OctreeNode, 16>());
     level++;
-    for (int i=0; i<8; i++) {
+    for (int i=0; i<16; i++) {
       OctreeNode& child = (*m_children)[i];
 
       child.fillMostSignificantNodes(level);
@@ -86,18 +87,18 @@ void OctreeNode::fillMostSignificantNodes(int level)
   }
 }
 
-int OctreeNode::mapColor(int r, int g, int b, int level) const
+int OctreeNode::mapColor(int r, int g, int b, int a, int level) const
 {
-  OctreeNode& child = (*m_children)[getOctet(rgba(r, g, b, 0), level)];
+  OctreeNode& child = (*m_children)[getHextet(rgba(r, g, b, a), level)];
   if (child.hasChildren())
-    return child.mapColor(r, g, b, level+1);
+    return child.mapColor(r, g, b, a, level+1);
   else
     return child.m_paletteIndex;
 }
 
 void OctreeNode::collectLeafNodes(OctreeNodes& leavesVector, int& paletteIndex)
 {
-  for (int i=0; i<8; i++) {
+  for (int i=0; i<16; i++) {
     OctreeNode& child = (*m_children)[i];
 
     if (child.isLeaf()) {
@@ -115,12 +116,11 @@ void OctreeNode::collectLeafNodes(OctreeNodes& leavesVector, int& paletteIndex)
 // auxParentVector: i/o addreess of an auxiliary parent leaf Vector from outside this function.
 // rootLeavesVector: i/o address of the m_root->m_leavesVector
 int OctreeNode::removeLeaves(OctreeNodes& auxParentVector,
-                             OctreeNodes& rootLeavesVector,
+                             OctreeNodes& rootLeavesVector)
-                             int octreeDeep)
 {
   // Apply to OctreeNode which has children which are leaf nodes
   int result = 0;
-  for (int i=octreeDeep; i>=0; i--) {
+  for (int i=15; i>=0; i--) {
     OctreeNode& child = (*m_children)[i];
 
     if (child.isLeaf()) {
@@ -135,26 +135,28 @@ int OctreeNode::removeLeaves(OctreeNodes& auxParentVector,
 }
 
 // static
-int OctreeNode::getOctet(color_t c, int level)
+int OctreeNode::getHextet(color_t c, int level)
 {
   return ((c & (0x00000080 >> level)) ? 1 : 0) |
          ((c & (0x00008000 >> level)) ? 2 : 0) |
-         ((c & (0x00800000 >> level)) ? 4 : 0);
+         ((c & (0x00800000 >> level)) ? 4 : 0) |
+         ((c & (0x80000000 >> level)) ? 8 : 0);
 }
 
 // static
-color_t OctreeNode::octetToBranchColor(int octet, int level)
+color_t OctreeNode::hextetToBranchColor(int hextet, int level)
 {
-  return ((octet & 1) ? 0x00000080 >> level : 0) |
+  return ((hextet & 1) ? 0x00000080 >> level : 0) |
-         ((octet & 2) ? 0x00008000 >> level : 0) |
+         ((hextet & 2) ? 0x00008000 >> level : 0) |
-         ((octet & 4) ? 0x00800000 >> level : 0);
+         ((hextet & 4) ? 0x00800000 >> level : 0) |
+         ((hextet & 8) ? 0x80000000 >> level : 0);
 }
 
 //////////////////////////////////////////////////////////////////////
 // OctreeMap
 
 bool OctreeMap::makePalette(Palette* palette,
-                            const int colorCount,
+                            int colorCount,
                             const int levelDeep)
 {
   if (m_root.hasChildren()) {
@@ -165,6 +167,9 @@ bool OctreeMap::makePalette(Palette* palette,
     m_root.collectLeafNodes(m_leavesVector, paletteIndex);
   }
 
+  if (m_maskColor != DOC_OCTREE_IS_OPAQUE)
+    colorCount--;
+
   // If we can improve the octree accuracy, makePalette returns false, then
   // outside from this function we must re-construct the octreeMap all again with
   // deep level equal to 8.
@@ -184,14 +189,14 @@ bool OctreeMap::makePalette(Palette* palette,
         break;
       }
       else if (m_leavesVector.size() == 0) {
-        // When colorCount is < 8, auxLeavesVector->size() could reach the 8 size,
+        // When colorCount is < 16, auxLeavesVector->size() could reach the 16 size,
         // if this is true and we don't stop the regular removeLeaves algorithm,
-        // the 8 remains colors will collapse in one.
+        // the 16 remains colors will collapse in one.
         // So, we have to reduce color with other method:
         // Sort colors by pixelCount (most pixelCount on front of sortedVector),
         // then:
         // Blend in pairs from the least pixelCount colors.
-        if (auxLeavesVector.size() <= 8 && colorCount < 8 && colorCount > 0) {
+        if (auxLeavesVector.size() <= 16 && colorCount < 16 && colorCount > 0) {
           // Sort colors:
           OctreeNodes sortedVector;
           int auxVectorSize = auxLeavesVector.size();
@@ -241,7 +246,7 @@ bool OctreeMap::makePalette(Palette* palette,
   }
   int leafCount = m_leavesVector.size();
   int aux = 0;
-  if (m_maskColor == 0x00FFFFFF)
+  if (m_maskColor == DOC_OCTREE_IS_OPAQUE)
     palette->resize(leafCount);
   else {
     palette->resize(leafCount + 1);
@@ -262,37 +267,44 @@ void OctreeMap::fillOrphansNodes(const Palette* palette)
 }
 
 void OctreeMap::feedWithImage(const Image* image,
+                              const bool withAlpha,
                               const color_t maskColor,
                               const int levelDeep)
 {
   ASSERT(image);
   ASSERT(image->pixelFormat() == IMAGE_RGB || image->pixelFormat() == IMAGE_GRAYSCALE);
-  uint32_t color;
+  color_t forceFullOpacity;
-  if (image->pixelFormat() == IMAGE_RGB) {
+  color_t alpha = 0;
-    const LockImageBits<RgbTraits> bits(image);
+  const bool imageIsRGBA = image->pixelFormat() == IMAGE_RGB;
-    auto it = bits.begin(), end = bits.end();
 
-    for (; it != end; ++it) {
+  auto add_color_to_octree =
-      color = *it;
+    [this, &forceFullOpacity, &maskColor, &alpha, &levelDeep, &imageIsRGBA](color_t color) {
-      if (rgba_geta(color) > 0)
+      if (color != maskColor) {
-        addColor(color, levelDeep);
+        color |= forceFullOpacity;
+        alpha = (imageIsRGBA ? rgba_geta(color) : graya_geta(color));
+        if (alpha >= MIN_ALPHA_THRESHOLD) { // Colors which alpha is less than
+                                            // MIN_ALPHA_THRESHOLD will not registered
+          color = (imageIsRGBA ? color : rgba(graya_getv(color),
+                                              graya_getv(color),
+                                              graya_getv(color),
+                                              graya_geta(color)));
+          addColor(color, levelDeep);
+        }
+      }
+    };
+
+  switch (image->pixelFormat()) {
+    case IMAGE_RGB: {
+      forceFullOpacity = (withAlpha) ? 0 : rgba_a_mask;
+      doc::for_each_pixel<RgbTraits>(image, add_color_to_octree);
+      break;
+    }
+    case IMAGE_GRAYSCALE: {
+      forceFullOpacity = (withAlpha) ? 0 : graya_a_mask;
+      doc::for_each_pixel<GrayscaleTraits>(image, add_color_to_octree);
+      break;
     }
   }
-  else {
-    const LockImageBits<GrayscaleTraits> bits(image);
-    auto it = bits.begin(), end = bits.end();
-
-    for (; it != end; ++it) {
-      color = *it;
-      if (graya_geta(color) > 0)
-        addColor(rgba(graya_getv(color),
-                      graya_getv(color),
-                      graya_getv(color),
-                      255), levelDeep);
-    }
-  }
-
-
   m_maskColor = maskColor;
 }
 
@@ -301,7 +313,8 @@ int OctreeMap::mapColor(color_t rgba) const
   if (m_root.hasChildren())
     return m_root.mapColor(rgba_getr(rgba),
                            rgba_getg(rgba),
-                           rgba_getb(rgba), 0);
+                           rgba_getb(rgba),
+                           rgba_geta(rgba), 0);
   else
     return -1;
 }
@@ -323,14 +336,15 @@ void OctreeMap::regenerateMap(const Palette* palette, const int maskIndex)
   m_maskIndex = maskIndex;
   int maskColorBestFitIndex;
   if (maskIndex < 0) {
-    m_maskColor = 0x00ffffff;
+    m_maskColor = DOC_OCTREE_IS_OPAQUE;
     maskColorBestFitIndex = -1;
   }
   else {
     m_maskColor = palette->getEntry(maskIndex);
     maskColorBestFitIndex = palette->findBestfit(rgba_getr(m_maskColor),
                                                  rgba_getg(m_maskColor),
-                                                 rgba_getb(m_maskColor), 255, maskIndex);
+                                                 rgba_getb(m_maskColor),
+                                                 rgba_geta(m_maskColor), maskIndex);
   }
 
   for (int i=0; i<palette->size(); i++) {

src/doc/octree_map.h

@@ -17,6 +17,12 @@
 #include <memory>
 #include <vector>
 
+// When this DOC_OCTREE_IS_OPAQUE 'color' is asociated with
+// some variable which represents a mask color, it tells us that
+// there isn't any transparent color in the sprite, i.e.
+// there is a background layer in the sprite.
+#define DOC_OCTREE_IS_OPAQUE 0x00FFFFFF
+
 namespace doc {
 
 class OctreeNode;
@@ -30,13 +36,15 @@ private:
       m_r(0),
       m_g(0),
       m_b(0),
+      m_a(0),
       m_pixelCount(0) {
     }
 
-    LeafColor(int r, int g, int b, size_t pixelCount) :
+    LeafColor(int r, int g, int b, int a, size_t pixelCount) :
       m_r((double)r),
       m_g((double)g),
       m_b((double)b),
+      m_a((double)a),
       m_pixelCount(pixelCount) {
     }
 
@@ -44,6 +52,7 @@ private:
       m_r += rgba_getr(c);
       m_g += rgba_getg(c);
       m_b += rgba_getb(c);
+      m_a += rgba_geta(c);
       ++m_pixelCount;
     }
 
@@ -51,6 +60,7 @@ private:
       m_r += leafColor.m_r;
       m_g += leafColor.m_g;
       m_b += leafColor.m_b;
+      m_a += leafColor.m_a;
       m_pixelCount += leafColor.m_pixelCount;
     }
 
@@ -58,9 +68,11 @@ private:
       int auxR = (((int)m_r) % m_pixelCount > m_pixelCount / 2) ? 1: 0;
       int auxG = (((int)m_g) % m_pixelCount > m_pixelCount / 2) ? 1: 0;
       int auxB = (((int)m_b) % m_pixelCount > m_pixelCount / 2) ? 1: 0;
+      int auxA = (((int)m_a) % m_pixelCount > m_pixelCount / 2) ? 1: 0;
       return rgba(int(m_r / m_pixelCount + auxR),
                   int(m_g / m_pixelCount + auxG),
-                  int(m_b / m_pixelCount + auxB), 255);
+                  int(m_b / m_pixelCount + auxB),
+                  int(m_a / m_pixelCount + auxA));
     }
 
     size_t pixelCount() const { return m_pixelCount; }
@@ -69,6 +81,7 @@ private:
     double m_r;
     double m_g;
     double m_b;
+    double m_a;
     size_t m_pixelCount;
   };
 
@@ -86,7 +99,7 @@ public:
 
   void fillMostSignificantNodes(int level);
 
-  int mapColor(int r, int g, int b, int level) const;
+  int mapColor(int r, int g, int b, int a, int level) const;
 
   void collectLeafNodes(OctreeNodes& leavesVector, int& paletteIndex);
 
@@ -94,19 +107,18 @@ public:
   // auxParentVector: i/o addreess of an auxiliary parent leaf Vector from outside.
   // rootLeavesVector: i/o address of the m_root->m_leavesVector
   int removeLeaves(OctreeNodes& auxParentVector,
-                   OctreeNodes& rootLeavesVector,
+                   OctreeNodes& rootLeavesVector);
-                   int octreeDeep = 7);
 
 private:
   bool isLeaf() { return m_leafColor.pixelCount() > 0; }
   void paletteIndex(int index) { m_paletteIndex = index; }
 
-  static int getOctet(color_t c, int level);
+  static int getHextet(color_t c, int level);
-  static color_t octetToBranchColor(int octet, int level);
+  static color_t hextetToBranchColor(int hextet, int level);
 
   LeafColor m_leafColor;
   int m_paletteIndex = 0;
-  std::unique_ptr<std::array<OctreeNode, 8>> m_children;
+  std::unique_ptr<std::array<OctreeNode, 16>> m_children;
   OctreeNode* m_parent = nullptr;
 };
 
@@ -119,10 +131,11 @@ public:
   // makePalette returns true if a 7 level octreeDeep is OK, and false
   // if we can add ONE level deep.
   bool makePalette(Palette* palette,
-                   const int colorCount,
+                   int colorCount,
                    const int levelDeep = 7);
 
   void feedWithImage(const Image* image,
+                     const bool withAlpha,
                      const color_t maskColor,
                      const int levelDeep = 7);
 

src/doc/palette.cpp

@@ -369,11 +369,12 @@ int Palette::findBestfit(int r, int g, int b, int a, int mask_index) const
   return bestfit;
 }
 
-int Palette::findBestfit2(int r, int g, int b) const
+int Palette::findBestfit2(int r, int g, int b, int a) const
 {
   ASSERT(r >= 0 && r <= 255);
   ASSERT(g >= 0 && g <= 255);
   ASSERT(b >= 0 && b <= 255);
+  ASSERT(a >= 0 && a <= 255);
 
   int bestfit = 0;
   int lowest = std::numeric_limits<int>::max();
@@ -384,11 +385,15 @@ int Palette::findBestfit2(int r, int g, int b) const
     int rDiff = r - rgba_getr(rgb);
     int gDiff = g - rgba_getg(rgb);
     int bDiff = b - rgba_getb(rgb);
+    int aDiff = a - rgba_geta(rgb);
 
     // TODO We should have two different ways to calculate the
     // distance between colors, like "Perceptual" and "Linear", or a
     // way to configure these coefficients.
-    int diff = rDiff * rDiff * 900 + gDiff * gDiff * 3481 + bDiff * bDiff * 121;
+    int diff = rDiff * rDiff * 900  +
+               gDiff * gDiff * 3481 +
+               bDiff * bDiff * 121  +
+               aDiff * aDiff * 900; // there is no scientific reason to choose this value.
     if (diff < lowest) {
       lowest = diff;
       bestfit = i;

src/doc/palette.h

@@ -99,7 +99,7 @@ namespace doc {
     int findExactMatch(int r, int g, int b, int a, int mask_index) const;
     bool findExactMatch(color_t color) const;
     int findBestfit(int r, int g, int b, int a, int mask_index) const;
-    int findBestfit2(int r, int g, int b) const;
+    int findBestfit2(int r, int g, int b, int a) const;
 
     void applyRemap(const Remap& remap);
 

src/doc/rgbmap_algorithm.h

@@ -11,9 +11,9 @@
 namespace doc {
 
   enum class RgbMapAlgorithm {
-    DEFAULT, // Select best algorithm (generally octree when alpha is=255 in all colors)
+    DEFAULT = 0,
-    RGB5A3,
+    RGB5A3 = 1,
-    OCTREE,
+    OCTREE = 2,
   };
 
 } // namespace doc

src/doc/sprite.cpp

@@ -393,20 +393,11 @@ RgbMap* Sprite::rgbMap(const frame_t frame,
   int maskIndex = (forLayer == RgbMapFor::OpaqueLayer ?
                    -1: transparentColor());
 
-  if (mapAlgo == RgbMapAlgorithm::DEFAULT) {
-    mapAlgo = RgbMapAlgorithm::OCTREE;
-    for (const auto& pal : getPalettes()) {
-      if (pal->hasSemiAlpha()) {
-        mapAlgo = RgbMapAlgorithm::RGB5A3;
-        break;
-      }
-    }
-  }
-
   if (!m_rgbMap || m_rgbMapAlgorithm != mapAlgo) {
     m_rgbMapAlgorithm = mapAlgo;
     switch (m_rgbMapAlgorithm) {
       case RgbMapAlgorithm::RGB5A3: m_rgbMap.reset(new RgbMapRGB5A3); break;
+      case RgbMapAlgorithm::DEFAULT:
       case RgbMapAlgorithm::OCTREE: m_rgbMap.reset(new OctreeMap); break;
       default:
         m_rgbMap.reset(nullptr);

src/render/quantization.cpp

@@ -46,10 +46,13 @@ Palette* create_palette_from_sprite(
   RgbMapAlgorithm mapAlgo,
   const bool calculateWithTransparent)
 {
+   if (mapAlgo == doc::RgbMapAlgorithm::DEFAULT)
+     mapAlgo = doc::RgbMapAlgorithm::OCTREE;
+
   PaletteOptimizer optimizer;
   OctreeMap octreemap;
   const color_t maskColor = (sprite->backgroundLayer()
-                             && sprite->allLayersCount() == 1) ? 0x00FFFFFF:
+                             && sprite->allLayersCount() == 1) ? DOC_OCTREE_IS_OPAQUE:
                                                                  sprite->transparentColor();
 
   if (!palette)
@@ -62,24 +65,6 @@ Palette* create_palette_from_sprite(
   render::Render render;
   render.setNewBlend(newBlend);
 
-  // Use octree if there are no semi-transparent pixels.
-  if (mapAlgo == RgbMapAlgorithm::DEFAULT) {
-    for (frame_t frame=fromFrame;
-         frame<=toFrame && mapAlgo == RgbMapAlgorithm::DEFAULT;
-         ++frame) {
-      render.renderSprite(flat_image.get(), sprite, frame);
-      doc::for_each_pixel<RgbTraits>(
-        flat_image.get(),
-        [&mapAlgo](const color_t p) {
-          if (rgba_geta(p) > 0 && rgba_geta(p) < 255) {
-            mapAlgo = RgbMapAlgorithm::RGB5A3;
-          }
-        });
-    }
-    if (mapAlgo == RgbMapAlgorithm::DEFAULT)
-      mapAlgo = RgbMapAlgorithm::OCTREE;
-  }
-
   // Feed the optimizer with all rendered frames
   for (frame_t frame=fromFrame; frame<=toFrame; ++frame) {
     render.renderSprite(flat_image.get(), sprite, frame);
@@ -89,7 +74,7 @@ Palette* create_palette_from_sprite(
         optimizer.feedWithImage(flat_image.get(), withAlpha);
         break;
       case RgbMapAlgorithm::OCTREE:
-        octreemap.feedWithImage(flat_image.get(), maskColor);
+        octreemap.feedWithImage(flat_image.get(), withAlpha, maskColor);
         break;
       default:
         ASSERT(false);
@@ -126,7 +111,7 @@ Palette* create_palette_from_sprite(
         octreemap = OctreeMap();
         for (frame_t frame=fromFrame; frame<=toFrame; ++frame) {
           render.renderSprite(flat_image.get(), sprite, frame);
-          octreemap.feedWithImage(flat_image.get(), maskColor , 8);
+          octreemap.feedWithImage(flat_image.get(), withAlpha, maskColor , 8);
           if (delegate) {
             if (!delegate->continueTask())
               return nullptr;