New default RgbMapAlgorithm to select what could works better at a given time

data/pref.xml

@@ -221,7 +221,6 @@
       <option id="load_wintab_driver" type="bool" default="true" />
       <option id="flash_layer" type="bool" default="false" />
       <option id="nonactive_layers_opacity" type="int" default="255" />
-      <option id="rgbmap_algorithm" type="doc::RgbMapAlgorithm" default="doc::RgbMapAlgorithm::DEFAULT" />
     </section>
     <section id="news">
       <option id="cache_file" type="std::string" />
@@ -295,6 +294,7 @@
       <option id="dithering_factor" type="int" default="100" />
       <option id="to_gray" type="ToGrayAlgorithm" default="ToGrayAlgorithm::DEFAULT" />
       <option id="advanced" type="bool" default="false" />
+      <option id="rgbmap_algorithm" type="doc::RgbMapAlgorithm" default="doc::RgbMapAlgorithm::DEFAULT" />
     </section>
     <section id="eyedropper" text="Editor">
       <option id="channel" type="EyedropperChannel" default="EyedropperChannel::COLOR_ALPHA" />

data/strings/en.ini

@@ -1076,6 +1076,7 @@ double_high = Double-high Pixels (1:2)
 
 [rgbmap_algorithm_selector]
 label = RGB to palette index mapping:
+default = Default (chose best algorithm automatically)
 rgb5a3 = Table RGB 5 bits + Alpha 3 bits
 octree = Octree without Alpha
 

src/app/commands/cmd_change_pixel_format.cpp

@@ -229,7 +229,7 @@ public:
       }
 
       // Select default RgbMap algorithm
-      m_mapAlgorithmSelector->algorithm(pref.experimental.rgbmapAlgorithm());
+      m_mapAlgorithmSelector->algorithm(pref.quantization.rgbmapAlgorithm());
 
       ditheringPlaceholder()->addChild(m_ditheringSelector);
       rgbmapAlgorithmPlaceholder()->addChild(m_mapAlgorithmSelector);
@@ -548,9 +548,11 @@ void ChangePixelFormatCommand::onLoadParams(const Params& params)
     m_rgbmap = doc::RgbMapAlgorithm::OCTREE;
   else if (rgbmap == "rgb5a3")
     m_rgbmap = doc::RgbMapAlgorithm::RGB5A3;
+  else if (rgbmap == "default")
+    m_rgbmap = doc::RgbMapAlgorithm::DEFAULT;
   else {
     // Use the configured algorithm by default.
-    m_rgbmap = Preferences::instance().experimental.rgbmapAlgorithm();
+    m_rgbmap = Preferences::instance().quantization.rgbmapAlgorithm();
   }
 
   std::string toGray = params.get("toGray");

src/app/commands/cmd_color_quantization.cpp

@@ -57,6 +57,7 @@ public:
     m_algoSelector.Change.connect(
       [this](){
         switch (algorithm()) {
+          case RgbMapAlgorithm::DEFAULT:
           case RgbMapAlgorithm::RGB5A3:
             alphaChannel()->setEnabled(true);
             break;
@@ -126,7 +127,7 @@ void ColorQuantizationCommand::onExecute(Context* ctx)
       const Palette* curPalette = site.sprite()->palette(site.frame());
 
       if (!params().algorithm.isSet())
-        algorithm = pref.experimental.rgbmapAlgorithm();
+        algorithm = pref.quantization.rgbmapAlgorithm();
       if (!params().withAlpha.isSet())
         withAlpha = pref.quantization.withAlpha();
 

src/app/commands/cmd_new_file.cpp

@@ -264,7 +264,7 @@ void NewFileCommand::onExecute(Context* ctx)
         render::create_palette_from_sprite(
           sprite.get(), 0, sprite->lastFrame(), true,
           &clipboardPalette, nullptr, true,
-          Preferences::instance().experimental.rgbmapAlgorithm());
+          Preferences::instance().quantization.rgbmapAlgorithm());
       }
       sprite->setPalette(&clipboardPalette, false);
     }

src/app/commands/cmd_options.cpp

@@ -403,7 +403,7 @@ public:
 
     rgbmapAlgorithmPlaceholder()->addChild(&m_rgbmapAlgorithmSelector);
     m_rgbmapAlgorithmSelector.setExpansive(true);
-    m_rgbmapAlgorithmSelector.algorithm(m_pref.experimental.rgbmapAlgorithm());
+    m_rgbmapAlgorithmSelector.algorithm(m_pref.quantization.rgbmapAlgorithm());
 
     if (m_pref.editor.showScrollbars())
       showScrollbars()->setSelected(true);
@@ -703,7 +703,7 @@ public:
     m_pref.experimental.useNativeFileDialog(nativeFileDialog()->isSelected());
     m_pref.experimental.flashLayer(flashLayer()->isSelected());
     m_pref.experimental.nonactiveLayersOpacity(nonactiveLayersOpacity()->getValue());
-    m_pref.experimental.rgbmapAlgorithm(m_rgbmapAlgorithmSelector.algorithm());
+    m_pref.quantization.rgbmapAlgorithm(m_rgbmapAlgorithmSelector.algorithm());
 
 #ifdef _WIN32
     {

src/app/file/file_op_config.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2019  Igara Studio S.A.
+// Copyright (C) 2019-2021  Igara Studio S.A.
 //
 // This program is distributed under the terms of
 // the End-User License Agreement for Aseprite.
@@ -22,7 +22,7 @@ void FileOpConfig::fillFromPreferences()
   newBlend = Preferences::instance().experimental.newBlend();
   defaultSliceColor = Preferences::instance().slices.defaultColor();
   workingCS = get_working_rgb_space_from_preferences();
-  rgbMapAlgorithm = Preferences::instance().experimental.rgbmapAlgorithm();
+  rgbMapAlgorithm = Preferences::instance().quantization.rgbmapAlgorithm();
 }
 
 } // namespace app

src/app/pref/preferences.cpp

@@ -56,11 +56,11 @@ Preferences::Preferences()
 
   // Create a connection with the default RgbMapAlgorithm preferences
   // to change the default algorithm in the "doc" layer.
-  experimental.rgbmapAlgorithm.AfterChange.connect(
+  quantization.rgbmapAlgorithm.AfterChange.connect(
     [](const doc::RgbMapAlgorithm& newValue){
       doc::Sprite::SetDefaultRgbMapAlgorithm(newValue);
     });
-  doc::Sprite::SetDefaultRgbMapAlgorithm(experimental.rgbmapAlgorithm());
+  doc::Sprite::SetDefaultRgbMapAlgorithm(quantization.rgbmapAlgorithm());
 
   // Create a connection with the default document preferences grid
   // bounds to sync the default grid bounds for new sprites in the

src/app/ui/rgbmap_algorithm_selector.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2020  Igara Studio S.A.
+// Copyright (C) 2020-2021  Igara Studio S.A.
 //
 // This program is distributed under the terms of
 // the End-User License Agreement for Aseprite.
@@ -17,10 +17,12 @@ namespace app {
 RgbMapAlgorithmSelector::RgbMapAlgorithmSelector()
 {
   // addItem() must match the RgbMapAlgorithm enum
-  static_assert(int(doc::RgbMapAlgorithm::RGB5A3) == 0 &&
-                int(doc::RgbMapAlgorithm::OCTREE) == 1,
+  static_assert(int(doc::RgbMapAlgorithm::DEFAULT) == 0 &&
+                int(doc::RgbMapAlgorithm::RGB5A3) == 1 &&
+                int(doc::RgbMapAlgorithm::OCTREE) == 2,
                 "Unexpected doc::RgbMapAlgorithm values");
 
+  addItem(Strings::rgbmap_algorithm_selector_default());
   addItem(Strings::rgbmap_algorithm_selector_rgb5a3());
   addItem(Strings::rgbmap_algorithm_selector_octree());
 

src/doc/palette.cpp

@@ -108,6 +108,16 @@ bool Palette::hasAlpha() const
   return false;
 }
 
+bool Palette::hasSemiAlpha() const
+{
+  for (int i=0; i<(int)m_colors.size(); ++i) {
+    int a = rgba_geta(getEntry(i));
+    if (a > 0 && a < 255)
+      return true;
+  }
+  return false;
+}
+
 void Palette::setFrame(frame_t frame)
 {
   ASSERT(frame >= 0);

src/doc/palette.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (C) 2020  Igara Studio S.A.
+// Copyright (C) 2020-2021  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -53,6 +53,9 @@ namespace doc {
     // Return true if the palette has alpha != 255 in some entry
     bool hasAlpha() const;
 
+    // Return true if the palette has an alpha value between > 0 and < 255.
+    bool hasSemiAlpha() const;
+
     frame_t frame() const { return m_frame; }
     void setFrame(frame_t frame);
 

src/doc/rgbmap_algorithm.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2020 Igara Studio S.A.
+// Copyright (c) 2020-2021 Igara Studio S.A.
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -11,9 +11,9 @@
 namespace doc {
 
   enum class RgbMapAlgorithm {
+    DEFAULT, // Select best algorithm (generally octree when alpha is=255 in all colors)
     RGB5A3,
     OCTREE,
-    DEFAULT = RGB5A3
   };
 
 } // namespace doc

src/doc/sprite.cpp

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (C) 2018-2020  Igara Studio S.A.
+// Copyright (C) 2018-2021  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -388,11 +388,21 @@ RgbMap* Sprite::rgbMap(const frame_t frame,
 
 RgbMap* Sprite::rgbMap(const frame_t frame,
                        const RgbMapFor forLayer,
-                       const RgbMapAlgorithm mapAlgo) const
+                       RgbMapAlgorithm mapAlgo) const
 {
   int maskIndex = (forLayer == RgbMapFor::OpaqueLayer ?
                    -1: transparentColor());
 
+  if (mapAlgo == RgbMapAlgorithm::DEFAULT) {
+    mapAlgo = RgbMapAlgorithm::OCTREE;
+    for (const auto& pal : getPalettes()) {
+      if (pal->hasSemiAlpha()) {
+        mapAlgo = RgbMapAlgorithm::RGB5A3;
+        break;
+      }
+    }
+  }
+
   if (!m_rgbMap || m_rgbMapAlgorithm != mapAlgo) {
     m_rgbMapAlgorithm = mapAlgo;
     switch (m_rgbMapAlgorithm) {

src/doc/sprite.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (C) 2018-2020  Igara Studio S.A.
+// Copyright (C) 2018-2021  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -149,7 +149,7 @@ namespace doc {
                    const RgbMapFor forLayer) const;
     RgbMap* rgbMap(const frame_t frame,
                    const RgbMapFor forLayer,
-                   const RgbMapAlgorithm mapAlgo) const;
+                   RgbMapAlgorithm mapAlgo) const;
 
     ////////////////////////////////////////
     // Frames

src/render/quantization.cpp

@@ -1,5 +1,5 @@
 // Aseprite Render Library
-// Copyright (c) 2019-2020  Igara Studio S.A.
+// Copyright (c) 2019-2021  Igara Studio S.A.
 // Copyright (c) 2001-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -43,7 +43,7 @@ Palette* create_palette_from_sprite(
   Palette* palette,
   TaskDelegate* delegate,
   const bool newBlend,
-  const RgbMapAlgorithm mappingAlgorithm,
+  RgbMapAlgorithm mapAlgo,
   const bool calculateWithTransparent)
 {
   PaletteOptimizer optimizer;
@@ -59,19 +59,41 @@ Palette* create_palette_from_sprite(
   ImageRef flat_image(Image::create(IMAGE_RGB,
       sprite->width(), sprite->height()));
 
-  // Feed the optimizer with all rendered frames
   render::Render render;
   render.setNewBlend(newBlend);
+
+  // Use octree if there are no semi-transparent pixels.
+  if (mapAlgo == RgbMapAlgorithm::DEFAULT) {
+    for (frame_t frame=fromFrame;
+         frame<=toFrame && mapAlgo == RgbMapAlgorithm::DEFAULT;
+         ++frame) {
+      render.renderSprite(flat_image.get(), sprite, frame);
+      doc::for_each_pixel<RgbTraits>(
+        flat_image.get(),
+        [&mapAlgo](const color_t p) {
+          if (rgba_geta(p) > 0 && rgba_geta(p) < 255) {
+            mapAlgo = RgbMapAlgorithm::RGB5A3;
+          }
+        });
+    }
+    if (mapAlgo == RgbMapAlgorithm::DEFAULT)
+      mapAlgo = RgbMapAlgorithm::OCTREE;
+  }
+
+  // Feed the optimizer with all rendered frames
   for (frame_t frame=fromFrame; frame<=toFrame; ++frame) {
     render.renderSprite(flat_image.get(), sprite, frame);
 
-    switch (mappingAlgorithm) {
+    switch (mapAlgo) {
       case RgbMapAlgorithm::RGB5A3:
         optimizer.feedWithImage(flat_image.get(), withAlpha);
         break;
       case RgbMapAlgorithm::OCTREE:
         octreemap.feedWithImage(flat_image.get(), maskColor);
         break;
+      default:
+        ASSERT(false);
+        break;
     }
 
     if (delegate) {
@@ -83,7 +105,8 @@ Palette* create_palette_from_sprite(
     }
   }
 
-  switch (mappingAlgorithm) {
+  switch (mapAlgo) {
+
     case RgbMapAlgorithm::RGB5A3:
       // Generate an optimized palette
       optimizer.calculate(
@@ -93,6 +116,7 @@ Palette* create_palette_from_sprite(
           sprite->allLayersCount() == 1) ||
          !calculateWithTransparent)? -1: sprite->transparentColor());
       break;
+
     case RgbMapAlgorithm::OCTREE:
       // TODO check calculateWithTransparent flag
 

src/render/quantization.h

@@ -1,5 +1,5 @@
 // Aseprite Rener Library
-// Copyright (c) 2019-2020  Igara Studio S.A.
+// Copyright (c) 2019-2021  Igara Studio S.A.
 // Copyright (c) 2001-2017  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -53,7 +53,7 @@ namespace render {
     doc::Palette* newPalette, // Can be NULL to create a new palette
     TaskDelegate* delegate,
     const bool newBlend,
-    const RgbMapAlgorithm mappingAlgorithm,
+    RgbMapAlgorithm mapAlgo,
     const bool calculateWithTransparent = true);
 
   // Changes the image pixel format. The dithering method is used only