Fix changing random values to transparent color for non-indexed sprite from cmd::SetPalette

Possible fix for #2970
Possible regression introduced in dd2d226264

src/app/cmd/set_palette.cpp

@@ -53,13 +53,19 @@ SetPalette::SetPalette(Sprite* sprite, frame_t frame, const Palette* newPalette)
         m_newColors[i] = newPalette->getEntry(m_from+i);
     }
   }
+
   if (sprite->pixelFormat() == IMAGE_INDEXED) {
     m_oldTransparentIndex = sprite->transparentColor();
     if (m_oldTransparentIndex >= newPalette->size())
-      m_newTransparentIndex = newPalette->size() -1;
+      m_newTransparentIndex = newPalette->size() - 1;
     else
       m_newTransparentIndex = m_oldTransparentIndex;
   }
+  else {
+    ASSERT(sprite->transparentColor() == 0);
+    m_oldTransparentIndex = 0;
+    m_newTransparentIndex = 0;
+  }
 }
 
 void SetPalette::onExecute()
@@ -71,7 +77,9 @@ void SetPalette::onExecute()
   for (size_t i=0; i<m_newColors.size(); ++i)
     palette->setEntry(m_from+i, m_newColors[i]);
 
-  sprite->setTransparentColor(m_newTransparentIndex);
+  if (m_newTransparentIndex != m_oldTransparentIndex)
+    sprite->setTransparentColor(m_newTransparentIndex);
+
   palette->incrementVersion();
 }
 
@@ -84,7 +92,9 @@ void SetPalette::onUndo()
   for (size_t i=0; i<m_oldColors.size(); ++i)
     palette->setEntry(m_from+i, m_oldColors[i]);
 
-  sprite->setTransparentColor(m_oldTransparentIndex);
+  if (m_newTransparentIndex != m_oldTransparentIndex)
+    sprite->setTransparentColor(m_oldTransparentIndex);
+
   palette->incrementVersion();
 }
 

src/dio/aseprite_decoder.cpp

@@ -1,5 +1,5 @@
 // Aseprite Document IO Library
-// Copyright (c) 2018-2020 Igara Studio S.A.
+// Copyright (c) 2018-2021 Igara Studio S.A.
 // Copyright (c) 2001-2018 David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -277,6 +277,9 @@ bool AsepriteDecoder::readHeader(AsepriteHeader* header)
   header->grid_width   = read16();
   header->grid_height  = read16();
 
+  if (header->depth != 8)       // Transparent index only valid for indexed images
+    header->transparent_index = 0;
+
   if (header->ncolors == 0)     // 0 means 256 (old .ase files)
     header->ncolors = 256;
 

src/doc/sprite.cpp

@@ -195,6 +195,12 @@ bool Sprite::supportAlpha() const
 
 void Sprite::setTransparentColor(color_t color)
 {
+#if _DEBUG
+  if (colorMode() != ColorMode::INDEXED) {
+    ASSERT(color == 0);
+  }
+#endif // _DEBUG
+
   m_spec.setMaskColor(color);
 
   // Change the mask color of all images.

src/render/quantization.cpp

@@ -1,5 +1,5 @@
 // Aseprite Render Library
-// Copyright (c) 2019-2020  Igara Studio S.A.
+// Copyright (c) 2019-2021  Igara Studio S.A.
 // Copyright (c) 2001-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -69,12 +69,19 @@ Palette* create_palette_from_sprite(
     }
   }
 
+  // Transparent color is needed if we have transparent layers
+  int maskIndex;
+  if (sprite->backgroundLayer() && sprite->allLayersCount() == 1)
+    maskIndex = -1;
+  else if (sprite->colorMode() == ColorMode::INDEXED)
+    maskIndex = sprite->transparentColor();
+  else {
+    ASSERT(sprite->transparentColor() == 0);
+    maskIndex = 0; // For RGB/Grayscale images we use index 0 as the transparent index by default
+  }
+
   // Generate an optimized palette
-  optimizer.calculate(
-    palette,
-    // Transparent color is needed if we have transparent layers
-    (sprite->backgroundLayer() &&
-     sprite->allLayersCount() == 1 ? -1: sprite->transparentColor()));
+  optimizer.calculate(palette, maskIndex);
 
   return palette;
 }