From d7ddb7feed2e377a9d3069448047681d78fbe661 Mon Sep 17 00:00:00 2001
From: David Capello <david@igarastudio.com>
Date: Wed, 29 Sep 2021 14:42:55 -0300
Subject: [PATCH] Fix changing random values to transparent color for
 non-indexed sprite from cmd::SetPalette

Possible fix for #2970
Possible regression introduced in dd2d226264b0960145a7e04a97a4aa3669330ad3
---
 src/app/cmd/set_palette.cpp  | 16 +++++++++++++---
 src/dio/aseprite_decoder.cpp |  5 ++++-
 src/doc/sprite.cpp           |  6 ++++++
 src/render/quantization.cpp  | 19 +++++++++++++------
 4 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/src/app/cmd/set_palette.cpp b/src/app/cmd/set_palette.cpp
index bfda55dd5..7e893c381 100644
--- a/src/app/cmd/set_palette.cpp
+++ b/src/app/cmd/set_palette.cpp
@@ -53,13 +53,19 @@ SetPalette::SetPalette(Sprite* sprite, frame_t frame, const Palette* newPalette)
         m_newColors[i] = newPalette->getEntry(m_from+i);
     }
   }
+
   if (sprite->pixelFormat() == IMAGE_INDEXED) {
     m_oldTransparentIndex = sprite->transparentColor();
     if (m_oldTransparentIndex >= newPalette->size())
-      m_newTransparentIndex = newPalette->size() -1;
+      m_newTransparentIndex = newPalette->size() - 1;
     else
       m_newTransparentIndex = m_oldTransparentIndex;
   }
+  else {
+    ASSERT(sprite->transparentColor() == 0);
+    m_oldTransparentIndex = 0;
+    m_newTransparentIndex = 0;
+  }
 }
 
 void SetPalette::onExecute()
@@ -71,7 +77,9 @@ void SetPalette::onExecute()
   for (size_t i=0; i<m_newColors.size(); ++i)
     palette->setEntry(m_from+i, m_newColors[i]);
 
-  sprite->setTransparentColor(m_newTransparentIndex);
+  if (m_newTransparentIndex != m_oldTransparentIndex)
+    sprite->setTransparentColor(m_newTransparentIndex);
+
   palette->incrementVersion();
 }
 
@@ -84,7 +92,9 @@ void SetPalette::onUndo()
   for (size_t i=0; i<m_oldColors.size(); ++i)
     palette->setEntry(m_from+i, m_oldColors[i]);
 
-  sprite->setTransparentColor(m_oldTransparentIndex);
+  if (m_newTransparentIndex != m_oldTransparentIndex)
+    sprite->setTransparentColor(m_oldTransparentIndex);
+
   palette->incrementVersion();
 }
 
diff --git a/src/dio/aseprite_decoder.cpp b/src/dio/aseprite_decoder.cpp
index be2ab6bfb..56547a2b4 100644
--- a/src/dio/aseprite_decoder.cpp
+++ b/src/dio/aseprite_decoder.cpp
@@ -1,5 +1,5 @@
 // Aseprite Document IO Library
-// Copyright (c) 2018-2020 Igara Studio S.A.
+// Copyright (c) 2018-2021 Igara Studio S.A.
 // Copyright (c) 2001-2018 David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -277,6 +277,9 @@ bool AsepriteDecoder::readHeader(AsepriteHeader* header)
   header->grid_width   = read16();
   header->grid_height  = read16();
 
+  if (header->depth != 8)       // Transparent index only valid for indexed images
+    header->transparent_index = 0;
+
   if (header->ncolors == 0)     // 0 means 256 (old .ase files)
     header->ncolors = 256;
 
diff --git a/src/doc/sprite.cpp b/src/doc/sprite.cpp
index a43be3661..f9b430ce2 100644
--- a/src/doc/sprite.cpp
+++ b/src/doc/sprite.cpp
@@ -195,6 +195,12 @@ bool Sprite::supportAlpha() const
 
 void Sprite::setTransparentColor(color_t color)
 {
+#if _DEBUG
+  if (colorMode() != ColorMode::INDEXED) {
+    ASSERT(color == 0);
+  }
+#endif // _DEBUG
+
   m_spec.setMaskColor(color);
 
   // Change the mask color of all images.
diff --git a/src/render/quantization.cpp b/src/render/quantization.cpp
index 498838bc0..504fee6e3 100644
--- a/src/render/quantization.cpp
+++ b/src/render/quantization.cpp
@@ -1,5 +1,5 @@
 // Aseprite Render Library
-// Copyright (c) 2019-2020  Igara Studio S.A.
+// Copyright (c) 2019-2021  Igara Studio S.A.
 // Copyright (c) 2001-2018  David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -69,12 +69,19 @@ Palette* create_palette_from_sprite(
     }
   }
 
+  // Transparent color is needed if we have transparent layers
+  int maskIndex;
+  if (sprite->backgroundLayer() && sprite->allLayersCount() == 1)
+    maskIndex = -1;
+  else if (sprite->colorMode() == ColorMode::INDEXED)
+    maskIndex = sprite->transparentColor();
+  else {
+    ASSERT(sprite->transparentColor() == 0);
+    maskIndex = 0; // For RGB/Grayscale images we use index 0 as the transparent index by default
+  }
+
   // Generate an optimized palette
-  optimizer.calculate(
-    palette,
-    // Transparent color is needed if we have transparent layers
-    (sprite->backgroundLayer() &&
-     sprite->allLayersCount() == 1 ? -1: sprite->transparentColor()));
+  optimizer.calculate(palette, maskIndex);
 
   return palette;
 }