From 5798e27993e9a91ddd01e030944d87b898a542ff Mon Sep 17 00:00:00 2001
From: Gaspar Capello <gaspar@igara.com>
Date: Wed, 12 Jun 2024 14:57:53 -0300
Subject: [PATCH] Fix mask color turns to opaque on RGBA->INDEXED conversion
 (fix #4438)

Original issue title: When using a background layer, switching to
Indexed Color Mode fills all layer bounding rectangles with
Color 0.
Conditions to reproduce the original issue:
- Opaque RGBA sprite, i.e. the bottom layer is 'Background'.
- There is a second layer with an ellipse (for example).
- There is a mask color #000000 alpha=0 is in the palette.
- The mask color index is greater and not equal than 0.
- Go to Sprite > Color Mode > Indexed.
Result: the transparent color of the second layer will change to
index color = 0 (usually black).

Also added test for RGBA->INDEXED conversion
---
 src/doc/sprite.cpp              | 11 +++------
 tests/scripts/color_convert.lua | 40 ++++++++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/src/doc/sprite.cpp b/src/doc/sprite.cpp
index 97dff921c..c11a56a26 100644
--- a/src/doc/sprite.cpp
+++ b/src/doc/sprite.cpp
@@ -455,14 +455,9 @@ RgbMap* Sprite::rgbMap(const frame_t frame,
     }
     m_rgbMap->fitCriteria(fitCriteria);
   }
-  int maskIndex;
-  if (forLayer == RgbMapFor::OpaqueLayer)
-    maskIndex = -1;
-  else {
-    maskIndex = palette(frame)->findMaskColor();
-    if (maskIndex == -1)
-      maskIndex = 0;
-  }
+  int maskIndex = palette(frame)->findMaskColor();
+  maskIndex = (maskIndex == -1 ? (forLayer == RgbMapFor::OpaqueLayer ? -1: 0):
+                                 maskIndex);
   m_rgbMap->regenerateMap(palette(frame), maskIndex, fitCriteria);
   return m_rgbMap.get();
 }
diff --git a/tests/scripts/color_convert.lua b/tests/scripts/color_convert.lua
index 3babcef41..ac41ef577 100644
--- a/tests/scripts/color_convert.lua
+++ b/tests/scripts/color_convert.lua
@@ -1,4 +1,4 @@
--- Copyright (C) 2019-2022  Igara Studio S.A.
+-- Copyright (C) 2019-2024  Igara Studio S.A.
 --
 -- This file is released under the terms of the MIT license.
 -- Read LICENSE.txt for more information.
@@ -49,3 +49,41 @@ do
     end
   end
 end
+
+-- Test of mask color conversion in a opaque sprite with
+-- extra non-opaque layers.
+-- Conditions:
+-- + There is a background layer
+-- + There is an extra layer drawn
+-- + The mask color is in the palette and whose index is greater than 0
+-- + RGBA->INDEXED conversion
+do
+  local sprite = Sprite(3, 3, ColorMode.RGB)
+  app.command.BackgroundFromLayer()
+  local pal = sprite.palettes[1]
+  local backgroundLayer = sprite.layers[1]
+
+  assert(sprite.layers[1].isBackground)
+  assert(sprite.colorMode == ColorMode.RGB)
+  assert(sprite.layers[1]:cel(1).image:getPixel(0, 0) == app.pixelColor.rgba(0,0,0,255))
+  assert(#pal == 256)
+
+  pal:setColor(0, Color{ r=255, g=0  , b=0  , a=255 })
+  pal:setColor(1, Color{ r=0  , g=0  , b=0  , a=0 })
+  pal:setColor(2, Color{ r=0  , g=255, b=0  , a=255 })
+  pal:setColor(3, Color{ r=0  , g=0  , b=255, a=255 })
+  local layer = sprite:newLayer()
+  app.useTool {
+    tool='pencil',
+    color=pal:getColor(2),
+    points={ Point(0, 1), Point(1, 0) },
+    layer=Layer
+  }
+
+  app.command.ChangePixelFormat {
+    format="indexed"
+  }
+
+  assert(pal:getColor(1) == Color{r=0  , g=0  , b=0  , a=0})
+  assert(layer:cel(1).image:getPixel(0, 0) == 1)
+end
\ No newline at end of file