Fix when using a background layer, switching to Indexed Color Mode

fills all layer bounding rectangles with Color 0 (fix aseprite#4438)

Conditions to reproduce the original issue:
- Opaque RGBA sprite, i.e. the bottom layer is 'Background'.
- There is a second layer with an ellipse (for example).
- There is a mask color #000000 alpha=0 is in the palette.
- The mask color index is greater and not equal than 0.
- Go to Sprite > Color Mode > Indexed.
Result: the transparent color of the second layer will change to
index color = 0 (usually black).

Also added a test.

src/doc/sprite.cpp

@@ -456,14 +456,10 @@ RgbMap* Sprite::rgbMap(const frame_t frame,
         return nullptr;
     }
   }
-  int maskIndex;
-  if (forLayer == RgbMapFor::OpaqueLayer)
-    maskIndex = -1;
-  else {
-    maskIndex = palette(frame)->findMaskColor();
-    if (maskIndex == -1)
-      maskIndex = 0;
-  }
+  int maskIndex = palette(frame)->findMaskColor();
+  maskIndex = (maskIndex == -1 ?
+                (forLayer == RgbMapFor::OpaqueLayer ? -1 : 0)
+                : maskIndex);
   m_rgbMap->regenerateMap(palette(frame), maskIndex);
   return m_rgbMap.get();
 }

tests/scripts/color_convert.lua

@@ -1,4 +1,4 @@
--- Copyright (C) 2019-2022  Igara Studio S.A.
+-- Copyright (C) 2019-2024  Igara Studio S.A.
 --
 -- This file is released under the terms of the MIT license.
 -- Read LICENSE.txt for more information.
@@ -49,3 +49,41 @@ do
     end
   end
 end
+
+-- Test of mask color conversion in a opaque sprite with
+-- extra non-opaque layers.
+-- Conditions:
+-- + There is a background layer
+-- + There is an extra layer drawn
+-- + The mask color is in the palette and whose index is greater than 0
+-- + RGBA->INDEXED conversion
+do
+  local sprite = Sprite(3, 3, ColorMode.RGB)
+  app.command.BackgroundFromLayer()
+  local pal = sprite.palettes[1]
+  local backgroundLayer = sprite.layers[1]
+
+  assert(sprite.layers[1].isBackground)
+  assert(sprite.colorMode == ColorMode.RGB)
+  assert(sprite.layers[1]:cel(1).image:getPixel(0, 0) == app.pixelColor.rgba(0,0,0,255))
+  assert(#pal == 256)
+
+  pal:setColor(0, Color{ r=255, g=0  , b=0  , a=255 })
+  pal:setColor(1, Color{ r=0  , g=0  , b=0  , a=0 })
+  pal:setColor(2, Color{ r=0  , g=255, b=0  , a=255 })
+  pal:setColor(3, Color{ r=0  , g=0  , b=255, a=255 })
+  local layer = sprite:newLayer()
+  app.useTool {
+    tool='pencil',
+    color=pal:getColor(2),
+    points={ Point(0, 1), Point(1, 0) },
+    layer=Layer
+  }
+
+  app.command.ChangePixelFormat {
+    format="indexed"
+  }
+
+  assert(pal:getColor(1) == Color{r=0  , g=0  , b=0  , a=0})
+  assert(layer:cel(1).image:getPixel(0, 0) == 1)
+end