Add support for alpha channel in RgbMap and Palette::findBestfit()

This include several changes:
- Color::getIndex() can return palette values with alpha != 255
- Fix Transparent and Blurs ink for indexed images to make better use
  of palette entries with alpha values
- Fix bilinear resize algorithm for indexed images with alpha
- New RgbMap with four parameters: R, G, B, A
- Add one extra color scale function used in the alpha channel of the
  new RgbMap
- Fix color curve, convolution matrix, invert color, and median filters
  to take care of this new alpha channel on indexed images
- Fix ordered dithering and quantization

Related to #286

src/app/color.cpp

@@ -616,8 +616,13 @@ int Color::getIndex() const
 
     case Color::RgbType:
     case Color::HsvType:
-    case Color::GrayType:
+    case Color::GrayType: {
-      return get_current_palette()->findBestfit(getRed(), getGreen(), getBlue(), getAlpha(), 0);
+      int i = get_current_palette()->findExactMatch(getRed(), getGreen(), getBlue(), getAlpha());
+      if (i >= 0)
+        return i;
+      else
+        return get_current_palette()->findBestfit(getRed(), getGreen(), getBlue(), getAlpha(), 0);
+    }
 
     case Color::IndexType:
       return m_value.index;

src/app/commands/cmd_sprite_size.cpp

@@ -28,6 +28,7 @@
 #include "doc/cel.h"
 #include "doc/cels_range.h"
 #include "doc/image.h"
+#include "doc/layer.h"
 #include "doc/mask.h"
 #include "doc/primitives.h"
 #include "doc/sprite.h"
@@ -95,10 +96,12 @@ protected:
         ImageRef new_image(Image::create(image->pixelFormat(), MAX(1, w), MAX(1, h)));
 
         doc::algorithm::fixup_image_transparent_colors(image);
-        doc::algorithm::resize_image(image, new_image.get(),
+        doc::algorithm::resize_image(
+          image, new_image.get(),
           m_resize_method,
           m_sprite->palette(cel->frame()),
-          m_sprite->rgbMap(cel->frame()));
+          m_sprite->rgbMap(cel->frame()),
+          (cel->layer()->isBackground() ? -1: m_sprite->transparentColor()));
 
         api.replaceImage(m_sprite, cel->imageRef(), new_image);
       }
@@ -125,10 +128,12 @@ protected:
         gfx::Rect(
           scale_x(m_document->mask()->bounds().x-1),
           scale_y(m_document->mask()->bounds().y-1), MAX(1, w), MAX(1, h)));
-      algorithm::resize_image(old_bitmap.get(), new_mask->bitmap(),
+      algorithm::resize_image(
-          m_resize_method,
+        old_bitmap.get(), new_mask->bitmap(),
-          m_sprite->palette(0), // Ignored
+        m_resize_method,
-          m_sprite->rgbMap(0)); // Ignored
+        m_sprite->palette(0), // Ignored
+        m_sprite->rgbMap(0),  // Ignored
+        -1);                  // Ignored
 
       // Reshrink
       new_mask->intersect(new_mask->bounds());

src/app/commands/filters/filter_target_buttons.cpp

@@ -61,17 +61,15 @@ FilterTargetButtons::FilterTargetButtons(int imgtype, bool withChannels)
       case IMAGE_INDEXED:
         r = check_button_new("R", 2, 0, 0, 0);
         g = check_button_new("G", 0, 0, 0, 0);
-        b = check_button_new("B", 0, (imgtype == IMAGE_RGB) ? 0: 2, 0, 0);
+        b = check_button_new("B", 0, 0, 0, 0);
+        a = check_button_new("A", 0, 2, 0, 0);
 
         r->setId("r");
         g->setId("g");
         b->setId("b");
+        a->setId("a");
 
-        if (imgtype == IMAGE_RGB) {
+        if (imgtype == IMAGE_INDEXED) {
-          a = check_button_new("A", 0, 2, 0, 0);
-          a->setId("a");
-        }
-        else {
           index = check_button_new("Index", 0, 0, 0, 0);
           index->setId("i");
         }

src/app/tools/ink_processing.h

@@ -10,6 +10,7 @@
 #include "app/tools/shading_options.h"
 #include "doc/blend_funcs.h"
 #include "doc/image_impl.h"
+#include "doc/layer.h"
 #include "doc/palette.h"
 #include "doc/rgbmap.h"
 #include "doc/sprite.h"
@@ -241,14 +242,22 @@ public:
     m_palette(get_current_palette()),
     m_rgbmap(loop->getRgbMap()),
     m_opacity(loop->getOpacity()),
-    m_color(m_palette->getEntry(loop->getPrimaryColor())) {
+    m_color(m_palette->getEntry(loop->getPrimaryColor())),
+    m_maskColor(loop->getLayer()->isBackground() ? -1: loop->sprite()->transparentColor()) {
   }
 
   void processPixel(int x, int y) {
-    color_t c = rgba_blender_normal(m_palette->getEntry(*m_srcAddress), m_color, m_opacity);
+    color_t c = *m_srcAddress;
+    if (c == m_maskColor)
+      c = m_palette->getEntry(c) & rgba_rgb_mask;  // Alpha = 0
+    else
+      c = m_palette->getEntry(c);
+
+    c = rgba_blender_normal(c, m_color, m_opacity);
     *m_dstAddress = m_rgbmap->mapColor(rgba_getr(c),
                                        rgba_getg(c),
-                                       rgba_getb(c));
+                                       rgba_getb(c),
+                                       rgba_geta(c));
   }
 
 private:
@@ -256,6 +265,7 @@ private:
   const RgbMap* m_rgbmap;
   int m_opacity;
   color_t m_color;
+  color_t m_maskColor;
 };
 
 //////////////////////////////////////////////////////////////////////
@@ -387,24 +397,27 @@ public:
     m_opacity(loop->getOpacity()),
     m_tiledMode(loop->getTiledMode()),
     m_srcImage(loop->getSrcImage()),
-    m_area(get_current_palette()) {
+    m_area(get_current_palette(),
+           loop->getLayer()->isBackground() ? -1: loop->sprite()->transparentColor()) {
   }
 
   void processPixel(int x, int y) {
     m_area.reset();
     get_neighboring_pixels<IndexedTraits>(m_srcImage, x, y, 3, 3, 1, 1, m_tiledMode, m_area);
 
-    if (m_area.count > 0 && m_area.a/9 >= 128) {
+    if (m_area.count > 0) {
       m_area.r /= m_area.count;
       m_area.g /= m_area.count;
       m_area.b /= m_area.count;
+      m_area.a /= 9;
 
       uint32_t color32 = m_palette->getEntry(*m_srcAddress);
       m_area.r = rgba_getr(color32) + (m_area.r-rgba_getr(color32)) * m_opacity / 255;
       m_area.g = rgba_getg(color32) + (m_area.g-rgba_getg(color32)) * m_opacity / 255;
       m_area.b = rgba_getb(color32) + (m_area.b-rgba_getb(color32)) * m_opacity / 255;
+      m_area.a = rgba_geta(color32) + (m_area.a-rgba_geta(color32)) * m_opacity / 255;
 
-      *m_dstAddress = m_rgbmap->mapColor(m_area.r, m_area.g, m_area.b);
+      *m_dstAddress = m_rgbmap->mapColor(m_area.r, m_area.g, m_area.b, m_area.a);
     }
     else {
       *m_dstAddress = *m_srcAddress;
@@ -415,20 +428,27 @@ private:
   struct GetPixelsDelegate {
     const Palette* pal;
     int count, r, g, b, a;
+    color_t maskColor;
 
-    GetPixelsDelegate(const Palette* pal) : pal(pal) { }
+    GetPixelsDelegate(const Palette* pal,
+                      color_t maskColor)
+      : pal(pal), maskColor(maskColor) { }
 
     void reset() { count = r = g = b = a = 0; }
 
     void operator()(IndexedTraits::pixel_t color)
     {
-      a += (color == 0 ? 0: 255);
+      if (color == maskColor)
+        return;
 
       uint32_t color32 = pal->getEntry(color);
-      r += rgba_getr(color32);
+      if (rgba_geta(color32) > 0) {
-      g += rgba_getg(color32);
+        r += rgba_getr(color32);
-      b += rgba_getb(color32);
+        g += rgba_getg(color32);
-      count++;
+        b += rgba_getb(color32);
+        a += rgba_geta(color32);
+        ++count;
+      }
     }
   };
 
@@ -510,7 +530,7 @@ public:
           m_palette->getEntry(*m_srcAddress), m_color2, m_opacity);
 
         *m_dstAddress = m_rgbmap->mapColor(
-          rgba_getr(c), rgba_getg(c), rgba_getb(c));
+          rgba_getr(c), rgba_getg(c), rgba_getb(c), rgba_geta(c));
       }
     }
   }
@@ -610,7 +630,8 @@ void JumbleInkProcessing<IndexedTraits>::processPixel(int x, int y)
   if (rgba_geta(c) >= 128)
     *m_dstAddress = m_rgbmap->mapColor(rgba_getr(c),
                                        rgba_getg(c),
-                                       rgba_getb(c));
+                                       rgba_getb(c),
+                                       rgba_geta(c));
   else
     *m_dstAddress = 0;
 }
@@ -692,7 +713,8 @@ public:
     color_t c = rgba_blender_neg_bw(m_palette->getEntry(*m_srcAddress), m_color, 255);
     *m_dstAddress = m_rgbmap->mapColor(rgba_getr(c),
                                        rgba_getg(c),
-                                       rgba_getb(c));
+                                       rgba_getb(c),
+                                       rgba_geta(c));
   }
 
 private:

src/doc/CMakeLists.txt

@@ -17,7 +17,6 @@ add_library(doc-lib
   cel_data_io.cpp
   cel_io.cpp
   cels_range.cpp
-  color_scales.cpp
   compressed_image.cpp
   context.cpp
   conversion_she.cpp

src/doc/algorithm/resize_image.cpp

@@ -18,7 +18,7 @@
 namespace doc {
 namespace algorithm {
 
-void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palette* pal, const RgbMap* rgbmap)
+void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palette* pal, const RgbMap* rgbmap, color_t maskColor)
 {
   switch (method) {
 
@@ -111,15 +111,23 @@ void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palet
               break;
             }
             case IMAGE_INDEXED: {
-              int r = int((rgba_getr(pal->getEntry(color[0]))*u2 + rgba_getr(pal->getEntry(color[1]))*u1)*v2 +
+              // Convert index to RGBA values
-                          (rgba_getr(pal->getEntry(color[2]))*u2 + rgba_getr(pal->getEntry(color[3]))*u1)*v1);
+              for (int i=0; i<4; ++i) {
-              int g = int((rgba_getg(pal->getEntry(color[0]))*u2 + rgba_getg(pal->getEntry(color[1]))*u1)*v2 +
+                if (color[i] == maskColor)
-                          (rgba_getg(pal->getEntry(color[2]))*u2 + rgba_getg(pal->getEntry(color[3]))*u1)*v1);
+                  color[i] = pal->getEntry(color[i]) & rgba_rgb_mask; // Set alpha = 0
-              int b = int((rgba_getb(pal->getEntry(color[0]))*u2 + rgba_getb(pal->getEntry(color[1]))*u1)*v2 +
+                else
-                          (rgba_getb(pal->getEntry(color[2]))*u2 + rgba_getb(pal->getEntry(color[3]))*u1)*v1);
+                  color[i] = pal->getEntry(color[i]);
-              int a = int(((color[0] == 0 ? 0: 255)*u2 + (color[1] == 0 ? 0: 255)*u1)*v2 +
+              }
-                          ((color[2] == 0 ? 0: 255)*u2 + (color[3] == 0 ? 0: 255)*u1)*v1);
+
-              dst_color = a > 127 ? rgbmap->mapColor(r, g, b): 0;
+              int r = int((rgba_getr(color[0])*u2 + rgba_getr(color[1])*u1)*v2 +
+                          (rgba_getr(color[2])*u2 + rgba_getr(color[3])*u1)*v1);
+              int g = int((rgba_getg(color[0])*u2 + rgba_getg(color[1])*u1)*v2 +
+                          (rgba_getg(color[2])*u2 + rgba_getg(color[3])*u1)*v1);
+              int b = int((rgba_getb(color[0])*u2 + rgba_getb(color[1])*u1)*v2 +
+                          (rgba_getb(color[2])*u2 + rgba_getb(color[3])*u1)*v1);
+              int a = int((rgba_geta(color[0])*u2 + rgba_geta(color[1])*u1)*v2 +
+                          (rgba_geta(color[2])*u2 + rgba_geta(color[3])*u1)*v1);
+              dst_color = rgbmap->mapColor(r, g, b, a);
               break;
             }
           }

src/doc/algorithm/resize_image.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2001-2014 David Capello
+// Copyright (c) 2001-2015 David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -8,6 +8,7 @@
 #define DOC_ALGORITHM_RESIZE_IMAGE_H_INCLUDED
 #pragma once
 
+#include "doc/color.h"
 #include "gfx/fwd.h"
 
 namespace doc {
@@ -27,7 +28,8 @@ namespace doc {
     // Warning: If you are using the RESIZE_METHOD_BILINEAR, it is
     // recommended to use 'fixup_image_transparent_colors' function
     // over the source image 'src' BEFORE using this routine.
-    void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palette* palette, const RgbMap* rgbmap);
+    void resize_image(const Image* src, Image* dst, ResizeMethod method, const Palette* palette, const RgbMap* rgbmap,
+                      color_t maskColor);
 
     // It does not modify the image to the human eye, but internally
     // tries to fixup all colors that are completelly transparent

src/doc/color_scales.cpp

@@ -1,48 +0,0 @@
-// Aseprite Document Library
-// Copyright (c) 2001-2014 David Capello
-//
-// This file is released under the terms of the MIT license.
-// Read LICENSE.txt for more information.
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include "base/clamp.h"
-
-// Based on Allegro _rgb_scale_5 and _rgb_scale_6 tables
-
-namespace doc {
-
-int scale_5bits_to_8bits(int channel5bits)
-{
-  static int scale[32] = {
-    0,   8,   16,  24,  33,  41,  49,  57,
-    66,  74,  82,  90,  99,  107, 115, 123,
-    132, 140, 148, 156, 165, 173, 181, 189,
-    198, 206, 214, 222, 231, 239, 247, 255
-  };
-
-  ASSERT(channel5bits >= 0);
-  ASSERT(channel5bits < 32);
-  return scale[channel5bits];
-}
-
-int scale_6bits_to_8bits(int channel6bits)
-{
-  static int scale[64] = {
-    0,   4,   8,   12,  16,  20,  24,  28,
-    32,  36,  40,  44,  48,  52,  56,  60,
-    65,  69,  73,  77,  81,  85,  89,  93,
-    97,  101, 105, 109, 113, 117, 121, 125,
-    130, 134, 138, 142, 146, 150, 154, 158,
-    162, 166, 170, 174, 178, 182, 186, 190,
-    195, 199, 203, 207, 211, 215, 219, 223,
-    227, 231, 235, 239, 243, 247, 251, 255
-  };
-  ASSERT(channel6bits >= 0);
-  ASSERT(channel6bits < 64);
-  return scale[channel6bits];
-}
-
-} // namespace doc

src/doc/color_scales.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2001-2014 David Capello
+// Copyright (c) 2001-2015 David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -15,8 +15,64 @@
 
 namespace doc {
 
-  int scale_5bits_to_8bits(int channel5bits);
+  inline int scale_2bits_to_8bits(int channel2bits) {
-  int scale_6bits_to_8bits(int channel6bits);
+    static int scale[4] = {
+      0,   85,  170,  255
+    };
+    ASSERT(channel2bits >= 0);
+    ASSERT(channel2bits < 4);
+    return scale[channel2bits];
+  }
+
+  inline int scale_3bits_to_8bits(int channel3bits) {
+    static int scale[8] = {
+      0,   36,  72,  109,
+      145, 182, 218, 255
+    };
+    ASSERT(channel3bits >= 0);
+    ASSERT(channel3bits < 8);
+    return scale[channel3bits];
+  }
+
+  inline int scale_4bits_to_8bits(int channel4bits) {
+    static int scale[16] = {
+      0,   16,  34,  51,
+      68,  85,  102, 119,
+      136, 153, 170, 187,
+      204, 221, 238, 255
+    };
+    ASSERT(channel4bits >= 0);
+    ASSERT(channel4bits < 16);
+    return scale[channel4bits];
+  }
+
+  inline int scale_5bits_to_8bits(int channel5bits) {
+    static int scale[32] = {
+      0,   8,   16,  24,  33,  41,  49,  57,
+      66,  74,  82,  90,  99,  107, 115, 123,
+      132, 140, 148, 156, 165, 173, 181, 189,
+      198, 206, 214, 222, 231, 239, 247, 255
+    };
+    ASSERT(channel5bits >= 0);
+    ASSERT(channel5bits < 32);
+    return scale[channel5bits];
+  }
+
+  inline int scale_6bits_to_8bits(int channel6bits) {
+    static int scale[64] = {
+      0,   4,   8,   12,  16,  20,  24,  28,
+      32,  36,  40,  44,  48,  52,  56,  60,
+      65,  69,  73,  77,  81,  85,  89,  93,
+      97,  101, 105, 109, 113, 117, 121, 125,
+      130, 134, 138, 142, 146, 150, 154, 158,
+      162, 166, 170, 174, 178, 182, 186, 190,
+      195, 199, 203, 207, 211, 215, 219, 223,
+      227, 231, 235, 239, 243, 247, 251, 255
+    };
+    ASSERT(channel6bits >= 0);
+    ASSERT(channel6bits < 64);
+    return scale[channel6bits];
+  }
 
 } // namespace doc
 

src/doc/palette.cpp

@@ -189,57 +189,64 @@ int Palette::findExactMatch(int r, int g, int b, int a) const
 //////////////////////////////////////////////////////////////////////
 // Based on Allegro's bestfit_color
 
-static unsigned int col_diff[3*128];
+static std::vector<unsigned int> col_diff;
 
-static void bestfit_init()
+static void initBestfit()
 {
-  int i, k;
+  col_diff.resize(4*128, 0);
 
-  for (i=1; i<64; i++) {
+  for (int i=1; i<64; ++i) {
-    k = i * i;
+    int k = i * i;
-    col_diff[0  +i] = col_diff[0  +128-i] = k * (59 * 59);
+    col_diff[0  +i] = col_diff[0  +128-i] = k * 59 * 59;
-    col_diff[128+i] = col_diff[128+128-i] = k * (30 * 30);
+    col_diff[128+i] = col_diff[128+128-i] = k * 30 * 30;
-    col_diff[256+i] = col_diff[256+128-i] = k * (11 * 11);
+    col_diff[256+i] = col_diff[256+128-i] = k * 11 * 11;
+    col_diff[384+i] = col_diff[384+128-i] = k * 8 * 8;
   }
 }
 
 int Palette::findBestfit(int r, int g, int b, int a, int mask_index) const
 {
-  int i, bestfit, coldiff, lowest;
-
   ASSERT(r >= 0 && r <= 255);
   ASSERT(g >= 0 && g <= 255);
   ASSERT(b >= 0 && b <= 255);
   ASSERT(a >= 0 && a <= 255);
 
-  if (col_diff[1] == 0)
+  if (col_diff.empty())
-    bestfit_init();
+    initBestfit();
-
-  bestfit = 0;
-  lowest = std::numeric_limits<int>::max();
 
   r >>= 3;
   g >>= 3;
   b >>= 3;
+  a >>= 3;
 
-  i = 0;
+  // Mask index is like alpha = 0, so we can use it as transparent color.
-  while (i < size()) {
+  if (a == 0 && mask_index >= 0)
+    return mask_index;
+
+  int bestfit = 0;
+  int lowest = std::numeric_limits<int>::max();
+  int size = MIN(256, m_colors.size());
+
+  for (int i=0; i<size; ++i) {
     color_t rgb = m_colors[i];
 
-    coldiff = (col_diff + 0) [ ((rgba_getg(rgb)>>3) - g) & 0x7F ];
+    int coldiff = col_diff[((rgba_getg(rgb)>>3) - g) & 0x7F];
     if (coldiff < lowest) {
-      coldiff += (col_diff + 128) [ ((rgba_getr(rgb)>>3) - r) & 0x7F ];
+      coldiff += col_diff[128 + (((rgba_getr(rgb)>>3) - r) & 0x7F)];
       if (coldiff < lowest) {
-        coldiff += (col_diff + 256) [ ((rgba_getb(rgb)>>3) - b) & 0x7F ];
+        coldiff += col_diff[256 + (((rgba_getb(rgb)>>3) - b) & 0x7F)];
-        if (coldiff < lowest && i != mask_index) {
+        if (coldiff < lowest) {
-          bestfit = i;
+          coldiff += col_diff[384 + (((rgba_geta(rgb)>>3) - a) & 0x7F)];
-          if (coldiff == 0)
+          if (coldiff < lowest && i != mask_index) {
-            return bestfit;
+            if (coldiff == 0)
-          lowest = coldiff;
+              return i;
+
+            bestfit = i;
+            lowest = coldiff;
+          }
         }
       }
     }
-    i++;
   }
 
   return bestfit;

src/doc/resize_image_tests.cpp

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2001-2014 David Capello
+// Copyright (c) 2001-2015 David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -56,7 +56,7 @@ color_t test_image_scaled_9x9_bilinear[81] =
   0x000000, 0x000000, 0x565656, 0xa9a9a9, 0xffffff, 0xa9a9a9, 0x565656, 0x000000, 0x000000
 };
 
-Image* create_image_from_data(PixelFormat format, color_t* data, int width, int height) 
+Image* create_image_from_data(PixelFormat format, color_t* data, int width, int height)
 {
   Image* new_image = Image::create(format, width, height);
   for (int i = 0; i < width * height; i++) {
@@ -72,7 +72,7 @@ TEST(ResizeImage, NearestNeighborInterp)
   Image* dst_expected = create_image_from_data(IMAGE_RGB, test_image_scaled_9x9_nearest, 9, 9);
 
   Image* dst = Image::create(IMAGE_RGB, 9, 9);
-  algorithm::resize_image(src, dst, algorithm::RESIZE_METHOD_NEAREST_NEIGHBOR, NULL, NULL);
+  algorithm::resize_image(src, dst, algorithm::RESIZE_METHOD_NEAREST_NEIGHBOR, NULL, NULL, -1);
 
   ASSERT_EQ(0, count_diff_between_images(dst, dst_expected));
 }

src/doc/rgbmap.cpp

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2001-2014 David Capello
+// Copyright (c) 2001-2015 David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -15,7 +15,11 @@
 
 namespace doc {
 
-#define MAPSIZE 32*32*32
+#define RSIZE   32
+#define GSIZE   32
+#define BSIZE   32
+#define ASIZE   8
+#define MAPSIZE (RSIZE*GSIZE*BSIZE*ASIZE)
 
 RgbMap::RgbMap()
   : Object(ObjectType::RgbMap)
@@ -36,26 +40,23 @@ void RgbMap::regenerate(const Palette* palette, int mask_index)
   m_palette = palette;
   m_modifications = palette->getModifications();
 
+  // TODO This is slow for 256 colors 32*32*32*8 findBestfit calls
+
   int i = 0;
-  for (int r=0; r<32; ++r) {
+  for (int r=0; r<RSIZE; ++r) {
-    for (int g=0; g<32; ++g) {
+    for (int g=0; g<GSIZE; ++g) {
-      for (int b=0; b<32; ++b) {
+      for (int b=0; b<BSIZE; ++b) {
-        m_map[i++] =
+        for (int a=0; a<ASIZE; ++a) {
-          palette->findBestfit(
+          m_map[i++] =
-            scale_5bits_to_8bits(r),
+            palette->findBestfit(
-            scale_5bits_to_8bits(g),
+              scale_5bits_to_8bits(r),
-            scale_5bits_to_8bits(b), 255, mask_index);
+              scale_5bits_to_8bits(g),
+              scale_5bits_to_8bits(b),
+              scale_3bits_to_8bits(a), mask_index);
+        }
       }
     }
   }
 }
 
-int RgbMap::mapColor(int r, int g, int b) const
-{
-  ASSERT(r >= 0 && r < 256);
-  ASSERT(g >= 0 && g < 256);
-  ASSERT(b >= 0 && b < 256);
-  return m_map[((r>>3) << 10) + ((g>>3) << 5) + (b>>3)];
-}
-
 } // namespace doc

src/doc/rgbmap.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2001-2014 David Capello
+// Copyright (c) 2001-2015 David Capello
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -24,7 +24,14 @@ namespace doc {
     bool match(const Palette* palette) const;
     void regenerate(const Palette* palette, int mask_index);
 
-    int mapColor(int r, int g, int b) const;
+    int mapColor(int r, int g, int b, int a) const {
+      ASSERT(r >= 0 && r < 256);
+      ASSERT(g >= 0 && g < 256);
+      ASSERT(b >= 0 && b < 256);
+      ASSERT(a >= 0 && a < 256);
+      // bits -> bbbbbgggggrrrrraaa
+      return m_map[(a>>5) | ((b>>3) << 3) | ((g>>3) << 8) | ((r>>3) << 13)];
+    }
 
   private:
     std::vector<uint8_t> m_map;

src/filters/color_curve_filter.cpp

@@ -114,7 +114,7 @@ void ColorCurveFilter::applyToIndexed(FilterManager* filterMgr)
   Target target = filterMgr->getTarget();
   const Palette* pal = filterMgr->getIndexedData()->getPalette();
   const RgbMap* rgbmap = filterMgr->getIndexedData()->getRgbMap();
-  int x, c, r, g, b;
+  int x, c, r, g, b, a;
 
   for (x=0; x<w; x++) {
     if (filterMgr->skipPixel()) {
@@ -129,15 +129,18 @@ void ColorCurveFilter::applyToIndexed(FilterManager* filterMgr)
       c = m_cmap[c];
     }
     else {
-      r = rgba_getr(pal->getEntry(c));
+      c = pal->getEntry(c);
-      g = rgba_getg(pal->getEntry(c));
+      r = rgba_getr(c);
-      b = rgba_getb(pal->getEntry(c));
+      g = rgba_getg(c);
+      b = rgba_getb(c);
+      a = rgba_geta(c);
 
-      if (target & TARGET_RED_CHANNEL) r = m_cmap[r];
+      if (target & TARGET_RED_CHANNEL  ) r = m_cmap[r];
       if (target & TARGET_GREEN_CHANNEL) g = m_cmap[g];
-      if (target & TARGET_BLUE_CHANNEL) b = m_cmap[b];
+      if (target & TARGET_BLUE_CHANNEL ) b = m_cmap[b];
+      if (target & TARGET_ALPHA_CHANNEL) a = m_cmap[a];
 
-      c = rgbmap->mapColor(r, g, b);
+      c = rgbmap->mapColor(r, g, b, a);
     }
 
     *(dst_address++) = MID(0, c, pal->size()-1);

src/filters/convolution_matrix_filter.cpp

@@ -87,22 +87,28 @@ namespace {
 
   struct GetPixelsDelegateIndexed : public GetPixelsDelegate {
     const Palette* pal;
-    int r, g, b, index;
+    int r, g, b, a, index;
 
     GetPixelsDelegateIndexed(const Palette* pal) : pal(pal) { }
 
     void reset(const ConvolutionMatrix* matrix) {
       GetPixelsDelegate::reset(matrix);
-      r = g = b = index = 0;
+      r = g = b = a = index = 0;
     }
 
-    void operator()(GrayscaleTraits::pixel_t color)
+    void operator()(IndexedTraits::pixel_t color)
     {
       if (*matrixData) {
-        r += rgba_getr(pal->getEntry(color)) * (*matrixData);
-        g += rgba_getg(pal->getEntry(color)) * (*matrixData);
-        b += rgba_getb(pal->getEntry(color)) * (*matrixData);
         index += color * (*matrixData);
+        color_t rgba = pal->getEntry(color);
+        if (rgba_geta(rgba) == 0)
+          div -= *matrixData;
+        else {
+          r += rgba_getr(rgba) * (*matrixData);
+          g += rgba_getg(rgba) * (*matrixData);
+          b += rgba_getb(rgba) * (*matrixData);
+          a += rgba_geta(rgba) * (*matrixData);
+        }
       }
       matrixData++;
     }
@@ -297,28 +303,37 @@ void ConvolutionMatrixFilter::applyToIndexed(FilterManager* filterMgr)
       *(dst_address++) = delegate.index;
     }
     else {
+      color = pal->getEntry(color);
+
       if (target & TARGET_RED_CHANNEL) {
         delegate.r = delegate.r / delegate.div + m_matrix->getBias();
         delegate.r = MID(0, delegate.r, 255);
       }
       else
-        delegate.r = rgba_getr(pal->getEntry(color));
+        delegate.r = rgba_getr(color);
 
       if (target & TARGET_GREEN_CHANNEL) {
         delegate.g =  delegate.g / delegate.div + m_matrix->getBias();
         delegate.g = MID(0, delegate.g, 255);
       }
       else
-        delegate.g = rgba_getg(pal->getEntry(color));
+        delegate.g = rgba_getg(color);
 
       if (target & TARGET_BLUE_CHANNEL) {
         delegate.b = delegate.b / delegate.div + m_matrix->getBias();
         delegate.b = MID(0, delegate.b, 255);
       }
       else
-        delegate.b = rgba_getb(pal->getEntry(color));
+        delegate.b = rgba_getb(color);
 
-      *(dst_address++) = rgbmap->mapColor(delegate.r, delegate.g, delegate.b);
+      if (target & TARGET_ALPHA_CHANNEL) {
+        delegate.a = delegate.a / delegate.div + m_matrix->getBias();
+        delegate.a = MID(0, delegate.a, 255);
+      }
+      else
+        delegate.a = rgba_geta(color);
+
+      *(dst_address++) = rgbmap->mapColor(delegate.r, delegate.g, delegate.b, delegate.a);
     }
   }
 }

src/filters/invert_color_filter.cpp

@@ -92,7 +92,7 @@ void InvertColorFilter::applyToIndexed(FilterManager* filterMgr)
   const RgbMap* rgbmap = filterMgr->getIndexedData()->getRgbMap();
   int w = filterMgr->getWidth();
   Target target = filterMgr->getTarget();
-  int x, c, r, g, b;
+  int x, c, r, g, b, a;
 
   for (x=0; x<w; x++) {
     if (filterMgr->skipPixel()) {
@@ -106,15 +106,18 @@ void InvertColorFilter::applyToIndexed(FilterManager* filterMgr)
     if (target & TARGET_INDEX_CHANNEL)
       c ^= 0xff;
     else {
-      r = rgba_getr(pal->getEntry(c));
+      c = pal->getEntry(c);
-      g = rgba_getg(pal->getEntry(c));
+      r = rgba_getr(c);
-      b = rgba_getb(pal->getEntry(c));
+      g = rgba_getg(c);
+      b = rgba_getb(c);
+      a = rgba_geta(c);
 
       if (target & TARGET_RED_CHANNEL  ) r ^= 0xff;
       if (target & TARGET_GREEN_CHANNEL) g ^= 0xff;
       if (target & TARGET_BLUE_CHANNEL ) b ^= 0xff;
+      if (target & TARGET_ALPHA_CHANNEL) a ^= 0xff;
 
-      c = rgbmap->mapColor(r, g, b);
+      c = rgbmap->mapColor(r, g, b, a);
     }
 
     *(dst_address++) = c;

src/filters/median_filter.cpp

@@ -78,9 +78,11 @@ namespace {
         channel[0][c] = color;
       }
       else {
-        channel[0][c] = rgba_getr(pal->getEntry(color));
+        color_t rgb = pal->getEntry(color);
-        channel[1][c] = rgba_getg(pal->getEntry(color));
+        channel[0][c] = rgba_getr(rgb);
-        channel[2][c] = rgba_getb(pal->getEntry(color));
+        channel[1][c] = rgba_getg(rgb);
+        channel[2][c] = rgba_getb(rgb);
+        channel[3][c] = rgba_geta(rgb);
       }
       c++;
     }
@@ -222,7 +224,7 @@ void MedianFilter::applyToIndexed(FilterManager* filterMgr)
   const Palette* pal = filterMgr->getIndexedData()->getPalette();
   const RgbMap* rgbmap = filterMgr->getIndexedData()->getRgbMap();
   Target target = filterMgr->getTarget();
-  int color, r, g, b;
+  int color, r, g, b, a;
   GetPixelsDelegateIndexed delegate(pal, m_channel, target);
   int x = filterMgr->x();
   int x2 = x+filterMgr->getWidth();
@@ -245,13 +247,14 @@ void MedianFilter::applyToIndexed(FilterManager* filterMgr)
     }
     else {
       color = get_pixel_fast<IndexedTraits>(src, x, y);
+      color = pal->getEntry(color);
 
       if (target & TARGET_RED_CHANNEL) {
         std::sort(m_channel[0].begin(), m_channel[0].end());
         r = m_channel[0][m_ncolors/2];
       }
       else
-        r = rgba_getr(pal->getEntry(color));
+        r = rgba_getr(color);
 
       if (target & TARGET_GREEN_CHANNEL) {
         std::sort(m_channel[1].begin(), m_channel[1].end());
@@ -265,9 +268,16 @@ void MedianFilter::applyToIndexed(FilterManager* filterMgr)
         b = m_channel[2][m_ncolors/2];
       }
       else
-        b = rgba_getb(pal->getEntry(color));
+        b = rgba_getb(color);
 
-      *(dst_address++) = rgbmap->mapColor(r, g, b);
+      if (target & TARGET_ALPHA_CHANNEL) {
+        std::sort(m_channel[3].begin(), m_channel[3].end());
+        a = m_channel[3][m_ncolors/2];
+      }
+      else
+        a = rgba_geta(color);
+
+      *(dst_address++) = rgbmap->mapColor(r, g, b, a);
     }
   }
 }

src/render/ordered_dither.h

@@ -60,12 +60,13 @@ namespace render {
                                 3, 1 };
 
   class OrderedDither {
-    static int colorDistance(int r1, int g1, int b1,
+    static int colorDistance(int r1, int g1, int b1, int a1,
-                             int r2, int g2, int b2) {
+                             int r2, int g2, int b2, int a2) {
       // The factor for RGB components came from doc::rba_luma()
       return int((r1-r2) * (r1-r2) * 21 + // 2126
                  (g1-g2) * (g1-g2) * 71 + // 7152
-                 (b1-b2) * (b1-b2) *  7); //  722
+                 (b1-b2) * (b1-b2) *  7 + //  722
+                 (a1-a2) * (a1-a2));
     }
 
   public:
@@ -80,7 +81,7 @@ namespace render {
       const doc::RgbMap* rgbmap,
       const doc::Palette* palette) {
       // Alpha=0, output transparent color
-      if (!doc::rgba_geta(color))
+      if (m_transparentIndex >= 0 && !doc::rgba_geta(color))
         return m_transparentIndex;
 
       // Get the nearest color in the palette with the given RGB
@@ -88,14 +89,16 @@ namespace render {
       int r = doc::rgba_getr(color);
       int g = doc::rgba_getg(color);
       int b = doc::rgba_getb(color);
+      int a = doc::rgba_geta(color);
       doc::color_t nearest1idx =
-        (rgbmap ? rgbmap->mapColor(r, g, b):
+        (rgbmap ? rgbmap->mapColor(r, g, b, a):
-                  palette->findBestfit(r, g, b, 255, m_transparentIndex));
+                  palette->findBestfit(r, g, b, a, m_transparentIndex));
 
       doc::color_t nearest1rgb = palette->getEntry(nearest1idx);
       int r1 = doc::rgba_getr(nearest1rgb);
       int g1 = doc::rgba_getg(nearest1rgb);
       int b1 = doc::rgba_getb(nearest1rgb);
+      int a1 = doc::rgba_geta(nearest1rgb);
 
       // Between the original color ('color' parameter) and 'nearest'
       // index, we have an error (r1-r, g1-g, b1-b). Here we try to
@@ -104,12 +107,14 @@ namespace render {
       int r2 = r - (r1-r);
       int g2 = g - (g1-g);
       int b2 = b - (b1-b);
+      int a2 = a - (a1-a);
       r2 = MID(0, r2, 255);
       g2 = MID(0, g2, 255);
       b2 = MID(0, b2, 255);
+      a2 = MID(0, a2, 255);
       doc::color_t nearest2idx =
-        (rgbmap ? rgbmap->mapColor(r2, g2, b2):
+        (rgbmap ? rgbmap->mapColor(r2, g2, b2, a2):
-                  palette->findBestfit(r2, g2, b2, 255, m_transparentIndex));
+                  palette->findBestfit(r2, g2, b2, a2, m_transparentIndex));
 
       // If both possible RGB colors use the same index, we cannot
       // make any dither with these two colors.
@@ -120,12 +125,13 @@ namespace render {
       r2 = doc::rgba_getr(nearest2rgb);
       g2 = doc::rgba_getg(nearest2rgb);
       b2 = doc::rgba_getb(nearest2rgb);
+      a2 = doc::rgba_geta(nearest2rgb);
 
       // Here we calculate the distance between the original 'color'
       // and 'nearest1rgb'. The maximum possible distance is given by
       // the distance between 'nearest1rgb' and 'nearest2rgb'.
-      int d = colorDistance(r1, g1, b1, r, g, b);
+      int d = colorDistance(r1, g1, b1, a1, r, g, b, a);
-      int D = colorDistance(r1, g1, b1, r2, g2, b2);
+      int D = colorDistance(r1, g1, b1, a1, r2, g2, b2, a2);
       if (D == 0)
         return nearest1idx;
 

src/render/quantization.cpp

@@ -137,11 +137,12 @@ Image* convert_pixel_format(
             r = rgba_getr(c);
             g = rgba_getg(c);
             b = rgba_getb(c);
+            a = rgba_geta(c);
 
-            if (rgba_geta(c) == 0)
+            if (a == 0)
-              *dst_it = 0;
+              *dst_it = 0;      // TODO why 0 is mask color and not a param?
             else
-              *dst_it = rgbmap->mapColor(r, g, b);
+              *dst_it = rgbmap->mapColor(r, g, b, a);
           }
           ASSERT(dst_it == dst_end);
           break;
@@ -192,11 +193,13 @@ Image* convert_pixel_format(
           for (; src_it != src_end; ++src_it, ++dst_it) {
             ASSERT(dst_it != dst_end);
             c = *src_it;
+            a = graya_geta(c);
+            c = graya_getv(c);
 
-            if (graya_geta(c) == 0)
+            if (a == 0)
-              *dst_it = 0;
+              *dst_it = 0;      // TODO why 0 is mask color and not a param?
             else
-              *dst_it = graya_getv(c);
+              *dst_it = rgbmap->mapColor(c, c, c, a);
           }
           ASSERT(dst_it == dst_end);
           break;
@@ -277,11 +280,12 @@ Image* convert_pixel_format(
             if (!is_background && c == image->maskColor())
               *dst_it = dstMaskColor;
             else {
-              r = rgba_getr(palette->getEntry(c));
+              c = palette->getEntry(c);
-              g = rgba_getg(palette->getEntry(c));
+              r = rgba_getr(c);
-              b = rgba_getb(palette->getEntry(c));
+              g = rgba_getg(c);
-
+              b = rgba_getb(c);
-              *dst_it = rgbmap->mapColor(r, g, b);
+              a = rgba_geta(c);
+              *dst_it = rgbmap->mapColor(r, g, b, a);
             }
           }
           ASSERT(dst_it == dst_end);