Fix eyedropper in Tiles mode to pick tiles that are flipped and have masked areas

Sprite::pickCels() wasn't taking care of the tile flags to pick
tilemap layers correctly with flipped tiles.

src/app/color_picker.cpp

@@ -22,14 +22,13 @@
 #include "doc/render_plan.h"
 #include "doc/sprite.h"
 #include "doc/tile.h"
+#include "doc/tile_primitives.h"
 #include "doc/tileset.h"
 #include "gfx/point.h"
 #include "render/get_sprite_pixel.h"
 
 #include <algorithm>
 
-#define PICKER_TRACE(...) // TRACE
-
 namespace app {
 
 namespace {
@@ -57,37 +56,9 @@ bool get_cel_pixel(const Cel* cel,
   if (image->pixelFormat() == IMAGE_TILEMAP) {
     ASSERT(cel->layer()->isTilemap());
 
-    auto layerTilemap = static_cast<doc::LayerTilemap*>(cel->layer());
-    doc::Grid grid = layerTilemap->tileset()->grid();
-    grid.origin(grid.origin() + cel->position());
-
-    gfx::Point tilePos = grid.canvasToTile(gfx::Point(pos));
-    PICKER_TRACE("PICKER: tilePos=(%d %d)\n", tilePos.x,tilePos.y);
-    if (!image->bounds().contains(tilePos))
-      return false;
-
-    const doc::tile_index t =
-      get_pixel(image, tilePos.x, tilePos.y);
-    const doc::tile_index ti = doc::tile_geti(t);
-    const doc::tile_index tf = doc::tile_getf(t);
-
-    PICKER_TRACE("PICKER: tile=%d index=%d flags=%d\n", t, ti, tf);
-
-    doc::ImageRef tile = layerTilemap->tileset()->get(ti);
-    if (!tile)
-      return false;
-
-    gfx::Point ipos =
-      gfx::Point(pos) - grid.tileToCanvas(tilePos);
-    if (tf & doc::tile_f_xflip) { ipos.x = tile->width()-ipos.x-1; }
-    if (tf & doc::tile_f_yflip) { ipos.y = tile->height()-ipos.y-1; }
-    if (tf & doc::tile_f_dflip) { std::swap(ipos.x, ipos.y); }
-
-    PICKER_TRACE("PICKER: ipos=%d %d\n", ipos.x, ipos.y);
-
-    output = get_pixel(tile.get(), ipos.x, ipos.y);
-    PICKER_TRACE("PICKER: output=%d\n", output);
-    return true;
+    doc::tile_index ti;
+    doc::tile_flags tf;
+    return get_tile_pixel(cel, pos, ti, tf, output);
   }
   // Regular images
   else {
@@ -145,12 +116,18 @@ void ColorPicker::pickColor(const Site& site,
         m_layer = cels.front()->layer();
 
       if (site.tilemapMode() == TilemapMode::Tiles) {
-        if (cels.empty() || !cels.front()->image()->isTilemap())
+        if (cels.empty())
           return;
 
-        const gfx::Point tilePos = site.grid().canvasToTile(gfx::Point(pos));
-        if (cels.front()->image()->bounds().contains(tilePos)) {
-          m_tile = doc::get_pixel(cels.front()->image(), tilePos.x, tilePos.y);
+        const Cel* cel = cels.front();
+        if (!cel->image()->isTilemap())
+          return;
+
+        doc::tile_index ti;
+        doc::tile_flags tf;
+        doc::color_t pixelColor;
+        if (get_tile_pixel(cel, pos, ti, tf, pixelColor)) {
+          m_tile = doc::tile(ti, tf);
           m_color = app::Color::fromTile(m_tile);
         }
       }
@@ -171,9 +148,12 @@ void ColorPicker::pickColor(const Site& site,
         return;
 
       if (site.tilemapMode() == TilemapMode::Tiles) {
-        const gfx::Point tilePos = site.grid().canvasToTile(gfx::Point(pos));
-        if (cel->image()->bounds().contains(tilePos)) {
-          m_tile = doc::get_pixel(cel->image(), tilePos.x, tilePos.y);
+        doc::tile_index ti;
+        doc::tile_flags tf;
+        doc::color_t pixelColor;
+        if (cel->layer()->isTilemap() &&
+            get_tile_pixel(cel, pos, ti, tf, pixelColor)) {
+          m_tile = doc::tile(ti, tf);
           m_color = app::Color::fromTile(m_tile);
         }
       }

src/doc/CMakeLists.txt

@@ -73,6 +73,7 @@ add_library(doc-lib
   tag.cpp
   tag_io.cpp
   tags.cpp
+  tile_primitives.cpp
   tileset.cpp
   tileset_io.cpp
   tilesets.cpp

src/doc/sprite.cpp

@@ -25,6 +25,7 @@
 #include "doc/render_plan.h"
 #include "doc/rgbmap_rgb5a3.h"
 #include "doc/tag.h"
+#include "doc/tile_primitives.h"
 #include "doc/tilesets.h"
 
 #include <algorithm>
@@ -691,48 +692,31 @@ void Sprite::pickCels(const gfx::PointF& pos,
     if (!celBounds.contains(pos))
       continue;
 
-    gfx::Point ipos;
+    color_t color = 0;
     if (image->isTilemap()) {
-      Tileset* tileset = static_cast<LayerTilemap*>(cel->layer())->tileset();
-      if (!tileset)
+      tile_index ti;
+      tile_index tf;
+      if (!get_tile_pixel(cel, pos, ti, tf, color))
         continue;
-
-      const Grid grid = cel->grid();
-
-      tile_t tile = notile;
-      const gfx::Point tilePos = grid.canvasToTile(gfx::Point(pos));
-      if (image->bounds().contains(tilePos.x, tilePos.y))
-        tile = image->getPixel(tilePos.x, tilePos.y);
-      if (tile == notile)
-        continue;
-
-      const tile_index ti = tile_geti(tile);
-      image = tileset->get(ti).get();
-      if (!image)
-        continue;
-
-      gfx::Point tileStart = grid.tileToCanvas(tilePos);
-      ipos = gfx::Point(pos.x - tileStart.x,
-                        pos.y - tileStart.y);
     }
     else {
-      ipos = gfx::Point(
+      gfx::Point ipos(
         int((pos.x-celBounds.x)*image->width()/celBounds.w),
         int((pos.y-celBounds.y)*image->height()/celBounds.h));
+      if (!image->bounds().contains(ipos))
+        continue;
+
+      color = get_pixel(image, ipos.x, ipos.y);
     }
 
-    if (!image->bounds().contains(ipos))
-      continue;
-
-    const color_t color = get_pixel(image, ipos.x, ipos.y);
     bool isOpaque = true;
 
-    switch (image->pixelFormat()) {
+    switch (pixelFormat()) {
       case IMAGE_RGB:
         isOpaque = (rgba_geta(color) >= opacityThreshold);
         break;
       case IMAGE_INDEXED:
-        isOpaque = (color != image->maskColor());
+        isOpaque = (color != transparentColor());
         break;
       case IMAGE_GRAYSCALE:
         isOpaque = (graya_geta(color) >= opacityThreshold);

src/doc/tile_primitives.cpp

@@ -0,0 +1,85 @@
+// Aseprite Document Library
+// Copyright (c) 2023 Igara Studio S.A.
+//
+// This file is released under the terms of the MIT license.
+// Read LICENSE.txt for more information.
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "doc/tile_primitives.h"
+
+#include "doc/cel.h"
+#include "doc/color.h"
+#include "doc/grid.h"
+#include "doc/image.h"
+#include "doc/layer_tilemap.h"
+
+#define TILE_TRACE(...) // TRACE(__VA_ARGS__)
+
+namespace doc {
+
+bool get_tile_pixel(
+  // Input
+  const Image* tilemapImage,
+  const Tileset* tileset,
+  const Grid& grid,
+  const gfx::PointF& canvasPos,
+  // Output
+  tile_index& ti,
+  tile_index& tf,
+  color_t& tileImageColor)
+{
+  const gfx::Point tilePos = grid.canvasToTile(gfx::Point(canvasPos));
+  TILE_TRACE("TILE: tilePos=(%d %d)\n", tilePos.x, tilePos.y);
+  if (!tilemapImage->bounds().contains(tilePos))
+    return false;
+
+  const doc::tile_t t = doc::get_pixel(tilemapImage, tilePos.x, tilePos.y);
+  ti = doc::tile_geti(t);
+  tf = doc::tile_getf(t);
+
+  TILE_TRACE("TILE: tile=0x%08x index=%d flags=0x%08x\n", t, ti, tf);
+
+  const doc::ImageRef tile = tileset->get(ti);
+  if (!tile)
+    return false;
+
+  const gfx::Point tileStart = grid.tileToCanvas(tilePos);
+  gfx::Point ipos = gfx::Point(canvasPos) - tileStart;
+  if (tf & doc::tile_f_xflip) { ipos.x = tile->width()-ipos.x-1; }
+  if (tf & doc::tile_f_yflip) { ipos.y = tile->height()-ipos.y-1; }
+  if (tf & doc::tile_f_dflip) { std::swap(ipos.x, ipos.y); }
+
+  tileImageColor = get_pixel(tile.get(), ipos.x, ipos.y);
+
+  TILE_TRACE("TILE: tileImagePos=%d %d\n", ipos.x, ipos.y);
+  TILE_TRACE("TILE: tileImageColor=%d\n", tileImageColor);
+  return true;
+}
+
+bool get_tile_pixel(
+  // Input
+  const Cel* cel,
+  const gfx::PointF& canvasPos,
+  // Output
+  tile_index& ti,
+  tile_index& tf,
+  color_t& tileImageColor)
+{
+  if (!cel ||
+      !cel->layer()->isTilemap() ||
+      !cel->image()->isTilemap())
+    return false;
+
+  Tileset* tileset = static_cast<LayerTilemap*>(cel->layer())->tileset();
+  if (!tileset)
+    return false;
+
+  return get_tile_pixel(
+    cel->image(), tileset, cel->grid(), canvasPos,
+    ti, tf, tileImageColor);
+}
+
+} // namespace doc

src/doc/tile_primitives.h

@@ -0,0 +1,58 @@
+// Aseprite Document Library
+// Copyright (c) 2023 Igara Studio S.A.
+//
+// This file is released under the terms of the MIT license.
+// Read LICENSE.txt for more information.
+
+#ifndef DOC_TILE_PRIMITIVES_H_INCLUDED
+#define DOC_TILE_PRIMITIVES_H_INCLUDED
+#pragma once
+
+#include "doc/color.h"
+#include "doc/tile.h"
+#include "gfx/point.h"
+
+namespace doc {
+
+class Cel;
+class Grid;
+class Image;
+class Tileset;
+
+// Returns true if "canvasPos" is inside the given tilemap with the
+// given grid settings, and the tile in that "canvasPos" is.
+//
+// Input:
+// * tilemapImage: a tilemap layer
+// * tileset: a tileset for the tilemap to get tiles
+// * grid: the grid settings (e.g. cel->grid())
+// * canvasPos: a position on the sprite
+//
+// Output:
+// * ti: the tile index in the "canvasPos"
+// * tf: the flags/flips of that tile
+// * tileImageColor: the pixel color above the tile image
+//   on "canvasPos" after applying the tile flips (tf)
+bool get_tile_pixel(
+  // Input
+  const Image* tilemapImage,
+  const Tileset* tileset,
+  const Grid& grid,
+  const gfx::PointF& canvasPos,
+  // Output
+  tile_index& ti,
+  tile_index& tf,
+  color_t& tileImageColor);
+
+bool get_tile_pixel(
+  // Input
+  const Cel* cel,
+  const gfx::PointF& canvasPos,
+  // Output
+  tile_index& ti,
+  tile_index& tf,
+  color_t& tileImageColor);
+
+} // namespace doc
+
+#endif