Add support to match flipped tiles automatically in Auto/Stack modes

By default Aseprite will not try to match flipped versions of the
tiles (as it requires more CPU), but when we create a tileset we can
specify which flips can be matched automatically (new
Tileset::matchFlags() property).

These flags are just for the Auto mode, if we manually insert a
flipped tile, that is always supported, even when the matchFlags() are
not specified.

data/strings/en.ini

@@ -1425,6 +1425,11 @@ Visible aid to see the first tile with content from the tileset
 as index 1 (by default, one-based index) or other value.
 E.g. you can use 0 here for zero-based indexing.
 END
+allow_flipped_tiles = Allow Flipped Tiles:
+allow_flipped_tiles_tooltip = <<<END
+Aseprite can reuse tiles matching automatically with their flipped
+versions (in X, Y, or Diagonal axes) in Auto/Stack modes.
+END
 
 [tileset_selector_window]
 title = Tileset

data/widgets/tileset_selector.xml

@@ -19,5 +19,14 @@
     <expr id="base_index" text="1" tooltip="@.base_tooltip" />
     <boxfiller cell_hspan="2" />
   </grid>
+
+  <hbox>
+    <label text="@.allow_flipped_tiles" />
+    <buttonset id="flipped_tiles" columns="3" multiple="true">
+      <item id="xflip" text="X" minwidth="20" tooltip="@.allow_flipped_tiles_tooltip" tooltip_dir="bottom" />
+      <item id="yflip" text="Y" minwidth="20" tooltip="@.allow_flipped_tiles_tooltip" tooltip_dir="bottom" />
+      <item id="dflip" text="D" minwidth="20" tooltip="@.allow_flipped_tiles_tooltip" tooltip_dir="bottom" />
+    </buttonset>
+  </hbox>
 </vbox>
 </gui>

docs/ase-file-specs.md

@@ -485,6 +485,11 @@ The data of this chunk is as follows:
                       (this is the new format). In rare cases this bit is off,
                       and the empty tile will be equal to 0xffffffff (used in
                       internal versions of Aseprite)
+                  8 - Aseprite will try to match modified tiles with their X
+                      flipped version automatically in Auto mode when using
+                      this tileset.
+                  16 - Same for Y flips
+                  32 - Same for D(iagonal) flips
     DWORD       Number of tiles
     WORD        Tile Width
     WORD        Tile Height

src/app/CMakeLists.txt

@@ -541,6 +541,7 @@ add_library(app-lib
   cmd/set_tile_data_properties.cpp
   cmd/set_tile_data_property.cpp
   cmd/set_tileset_base_index.cpp
+  cmd/set_tileset_match_flags.cpp
   cmd/set_tileset_name.cpp
   cmd/set_total_frames.cpp
   cmd/set_transparent_color.cpp

src/app/cmd/set_tileset_match_flags.cpp

@@ -0,0 +1,45 @@
+// Aseprite
+// Copyright (C) 2023  Igara Studio S.A.
+//
+// This program is distributed under the terms of
+// the End-User License Agreement for Aseprite.
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "app/cmd/set_tileset_match_flags.h"
+
+#include "app/doc.h"
+#include "app/doc_event.h"
+#include "doc/tileset.h"
+
+namespace app {
+namespace cmd {
+
+SetTilesetMatchFlags::SetTilesetMatchFlags(Tileset* tileset,
+                                           const tile_flags matchFlags)
+  : WithTileset(tileset)
+  , m_oldMatchFlags(tileset->matchFlags())
+  , m_newMatchFlags(matchFlags)
+{
+}
+
+void SetTilesetMatchFlags::onExecute()
+{
+  auto ts = tileset();
+  ts->setMatchFlags(m_newMatchFlags);
+  ts->incrementVersion();
+  ts->sprite()->incrementVersion();
+}
+
+void SetTilesetMatchFlags::onUndo()
+{
+  auto ts = tileset();
+  ts->setMatchFlags(m_oldMatchFlags);
+  ts->incrementVersion();
+  ts->sprite()->incrementVersion();
+}
+
+} // namespace cmd
+} // namespace app

src/app/cmd/set_tileset_match_flags.h

@@ -0,0 +1,40 @@
+// Aseprite
+// Copyright (C) 2023  Igara Studio S.A.
+//
+// This program is distributed under the terms of
+// the End-User License Agreement for Aseprite.
+
+#ifndef APP_CMD_SET_TILESET_MATCH_FLAGS_H_INCLUDED
+#define APP_CMD_SET_TILESET_MATCH_FLAGS_H_INCLUDED
+#pragma once
+
+#include "app/cmd.h"
+#include "app/cmd/with_tileset.h"
+#include "doc/tile.h"
+
+namespace app {
+namespace cmd {
+  using namespace doc;
+
+  class SetTilesetMatchFlags : public Cmd
+                            , public WithTileset {
+  public:
+    SetTilesetMatchFlags(Tileset* tileset,
+                         const tile_flags matchFlags);
+
+  protected:
+    void onExecute() override;
+    void onUndo() override;
+    size_t onMemSize() const override {
+      return sizeof(*this);
+    }
+
+  private:
+    tile_flags m_oldMatchFlags;
+    tile_flags m_newMatchFlags;
+  };
+
+} // namespace cmd
+} // namespace app
+
+#endif

src/app/commands/cmd_layer_properties.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2020-2022  Igara Studio S.A.
+// Copyright (C) 2020-2023  Igara Studio S.A.
 // Copyright (C) 2001-2018  David Capello
 //
 // This program is distributed under the terms of
@@ -15,6 +15,7 @@
 #include "app/cmd/set_layer_opacity.h"
 #include "app/cmd/set_layer_tileset.h"
 #include "app/cmd/set_tileset_base_index.h"
+#include "app/cmd/set_tileset_match_flags.h"
 #include "app/cmd/set_tileset_name.h"
 #include "app/cmd/set_user_data.h"
 #include "app/commands/command.h"
@@ -362,6 +363,7 @@ private:
     tilesetInfo.grid = tileset->grid();
     tilesetInfo.name = tileset->name();
     tilesetInfo.baseIndex = tileset->baseIndex();
+    tilesetInfo.matchFlags = tileset->matchFlags();
     tilesetInfo.tsi = tilemap->tilesetIndex();
 
     try {
@@ -376,6 +378,7 @@ private:
 
       if (tileset->name() != tilesetInfo.name ||
           tileset->baseIndex() != tilesetInfo.baseIndex ||
+          tileset->matchFlags() != tilesetInfo.matchFlags ||
           tilesetInfo.tsi != tilemap->tilesetIndex()) {
         ContextWriter writer(UIContext::instance());
         Tx tx(writer.context(), "Set Tileset Properties");
@@ -388,6 +391,8 @@ private:
           tx(new cmd::SetTilesetName(tileset, tilesetInfo.name));
         if (tileset->baseIndex() != tilesetInfo.baseIndex)
           tx(new cmd::SetTilesetBaseIndex(tileset, tilesetInfo.baseIndex));
+        if (tileset->matchFlags() != tilesetInfo.matchFlags)
+          tx(new cmd::SetTilesetMatchFlags(tileset, tilesetInfo.matchFlags));
         // TODO catch the tileset base index modification from the editor
         App::instance()->mainWindow()->invalidate();
         tx.commit();

src/app/commands/cmd_new_layer.cpp

@@ -208,6 +208,7 @@ void NewLayerCommand::onExecute(Context* context)
                       context->activeSite().grid():
                       doc::Grid(params().gridBounds()));
   tilesetInfo.baseIndex = 1;
+  tilesetInfo.matchFlags = 0;   // TODO default flags?
 
 #ifdef ENABLE_UI
   // If params specify to ask the user about the name...
@@ -236,8 +237,8 @@ void NewLayerCommand::onExecute(Context* context)
 
     name = window.name()->text();
     if (tilesetSelector) {
-      pref.tileset.baseIndex(tilesetSelector->getInfo().baseIndex);
       tilesetInfo = tilesetSelector->getInfo();
+      pref.tileset.baseIndex(tilesetInfo.baseIndex);
     }
   }
 #endif
@@ -286,6 +287,7 @@ void NewLayerCommand::onExecute(Context* context)
         if (tilesetInfo.newTileset) {
           auto tileset = new Tileset(sprite, tilesetInfo.grid, 1);
           tileset->setBaseIndex(tilesetInfo.baseIndex);
+          tileset->setMatchFlags(tilesetInfo.matchFlags);
           tileset->setName(tilesetInfo.name);
 
           auto addTileset = new cmd::AddTileset(sprite, tileset);

src/app/file/ase_format.cpp

@@ -1437,6 +1437,11 @@ static void ase_file_write_tileset_chunk(FILE* f, FileOp* fop,
   else
     flags |= ASE_TILESET_FLAG_EMBEDDED;
 
+  doc::tile_flags tf = tileset->matchFlags();
+  if (tf & doc::tile_f_xflip) flags |= ASE_TILESET_FLAG_MATCH_XFLIP;
+  if (tf & doc::tile_f_yflip) flags |= ASE_TILESET_FLAG_MATCH_YFLIP;
+  if (tf & doc::tile_f_dflip) flags |= ASE_TILESET_FLAG_MATCH_DFLIP;
+
   fputl(si, f);         // Tileset ID
   fputl(flags, f);      // Tileset Flags
   fputl(tileset->size(), f);

src/app/ui/tileset_selector.cpp

@@ -26,10 +26,10 @@ TilesetSelector::TilesetSelector(const doc::Sprite* sprite,
 {
   initTheme();
 
-  name()->setText(m_info.name);
-  gridWidth()->setTextf("%d", m_info.grid.tileSize().w);
-  gridHeight()->setTextf("%d", m_info.grid.tileSize().h);
-  baseIndex()->setTextf("%d", m_info.baseIndex);
+  fillControls(m_info.name,
+               m_info.grid.tileSize(),
+               m_info.baseIndex,
+               m_info.matchFlags);
 
   if (!m_info.allowNewTileset) {
     tilesets()->deleteAllItems();
@@ -63,22 +63,35 @@ TilesetSelector::TilesetSelector(const doc::Sprite* sprite,
   updateControlsState(sprite->tilesets());
 }
 
+void TilesetSelector::fillControls(const std::string& nameValue,
+                                   const gfx::Size& gridSize,
+                                   const int baseIndexValue,
+                                   const doc::tile_flags matchFlags)
+{
+  name()->setText(nameValue);
+  gridWidth()->setTextf("%d", gridSize.w);
+  gridHeight()->setTextf("%d", gridSize.h);
+  baseIndex()->setTextf("%d", baseIndexValue);
+  xflip()->setSelected((matchFlags & doc::tile_f_xflip) ? true: false);
+  yflip()->setSelected((matchFlags & doc::tile_f_yflip) ? true: false);
+  dflip()->setSelected((matchFlags & doc::tile_f_dflip) ? true: false);
+}
+
 void TilesetSelector::updateControlsState(const doc::Tilesets* spriteTilesets)
 {
   if (m_info.enabled) {
-    int index = getSelectedItemIndex();
-    bool isNewTileset = (index == 0);
+    const int index = getSelectedItemIndex();
+    const bool isNewTileset = (index == 0);
     if (isNewTileset) {
       name()->setText("");
       baseIndex()->setTextf("%d", 1);
     }
     else {
-      doc::Tileset* ts = spriteTilesets->get(index-1);
-      doc::Grid grid = ts->grid();
-      name()->setText(ts->name());
-      gridWidth()->setTextf("%d", grid.tileSize().w);
-      gridHeight()->setTextf("%d", grid.tileSize().h);
-      baseIndex()->setTextf("%d", ts->baseIndex());
+      const doc::Tileset* ts = spriteTilesets->get(index-1);
+      fillControls(ts->name(),
+                   ts->grid().tileSize(),
+                   ts->baseIndex(),
+                   ts->matchFlags());
     }
 
     name()->setEnabled(isNewTileset || !m_info.allowNewTileset);
@@ -90,6 +103,10 @@ void TilesetSelector::updateControlsState(const doc::Tilesets* spriteTilesets)
     tilesets()->setEnabled(false);
     gridWidth()->setEnabled(false);
     gridHeight()->setEnabled(false);
+    baseIndex()->setEnabled(false);
+    xflip()->setEnabled(false);
+    yflip()->setEnabled(false);
+    dflip()->setEnabled(false);
   }
 }
 
@@ -110,6 +127,11 @@ TilesetSelector::Info TilesetSelector::getInfo()
   }
   info.name = name()->text();
   info.baseIndex = baseIndex()->textInt();
+  info.matchFlags =
+    (xflip()->isSelected() ? doc::tile_f_xflip: 0) |
+    (yflip()->isSelected() ? doc::tile_f_yflip: 0) |
+    (dflip()->isSelected() ? doc::tile_f_dflip: 0);
+
   return info;
 }
 

src/app/ui/tileset_selector.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (c) 2019-2020  Igara Studio S.A.
+// Copyright (c) 2019-2023  Igara Studio S.A.
 //
 // This program is distributed under the terms of
 // the End-User License Agreement for Aseprite.
@@ -37,6 +37,7 @@ namespace app {
       std::string name;
       doc::Grid grid;
       int baseIndex = 1;
+      doc::tile_flags matchFlags = 0;
       doc::tileset_index tsi = -1;
     };
 
@@ -47,6 +48,10 @@ namespace app {
     Info getInfo();
 
   private:
+    void fillControls(const std::string& name,
+                      const gfx::Size& gridSize,
+                      const int baseIndex,
+                      const doc::tile_flags matchFlags);
     void updateControlsState(const doc::Tilesets* spriteTilesets);
 
     // Returns the selected item index as if the combobox always has the "New Tileset"

src/app/util/cel_ops.cpp

@@ -120,6 +120,136 @@ struct Mod {
   gfx::Region tileRgn;
 };
 
+class DoFlip {
+public:
+  DoFlip(const doc::ImageRef& image,
+         const doc::algorithm::FlipType flipType)
+    : m_image(image.get())
+    , m_flipType(flipType) {
+  }
+  ~DoFlip() {
+    reset();
+  }
+  void flip() {
+    m_flipped = !m_flipped;
+    doc::algorithm::flip_image(m_image,
+                               m_image->bounds(),
+                               m_flipType);
+  }
+  void reset() {
+    if (m_flipped)
+      flip();
+  }
+  bool operator!() const {
+    return !m_flipped;
+  }
+private:
+  doc::Image* m_image;
+  bool m_flipped = false;
+  doc::algorithm::FlipType m_flipType;
+};
+
+// This is a terrible way to find tiles, i.e. flipping several times
+// the image, instead of searching for flipped hashes. In the future
+// we could try to improve it.
+bool find_tile(doc::Tileset* tileset,
+               doc::ImageRef& tileImage,
+               doc::tile_index& tileIndex,
+               doc::tile_flags& tileFlags)
+{
+  // Find without flags
+  if (tileset->findTileIndex(tileImage, tileIndex)) {
+    tileFlags = 0;
+    return true;
+  }
+
+  if (tileset->matchFlags() == 0) // In case we don't allow flipped tiles
+    return false;
+
+  DoFlip x(tileImage, doc::algorithm::FlipHorizontal);
+
+  // Find with X flip
+  if (tileset->matchFlags() & doc::tile_f_xflip) {
+    x.flip();
+    if (tileset->findTileIndex(tileImage, tileIndex)) {
+      tileFlags = doc::tile_f_xflip;
+      return true;
+    }
+    x.reset();
+  }
+
+  // Find with Y flip
+  DoFlip y(tileImage, doc::algorithm::FlipVertical);
+  if (tileset->matchFlags() & doc::tile_f_yflip) {
+    y.flip();
+    if (tileset->findTileIndex(tileImage, tileIndex)) {
+      tileFlags = doc::tile_f_yflip;
+      return true;
+    }
+
+    if (tileset->matchFlags() & doc::tile_f_xflip) {
+      // Find with X+Y flip
+      x.flip();
+      if (tileset->findTileIndex(tileImage, tileIndex)) {
+        tileFlags = doc::tile_f_xflip | doc::tile_f_yflip;
+        return true;
+      }
+      x.reset();
+    }
+    y.reset();
+  }
+
+  // Check if we can match diagonal flips
+  if ((tileset->matchFlags() & doc::tile_f_dflip) == 0)
+    return false;
+
+  // Find with D flip
+  DoFlip d(tileImage, doc::algorithm::FlipDiagonal);
+  d.flip();
+  if (tileset->findTileIndex(tileImage, tileIndex)) {
+    tileFlags = doc::tile_f_dflip;
+    return true;
+  }
+
+  // Find with X+D flip
+  if (tileset->matchFlags() & doc::tile_f_xflip) {
+    d.reset();
+    x.flip();
+    d.flip();
+    if (tileset->findTileIndex(tileImage, tileIndex)) {
+      tileFlags = doc::tile_f_xflip | doc::tile_f_dflip;
+      return true;
+    }
+
+    // Find with X+Y+D flip
+    if (tileset->matchFlags() & doc::tile_f_yflip) {
+      d.reset();
+      y.flip();
+      d.flip();
+      if (tileset->findTileIndex(tileImage, tileIndex)) {
+        tileFlags = doc::tile_f_xflip | doc::tile_f_yflip | doc::tile_f_dflip;
+        return true;
+      }
+    }
+  }
+
+  // Find with Y+D flip only
+  if (tileset->matchFlags() & doc::tile_f_yflip) {
+    d.reset();
+    x.reset();
+    if (!y)
+      y.flip();
+    d.flip();
+    if (tileset->findTileIndex(tileImage, tileIndex)) {
+      tileFlags =  doc::tile_f_yflip | doc::tile_f_dflip;
+      return true;
+    }
+  }
+
+  // DoFlip destructors will reset the image.
+  return false;
+}
+
 } // anonymous namespace
 
 void create_region_with_differences(const Image* a,
@@ -411,7 +541,9 @@ void draw_image_into_new_tilemap_cel(
     preprocess_transparent_pixels(tileImage.get());
 
     doc::tile_index tileIndex;
-    if (!tileset->findTileIndex(tileImage, tileIndex)) {
+    doc::tile_flags tileFlag = 0;
+
+    if (!find_tile(tileset, tileImage, tileIndex, tileFlag)) {
       auto addTile = new cmd::AddTile(tileset, tileImage);
 
       if (cmds)
@@ -436,7 +568,8 @@ void draw_image_into_new_tilemap_cel(
       const int u = tilePt.x-tilemapBounds.x;
       const int v = tilePt.y-tilemapBounds.y;
       ASSERT((u >= 0) && (v >= 0) && (u < newTilemap->width()) && (v < newTilemap->height()));
-      doc::put_pixel(newTilemap.get(), u, v, tileIndex);
+      doc::put_pixel(newTilemap.get(), u, v,
+                     doc::tile(tileIndex, tileFlag));
     }
   }
 
@@ -555,8 +688,10 @@ void modify_tilemap_cel_region(
 
       preprocess_transparent_pixels(tileImage.get());
 
-      tile_index tileIndex;
-      if (tileset->findTileIndex(tileImage, tileIndex)) {
+      doc::tile_index tileIndex;
+      doc::tile_flags tileFlag = 0;
+
+      if (find_tile(tileset, tileImage, tileIndex, tileFlag)) {
         // We can re-use an existent tile (tileIndex) from the tileset
       }
       else if (tilesetMode == TilesetMode::Auto &&
@@ -602,7 +737,7 @@ void modify_tilemap_cel_region(
                 (t == doc::notile ? -1: ti),
                 tileIndex);
 
-      const doc::tile_t tile = doc::tile(tileIndex, 0);
+      const doc::tile_t tile = doc::tile(tileIndex, tileFlag);
       if (t != tile) {
         newTilemap->putPixel(u, v, tile);
         tilePtsRgn |= gfx::Region(gfx::Rect(u, v, 1, 1));

src/dio/aseprite_common.h

@@ -64,6 +64,9 @@
 #define ASE_TILESET_FLAG_EXTERNAL_FILE      1
 #define ASE_TILESET_FLAG_EMBEDDED           2
 #define ASE_TILESET_FLAG_ZERO_IS_NOTILE     4
+#define ASE_TILESET_FLAG_MATCH_XFLIP        8
+#define ASE_TILESET_FLAG_MATCH_YFLIP        16
+#define ASE_TILESET_FLAG_MATCH_DFLIP        32
 
 #define ASE_EXTERNAL_FILE_PALETTE           0
 #define ASE_EXTERNAL_FILE_TILESET           1

src/dio/aseprite_decoder.cpp

@@ -1340,6 +1340,11 @@ doc::Tileset* AsepriteDecoder::readTilesetChunk(
     sprite->tilesets()->set(id, tileset);
   }
 
+  tileset->setMatchFlags(
+    (flags & ASE_TILESET_FLAG_MATCH_XFLIP ? doc::tile_f_xflip: 0) |
+    (flags & ASE_TILESET_FLAG_MATCH_YFLIP ? doc::tile_f_yflip: 0) |
+    (flags & ASE_TILESET_FLAG_MATCH_DFLIP ? doc::tile_f_dflip: 0));
+
   if (id >= m_tilesetFlags.size())
     m_tilesetFlags.resize(id+1, 0);
   m_tilesetFlags[id] = flags;

src/doc/tileset.h

@@ -59,6 +59,11 @@ namespace doc {
     int baseIndex() const { return m_baseIndex; }
     void setBaseIndex(int index) { m_baseIndex = index; }
 
+    // Allow to match tiles with the given flags/flips automatically
+    // in Auto/Stack modes.
+    tile_flags matchFlags() const { return m_matchFlags; }
+    void setMatchFlags(const tile_flags tf) { m_matchFlags = tf; }
+
     // Cached compressed tileset read/writen directly from .aseprite
     // files.
     void discardCompressedData();
@@ -153,6 +158,7 @@ namespace doc {
     TilesetHashTable m_hash;
     std::string m_name;
     int m_baseIndex = 1;
+    tile_flags m_matchFlags = 0;
     struct External {
       std::string filename;
       tileset_index tileset;