New tile flags meaning (x/y/diagonal flip) + serialize then correctly

* Changed the "90cw" flag to "diagonal flip" (the tile should be
  rendered with X/Y axis switched in this case)
* Each time we read/write an .aseprite file we have to convert
  the mask/shift from the file to the values expected in
  memory (tile_f_xflip/yflip/dflip)

docs/ase-file-specs.md

@@ -235,7 +235,7 @@ This chunk determine where to put a cel in the specified layer/frame.
       DWORD     Bitmask for tile ID (e.g. 0x1fffffff for 32-bit tiles)
       DWORD     Bitmask for X flip
       DWORD     Bitmask for Y flip
-      DWORD     Bitmask for 90CW rotation
+      DWORD     Bitmask for diagonal flip (swap X/Y axis)
       BYTE[10]  Reserved
       TILE[]    Row by row, from top to bottom tile by tile
                 compressed with ZLIB method (see NOTE.3)

src/app/file/ase_format.cpp

@@ -1045,9 +1045,9 @@ static void ase_file_write_cel_chunk(FILE* f, dio::AsepriteFrameHeader* frame_he
       fputw(image->height(), f);
       fputw(32, f);             // TODO use different bpp when possible
       fputl(tile_i_mask, f);
-      fputl(tile_f_flipx, f);
-      fputl(tile_f_flipy, f);
-      fputl(tile_f_90cw, f);
+      fputl(tile_f_xflip, f);
+      fputl(tile_f_yflip, f);
+      fputl(tile_f_dflip, f);
       ase_file_write_padding(f, 10);
 
       ImageScanlines scan(image);

src/app/ui/status_bar.cpp

@@ -519,9 +519,10 @@ public:
       else
         str += fmt::format("{}", ti + baseIndex - 1);
       if (tf) {
-        if (tf & doc::tile_f_flipx) str += " FlipX";
-        if (tf & doc::tile_f_flipy) str += " FlipY";
-        if (tf & doc::tile_f_90cw) str += " Rot90CW";
+        str += " Flip ";
+        if (tf & doc::tile_f_xflip) str += "X";
+        if (tf & doc::tile_f_yflip) str += "Y";
+        if (tf & doc::tile_f_dflip) str += "D";
       }
     }
     m_indicators->addTextIndicator(str.c_str());

src/dio/aseprite_decoder.cpp

@@ -15,6 +15,7 @@
 #include "base/exception.h"
 #include "base/file_handle.h"
 #include "base/fs.h"
+#include "base/mask_shift.h"
 #include "dio/aseprite_common.h"
 #include "dio/decode_delegate.h"
 #include "dio/file_interface.h"
@@ -948,10 +949,11 @@ doc::Cel* AsepriteDecoder::readCelChunk(doc::Sprite* sprite,
       int h = read16();
       int bitsPerTile = read16();
       uint32_t tileIDMask = read32();
-      uint32_t flipxMask = read32();
-      uint32_t flipyMask = read32();
-      uint32_t rot90Mask = read32();
-      uint32_t flagsMask = (flipxMask | flipyMask | rot90Mask);
+      uint32_t tileIDShift = base::mask_shift(tileIDMask);
+      uint32_t xflipMask = read32();
+      uint32_t yflipMask = read32();
+      uint32_t dflipMask = read32();
+      uint32_t flagsMask = (xflipMask | yflipMask | dflipMask);
       readPadding(10);
 
       // We only support 32bpp at the moment
@@ -980,12 +982,37 @@ doc::Cel* AsepriteDecoder::readCelChunk(doc::Sprite* sprite,
           doc::fix_old_tilemap(image.get(), ts, tileIDMask, flagsMask);
         }
 
-        // normalize the tile, so its value is never out of bounds
-        const doc::tile_index tilesetSize = ts->size();
+        // Convert the tile index and masks to a proper in-memory
+        // representation for the doc-lib.
         doc::transform_image<doc::TilemapTraits>(
           image.get(),
-          [tilesetSize](const doc::tile_t& tile){
-            return doc::tile_geti(tile) >= tilesetSize ? doc::notile : tile;
+          [ts, tileIDMask, tileIDShift,
+           xflipMask, yflipMask, dflipMask, flagsMask]
+          (doc::tile_t tile) {
+            // Get the tile index.
+            doc::tile_index ti = ((tile & tileIDMask) >> tileIDShift);
+
+            // If the index is out of bounds from the tileset, we
+            // allow to keep some small values in-memory, but if the
+            // index is too big, we consider it as a broken file and
+            // remove the tile (as an huge index bring some lag
+            // problems in the remove_unused_tiles_from_tileset()
+            // creating a big Remap structure).
+            //
+            // Related to https://github.com/aseprite/aseprite/issues/2877
+            if (ti > ts->size() &&
+                ti > 0xffffff) {
+              return doc::notile;
+            }
+
+            // Convert read index to doc::tile_i_mask, and flags to doc::tile_f_mask
+            tile = doc::tile(
+              ti,
+              ((tile & xflipMask) == xflipMask ? doc::tile_f_xflip: 0) |
+              ((tile & yflipMask) == yflipMask ? doc::tile_f_yflip: 0) |
+              ((tile & dflipMask) == dflipMask ? doc::tile_f_dflip: 0));
+
+            return tile;
           });
 
         cel = std::make_unique<doc::Cel>(frame, image);

src/doc/tile.h

@@ -1,5 +1,5 @@
 // Aseprite Document Library
-// Copyright (c) 2019-2020  Igara Studio S.A.
+// Copyright (c) 2019-2023  Igara Studio S.A.
 //
 // This file is released under the terms of the MIT license.
 // Read LICENSE.txt for more information.
@@ -18,14 +18,14 @@ namespace doc {
   typedef uint32_t tile_flags;
 
   const uint32_t tile_i_shift = 0;  // Tile index
-  const uint32_t tile_f_shift = 28; // Flags (flip, rotation)
+  const uint32_t tile_f_shift = 28; // Tile flags (flips)
 
   const uint32_t notile       = 0;
   const uint32_t tile_i_mask  = 0x1fffffff;
   const uint32_t tile_f_mask  = 0xe0000000; // 3 flags
-  const uint32_t tile_f_flipx = 0x20000000;
-  const uint32_t tile_f_flipy = 0x40000000;
-  const uint32_t tile_f_90cw  = 0x80000000;
+  const uint32_t tile_f_xflip = 0x80000000;
+  const uint32_t tile_f_yflip = 0x40000000;
+  const uint32_t tile_f_dflip = 0x20000000;
 
   inline tile_index tile_geti(const tile_t t) {
     return ((t & tile_i_mask) >> tile_i_shift);