Cache compressed tileset information from .aseprite files on memory

We've added an experimental option (enabled by default) to keep the
compressed tileset data when we load/save a .aseprite file to avoid
recompressing each time we save (and only compressing the tileset if
tiles are modified).

This is an attempt to make the save operation faster when we use
sprites with several tilemap layers + large tilesets (many tiles, with
big tiles).

Reference: https://github.com/aseprite/Attachment-System/issues/54

data/pref.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <!-- Aseprite -->
-<!-- Copyright (C) 2018-2022  Igara Studio S.A. -->
+<!-- Copyright (C) 2018-2023  Igara Studio S.A. -->
 <!-- Copyright (C) 2014-2018  David Capello -->
 <preferences>
 
@@ -427,6 +427,7 @@
     </section>
     <section id="tileset">
       <option id="base_index" type="int" default="1" />
+      <option id="cache_compressed_tilesets" type="bool" default="true" />
     </section>
   </global>
 

data/strings/en.ini

@@ -1701,6 +1701,7 @@ native_clipboard = Use native clipboard
 native_file_dialog = Use native file dialog
 shaders_for_color_selectors = Use shaders for color selectors
 hue_with_sat_value = Apply Saturation/Value to Hue slider on Tint/Shade/Tone selector
+cache_compressed_tilesets = Cache compressed tilesets for faster save (uses more memory)
 one_finger_as_mouse_movement = Interpret one finger as mouse movement
 one_finger_as_mouse_movement_tooltip = <<<END
 Only for Windows 8/10 Pointer API: Interprets one finger as mouse movement

data/widgets/options.xml

@@ -1,5 +1,5 @@
 <!-- Aseprite -->
-<!-- Copyright (C) 2018-2022  Igara Studio S.A. -->
+<!-- Copyright (C) 2018-2023  Igara Studio S.A. -->
 <!-- Copyright (C) 2001-2018  David Capello -->
 <gui>
   <window id="options" text="@.title">
@@ -522,6 +522,9 @@
           <check id="tint_shade_tone_hue_with_sat_value"
                  text="@.hue_with_sat_value"
                  pref="experimental.hue_with_sat_value_for_color_selector" />
+          <check id="cache_compressed_tilesets"
+                 text="@.cache_compressed_tilesets"
+                 pref="tileset.cache_compressed_tilesets" />
           <hbox id="load_wintab_driver_box">
             <check id="load_wintab_driver2"
                    text="@.load_wintab_driver"

src/app/file/ase_format.cpp

@@ -18,6 +18,7 @@
 #include "base/exception.h"
 #include "base/file_handle.h"
 #include "base/fs.h"
+#include "base/mem_utils.h"
 #include "dio/aseprite_common.h"
 #include "dio/aseprite_decoder.h"
 #include "dio/decode_delegate.h"
@@ -33,6 +34,8 @@
 #include <deque>
 #include <variant>
 
+#define ASEFILE_TRACE(...) // TRACE(__VA_ARGS__)
+
 namespace app {
 
 using namespace base;
@@ -77,6 +80,10 @@ public:
 
   doc::Sprite* sprite() { return m_sprite; }
 
+  bool cacheCompressedTilesets() const {
+    return m_fop->config().cacheCompressedTilesets;
+  }
+
 private:
   FileOp* m_fop;
   doc::Sprite* m_sprite;
@@ -855,7 +862,9 @@ static void write_raw_image(FILE* f, ScanlinesGen* gen, PixelFormat pixelFormat)
 //////////////////////////////////////////////////////////////////////
 
 template<typename ImageTraits>
-static void write_compressed_image_templ(FILE* f, ScanlinesGen* gen)
+static void write_compressed_image_templ(FILE* f,
+                                         ScanlinesGen* gen,
+                                         base::buffer* compressedOutput)
 {
   PixelIO<ImageTraits> pixel_io;
   z_stream zstream;
@@ -896,6 +905,17 @@ static void write_compressed_image_templ(FILE* f, ScanlinesGen* gen)
         if ((fwrite(&compressed[0], 1, output_bytes, f) != (size_t)output_bytes)
             || ferror(f))
           throw base::Exception("Error writing compressed image pixels.\n");
+
+        // Save the whole compressed buffer to re-use in following
+        // save options (so we don't have to re-compress the whole
+        // tileset)
+        if (compressedOutput) {
+          std::size_t n = compressedOutput->size();
+          compressedOutput->resize(n + output_bytes);
+          std::copy(compressed.begin(),
+                    compressed.begin() + output_bytes,
+                    compressedOutput->begin() + n);
+        }
       }
     } while (zstream.avail_out == 0);
   }
@@ -905,23 +925,26 @@ static void write_compressed_image_templ(FILE* f, ScanlinesGen* gen)
     throw base::Exception("ZLib error %d in deflateEnd().", err);
 }
 
-static void write_compressed_image(FILE* f, ScanlinesGen* gen, PixelFormat pixelFormat)
+static void write_compressed_image(FILE* f,
+                                   ScanlinesGen* gen,
+                                   PixelFormat pixelFormat,
+                                   base::buffer* compressedOutput = nullptr)
 {
   switch (pixelFormat) {
     case IMAGE_RGB:
-      write_compressed_image_templ<RgbTraits>(f, gen);
+      write_compressed_image_templ<RgbTraits>(f, gen, compressedOutput);
       break;
 
     case IMAGE_GRAYSCALE:
-      write_compressed_image_templ<GrayscaleTraits>(f, gen);
+      write_compressed_image_templ<GrayscaleTraits>(f, gen, compressedOutput);
       break;
 
     case IMAGE_INDEXED:
-      write_compressed_image_templ<IndexedTraits>(f, gen);
+      write_compressed_image_templ<IndexedTraits>(f, gen, compressedOutput);
       break;
 
     case IMAGE_TILEMAP:
-      write_compressed_image_templ<TilemapTraits>(f, gen);
+      write_compressed_image_templ<TilemapTraits>(f, gen, compressedOutput);
       break;
   }
 }
@@ -1436,14 +1459,43 @@ static void ase_file_write_tileset_chunk(FILE* f, FileOp* fop,
   // Flag 2 = tileset
   if (flags & ASE_TILESET_FLAG_EMBEDDED) {
     size_t beg = ftell(f);
-    fputl(0, f);                  // Field for compressed data length (completed later)
-    TilesetScanlines gen(tileset);
-    write_compressed_image(f, &gen, tileset->sprite()->pixelFormat());
 
-    size_t end = ftell(f);
-    fseek(f, beg, SEEK_SET);
-    fputl(end-beg-4, f);          // Save the compressed data length
-    fseek(f, end, SEEK_SET);
+    // Save the cached tileset compressed data
+    if (!tileset->compressedData().empty() &&
+        tileset->compressedDataVersion() == tileset->version()) {
+      const base::buffer& data = tileset->compressedData();
+
+      ASEFILE_TRACE("[%d] saving compressed tileset (%s)\n",
+                    tileset->id(), base::get_pretty_memory_size(data.size()).c_str());
+
+      fputl(data.size(), f); // Compressed data length
+      fwrite(&data[0], 1, data.size(), f);
+    }
+    // Compress and save the tileset now
+    else {
+      fputl(0, f);                  // Field for compressed data length (completed later)
+      TilesetScanlines gen(tileset);
+
+      ASEFILE_TRACE("[%d] recompressing tileset\n", tileset->id());
+
+      base::buffer compressedData;
+      base::buffer* compressedDataPtr = nullptr;
+      if (fop->config().cacheCompressedTilesets)
+        compressedDataPtr = &compressedData;
+
+      write_compressed_image(f, &gen, tileset->sprite()->pixelFormat(),
+                             compressedDataPtr);
+
+      // As we've just compressed the tileset, we can cache this same
+      // data (so saving the file again will not need recompressing).
+      if (compressedDataPtr)
+        tileset->setCompressedData(compressedData);
+
+      size_t end = ftell(f);
+      fseek(f, beg, SEEK_SET);
+      fputl(end-beg-4, f);          // Save the compressed data length
+      fseek(f, end, SEEK_SET);
+    }
   }
 }
 

src/app/file/file_op_config.cpp

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2019-2021  Igara Studio S.A.
+// Copyright (C) 2019-2023  Igara Studio S.A.
 //
 // This program is distributed under the terms of
 // the End-User License Agreement for Aseprite.
@@ -25,6 +25,7 @@ void FileOpConfig::fillFromPreferences()
   defaultSliceColor = pref.slices.defaultColor();
   workingCS = get_working_rgb_space_from_preferences();
   rgbMapAlgorithm = pref.quantization.rgbmapAlgorithm();
+  cacheCompressedTilesets = pref.tileset.cacheCompressedTilesets();
 }
 
 } // namespace app

src/app/file/file_op_config.h

@@ -1,5 +1,5 @@
 // Aseprite
-// Copyright (C) 2019-2020  Igara Studio S.A.
+// Copyright (C) 2019-2023  Igara Studio S.A.
 //
 // This program is distributed under the terms of
 // the End-User License Agreement for Aseprite.
@@ -36,6 +36,12 @@ namespace app {
     // Algorithm used to create a palette from RGB files.
     doc::RgbMapAlgorithm rgbMapAlgorithm = doc::RgbMapAlgorithm::DEFAULT;
 
+    // Cache compressed tilesets. When we load a tileset from a
+    // .aseprite file, the compressed data will be stored on memory to
+    // make the save operation faster (as we can re-use the already
+    // compressed data that was loaded as-is).
+    bool cacheCompressedTilesets = true;
+
     void fillFromPreferences();
   };
 

src/dio/aseprite_decoder.cpp

@@ -1181,9 +1181,10 @@ doc::Slice* AsepriteDecoder::readSliceChunk(doc::Slices& slices)
   return slice.release();
 }
 
-doc::Tileset* AsepriteDecoder::readTilesetChunk(doc::Sprite* sprite,
-                                       const AsepriteHeader* header,
-                                       const AsepriteExternalFiles& extFiles)
+doc::Tileset* AsepriteDecoder::readTilesetChunk(
+  doc::Sprite* sprite,
+  const AsepriteHeader* header,
+  const AsepriteExternalFiles& extFiles)
 {
   const doc::tileset_index id = read32();
   const uint32_t flags = read32();
@@ -1228,6 +1229,14 @@ doc::Tileset* AsepriteDecoder::readTilesetChunk(doc::Sprite* sprite,
       const size_t dataBeg = f()->tell();
       const size_t dataEnd = dataBeg+dataSize;
 
+      base::buffer compressed;
+      if (delegate()->cacheCompressedTilesets() &&
+          dataSize > 0) {
+        compressed.resize(dataSize);
+        f()->readBytes(&compressed[0], dataSize);
+        f()->seek(dataBeg);
+      }
+
       doc::ImageRef alltiles(doc::Image::create(sprite->pixelFormat(), w, h*ntiles));
       alltiles->setMaskColor(sprite->transparentColor());
 
@@ -1242,6 +1251,9 @@ doc::Tileset* AsepriteDecoder::readTilesetChunk(doc::Sprite* sprite,
       // If we are reading and old .aseprite file (where empty tile is not the zero]
       if ((flags & ASE_TILESET_FLAG_ZERO_IS_NOTILE) == 0)
         doc::fix_old_tileset(tileset);
+
+      if (!compressed.empty())
+        tileset->setCompressedData(compressed);
     }
     sprite->tilesets()->set(id, tileset);
   }

src/dio/decode_delegate.h

@@ -1,4 +1,5 @@
 // Aseprite Document IO Library
+// Copyright (c) 2023 Igara Studio S.A.
 // Copyright (c) 2017 David Capello
 //
 // This file is released under the terms of the MIT license.
@@ -44,6 +45,13 @@ public:
     // sprite and then discard it when you don't need it anymore.
     delete sprite;
   }
+
+  // Returns true if we want to cache the read compressed data of
+  // tilesets exactly as they are in the disk (so we can save it
+  // without re-compressing).
+  virtual bool cacheCompressedTilesets() const {
+    return false;
+  }
 };
 
 } // namespace dio

src/doc/tileset.cpp

@@ -10,6 +10,7 @@
 
 #include "doc/tileset.h"
 
+#include "base/mem_utils.h"
 #include "doc/primitives.h"
 #include "doc/remap.h"
 #include "doc/sprite.h"
@@ -86,6 +87,27 @@ Tileset* Tileset::MakeCopyCopyingImages(const Tileset* tileset)
   return copy.release();
 }
 
+void Tileset::discardCompressedData()
+{
+  if (!m_compressedData.empty()) {
+    TS_TRACE("TS: [%d] discardCompressedData\n", id());
+
+    m_compressedData.clear();
+    m_compressedDataVersion = 0;
+  }
+}
+
+void Tileset::setCompressedData(const base::buffer& buffer) const
+{
+  if (!buffer.empty()) {
+    TS_TRACE("TS: [%d] setCompressedData (%s)\n", id(),
+             base::get_pretty_memory_size(buffer.size()).c_str());
+
+    m_compressedData = buffer;
+    m_compressedDataVersion = version();
+  }
+}
+
 int Tileset::getMemSize() const
 {
   int size = sizeof(Tileset) + m_name.size();
@@ -112,7 +134,7 @@ void Tileset::remap(const Remap& remap)
   ASSERT(remap[0] == 0);
 
   for (tile_index ti=1; ti<size(); ++ti) {
-    TS_TRACE("m_tiles[%d] = tmp[%d]\n", remap[ti], ti);
+    TS_TRACE("TS: m_tiles[%d] = tmp[%d]\n", remap[ti], ti);
 
     ASSERT(remap[ti] >= 0);
     ASSERT(remap[ti] < m_tiles.size());
@@ -367,6 +389,10 @@ void Tileset::rehash()
   // Clear the hash table, we'll lazy-rehash it when
   // hashTable()/findTileIndex() is used.
   m_hash.clear();
+
+  // Reset the compressed data (just in case we have cached the data
+  // from a loaded .aseprite file or when saving the file).
+  discardCompressedData();
 }
 
 TilesetHashTable& Tileset::hashTable()

src/doc/tileset.h

@@ -8,6 +8,7 @@
 #define DOC_TILESET_H_INCLUDED
 #pragma once
 
+#include "base/buffer.h"
 #include "doc/grid.h"
 #include "doc/image_ref.h"
 #include "doc/object.h"
@@ -57,6 +58,13 @@ namespace doc {
     int baseIndex() const { return m_baseIndex; }
     void setBaseIndex(int index) { m_baseIndex = index; }
 
+    // Cached compressed tileset read/writen directly from .aseprite
+    // files.
+    void discardCompressedData();
+    void setCompressedData(const base::buffer& buffer) const;
+    const base::buffer& compressedData() const { return m_compressedData; }
+    ObjectVersion compressedDataVersion() const { return m_compressedDataVersion; }
+
     int getMemSize() const override;
 
     iterator begin() { return m_tiles.begin(); }
@@ -145,6 +153,18 @@ namespace doc {
       std::string filename;
       tileset_index tileset;
     } m_external;
+
+    // This is a cached version of the compressed tileset data
+    // directly read from an .aseprite file. It's used to save the
+    // tileset as-is (without re-compressing). When we modify the
+    // tileset (at least one tile), the compressed data is discarded,
+    // and the recompressiong must be done.
+    //
+    // This was added to improve the performance of saving a sprite
+    // when tilesets are not modified (generally useful when a sprite
+    // contains several layers with tilesets).
+    mutable base::buffer m_compressedData;
+    mutable doc::ObjectVersion m_compressedDataVersion;
   };
 
 } // namespace doc