reduce allocations while reading chunks data

src/coders/compression.cpp

@@ -85,12 +85,12 @@ std::unique_ptr<ubyte[]> compression::decompress(
             throw std::invalid_argument("compression method is NONE");
         case Method::EXTRLE8: {
             auto decompressed = std::make_unique<ubyte[]>(dstlen);
-            extrle::decode(src, srclen, decompressed.get());
+            extrle::decode(src, srclen, decompressed.get(), dstlen);
             return decompressed;
         }
         case Method::EXTRLE16: {
             auto decompressed = std::make_unique<ubyte[]>(dstlen);
-            size_t decoded = extrle::decode16(src, srclen, decompressed.get());
+            size_t decoded = extrle::decode16(src, srclen, decompressed.get(), dstlen);
             if (decoded != dstlen) {
                 throw std::runtime_error(
                     "expected decompressed size " + std::to_string(dstlen) +
@@ -110,6 +110,39 @@ std::unique_ptr<ubyte[]> compression::decompress(
             return decompressed;
         }
         default:
-            throw std::runtime_error("not implemented");
+            throw std::runtime_error("method not implemented");
+    }
+}
+
+void compression::decompress(const util::span<ubyte> src, ubyte* dst, size_t dstlen, Method method) {
+    switch (method) {
+        case Method::NONE:
+            throw std::invalid_argument("compression method is NONE");
+        case Method::EXTRLE8:
+            extrle::decode(src.data(), src.size(), dst, dstlen);
+            break;
+        case Method::EXTRLE16: {
+            size_t decoded =
+                extrle::decode16(src.data(), src.size(), dst, dstlen);
+            if (decoded != dstlen) {
+                throw std::runtime_error(
+                    "expected decompressed size " + std::to_string(dstlen) +
+                    " got " + std::to_string(decoded)
+                );
+            }
+            break;
+        }
+        case Method::GZIP: {
+            auto buffer = gzip::decompress(src.data(), src.size());
+            if (buffer.size() != dstlen) {
+                throw std::runtime_error(
+                    "expected decompressed size " + std::to_string(dstlen) +
+                    " got " + std::to_string(buffer.size())
+                );
+            }
+            std::memcpy(dst, buffer.data(), buffer.size());
+        }
+        default:
+            throw std::runtime_error("method not implemented");
     }
 }

src/coders/compression.hpp

@@ -3,6 +3,7 @@
 #include <memory>
 
 #include "typedefs.hpp"
+#include "util/span.hpp"
 
 namespace compression {
     enum class Method {
@@ -27,4 +28,8 @@ namespace compression {
     /// @return decompressed bytes array
     std::unique_ptr<ubyte[]> decompress(
         const ubyte* src, size_t srclen, size_t dstlen, Method method);
+
+    void decompress(
+        const util::span<ubyte> src, ubyte* dst, size_t dstlen, Method method
+    );
 }

src/coders/rle.cpp

@@ -1,12 +1,17 @@
 #include "rle.hpp"
 
+#include <stdexcept>
+
 #include "util/data_io.hpp"
 
-size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst) {
+size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) {
     size_t offset = 0;
     for (size_t i = 0; i < srclen;) {
         ubyte len = src[i++];
         ubyte c = src[i++];
+        if (offset + len >= dstLength) {
+            throw std::runtime_error("buffer overflow");
+        }
         for (size_t j = 0; j <= len; j++) {
             dst[offset++] = c;
         }
@@ -37,13 +42,16 @@ size_t rle::encode(const ubyte* src, size_t srclen, ubyte* dst) {
     return offset;
 }
 
-size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst) {
+size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) {
     auto src16 = reinterpret_cast<const uint16_t*>(src);
     auto dst16 = reinterpret_cast<uint16_t*>(dst);
     size_t offset = 0;
     for (size_t i = 0; i < srclen / 2;) {
         uint16_t len = dataio::le2h(src16[i++]);
         uint16_t c = dataio::le2h(src16[i++]);
+        if (offset + len >= dstLength) {
+            throw std::runtime_error("buffer overflow");
+        }
         for (size_t j = 0; j <= len; j++) {
             dst16[offset++] = c;
         }
@@ -76,7 +84,7 @@ size_t rle::encode16(const ubyte* src, size_t srclen, ubyte* dst) {
     return offset * 2;
 }
 
-size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) {
+size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) {
     size_t offset = 0;
     for (size_t i = 0; i < srclen;) {
         uint len = src[i++];
@@ -85,6 +93,9 @@ size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) {
             len |= (static_cast<uint>(src[i++])) << 7;
         }
         ubyte c = src[i++];
+        if (offset + len >= dstLength) {
+            throw std::runtime_error("buffer overflow");
+        }
         for (size_t j = 0; j <= len; j++) {
             dst[offset++] = c;
         }
@@ -125,7 +136,7 @@ size_t extrle::encode(const ubyte* src, size_t srclen, ubyte* dst) {
     return offset;
 }
 
-size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) {
+size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8, size_t dstLength) {
     auto dst = reinterpret_cast<uint16_t*>(dst8);
     size_t offset = 0;
     for (size_t i = 0; i < srclen;) {
@@ -141,6 +152,9 @@ size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) {
         if (widechar) {
             c |= ((static_cast<uint>(src[i++])) << 8);
         }
+        if (offset + len >= dstLength) {
+            throw std::runtime_error("buffer overflow");
+        }
         for (size_t j = 0; j <= len; j++) {
             dst[offset++] = c;
         }

src/coders/rle.hpp

@@ -4,18 +4,18 @@
 
 namespace rle {
     size_t encode(const ubyte* src, size_t length, ubyte* dst);
-    size_t decode(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode(const ubyte* src, size_t length, ubyte* dst, size_t dstLength);
 
     size_t encode16(const ubyte* src, size_t length, ubyte* dst);
-    size_t decode16(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode16(const ubyte* src, size_t length, ubyte* dst, size_t dstLength);
 }
 
 namespace extrle {
     constexpr uint max_sequence = 0x7FFF;
     size_t encode(const ubyte* src, size_t length, ubyte* dst);
-    size_t decode(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode(const ubyte* src, size_t length, ubyte* dst, size_t dstLength);
 
     constexpr uint max_sequence16 = 0x3FFF;
     size_t encode16(const ubyte* src, size_t length, ubyte* dst);
-    size_t decode16(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode16(const ubyte* src, size_t length, ubyte* dst, size_t dstLength);
 }

src/lighting/Lightmap.cpp

@@ -23,12 +23,10 @@ std::unique_ptr<ubyte[]> Lightmap::encode() const {
     return buffer;
 }
 
-std::unique_ptr<light_t[]> Lightmap::decode(const ubyte* buffer) {
-    auto lights = std::make_unique<light_t[]>(CHUNK_VOL);
+void Lightmap::decode(const ubyte* src) {
     for (uint i = 0; i < CHUNK_VOL; i+=2) {
-        ubyte b = buffer[i/2];
-        lights[i] = ((b & 0xF) << 12);
-        lights[i+1] = ((b & 0xF0) << 8);
+        ubyte b = src[i/2];
+        map[i] = ((b & 0xF) << 12);
+        map[i+1] = ((b & 0xF0) << 8);
     } 
-    return lights;
 }

src/lighting/Lightmap.hpp

@@ -88,7 +88,7 @@ public:
     }
 
     std::unique_ptr<ubyte[]> encode() const;
-    static std::unique_ptr<light_t[]> decode(const ubyte* buffer);
+    void decode(const ubyte* src);
 
     static inline light_t SUN_LIGHT_ONLY = combine(0U, 0U, 0U, 15U);
 };

src/logic/scripting/lua/libs/libworld.cpp

@@ -130,11 +130,11 @@ static int l_get_chunk_data(lua::State* L) {
     int z = static_cast<int>(lua::tointeger(L, 2));
     const auto& chunk = level->chunks->getChunk(x, z);
 
+    auto voxelData = std::make_unique<ubyte[]>(CHUNK_DATA_LEN);
     std::vector<ubyte> chunkData;
     if (chunk == nullptr) {
         auto& regions = level->getWorld()->wfile->getRegions();
-        auto voxelData = regions.getVoxels(x, z);
-        if (voxelData == nullptr) {
+        if (!regions.getVoxels(x, z, voxelData.get())) {
             return 0;
         }
         static util::Buffer<ubyte> rleBuffer(CHUNK_DATA_LEN * 2);

src/voxels/GlobalChunks.cpp

@@ -98,6 +98,10 @@ std::shared_ptr<Chunk> GlobalChunks::create(int x, int z, bool lighting) {
     if (found != chunksMap.end()) {
         return found->second;
     }
+    static std::unique_ptr<ubyte[]> voxelDataBuffer = nullptr;
+    if (voxelDataBuffer == nullptr) {
+        voxelDataBuffer = std::make_unique<ubyte[]>(CHUNK_DATA_LEN);
+    }
 
     auto chunk =
         chunks_pool.create(x, z, lighting ? lightmaps_pool.create() : nullptr);
@@ -106,10 +110,10 @@ std::shared_ptr<Chunk> GlobalChunks::create(int x, int z, bool lighting) {
     World& world = *level.getWorld();
     auto& regions = world.wfile.get()->getRegions();
 
-    if (auto data = regions.getVoxels(chunk->x, chunk->z)) {
+    if (regions.getVoxels(chunk->x, chunk->z, voxelDataBuffer.get())) {
         const auto& indices = *level.content.getIndices();
 
-        chunk->decode(data.get());
+        chunk->decode(voxelDataBuffer.get());
         check_voxels(indices, *chunk);
 
         chunk->setBlockInventories(
@@ -128,8 +132,8 @@ std::shared_ptr<Chunk> GlobalChunks::create(int x, int z, bool lighting) {
         }
     }
     if (chunk->lightmap) {
-        if (auto lights = regions.getLights(chunk->x, chunk->z)) {
-            chunk->lightmap->set(lights.get());
+        if (regions.getLights(chunk->x, chunk->z, voxelDataBuffer.get())) {
+            chunk->lightmap->decode(voxelDataBuffer.get());
             chunk->flags.loadedLights = true;
         }
     }

src/voxels/compressed_chunks.cpp

@@ -46,7 +46,7 @@ static void read_voxel_data(ByteReader& reader, util::Buffer<ubyte>& dst) {
     auto rleData = gzip::decompress(reader.pointer(), gzipCompressedSize);
     reader.skip(gzipCompressedSize);
 
-    extrle::decode16(rleData.data(), rleData.size(), dst.data());
+    extrle::decode16(rleData.data(), rleData.size(), dst.data(), dst.size());
 }
 
 void compressed_chunks::decode(

src/world/files/WorldRegions.cpp

@@ -215,33 +215,29 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
     }
 }
 
-std::unique_ptr<ubyte[]> WorldRegions::getVoxels(int x, int z) {
+bool WorldRegions::getVoxels(int x, int z, ubyte* dst) {
     uint32_t size;
     uint32_t srcSize;
     auto& layer = layers[REGION_LAYER_VOXELS];
     auto* data = layer.getData(x, z, size, srcSize);
     if (data == nullptr) {
-        return nullptr;
+        return false;
     }
     assert(srcSize == CHUNK_DATA_LEN);
-    return compression::decompress(data, size, srcSize, layer.compression);
+    compression::decompress({data, size}, dst, CHUNK_DATA_LEN, layer.compression);
+    return true;
 }
 
-std::unique_ptr<light_t[]> WorldRegions::getLights(int x, int z) {
+bool WorldRegions::getLights(int x, int z, ubyte* dst) {
     uint32_t size;
     uint32_t srcSize;
     auto& layer = layers[REGION_LAYER_LIGHTS];
     auto* bytes = layer.getData(x, z, size, srcSize);
     if (bytes == nullptr) {
-        return nullptr;
+        return false;
     }
-    auto data = compression::decompress(
-        bytes, size, srcSize, layer.compression
-    );
-    if (srcSize == LIGHTMAP_DATA_LEN) {
-        return Lightmap::decode(data.get());
-    }
-    return nullptr;
+    compression::decompress({bytes, size}, dst, srcSize, layer.compression);
+    return true;
 }
 
 ChunkInventoriesMap WorldRegions::fetchInventories(int x, int z) {

src/world/files/WorldRegions.hpp

@@ -207,12 +207,12 @@ public:
     /// @brief Get chunk voxels data
     /// @param x chunk.x
     /// @param z chunk.z
-    /// @return voxels data buffer or nullptr
-    std::unique_ptr<ubyte[]> getVoxels(int x, int z);
+    /// @return true if data read
+    bool getVoxels(int x, int z, ubyte* dst);
 
     /// @brief Get cached lights for chunk at x,z
-    /// @return lights data or nullptr
-    std::unique_ptr<light_t[]> getLights(int x, int z);
+    /// @return true if data read
+    bool getLights(int x, int z, ubyte* dst);
     
     ChunkInventoriesMap fetchInventories(int x, int z);
 

test/coders/rle.cpp

@@ -5,7 +5,7 @@
 
 static void test_encode_decode(
     size_t(*encodefunc)(const ubyte*, size_t, ubyte*),
-    size_t(*decodefunc)(const ubyte*, size_t, ubyte*),
+    size_t(*decodefunc)(const ubyte*, size_t, ubyte*, size_t),
     int dencity
 ) {
     const size_t initial_size = 50'000;
@@ -20,7 +20,7 @@ static void test_encode_decode(
     uint8_t encoded[initial_size * 2];
     size_t encoded_size = encodefunc(initial, initial_size, encoded);
     uint8_t decoded[initial_size * 2];
-    size_t decoded_size = decodefunc(encoded, encoded_size, decoded);
+    size_t decoded_size = decodefunc(encoded, encoded_size, decoded, initial_size);
     
     EXPECT_EQ(decoded_size, initial_size);