From 9dc944190276bdc6a63b764b68eccc0b937b6a20 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 6 Dec 2025 00:58:13 +0300 Subject: [PATCH 1/8] reduce allocations while reading chunks data --- src/coders/compression.cpp | 39 +++++++++++++++++++++-- src/coders/compression.hpp | 5 +++ src/coders/rle.cpp | 22 ++++++++++--- src/coders/rle.hpp | 8 ++--- src/lighting/Lightmap.cpp | 10 +++--- src/lighting/Lightmap.hpp | 2 +- src/logic/scripting/lua/libs/libworld.cpp | 4 +-- src/voxels/GlobalChunks.cpp | 12 ++++--- src/voxels/compressed_chunks.cpp | 2 +- src/world/files/WorldRegions.cpp | 20 +++++------- src/world/files/WorldRegions.hpp | 8 ++--- test/coders/rle.cpp | 4 +-- 12 files changed, 93 insertions(+), 43 deletions(-) diff --git a/src/coders/compression.cpp b/src/coders/compression.cpp index a3af09cea..d7359fd27 100644 --- a/src/coders/compression.cpp +++ b/src/coders/compression.cpp @@ -85,12 +85,12 @@ std::unique_ptr compression::decompress( throw std::invalid_argument("compression method is NONE"); case Method::EXTRLE8: { auto decompressed = std::make_unique(dstlen); - extrle::decode(src, srclen, decompressed.get()); + extrle::decode(src, srclen, decompressed.get(), dstlen); return decompressed; } case Method::EXTRLE16: { auto decompressed = std::make_unique(dstlen); - size_t decoded = extrle::decode16(src, srclen, decompressed.get()); + size_t decoded = extrle::decode16(src, srclen, decompressed.get(), dstlen); if (decoded != dstlen) { throw std::runtime_error( "expected decompressed size " + std::to_string(dstlen) + @@ -110,6 +110,39 @@ std::unique_ptr compression::decompress( return decompressed; } default: - throw std::runtime_error("not implemented"); + throw std::runtime_error("method not implemented"); + } +} + +void compression::decompress(const util::span src, ubyte* dst, size_t dstlen, Method method) { + switch (method) { + case Method::NONE: + throw std::invalid_argument("compression method is NONE"); + case Method::EXTRLE8: + extrle::decode(src.data(), src.size(), dst, dstlen); + break; + case Method::EXTRLE16: { + size_t decoded = + extrle::decode16(src.data(), src.size(), dst, dstlen); + if (decoded != dstlen) { + throw std::runtime_error( + "expected decompressed size " + std::to_string(dstlen) + + " got " + std::to_string(decoded) + ); + } + break; + } + case Method::GZIP: { + auto buffer = gzip::decompress(src.data(), src.size()); + if (buffer.size() != dstlen) { + throw std::runtime_error( + "expected decompressed size " + std::to_string(dstlen) + + " got " + std::to_string(buffer.size()) + ); + } + std::memcpy(dst, buffer.data(), buffer.size()); + } + default: + throw std::runtime_error("method not implemented"); } } diff --git a/src/coders/compression.hpp b/src/coders/compression.hpp index 98b7a0e3e..461a42799 100644 --- a/src/coders/compression.hpp +++ b/src/coders/compression.hpp @@ -3,6 +3,7 @@ #include #include "typedefs.hpp" +#include "util/span.hpp" namespace compression { enum class Method { @@ -27,4 +28,8 @@ namespace compression { /// @return decompressed bytes array std::unique_ptr decompress( const ubyte* src, size_t srclen, size_t dstlen, Method method); + + void decompress( + const util::span src, ubyte* dst, size_t dstlen, Method method + ); } diff --git a/src/coders/rle.cpp b/src/coders/rle.cpp index e99006e5c..943219177 100644 --- a/src/coders/rle.cpp +++ b/src/coders/rle.cpp @@ -1,12 +1,17 @@ #include "rle.hpp" +#include + #include "util/data_io.hpp" -size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst) { +size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) { size_t offset = 0; for (size_t i = 0; i < srclen;) { ubyte len = src[i++]; ubyte c = src[i++]; + if (offset + len >= dstLength) { + throw std::runtime_error("buffer overflow"); + } for (size_t j = 0; j <= len; j++) { dst[offset++] = c; } @@ -37,13 +42,16 @@ size_t rle::encode(const ubyte* src, size_t srclen, ubyte* dst) { return offset; } -size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst) { +size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) { auto src16 = reinterpret_cast(src); auto dst16 = reinterpret_cast(dst); size_t offset = 0; for (size_t i = 0; i < srclen / 2;) { uint16_t len = dataio::le2h(src16[i++]); uint16_t c = dataio::le2h(src16[i++]); + if (offset + len >= dstLength) { + throw std::runtime_error("buffer overflow"); + } for (size_t j = 0; j <= len; j++) { dst16[offset++] = c; } @@ -76,7 +84,7 @@ size_t rle::encode16(const ubyte* src, size_t srclen, ubyte* dst) { return offset * 2; } -size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) { +size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) { size_t offset = 0; for (size_t i = 0; i < srclen;) { uint len = src[i++]; @@ -85,6 +93,9 @@ size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) { len |= (static_cast(src[i++])) << 7; } ubyte c = src[i++]; + if (offset + len >= dstLength) { + throw std::runtime_error("buffer overflow"); + } for (size_t j = 0; j <= len; j++) { dst[offset++] = c; } @@ -125,7 +136,7 @@ size_t extrle::encode(const ubyte* src, size_t srclen, ubyte* dst) { return offset; } -size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) { +size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8, size_t dstLength) { auto dst = reinterpret_cast(dst8); size_t offset = 0; for (size_t i = 0; i < srclen;) { @@ -141,6 +152,9 @@ size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) { if (widechar) { c |= ((static_cast(src[i++])) << 8); } + if (offset + len >= dstLength) { + throw std::runtime_error("buffer overflow"); + } for (size_t j = 0; j <= len; j++) { dst[offset++] = c; } diff --git a/src/coders/rle.hpp b/src/coders/rle.hpp index c087ac696..42fa42c20 100644 --- a/src/coders/rle.hpp +++ b/src/coders/rle.hpp @@ -4,18 +4,18 @@ namespace rle { size_t encode(const ubyte* src, size_t length, ubyte* dst); - size_t decode(const ubyte* src, size_t length, ubyte* dst); + size_t decode(const ubyte* src, size_t length, ubyte* dst, size_t dstLength); size_t encode16(const ubyte* src, size_t length, ubyte* dst); - size_t decode16(const ubyte* src, size_t length, ubyte* dst); + size_t decode16(const ubyte* src, size_t length, ubyte* dst, size_t dstLength); } namespace extrle { constexpr uint max_sequence = 0x7FFF; size_t encode(const ubyte* src, size_t length, ubyte* dst); - size_t decode(const ubyte* src, size_t length, ubyte* dst); + size_t decode(const ubyte* src, size_t length, ubyte* dst, size_t dstLength); constexpr uint max_sequence16 = 0x3FFF; size_t encode16(const ubyte* src, size_t length, ubyte* dst); - size_t decode16(const ubyte* src, size_t length, ubyte* dst); + size_t decode16(const ubyte* src, size_t length, ubyte* dst, size_t dstLength); } diff --git a/src/lighting/Lightmap.cpp b/src/lighting/Lightmap.cpp index 24581e246..a86d5b29d 100644 --- a/src/lighting/Lightmap.cpp +++ b/src/lighting/Lightmap.cpp @@ -23,12 +23,10 @@ std::unique_ptr Lightmap::encode() const { return buffer; } -std::unique_ptr Lightmap::decode(const ubyte* buffer) { - auto lights = std::make_unique(CHUNK_VOL); +void Lightmap::decode(const ubyte* src) { for (uint i = 0; i < CHUNK_VOL; i+=2) { - ubyte b = buffer[i/2]; - lights[i] = ((b & 0xF) << 12); - lights[i+1] = ((b & 0xF0) << 8); + ubyte b = src[i/2]; + map[i] = ((b & 0xF) << 12); + map[i+1] = ((b & 0xF0) << 8); } - return lights; } diff --git a/src/lighting/Lightmap.hpp b/src/lighting/Lightmap.hpp index b376c0db3..858e37388 100644 --- a/src/lighting/Lightmap.hpp +++ b/src/lighting/Lightmap.hpp @@ -88,7 +88,7 @@ public: } std::unique_ptr encode() const; - static std::unique_ptr decode(const ubyte* buffer); + void decode(const ubyte* src); static inline light_t SUN_LIGHT_ONLY = combine(0U, 0U, 0U, 15U); }; diff --git a/src/logic/scripting/lua/libs/libworld.cpp b/src/logic/scripting/lua/libs/libworld.cpp index 7a53c4cb5..9b28c719f 100644 --- a/src/logic/scripting/lua/libs/libworld.cpp +++ b/src/logic/scripting/lua/libs/libworld.cpp @@ -130,11 +130,11 @@ static int l_get_chunk_data(lua::State* L) { int z = static_cast(lua::tointeger(L, 2)); const auto& chunk = level->chunks->getChunk(x, z); + auto voxelData = std::make_unique(CHUNK_DATA_LEN); std::vector chunkData; if (chunk == nullptr) { auto& regions = level->getWorld()->wfile->getRegions(); - auto voxelData = regions.getVoxels(x, z); - if (voxelData == nullptr) { + if (!regions.getVoxels(x, z, voxelData.get())) { return 0; } static util::Buffer rleBuffer(CHUNK_DATA_LEN * 2); diff --git a/src/voxels/GlobalChunks.cpp b/src/voxels/GlobalChunks.cpp index cbde6f9c6..5e692ad69 100644 --- a/src/voxels/GlobalChunks.cpp +++ b/src/voxels/GlobalChunks.cpp @@ -98,6 +98,10 @@ std::shared_ptr GlobalChunks::create(int x, int z, bool lighting) { if (found != chunksMap.end()) { return found->second; } + static std::unique_ptr voxelDataBuffer = nullptr; + if (voxelDataBuffer == nullptr) { + voxelDataBuffer = std::make_unique(CHUNK_DATA_LEN); + } auto chunk = chunks_pool.create(x, z, lighting ? lightmaps_pool.create() : nullptr); @@ -106,10 +110,10 @@ std::shared_ptr GlobalChunks::create(int x, int z, bool lighting) { World& world = *level.getWorld(); auto& regions = world.wfile.get()->getRegions(); - if (auto data = regions.getVoxels(chunk->x, chunk->z)) { + if (regions.getVoxels(chunk->x, chunk->z, voxelDataBuffer.get())) { const auto& indices = *level.content.getIndices(); - chunk->decode(data.get()); + chunk->decode(voxelDataBuffer.get()); check_voxels(indices, *chunk); chunk->setBlockInventories( @@ -128,8 +132,8 @@ std::shared_ptr GlobalChunks::create(int x, int z, bool lighting) { } } if (chunk->lightmap) { - if (auto lights = regions.getLights(chunk->x, chunk->z)) { - chunk->lightmap->set(lights.get()); + if (regions.getLights(chunk->x, chunk->z, voxelDataBuffer.get())) { + chunk->lightmap->decode(voxelDataBuffer.get()); chunk->flags.loadedLights = true; } } diff --git a/src/voxels/compressed_chunks.cpp b/src/voxels/compressed_chunks.cpp index 46ba800dd..52c5616fe 100644 --- a/src/voxels/compressed_chunks.cpp +++ b/src/voxels/compressed_chunks.cpp @@ -46,7 +46,7 @@ static void read_voxel_data(ByteReader& reader, util::Buffer& dst) { auto rleData = gzip::decompress(reader.pointer(), gzipCompressedSize); reader.skip(gzipCompressedSize); - extrle::decode16(rleData.data(), rleData.size(), dst.data()); + extrle::decode16(rleData.data(), rleData.size(), dst.data(), dst.size()); } void compressed_chunks::decode( diff --git a/src/world/files/WorldRegions.cpp b/src/world/files/WorldRegions.cpp index 6cfe55d68..0f8535e12 100644 --- a/src/world/files/WorldRegions.cpp +++ b/src/world/files/WorldRegions.cpp @@ -215,33 +215,29 @@ void WorldRegions::put(Chunk* chunk, std::vector entitiesData) { } } -std::unique_ptr WorldRegions::getVoxels(int x, int z) { +bool WorldRegions::getVoxels(int x, int z, ubyte* dst) { uint32_t size; uint32_t srcSize; auto& layer = layers[REGION_LAYER_VOXELS]; auto* data = layer.getData(x, z, size, srcSize); if (data == nullptr) { - return nullptr; + return false; } assert(srcSize == CHUNK_DATA_LEN); - return compression::decompress(data, size, srcSize, layer.compression); + compression::decompress({data, size}, dst, CHUNK_DATA_LEN, layer.compression); + return true; } -std::unique_ptr WorldRegions::getLights(int x, int z) { +bool WorldRegions::getLights(int x, int z, ubyte* dst) { uint32_t size; uint32_t srcSize; auto& layer = layers[REGION_LAYER_LIGHTS]; auto* bytes = layer.getData(x, z, size, srcSize); if (bytes == nullptr) { - return nullptr; + return false; } - auto data = compression::decompress( - bytes, size, srcSize, layer.compression - ); - if (srcSize == LIGHTMAP_DATA_LEN) { - return Lightmap::decode(data.get()); - } - return nullptr; + compression::decompress({bytes, size}, dst, srcSize, layer.compression); + return true; } ChunkInventoriesMap WorldRegions::fetchInventories(int x, int z) { diff --git a/src/world/files/WorldRegions.hpp b/src/world/files/WorldRegions.hpp index 8bc10ba23..3b4a63fa6 100644 --- a/src/world/files/WorldRegions.hpp +++ b/src/world/files/WorldRegions.hpp @@ -207,12 +207,12 @@ public: /// @brief Get chunk voxels data /// @param x chunk.x /// @param z chunk.z - /// @return voxels data buffer or nullptr - std::unique_ptr getVoxels(int x, int z); + /// @return true if data read + bool getVoxels(int x, int z, ubyte* dst); /// @brief Get cached lights for chunk at x,z - /// @return lights data or nullptr - std::unique_ptr getLights(int x, int z); + /// @return true if data read + bool getLights(int x, int z, ubyte* dst); ChunkInventoriesMap fetchInventories(int x, int z); diff --git a/test/coders/rle.cpp b/test/coders/rle.cpp index e393f8dd9..919dd532a 100644 --- a/test/coders/rle.cpp +++ b/test/coders/rle.cpp @@ -5,7 +5,7 @@ static void test_encode_decode( size_t(*encodefunc)(const ubyte*, size_t, ubyte*), - size_t(*decodefunc)(const ubyte*, size_t, ubyte*), + size_t(*decodefunc)(const ubyte*, size_t, ubyte*, size_t), int dencity ) { const size_t initial_size = 50'000; @@ -20,7 +20,7 @@ static void test_encode_decode( uint8_t encoded[initial_size * 2]; size_t encoded_size = encodefunc(initial, initial_size, encoded); uint8_t decoded[initial_size * 2]; - size_t decoded_size = decodefunc(encoded, encoded_size, decoded); + size_t decoded_size = decodefunc(encoded, encoded_size, decoded, initial_size); EXPECT_EQ(decoded_size, initial_size); From 43ec32662cdce36546493cadf975dd5759445e26 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sun, 7 Dec 2025 15:35:39 +0300 Subject: [PATCH 2/8] fix --- src/coders/compression.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coders/compression.cpp b/src/coders/compression.cpp index d7359fd27..569834301 100644 --- a/src/coders/compression.cpp +++ b/src/coders/compression.cpp @@ -141,6 +141,7 @@ void compression::decompress(const util::span src, ubyte* dst, size_t dst ); } std::memcpy(dst, buffer.data(), buffer.size()); + break; } default: throw std::runtime_error("method not implemented"); From 677b7c67dedf5d8d17a57fa8591d3132355d1bbc Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sun, 7 Dec 2025 18:27:12 +0300 Subject: [PATCH 3/8] minor optimization --- src/logic/BlocksController.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/logic/BlocksController.cpp b/src/logic/BlocksController.cpp index ee74bb955..616060951 100644 --- a/src/logic/BlocksController.cpp +++ b/src/logic/BlocksController.cpp @@ -151,8 +151,12 @@ void BlocksController::randomTick( for (int s = 0; s < segments; s++) { for (int i = 0; i < 4; i++) { + int segmentY = s * segheight; + if (segmentY > chunk.top) { + break; + } int bx = random.rand() % CHUNK_W; - int by = random.rand() % segheight + s * segheight; + int by = random.rand() % segheight + segmentY; int bz = random.rand() % CHUNK_D; const voxel& vox = chunk.voxels[vox_index(bx, by, bz)]; auto& block = indices->blocks.require(vox.id); From b4bd2e4492a65ab5063575f58bbcd35c117e475a Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sun, 7 Dec 2025 20:12:52 +0300 Subject: [PATCH 4/8] fix --- src/util/platform.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/util/platform.cpp b/src/util/platform.cpp index 16adbf89f..dce6f89b7 100644 --- a/src/util/platform.cpp +++ b/src/util/platform.cpp @@ -102,7 +102,11 @@ std::string platform::detect_locale() { if (programLocaleName && preferredLocaleName) { setlocale(LC_ALL, programLocaleName); - return std::string(preferredLocaleName); + if (std::strlen(preferredLocaleName) >= 5) { + return std::string(preferredLocaleName, 5); + } else { + return std::string(preferredLocaleName); + } } return langs::FALLBACK_DEFAULT; } From 4785d757faa54c68f017fdeaa1e50a9098b29b9b Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 8 Dec 2025 22:14:35 +0300 Subject: [PATCH 5/8] add gui.show_input_dialog --- res/modules/internal/gui_util.lua | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/res/modules/internal/gui_util.lua b/res/modules/internal/gui_util.lua index 4766cbfd9..81e50b21e 100644 --- a/res/modules/internal/gui_util.lua +++ b/res/modules/internal/gui_util.lua @@ -136,6 +136,36 @@ function gui.show_message(text, actual_callback) input.add_callback("key:escape", callback, gui.root[id]) end +function gui.show_input_dialog(text, actual_callback, validator, confirm_text) + local id = "dialog_"..random.uuid() + + local callback = function() + if not gui.root[id.."_input"].valid then + return + end + gui.root[id]:destruct() + if actual_callback then + actual_callback(gui.root[id.."_input"].text) + end + end + gui.root.root:add(string.format([[ + + + + + + + + ]], id, string.escape_xml(text), id, string.escape_xml(confirm_text or gui.str("OK"))), { + callback=callback, + validator=validator or function() return #text > 0 end + }) + input.add_callback("key:escape", callback, gui.root[id]) + gui.root[id.."_input"].focused = true +end + + function gui.ask(text, on_yes, on_no) on_yes = on_yes or function() end on_no = on_no or function() end From 2e44aa7d19a2ffd0456c9b342632b376bf94f479 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 8 Dec 2025 22:24:57 +0300 Subject: [PATCH 6/8] add 'enter' key support to gui.show_input_dialog --- res/modules/internal/gui_util.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/res/modules/internal/gui_util.lua b/res/modules/internal/gui_util.lua index 81e50b21e..793dc8bf1 100644 --- a/res/modules/internal/gui_util.lua +++ b/res/modules/internal/gui_util.lua @@ -162,6 +162,7 @@ function gui.show_input_dialog(text, actual_callback, validator, confirm_text) validator=validator or function() return #text > 0 end }) input.add_callback("key:escape", callback, gui.root[id]) + input.add_callback("key:enter", callback, gui.root[id]) gui.root[id.."_input"].focused = true end From 1f73dc5d5e5bb49ff9a4648b424efe50b69e239c Mon Sep 17 00:00:00 2001 From: MihailRis Date: Tue, 9 Dec 2025 01:20:41 +0300 Subject: [PATCH 7/8] feat: pre-read region chunks offsets table --- src/world/files/RegionsLayer.cpp | 20 ++++++++++++++++---- src/world/files/WorldRegions.hpp | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/world/files/RegionsLayer.cpp b/src/world/files/RegionsLayer.cpp index 2da2bd011..04821d463 100644 --- a/src/world/files/RegionsLayer.cpp +++ b/src/world/files/RegionsLayer.cpp @@ -50,6 +50,20 @@ regfile::regfile(io::path filename) : file(filename), filename(filename) { " is not supported in " + filename.string() ); } + + size_t file_size = file.length(); + size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; + + file.seekg(table_offset); + file.read( + reinterpret_cast(offsets.data()), + sizeof(uint32_t) * REGION_CHUNKS_COUNT + ); + if (dataio::is_big_endian()) { + for (size_t i = 0; i < offsets.size(); i++) { + offsets[i] = dataio::le2h(i); + } + } } std::unique_ptr regfile::read( @@ -58,14 +72,12 @@ std::unique_ptr regfile::read( size_t file_size = file.length(); size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4; - uint32_t buff32; - file.seekg(table_offset + index * 4); - file.read(reinterpret_cast(&buff32), 4); - uint32_t offset = dataio::le2h(buff32); + uint32_t offset = offsets.at(index); if (offset == 0) { return nullptr; } + uint32_t buff32; file.seekg(offset); file.read(reinterpret_cast(&buff32), 4); size = dataio::le2h(buff32); diff --git a/src/world/files/WorldRegions.hpp b/src/world/files/WorldRegions.hpp index 3b4a63fa6..e3bf2774a 100644 --- a/src/world/files/WorldRegions.hpp +++ b/src/world/files/WorldRegions.hpp @@ -55,6 +55,7 @@ struct regfile { io::path filename; int version; bool inUse = false; + std::array offsets; regfile(io::path filename); regfile(const regfile&) = delete; From 5e2774736caee65a62e07b88b8a7d3b4f5b94e9e Mon Sep 17 00:00:00 2001 From: MihailRis Date: Tue, 9 Dec 2025 07:44:53 +0300 Subject: [PATCH 8/8] fix missing include --- src/world/files/WorldRegions.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/world/files/WorldRegions.hpp b/src/world/files/WorldRegions.hpp index e3bf2774a..360c54978 100644 --- a/src/world/files/WorldRegions.hpp +++ b/src/world/files/WorldRegions.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -7,12 +8,12 @@ #include #include +#include "coders/compression.hpp" +#include "io/io.hpp" +#include "maths/voxmaths.hpp" #include "typedefs.hpp" #include "util/BufferPool.hpp" #include "voxels/Chunk.hpp" -#include "maths/voxmaths.hpp" -#include "coders/compression.hpp" -#include "io/io.hpp" #include "world_regions_fwd.hpp" #define GLM_ENABLE_EXPERIMENTAL