Merge pull request #728 from MihailRis/world-regions-works

chunks loading pipeline works

res/modules/internal/gui_util.lua

@@ -136,6 +136,37 @@ function gui.show_message(text, actual_callback)
     input.add_callback("key:escape", callback, gui.root[id])
 end
 
+function gui.show_input_dialog(text, actual_callback, validator, confirm_text)
+    local id = "dialog_"..random.uuid()
+
+    local callback = function()
+        if not gui.root[id.."_input"].valid then
+            return
+        end
+        gui.root[id]:destruct()
+        if actual_callback then
+            actual_callback(gui.root[id.."_input"].text)
+        end
+    end
+    gui.root.root:add(string.format([[
+        <container id='%s' color='#00000080' size-func='-1,-1' z-index='10'>
+            <panel color='#507090E0' size='600' padding='16'
+                   gravity='center-center' interval='4'>
+                <label>%s</label>
+                <textbox id='%s_input' validator='DATA.validator'/>
+                <button onclick='DATA.callback()'>%s</button>
+            </panel>
+        </container>
+    ]], id, string.escape_xml(text), id, string.escape_xml(confirm_text or gui.str("OK"))), {
+        callback=callback,
+        validator=validator or function() return #text > 0 end
+    })
+    input.add_callback("key:escape", callback, gui.root[id])
+    input.add_callback("key:enter", callback, gui.root[id])
+    gui.root[id.."_input"].focused = true
+end
+
+
 function gui.ask(text, on_yes, on_no)
     on_yes = on_yes or function() end
     on_no = on_no or function() end

src/coders/compression.cpp

@@ -85,12 +85,12 @@ std::unique_ptr<ubyte[]> compression::decompress(
             throw std::invalid_argument("compression method is NONE");
         case Method::EXTRLE8: {
             auto decompressed = std::make_unique<ubyte[]>(dstlen);
-            extrle::decode(src, srclen, decompressed.get());
+            extrle::decode(src, srclen, decompressed.get(), dstlen);
             return decompressed;
         }
         case Method::EXTRLE16: {
             auto decompressed = std::make_unique<ubyte[]>(dstlen);
-            size_t decoded = extrle::decode16(src, srclen, decompressed.get());
+            size_t decoded = extrle::decode16(src, srclen, decompressed.get(), dstlen);
             if (decoded != dstlen) {
                 throw std::runtime_error(
                     "expected decompressed size " + std::to_string(dstlen) +
@@ -110,6 +110,40 @@ std::unique_ptr<ubyte[]> compression::decompress(
             return decompressed;
         }
         default:
-            throw std::runtime_error("not implemented");
+            throw std::runtime_error("method not implemented");
+    }
+}
+
+void compression::decompress(const util::span<ubyte> src, ubyte* dst, size_t dstlen, Method method) {
+    switch (method) {
+        case Method::NONE:
+            throw std::invalid_argument("compression method is NONE");
+        case Method::EXTRLE8:
+            extrle::decode(src.data(), src.size(), dst, dstlen);
+            break;
+        case Method::EXTRLE16: {
+            size_t decoded =
+                extrle::decode16(src.data(), src.size(), dst, dstlen);
+            if (decoded != dstlen) {
+                throw std::runtime_error(
+                    "expected decompressed size " + std::to_string(dstlen) +
+                    " got " + std::to_string(decoded)
+                );
+            }
+            break;
+        }
+        case Method::GZIP: {
+            auto buffer = gzip::decompress(src.data(), src.size());
+            if (buffer.size() != dstlen) {
+                throw std::runtime_error(
+                    "expected decompressed size " + std::to_string(dstlen) +
+                    " got " + std::to_string(buffer.size())
+                );
+            }
+            std::memcpy(dst, buffer.data(), buffer.size());
+            break;
+        }
+        default:
+            throw std::runtime_error("method not implemented");
     }
 }

src/coders/compression.hpp

@@ -3,6 +3,7 @@
 #include <memory>
 
 #include "typedefs.hpp"
+#include "util/span.hpp"
 
 namespace compression {
     enum class Method {
@@ -27,4 +28,8 @@ namespace compression {
     /// @return decompressed bytes array
     std::unique_ptr<ubyte[]> decompress(
         const ubyte* src, size_t srclen, size_t dstlen, Method method);
+
+    void decompress(
+        const util::span<ubyte> src, ubyte* dst, size_t dstlen, Method method
+    );
 }

src/coders/rle.cpp

@@ -1,12 +1,17 @@
 #include "rle.hpp"
 
+#include <stdexcept>
+
 #include "util/data_io.hpp"
 
-size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst) {
+size_t rle::decode(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) {
     size_t offset = 0;
     for (size_t i = 0; i < srclen;) {
         ubyte len = src[i++];
         ubyte c = src[i++];
+        if (offset + len >= dstLength) {
+            throw std::runtime_error("buffer overflow");
+        }
         for (size_t j = 0; j <= len; j++) {
             dst[offset++] = c;
         }
@@ -37,13 +42,16 @@ size_t rle::encode(const ubyte* src, size_t srclen, ubyte* dst) {
     return offset;
 }
 
-size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst) {
+size_t rle::decode16(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) {
     auto src16 = reinterpret_cast<const uint16_t*>(src);
     auto dst16 = reinterpret_cast<uint16_t*>(dst);
     size_t offset = 0;
     for (size_t i = 0; i < srclen / 2;) {
         uint16_t len = dataio::le2h(src16[i++]);
         uint16_t c = dataio::le2h(src16[i++]);
+        if (offset + len >= dstLength) {
+            throw std::runtime_error("buffer overflow");
+        }
         for (size_t j = 0; j <= len; j++) {
             dst16[offset++] = c;
         }
@@ -76,7 +84,7 @@ size_t rle::encode16(const ubyte* src, size_t srclen, ubyte* dst) {
     return offset * 2;
 }
 
-size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) {
+size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst, size_t dstLength) {
     size_t offset = 0;
     for (size_t i = 0; i < srclen;) {
         uint len = src[i++];
@@ -85,6 +93,9 @@ size_t extrle::decode(const ubyte* src, size_t srclen, ubyte* dst) {
             len |= (static_cast<uint>(src[i++])) << 7;
         }
         ubyte c = src[i++];
+        if (offset + len >= dstLength) {
+            throw std::runtime_error("buffer overflow");
+        }
         for (size_t j = 0; j <= len; j++) {
             dst[offset++] = c;
         }
@@ -125,7 +136,7 @@ size_t extrle::encode(const ubyte* src, size_t srclen, ubyte* dst) {
     return offset;
 }
 
-size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) {
+size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8, size_t dstLength) {
     auto dst = reinterpret_cast<uint16_t*>(dst8);
     size_t offset = 0;
     for (size_t i = 0; i < srclen;) {
@@ -141,6 +152,9 @@ size_t extrle::decode16(const ubyte* src, size_t srclen, ubyte* dst8) {
         if (widechar) {
             c |= ((static_cast<uint>(src[i++])) << 8);
         }
+        if (offset + len >= dstLength) {
+            throw std::runtime_error("buffer overflow");
+        }
         for (size_t j = 0; j <= len; j++) {
             dst[offset++] = c;
         }

src/coders/rle.hpp

@@ -4,18 +4,18 @@
 
 namespace rle {
     size_t encode(const ubyte* src, size_t length, ubyte* dst);
-    size_t decode(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode(const ubyte* src, size_t length, ubyte* dst, size_t dstLength);
 
     size_t encode16(const ubyte* src, size_t length, ubyte* dst);
-    size_t decode16(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode16(const ubyte* src, size_t length, ubyte* dst, size_t dstLength);
 }
 
 namespace extrle {
     constexpr uint max_sequence = 0x7FFF;
     size_t encode(const ubyte* src, size_t length, ubyte* dst);
-    size_t decode(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode(const ubyte* src, size_t length, ubyte* dst, size_t dstLength);
 
     constexpr uint max_sequence16 = 0x3FFF;
     size_t encode16(const ubyte* src, size_t length, ubyte* dst);
-    size_t decode16(const ubyte* src, size_t length, ubyte* dst);
+    size_t decode16(const ubyte* src, size_t length, ubyte* dst, size_t dstLength);
 }

src/lighting/Lightmap.cpp

@@ -23,12 +23,10 @@ std::unique_ptr<ubyte[]> Lightmap::encode() const {
     return buffer;
 }
 
-std::unique_ptr<light_t[]> Lightmap::decode(const ubyte* buffer) {
-    auto lights = std::make_unique<light_t[]>(CHUNK_VOL);
+void Lightmap::decode(const ubyte* src) {
     for (uint i = 0; i < CHUNK_VOL; i+=2) {
-        ubyte b = buffer[i/2];
-        lights[i] = ((b & 0xF) << 12);
-        lights[i+1] = ((b & 0xF0) << 8);
+        ubyte b = src[i/2];
+        map[i] = ((b & 0xF) << 12);
+        map[i+1] = ((b & 0xF0) << 8);
     } 
-    return lights;
 }

src/lighting/Lightmap.hpp

@@ -88,7 +88,7 @@ public:
     }
 
     std::unique_ptr<ubyte[]> encode() const;
-    static std::unique_ptr<light_t[]> decode(const ubyte* buffer);
+    void decode(const ubyte* src);
 
     static inline light_t SUN_LIGHT_ONLY = combine(0U, 0U, 0U, 15U);
 };

src/logic/BlocksController.cpp

@@ -151,8 +151,12 @@ void BlocksController::randomTick(
 
     for (int s = 0; s < segments; s++) {
         for (int i = 0; i < 4; i++) {
+            int segmentY = s * segheight;
+            if (segmentY  > chunk.top) {
+                break;
+            }
             int bx = random.rand() % CHUNK_W;
-            int by = random.rand() % segheight + s * segheight;
+            int by = random.rand() % segheight + segmentY;
             int bz = random.rand() % CHUNK_D;
             const voxel& vox = chunk.voxels[vox_index(bx, by, bz)];
             auto& block = indices->blocks.require(vox.id);

src/logic/scripting/lua/libs/libworld.cpp

@@ -130,11 +130,11 @@ static int l_get_chunk_data(lua::State* L) {
     int z = static_cast<int>(lua::tointeger(L, 2));
     const auto& chunk = level->chunks->getChunk(x, z);
 
+    auto voxelData = std::make_unique<ubyte[]>(CHUNK_DATA_LEN);
     std::vector<ubyte> chunkData;
     if (chunk == nullptr) {
         auto& regions = level->getWorld()->wfile->getRegions();
-        auto voxelData = regions.getVoxels(x, z);
-        if (voxelData == nullptr) {
+        if (!regions.getVoxels(x, z, voxelData.get())) {
             return 0;
         }
         static util::Buffer<ubyte> rleBuffer(CHUNK_DATA_LEN * 2);

src/util/platform.cpp

@@ -102,7 +102,11 @@ std::string platform::detect_locale() {
     if (programLocaleName && preferredLocaleName) {
         setlocale(LC_ALL, programLocaleName);
 
-        return std::string(preferredLocaleName);
+        if (std::strlen(preferredLocaleName) >= 5) {
+            return std::string(preferredLocaleName, 5);
+        } else {
+            return std::string(preferredLocaleName);
+        }
     }
     return langs::FALLBACK_DEFAULT;
 }

src/voxels/GlobalChunks.cpp

@@ -98,6 +98,10 @@ std::shared_ptr<Chunk> GlobalChunks::create(int x, int z, bool lighting) {
     if (found != chunksMap.end()) {
         return found->second;
     }
+    static std::unique_ptr<ubyte[]> voxelDataBuffer = nullptr;
+    if (voxelDataBuffer == nullptr) {
+        voxelDataBuffer = std::make_unique<ubyte[]>(CHUNK_DATA_LEN);
+    }
 
     auto chunk =
         chunks_pool.create(x, z, lighting ? lightmaps_pool.create() : nullptr);
@@ -106,10 +110,10 @@ std::shared_ptr<Chunk> GlobalChunks::create(int x, int z, bool lighting) {
     World& world = *level.getWorld();
     auto& regions = world.wfile.get()->getRegions();
 
-    if (auto data = regions.getVoxels(chunk->x, chunk->z)) {
+    if (regions.getVoxels(chunk->x, chunk->z, voxelDataBuffer.get())) {
         const auto& indices = *level.content.getIndices();
 
-        chunk->decode(data.get());
+        chunk->decode(voxelDataBuffer.get());
         check_voxels(indices, *chunk);
 
         chunk->setBlockInventories(
@@ -128,8 +132,8 @@ std::shared_ptr<Chunk> GlobalChunks::create(int x, int z, bool lighting) {
         }
     }
     if (chunk->lightmap) {
-        if (auto lights = regions.getLights(chunk->x, chunk->z)) {
-            chunk->lightmap->set(lights.get());
+        if (regions.getLights(chunk->x, chunk->z, voxelDataBuffer.get())) {
+            chunk->lightmap->decode(voxelDataBuffer.get());
             chunk->flags.loadedLights = true;
         }
     }

src/voxels/compressed_chunks.cpp

@@ -46,7 +46,7 @@ static void read_voxel_data(ByteReader& reader, util::Buffer<ubyte>& dst) {
     auto rleData = gzip::decompress(reader.pointer(), gzipCompressedSize);
     reader.skip(gzipCompressedSize);
 
-    extrle::decode16(rleData.data(), rleData.size(), dst.data());
+    extrle::decode16(rleData.data(), rleData.size(), dst.data(), dst.size());
 }
 
 void compressed_chunks::decode(

src/world/files/RegionsLayer.cpp

@@ -50,6 +50,20 @@ regfile::regfile(io::path filename) : file(filename), filename(filename) {
             " is not supported in " + filename.string()
         );
     }
+
+    size_t file_size = file.length();
+    size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
+
+    file.seekg(table_offset);
+    file.read(
+        reinterpret_cast<char*>(offsets.data()),
+        sizeof(uint32_t) * REGION_CHUNKS_COUNT
+    );
+    if (dataio::is_big_endian()) {
+        for (size_t i = 0; i < offsets.size(); i++) {
+            offsets[i] = dataio::le2h(i);
+        }
+    }
 }
 
 std::unique_ptr<ubyte[]> regfile::read(
@@ -58,14 +72,12 @@ std::unique_ptr<ubyte[]> regfile::read(
     size_t file_size = file.length();
     size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
 
-    uint32_t buff32;
-    file.seekg(table_offset + index * 4);
-    file.read(reinterpret_cast<char*>(&buff32), 4);
-    uint32_t offset = dataio::le2h(buff32);
+    uint32_t offset = offsets.at(index);
     if (offset == 0) {
         return nullptr;
     }
 
+    uint32_t buff32;
     file.seekg(offset);
     file.read(reinterpret_cast<char*>(&buff32), 4);
     size = dataio::le2h(buff32);

src/world/files/WorldRegions.cpp

@@ -215,33 +215,29 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
     }
 }
 
-std::unique_ptr<ubyte[]> WorldRegions::getVoxels(int x, int z) {
+bool WorldRegions::getVoxels(int x, int z, ubyte* dst) {
     uint32_t size;
     uint32_t srcSize;
     auto& layer = layers[REGION_LAYER_VOXELS];
     auto* data = layer.getData(x, z, size, srcSize);
     if (data == nullptr) {
-        return nullptr;
+        return false;
     }
     assert(srcSize == CHUNK_DATA_LEN);
-    return compression::decompress(data, size, srcSize, layer.compression);
+    compression::decompress({data, size}, dst, CHUNK_DATA_LEN, layer.compression);
+    return true;
 }
 
-std::unique_ptr<light_t[]> WorldRegions::getLights(int x, int z) {
+bool WorldRegions::getLights(int x, int z, ubyte* dst) {
     uint32_t size;
     uint32_t srcSize;
     auto& layer = layers[REGION_LAYER_LIGHTS];
     auto* bytes = layer.getData(x, z, size, srcSize);
     if (bytes == nullptr) {
-        return nullptr;
+        return false;
     }
-    auto data = compression::decompress(
-        bytes, size, srcSize, layer.compression
-    );
-    if (srcSize == LIGHTMAP_DATA_LEN) {
-        return Lightmap::decode(data.get());
-    }
-    return nullptr;
+    compression::decompress({bytes, size}, dst, srcSize, layer.compression);
+    return true;
 }
 
 ChunkInventoriesMap WorldRegions::fetchInventories(int x, int z) {

src/world/files/WorldRegions.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <array>
 #include <condition_variable>
 #include <functional>
 #include <glm/glm.hpp>
@@ -7,12 +8,12 @@
 #include <mutex>
 #include <unordered_map>
 
+#include "coders/compression.hpp"
+#include "io/io.hpp"
+#include "maths/voxmaths.hpp"
 #include "typedefs.hpp"
 #include "util/BufferPool.hpp"
 #include "voxels/Chunk.hpp"
-#include "maths/voxmaths.hpp"
-#include "coders/compression.hpp"
-#include "io/io.hpp"
 #include "world_regions_fwd.hpp"
 
 #define GLM_ENABLE_EXPERIMENTAL
@@ -55,6 +56,7 @@ struct regfile {
     io::path filename;
     int version;
     bool inUse = false;
+    std::array<uint32_t, REGION_CHUNKS_COUNT> offsets;
 
     regfile(io::path filename);
     regfile(const regfile&) = delete;
@@ -207,12 +209,12 @@ public:
     /// @brief Get chunk voxels data
     /// @param x chunk.x
     /// @param z chunk.z
-    /// @return voxels data buffer or nullptr
-    std::unique_ptr<ubyte[]> getVoxels(int x, int z);
+    /// @return true if data read
+    bool getVoxels(int x, int z, ubyte* dst);
 
     /// @brief Get cached lights for chunk at x,z
-    /// @return lights data or nullptr
-    std::unique_ptr<light_t[]> getLights(int x, int z);
+    /// @return true if data read
+    bool getLights(int x, int z, ubyte* dst);
     
     ChunkInventoriesMap fetchInventories(int x, int z);
 

test/coders/rle.cpp

@@ -5,7 +5,7 @@
 
 static void test_encode_decode(
     size_t(*encodefunc)(const ubyte*, size_t, ubyte*),
-    size_t(*decodefunc)(const ubyte*, size_t, ubyte*),
+    size_t(*decodefunc)(const ubyte*, size_t, ubyte*, size_t),
     int dencity
 ) {
     const size_t initial_size = 50'000;
@@ -20,7 +20,7 @@ static void test_encode_decode(
     uint8_t encoded[initial_size * 2];
     size_t encoded_size = encodefunc(initial, initial_size, encoded);
     uint8_t decoded[initial_size * 2];
-    size_t decoded_size = decodefunc(encoded, encoded_size, decoded);
+    size_t decoded_size = decodefunc(encoded, encoded_size, decoded, initial_size);
     
     EXPECT_EQ(decoded_size, initial_size);