From ec51848447727860d100b8b4b080093a519bb562 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 10 Oct 2026 16:31:36 +0300 Subject: [PATCH 1/6] update memory_ostream --- src/io/memory_ostream.hpp | 133 ++++++++++++++++++++++++------------- test/io/memory_ostream.cpp | 22 ++++++ 2 files changed, 110 insertions(+), 45 deletions(-) create mode 100644 test/io/memory_ostream.cpp diff --git a/src/io/memory_ostream.hpp b/src/io/memory_ostream.hpp index 3ff14a994..7568b8934 100644 --- a/src/io/memory_ostream.hpp +++ b/src/io/memory_ostream.hpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -13,81 +15,118 @@ public: explicit memory_buffer(size_t initial_size = 64) : buffer(std::make_unique(initial_size)), capacity(initial_size) { - setp(buffer.get(), buffer.get() + initial_size); + setp(buffer.get(), buffer.get() + capacity); } std::string_view view() const { - return std::string_view(pbase(), pptr() - pbase()); + return std::string_view(pbase(), size()); } util::Buffer release() { - return {std::move(buffer), size()}; + size_t n = size(); + return {std::move(buffer), n}; } size_t size() const { - return pptr() - pbase(); + return std::max(length, position()); + } + + void set_sync_callback(std::function callback) { + on_sync = std::move(callback); } protected: int_type overflow(int_type c) override { if (c == traits_type::eof()) return traits_type::eof(); - const size_t data_size = pptr() - pbase(); - const size_t new_capacity = std::max(capacity * 2, data_size + 1); - auto new_buffer = std::make_unique(new_capacity); - - std::memcpy(new_buffer.get(), pbase(), data_size); - - buffer = std::move(new_buffer); - capacity = new_capacity; - - setp(buffer.get(), buffer.get() + new_capacity); - pbump(data_size); - + reserve(position() + 1); *pptr() = traits_type::to_char_type(c); pbump(1); - return c; } std::streamsize xsputn(const char* s, std::streamsize count) override { - const std::streamsize avail = epptr() - pptr(); + reserve(position() + count); + std::memcpy(pptr(), s, count); + advance(count); + return count; + } - if (avail >= count) { - std::memcpy(pptr(), s, count); - pbump(count); - return count; + pos_type seekoff( + off_type off, + std::ios_base::seekdir dir, + std::ios_base::openmode which + ) override { + if (!(which & std::ios_base::out)) { + return pos_type(off_type(-1)); + } + length = size(); + + off_type base; + switch (dir) { + case std::ios_base::beg: base = 0; break; + case std::ios_base::cur: base = position(); break; + case std::ios_base::end: base = length; break; + default: return pos_type(off_type(-1)); } - std::streamsize written = 0; - if (avail > 0) { - std::memcpy(pptr(), s, avail); - written += avail; - s += avail; - count -= avail; - pbump(avail); + off_type target = base + off; + if (target < 0 || target > static_cast(length)) { + return pos_type(off_type(-1)); } + set_position(target); + return pos_type(target); + } - const size_t data_size = pptr() - pbase(); - const size_t required_capacity = data_size + count; - const size_t new_capacity = std::max(capacity * 2, required_capacity); - auto new_buffer = std::make_unique(new_capacity); - - std::memcpy(new_buffer.get(), pbase(), data_size); - std::memcpy(new_buffer.get() + data_size, s, count); - - buffer = std::move(new_buffer); - capacity = new_capacity; - - setp(buffer.get(), buffer.get() + new_capacity); - pbump(data_size + count); - written += count; - - return written; + pos_type seekpos(pos_type pos, std::ios_base::openmode which) override { + return seekoff(off_type(pos), std::ios_base::beg, which); } private: std::unique_ptr buffer; size_t capacity; + size_t length = 0; + + size_t position() const { + return pptr() - pbase(); + } + + void advance(size_t n) { + while (n > 0) { + int step = static_cast(std::min(n, INT_MAX)); + pbump(step); + n -= step; + } + } + + void set_position(size_t pos) { + setp(buffer.get(), buffer.get() + capacity); + advance(pos); + } + + void reserve(size_t required) { + if (required <= capacity) { + return; + } + size_t pos = position(); + length = size(); + + size_t new_capacity = std::max(capacity * 2, required); + auto new_buffer = std::make_unique(new_capacity); + std::memcpy(new_buffer.get(), buffer.get(), length); + + buffer = std::move(new_buffer); + capacity = new_capacity; + set_position(pos); + } + + int sync() override { + if (on_sync) { + on_sync(view()); + } + return 0; + } + + std::function on_sync; }; class memory_ostream : public std::ostream { @@ -102,6 +141,10 @@ public: util::Buffer release() { return buffer.release(); } + + void set_sync_callback(std::function callback) { + buffer.set_sync_callback(std::move(callback)); + } private: memory_buffer buffer; }; diff --git a/test/io/memory_ostream.cpp b/test/io/memory_ostream.cpp new file mode 100644 index 000000000..c1f83e0f8 --- /dev/null +++ b/test/io/memory_ostream.cpp @@ -0,0 +1,22 @@ +#include + +#include "io/memory_ostream.hpp" + +TEST(io, memory_ostream) { + const char data[] = "Hello, world!"; + const int n = std::strlen(data); + + memory_ostream stream; + ASSERT_TRUE(stream.good()); + + stream.write(data, n); + ASSERT_EQ(std::string(stream.view()), std::string(data)); + + stream.seekp(0); + ASSERT_TRUE(stream.good()); + stream.write("Remove", 6); + ASSERT_EQ(std::string(stream.view()), "Remove world!"); + + auto buffer = stream.release(); + ASSERT_EQ(std::string(buffer.data(), buffer.size()), "Remove world!"); +} From 682ca665c9f11741f07f56579e09c41aa0e94d43 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 10 Oct 2026 16:32:15 +0300 Subject: [PATCH 2/6] make memory device create file on flush --- src/io/devices/MemoryDevice.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/io/devices/MemoryDevice.cpp b/src/io/devices/MemoryDevice.cpp index f9f4760ee..ae08b0da3 100644 --- a/src/io/devices/MemoryDevice.cpp +++ b/src/io/devices/MemoryDevice.cpp @@ -14,8 +14,13 @@ std::filesystem::path io::MemoryDevice::resolve(std::string_view path) { std::unique_ptr io::MemoryDevice::write(std::string_view path) { std::string filePath = std::string(path); + + auto memoryStream = std::make_unique(); + memoryStream->set_sync_callback([this, filePath](std::string_view data) { + createFile(std::move(filePath), util::Buffer(data.data(), data.size())); + }); return std::make_unique( - std::make_unique(), + std::move(memoryStream), [this, filePath](auto ostream) { auto& memoryStream = dynamic_cast(*ostream); createFile(std::move(filePath), memoryStream.release()); From 5fd3ce76da65c5d1f0b7454fbf8078f28afbd692 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 10 Oct 2026 16:32:33 +0300 Subject: [PATCH 3/6] fix --- src/logic/scripting/lua/libs/libfile.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/logic/scripting/lua/libs/libfile.cpp b/src/logic/scripting/lua/libs/libfile.cpp index 1bcceb6d3..61a7e2191 100644 --- a/src/logic/scripting/lua/libs/libfile.cpp +++ b/src/logic/scripting/lua/libs/libfile.cpp @@ -305,13 +305,14 @@ static int l_read_descriptor(lua::State* L) { int maxlen = lua::tointeger(L, 2); auto& stream = io_descriptors::require_input(descriptor); - if (stream.eof()) { - stream.clear(); - } util::Buffer buffer(maxlen); stream.read(buffer.data(), maxlen); - std::streamsize read_len = stream.gcount(); + std::streamsize read_len = stream.gcount(); + + if (stream.eof()) { + stream.clear(stream.rdstate() & std::ios::badbit); + } return lua::create_bytearray(L, buffer.data(), read_len); } From 48680d975fc70bb7554b16819bcebd1df5bdc2fe Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 10 Oct 2026 17:05:03 +0300 Subject: [PATCH 4/6] fix missing include --- src/io/memory_ostream.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/io/memory_ostream.hpp b/src/io/memory_ostream.hpp index 7568b8934..a4752e251 100644 --- a/src/io/memory_ostream.hpp +++ b/src/io/memory_ostream.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include From 3617e01d21f26d181762b01df2a4124ff3b5aa4f Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 10 Oct 2026 17:24:17 +0300 Subject: [PATCH 5/6] refactor: split io_stream --- res/modules/internal/stream/binary.lua | 72 ++++ res/modules/internal/stream/blocking.lua | 19 + res/modules/internal/stream/buffered.lua | 90 +++++ res/modules/internal/stream/raw.lua | 49 +++ res/modules/internal/stream/text.lua | 102 +++++ res/modules/internal/stream/wrapper.lua | 26 ++ res/modules/io_stream.lua | 476 ++++------------------- 7 files changed, 428 insertions(+), 406 deletions(-) create mode 100644 res/modules/internal/stream/binary.lua create mode 100644 res/modules/internal/stream/blocking.lua create mode 100644 res/modules/internal/stream/buffered.lua create mode 100644 res/modules/internal/stream/raw.lua create mode 100644 res/modules/internal/stream/text.lua create mode 100644 res/modules/internal/stream/wrapper.lua diff --git a/res/modules/internal/stream/binary.lua b/res/modules/internal/stream/binary.lua new file mode 100644 index 000000000..9d38f3b9b --- /dev/null +++ b/res/modules/internal/stream/binary.lua @@ -0,0 +1,72 @@ +local binary = {} +binary.__index = binary + +local DEFAULT_CHUNK = 8192 + +function binary.new(stream) + return setmetatable({ stream = stream }, binary) +end + +local function to_table(bytes) + local t = {} + for i = 1, #bytes do + t[i] = bytes[i] + end + return t +end + +function binary.read_all(stream) + local result = Bytearray() + local avail = stream:available() + local chunk = avail > 0 and avail or DEFAULT_CHUNK + + while true do + local part = stream:read_some(chunk) + if #part == 0 then break end + result:append(part) + end + + return result +end + +function binary:read(arg, as_table) + local t = type(arg) + + if t == "number" then + local bytes = self.stream:read(arg) + return as_table and to_table(bytes) or bytes + elseif t == "string" then + local size = byteutil.get_size(arg) + local bytes = self.stream:read(size) + if #bytes < size then + error(string.format( + "unexpected end of stream: format %q needs %d bytes, got %d", + arg, size, #bytes), 2) + end + return byteutil.unpack(arg, bytes) + elseif t == "nil" then + error("in binary mode the first argument must be a byteutil format string" + .. " or the number of bytes to read") + else + error("unknown argument type: " .. t) + end +end + +function binary:write(arg, ...) + local t = type(arg) + + if t == "string" then + self.stream:write(byteutil.pack(arg, ...)) + elseif t == "table" then + self.stream:write(Bytearray(arg)) + else + self.stream:write(arg) + end +end + +function binary:read_fully(as_table) + local bytes = binary.read_all(self.stream) + return as_table and to_table(bytes) or bytes +end + +return binary diff --git a/res/modules/internal/stream/blocking.lua b/res/modules/internal/stream/blocking.lua new file mode 100644 index 000000000..a722a733e --- /dev/null +++ b/res/modules/internal/stream/blocking.lua @@ -0,0 +1,19 @@ +local wrapper = require "internal/stream/wrapper" + +local blocking = wrapper.extend({}) + +function blocking.new(inner) + return setmetatable({ inner = inner }, blocking) +end + +function blocking:read(n) + local result = self.inner:read(n) + while #result < n do + coroutine.yield() + if not self.inner:is_alive() then break end + result:append(self.inner:read(n - #result)) + end + return result +end + +return blocking diff --git a/res/modules/internal/stream/buffered.lua b/res/modules/internal/stream/buffered.lua new file mode 100644 index 000000000..621e78ebd --- /dev/null +++ b/res/modules/internal/stream/buffered.lua @@ -0,0 +1,90 @@ +local wrapper = require "internal/stream/wrapper" + +local buffered = wrapper.extend({}) + +buffered.DEFAULT_SIZE = 8192 + +function buffered.new(inner, size, flush_inner) + return setmetatable({ + inner = inner, + size = size or buffered.DEFAULT_SIZE, + flush_inner = flush_inner ~= false, + rbuf = Bytearray(), + wbuf = Bytearray(), + }, buffered) +end + +function buffered:_take(n) + local buf = self.rbuf + if n >= #buf then + self.rbuf = Bytearray() + return buf + end + local part = buf:slice(1, n) + buf:remove(1, n) + return part +end + +function buffered:flush_buffer() + if #self.wbuf > 0 then + local data = self.wbuf + self.wbuf = Bytearray() + self.inner:write(data) + end +end + +function buffered:read(n) + local have = #self.rbuf + if have < n then + if have == 0 and n >= self.size then + return self.inner:read(n) + end + self.rbuf:append(self.inner:read(self.size - have)) + end + return self:_take(n) +end + +buffered.read_some = buffered.read + +function buffered:write(data) + if #self.wbuf + #data > self.size then + self:flush_buffer() + end + if #data >= self.size then + self.inner:write(data) + else + self.wbuf:append(data) + end +end + +function buffered:flush() + self:flush_buffer() + if self.flush_inner then + self.inner:flush() + end +end + +function buffered:available() + return self.inner:available() + #self.rbuf +end + +function buffered:seek(mode, offset) + self:flush_buffer() + if mode == "c" then + offset = offset - #self.rbuf + end + self.rbuf = Bytearray() + self.inner:seek(mode, offset) +end + +function buffered:tell() + return self.inner:tell() - #self.rbuf + #self.wbuf +end + +function buffered:close() + self:flush_buffer() + self.rbuf = Bytearray() + return self.inner:close() +end + +return buffered diff --git a/res/modules/internal/stream/raw.lua b/res/modules/internal/stream/raw.lua new file mode 100644 index 000000000..b7db77919 --- /dev/null +++ b/res/modules/internal/stream/raw.lua @@ -0,0 +1,49 @@ +local raw = {} +raw.__index = raw + +function raw.new(descriptor, io_lib) + return setmetatable({ descriptor = descriptor, io_lib = io_lib }, raw) +end + +function raw:read(n) + return self.io_lib.read(self.descriptor, n) +end + +raw.read_some = raw.read + +function raw:write(data) + self.io_lib.write(self.descriptor, data) +end + +function raw:flush() + if self.io_lib.flush then + self.io_lib.flush(self.descriptor) + elseif not self:is_alive() then + error("stream is closed") + end +end + +function raw:available() + local f = self.io_lib.available + return f and f(self.descriptor) or 0 +end + +function raw:seek(mode, offset) + if not self.io_lib.seek then error("cannot seek this stream") end + self.io_lib.seek(self.descriptor, mode, offset) +end + +function raw:tell() + if not self.io_lib.tell then error("cannot tell this stream") end + return self.io_lib.tell(self.descriptor) +end + +function raw:is_alive() + return self.io_lib.is_alive(self.descriptor) +end + +function raw:close() + return self.io_lib.close(self.descriptor) +end + +return raw diff --git a/res/modules/internal/stream/text.lua b/res/modules/internal/stream/text.lua new file mode 100644 index 000000000..835340b47 --- /dev/null +++ b/res/modules/internal/stream/text.lua @@ -0,0 +1,102 @@ +local binary = require "internal/stream/binary" + +local text = {} +text.__index = text + +local CR = string.byte('\r') +local LF = string.byte('\n') + +function text.new(stream) + return setmetatable({stream = stream}, text) +end + +function text:_read_byte() + return self.stream:read(1)[1] +end + +function text:read_line() + local line = Bytearray() + local got_any = false + + while true do + local b = self:_read_byte() + if b == nil then break end + got_any = true + + if b == LF then + break + elseif b == CR then + local nb = self:_read_byte() + if nb == LF then break end + line:append(CR) + if nb == nil then break end + line:append(nb) + else + line:append(b) + end + end + + if not got_any then return nil end + return utf8.tostring(line) +end + +local function trim_empty(lines) + while #lines > 0 and lines[#lines] == "" do + lines[#lines] = nil + end + while #lines > 0 and lines[1] == "" do + table.remove(lines, 1) + end +end + +function text:read_lines(count, trim) + if count < 0 then error("count of lines to read must be positive") end + + local lines = {} + for i = 1, count do + local line = self:read_line() + if line == nil then break end + lines[i] = line + end + + if trim ~= false then trim_empty(lines) end + return lines +end + +function text:read(count, trim) + if count == nil then + return self:read_line() + end + return self:read_lines(count, trim) +end + +function text:read_fully(as_lines) + if as_lines then + local lines = {} + for line in self.read_line, self do + lines[#lines + 1] = line + end + return lines + end + return utf8.tostring(binary.read_all(self.stream)) +end + +function text:write_line(str) + self.stream:write(utf8.tobytes(str .. "\n")) +end + +-- write(string) | write({ lines... }) +function text:write(arg) + local t = type(arg) + if t == "string" then + self:write_line(arg) + elseif t == "table" then + for i = 1, #arg do + self:write_line(arg[i]) + end + else + error("unknown argument type: " .. t) + end +end + +return text diff --git a/res/modules/internal/stream/wrapper.lua b/res/modules/internal/stream/wrapper.lua new file mode 100644 index 000000000..c6c838aff --- /dev/null +++ b/res/modules/internal/stream/wrapper.lua @@ -0,0 +1,26 @@ +local wrapper = {} +wrapper.__index = wrapper + +function wrapper.extend(cls) + cls.__index = cls + return setmetatable(cls, { __index = wrapper }) +end +function wrapper:read(n) return self.inner:read(n) end + +function wrapper:read_some(n) return self.inner:read_some(n) end + +function wrapper:write(data) return self.inner:write(data) end + +function wrapper:flush() return self.inner:flush() end + +function wrapper:available() return self.inner:available() end + +function wrapper:seek(mode, off) return self.inner:seek(mode, off) end + +function wrapper:tell() return self.inner:tell() end + +function wrapper:is_alive() return self.inner:is_alive() end + +function wrapper:close() return self.inner:close() end + +return wrapper diff --git a/res/modules/io_stream.lua b/res/modules/io_stream.lua index 8776ca810..cd318c72c 100644 --- a/res/modules/io_stream.lua +++ b/res/modules/io_stream.lua @@ -1,444 +1,108 @@ -local io_stream = { } +local raw = require "internal/stream/raw" +local buffered = require "internal/stream/buffered" +local blocking = require "internal/stream/blocking" +local binary = require "internal/stream/binary" +local text = require "internal/stream/text" +local io_stream = {} io_stream.__index = io_stream -local MAX_BUFFER_SIZE = 8192 - -local DEFAULT_MODE = "default" -local BUFFERED_MODE = "buffered" -local YIELD_MODE = "yield" - -local ALL_MODES = { - DEFAULT_MODE, - BUFFERED_MODE, - YIELD_MODE -} - -local FLUSH_MODE_ALL = "all" -local FLUSH_MODE_ONLY_BUFFER = "buffer" - -local ALL_FLUSH_MODES = { - FLUSH_MODE_ALL, - FLUSH_MODE_ONLY_BUFFER -} - -local CR = string.byte('\r') -local LF = string.byte('\n') - -local function read_fully(result, read_func) - local isTable = type(result) == "table" - - local buf - - repeat - buf = read_func(MAX_BUFFER_SIZE) - - if isTable then - for i = 1, #buf do - result[#result + 1] = buf[i] - end - else result:append(buf) end - until #buf == 0 -end - ---[[ -descriptor - descriptor of stream for provided I/O library -binaryMode - if enabled, most methods will expect bytes instead of strings -ioLib - I/O library. Should include the following functions: - read(descriptor: int, length: int) -> Bytearray - May return bytearray with a smaller size if bytes have not arrived yet or have run out - May throw error if descriptor isn't readable - write(descriptor: int, data: Bytearray) - May throw error if descriptor isn't writeable - [optional] seek(descriptor: int, mode: string, offset: int) - Mode may be 'b' (relative begin), 'c' (relative current), 'e' (relative end + 1) - [optional] tell(descriptor: int) -> int - [optional] flush(descriptor: int) - [optional] available(descriptor: int) -> int - is_alive(descriptor: int) -> bool - close(descriptor: int) ---]] +local MODES = { default = true, buffered = true, yield = true } +local FLUSH_MODES = { all = true, buffer = true } function io_stream.new(descriptor, binary_mode, io_lib, mode, flush_mode) - mode = mode or DEFAULT_MODE - flush_mode = flush_mode or FLUSH_MODE_ALL - - local self = setmetatable({}, io_stream) - - self.descriptor = descriptor - self.binary_mode = binary_mode - self.max_buffer_size = MAX_BUFFER_SIZE - self.io_lib = io_lib - - self:set_mode(mode) - self:set_flush_mode(flush_mode) + local self = setmetatable({ + descriptor = descriptor, + io_lib = io_lib, + binary_mode = binary_mode and true or false, + max_buffer_size = buffered.DEFAULT_SIZE, + }, io_stream) + self:set_flush_mode(flush_mode or "all") + self:set_mode(mode or "default") return self end -function io_stream:is_binary_mode() - return self.binary_mode +function io_stream:_rebuild() + local old = self.stream + if old and old.flush_buffer and old:is_alive() then + old:flush_buffer() + end + + local s = raw.new(self.descriptor, self.io_lib) + if self.mode == "buffered" then + s = buffered.new(s, self.max_buffer_size, self.flush_mode == "all") + elseif self.mode == "yield" then + s = blocking.new(s) + end + + self.stream = s + self.text = text.new(s) + self.binary = binary.new(s) end -function io_stream:set_binary_mode(binary_mode) - self.binary_mode = binary_mode ~= nil +function io_stream:_format() + return self.binary_mode and self.binary or self.text end -function io_stream:get_mode() - return self.mode -end +function io_stream:is_binary_mode() return self.binary_mode end +function io_stream:set_binary_mode(v) self.binary_mode = v and true or false end + +function io_stream:get_mode() return self.mode end function io_stream:set_mode(mode) - if not table.has(ALL_MODES, mode) then - error("invalid stream mode: "..mode) - end - - if self.write_buffer then - self.write_buffer:clear() - self.write_buffer = nil - end - - if self.read_buffer then - self.read_buffer:clear() - self.read_buffer = nil - end - - if mode == BUFFERED_MODE then - self.write_buffer = Bytearray(self.max_buffer_size) - self.read_buffer = Bytearray(self.max_buffer_size) - end - + if not MODES[mode] then error("invalid stream mode: " .. tostring(mode)) end self.mode = mode + self:_rebuild() end -function io_stream:get_flush_mode() - return self.flush_mode -end +function io_stream:get_flush_mode() return self.flush_mode end function io_stream:set_flush_mode(flush_mode) - if not table.has(ALL_FLUSH_MODES, flush_mode) then - error("invalid flush mode: " .. flush_mode) + if not FLUSH_MODES[flush_mode] then + error("invalid flush mode: " .. tostring(flush_mode)) end - self.flush_mode = flush_mode + if self.mode == "buffered" then + self.stream.flush_inner = flush_mode == "all" + end end -function io_stream:get_max_buffer_size() - return self.max_buffer_size +function io_stream:get_max_buffer_size() return self.max_buffer_size end + +function io_stream:set_max_buffer_size(size) + self.max_buffer_size = size + if self.mode == "buffered" then self:_rebuild() end end -function io_stream:set_max_buffer_size(max_buffer_size) - self.max_buffer_size = max_buffer_size +function io_stream:read(arg, use_table) return self:_format():read(arg, use_table) end - self.write_buffer = Bytearray(self.max_buffer_size) - self.read_buffer = Bytearray(self.max_buffer_size) -end +function io_stream:write(arg, ...) return self:_format():write(arg, ...) end + +function io_stream:read_fully(use_table) return self:_format():read_fully(use_table) end + +function io_stream:read_line() return self.text:read_line() end + +function io_stream:write_line(str) return self.text:write_line(str) end function io_stream:available(length) - local available = self.io_lib.available and self.io_lib.available(self.descriptor) or 0 - - if self.mode == BUFFERED_MODE then - available = available + #self.read_buffer - end - - if not length then - return available - else - return available >= length - end + local n = self.stream:available() + if length then return n >= length end + return n end -function io_stream:__read(length, from_read_fully) - if self.mode == YIELD_MODE then - if from_read_fully then - return self.io_lib.read(self.descriptor, length) - end +function io_stream:seek(mode, offset) return self.stream:seek(mode, offset) end - local buffer = Bytearray() +function io_stream:tell() return self.stream:tell() end - while #buffer < length do - buffer:append(self.io_lib.read(self.descriptor, length - #buffer)) +function io_stream:is_alive() return self.stream:is_alive() end - if #buffer < length then coroutine.yield() end - end +function io_stream:is_closed() return not self.stream:is_alive() end - return buffer - elseif self.mode == BUFFERED_MODE then - local buf_len = #self.read_buffer - - if buf_len < length then - self.read_buffer:append( - self.io_lib.read(self.descriptor, self.max_buffer_size - buf_len) - ) - end - - buf_len = #self.read_buffer - - length = math.min(buf_len, length) - - local copy = self.read_buffer:slice(1, length) - - if buf_len == length then - self.read_buffer:clear() - else - self.read_buffer:remove(1, length) - end - - return copy - elseif self.mode == DEFAULT_MODE then - return self.io_lib.read(self.descriptor, length) - end -end - -function io_stream:__write(data) - if self.mode == BUFFERED_MODE then - local data_length = #data - - if #self.write_buffer + data_length > self.max_buffer_size then - self:flush() - end - - if data_length > self.max_buffer_size then - local to_write = math.floor(data_length / self.max_buffer_size) * self.max_buffer_size - local to_save = data_length - to_write - - self.io_lib.write(self.descriptor, data:slice(1, to_write)) - - self:flush() - - self.write_buffer = data:slice(to_write + 1, to_save) - else self.write_buffer:append(data) end - elseif self.mode == DEFAULT_MODE or self.mode == YIELD_MODE then - return self.io_lib.write(self.descriptor, data) - end -end - -function io_stream:read_fully(use_table) - if self.binary_mode then - local result = use_table and { } or Bytearray() - - local avail = self:available() - - if avail == 0 then - avail = self.max_buffer_size - end - - read_fully(result, function() return self:__read(avail, true) end) - - return result - else - if use_table then - local lines = { } - - local line - - repeat - line = self:read_line() - - lines[#lines + 1] = line - until not line - - return lines - else - local result = Bytearray() - - read_fully(result, function() return self:__read(self.max_buffer_size) end) - - return utf8.tostring(result) - end - end -end - -function io_stream:read_line() - local result = Bytearray() - - local first = true - - while true do - local char = self:__read(1) - - if #char == 0 then - if first then return else break end - end - - char = char[1] - - if char == LF then break - elseif char == CR then - char = self:__read(1) - - if char[1] == LF then break - else - result:append(CR) - result:append(char[1]) - end - else result:append(char) end - - first = false - end - - return utf8.tostring(result) -end - -function io_stream:write_line(str) - self:__write(utf8.tobytes(str .. "\n")) -end - -function io_stream:read(arg, use_table) - local arg_type = type(arg) - - if self.binary_mode then - local byte_arr - - if arg_type == "number" then - -- using 'arg' as length - - byte_arr = self:__read(arg) - - if use_table == true then - local t = { } - - for i = 1, #byte_arr do - t[i] = byte_arr[i] - end - - return t - else - return byte_arr - end - elseif arg_type == "string" then - return byteutil.unpack( - arg, - self:__read(byteutil.get_size(arg)) - ) - elseif arg_type == "nil" then - error( - "in binary mode the first argument must be a string data format".. - " for the library \"byteutil\" or the number of bytes to read" - ) - else - error("unknown argument type: "..arg_type) - end - else - if not arg then - return self:read_line() - else - local lines_count = arg - local trim_last_empty_lines = use_table - - if use_table == nil then - trim_last_empty_lines = true - end - - if lines_count < 0 then error "count of lines to read must be positive" end - - local result = { } - - for i = 1, lines_count do - result[i] = self:read_line() - end - - if trim_last_empty_lines then - local i = #result - - while i >= 0 do - local length = utf8.length(result[i]) - - if length > 0 then break - else result[i] = nil end - - i = i - 1 - end - - local i = 1 - - while #result > 0 do - local length = utf8.length(result[i]) - - if length > 0 then break - else table.remove(result, i) end - end - end - - return result - end - end -end - -function io_stream:write(arg, ...) - local arg_type = type(arg) - - if self.binary_mode then - local byte_arr - - if arg_type ~= "string" then - -- using arg as bytes table/bytearray - - if arg_type == "table" then - byte_arr = Bytearray(arg) - else - byte_arr = arg - end - else - byte_arr = byteutil.pack(arg, ...) - end - - self:__write(byte_arr) - else - if arg_type == "string" then - self:write_line(arg) - elseif arg_type == "table" then - for i = 1, #arg do - self:write_line(arg[i]) - end - else error("unknown argument type: "..arg_type) end - end -end - -function io_stream:seek(mode, offset) - if not self.io_lib.seek then - error("cannot seek this stream") - end - - self.io_lib.seek(self.descriptor, mode, offset) -end - -function io_stream:tell() - if not self.io_lib.tell then - error("cannot tell this stream") - end - - return self.io_lib.tell(self.descriptor) -end - -function io_stream:is_alive() - return self.io_lib.is_alive(self.descriptor) -end - -function io_stream:is_closed() - return not self:is_alive() -end - -function io_stream:close() - if self.mode == BUFFERED_MODE then - self.read_buffer:clear() - self.write_buffer:clear() - end - - return self.io_lib.close(self.descriptor) -end +function io_stream:close() return self.stream:close() end function io_stream:flush() - if self.mode == BUFFERED_MODE and #self.write_buffer > 0 then - self.io_lib.write(self.descriptor, self.write_buffer) - self.write_buffer:clear() - end - - if self.flush_mode ~= FLUSH_MODE_ONLY_BUFFER then - if self.io_lib.flush then - self.io_lib.flush(self.descriptor) - elseif self:is_closed() then error("stream is closed") end - end + if self.mode ~= "buffered" and self.flush_mode == "buffer" then return end + self.stream:flush() end - -return io_stream \ No newline at end of file +return io_stream From 2a4feaf1c4b2e4fa02fc582c5ca2942e54be6feb Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 10 Oct 2026 17:35:07 +0300 Subject: [PATCH 6/6] update io_stream lua test --- dev/tests/io_stream.lua | 403 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 377 insertions(+), 26 deletions(-) diff --git a/dev/tests/io_stream.lua b/dev/tests/io_stream.lua index d08c68b0c..da2a2efb0 100644 --- a/dev/tests/io_stream.lua +++ b/dev/tests/io_stream.lua @@ -1,34 +1,385 @@ local device = file.create_memory_device() -file.write(device..":test.txt", "Hello\nWorld") -file.write_bytes(device..":test.bin", Bytearray({20, 30, 100, 200, 255})) -local stream = file.open(device..":test.txt", 'r') +local function path(name) return device .. ":" .. name end -local line1 = stream:read_line() -local line2 = stream:read_line() +local function eq_list(actual, expected, msg) + asserts.equals(#actual, #expected, (msg or "list") .. " length") + for i = 1, #expected do + asserts.equals(actual[i], expected[i], (msg or "list") .. "[" .. i .. "]") + end +end -assert(line1 == "Hello") -assert(line2 == "World") +file.write(path("test.txt"), "Hello\nWorld") +file.write_bytes(path("test.bin"), Bytearray({20, 30, 100, 200, 255})) -local stream2 = file.open(device..":test.bin", 'rb') -local data = stream2:read(5) -assert(data[1] == 20) -assert(data[2] == 30) -assert(data[3] == 100) -assert(data[4] == 200) -assert(data[5] == 255) +do + local stream = file.open(path("test.txt"), 'r') + asserts.equals(stream:read_line(), "Hello") + asserts.equals(stream:read_line(), "World") + stream:close() +end -local stream3 = file.open(device..":test.txt", 'r') -assert(stream3:is_alive()) -stream3:close() -assert(not stream3:is_alive()) +do + local stream = file.open(path("test.bin"), 'rb') + local data = stream:read(5) + asserts.equals(data[1], 20); asserts.equals(data[2], 30); asserts.equals(data[3], 100); asserts.equals(data[4], 200); asserts + .equals(data[5], 255) + stream:close() +end -local stream4 = file.open(device..":test.txt", 'r') -stream4:seek('b', 6) -local line = stream4:read_line() -assert(line == "World") +do + local stream = file.open(path("test.txt"), 'r') + assert(stream:is_alive()) + assert(not stream:is_closed()) + stream:close() + assert(not stream:is_alive()) + assert(stream:is_closed()) +end -local stream5 = file.open(device..":test.txt", 'r') -stream5:seek('e', 0) -local pos = stream5:tell() -assert(pos == 11) +do + local stream = file.open(path("test.txt"), 'r') + stream:seek('b', 6) + asserts.equals(stream:read_line(), "World") + stream:close() +end + +do + local stream = file.open(path("test.txt"), 'r') + stream:seek('e', 0) + asserts.equals(stream:tell(), 11) + stream:close() +end + +------------------------------------------------------------------------------- +-- text: line endings and end of stream +------------------------------------------------------------------------------- + +do + local stream = file.open(path("test.txt"), 'r') + stream:read_line(); stream:read_line() + asserts.equals(stream:read_line(), nil, "after last line") + asserts.equals(stream:read_line(), nil, "after last line, again") + stream:close() +end + +do + file.write(path("empty.txt"), "") + local stream = file.open(path("empty.txt"), 'r') + asserts.equals(stream:read_line(), nil, "empty file") + stream:close() +end + +do + file.write(path("crlf.txt"), "a\r\nb\rc\nd\r") + local stream = file.open(path("crlf.txt"), 'r') + asserts.equals(stream:read_line(), "a", "CRLF") + asserts.equals(stream:read_line(), "b\rc", "lone CR is kept") + asserts.equals(stream:read_line(), "d\r", "CR at end of stream") + asserts.equals(stream:read_line(), nil) + stream:close() +end + +------------------------------------------------------------------------------- +-- text: read(count) and read_fully +------------------------------------------------------------------------------- + +do + file.write(path("blank.txt"), "\n\nA\nB\n\n") + local stream = file.open(path("blank.txt"), 'r') + eq_list(stream:read(10), {"A", "B"}, "trimmed") + stream:close() +end + +do + local stream = file.open(path("blank.txt"), 'r') + eq_list(stream:read(10, false), {"", "", "A", "B", ""}, "untrimmed") + stream:close() +end + +do + file.write(path("only_blank.txt"), "\n\n\n") + local stream = file.open(path("only_blank.txt"), 'r') + eq_list(stream:read(5), {}, "only empty lines") + stream:close() +end + +do + local stream = file.open(path("test.txt"), 'r') + eq_list(stream:read(5), {"Hello", "World"}, "past end of stream") + stream:close() +end + +do + local stream = file.open(path("test.txt"), 'r') + assert(not pcall(stream.read, stream, -1), "negative count must fail") + stream:close() +end + +do + local stream = file.open(path("test.txt"), 'r') + asserts.equals(stream:read(), "Hello") + stream:close() +end + +do + local stream = file.open(path("test.txt"), 'r') + asserts.equals(stream:read_fully(), "Hello\nWorld", "read_fully as string") + stream:close() +end + +do + local stream = file.open(path("test.txt"), 'r') + eq_list(stream:read_fully(true), {"Hello", "World"}, "read_fully as lines") + stream:close() +end + +------------------------------------------------------------------------------- +-- text: writing +------------------------------------------------------------------------------- + +do + local stream = file.open(path("out.txt"), 'w') + stream:write("one") + stream:write({"two", "three"}) + stream:write_line("four") + stream:close() + asserts.equals(file.read(path("out.txt")), "one\ntwo\nthree\nfour\n", "text write") +end + +do + local stream = file.open(path("out.txt"), 'w') + assert(not pcall(stream.write, stream, 42), "writing a number in text mode must fail") + stream:close() +end + +------------------------------------------------------------------------------- +-- binary +------------------------------------------------------------------------------- + +do + local stream = file.open(path("test.bin"), 'rb') + eq_list(stream:read(5, true), {20, 30, 100, 200, 255}, "read as table") + stream:close() +end + +do + local stream = file.open(path("test.bin"), 'rb') + stream:read(3) + asserts.equals(#stream:read(10), 2, "short read") + asserts.equals(#stream:read(10), 0, "read at end of stream") + stream:close() +end + +do + local stream = file.open(path("test.bin"), 'rb') + local data = stream:read_fully() + asserts.equals(#data, 5); asserts.equals(data[1], 20); asserts.equals(data[5], 255) + stream:close() +end + +do + local stream = file.open(path("test.bin"), 'rb') + eq_list(stream:read_fully(true), {20, 30, 100, 200, 255}, "read_fully as table") + stream:close() +end + +do + local stream = file.open(path("out.bin"), 'wb') + stream:write({1, 2, 3}) + stream:write(Bytearray({4, 5})) + stream:close() + + local data = file.read_bytes(path("out.bin")) + asserts.equals(#data, 5, "binary write length") + for i = 1, 5 do asserts.equals(data[i], i, "binary write byte " .. i) end +end + +do + local stream = file.open(path("fmt.bin"), 'wb') + stream:write("