Merge pull request #1027 from ostermine/fix/zip-paths

fix zip archives paths and add zip files test

dev/tests/memory_filesystem.lua

@@ -41,6 +41,17 @@ assert(entries[1] == "memtest:dir/subdir/a.txt")
 assert(entries[2] == "memtest:dir/subdir/b.txt")
 assert(entries[3] == "memtest:dir/subdir/other")
 
+debug.log("create sibling directory")
+file.mkdir("memtest:dir/subdir/sibling")
+assert(file.isfile("memtest:dir/subdir/a.txt"))
+assert(file.isdir("memtest:dir/subdir/other"))
+asserts.equals(4, #file.list("memtest:dir/subdir"))
+
+debug.log("overwrite file")
+file.write("memtest:dir/subdir/a.txt", "overwritten")
+asserts.equals("overwritten", file.read("memtest:dir/subdir/a.txt"))
+asserts.equals(4, #file.list("memtest:dir/subdir"))
+
 debug.log("remove tree")
 file.remove_tree("memtest:dir")
 assert(not file.isdir("memtest:dir"))

dev/tests/zip_files.lua

@@ -0,0 +1,118 @@
+local function sorted_list(path)
+    local entries = file.list(path)
+    table.sort(entries)
+    return entries
+end
+
+local function check_list(expected, path)
+    local entries = sorted_list(path)
+    asserts.equals(#expected, #entries)
+    for i, name in ipairs(expected) do
+        asserts.equals(name, entries[i])
+    end
+end
+
+local function join(root, name)
+    if root:sub(-1) == ":" then
+        return root..name
+    end
+    return root.."/"..name
+end
+
+local bytes = {0xDE, 0xAD, 0x00, 0xC0, 0xDE}
+
+local function create_tree(root)
+    file.mkdirs(join(root, "sub/deep"))
+    file.mkdir(join(root, "empty"))
+    file.write(join(root, "root.txt"), "root file")
+    file.write(join(root, "sub/a.txt"), "example, пример")
+    file.write_bytes(join(root, "sub/deep/binary"), bytes)
+end
+
+local function check_tree(m)
+    debug.log("check directories")
+    assert(file.exists(m..":"))
+    assert(file.isdir(m..":"))
+    assert(file.isdir(m..":sub"))
+    assert(file.isdir(m..":sub/deep"))
+    assert(file.isdir(m..":empty"))
+    assert(not file.isfile(m..":sub"))
+
+    debug.log("check files")
+    assert(file.isfile(m..":root.txt"))
+    assert(file.isfile(m..":sub/a.txt"))
+    assert(not file.isdir(m..":sub/a.txt"))
+    assert(not file.exists(m..":missing.txt"))
+    assert(not file.exists(m..":sub/missing.txt"))
+
+    debug.log("read files")
+    asserts.equals("root file", file.read(m..":root.txt"))
+    asserts.equals("example, пример", file.read(m..":sub/a.txt"))
+    asserts.equals(#"root file", file.length(m..":root.txt"))
+
+    local rbytes = file.read_bytes(m..":sub/deep/binary")
+    asserts.equals(#bytes, #rbytes)
+    for i, b in ipairs(bytes) do
+        asserts.equals(b, rbytes[i])
+    end
+
+    debug.log("read file with io_stream")
+    local stream = file.open(m..":root.txt", "r")
+    asserts.equals("root file", stream:read_line())
+    stream:close()
+
+    debug.log("list directories")
+    check_list({m..":empty", m..":root.txt", m..":sub"}, m..":")
+    check_list({m..":sub/a.txt", m..":sub/deep"}, m..":sub")
+    check_list({m..":sub/deep/binary"}, m..":sub/deep")
+    check_list({}, m..":empty")
+
+    debug.log("check read-only")
+    assert(not file.is_writeable(m..":"))
+    assert(not pcall(file.write, m..":new.txt", "text"))
+    local ok, created = pcall(file.mkdir, m..":newdir")
+    assert(not (ok and created))
+    assert(not file.exists(m..":newdir"))
+end
+
+local function check_zip(src, zipfile)
+    debug.log("create zip "..zipfile.." from "..src)
+    file.create_zip(src, zipfile)
+    assert(file.isfile(zipfile))
+
+    debug.log("mount "..zipfile)
+    local m = file.mount(zipfile)
+    check_tree(m)
+
+    debug.log("unmount "..m)
+    file.unmount(m)
+    assert(not pcall(file.read, m..":root.txt"))
+    assert(not pcall(file.unmount, m))
+end
+
+local mem = file.create_memory_device()
+local out = file.create_memory_device()
+
+debug.log("zip memory device root")
+create_tree(mem..":")
+check_zip(mem..":", out..":root.zip")
+
+debug.log("zip memory device subdirectory")
+create_tree(mem..":pack")
+check_zip(mem..":pack", out..":pack.zip")
+
+debug.log("zip nested subdirectory")
+create_tree(mem..":a/b/pack")
+check_zip(mem..":a/b/pack", out..":nested.zip")
+
+debug.log("zip real filesystem directory")
+if file.exists("config:ziptest") then
+    file.remove_tree("config:ziptest")
+end
+create_tree("config:ziptest/pack")
+check_zip("config:ziptest/pack", "config:ziptest/pack.zip")
+file.remove_tree("config:ziptest")
+
+debug.log("mount non-zip file")
+file.write(out..":text.txt", "not an archive")
+assert(not pcall(file.mount, out..":text.txt"))

src/io/devices/MemoryDevice.cpp

@@ -180,6 +180,10 @@ std::unique_ptr<io::PathsGenerator> io::MemoryDevice::list(std::string_view path
 }
 
 io::MemoryDevice::Dir* io::MemoryDevice::createDir(std::string path) {
+    const auto& found = nodes.find(path);
+    if (found != nodes.end()) {
+        return found->second.get_if<Dir>();
+    }
     io::path filePath = path;
     io::path parent = filePath.parent();
     auto parentDir = getDir(parent.string());
@@ -195,6 +199,15 @@ io::MemoryDevice::Dir* io::MemoryDevice::createDir(std::string path) {
 io::MemoryDevice::Node* io::MemoryDevice::createFile(
     std::string path, util::Buffer<char>&& content
 ) {
+    const auto& found = nodes.find(path);
+    if (found != nodes.end()) {
+        auto& node = found->second;
+        if (!node.holds_alternative<File>()) {
+            return nullptr;
+        }
+        node.data = File {std::move(content)};
+        return &node;
+    }
     io::path filePath = path;
     io::path parent = filePath.parent();
     auto dir = getDir(parent.string());

src/io/devices/ZipFileDevice.cpp

@@ -58,11 +58,20 @@ namespace {
     }
 
     uint32_t to_ms_dos_timestamp(const file_time_type& fileTime) {
+        // 1980-01-01 00:00, the earliest MS-DOS date
+        constexpr uint32_t DOS_EPOCH = (0 << 9 | 1 << 5 | 1) << 16;
+        if (fileTime == file_time_type::min()) {
+            return DOS_EPOCH;
+        }
         auto timePoint = time_point_cast<system_clock::duration>(
             fileTime - file_time_type::clock::now() + system_clock::now()
         );
         std::time_t timeT = system_clock::to_time_t(timePoint);
-        std::tm tm = *std::localtime(&timeT);
+        const std::tm* tmPtr = std::localtime(&timeT);
+        if (tmPtr == nullptr || tmPtr->tm_year < 80) {
+            return DOS_EPOCH;
+        }
+        const std::tm& tm = *tmPtr;
         uint16_t date = (tm.tm_year - 80) << 9 | (tm.tm_mon + 1) << 5 | tm.tm_mday;
         uint16_t time = (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec / 2);
         return (date << 16) | time;
@@ -173,7 +182,13 @@ ZipFileDevice::ZipFileDevice(
         entries[entry.fileName] = std::move(entry);
     }
 
-    for (auto& [name, _] : entries) {
+    // Add missing parent directories entries
+    std::vector<std::string> names;
+    names.reserve(entries.size());
+    for (const auto& [name, _] : entries) {
+        names.push_back(name);
+    }
+    for (const auto& name : names) {
         io::path path = name;
 
         while (!(path = path.parent()).pathPart().empty()) {
@@ -184,7 +199,6 @@ ZipFileDevice::ZipFileDevice(
             entry.isDirectory = true;
             entries[path.pathPart()] = entry;
         }
-        break;
     }
 
     for (auto& [_, entry] : entries) {
@@ -253,6 +267,9 @@ file_time_type ZipFileDevice::lastWriteTime(std::string_view path) {
 }
 
 bool ZipFileDevice::exists(std::string_view path) {
+    if (path.empty()) {
+        return true;
+    }
     return entries.find(std::string(path)) != entries.end();
 }
 
@@ -441,7 +458,11 @@ static size_t write_zip(
 void io::write_zip(const path& folder, const path& file) {
     ByteBuilder central_dir;
     auto out = io::write(file);
-    size_t entries = write_zip(folder.pathPart(), folder, *out, central_dir);
+    auto root = folder.pathPart();
+    if (!root.empty() && root.back() != '/') {
+        root += '/';
+    }
+    size_t entries = write_zip(root, folder, *out, central_dir);
 
     size_t central_dir_offset = out->tellp();
     out->write(reinterpret_cast<const char*>(central_dir.data()), central_dir.size());

src/io/memory_ostream.hpp

@@ -50,6 +50,19 @@ protected:
         return c;
     }
 
+    pos_type seekoff(
+        off_type off,
+        std::ios_base::seekdir way,
+        std::ios_base::openmode mode = std::ios_base::out
+    ) override {
+        // only position query (tellp) is supported
+        if (off == 0 && way == std::ios_base::cur &&
+            (mode & std::ios_base::out)) {
+            return pos_type(off_type(size()));
+        }
+        return pos_type(off_type(-1));
+    }
+
     std::streamsize xsputn(const char* s, std::streamsize count) override {
         const std::streamsize avail = epptr() - pptr();