voxelcore/test/coders/lua_parsing.cpp
MihailRis 11cff72508 Fix syntax highlighting unicode support (#475)
* convert BasicParser to a template

* fix syntax hightlighting with unicode characters
2025-02-22 01:04:34 +03:00

22 lines
715 B
C++

#include <gtest/gtest.h>
#include "coders/commons.hpp"
#include "coders/lua_parsing.hpp"
#include "files/files.hpp"
#include "util/stringutil.hpp"
TEST(lua_parsing, Tokenizer) {
auto filename = "../../res/scripts/stdlib.lua";
auto source = files::read_string(std::filesystem::u8path(filename));
try {
auto tokens = lua::tokenize(filename, util::str2wstr_utf8(source));
for (const auto& token : tokens) {
std::cout << (int)token.tag << " "
<< util::quote(util::wstr2str_utf8(token.text))
<< std::endl;
}
} catch (const parsing_error& err) {
std::cerr << err.errorLog() << std::endl;
throw err;
}
}