Merge pull request #960 from MihailRis/assets-hot-reload

assets hot reload
This commit is contained in:
MihailRis 2026-08-31 22:50:16 +03:00 • committed by GitHub
commit c19733b594
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 310 additions and 209 deletions

View file

@ -1,4 +1,4 @@
# VC-DBG protocol v1
# VC-DBG protocol v1.1
## Notes
@ -116,6 +116,15 @@ Then `value` is:
}
```
### File hot-reload signal
```json
{
"type": "hot-reload",
"file": "string"
}
```
## Server-to-client
### Response signals

View file

@ -208,7 +208,9 @@ void SkeletonConfig::render(
model,
&skeleton.textures
);
}
} else if (!node->model.name.empty()) {
node->model.updateFlag = true;
}
}
}

View file

@ -1,6 +1,8 @@
#include "Assets.hpp"
#include "AssetsLoader.hpp"
#include "io/path.hpp"
Assets::Assets(util::ObjectsKeeper* vault) : vault(vault) {
Assets::Assets(util::ObjectsKeeper* vault) : vault(vault), assetsLoadInfo(std::make_unique<AssetsLoadInfo>()) {
}
Assets::~Assets() {

View file

@ -3,6 +3,7 @@
#include "util/stringutil.hpp"
#include "util/ObjectsKeeper.hpp"
#include "graphics/core/TextureAnimation.hpp"
#include "io/fwd.hpp"
#include <functional>
#include <memory>
@ -16,6 +17,8 @@
#include <vector>
class Assets;
struct AssetsLoadInfo;
class AssetsLoader;
enum class AssetType {
ANIMATION,
@ -34,8 +37,6 @@ namespace assetload {
/// @brief final work to do in the main thread
using postfunc = std::function<void(Assets&)>;
using setupfunc = std::function<void(const Assets&)>;
template <class T>
void assets_setup(const Assets&);
@ -63,16 +64,13 @@ namespace assetload {
return reason;
}
};
}
class Assets {
util::ObjectsKeeper* vault;
std::vector<TextureAnimation> animations;
using assets_map = std::unordered_map<std::string, std::shared_ptr<void>>;
std::unordered_map<std::type_index, assets_map> assets;
std::vector<assetload::setupfunc> setupFuncs;
public:
using assets_map = std::unordered_map<std::string, std::shared_ptr<void>>;
Assets(util::ObjectsKeeper* vault);
Assets(const Assets&) = delete;
~Assets();
@ -144,15 +142,15 @@ public:
return &mapIter->second;
}
void setup() {
for (auto& setupFunc : setupFuncs) {
setupFunc(*this);
}
AssetsLoadInfo& getLoadInfo() const {
return *assetsLoadInfo;
}
private:
util::ObjectsKeeper* vault;
std::vector<TextureAnimation> animations;
void addSetupFunc(assetload::setupfunc setupfunc) {
setupFuncs.push_back(setupfunc);
}
std::unordered_map<std::type_index, assets_map> assets;
std::unique_ptr<AssetsLoadInfo> assetsLoadInfo;
};
template <class T>

View file

@ -29,7 +29,10 @@ static debug::Logger logger("assets-loader");
AssetsLoader::AssetsLoader(
Engine& engine, Assets& assets, const ResPaths& paths
)
: engine(engine), assets(assets), paths(paths) {
: engine(engine),
assets(assets),
paths(paths),
assetsLoadInfo(assets.getLoadInfo()) {
addLoader(AssetType::ANIMATION, assetload::animation);
addLoader(AssetType::ATLAS, assetload::atlas);
addLoader(AssetType::FONT, assetload::font);
@ -53,11 +56,11 @@ void AssetsLoader::add(
std::shared_ptr<AssetCfg> settings,
bool overwrite
) {
if (!overwrite && enqueued.find({tag, alias}) != enqueued.end()){
if (!overwrite && enqueued.find({alias, tag}) != enqueued.end()){
return;
}
entries.push(aloader_entry {tag, filename, alias, std::move(settings)});
enqueued.insert({tag, alias});
enqueued.insert({alias, tag});
}
bool AssetsLoader::hasNext() const {
@ -75,7 +78,7 @@ aloader_func AssetsLoader::getLoader(AssetType tag) {
}
void AssetsLoader::loadNext() {
const aloader_entry& entry = entries.front();
aloader_entry& entry = entries.front();
logger.info() << "loading " << entry.filename << " as " << entry.alias;
std::string error {};
@ -96,6 +99,8 @@ void AssetsLoader::loadNext() {
entries.pop();
throw assetload::error(tag, std::move(filename), std::move(error));
}
assetsLoadInfo.processedEntries[{entry.alias, entry.tag}] =
std::move(entry);
entries.pop();
}
@ -120,14 +125,6 @@ static void add_layouts(
}
}
void AssetsLoader::tryAddSound(const std::string& name) {
if (name.empty()) {
return;
}
std::string file = SOUNDS_FOLDER + "/" + name;
add(AssetType::SOUND, file, name);
}
static std::string assets_def_folder(AssetType tag) {
switch (tag) {
case AssetType::ANIMATION:
@ -165,7 +162,8 @@ void AssetsLoader::processPreload(
}
std::shared_ptr<AssetCfg> config = nullptr;
map.at("path").get(path);
logger.debug() << "processing preload " << util::quote(name) << " path: " << util::quote(path);
logger.debug() << "processing preload " << util::quote(name)
<< " path: " << util::quote(path);
switch (tag) {
case AssetType::SOUND: {
bool keepPCM = false;
@ -269,65 +267,73 @@ static void add_variant(AssetsLoader& loader, const Variant& variant) {
}
}
void AssetsLoader::addDefaults(AssetsLoader& loader, const Content* content) {
loader.processPreloadConfigs(content);
if (content) {
for (auto& entry : content->getBlockMaterials()) {
auto& material = *entry.second;
loader.tryAddSound(material.stepsSound);
loader.tryAddSound(material.placeSound);
loader.tryAddSound(material.breakSound);
loader.tryAddSound(material.hitSound);
void AssetsLoader::addDefaults(const Content* content) {
processPreloadConfigs(content);
if (content == nullptr) {
return;
}
auto tryAddSound = [this](const std::string& name){
if (name.empty()) {
return;
}
std::string file = SOUNDS_FOLDER + "/" + name;
add(AssetType::SOUND, file, name);
};
for (auto& entry : content->getBlockMaterials()) {
auto& material = *entry.second;
tryAddSound(material.stepsSound);
tryAddSound(material.placeSound);
tryAddSound(material.breakSound);
tryAddSound(material.hitSound);
}
for (auto& entry : content->getPacks()) {
auto pack = entry.second.get();
auto& info = pack->getInfo();
io::path folder = info.folder / "layouts";
add_layouts(pack->getEnvironment(), info.id, folder, loader);
}
for (auto& entry : content->getPacks()) {
auto pack = entry.second.get();
auto& info = pack->getInfo();
io::path folder = info.folder / "layouts";
add_layouts(pack->getEnvironment(), info.id, folder, *this);
}
for (const auto& entry : content->getPacks()) {
io::path skeletonsDir = entry.first + ":skeletons";
if (!io::is_directory(skeletonsDir)) {
continue;
}
for (const auto& file : io::directory_iterator(skeletonsDir)) {
loader.add(
AssetType::SKELETON,
(file.parent() / file.stem()).string(),
entry.first + ":" + file.stem()
);
}
for (const auto& entry : content->getPacks()) {
io::path skeletonsDir = entry.first + ":skeletons";
if (!io::is_directory(skeletonsDir)) {
continue;
}
for (const auto& file : io::directory_iterator(skeletonsDir)) {
add(
AssetType::SKELETON,
(file.parent() / file.stem()).string(),
entry.first + ":" + file.stem()
);
}
}
for (const auto& [_, def] : content->blocks.getDefs()) {
if (def->variants) {
for (const auto& variant : def->variants->variants) {
add_variant(loader, variant);
}
} else {
add_variant(loader, def->defaults);
for (const auto& [_, def] : content->blocks.getDefs()) {
if (def->variants) {
for (const auto& variant : def->variants->variants) {
add_variant(*this, variant);
}
} else {
add_variant(*this, def->defaults);
}
for (const auto& [_, def] : content->items.getDefs()) {
if (def->modelName.find(':') == std::string::npos) {
loader.add(
AssetType::MODEL,
MODELS_FOLDER + "/" + def->modelName,
def->modelName
);
}
}
for (const auto& [_, def] : content->items.getDefs()) {
if (def->modelName.find(':') == std::string::npos) {
add(
AssetType::MODEL,
MODELS_FOLDER + "/" + def->modelName,
def->modelName
);
}
for (const auto& [_, def] : content->entities.getDefs()) {
if (def->skeletonName.find(':') == std::string::npos) {
// expecting a VCM with skeleton
loader.add(
AssetType::MODEL,
MODELS_FOLDER + "/" + def->skeletonName,
def->skeletonName
);
}
}
for (const auto& [_, def] : content->entities.getDefs()) {
if (def->skeletonName.find(':') == std::string::npos) {
// expecting a VCM with skeleton
add(
AssetType::MODEL,
MODELS_FOLDER + "/" + def->skeletonName,
def->skeletonName
);
}
}
}
@ -410,3 +416,22 @@ std::shared_ptr<Task> AssetsLoader::startTask(runnable onDone, int maxWorkers) {
}
return pool;
}
void AssetsLoader::attachToFile(const io::path& file, AssetFullId assetId) {
assetsLoadInfo.referencedAssets.insert({file, std::move(assetId)});
}
int AssetsLoader::addReload(const io::path& path) {
int added = 0;
auto range = assetsLoadInfo.referencedAssets.equal_range(path);
for (auto it = range.first; it != range.second; ++it) {
const auto& recipe = assetsLoadInfo.processedEntries.find(it->second);
if (recipe != assetsLoadInfo.processedEntries.end()) {
entries.push(recipe->second);
added++;
}
}
return added;
}

View file

@ -86,22 +86,14 @@ struct aloader_entry {
std::shared_ptr<AssetCfg> config;
};
using AssetFullId = std::pair<std::string, AssetType>;
struct AssetsLoadInfo {
std::map<AssetFullId, aloader_entry> processedEntries;
std::multimap<io::path, AssetFullId> referencedAssets;
};
class AssetsLoader {
Engine& engine;
Assets& assets;
std::map<AssetType, aloader_func> loaders;
std::queue<aloader_entry> entries;
std::set<std::pair<AssetType, std::string>> enqueued;
const ResPaths& paths;
void tryAddSound(const std::string& name);
void processPreload(
AssetType tag, const std::string& name, const dv::value& map
);
void processPreloadList(AssetType tag, const dv::value& list);
void processPreloadConfig(const io::path& file);
void processPreloadConfigs(const Content* content);
public:
AssetsLoader(Engine& engine, Assets& assets, const ResPaths& paths);
AssetsLoader(const AssetsLoader&) = delete;
@ -132,9 +124,8 @@ public:
aloader_func getLoader(AssetType tag);
/// @brief Enqueue core and content assets
/// @param loader target loader
/// @param content engine content
static void addDefaults(AssetsLoader& loader, const Content* content);
void addDefaults(const Content* content);
static bool loadExternalTexture(
AssetsLoader& loader,
@ -142,6 +133,25 @@ public:
const std::vector<io::path>& alternatives
);
int addReload(const io::path& path);
void attachToFile(const io::path& file, AssetFullId assetId);
Assets& getAssets();
Engine& getEngine();
private:
Engine& engine;
Assets& assets;
std::map<AssetType, aloader_func> loaders;
std::queue<aloader_entry> entries;
std::set<AssetFullId> enqueued;
const ResPaths& paths;
AssetsLoadInfo& assetsLoadInfo;
void processPreload(
AssetType tag, const std::string& name, const dv::value& map
);
void processPreloadList(AssetType tag, const dv::value& list);
void processPreloadConfig(const io::path& file);
void processPreloadConfigs(const Content* content);
};

View file

@ -29,7 +29,6 @@
#include <array>
#include <filesystem>
#include <iostream>
#include <stdexcept>
static debug::Logger logger("assetload-funcs");
@ -46,13 +45,15 @@ static bool load_animated_texture(
);
assetload::postfunc assetload::animation(
AssetsLoader&,
AssetsLoader& loader,
const ResPaths& paths,
const std::string& filename,
const std::string& name,
const std::shared_ptr<AssetCfg>&
) {
auto path = paths.find(filename + ".vca");
loader.attachToFile(path, {name, AssetType::ANIMATION});
if (io::is_regular_file(path)) {
scripting::load_vca_animation(path, name);
}
@ -60,20 +61,21 @@ assetload::postfunc assetload::animation(
}
assetload::postfunc assetload::texture(
AssetsLoader&,
AssetsLoader& loader,
const ResPaths& paths,
const std::string& filename,
const std::string& name,
const std::shared_ptr<AssetCfg>&
) {
auto actualFile = paths.find(filename + ".png");
auto file = paths.find(filename + ".png");
loader.attachToFile(file, {name, AssetType::TEXTURE});
try {
std::shared_ptr<ImageData> image(imageio::read(actualFile));
return [name, image, actualFile](auto& assets) {
std::shared_ptr<ImageData> image(imageio::read(file));
return [name, image, file](auto& assets) {
assets.store(Texture::from(image.get()), name);
};
} catch (const std::runtime_error& err) {
logger.error() << actualFile.string() << ": " << err.what();
logger.error() << file.string() << ": " << err.what();
return [](auto&) {};
}
}
@ -126,13 +128,15 @@ assetload::postfunc assetload::shader(
}
assetload::postfunc assetload::posteffect(
AssetsLoader&,
AssetsLoader& loader,
const ResPaths& paths,
const std::string& file,
const std::string& name,
const std::shared_ptr<AssetCfg>& settings
) {
io::path effectFile = paths.find(file + ".glsl");
loader.attachToFile(effectFile, {name, AssetType::POST_EFFECT});
std::string effectSource = io::read_string(effectFile);
auto& preprocessor = *Shader::preprocessor;
@ -219,7 +223,7 @@ assetload::postfunc assetload::font(
) {
auto cfg = std::dynamic_pointer_cast<FontCfg>(config);
auto ext = fs::path(filename).extension().string();
if (ext == ".ttf" || ext == ".otf") {
if (!ext.empty()) {
logger.info() << "loading vector font " << util::quote(filename);
return [=](Assets& assets) {
using FontFile = vector_fonts::FontFile;
@ -339,6 +343,128 @@ static void request_textures(AssetsLoader& loader, const model::Model& model) {
}
}
static assetload::postfunc load_vec3_model(
const io::path& file,
AssetsLoader& loader,
const std::string& name,
const std::shared_ptr<ModelCfg>& config
) {
loader.attachToFile(file, {name, AssetType::MODEL});
auto bytes = io::read_bytes_buffer(file);
auto modelVEC3 = std::make_shared<vec3::File>(vec3::load(file.string(), bytes));
return [&loader, name, file, config, modelVEC3 = std::move(modelVEC3)](
Assets& assets
) {
loader.attachToFile(file, {name, AssetType::MODEL});
if (config && config->squashed) {
model::Model fullModel;
for (auto& entry : modelVEC3->models) {
auto& vec3model = entry.second;
auto& model = vec3model.model;
model.translate(vec3model.origin);
fullModel.merge(std::move(model));
}
request_textures(loader, fullModel);
assets.store(
std::make_unique<model::Model>(fullModel),
name
);
logger.info() << "store model " << util::quote(name);
return;
}
for (auto& [modelName, model] : modelVEC3->models) {
request_textures(loader, model.model);
std::string fullName = name;
if (name != modelName) {
fullName += "." + modelName;
}
assets.store(
std::make_unique<model::Model>(model.model),
fullName
);
logger.info() << "store model " << util::quote(modelName)
<< " as " << util::quote(fullName);
}
};
}
static assetload::postfunc load_obj_model(
const io::path& file,
AssetsLoader& loader,
const std::string& name,
const std::shared_ptr<ModelCfg>&
) {
loader.attachToFile(file, {name, AssetType::MODEL});
auto text = io::read_string(file);
try {
auto model = obj::parse(file.string(), text).release();
return [=, &loader](Assets& assets) {
request_textures(loader, *model);
assets.store(std::unique_ptr<model::Model>(model), name);
};
} catch (const parsing_error& err) {
logger.error() << "error on loading model " << name << ": "
<< err.errorLog();
throw;
}
}
static assetload::postfunc load_vcm_model(
const io::path& file,
AssetsLoader& loader,
const std::string& name,
const std::shared_ptr<ModelCfg>& cfg
) {
loader.attachToFile(file, {name, AssetType::MODEL});
auto text = io::read_string(file);
try {
auto vcmModel = vcm::parse(file.string(), text, file.extension() == ".xml");
assert(vcmModel.parts.size() > 0);
if (vcmModel.parts.size() == 1 || (cfg && cfg->squashed)) {
auto modelPtr =
std::make_unique<model::Model>(std::move(vcmModel.squash()))
.release();
return [=, &loader](Assets& assets) {
auto model = std::unique_ptr<model::Model>(modelPtr);
request_textures(loader, *model);
assets.store(std::move(model), name);
logger.info() << "store model " << util::quote(name);
};
}
auto vcmModelPtr =
std::make_unique<vcm::VcmModel>(std::move(vcmModel)).release();
return [=](Assets& assets) {
auto vcmModel = std::unique_ptr<vcm::VcmModel>(vcmModelPtr);
for (auto& [partName, model] : vcmModel->parts) {
auto fullName = name + "." + partName;
logger.info()
<< "store model part " << util::quote(partName)
<< " as " << util::quote(fullName);
assets.store(
std::make_unique<model::Model>(std::move(model)),
fullName
);
}
for (auto& bone : vcmModel->skeleton->getBones()) {
bone->setModel(name + "." + bone->model.name);
}
logger.info() << "store skeleton " << util::quote(name);
assets.store<rigging::SkeletonConfig>(
std::make_unique<rigging::SkeletonConfig>(
std::move(*vcmModel->skeleton)
),
name
);
};
} catch (const parsing_error& err) {
logger.error() << "error on loading model " << name << ": "
<< err.errorLog();
throw;
}
}
assetload::postfunc assetload::model(
AssetsLoader& loader,
const ResPaths& paths,
@ -346,59 +472,15 @@ assetload::postfunc assetload::model(
const std::string& name,
const std::shared_ptr<AssetCfg>& config
) {
auto cfg = std::dynamic_pointer_cast<ModelCfg>(config);
auto modelConfig = std::dynamic_pointer_cast<ModelCfg>(config);
auto path = paths.find(file + ".vec3");
if (io::exists(path)) {
auto bytes = io::read_bytes_buffer(path);
auto modelVEC3 = std::make_shared<vec3::File>(vec3::load(path.string(), bytes));
return [&loader, name, cfg, modelVEC3 = std::move(modelVEC3)](
Assets& assets
) {
if (cfg && cfg->squashed) {
model::Model fullModel;
for (auto& entry : modelVEC3->models) {
auto& vec3model = entry.second;
auto& model = vec3model.model;
model.translate(vec3model.origin);
fullModel.merge(std::move(model));
}
request_textures(loader, fullModel);
assets.store(
std::make_unique<model::Model>(fullModel),
name
);
logger.info() << "store model " << util::quote(name);
return;
}
for (auto& [modelName, model] : modelVEC3->models) {
request_textures(loader, model.model);
std::string fullName = name;
if (name != modelName) {
fullName += "." + modelName;
}
assets.store(
std::make_unique<model::Model>(model.model),
fullName
);
logger.info() << "store model " << util::quote(modelName)
<< " as " << util::quote(fullName);
}
};
return load_vec3_model(path, loader, name, modelConfig);
}
path = paths.find(file + ".obj");
if (io::exists(path)) {
auto text = io::read_string(path);
try {
auto model = obj::parse(path.string(), text).release();
return [=, &loader](Assets& assets) {
request_textures(loader, *model);
assets.store(std::unique_ptr<model::Model>(model), name);
};
} catch (const parsing_error& err) {
std::cerr << err.errorLog() << std::endl;
throw;
}
return load_obj_model(path, loader, name, modelConfig);
}
std::array<std::string, 2> extensions {
@ -418,50 +500,7 @@ assetload::postfunc assetload::model(
throw std::runtime_error("could not to find model " + util::quote(file));
}
auto text = io::read_string(path);
try {
auto vcmModel = vcm::parse(path.string(), text, path.extension() == ".xml");
assert(vcmModel.parts.size() > 0);
if (vcmModel.parts.size() == 1 || (cfg && cfg->squashed)) {
auto modelPtr = std::make_unique<model::Model>(std::move(vcmModel.squash())).release();
return [=, &loader](Assets& assets) {
auto model = std::unique_ptr<model::Model>(modelPtr);
request_textures(loader, *model);
assets.store(std::move(model), name);
logger.info() << "store model " << util::quote(name);
};
} else {
auto vcmModelPtr = std::make_unique<vcm::VcmModel>(std::move(vcmModel)).release();
return [=](Assets& assets) {
auto vcmModel = std::unique_ptr<vcm::VcmModel>(vcmModelPtr);
for (auto& [partName, model] : vcmModel->parts) {
auto fullName = name + "." + partName;
logger.info()
<< "store model part " << util::quote(partName)
<< " as " << util::quote(fullName);
assets.store(
std::make_unique<model::Model>(std::move(model)),
fullName
);
}
for (auto& bone : vcmModel->skeleton->getBones()) {
bone->setModel(name + "." + bone->model.name);
}
logger.info() << "store skeleton " << util::quote(name);
assets.store<rigging::SkeletonConfig>(
std::make_unique<rigging::SkeletonConfig>(
std::move(*vcmModel->skeleton)
),
name
);
};
}
} catch (const parsing_error& err) {
std::cerr << err.errorLog() << std::endl;
throw;
}
return load_vcm_model(path, loader, name, modelConfig);
}
assetload::postfunc assetload::skeleton(
@ -593,8 +632,7 @@ static bool load_animated_texture(
read_anim_file(animFile, frameList);
}
for (const auto& file : paths.listdir(animsDir + "/" + name)) {
if (!frameList.empty() &&
!contains(frameList, file.stem())) {
if (!frameList.empty() && !contains(frameList, file.stem())) {
continue;
}
if (!append_atlas(builder, file)) continue;

View file

@ -1,9 +1,13 @@
#include "DebuggingServer.hpp"
#include "engine/Engine.hpp"
#include "network/Network.hpp"
#include "debug/Logger.hpp"
#include "assets/AssetsLoader.hpp"
#include "coders/json.hpp"
#include "data/dv.hpp"
#include "debug/Logger.hpp"
#include "engine/AssetsManagement.hpp"
#include "engine/Engine.hpp"
#include "io/path.hpp"
#include "network/Network.hpp"
using namespace devtools;
@ -260,6 +264,16 @@ bool DebuggingServer::performCommand(
}
});
return true;
} else if (type == "hot-reload") {
if (!map.has("file")) {
return false;
}
logger.info() << "hot-reload: " << map["file"].asString();
int count = engine.getAssetsManagement().acquireBackgroundLoader().addReload(
map["file"].asString()
);
logger.info() << "enqueued to reload: " << count;
return false;
} else {
logger.error() << "unsupported command '" << type << "'";
}

View file

@ -53,11 +53,11 @@ void AssetsManagement::loadAssets(Content* content) {
logger.info() << "loading assets";
Shader::preprocessor->setPaths(&paths.resPaths);
auto new_assets = std::make_unique<Assets>(
auto newAssets = std::make_unique<Assets>(
settings.system.preserveAssetsDuringFrame.get() ? &assetsVault : nullptr
);
AssetsLoader loader(engine, *new_assets, paths.resPaths);
AssetsLoader::addDefaults(loader, content);
AssetsLoader loader(engine, *newAssets, paths.resPaths);
loader.addDefaults(content);
// no need
// correct log messages order is more useful
@ -73,11 +73,10 @@ void AssetsManagement::loadAssets(Content* content) {
loader.loadNext();
}
}
assets = std::move(new_assets);
assets = std::move(newAssets);
if (content) {
ModelsGenerator::prepare(*content, *assets);
}
assets->setup();
engine.getGUI().onAssetsLoad(assets.get());
}

View file

@ -190,5 +190,9 @@ public:
return *appScripts;
}
AssetsManagement& getAssetsManagement() const {
return *assets;
}
void detachDebugger();
};