Merge pull request #940 from MihailRis/fix-assets-pipeline

update assets pipeline (moving to animation branch)

doc/en/scripting/builtins/libapp.md

@@ -9,21 +9,22 @@ local filename = "script:"..app.script..".lua"
 
 Since the control script may not belong to any of the packs, it does not belongs to its own package and has its own global namespace in which all global functions and tables are available, as well as the `app` library.
 
-## Functions
+
+## Common functions
 
 ```lua
+-- Performs one tick of the main engine loop.
 app.tick()
-```
 
-Performs one tick of the main engine loop.
+-- Waits for the specified time in seconds, performing the main engine loop.
-
-```lua
 app.sleep(time: number)
-```
 
-Waits for the specified time in seconds, performing the main engine loop.
+-- Terminates the engine, printing the call stack
+-- to trace the function call location.
+app.quit()
 
-```lua
+-- Waits for the condition checked by the function to be true,
+-- performing the main engine loop.
 app.sleep_until(
     -- function that checks the condition for ending the wait
     predicate: function() -> bool,
@@ -36,51 +37,54 @@ app.sleep_until(
 )
 ```
 
-Waits for the condition checked by the function to be true, performing the main engine loop.
+## Content
 
 ```lua
-app.quit()
+-- Checks if content is loaded.
-```
+app.is_content_loaded() -> bool
 
-Terminates the engine, printing the call stack to trace the function call location.
+-- Returns the current content configuration
+-- (list of packs IDs in loading order)
+app.get_content() -> table<string>
 
-```lua
+-- Loads content based on the current configuration.
+-- Cannot be used if content is already loaded (see app.reset_content).
+app.load_content()
+
+-- Unloads all content, resetting it to a single core pack.
+app.reset_content(
+    -- Packs for which modules, events, and environment will not be reset
+    [optional] non_reset_packs: table
+)
+
+-- Updates the packs configuration, checking its correctness
+-- (dependencies and availability of packs).
+-- Automatically adds and reorders packs based on dependencies.
 app.reconfig_packs(
     -- packs to add
     add_packs: table,
     -- packs to remove
     remove_packs: table
 )
-```
 
-Updates the packs configuration, checking its correctness (dependencies and availability of packs).
+-- To remove all packs from the configuration,you can use `pack.get_installed()`:
-Automatically adds dependencies.
-
-To remove all packs from the configuration, you can use `pack.get_installed()`:
-
-```lua
 app.reconfig_packs({}, pack.get_installed())
-```
+-- In this case, `base` will also be removed from the configuration.
 
-In this case, `base` will also be removed from the configuration.
+-- Updates the packs configuration, automatically removing unspecified ones,
-
+-- adding those missing in the previous configuration.
-```lua
+-- Uses app.reconfig_packs.
 app.config_packs(
     -- expected set of packs (excluding dependencies)
     packs: table
 )
 ```
 
-Updates the packs configuration, automatically removing unspecified ones, adding those missing in the previous configuration.
+## Worlds
-Uses app.reconfig_packs.
+
-
-```lua
-app.is_content_loaded() -> bool
-```
-
-Checks if content is loaded.
 
 ```lua
+-- Creates a new world and opens it.
 app.new_world(
     -- world name, empty string will create a nameless world
     name: str,
@@ -91,62 +95,42 @@ app.new_world(
     -- local player id
     [optional] local_player: int=0
 )
-```
 
-Creates a new world and opens it.
+-- Deletes a world by name.
+app.delete_world(name: str)
 
-```lua
+-- Opens a world by name.
 app.open_world(name: str)
-```
 
-Opens a world by name.
+-- Reopens the world.
-
-```lua
 app.reopen_world()
-```
 
-Reopens the world.
+-- Saves the world.
-
-```lua
 app.save_world()
-```
 
-Saves the world.
+-- Closes the world.
-
-```lua
 app.close_world(
     -- save the world before closing
     [optional] save_world: bool=false
 )
 ```
 
-Closes the world.
+## Engine settings and information
-
-```lua
-app.delete_world(name: str)
-```
-
-Deletes a world by name.
 
 ```lua
+-- Returns the major and minor versions of the engine.
 app.get_version() -> int, int
-```
 
-Returns the major and minor versions of the engine.
+-- Returns the value of a setting.
-
+-- Throws an exception if the setting does not exist.
-```lua
 app.get_setting(name: str) -> value
-```
 
-Returns the value of a setting. Throws an exception if the setting does not exist.
+-- Sets the value of a setting.
-
+-- Throws an exception if the setting does not exist.
-```lua
 app.set_setting(name: str, value: value)
-```
 
-Sets the value of a setting. Throws an exception if the setting does not exist.
+-- Returns a table with information about a setting.
-
+-- Throws an exception if the setting does not exist.
-```lua
 app.get_setting_info(name: str) -> {
     -- default value
     def: value,
@@ -155,43 +139,30 @@ app.get_setting_info(name: str) -> {
     -- maximum value
     [only for numeric settings] max: number
 }
-```
 
-Returns a table with information about a setting. Throws an exception if the setting does not exist.
+-- Brings the window to front and sets input focus.
-
-```lua
 app.focus()
 ```
 
-Brings the window to front and sets input focus.
+## Paths and entry points
 
 ```lua
+-- Creates an in-memory filesystem.
 app.create_memory_device(
     -- entry-point name
     name: str
 )
-```
 
-Creates an in-memory filesystem.
+-- Returns a list of content sources (paths), in descending priority order.
-
-```lua
 app.get_content_sources() -> table<string>
-```
 
-Returns a list of content sources (paths), in descending priority order.
+-- Sets a list of content sources (paths). Specified in descending priority order.
-
-```lua
 app.set_content_sources(sources: table<string>)
-```
 
-Sets a list of content sources (paths). Specified in descending priority order.
+-- Resets content sources.
-
-```lua
 app.reset_content_sources()
 ```
 
-Resets content sources.
-
 ## Sub-instances
 
 ```lua
@@ -209,5 +180,5 @@ app.is_instance_alive(handle: int) -> boolean
 
 -- Stops the engine sub-instance.
 -- Returns true if the instance was alive at the time of the call.
-app.terminate(handle: int) -> boolean
+app.terminate_instance(handle: int) -> boolean
 ```

doc/ru/scripting/builtins/libapp.md

@@ -28,7 +28,8 @@ app.sleep(time: number)
 -- Завершает выполнение движка, выводя стек вызовов для ослеживания места вызова функции.
 app.quit()
 
--- Ожидает истинности утверждения (условия), проверяемого функцией, выполняя основной цикл движка.
+-- Ожидает истинности утверждения (условия), проверяемого функцией,
+-- выполняя основной цикл движка.
 app.sleep_until(
     -- функция, проверяющее условия завершения ожидания
     predicate: function() -> boolean,
@@ -46,7 +47,11 @@ app.sleep_until(
 -- Проверяет, загружен ли контент.
 app.is_content_loaded() -> boolean
 
--- Загружает контент из конфига, нельзя использовать, если контент уже загружен
+-- Возвращает текущую конфигурацию контента (список id паков в порядке загрузки)
+app.get_content() -> table<string>
+
+-- Загружает контент на основе текущей конфигурации.
+-- Не может быть использован, если контент уже загружен (см. app.reset_content).
 app.load_content()
 
 -- Выгружает весь контент, сбрасывая до единственного пака ядра (`core`).
@@ -56,7 +61,7 @@ app.reset_content(
 )
 
 -- Обновляет конфигурацию паков, проверяя её корректность (зависимости и доступность паков). 
--- Автоматически добавляет зависимости.
+-- Автоматически добавляет и упорядочивает паки в соответствии с зависимостями.
 -- Для удаления ВСЕХ паков из конфигурации можно использовать `pack.get_installed()`
 app.reconfig_packs(
     -- добавляемые паки
@@ -107,19 +112,22 @@ app.close_world(
 )
 ```
 
-
 ## Свойства и настройки движка
+
 ```lua
 -- Возвращает мажорную и минорную версии движка.
 app.get_version() -> int, int
 
--- Возвращает значение настройки. Бросает исключение, если настройки не существует.
+-- Возвращает значение настройки.
+-- Бросает исключение, если настройки не существует.
 app.get_setting(name: string) -> any
 
--- Устанавливает значение настройки. Бросает исключение, если настройки не существует.
+-- Устанавливает значение настройки.
+-- Бросает исключение, если настройки не существует.
 app.set_setting(name: string, value: any)
 
--- Возвращает таблицу с информацией о настройке. Бросает исключение, если настройки не существует.
+-- Возвращает таблицу с информацией о настройке.
+-- Бросает исключение, если настройки не существует.
 app.get_setting_info(name: string) -> table
 
 -- Переводит окно на передний план и устанавливает фокус ввода.
@@ -127,6 +135,7 @@ app.focus()
 ```
 
 ## Точки входа и пути
+
 ```lua
 -- Создаёт файловую систему в памяти.
 app.create_memory_device(

res/layouts/pages/settings_audio.xml.lua

@@ -31,6 +31,14 @@ function update_checkbox_setting(id, value)
     document.input_device_select.enabled = recording_enabled
     document.input_volume_inner.enabled = recording_enabled
     document.input_volume_outer.enabled = recording_enabled
+
+    local selectbox = document.input_device_select
+    local info = audio.input.get_input_info()
+    if info then
+        selectbox.value = info.device_specifier
+    else
+        selectbox.value = app.get_setting("audio.input-device")
+    end
 end
 
 function create_checkbox(id, name, tooltip)
@@ -84,6 +92,7 @@ function on_open()
         table.insert(devices, {value=name, text=name})
     end
     selectbox.options = devices
+
     local info = audio.input.get_input_info()
     if info then
         selectbox.value = info.device_specifier

res/scripts/post_content.lua

@@ -1,3 +1,6 @@
+local app = __vc_app
+local internals = __vc_internals
+
 local user_props = file.read_combined_object("config/user-props.toml")
 local names = {
     "parent", "caption", "texture", "texture-faces", "model", "model-name",
@@ -85,3 +88,33 @@ cache_names(block)
 cache_names(item)
 
 __vc_scripts_registry.build_registry()
+
+local packs = app.get_content()
+table.insert(packs, 1, "core")
+
+for i, packid in ipairs(packs) do
+    local loaders_dir = packid..":scripts/loaders"
+    if not file.isdir(loaders_dir) then
+        goto continue
+    end
+    local files = file.list(loaders_dir)
+    for _, path in ipairs(files) do
+        if file.ext(path) == "json" then
+            debug.log("registering loader "..path)
+            local dto = json.parse(file.read(path))
+            local module = setmetatable({}, {__index=internals.get_pack_env(packid)})
+            local module_path = file.join(file.parent(path), file.stem(path)..".lua")
+            if not file.exists(module_path) then
+                error("loader "..path.." missing module "..module_path)
+            end
+            load(file.read(module_path), module_path, "t", module)()
+            if rawget(module, "execute") == nil then
+                error("loader "..path.." missing function 'execute'")
+            end
+            if dto.type == "compiler" then
+                internals.register_compiler(packid, dto.extensions, module)
+            end
+        end
+    end
+    ::continue::
+end

res/scripts/stdlib.lua

@@ -392,6 +392,7 @@ end
 
 ffi = nil
 __vc_app = nil
+__vc_internals = nil
 __vc_lock_internal_modules()
 __vc_lock_internal_modules = nil
 __vc_update_coroutines = nil

res/scripts/stdmin.lua

@@ -154,6 +154,20 @@ package = {
 }
 local __cached_scripts = {}
 local __warnings_hidden = {}
+local __compilers = {}
+
+function __vc_internals.register_compiler(sourcepack, extensions, module)
+    local compiler = {
+        packid = sourcepack,
+        extensions = extensions,
+        module = module,
+    }
+    for i, ext in ipairs(extensions) do
+        if not __compilers[ext] then
+            __compilers[ext] = compiler
+        end
+    end
+end
 
 function on_deprecated_call(name, alternatives)
     if __warnings_hidden[name] then
@@ -200,6 +214,21 @@ end
 
 local __internal_locked = false
 
+local default_compiler = {
+    module = {
+        execute = function(code, path, env)
+            local script, err = load(code, path)
+            if script == nil then
+                error(err)
+            end
+            if env then
+                script = setfenv(script, env)
+            end
+            return script
+        end
+    }
+}
+
 -- Load script with caching
 --
 -- path - script path `contentpack:filename`. 
@@ -211,6 +240,9 @@ function __load_script(path, nocache, env)
     local is_internal = (packname == "res" or packname == "core")
        and filename:find("modules/internal") == 1
 
+    local ext = path:match("%.([^:/\\]+)$")
+    local compiler = __compilers[ext] or default_compiler
+
     nocache = nocache or is_internal
 
     if is_internal and __internal_locked then
@@ -225,13 +257,7 @@ function __load_script(path, nocache, env)
         error("script '"..filename.."' not found in '"..packname.."'")
     end
 
-    local script, err = load(file.read(path), path)
+    local script = compiler.module.execute(file.read(path), path, env)
-    if script == nil then
-        error(err)
-    end
-    if env then
-        script = setfenv(script, env)
-    end
     local result = script()
     if not nocache then
         __cached_scripts[path] = script
@@ -245,6 +271,9 @@ function __vc_lock_internal_modules()
 end
 
 local __pack_envs = __vc__pack_envs
+function __vc_internals.get_pack_env(packid)
+    return __pack_envs[packid]
+end
 __vc__pack_envs = nil
 
 function require(path)
@@ -277,6 +306,13 @@ function __scripts_cleanup(non_reset_packs)
         __pack_envs[packname] = nil
         ::continue::
     end
+    for ext, compiler in pairs(__compilers) do
+        if table.has(non_reset_packs, compiler.packid) then
+            goto continue
+        end
+        __compilers[ext] = nil
+        ::continue::
+    end
 end
 
 function __vc__error(msg, frame, n, lastn)

src/assets/AssetsLoader.cpp

@@ -47,9 +47,10 @@ void AssetsLoader::add(
     AssetType tag,
     const std::string& filename,
     const std::string& alias,
-    std::shared_ptr<AssetCfg> settings
+    std::shared_ptr<AssetCfg> settings,
+    bool overwrite
 ) {
-    if (enqueued.find({tag, alias}) != enqueued.end()){
+    if (!overwrite && enqueued.find({tag, alias}) != enqueued.end()){
         return;
     }
     entries.push(aloader_entry {tag, filename, alias, std::move(settings)});
@@ -154,11 +155,12 @@ void AssetsLoader::processPreload(
     std::string defFolder = assets_def_folder(tag);
     std::string path = defFolder + "/" + name;
     if (map == nullptr) {
-        add(tag, path, name);
+        add(tag, path, name, nullptr, true);
         return;
     }
     std::shared_ptr<AssetCfg> config = nullptr;
     map.at("path").get(path);
+    logger.debug() << "processing preload " << util::quote(name) << " path: " << util::quote(path);
     switch (tag) {
         case AssetType::SOUND: {
             bool keepPCM = false;
@@ -196,7 +198,7 @@ void AssetsLoader::processPreload(
         default:
             break;
     }
-    add(tag, path, name, std::move(config));
+    add(tag, path, name, std::move(config), true);
 }
 
 void AssetsLoader::processPreloadList(AssetType tag, const dv::value& list) {

src/assets/AssetsLoader.hpp

@@ -110,7 +110,8 @@ public:
         AssetType tag,
         const std::string& filename,
         const std::string& alias,
-        std::shared_ptr<AssetCfg> settings = nullptr
+        std::shared_ptr<AssetCfg> settings = nullptr,
+        bool overwrite = false
     );
 
     bool hasNext() const;

src/audio/AL/ALAudio.cpp

@@ -5,6 +5,7 @@
 
 #include "debug/Logger.hpp"
 #include "alutil.hpp"
+#include "util/stringutil.hpp"
 #include "../MemoryPCMStream.hpp"
 
 static debug::Logger logger("al-audio");
@@ -99,6 +100,8 @@ ALInputDevice::ALInputDevice(
 
     if (deviceName) {
         deviceSpecifier = std::string(deviceName);
+        logger.info() << "created input device with specifier "
+                      << util::quote(deviceSpecifier);
     } else {
         logger.warning() << "could not retrieve input device specifier";
     }

src/audio/audio.cpp

@@ -211,12 +211,15 @@ void audio::initialize(
             return;
         }
         if (enabled) {
-            logger.info() << "recording enabled; input device is "
+            const auto& targetDevice = settings.inputDevice.get();
-                          << settings.inputDevice.get();
+            logger.info() << "recording enabled; target input device is "
+                          << (targetDevice.empty() ? "not specified"
+                                                   : targetDevice);
             ::input_device = backend->openInputDevice(
                 settings.inputDevice.get(), 44100, 1, 16
             );
             if (::input_device != nullptr) {
+                logger.error() << "could not create input device";
                 ::input_device->startCapture();
             }
         } else {

src/logic/scripting/lua/libs/libapp.cpp

@@ -92,6 +92,17 @@ static int l_reconfig_packs(lua::State* L) {
     return 0;
 }
 
+/// @brief Get current content packs configuration
+static int l_get_content(lua::State* L) {
+    const auto& configuration = content_control->getContentPacks();
+    lua::createtable(L, configuration.size(), 0);
+    for (int i = 0; i < configuration.size(); i++) {
+        lua::pushlstring(L, configuration[i].id);
+        lua::rawseti(L, i + 1);
+    }
+    return 1;
+}
+
 /// @brief Get content sources list
 static int l_get_content_sources(lua::State* L) {
     const auto& sources = engine->getContentControl().getContentSources();
@@ -453,6 +464,7 @@ const luaL_Reg applib[] = {
     {"load_content", lua::wrap<l_load_content>},
     {"reset_content", lua::wrap<l_reset_content>},
     {"reconfig_packs", lua::wrap<l_reconfig_packs>},
+    {"get_content", lua::wrap<l_get_content>},
     {"get_content_sources", lua::wrap<l_get_content_sources>},
     {"set_content_sources", lua::wrap<l_set_content_sources>},
     {"reset_content_sources", lua::wrap<l_reset_content_sources>},

src/logic/scripting/lua/lua_engine.cpp

@@ -74,6 +74,11 @@ static void create_libs(State* L, StateType stateType) {
     getglobal(L, "__vc_app");
     setregistry(L, "app");
 
+    createtable(L, 0, 0);
+    pushvalue(L, -1);
+    setglobal(L, "__vc_internals");
+    setregistry(L, "internals");
+
     if (stateType == StateType::SCRIPT) {
         getregistry(L, "app");
         setglobal(L, "app");

src/logic/scripting/scripting.cpp

@@ -288,7 +288,23 @@ void scripting::on_content_load(Content* content) {
         lua::setfield(L, "properties");
         lua::pop(L);
     }
+
+    lua::getregistry(L, "app");
+    lua::setglobal(L, "__vc_app");
+    lua::getregistry(L, "internals");
+    lua::setglobal(L, "__vc_internals");
+    try {
         load_script("post_content.lua", true);
+    } catch (const std::exception&) {
+        lua::pushnil(L);
+        lua::setglobal(L, "__vc_app");
+        throw;
+    }
+    lua::pushnil(L);
+    lua::setglobal(L, "__vc_app");
+    lua::pushnil(L);
+    lua::setglobal(L, "__vc_internals");
+
     load_script("stdcmd.lua", true);
 }
 

src/settings.hpp

@@ -18,7 +18,7 @@ struct AudioSettings {
     NumberSetting volumeAmbient {1.0f, 0.0f, 1.0f, setting_format::percent};
     NumberSetting volumeMusic {1.0f, 0.0f, 1.0f, setting_format::percent};
 
-    StringSetting inputDevice {"auto"};
+    StringSetting inputDevice {""};
 
     FlagSetting acousticEffects {true};
 };