--- /dev/null
+core.log("info", "Initializing asynchronous environment (game)")
+
+local function pack2(...)
+ return {n=select('#', ...), ...}
+end
+
+-- Entrypoint to run async jobs, called by C++
+function core.job_processor(func, params)
+ local retval = pack2(func(unpack(params, 1, params.n)))
+
+ return retval
+end
+
+-- Import a bunch of individual files from builtin/game/
+local gamepath = core.get_builtin_path() .. "game" .. DIR_DELIM
+
+dofile(gamepath .. "constants.lua")
+dofile(gamepath .. "item_s.lua")
+dofile(gamepath .. "misc_s.lua")
+dofile(gamepath .. "features.lua")
+dofile(gamepath .. "voxelarea.lua")
+
+-- Transfer of globals
+do
+ assert(core.transferred_globals)
+ local all = core.deserialize(core.transferred_globals, true)
+ core.transferred_globals = nil
+
+ -- reassemble other tables
+ all.registered_nodes = {}
+ all.registered_craftitems = {}
+ all.registered_tools = {}
+ for k, v in pairs(all.registered_items) do
+ if v.type == "node" then
+ all.registered_nodes[k] = v
+ elseif v.type == "craftitem" then
+ all.registered_craftitems[k] = v
+ elseif v.type == "tool" then
+ all.registered_tools[k] = v
+ end
+ end
+
+ for k, v in pairs(all) do
+ core[k] = v
+ end
+end
+++ /dev/null
-
-core.log("info", "Initializing Asynchronous environment")
-
-function core.job_processor(func, serialized_param)
- local param = core.deserialize(serialized_param)
-
- local retval = core.serialize(func(param))
-
- return retval or core.serialize(nil)
-end
-
--- /dev/null
+core.log("info", "Initializing asynchronous environment")
+
+function core.job_processor(func, serialized_param)
+ local param = core.deserialize(serialized_param)
+
+ local retval = core.serialize(func(param))
+
+ return retval or core.serialize(nil)
+end
--- /dev/null
+
+core.async_jobs = {}
+
+function core.async_event_handler(jobid, retval)
+ local callback = core.async_jobs[jobid]
+ assert(type(callback) == "function")
+ callback(unpack(retval, 1, retval.n))
+ core.async_jobs[jobid] = nil
+end
+
+function core.handle_async(func, callback, ...)
+ assert(type(func) == "function" and type(callback) == "function",
+ "Invalid minetest.handle_async invocation")
+ local args = {n = select("#", ...), ...}
+ local mod_origin = core.get_last_run_mod()
+
+ local jobid = core.do_async_callback(func, args, mod_origin)
+ core.async_jobs[jobid] = callback
+
+ return true
+end
+
dofile(gamepath .. "forceloading.lua")
dofile(gamepath .. "statbars.lua")
dofile(gamepath .. "knockback.lua")
+dofile(gamepath .. "async.lua")
profiler = nil
-- Used for callback handling with dynamic_add_media
core.dynamic_media_callbacks = {}
+
+
+-- Transfer of certain globals into async environment
+-- see builtin/async/game.lua for the other side
+
+local function copy_filtering(t, seen)
+ if type(t) == "userdata" or type(t) == "function" then
+ return true -- don't use nil so presence can still be detected
+ elseif type(t) ~= "table" then
+ return t
+ end
+ local n = {}
+ seen = seen or {}
+ seen[t] = n
+ for k, v in pairs(t) do
+ local k_ = seen[k] or copy_filtering(k, seen)
+ local v_ = seen[v] or copy_filtering(v, seen)
+ n[k_] = v_
+ end
+ return n
+end
+
+function core.get_globals_to_transfer()
+ local all = {
+ registered_items = copy_filtering(core.registered_items),
+ registered_aliases = core.registered_aliases,
+ }
+ return core.serialize(all)
+end
if not custom_loaded then
dofile(core.get_mainmenu_path() .. DIR_DELIM .. "init.lua")
end
-elseif INIT == "async" then
- dofile(asyncpath .. "init.lua")
+elseif INIT == "async" then
+ dofile(asyncpath .. "mainmenu.lua")
+elseif INIT == "async_game" then
+ dofile(asyncpath .. "game.lua")
elseif INIT == "client" then
dofile(clientpath .. "init.lua")
else
* `job:cancel()`
* Cancels the job function from being called
+Async environment
+-----------------
+
+The engine allows you to submit jobs to be ran in an isolated environment
+concurrently with normal server operation.
+A job consists of a function to be ran in the async environment, any amount of
+arguments (will be serialized) and a callback that will be called with the return
+value of the job function once it is finished.
+
+The async environment does *not* have access to the map, entities, players or any
+globals defined in the 'usual' environment. Consequently, functions like
+`minetest.get_node()` or `minetest.get_player_by_name()` simply do not exist in it.
+
+Arguments and return values passed through this can contain certain userdata
+objects that will be seamlessly copied (not shared) to the async environment.
+This allows you easy interoperability for delegating work to jobs.
+
+* `minetest.handle_async(func, callback, ...)`:
+ * Queue the function `func` to be ran in an async environment.
+ Note that there are multiple persistent workers and any of them may
+ end up running a given job. The engine will scale the amount of
+ worker threads automatically.
+ * When `func` returns the callback is called (in the normal environment)
+ with all of the return values as arguments.
+ * Optional: Variable number of arguments that are passed to `func`
+* `minetest.register_async_dofile(path)`:
+ * Register a path to a Lua file to be imported when an async environment
+ is initialized. You can use this to preload code which you can then call
+ later using `minetest.handle_async()`.
+
+### List of APIs available in an async environment
+
+Classes:
+* `ItemStack`
+* `PerlinNoise`
+* `PerlinNoiseMap`
+* `PseudoRandom`
+* `PcgRandom`
+* `SecureRandom`
+* `VoxelArea`
+* `VoxelManip`
+ * only if transferred into environment; can't read/write to map
+* `Settings`
+
+Class instances that can be transferred between environments:
+* `ItemStack`
+* `PerlinNoise`
+* `PerlinNoiseMap`
+* `VoxelManip`
+
+Functions:
+* Standalone helpers such as logging, filesystem, encoding,
+ hashing or compression APIs
+* `minetest.request_insecure_environment` (same restrictions apply)
+
+Variables:
+* `minetest.settings`
+* `minetest.registered_items`, `registered_nodes`, `registered_tools`,
+ `registered_craftitems` and `registered_aliases`
+ * with all functions and userdata values replaced by `true`, calling any
+ callbacks here is obviously not possible
+
Server
------
--- /dev/null
+-- helper
+
+core.register_async_dofile(core.get_modpath(core.get_current_modname()) ..
+ DIR_DELIM .. "inside_async_env.lua")
+
+local function deepequal(a, b)
+ if type(a) == "function" then
+ return type(b) == "function"
+ elseif type(a) ~= "table" then
+ return a == b
+ elseif type(b) ~= "table" then
+ return false
+ end
+ for k, v in pairs(a) do
+ if not deepequal(v, b[k]) then
+ return false
+ end
+ end
+ for k, v in pairs(b) do
+ if not deepequal(a[k], v) then
+ return false
+ end
+ end
+ return true
+end
+
+-- Object Passing / Serialization
+
+local test_object = {
+ name = "stairs:stair_glass",
+ type = "node",
+ groups = {oddly_breakable_by_hand = 3, cracky = 3, stair = 1},
+ description = "Glass Stair",
+ sounds = {
+ dig = {name = "default_glass_footstep", gain = 0.5},
+ footstep = {name = "default_glass_footstep", gain = 0.3},
+ dug = {name = "default_break_glass", gain = 1}
+ },
+ node_box = {
+ fixed = {
+ {-0.5, -0.5, -0.5, 0.5, 0, 0.5},
+ {-0.5, 0, 0, 0.5, 0.5, 0.5}
+ },
+ type = "fixed"
+ },
+ tiles = {
+ {name = "stairs_glass_split.png", backface_culling = true},
+ {name = "default_glass.png", backface_culling = true},
+ {name = "stairs_glass_stairside.png^[transformFX", backface_culling = true}
+ },
+ on_place = function(itemstack, placer)
+ return core.is_player(placer)
+ end,
+ sunlight_propagates = true,
+ is_ground_content = false,
+ light_source = 0,
+}
+
+local function test_object_passing()
+ local tmp = core.serialize_roundtrip(test_object)
+ assert(deepequal(test_object, tmp))
+
+ -- Circular key, should error
+ tmp = {"foo", "bar"}
+ tmp[tmp] = true
+ assert(not pcall(core.serialize_roundtrip, tmp))
+
+ -- Circular value, should error
+ tmp = {"foo"}
+ tmp[2] = tmp
+ assert(not pcall(core.serialize_roundtrip, tmp))
+end
+unittests.register("test_object_passing", test_object_passing)
+
+local function test_userdata_passing(_, pos)
+ -- basic userdata passing
+ local obj = table.copy(test_object.tiles[1])
+ obj.test = ItemStack("default:cobble 99")
+ local tmp = core.serialize_roundtrip(obj)
+ assert(type(tmp.test) == "userdata")
+ assert(obj.test:to_string() == tmp.test:to_string())
+
+ -- object can't be passed, should error
+ obj = core.raycast(pos, pos)
+ assert(not pcall(core.serialize_roundtrip, obj))
+
+ -- VManip
+ local vm = core.get_voxel_manip(pos, pos)
+ local expect = vm:get_node_at(pos)
+ local vm2 = core.serialize_roundtrip(vm)
+ assert(deepequal(vm2:get_node_at(pos), expect))
+end
+unittests.register("test_userdata_passing", test_userdata_passing, {map=true})
+
+-- Asynchronous jobs
+
+local function test_handle_async(cb)
+ -- Basic test including mod name tracking and unittests.async_test()
+ -- which is defined inside_async_env.lua
+ local func = function(x)
+ return core.get_last_run_mod(), _VERSION, unittests[x]()
+ end
+ local expect = {core.get_last_run_mod(), _VERSION, true}
+
+ core.handle_async(func, function(...)
+ if not deepequal(expect, {...}) then
+ cb("Values did not equal")
+ end
+ if core.get_last_run_mod() ~= expect[1] then
+ cb("Mod name not tracked correctly")
+ end
+
+ -- Test passing of nil arguments and return values
+ core.handle_async(function(a, b)
+ return a, b
+ end, function(a, b)
+ if b ~= 123 then
+ cb("Argument went missing")
+ end
+ cb()
+ end, nil, 123)
+ end, "async_test")
+end
+unittests.register("test_handle_async", test_handle_async, {async=true})
+
+local function test_userdata_passing2(cb, _, pos)
+ -- VManip: check transfer into other env
+ local vm = core.get_voxel_manip(pos, pos)
+ local expect = vm:get_node_at(pos)
+
+ core.handle_async(function(vm_, pos_)
+ return vm_:get_node_at(pos_)
+ end, function(ret)
+ if not deepequal(expect, ret) then
+ cb("Node data mismatch (one-way)")
+ end
+
+ -- VManip: test a roundtrip
+ core.handle_async(function(vm_)
+ return vm_
+ end, function(vm2)
+ if not deepequal(expect, vm2:get_node_at(pos)) then
+ cb("Node data mismatch (roundtrip)")
+ end
+ cb()
+ end, vm)
+ end, vm, pos)
+end
+unittests.register("test_userdata_passing2", test_userdata_passing2, {map=true, async=true})
dofile(modpath .. "/player.lua")
dofile(modpath .. "/crafting.lua")
dofile(modpath .. "/itemdescription.lua")
+dofile(modpath .. "/async_env.lua")
--------------
--- /dev/null
+unittests = {}
+
+core.log("info", "Hello World")
+
+function unittests.async_test()
+ assert(core == minetest)
+ -- stuff that should not be here
+ assert(not core.get_player_by_name)
+ assert(not core.set_node)
+ assert(not core.object_refs)
+ -- stuff that should be here
+ assert(ItemStack)
+ assert(core.registered_items[""])
+ return true
+end
VoxelManipulator(),
m_map(map)
{
+ assert(map);
}
void MMVManip::initialEmerge(v3s16 blockpos_min, v3s16 blockpos_max,
{
TimeTaker timer1("initialEmerge", &emerge_time);
+ assert(m_map);
+
// Units of these are MapBlocks
v3s16 p_min = blockpos_min;
v3s16 p_max = blockpos_max;
{
if(m_area.getExtent() == v3s16(0,0,0))
return;
+ assert(m_map);
/*
Copy data of all blocks
}
}
+MMVManip *MMVManip::clone() const
+{
+ MMVManip *ret = new MMVManip();
+
+ const s32 size = m_area.getVolume();
+ ret->m_area = m_area;
+ if (m_data) {
+ ret->m_data = new MapNode[size];
+ memcpy(ret->m_data, m_data, size * sizeof(MapNode));
+ }
+ if (m_flags) {
+ ret->m_flags = new u8[size];
+ memcpy(ret->m_flags, m_flags, size * sizeof(u8));
+ }
+
+ ret->m_is_dirty = m_is_dirty;
+ // Even if the copy is disconnected from a map object keep the information
+ // needed to write it back to one
+ ret->m_loaded_blocks = m_loaded_blocks;
+
+ return ret;
+}
+
+void MMVManip::reparent(Map *map)
+{
+ assert(map && !m_map);
+ m_map = map;
+}
+
//END
void blitBackAll(std::map<v3s16, MapBlock*> * modified_blocks,
bool overwrite_generated = true);
+ /*
+ Creates a copy of this VManip including contents, the copy will not be
+ associated with a Map.
+ */
+ MMVManip *clone() const;
+
+ // Reassociates a copied VManip to a map
+ void reparent(Map *map);
+
+ // Is it impossible to call initialEmerge / blitBackAll?
+ inline bool isOrphan() const { return !m_map; }
+
bool m_is_dirty = false;
protected:
- Map *m_map;
+ MMVManip() {};
+
+ // may be null
+ Map *m_map = nullptr;
/*
key = blockpos
value = flags describing the block
${CMAKE_CURRENT_SOURCE_DIR}/c_converter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/c_types.cpp
${CMAKE_CURRENT_SOURCE_DIR}/c_internal.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/c_packer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/helper.cpp
PARENT_SCOPE)
infostream << script_get_backtrace(L) << std::endl;
}
+void call_string_dump(lua_State *L, int idx)
+{
+ // Retrieve string.dump from insecure env to avoid it being tampered with
+ lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_GLOBALS_BACKUP);
+ if (!lua_isnil(L, -1))
+ lua_getfield(L, -1, "string");
+ else
+ lua_getglobal(L, "string");
+ lua_getfield(L, -1, "dump");
+ lua_remove(L, -2); // remove _G
+ lua_remove(L, -2); // remove 'string' table
+ lua_pushvalue(L, idx);
+ lua_call(L, 1, 1);
+}
#define CUSTOM_RIDX_BACKTRACE (CUSTOM_RIDX_BASE + 3)
#define CUSTOM_RIDX_HTTP_API_LUA (CUSTOM_RIDX_BASE + 4)
#define CUSTOM_RIDX_VECTOR_METATABLE (CUSTOM_RIDX_BASE + 5)
+#define CUSTOM_RIDX_METATABLE_MAP (CUSTOM_RIDX_BASE + 6)
// Determine if CUSTOM_RIDX_SCRIPTAPI will hold a light or full userdata
* @param stack_depth How far on the stack to the first user function (ie: not builtin or core)
*/
void log_deprecated(lua_State *L, std::string message, int stack_depth = 1);
+
+// Safely call string.dump on a function value
+// (does not pop, leaves one value on stack)
+void call_string_dump(lua_State *L, int idx);
--- /dev/null
+/*
+Minetest
+Copyright (C) 2022 sfan5 <sfan5@live.de>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include <cstdio>
+#include <cstring>
+#include <cmath>
+#include <cassert>
+#include <unordered_set>
+#include <unordered_map>
+#include "c_packer.h"
+#include "c_internal.h"
+#include "log.h"
+#include "debug.h"
+#include "threading/mutex_auto_lock.h"
+
+extern "C" {
+#include <lauxlib.h>
+}
+
+//
+// Helpers
+//
+
+// convert negative index to absolute position on Lua stack
+static inline int absidx(lua_State *L, int idx)
+{
+ assert(idx < 0);
+ return lua_gettop(L) + idx + 1;
+}
+
+// does the type put anything into PackedInstr::sdata?
+static inline bool uses_sdata(int type)
+{
+ switch (type) {
+ case LUA_TSTRING:
+ case LUA_TFUNCTION:
+ case LUA_TUSERDATA:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// does the type put anything into PackedInstr::<union>?
+static inline bool uses_union(int type)
+{
+ switch (type) {
+ case LUA_TNIL:
+ case LUA_TSTRING:
+ case LUA_TFUNCTION:
+ return false;
+ default:
+ return true;
+ }
+}
+
+static inline bool can_set_into(int ktype, int vtype)
+{
+ switch (ktype) {
+ case LUA_TNUMBER:
+ return !uses_union(vtype);
+ case LUA_TSTRING:
+ return !uses_sdata(vtype);
+ default:
+ return false;
+ }
+}
+
+// is the key suitable for use with set_into?
+static inline bool suitable_key(lua_State *L, int idx)
+{
+ if (lua_type(L, idx) == LUA_TSTRING) {
+ // strings may not have a NULL byte (-> lua_setfield)
+ size_t len;
+ const char *str = lua_tolstring(L, idx, &len);
+ return strlen(str) == len;
+ } else {
+ assert(lua_type(L, idx) == LUA_TNUMBER);
+ // numbers must fit into an s32 and be integers (-> lua_rawseti)
+ lua_Number n = lua_tonumber(L, idx);
+ return std::floor(n) == n && n >= S32_MIN && n <= S32_MAX;
+ }
+}
+
+namespace {
+ // checks if you left any values on the stack, for debugging
+ class StackChecker {
+ lua_State *L;
+ int top;
+ public:
+ StackChecker(lua_State *L) : L(L), top(lua_gettop(L)) {}
+ ~StackChecker() {
+ assert(lua_gettop(L) >= top);
+ if (lua_gettop(L) > top) {
+ rawstream << "Lua stack not cleaned up: "
+ << lua_gettop(L) << " != " << top
+ << " (false-positive if exception thrown)" << std::endl;
+ }
+ }
+ };
+
+ // Since an std::vector may reallocate, this is the only safe way to keep
+ // a reference to a particular element.
+ template <typename T>
+ class VectorRef {
+ std::vector<T> *vec;
+ size_t idx;
+ VectorRef(std::vector<T> *vec, size_t idx) : vec(vec), idx(idx) {}
+ public:
+ static VectorRef<T> front(std::vector<T> &vec) {
+ return VectorRef(&vec, 0);
+ }
+ static VectorRef<T> back(std::vector<T> &vec) {
+ return VectorRef(&vec, vec.size() - 1);
+ }
+ T &operator*() { return (*vec)[idx]; }
+ T *operator->() { return &(*vec)[idx]; }
+ };
+
+ struct Packer {
+ PackInFunc fin;
+ PackOutFunc fout;
+ };
+
+ typedef std::pair<std::string, Packer> PackerTuple;
+}
+
+static inline auto emplace(PackedValue &pv, s16 type)
+{
+ pv.i.emplace_back();
+ auto ref = VectorRef<PackedInstr>::back(pv.i);
+ ref->type = type;
+ // Initialize fields that may be left untouched
+ if (type == LUA_TTABLE) {
+ ref->uidata1 = 0;
+ ref->uidata2 = 0;
+ } else if (type == LUA_TUSERDATA) {
+ ref->ptrdata = nullptr;
+ } else if (type == INSTR_POP) {
+ ref->sidata2 = 0;
+ }
+ return ref;
+}
+
+//
+// Management of registered packers
+//
+
+static std::unordered_map<std::string, Packer> g_packers;
+static std::mutex g_packers_lock;
+
+void script_register_packer(lua_State *L, const char *regname,
+ PackInFunc fin, PackOutFunc fout)
+{
+ // Store away callbacks
+ {
+ MutexAutoLock autolock(g_packers_lock);
+ auto it = g_packers.find(regname);
+ if (it == g_packers.end()) {
+ auto &ref = g_packers[regname];
+ ref.fin = fin;
+ ref.fout = fout;
+ } else {
+ FATAL_ERROR_IF(it->second.fin != fin || it->second.fout != fout,
+ "Packer registered twice with mismatching callbacks");
+ }
+ }
+
+ // Save metatable so we can identify instances later
+ lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+ if (lua_isnil(L, -1)) {
+ lua_newtable(L);
+ lua_pushvalue(L, -1);
+ lua_rawseti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+ }
+
+ luaL_getmetatable(L, regname);
+ FATAL_ERROR_IF(lua_isnil(L, -1), "No metatable registered with that name");
+
+ // CUSTOM_RIDX_METATABLE_MAP contains { [metatable] = "regname", ... }
+ // check first
+ lua_pushstring(L, regname);
+ lua_rawget(L, -3);
+ if (!lua_isnil(L, -1)) {
+ FATAL_ERROR_IF(lua_topointer(L, -1) != lua_topointer(L, -2),
+ "Packer registered twice with inconsistent metatable");
+ }
+ lua_pop(L, 1);
+ // then set
+ lua_pushstring(L, regname);
+ lua_rawset(L, -3);
+
+ lua_pop(L, 1);
+}
+
+static bool find_packer(const char *regname, PackerTuple &out)
+{
+ MutexAutoLock autolock(g_packers_lock);
+ auto it = g_packers.find(regname);
+ if (it == g_packers.end())
+ return false;
+ // copy data for thread safety
+ out.first = it->first;
+ out.second = it->second;
+ return true;
+}
+
+static bool find_packer(lua_State *L, int idx, PackerTuple &out)
+{
+#ifndef NDEBUG
+ StackChecker checker(L);
+#endif
+
+ // retrieve metatable of the object
+ if (lua_getmetatable(L, idx) != 1)
+ return false;
+
+ // use our global table to map it to the registry name
+ lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_METATABLE_MAP);
+ assert(lua_istable(L, -1));
+ lua_pushvalue(L, -2);
+ lua_rawget(L, -2);
+ if (lua_isnil(L, -1)) {
+ lua_pop(L, 3);
+ return false;
+ }
+
+ // load the associated data
+ bool found = find_packer(lua_tostring(L, -1), out);
+ FATAL_ERROR_IF(!found, "Inconsistent internal state");
+ lua_pop(L, 3);
+ return true;
+}
+
+//
+// Packing implementation
+//
+
+// recursively goes through the structure and ensures there are no circular references
+static void pack_validate(lua_State *L, int idx, std::unordered_set<const void*> &seen)
+{
+#ifndef NDEBUG
+ StackChecker checker(L);
+ assert(idx > 0);
+#endif
+
+ if (lua_type(L, idx) != LUA_TTABLE)
+ return;
+
+ const void *ptr = lua_topointer(L, idx);
+ assert(ptr);
+
+ if (seen.find(ptr) != seen.end())
+ throw LuaError("Circular references cannot be packed (yet)");
+ seen.insert(ptr);
+
+ lua_checkstack(L, 5);
+ lua_pushnil(L);
+ while (lua_next(L, idx) != 0) {
+ // key at -2, value at -1
+ pack_validate(L, absidx(L, -2), seen);
+ pack_validate(L, absidx(L, -1), seen);
+
+ lua_pop(L, 1);
+ }
+
+ seen.erase(ptr);
+}
+
+static VectorRef<PackedInstr> pack_inner(lua_State *L, int idx, int vidx, PackedValue &pv)
+{
+#ifndef NDEBUG
+ StackChecker checker(L);
+ assert(idx > 0);
+ assert(vidx > 0);
+#endif
+
+ switch (lua_type(L, idx)) {
+ case LUA_TNONE:
+ case LUA_TNIL:
+ return emplace(pv, LUA_TNIL);
+ case LUA_TBOOLEAN: {
+ auto r = emplace(pv, LUA_TBOOLEAN);
+ r->bdata = lua_toboolean(L, idx);
+ return r;
+ }
+ case LUA_TNUMBER: {
+ auto r = emplace(pv, LUA_TNUMBER);
+ r->ndata = lua_tonumber(L, idx);
+ return r;
+ }
+ case LUA_TSTRING: {
+ auto r = emplace(pv, LUA_TSTRING);
+ size_t len;
+ const char *str = lua_tolstring(L, idx, &len);
+ assert(str);
+ r->sdata.assign(str, len);
+ return r;
+ }
+ case LUA_TTABLE:
+ break; // execution continues
+ case LUA_TFUNCTION: {
+ auto r = emplace(pv, LUA_TFUNCTION);
+ call_string_dump(L, idx);
+ size_t len;
+ const char *str = lua_tolstring(L, -1, &len);
+ assert(str);
+ r->sdata.assign(str, len);
+ lua_pop(L, 1);
+ return r;
+ }
+ case LUA_TUSERDATA: {
+ PackerTuple ser;
+ if (!find_packer(L, idx, ser))
+ throw LuaError("Cannot serialize unsupported userdata");
+ pv.contains_userdata = true;
+ auto r = emplace(pv, LUA_TUSERDATA);
+ r->sdata = ser.first;
+ r->ptrdata = ser.second.fin(L, idx);
+ return r;
+ }
+ default: {
+ std::string err = "Cannot serialize type ";
+ err += lua_typename(L, lua_type(L, idx));
+ throw LuaError(err);
+ }
+ }
+
+ // LUA_TTABLE
+ lua_checkstack(L, 5);
+
+ auto rtable = emplace(pv, LUA_TTABLE);
+ const int vi_table = vidx++;
+
+ lua_pushnil(L);
+ while (lua_next(L, idx) != 0) {
+ // key at -2, value at -1
+ const int ktype = lua_type(L, -2), vtype = lua_type(L, -1);
+ if (ktype == LUA_TNUMBER)
+ rtable->uidata1++; // narr
+ else
+ rtable->uidata2++; // nrec
+
+ // check if we can use a shortcut
+ if (can_set_into(ktype, vtype) && suitable_key(L, -2)) {
+ // push only the value
+ auto rval = pack_inner(L, absidx(L, -1), vidx, pv);
+ rval->pop = vtype != LUA_TTABLE;
+ // and where to put it:
+ rval->set_into = vi_table;
+ if (ktype == LUA_TSTRING)
+ rval->sdata = lua_tostring(L, -2);
+ else
+ rval->sidata1 = lua_tointeger(L, -2);
+ // pop tables after the fact
+ if (!rval->pop) {
+ auto ri1 = emplace(pv, INSTR_POP);
+ ri1->sidata1 = vidx;
+ }
+ } else {
+ // push the key and value
+ pack_inner(L, absidx(L, -2), vidx, pv);
+ vidx++;
+ pack_inner(L, absidx(L, -1), vidx, pv);
+ vidx++;
+ // push an instruction to set them
+ auto ri1 = emplace(pv, INSTR_SETTABLE);
+ ri1->set_into = vi_table;
+ ri1->sidata1 = vidx - 2;
+ ri1->sidata2 = vidx - 1;
+ ri1->pop = true;
+ vidx -= 2;
+ }
+
+ lua_pop(L, 1);
+ }
+
+ assert(vidx == vi_table + 1);
+ return rtable;
+}
+
+PackedValue *script_pack(lua_State *L, int idx)
+{
+ if (idx < 0)
+ idx = absidx(L, idx);
+
+ std::unordered_set<const void*> seen;
+ pack_validate(L, idx, seen);
+ assert(seen.size() == 0);
+
+ // Actual serialization
+ PackedValue pv;
+ pack_inner(L, idx, 1, pv);
+
+ return new PackedValue(std::move(pv));
+}
+
+//
+// Unpacking implementation
+//
+
+void script_unpack(lua_State *L, PackedValue *pv)
+{
+ const int top = lua_gettop(L);
+ int ctr = 0;
+
+ for (auto &i : pv->i) {
+ // If leaving values on stack make sure there's space (every 5th iteration)
+ if (!i.pop && (ctr++) >= 5) {
+ lua_checkstack(L, 5);
+ ctr = 0;
+ }
+
+ /* Instructions */
+ switch (i.type) {
+ case INSTR_SETTABLE:
+ lua_pushvalue(L, top + i.sidata1); // key
+ lua_pushvalue(L, top + i.sidata2); // value
+ lua_rawset(L, top + i.set_into);
+ if (i.pop) {
+ if (i.sidata1 != i.sidata2) {
+ // removing moves indices so pop higher index first
+ lua_remove(L, top + std::max(i.sidata1, i.sidata2));
+ lua_remove(L, top + std::min(i.sidata1, i.sidata2));
+ } else {
+ lua_remove(L, top + i.sidata1);
+ }
+ }
+ continue;
+ case INSTR_POP:
+ lua_remove(L, top + i.sidata1);
+ if (i.sidata2 > 0)
+ lua_remove(L, top + i.sidata2);
+ continue;
+ default:
+ break;
+ }
+
+ /* Lua types */
+ switch (i.type) {
+ case LUA_TNIL:
+ lua_pushnil(L);
+ break;
+ case LUA_TBOOLEAN:
+ lua_pushboolean(L, i.bdata);
+ break;
+ case LUA_TNUMBER:
+ lua_pushnumber(L, i.ndata);
+ break;
+ case LUA_TSTRING:
+ lua_pushlstring(L, i.sdata.data(), i.sdata.size());
+ break;
+ case LUA_TTABLE:
+ lua_createtable(L, i.uidata1, i.uidata2);
+ break;
+ case LUA_TFUNCTION:
+ luaL_loadbuffer(L, i.sdata.data(), i.sdata.size(), nullptr);
+ break;
+ case LUA_TUSERDATA: {
+ PackerTuple ser;
+ sanity_check(find_packer(i.sdata.c_str(), ser));
+ ser.second.fout(L, i.ptrdata);
+ i.ptrdata = nullptr; // ownership taken by callback
+ break;
+ }
+ default:
+ assert(0);
+ break;
+ }
+
+ if (i.set_into) {
+ if (!i.pop)
+ lua_pushvalue(L, -1);
+ if (uses_sdata(i.type))
+ lua_rawseti(L, top + i.set_into, i.sidata1);
+ else
+ lua_setfield(L, top + i.set_into, i.sdata.c_str());
+ } else {
+ if (i.pop)
+ lua_pop(L, 1);
+ }
+ }
+
+ // as part of the unpacking process we take ownership of all userdata
+ pv->contains_userdata = false;
+ // leave exactly one value on the stack
+ lua_settop(L, top+1);
+}
+
+//
+// PackedValue
+//
+
+PackedValue::~PackedValue()
+{
+ if (!contains_userdata)
+ return;
+ for (auto &i : this->i) {
+ if (i.type == LUA_TUSERDATA && i.ptrdata) {
+ PackerTuple ser;
+ if (find_packer(i.sdata.c_str(), ser)) {
+ // tell it to deallocate object
+ ser.second.fout(nullptr, i.ptrdata);
+ } else {
+ assert(false);
+ }
+ }
+ }
+}
+
+//
+// script_dump_packed
+//
+
+#ifndef NDEBUG
+void script_dump_packed(const PackedValue *val)
+{
+ printf("instruction stream: [\n");
+ for (const auto &i : val->i) {
+ printf("\t(");
+ switch (i.type) {
+ case INSTR_SETTABLE:
+ printf("SETTABLE(%d, %d)", i.sidata1, i.sidata2);
+ break;
+ case INSTR_POP:
+ printf(i.sidata2 ? "POP(%d, %d)" : "POP(%d)", i.sidata1, i.sidata2);
+ break;
+ case LUA_TNIL:
+ printf("nil");
+ break;
+ case LUA_TBOOLEAN:
+ printf(i.bdata ? "true" : "false");
+ break;
+ case LUA_TNUMBER:
+ printf("%f", i.ndata);
+ break;
+ case LUA_TSTRING:
+ printf("\"%s\"", i.sdata.c_str());
+ break;
+ case LUA_TTABLE:
+ printf("table(%d, %d)", i.uidata1, i.uidata2);
+ break;
+ case LUA_TFUNCTION:
+ printf("function(%d byte)", i.sdata.size());
+ break;
+ case LUA_TUSERDATA:
+ printf("userdata %s %p", i.sdata.c_str(), i.ptrdata);
+ break;
+ default:
+ printf("!!UNKNOWN!!");
+ break;
+ }
+ if (i.set_into) {
+ if (i.type >= 0 && uses_sdata(i.type))
+ printf(", k=%d, into=%d", i.sidata1, i.set_into);
+ else if (i.type >= 0)
+ printf(", k=\"%s\", into=%d", i.sdata.c_str(), i.set_into);
+ else
+ printf(", into=%d", i.set_into);
+ }
+ if (i.pop)
+ printf(", pop");
+ printf(")\n");
+ }
+ printf("]\n");
+}
+#endif
--- /dev/null
+/*
+Minetest
+Copyright (C) 2022 sfan5 <sfan5@live.de>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include "irrlichttypes.h"
+#include "util/basic_macros.h"
+
+extern "C" {
+#include <lua.h>
+}
+
+/*
+ This file defines an in-memory representation of Lua objects including
+ support for functions and userdata. It it used to move data between Lua
+ states and cannot be used for persistence or network transfer.
+*/
+
+#define INSTR_SETTABLE (-10)
+#define INSTR_POP (-11)
+
+/**
+ * Represents a single instruction that pushes a new value or works with existing ones.
+ */
+struct PackedInstr
+{
+ s16 type; // LUA_T* or INSTR_*
+ u16 set_into; // set into table on stack
+ bool pop; // remove from stack?
+ union {
+ bool bdata; // boolean: value
+ lua_Number ndata; // number: value
+ struct {
+ u16 uidata1, uidata2; // table: narr, nrec
+ };
+ struct {
+ /*
+ SETTABLE: key index, value index
+ POP: indices to remove
+ otherwise w/ set_into: numeric key, -
+ */
+ s32 sidata1, sidata2;
+ };
+ void *ptrdata; // userdata: implementation defined
+ };
+ /*
+ - string: value
+ - function: buffer
+ - w/ set_into: string key (no null bytes!)
+ - userdata: name in registry
+ */
+ std::string sdata;
+
+ PackedInstr() : type(0), set_into(0), pop(false) {}
+};
+
+/**
+ * A packed value can be a primitive like a string or number but also a table
+ * including all of its contents. It is made up of a linear stream of
+ * 'instructions' that build the final value when executed.
+ */
+struct PackedValue
+{
+ std::vector<PackedInstr> i;
+ // Indicates whether there are any userdata pointers that need to be deallocated
+ bool contains_userdata = false;
+
+ PackedValue() = default;
+ ~PackedValue();
+
+ DISABLE_CLASS_COPY(PackedValue)
+
+ ALLOW_CLASS_MOVE(PackedValue)
+};
+
+/*
+ * Packing callback: Turns a Lua value at given index into a void*
+ */
+typedef void *(*PackInFunc)(lua_State *L, int idx);
+/*
+ * Unpacking callback: Turns a void* back into the Lua value (left on top of stack)
+ *
+ * Note that this function must take ownership of the pointer, so make sure
+ * to free or keep the memory.
+ * `L` can be nullptr to indicate that data should just be discarded.
+ */
+typedef void (*PackOutFunc)(lua_State *L, void *ptr);
+/*
+ * Register a packable type with the name of its metatable.
+ *
+ * Even though the callbacks are global this must be called for every Lua state
+ * that supports objects of this type.
+ * This function is thread-safe.
+ */
+void script_register_packer(lua_State *L, const char *regname,
+ PackInFunc fin, PackOutFunc fout);
+
+// Pack a Lua value
+PackedValue *script_pack(lua_State *L, int idx);
+// Unpack a Lua value (left on top of stack)
+// Note that this may modify the PackedValue, you can't reuse it!
+void script_unpack(lua_State *L, PackedValue *val);
+
+// Dump contents of PackedValue to stdout for debugging
+void script_dump_packed(const PackedValue *val);
#include <cstdlib>
extern "C" {
-#include "lua.h"
-#include "lauxlib.h"
-#include "lualib.h"
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
}
#include "server.h"
#include "filesys.h"
#include "porting.h"
#include "common/c_internal.h"
+#include "common/c_packer.h"
#include "lua_api/l_base.h"
/******************************************************************************/
{
initDone = true;
- for (unsigned int i = 0; i < numEngines; i++) {
- AsyncWorkerThread *toAdd = new AsyncWorkerThread(this,
- std::string("AsyncWorker-") + itos(i));
- workerThreads.push_back(toAdd);
- toAdd->start();
+ if (numEngines == 0) {
+ // Leave one core for the main thread and one for whatever else
+ autoscaleMaxWorkers = Thread::getNumberOfProcessors();
+ if (autoscaleMaxWorkers >= 2)
+ autoscaleMaxWorkers -= 2;
+ infostream << "AsyncEngine: using at most " << autoscaleMaxWorkers
+ << " threads with automatic scaling" << std::endl;
+
+ addWorkerThread();
+ } else {
+ for (unsigned int i = 0; i < numEngines; i++)
+ addWorkerThread();
}
}
+void AsyncEngine::addWorkerThread()
+{
+ AsyncWorkerThread *toAdd = new AsyncWorkerThread(this,
+ std::string("AsyncWorker-") + itos(workerThreads.size()));
+ workerThreads.push_back(toAdd);
+ toAdd->start();
+}
+
/******************************************************************************/
u32 AsyncEngine::queueAsyncJob(std::string &&func, std::string &¶ms,
const std::string &mod_origin)
{
- jobQueueMutex.lock();
+ MutexAutoLock autolock(jobQueueMutex);
u32 jobId = jobIdCounter++;
jobQueue.emplace_back();
to_add.mod_origin = mod_origin;
jobQueueCounter.post();
- jobQueueMutex.unlock();
+ return jobId;
+}
+
+u32 AsyncEngine::queueAsyncJob(std::string &&func, PackedValue *params,
+ const std::string &mod_origin)
+{
+ MutexAutoLock autolock(jobQueueMutex);
+ u32 jobId = jobIdCounter++;
+
+ jobQueue.emplace_back();
+ auto &to_add = jobQueue.back();
+ to_add.id = jobId;
+ to_add.function = std::move(func);
+ to_add.params_ext.reset(params);
+ to_add.mod_origin = mod_origin;
+
+ jobQueueCounter.post();
return jobId;
}
/******************************************************************************/
void AsyncEngine::step(lua_State *L)
+{
+ stepJobResults(L);
+ stepAutoscale();
+}
+
+void AsyncEngine::stepJobResults(lua_State *L)
{
int error_handler = PUSH_ERROR_HANDLER(L);
lua_getglobal(L, "core");
luaL_checktype(L, -1, LUA_TFUNCTION);
lua_pushinteger(L, j.id);
- lua_pushlstring(L, j.result.data(), j.result.size());
+ if (j.result_ext)
+ script_unpack(L, j.result_ext.get());
+ else
+ lua_pushlstring(L, j.result.data(), j.result.size());
// Call handler
const char *origin = j.mod_origin.empty() ? nullptr : j.mod_origin.c_str();
lua_pop(L, 2); // Pop core and error handler
}
+void AsyncEngine::stepAutoscale()
+{
+ if (workerThreads.size() >= autoscaleMaxWorkers)
+ return;
+
+ MutexAutoLock autolock(jobQueueMutex);
+
+ // 2) If the timer elapsed, check again
+ if (autoscaleTimer && porting::getTimeMs() >= autoscaleTimer) {
+ autoscaleTimer = 0;
+ // Determine overlap with previous snapshot
+ unsigned int n = 0;
+ for (const auto &it : jobQueue)
+ n += autoscaleSeenJobs.count(it.id);
+ autoscaleSeenJobs.clear();
+ infostream << "AsyncEngine: " << n << " jobs were still waiting after 1s" << std::endl;
+ // Start this many new threads
+ while (workerThreads.size() < autoscaleMaxWorkers && n > 0) {
+ addWorkerThread();
+ n--;
+ }
+ return;
+ }
+
+ // 1) Check if there's anything in the queue
+ if (!autoscaleTimer && !jobQueue.empty()) {
+ // Take a snapshot of all jobs we have seen
+ for (const auto &it : jobQueue)
+ autoscaleSeenJobs.emplace(it.id);
+ // and set a timer for 1 second
+ autoscaleTimer = porting::getTimeMs() + 1000;
+ }
+}
+
/******************************************************************************/
-void AsyncEngine::prepareEnvironment(lua_State* L, int top)
+bool AsyncEngine::prepareEnvironment(lua_State* L, int top)
{
for (StateInitializer &stateInitializer : stateInitializers) {
stateInitializer(L, top);
}
+
+ auto *script = ModApiBase::getScriptApiBase(L);
+ try {
+ script->loadMod(Server::getBuiltinLuaPath() + DIR_DELIM + "init.lua",
+ BUILTIN_MOD_NAME);
+ } catch (const ModError &e) {
+ errorstream << "Execution of async base environment failed: "
+ << e.what() << std::endl;
+ FATAL_ERROR("Execution of async base environment failed");
+ }
+
+ // Load per mod stuff
+ if (server) {
+ const auto &list = server->m_async_init_files;
+ try {
+ for (auto &it : list)
+ script->loadMod(it.second, it.first);
+ } catch (const ModError &e) {
+ errorstream << "Failed to load mod script inside async environment." << std::endl;
+ server->setAsyncFatalError(e.what());
+ return false;
+ }
+ }
+
+ return true;
}
/******************************************************************************/
{
lua_State *L = getStack();
+ if (jobDispatcher->server) {
+ setGameDef(jobDispatcher->server);
+
+ if (g_settings->getBool("secure.enable_security"))
+ initializeSecurity();
+ }
+
// Prepare job lua environment
lua_getglobal(L, "core");
int top = lua_gettop(L);
// Push builtin initialization type
- lua_pushstring(L, "async");
+ lua_pushstring(L, jobDispatcher->server ? "async_game" : "async");
lua_setglobal(L, "INIT");
- jobDispatcher->prepareEnvironment(L, top);
+ if (!jobDispatcher->prepareEnvironment(L, top)) {
+ // can't throw from here so we're stuck with this
+ isErrored = true;
+ }
}
/******************************************************************************/
/******************************************************************************/
void* AsyncWorkerThread::run()
{
- lua_State *L = getStack();
+ if (isErrored)
+ return nullptr;
- try {
- loadMod(getServer()->getBuiltinLuaPath() + DIR_DELIM + "init.lua",
- BUILTIN_MOD_NAME);
- } catch (const ModError &e) {
- errorstream << "Execution of async base environment failed: "
- << e.what() << std::endl;
- FATAL_ERROR("Execution of async base environment failed");
- }
+ lua_State *L = getStack();
int error_handler = PUSH_ERROR_HANDLER(L);
+ auto report_error = [this] (const ModError &e) {
+ if (jobDispatcher->server)
+ jobDispatcher->server->setAsyncFatalError(e.what());
+ else
+ errorstream << e.what() << std::endl;
+ };
+
lua_getglobal(L, "core");
if (lua_isnil(L, -1)) {
FATAL_ERROR("Unable to find core within async environment!");
if (!jobDispatcher->getJob(&j) || stopRequested())
continue;
+ const bool use_ext = !!j.params_ext;
+
lua_getfield(L, -1, "job_processor");
if (lua_isnil(L, -1))
FATAL_ERROR("Unable to get async job processor!");
errorstream << "ASYNC WORKER: Unable to deserialize function" << std::endl;
lua_pushnil(L);
}
- lua_pushlstring(L, j.params.data(), j.params.size());
+ if (use_ext)
+ script_unpack(L, j.params_ext.get());
+ else
+ lua_pushlstring(L, j.params.data(), j.params.size());
// Call it
setOriginDirect(j.mod_origin.empty() ? nullptr : j.mod_origin.c_str());
try {
scriptError(result, "<async>");
} catch (const ModError &e) {
- errorstream << e.what() << std::endl;
+ report_error(e);
}
} else {
// Fetch result
- size_t length;
- const char *retval = lua_tolstring(L, -1, &length);
- j.result.assign(retval, length);
+ if (use_ext) {
+ try {
+ j.result_ext.reset(script_pack(L, -1));
+ } catch (const ModError &e) {
+ report_error(e);
+ result = LUA_ERRERR;
+ }
+ } else {
+ size_t length;
+ const char *retval = lua_tolstring(L, -1, &length);
+ j.result.assign(retval, length);
+ }
}
lua_pop(L, 1); // Pop retval
// Put job result
- if (!j.result.empty())
+ if (result == 0)
jobDispatcher->putJobResult(std::move(j));
}
#include <vector>
#include <deque>
+#include <unordered_set>
+#include <memory>
+#include <lua.h>
#include "threading/semaphore.h"
#include "threading/thread.h"
-#include "lua.h"
+#include "common/c_packer.h"
#include "cpp_api/s_base.h"
+#include "cpp_api/s_security.h"
// Forward declarations
class AsyncEngine;
std::string function;
// Parameter to be passed to function (serialized)
std::string params;
+ // Alternative parameters
+ std::unique_ptr<PackedValue> params_ext;
// Result of function call (serialized)
std::string result;
+ // Alternative result
+ std::unique_ptr<PackedValue> result_ext;
// Name of the mod who invoked this call
std::string mod_origin;
// JobID used to identify a job and match it to callback
};
// Asynchronous working environment
-class AsyncWorkerThread : public Thread, virtual public ScriptApiBase {
+class AsyncWorkerThread : public Thread,
+ virtual public ScriptApiBase, public ScriptApiSecurity {
friend class AsyncEngine;
public:
virtual ~AsyncWorkerThread();
private:
AsyncEngine *jobDispatcher = nullptr;
+ bool isErrored = false;
};
// Asynchornous thread and job management
typedef void (*StateInitializer)(lua_State *L, int top);
public:
AsyncEngine() = default;
+ AsyncEngine(Server *server) : server(server) {};
~AsyncEngine();
/**
/**
* Create async engine tasks and lock function registration
- * @param numEngines Number of async threads to be started
+ * @param numEngines Number of worker threads, 0 for automatic scaling
*/
void initialize(unsigned int numEngines);
u32 queueAsyncJob(std::string &&func, std::string &¶ms,
const std::string &mod_origin = "");
+ /**
+ * Queue an async job
+ * @param func Serialized lua function
+ * @param params Serialized parameters (takes ownership!)
+ * @return ID of queued job
+ */
+ u32 queueAsyncJob(std::string &&func, PackedValue *params,
+ const std::string &mod_origin = "");
+
/**
* Engine step to process finished jobs
- * the engine step is one way to pass events back, PushFinishedJobs another
* @param L The Lua stack
*/
void step(lua_State *L);
*/
void putJobResult(LuaJobInfo &&result);
+ /**
+ * Start an additional worker thread
+ */
+ void addWorkerThread();
+
+ /**
+ * Process finished jobs callbacks
+ */
+ void stepJobResults(lua_State *L);
+
+ /**
+ * Handle automatic scaling of worker threads
+ */
+ void stepAutoscale();
+
/**
* Initialize environment with current registred functions
* this function adds all functions registred by registerFunction to the
* passed lua stack
* @param L Lua stack to initialize
* @param top Stack position
+ * @return false if a mod error ocurred
*/
- void prepareEnvironment(lua_State* L, int top);
+ bool prepareEnvironment(lua_State* L, int top);
private:
// Variable locking the engine against further modification
bool initDone = false;
+ // Maximum number of worker threads for automatic scaling
+ // 0 if disabled
+ unsigned int autoscaleMaxWorkers = 0;
+ u64 autoscaleTimer = 0;
+ std::unordered_set<u32> autoscaleSeenJobs;
+
+ // Only set for the server async environment (duh)
+ Server *server = nullptr;
+
// Internal store for registred state initializers
std::vector<StateInitializer> stateInitializers;
API_FCT(register_craft);
API_FCT(clear_craft);
}
+
+void ModApiCraft::InitializeAsync(lua_State *L, int top)
+{
+ // all read-only functions
+ API_FCT(get_all_craft_recipes);
+ API_FCT(get_craft_recipe);
+ API_FCT(get_craft_result);
+}
public:
static void Initialize(lua_State *L, int top);
+ static void InitializeAsync(lua_State *L, int top);
};
// Retrieve Environment pointer as `env` (no map lock)
#define GET_PLAIN_ENV_PTR_NO_MAP_LOCK \
- Environment *env = (Environment *)getEnv(L); \
+ Environment *env = getEnv(L); \
if (env == NULL) \
return 0
#include "lua_api/l_internal.h"
#include "common/c_converter.h"
#include "common/c_content.h"
+#include "common/c_packer.h"
#include "itemdef.h"
#include "nodedef.h"
#include "server.h"
lua_setmetatable(L, -2);
return 1;
}
+
// Not callable from Lua
int LuaItemStack::create(lua_State *L, const ItemStack &item)
{
return *(LuaItemStack **)luaL_checkudata(L, narg, className);
}
+void *LuaItemStack::packIn(lua_State *L, int idx)
+{
+ LuaItemStack *o = checkobject(L, idx);
+ return new ItemStack(o->getItem());
+}
+
+void LuaItemStack::packOut(lua_State *L, void *ptr)
+{
+ ItemStack *stack = reinterpret_cast<ItemStack*>(ptr);
+ if (L)
+ create(L, *stack);
+ delete stack;
+}
+
void LuaItemStack::Register(lua_State *L)
{
lua_newtable(L);
// Can be created from Lua (ItemStack(itemstack or itemstring or table or nil))
lua_register(L, className, create_object);
+
+ script_register_packer(L, className, packIn, packOut);
}
const char LuaItemStack::className[] = "ItemStack";
API_FCT(get_content_id);
API_FCT(get_name_from_content_id);
}
+
+void ModApiItemMod::InitializeAsync(lua_State *L, int top)
+{
+ // all read-only functions
+ API_FCT(get_content_id);
+ API_FCT(get_name_from_content_id);
+}
// Not callable from Lua
static int create(lua_State *L, const ItemStack &item);
static LuaItemStack* checkobject(lua_State *L, int narg);
- static void Register(lua_State *L);
+ static void *packIn(lua_State *L, int idx);
+ static void packOut(lua_State *L, void *ptr);
+
+ static void Register(lua_State *L);
};
class ModApiItemMod : public ModApiBase {
static int l_register_alias_raw(lua_State *L);
static int l_get_content_id(lua_State *L);
static int l_get_name_from_content_id(lua_State *L);
+
public:
static void Initialize(lua_State *L, int top);
+ static void InitializeAsync(lua_State *L, int top);
};
#include "lua_api/l_internal.h"
#include "common/c_converter.h"
#include "common/c_content.h"
+#include "common/c_packer.h"
#include "log.h"
#include "porting.h"
#include "util/numeric.h"
}
+void *LuaPerlinNoise::packIn(lua_State *L, int idx)
+{
+ LuaPerlinNoise *o = checkobject(L, idx);
+ return new NoiseParams(o->np);
+}
+
+void LuaPerlinNoise::packOut(lua_State *L, void *ptr)
+{
+ NoiseParams *np = reinterpret_cast<NoiseParams*>(ptr);
+ if (L) {
+ LuaPerlinNoise *o = new LuaPerlinNoise(np);
+ *(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+ luaL_getmetatable(L, className);
+ lua_setmetatable(L, -2);
+ }
+ delete np;
+}
+
+
void LuaPerlinNoise::Register(lua_State *L)
{
lua_newtable(L);
lua_pop(L, 1);
lua_register(L, className, create_object);
+
+ script_register_packer(L, className, packIn, packOut);
}
}
+struct NoiseMapParams {
+ NoiseParams np;
+ s32 seed;
+ v3s16 size;
+};
+
+void *LuaPerlinNoiseMap::packIn(lua_State *L, int idx)
+{
+ LuaPerlinNoiseMap *o = checkobject(L, idx);
+ NoiseMapParams *ret = new NoiseMapParams();
+ ret->np = o->noise->np;
+ ret->seed = o->noise->seed;
+ ret->size = v3s16(o->noise->sx, o->noise->sy, o->noise->sz);
+ return ret;
+}
+
+void LuaPerlinNoiseMap::packOut(lua_State *L, void *ptr)
+{
+ NoiseMapParams *p = reinterpret_cast<NoiseMapParams*>(ptr);
+ if (L) {
+ LuaPerlinNoiseMap *o = new LuaPerlinNoiseMap(&p->np, p->seed, p->size);
+ *(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+ luaL_getmetatable(L, className);
+ lua_setmetatable(L, -2);
+ }
+ delete p;
+}
+
+
void LuaPerlinNoiseMap::Register(lua_State *L)
{
lua_newtable(L);
lua_pop(L, 1);
lua_register(L, className, create_object);
+
+ script_register_packer(L, className, packIn, packOut);
}
static LuaPerlinNoise *checkobject(lua_State *L, int narg);
+ static void *packIn(lua_State *L, int idx);
+ static void packOut(lua_State *L, void *ptr);
+
static void Register(lua_State *L);
};
static LuaPerlinNoiseMap *checkobject(lua_State *L, int narg);
+ static void *packIn(lua_State *L, int idx);
+ static void packOut(lua_State *L, void *ptr);
+
static void Register(lua_State *L);
};
#include "lua_api/l_internal.h"
#include "common/c_converter.h"
#include "common/c_content.h"
+#include "common/c_packer.h"
#include "cpp_api/s_base.h"
#include "cpp_api/s_security.h"
#include "scripting_server.h"
return 0;
}
+// do_async_callback(func, params, mod_origin)
+int ModApiServer::l_do_async_callback(lua_State *L)
+{
+ NO_MAP_LOCK_REQUIRED;
+ ServerScripting *script = getScriptApi<ServerScripting>(L);
+
+ luaL_checktype(L, 1, LUA_TFUNCTION);
+ luaL_checktype(L, 2, LUA_TTABLE);
+ luaL_checktype(L, 3, LUA_TSTRING);
+
+ call_string_dump(L, 1);
+ size_t func_length;
+ const char *serialized_func_raw = lua_tolstring(L, -1, &func_length);
+
+ PackedValue *param = script_pack(L, 2);
+
+ std::string mod_origin = readParam<std::string>(L, 3);
+
+ u32 jobId = script->queueAsync(
+ std::string(serialized_func_raw, func_length),
+ param, mod_origin);
+
+ lua_settop(L, 0);
+ lua_pushinteger(L, jobId);
+ return 1;
+}
+
+// register_async_dofile(path)
+int ModApiServer::l_register_async_dofile(lua_State *L)
+{
+ NO_MAP_LOCK_REQUIRED;
+
+ std::string path = readParam<std::string>(L, 1);
+ CHECK_SECURE_PATH(L, path.c_str(), false);
+
+ // Find currently running mod name (only at init time)
+ lua_rawgeti(L, LUA_REGISTRYINDEX, CUSTOM_RIDX_CURRENT_MOD_NAME);
+ if (!lua_isstring(L, -1))
+ return 0;
+ std::string modname = readParam<std::string>(L, -1);
+
+ getServer(L)->m_async_init_files.emplace_back(modname, path);
+ lua_pushboolean(L, true);
+ return 1;
+}
+
+// serialize_roundtrip(value)
+// Meant for unit testing the packer from Lua
+int ModApiServer::l_serialize_roundtrip(lua_State *L)
+{
+ NO_MAP_LOCK_REQUIRED;
+
+ int top = lua_gettop(L);
+ auto *pv = script_pack(L, 1);
+ if (top != lua_gettop(L))
+ throw LuaError("stack values leaked");
+
+#ifndef NDEBUG
+ script_dump_packed(pv);
+#endif
+
+ top = lua_gettop(L);
+ script_unpack(L, pv);
+ delete pv;
+ if (top + 1 != lua_gettop(L))
+ throw LuaError("stack values leaked");
+
+ return 1;
+}
+
void ModApiServer::Initialize(lua_State *L, int top)
{
API_FCT(request_shutdown);
API_FCT(remove_player);
API_FCT(unban_player_or_ip);
API_FCT(notify_authentication_modified);
+
+ API_FCT(do_async_callback);
+ API_FCT(register_async_dofile);
+ API_FCT(serialize_roundtrip);
+}
+
+void ModApiServer::InitializeAsync(lua_State *L, int top)
+{
+ API_FCT(get_worldpath);
+ API_FCT(is_singleplayer);
+
+ API_FCT(get_current_modname);
+ API_FCT(get_modpath);
+ API_FCT(get_modnames);
}
// notify_authentication_modified(name)
static int l_notify_authentication_modified(lua_State *L);
+ // do_async_callback(func, params, mod_origin)
+ static int l_do_async_callback(lua_State *L);
+
+ // register_async_dofile(path)
+ static int l_register_async_dofile(lua_State *L);
+
+ // serialize_roundtrip(obj)
+ static int l_serialize_roundtrip(lua_State *L);
+
public:
static void Initialize(lua_State *L, int top);
+ static void InitializeAsync(lua_State *L, int top);
};
API_FCT(cpdir);
API_FCT(mvdir);
API_FCT(get_dir_list);
+ API_FCT(safe_file_write);
+
+ API_FCT(request_insecure_environment);
API_FCT(encode_base64);
API_FCT(decode_base64);
API_FCT(colorspec_to_colorstring);
API_FCT(colorspec_to_bytes);
+ API_FCT(encode_png);
+
API_FCT(get_last_run_mod);
API_FCT(set_last_run_mod);
static void Initialize(lua_State *L, int top);
static void InitializeAsync(lua_State *L, int top);
static void InitializeClient(lua_State *L, int top);
-
- static void InitializeAsync(AsyncEngine &engine);
};
#include "lua_api/l_internal.h"
#include "common/c_content.h"
#include "common/c_converter.h"
+#include "common/c_packer.h"
#include "emerge.h"
#include "environment.h"
#include "map.h"
LuaVoxelManip *o = checkobject(L, 1);
MMVManip *vm = o->vm;
+ if (vm->isOrphan())
+ return 0;
v3s16 bp1 = getNodeBlockPos(check_v3s16(L, 2));
v3s16 bp2 = getNodeBlockPos(check_v3s16(L, 3));
return *(LuaVoxelManip **)ud; // unbox pointer
}
+void *LuaVoxelManip::packIn(lua_State *L, int idx)
+{
+ LuaVoxelManip *o = checkobject(L, idx);
+
+ if (o->is_mapgen_vm)
+ throw LuaError("nope");
+ return o->vm->clone();
+}
+
+void LuaVoxelManip::packOut(lua_State *L, void *ptr)
+{
+ MMVManip *vm = reinterpret_cast<MMVManip*>(ptr);
+ if (!L) {
+ delete vm;
+ return;
+ }
+
+ // Associate vmanip with map if the Lua env has one
+ Environment *env = getEnv(L);
+ if (env)
+ vm->reparent(&(env->getMap()));
+
+ LuaVoxelManip *o = new LuaVoxelManip(vm, false);
+ *(void **)(lua_newuserdata(L, sizeof(void *))) = o;
+ luaL_getmetatable(L, className);
+ lua_setmetatable(L, -2);
+}
+
void LuaVoxelManip::Register(lua_State *L)
{
lua_newtable(L);
// Can be created from Lua (VoxelManip())
lua_register(L, className, create_object);
+
+ script_register_packer(L, className, packIn, packOut);
}
const char LuaVoxelManip::className[] = "VoxelManip";
static LuaVoxelManip *checkobject(lua_State *L, int narg);
+ static void *packIn(lua_State *L, int idx);
+ static void packOut(lua_State *L, void *ptr);
+
static void Register(lua_State *L);
};
#include "lua_api/l_storage.h"
extern "C" {
-#include "lualib.h"
+#include <lualib.h>
}
ServerScripting::ServerScripting(Server* server):
- ScriptApiBase(ScriptingType::Server)
+ ScriptApiBase(ScriptingType::Server),
+ asyncEngine(server)
{
setGameDef(server);
infostream << "SCRIPTAPI: Initialized game modules" << std::endl;
}
+void ServerScripting::initAsync()
+{
+ // Save globals to transfer
+ {
+ lua_State *L = getStack();
+ lua_getglobal(L, "core");
+ luaL_checktype(L, -1, LUA_TTABLE);
+ lua_getfield(L, -1, "get_globals_to_transfer");
+ lua_call(L, 0, 1);
+ luaL_checktype(L, -1, LUA_TSTRING);
+ getServer()->m_async_globals_data.set(readParam<std::string>(L, -1));
+ lua_pushnil(L);
+ lua_setfield(L, -3, "get_globals_to_transfer"); // unset function too
+ lua_pop(L, 2); // pop 'core', return value
+ }
+
+ infostream << "SCRIPTAPI: Initializing async engine" << std::endl;
+ asyncEngine.registerStateInitializer(InitializeAsync);
+ asyncEngine.registerStateInitializer(ModApiUtil::InitializeAsync);
+ asyncEngine.registerStateInitializer(ModApiCraft::InitializeAsync);
+ asyncEngine.registerStateInitializer(ModApiItemMod::InitializeAsync);
+ asyncEngine.registerStateInitializer(ModApiServer::InitializeAsync);
+ // not added: ModApiMapgen is a minefield for thread safety
+ // not added: ModApiHttp async api can't really work together with our jobs
+ // not added: ModApiStorage is probably not thread safe(?)
+
+ asyncEngine.initialize(0);
+}
+
+void ServerScripting::stepAsync()
+{
+ asyncEngine.step(getStack());
+}
+
+u32 ServerScripting::queueAsync(std::string &&serialized_func,
+ PackedValue *param, const std::string &mod_origin)
+{
+ return asyncEngine.queueAsyncJob(std::move(serialized_func),
+ param, mod_origin);
+}
+
void ServerScripting::InitializeModApi(lua_State *L, int top)
{
// Register reference classes (userdata)
ModApiStorage::Initialize(L, top);
ModApiChannels::Initialize(L, top);
}
+
+void ServerScripting::InitializeAsync(lua_State *L, int top)
+{
+ // classes
+ LuaItemStack::Register(L);
+ LuaPerlinNoise::Register(L);
+ LuaPerlinNoiseMap::Register(L);
+ LuaPseudoRandom::Register(L);
+ LuaPcgRandom::Register(L);
+ LuaSecureRandom::Register(L);
+ LuaVoxelManip::Register(L);
+ LuaSettings::Register(L);
+
+ // globals data
+ lua_getglobal(L, "core");
+ luaL_checktype(L, -1, LUA_TTABLE);
+ std::string s = ModApiBase::getServer(L)->m_async_globals_data.get();
+ lua_pushlstring(L, s.c_str(), s.size());
+ lua_setfield(L, -2, "transferred_globals");
+ lua_pop(L, 1); // pop 'core'
+}
#include "cpp_api/s_player.h"
#include "cpp_api/s_server.h"
#include "cpp_api/s_security.h"
+#include "cpp_api/s_async.h"
+
+struct PackedValue;
/*****************************************************************************/
/* Scripting <-> Server Game Interface */
// use ScriptApiBase::loadMod() to load mods
+ // Initialize async engine, call this AFTER loading all mods
+ void initAsync();
+
+ // Global step handler to collect async results
+ void stepAsync();
+
+ // Pass job to async threads
+ u32 queueAsync(std::string &&serialized_func,
+ PackedValue *param, const std::string &mod_origin);
+
private:
void InitializeModApi(lua_State *L, int top);
+
+ static void InitializeAsync(lua_State *L, int top);
+
+ AsyncEngine asyncEngine;
};
m_clients(m_con),
m_admin_chat(iface),
m_on_shutdown_errmsg(on_shutdown_errmsg),
+ m_async_globals_data(""),
m_modchannel_mgr(new ModChannelMgr())
{
if (m_path_world.empty())
// Give environment reference to scripting api
m_script->initializeEnvironment(m_env);
+ // Do this after regular script init is done
+ m_script->initAsync();
+
// Register us to receive map edit events
servermap->addEventReceiver(this);
virtual const std::vector<ModSpec> &getMods() const;
virtual const ModSpec* getModSpec(const std::string &modname) const;
- std::string getBuiltinLuaPath();
+ static std::string getBuiltinLuaPath();
virtual std::string getWorldPath() const { return m_path_world; }
inline bool isSingleplayer() const
static bool migrateModStorageDatabase(const GameParams &game_params,
const Settings &cmd_args);
+ // Lua files registered for init of async env, pair of modname + path
+ std::vector<std::pair<std::string, std::string>> m_async_init_files;
+
+ // Serialized data transferred into async envs at init time
+ MutexedVariable<std::string> m_async_globals_data;
+
// Bind address
Address m_bind_addr;
*/
m_script->environment_Step(dtime);
+ m_script->stepAsync();
+
/*
Step active objects
*/
#define CONTAINS(c, v) (std::find((c).begin(), (c).end(), (v)) != (c).end())
// To disable copy constructors and assignment operations for some class
-// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) as a private member.
+// 'Foobar', add the macro DISABLE_CLASS_COPY(Foobar) in the class definition.
// Note this also disables copying for any classes derived from 'Foobar' as well
// as classes having a 'Foobar' member.
#define DISABLE_CLASS_COPY(C) \
C(const C &) = delete; \
C &operator=(const C &) = delete;
+// If you have used DISABLE_CLASS_COPY with a class but still want to permit moving
+// use this macro to add the default move constructors back.
+#define ALLOW_CLASS_MOVE(C) \
+ C(C &&other) = default; \
+ C &operator=(C &&) = default;
+
#ifndef _MSC_VER
#define UNUSED_ATTRIBUTE __attribute__ ((unused))
#else
projects / dragonfireclient.git / commitdiff