end
--- Callback handling for dynamic_add_media
-
-local dynamic_add_media_raw = core.dynamic_add_media_raw
-core.dynamic_add_media_raw = nil
-function core.dynamic_add_media(filepath, callback)
- local ret = dynamic_add_media_raw(filepath)
- if ret == false then
- return ret
- end
- if callback == nil then
- core.log("deprecated", "Calling minetest.dynamic_add_media without "..
- "a callback is deprecated and will stop working in future versions.")
- else
- -- At the moment async loading is not actually implemented, so we
- -- immediately call the callback ourselves
- for _, name in ipairs(ret) do
- callback(name)
- end
- end
- return true
-end
+-- Used for callback handling with dynamic_add_media
+core.dynamic_media_callbacks = {}
-- PNG encoder safety wrapper
* Returns a code (0: successful, 1: no such player, 2: player is connected)
* `minetest.remove_player_auth(name)`: remove player authentication data
* Returns boolean indicating success (false if player nonexistant)
-* `minetest.dynamic_add_media(filepath, callback)`
- * `filepath`: path to a media file on the filesystem
- * `callback`: function with arguments `name`, where name is a player name
- (previously there was no callback argument; omitting it is deprecated)
- * Adds the file to the media sent to clients by the server on startup
- and also pushes this file to already connected clients.
- The file must be a supported image, sound or model format. It must not be
- modified, deleted, moved or renamed after calling this function.
- The list of dynamically added media is not persisted.
+* `minetest.dynamic_add_media(options, callback)`
+ * `options`: table containing the following parameters
+ * `filepath`: path to a media file on the filesystem
+ * `to_player`: name of the player the media should be sent to instead of
+ all players (optional)
+ * `ephemeral`: boolean that marks the media as ephemeral,
+ it will not be cached on the client (optional, default false)
+ * `callback`: function with arguments `name`, which is a player name
+ * Pushes the specified media file to client(s). (details below)
+ The file must be a supported image, sound or model format.
+ Dynamically added media is not persisted between server restarts.
* Returns false on error, true if the request was accepted
* The given callback will be called for every player as soon as the
media is available on the client.
- Old clients that lack support for this feature will not see the media
- unless they reconnect to the server. (callback won't be called)
- * Since media transferred this way currently does not use client caching
- or HTTP transfers, dynamic media should not be used with big files.
+ * Details/Notes:
+ * If `ephemeral`=false and `to_player` is unset the file is added to the media
+ sent to clients on startup, this means the media will appear even on
+ old clients if they rejoin the server.
+ * If `ephemeral`=false the file must not be modified, deleted, moved or
+ renamed after calling this function.
+ * Regardless of any use of `ephemeral`, adding media files with the same
+ name twice is not possible/guaranteed to work. An exception to this is the
+ use of `to_player` to send the same, already existent file to multiple
+ chosen players.
+ * Clients will attempt to fetch files added this way via remote media,
+ this can make transfer of bigger files painless (if set up). Nevertheless
+ it is advised not to use dynamic media for big media files.
Bans
----
m_media_downloader = NULL;
}
}
+ {
+ // Acknowledge dynamic media downloads to server
+ std::vector<u32> done;
+ for (auto it = m_pending_media_downloads.begin();
+ it != m_pending_media_downloads.end();) {
+ assert(it->second->isStarted());
+ it->second->step(this);
+ if (it->second->isDone()) {
+ done.emplace_back(it->first);
+
+ it = m_pending_media_downloads.erase(it);
+ } else {
+ it++;
+ }
+
+ if (done.size() == 255) { // maximum in one packet
+ sendHaveMedia(done);
+ done.clear();
+ }
+ }
+ if (!done.empty())
+ sendHaveMedia(done);
+ }
/*
If the server didn't update the inventory in a while, revert
Send(&pkt);
infostream << "Client: Sending media request list to server ("
- << file_requests.size() << " files. packet size)" << std::endl;
+ << file_requests.size() << " files, packet size "
+ << pkt.getSize() << ")" << std::endl;
}
void Client::initLocalMapSaving(const Address &address,
Send(&pkt);
}
+void Client::sendHaveMedia(const std::vector<u32> &tokens)
+{
+ NetworkPacket pkt(TOSERVER_HAVE_MEDIA, 1 + tokens.size() * 4);
+
+ sanity_check(tokens.size() < 256);
+
+ pkt << static_cast<u8>(tokens.size());
+ for (u32 token : tokens)
+ pkt << token;
+
+ Send(&pkt);
+}
+
void Client::removeNode(v3s16 p)
{
std::map<v3s16, MapBlock*> modified_blocks;
class NodeDefManager;
//class IWritableCraftDefManager;
class ClientMediaDownloader;
+class SingleMediaDownloader;
struct MapDrawControl;
class ModChannelMgr;
class MtEventManager;
void sendDamage(u16 damage);
void sendRespawn();
void sendReady();
+ void sendHaveMedia(const std::vector<u32> &tokens);
ClientEnvironment& getEnv() { return m_env; }
ITextureSource *tsrc() { return getTextureSource(); }
bool m_activeobjects_received = false;
bool m_mods_loaded = false;
+ std::vector<std::string> m_remote_media_servers;
+ // Media downloader, only exists during init
ClientMediaDownloader *m_media_downloader;
// Set of media filenames pushed by server at runtime
std::unordered_set<std::string> m_media_pushed_files;
+ // Pending downloads of dynamic media (key: token)
+ std::vector<std::pair<u32, std::unique_ptr<SingleMediaDownloader>>> m_pending_media_downloads;
// time_of_day speed approximation for old protocol
bool m_time_of_day_set = false;
*/
ClientMediaDownloader::ClientMediaDownloader():
- m_media_cache(getMediaCacheDir()),
m_httpfetch_caller(HTTPFETCH_DISCARD)
{
}
delete remote;
}
+bool ClientMediaDownloader::loadMedia(Client *client, const std::string &data,
+ const std::string &name)
+{
+ return client->loadMedia(data, name);
+}
+
void ClientMediaDownloader::addFile(const std::string &name, const std::string &sha1)
{
assert(!m_initial_step_done); // pre-condition
{
assert(!m_initial_step_done); // pre-condition
- #ifdef USE_CURL
+#ifdef USE_CURL
if (g_settings->getBool("enable_remote_media_server")) {
infostream << "Client: Adding remote server \""
m_remotes.push_back(remote);
}
- #else
+#else
infostream << "Client: Ignoring remote server \""
<< baseurl << "\" because cURL support is not compiled in"
<< std::endl;
- #endif
+#endif
}
void ClientMediaDownloader::step(Client *client)
// Check media cache
m_uncached_count = m_files.size();
for (auto &file_it : m_files) {
- std::string name = file_it.first;
+ const std::string &name = file_it.first;
FileStatus *filestatus = file_it.second;
const std::string &sha1 = filestatus->sha1;
- std::ostringstream tmp_os(std::ios_base::binary);
- bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
-
- // If found in cache, try to load it from there
- if (found_in_cache) {
- bool success = checkAndLoad(name, sha1,
- tmp_os.str(), true, client);
- if (success) {
- filestatus->received = true;
- m_uncached_count--;
- }
+ if (tryLoadFromCache(name, sha1, client)) {
+ filestatus->received = true;
+ m_uncached_count--;
}
}
assert(m_uncached_received_count == 0);
// Create the media cache dir if we are likely to write to it
- if (m_uncached_count != 0) {
- bool did = fs::CreateAllDirs(getMediaCacheDir());
- if (!did) {
- errorstream << "Client: "
- << "Could not create media cache directory: "
- << getMediaCacheDir()
- << std::endl;
- }
- }
+ if (m_uncached_count != 0)
+ createCacheDirs();
// If we found all files in the cache, report this fact to the server.
// If the server reported no remote servers, immediately start
// available on this server, add this server
// to the available_remotes array
- for(std::map<std::string, FileStatus*>::iterator
- it = m_files.upper_bound(m_name_bound);
+ for(auto it = m_files.upper_bound(m_name_bound);
it != m_files.end(); ++it) {
FileStatus *f = it->second;
if (!f->received && sha1_set.count(f->sha1))
std::string name;
{
- std::unordered_map<unsigned long, std::string>::iterator it =
- m_remote_file_transfers.find(fetch_result.request_id);
+ auto it = m_remote_file_transfers.find(fetch_result.request_id);
assert(it != m_remote_file_transfers.end());
name = it->second;
m_remote_file_transfers.erase(it);
{
bool changing_name_bound = true;
- for (std::map<std::string, FileStatus*>::iterator
- files_iter = m_files.upper_bound(m_name_bound);
+ for (auto files_iter = m_files.upper_bound(m_name_bound);
files_iter != m_files.end(); ++files_iter) {
// Abort if active fetch limit is exceeded
}
}
-void ClientMediaDownloader::conventionalTransferDone(
+bool ClientMediaDownloader::conventionalTransferDone(
const std::string &name,
const std::string &data,
Client *client)
{
// Check that file was announced
- std::map<std::string, FileStatus*>::iterator
- file_iter = m_files.find(name);
+ auto file_iter = m_files.find(name);
if (file_iter == m_files.end()) {
errorstream << "Client: server sent media file that was"
<< "not announced, ignoring it: \"" << name << "\""
<< std::endl;
- return;
+ return false;
}
FileStatus *filestatus = file_iter->second;
assert(filestatus != NULL);
errorstream << "Client: server sent media file that we already"
<< "received, ignoring it: \"" << name << "\""
<< std::endl;
- return;
+ return true;
}
// Mark file as received, regardless of whether loading it works and
// Check that received file matches announced checksum
// If so, load it
checkAndLoad(name, filestatus->sha1, data, false, client);
+
+ return true;
+}
+
+/*
+ IClientMediaDownloader
+*/
+
+IClientMediaDownloader::IClientMediaDownloader():
+ m_media_cache(getMediaCacheDir()), m_write_to_cache(true)
+{
}
-bool ClientMediaDownloader::checkAndLoad(
+void IClientMediaDownloader::createCacheDirs()
+{
+ if (!m_write_to_cache)
+ return;
+
+ std::string path = getMediaCacheDir();
+ if (!fs::CreateAllDirs(path)) {
+ errorstream << "Client: Could not create media cache directory: "
+ << path << std::endl;
+ }
+}
+
+bool IClientMediaDownloader::tryLoadFromCache(const std::string &name,
+ const std::string &sha1, Client *client)
+{
+ std::ostringstream tmp_os(std::ios_base::binary);
+ bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
+
+ // If found in cache, try to load it from there
+ if (found_in_cache)
+ return checkAndLoad(name, sha1, tmp_os.str(), true, client);
+
+ return false;
+}
+
+bool IClientMediaDownloader::checkAndLoad(
const std::string &name, const std::string &sha1,
const std::string &data, bool is_from_cache, Client *client)
{
}
// Checksum is ok, try loading the file
- bool success = client->loadMedia(data, name);
+ bool success = loadMedia(client, data, name);
if (!success) {
infostream << "Client: "
<< "Failed to load " << cached_or_received << " media: "
<< std::endl;
// Update cache (unless we just loaded the file from the cache)
- if (!is_from_cache)
+ if (!is_from_cache && m_write_to_cache)
m_media_cache.update(sha1_hex, data);
return true;
// Write list of hashes of files that have not been
// received (found in cache) yet
- for (std::map<std::string, FileStatus*>::iterator
- it = m_files.begin();
- it != m_files.end(); ++it) {
- if (!it->second->received) {
- FATAL_ERROR_IF(it->second->sha1.size() != 20, "Invalid SHA1 size");
- os << it->second->sha1;
+ for (const auto &it : m_files) {
+ if (!it.second->received) {
+ FATAL_ERROR_IF(it.second->sha1.size() != 20, "Invalid SHA1 size");
+ os << it.second->sha1;
}
}
result.insert(data.substr(pos, 20));
}
}
+
+/*
+ SingleMediaDownloader
+*/
+
+SingleMediaDownloader::SingleMediaDownloader(bool write_to_cache):
+ m_httpfetch_caller(HTTPFETCH_DISCARD)
+{
+ m_write_to_cache = write_to_cache;
+}
+
+SingleMediaDownloader::~SingleMediaDownloader()
+{
+ if (m_httpfetch_caller != HTTPFETCH_DISCARD)
+ httpfetch_caller_free(m_httpfetch_caller);
+}
+
+bool SingleMediaDownloader::loadMedia(Client *client, const std::string &data,
+ const std::string &name)
+{
+ return client->loadMedia(data, name, true);
+}
+
+void SingleMediaDownloader::addFile(const std::string &name, const std::string &sha1)
+{
+ assert(m_stage == STAGE_INIT); // pre-condition
+
+ assert(!name.empty());
+ assert(sha1.size() == 20);
+
+ FATAL_ERROR_IF(!m_file_name.empty(), "Cannot add a second file");
+ m_file_name = name;
+ m_file_sha1 = sha1;
+}
+
+void SingleMediaDownloader::addRemoteServer(const std::string &baseurl)
+{
+ assert(m_stage == STAGE_INIT); // pre-condition
+
+ if (g_settings->getBool("enable_remote_media_server"))
+ m_remotes.emplace_back(baseurl);
+}
+
+void SingleMediaDownloader::step(Client *client)
+{
+ if (m_stage == STAGE_INIT) {
+ m_stage = STAGE_CACHE_CHECKED;
+ initialStep(client);
+ }
+
+ // Remote media: check for completion of fetches
+ if (m_httpfetch_caller != HTTPFETCH_DISCARD) {
+ HTTPFetchResult fetch_result;
+ while (httpfetch_async_get(m_httpfetch_caller, fetch_result)) {
+ remoteMediaReceived(fetch_result, client);
+ }
+ }
+}
+
+bool SingleMediaDownloader::conventionalTransferDone(const std::string &name,
+ const std::string &data, Client *client)
+{
+ if (name != m_file_name)
+ return false;
+
+ // Mark file as received unconditionally and try to load it
+ m_stage = STAGE_DONE;
+ checkAndLoad(name, m_file_sha1, data, false, client);
+ return true;
+}
+
+void SingleMediaDownloader::initialStep(Client *client)
+{
+ if (tryLoadFromCache(m_file_name, m_file_sha1, client))
+ m_stage = STAGE_DONE;
+ if (isDone())
+ return;
+
+ createCacheDirs();
+
+ // If the server reported no remote servers, immediately fall back to
+ // conventional transfer.
+ if (!USE_CURL || m_remotes.empty()) {
+ startConventionalTransfer(client);
+ } else {
+ // Otherwise start by requesting the file from the first remote media server
+ m_httpfetch_caller = httpfetch_caller_alloc();
+ m_current_remote = 0;
+ startRemoteMediaTransfer();
+ }
+}
+
+void SingleMediaDownloader::remoteMediaReceived(
+ const HTTPFetchResult &fetch_result, Client *client)
+{
+ sanity_check(!isDone());
+ sanity_check(m_current_remote >= 0);
+
+ // If fetch succeeded, try to load it
+ if (fetch_result.succeeded) {
+ bool success = checkAndLoad(m_file_name, m_file_sha1,
+ fetch_result.data, false, client);
+ if (success) {
+ m_stage = STAGE_DONE;
+ return;
+ }
+ }
+
+ // Otherwise try the next remote server or fall back to conventional transfer
+ m_current_remote++;
+ if (m_current_remote >= (int)m_remotes.size()) {
+ infostream << "Client: Failed to remote-fetch \"" << m_file_name
+ << "\". Requesting it the usual way." << std::endl;
+ m_current_remote = -1;
+ startConventionalTransfer(client);
+ } else {
+ startRemoteMediaTransfer();
+ }
+}
+
+void SingleMediaDownloader::startRemoteMediaTransfer()
+{
+ std::string url = m_remotes.at(m_current_remote) + hex_encode(m_file_sha1);
+ verbosestream << "Client: Requesting remote media file "
+ << "\"" << m_file_name << "\" " << "\"" << url << "\"" << std::endl;
+
+ HTTPFetchRequest fetch_request;
+ fetch_request.url = url;
+ fetch_request.caller = m_httpfetch_caller;
+ fetch_request.request_id = m_httpfetch_next_id;
+ fetch_request.timeout = g_settings->getS32("curl_file_download_timeout");
+ httpfetch_async(fetch_request);
+
+ m_httpfetch_next_id++;
+}
+
+void SingleMediaDownloader::startConventionalTransfer(Client *client)
+{
+ std::vector<std::string> requests;
+ requests.emplace_back(m_file_name);
+ client->request_media(requests);
+}
#include "irrlichttypes.h"
#include "filecache.h"
+#include "util/basic_macros.h"
#include <ostream>
#include <map>
#include <set>
bool clientMediaUpdateCache(const std::string &raw_hash,
const std::string &filedata);
-class ClientMediaDownloader
+// more of a base class than an interface but this name was most convenient...
+class IClientMediaDownloader
+{
+public:
+ DISABLE_CLASS_COPY(IClientMediaDownloader)
+
+ virtual bool isStarted() const = 0;
+
+ // If this returns true, the downloader is done and can be deleted
+ virtual bool isDone() const = 0;
+
+ // Add a file to the list of required file (but don't fetch it yet)
+ virtual void addFile(const std::string &name, const std::string &sha1) = 0;
+
+ // Add a remote server to the list; ignored if not built with cURL
+ virtual void addRemoteServer(const std::string &baseurl) = 0;
+
+ // Steps the media downloader:
+ // - May load media into client by calling client->loadMedia()
+ // - May check media cache for files
+ // - May add files to media cache
+ // - May start remote transfers by calling httpfetch_async
+ // - May check for completion of current remote transfers
+ // - May start conventional transfers by calling client->request_media()
+ // - May inform server that all media has been loaded
+ // by calling client->received_media()
+ // After step has been called once, don't call addFile/addRemoteServer.
+ virtual void step(Client *client) = 0;
+
+ // Must be called for each file received through TOCLIENT_MEDIA
+ // returns true if this file belongs to this downloader
+ virtual bool conventionalTransferDone(const std::string &name,
+ const std::string &data, Client *client) = 0;
+
+protected:
+ IClientMediaDownloader();
+ virtual ~IClientMediaDownloader() = default;
+
+ // Forwards the call to the appropriate Client method
+ virtual bool loadMedia(Client *client, const std::string &data,
+ const std::string &name) = 0;
+
+ void createCacheDirs();
+
+ bool tryLoadFromCache(const std::string &name, const std::string &sha1,
+ Client *client);
+
+ bool checkAndLoad(const std::string &name, const std::string &sha1,
+ const std::string &data, bool is_from_cache, Client *client);
+
+ // Filesystem-based media cache
+ FileCache m_media_cache;
+ bool m_write_to_cache;
+};
+
+class ClientMediaDownloader : public IClientMediaDownloader
{
public:
ClientMediaDownloader();
return 0.0f;
}
- bool isStarted() const {
+ bool isStarted() const override {
return m_initial_step_done;
}
- // If this returns true, the downloader is done and can be deleted
- bool isDone() const {
+ bool isDone() const override {
return m_initial_step_done &&
m_uncached_received_count == m_uncached_count;
}
- // Add a file to the list of required file (but don't fetch it yet)
- void addFile(const std::string &name, const std::string &sha1);
+ void addFile(const std::string &name, const std::string &sha1) override;
- // Add a remote server to the list; ignored if not built with cURL
- void addRemoteServer(const std::string &baseurl);
+ void addRemoteServer(const std::string &baseurl) override;
- // Steps the media downloader:
- // - May load media into client by calling client->loadMedia()
- // - May check media cache for files
- // - May add files to media cache
- // - May start remote transfers by calling httpfetch_async
- // - May check for completion of current remote transfers
- // - May start conventional transfers by calling client->request_media()
- // - May inform server that all media has been loaded
- // by calling client->received_media()
- // After step has been called once, don't call addFile/addRemoteServer.
- void step(Client *client);
+ void step(Client *client) override;
- // Must be called for each file received through TOCLIENT_MEDIA
- void conventionalTransferDone(
+ bool conventionalTransferDone(
const std::string &name,
const std::string &data,
- Client *client);
+ Client *client) override;
+
+protected:
+ bool loadMedia(Client *client, const std::string &data,
+ const std::string &name) override;
private:
struct FileStatus {
void startRemoteMediaTransfers();
void startConventionalTransfers(Client *client);
- bool checkAndLoad(const std::string &name, const std::string &sha1,
- const std::string &data, bool is_from_cache,
- Client *client);
-
- std::string serializeRequiredHashSet();
static void deSerializeHashSet(const std::string &data,
std::set<std::string> &result);
+ std::string serializeRequiredHashSet();
// Maps filename to file status
std::map<std::string, FileStatus*> m_files;
// Array of remote media servers
std::vector<RemoteServerStatus*> m_remotes;
- // Filesystem-based media cache
- FileCache m_media_cache;
-
// Has an attempt been made to load media files from the file cache?
// Have hash sets been requested from remote servers?
bool m_initial_step_done = false;
std::string m_name_bound = "";
};
+
+// A media downloader that only downloads a single file.
+// It does/doesn't do several things the normal downloader does:
+// - won't fetch hash sets from remote servers
+// - will mark loaded media as coming from file push
+// - writing to file cache is optional
+class SingleMediaDownloader : public IClientMediaDownloader
+{
+public:
+ SingleMediaDownloader(bool write_to_cache);
+ ~SingleMediaDownloader();
+
+ bool isStarted() const override {
+ return m_stage > STAGE_INIT;
+ }
+
+ bool isDone() const override {
+ return m_stage >= STAGE_DONE;
+ }
+
+ void addFile(const std::string &name, const std::string &sha1) override;
+
+ void addRemoteServer(const std::string &baseurl) override;
+
+ void step(Client *client) override;
+
+ bool conventionalTransferDone(const std::string &name,
+ const std::string &data, Client *client) override;
+
+protected:
+ bool loadMedia(Client *client, const std::string &data,
+ const std::string &name) override;
+
+private:
+ void initialStep(Client *client);
+ void remoteMediaReceived(const HTTPFetchResult &fetch_result, Client *client);
+ void startRemoteMediaTransfer();
+ void startConventionalTransfer(Client *client);
+
+ enum Stage {
+ STAGE_INIT,
+ STAGE_CACHE_CHECKED, // we have tried to load the file from cache
+ STAGE_DONE
+ };
+
+ // Information about the one file we want to fetch
+ std::string m_file_name;
+ std::string m_file_sha1;
+ s32 m_current_remote;
+
+ // Array of remote media servers
+ std::vector<std::string> m_remotes;
+
+ enum Stage m_stage = STAGE_INIT;
+
+ // Status of remote transfers
+ unsigned long m_httpfetch_caller;
+ unsigned long m_httpfetch_next_id = 0;
+
+};
#include "util/string.h"
#include <iostream>
#include <cstdio>
+#include <cstdlib>
#include <cstring>
#include <cerrno>
+#include <unistd.h>
#include <fstream>
#include "log.h"
#include "config.h"
return rename(from.c_str(), to.c_str()) == 0;
}
+std::string CreateTempFile()
+{
+ std::string path = TempPath() + DIR_DELIM "MT_XXXXXX";
+ int fd = mkstemp(&path[0]); // modifies path
+ if (fd == -1)
+ return "";
+ close(fd);
+ return path;
+}
+
} // namespace fs
// Returns path to temp directory, can return "" on error
std::string TempPath();
+// Returns path to securely-created temporary file (will already exist when this function returns)
+// can return "" on error
+std::string CreateTempFile();
+
/* Returns a list of subdirectories, including the path itself, but excluding
hidden directories (whose names start with . or _)
*/
null_command_factory, // 0x3e
null_command_factory, // 0x3f
{ "TOSERVER_REQUEST_MEDIA", 1, true }, // 0x40
- null_command_factory, // 0x41
+ { "TOSERVER_HAVE_MEDIA", 2, true }, // 0x41
null_command_factory, // 0x42
{ "TOSERVER_CLIENT_READY", 1, true }, // 0x43
null_command_factory, // 0x44
m_media_downloader->addFile(name, sha1_raw);
}
- try {
+ {
std::string str;
-
*pkt >> str;
Strfnd sf(str);
- while(!sf.at_end()) {
+ while (!sf.at_end()) {
std::string baseurl = trim(sf.next(","));
- if (!baseurl.empty())
+ if (!baseurl.empty()) {
+ m_remote_media_servers.emplace_back(baseurl);
m_media_downloader->addRemoteServer(baseurl);
+ }
}
}
- catch(SerializationError& e) {
- // not supported by server or turned off
- }
m_media_downloader->step(this);
}
if (num_files == 0)
return;
- if (!m_media_downloader || !m_media_downloader->isStarted()) {
- const char *problem = m_media_downloader ?
- "media has not been requested" :
- "all media has been received already";
- errorstream << "Client: Received media but "
- << problem << "! "
- << " bunch " << bunch_i << "/" << num_bunches
- << " files=" << num_files
- << " size=" << pkt->getSize() << std::endl;
- return;
- }
+ bool init_phase = m_media_downloader && m_media_downloader->isStarted();
- // Mesh update thread must be stopped while
- // updating content definitions
- sanity_check(!m_mesh_update_thread.isRunning());
+ if (init_phase) {
+ // Mesh update thread must be stopped while
+ // updating content definitions
+ sanity_check(!m_mesh_update_thread.isRunning());
+ }
- for (u32 i=0; i < num_files; i++) {
- std::string name;
+ for (u32 i = 0; i < num_files; i++) {
+ std::string name, data;
*pkt >> name;
+ data = pkt->readLongString();
- std::string data = pkt->readLongString();
-
- m_media_downloader->conventionalTransferDone(
- name, data, this);
+ bool ok = false;
+ if (init_phase) {
+ ok = m_media_downloader->conventionalTransferDone(name, data, this);
+ } else {
+ // Check pending dynamic transfers, one of them must be it
+ for (const auto &it : m_pending_media_downloads) {
+ if (it.second->conventionalTransferDone(name, data, this)) {
+ ok = true;
+ break;
+ }
+ }
+ }
+ if (!ok) {
+ errorstream << "Client: Received media \"" << name
+ << "\" but no downloads pending. " << num_bunches << " bunches, "
+ << num_files << " in this one. (init_phase=" << init_phase
+ << ")" << std::endl;
+ }
}
}
void Client::handleCommand_MediaPush(NetworkPacket *pkt)
{
std::string raw_hash, filename, filedata;
+ u32 token;
bool cached;
*pkt >> raw_hash >> filename >> cached;
- filedata = pkt->readLongString();
+ if (m_proto_ver >= 40)
+ *pkt >> token;
+ else
+ filedata = pkt->readLongString();
- if (raw_hash.size() != 20 || filedata.empty() || filename.empty() ||
+ if (raw_hash.size() != 20 || filename.empty() ||
+ (m_proto_ver < 40 && filedata.empty()) ||
!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) {
throw PacketError("Illegal filename, data or hash");
}
- verbosestream << "Server pushes media file \"" << filename << "\" with "
- << filedata.size() << " bytes of data (cached=" << cached
- << ")" << std::endl;
+ verbosestream << "Server pushes media file \"" << filename << "\" ";
+ if (filedata.empty())
+ verbosestream << "to be fetched ";
+ else
+ verbosestream << "with " << filedata.size() << " bytes ";
+ verbosestream << "(cached=" << cached << ")" << std::endl;
if (m_media_pushed_files.count(filename) != 0) {
- // Silently ignore for synchronization purposes
+ // Ignore (but acknowledge). Previously this was for sync purposes,
+ // but even in new versions media cannot be replaced at runtime.
+ if (m_proto_ver >= 40)
+ sendHaveMedia({ token });
return;
}
- // Compute and check checksum of data
- std::string computed_hash;
- {
- SHA1 ctx;
- ctx.addBytes(filedata.c_str(), filedata.size());
- unsigned char *buf = ctx.getDigest();
- computed_hash.assign((char*) buf, 20);
- free(buf);
- }
- if (raw_hash != computed_hash) {
- verbosestream << "Hash of file data mismatches, ignoring." << std::endl;
+ if (!filedata.empty()) {
+ // LEGACY CODEPATH
+ // Compute and check checksum of data
+ std::string computed_hash;
+ {
+ SHA1 ctx;
+ ctx.addBytes(filedata.c_str(), filedata.size());
+ unsigned char *buf = ctx.getDigest();
+ computed_hash.assign((char*) buf, 20);
+ free(buf);
+ }
+ if (raw_hash != computed_hash) {
+ verbosestream << "Hash of file data mismatches, ignoring." << std::endl;
+ return;
+ }
+
+ // Actually load media
+ loadMedia(filedata, filename, true);
+ m_media_pushed_files.insert(filename);
+
+ // Cache file for the next time when this client joins the same server
+ if (cached)
+ clientMediaUpdateCache(raw_hash, filedata);
return;
}
- // Actually load media
- loadMedia(filedata, filename, true);
m_media_pushed_files.insert(filename);
- // Cache file for the next time when this client joins the same server
- if (cached)
- clientMediaUpdateCache(raw_hash, filedata);
+ // create a downloader for this file
+ auto downloader = new SingleMediaDownloader(cached);
+ m_pending_media_downloads.emplace_back(token, downloader);
+ downloader->addFile(filename, raw_hash);
+ for (const auto &baseurl : m_remote_media_servers)
+ downloader->addRemoteServer(baseurl);
+
+ downloader->step(this);
}
/*
Minimap modes
PROTOCOL VERSION 40:
Added 'basic_debug' privilege
+ TOCLIENT_MEDIA_PUSH changed, TOSERVER_HAVE_MEDIA added
*/
#define LATEST_PROTOCOL_VERSION 40
/*
std::string raw_hash
std::string filename
+ u32 callback_token
bool should_be_cached
- u32 len
- char filedata[len]
*/
// (oops, there is some gap here)
}
*/
- TOSERVER_RECEIVED_MEDIA = 0x41, // Obsolete
+ TOSERVER_HAVE_MEDIA = 0x41,
+ /*
+ u8 number of callback tokens
+ for each:
+ u32 token
+ */
+
TOSERVER_BREATH = 0x42, // Obsolete
TOSERVER_CLIENT_READY = 0x43,
null_command_handler, // 0x3e
null_command_handler, // 0x3f
{ "TOSERVER_REQUEST_MEDIA", TOSERVER_STATE_STARTUP, &Server::handleCommand_RequestMedia }, // 0x40
- null_command_handler, // 0x41
+ { "TOSERVER_HAVE_MEDIA", TOSERVER_STATE_INGAME, &Server::handleCommand_HaveMedia }, // 0x41
null_command_handler, // 0x42
{ "TOSERVER_CLIENT_READY", TOSERVER_STATE_STARTUP, &Server::handleCommand_ClientReady }, // 0x43
null_command_handler, // 0x44
{ "TOCLIENT_TIME_OF_DAY", 0, true }, // 0x29
{ "TOCLIENT_CSM_RESTRICTION_FLAGS", 0, true }, // 0x2A
{ "TOCLIENT_PLAYER_SPEED", 0, true }, // 0x2B
- { "TOCLIENT_MEDIA_PUSH", 0, true }, // 0x2C (sent over channel 1 too)
+ { "TOCLIENT_MEDIA_PUSH", 0, true }, // 0x2C (sent over channel 1 too if legacy)
null_command_factory, // 0x2D
null_command_factory, // 0x2E
{ "TOCLIENT_CHAT_MESSAGE", 0, true }, // 0x2F
session_t peer_id = pkt->getPeerId();
infostream << "Sending " << numfiles << " files to " <<
getPlayerName(peer_id) << std::endl;
- verbosestream << "TOSERVER_REQUEST_MEDIA: " << std::endl;
+ verbosestream << "TOSERVER_REQUEST_MEDIA: requested file(s)" << std::endl;
for (u16 i = 0; i < numfiles; i++) {
std::string name;
*pkt >> name;
- tosend.push_back(name);
- verbosestream << "TOSERVER_REQUEST_MEDIA: requested file "
- << name << std::endl;
+ tosend.emplace_back(name);
+ verbosestream << " " << name << std::endl;
}
sendRequestedMedia(peer_id, tosend);
broadcastModChannelMessage(channel_name, channel_msg, peer_id);
}
+
+void Server::handleCommand_HaveMedia(NetworkPacket *pkt)
+{
+ std::vector<u32> tokens;
+ u8 numtokens;
+
+ *pkt >> numtokens;
+ for (u16 i = 0; i < numtokens; i++) {
+ u32 n;
+ *pkt >> n;
+ tokens.emplace_back(n);
+ }
+
+ const session_t peer_id = pkt->getPeerId();
+ auto player = m_env->getPlayer(peer_id);
+
+ for (const u32 token : tokens) {
+ auto it = m_pending_dyn_media.find(token);
+ if (it == m_pending_dyn_media.end())
+ continue;
+ if (it->second.waiting_players.count(peer_id)) {
+ it->second.waiting_players.erase(peer_id);
+ if (player)
+ getScriptIface()->on_dynamic_media_added(token, player->getName());
+ }
+ }
+}
#include "cpp_api/s_server.h"
#include "cpp_api/s_internal.h"
#include "common/c_converter.h"
+#include "util/numeric.h" // myrand
bool ScriptApiServer::getAuth(const std::string &playername,
std::string *dst_password,
return ret;
}
+
+u32 ScriptApiServer::allocateDynamicMediaCallback(int f_idx)
+{
+ lua_State *L = getStack();
+
+ if (f_idx < 0)
+ f_idx = lua_gettop(L) + f_idx + 1;
+
+ lua_getglobal(L, "core");
+ lua_getfield(L, -1, "dynamic_media_callbacks");
+ luaL_checktype(L, -1, LUA_TTABLE);
+
+ // Find a randomly generated token that doesn't exist yet
+ int tries = 100;
+ u32 token;
+ while (1) {
+ token = myrand();
+ lua_rawgeti(L, -2, token);
+ bool is_free = lua_isnil(L, -1);
+ lua_pop(L, 1);
+ if (is_free)
+ break;
+ if (--tries < 0)
+ FATAL_ERROR("Ran out of callbacks IDs?!");
+ }
+
+ // core.dynamic_media_callbacks[token] = callback_func
+ lua_pushvalue(L, f_idx);
+ lua_rawseti(L, -2, token);
+
+ lua_pop(L, 2);
+
+ verbosestream << "allocateDynamicMediaCallback() = " << token << std::endl;
+ return token;
+}
+
+void ScriptApiServer::freeDynamicMediaCallback(u32 token)
+{
+ lua_State *L = getStack();
+
+ verbosestream << "freeDynamicMediaCallback(" << token << ")" << std::endl;
+
+ // core.dynamic_media_callbacks[token] = nil
+ lua_getglobal(L, "core");
+ lua_getfield(L, -1, "dynamic_media_callbacks");
+ luaL_checktype(L, -1, LUA_TTABLE);
+ lua_pushnil(L);
+ lua_rawseti(L, -2, token);
+ lua_pop(L, 2);
+}
+
+void ScriptApiServer::on_dynamic_media_added(u32 token, const char *playername)
+{
+ SCRIPTAPI_PRECHECKHEADER
+
+ int error_handler = PUSH_ERROR_HANDLER(L);
+ lua_getglobal(L, "core");
+ lua_getfield(L, -1, "dynamic_media_callbacks");
+ luaL_checktype(L, -1, LUA_TTABLE);
+ lua_rawgeti(L, -1, token);
+ luaL_checktype(L, -1, LUA_TFUNCTION);
+
+ lua_pushstring(L, playername);
+ PCALL_RES(lua_pcall(L, 1, 0, error_handler));
+}
const std::string &password);
bool setPassword(const std::string &playername,
const std::string &password);
+
+ /* dynamic media handling */
+ u32 allocateDynamicMediaCallback(int f_idx);
+ void freeDynamicMediaCallback(u32 token);
+ void on_dynamic_media_added(u32 token, const char *playername);
+
private:
void getAuthHandler();
void readPrivileges(int index, std::set<std::string> &result);
}
// dynamic_add_media(filepath)
-int ModApiServer::l_dynamic_add_media_raw(lua_State *L)
+int ModApiServer::l_dynamic_add_media(lua_State *L)
{
NO_MAP_LOCK_REQUIRED;
if (!getEnv(L))
throw LuaError("Dynamic media cannot be added before server has started up");
+ Server *server = getServer(L);
- std::string filepath = readParam<std::string>(L, 1);
- CHECK_SECURE_PATH(L, filepath.c_str(), false);
+ std::string filepath;
+ std::string to_player;
+ bool ephemeral = false;
- std::vector<RemotePlayer*> sent_to;
- bool ok = getServer(L)->dynamicAddMedia(filepath, sent_to);
- if (ok) {
- // (see wrapper code in builtin)
- lua_createtable(L, sent_to.size(), 0);
- int i = 0;
- for (RemotePlayer *player : sent_to) {
- lua_pushstring(L, player->getName());
- lua_rawseti(L, -2, ++i);
- }
+ if (lua_istable(L, 1)) {
+ getstringfield(L, 1, "filepath", filepath);
+ getstringfield(L, 1, "to_player", to_player);
+ getboolfield(L, 1, "ephemeral", ephemeral);
} else {
- lua_pushboolean(L, false);
+ filepath = readParam<std::string>(L, 1);
}
+ if (filepath.empty())
+ luaL_typerror(L, 1, "non-empty string");
+ luaL_checktype(L, 2, LUA_TFUNCTION);
+
+ CHECK_SECURE_PATH(L, filepath.c_str(), false);
+
+ u32 token = server->getScriptIface()->allocateDynamicMediaCallback(2);
+
+ bool ok = server->dynamicAddMedia(filepath, token, to_player, ephemeral);
+ if (!ok)
+ server->getScriptIface()->freeDynamicMediaCallback(token);
+ lua_pushboolean(L, ok);
return 1;
}
API_FCT(sound_play);
API_FCT(sound_stop);
API_FCT(sound_fade);
- API_FCT(dynamic_add_media_raw);
+ API_FCT(dynamic_add_media);
API_FCT(get_player_information);
API_FCT(get_player_privs);
static int l_sound_fade(lua_State *L);
// dynamic_add_media(filepath)
- static int l_dynamic_add_media_raw(lua_State *L);
+ static int l_dynamic_add_media(lua_State *L);
// get_player_privs(name, text)
static int l_get_player_privs(lua_State *L);
} else {
m_lag_gauge->increment(dtime/100);
}
+
+ {
+ float &counter = m_step_pending_dyn_media_timer;
+ counter += dtime;
+ if (counter >= 5.0f) {
+ stepPendingDynMediaCallbacks(counter);
+ counter = 0;
+ }
+ }
+
+
#if USE_CURL
// send masterserver announce
{
std::string lang_suffix;
lang_suffix.append(".").append(lang_code).append(".tr");
for (const auto &i : m_media) {
+ if (i.second.no_announce)
+ continue;
if (str_ends_with(i.first, ".tr") && !str_ends_with(i.first, lang_suffix))
continue;
media_sent++;
pkt << media_sent;
for (const auto &i : m_media) {
+ if (i.second.no_announce)
+ continue;
if (str_ends_with(i.first, ".tr") && !str_ends_with(i.first, lang_suffix))
continue;
pkt << i.first << i.second.sha1_digest;
std::string path;
std::string data;
- SendableMedia(const std::string &name_="", const std::string &path_="",
- const std::string &data_=""):
- name(name_),
- path(path_),
- data(data_)
+ SendableMedia(const std::string &name, const std::string &path,
+ std::string &&data):
+ name(name), path(path), data(std::move(data))
{}
};
continue;
}
- //TODO get path + name
- std::string tpath = m_media[name].path;
+ const auto &m = m_media[name];
// Read data
- std::ifstream fis(tpath.c_str(), std::ios_base::binary);
- if(!fis.good()){
- errorstream<<"Server::sendRequestedMedia(): Could not open \""
- <<tpath<<"\" for reading"<<std::endl;
- continue;
- }
- std::ostringstream tmp_os(std::ios_base::binary);
- bool bad = false;
- for(;;) {
- char buf[1024];
- fis.read(buf, 1024);
- std::streamsize len = fis.gcount();
- tmp_os.write(buf, len);
- file_size_bunch_total += len;
- if(fis.eof())
- break;
- if(!fis.good()) {
- bad = true;
- break;
- }
- }
- if (bad) {
- errorstream<<"Server::sendRequestedMedia(): Failed to read \""
- <<name<<"\""<<std::endl;
+ std::string data;
+ if (!fs::ReadFile(m.path, data)) {
+ errorstream << "Server::sendRequestedMedia(): Failed to read \""
+ << name << "\"" << std::endl;
continue;
}
- /*infostream<<"Server::sendRequestedMedia(): Loaded \""
- <<tname<<"\""<<std::endl;*/
+ file_size_bunch_total += data.size();
+
// Put in list
- file_bunches[file_bunches.size()-1].emplace_back(name, tpath, tmp_os.str());
+ file_bunches.back().emplace_back(name, m.path, std::move(data));
// Start next bunch if got enough data
if(file_size_bunch_total >= bytes_per_bunch) {
}
}
+void Server::stepPendingDynMediaCallbacks(float dtime)
+{
+ MutexAutoLock lock(m_env_mutex);
+
+ for (auto it = m_pending_dyn_media.begin(); it != m_pending_dyn_media.end();) {
+ it->second.expiry_timer -= dtime;
+ bool del = it->second.waiting_players.empty() || it->second.expiry_timer < 0;
+
+ if (!del) {
+ it++;
+ continue;
+ }
+
+ const auto &name = it->second.filename;
+ if (!name.empty()) {
+ assert(m_media.count(name));
+ // if no_announce isn't set we're definitely deleting the wrong file!
+ sanity_check(m_media[name].no_announce);
+
+ fs::DeleteSingleFileOrEmptyDirectory(m_media[name].path);
+ m_media.erase(name);
+ }
+ getScriptIface()->freeDynamicMediaCallback(it->first);
+ it = m_pending_dyn_media.erase(it);
+ }
+}
+
void Server::SendMinimapModes(session_t peer_id,
std::vector<MinimapMode> &modes, size_t wanted_mode)
{
SendDeleteParticleSpawner(peer_id, id);
}
-bool Server::dynamicAddMedia(const std::string &filepath,
- std::vector<RemotePlayer*> &sent_to)
+bool Server::dynamicAddMedia(std::string filepath,
+ const u32 token, const std::string &to_player, bool ephemeral)
{
std::string filename = fs::GetFilenameFromPath(filepath.c_str());
- if (m_media.find(filename) != m_media.end()) {
- errorstream << "Server::dynamicAddMedia(): file \"" << filename
- << "\" already exists in media cache" << std::endl;
- return false;
+ auto it = m_media.find(filename);
+ if (it != m_media.end()) {
+ // Allow the same path to be "added" again in certain conditions
+ if (ephemeral || it->second.path != filepath) {
+ errorstream << "Server::dynamicAddMedia(): file \"" << filename
+ << "\" already exists in media cache" << std::endl;
+ return false;
+ }
}
// Load the file and add it to our media cache
if (!ok)
return false;
+ if (ephemeral) {
+ // Create a copy of the file and swap out the path, this removes the
+ // requirement that mods keep the file accessible at the original path.
+ filepath = fs::CreateTempFile();
+ bool ok = ([&] () -> bool {
+ if (filepath.empty())
+ return false;
+ std::ofstream os(filepath.c_str(), std::ios::binary);
+ if (!os.good())
+ return false;
+ os << filedata;
+ os.close();
+ return !os.fail();
+ })();
+ if (!ok) {
+ errorstream << "Server: failed to create a copy of media file "
+ << "\"" << filename << "\"" << std::endl;
+ m_media.erase(filename);
+ return false;
+ }
+ verbosestream << "Server: \"" << filename << "\" temporarily copied to "
+ << filepath << std::endl;
+
+ m_media[filename].path = filepath;
+ m_media[filename].no_announce = true;
+ // stepPendingDynMediaCallbacks will clean this up later.
+ } else if (!to_player.empty()) {
+ m_media[filename].no_announce = true;
+ }
+
// Push file to existing clients
NetworkPacket pkt(TOCLIENT_MEDIA_PUSH, 0);
- pkt << raw_hash << filename << (bool) true;
- pkt.putLongString(filedata);
+ pkt << raw_hash << filename << (bool)ephemeral;
+
+ NetworkPacket legacy_pkt = pkt;
+ // Newer clients get asked to fetch the file (asynchronous)
+ pkt << token;
+ // Older clients have an awful hack that just throws the data at them
+ legacy_pkt.putLongString(filedata);
+
+ std::unordered_set<session_t> delivered, waiting;
m_clients.lock();
for (auto &pair : m_clients.getClientList()) {
if (pair.second->getState() < CS_DefinitionsSent)
continue;
- if (pair.second->net_proto_version < 39)
+ const auto proto_ver = pair.second->net_proto_version;
+ if (proto_ver < 39)
continue;
- if (auto player = m_env->getPlayer(pair.second->peer_id))
- sent_to.emplace_back(player);
- /*
- FIXME: this is a very awful hack
- The network layer only guarantees ordered delivery inside a channel.
- Since the very next packet could be one that uses the media, we have
- to push the media over ALL channels to ensure it is processed before
- it is used.
- In practice this means we have to send it twice:
- - channel 1 (HUD)
- - channel 0 (everything else: e.g. play_sound, object messages)
- */
- m_clients.send(pair.second->peer_id, 1, &pkt, true);
- m_clients.send(pair.second->peer_id, 0, &pkt, true);
+ const session_t peer_id = pair.second->peer_id;
+ if (!to_player.empty() && getPlayerName(peer_id) != to_player)
+ continue;
+
+ if (proto_ver < 40) {
+ delivered.emplace(peer_id);
+ /*
+ The network layer only guarantees ordered delivery inside a channel.
+ Since the very next packet could be one that uses the media, we have
+ to push the media over ALL channels to ensure it is processed before
+ it is used. In practice this means channels 1 and 0.
+ */
+ m_clients.send(peer_id, 1, &legacy_pkt, true);
+ m_clients.send(peer_id, 0, &legacy_pkt, true);
+ } else {
+ waiting.emplace(peer_id);
+ Send(peer_id, &pkt);
+ }
}
m_clients.unlock();
+ // Run callback for players that already had the file delivered (legacy-only)
+ for (session_t peer_id : delivered) {
+ if (auto player = m_env->getPlayer(peer_id))
+ getScriptIface()->on_dynamic_media_added(token, player->getName());
+ }
+
+ // Save all others in our pending state
+ auto &state = m_pending_dyn_media[token];
+ state.waiting_players = std::move(waiting);
+ // regardless of success throw away the callback after a while
+ state.expiry_timer = 60.0f;
+ if (ephemeral)
+ state.filename = filename;
+
return true;
}
#include <list>
#include <map>
#include <vector>
+#include <unordered_set>
class ChatEvent;
struct ChatEventChat;
struct MediaInfo
{
std::string path;
- std::string sha1_digest;
+ std::string sha1_digest; // base64-encoded
+ bool no_announce; // true: not announced in TOCLIENT_ANNOUNCE_MEDIA (at player join)
MediaInfo(const std::string &path_="",
const std::string &sha1_digest_=""):
path(path_),
- sha1_digest(sha1_digest_)
+ sha1_digest(sha1_digest_),
+ no_announce(false)
{
}
};
void handleCommand_FirstSrp(NetworkPacket* pkt);
void handleCommand_SrpBytesA(NetworkPacket* pkt);
void handleCommand_SrpBytesM(NetworkPacket* pkt);
+ void handleCommand_HaveMedia(NetworkPacket *pkt);
void ProcessData(NetworkPacket *pkt);
void deleteParticleSpawner(const std::string &playername, u32 id);
- bool dynamicAddMedia(const std::string &filepath, std::vector<RemotePlayer*> &sent_to);
+ bool dynamicAddMedia(std::string filepath, u32 token,
+ const std::string &to_player, bool ephemeral);
ServerInventoryManager *getInventoryMgr() const { return m_inventory_mgr.get(); }
void sendDetachedInventory(Inventory *inventory, const std::string &name, session_t peer_id);
float m_timer = 0.0f;
};
+ struct PendingDynamicMediaCallback {
+ std::string filename; // only set if media entry and file is to be deleted
+ float expiry_timer;
+ std::unordered_set<session_t> waiting_players;
+ };
+
void init();
void SendMovement(session_t peer_id);
void sendMediaAnnouncement(session_t peer_id, const std::string &lang_code);
void sendRequestedMedia(session_t peer_id,
const std::vector<std::string> &tosend);
+ void stepPendingDynMediaCallbacks(float dtime);
// Adds a ParticleSpawner on peer with peer_id (PEER_ID_INEXISTENT == all)
void SendAddParticleSpawner(session_t peer_id, u16 protocol_version,
// media files known to server
std::unordered_map<std::string, MediaInfo> m_media;
+ // pending dynamic media callbacks, clients inform the server when they have a file fetched
+ std::unordered_map<u32, PendingDynamicMediaCallback> m_pending_dyn_media;
+ float m_step_pending_dyn_media_timer = 0.0f;
+
/*
Sounds
*/
projects / dragonfireclient.git / commitdiff