src/client/clientmedia.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "clientmedia.h"
  21 #include "httpfetch.h"
  22 #include "client.h"
  23 #include "filecache.h"
  24 #include "filesys.h"
  25 #include "log.h"
  26 #include "porting.h"
  27 #include "settings.h"
  28 #include "util/hex.h"
  29 #include "util/serialize.h"
  30 #include "util/sha1.h"
  31 #include "util/string.h"
  32
  33 static std::string getMediaCacheDir()
  34 {
  35         return porting::path_cache + DIR_DELIM + "media";
  36 }
  37
  38 /*
  39         ClientMediaDownloader
  40 */
  41
  42 ClientMediaDownloader::ClientMediaDownloader():
  43         m_media_cache(getMediaCacheDir()),
  44         m_httpfetch_caller(HTTPFETCH_DISCARD)
  45 {
  46 }
  47
  48 ClientMediaDownloader::~ClientMediaDownloader()
  49 {
  50         if (m_httpfetch_caller != HTTPFETCH_DISCARD)
  51                 httpfetch_caller_free(m_httpfetch_caller);
  52
  53         for (auto &file_it : m_files)
  54                 delete file_it.second;
  55
  56         for (auto &remote : m_remotes)
  57                 delete remote;
  58 }
  59
  60 void ClientMediaDownloader::addFile(const std::string &name, const std::string &sha1)
  61 {
  62         assert(!m_initial_step_done); // pre-condition
  63
  64         // if name was already announced, ignore the new announcement
  65         if (m_files.count(name) != 0) {
  66                 errorstream << "Client: ignoring duplicate media announcement "
  67                                 << "sent by server: \"" << name << "\""
  68                                 << std::endl;
  69                 return;
  70         }
  71
  72         // if name is empty or contains illegal characters, ignore the file
  73         if (name.empty() || !string_allowed(name, TEXTURENAME_ALLOWED_CHARS)) {
  74                 errorstream << "Client: ignoring illegal file name "
  75                                 << "sent by server: \"" << name << "\""
  76                                 << std::endl;
  77                 return;
  78         }
  79
  80         // length of sha1 must be exactly 20 (160 bits), else ignore the file
  81         if (sha1.size() != 20) {
  82                 errorstream << "Client: ignoring illegal SHA1 sent by server: "
  83                                 << hex_encode(sha1) << " \"" << name << "\""
  84                                 << std::endl;
  85                 return;
  86         }
  87
  88         FileStatus *filestatus = new FileStatus();
  89         filestatus->received = false;
  90         filestatus->sha1 = sha1;
  91         filestatus->current_remote = -1;
  92         m_files.insert(std::make_pair(name, filestatus));
  93 }
  94
  95 void ClientMediaDownloader::addRemoteServer(const std::string &baseurl)
  96 {
  97         assert(!m_initial_step_done);   // pre-condition
  98
  99         #ifdef USE_CURL
 100
 101         if (g_settings->getBool("enable_remote_media_server")) {
 102                 infostream << "Client: Adding remote server \""
 103                         << baseurl << "\" for media download" << std::endl;
 104
 105                 RemoteServerStatus *remote = new RemoteServerStatus();
 106                 remote->baseurl = baseurl;
 107                 remote->active_count = 0;
 108                 m_remotes.push_back(remote);
 109         }
 110
 111         #else
 112
 113         infostream << "Client: Ignoring remote server \""
 114                 << baseurl << "\" because cURL support is not compiled in"
 115                 << std::endl;
 116
 117         #endif
 118 }
 119
 120 void ClientMediaDownloader::step(Client *client)
 121 {
 122         if (!m_initial_step_done) {
 123                 initialStep(client);
 124                 m_initial_step_done = true;
 125         }
 126
 127         // Remote media: check for completion of fetches
 128         if (m_httpfetch_active) {
 129                 bool fetched_something = false;
 130                 HTTPFetchResult fetch_result;
 131
 132                 while (httpfetch_async_get(m_httpfetch_caller, fetch_result)) {
 133                         m_httpfetch_active--;
 134                         fetched_something = true;
 135
 136                         // Is this a hashset (index.mth) or a media file?
 137                         if (fetch_result.request_id < m_remotes.size())
 138                                 remoteHashSetReceived(fetch_result);
 139                         else
 140                                 remoteMediaReceived(fetch_result, client);
 141                 }
 142
 143                 if (fetched_something)
 144                         startRemoteMediaTransfers();
 145
 146                 // Did all remote transfers end and no new ones can be started?
 147                 // If so, request still missing files from the minetest server
 148                 // (Or report that we have all files.)
 149                 if (m_httpfetch_active == 0) {
 150                         if (m_uncached_received_count < m_uncached_count) {
 151                                 infostream << "Client: Failed to remote-fetch "
 152                                         << (m_uncached_count-m_uncached_received_count)
 153                                         << " files. Requesting them"
 154                                         << " the usual way." << std::endl;
 155                         }
 156                         startConventionalTransfers(client);
 157                 }
 158         }
 159 }
 160
 161 void ClientMediaDownloader::initialStep(Client *client)
 162 {
 163         // Check media cache
 164         m_uncached_count = m_files.size();
 165         for (auto &file_it : m_files) {
 166                 std::string name = file_it.first;
 167                 FileStatus *filestatus = file_it.second;
 168                 const std::string &sha1 = filestatus->sha1;
 169
 170                 std::ostringstream tmp_os(std::ios_base::binary);
 171                 bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
 172
 173                 // If found in cache, try to load it from there
 174                 if (found_in_cache) {
 175                         bool success = checkAndLoad(name, sha1,
 176                                         tmp_os.str(), true, client);
 177                         if (success) {
 178                                 filestatus->received = true;
 179                                 m_uncached_count--;
 180                         }
 181                 }
 182         }
 183
 184         assert(m_uncached_received_count == 0);
 185
 186         // Create the media cache dir if we are likely to write to it
 187         if (m_uncached_count != 0) {
 188                 bool did = fs::CreateAllDirs(getMediaCacheDir());
 189                 if (!did) {
 190                         errorstream << "Client: "
 191                                 << "Could not create media cache directory: "
 192                                 << getMediaCacheDir()
 193                                 << std::endl;
 194                 }
 195         }
 196
 197         // If we found all files in the cache, report this fact to the server.
 198         // If the server reported no remote servers, immediately start
 199         // conventional transfers. Note: if cURL support is not compiled in,
 200         // m_remotes is always empty, so "!USE_CURL" is redundant but may
 201         // reduce the size of the compiled code
 202         if (!USE_CURL || m_uncached_count == 0 || m_remotes.empty()) {
 203                 startConventionalTransfers(client);
 204         }
 205         else {
 206                 // Otherwise start off by requesting each server's sha1 set
 207
 208                 // This is the first time we use httpfetch, so alloc a caller ID
 209                 m_httpfetch_caller = httpfetch_caller_alloc();
 210                 m_httpfetch_timeout = g_settings->getS32("curl_timeout");
 211
 212                 // Set the active fetch limit to curl_parallel_limit or 84,
 213                 // whichever is greater. This gives us some leeway so that
 214                 // inefficiencies in communicating with the httpfetch thread
 215                 // don't slow down fetches too much. (We still want some limit
 216                 // so that when the first remote server returns its hash set,
 217                 // not all files are requested from that server immediately.)
 218                 // One such inefficiency is that ClientMediaDownloader::step()
 219                 // is only called a couple times per second, while httpfetch
 220                 // might return responses much faster than that.
 221                 // Note that httpfetch strictly enforces curl_parallel_limit
 222                 // but at no inter-thread communication cost. This however
 223                 // doesn't help with the aforementioned inefficiencies.
 224                 // The signifance of 84 is that it is 2*6*9 in base 13.
 225                 m_httpfetch_active_limit = g_settings->getS32("curl_parallel_limit");
 226                 m_httpfetch_active_limit = MYMAX(m_httpfetch_active_limit, 84);
 227
 228                 // Write a list of hashes that we need. This will be POSTed
 229                 // to the server using Content-Type: application/octet-stream
 230                 std::string required_hash_set = serializeRequiredHashSet();
 231
 232                 // minor fixme: this loop ignores m_httpfetch_active_limit
 233
 234                 // another minor fixme, unlikely to matter in normal usage:
 235                 // these index.mth fetches do (however) count against
 236                 // m_httpfetch_active_limit when starting actual media file
 237                 // requests, so if there are lots of remote servers that are
 238                 // not responding, those will stall new media file transfers.
 239
 240                 for (u32 i = 0; i < m_remotes.size(); ++i) {
 241                         assert(m_httpfetch_next_id == i);
 242
 243                         RemoteServerStatus *remote = m_remotes[i];
 244                         actionstream << "Client: Contacting remote server \""
 245                                 << remote->baseurl << "\"" << std::endl;
 246
 247                         HTTPFetchRequest fetch_request;
 248                         fetch_request.url =
 249                                 remote->baseurl + MTHASHSET_FILE_NAME;
 250                         fetch_request.caller = m_httpfetch_caller;
 251                         fetch_request.request_id = m_httpfetch_next_id; // == i
 252                         fetch_request.timeout = m_httpfetch_timeout;
 253                         fetch_request.connect_timeout = m_httpfetch_timeout;
 254                         fetch_request.post_data = required_hash_set;
 255                         fetch_request.extra_headers.emplace_back(
 256                                 "Content-Type: application/octet-stream");
 257                         httpfetch_async(fetch_request);
 258
 259                         m_httpfetch_active++;
 260                         m_httpfetch_next_id++;
 261                         m_outstanding_hash_sets++;
 262                 }
 263         }
 264 }
 265
 266 void ClientMediaDownloader::remoteHashSetReceived(
 267                 const HTTPFetchResult &fetch_result)
 268 {
 269         u32 remote_id = fetch_result.request_id;
 270         assert(remote_id < m_remotes.size());
 271         RemoteServerStatus *remote = m_remotes[remote_id];
 272
 273         m_outstanding_hash_sets--;
 274
 275         if (fetch_result.succeeded) {
 276                 try {
 277                         // Server sent a list of file hashes that are
 278                         // available on it, try to parse the list
 279
 280                         std::set<std::string> sha1_set;
 281                         deSerializeHashSet(fetch_result.data, sha1_set);
 282
 283                         // Parsing succeeded: For every file that is
 284                         // available on this server, add this server
 285                         // to the available_remotes array
 286
 287                         for(std::map<std::string, FileStatus*>::iterator
 288                                         it = m_files.upper_bound(m_name_bound);
 289                                         it != m_files.end(); ++it) {
 290                                 FileStatus *f = it->second;
 291                                 if (!f->received && sha1_set.count(f->sha1))
 292                                         f->available_remotes.push_back(remote_id);
 293                         }
 294                 }
 295                 catch (SerializationError &e) {
 296                         infostream << "Client: Remote server \""
 297                                 << remote->baseurl << "\" sent invalid hash set: "
 298                                 << e.what() << std::endl;
 299                 }
 300         }
 301 }
 302
 303 void ClientMediaDownloader::remoteMediaReceived(
 304                 const HTTPFetchResult &fetch_result,
 305                 Client *client)
 306 {
 307         // Some remote server sent us a file.
 308         // -> decrement number of active fetches
 309         // -> mark file as received if fetch succeeded
 310         // -> try to load media
 311
 312         std::string name;
 313         {
 314                 std::unordered_map<unsigned long, std::string>::iterator it =
 315                         m_remote_file_transfers.find(fetch_result.request_id);
 316                 assert(it != m_remote_file_transfers.end());
 317                 name = it->second;
 318                 m_remote_file_transfers.erase(it);
 319         }
 320
 321         sanity_check(m_files.count(name) != 0);
 322
 323         FileStatus *filestatus = m_files[name];
 324         sanity_check(!filestatus->received);
 325         sanity_check(filestatus->current_remote >= 0);
 326
 327         RemoteServerStatus *remote = m_remotes[filestatus->current_remote];
 328
 329         filestatus->current_remote = -1;
 330         remote->active_count--;
 331
 332         // If fetch succeeded, try to load media file
 333
 334         if (fetch_result.succeeded) {
 335                 bool success = checkAndLoad(name, filestatus->sha1,
 336                                 fetch_result.data, false, client);
 337                 if (success) {
 338                         filestatus->received = true;
 339                         assert(m_uncached_received_count < m_uncached_count);
 340                         m_uncached_received_count++;
 341                 }
 342         }
 343 }
 344
 345 s32 ClientMediaDownloader::selectRemoteServer(FileStatus *filestatus)
 346 {
 347         // Pre-conditions
 348         assert(filestatus != NULL);
 349         assert(!filestatus->received);
 350         assert(filestatus->current_remote < 0);
 351
 352         if (filestatus->available_remotes.empty())
 353                 return -1;
 354
 355         // Of all servers that claim to provide the file (and haven't
 356         // been unsuccessfully tried before), find the one with the
 357         // smallest number of currently active transfers
 358
 359         s32 best = 0;
 360         s32 best_remote_id = filestatus->available_remotes[best];
 361         s32 best_active_count = m_remotes[best_remote_id]->active_count;
 362
 363         for (u32 i = 1; i < filestatus->available_remotes.size(); ++i) {
 364                 s32 remote_id = filestatus->available_remotes[i];
 365                 s32 active_count = m_remotes[remote_id]->active_count;
 366                 if (active_count < best_active_count) {
 367                         best = i;
 368                         best_remote_id = remote_id;
 369                         best_active_count = active_count;
 370                 }
 371         }
 372
 373         filestatus->available_remotes.erase(
 374                         filestatus->available_remotes.begin() + best);
 375
 376         return best_remote_id;
 377
 378 }
 379
 380 void ClientMediaDownloader::startRemoteMediaTransfers()
 381 {
 382         bool changing_name_bound = true;
 383
 384         for (std::map<std::string, FileStatus*>::iterator
 385                         files_iter = m_files.upper_bound(m_name_bound);
 386                         files_iter != m_files.end(); ++files_iter) {
 387
 388                 // Abort if active fetch limit is exceeded
 389                 if (m_httpfetch_active >= m_httpfetch_active_limit)
 390                         break;
 391
 392                 const std::string &name = files_iter->first;
 393                 FileStatus *filestatus = files_iter->second;
 394
 395                 if (!filestatus->received && filestatus->current_remote < 0) {
 396                         // File has not been received yet and is not currently
 397                         // being transferred. Choose a server for it.
 398                         s32 remote_id = selectRemoteServer(filestatus);
 399                         if (remote_id >= 0) {
 400                                 // Found a server, so start fetching
 401                                 RemoteServerStatus *remote =
 402                                         m_remotes[remote_id];
 403
 404                                 std::string url = remote->baseurl +
 405                                         hex_encode(filestatus->sha1);
 406                                 verbosestream << "Client: "
 407                                         << "Requesting remote media file "
 408                                         << "\"" << name << "\" "
 409                                         << "\"" << url << "\"" << std::endl;
 410
 411                                 HTTPFetchRequest fetch_request;
 412                                 fetch_request.url = url;
 413                                 fetch_request.caller = m_httpfetch_caller;
 414                                 fetch_request.request_id = m_httpfetch_next_id;
 415                                 fetch_request.timeout = 0; // no data timeout!
 416                                 fetch_request.connect_timeout =
 417                                         m_httpfetch_timeout;
 418                                 httpfetch_async(fetch_request);
 419
 420                                 m_remote_file_transfers.insert(std::make_pair(
 421                                                         m_httpfetch_next_id,
 422                                                         name));
 423
 424                                 filestatus->current_remote = remote_id;
 425                                 remote->active_count++;
 426                                 m_httpfetch_active++;
 427                                 m_httpfetch_next_id++;
 428                         }
 429                 }
 430
 431                 if (filestatus->received ||
 432                                 (filestatus->current_remote < 0 &&
 433                                  !m_outstanding_hash_sets)) {
 434                         // If we arrive here, we conclusively know that we
 435                         // won't fetch this file from a remote server in the
 436                         // future. So update the name bound if possible.
 437                         if (changing_name_bound)
 438                                 m_name_bound = name;
 439                 }
 440                 else
 441                         changing_name_bound = false;
 442         }
 443
 444 }
 445
 446 void ClientMediaDownloader::startConventionalTransfers(Client *client)
 447 {
 448         assert(m_httpfetch_active == 0);        // pre-condition
 449
 450         if (m_uncached_received_count != m_uncached_count) {
 451                 // Some media files have not been received yet, use the
 452                 // conventional slow method (minetest protocol) to get them
 453                 std::vector<std::string> file_requests;
 454                 for (auto &file : m_files) {
 455                         if (!file.second->received)
 456                                 file_requests.push_back(file.first);
 457                 }
 458                 assert((s32) file_requests.size() ==
 459                                 m_uncached_count - m_uncached_received_count);
 460                 client->request_media(file_requests);
 461         }
 462 }
 463
 464 void ClientMediaDownloader::conventionalTransferDone(
 465                 const std::string &name,
 466                 const std::string &data,
 467                 Client *client)
 468 {
 469         // Check that file was announced
 470         std::map<std::string, FileStatus*>::iterator
 471                 file_iter = m_files.find(name);
 472         if (file_iter == m_files.end()) {
 473                 errorstream << "Client: server sent media file that was"
 474                         << "not announced, ignoring it: \"" << name << "\""
 475                         << std::endl;
 476                 return;
 477         }
 478         FileStatus *filestatus = file_iter->second;
 479         assert(filestatus != NULL);
 480
 481         // Check that file hasn't already been received
 482         if (filestatus->received) {
 483                 errorstream << "Client: server sent media file that we already"
 484                         << "received, ignoring it: \"" << name << "\""
 485                         << std::endl;
 486                 return;
 487         }
 488
 489         // Mark file as received, regardless of whether loading it works and
 490         // whether the checksum matches (because at this point there is no
 491         // other server that could send a replacement)
 492         filestatus->received = true;
 493         assert(m_uncached_received_count < m_uncached_count);
 494         m_uncached_received_count++;
 495
 496         // Check that received file matches announced checksum
 497         // If so, load it
 498         checkAndLoad(name, filestatus->sha1, data, false, client);
 499 }
 500
 501 bool ClientMediaDownloader::checkAndLoad(
 502                 const std::string &name, const std::string &sha1,
 503                 const std::string &data, bool is_from_cache, Client *client)
 504 {
 505         const char *cached_or_received = is_from_cache ? "cached" : "received";
 506         const char *cached_or_received_uc = is_from_cache ? "Cached" : "Received";
 507         std::string sha1_hex = hex_encode(sha1);
 508
 509         // Compute actual checksum of data
 510         std::string data_sha1;
 511         {
 512                 SHA1 data_sha1_calculator;
 513                 data_sha1_calculator.addBytes(data.c_str(), data.size());
 514                 unsigned char *data_tmpdigest = data_sha1_calculator.getDigest();
 515                 data_sha1.assign((char*) data_tmpdigest, 20);
 516                 free(data_tmpdigest);
 517         }
 518
 519         // Check that received file matches announced checksum
 520         if (data_sha1 != sha1) {
 521                 std::string data_sha1_hex = hex_encode(data_sha1);
 522                 infostream << "Client: "
 523                         << cached_or_received_uc << " media file "
 524                         << sha1_hex << " \"" << name << "\" "
 525                         << "mismatches actual checksum " << data_sha1_hex
 526                         << std::endl;
 527                 return false;
 528         }
 529
 530         // Checksum is ok, try loading the file
 531         bool success = client->loadMedia(data, name);
 532         if (!success) {
 533                 infostream << "Client: "
 534                         << "Failed to load " << cached_or_received << " media: "
 535                         << sha1_hex << " \"" << name << "\""
 536                         << std::endl;
 537                 return false;
 538         }
 539
 540         verbosestream << "Client: "
 541                 << "Loaded " << cached_or_received << " media: "
 542                 << sha1_hex << " \"" << name << "\""
 543                 << std::endl;
 544
 545         // Update cache (unless we just loaded the file from the cache)
 546         if (!is_from_cache)
 547                 m_media_cache.update(sha1_hex, data);
 548
 549         return true;
 550 }
 551
 552
 553 /*
 554         Minetest Hashset File Format
 555
 556         All values are stored in big-endian byte order.
 557         [u32] signature: 'MTHS'
 558         [u16] version: 1
 559         For each hash in set:
 560                 [u8*20] SHA1 hash
 561
 562         Version changes:
 563         1 - Initial version
 564 */
 565
 566 std::string ClientMediaDownloader::serializeRequiredHashSet()
 567 {
 568         std::ostringstream os(std::ios::binary);
 569
 570         writeU32(os, MTHASHSET_FILE_SIGNATURE); // signature
 571         writeU16(os, 1);                        // version
 572
 573         // Write list of hashes of files that have not been
 574         // received (found in cache) yet
 575         for (std::map<std::string, FileStatus*>::iterator
 576                         it = m_files.begin();
 577                         it != m_files.end(); ++it) {
 578                 if (!it->second->received) {
 579                         FATAL_ERROR_IF(it->second->sha1.size() != 20, "Invalid SHA1 size");
 580                         os << it->second->sha1;
 581                 }
 582         }
 583
 584         return os.str();
 585 }
 586
 587 void ClientMediaDownloader::deSerializeHashSet(const std::string &data,
 588                 std::set<std::string> &result)
 589 {
 590         if (data.size() < 6 || data.size() % 20 != 6) {
 591                 throw SerializationError(
 592                                 "ClientMediaDownloader::deSerializeHashSet: "
 593                                 "invalid hash set file size");
 594         }
 595
 596         const u8 *data_cstr = (const u8*) data.c_str();
 597
 598         u32 signature = readU32(&data_cstr[0]);
 599         if (signature != MTHASHSET_FILE_SIGNATURE) {
 600                 throw SerializationError(
 601                                 "ClientMediaDownloader::deSerializeHashSet: "
 602                                 "invalid hash set file signature");
 603         }
 604
 605         u16 version = readU16(&data_cstr[4]);
 606         if (version != 1) {
 607                 throw SerializationError(
 608                                 "ClientMediaDownloader::deSerializeHashSet: "
 609                                 "unsupported hash set file version");
 610         }
 611
 612         for (u32 pos = 6; pos < data.size(); pos += 20) {
 613                 result.insert(data.substr(pos, 20));
 614         }
 615 }