src/clientmedia.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "clientmedia.h"
  21 #include "httpfetch.h"
  22 #include "client.h"
  23 #include "clientserver.h"
  24 #include "filecache.h"
  25 #include "hex.h"
  26 #include "sha1.h"
  27 #include "debug.h"
  28 #include "log.h"
  29 #include "porting.h"
  30 #include "settings.h"
  31 #include "main.h"
  32 #include "util/serialize.h"
  33 #include "util/string.h"
  34
  35 static std::string getMediaCacheDir()
  36 {
  37         return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media";
  38 }
  39
  40 /*
  41         ClientMediaDownloader
  42 */
  43
  44 ClientMediaDownloader::ClientMediaDownloader():
  45         m_media_cache(getMediaCacheDir())
  46 {
  47         m_initial_step_done = false;
  48         m_name_bound = "";  // works because "" is an invalid file name
  49         m_uncached_count = 0;
  50         m_uncached_received_count = 0;
  51         m_httpfetch_caller = HTTPFETCH_DISCARD;
  52         m_httpfetch_active = 0;
  53         m_httpfetch_active_limit = 0;
  54         m_httpfetch_next_id = 0;
  55         m_httpfetch_timeout = 0;
  56         m_outstanding_hash_sets = 0;
  57 }
  58
  59 ClientMediaDownloader::~ClientMediaDownloader()
  60 {
  61         if (m_httpfetch_caller != HTTPFETCH_DISCARD)
  62                 httpfetch_caller_free(m_httpfetch_caller);
  63
  64         for (std::map<std::string, FileStatus*>::iterator it = m_files.begin();
  65                         it != m_files.end(); ++it)
  66                 delete it->second;
  67
  68         for (u32 i = 0; i < m_remotes.size(); ++i)
  69                 delete m_remotes[i];
  70 }
  71
  72 void ClientMediaDownloader::addFile(std::string name, std::string sha1)
  73 {
  74         assert(!m_initial_step_done);
  75
  76         // if name was already announced, ignore the new announcement
  77         if (m_files.count(name) != 0) {
  78                 errorstream << "Client: ignoring duplicate media announcement "
  79                                 << "sent by server: \"" << name << "\""
  80                                 << std::endl;
  81                 return;
  82         }
  83
  84         // if name is empty or contains illegal characters, ignore the file
  85         if (name.empty() || !string_allowed(name, TEXTURENAME_ALLOWED_CHARS)) {
  86                 errorstream << "Client: ignoring illegal file name "
  87                                 << "sent by server: \"" << name << "\""
  88                                 << std::endl;
  89                 return;
  90         }
  91
  92         // length of sha1 must be exactly 20 (160 bits), else ignore the file
  93         if (sha1.size() != 20) {
  94                 errorstream << "Client: ignoring illegal SHA1 sent by server: "
  95                                 << hex_encode(sha1) << " \"" << name << "\""
  96                                 << std::endl;
  97                 return;
  98         }
  99
 100         FileStatus *filestatus = new FileStatus;
 101         filestatus->received = false;
 102         filestatus->sha1 = sha1;
 103         filestatus->current_remote = -1;
 104         m_files.insert(std::make_pair(name, filestatus));
 105 }
 106
 107 void ClientMediaDownloader::addRemoteServer(std::string baseurl)
 108 {
 109         assert(!m_initial_step_done);
 110
 111         #ifdef USE_CURL
 112
 113         infostream << "Client: Adding remote server \""
 114                 << baseurl << "\" for media download" << std::endl;
 115
 116         RemoteServerStatus *remote = new RemoteServerStatus;
 117         remote->baseurl = baseurl;
 118         remote->active_count = 0;
 119         remote->request_by_filename = false;
 120         m_remotes.push_back(remote);
 121
 122         #else
 123
 124         infostream << "Client: Ignoring remote server \""
 125                 << baseurl << "\" because cURL support is not compiled in"
 126                 << std::endl;
 127
 128         #endif
 129 }
 130
 131 void ClientMediaDownloader::step(Client *client)
 132 {
 133         if (!m_initial_step_done) {
 134                 initialStep(client);
 135                 m_initial_step_done = true;
 136         }
 137
 138         // Remote media: check for completion of fetches
 139         if (m_httpfetch_active) {
 140                 bool fetched_something = false;
 141                 HTTPFetchResult fetchresult;
 142
 143                 while (httpfetch_async_get(m_httpfetch_caller, fetchresult)) {
 144                         m_httpfetch_active--;
 145                         fetched_something = true;
 146
 147                         // Is this a hashset (index.mth) or a media file?
 148                         if (fetchresult.request_id < m_remotes.size())
 149                                 remoteHashSetReceived(fetchresult);
 150                         else
 151                                 remoteMediaReceived(fetchresult, client);
 152                 }
 153
 154                 if (fetched_something)
 155                         startRemoteMediaTransfers();
 156
 157                 // Did all remote transfers end and no new ones can be started?
 158                 // If so, request still missing files from the minetest server
 159                 // (Or report that we have all files.)
 160                 if (m_httpfetch_active == 0) {
 161                         if (m_uncached_received_count < m_uncached_count) {
 162                                 infostream << "Client: Failed to remote-fetch "
 163                                         << (m_uncached_count-m_uncached_received_count)
 164                                         << " files. Requesting them"
 165                                         << " the usual way." << std::endl;
 166                         }
 167                         startConventionalTransfers(client);
 168                 }
 169         }
 170 }
 171
 172 void ClientMediaDownloader::initialStep(Client *client)
 173 {
 174         // Check media cache
 175         m_uncached_count = m_files.size();
 176         for (std::map<std::string, FileStatus*>::iterator
 177                         it = m_files.begin();
 178                         it != m_files.end(); ++it) {
 179                 std::string name = it->first;
 180                 FileStatus *filestatus = it->second;
 181                 const std::string &sha1 = filestatus->sha1;
 182
 183                 std::ostringstream tmp_os(std::ios_base::binary);
 184                 bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
 185
 186                 // If found in cache, try to load it from there
 187                 if (found_in_cache) {
 188                         bool success = checkAndLoad(name, sha1,
 189                                         tmp_os.str(), true, client);
 190                         if (success) {
 191                                 filestatus->received = true;
 192                                 m_uncached_count--;
 193                         }
 194                 }
 195         }
 196
 197         assert(m_uncached_received_count == 0);
 198
 199         // Create the media cache dir if we are likely to write to it
 200         if (m_uncached_count != 0) {
 201                 bool did = fs::CreateAllDirs(getMediaCacheDir());
 202                 if (!did) {
 203                         errorstream << "Client: "
 204                                 << "Could not create media cache directory: "
 205                                 << getMediaCacheDir()
 206                                 << std::endl;
 207                 }
 208         }
 209
 210         // If we found all files in the cache, report this fact to the server.
 211         // If the server reported no remote servers, immediately start
 212         // conventional transfers. Note: if cURL support is not compiled in,
 213         // m_remotes is always empty, so "!USE_CURL" is redundant but may
 214         // reduce the size of the compiled code
 215         if (!USE_CURL || m_uncached_count == 0 || m_remotes.empty()) {
 216                 startConventionalTransfers(client);
 217         }
 218         else {
 219                 // Otherwise start off by requesting each server's sha1 set
 220
 221                 // This is the first time we use httpfetch, so alloc a caller ID
 222                 m_httpfetch_caller = httpfetch_caller_alloc();
 223                 m_httpfetch_timeout = g_settings->getS32("curl_timeout");
 224
 225                 // Set the active fetch limit to curl_parallel_limit or 84,
 226                 // whichever is greater. This gives us some leeway so that
 227                 // inefficiencies in communicating with the httpfetch thread
 228                 // don't slow down fetches too much. (We still want some limit
 229                 // so that when the first remote server returns its hash set,
 230                 // not all files are requested from that server immediately.)
 231                 // One such inefficiency is that ClientMediaDownloader::step()
 232                 // is only called a couple times per second, while httpfetch
 233                 // might return responses much faster than that.
 234                 // Note that httpfetch strictly enforces curl_parallel_limit
 235                 // but at no inter-thread communication cost. This however
 236                 // doesn't help with the aforementioned inefficiencies.
 237                 // The signifance of 84 is that it is 2*6*9 in base 13.
 238                 m_httpfetch_active_limit = g_settings->getS32("curl_parallel_limit");
 239                 m_httpfetch_active_limit = MYMAX(m_httpfetch_active_limit, 84);
 240
 241                 // Write a list of hashes that we need. This will be POSTed
 242                 // to the server using Content-Type: application/octet-stream
 243                 std::string required_hash_set = serializeRequiredHashSet();
 244
 245                 // minor fixme: this loop ignores m_httpfetch_active_limit
 246
 247                 // another minor fixme, unlikely to matter in normal usage:
 248                 // these index.mth fetches do (however) count against
 249                 // m_httpfetch_active_limit when starting actual media file
 250                 // requests, so if there are lots of remote servers that are
 251                 // not responding, those will stall new media file transfers.
 252
 253                 for (u32 i = 0; i < m_remotes.size(); ++i) {
 254                         assert(m_httpfetch_next_id == i);
 255
 256                         RemoteServerStatus *remote = m_remotes[i];
 257                         actionstream << "Client: Contacting remote server \""
 258                                 << remote->baseurl << "\"" << std::endl;
 259
 260                         HTTPFetchRequest fetchrequest;
 261                         fetchrequest.url =
 262                                 remote->baseurl + MTHASHSET_FILE_NAME;
 263                         fetchrequest.caller = m_httpfetch_caller;
 264                         fetchrequest.request_id = m_httpfetch_next_id; // == i
 265                         fetchrequest.timeout = m_httpfetch_timeout;
 266                         fetchrequest.connect_timeout = m_httpfetch_timeout;
 267                         fetchrequest.post_fields = required_hash_set;
 268                         fetchrequest.extra_headers.push_back(
 269                                 "Content-Type: application/octet-stream");
 270                         httpfetch_async(fetchrequest);
 271
 272                         m_httpfetch_active++;
 273                         m_httpfetch_next_id++;
 274                         m_outstanding_hash_sets++;
 275                 }
 276         }
 277 }
 278
 279 void ClientMediaDownloader::remoteHashSetReceived(
 280                 const HTTPFetchResult &fetchresult)
 281 {
 282         u32 remote_id = fetchresult.request_id;
 283         assert(remote_id < m_remotes.size());
 284         RemoteServerStatus *remote = m_remotes[remote_id];
 285
 286         m_outstanding_hash_sets--;
 287
 288         if (fetchresult.succeeded) {
 289                 try {
 290                         // Server sent a list of file hashes that are
 291                         // available on it, try to parse the list
 292
 293                         std::set<std::string> sha1_set;
 294                         deSerializeHashSet(fetchresult.data, sha1_set);
 295
 296                         // Parsing succeeded: For every file that is
 297                         // available on this server, add this server
 298                         // to the available_remotes array
 299
 300                         for(std::map<std::string, FileStatus*>::iterator
 301                                         it = m_files.upper_bound(m_name_bound);
 302                                         it != m_files.end(); ++it) {
 303                                 FileStatus *f = it->second;
 304                                 if (!f->received && sha1_set.count(f->sha1))
 305                                         f->available_remotes.push_back(remote_id);
 306                         }
 307                 }
 308                 catch (SerializationError &e) {
 309                         infostream << "Client: Remote server \""
 310                                 << remote->baseurl << "\" sent invalid hash set: "
 311                                 << e.what() << std::endl;
 312                 }
 313         }
 314
 315         // For compatibility: If index.mth is not found, assume that the
 316         // server contains files named like the original files (not their sha1)
 317
 318         // Do NOT check for any particular response code (e.g. 404) here,
 319         // because different servers respond differently
 320
 321         if (!fetchresult.succeeded && !fetchresult.timeout) {
 322                 infostream << "Client: Enabling compatibility mode for remote "
 323                         << "server \"" << remote->baseurl << "\"" << std::endl;
 324                 remote->request_by_filename = true;
 325
 326                 // Assume every file is available on this server
 327
 328                 for(std::map<std::string, FileStatus*>::iterator
 329                                 it = m_files.upper_bound(m_name_bound);
 330                                 it != m_files.end(); ++it) {
 331                         FileStatus *f = it->second;
 332                         if (!f->received)
 333                                 f->available_remotes.push_back(remote_id);
 334                 }
 335         }
 336 }
 337
 338 void ClientMediaDownloader::remoteMediaReceived(
 339                 const HTTPFetchResult &fetchresult,
 340                 Client *client)
 341 {
 342         // Some remote server sent us a file.
 343         // -> decrement number of active fetches
 344         // -> mark file as received if fetch succeeded
 345         // -> try to load media
 346
 347         std::string name;
 348         {
 349                 std::map<unsigned long, std::string>::iterator it =
 350                         m_remote_file_transfers.find(fetchresult.request_id);
 351                 assert(it != m_remote_file_transfers.end());
 352                 name = it->second;
 353                 m_remote_file_transfers.erase(it);
 354         }
 355
 356         assert(m_files.count(name) != 0);
 357
 358         FileStatus *filestatus = m_files[name];
 359         assert(!filestatus->received);
 360         assert(filestatus->current_remote >= 0);
 361
 362         RemoteServerStatus *remote = m_remotes[filestatus->current_remote];
 363
 364         filestatus->current_remote = -1;
 365         remote->active_count--;
 366
 367         // If fetch succeeded, try to load media file
 368
 369         if (fetchresult.succeeded) {
 370                 bool success = checkAndLoad(name, filestatus->sha1,
 371                                 fetchresult.data, false, client);
 372                 if (success) {
 373                         filestatus->received = true;
 374                         assert(m_uncached_received_count < m_uncached_count);
 375                         m_uncached_received_count++;
 376                 }
 377         }
 378 }
 379
 380 s32 ClientMediaDownloader::selectRemoteServer(FileStatus *filestatus)
 381 {
 382         assert(filestatus != NULL);
 383         assert(!filestatus->received);
 384         assert(filestatus->current_remote < 0);
 385
 386         if (filestatus->available_remotes.empty())
 387                 return -1;
 388         else {
 389                 // Of all servers that claim to provide the file (and haven't
 390                 // been unsuccessfully tried before), find the one with the
 391                 // smallest number of currently active transfers
 392
 393                 s32 best = 0;
 394                 s32 best_remote_id = filestatus->available_remotes[best];
 395                 s32 best_active_count = m_remotes[best_remote_id]->active_count;
 396
 397                 for (u32 i = 1; i < filestatus->available_remotes.size(); ++i) {
 398                         s32 remote_id = filestatus->available_remotes[i];
 399                         s32 active_count = m_remotes[remote_id]->active_count;
 400                         if (active_count < best_active_count) {
 401                                 best = i;
 402                                 best_remote_id = remote_id;
 403                                 best_active_count = active_count;
 404                         }
 405                 }
 406
 407                 filestatus->available_remotes.erase(
 408                                 filestatus->available_remotes.begin() + best);
 409
 410                 return best_remote_id;
 411         }
 412 }
 413
 414 void ClientMediaDownloader::startRemoteMediaTransfers()
 415 {
 416         bool changing_name_bound = true;
 417
 418         for (std::map<std::string, FileStatus*>::iterator
 419                         files_iter = m_files.upper_bound(m_name_bound);
 420                         files_iter != m_files.end(); ++files_iter) {
 421
 422                 // Abort if active fetch limit is exceeded
 423                 if (m_httpfetch_active >= m_httpfetch_active_limit)
 424                         break;
 425
 426                 const std::string &name = files_iter->first;
 427                 FileStatus *filestatus = files_iter->second;
 428
 429                 if (!filestatus->received && filestatus->current_remote < 0) {
 430                         // File has not been received yet and is not currently
 431                         // being transferred. Choose a server for it.
 432                         s32 remote_id = selectRemoteServer(filestatus);
 433                         if (remote_id >= 0) {
 434                                 // Found a server, so start fetching
 435                                 RemoteServerStatus *remote =
 436                                         m_remotes[remote_id];
 437
 438                                 std::string url = remote->baseurl +
 439                                         (remote->request_by_filename ? name :
 440                                         hex_encode(filestatus->sha1));
 441                                 verbosestream << "Client: "
 442                                         << "Requesting remote media file "
 443                                         << "\"" << name << "\" "
 444                                         << "\"" << url << "\"" << std::endl;
 445
 446                                 HTTPFetchRequest fetchrequest;
 447                                 fetchrequest.url = url;
 448                                 fetchrequest.caller = m_httpfetch_caller;
 449                                 fetchrequest.request_id = m_httpfetch_next_id;
 450                                 fetchrequest.timeout = 0; // no data timeout!
 451                                 fetchrequest.connect_timeout =
 452                                         m_httpfetch_timeout;
 453                                 httpfetch_async(fetchrequest);
 454
 455                                 m_remote_file_transfers.insert(std::make_pair(
 456                                                         m_httpfetch_next_id,
 457                                                         name));
 458
 459                                 filestatus->current_remote = remote_id;
 460                                 remote->active_count++;
 461                                 m_httpfetch_active++;
 462                                 m_httpfetch_next_id++;
 463                         }
 464                 }
 465
 466                 if (filestatus->received ||
 467                                 (filestatus->current_remote < 0 &&
 468                                  !m_outstanding_hash_sets)) {
 469                         // If we arrive here, we conclusively know that we
 470                         // won't fetch this file from a remote server in the
 471                         // future. So update the name bound if possible.
 472                         if (changing_name_bound)
 473                                 m_name_bound = name;
 474                 }
 475                 else
 476                         changing_name_bound = false;
 477         }
 478
 479 }
 480
 481 void ClientMediaDownloader::startConventionalTransfers(Client *client)
 482 {
 483         assert(m_httpfetch_active == 0);
 484
 485         if (m_uncached_received_count != m_uncached_count) {
 486                 // Some media files have not been received yet, use the
 487                 // conventional slow method (minetest protocol) to get them
 488                 std::list<std::string> file_requests;
 489                 for (std::map<std::string, FileStatus*>::iterator
 490                                 it = m_files.begin();
 491                                 it != m_files.end(); ++it) {
 492                         if (!it->second->received)
 493                                 file_requests.push_back(it->first);
 494                 }
 495                 assert((s32) file_requests.size() ==
 496                                 m_uncached_count - m_uncached_received_count);
 497                 client->request_media(file_requests);
 498         }
 499 }
 500
 501 void ClientMediaDownloader::conventionalTransferDone(
 502                 const std::string &name,
 503                 const std::string &data,
 504                 Client *client)
 505 {
 506         // Check that file was announced
 507         std::map<std::string, FileStatus*>::iterator
 508                 file_iter = m_files.find(name);
 509         if (file_iter == m_files.end()) {
 510                 errorstream << "Client: server sent media file that was"
 511                         << "not announced, ignoring it: \"" << name << "\""
 512                         << std::endl;
 513                 return;
 514         }
 515         FileStatus *filestatus = file_iter->second;
 516         assert(filestatus != NULL);
 517
 518         // Check that file hasn't already been received
 519         if (filestatus->received) {
 520                 errorstream << "Client: server sent media file that we already"
 521                         << "received, ignoring it: \"" << name << "\""
 522                         << std::endl;
 523                 return;
 524         }
 525
 526         // Mark file as received, regardless of whether loading it works and
 527         // whether the checksum matches (because at this point there is no
 528         // other server that could send a replacement)
 529         filestatus->received = true;
 530         assert(m_uncached_received_count < m_uncached_count);
 531         m_uncached_received_count++;
 532
 533         // Check that received file matches announced checksum
 534         // If so, load it
 535         checkAndLoad(name, filestatus->sha1, data, false, client);
 536 }
 537
 538 bool ClientMediaDownloader::checkAndLoad(
 539                 const std::string &name, const std::string &sha1,
 540                 const std::string &data, bool is_from_cache, Client *client)
 541 {
 542         const char *cached_or_received = is_from_cache ? "cached" : "received";
 543         const char *cached_or_received_uc = is_from_cache ? "Cached" : "Received";
 544         std::string sha1_hex = hex_encode(sha1);
 545
 546         // Compute actual checksum of data
 547         std::string data_sha1;
 548         {
 549                 SHA1 data_sha1_calculator;
 550                 data_sha1_calculator.addBytes(data.c_str(), data.size());
 551                 unsigned char *data_tmpdigest = data_sha1_calculator.getDigest();
 552                 data_sha1.assign((char*) data_tmpdigest, 20);
 553                 free(data_tmpdigest);
 554         }
 555
 556         // Check that received file matches announced checksum
 557         if (data_sha1 != sha1) {
 558                 std::string data_sha1_hex = hex_encode(data_sha1);
 559                 infostream << "Client: "
 560                         << cached_or_received_uc << " media file "
 561                         << sha1_hex << " \"" << name << "\" "
 562                         << "mismatches actual checksum " << data_sha1_hex
 563                         << std::endl;
 564                 return false;
 565         }
 566
 567         // Checksum is ok, try loading the file
 568         bool success = client->loadMedia(data, name);
 569         if (!success) {
 570                 infostream << "Client: "
 571                         << "Failed to load " << cached_or_received << " media: "
 572                         << sha1_hex << " \"" << name << "\""
 573                         << std::endl;
 574                 return false;
 575         }
 576
 577         verbosestream << "Client: "
 578                 << "Loaded " << cached_or_received << " media: "
 579                 << sha1_hex << " \"" << name << "\""
 580                 << std::endl;
 581
 582         // Update cache (unless we just loaded the file from the cache)
 583         if (!is_from_cache)
 584                 m_media_cache.update(sha1_hex, data);
 585
 586         return true;
 587 }
 588
 589
 590 /*
 591         Minetest Hashset File Format
 592
 593         All values are stored in big-endian byte order.
 594         [u32] signature: 'MTHS'
 595         [u16] version: 1
 596         For each hash in set:
 597                 [u8*20] SHA1 hash
 598
 599         Version changes:
 600         1 - Initial version
 601 */
 602
 603 std::string ClientMediaDownloader::serializeRequiredHashSet()
 604 {
 605         std::ostringstream os(std::ios::binary);
 606
 607         writeU32(os, MTHASHSET_FILE_SIGNATURE); // signature
 608         writeU16(os, 1);                        // version
 609
 610         // Write list of hashes of files that have not been
 611         // received (found in cache) yet
 612         for (std::map<std::string, FileStatus*>::iterator
 613                         it = m_files.begin();
 614                         it != m_files.end(); ++it) {
 615                 if (!it->second->received) {
 616                         assert(it->second->sha1.size() == 20);
 617                         os << it->second->sha1;
 618                 }
 619         }
 620
 621         return os.str();
 622 }
 623
 624 void ClientMediaDownloader::deSerializeHashSet(const std::string &data,
 625                 std::set<std::string> &result)
 626 {
 627         if (data.size() < 6 || data.size() % 20 != 6) {
 628                 throw SerializationError(
 629                                 "ClientMediaDownloader::deSerializeHashSet: "
 630                                 "invalid hash set file size");
 631         }
 632
 633         const u8 *data_cstr = (const u8*) data.c_str();
 634
 635         u32 signature = readU32(&data_cstr[0]);
 636         if (signature != MTHASHSET_FILE_SIGNATURE) {
 637                 throw SerializationError(
 638                                 "ClientMediaDownloader::deSerializeHashSet: "
 639                                 "invalid hash set file signature");
 640         }
 641
 642         u16 version = readU16(&data_cstr[4]);
 643         if (version != 1) {
 644                 throw SerializationError(
 645                                 "ClientMediaDownloader::deSerializeHashSet: "
 646                                 "unsupported hash set file version");
 647         }
 648
 649         for (u32 pos = 6; pos < data.size(); pos += 20) {
 650                 result.insert(data.substr(pos, 20));
 651         }
 652 }