src/client/clientmedia.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "clientmedia.h"
  21 #include "httpfetch.h"
  22 #include "client.h"
  23 #include "filecache.h"
  24 #include "filesys.h"
  25 #include "log.h"
  26 #include "porting.h"
  27 #include "settings.h"
  28 #include "util/hex.h"
  29 #include "util/serialize.h"
  30 #include "util/sha1.h"
  31 #include "util/string.h"
  32
  33 static std::string getMediaCacheDir()
  34 {
  35         return porting::path_cache + DIR_DELIM + "media";
  36 }
  37
  38 bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &filedata)
  39 {
  40         FileCache media_cache(getMediaCacheDir());
  41         std::string sha1_hex = hex_encode(raw_hash);
  42         if (!media_cache.exists(sha1_hex))
  43                 return media_cache.update(sha1_hex, filedata);
  44         return true;
  45 }
  46
  47 /*
  48         ClientMediaDownloader
  49 */
  50
  51 ClientMediaDownloader::ClientMediaDownloader():
  52         m_media_cache(getMediaCacheDir()),
  53         m_httpfetch_caller(HTTPFETCH_DISCARD)
  54 {
  55 }
  56
  57 ClientMediaDownloader::~ClientMediaDownloader()
  58 {
  59         if (m_httpfetch_caller != HTTPFETCH_DISCARD)
  60                 httpfetch_caller_free(m_httpfetch_caller);
  61
  62         for (auto &file_it : m_files)
  63                 delete file_it.second;
  64
  65         for (auto &remote : m_remotes)
  66                 delete remote;
  67 }
  68
  69 void ClientMediaDownloader::addFile(const std::string &name, const std::string &sha1)
  70 {
  71         assert(!m_initial_step_done); // pre-condition
  72
  73         // if name was already announced, ignore the new announcement
  74         if (m_files.count(name) != 0) {
  75                 errorstream << "Client: ignoring duplicate media announcement "
  76                                 << "sent by server: \"" << name << "\""
  77                                 << std::endl;
  78                 return;
  79         }
  80
  81         // if name is empty or contains illegal characters, ignore the file
  82         if (name.empty() || !string_allowed(name, TEXTURENAME_ALLOWED_CHARS)) {
  83                 errorstream << "Client: ignoring illegal file name "
  84                                 << "sent by server: \"" << name << "\""
  85                                 << std::endl;
  86                 return;
  87         }
  88
  89         // length of sha1 must be exactly 20 (160 bits), else ignore the file
  90         if (sha1.size() != 20) {
  91                 errorstream << "Client: ignoring illegal SHA1 sent by server: "
  92                                 << hex_encode(sha1) << " \"" << name << "\""
  93                                 << std::endl;
  94                 return;
  95         }
  96
  97         FileStatus *filestatus = new FileStatus();
  98         filestatus->received = false;
  99         filestatus->sha1 = sha1;
 100         filestatus->current_remote = -1;
 101         m_files.insert(std::make_pair(name, filestatus));
 102 }
 103
 104 void ClientMediaDownloader::addRemoteServer(const std::string &baseurl)
 105 {
 106         assert(!m_initial_step_done);   // pre-condition
 107
 108         #ifdef USE_CURL
 109
 110         if (g_settings->getBool("enable_remote_media_server")) {
 111                 infostream << "Client: Adding remote server \""
 112                         << baseurl << "\" for media download" << std::endl;
 113
 114                 RemoteServerStatus *remote = new RemoteServerStatus();
 115                 remote->baseurl = baseurl;
 116                 remote->active_count = 0;
 117                 m_remotes.push_back(remote);
 118         }
 119
 120         #else
 121
 122         infostream << "Client: Ignoring remote server \""
 123                 << baseurl << "\" because cURL support is not compiled in"
 124                 << std::endl;
 125
 126         #endif
 127 }
 128
 129 void ClientMediaDownloader::step(Client *client)
 130 {
 131         if (!m_initial_step_done) {
 132                 initialStep(client);
 133                 m_initial_step_done = true;
 134         }
 135
 136         // Remote media: check for completion of fetches
 137         if (m_httpfetch_active) {
 138                 bool fetched_something = false;
 139                 HTTPFetchResult fetch_result;
 140
 141                 while (httpfetch_async_get(m_httpfetch_caller, fetch_result)) {
 142                         m_httpfetch_active--;
 143                         fetched_something = true;
 144
 145                         // Is this a hashset (index.mth) or a media file?
 146                         if (fetch_result.request_id < m_remotes.size())
 147                                 remoteHashSetReceived(fetch_result);
 148                         else
 149                                 remoteMediaReceived(fetch_result, client);
 150                 }
 151
 152                 if (fetched_something)
 153                         startRemoteMediaTransfers();
 154
 155                 // Did all remote transfers end and no new ones can be started?
 156                 // If so, request still missing files from the minetest server
 157                 // (Or report that we have all files.)
 158                 if (m_httpfetch_active == 0) {
 159                         if (m_uncached_received_count < m_uncached_count) {
 160                                 infostream << "Client: Failed to remote-fetch "
 161                                         << (m_uncached_count-m_uncached_received_count)
 162                                         << " files. Requesting them"
 163                                         << " the usual way." << std::endl;
 164                         }
 165                         startConventionalTransfers(client);
 166                 }
 167         }
 168 }
 169
 170 void ClientMediaDownloader::initialStep(Client *client)
 171 {
 172         // Check media cache
 173         m_uncached_count = m_files.size();
 174         for (auto &file_it : m_files) {
 175                 std::string name = file_it.first;
 176                 FileStatus *filestatus = file_it.second;
 177                 const std::string &sha1 = filestatus->sha1;
 178
 179                 std::ostringstream tmp_os(std::ios_base::binary);
 180                 bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
 181
 182                 // If found in cache, try to load it from there
 183                 if (found_in_cache) {
 184                         bool success = checkAndLoad(name, sha1,
 185                                         tmp_os.str(), true, client);
 186                         if (success) {
 187                                 filestatus->received = true;
 188                                 m_uncached_count--;
 189                         }
 190                 }
 191         }
 192
 193         assert(m_uncached_received_count == 0);
 194
 195         // Create the media cache dir if we are likely to write to it
 196         if (m_uncached_count != 0) {
 197                 bool did = fs::CreateAllDirs(getMediaCacheDir());
 198                 if (!did) {
 199                         errorstream << "Client: "
 200                                 << "Could not create media cache directory: "
 201                                 << getMediaCacheDir()
 202                                 << std::endl;
 203                 }
 204         }
 205
 206         // If we found all files in the cache, report this fact to the server.
 207         // If the server reported no remote servers, immediately start
 208         // conventional transfers. Note: if cURL support is not compiled in,
 209         // m_remotes is always empty, so "!USE_CURL" is redundant but may
 210         // reduce the size of the compiled code
 211         if (!USE_CURL || m_uncached_count == 0 || m_remotes.empty()) {
 212                 startConventionalTransfers(client);
 213         }
 214         else {
 215                 // Otherwise start off by requesting each server's sha1 set
 216
 217                 // This is the first time we use httpfetch, so alloc a caller ID
 218                 m_httpfetch_caller = httpfetch_caller_alloc();
 219
 220                 // Set the active fetch limit to curl_parallel_limit or 84,
 221                 // whichever is greater. This gives us some leeway so that
 222                 // inefficiencies in communicating with the httpfetch thread
 223                 // don't slow down fetches too much. (We still want some limit
 224                 // so that when the first remote server returns its hash set,
 225                 // not all files are requested from that server immediately.)
 226                 // One such inefficiency is that ClientMediaDownloader::step()
 227                 // is only called a couple times per second, while httpfetch
 228                 // might return responses much faster than that.
 229                 // Note that httpfetch strictly enforces curl_parallel_limit
 230                 // but at no inter-thread communication cost. This however
 231                 // doesn't help with the aforementioned inefficiencies.
 232                 // The signifance of 84 is that it is 2*6*9 in base 13.
 233                 m_httpfetch_active_limit = g_settings->getS32("curl_parallel_limit");
 234                 m_httpfetch_active_limit = MYMAX(m_httpfetch_active_limit, 84);
 235
 236                 // Write a list of hashes that we need. This will be POSTed
 237                 // to the server using Content-Type: application/octet-stream
 238                 std::string required_hash_set = serializeRequiredHashSet();
 239
 240                 // minor fixme: this loop ignores m_httpfetch_active_limit
 241
 242                 // another minor fixme, unlikely to matter in normal usage:
 243                 // these index.mth fetches do (however) count against
 244                 // m_httpfetch_active_limit when starting actual media file
 245                 // requests, so if there are lots of remote servers that are
 246                 // not responding, those will stall new media file transfers.
 247
 248                 for (u32 i = 0; i < m_remotes.size(); ++i) {
 249                         assert(m_httpfetch_next_id == i);
 250
 251                         RemoteServerStatus *remote = m_remotes[i];
 252                         actionstream << "Client: Contacting remote server \""
 253                                 << remote->baseurl << "\"" << std::endl;
 254
 255                         HTTPFetchRequest fetch_request;
 256                         fetch_request.url =
 257                                 remote->baseurl + MTHASHSET_FILE_NAME;
 258                         fetch_request.caller = m_httpfetch_caller;
 259                         fetch_request.request_id = m_httpfetch_next_id; // == i
 260                         fetch_request.method = HTTP_POST;
 261                         fetch_request.raw_data = required_hash_set;
 262                         fetch_request.extra_headers.emplace_back(
 263                                 "Content-Type: application/octet-stream");
 264
 265                         // Encapsulate possible IPv6 plain address in []
 266                         std::string addr = client->getAddressName();
 267                         if (addr.find(':', 0) != std::string::npos)
 268                                 addr = '[' + addr + ']';
 269                         fetch_request.extra_headers.emplace_back(
 270                                 std::string("Referer: minetest://") +
 271                                 addr + ":" +
 272                                 std::to_string(client->getServerAddress().getPort()));
 273
 274                         httpfetch_async(fetch_request);
 275
 276                         m_httpfetch_active++;
 277                         m_httpfetch_next_id++;
 278                         m_outstanding_hash_sets++;
 279                 }
 280         }
 281 }
 282
 283 void ClientMediaDownloader::remoteHashSetReceived(
 284                 const HTTPFetchResult &fetch_result)
 285 {
 286         u32 remote_id = fetch_result.request_id;
 287         assert(remote_id < m_remotes.size());
 288         RemoteServerStatus *remote = m_remotes[remote_id];
 289
 290         m_outstanding_hash_sets--;
 291
 292         if (fetch_result.succeeded) {
 293                 try {
 294                         // Server sent a list of file hashes that are
 295                         // available on it, try to parse the list
 296
 297                         std::set<std::string> sha1_set;
 298                         deSerializeHashSet(fetch_result.data, sha1_set);
 299
 300                         // Parsing succeeded: For every file that is
 301                         // available on this server, add this server
 302                         // to the available_remotes array
 303
 304                         for(std::map<std::string, FileStatus*>::iterator
 305                                         it = m_files.upper_bound(m_name_bound);
 306                                         it != m_files.end(); ++it) {
 307                                 FileStatus *f = it->second;
 308                                 if (!f->received && sha1_set.count(f->sha1))
 309                                         f->available_remotes.push_back(remote_id);
 310                         }
 311                 }
 312                 catch (SerializationError &e) {
 313                         infostream << "Client: Remote server \""
 314                                 << remote->baseurl << "\" sent invalid hash set: "
 315                                 << e.what() << std::endl;
 316                 }
 317         }
 318 }
 319
 320 void ClientMediaDownloader::remoteMediaReceived(
 321                 const HTTPFetchResult &fetch_result,
 322                 Client *client)
 323 {
 324         // Some remote server sent us a file.
 325         // -> decrement number of active fetches
 326         // -> mark file as received if fetch succeeded
 327         // -> try to load media
 328
 329         std::string name;
 330         {
 331                 std::unordered_map<unsigned long, std::string>::iterator it =
 332                         m_remote_file_transfers.find(fetch_result.request_id);
 333                 assert(it != m_remote_file_transfers.end());
 334                 name = it->second;
 335                 m_remote_file_transfers.erase(it);
 336         }
 337
 338         sanity_check(m_files.count(name) != 0);
 339
 340         FileStatus *filestatus = m_files[name];
 341         sanity_check(!filestatus->received);
 342         sanity_check(filestatus->current_remote >= 0);
 343
 344         RemoteServerStatus *remote = m_remotes[filestatus->current_remote];
 345
 346         filestatus->current_remote = -1;
 347         remote->active_count--;
 348
 349         // If fetch succeeded, try to load media file
 350
 351         if (fetch_result.succeeded) {
 352                 bool success = checkAndLoad(name, filestatus->sha1,
 353                                 fetch_result.data, false, client);
 354                 if (success) {
 355                         filestatus->received = true;
 356                         assert(m_uncached_received_count < m_uncached_count);
 357                         m_uncached_received_count++;
 358                 }
 359         }
 360 }
 361
 362 s32 ClientMediaDownloader::selectRemoteServer(FileStatus *filestatus)
 363 {
 364         // Pre-conditions
 365         assert(filestatus != NULL);
 366         assert(!filestatus->received);
 367         assert(filestatus->current_remote < 0);
 368
 369         if (filestatus->available_remotes.empty())
 370                 return -1;
 371
 372         // Of all servers that claim to provide the file (and haven't
 373         // been unsuccessfully tried before), find the one with the
 374         // smallest number of currently active transfers
 375
 376         s32 best = 0;
 377         s32 best_remote_id = filestatus->available_remotes[best];
 378         s32 best_active_count = m_remotes[best_remote_id]->active_count;
 379
 380         for (u32 i = 1; i < filestatus->available_remotes.size(); ++i) {
 381                 s32 remote_id = filestatus->available_remotes[i];
 382                 s32 active_count = m_remotes[remote_id]->active_count;
 383                 if (active_count < best_active_count) {
 384                         best = i;
 385                         best_remote_id = remote_id;
 386                         best_active_count = active_count;
 387                 }
 388         }
 389
 390         filestatus->available_remotes.erase(
 391                         filestatus->available_remotes.begin() + best);
 392
 393         return best_remote_id;
 394
 395 }
 396
 397 void ClientMediaDownloader::startRemoteMediaTransfers()
 398 {
 399         bool changing_name_bound = true;
 400
 401         for (std::map<std::string, FileStatus*>::iterator
 402                         files_iter = m_files.upper_bound(m_name_bound);
 403                         files_iter != m_files.end(); ++files_iter) {
 404
 405                 // Abort if active fetch limit is exceeded
 406                 if (m_httpfetch_active >= m_httpfetch_active_limit)
 407                         break;
 408
 409                 const std::string &name = files_iter->first;
 410                 FileStatus *filestatus = files_iter->second;
 411
 412                 if (!filestatus->received && filestatus->current_remote < 0) {
 413                         // File has not been received yet and is not currently
 414                         // being transferred. Choose a server for it.
 415                         s32 remote_id = selectRemoteServer(filestatus);
 416                         if (remote_id >= 0) {
 417                                 // Found a server, so start fetching
 418                                 RemoteServerStatus *remote =
 419                                         m_remotes[remote_id];
 420
 421                                 std::string url = remote->baseurl +
 422                                         hex_encode(filestatus->sha1);
 423                                 verbosestream << "Client: "
 424                                         << "Requesting remote media file "
 425                                         << "\"" << name << "\" "
 426                                         << "\"" << url << "\"" << std::endl;
 427
 428                                 HTTPFetchRequest fetch_request;
 429                                 fetch_request.url = url;
 430                                 fetch_request.caller = m_httpfetch_caller;
 431                                 fetch_request.request_id = m_httpfetch_next_id;
 432                                 fetch_request.timeout =
 433                                         g_settings->getS32("curl_file_download_timeout");
 434                                 httpfetch_async(fetch_request);
 435
 436                                 m_remote_file_transfers.insert(std::make_pair(
 437                                                         m_httpfetch_next_id,
 438                                                         name));
 439
 440                                 filestatus->current_remote = remote_id;
 441                                 remote->active_count++;
 442                                 m_httpfetch_active++;
 443                                 m_httpfetch_next_id++;
 444                         }
 445                 }
 446
 447                 if (filestatus->received ||
 448                                 (filestatus->current_remote < 0 &&
 449                                  !m_outstanding_hash_sets)) {
 450                         // If we arrive here, we conclusively know that we
 451                         // won't fetch this file from a remote server in the
 452                         // future. So update the name bound if possible.
 453                         if (changing_name_bound)
 454                                 m_name_bound = name;
 455                 }
 456                 else
 457                         changing_name_bound = false;
 458         }
 459
 460 }
 461
 462 void ClientMediaDownloader::startConventionalTransfers(Client *client)
 463 {
 464         assert(m_httpfetch_active == 0);        // pre-condition
 465
 466         if (m_uncached_received_count != m_uncached_count) {
 467                 // Some media files have not been received yet, use the
 468                 // conventional slow method (minetest protocol) to get them
 469                 std::vector<std::string> file_requests;
 470                 for (auto &file : m_files) {
 471                         if (!file.second->received)
 472                                 file_requests.push_back(file.first);
 473                 }
 474                 assert((s32) file_requests.size() ==
 475                                 m_uncached_count - m_uncached_received_count);
 476                 client->request_media(file_requests);
 477         }
 478 }
 479
 480 void ClientMediaDownloader::conventionalTransferDone(
 481                 const std::string &name,
 482                 const std::string &data,
 483                 Client *client)
 484 {
 485         // Check that file was announced
 486         std::map<std::string, FileStatus*>::iterator
 487                 file_iter = m_files.find(name);
 488         if (file_iter == m_files.end()) {
 489                 errorstream << "Client: server sent media file that was"
 490                         << "not announced, ignoring it: \"" << name << "\""
 491                         << std::endl;
 492                 return;
 493         }
 494         FileStatus *filestatus = file_iter->second;
 495         assert(filestatus != NULL);
 496
 497         // Check that file hasn't already been received
 498         if (filestatus->received) {
 499                 errorstream << "Client: server sent media file that we already"
 500                         << "received, ignoring it: \"" << name << "\""
 501                         << std::endl;
 502                 return;
 503         }
 504
 505         // Mark file as received, regardless of whether loading it works and
 506         // whether the checksum matches (because at this point there is no
 507         // other server that could send a replacement)
 508         filestatus->received = true;
 509         assert(m_uncached_received_count < m_uncached_count);
 510         m_uncached_received_count++;
 511
 512         // Check that received file matches announced checksum
 513         // If so, load it
 514         checkAndLoad(name, filestatus->sha1, data, false, client);
 515 }
 516
 517 bool ClientMediaDownloader::checkAndLoad(
 518                 const std::string &name, const std::string &sha1,
 519                 const std::string &data, bool is_from_cache, Client *client)
 520 {
 521         const char *cached_or_received = is_from_cache ? "cached" : "received";
 522         const char *cached_or_received_uc = is_from_cache ? "Cached" : "Received";
 523         std::string sha1_hex = hex_encode(sha1);
 524
 525         // Compute actual checksum of data
 526         std::string data_sha1;
 527         {
 528                 SHA1 data_sha1_calculator;
 529                 data_sha1_calculator.addBytes(data.c_str(), data.size());
 530                 unsigned char *data_tmpdigest = data_sha1_calculator.getDigest();
 531                 data_sha1.assign((char*) data_tmpdigest, 20);
 532                 free(data_tmpdigest);
 533         }
 534
 535         // Check that received file matches announced checksum
 536         if (data_sha1 != sha1) {
 537                 std::string data_sha1_hex = hex_encode(data_sha1);
 538                 infostream << "Client: "
 539                         << cached_or_received_uc << " media file "
 540                         << sha1_hex << " \"" << name << "\" "
 541                         << "mismatches actual checksum " << data_sha1_hex
 542                         << std::endl;
 543                 return false;
 544         }
 545
 546         // Checksum is ok, try loading the file
 547         bool success = client->loadMedia(data, name);
 548         if (!success) {
 549                 infostream << "Client: "
 550                         << "Failed to load " << cached_or_received << " media: "
 551                         << sha1_hex << " \"" << name << "\""
 552                         << std::endl;
 553                 return false;
 554         }
 555
 556         verbosestream << "Client: "
 557                 << "Loaded " << cached_or_received << " media: "
 558                 << sha1_hex << " \"" << name << "\""
 559                 << std::endl;
 560
 561         // Update cache (unless we just loaded the file from the cache)
 562         if (!is_from_cache)
 563                 m_media_cache.update(sha1_hex, data);
 564
 565         return true;
 566 }
 567
 568 /*
 569         Minetest Hashset File Format
 570
 571         All values are stored in big-endian byte order.
 572         [u32] signature: 'MTHS'
 573         [u16] version: 1
 574         For each hash in set:
 575                 [u8*20] SHA1 hash
 576
 577         Version changes:
 578         1 - Initial version
 579 */
 580
 581 std::string ClientMediaDownloader::serializeRequiredHashSet()
 582 {
 583         std::ostringstream os(std::ios::binary);
 584
 585         writeU32(os, MTHASHSET_FILE_SIGNATURE); // signature
 586         writeU16(os, 1);                        // version
 587
 588         // Write list of hashes of files that have not been
 589         // received (found in cache) yet
 590         for (std::map<std::string, FileStatus*>::iterator
 591                         it = m_files.begin();
 592                         it != m_files.end(); ++it) {
 593                 if (!it->second->received) {
 594                         FATAL_ERROR_IF(it->second->sha1.size() != 20, "Invalid SHA1 size");
 595                         os << it->second->sha1;
 596                 }
 597         }
 598
 599         return os.str();
 600 }
 601
 602 void ClientMediaDownloader::deSerializeHashSet(const std::string &data,
 603                 std::set<std::string> &result)
 604 {
 605         if (data.size() < 6 || data.size() % 20 != 6) {
 606                 throw SerializationError(
 607                                 "ClientMediaDownloader::deSerializeHashSet: "
 608                                 "invalid hash set file size");
 609         }
 610
 611         const u8 *data_cstr = (const u8*) data.c_str();
 612
 613         u32 signature = readU32(&data_cstr[0]);
 614         if (signature != MTHASHSET_FILE_SIGNATURE) {
 615                 throw SerializationError(
 616                                 "ClientMediaDownloader::deSerializeHashSet: "
 617                                 "invalid hash set file signature");
 618         }
 619
 620         u16 version = readU16(&data_cstr[4]);
 621         if (version != 1) {
 622                 throw SerializationError(
 623                                 "ClientMediaDownloader::deSerializeHashSet: "
 624                                 "unsupported hash set file version");
 625         }
 626
 627         for (u32 pos = 6; pos < data.size(); pos += 20) {
 628                 result.insert(data.substr(pos, 20));
 629         }
 630 }