3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU Lesser General Public License as published by
7 the Free Software Foundation; either version 2.1 of the License, or
8 (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public License along
16 with this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 #include "irrlichttypes.h"
23 #include "filecache.h"
24 #include "util/basic_macros.h"
29 #include <unordered_map>
32 struct HTTPFetchResult;
34 #define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS'
35 #define MTHASHSET_FILE_NAME "index.mth"
37 // Store file into media cache (unless it exists already)
38 // Validating the hash is responsibility of the caller
39 bool clientMediaUpdateCache(const std::string &raw_hash,
40 const std::string &filedata);
42 // more of a base class than an interface but this name was most convenient...
43 class IClientMediaDownloader
46 DISABLE_CLASS_COPY(IClientMediaDownloader)
48 virtual bool isStarted() const = 0;
50 // If this returns true, the downloader is done and can be deleted
51 virtual bool isDone() const = 0;
53 // Add a file to the list of required file (but don't fetch it yet)
54 virtual void addFile(const std::string &name, const std::string &sha1) = 0;
56 // Add a remote server to the list; ignored if not built with cURL
57 virtual void addRemoteServer(const std::string &baseurl) = 0;
59 // Steps the media downloader:
60 // - May load media into client by calling client->loadMedia()
61 // - May check media cache for files
62 // - May add files to media cache
63 // - May start remote transfers by calling httpfetch_async
64 // - May check for completion of current remote transfers
65 // - May start conventional transfers by calling client->request_media()
66 // - May inform server that all media has been loaded
67 // by calling client->received_media()
68 // After step has been called once, don't call addFile/addRemoteServer.
69 virtual void step(Client *client) = 0;
71 // Must be called for each file received through TOCLIENT_MEDIA
72 // returns true if this file belongs to this downloader
73 virtual bool conventionalTransferDone(const std::string &name,
74 const std::string &data, Client *client) = 0;
77 IClientMediaDownloader();
78 virtual ~IClientMediaDownloader() = default;
80 // Forwards the call to the appropriate Client method
81 virtual bool loadMedia(Client *client, const std::string &data,
82 const std::string &name) = 0;
84 void createCacheDirs();
86 bool tryLoadFromCache(const std::string &name, const std::string &sha1,
89 bool checkAndLoad(const std::string &name, const std::string &sha1,
90 const std::string &data, bool is_from_cache, Client *client);
92 // Filesystem-based media cache
93 FileCache m_media_cache;
94 bool m_write_to_cache;
97 class ClientMediaDownloader : public IClientMediaDownloader
100 ClientMediaDownloader();
101 ~ClientMediaDownloader();
103 float getProgress() const {
104 if (m_uncached_count >= 1)
105 return 1.0f * m_uncached_received_count /
111 bool isStarted() const override {
112 return m_initial_step_done;
115 bool isDone() const override {
116 return m_initial_step_done &&
117 m_uncached_received_count == m_uncached_count;
120 void addFile(const std::string &name, const std::string &sha1) override;
122 void addRemoteServer(const std::string &baseurl) override;
124 void step(Client *client) override;
126 bool conventionalTransferDone(
127 const std::string &name,
128 const std::string &data,
129 Client *client) override;
132 bool loadMedia(Client *client, const std::string &data,
133 const std::string &name) override;
140 std::vector<s32> available_remotes;
143 struct RemoteServerStatus {
148 void initialStep(Client *client);
149 void remoteHashSetReceived(const HTTPFetchResult &fetch_result);
150 void remoteMediaReceived(const HTTPFetchResult &fetch_result,
152 s32 selectRemoteServer(FileStatus *filestatus);
153 void startRemoteMediaTransfers();
154 void startConventionalTransfers(Client *client);
156 static void deSerializeHashSet(const std::string &data,
157 std::set<std::string> &result);
158 std::string serializeRequiredHashSet();
160 // Maps filename to file status
161 std::map<std::string, FileStatus*> m_files;
163 // Array of remote media servers
164 std::vector<RemoteServerStatus*> m_remotes;
166 // Has an attempt been made to load media files from the file cache?
167 // Have hash sets been requested from remote servers?
168 bool m_initial_step_done = false;
170 // Total number of media files to load
171 s32 m_uncached_count = 0;
173 // Number of media files that have been received
174 s32 m_uncached_received_count = 0;
176 // Status of remote transfers
177 u64 m_httpfetch_caller;
178 u64 m_httpfetch_next_id = 0;
179 s32 m_httpfetch_active = 0;
180 s32 m_httpfetch_active_limit = 0;
181 s32 m_outstanding_hash_sets = 0;
182 std::unordered_map<u64, std::string> m_remote_file_transfers;
184 // All files up to this name have either been received from a
185 // remote server or failed on all remote servers, so those files
186 // don't need to be looked at again
187 // (use m_files.upper_bound(m_name_bound) to get an iterator)
188 std::string m_name_bound = "";
192 // A media downloader that only downloads a single file.
193 // It does/doesn't do several things the normal downloader does:
194 // - won't fetch hash sets from remote servers
195 // - will mark loaded media as coming from file push
196 // - writing to file cache is optional
197 class SingleMediaDownloader : public IClientMediaDownloader
200 SingleMediaDownloader(bool write_to_cache);
201 ~SingleMediaDownloader();
203 bool isStarted() const override {
204 return m_stage > STAGE_INIT;
207 bool isDone() const override {
208 return m_stage >= STAGE_DONE;
211 void addFile(const std::string &name, const std::string &sha1) override;
213 void addRemoteServer(const std::string &baseurl) override;
215 void step(Client *client) override;
217 bool conventionalTransferDone(const std::string &name,
218 const std::string &data, Client *client) override;
221 bool loadMedia(Client *client, const std::string &data,
222 const std::string &name) override;
225 void initialStep(Client *client);
226 void remoteMediaReceived(const HTTPFetchResult &fetch_result, Client *client);
227 void startRemoteMediaTransfer();
228 void startConventionalTransfer(Client *client);
232 STAGE_CACHE_CHECKED, // we have tried to load the file from cache
236 // Information about the one file we want to fetch
237 std::string m_file_name;
238 std::string m_file_sha1;
239 s32 m_current_remote;
241 // Array of remote media servers
242 std::vector<std::string> m_remotes;
244 enum Stage m_stage = STAGE_INIT;
246 // Status of remote transfers
247 unsigned long m_httpfetch_caller;
248 unsigned long m_httpfetch_next_id = 0;