*/
#include "socket.h" // for select()
-#include "porting.h" // for sleep_ms()
+#include "porting.h" // for sleep_ms(), get_sysinfo()
#include "httpfetch.h"
#include <iostream>
#include <sstream>
#include <list>
#include <map>
#include <errno.h>
-#ifndef _MSC_VER
-#include <sys/utsname.h>
-#endif
#include "jthread/jevent.h"
#include "config.h"
#include "exceptions.h"
#include "settings.h"
JMutex g_httpfetch_mutex;
-std::map<unsigned long, std::list<HTTPFetchResult> > g_httpfetch_results;
+std::map<unsigned long, std::queue<HTTPFetchResult> > g_httpfetch_results;
- HTTPFetchRequest::HTTPFetchRequest()
- {
- url = "";
- caller = HTTPFETCH_DISCARD;
- request_id = 0;
- timeout = g_settings->getS32("curl_timeout");
- connect_timeout = timeout * 5;
-
- useragent = std::string("Minetest ") + minetest_version_hash;
-#ifdef _MSC_VER
- useragent += "Windows";
-#else
- struct utsname osinfo;
- uname(&osinfo);
- useragent += std::string(" (") + osinfo.sysname + "; " + osinfo.release + "; " + osinfo.machine + ")";
-#endif
- }
+HTTPFetchRequest::HTTPFetchRequest()
+{
+ url = "";
+ caller = HTTPFETCH_DISCARD;
+ request_id = 0;
+ timeout = g_settings->getS32("curl_timeout");
+ connect_timeout = timeout;
+ multipart = false;
+
+ useragent = std::string("Minetest/") + minetest_version_hash + " (" + porting::get_sysinfo() + ")";
+}
-static void httpfetch_deliver_result(const HTTPFetchResult &fetchresult)
+static void httpfetch_deliver_result(const HTTPFetchResult &fetch_result)
{
- unsigned long caller = fetchresult.caller;
+ unsigned long caller = fetch_result.caller;
if (caller != HTTPFETCH_DISCARD) {
JMutexAutoLock lock(g_httpfetch_mutex);
- g_httpfetch_results[caller].push_back(fetchresult);
+ g_httpfetch_results[caller].push(fetch_result);
}
}
// Check each caller ID except HTTPFETCH_DISCARD
const unsigned long discard = HTTPFETCH_DISCARD;
for (unsigned long caller = discard + 1; caller != discard; ++caller) {
- std::map<unsigned long, std::list<HTTPFetchResult> >::iterator
+ std::map<unsigned long, std::queue<HTTPFetchResult> >::iterator
it = g_httpfetch_results.find(caller);
if (it == g_httpfetch_results.end()) {
- verbosestream<<"httpfetch_caller_alloc: allocating "
- <<caller<<std::endl;
+ verbosestream << "httpfetch_caller_alloc: allocating "
+ << caller << std::endl;
// Access element to create it
g_httpfetch_results[caller];
return caller;
}
}
- assert("httpfetch_caller_alloc: ran out of caller IDs" == 0);
+ FATAL_ERROR("httpfetch_caller_alloc: ran out of caller IDs");
return discard;
}
}
}
-bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetchresult)
+bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result)
{
JMutexAutoLock lock(g_httpfetch_mutex);
// Check that caller exists
- std::map<unsigned long, std::list<HTTPFetchResult> >::iterator
+ std::map<unsigned long, std::queue<HTTPFetchResult> >::iterator
it = g_httpfetch_results.find(caller);
if (it == g_httpfetch_results.end())
return false;
// Check that result queue is nonempty
- std::list<HTTPFetchResult> &callerresults = it->second;
- if (callerresults.empty())
+ std::queue<HTTPFetchResult> &caller_results = it->second;
+ if (caller_results.empty())
return false;
// Pop first result
- fetchresult = callerresults.front();
- callerresults.pop_front();
+ fetch_result = caller_results.front();
+ caller_results.pop();
return true;
}
}
};
-struct HTTPFetchOngoing
+class HTTPFetchOngoing
{
+public:
+ HTTPFetchOngoing(HTTPFetchRequest request, CurlHandlePool *pool);
+ ~HTTPFetchOngoing();
+
+ CURLcode start(CURLM *multi);
+ const HTTPFetchResult * complete(CURLcode res);
+
+ const HTTPFetchRequest &getRequest() const { return request; };
+ const CURL *getEasyHandle() const { return curl; };
+
+private:
CurlHandlePool *pool;
CURL *curl;
CURLM *multi;
HTTPFetchRequest request;
HTTPFetchResult result;
std::ostringstream oss;
- char *post_fields;
- struct curl_slist *httpheader;
-
- HTTPFetchOngoing(HTTPFetchRequest request_, CurlHandlePool *pool_):
- pool(pool_),
- curl(NULL),
- multi(NULL),
- request(request_),
- result(request_),
- oss(std::ios::binary),
- httpheader(NULL)
- {
- curl = pool->alloc();
- if (curl != NULL) {
- // Set static cURL options
- curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
- curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
- curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
- curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 1);
+ struct curl_slist *http_header;
+ curl_httppost *post;
+};
+
+
+HTTPFetchOngoing::HTTPFetchOngoing(HTTPFetchRequest request_, CurlHandlePool *pool_):
+ pool(pool_),
+ curl(NULL),
+ multi(NULL),
+ request(request_),
+ result(request_),
+ oss(std::ios::binary),
+ http_header(NULL),
+ post(NULL)
+{
+ curl = pool->alloc();
+ if (curl == NULL) {
+ return;
+ }
+
+ // Set static cURL options
+ curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
+ curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+ curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 1);
+
+ std::string bind_address = g_settings->get("bind_address");
+ if (!bind_address.empty()) {
+ curl_easy_setopt(curl, CURLOPT_INTERFACE, bind_address.c_str());
+ }
#if LIBCURL_VERSION_NUM >= 0x071304
- // Restrict protocols so that curl vulnerabilities in
- // other protocols don't affect us.
- // These settings were introduced in curl 7.19.4.
- long protocols =
- CURLPROTO_HTTP |
- CURLPROTO_HTTPS |
- CURLPROTO_FTP |
- CURLPROTO_FTPS;
- curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols);
- curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols);
+ // Restrict protocols so that curl vulnerabilities in
+ // other protocols don't affect us.
+ // These settings were introduced in curl 7.19.4.
+ long protocols =
+ CURLPROTO_HTTP |
+ CURLPROTO_HTTPS |
+ CURLPROTO_FTP |
+ CURLPROTO_FTPS;
+ curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols);
+ curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols);
#endif
- // Set cURL options based on HTTPFetchRequest
- curl_easy_setopt(curl, CURLOPT_URL,
- request.url.c_str());
- curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS,
- request.timeout);
- curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
- request.connect_timeout);
-
- if (request.useragent != "")
- curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str());
-
- // Set up a write callback that writes to the
- // ostringstream ongoing->oss, unless the data
- // is to be discarded
- if (request.caller == HTTPFETCH_DISCARD) {
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
- httpfetch_discardfunction);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
- }
- else {
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
- httpfetch_writefunction);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss);
- }
- // Set POST (or GET) data
- if (request.post_fields.empty()) {
- curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
- }
- else {
- curl_easy_setopt(curl, CURLOPT_POST, 1);
- curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
- request.post_fields.size());
- curl_easy_setopt(curl, CURLOPT_POSTFIELDS,
- request.post_fields.c_str());
- // request.post_fields must now *never* be
- // modified until CURLOPT_POSTFIELDS is cleared
- }
- // Set additional HTTP headers
- for (size_t i = 0; i < request.extra_headers.size(); ++i) {
- httpheader = curl_slist_append(
- httpheader,
- request.extra_headers[i].c_str());
- }
- curl_easy_setopt(curl, CURLOPT_HTTPHEADER, httpheader);
- }
+ // Set cURL options based on HTTPFetchRequest
+ curl_easy_setopt(curl, CURLOPT_URL,
+ request.url.c_str());
+ curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS,
+ request.timeout);
+ curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
+ request.connect_timeout);
+
+ if (request.useragent != "")
+ curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str());
+
+ // Set up a write callback that writes to the
+ // ostringstream ongoing->oss, unless the data
+ // is to be discarded
+ if (request.caller == HTTPFETCH_DISCARD) {
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+ httpfetch_discardfunction);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
+ } else {
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+ httpfetch_writefunction);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss);
}
- CURLcode start(CURLM *multi_)
- {
- if (curl == NULL)
- return CURLE_FAILED_INIT;
-
- if (multi_) {
- // Multi interface (async)
- CURLMcode mres = curl_multi_add_handle(multi_, curl);
- if (mres != CURLM_OK) {
- errorstream<<"curl_multi_add_handle"
- <<" returned error code "<<mres
- <<std::endl;
- return CURLE_FAILED_INIT;
- }
- multi = multi_; // store for curl_multi_remove_handle
- return CURLE_OK;
+ // Set POST (or GET) data
+ if (request.post_fields.empty()) {
+ curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
+ } else if (request.multipart) {
+ curl_httppost *last = NULL;
+ for (std::map<std::string, std::string>::iterator it =
+ request.post_fields.begin();
+ it != request.post_fields.end(); ++it) {
+ curl_formadd(&post, &last,
+ CURLFORM_NAMELENGTH, it->first.size(),
+ CURLFORM_PTRNAME, it->first.c_str(),
+ CURLFORM_CONTENTSLENGTH, it->second.size(),
+ CURLFORM_PTRCONTENTS, it->second.c_str(),
+ CURLFORM_END);
}
- else {
- // Easy interface (sync)
- return curl_easy_perform(curl);
+ curl_easy_setopt(curl, CURLOPT_HTTPPOST, post);
+ // request.post_fields must now *never* be
+ // modified until CURLOPT_HTTPPOST is cleared
+ } else if (request.post_data.empty()) {
+ curl_easy_setopt(curl, CURLOPT_POST, 1);
+ std::string str;
+ for (std::map<std::string, std::string>::iterator it =
+ request.post_fields.begin();
+ it != request.post_fields.end();
+ ++it) {
+ if (str != "")
+ str += "&";
+ str += urlencode(it->first);
+ str += "=";
+ str += urlencode(it->second);
}
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
+ str.size());
+ curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS,
+ str.c_str());
+ } else {
+ curl_easy_setopt(curl, CURLOPT_POST, 1);
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
+ request.post_data.size());
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDS,
+ request.post_data.c_str());
+ // request.post_data must now *never* be
+ // modified until CURLOPT_POSTFIELDS is cleared
}
+ // Set additional HTTP headers
+ for (std::vector<std::string>::iterator it = request.extra_headers.begin();
+ it != request.extra_headers.end(); ++it) {
+ http_header = curl_slist_append(http_header, it->c_str());
+ }
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, http_header);
- void complete(CURLcode res)
- {
- result.succeeded = (res == CURLE_OK);
- result.timeout = (res == CURLE_OPERATION_TIMEDOUT);
- result.data = oss.str();
+ if (!g_settings->getBool("curl_verify_cert")) {
+ curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
+ }
+}
+
+CURLcode HTTPFetchOngoing::start(CURLM *multi_)
+{
+ if (!curl)
+ return CURLE_FAILED_INIT;
+
+ if (!multi_) {
+ // Easy interface (sync)
+ return curl_easy_perform(curl);
+ }
+
+ // Multi interface (async)
+ CURLMcode mres = curl_multi_add_handle(multi_, curl);
+ if (mres != CURLM_OK) {
+ errorstream << "curl_multi_add_handle"
+ << " returned error code " << mres
+ << std::endl;
+ return CURLE_FAILED_INIT;
+ }
+ multi = multi_; // store for curl_multi_remove_handle
+ return CURLE_OK;
+}
- // Get HTTP/FTP response code
+const HTTPFetchResult * HTTPFetchOngoing::complete(CURLcode res)
+{
+ result.succeeded = (res == CURLE_OK);
+ result.timeout = (res == CURLE_OPERATION_TIMEDOUT);
+ result.data = oss.str();
+
+ // Get HTTP/FTP response code
+ result.response_code = 0;
+ if (curl && (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE,
+ &result.response_code) != CURLE_OK)) {
+ // We failed to get a return code, make sure it is still 0
result.response_code = 0;
- if (curl != NULL) {
- if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE,
- &result.response_code) != CURLE_OK) {
- //we failed to get a return code make sure it is still 0
- result.response_code = 0;
- }
- }
+ }
- if (res != CURLE_OK) {
- infostream<<request.url<<" not found ("
- <<curl_easy_strerror(res)<<")"
- <<" (response code "<<result.response_code<<")"
- <<std::endl;
- }
+ if (res != CURLE_OK) {
+ errorstream << request.url << " not found ("
+ << curl_easy_strerror(res) << ")"
+ << " (response code " << result.response_code << ")"
+ << std::endl;
}
- ~HTTPFetchOngoing()
- {
- if (multi != NULL) {
- CURLMcode mres = curl_multi_remove_handle(multi, curl);
- if (mres != CURLM_OK) {
- errorstream<<"curl_multi_remove_handle"
- <<" returned error code "<<mres
- <<std::endl;
- }
- }
+ return &result;
+}
- // Set safe options for the reusable cURL handle
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
- httpfetch_discardfunction);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
- curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL);
- if (httpheader != NULL) {
- curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
- curl_slist_free_all(httpheader);
+HTTPFetchOngoing::~HTTPFetchOngoing()
+{
+ if (multi) {
+ CURLMcode mres = curl_multi_remove_handle(multi, curl);
+ if (mres != CURLM_OK) {
+ errorstream << "curl_multi_remove_handle"
+ << " returned error code " << mres
+ << std::endl;
}
+ }
- // Store the cURL handle for reuse
- pool->free(curl);
+ // Set safe options for the reusable cURL handle
+ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+ httpfetch_discardfunction);
+ curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL);
+ if (http_header) {
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
+ curl_slist_free_all(http_header);
}
-};
+ if (post) {
+ curl_easy_setopt(curl, CURLOPT_HTTPPOST, NULL);
+ curl_formfree(post);
+ }
+
+ // Store the cURL handle for reuse
+ pool->free(curl);
+}
+
class CurlFetchThread : public JThread
{
struct Request {
RequestType type;
- HTTPFetchRequest fetchrequest;
+ HTTPFetchRequest fetch_request;
Event *event;
};
m_parallel_limit = 1;
}
- void requestFetch(const HTTPFetchRequest &fetchrequest)
+ void requestFetch(const HTTPFetchRequest &fetch_request)
{
Request req;
req.type = RT_FETCH;
- req.fetchrequest = fetchrequest;
+ req.fetch_request = fetch_request;
req.event = NULL;
m_requests.push_back(req);
}
{
Request req;
req.type = RT_CLEAR;
- req.fetchrequest.caller = caller;
+ req.fetch_request.caller = caller;
req.event = event;
m_requests.push_back(req);
}
if (req.type == RT_FETCH) {
// New fetch, queue until there are less
// than m_parallel_limit ongoing fetches
- m_queued_fetches.push_back(req.fetchrequest);
+ m_queued_fetches.push_back(req.fetch_request);
// see processQueued() for what happens next
}
else if (req.type == RT_CLEAR) {
- unsigned long caller = req.fetchrequest.caller;
+ unsigned long caller = req.fetch_request.caller;
// Abort all ongoing fetches for the caller
for (std::vector<HTTPFetchOngoing*>::iterator
it = m_all_ongoing.begin();
it != m_all_ongoing.end();) {
- if ((*it)->request.caller == caller) {
+ if ((*it)->getRequest().caller == caller) {
delete (*it);
it = m_all_ongoing.erase(it);
- }
- else
+ } else {
++it;
+ }
}
// Also abort all queued fetches for the caller
m_all_ongoing.push_back(ongoing);
}
else {
- ongoing->complete(res);
- httpfetch_deliver_result(ongoing->result);
+ httpfetch_deliver_result(*ongoing->complete(res));
delete ongoing;
}
}
size_t i = 0;
bool found = false;
for (i = 0; i < m_all_ongoing.size(); ++i) {
- if (m_all_ongoing[i]->curl == msg->easy_handle) {
+ if (m_all_ongoing[i]->getEasyHandle() == msg->easy_handle) {
found = true;
break;
}
if (msg->msg == CURLMSG_DONE && found) {
// m_all_ongoing[i] succeeded or failed.
HTTPFetchOngoing *ongoing = m_all_ongoing[i];
- ongoing->complete(msg->data.result);
- httpfetch_deliver_result(ongoing->result);
+ httpfetch_deliver_result(*ongoing->complete(msg->data.result));
delete ongoing;
m_all_ongoing.erase(m_all_ongoing.begin() + i);
}
if (select_timeout > 0) {
// in Winsock it is forbidden to pass three empty
// fd_sets to select(), so in that case use sleep_ms
- if (max_fd == -1) {
+ if (max_fd != -1) {
select_tv.tv_sec = select_timeout / 1000;
select_tv.tv_usec = (select_timeout % 1000) * 1000;
int retval = select(max_fd + 1, &read_fd_set,
log_register_thread("CurlFetchThread");
DSTACK(__FUNCTION_NAME);
+ porting::setThreadName("CurlFetchThread");
+
CurlHandlePool pool;
m_multi = curl_multi_init();
return NULL;
}
- assert(m_all_ongoing.empty());
+ FATAL_ERROR_IF(!m_all_ongoing.empty(), "Expected empty");
while (!StopRequested()) {
BEGIN_DEBUG_EXCEPTION_HANDLER
*/
while (!m_requests.empty()) {
- Request req = m_requests.pop_front();
+ Request req = m_requests.pop_frontNoEx();
processRequest(req);
}
processQueued(&pool);
<<std::endl;
CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT);
- assert(res == CURLE_OK);
+ FATAL_ERROR_IF(res != CURLE_OK, "CURL init failed");
g_httpfetch_thread = new CurlFetchThread(parallel_limit);
}
curl_global_cleanup();
}
-void httpfetch_async(const HTTPFetchRequest &fetchrequest)
+void httpfetch_async(const HTTPFetchRequest &fetch_request)
{
- g_httpfetch_thread->requestFetch(fetchrequest);
+ g_httpfetch_thread->requestFetch(fetch_request);
if (!g_httpfetch_thread->IsRunning())
g_httpfetch_thread->Start();
}
}
}
-void httpfetch_sync(const HTTPFetchRequest &fetchrequest,
- HTTPFetchResult &fetchresult)
+void httpfetch_sync(const HTTPFetchRequest &fetch_request,
+ HTTPFetchResult &fetch_result)
{
// Create ongoing fetch data and make a cURL handle
// Set cURL options based on HTTPFetchRequest
CurlHandlePool pool;
- HTTPFetchOngoing ongoing(fetchrequest, &pool);
+ HTTPFetchOngoing ongoing(fetch_request, &pool);
// Do the fetch (curl_easy_perform)
CURLcode res = ongoing.start(NULL);
- // Update fetchresult
- ongoing.complete(res);
- fetchresult = ongoing.result;
+ // Update fetch result
+ fetch_result = *ongoing.complete(res);
}
#else // USE_CURL
{
}
-void httpfetch_async(const HTTPFetchRequest &fetchrequest)
+void httpfetch_async(const HTTPFetchRequest &fetch_request)
{
- errorstream<<"httpfetch_async: unable to fetch "<<fetchrequest.url
- <<" because USE_CURL=0"<<std::endl;
+ errorstream << "httpfetch_async: unable to fetch " << fetch_request.url
+ << " because USE_CURL=0" << std::endl;
- HTTPFetchResult fetchresult(fetchrequest); // sets succeeded = false etc.
- httpfetch_deliver_result(fetchresult);
+ HTTPFetchResult fetch_result(fetch_request); // sets succeeded = false etc.
+ httpfetch_deliver_result(fetch_result);
}
static void httpfetch_request_clear(unsigned long caller)
{
}
-void httpfetch_sync(const HTTPFetchRequest &fetchrequest,
- HTTPFetchResult &fetchresult)
+void httpfetch_sync(const HTTPFetchRequest &fetch_request,
+ HTTPFetchResult &fetch_result)
{
- errorstream<<"httpfetch_sync: unable to fetch "<<fetchrequest.url
- <<" because USE_CURL=0"<<std::endl;
+ errorstream << "httpfetch_sync: unable to fetch " << fetch_request.url
+ << " because USE_CURL=0" << std::endl;
- fetchresult = HTTPFetchResult(fetchrequest); // sets succeeded = false etc.
+ fetch_result = HTTPFetchResult(fetch_request); // sets succeeded = false etc.
}
#endif // USE_CURL
projects / minetest.git / blobdiff