X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=src%2Fhttpfetch.cpp;h=65202ce3ebac5ff72c6d14777b2e94cdc0aee35b;hb=0f74c7a977c412a81890926548e2a5c8dae5f6eb;hp=25474c725bee6721f99dc2240562842d7b068fdd;hpb=b03135548bbd9bcb73d14b067b96ec913404dd5f;p=dragonfireclient.git

diff --git a/src/httpfetch.cpp b/src/httpfetch.cpp
index 25474c725..65202ce3e 100644
--- a/src/httpfetch.cpp
+++ b/src/httpfetch.cpp
@@ -18,29 +18,43 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 */
 
 #include "httpfetch.h"
+#include "porting.h" // for sleep_ms(), get_sysinfo(), secure_rand_fill_buf()
 #include <iostream>
 #include <sstream>
 #include <list>
 #include <map>
-#include <errno.h>
-#include "jthread/jevent.h"
+#include <cerrno>
+#include <mutex>
+#include "network/socket.h" // for select()
+#include "threading/event.h"
 #include "config.h"
 #include "exceptions.h"
 #include "debug.h"
 #include "log.h"
 #include "util/container.h"
 #include "util/thread.h"
-#include "socket.h" // for select()
+#include "version.h"
+#include "settings.h"
+#include "noise.h"
+
+std::mutex g_httpfetch_mutex;
+std::map<unsigned long, std::queue<HTTPFetchResult> > g_httpfetch_results;
+PcgRandom g_callerid_randomness;
+
+HTTPFetchRequest::HTTPFetchRequest() :
+	timeout(g_settings->getS32("curl_timeout")),
+	connect_timeout(timeout),
+	useragent(std::string(PROJECT_NAME_C "/") + g_version_hash + " (" + porting::get_sysinfo() + ")")
+{
+}
 
-JMutex g_httpfetch_mutex;
-std::map<unsigned long, std::list<HTTPFetchResult> > g_httpfetch_results;
 
-static void httpfetch_deliver_result(const HTTPFetchResult &fetchresult)
+static void httpfetch_deliver_result(const HTTPFetchResult &fetch_result)
 {
-	unsigned long caller = fetchresult.caller;
+	unsigned long caller = fetch_result.caller;
 	if (caller != HTTPFETCH_DISCARD) {
-		JMutexAutoLock lock(g_httpfetch_mutex);
-		g_httpfetch_results[caller].push_back(fetchresult);
+		MutexAutoLock lock(g_httpfetch_mutex);
+		g_httpfetch_results[caller].push(fetch_result);
 	}
 }
 
@@ -48,26 +62,54 @@ static void httpfetch_request_clear(unsigned long caller);
 
 unsigned long httpfetch_caller_alloc()
 {
-	JMutexAutoLock lock(g_httpfetch_mutex);
+	MutexAutoLock lock(g_httpfetch_mutex);
 
 	// Check each caller ID except HTTPFETCH_DISCARD
 	const unsigned long discard = HTTPFETCH_DISCARD;
 	for (unsigned long caller = discard + 1; caller != discard; ++caller) {
-		std::map<unsigned long, std::list<HTTPFetchResult> >::iterator
+		std::map<unsigned long, std::queue<HTTPFetchResult> >::iterator
 			it = g_httpfetch_results.find(caller);
 		if (it == g_httpfetch_results.end()) {
-			verbosestream<<"httpfetch_caller_alloc: allocating "
-					<<caller<<std::endl;
+			verbosestream << "httpfetch_caller_alloc: allocating "
+					<< caller << std::endl;
 			// Access element to create it
 			g_httpfetch_results[caller];
 			return caller;
 		}
 	}
 
-	assert("httpfetch_caller_alloc: ran out of caller IDs" == 0);
+	FATAL_ERROR("httpfetch_caller_alloc: ran out of caller IDs");
 	return discard;
 }
 
+unsigned long httpfetch_caller_alloc_secure()
+{
+	MutexAutoLock lock(g_httpfetch_mutex);
+
+	// Generate random caller IDs and make sure they're not
+	// already used or equal to HTTPFETCH_DISCARD
+	// Give up after 100 tries to prevent infinite loop
+	u8 tries = 100;
+	unsigned long caller;
+
+	do {
+		caller = (((u64) g_callerid_randomness.next()) << 32) |
+				g_callerid_randomness.next();
+
+		if (--tries < 1) {
+			FATAL_ERROR("httpfetch_caller_alloc_secure: ran out of caller IDs");
+			return HTTPFETCH_DISCARD;
+		}
+	} while (g_httpfetch_results.find(caller) != g_httpfetch_results.end());
+
+	verbosestream << "httpfetch_caller_alloc_secure: allocating "
+		<< caller << std::endl;
+
+	// Access element to create it
+	g_httpfetch_results[caller];
+	return caller;
+}
+
 void httpfetch_caller_free(unsigned long caller)
 {
 	verbosestream<<"httpfetch_caller_free: freeing "
@@ -75,29 +117,29 @@ void httpfetch_caller_free(unsigned long caller)
 
 	httpfetch_request_clear(caller);
 	if (caller != HTTPFETCH_DISCARD) {
-		JMutexAutoLock lock(g_httpfetch_mutex);
+		MutexAutoLock lock(g_httpfetch_mutex);
 		g_httpfetch_results.erase(caller);
 	}
 }
 
-bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetchresult)
+bool httpfetch_async_get(unsigned long caller, HTTPFetchResult &fetch_result)
 {
-	JMutexAutoLock lock(g_httpfetch_mutex);
+	MutexAutoLock lock(g_httpfetch_mutex);
 
 	// Check that caller exists
-	std::map<unsigned long, std::list<HTTPFetchResult> >::iterator
+	std::map<unsigned long, std::queue<HTTPFetchResult> >::iterator
 		it = g_httpfetch_results.find(caller);
 	if (it == g_httpfetch_results.end())
 		return false;
 
 	// Check that result queue is nonempty
-	std::list<HTTPFetchResult> &callerresults = it->second;
-	if (callerresults.empty())
+	std::queue<HTTPFetchResult> &caller_results = it->second;
+	if (caller_results.empty())
 		return false;
 
 	// Pop first result
-	fetchresult = callerresults.front();
-	callerresults.pop_front();
+	fetch_result = caller_results.front();
+	caller_results.pop();
 	return true;
 }
 
@@ -128,7 +170,8 @@ class CurlHandlePool
 	std::list<CURL*> handles;
 
 public:
-	CurlHandlePool() {}
+	CurlHandlePool() = default;
+
 	~CurlHandlePool()
 	{
 		for (std::list<CURL*>::iterator it = handles.begin();
@@ -158,167 +201,239 @@ class CurlHandlePool
 	}
 };
 
-struct HTTPFetchOngoing
+class HTTPFetchOngoing
 {
+public:
+	HTTPFetchOngoing(const HTTPFetchRequest &request, CurlHandlePool *pool);
+	~HTTPFetchOngoing();
+
+	CURLcode start(CURLM *multi);
+	const HTTPFetchResult * complete(CURLcode res);
+
+	const HTTPFetchRequest &getRequest()    const { return request; };
+	const CURL             *getEasyHandle() const { return curl; };
+
+private:
 	CurlHandlePool *pool;
 	CURL *curl;
 	CURLM *multi;
 	HTTPFetchRequest request;
 	HTTPFetchResult result;
 	std::ostringstream oss;
-	char *post_fields;
-	struct curl_slist *httpheader;
-
-	HTTPFetchOngoing(HTTPFetchRequest request_, CurlHandlePool *pool_):
-		pool(pool_),
-		curl(NULL),
-		multi(NULL),
-		request(request_),
-		result(request_),
-		oss(std::ios::binary),
-		httpheader(NULL)
-	{
-		curl = pool->alloc();
-		if (curl != NULL) {
-			// Set static cURL options
-			curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
-			curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
-			curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
-			curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 1);
+	struct curl_slist *http_header;
+	curl_httppost *post;
+};
+
+
+HTTPFetchOngoing::HTTPFetchOngoing(const HTTPFetchRequest &request_,
+		CurlHandlePool *pool_):
+	pool(pool_),
+	curl(NULL),
+	multi(NULL),
+	request(request_),
+	result(request_),
+	oss(std::ios::binary),
+	http_header(NULL),
+	post(NULL)
+{
+	curl = pool->alloc();
+	if (curl == NULL) {
+		return;
+	}
+
+	// Set static cURL options
+	curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1);
+	curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+	curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+	curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 3);
+	curl_easy_setopt(curl, CURLOPT_ENCODING, "gzip");
+
+	std::string bind_address = g_settings->get("bind_address");
+	if (!bind_address.empty()) {
+		curl_easy_setopt(curl, CURLOPT_INTERFACE, bind_address.c_str());
+	}
+
+	if (!g_settings->getBool("enable_ipv6")) {
+		curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
+	}
 
 #if LIBCURL_VERSION_NUM >= 0x071304
-			// Restrict protocols so that curl vulnerabilities in
-			// other protocols don't affect us.
-			// These settings were introduced in curl 7.19.4.
-			long protocols =
-				CURLPROTO_HTTP |
-				CURLPROTO_HTTPS |
-				CURLPROTO_FTP |
-				CURLPROTO_FTPS;
-			curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols);
-			curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols);
+	// Restrict protocols so that curl vulnerabilities in
+	// other protocols don't affect us.
+	// These settings were introduced in curl 7.19.4.
+	long protocols =
+		CURLPROTO_HTTP |
+		CURLPROTO_HTTPS |
+		CURLPROTO_FTP |
+		CURLPROTO_FTPS;
+	curl_easy_setopt(curl, CURLOPT_PROTOCOLS, protocols);
+	curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, protocols);
 #endif
 
-			// Set cURL options based on HTTPFetchRequest
-			curl_easy_setopt(curl, CURLOPT_URL,
-					request.url.c_str());
-			curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS,
-					request.timeout);
-			curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
-					request.connect_timeout);
-
-			if (request.useragent != "")
-				curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str());
-
-			// Set up a write callback that writes to the
-			// ostringstream ongoing->oss, unless the data
-			// is to be discarded
-			if (request.caller == HTTPFETCH_DISCARD) {
-				curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
-						httpfetch_discardfunction);
-				curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
-			}
-			else {
-				curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
-						httpfetch_writefunction);
-				curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss);
-			}
-			// Set POST (or GET) data
-			if (request.post_fields.empty()) {
-				curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
-			}
-			else {
-				curl_easy_setopt(curl, CURLOPT_POST, 1);
+	// Set cURL options based on HTTPFetchRequest
+	curl_easy_setopt(curl, CURLOPT_URL,
+			request.url.c_str());
+	curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS,
+			request.timeout);
+	curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS,
+			request.connect_timeout);
+
+	if (!request.useragent.empty())
+		curl_easy_setopt(curl, CURLOPT_USERAGENT, request.useragent.c_str());
+
+	// Set up a write callback that writes to the
+	// ostringstream ongoing->oss, unless the data
+	// is to be discarded
+	if (request.caller == HTTPFETCH_DISCARD) {
+		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+				httpfetch_discardfunction);
+		curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
+	} else {
+		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+				httpfetch_writefunction);
+		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss);
+	}
+
+	// Set data from fields or raw_data
+	if (request.multipart) {
+		curl_httppost *last = NULL;
+		for (StringMap::iterator it = request.fields.begin();
+				it != request.fields.end(); ++it) {
+			curl_formadd(&post, &last,
+					CURLFORM_NAMELENGTH, it->first.size(),
+					CURLFORM_PTRNAME, it->first.c_str(),
+					CURLFORM_CONTENTSLENGTH, it->second.size(),
+					CURLFORM_PTRCONTENTS, it->second.c_str(),
+					CURLFORM_END);
+		}
+		curl_easy_setopt(curl, CURLOPT_HTTPPOST, post);
+		// request.post_fields must now *never* be
+		// modified until CURLOPT_HTTPPOST is cleared
+	} else {
+		switch (request.method) {
+		case HTTP_GET:
+			curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
+			break;
+		case HTTP_POST:
+			curl_easy_setopt(curl, CURLOPT_POST, 1);
+			break;
+		case HTTP_PUT:
+			curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
+			break;
+		case HTTP_DELETE:
+			curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
+			break;
+		}
+		if (request.method != HTTP_GET) {
+			if (!request.raw_data.empty()) {
 				curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
-						request.post_fields.size());
+						request.raw_data.size());
 				curl_easy_setopt(curl, CURLOPT_POSTFIELDS,
-						request.post_fields.c_str());
-				// request.post_fields must now *never* be
-				// modified until CURLOPT_POSTFIELDS is cleared
-			}
-			// Set additional HTTP headers
-			for (size_t i = 0; i < request.extra_headers.size(); ++i) {
-				httpheader = curl_slist_append(
-					httpheader,
-					request.extra_headers[i].c_str());
+						request.raw_data.c_str());
+			} else if (!request.fields.empty()) {
+				std::string str;
+				for (auto &field : request.fields) {
+					if (!str.empty())
+						str += "&";
+					str += urlencode(field.first);
+					str += "=";
+					str += urlencode(field.second);
+				}
+				curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE,
+						str.size());
+				curl_easy_setopt(curl, CURLOPT_COPYPOSTFIELDS,
+						str.c_str());
 			}
-			curl_easy_setopt(curl, CURLOPT_HTTPHEADER, httpheader);
 		}
 	}
+	// Set additional HTTP headers
+	for (const std::string &extra_header : request.extra_headers) {
+		http_header = curl_slist_append(http_header, extra_header.c_str());
+	}
+	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, http_header);
 
-	CURLcode start(CURLM *multi_)
-	{
-		if (curl == NULL)
-			return CURLE_FAILED_INIT;
-
-		if (multi_) {
-			// Multi interface (async)
-			CURLMcode mres = curl_multi_add_handle(multi_, curl);
-			if (mres != CURLM_OK) {
-				errorstream<<"curl_multi_add_handle"
-					<<" returned error code "<<mres
-					<<std::endl;
-				return CURLE_FAILED_INIT;
-			}
-			multi = multi_; // store for curl_multi_remove_handle
-			return CURLE_OK;
-		}
-		else {
-			// Easy interface (sync)
-			return curl_easy_perform(curl);
-		}
+	if (!g_settings->getBool("curl_verify_cert")) {
+		curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, false);
 	}
+}
 
-	void complete(CURLcode res)
-	{
-		result.succeeded = (res == CURLE_OK);
-		result.timeout = (res == CURLE_OPERATION_TIMEDOUT);
-		result.data = oss.str();
+CURLcode HTTPFetchOngoing::start(CURLM *multi_)
+{
+	if (!curl)
+		return CURLE_FAILED_INIT;
+
+	if (!multi_) {
+		// Easy interface (sync)
+		return curl_easy_perform(curl);
+	}
 
-		// Get HTTP/FTP response code
+	// Multi interface (async)
+	CURLMcode mres = curl_multi_add_handle(multi_, curl);
+	if (mres != CURLM_OK) {
+		errorstream << "curl_multi_add_handle"
+			<< " returned error code " << mres
+			<< std::endl;
+		return CURLE_FAILED_INIT;
+	}
+	multi = multi_; // store for curl_multi_remove_handle
+	return CURLE_OK;
+}
+
+const HTTPFetchResult * HTTPFetchOngoing::complete(CURLcode res)
+{
+	result.succeeded = (res == CURLE_OK);
+	result.timeout = (res == CURLE_OPERATION_TIMEDOUT);
+	result.data = oss.str();
+
+	// Get HTTP/FTP response code
+	result.response_code = 0;
+	if (curl && (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE,
+				&result.response_code) != CURLE_OK)) {
+		// We failed to get a return code, make sure it is still 0
 		result.response_code = 0;
-		if (curl != NULL) {
-			if (curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE,
-					&result.response_code) != CURLE_OK) {
-				result.response_code = 0;
-			}
-		}
+	}
 
-		if (res != CURLE_OK) {
-			infostream<<request.url<<" not found ("
-				<<curl_easy_strerror(res)<<")"
-				<<" (response code "<<result.response_code<<")"
-				<<std::endl;
-		}
+	if (res != CURLE_OK) {
+		errorstream << request.url << " not found ("
+			<< curl_easy_strerror(res) << ")"
+			<< " (response code " << result.response_code << ")"
+			<< std::endl;
 	}
 
-	~HTTPFetchOngoing()
-	{
-		if (multi != NULL) {
-			CURLMcode mres = curl_multi_remove_handle(multi, curl);
-			if (mres != CURLM_OK) {
-				errorstream<<"curl_multi_remove_handle"
-					<<" returned error code "<<mres
-					<<std::endl;
-			}
-		}
+	return &result;
+}
 
-		// Set safe options for the reusable cURL handle
-		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
-				httpfetch_discardfunction);
-		curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
-		curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL);
-		if (httpheader != NULL) {
-			curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
-			curl_slist_free_all(httpheader);
+HTTPFetchOngoing::~HTTPFetchOngoing()
+{
+	if (multi) {
+		CURLMcode mres = curl_multi_remove_handle(multi, curl);
+		if (mres != CURLM_OK) {
+			errorstream << "curl_multi_remove_handle"
+				<< " returned error code " << mres
+				<< std::endl;
 		}
+	}
 
-		// Store the cURL handle for reuse
-		pool->free(curl);
+	// Set safe options for the reusable cURL handle
+	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+			httpfetch_discardfunction);
+	curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
+	curl_easy_setopt(curl, CURLOPT_POSTFIELDS, NULL);
+	if (http_header) {
+		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
+		curl_slist_free_all(http_header);
 	}
-};
+	if (post) {
+		curl_easy_setopt(curl, CURLOPT_HTTPPOST, NULL);
+		curl_formfree(post);
+	}
+
+	// Store the cURL handle for reuse
+	pool->free(curl);
+}
+
 
-class CurlFetchThread : public SimpleThread
+class CurlFetchThread : public Thread
 {
 protected:
 	enum RequestType {
@@ -329,7 +444,7 @@ class CurlFetchThread : public SimpleThread
 
 	struct Request {
 		RequestType type;
-		HTTPFetchRequest fetchrequest;
+		HTTPFetchRequest fetch_request;
 		Event *event;
 	};
 
@@ -342,7 +457,8 @@ class CurlFetchThread : public SimpleThread
 	std::list<HTTPFetchRequest> m_queued_fetches;
 
 public:
-	CurlFetchThread(int parallel_limit)
+	CurlFetchThread(int parallel_limit) :
+		Thread("CurlFetch")
 	{
 		if (parallel_limit >= 1)
 			m_parallel_limit = parallel_limit;
@@ -350,11 +466,11 @@ class CurlFetchThread : public SimpleThread
 			m_parallel_limit = 1;
 	}
 
-	void requestFetch(const HTTPFetchRequest &fetchrequest)
+	void requestFetch(const HTTPFetchRequest &fetch_request)
 	{
 		Request req;
 		req.type = RT_FETCH;
-		req.fetchrequest = fetchrequest;
+		req.fetch_request = fetch_request;
 		req.event = NULL;
 		m_requests.push_back(req);
 	}
@@ -363,7 +479,7 @@ class CurlFetchThread : public SimpleThread
 	{
 		Request req;
 		req.type = RT_CLEAR;
-		req.fetchrequest.caller = caller;
+		req.fetch_request.caller = caller;
 		req.event = event;
 		m_requests.push_back(req);
 	}
@@ -384,24 +500,24 @@ class CurlFetchThread : public SimpleThread
 		if (req.type == RT_FETCH) {
 			// New fetch, queue until there are less
 			// than m_parallel_limit ongoing fetches
-			m_queued_fetches.push_back(req.fetchrequest);
+			m_queued_fetches.push_back(req.fetch_request);
 
 			// see processQueued() for what happens next
 
 		}
 		else if (req.type == RT_CLEAR) {
-			unsigned long caller = req.fetchrequest.caller;
+			unsigned long caller = req.fetch_request.caller;
 
 			// Abort all ongoing fetches for the caller
 			for (std::vector<HTTPFetchOngoing*>::iterator
 					it = m_all_ongoing.begin();
 					it != m_all_ongoing.end();) {
-				if ((*it)->request.caller == caller) {
+				if ((*it)->getRequest().caller == caller) {
 					delete (*it);
 					it = m_all_ongoing.erase(it);
-				}
-				else
+				} else {
 					++it;
+				}
 			}
 
 			// Also abort all queued fetches for the caller
@@ -441,8 +557,7 @@ class CurlFetchThread : public SimpleThread
 				m_all_ongoing.push_back(ongoing);
 			}
 			else {
-				ongoing->complete(res);
-				httpfetch_deliver_result(ongoing->result);
+				httpfetch_deliver_result(*ongoing->complete(res));
 				delete ongoing;
 			}
 		}
@@ -455,7 +570,7 @@ class CurlFetchThread : public SimpleThread
 		size_t i = 0;
 		bool found = false;
 		for (i = 0; i < m_all_ongoing.size(); ++i) {
-			if (m_all_ongoing[i]->curl == msg->easy_handle) {
+			if (m_all_ongoing[i]->getEasyHandle() == msg->easy_handle) {
 				found = true;
 				break;
 			}
@@ -463,8 +578,7 @@ class CurlFetchThread : public SimpleThread
 		if (msg->msg == CURLMSG_DONE && found) {
 			// m_all_ongoing[i] succeeded or failed.
 			HTTPFetchOngoing *ongoing = m_all_ongoing[i];
-			ongoing->complete(msg->data.result);
-			httpfetch_deliver_result(ongoing->result);
+			httpfetch_deliver_result(*ongoing->complete(msg->data.result));
 			delete ongoing;
 			m_all_ongoing.erase(m_all_ongoing.begin() + i);
 		}
@@ -519,29 +633,32 @@ class CurlFetchThread : public SimpleThread
 			select_timeout = timeout;
 
 		if (select_timeout > 0) {
-			select_tv.tv_sec = select_timeout / 1000;
-			select_tv.tv_usec = (select_timeout % 1000) * 1000;
-			int retval = select(max_fd + 1, &read_fd_set,
-					&write_fd_set, &exc_fd_set,
-					&select_tv);
-			if (retval == -1) {
-				#ifdef _WIN32
-				errorstream<<"select returned error code "
-					<<WSAGetLastError()<<std::endl;
-				#else
-				errorstream<<"select returned error code "
-					<<errno<<std::endl;
-				#endif
+			// in Winsock it is forbidden to pass three empty
+			// fd_sets to select(), so in that case use sleep_ms
+			if (max_fd != -1) {
+				select_tv.tv_sec = select_timeout / 1000;
+				select_tv.tv_usec = (select_timeout % 1000) * 1000;
+				int retval = select(max_fd + 1, &read_fd_set,
+						&write_fd_set, &exc_fd_set,
+						&select_tv);
+				if (retval == -1) {
+					#ifdef _WIN32
+					errorstream<<"select returned error code "
+						<<WSAGetLastError()<<std::endl;
+					#else
+					errorstream<<"select returned error code "
+						<<errno<<std::endl;
+					#endif
+				}
+			}
+			else {
+				sleep_ms(select_timeout);
 			}
 		}
 	}
 
-	void * Thread()
+	void *run()
 	{
-		ThreadStarted();
-		log_register_thread("CurlFetchThread");
-		DSTACK(__FUNCTION_NAME);
-
 		CurlHandlePool pool;
 
 		m_multi = curl_multi_init();
@@ -550,9 +667,9 @@ class CurlFetchThread : public SimpleThread
 			return NULL;
 		}
 
-		assert(m_all_ongoing.empty());
+		FATAL_ERROR_IF(!m_all_ongoing.empty(), "Expected empty");
 
-		while (getRun()) {
+		while (!stopRequested()) {
 			BEGIN_DEBUG_EXCEPTION_HANDLER
 
 			/*
@@ -560,7 +677,7 @@ class CurlFetchThread : public SimpleThread
 			*/
 
 			while (!m_requests.empty()) {
-				Request req = m_requests.pop_front();
+				Request req = m_requests.pop_frontNoEx();
 				processRequest(req);
 			}
 			processQueued(&pool);
@@ -601,12 +718,12 @@ class CurlFetchThread : public SimpleThread
 			else
 				waitForIO(100);
 
-			END_DEBUG_EXCEPTION_HANDLER(errorstream)
+			END_DEBUG_EXCEPTION_HANDLER
 		}
 
 		// Call curl_multi_remove_handle and cleanup easy handles
-		for (size_t i = 0; i < m_all_ongoing.size(); ++i) {
-			delete m_all_ongoing[i];
+		for (HTTPFetchOngoing *i : m_all_ongoing) {
+			delete i;
 		}
 		m_all_ongoing.clear();
 
@@ -631,54 +748,57 @@ void httpfetch_init(int parallel_limit)
 			<<std::endl;
 
 	CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT);
-	assert(res == CURLE_OK);
+	FATAL_ERROR_IF(res != CURLE_OK, "CURL init failed");
 
 	g_httpfetch_thread = new CurlFetchThread(parallel_limit);
+
+	// Initialize g_callerid_randomness for httpfetch_caller_alloc_secure
+	u64 randbuf[2];
+	porting::secure_rand_fill_buf(randbuf, sizeof(u64) * 2);
+	g_callerid_randomness = PcgRandom(randbuf[0], randbuf[1]);
 }
 
 void httpfetch_cleanup()
 {
 	verbosestream<<"httpfetch_cleanup: cleaning up"<<std::endl;
 
-	g_httpfetch_thread->setRun(false);
-	g_httpfetch_thread->requestWakeUp();
 	g_httpfetch_thread->stop();
+	g_httpfetch_thread->requestWakeUp();
+	g_httpfetch_thread->wait();
 	delete g_httpfetch_thread;
 
 	curl_global_cleanup();
 }
 
-void httpfetch_async(const HTTPFetchRequest &fetchrequest)
+void httpfetch_async(const HTTPFetchRequest &fetch_request)
 {
-	g_httpfetch_thread->requestFetch(fetchrequest);
-	if (!g_httpfetch_thread->IsRunning())
-		g_httpfetch_thread->Start();
+	g_httpfetch_thread->requestFetch(fetch_request);
+	if (!g_httpfetch_thread->isRunning())
+		g_httpfetch_thread->start();
 }
 
 static void httpfetch_request_clear(unsigned long caller)
 {
-	if (g_httpfetch_thread->IsRunning()) {
+	if (g_httpfetch_thread->isRunning()) {
 		Event event;
 		g_httpfetch_thread->requestClear(caller, &event);
 		event.wait();
-	}
-	else {
+	} else {
 		g_httpfetch_thread->requestClear(caller, NULL);
 	}
 }
 
-void httpfetch_sync(const HTTPFetchRequest &fetchrequest,
-		HTTPFetchResult &fetchresult)
+void httpfetch_sync(const HTTPFetchRequest &fetch_request,
+		HTTPFetchResult &fetch_result)
 {
 	// Create ongoing fetch data and make a cURL handle
 	// Set cURL options based on HTTPFetchRequest
 	CurlHandlePool pool;
-	HTTPFetchOngoing ongoing(fetchrequest, &pool);
+	HTTPFetchOngoing ongoing(fetch_request, &pool);
 	// Do the fetch (curl_easy_perform)
 	CURLcode res = ongoing.start(NULL);
-	// Update fetchresult
-	ongoing.complete(res);
-	fetchresult = ongoing.result;
+	// Update fetch result
+	fetch_result = *ongoing.complete(res);
 }
 
 #else  // USE_CURL
@@ -697,26 +817,26 @@ void httpfetch_cleanup()
 {
 }
 
-void httpfetch_async(const HTTPFetchRequest &fetchrequest)
+void httpfetch_async(const HTTPFetchRequest &fetch_request)
 {
-	errorstream<<"httpfetch_async: unable to fetch "<<fetchrequest.url
-			<<" because USE_CURL=0"<<std::endl;
+	errorstream << "httpfetch_async: unable to fetch " << fetch_request.url
+			<< " because USE_CURL=0" << std::endl;
 
-	HTTPFetchResult fetchresult(fetchrequest); // sets succeeded = false etc.
-	httpfetch_deliver_result(fetchresult);
+	HTTPFetchResult fetch_result(fetch_request); // sets succeeded = false etc.
+	httpfetch_deliver_result(fetch_result);
 }
 
 static void httpfetch_request_clear(unsigned long caller)
 {
 }
 
-void httpfetch_sync(const HTTPFetchRequest &fetchrequest,
-		HTTPFetchResult &fetchresult)
+void httpfetch_sync(const HTTPFetchRequest &fetch_request,
+		HTTPFetchResult &fetch_result)
 {
-	errorstream<<"httpfetch_sync: unable to fetch "<<fetchrequest.url
-			<<" because USE_CURL=0"<<std::endl;
+	errorstream << "httpfetch_sync: unable to fetch " << fetch_request.url
+			<< " because USE_CURL=0" << std::endl;
 
-	fetchresult = HTTPFetchResult(fetchrequest); // sets succeeded = false etc.
+	fetch_result = HTTPFetchResult(fetch_request); // sets succeeded = false etc.
 }
 
 #endif  // USE_CURL