X-Git-Url: https://git.lizzy.rs/?a=blobdiff_plain;f=src%2Fdatabase-sqlite3.cpp;h=095d485c0f72523d91196fd2d8f04da5681e96d8;hb=de83c29ba0a517630b361ffdbebd1a1f0b5eed4e;hp=38fb2ca1931b62f7c3b37d4573ac761e97f66ea4;hpb=068dd796f51e59953d1aff9f05510847e2bfe872;p=dragonfireclient.git

diff --git a/src/database-sqlite3.cpp b/src/database-sqlite3.cpp
index 38fb2ca19..095d485c0 100644
--- a/src/database-sqlite3.cpp
+++ b/src/database-sqlite3.cpp
@@ -18,334 +18,283 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 */
 
 /*
-	SQLite format specification:
-	- Initially only replaces sectors/ and sectors2/
-	
-	If map.sqlite does not exist in the save dir
-	or the block was not found in the database
-	the map will try to load from sectors folder.
-	In either case, map.sqlite will be created
-	and all future saves will save there.
-	
-	Structure of map.sqlite:
-	Tables:
-		blocks
-			(PK) INT pos
-			BLOB data
+SQLite format specification:
+	blocks:
+		(PK) INT id
+		BLOB data
 */
 
 
 #include "database-sqlite3.h"
 
-#include "map.h"
-#include "mapsector.h"
-#include "mapblock.h"
-#include "serialization.h"
-#include "main.h"
-#include "settings.h"
 #include "log.h"
+#include "filesys.h"
+#include "exceptions.h"
+#include "settings.h"
+#include "porting.h"
+#include "util/string.h"
+
+#include <cassert>
+
+// When to print messages when the database is being held locked by another process
+// Note: I've seen occasional delays of over 250ms while running minetestmapper.
+#define BUSY_INFO_TRESHOLD	100	// Print first informational message after 100ms.
+#define BUSY_WARNING_TRESHOLD	250	// Print warning message after 250ms. Lag is increased.
+#define BUSY_ERROR_TRESHOLD	1000	// Print error message after 1000ms. Significant lag.
+#define BUSY_FATAL_TRESHOLD	3000	// Allow SQLITE_BUSY to be returned, which will cause a minetest crash.
+#define BUSY_ERROR_INTERVAL	10000	// Safety net: report again every 10 seconds
+
+
+#define SQLRES(s, r, m) \
+	if ((s) != (r)) { \
+		throw DatabaseException(std::string(m) + ": " +\
+				sqlite3_errmsg(m_database)); \
+	}
+#define SQLOK(s, m) SQLRES(s, SQLITE_OK, m)
 
-Database_SQLite3::Database_SQLite3(ServerMap *map, std::string savedir)
+#define PREPARE_STATEMENT(name, query) \
+	SQLOK(sqlite3_prepare_v2(m_database, query, -1, &m_stmt_##name, NULL),\
+		"Failed to prepare query '" query "'")
+
+#define SQLOK_ERRSTREAM(s, m)                           \
+	if ((s) != SQLITE_OK) {                             \
+		errorstream << (m) << ": "                      \
+			<< sqlite3_errmsg(m_database) << std::endl; \
+	}
+
+#define FINALIZE_STATEMENT(statement) SQLOK_ERRSTREAM(sqlite3_finalize(statement), \
+	"Failed to finalize " #statement)
+
+int Database_SQLite3::busyHandler(void *data, int count)
 {
-	m_database = NULL;
-	m_database_read = NULL;
-	m_database_write = NULL;
-	m_database_list = NULL;
-	m_savedir = savedir;
-	srvmap = map;
+	s64 &first_time = reinterpret_cast<s64 *>(data)[0];
+	s64 &prev_time = reinterpret_cast<s64 *>(data)[1];
+	s64 cur_time = getTimeMs();
+
+	if (count == 0) {
+		first_time = cur_time;
+		prev_time = first_time;
+	} else {
+		while (cur_time < prev_time)
+			cur_time += s64(1)<<32;
+	}
+
+	if (cur_time - first_time < BUSY_INFO_TRESHOLD) {
+		; // do nothing
+	} else if (cur_time - first_time >= BUSY_INFO_TRESHOLD &&
+			prev_time - first_time < BUSY_INFO_TRESHOLD) {
+		infostream << "SQLite3 database has been locked for "
+			<< cur_time - first_time << " ms." << std::endl;
+	} else if (cur_time - first_time >= BUSY_WARNING_TRESHOLD &&
+			prev_time - first_time < BUSY_WARNING_TRESHOLD) {
+		warningstream << "SQLite3 database has been locked for "
+			<< cur_time - first_time << " ms." << std::endl;
+	} else if (cur_time - first_time >= BUSY_ERROR_TRESHOLD &&
+			prev_time - first_time < BUSY_ERROR_TRESHOLD) {
+		errorstream << "SQLite3 database has been locked for "
+			<< cur_time - first_time << " ms; this causes lag." << std::endl;
+	} else if (cur_time - first_time >= BUSY_FATAL_TRESHOLD &&
+			prev_time - first_time < BUSY_FATAL_TRESHOLD) {
+		errorstream << "SQLite3 database has been locked for "
+			<< cur_time - first_time << " ms - giving up!" << std::endl;
+	} else if ((cur_time - first_time) / BUSY_ERROR_INTERVAL !=
+			(prev_time - first_time) / BUSY_ERROR_INTERVAL) {
+		// Safety net: keep reporting every BUSY_ERROR_INTERVAL
+		errorstream << "SQLite3 database has been locked for "
+			<< (cur_time - first_time) / 1000 << " seconds!" << std::endl;
+	}
+
+	prev_time = cur_time;
+
+	// Make sqlite transaction fail if delay exceeds BUSY_FATAL_TRESHOLD
+	return cur_time - first_time < BUSY_FATAL_TRESHOLD;
 }
 
-int Database_SQLite3::Initialized(void)
+
+Database_SQLite3::Database_SQLite3(const std::string &savedir) :
+	m_initialized(false),
+	m_savedir(savedir),
+	m_database(NULL),
+	m_stmt_read(NULL),
+	m_stmt_write(NULL),
+	m_stmt_list(NULL),
+	m_stmt_delete(NULL),
+	m_stmt_begin(NULL),
+	m_stmt_end(NULL)
 {
-	return m_database ? 1 : 0;
 }
 
 void Database_SQLite3::beginSave() {
 	verifyDatabase();
-	if(sqlite3_exec(m_database, "BEGIN;", NULL, NULL, NULL) != SQLITE_OK)
-		infostream<<"WARNING: beginSave() failed, saving might be slow.";
+	SQLRES(sqlite3_step(m_stmt_begin), SQLITE_DONE,
+		"Failed to start SQLite3 transaction");
+	sqlite3_reset(m_stmt_begin);
 }
 
 void Database_SQLite3::endSave() {
 	verifyDatabase();
-	if(sqlite3_exec(m_database, "COMMIT;", NULL, NULL, NULL) != SQLITE_OK)
-		infostream<<"WARNING: endSave() failed, map might not have saved.";
+	SQLRES(sqlite3_step(m_stmt_end), SQLITE_DONE,
+		"Failed to commit SQLite3 transaction");
+	sqlite3_reset(m_stmt_end);
 }
 
-void Database_SQLite3::createDirs(std::string path)
+void Database_SQLite3::openDatabase()
 {
-	if(fs::CreateAllDirs(path) == false)
-	{
-		infostream<<DTIME<<"Database_SQLite3: Failed to create directory "
-				<<"\""<<path<<"\""<<std::endl;
-		throw BaseException("Database_SQLite3 failed to create directory");
+	if (m_database) return;
+
+	std::string dbp = m_savedir + DIR_DELIM + "map.sqlite";
+
+	// Open the database connection
+
+	if (!fs::CreateAllDirs(m_savedir)) {
+		infostream << "Database_SQLite3: Failed to create directory \""
+			<< m_savedir << "\"" << std::endl;
+		throw FileNotGoodException("Failed to create database "
+				"save directory");
 	}
-}
 
-void Database_SQLite3::verifyDatabase() {
-	if(m_database)
-		return;
-	
-	{
-		std::string dbp = m_savedir + DIR_DELIM + "map.sqlite";
-		bool needs_create = false;
-		int d;
-		
-		/*
-			Open the database connection
-		*/
-	
-		createDirs(m_savedir); // ?
-	
-		if(!fs::PathExists(dbp))
-			needs_create = true;
-	
-		d = sqlite3_open_v2(dbp.c_str(), &m_database, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL);
-		if(d != SQLITE_OK) {
-			infostream<<"WARNING: SQLite3 database failed to open: "<<sqlite3_errmsg(m_database)<<std::endl;
-			throw FileNotGoodException("Cannot open database file");
-		}
-		
-		if(needs_create)
-			createDatabase();
-
-		std::string querystr = std::string("PRAGMA synchronous = ")
-				 + itos(g_settings->getU16("sqlite_synchronous"));
-		d = sqlite3_exec(m_database, querystr.c_str(), NULL, NULL, NULL);
-		if(d != SQLITE_OK) {
-			infostream<<"WARNING: Database pragma set failed: "
-					<<sqlite3_errmsg(m_database)<<std::endl;
-			throw FileNotGoodException("Cannot set pragma");
-		}
-
-		d = sqlite3_prepare(m_database, "SELECT `data` FROM `blocks` WHERE `pos`=? LIMIT 1", -1, &m_database_read, NULL);
-		if(d != SQLITE_OK) {
-			infostream<<"WARNING: SQLite3 database read statment failed to prepare: "<<sqlite3_errmsg(m_database)<<std::endl;
-			throw FileNotGoodException("Cannot prepare read statement");
-		}
-		
-		d = sqlite3_prepare(m_database, "REPLACE INTO `blocks` VALUES(?, ?)", -1, &m_database_write, NULL);
-		if(d != SQLITE_OK) {
-			infostream<<"WARNING: SQLite3 database write statment failed to prepare: "<<sqlite3_errmsg(m_database)<<std::endl;
-			throw FileNotGoodException("Cannot prepare write statement");
-		}
-		
-		d = sqlite3_prepare(m_database, "SELECT `pos` FROM `blocks`", -1, &m_database_list, NULL);
-		if(d != SQLITE_OK) {
-			infostream<<"WARNING: SQLite3 database list statment failed to prepare: "<<sqlite3_errmsg(m_database)<<std::endl;
-			throw FileNotGoodException("Cannot prepare read statement");
-		}
-		
-		infostream<<"ServerMap: SQLite3 database opened"<<std::endl;
+	bool needs_create = !fs::PathExists(dbp);
+
+	SQLOK(sqlite3_open_v2(dbp.c_str(), &m_database,
+			SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL),
+		std::string("Failed to open SQLite3 database file ") + dbp);
+
+	SQLOK(sqlite3_busy_handler(m_database, Database_SQLite3::busyHandler,
+		m_busy_handler_data), "Failed to set SQLite3 busy handler");
+
+	if (needs_create) {
+		createDatabase();
 	}
+
+	std::string query_str = std::string("PRAGMA synchronous = ")
+			 + itos(g_settings->getU16("sqlite_synchronous"));
+	SQLOK(sqlite3_exec(m_database, query_str.c_str(), NULL, NULL, NULL),
+		"Failed to modify sqlite3 synchronous mode");
 }
 
-void Database_SQLite3::saveBlock(MapBlock *block)
+void Database_SQLite3::verifyDatabase()
 {
-	DSTACK(__FUNCTION_NAME);
-	/*
-		Dummy blocks are not written
-	*/
-	if(block->isDummy())
-	{
-		/*v3s16 p = block->getPos();
-		infostream<<"Database_SQLite3::saveBlock(): WARNING: Not writing dummy block "
-				<<"("<<p.X<<","<<p.Y<<","<<p.Z<<")"<<std::endl;*/
-		return;
-	}
+	if (m_initialized) return;
 
-	// Format used for writing
-	u8 version = SER_FMT_VER_HIGHEST_WRITE;
-	// Get destination
-	v3s16 p3d = block->getPos();
-	
-	
-#if 0
-	v2s16 p2d(p3d.X, p3d.Z);
-	std::string sectordir = getSectorDir(p2d);
-
-	createDirs(sectordir);
-
-	std::string fullpath = sectordir+DIR_DELIM+getBlockFilename(p3d);
-	std::ofstream o(fullpath.c_str(), std::ios_base::binary);
-	if(o.good() == false)
-		throw FileNotGoodException("Cannot open block data");
+	openDatabase();
+
+	PREPARE_STATEMENT(begin, "BEGIN");
+	PREPARE_STATEMENT(end, "COMMIT");
+	PREPARE_STATEMENT(read, "SELECT `data` FROM `blocks` WHERE `pos` = ? LIMIT 1");
+#ifdef __ANDROID__
+	PREPARE_STATEMENT(write,  "INSERT INTO `blocks` (`pos`, `data`) VALUES (?, ?)");
+#else
+	PREPARE_STATEMENT(write, "REPLACE INTO `blocks` (`pos`, `data`) VALUES (?, ?)");
 #endif
-	/*
-		[0] u8 serialization version
-		[1] data
-	*/
-	
+	PREPARE_STATEMENT(delete, "DELETE FROM `blocks` WHERE `pos` = ?");
+	PREPARE_STATEMENT(list, "SELECT `pos` FROM `blocks`");
+
+	m_initialized = true;
+
+	verbosestream << "ServerMap: SQLite3 database opened." << std::endl;
+}
+
+inline void Database_SQLite3::bindPos(sqlite3_stmt *stmt, const v3s16 &pos, int index)
+{
+	SQLOK(sqlite3_bind_int64(stmt, index, getBlockAsInteger(pos)),
+		"Internal error: failed to bind query at " __FILE__ ":" TOSTRING(__LINE__));
+}
+
+bool Database_SQLite3::deleteBlock(const v3s16 &pos)
+{
 	verifyDatabase();
-	
-	std::ostringstream o(std::ios_base::binary);
-	
-	o.write((char*)&version, 1);
-	
-	// Write basic data
-	block->serialize(o, version, true);
-	
-	// Write block to database
-	
-	std::string tmp = o.str();
-	const char *bytes = tmp.c_str();
-	
-	if(sqlite3_bind_int64(m_database_write, 1, getBlockAsInteger(p3d)) != SQLITE_OK)
-		infostream<<"WARNING: Block position failed to bind: "<<sqlite3_errmsg(m_database)<<std::endl;
-	if(sqlite3_bind_blob(m_database_write, 2, (void *)bytes, o.tellp(), NULL) != SQLITE_OK) // TODO this mught not be the right length
-		infostream<<"WARNING: Block data failed to bind: "<<sqlite3_errmsg(m_database)<<std::endl;
-	int written = sqlite3_step(m_database_write);
-	if(written != SQLITE_DONE)
-		infostream<<"WARNING: Block failed to save ("<<p3d.X<<", "<<p3d.Y<<", "<<p3d.Z<<") "
-		<<sqlite3_errmsg(m_database)<<std::endl;
-	// Make ready for later reuse
-	sqlite3_reset(m_database_write);
-	
-	// We just wrote it to the disk so clear modified flag
-	block->resetModified();
+
+	bindPos(m_stmt_delete, pos);
+
+	bool good = sqlite3_step(m_stmt_delete) == SQLITE_DONE;
+	sqlite3_reset(m_stmt_delete);
+
+	if (!good) {
+		warningstream << "deleteBlock: Block failed to delete "
+			<< PP(pos) << ": " << sqlite3_errmsg(m_database) << std::endl;
+	}
+	return good;
 }
 
-MapBlock* Database_SQLite3::loadBlock(v3s16 blockpos)
+bool Database_SQLite3::saveBlock(const v3s16 &pos, const std::string &data)
 {
-	v2s16 p2d(blockpos.X, blockpos.Z);
 	verifyDatabase();
 
-	if (sqlite3_bind_int64(m_database_read, 1, getBlockAsInteger(blockpos)) != SQLITE_OK) {
-		infostream << "WARNING: Could not bind block position for load: "
-			<< sqlite3_errmsg(m_database)<<std::endl;
+#ifdef __ANDROID__
+	/**
+	 * Note: For some unknown reason SQLite3 fails to REPLACE blocks on Android,
+	 * deleting them and then inserting works.
+	 */
+	bindPos(m_stmt_read, pos);
+
+	if (sqlite3_step(m_stmt_read) == SQLITE_ROW) {
+		deleteBlock(pos);
 	}
+	sqlite3_reset(m_stmt_read);
+#endif
+
+	bindPos(m_stmt_write, pos);
+	SQLOK(sqlite3_bind_blob(m_stmt_write, 2, data.data(), data.size(), NULL),
+		"Internal error: failed to bind query at " __FILE__ ":" TOSTRING(__LINE__));
+
+	SQLRES(sqlite3_step(m_stmt_write), SQLITE_DONE, "Failed to save block")
+	sqlite3_reset(m_stmt_write);
+
+	return true;
+}
+
+void Database_SQLite3::loadBlock(const v3s16 &pos, std::string *block)
+{
+	verifyDatabase();
+
+	bindPos(m_stmt_read, pos);
 
-	if (sqlite3_step(m_database_read) == SQLITE_ROW) {
-		/*
-			Make sure sector is loaded
-		*/
-		MapSector *sector = srvmap->createSector(p2d);
-
-		/*
-			Load block
-		*/
-		const char *data = (const char *)sqlite3_column_blob(m_database_read, 0);
-		size_t len = sqlite3_column_bytes(m_database_read, 0);
-		if (data == NULL || len == 0) {
-			errorstream << "Blank block data in database (data == NULL || len"
-				" == 0) (" << blockpos.X << "," << blockpos.Y << ","
-				<< blockpos.Z << ")" << std::endl;
-
-			if (g_settings->getBool("ignore_world_load_errors")) {
-				errorstream << "Ignoring block load error. Duck and cover! "
-					<< "(ignore_world_load_errors)" << std::endl;
-			} else {
-				throw SerializationError("Blank block data in database");
-			}
-			return NULL;
-		}
-
-		std::string datastr(data, len);
-
-		//srvmap->loadBlock(&datastr, blockpos, sector, false);
-
-		try {
-			std::istringstream is(datastr, std::ios_base::binary);
-
-			u8 version = SER_FMT_VER_INVALID;
-			is.read((char *)&version, 1);
-
-			if (is.fail())
-				throw SerializationError("ServerMap::loadBlock(): Failed"
-					 " to read MapBlock version");
-
-			MapBlock *block = NULL;
-			bool created_new = false;
-			block = sector->getBlockNoCreateNoEx(blockpos.Y);
-			if (block == NULL)
-			{
-				block = sector->createBlankBlockNoInsert(blockpos.Y);
-				created_new = true;
-			}
-
-			// Read basic data
-			block->deSerialize(is, version, true);
-
-			// If it's a new block, insert it to the map
-			if (created_new)
-				sector->insertBlock(block);
-
-			/*
-				Save blocks loaded in old format in new format
-			*/
-			//if(version < SER_FMT_VER_HIGHEST || save_after_load)
-			// Only save if asked to; no need to update version
-			//if(save_after_load)
-			//     	saveBlock(block);
-
-			// We just loaded it from, so it's up-to-date.
-			block->resetModified();
-		}
-		catch (SerializationError &e)
-		{
-			errorstream << "Invalid block data in database"
-				<< " (" << blockpos.X << "," << blockpos.Y << "," << blockpos.Z << ")"
-				<< " (SerializationError): " << e.what() << std::endl;
-
-			// TODO: Block should be marked as invalid in memory so that it is
-			// not touched but the game can run
-
-			if (g_settings->getBool("ignore_world_load_errors")) {
-				errorstream << "Ignoring block load error. Duck and cover! "
-					<< "(ignore_world_load_errors)" << std::endl;
-			} else {
-				throw SerializationError("Invalid block data in database");
-				//assert(0);
-			}
-		}
-
-		sqlite3_step(m_database_read);
-		// We should never get more than 1 row, so ok to reset
-		sqlite3_reset(m_database_read);
-
-		return srvmap->getBlockNoCreateNoEx(blockpos);  // should not be using this here
+	if (sqlite3_step(m_stmt_read) != SQLITE_ROW) {
+		sqlite3_reset(m_stmt_read);
+		return;
 	}
-	sqlite3_reset(m_database_read);
-	return NULL;
+
+	const char *data = (const char *) sqlite3_column_blob(m_stmt_read, 0);
+	size_t len = sqlite3_column_bytes(m_stmt_read, 0);
+
+	*block = (data) ? std::string(data, len) : "";
+
+	sqlite3_step(m_stmt_read);
+	// We should never get more than 1 row, so ok to reset
+	sqlite3_reset(m_stmt_read);
 }
 
 void Database_SQLite3::createDatabase()
 {
-	int e;
-	assert(m_database);
-	e = sqlite3_exec(m_database,
-		"CREATE TABLE IF NOT EXISTS `blocks` ("
-			"`pos` INT NOT NULL PRIMARY KEY,"
-			"`data` BLOB"
-		");"
-	, NULL, NULL, NULL);
-	if(e == SQLITE_ABORT)
-		throw FileNotGoodException("Could not create sqlite3 database structure");
-	else
-		infostream<<"ServerMap: SQLite3 database structure was created";
-	
+	assert(m_database); // Pre-condition
+	SQLOK(sqlite3_exec(m_database,
+		"CREATE TABLE IF NOT EXISTS `blocks` (\n"
+		"	`pos` INT PRIMARY KEY,\n"
+		"	`data` BLOB\n"
+		");\n",
+		NULL, NULL, NULL),
+		"Failed to create database table");
 }
 
-void Database_SQLite3::listAllLoadableBlocks(std::list<v3s16> &dst)
+void Database_SQLite3::listAllLoadableBlocks(std::vector<v3s16> &dst)
 {
 	verifyDatabase();
-	
-	while(sqlite3_step(m_database_list) == SQLITE_ROW)
-	{
-		sqlite3_int64 block_i = sqlite3_column_int64(m_database_list, 0);
-		v3s16 p = getIntegerAsBlock(block_i);
-		//dstream<<"block_i="<<block_i<<" p="<<PP(p)<<std::endl;
-		dst.push_back(p);
+
+	while (sqlite3_step(m_stmt_list) == SQLITE_ROW) {
+		dst.push_back(getIntegerAsBlock(sqlite3_column_int64(m_stmt_list, 0)));
 	}
+	sqlite3_reset(m_stmt_list);
 }
 
 Database_SQLite3::~Database_SQLite3()
 {
-	if(m_database_read)
-		sqlite3_finalize(m_database_read);
-	if(m_database_write)
-		sqlite3_finalize(m_database_write);
-	if(m_database)
-		sqlite3_close(m_database);
+	FINALIZE_STATEMENT(m_stmt_read)
+	FINALIZE_STATEMENT(m_stmt_write)
+	FINALIZE_STATEMENT(m_stmt_list)
+	FINALIZE_STATEMENT(m_stmt_begin)
+	FINALIZE_STATEMENT(m_stmt_end)
+	FINALIZE_STATEMENT(m_stmt_delete)
+
+	SQLOK_ERRSTREAM(sqlite3_close(m_database), "Failed to close database");
 }