Rewrite serializable to have field level granularity

Represent serializable objects in a digraph, and as a result made most
object relationships implicitly defined, and use the graph to trace
references between objects to determine relationships. Edges may
also be marked as having a dependency of the object they point to,
which allows for automatic cleanup and deletion of most objects when
no longer needed.

Additionally, this allows not having to require in-memory copies of
everything when using external databases. db_sql has been rewritten
for this and now always requires a database to function. db_sql with
MySQL now requires InnoDB to make use of transactions and foreign
key constraints.

12 files changed, 382 insertions, 249 deletions

diff --git a/modules/extra/m_ldap_authentication.cpp b/modules/extra/m_ldap_authentication.cpp
index 1b8b36fb4..dbd0344de 100644
--- a/modules/extra/m_ldap_authentication.cpp
+++ b/modules/extra/m_ldap_authentication.cpp
@@ -104,16 +104,16 @@ class IdentifyInterface : public LDAPInterface
 					if (na == NULL)
 					{
 						na = new NickServ::Nick(ii->req->GetAccount(), new NickServ::Account(ii->req->GetAccount()));
-						na->last_realname = ii->user ? ii->user->realname : ii->req->GetAccount();
+						na->SetLastRealname(ii->user ? ii->user->realname : ii->req->GetAccount());
 						NickServ::Event::OnNickRegister(&NickServ::Event::NickRegister::OnNickRegister, ii->user, na, ii->req->GetPassword());;
-						BotInfo *NickServ = Config->GetClient("NickServ");
+						ServiceBot *NickServ = Config->GetClient("NickServ");
 						if (ii->user && NickServ)
-							ii->user->SendMessage(NickServ, _("Your account \002%s\002 has been successfully created."), na->nick.c_str());
+							ii->user->SendMessage(NickServ, _("Your account \002%s\002 has been successfully created."), na->GetNick().c_str());
 					}
 					// encrypt and store the password in the nickcore
-					Anope::Encrypt(ii->req->GetPassword(), na->nc->pass);
+					Anope::Encrypt(ii->req->GetPassword(), na->GetAccount()->pass);
 
-					na->nc->Extend<Anope::string>("m_ldap_authentication_dn", ii->dn);
+					na->GetAccount()->Extend<Anope::string>("m_ldap_authentication_dn", ii->dn);
 					ii->req->Success(me);
 				}
 				break;
@@ -164,13 +164,13 @@ class OnIdentifyInterface : public LDAPInterface
 			const LDAPAttributes &attr = r.get(0);
 			Anope::string email = attr.get(email_attribute);
 
-			if (!email.equals_ci(u->Account()->email))
+			if (!email.equals_ci(u->Account()->GetEmail()))
 			{
-				u->Account()->email = email;
-				BotInfo *NickServ = Config->GetClient("NickServ");
+				u->Account()->GetEmail() = email;
+				ServiceBot *NickServ = Config->GetClient("NickServ");
 				if (NickServ)
 					u->SendMessage(NickServ, _("Your email has been updated to \002%s\002"), email.c_str());
-				Log(this->owner) << "Updated email address for " << u->nick << " (" << u->Account()->display << ") to " << email;
+				Log(this->owner) << "Updated email address for " << u->nick << " (" << u->Account()->GetDisplay() << ") to " << email;
 			}
 		}
 		catch (const LDAPException &ex)
@@ -332,18 +332,18 @@ class NSIdentifyLDAP : public Module
 			attributes[0].values.push_back(object_class);
 
 			attributes[1].name = username_attribute;
-			attributes[1].values.push_back(na->nick);
+			attributes[1].values.push_back(na->GetNick());
 
-			if (!na->nc->email.empty())
+			if (!na->GetAccount()->GetEmail().empty())
 			{
 				attributes[2].name = email_attribute;
-				attributes[2].values.push_back(na->nc->email);
+				attributes[2].values.push_back(na->GetAccount()->GetEmail());
 			}
 
 			attributes[3].name = this->password_attribute;
 			attributes[3].values.push_back(pass);
 
-			Anope::string new_dn = username_attribute + "=" + na->nick + "," + basedn;
+			Anope::string new_dn = username_attribute + "=" + na->GetNick() + "," + basedn;
 			this->ldap->Add(&this->orinterface, new_dn, attributes);
 		}
 		catch (const LDAPException &ex)
diff --git a/modules/extra/m_ldap_oper.cpp b/modules/extra/m_ldap_oper.cpp
index ca6020c6a..0445149c1 100644
--- a/modules/extra/m_ldap_oper.cpp
+++ b/modules/extra/m_ldap_oper.cpp
@@ -50,7 +50,7 @@ class IdentifyInterface : public LDAPInterface
 				o = new Oper(u->nick, ot);
 				my_opers.insert(o);
 				nc->o = o;
-				Log(this->owner) << "m_ldap_oper: Tied " << u->nick << " (" << nc->display << ") to opertype " << ot->GetName();
+				Log(this->owner) << "m_ldap_oper: Tied " << u->nick << " (" << nc->GetDisplay() << ") to opertype " << ot->GetName();
 			}
 		}
 		catch (const LDAPException &ex)
@@ -64,7 +64,7 @@ class IdentifyInterface : public LDAPInterface
 				}
 				nc->o = NULL;
 
-				Log() << "Removed services operator from " << u->nick << " (" << nc->display << ")";
+				Log() << "Removed services operator from " << u->nick << " (" << nc->GetDisplay() << ")";
 			}
 		}
 	}
@@ -121,8 +121,8 @@ class LDAPOper : public Module
 				throw LDAPException("Could not search LDAP for opertype settings, invalid configuration.");
 
 			if (!this->binddn.empty())
-				this->ldap->Bind(NULL, this->binddn.replace_all_cs("%a", u->Account()->display), this->password.c_str());
-			LDAPQuery id = this->ldap->Search(&this->iinterface, this->basedn, this->filter.replace_all_cs("%a", u->Account()->display));
+				this->ldap->Bind(NULL, this->binddn.replace_all_cs("%a", u->Account()->GetDisplay()), this->password.c_str());
+			LDAPQuery id = this->ldap->Search(&this->iinterface, this->basedn, this->filter.replace_all_cs("%a", u->Account()->GetDisplay()));
 			this->iinterface.Add(id, u->nick);
 		}
 		catch (const LDAPException &ex)
diff --git a/modules/extra/m_mysql.cpp b/modules/extra/m_mysql.cpp
index c00788e82..2a6e5f8d8 100644
--- a/modules/extra/m_mysql.cpp
+++ b/modules/extra/m_mysql.cpp
@@ -58,31 +58,35 @@ class MySQLResult : public Result
  public:
 	MySQLResult(unsigned int i, const Query &q, const Anope::string &fq, MYSQL_RES *r) : Result(i, q, fq), res(r)
 	{
-		unsigned num_fields = res ? mysql_num_fields(res) : 0;
+		if (!res)
+			return;
+
+		unsigned num_fields = mysql_num_fields(res);
+		MYSQL_FIELD *fields = mysql_fetch_fields(res);
 
 		/* It is not thread safe to log anything here using Log(this->owner) now :( */
 
-		if (!num_fields)
+		if (!num_fields || !fields)
 			return;
 
+		for (unsigned field_count = 0; field_count < num_fields; ++field_count)
+			columns.push_back(fields[field_count].name ? fields[field_count].name : "");
+
 		for (MYSQL_ROW row; (row = mysql_fetch_row(res));)
 		{
-			MYSQL_FIELD *fields = mysql_fetch_fields(res);
+			std::vector<Value> values;
 
-			if (fields)
+			for (unsigned field_count = 0; field_count < num_fields; ++field_count)
 			{
-				std::map<Anope::string, Anope::string> items;
-
-				for (unsigned field_count = 0; field_count < num_fields; ++field_count)
-				{
-					Anope::string column = (fields[field_count].name ? fields[field_count].name : "");
-					Anope::string data = (row[field_count] ? row[field_count] : "");
+				const char *data = row[field_count];
 
-					items[column] = data;
-				}
-
-				this->entries.push_back(items);
+				Value v;
+				v.null = !data;
+				v.value = data ? data : "";
+				values.push_back(v);
 			}
+
+			this->values.push_back(values);
 		}
 	}
 
@@ -101,7 +105,7 @@ class MySQLResult : public Result
  */
 class MySQLService : public Provider
 {
-	std::map<Anope::string, std::set<Anope::string> > active_schema;
+	std::map<Anope::string, std::set<Anope::string> > active_schema, indexes;
 
 	Anope::string database;
 	Anope::string server;
@@ -131,9 +135,16 @@ class MySQLService : public Provider
 
 	Result RunQuery(const Query &query) override;
 
-	std::vector<Query> CreateTable(const Anope::string &table, const Data &data) override;
+	std::vector<Query> InitSchema(const Anope::string &prefix) override;
+	std::vector<Query> Replace(const Anope::string &table, const Query &, const std::set<Anope::string> &) override;
+	std::vector<Query> CreateTable(const Anope::string &prefix, const Anope::string &table) override;
+	std::vector<Query> AlterTable(const Anope::string &, const Anope::string &table, const Anope::string &field, bool) override;
+	std::vector<Query> CreateIndex(const Anope::string &table, const Anope::string &field) override;
+
+	Query BeginTransaction() override;
+	Query Commit() override;
 
-	Query BuildInsert(const Anope::string &table, unsigned int id, Data &data) override;
+	Serialize::ID GetID(const Anope::string &) override;
 
 	Query GetTables(const Anope::string &prefix) override;
 
@@ -142,8 +153,6 @@ class MySQLService : public Provider
 	bool CheckConnection();
 
 	Anope::string BuildQuery(const Query &q);
-
-	Anope::string FromUnixtime(time_t);
 };
 
 /** The SQL thread used to execute queries
@@ -361,90 +370,146 @@ Result MySQLService::RunQuery(const Query &query)
 	}
 }
 
-std::vector<Query> MySQLService::CreateTable(const Anope::string &table, const Data &data)
+std::vector<Query> MySQLService::InitSchema(const Anope::string &prefix)
+{
+	std::vector<Query> queries;
+
+	Query t = "CREATE TABLE IF NOT EXISTS `" + prefix + "id` ("
+		"`id` bigint(20) NOT NULL"
+		") ENGINE=InnoDB";
+	queries.push_back(t);
+
+	t = "CREATE TABLE IF NOT EXISTS `" + prefix + "objects` (`id` bigint(20) NOT NULL PRIMARY KEY, `type` varchar(256)) ENGINE=InnoDB";
+	queries.push_back(t);
+
+	t = "CREATE TABLE IF NOT EXISTS `" + prefix + "edges` ("
+		"`id` bigint(20) NOT NULL,"
+		"`field` varchar(64) NOT NULL,"
+		"`other_id` bigint(20) NOT NULL,"
+		"PRIMARY KEY (`id`, `field`),"
+		"KEY `other` (`other_id`),"
+		"CONSTRAINT `edges_id_fk` FOREIGN KEY (`id`) REFERENCES `" + prefix + "objects` (`id`),"
+		"CONSTRAINT `edges_other_id_fk` FOREIGN KEY (`other_id`) REFERENCES `" + prefix + "objects` (`id`)"
+		") ENGINE=InnoDB";
+	queries.push_back(t);
+
+	return queries;
+}
+
+std::vector<Query> MySQLService::Replace(const Anope::string &table, const Query &q, const std::set<Anope::string> &keys)
+{
+	std::vector<Query> queries;
+
+	Anope::string query_text = "INSERT INTO `" + table + "` (";
+	for (const std::pair<Anope::string, QueryData> &p : q.parameters)
+		query_text += "`" + p.first + "`,";
+	query_text.erase(query_text.length() - 1);
+	query_text += ") VALUES (";
+	for (const std::pair<Anope::string, QueryData> &p : q.parameters)
+		query_text += "@" + p.first + "@,";
+	query_text.erase(query_text.length() - 1);
+	query_text += ") ON DUPLICATE KEY UPDATE ";
+	for (const std::pair<Anope::string, QueryData> &p : q.parameters)
+		if (!keys.count(p.first))
+			query_text += "`" + p.first + "` = VALUES(`" + p.first + "`),";
+	query_text.erase(query_text.length() - 1);
+
+	Query query(query_text);
+	query.parameters = q.parameters;
+
+	queries.push_back(query);
+
+	return queries;
+}
+
+std::vector<Query> MySQLService::CreateTable(const Anope::string &prefix, const Anope::string &table)
 {
 	std::vector<Query> queries;
-	std::set<Anope::string> &known_cols = this->active_schema[table];
 
-	if (known_cols.empty())
+	if (active_schema.find(prefix + table) == active_schema.end())
 	{
-		Log(LOG_DEBUG) << "m_mysql: Fetching columns for " << table;
+		Query t = "CREATE TABLE IF NOT EXISTS `" + prefix + table + "` (`id` bigint(20) NOT NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB";
+		queries.push_back(t);
 
-		Result columns = this->RunQuery("SHOW COLUMNS FROM `" + table + "`");
-		for (int i = 0; i < columns.Rows(); ++i)
-		{
-			const Anope::string &column = columns.Get(i, "Field");
+		t = "ALTER TABLE `" + prefix + table + "` "
+			"ADD CONSTRAINT `" + table + "_id_fk` FOREIGN KEY (`id`) REFERENCES `" + prefix + "objects` (`id`)";
+		queries.push_back(t);
 
-			Log(LOG_DEBUG) << "m_mysql: Column #" << i << " for " << table << ": " << column;
-			known_cols.insert(column);
-		}
+		active_schema[prefix + table];
 	}
 
-	if (known_cols.empty())
-	{
-		Anope::string query_text = "CREATE TABLE `" + table + "` (`id` int(10) unsigned NOT NULL AUTO_INCREMENT,"
-			" `timestamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP";
-		for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		{
-			known_cols.insert(it->first);
+	return queries;
+}
 
-			query_text += ", `" + it->first + "` ";
-			if (data.GetType(it->first) == Serialize::Data::DT_INT)
-				query_text += "int(11)";
-			else
-				query_text += "text";
-		}
-		query_text += ", PRIMARY KEY (`id`), KEY `timestamp_idx` (`timestamp`))";
-		queries.push_back(query_text);
+std::vector<Query> MySQLService::AlterTable(const Anope::string &prefix, const Anope::string &table, const Anope::string &field, bool object)
+{
+	std::vector<Query> queries;
+	std::set<Anope::string> &s = active_schema[prefix + table];
+
+	if (!s.count(field))
+	{
+		Query column;
+		if (!object)
+			column = "ALTER TABLE `" + prefix + table + "` ADD COLUMN `" + field + "` TINYTEXT";
+		else
+			column = "ALTER TABLE `" + prefix + table + "` "
+				"ADD COLUMN `" + field + "` bigint(20), "
+				"ADD CONSTRAINT `" + table + "_" + field + "_fk` FOREIGN KEY (`" + field + "`) REFERENCES `" + prefix + "objects` (`id`)";
+		queries.push_back(column);
+		s.insert(field);
 	}
-	else
-		for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		{
-			if (known_cols.count(it->first) > 0)
-				continue;
 
-			known_cols.insert(it->first);
+	return queries;
+}
+
+std::vector<Query> MySQLService::CreateIndex(const Anope::string &table, const Anope::string &field)
+{
+	std::vector<Query> queries;
 
-			Anope::string query_text = "ALTER TABLE `" + table + "` ADD `" + it->first + "` ";
-			if (data.GetType(it->first) == Serialize::Data::DT_INT)
-				query_text += "int(11)";
-			else
-				query_text += "text";
+	if (indexes[table].count(field))
+		return queries;
 
-			queries.push_back(query_text);
-		}
+	Query t = "ALTER TABLE `" + table + "` ADD KEY `idx_" + field + "` (`" + field + "`(512))";
+	queries.push_back(t);
+
+	indexes[table].insert(field);
 
 	return queries;
 }
 
-Query MySQLService::BuildInsert(const Anope::string &table, unsigned int id, Data &data)
+Query MySQLService::BeginTransaction()
 {
-	/* Empty columns not present in the data set */
-	const std::set<Anope::string> &known_cols = this->active_schema[table];
-	for (std::set<Anope::string>::iterator it = known_cols.begin(), it_end = known_cols.end(); it != it_end; ++it)
-		if (*it != "id" && *it != "timestamp" && data.data.count(*it) == 0)
-			data[*it] << "";
-
-	Anope::string query_text = "INSERT INTO `" + table + "` (`id`";
-	for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		query_text += ",`" + it->first + "`";
-	query_text += ") VALUES (" + stringify(id);
-	for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		query_text += ",@" + it->first + "@";
-	query_text += ") ON DUPLICATE KEY UPDATE ";
-	for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		query_text += "`" + it->first + "`=VALUES(`" + it->first + "`),";
-	query_text.erase(query_text.end() - 1);
+	return Query("START TRANSACTION WITH CONSISTENT SNAPSHOT");
+}
 
-	Query query(query_text);
-	for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
+Query MySQLService::Commit()
+{
+	return Query("COMMIT");
+}
+
+Serialize::ID MySQLService::GetID(const Anope::string &prefix)
+{
+	Query query("SELECT `id` FROM `" + prefix + "id` FOR UPDATE");
+	Serialize::ID id;
+
+	Result res = RunQuery(query);
+	if (res.Rows())
+	{
+		id = convertTo<Serialize::ID>(res.Get(0, "id"));
+
+		Query update_query("UPDATE `" + prefix + "id` SET `id` = `id` + 1");
+		RunQuery(update_query);
+	}
+	else
 	{
-		Anope::string buf;
-		*it->second >> buf;
-		query.SetValue(it->first, buf);
+		id = 0;
+
+		Query insert_query("INSERT INTO `" + prefix + "id` (id) VALUES(@id@)");
+		insert_query.SetValue("id", 1);
+		RunQuery(insert_query);
 	}
 
-	return query;
+	return id;
 }
 
 Query MySQLService::GetTables(const Anope::string &prefix)
@@ -497,14 +562,21 @@ Anope::string MySQLService::BuildQuery(const Query &q)
 	Anope::string real_query = q.query;
 
 	for (std::map<Anope::string, QueryData>::const_iterator it = q.parameters.begin(), it_end = q.parameters.end(); it != it_end; ++it)
-		real_query = real_query.replace_all_cs("@" + it->first + "@", (it->second.escape ? ("'" + this->Escape(it->second.data) + "'") : it->second.data));
+	{
+		const QueryData& qd = it->second;
+		Anope::string replacement;
 
-	return real_query;
-}
+		if (qd.null)
+			replacement = "NULL";
+		else if (!qd.escape)
+			replacement = qd.data;
+		else
+			replacement = "'" + this->Escape(qd.data) + "'";
 
-Anope::string MySQLService::FromUnixtime(time_t t)
-{
-	return "FROM_UNIXTIME(" + stringify(t) + ")";
+		real_query = real_query.replace_all_cs("@" + it->first + "@", replacement);
+	}
+
+	return real_query;
 }
 
 void DispatcherThread::Run()
diff --git a/modules/extra/m_sql_authentication.cpp b/modules/extra/m_sql_authentication.cpp
index cca8b9af1..d0210ec54 100644
--- a/modules/extra/m_sql_authentication.cpp
+++ b/modules/extra/m_sql_authentication.cpp
@@ -39,18 +39,18 @@ class SQLAuthenticationResult : public SQL::Interface
 		catch (const SQL::Exception &) { }
 
 		NickServ::Nick *na = NickServ::FindNick(req->GetAccount());
-		BotInfo *NickServ = Config->GetClient("NickServ");
+		ServiceBot *NickServ = Config->GetClient("NickServ");
 		if (na == NULL)
 		{
 			na = new NickServ::Nick(req->GetAccount(), new NickServ::Account(req->GetAccount()));
 			NickServ::Event::OnNickRegister(&NickServ::Event::NickRegister::OnNickRegister, user, na, "");
 			if (user && NickServ)
-				user->SendMessage(NickServ, _("Your account \002%s\002 has been successfully created."), na->nick.c_str());
+				user->SendMessage(NickServ, _("Your account \002%s\002 has been successfully created."), na->GetNick().c_str());
 		}
 
-		if (!email.empty() && email != na->nc->email)
+		if (!email.empty() && email != na->GetAccount()->GetEmail())
 		{
-			na->nc->email = email;
+			na->GetAccount()->GetEmail() = email;
 			if (user && NickServ)
 				user->SendMessage(NickServ, _("Your email has been updated to \002%s\002."), email.c_str());
 		}
diff --git a/modules/extra/m_sql_log.cpp b/modules/extra/m_sql_log.cpp
index 496ecb5e6..2be893cba 100644
--- a/modules/extra/m_sql_log.cpp
+++ b/modules/extra/m_sql_log.cpp
@@ -99,9 +99,9 @@ class SQLLog : public Module
 		}
 
 		insert.SetValue("user", l->u ? l->u->nick : "");
-		insert.SetValue("acc", l->nc ? l->nc->display : "");
+		insert.SetValue("acc", l->nc ? l->nc->GetDisplay() : "");
 		insert.SetValue("command", l->c ? l->c->name : "");
-		insert.SetValue("channel", l->ci ? l->ci->name : "");
+		insert.SetValue("channel", l->ci ? l->ci->GetName() : "");
 		insert.SetValue("msg", msg);
 
 		SQL->Run(NULL, insert);
diff --git a/modules/extra/m_sql_oper.cpp b/modules/extra/m_sql_oper.cpp
index 3e056c1d7..fe62111ec 100644
--- a/modules/extra/m_sql_oper.cpp
+++ b/modules/extra/m_sql_oper.cpp
@@ -46,7 +46,7 @@ class SQLOperResult : public SQL::Interface
 		}
 		catch (const SQL::Exception &) { }
 
-		BotInfo *OperServ = Config->GetClient("OperServ");
+		ServiceBot *OperServ = Config->GetClient("OperServ");
 		if (opertype.empty())
 		{
 			if (user->Account() && user->Account()->o && dynamic_cast<SQLOper *>(user->Account()->o))
@@ -54,7 +54,7 @@ class SQLOperResult : public SQL::Interface
 				delete user->Account()->o;
 				user->Account()->o = NULL;
 
-				Log(this->owner) << "m_sql_oper: Removed services operator from " << user->nick << " (" << user->Account()->display << ")";
+				Log(this->owner) << "m_sql_oper: Removed services operator from " << user->nick << " (" << user->Account()->GetDisplay() << ")";
 				user->RemoveMode(OperServ, "OPER"); // Probably not set, just incase
 			}
 			return;
@@ -70,7 +70,7 @@ class SQLOperResult : public SQL::Interface
 		if (!user->Account()->o || user->Account()->o->ot != ot)
 		{
 			Log(this->owner) << "m_sql_oper: Tieing oper " << user->nick << " to type " << opertype;
-			user->Account()->o = new SQLOper(user->Account()->display, ot);
+			user->Account()->o = new SQLOper(user->Account()->GetDisplay(), ot);
 		}
 
 		if (!user->HasMode("OPER"))
@@ -136,12 +136,12 @@ class ModuleSQLOper : public Module
 		}
 
 		SQL::Query q(this->query);
-		q.SetValue("a", u->Account()->display);
+		q.SetValue("a", u->Account()->GetDisplay());
 		q.SetValue("i", u->ip.addr());
 
 		this->SQL->Run(new SQLOperResult(this, u), q);
 
-		Log(LOG_DEBUG) << "m_sql_oper: Checking authentication for " << u->Account()->display;
+		Log(LOG_DEBUG) << "m_sql_oper: Checking authentication for " << u->Account()->GetDisplay();
 	}
 };
 
diff --git a/modules/extra/m_sqlite.cpp b/modules/extra/m_sqlite.cpp
index f07f6dbc2..19d95c248 100644
--- a/modules/extra/m_sqlite.cpp
+++ b/modules/extra/m_sqlite.cpp
@@ -14,17 +14,38 @@ using namespace SQL;
 class SQLiteResult : public Result
 {
  public:
-	SQLiteResult(unsigned int i, const Query &q, const Anope::string &fq) : Result(i, q, fq)
+	SQLiteResult(sqlite3 *sql, unsigned int id, const Query &q, const Anope::string &fq, sqlite3_stmt *stmt) : Result(id, q, fq)
 	{
-	}
+		int cols = sqlite3_column_count(stmt);
+		for (int i = 0; i < cols; ++i)
+			this->columns.push_back(sqlite3_column_name(stmt, i));
 
-	SQLiteResult(const Query &q, const Anope::string &fq, const Anope::string &err) : Result(0, q, fq, err)
-	{
+		int err;
+		while ((err = sqlite3_step(stmt)) == SQLITE_ROW)
+		{
+			std::vector<SQL::Result::Value> values;
+
+			for (int i = 0; i < cols; ++i)
+			{
+				const char *data = reinterpret_cast<const char *>(sqlite3_column_text(stmt, i));
+
+				Value v;
+				v.null = !data;
+				v.value = data ? data : "";
+				values.push_back(v);
+			}
+
+			this->values.push_back(values);
+		}
+
+		if (err != SQLITE_DONE)
+		{
+			error = sqlite3_errmsg(sql);
+		}
 	}
 
-	void AddRow(const std::map<Anope::string, Anope::string> &data)
+	SQLiteResult(const Query &q, const Anope::string &fq, const Anope::string &err) : Result(0, q, fq, err)
 	{
-		this->entries.push_back(data);
 	}
 };
 
@@ -32,7 +53,7 @@ class SQLiteResult : public Result
  */
 class SQLiteService : public Provider
 {
-	std::map<Anope::string, std::set<Anope::string> > active_schema;
+	std::map<Anope::string, std::set<Anope::string> > active_schema, indexes;
 
 	Anope::string database;
 
@@ -49,15 +70,20 @@ class SQLiteService : public Provider
 
 	Result RunQuery(const Query &query);
 
-	std::vector<Query> CreateTable(const Anope::string &table, const Data &data) override;
+	std::vector<Query> InitSchema(const Anope::string &prefix) override;
+	std::vector<Query> Replace(const Anope::string &table, const Query &, const std::set<Anope::string> &) override;
+	std::vector<Query> CreateTable(const Anope::string &, const Anope::string &table) override;
+	std::vector<Query> AlterTable(const Anope::string &, const Anope::string &table, const Anope::string &field, bool) override;
+	std::vector<Query> CreateIndex(const Anope::string &table, const Anope::string &field) override;
 
-	Query BuildInsert(const Anope::string &table, unsigned int id, Data &data);
+	Query BeginTransaction() override;
+	Query Commit() override;
+
+	Serialize::ID GetID(const Anope::string &) override;
 
 	Query GetTables(const Anope::string &prefix);
 
 	Anope::string BuildQuery(const Query &q);
-
-	Anope::string FromUnixtime(time_t);
 };
 
 class ModuleSQLite : public Module
@@ -155,136 +181,164 @@ Result SQLiteService::RunQuery(const Query &query)
 	sqlite3_stmt *stmt;
 	int err = sqlite3_prepare_v2(this->sql, real_query.c_str(), real_query.length(), &stmt, NULL);
 	if (err != SQLITE_OK)
+	{
 		return SQLiteResult(query, real_query, sqlite3_errmsg(this->sql));
+	}
 
-	std::vector<Anope::string> columns;
-	int cols = sqlite3_column_count(stmt);
-	columns.resize(cols);
-	for (int i = 0; i < cols; ++i)
-		columns[i] = sqlite3_column_name(stmt, i);
+	int id = sqlite3_last_insert_rowid(this->sql);
+	SQLiteResult result(this->sql, id, query, real_query, stmt);
 
-	SQLiteResult result(0, query, real_query);
+	sqlite3_finalize(stmt);
 
-	while ((err = sqlite3_step(stmt)) == SQLITE_ROW)
-	{
-		std::map<Anope::string, Anope::string> items;
-		for (int i = 0; i < cols; ++i)
-		{
-			const char *data = reinterpret_cast<const char *>(sqlite3_column_text(stmt, i));
-			if (data && *data)
-				items[columns[i]] = data;
-		}
-		result.AddRow(items);
-	}
+	return result;
+}
+
+std::vector<Query> SQLiteService::InitSchema(const Anope::string &prefix)
+{
+	std::vector<Query> queries;
 
-	result.id = sqlite3_last_insert_rowid(this->sql);
+	Query t = "CREATE TABLE IF NOT EXISTS `" + prefix + "id` ("
+		"`id`"
+		")";
+	queries.push_back(t);
 
-	sqlite3_finalize(stmt);
+	t = "CREATE TABLE IF NOT EXISTS `" + prefix + "objects` (`id` PRIMARY KEY, `type`)";
+	queries.push_back(t);
 
-	if (err != SQLITE_DONE)
-		return SQLiteResult(query, real_query, sqlite3_errmsg(this->sql));
+	t = "CREATE TABLE IF NOT EXISTS `" + prefix + "edges` ("
+		"`id`,"
+		"`field`,"
+		"`other_id`,"
+		"PRIMARY KEY (`id`, `field`)"
+		")";
+	queries.push_back(t);
 
-	return result;
+	t = "CREATE INDEX IF NOT EXISTS idx_edge ON `" + prefix + "edges` (other_id)";
+	queries.push_back(t);
+
+	return queries;
 }
 
-std::vector<Query> SQLiteService::CreateTable(const Anope::string &table, const Data &data)
+std::vector<Query> SQLiteService::Replace(const Anope::string &table, const Query &q, const std::set<Anope::string> &keys)
 {
 	std::vector<Query> queries;
-	std::set<Anope::string> &known_cols = this->active_schema[table];
 
-	if (known_cols.empty())
-	{
-		Log(LOG_DEBUG) << "m_sqlite: Fetching columns for " << table;
+	Anope::string query_text = "INSERT OR IGNORE INTO `" + table + "` (";
+	for (const std::pair<Anope::string, QueryData> &p : q.parameters)
+		query_text += "`" + p.first + "`,";
+	query_text.erase(query_text.length() - 1);
+	query_text += ") VALUES (";
+	for (const std::pair<Anope::string, QueryData> &p : q.parameters)
+		query_text += "@" + p.first + "@,";
+	query_text.erase(query_text.length() - 1);
+	query_text += ")";
 
-		Result columns = this->RunQuery("PRAGMA table_info(" + table + ")");
-		for (int i = 0; i < columns.Rows(); ++i)
-		{
-			const Anope::string &column = columns.Get(i, "name");
+	Query query(query_text);
+	query.parameters = q.parameters;
+	queries.push_back(query);
 
-			Log(LOG_DEBUG) << "m_sqlite: Column #" << i << " for " << table << ": " << column;
-			known_cols.insert(column);
-		}
+	query_text = "UPDATE `" + table + "` SET ";
+	for (const std::pair<Anope::string, QueryData> &p : q.parameters)
+		if (!keys.count(p.first))
+			query_text += "`" + p.first + "` = @" + p.first + "@,";
+	query_text.erase(query_text.length() - 1);
+	unsigned int i = 0;
+	for (const Anope::string &key : keys)
+	{
+		if (!i++)
+			query_text += " WHERE ";
+		else
+			query_text += " AND ";
+		query_text += "`" + key + "` = @" + key + "@";
 	}
 
-	if (known_cols.empty())
-	{
-		Anope::string query_text = "CREATE TABLE `" + table + "` (`id` INTEGER PRIMARY KEY, `timestamp` timestamp DEFAULT CURRENT_TIMESTAMP";
+	query = query_text;
+	query.parameters = q.parameters;
+	queries.push_back(query);
 
-		for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		{
-			known_cols.insert(it->first);
+	return queries;
+}
 
-			query_text += ", `" + it->first + "` ";
-			if (data.GetType(it->first) == Serialize::Data::DT_INT)
-				query_text += "int(11)";
-			else
-				query_text += "text";
-		}
+std::vector<Query> SQLiteService::CreateTable(const Anope::string &prefix, const Anope::string &table)
+{
+	std::vector<Query> queries;
 
-		query_text += ")";
+	if (active_schema.find(prefix + table) == active_schema.end())
+	{
+		Query t = "CREATE TABLE IF NOT EXISTS `" + prefix + table + "` (`id` bigint(20) NOT NULL, PRIMARY KEY (`id`))";
+		queries.push_back(t);
 
-		queries.push_back(query_text);
+		active_schema[prefix + table];
+	}
 
-		query_text = "CREATE UNIQUE INDEX `" + table + "_id_idx` ON `" + table + "` (`id`)";
-		queries.push_back(query_text);
+	return queries;
+}
 
-		query_text = "CREATE INDEX `" + table + "_timestamp_idx` ON `" + table + "` (`timestamp`)";
-		queries.push_back(query_text);
+std::vector<Query> SQLiteService::AlterTable(const Anope::string &prefix, const Anope::string &table, const Anope::string &field, bool)
+{
+	std::vector<Query> queries;
+	std::set<Anope::string> &s = active_schema[prefix + table];
 
-		query_text = "CREATE TRIGGER `" + table + "_trigger` AFTER UPDATE ON `" + table + "` FOR EACH ROW BEGIN UPDATE `" + table + "` SET `timestamp` = CURRENT_TIMESTAMP WHERE `id` = `old.id`; end;";
-		queries.push_back(query_text);
+	if (!s.count(field))
+	{
+		Query t =  "ALTER TABLE `" + prefix + table + "` ADD `" + field + "` COLLATE NOCASE";
+		queries.push_back(t);
+		s.insert(field);
 	}
-	else
-		for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		{
-			if (known_cols.count(it->first) > 0)
-				continue;
 
-			known_cols.insert(it->first);
+	return queries;
+}
 
-			Anope::string query_text = "ALTER TABLE `" + table + "` ADD `" + it->first + "` ";
-			if (data.GetType(it->first) == Serialize::Data::DT_INT)
-				query_text += "int(11)";
-			else
-				query_text += "text";
+std::vector<Query> SQLiteService::CreateIndex(const Anope::string &table, const Anope::string &field)
+{
+	std::vector<Query> queries;
 
-			queries.push_back(query_text);
-		}
+	if (indexes[table].count(field))
+		return queries;
+
+	Query t = "CREATE INDEX IF NOT EXISTS idx_" + field + " ON `" + table + "` (" + field + ")";
+	queries.push_back(t);
+
+	indexes[table].insert(field);
 
 	return queries;
 }
 
-Query SQLiteService::BuildInsert(const Anope::string &table, unsigned int id, Data &data)
+Query SQLiteService::BeginTransaction()
 {
-	/* Empty columns not present in the data set */
-	const std::set<Anope::string> &known_cols = this->active_schema[table];
-	for (std::set<Anope::string>::iterator it = known_cols.begin(), it_end = known_cols.end(); it != it_end; ++it)
-		if (*it != "id" && *it != "timestamp" && data.data.count(*it) == 0)
-			data[*it] << "";
-
-	Anope::string query_text = "REPLACE INTO `" + table + "` (";
-	if (id > 0)
-		query_text += "`id`,";
-	for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		query_text += "`" + it->first + "`,";
-	query_text.erase(query_text.length() - 1);
-	query_text += ") VALUES (";
-	if (id > 0)
-		query_text += stringify(id) + ",";
-	for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
-		query_text += "@" + it->first + "@,";
-	query_text.erase(query_text.length() - 1);
-	query_text += ")";
+	return Query("BEGIN TRANSACTION");
+}
 
-	Query query(query_text);
-	for (Data::Map::const_iterator it = data.data.begin(), it_end = data.data.end(); it != it_end; ++it)
+Query SQLiteService::Commit()
+{
+	return Query("COMMIT");
+}
+
+Serialize::ID SQLiteService::GetID(const Anope::string &prefix)
+{
+	/* must be in a deferred or reserved transaction here for atomic row update */
+
+	Query query("SELECT `id` FROM `" + prefix + "id`");
+	Serialize::ID id;
+
+	Result res = RunQuery(query);
+	if (res.Rows())
 	{
-		Anope::string buf;
-		*it->second >> buf;
-		query.SetValue(it->first, buf);
+		id = convertTo<Serialize::ID>(res.Get(0, "id"));
+
+		Query update_query("UPDATE `" + prefix + "id` SET `id` = `id` + 1");
+		RunQuery(update_query);
+	}
+	else
+	{
+		id = 0;
+
+		Query insert_query("INSERT INTO `" + prefix + "id` (id) VALUES(@id@)");
+		insert_query.SetValue("id", 1);
+		RunQuery(insert_query);
 	}
 
-	return query;
+	return id;
 }
 
 Query SQLiteService::GetTables(const Anope::string &prefix)
@@ -305,14 +359,21 @@ Anope::string SQLiteService::BuildQuery(const Query &q)
 	Anope::string real_query = q.query;
 
 	for (std::map<Anope::string, QueryData>::const_iterator it = q.parameters.begin(), it_end = q.parameters.end(); it != it_end; ++it)
-		real_query = real_query.replace_all_cs("@" + it->first + "@", (it->second.escape ? ("'" + this->Escape(it->second.data) + "'") : it->second.data));
+	{
+		const QueryData& qd = it->second;
+		Anope::string replacement;
 
-	return real_query;
-}
+		if (qd.null)
+			replacement = "NULL";
+		else if (!qd.escape)
+			replacement = qd.data;
+		else
+			replacement = "'" + this->Escape(qd.data) + "'";
 
-Anope::string SQLiteService::FromUnixtime(time_t t)
-{
-	return "datetime('" + stringify(t) + "', 'unixepoch')";
+		real_query = real_query.replace_all_cs("@" + it->first + "@", replacement);
+	}
+
+	return real_query;
 }
 
 MODULE_INIT(ModuleSQLite)
diff --git a/modules/extra/stats/cs_fantasy_stats.cpp b/modules/extra/stats/cs_fantasy_stats.cpp
index 4674e0892..a5ce54d33 100644
--- a/modules/extra/stats/cs_fantasy_stats.cpp
+++ b/modules/extra/stats/cs_fantasy_stats.cpp
@@ -114,7 +114,7 @@ class CSStats : public Module
 				else
 				{
 					if (NickServ::Nick *na = NickServ::FindNick(params[0]))
-						display = na->nc->display;
+						display = na->GetAccount()->GetDisplay();
 					else
 					{
 						source.Reply(_("%s not found."), params[0].c_str());
@@ -125,7 +125,7 @@ class CSStats : public Module
 		}
 
 		if (display.empty())
-			display = source.nc->display;
+			display = source.nc->GetDisplay();
 
 		try
 		{
diff --git a/modules/extra/stats/cs_fantasy_top.cpp b/modules/extra/stats/cs_fantasy_top.cpp
index 540a4f3d9..94197a0e5 100644
--- a/modules/extra/stats/cs_fantasy_top.cpp
+++ b/modules/extra/stats/cs_fantasy_top.cpp
@@ -122,7 +122,7 @@ class CSTop : public Module
 		if (!params.empty())
 			channel = params[0];
 		else if (source.c && source.c->ci)
-			channel = source.c->ci->name;
+			channel = source.c->ci->GetName();
 
 		if (!is_global && channel.empty())
 			is_global = true;
diff --git a/modules/extra/stats/irc2sql/irc2sql.cpp b/modules/extra/stats/irc2sql/irc2sql.cpp
index 8c0aa83ab..bc302c575 100644
--- a/modules/extra/stats/irc2sql/irc2sql.cpp
+++ b/modules/extra/stats/irc2sql/irc2sql.cpp
@@ -25,7 +25,7 @@ void IRC2SQL::OnReload(Configuration::Conf *conf)
 	const Anope::string &snick = block->Get<const Anope::string>("client");
 	if (snick.empty())
 		throw ConfigException(Module::name + ": <client> must be defined");
-	StatServ = BotInfo::Find(snick, true);
+	StatServ = ServiceBot::Find(snick, true);
 	if (!StatServ)
 		throw ConfigException(Module::name + ": no bot named " + snick);
 
@@ -99,7 +99,7 @@ void IRC2SQL::OnUserConnect(User *u, bool &exempt)
 	query.SetValue("ident", u->GetIdent());
 	query.SetValue("vident", u->GetVIdent());
 	query.SetValue("secure", u->HasMode("SSL") || u->HasExt("ssl") ? "Y" : "N");
-	query.SetValue("account", u->Account() ? u->Account()->display : "");
+	query.SetValue("account", u->Account() ? u->Account()->GetDisplay() : "");
 	query.SetValue("fingerprint", u->fingerprint);
 	query.SetValue("signon", u->signon);
 	query.SetValue("server", u->server->GetName());
@@ -158,7 +158,7 @@ void IRC2SQL::OnUserLogin(User *u)
 {
 	query = "UPDATE `" + prefix + "user` SET account=@account@ WHERE nick=@nick@";
 	query.SetValue("nick", u->nick);
-	query.SetValue("account", u->Account() ? u->Account()->display : "");
+	query.SetValue("account", u->Account() ? u->Account()->GetDisplay() : "");
 	this->RunQuery(query);
 }
 
@@ -255,7 +255,7 @@ void IRC2SQL::OnTopicUpdated(Channel *c, const Anope::string &user, const Anope:
 	this->RunQuery(query);
 }
 
-void IRC2SQL::OnBotNotice(User *u, BotInfo *bi, Anope::string &message)
+void IRC2SQL::OnBotNotice(User *u, ServiceBot *bi, Anope::string &message)
 {
 	Anope::string versionstr;
 	if (bi != StatServ)
diff --git a/modules/extra/stats/irc2sql/irc2sql.h b/modules/extra/stats/irc2sql/irc2sql.h
index 1ce118f38..175e37681 100644
--- a/modules/extra/stats/irc2sql/irc2sql.h
+++ b/modules/extra/stats/irc2sql/irc2sql.h
@@ -27,7 +27,7 @@ class IRC2SQL : public Module
 	std::vector<Anope::string> TableList, ProcedureList, EventList;
 	Anope::string prefix, GeoIPDB;
 	bool quitting, introduced_myself, ctcpuser, ctcpeob, firstrun;
-	BotInfo *StatServ;
+	ServiceBot *StatServ;
 	PrimitiveExtensibleItem<bool> versionreply;
 
 	void RunQuery(const SQL::Query &q);
@@ -71,7 +71,7 @@ class IRC2SQL : public Module
 
 	void OnTopicUpdated(Channel *c, const Anope::string &user, const Anope::string &topic) override;
 
-	void OnBotNotice(User *u, BotInfo *bi, Anope::string &message) override;
+	void OnBotNotice(User *u, ServiceBot *bi, Anope::string &message) override;
 };
 
 
diff --git a/modules/extra/stats/m_chanstats.cpp b/modules/extra/stats/m_chanstats.cpp
index 6238b749c..1d299ce7a 100644
--- a/modules/extra/stats/m_chanstats.cpp
+++ b/modules/extra/stats/m_chanstats.cpp
@@ -73,25 +73,25 @@ class CommandNSSetChanstats : public Command
 		}
 
 		EventReturn MOD_RESULT;
-		MOD_RESULT = Event::OnSetNickOption(&Event::SetNickOption::OnSetNickOption, source, this, na->nc, param);
+		MOD_RESULT = Event::OnSetNickOption(&Event::SetNickOption::OnSetNickOption, source, this, na->GetAccount(), param);
 		if (MOD_RESULT == EVENT_STOP)
 			return;
 
 		if (param.equals_ci("ON"))
 		{
-			Log(na->nc == source.GetAccount() ? LOG_COMMAND : LOG_ADMIN, source, this) << "to enable chanstats for " << na->nc->display;
-			na->nc->Extend<bool>("NS_STATS");
+			Log(na->GetAccount() == source.GetAccount() ? LOG_COMMAND : LOG_ADMIN, source, this) << "to enable chanstats for " << na->GetAccount()->GetDisplay();
+			na->GetAccount()->Extend<bool>("NS_STATS");
 			if (saset)
-				source.Reply(_("Chanstats statistics are now enabled for %s"), na->nc->display.c_str());
+				source.Reply(_("Chanstats statistics are now enabled for %s"), na->GetAccount()->GetDisplay().c_str());
 			else
 				source.Reply(_("Chanstats statistics are now enabled for your nick."));
 		}
 		else if (param.equals_ci("OFF"))
 		{
-			Log(na->nc == source.GetAccount() ? LOG_COMMAND : LOG_ADMIN, source, this) << "to disable chanstats for " << na->nc->display;
-			na->nc->Shrink<bool>("NS_STATS");
+			Log(na->GetAccount() == source.GetAccount() ? LOG_COMMAND : LOG_ADMIN, source, this) << "to disable chanstats for " << na->GetAccount()->GetDisplay();
+			na->GetAccount()->Shrink<bool>("NS_STATS");
 			if (saset)
-				source.Reply(_("Chanstats statistics are now disabled for %s"), na->nc->display.c_str());
+				source.Reply(_("Chanstats statistics are now disabled for %s"), na->GetAccount()->GetDisplay().c_str());
 			else
 				source.Reply(_("Chanstats statistics are now disabled for your nick."));
 		}
@@ -101,7 +101,7 @@ class CommandNSSetChanstats : public Command
 
 	void Execute(CommandSource &source, const std::vector<Anope::string> &params) override
 	{
-		this->Run(source, source.nc->display, params[0]);
+		this->Run(source, source.nc->GetDisplay(), params[0]);
 	}
 
 	bool OnHelp(CommandSource &source, const Anope::string &) override
@@ -156,7 +156,7 @@ class MySQLInterface : public SQL::Interface
 
 class MChanstats : public Module
 {
-	SerializableExtensibleItem<bool> cs_stats, ns_stats;
+	Serialize::Field<bool> cs_stats, ns_stats;
 
 	CommandCSSetChanstats commandcssetchanstats;
 
@@ -200,7 +200,7 @@ class MChanstats : public Module
 	const Anope::string GetDisplay(User *u)
 	{
 		if (u && u->Account() && ns_stats.HasExt(u->Account()))
-			return u->Account()->display;
+			return u->Account()->GetDisplay();
 		else
 			return "";
 	}
@@ -513,7 +513,7 @@ class MChanstats : public Module
 	{
 		if (!show_hidden)
 			return;
-		if (ns_stats.HasExt(na->nc))
+		if (ns_stats.HasExt(na->GetAccount()))
 			info.AddOption(_("Chanstats"));
 	}
 
@@ -613,14 +613,14 @@ class MChanstats : public Module
 	void OnDelCore(NickServ::Account *nc) override
 	{
 		query = "DELETE FROM `" + prefix + "chanstats` WHERE `nick` = @nick@;";
-		query.SetValue("nick", nc->display);
+		query.SetValue("nick", nc->GetDisplay());
 		this->RunQuery(query);
 	}
 
 	void OnChangeCoreDisplay(NickServ::Account *nc, const Anope::string &newdisplay) override
 	{
 		query = "CALL " + prefix + "chanstats_proc_chgdisplay(@old_display@, @new_display@);";
-		query.SetValue("old_display", nc->display);
+		query.SetValue("old_display", nc->GetDisplay());
 		query.SetValue("new_display", newdisplay);
 		this->RunQuery(query);
 	}
@@ -628,7 +628,7 @@ class MChanstats : public Module
 	void OnDelChan(ChanServ::Channel *ci) override
 	{
 		query = "DELETE FROM `" + prefix + "chanstats` WHERE `chan` = @channel@;";
-		query.SetValue("channel", ci->name);
+		query.SetValue("channel", ci->GetName());
 		this->RunQuery(query);
 	}
 
@@ -641,7 +641,7 @@ class MChanstats : public Module
 	void OnNickRegister(User *user, NickAlias *na, const Anope::string &)
 	{
 		if (NSDefChanstats)
-			na->nc->Extend<bool>("NS_STATS");
+			na->GetAccount()->Extend<bool>("NS_STATS");
 	}
 };