Rewrite serializable to have field level granularity

Represent serializable objects in a digraph, and as a result made most
object relationships implicitly defined, and use the graph to trace
references between objects to determine relationships. Edges may
also be marked as having a dependency of the object they point to,
which allows for automatic cleanup and deletion of most objects when
no longer needed.

Additionally, this allows not having to require in-memory copies of
everything when using external databases. db_sql has been rewritten
for this and now always requires a database to function. db_sql with
MySQL now requires InnoDB to make use of transactions and foreign
key constraints.

1 files changed, 291 insertions, 197 deletions

diff --git a/modules/database/db_sql.cpp b/modules/database/db_sql.cpp
index 5d213cb9a..3481d3a89 100644
--- a/modules/database/db_sql.cpp
+++ b/modules/database/db_sql.cpp
@@ -1,279 +1,373 @@
-/*
- * (C) 2003-2014 Anope Team
- * Contact us at team@anope.org
- *
- * Please read COPYING and README for further details.
- *
- * Based on the original code of Epona by Lara.
- * Based on the original code of Services by Andy Church.
- */
-
 #include "module.h"
 #include "modules/sql.h"
 
 using namespace SQL;
 
-class SQLSQLInterface : public Interface
+class DBMySQL : public Module, public Pipe
+	, public EventHook<Event::SerializeEvents>
 {
- public:
-	SQLSQLInterface(Module *o) : Interface(o) { }
+ private:
+	bool transaction = false;
+	bool inited = false;
+	Anope::string prefix;
+	ServiceReference<Provider> SQL;
 
-	void OnResult(const Result &r) override
+	Result Run(const Query &query)
 	{
-		Log(LOG_DEBUG) << "SQL successfully executed query: " << r.finished_query;
+		if (!SQL)
+			;//XXX
+
+		if (!inited)
+		{
+			inited = true;
+			for (const Query &q : SQL->InitSchema(prefix))
+				SQL->RunQuery(q);
+		}
+
+		return SQL->RunQuery(query);
 	}
 
-	void OnError(const Result &r) override
+	void StartTransaction()
 	{
-		if (!r.GetQuery().query.empty())
-			Log(LOG_DEBUG) << "Error executing query " << r.finished_query << ": " << r.GetError();
-		else
-			Log(LOG_DEBUG) << "Error executing query: " << r.GetError();
-	}
-};
+		if (!SQL || transaction)
+			return;
 
-class ResultSQLSQLInterface : public SQLSQLInterface
-{
-	Reference<Serializable> obj;
+		Run(SQL->BeginTransaction());
 
-public:
-	ResultSQLSQLInterface(Module *o, Serializable *ob) : SQLSQLInterface(o), obj(ob) { }
+		transaction = true;
+		Notify();
+	}
 
-	void OnResult(const Result &r) override
+	void Commit()
 	{
-		SQLSQLInterface::OnResult(r);
-		if (r.GetID() > 0 && this->obj)
-			this->obj->id = r.GetID();
-		delete this;
+		if (!SQL || !transaction)
+			return;
+
+		Run(SQL->Commit());
+
+		transaction = false;
 	}
 
-	void OnError(const Result &r) override
+ public:
+	DBMySQL(const Anope::string &modname, const Anope::string &creator) : Module(modname, creator, DATABASE | VENDOR)
+		, EventHook<Event::SerializeEvents>("OnSerialize")
 	{
-		SQLSQLInterface::OnError(r);
-		delete this;
 	}
-};
 
-class DBSQL : public Module, public Pipe
-	, public EventHook<Event::Shutdown>
-	, public EventHook<Event::Restart>
-	, public EventHook<Event::LoadDatabase>
-	, public EventHook<Event::SerializableConstruct>
-	, public EventHook<Event::SerializableDestruct>
-	, public EventHook<Event::SerializableUpdate>
-	, public EventHook<Event::SerializeTypeCreate>
-{
-	ServiceReference<Provider> sql;
-	SQLSQLInterface sqlinterface;
-	Anope::string prefix;
-	bool import;
+	void OnNotify() override
+	{
+		Commit();
+		Serialize::Clear();
+	}
 
-	std::set<Serializable *> updated_items;
-	bool shutting_down;
-	bool loading_databases;
-	bool loaded;
-	bool imported;
+	void OnReload(Configuration::Conf *conf) override
+	{
+		Configuration::Block *block = conf->GetModule(this);
+		this->SQL = ServiceReference<Provider>("SQL::Provider", block->Get<const Anope::string>("engine"));
+		this->prefix = block->Get<const Anope::string>("prefix", "anope_db_");
+		inited = false;
+	}
 
-	void RunBackground(const Query &q, Interface *iface = NULL)
+	EventReturn OnSerializeList(Serialize::TypeBase *type, std::vector<Serialize::ID> &ids) override
 	{
-		if (!this->sql)
+		StartTransaction();
+
+		ids.clear();
+
+		Query query = "SELECT `id` FROM `" + prefix + type->GetName() + "`";
+		Result res = Run(query);
+		for (int i = 0; i < res.Rows(); ++i)
 		{
-			static time_t last_warn = 0;
-			if (last_warn + 300 < Anope::CurTime)
+			Serialize::ID id = convertTo<Serialize::ID>(res.Get(i, "id"));
+			ids.push_back(id);
+		}
+
+		return EVENT_ALLOW;
+	}
+
+	EventReturn OnSerializeFind(Serialize::TypeBase *type, Serialize::FieldBase *field, const Anope::string &value, Serialize::ID &id) override
+	{
+		if (!SQL)
+			return EVENT_CONTINUE;
+
+		StartTransaction();
+
+		for (Query &q : SQL->CreateTable(prefix, type->GetName()))
+			Run(q);
+
+		for (Query &q : SQL->AlterTable(prefix, type->GetName(), field->GetName(), false))
+			Run(q);
+
+		for (const Query &q : SQL->CreateIndex(prefix + type->GetName(), field->GetName()))
+			Run(q);
+
+		Query query("SELECT `id` FROM `" + prefix + type->GetName() + "` WHERE `" + field->GetName() + "` = @value@");
+		query.SetValue("value", value);
+		Result res = Run(query);
+		if (res.Rows())
+			try
 			{
-				last_warn = Anope::CurTime;
-				Log(this) << "db_sql: Unable to execute query, is SQL configured correctly?";
+				id = convertTo<Serialize::ID>(res.Get(0, "id"));
+				return EVENT_ALLOW;
 			}
-		}
-		else if (!Anope::Quitting)
-		{
-			if (iface == NULL)
-				iface = &this->sqlinterface;
-			this->sql->Run(iface, q);
-		}
-		else
-			this->sql->RunQuery(q);
+			catch (const ConvertException &)
+			{
+			}
+		return EVENT_CONTINUE;
+	}
+
+ private:
+	bool GetValue(Serialize::Object *object, Serialize::FieldBase *field, SQL::Result::Value &v)
+	{
+		StartTransaction();
+
+		Query query = "SELECT `" + field->GetName() + "` FROM `" + prefix + object->GetSerializableType()->GetName() + "` WHERE `id` = @id@";
+		query.SetValue("id", object->id);
+		Result res = Run(query);
+
+		if (res.Rows() == 0)
+			return false;
+
+		v = res.GetValue(0, field->GetName());
+		return true;
 	}
 
  public:
-	DBSQL(const Anope::string &modname, const Anope::string &creator) : Module(modname, creator, DATABASE | VENDOR)
-		, EventHook<Event::Shutdown>("OnShutdown")
-		, EventHook<Event::Restart>("OnRestart")
-		, EventHook<Event::LoadDatabase>("OnLoadDatabase")
-		, EventHook<Event::SerializableConstruct>("OnSerializableConstruct")
-		, EventHook<Event::SerializableDestruct>("OnSerializableDestruct")
-		, EventHook<Event::SerializableUpdate>("OnSerializableUpdate")
-		, EventHook<Event::SerializeTypeCreate>("OnSerializeTypeCreate")
-		, sqlinterface(this)
-		, shutting_down(false)
-		, loading_databases(false)
-		, loaded(false)
-		, imported(false)
+	EventReturn OnSerializeGet(Serialize::Object *object, Serialize::FieldBase *field, Anope::string &value) override
 	{
+		SQL::Result::Value v;
 
+		if (!GetValue(object, field, v))
+		{
+			field->CacheMiss(object);
+			return EVENT_CONTINUE;
+		}
 
-		if (ModuleManager::FindModule("db_sql_live") != NULL)
-			throw ModuleException("db_sql can not be loaded after db_sql_live");
+		value = v.value;
+		return EVENT_ALLOW;
 	}
 
-	void OnNotify() override
+	EventReturn OnSerializeGetRefs(Serialize::Object *object, Serialize::TypeBase *type, std::vector<Serialize::Edge> &edges) override
 	{
-		for (std::set<Serializable *>::iterator it = this->updated_items.begin(), it_end = this->updated_items.end(); it != it_end; ++it)
+		StartTransaction();
+
+		edges.clear();
+
+		Query query;
+		if (type)
+			query = "SELECT field," + prefix + "edges.id,other_id,j1.type,j2.type AS other_type FROM `" + prefix + "edges` "
+				"JOIN `" + prefix + "objects` AS j1 ON " + prefix + "edges.id = j1.id "
+				"JOIN `" + prefix + "objects` AS j2 ON " + prefix + "edges.other_id = j2.id "
+				"WHERE "
+				"	(" + prefix + "edges.id = @id@ AND j2.type = @other_type@) "
+				"OR"
+				"	(other_id = @id@ AND j1.type = @other_type@)";
+		else
+			query = "SELECT field," + prefix + "edges.id,other_id,j1.type,j2.type AS other_type FROM `" + prefix + "edges` "
+				"JOIN `" + prefix + "objects` AS j1 ON " + prefix + "edges.id = j1.id "
+				"JOIN `" + prefix + "objects` AS j2 ON " + prefix + "edges.other_id = j2.id "
+				"WHERE " + prefix + "edges.id = @id@ OR other_id = @id@";
+
+		query.SetValue("type", object->GetSerializableType()->GetName());
+		query.SetValue("id", object->id);
+		if (type)
+			query.SetValue("other_type", type->GetName());
+
+		Result res = Run(query);
+		for (int i = 0; i < res.Rows(); ++i)
 		{
-			Serializable *obj = *it;
+			Serialize::ID id = convertTo<Serialize::ID>(res.Get(i, "id"));
+
+			if (id == object->id)
+			{
+				// we want other type, this is my edge
+				Anope::string t = res.Get(i, "other_type");
+				Anope::string f = res.Get(i, "field");
+				id = convertTo<Serialize::ID>(res.Get(i, "other_id"));
+
+				//XXX sanity checks
+				Serialize::FieldBase *obj_field = object->GetSerializableType()->GetField(f);
 
-			if (this->sql)
+				Serialize::TypeBase *obj_type = Serialize::TypeBase::Find(t);
+				Serialize::Object *other = obj_type->Require(id);
+
+				edges.emplace_back(other, obj_field, true);
+			}
+			else
 			{
-				Data data;
-				obj->Serialize(data);
-
-				if (obj->IsCached(data))
-					continue;
-
-				obj->UpdateCache(data);
-
-				/* If we didn't load these objects and we don't want to import just update the cache and continue */
-				if (!this->loaded && !this->imported && !this->import)
-					continue;
-
-				Serialize::Type *s_type = obj->GetSerializableType();
-				if (!s_type)
-					continue;
-
-				std::vector<Query> create = this->sql->CreateTable(this->prefix + s_type->GetName(), data);
-				for (unsigned i = 0; i < create.size(); ++i)
-					this->RunBackground(create[i]);
-
-				Query insert = this->sql->BuildInsert(this->prefix + s_type->GetName(), obj->id, data);
-				if (this->imported)
-					this->RunBackground(insert, new ResultSQLSQLInterface(this, obj));
-				else
-				{
-					/* We are importing objects from another database module, so don't do asynchronous
-					 * queries in case the core has to shut down, it will cut short the import
-					 */
-					Result r = this->sql->RunQuery(insert);
-					if (r.GetID() > 0)
-						obj->id = r.GetID();
-				}
+				// edge to me
+				Anope::string t = res.Get(i, "type");
+				Anope::string f = res.Get(i, "field");
+
+				//XXX sanity checks
+				Serialize::TypeBase *obj_type = Serialize::TypeBase::Find(t);
+				Serialize::FieldBase *obj_field = obj_type->GetField(f);
+				Serialize::Object *other = obj_type->Require(id);
+
+				// other type, other field,
+				edges.emplace_back(other, obj_field, false);
 			}
 		}
+		
+		return EVENT_ALLOW;
+	}
+
+	EventReturn OnSerializeDeref(Serialize::ID id, Serialize::TypeBase *type) override
+	{
+		StartTransaction();
 
-		this->updated_items.clear();
-		this->imported = true;
+		Query query = "SELECT `id` FROM `" + prefix + type->GetName() + "` WHERE `id` = @id@";
+		query.SetValue("id", id);
+		Result res = Run(query);
+		if (res.Rows() == 0)
+			return EVENT_CONTINUE;
+		return EVENT_ALLOW;
 	}
 
-	void OnReload(Configuration::Conf *conf) override
+	EventReturn OnSerializeGetSerializable(Serialize::Object *object, Serialize::FieldBase *field, Anope::string &type, Serialize::ID &value) override
 	{
-		Configuration::Block *block = conf->GetModule(this);
-		this->sql = ServiceReference<Provider>("SQL::Provider", block->Get<const Anope::string>("engine"));
-		this->prefix = block->Get<const Anope::string>("prefix", "anope_db_");
-		this->import = block->Get<bool>("import");
+		StartTransaction();
+
+		Query query = "SELECT `" + field->GetName() + "`,j1.type AS " + field->GetName() + "_type FROM `" + prefix + object->GetSerializableType()->GetName() + "` "
+			"JOIN `" + prefix + "objects` AS j1 ON " + prefix + object->GetSerializableType()->GetName() + "." + field->GetName() + " = j1.id "
+			"WHERE " + prefix + object->GetSerializableType()->GetName() + ".id = @id@";
+		query.SetValue("id", object->id);
+		Result res = Run(query);
+
+		if (res.Rows() == 0)
+			return EVENT_CONTINUE;
+
+		type = res.Get(0, field->GetName() + "_type");
+		try
+		{
+			value = convertTo<Serialize::ID>(res.Get(0, field->GetName()));
+		}
+		catch (const ConvertException &ex)
+		{
+			return EVENT_STOP;
+		}
+
+		return EVENT_ALLOW;
 	}
 
-	void OnShutdown() override
+ private:
+	void DoSet(Serialize::Object *object, Serialize::FieldBase *field, bool is_object, const Anope::string *value)
 	{
-		this->shutting_down = true;
-		this->OnNotify();
+		if (!SQL)
+			return;
+
+		StartTransaction();
+
+		for (Query &q : SQL->CreateTable(prefix, object->GetSerializableType()->GetName()))
+			Run(q);
+
+		for (Query &q : SQL->AlterTable(prefix, object->GetSerializableType()->GetName(), field->GetName(), is_object))
+			Run(q);
+
+		Query q;
+		q.SetValue("id", object->id);
+		if (value)
+			q.SetValue(field->GetName(), *value);
+		else
+			q.SetNull(field->GetName());
+
+		for (Query &q : SQL->Replace(prefix + object->GetSerializableType()->GetName(), q, { "id" }))
+			Run(q);
 	}
 
-	void OnRestart() override
+ public:
+	EventReturn OnSerializeSet(Serialize::Object *object, Serialize::FieldBase *field, const Anope::string &value) override
 	{
-		this->OnShutdown();
+		DoSet(object, field, false, &value);
+		return EVENT_STOP;
 	}
 
-	EventReturn OnLoadDatabase() override
+	EventReturn OnSerializeSetSerializable(Serialize::Object *object, Serialize::FieldBase *field, Serialize::Object *value) override
 	{
-		if (!this->sql)
-		{
-			Log(this) << "Unable to load databases, is SQL configured correctly?";
+		if (!SQL)
 			return EVENT_CONTINUE;
-		}
 
-		this->loading_databases = true;
+		StartTransaction();
 
-		const std::vector<Anope::string> type_order = Serialize::Type::GetTypeOrder();
-		for (unsigned i = 0; i < type_order.size(); ++i)
+		if (value)
 		{
-			Serialize::Type *sb = Serialize::Type::Find(type_order[i]);
-			this->OnSerializeTypeCreate(sb);
+			Anope::string v = stringify(value->id);
+			DoSet(object, field, true, &v);
+
+			Query query;
+			query.SetValue("field", field->GetName());
+			query.SetValue("id", object->id);
+			query.SetValue("other_id", value->id);
+
+			for (Query &q : SQL->Replace(prefix + "edges", query, { "id", "field" }))
+				Run(q);
 		}
+		else
+		{
+			DoSet(object, field, true, nullptr);
 
-		this->loading_databases = false;
-		this->loaded = true;
+			Query query("DELETE FROM `" + prefix + "edges` WHERE `id` = @id@ AND `field` = @field@");
+			query.SetValue("id", object->id);
+			query.SetValue("field", field->GetName());
+			Run(query);
+		}
 
 		return EVENT_STOP;
 	}
 
-	void OnSerializableConstruct(Serializable *obj) override
+	EventReturn OnSerializeUnset(Serialize::Object *object, Serialize::FieldBase *field) override
 	{
-		if (this->shutting_down || this->loading_databases)
-			return;
-		obj->UpdateTS();
-		this->updated_items.insert(obj);
-		this->Notify();
+		DoSet(object, field, false, nullptr);
+		field->CacheMiss(object);
+		return EVENT_STOP;
 	}
 
-	void OnSerializableDestruct(Serializable *obj) override
+	EventReturn OnSerializeUnsetSerializable(Serialize::Object *object, Serialize::FieldBase *field) override
 	{
-		if (this->shutting_down)
-			return;
-		Serialize::Type *s_type = obj->GetSerializableType();
-		if (s_type && obj->id > 0)
-			this->RunBackground("DELETE FROM `" + this->prefix + s_type->GetName() + "` WHERE `id` = " + stringify(obj->id));
-		this->updated_items.erase(obj);
+		DoSet(object, field, true, nullptr);
+		field->CacheMiss(object);
+
+		Query query("DELETE FROM `" + prefix + "edges` WHERE `id` = @id@ AND `field` = @field@");
+		query.SetValue("id", object->id);
+		query.SetValue("field", field->GetName());
+		Run(query);
+
+		return EVENT_STOP;
 	}
 
-	void OnSerializableUpdate(Serializable *obj) override
+	EventReturn OnSerializeHasField(Serialize::Object *object, Serialize::FieldBase *field) override
 	{
-		if (this->shutting_down || obj->IsTSCached())
-			return;
-		obj->UpdateTS();
-		this->updated_items.insert(obj);
-		this->Notify();
+		SQL::Result::Value v;
+
+		return GetValue(object, field, v) && !v.null ? EVENT_STOP : EVENT_CONTINUE;
 	}
 
-	void OnSerializeTypeCreate(Serialize::Type *sb) override
+	EventReturn OnSerializableGetId(Serialize::ID &id) override
 	{
-		if (!this->loading_databases && !this->loaded)
-			return;
+		StartTransaction();
 
-		Query query("SELECT * FROM `" + this->prefix + sb->GetName() + "`");
-		Result res = this->sql->RunQuery(query);
+		id = SQL->GetID(prefix);
+		return EVENT_ALLOW;
+	}
 
-		for (int j = 0; j < res.Rows(); ++j)
-		{
-			Data data;
+	void OnSerializableCreate(Serialize::Object *object) override
+	{
+		StartTransaction();
 
-			const std::map<Anope::string, Anope::string> &row = res.Row(j);
-			for (std::map<Anope::string, Anope::string>::const_iterator rit = row.begin(), rit_end = row.end(); rit != rit_end; ++rit)
-				data[rit->first] << rit->second;
+		Query q = Query("INSERT INTO `" + prefix + "objects` (`id`,`type`) VALUES (@id@, @type@)");
+		q.SetValue("id", object->id);
+		q.SetValue("type", object->GetSerializableType()->GetName());
+		Run(q);
+	}
 
-			Serializable *obj = sb->Unserialize(NULL, data);
-			try
-			{
-				if (obj)
-					obj->id = convertTo<unsigned int>(res.Get(j, "id"));
-			}
-			catch (const ConvertException &)
-			{
-				Log(this) << "Unable to convert id for object #" << j << " of type " << sb->GetName();
-			}
+	void OnSerializableDelete(Serialize::Object *object) override
+	{
+		StartTransaction();
 
-			if (obj)
-			{
-				/* The Unserialize operation is destructive so rebuild the data for UpdateCache.
-				 * Also the old data may contain columns that we don't use, so we reserialize the
-				 * object to know for sure our cache is consistent
-				 */
-
-				Data data2;
-				obj->Serialize(data2);
-				obj->UpdateCache(data2); /* We know this is the most up to date copy */
-			}
-		}
+		Query query("DELETE FROM `" + prefix + object->GetSerializableType()->GetName() + "` WHERE `id` = " + stringify(object->id));
+		Run(query);
 	}
 };
 
-MODULE_INIT(DBSQL)
+MODULE_INIT(DBMySQL)