diff options
author | Adam <Adam@anope.org> | 2012-12-13 06:12:56 -0500 |
---|---|---|
committer | Adam <Adam@anope.org> | 2012-12-13 06:12:56 -0500 |
commit | c1077faa281c5635f85b892e605e23bd6c8fcc3b (patch) | |
tree | 213b5f87a19f182e1efd6110f03ff10d5b10ebf6 /modules/database | |
parent | 76ba147c22944b67e8522cd2bb7b6e1bae498ced (diff) |
Optimize much of the database code and serialize code.
Diffstat (limited to 'modules/database')
-rw-r--r-- | modules/database/db_flatfile.cpp | 354 | ||||
-rw-r--r-- | modules/database/db_sql.cpp | 62 | ||||
-rw-r--r-- | modules/database/db_sql_live.cpp | 48 |
3 files changed, 290 insertions, 174 deletions
diff --git a/modules/database/db_flatfile.cpp b/modules/database/db_flatfile.cpp index acb287cee..66715b9f8 100644 --- a/modules/database/db_flatfile.cpp +++ b/modules/database/db_flatfile.cpp @@ -12,145 +12,202 @@ #include "module.h" -class DBFlatFile : public Module +class SaveData : public Serialize::Data { - Anope::string DatabaseFile; - Anope::string BackupFile; - /* Day the last backup was on */ - int LastDay; - /* Backup file names */ - std::list<Anope::string> Backups; + public: + std::fstream *fs; + + SaveData() : fs(NULL) { } + + std::iostream& operator[](const Anope::string &key) anope_override + { + *fs << "\nDATA " << key << " "; + return *fs; + } +}; +class LoadData : public Serialize::Data +{ public: - DBFlatFile(const Anope::string &modname, const Anope::string &creator) : Module(modname, creator, DATABASE) + std::fstream *fs; + std::map<Anope::string, Anope::string> data; + std::stringstream ss; + bool read; + + LoadData() : fs(NULL), read(false) { } + + std::iostream& operator[](const Anope::string &key) anope_override { - this->SetAuthor("Anope"); + if (!read) + { + for (Anope::string token; std::getline(*this->fs, token.str());) + { + if (token.find("DATA ") != 0) + break; - Implementation i[] = { I_OnReload, I_OnLoadDatabase, I_OnSaveDatabase }; - ModuleManager::Attach(i, this, sizeof(i) / sizeof(Implementation)); + size_t sp = token.find(' ', 5); // Skip DATA + if (sp != Anope::string::npos) + data[token.substr(5, sp - 5)] = token.substr(sp + 1); + } - OnReload(); + read = true; + } - LastDay = 0; + ss.clear(); + this->ss << this->data[key]; + return this->ss; } + + void Reset() + { + read = false; + data.clear(); + } +}; + +class DBFlatFile : public Module, public Pipe +{ + Anope::string database_file; + /* Day the last backup was on */ + int last_day; + /* Backup file names */ + std::map<Anope::string, std::list<Anope::string> > backups; + bool use_fork; void BackupDatabase() { - /* Do not backup a database that doesn't exist */ - if (!Anope::IsFile(DatabaseFile)) - return; - - time_t now = Anope::CurTime; - tm *tm = localtime(&now); + tm *tm = localtime(&Anope::CurTime); - if (tm->tm_mday != LastDay) + if (tm->tm_mday != last_day) { - LastDay = tm->tm_mday; - Anope::string newname = BackupFile + "." + stringify(tm->tm_year) + "." + stringify(tm->tm_mon) + "." + stringify(tm->tm_mday); + last_day = tm->tm_mday; - /* Backup already exists */ - if (IsFile(newname)) - return; + const std::vector<Anope::string> &type_order = Serialize::Type::GetTypeOrder(); - Log(LOG_DEBUG) << "db_flatfile: Attemping to rename " << DatabaseFile << " to " << newname; - if (rename(DatabaseFile.c_str(), newname.c_str())) - { - Log(this) << "Unable to back up database!"; + std::set<Anope::string> dbs; + dbs.insert(database_file); - if (!Config->NoBackupOkay) - Anope::Quitting = true; + for (unsigned i = 0; i < type_order.size(); ++i) + { + Serialize::Type *stype = Serialize::Type::Find(type_order[i]); - return; + if (stype && stype->GetOwner()) + dbs.insert("module_" + stype->GetOwner()->name + ".db"); } - Backups.push_back(newname); - if (Config->KeepBackups > 0 && Backups.size() > static_cast<unsigned>(Config->KeepBackups)) + for (std::set<Anope::string>::const_iterator it = dbs.begin(), it_end = dbs.end(); it != it_end; ++it) { - unlink(Backups.front().c_str()); - Backups.pop_front(); + const Anope::string &oldname = Anope::DataDir + "/" + *it; + Anope::string newname = Anope::DataDir + "/backups/" + *it + "." + stringify(tm->tm_year) + "." + stringify(tm->tm_mon) + "." + stringify(tm->tm_mday); + + /* Backup already exists */ + if (Anope::IsFile(newname)) + continue; + + Log(LOG_DEBUG) << "db_flatfile: Attemping to rename " << *it << " to " << newname; + if (rename(oldname.c_str(), newname.c_str())) + { + Log(this) << "Unable to back up database " << *it << "!"; + + if (!Config->NoBackupOkay) + Anope::Quitting = true; + + continue; + } + + backups[*it].push_back(newname); + + if (Config->KeepBackups > 0 && backups[*it].size() > static_cast<unsigned>(Config->KeepBackups)) + { + unlink(backups[*it].front().c_str()); + backups[*it].pop_front(); + } } } } - void OnReload() anope_override + public: + DBFlatFile(const Anope::string &modname, const Anope::string &creator) : Module(modname, creator, DATABASE), last_day(0), use_fork(false) { - ConfigReader config; - DatabaseFile = Anope::DataDir + "/" + config.ReadValue("db_flatfile", "database", "anope.db", 0); - BackupFile = Anope::DataDir + "/backups/" + config.ReadValue("db_flatfile", "database", "anope.db", 0); + this->SetAuthor("Anope"); + + Implementation i[] = { I_OnReload, I_OnLoadDatabase, I_OnSaveDatabase }; + ModuleManager::Attach(i, this, sizeof(i) / sizeof(Implementation)); + + OnReload(); } - EventReturn OnLoadDatabase() anope_override + void OnNotify() anope_override { - std::map<Module *, std::fstream *> databases; - databases[NULL] = new std::fstream(DatabaseFile.c_str(), std::ios_base::in); + char buf[512]; + int i = this->Read(buf, sizeof(buf) - 1); + if (i <= 0) + return; + buf[i] = 0; - if (!databases[NULL]->is_open()) + if (!*buf) { - delete databases[NULL]; - Log(this) << "Unable to open " << DatabaseFile << " for reading!"; - return EVENT_CONTINUE; + Log(this) << "Finished saving databases"; + return; } - const std::vector<Anope::string> type_order = Serialize::Type::GetTypeOrder(); + Log(this) << "Error saving databases: " << buf; - for (unsigned i = 0; i < type_order.size(); ++i) - { - Serialize::Type *stype = Serialize::Type::Find(type_order[i]); - if (stype && !databases.count(stype->GetOwner())) - { - Anope::string db_name = Anope::DataDir + "/module_" + stype->GetOwner()->name + ".db"; - databases[stype->GetOwner()] = new std::fstream(db_name.c_str(), std::ios_base::in); - } - } + if (!Config->NoBackupOkay) + Anope::Quitting = true; + } - std::multimap<Serialize::Type *, Serialize::Data> objects; - for (std::map<Module *, std::fstream *>::iterator it = databases.begin(), it_end = databases.end(); it != it_end; ++it) - { - std::fstream *db = it->second; - Serialize::Type *st = NULL; - Serialize::Data data; - for (Anope::string buf, token; std::getline(*db, buf.str());) - { - spacesepstream sep(buf); + void OnReload() anope_override + { + ConfigReader config; + database_file = config.ReadValue("db_flatfile", "database", "anope.db", 0); + use_fork = config.ReadFlag("db_flatfile", "fork", "no", 0); + } - if (!sep.GetToken(token)) - continue; + EventReturn OnLoadDatabase() anope_override + { + const std::vector<Anope::string> &type_order = Serialize::Type::GetTypeOrder(); + std::set<Anope::string> tried_dbs; - if (token == "OBJECT" && sep.GetToken(token)) - { - st = Serialize::Type::Find(token); - data.clear(); - } - else if (token == "DATA" && st != NULL && sep.GetToken(token)) - data[token] << sep.GetRemaining(); - else if (token == "END" && st != NULL) - { - objects.insert(std::make_pair(st, data)); + const Anope::string &db_name = Anope::DataDir + "/" + database_file; - st = NULL; - data.clear(); - } - } + std::fstream fd(db_name.c_str(), std::ios_base::in); + if (!fd.is_open()) + { + Log(this) << "Unable to open " << db_name << " for reading!"; + return EVENT_STOP; } + std::map<Anope::string, std::vector<std::streampos> > positions; + + for (Anope::string buf; std::getline(fd, buf.str());) + if (buf.find("OBJECT ") == 0) + positions[buf.substr(7)].push_back(fd.tellg()); + + LoadData ld; + ld.fs = &fd; + for (unsigned i = 0; i < type_order.size(); ++i) { Serialize::Type *stype = Serialize::Type::Find(type_order[i]); - - std::multimap<Serialize::Type *, Serialize::Data>::iterator it = objects.find(stype), it_end = objects.upper_bound(stype); - if (it == objects.end()) + if (!stype || stype->GetOwner()) continue; - for (; it != it_end; ++it) - it->first->Unserialize(NULL, it->second); - } + + std::vector<std::streampos> &pos = positions[stype->GetName()]; - for (std::map<Module *, std::fstream *>::iterator it = databases.begin(), it_end = databases.end(); it != it_end; ++it) - { - it->second->close(); - delete it->second; + for (unsigned j = 0; j < pos.size(); ++j) + { + fd.clear(); + fd.seekg(pos[j]); + + stype->Unserialize(NULL, ld); + ld.Reset(); + } } + fd.close(); + return EVENT_STOP; } @@ -159,61 +216,92 @@ class DBFlatFile : public Module { BackupDatabase(); - Anope::string tmp_db = DatabaseFile + ".tmp"; - if (IsFile(DatabaseFile)) - rename(DatabaseFile.c_str(), tmp_db.c_str()); - - std::map<Module *, std::fstream *> databases; - databases[NULL] = new std::fstream(DatabaseFile.c_str(), std::ios_base::out | std::ios_base::trunc); - if (!databases[NULL]->is_open()) + int i = -1; + if (use_fork) { - delete databases[NULL]; - Log(this) << "Unable to open " << DatabaseFile << " for writing"; - if (IsFile(tmp_db)) - rename(tmp_db.c_str(), DatabaseFile.c_str()); - return EVENT_CONTINUE; + i = fork(); + if (i > 0) + return EVENT_CONTINUE; + else if (i < 0) + Log(this) << "Unable to fork for database save"; } - const std::list<Serializable *> &items = Serializable::GetItems(); - for (std::list<Serializable *>::const_iterator it = items.begin(), it_end = items.end(); it != it_end; ++it) + try { - Serializable *base = *it; - Serialize::Type *s_type = base->GetSerializableType(); + std::map<Module *, std::fstream *> databases; - if (!s_type) - continue; + SaveData data; + const std::list<Serializable *> &items = Serializable::GetItems(); + for (std::list<Serializable *>::const_iterator it = items.begin(), it_end = items.end(); it != it_end; ++it) + { + Serializable *base = *it; + Serialize::Type *s_type = base->GetSerializableType(); + + if (!s_type) + continue; - Serialize::Data data = base->Serialize(); + data.fs = databases[s_type->GetOwner()]; - if (!databases.count(s_type->GetOwner())) + if (!data.fs) + { + Anope::string db_name; + if (s_type->GetOwner()) + db_name = Anope::DataDir + "/module_" + s_type->GetOwner()->name + ".db"; + else + db_name = Anope::DataDir + "/" + database_file; + + if (Anope::IsFile(db_name)) + rename(db_name.c_str(), (db_name + ".tmp").c_str()); + + data.fs = databases[s_type->GetOwner()] = new std::fstream(db_name.c_str(), std::ios_base::out | std::ios_base::trunc); + + if (!data.fs->is_open()) + { + Log(this) << "Unable to open " << db_name << " for writing"; + continue; + } + } + else if (!data.fs->is_open()) + continue; + + *data.fs << "OBJECT " << s_type->GetName(); + base->Serialize(data); + *data.fs << "\nEND\n"; + } + + for (std::map<Module *, std::fstream *>::iterator it = databases.begin(), it_end = databases.end(); it != it_end; ++it) { - Anope::string db_name = Anope::DataDir + "/module_" + s_type->GetOwner()->name + ".db"; - databases[s_type->GetOwner()] = new std::fstream(db_name.c_str(), std::ios_base::out | std::ios_base::trunc); + std::fstream *f = it->second; + const Anope::string &db_name = Anope::DataDir + "/" + (it->first ? (it->first->name + ".db") : database_file); + + if (!f->is_open() || !f->good()) + { + this->Write("Unable to write database " + db_name); + + f->close(); + + if (Anope::IsFile((db_name + ".tmp").c_str())) + rename((db_name + ".tmp").c_str(), db_name.c_str()); + } + else + { + f->close(); + unlink((db_name + ".tmp").c_str()); + } + + delete f; } - std::fstream *fd = databases[s_type->GetOwner()]; - - *fd << "OBJECT " << s_type->GetName() << "\n"; - for (Serialize::Data::iterator dit = data.begin(), dit_end = data.end(); dit != dit_end; ++dit) - *fd << "DATA " << dit->first << " " << dit->second.astr() << "\n"; - *fd << "END\n"; } - - if (databases[NULL]->good() == false) + catch (...) { - Log(this) << "Unable to write database"; - databases[NULL]->close(); - if (!Config->NoBackupOkay) - Anope::Quitting = true; - if (IsFile(tmp_db)) - rename(tmp_db.c_str(), DatabaseFile.c_str()); + if (i) + throw; } - else - unlink(tmp_db.c_str()); - for (std::map<Module *, std::fstream *>::iterator it = databases.begin(), it_end = databases.end(); it != it_end; ++it) + if (!i) { - it->second->close(); - delete it->second; + this->Notify(); + exit(0); } return EVENT_CONTINUE; diff --git a/modules/database/db_sql.cpp b/modules/database/db_sql.cpp index 60db11c0d..56db7b8a0 100644 --- a/modules/database/db_sql.cpp +++ b/modules/database/db_sql.cpp @@ -11,18 +11,19 @@ #include "module.h" #include "../extra/sql.h" -class SQLSQLInterface : public SQLInterface +using namespace SQL; + +class SQLSQLInterface : public Interface { public: - SQLSQLInterface(Module *o) : SQLInterface(o) { } - virtual ~SQLSQLInterface() { } + SQLSQLInterface(Module *o) : Interface(o) { } - void OnResult(const SQLResult &r) anope_override + void OnResult(const Result &r) anope_override { Log(LOG_DEBUG) << "SQL successfully executed query: " << r.finished_query; } - void OnError(const SQLResult &r) anope_override + void OnError(const Result &r) anope_override { if (!r.GetQuery().query.empty()) Log(LOG_DEBUG) << "Error executing query " << r.finished_query << ": " << r.GetError(); @@ -38,7 +39,7 @@ class ResultSQLSQLInterface : public SQLSQLInterface public: ResultSQLSQLInterface(Module *o, Serializable *ob) : SQLSQLInterface(o), obj(ob) { } - void OnResult(const SQLResult &r) anope_override + void OnResult(const Result &r) anope_override { SQLSQLInterface::OnResult(r); if (r.GetID() > 0 && this->obj) @@ -46,7 +47,7 @@ public: delete this; } - void OnError(const SQLResult &r) anope_override + void OnError(const Result &r) anope_override { SQLSQLInterface::OnError(r); delete this; @@ -55,13 +56,14 @@ public: class DBSQL : public Module, public Pipe { - ServiceReference<SQLProvider> sql; + ServiceReference<Provider> sql; SQLSQLInterface sqlinterface; Anope::string prefix; std::set<Reference<Serializable> > updated_items; bool shutting_down; + bool loading_databases; - void RunBackground(const SQLQuery &q, SQLInterface *iface = NULL) + void RunBackground(const Query &q, Interface *iface = NULL) { if (!this->sql) { @@ -83,7 +85,7 @@ class DBSQL : public Module, public Pipe } public: - DBSQL(const Anope::string &modname, const Anope::string &creator) : Module(modname, creator, DATABASE), sql("", ""), sqlinterface(this), shutting_down(false) + DBSQL(const Anope::string &modname, const Anope::string &creator) : Module(modname, creator, DATABASE), sql("", ""), sqlinterface(this), shutting_down(false), loading_databases(false) { this->SetAuthor("Anope"); @@ -101,21 +103,26 @@ class DBSQL : public Module, public Pipe if (obj && this->sql) { - if (obj->IsCached()) + Data *data = new Data(); + obj->Serialize(*data); + + if (obj->IsCached(data)) + { + delete data; continue; - obj->UpdateCache(); + } + + obj->UpdateCache(data); Serialize::Type *s_type = obj->GetSerializableType(); if (!s_type) continue; - Serialize::Data data = obj->Serialize(); - - std::vector<SQLQuery> create = this->sql->CreateTable(this->prefix + s_type->GetName(), data); + std::vector<Query> create = this->sql->CreateTable(this->prefix + s_type->GetName(), *data); for (unsigned i = 0; i < create.size(); ++i) this->RunBackground(create[i]); - SQLQuery insert = this->sql->BuildInsert(this->prefix + s_type->GetName(), obj->id, data); + Query insert = this->sql->BuildInsert(this->prefix + s_type->GetName(), obj->id, *data); this->RunBackground(insert, new ResultSQLSQLInterface(this, obj)); } } @@ -127,7 +134,7 @@ class DBSQL : public Module, public Pipe { ConfigReader config; Anope::string engine = config.ReadValue("db_sql", "engine", "", 0); - this->sql = ServiceReference<SQLProvider>("SQLProvider", engine); + this->sql = ServiceReference<Provider>("SQL::Provider", engine); this->prefix = config.ReadValue("db_sql", "prefix", "anope_db_", 0); } @@ -150,23 +157,25 @@ class DBSQL : public Module, public Pipe return EVENT_CONTINUE; } + this->loading_databases = true; + const std::vector<Anope::string> type_order = Serialize::Type::GetTypeOrder(); for (unsigned i = 0; i < type_order.size(); ++i) { Serialize::Type *sb = Serialize::Type::Find(type_order[i]); - SQLQuery query("SELECT * FROM `" + this->prefix + sb->GetName() + "`"); - SQLResult res = this->sql->RunQuery(query); + Query query("SELECT * FROM `" + this->prefix + sb->GetName() + "`"); + Result res = this->sql->RunQuery(query); for (int j = 0; j < res.Rows(); ++j) { - Serialize::Data data; + Data *data = new Data(); const std::map<Anope::string, Anope::string> &row = res.Row(j); for (std::map<Anope::string, Anope::string>::const_iterator rit = row.begin(), rit_end = row.end(); rit != rit_end; ++rit) - data[rit->first] << rit->second; + (*data)[rit->first] << rit->second; - Serializable *obj = sb->Unserialize(NULL, data); + Serializable *obj = sb->Unserialize(NULL, *data); try { if (obj) @@ -176,15 +185,22 @@ class DBSQL : public Module, public Pipe { Log(this) << "Unable to convert id for object #" << j << " of type " << sb->GetName(); } + + if (obj) + obj->UpdateCache(data); /* We know this is the most up to date copy */ + else + delete data; } } + this->loading_databases = false; + return EVENT_STOP; } void OnSerializableConstruct(Serializable *obj) anope_override { - if (this->shutting_down) + if (this->shutting_down || this->loading_databases) return; this->updated_items.insert(obj); this->Notify(); diff --git a/modules/database/db_sql_live.cpp b/modules/database/db_sql_live.cpp index 00561592f..8d8836055 100644 --- a/modules/database/db_sql_live.cpp +++ b/modules/database/db_sql_live.cpp @@ -2,12 +2,14 @@ #include "../extra/sql.h" #include "../commands/os_session.h" +using namespace SQL; + class DBMySQL : public Module, public Pipe { private: Anope::string engine; Anope::string prefix; - ServiceReference<SQLProvider> SQL; + ServiceReference<Provider> SQL; time_t lastwarn; bool ro; bool init; @@ -43,24 +45,24 @@ class DBMySQL : public Module, public Pipe return init && SQL; } - void RunQuery(const SQLQuery &query) + void RunQuery(const Query &query) { /* Can this be threaded? */ this->RunQueryResult(query); } - SQLResult RunQueryResult(const SQLQuery &query) + Result RunQueryResult(const Query &query) { if (this->CheckSQL()) { - SQLResult res = SQL->RunQuery(query); + Result res = SQL->RunQuery(query); if (!res.GetError().empty()) Log(LOG_DEBUG) << "SQL-live got error " << res.GetError() << " for " + res.finished_query; else Log(LOG_DEBUG) << "SQL-live got " << res.Rows() << " rows for " << res.finished_query; return res; } - throw SQLException("No SQL!"); + throw SQL::Exception("No SQL!"); } public: @@ -87,21 +89,26 @@ class DBMySQL : public Module, public Pipe if (obj && this->SQL) { - if (obj->IsCached()) + Data *data = new Data(); + obj->Serialize(*data); + + if (obj->IsCached(data)) + { + delete data; continue; - obj->UpdateCache(); + } + + obj->UpdateCache(data); Serialize::Type *s_type = obj->GetSerializableType(); if (!s_type) continue; - Serialize::Data data = obj->Serialize(); - - std::vector<SQLQuery> create = this->SQL->CreateTable(this->prefix + s_type->GetName(), data); + std::vector<Query> create = this->SQL->CreateTable(this->prefix + s_type->GetName(), *data); for (unsigned i = 0; i < create.size(); ++i) this->RunQueryResult(create[i]); - SQLResult res = this->RunQueryResult(this->SQL->BuildInsert(this->prefix + s_type->GetName(), obj->id, data)); + Result res = this->RunQueryResult(this->SQL->BuildInsert(this->prefix + s_type->GetName(), obj->id, *data)); if (obj->id != res.GetID()) { /* In this case obj is new, so place it into the object map */ @@ -129,7 +136,7 @@ class DBMySQL : public Module, public Pipe { ConfigReader config; this->engine = config.ReadValue("db_sql", "engine", "", 0); - this->SQL = ServiceReference<SQLProvider>("SQLProvider", this->engine); + this->SQL = ServiceReference<Provider>("SQL::Provider", this->engine); this->prefix = config.ReadValue("db_sql", "prefix", "anope_db_", 0); } @@ -157,11 +164,11 @@ class DBMySQL : public Module, public Pipe if (!this->CheckInit() || obj->GetTimestamp() == Anope::CurTime) return; - SQLQuery query("SELECT * FROM `" + this->prefix + obj->GetName() + "` WHERE (`timestamp` > " + this->SQL->FromUnixtime(obj->GetTimestamp()) + " OR `timestamp` IS NULL)"); + Query query("SELECT * FROM `" + this->prefix + obj->GetName() + "` WHERE (`timestamp` > " + this->SQL->FromUnixtime(obj->GetTimestamp()) + " OR `timestamp` IS NULL)"); obj->UpdateTimestamp(); - SQLResult res = this->RunQueryResult(query); + Result res = this->RunQueryResult(query); bool clear_null = false; for (int i = 0; i < res.Rows(); ++i) @@ -191,17 +198,17 @@ class DBMySQL : public Module, public Pipe } else { - Serialize::Data data; + Data *data = new Data(); for (std::map<Anope::string, Anope::string>::const_iterator it = row.begin(), it_end = row.end(); it != it_end; ++it) - data[it->first] << it->second; + (*data)[it->first] << it->second; Serializable *s = NULL; std::map<unsigned int, Serializable *>::iterator it = obj->objects.find(id); if (it != obj->objects.end()) s = it->second; - Serializable *new_s = obj->Unserialize(s, data); + Serializable *new_s = obj->Unserialize(s, *data); if (new_s) { // If s == new_s then s->id == new_s->id @@ -209,11 +216,16 @@ class DBMySQL : public Module, public Pipe { new_s->id = id; obj->objects[id] = new_s; - new_s->UpdateCache(); /* We know this is the most up to date copy */ + new_s->UpdateCache(data); /* We know this is the most up to date copy */ } + else + delete data; } else + { + delete data; s->Destroy(); + } } } |