diff options
| author | Sadie Powell <sadie@witchery.services> | 2024-08-14 02:40:48 +0100 |
|---|---|---|
| committer | Sadie Powell <sadie@witchery.services> | 2024-08-14 05:47:55 +0100 |
| commit | 528b5938ec71abed396248cf5e00f346e685aaea (patch) | |
| tree | e923fed32be3fb62f946d44c633cd85ce855550b | |
| parent | 03bee1706383d2766923e5d8edbb90d7ad1948b6 (diff) | |
Automatically determine SQL column type from the field.
Also add more column types to ensure we are storing data in the
best format in the database.
36 files changed, 331 insertions, 223 deletions
diff --git a/data/modules.example.conf b/data/modules.example.conf index 6becc7c36..5b0a578cb 100644 --- a/data/modules.example.conf +++ b/data/modules.example.conf @@ -755,7 +755,7 @@ module { name = "sasl" } name = "sqlite/main" /* The database name, it will be created if it does not exist. */ - database = "anope.db" + database = "anope.sqlite" } } diff --git a/include/extensible.h b/include/extensible.h index 7ff518516..f248e6db8 100644 --- a/include/extensible.h +++ b/include/extensible.h @@ -173,7 +173,7 @@ public: void ExtensibleSerialize(const Extensible *e, const Serializable *s, Serialize::Data &data) const override { T *t = this->Get(e); - data[this->name] << *t; + data.Store(this->name, *t); } void ExtensibleUnserialize(Extensible *e, Serializable *s, Serialize::Data &data) override @@ -194,8 +194,7 @@ public: void ExtensibleSerialize(const Extensible *e, const Serializable *s, Serialize::Data &data) const override { - data.SetType(this->name, Serialize::Data::DT_INT); - data[this->name] << true; + data.Store(this->name, true); } void ExtensibleUnserialize(Extensible *e, Serializable *s, Serialize::Data &data) override diff --git a/include/modules/os_oper.h b/include/modules/os_oper.h index 8a92efe45..034c2b2fd 100644 --- a/include/modules/os_oper.h +++ b/include/modules/os_oper.h @@ -16,8 +16,8 @@ struct MyOper final void Serialize(Serialize::Data &data) const override { - data["name"] << this->name; - data["type"] << this->ot->GetName(); + data.Store("name", this->name); + data.Store("type", this->ot->GetName()); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/include/modules/os_session.h b/include/modules/os_session.h index 4281ed086..0c1da71e6 100644 --- a/include/modules/os_session.h +++ b/include/modules/os_session.h @@ -62,12 +62,12 @@ static ServiceReference<SessionService> session_service("SessionService", "sessi void Exception::Serialize(Serialize::Data &data) const { - data["mask"] << this->mask; - data["limit"] << this->limit; - data["who"] << this->who; - data["reason"] << this->reason; - data["time"] << this->time; - data["expires"] << this->expires; + data.Store("mask", this->mask); + data.Store("limit", this->limit); + data.Store("who", this->who); + data.Store("reason", this->reason); + data.Store("time", this->time); + data.Store("expires", this->expires); } Serializable *Exception::Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/include/modules/sql.h b/include/modules/sql.h index 8818a5130..8ff5efc7c 100644 --- a/include/modules/sql.h +++ b/include/modules/sql.h @@ -19,7 +19,7 @@ namespace SQL public: typedef std::map<Anope::string, std::stringstream *> Map; Map data; - std::map<Anope::string, Type> types; + std::map<Anope::string, Serialize::DataType> types; ~Data() { @@ -60,17 +60,17 @@ namespace SQL this->data.clear(); } - void SetType(const Anope::string &key, Type t) override + void SetType(const Anope::string &key, Serialize::DataType dt) override { - this->types[key] = t; + this->types[key] = dt; } - Type GetType(const Anope::string &key) const override + Serialize::DataType GetType(const Anope::string &key) const override { - std::map<Anope::string, Type>::const_iterator it = this->types.find(key); + auto it = this->types.find(key); if (it != this->types.end()) return it->second; - return DT_TEXT; + return Serialize::DataType::TEXT; } }; diff --git a/include/serialize.h b/include/serialize.h index e53295dc0..ff8ed30d3 100644 --- a/include/serialize.h +++ b/include/serialize.h @@ -18,22 +18,44 @@ namespace Serialize { + enum class DataType + : uint8_t + { + BOOL, + FLOAT, + INT, + TEXT, + UINT, + }; + class Data { public: - enum Type - { - DT_TEXT, - DT_INT - }; - virtual ~Data() = default; virtual std::iostream &operator[](const Anope::string &key) = 0; + + template <typename T> + void Store(const Anope::string &key, const T &value) + { + using Type = std::remove_cv_t<std::remove_reference_t<T>>; + + if constexpr (std::is_same_v<Type, bool>) + SetType(key, DataType::BOOL); + else if constexpr (std::is_floating_point_v<Type>) + SetType(key, DataType::FLOAT); + else if constexpr (std::is_integral_v<Type> && std::is_signed_v<Type>) + SetType(key, DataType::INT); + else if constexpr (std::is_integral_v<Type> && std::is_unsigned_v<Type>) + SetType(key, DataType::UINT); + + this->operator[](key) << value; + } + virtual size_t Hash() const { throw CoreException("Not supported"); } - virtual void SetType(const Anope::string &key, Type t) { } - virtual Type GetType(const Anope::string &key) const { return DT_TEXT; } + virtual void SetType(const Anope::string &key, DataType dt) { } + virtual DataType GetType(const Anope::string &key) const { return DataType::TEXT; } }; extern void RegisterTypes(); diff --git a/modules/botserv/bs_badwords.cpp b/modules/botserv/bs_badwords.cpp index e72e32a6e..2396a91c6 100644 --- a/modules/botserv/bs_badwords.cpp +++ b/modules/botserv/bs_badwords.cpp @@ -21,9 +21,9 @@ struct BadWordImpl final void Serialize(Serialize::Data &data) const override { - data["ci"] << this->chan; - data["word"] << this->word; - data.SetType("type", Serialize::Data::DT_INT); data["type"] << this->type; + data.Store("ci", this->chan); + data.Store("word", this->word); + data.Store("type", this->type); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &); diff --git a/modules/botserv/bs_kick.cpp b/modules/botserv/bs_kick.cpp index f2a39dbf1..ba2ace0dd 100644 --- a/modules/botserv/bs_kick.cpp +++ b/modules/botserv/bs_kick.cpp @@ -53,25 +53,28 @@ struct KickerDataImpl final if (kd == NULL) return; - data.SetType("kickerdata:amsgs", Serialize::Data::DT_INT); data["kickerdata:amsgs"] << kd->amsgs; - data.SetType("kickerdata:badwords", Serialize::Data::DT_INT); data["kickerdata:badwords"] << kd->badwords; - data.SetType("kickerdata:bolds", Serialize::Data::DT_INT); data["kickerdata:bolds"] << kd->bolds; - data.SetType("kickerdata:caps", Serialize::Data::DT_INT); data["kickerdata:caps"] << kd->caps; - data.SetType("kickerdata:colors", Serialize::Data::DT_INT); data["kickerdata:colors"] << kd->colors; - data.SetType("kickerdata:flood", Serialize::Data::DT_INT); data["kickerdata:flood"] << kd->flood; - data.SetType("kickerdata:italics", Serialize::Data::DT_INT); data["kickerdata:italics"] << kd->italics; - data.SetType("kickerdata:repeat", Serialize::Data::DT_INT); data["kickerdata:repeat"] << kd->repeat; - data.SetType("kickerdata:reverses", Serialize::Data::DT_INT); data["kickerdata:reverses"] << kd->reverses; - data.SetType("kickerdata:underlines", Serialize::Data::DT_INT); data["kickerdata:underlines"] << kd->underlines; - data.SetType("capsmin", Serialize::Data::DT_INT); data["capsmin"] << kd->capsmin; - data.SetType("capspercent", Serialize::Data::DT_INT); data["capspercent"] << kd->capspercent; - data.SetType("floodlines", Serialize::Data::DT_INT); data["floodlines"] << kd->floodlines; - data.SetType("floodsecs", Serialize::Data::DT_INT); data["floodsecs"] << kd->floodsecs; - data.SetType("repeattimes", Serialize::Data::DT_INT); data["repeattimes"] << kd->repeattimes; - data.SetType("dontkickops", Serialize::Data::DT_INT); data["dontkickops"] << kd->dontkickops; - data.SetType("dontkickvoices", Serialize::Data::DT_INT); data["dontkickvoices"] << kd->dontkickvoices; + data.Store("kickerdata:amsgs", kd->amsgs); + data.Store("kickerdata:badwords", kd->badwords); + data.Store("kickerdata:bolds", kd->bolds); + data.Store("kickerdata:caps", kd->caps); + data.Store("kickerdata:colors", kd->colors); + data.Store("kickerdata:flood", kd->flood); + data.Store("kickerdata:italics", kd->italics); + data.Store("kickerdata:repeat", kd->repeat); + data.Store("kickerdata:reverses", kd->reverses); + data.Store("kickerdata:underlines", kd->underlines); + data.Store("capsmin", kd->capsmin); + data.Store("capspercent", kd->capspercent); + data.Store("floodlines", kd->floodlines); + data.Store("floodsecs", kd->floodsecs); + data.Store("repeattimes", kd->repeattimes); + data.Store("dontkickops", kd->dontkickops); + data.Store("dontkickvoices", kd->dontkickvoices); + + std::ostringstream oss; for (auto ttbtype : kd->ttb) - data["ttb"] << ttbtype << " "; + oss << ttbtype << " "; + data.Store("ttb", oss.str()); } void ExtensibleUnserialize(Extensible *e, Serializable *s, Serialize::Data &data) override diff --git a/modules/chanserv/cs_entrymsg.cpp b/modules/chanserv/cs_entrymsg.cpp index efe13d1aa..808582bdc 100644 --- a/modules/chanserv/cs_entrymsg.cpp +++ b/modules/chanserv/cs_entrymsg.cpp @@ -32,10 +32,10 @@ struct EntryMsgImpl final void Serialize(Serialize::Data &data) const override { - data["ci"] << this->chan; - data["creator"] << this->creator; - data["message"] << this->message; - data.SetType("when", Serialize::Data::DT_INT); data["when"] << this->when; + data.Store("ci", this->chan); + data.Store("creator", this->creator); + data.Store("message", this->message); + data.Store("when", this->when); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data); diff --git a/modules/chanserv/cs_log.cpp b/modules/chanserv/cs_log.cpp index defd38287..e06d3d42b 100644 --- a/modules/chanserv/cs_log.cpp +++ b/modules/chanserv/cs_log.cpp @@ -37,14 +37,14 @@ struct LogSettingImpl final void Serialize(Serialize::Data &data) const override { - data["ci"] << chan; - data["service_name"] << service_name; - data["command_service"] << command_service; - data["command_name"] << command_name; - data["method"] << method; - data["extra"] << extra; - data["creator"] << creator; - data.SetType("created", Serialize::Data::DT_INT); data["created"] << created; + data.Store("ci", chan); + data.Store("service_name", service_name); + data.Store("command_service", command_service); + data.Store("command_name", command_name); + data.Store("method", method); + data.Store("extra", extra); + data.Store("creator", creator); + data.Store("created", created); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/modules/chanserv/cs_mode.cpp b/modules/chanserv/cs_mode.cpp index 1ea0e9716..9962d1d2f 100644 --- a/modules/chanserv/cs_mode.cpp +++ b/modules/chanserv/cs_mode.cpp @@ -205,12 +205,12 @@ struct ModeLocksImpl final void ModeLockImpl::Serialize(Serialize::Data &data) const { - data["ci"] << this->ci; - data["set"] << this->set; - data["name"] << this->name; - data["param"] << this->param; - data["setter"] << this->setter; - data.SetType("created", Serialize::Data::DT_INT); data["created"] << this->created; + data.Store("ci", this->ci); + data.Store("set", this->set); + data.Store("name", this->name); + data.Store("param", this->param); + data.Store("setter", this->setter); + data.Store("created", this->created); } Serializable *ModeLockImpl::Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/modules/chanserv/cs_seen.cpp b/modules/chanserv/cs_seen.cpp index 1603983f5..97d23ee70 100644 --- a/modules/chanserv/cs_seen.cpp +++ b/modules/chanserv/cs_seen.cpp @@ -46,13 +46,13 @@ struct SeenInfo final void Serialize(Serialize::Data &data) const override { - data["nick"] << nick; - data["vhost"] << vhost; - data["type"] << type; - data["nick2"] << nick2; - data["channel"] << channel; - data["message"] << message; - data.SetType("last", Serialize::Data::DT_INT); data["last"] << last; + data.Store("nick", nick); + data.Store("vhost", vhost); + data.Store("type", type); + data.Store("nick2", nick2); + data.Store("channel", channel); + data.Store("message", message); + data.Store("last", last); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/modules/chanserv/cs_set.cpp b/modules/chanserv/cs_set.cpp index 932d1f43c..4f39574af 100644 --- a/modules/chanserv/cs_set.cpp +++ b/modules/chanserv/cs_set.cpp @@ -1068,7 +1068,7 @@ class CSSet final if (!last_value.empty()) modes += "," + last_value; } - data["last_modes"] << modes; + data.Store("last_modes", modes); } void ExtensibleUnserialize(Extensible *e, Serializable *s, Serialize::Data &data) override diff --git a/modules/chanserv/cs_set_misc.cpp b/modules/chanserv/cs_set_misc.cpp index 685ab8022..4ad0ee7a1 100644 --- a/modules/chanserv/cs_set_misc.cpp +++ b/modules/chanserv/cs_set_misc.cpp @@ -46,9 +46,9 @@ struct CSMiscData final void Serialize(Serialize::Data &sdata) const override { - sdata["ci"] << this->object; - sdata["name"] << this->name; - sdata["data"] << this->data; + sdata.Store("ci", this->object); + sdata.Store("name", this->name); + sdata.Store("data", this->data); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/modules/chanserv/cs_suspend.cpp b/modules/chanserv/cs_suspend.cpp index 03eb36c48..846f4fd59 100644 --- a/modules/chanserv/cs_suspend.cpp +++ b/modules/chanserv/cs_suspend.cpp @@ -20,11 +20,11 @@ struct CSSuspendInfo final void Serialize(Serialize::Data &data) const override { - data["chan"] << what; - data["by"] << by; - data["reason"] << reason; - data["time"] << when; - data["expires"] << expires; + data.Store("chan", what); + data.Store("by", by); + data.Store("reason", reason); + data.Store("time", when); + data.Store("expires", expires); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/modules/extra/mysql.cpp b/modules/extra/mysql.cpp index 193427fdb..caa02d8f0 100644 --- a/modules/extra/mysql.cpp +++ b/modules/extra/mysql.cpp @@ -158,6 +158,29 @@ public: Anope::string BuildQuery(const Query &q); Anope::string FromUnixtime(time_t) override; + + Anope::string GetColumnType(Serialize::DataType dt) + { + switch (dt) + { + case Serialize::DataType::BOOL: + return "TINYINT NOT NULL"; + + case Serialize::DataType::FLOAT: + return "DOUBLE PRECISION NOT NULL"; + + case Serialize::DataType::INT: + return "BIGINT NOT NULL"; + + case Serialize::DataType::TEXT: + return "TEXT"; + + case Serialize::DataType::UINT: + return "BIGINT UNSIGNED NOT NULL"; + } + + return "TEXT"; // Should never be reached + } }; /** The SQL thread used to execute queries @@ -410,17 +433,13 @@ std::vector<Query> MySQLService::CreateTable(const Anope::string &table, const D if (known_cols.empty()) { - Anope::string query_text = "CREATE TABLE `" + table + "` (`id` int(10) unsigned NOT NULL AUTO_INCREMENT," + Anope::string query_text = "CREATE TABLE `" + table + "` (`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE," " `timestamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"; for (const auto &[column, _] : data.data) { known_cols.insert(column); - query_text += ", `" + column + "` "; - if (data.GetType(column) == Serialize::Data::DT_INT) - query_text += "int"; - else - query_text += "text"; + query_text += ", `" + column + "` " + GetColumnType(data.GetType(column)); } query_text += ", PRIMARY KEY (`id`), KEY `timestamp_idx` (`timestamp`)) ROW_FORMAT=DYNAMIC"; queries.push_back(query_text); @@ -434,11 +453,7 @@ std::vector<Query> MySQLService::CreateTable(const Anope::string &table, const D known_cols.insert(column); - Anope::string query_text = "ALTER TABLE `" + table + "` ADD `" + column + "` "; - if (data.GetType(column) == Serialize::Data::DT_INT) - query_text += "int"; - else - query_text += "text"; + Anope::string query_text = "ALTER TABLE `" + table + "` ADD `" + column + "` " + GetColumnType(data.GetType(column)); queries.push_back(query_text); } @@ -474,11 +489,29 @@ Query MySQLService::BuildInsert(const Anope::string &table, unsigned int id, Dat Anope::string buf; *value >> buf; - bool escape = true; - if (buf.empty()) + auto escape = true; + switch (data.GetType(field)) { - buf = "NULL"; - escape = false; + case Serialize::DataType::BOOL: + case Serialize::DataType::FLOAT: + case Serialize::DataType::INT: + case Serialize::DataType::UINT: + { + if (buf.empty()) + buf = "0"; + escape = false; + break; + } + + case Serialize::DataType::TEXT: + { + if (buf.empty()) + { + buf = "NULL"; + escape = false; + } + break; + } } query.SetValue(field, buf, escape); diff --git a/modules/extra/sqlite.cpp b/modules/extra/sqlite.cpp index 2454c838c..2c6ccc6db 100644 --- a/modules/extra/sqlite.cpp +++ b/modules/extra/sqlite.cpp @@ -68,6 +68,31 @@ public: Anope::string BuildQuery(const Query &q); Anope::string FromUnixtime(time_t) override; + + Anope::string GetColumnType(Serialize::DataType dt) + { + switch (dt) + { + case Serialize::DataType::BOOL: + return "INTEGER"; + + case Serialize::DataType::FLOAT: + return "REAL"; + + case Serialize::DataType::INT: + return "INTEGER"; + + case Serialize::DataType::TEXT: + return "TEXT"; + + // SQLite lacks support for 64-bit unsigned integers so we have to + // store them as text columns instead. + case Serialize::DataType::UINT: + return "TEXT"; + } + + return "TEXT"; // Should never be reached + } }; class ModuleSQLite final @@ -237,11 +262,7 @@ std::vector<Query> SQLiteService::CreateTable(const Anope::string &table, const { known_cols.insert(column); - query_text += ", `" + column + "` "; - if (data.GetType(column) == Serialize::Data::DT_INT) - query_text += "int(11)"; - else - query_text += "text"; + query_text += ", `" + column + "` " + GetColumnType(data.GetType(column)); } query_text += ")"; @@ -266,11 +287,7 @@ std::vector<Query> SQLiteService::CreateTable(const Anope::string &table, const known_cols.insert(column); - Anope::string query_text = "ALTER TABLE `" + table + "` ADD `" + column + "` "; - if (data.GetType(column) == Serialize::Data::DT_INT) - query_text += "int(11)"; - else - query_text += "text"; + Anope::string query_text = "ALTER TABLE `" + table + "` ADD `" + column + "` " + GetColumnType(data.GetType(column));; queries.push_back(query_text); } @@ -307,7 +324,33 @@ Query SQLiteService::BuildInsert(const Anope::string &table, unsigned int id, Da { Anope::string buf; *value >> buf; - query.SetValue(field, buf); + + auto escape = true; + switch (data.GetType(field)) + { + case Serialize::DataType::BOOL: + case Serialize::DataType::FLOAT: + case Serialize::DataType::INT: + { + if (buf.empty()) + buf = "0"; + escape = false; + break; + } + + case Serialize::DataType::TEXT: + case Serialize::DataType::UINT: + { + if (buf.empty()) + { + buf = "NULL"; + escape = false; + } + break; + } + } + + query.SetValue(field, buf, escape); } return query; diff --git a/modules/hostserv/hs_request.cpp b/modules/hostserv/hs_request.cpp index c11a1ad90..e6f30fd70 100644 --- a/modules/hostserv/hs_request.cpp +++ b/modules/hostserv/hs_request.cpp @@ -32,10 +32,10 @@ struct HostRequestImpl final void Serialize(Serialize::Data &data) const override { - data["nick"] << this->nick; - data["ident"] << this->ident; - data["host"] << this->host; - data.SetType("time", Serialize::Data::DT_INT); data["time"] << this->time; + data.Store("nick", this->nick); + data.Store("ident", this->ident); + data.Store("host", this->host); + data.Store("time", this->time); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/modules/nickserv/ns_ajoin.cpp b/modules/nickserv/ns_ajoin.cpp index 91e700e4a..5e4419b85 100644 --- a/modules/nickserv/ns_ajoin.cpp +++ b/modules/nickserv/ns_ajoin.cpp @@ -40,14 +40,14 @@ struct AJoinEntry final } } - void Serialize(Serialize::Data &sd) const override + void Serialize(Serialize::Data &data) const override { if (!this->owner) return; - sd["owner"] << this->owner->display; - sd["channel"] << this->channel; - sd["key"] << this->key; + data.Store("owner", this->owner->display); + data.Store("channel", this->channel); + data.Store("key", this->key); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &sd) diff --git a/modules/nickserv/ns_cert.cpp b/modules/nickserv/ns_cert.cpp index a5567b8e8..5594b2e57 100644 --- a/modules/nickserv/ns_cert.cpp +++ b/modules/nickserv/ns_cert.cpp @@ -168,8 +168,10 @@ public: if (c == NULL || !c->GetCertCount()) return; + std::ostringstream oss; for (unsigned i = 0; i < c->GetCertCount(); ++i) - data["cert"] << c->GetCert(i) << " "; + oss << c->GetCert(i) << " "; + data.Store("cert", oss.str()); } void ExtensibleUnserialize(Extensible *e, Serializable *s, Serialize::Data &data) override diff --git a/modules/nickserv/ns_set.cpp b/modules/nickserv/ns_set.cpp index 5bd3a4d3d..b4dbe0a44 100644 --- a/modules/nickserv/ns_set.cpp +++ b/modules/nickserv/ns_set.cpp @@ -1108,7 +1108,7 @@ class NSSet final if (!last_value.empty()) modes += "," + last_value; } - data["last_modes"] << modes; + data.Store("last_modes", modes); } void ExtensibleUnserialize(Extensible *e, Serializable *s, Serialize::Data &data) override diff --git a/modules/nickserv/ns_set_misc.cpp b/modules/nickserv/ns_set_misc.cpp index 6f8ad0f4e..87b50ebc4 100644 --- a/modules/nickserv/ns_set_misc.cpp +++ b/modules/nickserv/ns_set_misc.cpp @@ -46,9 +46,9 @@ struct NSMiscData final void Serialize(Serialize::Data &sdata) const override { - sdata["nc"] << this->object; - sdata["name"] << this->name; - sdata["data"] << this->data; + sdata.Store("nc", this->object); + sdata.Store("name", this->name); + sdata.Store("data", this->data); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data) diff --git a/modules/nickserv/ns_suspend.cpp b/modules/nickserv/ns_suspend.cpp index cfe70861e..79c7b8a08 100644 --- a/modules/nickserv/ns_suspend.cpp +++ b/modules/nickserv/ns_suspend.cpp @@ -22,11 +22,11 @@ struct NSSuspendInfo final void Serialize(Serialize::Data &data) const override { - data["nick"] << what; - data["by"] << by; - data["reason"] << reason; - data["time"] << when; - data["expires"] << expires; + data.Store("nick", what); + data.Store("by", by); + data.Store("reason", reason); + data.Store("time", when); + data.Store("expires", expires); } static Serializable *Unserialize(Serializable *obj, Serialize::Data &data) |
