Address casemapping issues in sqlite/mysql

16 files changed, 434 insertions, 29 deletions

diff --git a/data/anope.example.conf b/data/anope.example.conf
index 9a0384e6f..571f84182 100644
--- a/data/anope.example.conf
+++ b/data/anope.example.conf
@@ -1108,8 +1108,8 @@ module
 	name = "database/sql"
 
 	/*
-	 * The SQL service database/sql should use, these are configured in modules.conf.
-	 * For MySQL, this should probably be mysql/main.
+	 * The SQL service database/sql should use, these are configured in modules.conf,
+	 * and are provided by other modules. For MySQL, this should probably be mysql/main.
 	 */
 	engine = "sqlite/main"
 
diff --git a/data/modules.example.conf b/data/modules.example.conf
index 89166e619..64fe7df5a 100644
--- a/data/modules.example.conf
+++ b/data/modules.example.conf
@@ -357,6 +357,15 @@ module { name = "help" }
  * mysql [EXTRA]
  *
  * This module allows other modules to use MySQL.
+ *
+ * If you use db_sql with a mysql service defined below, note that your default
+ * database collation should be the same as your configured casemap or locale
+ * in Anope.
+ *
+ * For ascii, this is ascii_general_ci.
+ *
+ * For rfc1459, a charset configuration file is provided in data/mysql/rfc1459.xml
+ * Instructions are provided in docs/MySQL.md on how to install it.
  */
 #module
 {
diff --git a/data/mysql/rfc1459.xml b/data/mysql/rfc1459.xml
new file mode 100644
index 000000000..e5c81fd0f
--- /dev/null
+++ b/data/mysql/rfc1459.xml
@@ -0,0 +1,158 @@
+<?xml version='1.0' encoding="utf-8"?>
+
+<charsets>
+
+<copyright>
+  Copyright (C) 2016 Anope Team &lt;team@anope.org&gt;
+
+  This file is part of Anope. Anope is free software; you can
+  redistribute it and/or modify it under the terms of the GNU
+  General Public License as published by the Free Software
+  Foundation, version 2.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, see see &lt;http://www.gnu.org/licenses/&gt;.
+</copyright>
+
+<charset name="rfc1459">
+
+<ctype>
+<map>
+ 00
+ 20 20 20 20 20 20 20 20 20 28 28 28 28 28 20 20
+ 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20
+ 48 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10
+ 84 84 84 84 84 84 84 84 84 84 10 10 10 10 10 10
+ 10 81 81 81 81 81 81 01 01 01 01 01 01 01 01 01
+ 01 01 01 01 01 01 01 01 01 01 01 10 10 10 10 10
+ 10 82 82 82 82 82 82 02 02 02 02 02 02 02 02 02
+ 02 02 02 02 02 02 02 02 02 02 02 10 10 10 10 20
+ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+</map>
+</ctype>
+
+
+<lower>
+<map>
+ 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F
+ 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F
+ 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 2F
+ 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F
+ 40 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F
+ 70 71 72 73 74 75 76 77 78 79 7A 5B 5C 5D 5E 5F
+ 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F
+ 70 71 72 73 74 75 76 77 78 79 7A 5B 5C 5D 5E 7F
+ 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F
+ 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F
+ A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF
+ B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF
+ C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF
+ D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF
+ E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF
+ F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF
+</map>
+</lower>
+
+
+<upper>
+<map>
+ 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F
+ 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F
+ 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 2F
+ 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F
+ 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F
+ 50 51 52 53 54 55 56 57 58 59 5A 7B 7C 7D 7E 5F
+ 60 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F
+ 50 51 52 53 54 55 56 57 58 59 5A 7B 7C 7D 7E 7F
+ 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F
+ 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F
+ A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF
+ B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF
+ C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF
+ D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF
+ E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF
+ F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF
+</map>
+</upper>
+
+
+<unicode>
+<map>
+0000 0001 0002 0003 0004 0005 0006 0007 0008 0009 000A 000B 000C 000D 000E 000F
+0010 0011 0012 0013 0014 0015 0016 0017 0018 0019 001A 001B 001C 001D 001E 001F
+0020 0021 0022 0023 0024 0025 0026 0027 0028 0029 002A 002B 002C 002D 002E 002F
+0030 0031 0032 0033 0034 0035 0036 0037 0038 0039 003A 003B 003C 003D 003E 003F
+0040 0041 0042 0043 0044 0045 0046 0047 0048 0049 004A 004B 004C 004D 004E 004F
+0050 0051 0052 0053 0054 0055 0056 0057 0058 0059 005A 005B 005C 005D 005E 005F
+0060 0061 0062 0063 0064 0065 0066 0067 0068 0069 006A 006B 006C 006D 006E 006F
+0070 0071 0072 0073 0074 0075 0076 0077 0078 0079 007A 007B 007C 007D 007E 007F
+0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
+0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
+0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
+0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
+0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
+0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
+0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
+0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
+</map>
+</unicode>
+
+
+<collation name="rfc1459_ci">
+<map>
+ 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F
+ 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F
+ 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 2F
+ 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F
+ 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F
+ 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F
+ 60 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F
+ 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 7F
+ 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F
+ 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F
+ A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF
+ B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF
+ C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF
+ D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF
+ E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF
+ F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF
+</map>
+</collation>
+
+<!-- like rfc1459_ci, but '~' != '^' -->
+<collation name="rfc1459_inspircd_ci">
+<map>
+ 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F
+ 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F
+ 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 2F
+ 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F
+ 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F
+ 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F
+ 60 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F
+ 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 7E 7F
+ 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F
+ 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F
+ A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF
+ B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF
+ C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF
+ D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF
+ E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF
+ F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF
+</map>
+</collation>
+
+</charset>
+
+</charsets>
diff --git a/data/rfc1459.conf b/data/rfc1459.conf
index 81ced1ac0..d1fd7fd40 100644
--- a/data/rfc1459.conf
+++ b/data/rfc1459.conf
@@ -22,3 +22,10 @@ casemap
 	upper = "\""
 }
 
+/* InspIRCd doesn't use this, but many other IRCds do. */
+#casemap
+{
+	lower = "^"
+	upper = "~"
+}
+
diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt
index fd93e81c0..7e135d207 100644
--- a/docs/CMakeLists.txt
+++ b/docs/CMakeLists.txt
@@ -7,7 +7,7 @@ if(WIN32)
     # Add README.txt to list of files for CPack to ignore
     add_to_cpack_ignored_files("README.txt$" TRUE)
   endif()
-  set(DOCS Changes Changes.conf DEFCON FAQ INSTALL LANGUAGE MODULES NEWS ${CMAKE_CURRENT_BINARY_DIR}/README.txt WIN32.txt)
+  set(DOCS Changes Changes.conf DEFCON FAQ INSTALL LANGUAGE MODULES MySQL.md NEWS ${CMAKE_CURRENT_BINARY_DIR}/README.txt SQLite.md WIN32.txt)
   install(FILES ${DOCS}
     DESTINATION ${DOC_DIR}
   )
diff --git a/docs/MySQL.md b/docs/MySQL.md
new file mode 100644
index 000000000..c463c1f00
--- /dev/null
+++ b/docs/MySQL.md
@@ -0,0 +1,24 @@
+# How to install rfc1459 character set into MySQL:
+
+Copy data/mysql/rfc1459.xml to your character_sets_dir, identified by
+
+```
+SHOW VARIABLES LIKE 'character_sets_dir';
+```
+
+Add the following charset to Index.xml in the character_sets_dir directory:
+
+```
+<charset name="rfc1459">
+  <collation name="rfc1459_ci" id="800"/>
+  <collation name="rfc1459_inspircd_ci" id="801" flag="primary"/>
+</charset>
+```
+
+Restart MySQL.
+
+Now create the database using the appropriate character set:
+
+```
+CREATE DATABASE anope DEFAULT CHARACTER SET rfc1459 DEFAULT COLLATE rfc1459_inspircd_ci;
+```
diff --git a/docs/SQLite.md b/docs/SQLite.md
new file mode 100644
index 000000000..2d5b1c7b8
--- /dev/null
+++ b/docs/SQLite.md
@@ -0,0 +1,32 @@
+# How to modify Anope sqlite databases
+
+Anope uses a custom SQLiite function (anope_canonicalize) for canonicalizing strings for case insensitivity comparisons.
+It does this by using SQLites support for having indexes on expressions https://www.sqlite.org/expridx.html.
+
+For example the account table could look like:
+
+```
+CREATE TABLE `anope_db_account` (
+	`display`,
+	`pass`,
+	`email`,
+	`language`,
+	`id` NOT NULL PRIMARY KEY,
+	FOREIGN KEY (id) REFERENCES anope_db_objects(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
+);
+CREATE INDEX idx_display ON `anope_db_account` (anope_canonicalize(display));
+```
+
+So, to do a SELECT which utilizes the indicies, Anope does something like:
+
+```
+SELECT id FROM `anope_db_account` WHERE anope_canonicalize(display) = anope_canonicalize('Adam');
+```
+
+If you are using your own SQLite instance, like the sqlite command line interface, the anope_canonicalize function
+will be missing. To fix this load one of libanope_sqlite_ascii.so, libanope_sqlite_rfc1459_inspircd.so,
+or libanope_sqlite_rfc1459.so into SQLite, depending on your casemap configuration.
+
+```
+sqlite> .load lib/libanope_sqlite_ascii.so
+```
diff --git a/include/anope.h b/include/anope.h
index 49e7c7ddc..dbaebd573 100644
--- a/include/anope.h
+++ b/include/anope.h
@@ -345,6 +345,8 @@ namespace Anope
 		bool operator()(const string &s1, const string &s2) const;
 	};
 
+	extern Anope::string transform(const Anope::string &);
+
 	template<typename T> using map = std::map<string, T, ci::less>;
 	template<typename T> using multimap = std::multimap<string, T, ci::less>;
 	template<typename T> using hash_map = std::unordered_map<string, T, hash_ci, compare_ci>;
diff --git a/include/hashcomp.h b/include/hashcomp.h
index d246bfd2b..c9fa77f61 100644
--- a/include/hashcomp.h
+++ b/include/hashcomp.h
@@ -160,6 +160,7 @@ namespace locale {
 extern std::locale generate(const std::string &);
 extern int compare(const std::string &, const std::string &);
 extern long hash(const std::string &);
+extern std::string transform(const std::string &);
 
 #endif
 
diff --git a/include/modules/sql.h b/include/modules/sql.h
index 6770f407e..263b99fd1 100644
--- a/include/modules/sql.h
+++ b/include/modules/sql.h
@@ -205,6 +205,7 @@ namespace SQL
 		virtual std::vector<Query> CreateTable(const Anope::string &prefix, Serialize::TypeBase *) anope_abstract;
 		virtual std::vector<Query> AlterTable(const Anope::string &, Serialize::TypeBase *, Serialize::FieldBase *) anope_abstract;
 		virtual std::vector<Query> CreateIndex(const Anope::string &table, const Anope::string &field) anope_abstract;
+		virtual Query SelectFind(const Anope::string &table, const Anope::string &field) anope_abstract;
 
 		virtual Query BeginTransaction() anope_abstract;
 		virtual Query Commit() anope_abstract;
diff --git a/modules/database/sql.cpp b/modules/database/sql.cpp
index aa1c94bc0..f88fc35d7 100644
--- a/modules/database/sql.cpp
+++ b/modules/database/sql.cpp
@@ -136,7 +136,8 @@ class DBSQL : public Module, public Pipe
 		for (const Query &q : SQL->CreateIndex(prefix + type->GetName(), field->serialize_name))
 			Run(q);
 
-		Query query("SELECT `id` FROM `" + prefix + type->GetName() + "` WHERE `" + field->serialize_name + "` = @value@");
+		Query query = SQL->SelectFind(prefix + type->GetName(), field->serialize_name);
+
 		query.SetValue("value", value);
 		Result res = Run(query);
 		if (res.Rows())
diff --git a/modules/extra/mysql.cpp b/modules/extra/mysql.cpp
index b5ff0c3d9..389124073 100644
--- a/modules/extra/mysql.cpp
+++ b/modules/extra/mysql.cpp
@@ -159,6 +159,7 @@ class MySQLService : public Provider
 	std::vector<Query> CreateTable(const Anope::string &prefix, Serialize::TypeBase *) override;
 	std::vector<Query> AlterTable(const Anope::string &, Serialize::TypeBase *, Serialize::FieldBase *) override;
 	std::vector<Query> CreateIndex(const Anope::string &table, const Anope::string &field) override;
+	Query SelectFind(const Anope::string &table, const Anope::string &field) override;
 
 	Query BeginTransaction() override;
 	Query Commit() override;
@@ -490,6 +491,11 @@ std::vector<Query> MySQLService::CreateIndex(const Anope::string &table, const A
 	return queries;
 }
 
+Query MySQLService::SelectFind(const Anope::string &table, const Anope::string &field)
+{
+	return Query("SELECT `id` FROM `" + table + "` WHERE `" + field + "` = @value@");
+}
+
 Query MySQLService::BeginTransaction()
 {
 	return Query("START TRANSACTION WITH CONSISTENT SNAPSHOT");
diff --git a/modules/extra/sqlite.cpp b/modules/extra/sqlite.cpp
index b9966af5f..881fe0b4a 100644
--- a/modules/extra/sqlite.cpp
+++ b/modules/extra/sqlite.cpp
@@ -94,6 +94,7 @@ class SQLiteService : public Provider
 	std::vector<Query> CreateTable(const Anope::string &, Serialize::TypeBase *) override;
 	std::vector<Query> AlterTable(const Anope::string &, Serialize::TypeBase *, Serialize::FieldBase *) override;
 	std::vector<Query> CreateIndex(const Anope::string &table, const Anope::string &field) override;
+	Query SelectFind(const Anope::string &table, const Anope::string &field) override;
 
 	Query BeginTransaction() override;
 	Query Commit() override;
@@ -103,6 +104,8 @@ class SQLiteService : public Provider
 	Query GetTables(const Anope::string &prefix);
 
 	Anope::string BuildQuery(const Query &q);
+
+	static void Canonicalize(sqlite3_context *context, int argc, sqlite3_value **argv);
 };
 
 class ModuleSQLite : public Module
@@ -186,6 +189,10 @@ SQLiteService::SQLiteService(Module *o, const Anope::string &n, const Anope::str
 		}
 		throw SQL::Exception(exstr);
 	}
+
+	int ret = sqlite3_create_function_v2(this->sql, "anope_canonicalize", 1, SQLITE_DETERMINISTIC, NULL, Canonicalize, NULL, NULL, NULL);
+	if (ret != SQLITE_OK)
+		Log(LOG_DEBUG) << "Unable to add anope_canonicalize function: " << sqlite3_errmsg(this->sql);
 }
 
 SQLiteService::~SQLiteService()
@@ -293,7 +300,7 @@ std::vector<Query> SQLiteService::CreateTable(const Anope::string &prefix, Seria
 
 		for (Serialize::FieldBase *field : base->GetFields())
 		{
-			query += "`" + field->serialize_name + "` COLLATE NOCASE";
+			query += "`" + field->serialize_name + "`";
 			fields.insert(field->serialize_name);
 
 			if (field->object)
@@ -333,7 +340,7 @@ std::vector<Query> SQLiteService::AlterTable(const Anope::string &prefix, Serial
 
 	if (!s.count(field->serialize_name))
 	{
-		Anope::string buf = "ALTER TABLE `" + prefix + table + "` ADD `" + field->serialize_name + "` COLLATE NOCASE";
+		Anope::string buf = "ALTER TABLE `" + prefix + table + "` ADD `" + field->serialize_name + "`";
 
 		if (field->object)
 		{
@@ -365,7 +372,7 @@ std::vector<Query> SQLiteService::CreateIndex(const Anope::string &table, const
 	if (indexes[table].count(field))
 		return queries;
 
-	Query t = "CREATE INDEX IF NOT EXISTS idx_" + field + " ON `" + table + "` (" + field + ")";
+	Query t = "CREATE INDEX IF NOT EXISTS idx_" + field + " ON `" + table + "` (anope_canonicalize(" + field + "))";
 	queries.push_back(t);
 
 	indexes[table].insert(field);
@@ -373,6 +380,11 @@ std::vector<Query> SQLiteService::CreateIndex(const Anope::string &table, const
 	return queries;
 }
 
+Query SQLiteService::SelectFind(const Anope::string &table, const Anope::string &field)
+{
+	return Query("SELECT `id` FROM `" + table + "` WHERE anope_canonicalize(`" + field + "`) = anope_canonicalize(@value@)");
+}
+
 Query SQLiteService::BeginTransaction()
 {
 	return Query("BEGIN TRANSACTION");
@@ -436,5 +448,18 @@ Anope::string SQLiteService::BuildQuery(const Query &q)
 	return real_query;
 }
 
+void SQLiteService::Canonicalize(sqlite3_context *context, int argc, sqlite3_value **argv)
+{
+	sqlite3_value *arg = argv[0];
+	const char *text = reinterpret_cast<const char *>(sqlite3_value_text(arg));
+
+	if (text == nullptr)
+		return;
+
+	const Anope::string &result = Anope::transform(text);
+
+	sqlite3_result_text(context, result.c_str(), -1, SQLITE_TRANSIENT);
+}
+
 MODULE_INIT(ModuleSQLite)
 
diff --git a/src/hashcomp.cpp b/src/hashcomp.cpp
index 265cb362b..72e729eba 100644
--- a/src/hashcomp.cpp
+++ b/src/hashcomp.cpp
@@ -212,6 +212,18 @@ bool Anope::compare_locale::operator()(const Anope::string &s1, const Anope::str
 	return Anope::compare_ci()(s1, s2);
 }
 
+Anope::string Anope::transform(const Anope::string &s)
+{
+#ifdef Boost_FOUND
+	if (Config != nullptr && Config->locale != nullptr)
+	{
+		return Anope::locale::transform(s.str());
+	}
+#endif
+
+	return s.lower();
+}
+
 #ifdef Boost_FOUND
 
 std::locale Anope::locale::generate(const std::string &name)
@@ -240,5 +252,11 @@ long Anope::locale::hash(const std::string &s)
 	return ct.hash(boost::locale::collator_base::secondary, s.data(), s.data() + s.length());
 }
 
+std::string Anope::locale::transform(const std::string &s)
+{
+	const boost::locale::collator<char> &ct = std::use_facet<boost::locale::collator<char>>(*Config->locale);
+	return ct.transform(boost::locale::collator_base::secondary, s);
+}
+
 #endif
 
diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
index f302c3ef7..ee445a00e 100644
--- a/src/tools/CMakeLists.txt
+++ b/src/tools/CMakeLists.txt
@@ -1,27 +1,16 @@
-# Find all the *.cpp files within the current source directory, and sort the list
-file(GLOB TOOLS_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cpp")
-list(SORT TOOLS_SRCS)
 
-# Set all the files to use C++ as well as set their compile flags
-set_source_files_properties(${TOOLS_SRCS} PROPERTIES LANGUAGE CXX COMPILE_FLAGS "${CXXFLAGS}")
-
-# Iterate through all the source files
-foreach(SRC ${TOOLS_SRCS})
-  # Convert the source file extension to have no extension
-  string(REGEX REPLACE "\\.cpp$" "" EXE ${SRC})
-  # Generate the executable and set its linker flags, also set it to depend on the main Anope executable to be built beforehand
-  add_executable(${EXE} ${SRC})
-  set_target_properties(${EXE} PROPERTIES LINKER_LANGUAGE CXX LINK_FLAGS "${LDFLAGS}")
-  add_dependencies(${EXE} ${PROGRAM_NAME})
-  # Set the executable to be installed to the bin directory under the main directory
-  install(TARGETS ${EXE}
-    DESTINATION ${BIN_DIR}
+function(build_sqlite LIBNAME)
+  add_library(tools_anope_sqlite_${LIBNAME} SHARED anope_sqlite.cpp)
+  set_source_files_properties(anope_sqlite.cpp PROPERTIES LANGUAGE CXX COMPILE_FLAGS "${CXXFLAGS}")
+  set_target_properties(tools_anope_sqlite_${LIBNAME} PROPERTIES LINKER_LANGUAGE CXX LINK_FLAGS "${LDFLAGS}" OUTPUT_NAME "anope_sqlite_${LIBNAME}")
+  install(TARGETS tools_anope_sqlite_${LIBNAME}
+    DESTINATION ${LIB_DIR}
   )
-  # Add the executable to the list of files for CPack to ignore
-  get_target_property(EXE_BINARY ${EXE} LOCATION)
-  get_filename_component(EXE_BINARY ${EXE_BINARY} NAME)
-  add_to_cpack_ignored_files("${EXE_BINARY}$" TRUE)
-endforeach()
+endfunction()
+
+build_sqlite(ascii)
+build_sqlite(rfc1459)
+build_sqlite(rfc1459_inspircd)
 
 # If not on Windows, generate anoperc and install it along with mydbgen
 if(NOT WIN32)
diff --git a/src/tools/anope_sqlite.cpp b/src/tools/anope_sqlite.cpp
new file mode 100644
index 000000000..e03f99bd2
--- /dev/null
+++ b/src/tools/anope_sqlite.cpp
@@ -0,0 +1,132 @@
+/*
+ * Anope IRC Services
+ *
+ * Copyright (C) 2016 Anope Team <team@anope.org>
+ *
+ * This file is part of Anope. Anope is free software; you can
+ * redistribute it and/or modify it under the terms of the GNU
+ * General Public License as published by the Free Software
+ * Foundation, version 2.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see see <http://www.gnu.org/licenses/>.
+ */
+
+#include <sqlite3ext.h>
+
+#include <algorithm>
+#include <string>
+
+#ifdef _WIN32
+# define DllExport __declspec(dllexport)
+#else
+# define DllExport
+#endif
+
+SQLITE_EXTENSION_INIT1
+
+namespace
+{
+
+enum collation
+{
+	ANOPE_ASCII,
+	ANOPE_RFC1459,
+	ANOPE_RFC1459_INSPIRCD
+};
+
+char (*anope_tolower)(char) = nullptr;
+
+char tolower_ascii(char c)
+{
+	if (c >= 'A' && c <= 'Z')
+		return c + 0x20;
+	else
+		return c;
+}
+
+char tolower_rfc1459_inspircd(char c)
+{
+	if (c == '[' || c == ']' || c == '\\')
+		return c + 0x20;
+	else
+		return tolower_ascii(c);
+}
+
+char tolower_rfc1459(char c)
+{
+	if (c == '[' || c == ']' || c == '\\' || c == '^')
+		return c + 0x20;
+	else
+		return tolower_ascii(c);
+}
+
+void anope_canonicalize(sqlite3_context *context, int argc, sqlite3_value **argv)
+{
+	sqlite3_value *arg = argv[0];
+	const unsigned char *text = sqlite3_value_text(arg);
+
+	if (text == nullptr || anope_tolower == nullptr)
+		return;
+
+	std::string result(reinterpret_cast<const char *>(text));
+	std::transform(result.begin(), result.end(), result.begin(), anope_tolower);
+
+	sqlite3_result_text(context, result.c_str(), -1, SQLITE_TRANSIENT);
+}
+
+int sqlite_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi, collation col)
+{
+	SQLITE_EXTENSION_INIT2(pApi);
+
+	switch (col)
+	{
+		case ANOPE_ASCII:
+			anope_tolower = tolower_ascii;
+			break;
+		case ANOPE_RFC1459:
+			anope_tolower = tolower_rfc1459;
+			break;
+		case ANOPE_RFC1459_INSPIRCD:
+			anope_tolower = tolower_rfc1459_inspircd;
+			break;
+		default:
+			return SQLITE_INTERNAL;
+	}
+
+	int rc = sqlite3_create_function_v2(db, "anope_canonicalize", 1, SQLITE_DETERMINISTIC, NULL, anope_canonicalize, NULL, NULL, NULL);
+
+	return rc;
+}
+
+} // namespace
+
+extern "C" DllExport
+int sqlite3_anopesqliteascii_init(sqlite3 *, char **, const sqlite3_api_routines *);
+
+extern "C" DllExport
+int sqlite3_anopesqliterfc1459_init(sqlite3 *, char **, const sqlite3_api_routines *);
+
+extern "C" DllExport
+int sqlite3_anopesqliterfc1459inspircd_init(sqlite3 *, char **, const sqlite3_api_routines *);
+
+int sqlite3_anopesqliteascii_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi)
+{
+	return sqlite_init(db, pzErrMsg, pApi, ANOPE_ASCII);
+}
+
+int sqlite3_anopesqliterfc1459_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi)
+{
+	return sqlite_init(db, pzErrMsg, pApi, ANOPE_RFC1459);
+}
+
+int sqlite3_anopesqliterfc1459inspircd_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi)
+{
+	return sqlite_init(db, pzErrMsg, pApi, ANOPE_RFC1459_INSPIRCD);
+}
+