Optionally allow using Boost.Locale for hashcomp

14 files changed, 189 insertions, 47 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 890e15969..1705ac733 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -54,6 +54,15 @@ endif()
 
 # Find gettext
 find_package(Gettext)
+# FindBoost.cmake warns even if you specify QUIET, so
+find_path(BOOST_LOCALE_HEADER_FILE NAMES boost/locale.hpp)
+if(BOOST_LOCALE_HEADER_FILE)
+  find_package(Boost COMPONENTS locale)
+endif()
+
+if(Boost_FOUND)
+  include_directories(${Boost_INCLUDE_DIRS})
+endif()
 
 # Use the following directories as includes
 include_directories(${Anope_BINARY_DIR}/include ${Anope_SOURCE_DIR}/include)
diff --git a/data/anope.example.conf b/data/anope.example.conf
index e75b01e74..7724c8125 100644
--- a/data/anope.example.conf
+++ b/data/anope.example.conf
@@ -357,6 +357,31 @@ options
 	#group = "anope"
 
 	/*
+	 * Chooses the configuration file used to configure casemaps for Anope.
+	 * Anope uses this case map to compare, with case insensitivity,
+	 * nick names and channel names.
+	 *
+	 * Two casemaps shipped with Anope are ascii and rfc1459.
+	 *
+	 * You may create your own instead by defining casemapping directives
+	 * similar to how ascii.conf and rfc1459.conf do. However, they are
+	 * limited to single byte characters only.
+	 *
+	 * Alternatively, if you have Boost.Locale installed, instead of configuring
+	 * a casemapping configuration file, you may configure a locale name. This
+	 * can support variable-length character encodings like utf-8.
+	 *
+	 * The casemapping you use should be set to what your IRCd uses, which is
+	 * probably rfc1459. However, Anope has always used ascii for comparison,
+	 * so the default is ascii.
+	 *
+	 * Changing this value once set is not recommended.
+	 */
+	casemap = "ascii"
+	#casemap = "rfc1459"
+	#locale = "utf-8"
+
+	/*
 	 * This key is used to initiate the random number generator. This number
 	 * MUST be random as you want your passcodes to be random. Don't give this
 	 * key to anyone! Keep it private!
@@ -520,26 +545,6 @@ options
 }
 
 /*
- * [REQUIRED] Casemapping configuration
- *
- * Includes the configuration for the casemap services should use.
- * Anope uses this case map to compare, with case insensitivity,
- * things such as nick names, channel names, etc.
- *
- * Two casemaps shipped with Anope are ascii and rfc1459.
- *
- * This value should be set to what your IRCd uses, which is probably rfc1459,
- * however Anope has always used ascii for comparison, so the default is ascii.
- *
- * Changing this value once set is not recommended.
- */
-include
-{
-	name = "ascii.conf"
-	#name = "rfc1459.conf"
-}
-
-/*
  * [OPTIONAL] BotServ
  *
  * Includes botserv.example.conf, which is necessary for BotServ functionality.
diff --git a/include/anope.h b/include/anope.h
index d5e935891..c298d5d86 100644
--- a/include/anope.h
+++ b/include/anope.h
@@ -315,33 +315,40 @@ namespace Anope
 	inline const string operator+(const char *_str, const string &str) { string tmp(_str); tmp += str; return tmp; }
 	inline const string operator+(const std::string &_str, const string &str) { string tmp(_str); tmp += str; return tmp; }
 
+	struct hash
+	{
+		size_t operator()(const string &s) const;
+	};
+
+	struct compare
+	{
+		bool operator()(const string &s1, const string &s2) const;
+	};
+
 	struct hash_ci
 	{
-		inline size_t operator()(const string &s) const
-		{
-			return std::hash<std::string>()(s.lower().str());
-		}
+		size_t operator()(const string &s) const;
 	};
 
-	struct hash_cs
+	struct compare_ci
 	{
-		inline size_t operator()(const string &s) const
-		{
-			return std::hash<std::string>()(s.str());
-		}
+		bool operator()(const string &s1, const string &s2) const;
 	};
 
-	struct compare
+	struct hash_locale
 	{
-		inline bool operator()(const string &s1, const string &s2) const
-		{
-			return s1.equals_ci(s2);
-		}
+		size_t operator()(const string &s) const;
+	};
+
+	struct compare_locale
+	{
+		bool operator()(const string &s1, const string &s2) const;
 	};
 
-	template<typename T> class map : public std::map<string, T, ci::less> { };
-	template<typename T> class multimap : public std::multimap<string, T, ci::less> { };
-	template<typename T> class hash_map : public std::unordered_map<string, T, hash_ci, compare> { };
+	template<typename T> using map = std::map<string, T, ci::less>;
+	template<typename T> using multimap = std::multimap<string, T, ci::less>;
+	template<typename T> using hash_map = std::unordered_map<string, T, hash_ci, compare_ci>;
+	template<typename T> using locale_hash_map = std::unordered_map<string, T, hash_locale, compare_locale>;
 
 	static const char *const compiled = __TIME__ " " __DATE__;
 
diff --git a/include/channels.h b/include/channels.h
index bfe2fc0d2..5888f84e3 100644
--- a/include/channels.h
+++ b/include/channels.h
@@ -24,7 +24,7 @@
 #include "modes.h"
 #include "serialize.h"
 
-typedef Anope::hash_map<Channel *> channel_map;
+using channel_map = Anope::locale_hash_map<Channel *>;
 
 extern CoreExport channel_map ChannelList;
 
diff --git a/include/config.h b/include/config.h
index 6d73e03ad..7ca8caaf4 100644
--- a/include/config.h
+++ b/include/config.h
@@ -142,6 +142,7 @@ namespace Configuration
 		std::vector<Usermode> Usermodes;
 		std::vector<Channelmode> Channelmodes;
 		unsigned char CaseMapUpper[256] = { 0 }, CaseMapLower[256] = { 0 };
+		std::locale *locale = nullptr;
 
 		/* module configuration blocks */
 		std::map<Anope::string, Block *> modules;
diff --git a/include/hashcomp.h b/include/hashcomp.h
index 470c054fd..d246bfd2b 100644
--- a/include/hashcomp.h
+++ b/include/hashcomp.h
@@ -152,3 +152,17 @@ inline bool operator!=(const std::string &leftval, const ci::string &rightval)
 	return !(leftval.c_str() == rightval);
 }
 
+namespace Anope {
+namespace locale {
+
+#ifdef Boost_FOUND
+
+extern std::locale generate(const std::string &);
+extern int compare(const std::string &, const std::string &);
+extern long hash(const std::string &);
+
+#endif
+
+}
+}
+
diff --git a/include/modules/chanserv.h b/include/modules/chanserv.h
index 8029782bb..8c5687cc2 100644
--- a/include/modules/chanserv.h
+++ b/include/modules/chanserv.h
@@ -96,7 +96,7 @@ namespace ChanServ
 	};
 
 	class Channel;
-	using registered_channel_map = Anope::hash_map<Channel *>;
+	using registered_channel_map = Anope::locale_hash_map<Channel *>;
 
 	class Level : public Serialize::Object
 	{
diff --git a/include/modules/nickserv.h b/include/modules/nickserv.h
index 047c2cf18..43e2a21ce 100644
--- a/include/modules/nickserv.h
+++ b/include/modules/nickserv.h
@@ -29,8 +29,8 @@ namespace NickServ
 	class Account;
 	class IdentifyRequestListener;
 
-	typedef Anope::hash_map<Nick *> nickalias_map;
-	typedef Anope::hash_map<Account *> nickcore_map;
+	using nickalias_map = Anope::locale_hash_map<Nick *>;
+	using nickcore_map = Anope::locale_hash_map<Account *>;
 
 	class NickServService : public Service
 	{
diff --git a/include/sysconf.h.cmake b/include/sysconf.h.cmake
index 8157bc0be..2d2ede9de 100644
--- a/include/sysconf.h.cmake
+++ b/include/sysconf.h.cmake
@@ -24,4 +24,5 @@
 #cmakedefine DEFUMASK @DEFUMASK@
 #cmakedefine HAVE_UMASK 1
 #cmakedefine GETTEXT_FOUND 1
+#cmakedefine Boost_FOUND 1
 
diff --git a/include/users.h b/include/users.h
index 046f41e8e..dacb7045e 100644
--- a/include/users.h
+++ b/include/users.h
@@ -27,9 +27,11 @@
 #include "commands.h"
 #include "sockets.h"
 
-typedef Anope::hash_map<User *> user_map;
+using user_map = Anope::locale_hash_map<User *>;
+using uid_map = std::unordered_map<Anope::string, User *, Anope::hash, Anope::compare>;
 
-extern CoreExport user_map UserListByNick, UserListByUID;
+extern CoreExport user_map UserListByNick;
+extern CoreExport uid_map UserListByUID;
 
 extern CoreExport int OperCount;
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 373677fe7..007e3605a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -49,10 +49,10 @@ add_executable(${PROGRAM_NAME} ${SRC_SRCS})
 set_target_properties(${PROGRAM_NAME} PROPERTIES LINKER_LANGUAGE CXX LINK_FLAGS "${LDFLAGS} ${EXTRA_LDFLAGS}" ENABLE_EXPORTS ON INSTALL_RPATH_USE_LINK_PATH ON BUILD_WITH_INSTALL_RPATH ON)
 # On Windows, also link Anope to the wsock32 and Ws2_32 library, as well as set the version
 if(WIN32)
-  target_link_libraries(${PROGRAM_NAME} wsock32 Ws2_32 ${LINK_LIBS} ${GETTEXT_LIBRARIES} ${WIN32_MEMORY})
+  target_link_libraries(${PROGRAM_NAME} wsock32 Ws2_32 ${LINK_LIBS} ${GETTEXT_LIBRARIES} ${Boost_LIBRARIES} ${WIN32_MEMORY})
   set_target_properties(${PROGRAM_NAME} PROPERTIES VERSION "${VERSION_DOTTED}")
 else()
-  target_link_libraries(${PROGRAM_NAME} ${LINK_LIBS} ${GETTEXT_LIBRARIES})
+  target_link_libraries(${PROGRAM_NAME} ${LINK_LIBS} ${GETTEXT_LIBRARIES} ${Boost_LIBRARIES})
 endif()
 # Building the Anope executable requires the version.h header to be generated
 add_dependencies(${PROGRAM_NAME} headers)
diff --git a/src/config.cpp b/src/config.cpp
index dbeb98f89..6d9cef04d 100644
--- a/src/config.cpp
+++ b/src/config.cpp
@@ -178,6 +178,7 @@ Conf::Conf() : Block("")
 			{"networkinfo", "userlen"},
 			{"networkinfo", "hostlen"},
 			{"networkinfo", "chanlen"},
+			{"options", "casemap"},
 		};
 
 		for (unsigned i = 0; i < sizeof(noreload) / sizeof(noreload[0]); ++i)
@@ -223,6 +224,27 @@ Conf::Conf() : Block("")
 	this->TimeoutCheck = options->Get<time_t>("timeoutcheck");
 	this->NickChars = networkinfo->Get<Anope::string>("nick_chars");
 
+	Anope::string locale = options->Get<Anope::string>("locale");
+	Anope::string casemap = options->Get<Anope::string>("casemap");
+
+	if (locale.empty() == casemap.empty())
+		throw ConfigException("One of options:locale and options:casemap must be set");
+
+	if (locale.empty())
+	{
+		// load locale conf
+		File f(casemap + ".conf", false);
+		this->LoadConf(f);
+	}
+	else
+	{
+#if Boost_FOUND
+		this->locale = new std::locale(Anope::locale::generate(locale.str()));
+#else
+		throw ConfigException("Boost.Locale is not enabled, cannot use locale " + locale);
+#endif
+	}
+
 	for (int i = 0; i < this->CountBlock("uplink"); ++i)
 	{
 		Block *uplink = this->GetBlock("uplink", i);
@@ -599,6 +621,8 @@ Conf::~Conf()
 {
 	for (unsigned i = 0; i < MyOperTypes.size(); ++i)
 		delete MyOperTypes[i];
+
+	delete locale;
 }
 
 void Conf::Post(Conf *old)
diff --git a/src/hashcomp.cpp b/src/hashcomp.cpp
index 03dda8d18..265cb362b 100644
--- a/src/hashcomp.cpp
+++ b/src/hashcomp.cpp
@@ -23,6 +23,10 @@
 #include "anope.h"
 #include "config.h"
 
+#ifdef Boost_FOUND
+#include <boost/locale.hpp>
+#endif
+
 unsigned char Anope::tolower(unsigned char c)
 {
 	if (!Config || !Config->CaseMapLower[c])
@@ -164,3 +168,77 @@ bool sepstream::StreamEnd()
 	return this->pos > this->tokens.length();
 }
 
+size_t Anope::hash::operator()(const Anope::string &s) const
+{
+	return std::hash<std::string>()(s.str());
+}
+
+bool Anope::compare::operator()(const Anope::string &s1, const Anope::string &s2) const
+{
+	return s1.equals_cs(s2);
+}
+
+size_t Anope::hash_ci::operator()(const Anope::string &s) const
+{
+	return std::hash<std::string>()(s.lower().str());
+}
+
+bool Anope::compare_ci::operator()(const Anope::string &s1, const Anope::string &s2) const
+{
+	return s1.equals_ci(s2);
+}
+
+size_t Anope::hash_locale::operator()(const Anope::string &s) const
+{
+#ifdef Boost_FOUND
+	if (Config != nullptr && Config->locale != nullptr)
+	{
+		return Anope::locale::hash(s.str());
+	}
+#endif
+
+	return Anope::hash_ci()(s);
+}
+
+bool Anope::compare_locale::operator()(const Anope::string &s1, const Anope::string &s2) const
+{
+#ifdef Boost_FOUND
+	if (Config != nullptr && Config->locale != nullptr)
+	{
+		return Anope::locale::compare(s1.str(), s2.str()) == 0;
+	}
+#endif
+
+	return Anope::compare_ci()(s1, s2);
+}
+
+#ifdef Boost_FOUND
+
+std::locale Anope::locale::generate(const std::string &name)
+{
+	boost::locale::generator gen;
+	return gen.generate(name);
+}
+
+int Anope::locale::compare(const std::string &s1, const std::string &s2)
+{
+	std::string c1 = boost::locale::conv::between(s1.data(), s1.data() + s1.length(),
+		Config->locale->name(),
+		"");
+
+	std::string c2 = boost::locale::conv::between(s2.data(), s2.data() + s2.length(),
+		Config->locale->name(),
+		"");
+
+	const boost::locale::collator<char> &ct = std::use_facet<boost::locale::collator<char>>(*Config->locale);
+	return ct.compare(boost::locale::collator_base::secondary, c1, c2);
+}
+
+long Anope::locale::hash(const std::string &s)
+{
+	const boost::locale::collator<char> &ct = std::use_facet<boost::locale::collator<char>>(*Config->locale);
+	return ct.hash(boost::locale::collator_base::secondary, s.data(), s.data() + s.length());
+}
+
+#endif
+
diff --git a/src/users.cpp b/src/users.cpp
index a8d7642c9..f107d8fb3 100644
--- a/src/users.cpp
+++ b/src/users.cpp
@@ -32,7 +32,8 @@
 #include "event.h"
 #include "modules/nickserv.h"
 
-user_map UserListByNick, UserListByUID;
+user_map UserListByNick;
+uid_map UserListByUID;
 
 int OperCount = 0;
 
@@ -832,7 +833,7 @@ User* User::Find(const Anope::string &name, bool nick_only)
 {
 	if (!nick_only && IRCD->RequiresID)
 	{
-		user_map::iterator it = UserListByUID.find(name);
+		uid_map::iterator it = UserListByUID.find(name);
 		if (it != UserListByUID.end())
 			return it->second;