Added command 'profanity'

changelog.txt

@@ -1,6 +1,26 @@
 EQEMu Changelog (Started on Sept 24, 2003 15:50)
 -------------------------------------------------------
 
+== 2/4/2019 ==
+Uleat: Added command 'profanity' (aliased 'prof')
+	- This is a server-based tool for redacting any language that an admin deems as profanity (socially unacceptable within their community)
+	- Five options are available under this command..
+	 -- 'list' - shows the current list of banned words
+	 -- 'clear' - clears the current list of banned words
+	 -- 'add <word>' - adds <word> to the banned word list
+	 -- 'del <word>' - deletes <word> from the banned word list
+	 -- 'reload' - forces a reload of the banned word list
+	- All actions are immediate and a world broadcast refreshes other active zones
+	- The system is in stand-by when the list is empty..just add a word to the list to begin censorship
+	- Redaction only occurs on genuine occurences of any banned word
+	 -- Banned words are replaced with a series of '*' characters
+	 -- Compounded words are ignored to avoid issues with allowed words containing a banned sub-string
+	 -- If 'test' is banned, 'testing' will not be banned .. it must be added separately
+	- Extreme care should be exercised when adding words to the banned list..
+	 -- Quest failures and limited social interactions may alienate players if they become inhibiting
+	 -- System commands are allowed to be processed before redaction occurs in the 'say' channel
+	- A longer list requires more clock cycles to process - so, try to keep them to the most offensible occurrences
+
 == 1/26/2019 ==
 Uleat: Fix for class Bot not honoring NPCType data reference
 	- Fixes bots not moving on spawn/grouping issue

common/CMakeLists.txt

@@ -55,6 +55,7 @@ SET(common_sources
 	perl_eqdb.cpp
 	perl_eqdb_res.cpp
 	proc_launcher.cpp
+	profanity_manager.cpp
 	ptimer.cpp
 	races.cpp
 	rdtsc.cpp
@@ -181,6 +182,7 @@ SET(common_headers
 	packet_functions.h
 	platform.h
 	proc_launcher.h
+	profanity_manager.h
 	profiler.h
 	ptimer.h
 	queue.h

common/profanity_manager.cpp

@@ -0,0 +1,249 @@
+/*	EQEMu: Everquest Server Emulator
+	
+	Copyright (C) 2001-2019 EQEMu Development Team (http://eqemulator.net)
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation; version 2 of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY except by those people which sell it, which
+	are required to give you total support for your newly bought product;
+	without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+	A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+	
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include "profanity_manager.h"
+#include "dbcore.h"
+
+#include <ctype.h>
+#include <algorithm>
+
+
+static std::list<std::string> profanity_list;
+static bool update_originator_flag = false;
+
+bool EQEmu::ProfanityManager::LoadProfanityList(DBcore *db) {
+	if (update_originator_flag == true) {
+		update_originator_flag = false;
+		return true;
+	}
+
+	if (!load_database_entries(db))
+		return false;
+
+	return true;
+}
+
+bool EQEmu::ProfanityManager::UpdateProfanityList(DBcore *db) {
+	if (!load_database_entries(db))
+		return false;
+
+	update_originator_flag = true;
+
+	return true;
+}
+
+bool EQEmu::ProfanityManager::DeleteProfanityList(DBcore *db) {
+	if (!clear_database_entries(db))
+		return false;
+
+	update_originator_flag = true;
+
+	return true;
+}
+
+bool EQEmu::ProfanityManager::AddProfanity(DBcore *db, const char *profanity) {
+	if (!db || !profanity)
+		return false;
+
+	std::string entry(profanity);
+
+	std::transform(entry.begin(), entry.end(), entry.begin(), [](unsigned char c) -> unsigned char { return tolower(c); });
+
+	if (check_for_existing_entry(entry.c_str()))
+		return true;
+
+	if (entry.length() < REDACTION_LENGTH_MIN)
+		return false;
+
+	profanity_list.push_back(entry);
+
+	std::string query = "REPLACE INTO `profanity_list` (`word`) VALUES ('";
+	query.append(entry);
+	query.append("')");
+	auto results = db->QueryDatabase(query);
+	if (!results.Success())
+		return false;
+
+	update_originator_flag = true;
+
+	return true;
+}
+
+bool EQEmu::ProfanityManager::RemoveProfanity(DBcore *db, const char *profanity) {
+	if (!db || !profanity)
+		return false;
+
+	std::string entry(profanity);
+
+	std::transform(entry.begin(), entry.end(), entry.begin(), [](unsigned char c) -> unsigned char { return tolower(c); });
+
+	if (!check_for_existing_entry(entry.c_str()))
+		return true;
+
+	profanity_list.remove(entry);
+
+	std::string query = "DELETE FROM `profanity_list` WHERE `word` LIKE '";
+	query.append(entry);
+	query.append("'");
+	auto results = db->QueryDatabase(query);
+	if (!results.Success())
+		return false;
+
+	update_originator_flag = true;
+
+	return true;
+}
+
+void EQEmu::ProfanityManager::RedactMessage(char *message) {
+	if (!message)
+		return;
+
+	std::string test_message(message);
+	// hard-coded max length based on channel message buffer size (4096 bytes)..
+	// ..will need to change or remove if other sources are used for redaction
+	if (test_message.length() < REDACTION_LENGTH_MIN || test_message.length() >= 4096)
+		return;
+
+	std::transform(test_message.begin(), test_message.end(), test_message.begin(), [](unsigned char c) -> unsigned char { return tolower(c); });
+	
+	for (const auto &iter : profanity_list) { // consider adding textlink checks if it becomes an issue
+		size_t pos = 0;
+		size_t start_pos = 0;
+
+		while (pos != std::string::npos) {
+			pos = test_message.find(iter, start_pos);
+			if (pos == std::string::npos)
+				continue;
+
+			if ((pos + iter.length()) == test_message.length() || !isalpha(test_message.at(pos + iter.length()))) {
+				if (pos == 0 || !isalpha(test_message.at(pos - 1)))
+					memset((message + pos), REDACTION_CHARACTER, iter.length());
+			}
+
+			start_pos = (pos + iter.length());
+		}
+	}
+}
+
+void EQEmu::ProfanityManager::RedactMessage(std::string &message) {
+	if (message.length() < REDACTION_LENGTH_MIN || message.length() >= 4096)
+		return;
+
+	std::string test_message(const_cast<const std::string&>(message));
+
+	std::transform(test_message.begin(), test_message.end(), test_message.begin(), [](unsigned char c) -> unsigned char { return tolower(c); });
+
+	for (const auto &iter : profanity_list) { // consider adding textlink checks if it becomes an issue
+		size_t pos = 0;
+		size_t start_pos = 0;
+
+		while (pos != std::string::npos) {
+			pos = test_message.find(iter, start_pos);
+			if (pos == std::string::npos)
+				continue;
+
+			if ((pos + iter.length()) == test_message.length() || !isalpha(test_message.at(pos + iter.length()))) {
+				if (pos == 0 || !isalpha(test_message.at(pos - 1)))
+					message.replace(pos, iter.length(), iter.length(), REDACTION_CHARACTER);
+			}
+
+			start_pos = (pos + iter.length());
+		}
+	}
+}
+
+bool EQEmu::ProfanityManager::ContainsCensoredLanguage(const char *message) {
+	if (!message)
+		return false;
+
+	return ContainsCensoredLanguage(std::string(message));
+}
+
+bool EQEmu::ProfanityManager::ContainsCensoredLanguage(const std::string &message) {
+	if (message.length() < REDACTION_LENGTH_MIN || message.length() >= 4096)
+		return false;
+
+	std::string test_message(message);
+
+	std::transform(test_message.begin(), test_message.end(), test_message.begin(), [](unsigned char c) -> unsigned char { return tolower(c); });
+
+	for (const auto &iter : profanity_list) {
+		if (test_message.find(iter) != std::string::npos)
+			return true;
+	}
+
+	return false;
+}
+
+const std::list<std::string> &EQEmu::ProfanityManager::GetProfanityList() {
+	return profanity_list;
+}
+
+bool EQEmu::ProfanityManager::IsCensorshipActive() {
+	return (profanity_list.size() != 0);
+}
+
+bool EQEmu::ProfanityManager::load_database_entries(DBcore *db) {
+	if (!db)
+		return false;
+
+	profanity_list.clear();
+
+	std::string query = "SELECT `word` FROM `profanity_list`";
+	auto results = db->QueryDatabase(query);
+	if (!results.Success())
+		return false;
+
+	for (auto row = results.begin(); row != results.end(); ++row) {
+		if (std::strlen(row[0]) >= REDACTION_LENGTH_MIN) {
+			std::string entry(row[0]);
+			std::transform(entry.begin(), entry.end(), entry.begin(), [](unsigned char c) -> unsigned char { return tolower(c); });
+			if (!check_for_existing_entry(entry.c_str()))
+				profanity_list.push_back((std::string)entry);
+		}
+	}
+
+	return true;
+}
+
+bool EQEmu::ProfanityManager::clear_database_entries(DBcore *db) {
+	if (!db)
+		return false;
+
+	profanity_list.clear();
+
+	std::string query = "DELETE FROM `profanity_list`";
+	auto results = db->QueryDatabase(query);
+	if (!results.Success())
+		return false;
+
+	return true;
+}
+
+bool EQEmu::ProfanityManager::check_for_existing_entry(const char *profanity) {
+	if (!profanity)
+		return false;
+
+	for (const auto &iter : profanity_list) {
+		if (iter.compare(profanity) == 0)
+			return true;
+	}
+	
+	return false;
+}

common/profanity_manager.h

@@ -0,0 +1,62 @@
+/*	EQEMu: Everquest Server Emulator
+	
+	Copyright (C) 2001-2019 EQEMu Development Team (http://eqemulator.net)
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation; version 2 of the License.
+	
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY except by those people which sell it, which
+	are required to give you total support for your newly bought product;
+	without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+	A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+	
+	You should have received a copy of the GNU General Public License
+	along with this program; if not, write to the Free Software
+	Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef COMMON_PROFANITY_MANAGER_H
+#define COMMON_PROFANITY_MANAGER_H
+
+#include <string>
+#include <list>
+
+
+class DBcore;
+
+namespace EQEmu
+{
+	class ProfanityManager {
+	public:
+		static bool LoadProfanityList(DBcore *db);
+		static bool UpdateProfanityList(DBcore *db);
+		static bool DeleteProfanityList(DBcore *db);
+
+		static bool AddProfanity(DBcore *db, const char *profanity);
+		static bool RemoveProfanity(DBcore *db, const char *profanity);
+		
+		static void RedactMessage(char *message);
+		static void RedactMessage(std::string &message);
+
+		static bool ContainsCensoredLanguage(const char *message);
+		static bool ContainsCensoredLanguage(const std::string &message);
+
+		static const std::list<std::string> &GetProfanityList();
+
+		static bool IsCensorshipActive();
+
+		static const char REDACTION_CHARACTER = '*';
+		static const int REDACTION_LENGTH_MIN = 3;
+
+	private:
+		static bool load_database_entries(DBcore *db);
+		static bool clear_database_entries(DBcore *db);
+		static bool check_for_existing_entry(const char *profanity);
+		
+	};
+
+} /*EQEmu*/
+
+#endif /*COMMON_PROFANITY_MANAGER_H*/

common/servertalk.h

@@ -159,6 +159,7 @@
 #define ServerOP_SetWorldTime		0x200B
 #define ServerOP_GetWorldTime		0x200C
 #define ServerOP_SyncWorldTime		0x200E
+#define ServerOP_RefreshCensorship	0x200F
 
 #define ServerOP_LSZoneInfo			0x3001
 #define ServerOP_LSZoneStart		0x3002

common/version.h

@@ -30,7 +30,7 @@
 	Manifest: https://github.com/EQEmu/Server/blob/master/utils/sql/db_update_manifest.txt
 */
 
-#define CURRENT_BINARY_DATABASE_VERSION 9135
+#define CURRENT_BINARY_DATABASE_VERSION 9136
 #ifdef BOTS
 	#define CURRENT_BINARY_BOTS_DATABASE_VERSION 9021
 #else

utils/sql/db_update_manifest.txt

@@ -389,6 +389,7 @@
 9133|2018_11_25_StuckBehavior.sql|SHOW COLUMNS FROM `npc_types` LIKE 'stuck_behavior'|empty|
 9134|2019_01_04_update_global_base_scaling.sql|SELECT * FROM db_version WHERE version >= 9134|empty|
 9135|2019_01_10_multi_version_spawns.sql|SHOW COLUMNS FROM `spawn2` LIKE 'version'|contains|unsigned|
+9136|2019_02_14_profanity_command.sql|SHOW TABLES LIKE 'profanity_list'|empty|
 
 # Upgrade conditions:
 # 	This won't be needed after this system is implemented, but it is used database that are not

utils/sql/git/required/2019_02_04_profanity_command.sql

@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS `profanity_list`;
+
+CREATE TABLE `profanity_list` (
+	`word` VARCHAR(16) NOT NULL
+)
+COLLATE='latin1_swedish_ci'
+ENGINE=InnoDB
+;
+
+REPLACE INTO `command_settings` VALUES ('profanity', 150, 'prof');

world/zoneserver.cpp

@@ -981,6 +981,10 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 		safe_delete(pack);
 		break;
 	}
+	case ServerOP_RefreshCensorship: {
+		zoneserver_list.SendPacket(pack);
+		break;
+	}
 	case ServerOP_SetWorldTime: {
 		Log(Logs::Detail, Logs::World_Server, "Received SetWorldTime");
 		eqTimeOfDay* newtime = (eqTimeOfDay*)pack->pBuffer;

zone/client.cpp

@@ -38,6 +38,7 @@ extern volatile bool RunLoops;
 #include "../common/rulesys.h"
 #include "../common/string_util.h"
 #include "../common/data_verification.h"
+#include "../common/profanity_manager.h"
 #include "data_bucket.h"
 #include "position.h"
 #include "net.h"
@@ -895,6 +896,10 @@ void Client::ChannelMessageReceived(uint8 chan_num, uint8 language, uint8 lang_s
 		language = 0; // No need for language when drunk
 	}
 
+	// Censor the message
+	if (EQEmu::ProfanityManager::IsCensorshipActive() && (chan_num != 8))
+		EQEmu::ProfanityManager::RedactMessage(message);
+
 	switch(chan_num)
 	{
 	case 0: { /* Guild Chat */
@@ -1092,6 +1097,9 @@ void Client::ChannelMessageReceived(uint8 chan_num, uint8 language, uint8 lang_s
 			break;
 		}
 
+		if (EQEmu::ProfanityManager::IsCensorshipActive())
+			EQEmu::ProfanityManager::RedactMessage(message);
+
 #ifdef BOTS
 		if (message[0] == BOT_COMMAND_CHAR) {
 			if (bot_command_dispatch(this, message) == -2) {

zone/command.cpp

@@ -54,6 +54,7 @@
 #include "../common/string_util.h"
 #include "../say_link.h"
 #include "../common/eqemu_logsys.h"
+#include "../common/profanity_manager.h"
 
 #include "data_bucket.h"
 #include "command.h"
@@ -307,6 +308,7 @@ int command_init(void)
 		command_add("petitioninfo", "[petition number] - Get info about a petition", 20, command_petitioninfo) ||
 		command_add("pf", "- Display additional mob coordinate and wandering data", 0, command_pf) ||
 		command_add("picklock",  "Analog for ldon pick lock for the newer clients since we still don't have it working.",  0, command_picklock) ||
+		command_add("profanity", "Manage censored language.", 150, command_profanity) ||
 
 #ifdef EQPROFILE
 		command_add("profiledump", "- Dump profiling info to logs", 250, command_profiledump) ||
@@ -11043,6 +11045,68 @@ void command_picklock(Client *c, const Seperator *sep)
 	}
 }
 
+void command_profanity(Client *c, const Seperator *sep)
+{
+	std::string arg1(sep->arg[1]);
+	
+	while (true) {
+		if (arg1.compare("list") == 0) {
+			// do nothing
+		}
+		else if (arg1.compare("clear") == 0) {
+			EQEmu::ProfanityManager::DeleteProfanityList(&database);
+			auto pack = new ServerPacket(ServerOP_RefreshCensorship);
+			worldserver.SendPacket(pack);
+			safe_delete(pack);
+		}
+		else if (arg1.compare("add") == 0) {
+			if (!EQEmu::ProfanityManager::AddProfanity(&database, sep->arg[2]))
+				c->Message(CC_Red, "Could not add '%s' to the profanity list.", sep->arg[2]);
+			auto pack = new ServerPacket(ServerOP_RefreshCensorship);
+			worldserver.SendPacket(pack);
+			safe_delete(pack);
+		}
+		else if (arg1.compare("del") == 0) {
+			if (!EQEmu::ProfanityManager::RemoveProfanity(&database, sep->arg[2]))
+				c->Message(CC_Red, "Could not delete '%s' from the profanity list.", sep->arg[2]);
+			auto pack = new ServerPacket(ServerOP_RefreshCensorship);
+			worldserver.SendPacket(pack);
+			safe_delete(pack);
+		}
+		else if (arg1.compare("reload") == 0) {
+			if (!EQEmu::ProfanityManager::UpdateProfanityList(&database))
+				c->Message(CC_Red, "Could not reload the profanity list.");
+			auto pack = new ServerPacket(ServerOP_RefreshCensorship);
+			worldserver.SendPacket(pack);
+			safe_delete(pack);
+		}
+		else {
+			break;
+		}
+
+		std::string popup;
+		const auto &list = EQEmu::ProfanityManager::GetProfanityList();
+		for (const auto &iter : list) {
+			popup.append(iter);
+			popup.append("<br>");
+		}
+		if (list.empty())
+			popup.append("** Censorship Inactive **<br>");
+		else
+			popup.append("** End of List **<br>");
+
+		c->SendPopupToClient("Profanity List", popup.c_str());
+		
+		return;
+	}
+	
+	c->Message(0, "Usage: #profanity [list] - shows profanity list");
+	c->Message(0, "Usage: #profanity [clear] - deletes all entries");
+	c->Message(0, "Usage: #profanity [add] [<word>] - adds entry");
+	c->Message(0, "Usage: #profanity [del] [<word>] - deletes entry");
+	c->Message(0, "Usage: #profanity [reload] - reloads profanity list");
+}
+
 void command_mysql(Client *c, const Seperator *sep)
 {
 	if(!sep->arg[1][0] || !sep->arg[2][0]) {

zone/command.h

@@ -210,6 +210,7 @@ void command_permagender(Client *c, const Seperator *sep);
 void command_permarace(Client *c, const Seperator *sep);
 void command_petitioninfo(Client *c, const Seperator *sep);
 void command_picklock(Client *c, const Seperator *sep);
+void command_profanity(Client *c, const Seperator *sep);
 
 #ifdef EQPROFILE
 void command_profiledump(Client *c, const Seperator *sep);

zone/net.cpp

@@ -32,6 +32,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 #include "../common/eq_stream_ident.h"
 #include "../common/patches/patches.h"
 #include "../common/rulesys.h"
+#include "../common/profanity_manager.h"
 #include "../common/misc_functions.h"
 #include "../common/string_util.h"
 #include "../common/platform.h"
@@ -350,6 +351,10 @@ int main(int argc, char** argv) {
 	Log(Logs::General, Logs::Zone_Server, "Loading corpse timers");
 	database.GetDecayTimes(npcCorpseDecayTimes);
 
+	Log(Logs::General, Logs::Zone_Server, "Loading profanity list");
+	if (!EQEmu::ProfanityManager::LoadProfanityList(&database))
+		Log(Logs::General, Logs::Error, "Loading profanity list FAILED!");
+
 	Log(Logs::General, Logs::Zone_Server, "Loading commands");
 	int retval = command_init();
 	if (retval<0)

zone/worldserver.cpp

@@ -36,6 +36,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 #include "../common/misc_functions.h"
 #include "../common/rulesys.h"
 #include "../common/servertalk.h"
+#include "../common/profanity_manager.h"
 
 #include "client.h"
 #include "corpse.h"
@@ -793,6 +794,11 @@ void WorldServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p)
 		}
 		break;
 	}
+	case ServerOP_RefreshCensorship: {
+		if (!EQEmu::ProfanityManager::LoadProfanityList(&database))
+			Log(Logs::General, Logs::Error, "Received request to refresh the profanity list..but, the action failed");
+		break;
+	}
 	case ServerOP_ChangeWID: {
 		if (pack->size != sizeof(ServerChangeWID_Struct)) {
 			std::cout << "Wrong size on ServerChangeWID_Struct. Got: " << pack->size << ", Expected: " << sizeof(ServerChangeWID_Struct) << std::endl;