[Logging] Implement Player Event Logging system (#2833)

* Plumbing

* Batch processing in world

* Cleanup

* Cleanup

* Update player_event_logs.cpp

* Add player zoning event

* Use generics

* Comments

* Add events

* Add more events

* AA_GAIN, AA_PURCHASE, FORAGE_SUCCESS, FORAGE_FAILURE

* FISH_SUCCESS, FISH_FAILURE, ITEM_DESTROY

* Add charges to ITEM_DESTROY

* WENT_ONLINE, WENT_OFFLINE

* LEVEL_GAIN, LEVEL_LOSS

* LOOT_ITEM

* MERCHANT_PURCHASE

* MERCHANT_SELL

* SKILL_UP

* Add events

* Add more events

* TASK_ACCEPT, TASK_COMPLETE, and TASK_UPDATE

* GROUNDSPAWN_PICKUP

* SAY

* REZ_ACCEPTED

* COMBINE_FAILURE and COMBINE_SUCCESS

* DROPPED_ITEM

* DEATH

* SPLIT_MONEY

* TRADER_PURCHASE and TRADER_SELL

* DISCOVER_ITEM

* Convert GM_COMMAND to use new macro

* Convert ZONING event to use macro

* Revert some code changes

* Revert "Revert some code changes"

This reverts commit d53682f997e89a053a660761085913245db91e9d.

* Add cereal generation support to repositories

* TRADE

* Formatting

* Cleanup

* Relocate discord_manager to discord folder

* Discord sending plumbing

* Rename UCS's Database class to UCSDatabase to be more specific and not collide with base Database class for repository usage

* More discord sending plumbing

* More discord message formatting work

* More discord formatting work

* Discord formatting of events

* Format WENT_ONLINE, WENT_OFFLINE

* Add merchant purchase event

* Handle Discord MERCHANT_SELL formatter

* Update player_event_discord_formatter.cpp

* Tweaks

* Implement retention truncation

* Put mutex locking on batch queue, put processor on its own thread

* Process on initial bootup

* Implement optional QS processing, implement keepalive from world to QS

* Reload player event settings when logs are reloaded in game

* Set settings defaults

* Update player_event_logs.cpp

* Update player_event_logs.cpp

* Set retention days on boot

* Update player_event_logs.cpp

* Player Handin Event Testing.

Testing player handin stuff.

* Cleanup.

* Finish NPC Handin.

* set a reference to the client inside of the trade object as well for plugins to process

* Fix for windows _inline

* Bump to cpp20 default, ignore excessive warnings on windows

* Bump FMT to 6.1.2 for cpp20 compat and swap fmt::join for Strings::Join

* Windows compile fixes

* Update CMakeLists.txt

* Update CMakeLists.txt

* Update CMakeLists.txt

* Create 2022_12_19_player_events_tables.sql

* [Formatters] Work on Discord Formatters

* Handin money.

* Format header

* [Formatters] Work on Discord Formatters

* Format

* Format

* [Formatters] More Formatter work, need to test further.

* [Formatters] More Work on Formatters.

* Add missing #endif

* [Formatters] Work on Formatters, fix Bot formatting in ^create help

* NPC Handin Discord Formatter

* Update player_event_logs.cpp

* Discover Item Discord Formatter

* Dropped Item Discord Formatter

* Split Money Discord Formatter

* Trader Discord Formatters

* Cleanup.

* Trade Event Discord Formatter Groundwork

* SAY don't record GM commands

* GM_Command don't record #help

* Update player_event_logs.cpp

* Fill in more event data

* Post rebase fixes

* Post rebase fix

* Discord formatting adjustments

* Add event deprecation or unimplemented tag support

* Trade events

* Add return money and sanity checks.

* Update schema

* Update ucs.cpp

* Update client.cpp

* Update 2022_12_19_player_events_tables.sql

* Implement archive single line

* Replace hackers table and functions with PossibleHack player event

* Replace very old eventlog table since the same events are covered by player event logs

* Update bot_command.cpp

* Record NPC kill events ALL / Named / Raid

* Add BatchEventProcessIntervalSeconds rule

* Naming

* Update CMakeLists.txt

* Update database_schema.h

* Remove logging function and methods

* DB version

* Cleanup SendPlayerHandinEvent

---------

Co-authored-by: Kinglykrab <kinglykrab@gmail.com>
Co-authored-by: Aeadoin <109764533+Aeadoin@users.noreply.github.com>

common/discord/discord.cpp

@@ -1,22 +1,17 @@
+#include <cereal/archives/json.hpp>
+#include <cereal/archives/binary.hpp>
 #include "discord.h"
 #include "../http/httplib.h"
 #include "../json/json.h"
 #include "../strings.h"
 #include "../eqemu_logsys.h"
+#include "../events/player_event_logs.h"
 
 constexpr int MAX_RETRIES = 10;
 
 void Discord::SendWebhookMessage(const std::string &message, const std::string &webhook_url)
 {
-	// validate
-	if (webhook_url.empty()) {
-		LogDiscord("[webhook_url] is empty");
-		return;
-	}
-
-	// validate
-	if (webhook_url.find("http://") == std::string::npos && webhook_url.find("https://") == std::string::npos) {
-		LogDiscord("[webhook_url] [{}] does not contain a valid http/s prefix.", webhook_url);
+	if (!ValidateWebhookUrl(webhook_url)) {
 		return;
 	}
 
@@ -28,7 +23,7 @@ void Discord::SendWebhookMessage(const std::string &message, const std::string &
 	std::string endpoint = Strings::Replace(webhook_url, base_url, "");
 
 	// client
-	httplib::Client cli(base_url.c_str());
+	httplib::Client cli(base_url);
 	cli.set_connection_timeout(0, 15000000); // 15 sec
 	cli.set_read_timeout(15, 0); // 15 seconds
 	cli.set_write_timeout(15, 0); // 15 seconds
@@ -46,9 +41,9 @@ void Discord::SendWebhookMessage(const std::string &message, const std::string &
 	int  retries     = 0;
 	int  retry_timer = 1000;
 	while (retry) {
-		if (auto res = cli.Post(endpoint.c_str(), payload.str(), "application/json")) {
+		if (auto res = cli.Post(endpoint, payload.str(), "application/json")) {
 			if (res->status != 200 && res->status != 204) {
-				LogError("Code [{}] Error [{}]", res->status, res->body);
+				LogError("[Discord Client] Code [{}] Error [{}]", res->status, res->body);
 			}
 			if (res->status == 429) {
 				if (!res->body.empty()) {
@@ -81,6 +76,74 @@ void Discord::SendWebhookMessage(const std::string &message, const std::string &
 	}
 }
 
+void Discord::SendPlayerEventMessage(
+	const PlayerEvent::PlayerEventContainer &e,
+	const std::string &webhook_url
+)
+{
+	if (!ValidateWebhookUrl(webhook_url)) {
+		return;
+	}
+
+	auto s = Strings::Split(webhook_url, '/');
+
+	// url
+	std::string base_url = fmt::format("{}//{}", s[0], s[2]);
+	std::string endpoint = Strings::Replace(webhook_url, base_url, "");
+
+	// client
+	httplib::Client cli(base_url);
+	cli.set_connection_timeout(0, 15000000); // 15 sec
+	cli.set_read_timeout(15, 0); // 15 seconds
+	cli.set_write_timeout(15, 0); // 15 seconds
+	httplib::Headers headers = {
+		{"Content-Type", "application/json"}
+	};
+
+	std::string payload = PlayerEventLogs::GetDiscordPayloadFromEvent(e);
+	if (payload.empty()) {
+		return;
+	}
+
+	bool retry       = true;
+	int  retries     = 0;
+	int  retry_timer = 1000;
+	while (retry) {
+		if (auto res = cli.Post(endpoint, payload, "application/json")) {
+			if (res->status != 200 && res->status != 204) {
+				LogError("Code [{}] Error [{}]", res->status, res->body);
+			}
+			if (res->status == 429) {
+				if (!res->body.empty()) {
+					std::stringstream ss(res->body);
+					Json::Value       response;
+
+					try {
+						ss >> response;
+					}
+					catch (std::exception const &ex) {
+						LogDiscord("JSON serialization failure [{}] via [{}]", ex.what(), res->body);
+					}
+
+					retry_timer = std::stoi(response["retry_after"].asString()) + 500;
+				}
+
+				LogDiscord("Rate limited... retrying message in [{}ms]", retry_timer);
+				std::this_thread::sleep_for(std::chrono::milliseconds(retry_timer + 500));
+			}
+			if (res->status == 204) {
+				retry = false;
+			}
+			if (retries > MAX_RETRIES) {
+				LogDiscord("Retries exceeded for player event message");
+				retry = false;
+			}
+
+			retries++;
+		}
+	}
+}
+
 std::string Discord::FormatDiscordMessage(uint16 category_id, const std::string &message)
 {
 	if (category_id == Logs::LogCategory::MySQLQuery) {
@@ -89,3 +152,20 @@ std::string Discord::FormatDiscordMessage(uint16 category_id, const std::string
 
 	return message + "\n";
 }
+
+bool Discord::ValidateWebhookUrl(const std::string &webhook_url)
+{
+	// validate
+	if (webhook_url.empty()) {
+		LogDiscord("[webhook_url] is empty");
+		return false;
+	}
+
+	// validate
+	if (!Strings::Contains(webhook_url, "http://") && !Strings::Contains(webhook_url, "https://")) {
+		LogDiscord("[webhook_url] [{}] does not contain a valid http/s prefix.", webhook_url);
+		return false;
+	}
+
+	return true;
+}

common/discord/discord.h

@@ -4,11 +4,16 @@
 
 #include <string>
 #include "../types.h"
+#include "../http/httplib.h"
+#include "../repositories/player_event_logs_repository.h"
+#include "../events/player_events.h"
 
 class Discord {
 public:
 	static void SendWebhookMessage(const std::string& message, const std::string& webhook_url);
 	static std::string FormatDiscordMessage(uint16 category_id, const std::string& message);
+	static void SendPlayerEventMessage(const PlayerEvent::PlayerEventContainer& e, const std::string &webhook_url);
+	static bool ValidateWebhookUrl(const std::string &webhook_url);
 };
 
 

common/discord/discord_manager.cpp

@@ -0,0 +1,75 @@
+#include "discord_manager.h"
+#include "../../common/discord/discord.h"
+#include "../events/player_event_logs.h"
+
+void DiscordManager::QueueWebhookMessage(uint32 webhook_id, const std::string &message)
+{
+	webhook_queue_lock.lock();
+	webhook_message_queue[webhook_id].emplace_back(message);
+	webhook_queue_lock.unlock();
+}
+
+constexpr int MAX_MESSAGE_LENGTH = 1900;
+
+void DiscordManager::ProcessMessageQueue()
+{
+	if (webhook_message_queue.empty()) {
+		return;
+	}
+
+	webhook_queue_lock.lock();
+	for (auto &q: webhook_message_queue) {
+		LogDiscord("Processing [{}] messages in queue for webhook ID [{}]...", q.second.size(), q.first);
+
+		if (q.first >= MAX_DISCORD_WEBHOOK_ID) {
+			LogDiscord("Out of bounds webhook ID [{}] max [{}]", q.first, MAX_DISCORD_WEBHOOK_ID);
+			continue;
+		}
+
+		auto        webhook  = LogSys.GetDiscordWebhooks()[q.first];
+		std::string message;
+
+		for (auto &m: q.second) {
+			// next message would become too large
+			bool next_message_too_large = ((int) m.length() + (int) message.length()) > MAX_MESSAGE_LENGTH;
+			if (next_message_too_large) {
+				Discord::SendWebhookMessage(
+					message,
+					webhook.webhook_url
+				);
+				message = "";
+			}
+
+			message += m;
+
+			// one single message was too large
+			// this should rarely happen but the message will need to be split
+			if ((int) message.length() > MAX_MESSAGE_LENGTH) {
+				for (unsigned mi = 0; mi < message.length(); mi += MAX_MESSAGE_LENGTH) {
+					Discord::SendWebhookMessage(
+						message.substr(mi, MAX_MESSAGE_LENGTH),
+						webhook.webhook_url
+					);
+				}
+				message = "";
+			}
+		}
+		// final flush
+		if (!message.empty()) {
+			Discord::SendWebhookMessage(
+				message,
+				webhook.webhook_url
+			);
+		}
+	}
+	webhook_message_queue.clear();
+	webhook_queue_lock.unlock();
+}
+
+void DiscordManager::QueuePlayerEventMessage(const PlayerEvent::PlayerEventContainer& e)
+{
+	auto w = player_event_logs.GetDiscordWebhookUrlFromEventType(e.player_event_log.event_type_id);
+	if (!w.empty()) {
+		Discord::SendPlayerEventMessage(e, w);
+	}
+}

common/discord/discord_manager.h

@@ -0,0 +1,22 @@
+#ifndef EQEMU_DISCORD_MANAGER_H
+#define EQEMU_DISCORD_MANAGER_H
+
+#include <mutex>
+#include <map>
+#include <vector>
+#include "../../common/types.h"
+#include "../repositories/player_event_logs_repository.h"
+#include "../events/player_events.h"
+
+class DiscordManager {
+public:
+	void QueueWebhookMessage(uint32 webhook_id, const std::string& message);
+	void ProcessMessageQueue();
+	void QueuePlayerEventMessage(const PlayerEvent::PlayerEventContainer& e);
+private:
+	std::mutex webhook_queue_lock{};
+	std::map<uint32, std::vector<std::string>> webhook_message_queue{};
+};
+
+
+#endif