[Player Event Logs] Migrate and Deprecate QS Legacy Logging (#4542)

* First pass of player_event_loot_items

* Second pass of player_event_loot_items

* Third pass of player_event_loot_items

* Example without RecordDetailEvent template

* Cleanup the removal of the template

* Fourth Pass

Add retention for etl tables
Rename tables/fields to etl nomenclature
Combine database work to one atomic load

* Reposition to reduce db tasks

* Refactor etl processing for easier additions

* Add merchant purchase event
testing passed though appears that the event itself has a few bugs.  Will fix them in another commit

* Fix PlayerEventMerchantPurchase in client_packet.cpp

* WIP - Handin

* Handin Event added

* Cleanup

* All a rentention period of 0 days which deletes all current records.

* Updates

Cleanup and refactor a few items.

* Cleanup and Formatting

Cleanup and Formatting

* Add etl for
Playerevent::Trade
PlayerEvent::Speech (new event to mirror functionality of qs_speech

* Add etl for
Playerevent::KilledNPC, KilledNamedNPC and KilledRaidNPC

* Add etl for Playerevent::AA_purchase

Add etl for Playerevent::AA_purchase

* Cleanup before PR

* Review comment updates.

* Add world cli etl:settings to output a json on all player event details.

* Add reserve for all etl_queues
Correct a failed test case for improper next id for etl tables when table is first created.

* Potential solution for a dedicated database connection for player events.

* Simple thread for player_events.  Likely there is a better way to do this.

* Add zone to qs communications for recordplayerevents

First pass of enabling zone to qs direct transport to allow for PlayerEvents to bypass world.

* Cleanup a linux compile issue

* Add augments to LOOT ITEM and DESTROY ITEM

* Add augments to ITEMCREATION, FORAGESUCCESS, FISHSUCCESS, DESTROYITEM, LOOTITEM, DROPPEDITEM, TRADERPURCHASE, TRADERSELL, GUILDTRIBUTEDONATE and cleaned up the naming convention of augments

* Formatting fixes

* Swap out GetNextTableId

* Statically load counter

* Add counter.clear() since the counter is static

* Upload optional QS conversion scripts

* Remove all qs_tables and code referencing them

* Update database.cpp

* Simplify ProcessBatchQueue

* Simplify PorcessBatchQueue

* Simplify event truncation

* Build event truncation to bulk query by retention groups

* Post rebase

* Update player_events.h

* Fix build

* Update npc.cpp

* First pass of direct zone to qs sending for player events

* Remove keepalive logic

* Fix event ordering

* Cleanup

* Update player_event_logs.cpp

* Wipe event data after ETL processed

* Split up database connections, hot reload logs for QS

* Load rules from database vs qs_database

* Update player_event_logs.cpp

* Hot toggle queryserv connect

---------

Co-authored-by: Akkadius <akkadius1@gmail.com>

world/cli/database_dump.cpp

@@ -14,7 +14,6 @@ void WorldserverCLI::DatabaseDump(int argc, char **argv, argh::parser &cmd, std:
 		"--merc-tables",
 		"--state-tables",
 		"--system-tables",
-		"--query-serv-tables",
 		"--static-instance-data",
 		"--table-structure-only",
 		"--table-lock",
@@ -45,7 +44,6 @@ void WorldserverCLI::DatabaseDump(int argc, char **argv, argh::parser &cmd, std:
 	s->SetDumpMercTables(cmd[{"--merc-tables"}] || dump_all);
 	s->SetDumpStateTables(cmd[{"--state-tables"}] || dump_all);
 	s->SetDumpSystemTables(cmd[{"--system-tables"}] || dump_all);
-	s->SetDumpQueryServerTables(cmd[{"--query-serv-tables"}] || dump_all);
 	s->SetDumpAllTables(dump_all);
 	s->SetDumpWithNoData(cmd[{"--table-structure-only"}]);
 	s->SetDumpTableLock(cmd[{"--table-lock"}]);

world/cli/etl_get_settings.cpp

@@ -0,0 +1,41 @@
+#include "../../common/events/player_event_logs.h"
+#include "../../common/json/json.h"
+
+void WorldserverCLI::EtlGetSettings(int argc, char **argv, argh::parser &cmd, std::string &description)
+{
+	description = "Displays server player event logs that are etl enabled";
+
+	if (cmd[{"-h", "--help"}]) {
+		return;
+	}
+
+	Json::Value etl_settings;
+	Json::Value player_events;
+
+	player_event_logs.SetDatabase(&database)->Init();
+	auto event_settings = player_event_logs.GetSettings();
+	auto etl_details    = player_event_logs.GetEtlSettings();
+
+	for (auto i = 0; i < PlayerEvent::EventType::MAX; i++) {
+		player_events["event_id"]    = event_settings[i].id;
+		player_events["enabled"]     = event_settings[i].event_enabled ? true : false;
+		player_events["retention"]   = event_settings[i].retention_days;
+		player_events["discord_id"]  = event_settings[i].discord_webhook_id;
+		player_events["etl_enabled"] = event_settings[i].etl_enabled ? true : false;
+		player_events["table_name"]  = "";
+
+		auto it = etl_details.find(static_cast<PlayerEvent::EventType>(event_settings[i].id));
+
+		if (it != std::end(etl_details)) {
+			player_events["table_name"]  = it->second.table_name;
+			player_events["etl_enabled"] = it->second.enabled;
+		}
+
+		etl_settings["etl_settings"].append(player_events);
+	}
+
+	std::stringstream payload;
+	payload << etl_settings;
+
+	std::cout << payload.str() << std::endl;
+}

world/eqemu_api_world_data_service.cpp

@@ -1,23 +1,3 @@
-/**
- * EQEmulator: Everquest Server Emulator
- * Copyright (C) 2001-2019 EQEmulator Development Team (https://github.com/EQEmu/Server)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY except by those people which sell it, which
- * are required to give you total support for your newly bought product;
- * without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
- *
- */
-
 #include <fmt/format.h>
 #include "clientlist.h"
 #include "cliententry.h"
@@ -29,10 +9,14 @@
 #include "worlddb.h"
 #include "wguild_mgr.h"
 #include "world_config.h"
+#include "ucs.h"
+#include "queryserv.h"
 
 extern ZSList            zoneserver_list;
 extern ClientList        client_list;
 extern WorldGuildManager guild_mgr;
+extern UCSConnection UCSLink;
+extern QueryServConnection QSLink;
 
 void callGetZoneList(Json::Value &response)
 {
@@ -237,6 +221,8 @@ void EQEmuApiWorldDataService::reload(Json::Value &r, const std::vector<std::str
 
 				if (c.opcode == ServerOP_ReloadLogs) {
 					LogSys.LoadLogDatabaseSettings();
+					QSLink.SendPacket(pack);
+					UCSLink.SendPacket(pack);
 				}
 				else if (c.opcode == ServerOP_ReloadRules) {
 					RuleManager::Instance()->LoadRules(&database, RuleManager::Instance()->GetActiveRuleset(), true);

world/main.cpp

@@ -376,7 +376,9 @@ int main(int argc, char **argv)
 	);
 
 	Timer player_event_process_timer(1000);
-	player_event_logs.SetDatabase(&database)->Init();
+	if (player_event_logs.LoadDatabaseConnection()) {
+		player_event_logs.Init();
+	}
 
 	auto loop_fn = [&](EQ::Timer* t) {
 		Timer::SetCurrentTime();
@@ -441,7 +443,7 @@ int main(int argc, char **argv)
 		}
 
 		if (player_event_process_timer.Check()) {
-			player_event_logs.Process();
+			std::jthread event_thread(&PlayerEventLogs::Process, &player_event_logs);
 		}
 
 		if (PurgeInstanceTimer.Check()) {

world/world_server_cli.cpp

@@ -34,6 +34,7 @@ void WorldserverCLI::CommandHandler(int argc, char **argv)
 	function_map["test:repository2"]            = &WorldserverCLI::TestRepository2;
 	function_map["test:db-concurrency"]         = &WorldserverCLI::TestDatabaseConcurrency;
 	function_map["test:string-benchmark"]       = &WorldserverCLI::TestStringBenchmarkCommand;
+	function_map["etl:settings"]                = &WorldserverCLI::EtlGetSettings;
 
 	EQEmuCommand::HandleMenu(function_map, cmd, argc, argv);
 }
@@ -56,3 +57,4 @@ void WorldserverCLI::CommandHandler(int argc, char **argv)
 #include "cli/test_repository_2.cpp"
 #include "cli/test_string_benchmark.cpp"
 #include "cli/version.cpp"
+#include "cli/etl_get_settings.cpp"

world/world_server_cli.h

@@ -25,6 +25,7 @@ public:
 	static void TestRepository2(int argc, char **argv, argh::parser &cmd, std::string &description);
 	static void TestDatabaseConcurrency(int argc, char **argv, argh::parser &cmd, std::string &description);
 	static void TestStringBenchmarkCommand(int argc, char **argv, argh::parser &cmd, std::string &description);
+	static void EtlGetSettings(int argc, char **argv, argh::parser &cmd, std::string &description);
 };
 
 

world/zoneserver.cpp

@@ -1353,15 +1353,6 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 			safe_delete(outapp);
 			break;
 		}
-		case ServerOP_Speech:
-		case ServerOP_QSSendQuery:
-		case ServerOP_QSPlayerLogDeletes:
-		case ServerOP_QSPlayerDropItem:
-		case ServerOP_QSPlayerLogHandins:
-		case ServerOP_QSPlayerLogMerchantTransactions:
-		case ServerOP_QSPlayerLogMoves:
-		case ServerOP_QSPlayerLogNPCKills:
-		case ServerOP_QSPlayerLogTrades:
 		case ServerOP_QueryServGeneric: {
 			QSLink.SendPacket(pack);
 			break;
@@ -1481,6 +1472,7 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 		}
 		case ServerOP_ReloadLogs: {
 			zoneserver_list.SendPacket(pack);
+			QSLink.SendPacket(pack);
 			UCSLink.SendPacket(pack);
 			LogSys.LoadLogDatabaseSettings();
 			player_event_logs.ReloadSettings();