[Player Event Logs] Migrate and Deprecate QS Legacy Logging (#4542)

* First pass of player_event_loot_items

* Second pass of player_event_loot_items

* Third pass of player_event_loot_items

* Example without RecordDetailEvent template

* Cleanup the removal of the template

* Fourth Pass

Add retention for etl tables
Rename tables/fields to etl nomenclature
Combine database work to one atomic load

* Reposition to reduce db tasks

* Refactor etl processing for easier additions

* Add merchant purchase event
testing passed though appears that the event itself has a few bugs.  Will fix them in another commit

* Fix PlayerEventMerchantPurchase in client_packet.cpp

* WIP - Handin

* Handin Event added

* Cleanup

* All a rentention period of 0 days which deletes all current records.

* Updates

Cleanup and refactor a few items.

* Cleanup and Formatting

Cleanup and Formatting

* Add etl for
Playerevent::Trade
PlayerEvent::Speech (new event to mirror functionality of qs_speech

* Add etl for
Playerevent::KilledNPC, KilledNamedNPC and KilledRaidNPC

* Add etl for Playerevent::AA_purchase

Add etl for Playerevent::AA_purchase

* Cleanup before PR

* Review comment updates.

* Add world cli etl:settings to output a json on all player event details.

* Add reserve for all etl_queues
Correct a failed test case for improper next id for etl tables when table is first created.

* Potential solution for a dedicated database connection for player events.

* Simple thread for player_events.  Likely there is a better way to do this.

* Add zone to qs communications for recordplayerevents

First pass of enabling zone to qs direct transport to allow for PlayerEvents to bypass world.

* Cleanup a linux compile issue

* Add augments to LOOT ITEM and DESTROY ITEM

* Add augments to ITEMCREATION, FORAGESUCCESS, FISHSUCCESS, DESTROYITEM, LOOTITEM, DROPPEDITEM, TRADERPURCHASE, TRADERSELL, GUILDTRIBUTEDONATE and cleaned up the naming convention of augments

* Formatting fixes

* Swap out GetNextTableId

* Statically load counter

* Add counter.clear() since the counter is static

* Upload optional QS conversion scripts

* Remove all qs_tables and code referencing them

* Update database.cpp

* Simplify ProcessBatchQueue

* Simplify PorcessBatchQueue

* Simplify event truncation

* Build event truncation to bulk query by retention groups

* Post rebase

* Update player_events.h

* Fix build

* Update npc.cpp

* First pass of direct zone to qs sending for player events

* Remove keepalive logic

* Fix event ordering

* Cleanup

* Update player_event_logs.cpp

* Wipe event data after ETL processed

* Split up database connections, hot reload logs for QS

* Load rules from database vs qs_database

* Update player_event_logs.cpp

* Hot toggle queryserv connect

---------

Co-authored-by: Akkadius <akkadius1@gmail.com>
This commit is contained in:
Mitch Freeman
2025-02-05 04:02:16 -04:00
committed by GitHub
parent 21d27a1122
commit 8f4f8368df
99 changed files with 8156 additions and 2451 deletions
-21
View File
@@ -136,11 +136,6 @@ std::string DatabaseDumpService::GetLoginTableList()
return Strings::Join(DatabaseSchema::GetLoginTables(), " ");
}
std::string DatabaseDumpService::GetQueryServTables()
{
return Strings::Join(DatabaseSchema::GetQueryServerTables(), " ");
}
std::string DatabaseDumpService::GetSystemTablesList()
{
auto system_tables = DatabaseSchema::GetServerTables();
@@ -272,11 +267,6 @@ void DatabaseDumpService::DatabaseDump()
tables_to_dump += GetLoginTableList() + " ";
dump_descriptor += "-login";
}
if (IsDumpQueryServerTables()) {
tables_to_dump += GetQueryServTables();
dump_descriptor += "-queryserv";
}
}
if (IsDumpStaticInstanceData()) {
@@ -401,7 +391,6 @@ void DatabaseDumpService::DatabaseDump()
// LogDebug("[{}] dump-to-console", IsDumpOutputToConsole());
// LogDebug("[{}] dump-path", GetSetDumpPath());
// LogDebug("[{}] compression", (IsDumpWithCompression() ? "true" : "false"));
// LogDebug("[{}] query-serv", (IsDumpQueryServerTables() ? "true" : "false"));
// LogDebug("[{}] has-compression-binary", (HasCompressionBinary() ? "true" : "false"));
// LogDebug("[{}] content", (IsDumpContentTables() ? "true" : "false"));
// LogDebug("[{}] no-data", (IsDumpWithNoData() ? "true" : "false"));
@@ -511,16 +500,6 @@ const std::string &DatabaseDumpService::GetDumpFileName() const
return dump_file_name;
}
bool DatabaseDumpService::IsDumpQueryServerTables() const
{
return dump_query_server_tables;
}
void DatabaseDumpService::SetDumpQueryServerTables(bool dump_query_server_tables)
{
DatabaseDumpService::dump_query_server_tables = dump_query_server_tables;
}
bool DatabaseDumpService::IsDumpOutputToConsole() const
{
return dump_output_to_console;
-3
View File
@@ -45,8 +45,6 @@ public:
void SetDumpPath(const std::string &dump_path);
const std::string &GetDumpFileName() const;
void SetDumpFileName(const std::string &dump_file_name);
bool IsDumpQueryServerTables() const;
void SetDumpQueryServerTables(bool dump_query_server_tables);
bool IsDumpOutputToConsole() const;
void SetDumpOutputToConsole(bool dump_output_to_console);
bool IsDumpDropTableSyntaxOnly() const;
@@ -96,7 +94,6 @@ private:
bool HasCompressionBinary();
std::string GetDumpFileNameWithPath();
std::string GetSetDumpPath();
std::string GetQueryServTables();
void RemoveSqlBackup();
void BuildCredentialsFile();
void RemoveCredentialsFile();
@@ -6505,6 +6505,247 @@ ALTER TABLE `npc_types`
ADD COLUMN `multiquest_enabled` tinyint(1) UNSIGNED NOT NULL DEFAULT 0 AFTER `is_parcel_merchant`;
)",
.content_schema_update = true
},
ManifestEntry{
.version = 9301,
.description = "2024_10_08_add_detail_player_event_logging.sql",
.check = "SHOW COLUMNS FROM `player_event_log_settings` LIKE 'etl_enabled'",
.condition = "empty",
.match = "",
.sql = R"(
ALTER TABLE `player_event_log_settings`
ADD COLUMN `etl_enabled` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0' AFTER `discord_webhook_id`;
ALTER TABLE `player_event_logs`
ADD COLUMN `etl_table_id` BIGINT(20) NOT NULL DEFAULT '0' AFTER `event_data`;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 14;
CREATE TABLE `player_event_loot_items` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`item_id` INT(10) UNSIGNED NULL DEFAULT NULL,
`item_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`charges` INT(11) NULL DEFAULT NULL,
`augment_1_id` INT UNSIGNED NULL DEFAULT '0',
`augment_2_id` INT UNSIGNED NULL DEFAULT '0',
`augment_3_id` INT UNSIGNED NULL DEFAULT '0',
`augment_4_id` INT UNSIGNED NULL DEFAULT '0',
`augment_5_id` INT UNSIGNED NULL DEFAULT '0',
`augment_6_id` INT UNSIGNED NULL DEFAULT '0',
`npc_id` INT(10) UNSIGNED NULL DEFAULT NULL,
`corpse_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `item_id_npc_id` (`item_id`, `npc_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
AUTO_INCREMENT=1;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 16;
CREATE TABLE `player_event_merchant_sell` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`npc_id` INT(10) UNSIGNED NULL DEFAULT '0',
`merchant_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`merchant_type` INT(10) UNSIGNED NULL DEFAULT '0',
`item_id` INT(10) UNSIGNED NULL DEFAULT '0',
`item_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`charges` INT(11) NULL DEFAULT '0',
`cost` INT(10) UNSIGNED NULL DEFAULT '0',
`alternate_currency_id` INT(10) UNSIGNED NULL DEFAULT '0',
`player_money_balance` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`player_currency_balance` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `item_id_npc_id` (`item_id`, `npc_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
AUTO_INCREMENT=1;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 15;
CREATE TABLE `player_event_merchant_purchase` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`npc_id` INT(10) UNSIGNED NULL DEFAULT '0',
`merchant_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`merchant_type` INT(10) UNSIGNED NULL DEFAULT '0',
`item_id` INT(10) UNSIGNED NULL DEFAULT '0',
`item_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`charges` INT(11) NULL DEFAULT '0',
`cost` INT(10) UNSIGNED NULL DEFAULT '0',
`alternate_currency_id` INT(10) UNSIGNED NULL DEFAULT '0',
`player_money_balance` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`player_currency_balance` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `item_id_npc_id` (`item_id`, `npc_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
AUTO_INCREMENT=1;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 22;
CREATE TABLE `player_event_npc_handin` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`npc_id` INT(10) UNSIGNED NULL DEFAULT '0',
`npc_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`handin_copper` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`handin_silver` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`handin_gold` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`handin_platinum` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`return_copper` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`return_silver` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`return_gold` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`return_platinum` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`is_quest_handin` TINYINT(3) UNSIGNED NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `npc_id_is_quest_handin` (`npc_id`, `is_quest_handin`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
AUTO_INCREMENT=1;
CREATE TABLE `player_event_npc_handin_entries` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`player_event_npc_handin_id` BIGINT(20) UNSIGNED NOT NULL DEFAULT '0',
`type` INT(10) UNSIGNED NULL DEFAULT NULL,
`item_id` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`charges` INT(11) NOT NULL DEFAULT '0',
`evolve_level` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`evolve_amount` BIGINT(20) UNSIGNED NOT NULL DEFAULT '0',
`augment_1_id` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`augment_2_id` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`augment_3_id` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`augment_4_id` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`augment_5_id` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`augment_6_id` INT(10) UNSIGNED NOT NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `type_item_id` (`type`, `item_id`) USING BTREE,
INDEX `player_event_npc_handin_id` (`player_event_npc_handin_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
AUTO_INCREMENT=1;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 27;
CREATE TABLE `player_event_trade` (
`id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`char1_id` INT(10) UNSIGNED NULL DEFAULT '0',
`char2_id` INT(10) UNSIGNED NULL DEFAULT '0',
`char1_copper` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`char1_silver` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`char1_gold` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`char1_platinum` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`char2_copper` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`char2_silver` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`char2_gold` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`char2_platinum` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `char1_id_char2_id` (`char1_id`, `char2_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
AUTO_INCREMENT=1;
CREATE TABLE `player_event_trade_entries` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`player_event_trade_id` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`char_id` INT(10) UNSIGNED NULL DEFAULT '0',
`slot` SMALLINT(6) NULL DEFAULT '0',
`item_id` INT(10) UNSIGNED NULL DEFAULT '0',
`charges` SMALLINT(6) NULL DEFAULT '0',
`augment_1_id` INT(10) UNSIGNED NULL DEFAULT '0',
`augment_2_id` INT(10) UNSIGNED NULL DEFAULT '0',
`augment_3_id` INT(10) UNSIGNED NULL DEFAULT '0',
`augment_4_id` INT(10) UNSIGNED NULL DEFAULT '0',
`augment_5_id` INT(10) UNSIGNED NULL DEFAULT '0',
`augment_6_id` INT(10) UNSIGNED NULL DEFAULT '0',
`in_bag` TINYINT(4) NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `player_event_trade_id` (`player_event_trade_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
AUTO_INCREMENT=1;
UPDATE `player_event_log_settings` SET `etl_enabled` = 0 WHERE `id` = 54;
CREATE TABLE `player_event_speech` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`to_char_id` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`from_char_id` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`guild_id` INT(10) UNSIGNED NULL DEFAULT '0',
`type` INT(10) UNSIGNED NULL DEFAULT '0',
`min_status` INT(10) UNSIGNED NULL DEFAULT '0',
`message` LONGTEXT NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `to_char_id_from_char_id` (`to_char_id`, `from_char_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
AUTO_INCREMENT=1;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 44;
CREATE TABLE `player_event_killed_npc` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`npc_id` INT(10) UNSIGNED NULL DEFAULT '0',
`npc_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`combat_time_seconds` INT(10) UNSIGNED NULL DEFAULT '0',
`total_damage_per_second_taken` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`total_heal_per_second_taken` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `npc_id` (`npc_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 45;
CREATE TABLE `player_event_killed_named_npc` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`npc_id` INT(10) UNSIGNED NULL DEFAULT '0',
`npc_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`combat_time_seconds` INT(10) UNSIGNED NULL DEFAULT '0',
`total_damage_per_second_taken` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`total_heal_per_second_taken` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `npc_id` (`npc_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 46;
CREATE TABLE `player_event_killed_raid_npc` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`npc_id` INT(10) UNSIGNED NULL DEFAULT '0',
`npc_name` VARCHAR(64) NULL DEFAULT NULL COLLATE 'latin1_swedish_ci',
`combat_time_seconds` INT(10) UNSIGNED NULL DEFAULT '0',
`total_damage_per_second_taken` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`total_heal_per_second_taken` BIGINT(20) UNSIGNED NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `npc_id` (`npc_id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB;
UPDATE `player_event_log_settings` SET `etl_enabled` = 1 WHERE `id` = 4;
CREATE TABLE `player_event_aa_purchase` (
`id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
`aa_ability_id` INT(11) NULL DEFAULT '0',
`cost` INT(11) NULL DEFAULT '0',
`previous_id` INT(11) NULL DEFAULT '0',
`next_id` INT(11) NULL DEFAULT '0',
`created_at` DATETIME NULL DEFAULT NULL,
PRIMARY KEY (`id`) USING BTREE,
INDEX `created_at` (`created_at`) USING BTREE
)
COLLATE='latin1_swedish_ci'
ENGINE=InnoDB
;
)"
}
// -- template; copy/paste this when you need to create a new entry
// ManifestEntry{