diff --git a/common/database/database_update_manifest.cpp b/common/database/database_update_manifest.cpp index 856852e88..f309bdd81 100644 --- a/common/database/database_update_manifest.cpp +++ b/common/database/database_update_manifest.cpp @@ -6838,6 +6838,39 @@ DROP TABLE `expeditions`; RENAME TABLE `expedition_lockouts` TO `dynamic_zone_lockouts`; )" }, + ManifestEntry{ + .version = 9306, + .description = "2025_02_16_data_buckets_zone_id_instance_id.sql", + .check = "SHOW COLUMNS FROM `data_buckets` LIKE 'zone_id'", + .condition = "empty", + .match = "", + .sql = R"( +-- โœ… Drop old indexes +DROP INDEX IF EXISTS `keys` ON `data_buckets`; +DROP INDEX IF EXISTS `idx_npc_expires` ON `data_buckets`; +DROP INDEX IF EXISTS `idx_bot_expires` ON `data_buckets`; + +-- Add zone_id, instance_id +ALTER TABLE `data_buckets` + MODIFY COLUMN `npc_id` int(11) NOT NULL DEFAULT 0 AFTER `character_id`, + MODIFY COLUMN `bot_id` int(11) NOT NULL DEFAULT 0 AFTER `npc_id`, + ADD COLUMN `zone_id` smallint(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `bot_id`, + ADD COLUMN `instance_id` smallint(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `zone_id`; + +ALTER TABLE `data_buckets` + MODIFY COLUMN `account_id` bigint(11) UNSIGNED NULL DEFAULT 0 AFTER `expires`, + MODIFY COLUMN `character_id` bigint(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `account_id`, + MODIFY COLUMN `npc_id` int(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `character_id`, + MODIFY COLUMN `bot_id` int(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `npc_id`; + +-- โœ… Create optimized unique index with `key` first +CREATE UNIQUE INDEX `keys` ON data_buckets (`key`, character_id, npc_id, bot_id, account_id, zone_id, instance_id); + +-- โœ… Create indexes for just instance_id (instance deletion) +CREATE INDEX idx_instance_id ON data_buckets (instance_id); +)", + .content_schema_update = false + }, // -- template; copy/paste this when you need to create a new entry // ManifestEntry{ // .version = 9228, diff --git a/common/database_instances.cpp b/common/database_instances.cpp index 2e3b4fd0a..c595a0221 100644 --- a/common/database_instances.cpp +++ b/common/database_instances.cpp @@ -30,7 +30,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #include "../common/repositories/respawn_times_repository.h" #include "../common/repositories/spawn_condition_values_repository.h" #include "repositories/spawn2_disabled_repository.h" - +#include "repositories/data_buckets_repository.h" #include "database.h" @@ -479,6 +479,7 @@ void Database::DeleteInstance(uint16 instance_id) DynamicZoneMembersRepository::DeleteByInstance(*this, instance_id); DynamicZonesRepository::DeleteWhere(*this, fmt::format("instance_id = {}", instance_id)); CharacterCorpsesRepository::BuryInstance(*this, instance_id); + DataBucketsRepository::DeleteWhere(*this, fmt::format("instance_id = {}", instance_id)); } void Database::FlagInstanceByGroupLeader(uint32 zone_id, int16 version, uint32 character_id, uint32 group_id) diff --git a/common/repositories/base/base_data_buckets_repository.h b/common/repositories/base/base_data_buckets_repository.h index 42c85336c..e111eaabf 100644 --- a/common/repositories/base/base_data_buckets_repository.h +++ b/common/repositories/base/base_data_buckets_repository.h @@ -23,10 +23,12 @@ public: std::string key_; std::string value; uint32_t expires; - int64_t account_id; - int64_t character_id; - int64_t npc_id; - int64_t bot_id; + uint64_t account_id; + uint64_t character_id; + uint32_t npc_id; + uint32_t bot_id; + uint16_t zone_id; + uint16_t instance_id; // cereal template @@ -40,7 +42,9 @@ public: CEREAL_NVP(account_id), CEREAL_NVP(character_id), CEREAL_NVP(npc_id), - CEREAL_NVP(bot_id) + CEREAL_NVP(bot_id), + CEREAL_NVP(zone_id), + CEREAL_NVP(instance_id) ); } }; @@ -61,6 +65,8 @@ public: "character_id", "npc_id", "bot_id", + "zone_id", + "instance_id", }; } @@ -75,6 +81,8 @@ public: "character_id", "npc_id", "bot_id", + "zone_id", + "instance_id", }; } @@ -123,6 +131,8 @@ public: e.character_id = 0; e.npc_id = 0; e.bot_id = 0; + e.zone_id = 0; + e.instance_id = 0; return e; } @@ -163,10 +173,12 @@ public: e.key_ = row[1] ? row[1] : ""; e.value = row[2] ? row[2] : ""; e.expires = row[3] ? static_cast(strtoul(row[3], nullptr, 10)) : 0; - e.account_id = row[4] ? strtoll(row[4], nullptr, 10) : 0; - e.character_id = row[5] ? strtoll(row[5], nullptr, 10) : 0; - e.npc_id = row[6] ? strtoll(row[6], nullptr, 10) : 0; - e.bot_id = row[7] ? strtoll(row[7], nullptr, 10) : 0; + e.account_id = row[4] ? strtoull(row[4], nullptr, 10) : 0; + e.character_id = row[5] ? strtoull(row[5], nullptr, 10) : 0; + e.npc_id = row[6] ? static_cast(strtoul(row[6], nullptr, 10)) : 0; + e.bot_id = row[7] ? static_cast(strtoul(row[7], nullptr, 10)) : 0; + e.zone_id = row[8] ? static_cast(strtoul(row[8], nullptr, 10)) : 0; + e.instance_id = row[9] ? static_cast(strtoul(row[9], nullptr, 10)) : 0; return e; } @@ -207,6 +219,8 @@ public: v.push_back(columns[5] + " = " + std::to_string(e.character_id)); v.push_back(columns[6] + " = " + std::to_string(e.npc_id)); v.push_back(columns[7] + " = " + std::to_string(e.bot_id)); + v.push_back(columns[8] + " = " + std::to_string(e.zone_id)); + v.push_back(columns[9] + " = " + std::to_string(e.instance_id)); auto results = db.QueryDatabase( fmt::format( @@ -236,6 +250,8 @@ public: v.push_back(std::to_string(e.character_id)); v.push_back(std::to_string(e.npc_id)); v.push_back(std::to_string(e.bot_id)); + v.push_back(std::to_string(e.zone_id)); + v.push_back(std::to_string(e.instance_id)); auto results = db.QueryDatabase( fmt::format( @@ -273,6 +289,8 @@ public: v.push_back(std::to_string(e.character_id)); v.push_back(std::to_string(e.npc_id)); v.push_back(std::to_string(e.bot_id)); + v.push_back(std::to_string(e.zone_id)); + v.push_back(std::to_string(e.instance_id)); insert_chunks.push_back("(" + Strings::Implode(",", v) + ")"); } @@ -310,10 +328,12 @@ public: e.key_ = row[1] ? row[1] : ""; e.value = row[2] ? row[2] : ""; e.expires = row[3] ? static_cast(strtoul(row[3], nullptr, 10)) : 0; - e.account_id = row[4] ? strtoll(row[4], nullptr, 10) : 0; - e.character_id = row[5] ? strtoll(row[5], nullptr, 10) : 0; - e.npc_id = row[6] ? strtoll(row[6], nullptr, 10) : 0; - e.bot_id = row[7] ? strtoll(row[7], nullptr, 10) : 0; + e.account_id = row[4] ? strtoull(row[4], nullptr, 10) : 0; + e.character_id = row[5] ? strtoull(row[5], nullptr, 10) : 0; + e.npc_id = row[6] ? static_cast(strtoul(row[6], nullptr, 10)) : 0; + e.bot_id = row[7] ? static_cast(strtoul(row[7], nullptr, 10)) : 0; + e.zone_id = row[8] ? static_cast(strtoul(row[8], nullptr, 10)) : 0; + e.instance_id = row[9] ? static_cast(strtoul(row[9], nullptr, 10)) : 0; all_entries.push_back(e); } @@ -342,10 +362,12 @@ public: e.key_ = row[1] ? row[1] : ""; e.value = row[2] ? row[2] : ""; e.expires = row[3] ? static_cast(strtoul(row[3], nullptr, 10)) : 0; - e.account_id = row[4] ? strtoll(row[4], nullptr, 10) : 0; - e.character_id = row[5] ? strtoll(row[5], nullptr, 10) : 0; - e.npc_id = row[6] ? strtoll(row[6], nullptr, 10) : 0; - e.bot_id = row[7] ? strtoll(row[7], nullptr, 10) : 0; + e.account_id = row[4] ? strtoull(row[4], nullptr, 10) : 0; + e.character_id = row[5] ? strtoull(row[5], nullptr, 10) : 0; + e.npc_id = row[6] ? static_cast(strtoul(row[6], nullptr, 10)) : 0; + e.bot_id = row[7] ? static_cast(strtoul(row[7], nullptr, 10)) : 0; + e.zone_id = row[8] ? static_cast(strtoul(row[8], nullptr, 10)) : 0; + e.instance_id = row[9] ? static_cast(strtoul(row[9], nullptr, 10)) : 0; all_entries.push_back(e); } @@ -428,6 +450,8 @@ public: v.push_back(std::to_string(e.character_id)); v.push_back(std::to_string(e.npc_id)); v.push_back(std::to_string(e.bot_id)); + v.push_back(std::to_string(e.zone_id)); + v.push_back(std::to_string(e.instance_id)); auto results = db.QueryDatabase( fmt::format( @@ -458,6 +482,8 @@ public: v.push_back(std::to_string(e.character_id)); v.push_back(std::to_string(e.npc_id)); v.push_back(std::to_string(e.bot_id)); + v.push_back(std::to_string(e.zone_id)); + v.push_back(std::to_string(e.instance_id)); insert_chunks.push_back("(" + Strings::Implode(",", v) + ")"); } diff --git a/common/version.h b/common/version.h index 56989f56c..8c6e8c052 100644 --- a/common/version.h +++ b/common/version.h @@ -42,7 +42,7 @@ * Manifest: https://github.com/EQEmu/Server/blob/master/utils/sql/db_update_manifest.txt */ -#define CURRENT_BINARY_DATABASE_VERSION 9305 +#define CURRENT_BINARY_DATABASE_VERSION 9306 #define CURRENT_BINARY_BOTS_DATABASE_VERSION 9054 #endif diff --git a/zone/cli/benchmark_databuckets.cpp b/zone/cli/benchmark_databuckets.cpp new file mode 100644 index 000000000..f22533d92 --- /dev/null +++ b/zone/cli/benchmark_databuckets.cpp @@ -0,0 +1,277 @@ +#include +#include +#include +#include "../../common/http/httplib.h" +#include "../../common/eqemu_logsys.h" +#include "../sidecar_api/sidecar_api.h" +#include "../../common/platform.h" +#include "../data_bucket.h" +#include "../zonedb.h" +#include "../../common/repositories/data_buckets_repository.h" + +void RunBenchmarkCycle(uint64_t target_rows) +{ + const size_t OPERATIONS_PER_TEST = 5000; + const std::string test_key_prefix = "test_key_"; + + std::cout << Strings::Repeat("-", 70) << "\n"; + std::cout << "๐Ÿ“Š Running Benchmark at " << Strings::Commify(target_rows) << " Rows...\n"; + std::cout << Strings::Repeat("-", 70) << "\n"; + + // ๐Ÿงน **Purge `test_key_*` Keys Before Each Run** + std::cout << "๐Ÿงน Purging test keys (`test_key_*`)...\n"; + auto purge_start = std::chrono::high_resolution_clock::now(); + DataBucketsRepository::DeleteWhere(database, "`key` LIKE '" + test_key_prefix + "%'"); + auto purge_end = std::chrono::high_resolution_clock::now(); + std::chrono::duration purge_time = purge_end - purge_start; + std::cout << "โœ… Purged test keys in " << purge_time.count() << " seconds.\n"; + + // ๐Ÿ“Š **Ensure the Table Contains At Least `target_rows`** + auto populate_start = std::chrono::high_resolution_clock::now(); + uint64_t current_count = DataBucketsRepository::Count(database); + if (current_count < target_rows) { + std::cout << "๐Ÿ“Œ Populating table to " << Strings::Commify(target_rows) << " rows...\n"; + std::mt19937 rng(std::random_device{}()); + std::uniform_int_distribution entity_type(1, 5); + std::uniform_int_distribution id_dist(1, 1000000); + std::uniform_int_distribution expiry_dist(0, 86400 * 30); // Expiry up to 30 days + + while (current_count < target_rows) { + std::vector batch; + for (size_t i = 0; i < 100000; ++i) { + if (i > target_rows - current_count) { + break; + } + + int entity_choice = entity_type(rng); + int entity_id = id_dist(rng); + std::string key = "test_key_" + std::to_string(current_count + i); + std::string value = "value_" + std::to_string(current_count + i); + int expires = static_cast(std::time(nullptr)) + expiry_dist(rng); + + DataBucketsRepository::DataBuckets e{}; + e.key_ = key; + e.value = value; + e.expires = expires; + e.account_id = (entity_choice == 1) ? entity_id : 0; + e.character_id = (entity_choice == 2) ? entity_id : 0; + e.npc_id = (entity_choice == 3) ? entity_id : 0; + e.bot_id = (entity_choice == 4) ? entity_id : 0; + e.zone_id = (entity_choice == 5) ? entity_id : 0; + e.instance_id = (entity_choice == 5) ? entity_id : 0; + + batch.emplace_back(e); + } + DataBucketsRepository::InsertMany(database, batch); + current_count += batch.size(); + } + } + else { + std::cout << "โœ… Table already has " << current_count << " rows, proceeding with benchmark.\n"; + } + + auto populate_end = std::chrono::high_resolution_clock::now(); + std::chrono::duration populate_time = populate_end - populate_start; + std::cout << "โœ… Populated table in " << populate_time.count() << " seconds.\n"; + + std::mt19937 rng(std::random_device{}()); + std::uniform_int_distribution id_dist(1, 1000); + + // ๐Ÿš€ **Measure Insert Performance** + std::vector inserted_keys = {}; + auto insert_start = std::chrono::high_resolution_clock::now(); + for (size_t i = 0; i < OPERATIONS_PER_TEST; ++i) { + std::string key = test_key_prefix + std::to_string(current_count + i); + std::string value = "value_" + std::to_string(current_count + i); + int expires = static_cast(std::time(nullptr)) + 3600; + + DataBucketKey e{ + .key = key, + .value = value, + .expires = std::to_string(expires), + .account_id = 0, + .character_id = 0, + .npc_id = 0, + .bot_id = 0 + }; + + // randomly set account_id, character_id, npc_id, or bot_id + switch (i % 4) { + case 0: + e.account_id = id_dist(rng); + break; + case 1: + e.character_id = id_dist(rng); + break; + case 2: + e.npc_id = id_dist(rng); + break; + case 3: + e.bot_id = id_dist(rng); + break; + case 4: + int entity_choice = id_dist(rng); + e.zone_id = entity_choice; + e.instance_id = entity_choice; + break; + } + + DataBucket::SetData(e); + + inserted_keys.emplace_back(e); + } + auto insert_end = std::chrono::high_resolution_clock::now(); + std::chrono::duration insert_time = insert_end - insert_start; + std::cout << "โœ… Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " inserts in " << insert_time.count() + << " seconds. (Individual Insert Performance)\n"; + + // โœ๏ธ **Measure Update Performance Using DataBucket** + auto update_start = std::chrono::high_resolution_clock::now(); + for (auto &key: inserted_keys) { + // ๐Ÿ” Retrieve existing bucket using scoped `GetData` + auto e = DataBucket::GetData(key); + if (e.id > 0) { + // create a new key object with the updated values + DataBucketKey bucket_entry_key{ + .key = e.key_, + .value = "some_new_value", + .expires = std::to_string(e.expires), + .account_id = e.account_id, + .character_id = e.character_id, + .npc_id = e.npc_id, + .bot_id = e.bot_id, + .zone_id = e.zone_id, + .instance_id = e.instance_id + }; + + // ๐Ÿ”„ Update using DataBucket class + DataBucket::SetData(bucket_entry_key); + } + } + auto update_end = std::chrono::high_resolution_clock::now(); + std::chrono::duration update_time = update_end - update_start; + std::cout << "โœ… Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " updates in " << update_time.count() + << " seconds. (Scoped Update Performance)\n"; + + + // ๐Ÿ” **Measure Cached Read Performance** + auto read_cached_start = std::chrono::high_resolution_clock::now(); + for (size_t i = 0; i < OPERATIONS_PER_TEST; ++i) { + std::string key = test_key_prefix + std::to_string(current_count + i); + DataBucketKey k{ + .key = key, + .account_id = 0, + .character_id = 0, + .npc_id = 0, + .bot_id = 0, + .zone_id = 0, + .instance_id = 0 + }; + + // randomly set account_id, character_id, npc_id, or bot_id + switch (i % 4) { + case 0: + k.account_id = id_dist(rng); + break; + case 1: + k.character_id = id_dist(rng); + break; + case 2: + k.npc_id = id_dist(rng); + break; + case 3: + k.bot_id = id_dist(rng); + break; + case 4: + int entity_choice = id_dist(rng); + k.zone_id = entity_choice; + k.instance_id = entity_choice; + } + + DataBucket::GetData(key); + } + auto read_cached_end = std::chrono::high_resolution_clock::now(); + std::chrono::duration read_cached_time = read_cached_end - read_cached_start; + std::cout << "โœ… Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " cached reads in " + << read_cached_time.count() << " seconds. (DataBucket::GetData)\n"; + + // ๐Ÿ” **Measure Non-Cached Read Performance (Direct Query)** + auto read_uncached_start = std::chrono::high_resolution_clock::now(); + for (size_t i = 0; i < OPERATIONS_PER_TEST; ++i) { + std::string key = test_key_prefix + std::to_string(current_count + i); + DataBucketsRepository::GetWhere(database, "`key` = '" + key + "'"); + } + auto read_uncached_end = std::chrono::high_resolution_clock::now(); + std::chrono::duration read_uncached_time = read_uncached_end - read_uncached_start; + std::cout << "โœ… Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " non-cached reads in " + << read_uncached_time.count() << " seconds. (DataBucketsRepository::GetWhere)\n"; + + // ๐Ÿ—‘๏ธ **Measure Delete Performance** + auto delete_start = std::chrono::high_resolution_clock::now(); + for (size_t i = 0; i < OPERATIONS_PER_TEST; ++i) { + std::string key = test_key_prefix + std::to_string(current_count + i); + + DataBucketKey k{ + .key = key, + .account_id = 0, + .character_id = 0, + .npc_id = 0, + .bot_id = 0, + .zone_id = 0, + .instance_id = 0 + }; + + // randomly set account_id, character_id, npc_id, or bot_id + switch (i % 4) { + case 0: + k.account_id = id_dist(rng); + break; + case 1: + k.character_id = id_dist(rng); + break; + case 2: + k.npc_id = id_dist(rng); + break; + case 3: + k.bot_id = id_dist(rng); + break; + case 4: + int entity_choice = id_dist(rng); + k.zone_id = entity_choice; + k.instance_id = entity_choice; + } + + DataBucket::DeleteData(k); + } + auto delete_end = std::chrono::high_resolution_clock::now(); + std::chrono::duration delete_time = delete_end - delete_start; + std::cout << "โœ… Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " deletes in " << delete_time.count() + << " seconds.\n"; +} + +void ZoneCLI::BenchmarkDatabuckets(int argc, char **argv, argh::parser &cmd, std::string &description) +{ + description = "Benchmark individual reads/writes/deletes in data_buckets at different table sizes."; + + if (cmd[{"-h", "--help"}]) { + std::cout << "Usage: BenchmarkDatabuckets\n"; + return; + } + + if (std::getenv("DEBUG")) { + LogSys.SetDatabase(&database)->LoadLogDatabaseSettings(); + } + + auto start_time = std::chrono::high_resolution_clock::now(); + + std::vector benchmark_sizes = {10000, 100000, 1000000}; + + for (auto size: benchmark_sizes) { + RunBenchmarkCycle(size); + } + + // ๐Ÿš€ **Total Benchmark Time** + auto end_time = std::chrono::high_resolution_clock::now(); + std::chrono::duration total_elapsed = end_time - start_time; + std::cout << "\n๐Ÿš€ Total Benchmark Time: " << total_elapsed.count() << " seconds\n"; +} diff --git a/zone/data_bucket.cpp b/zone/data_bucket.cpp index 8aeadef0c..1f0c377e0 100644 --- a/zone/data_bucket.cpp +++ b/zone/data_bucket.cpp @@ -19,10 +19,6 @@ void DataBucket::SetData(const std::string &bucket_key, const std::string &bucke .key = bucket_key, .value = bucket_value, .expires = expires_time, - .account_id = 0, - .character_id = 0, - .npc_id = 0, - .bot_id = 0 }; DataBucket::SetData(k); @@ -54,6 +50,9 @@ void DataBucket::SetData(const DataBucketKey &k_) } else if (k.bot_id > 0) { b.bot_id = k.bot_id; + } else if (k.zone_id > 0) { + b.zone_id = k.zone_id; + b.instance_id = k.instance_id; } const uint64 bucket_id = b.id; @@ -189,12 +188,14 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k_, } LogDataBuckets( - "Getting bucket key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}]", + "Getting bucket key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}] zone_id [{}] instance_id [{}]", k.key, k.bot_id, k.account_id, k.character_id, - k.npc_id + k.npc_id, + k.zone_id, + k.instance_id ); bool can_cache = CanCache(k); @@ -244,17 +245,21 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k_, .account_id = k.account_id, .character_id = k.character_id, .npc_id = k.npc_id, - .bot_id = k.bot_id + .bot_id = k.bot_id, + .zone_id = k.zone_id, + .instance_id = k.instance_id } ); LogDataBuckets( - "Key [{}] not found in database, adding to cache as a miss account_id [{}] character_id [{}] npc_id [{}] bot_id [{}] cache size before [{}] after [{}]", + "Key [{}] not found in database, adding to cache as a miss account_id [{}] character_id [{}] npc_id [{}] bot_id [{}] zone_id [{}] instance_id [{}] cache size before [{}] after [{}]", k.key, k.account_id, k.character_id, k.npc_id, k.bot_id, + k.zone_id, + k.instance_id, size_before, g_data_bucket_cache.size() ); @@ -347,12 +352,15 @@ bool DataBucket::DeleteData(const DataBucketKey &k) ); LogDataBuckets( - "Deleting bucket key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}] cache size before [{}] after [{}]", + "Deleting bucket key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}] bot_id [{}] zone_id [{}] instance_id [{}] cache size before [{}] after [{}]", k.key, k.bot_id, k.account_id, k.character_id, k.npc_id, + k.bot_id, + k.zone_id, + k.instance_id, size_before, g_data_bucket_cache.size() ); @@ -390,12 +398,15 @@ std::string DataBucket::GetDataExpires(const DataBucketKey &k) std::string DataBucket::GetDataRemaining(const DataBucketKey &k) { LogDataBuckets( - "Getting bucket remaining key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}]", + "Getting bucket remaining key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}] bot_id [{}] zone_id [{}] instance_id [{}]", k.key, k.bot_id, k.account_id, k.character_id, - k.npc_id + k.npc_id, + k.bot_id, + k.zone_id, + k.instance_id ); auto r = GetData(k); @@ -408,39 +419,46 @@ std::string DataBucket::GetDataRemaining(const DataBucketKey &k) std::string DataBucket::GetScopedDbFilters(const DataBucketKey &k) { - std::vector query = {}; + std::vector q = {}; if (k.character_id > 0) { - query.emplace_back(fmt::format("character_id = {}", k.character_id)); + q.emplace_back(fmt::format("character_id = {}", k.character_id)); } else { - query.emplace_back("character_id = 0"); + q.emplace_back("character_id = 0"); } if (k.account_id > 0) { - query.emplace_back(fmt::format("account_id = {}", k.account_id)); + q.emplace_back(fmt::format("account_id = {}", k.account_id)); } else { - query.emplace_back("account_id = 0"); + q.emplace_back("account_id = 0"); } if (k.npc_id > 0) { - query.emplace_back(fmt::format("npc_id = {}", k.npc_id)); + q.emplace_back(fmt::format("npc_id = {}", k.npc_id)); } else { - query.emplace_back("npc_id = 0"); + q.emplace_back("npc_id = 0"); } if (k.bot_id > 0) { - query.emplace_back(fmt::format("bot_id = {}", k.bot_id)); + q.emplace_back(fmt::format("bot_id = {}", k.bot_id)); } else { - query.emplace_back("bot_id = 0"); + q.emplace_back("bot_id = 0"); + } + + if (k.zone_id > 0) { + q.emplace_back(fmt::format("zone_id = {} AND instance_id = {}", k.zone_id, k.instance_id)); + } + else { + q.emplace_back("zone_id = 0 AND instance_id = 0"); } return fmt::format( "{} {}", - Strings::Join(query, " AND "), - !query.empty() ? "AND" : "" + Strings::Join(q, " AND "), + !q.empty() ? "AND" : "" ); } @@ -451,7 +469,52 @@ bool DataBucket::CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, dbe.bot_id == k.bot_id && dbe.account_id == k.account_id && dbe.character_id == k.character_id && - dbe.npc_id == k.npc_id + dbe.npc_id == k.npc_id && + dbe.zone_id == k.zone_id && + dbe.instance_id == k.instance_id + ); +} + +void DataBucket::LoadZoneCache(uint16 zone_id, uint16 instance_id) +{ + const auto &l = DataBucketsRepository::GetWhere( + database, + fmt::format( + "zone_id = {} AND instance_id = {} AND (`expires` > {} OR `expires` = 0)", + zone_id, + instance_id, + (long long) std::time(nullptr) + ) + ); + + if (l.empty()) { + return; + } + + LogDataBucketsDetail("cache size before [{}] l size [{}]", g_data_bucket_cache.size(), l.size()); + + uint32 added_count = 0; + + for (const auto &e: l) { + if (!ExistsInCache(e)) { + added_count++; + } + } + + for (const auto &e: l) { + if (!ExistsInCache(e)) { + LogDataBucketsDetail("bucket id [{}] bucket key [{}] bucket value [{}]", e.id, e.key_, e.value); + + g_data_bucket_cache.emplace_back(e); + } + } + + LogDataBucketsDetail("cache size after [{}]", g_data_bucket_cache.size()); + + LogDataBuckets( + "Loaded [{}] zone keys new cache size is [{}]", + l.size(), + g_data_bucket_cache.size() ); } @@ -541,7 +604,7 @@ void DataBucket::BulkLoadEntitiesToCache(DataBucketLoadType::Type t, std::vector ); } -void DataBucket::DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id) +void DataBucket::DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id, uint32 secondary_id) { size_t size_before = g_data_bucket_cache.size(); @@ -553,7 +616,8 @@ void DataBucket::DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id) return ( (type == DataBucketLoadType::Bot && e.bot_id == id) || (type == DataBucketLoadType::Account && e.account_id == id) || - (type == DataBucketLoadType::Client && e.character_id == id) + (type == DataBucketLoadType::Client && e.character_id == id) || + (type == DataBucketLoadType::Zone && e.zone_id == id && e.instance_id == secondary_id) ); } ), @@ -595,7 +659,9 @@ void DataBucket::DeleteFromMissesCache(DataBucketsRepository::DataBuckets e) ce.account_id == e.account_id && ce.character_id == e.character_id && ce.npc_id == e.npc_id && - ce.bot_id == e.bot_id; + ce.bot_id == e.bot_id && + ce.zone_id == e.zone_id && + ce.instance_id == e.instance_id; } ), g_data_bucket_cache.end() @@ -647,13 +713,42 @@ void DataBucket::DeleteFromCache(uint64 id, DataBucketLoadType::Type type) ); } +void DataBucket::DeleteZoneFromCache(uint16 zone_id, uint16 instance_id, DataBucketLoadType::Type type) +{ + size_t size_before = g_data_bucket_cache.size(); + + g_data_bucket_cache.erase( + std::remove_if( + g_data_bucket_cache.begin(), + g_data_bucket_cache.end(), + [&](DataBucketsRepository::DataBuckets &e) { + switch (type) { + case DataBucketLoadType::Zone: + return e.zone_id == zone_id && e.instance_id == instance_id; + default: + return false; + } + } + ), + g_data_bucket_cache.end() + ); + + LogDataBuckets( + "Deleted zone [{}] instance [{}] from cache size before [{}] after [{}]", + zone_id, + instance_id, + size_before, + g_data_bucket_cache.size() + ); +} + // CanCache returns whether a bucket can be cached or not // characters are only in one zone at a time so we can cache locally to the zone // bots (not implemented) are only in one zone at a time so we can cache locally to the zone // npcs (ids) can be in multiple zones so we can't cache locally to the zone bool DataBucket::CanCache(const DataBucketKey &key) { - if (key.character_id > 0 || key.account_id > 0 || key.bot_id > 0) { + if (key.character_id > 0 || key.account_id > 0 || key.bot_id > 0 || key.zone_id > 0) { return true; } diff --git a/zone/data_bucket.h b/zone/data_bucket.h index 1c4946442..ca2904a43 100644 --- a/zone/data_bucket.h +++ b/zone/data_bucket.h @@ -12,10 +12,12 @@ struct DataBucketKey { std::string key; std::string value; std::string expires; - int64_t account_id = 0; - int64_t character_id = 0; - int64_t npc_id = 0; - int64_t bot_id = 0; + uint64_t account_id = 0; + uint64_t character_id = 0; + uint32_t npc_id = 0; + uint32_t bot_id = 0; + uint16_t zone_id = 0; + uint16_t instance_id = 0; }; namespace DataBucketLoadType { @@ -23,6 +25,7 @@ namespace DataBucketLoadType { Bot, Account, Client, + Zone, MaxType }; @@ -30,6 +33,7 @@ namespace DataBucketLoadType { "Bot", "Account", "Client", + "Zone" }; } @@ -56,12 +60,14 @@ public: static bool CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, const DataBucketKey &k); static bool ExistsInCache(const DataBucketsRepository::DataBuckets &entry); + static void LoadZoneCache(uint16 zone_id, uint16 instance_id); static void BulkLoadEntitiesToCache(DataBucketLoadType::Type t, std::vector ids); - static void DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id); + static void DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id, uint32 secondary_id = 0); static void DeleteFromMissesCache(DataBucketsRepository::DataBuckets e); static void ClearCache(); static void DeleteFromCache(uint64 id, DataBucketLoadType::Type type); + static void DeleteZoneFromCache(uint16 zone_id, uint16 instance_id, DataBucketLoadType::Type type); static bool CanCache(const DataBucketKey &key); static DataBucketsRepository::DataBuckets ExtractNestedValue(const DataBucketsRepository::DataBuckets &bucket, const std::string &full_key); diff --git a/zone/lua_zone.cpp b/zone/lua_zone.cpp index 21d45b213..cfc709bed 100644 --- a/zone/lua_zone.cpp +++ b/zone/lua_zone.cpp @@ -685,6 +685,42 @@ void Lua_Zone::ShowZoneGlobalLoot(Lua_Client c) self->ShowZoneGlobalLoot(c); } +void Lua_Zone::SetBucket(const std::string& bucket_name, const std::string& bucket_value) +{ + Lua_Safe_Call_Void(); + self->SetBucket(bucket_name, bucket_value); +} + +void Lua_Zone::SetBucket(const std::string& bucket_name, const std::string& bucket_value, const std::string& expiration) +{ + Lua_Safe_Call_Void(); + self->SetBucket(bucket_name, bucket_value, expiration); +} + +void Lua_Zone::DeleteBucket(const std::string& bucket_name) +{ + Lua_Safe_Call_Void(); + self->DeleteBucket(bucket_name); +} + +std::string Lua_Zone::GetBucket(const std::string& bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucket(bucket_name); +} + +std::string Lua_Zone::GetBucketExpires(const std::string& bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucketExpires(bucket_name); +} + +std::string Lua_Zone::GetBucketRemaining(const std::string& bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucketRemaining(bucket_name); +} + luabind::scope lua_register_zone() { return luabind::class_("Zone") .def(luabind::constructor<>()) @@ -695,11 +731,15 @@ luabind::scope lua_register_zone() { .def("CanDoCombat", &Lua_Zone::CanDoCombat) .def("CanLevitate", &Lua_Zone::CanLevitate) .def("ClearSpawnTimers", &Lua_Zone::ClearSpawnTimers) + .def("DeleteBucket", (void(Lua_Zone::*)(const std::string&))&Lua_Zone::DeleteBucket) .def("Depop", (void(Lua_Zone::*)(void))&Lua_Zone::Depop) .def("Depop", (void(Lua_Zone::*)(bool))&Lua_Zone::Depop) .def("Despawn", &Lua_Zone::Despawn) .def("GetAAEXPModifier", &Lua_Zone::GetAAEXPModifier) .def("GetAAEXPModifierByCharacterID", &Lua_Zone::GetAAEXPModifierByCharacterID) + .def("GetBucket", (std::string(Lua_Zone::*)(const std::string&))&Lua_Zone::GetBucket) + .def("GetBucketExpires", (std::string(Lua_Zone::*)(const std::string&))&Lua_Zone::GetBucketExpires) + .def("GetBucketRemaining", (std::string(Lua_Zone::*)(const std::string&))&Lua_Zone::GetBucketRemaining) .def("GetContentFlags", &Lua_Zone::GetContentFlags) .def("GetContentFlagsDisabled", &Lua_Zone::GetContentFlagsDisabled) .def("GetExperienceMultiplier", &Lua_Zone::GetExperienceMultiplier) @@ -795,6 +835,8 @@ luabind::scope lua_register_zone() { .def("Repop", (void(Lua_Zone::*)(bool))&Lua_Zone::Repop) .def("SetAAEXPModifier", &Lua_Zone::SetAAEXPModifier) .def("SetAAEXPModifierByCharacterID", &Lua_Zone::SetAAEXPModifierByCharacterID) + .def("SetBucket", (void(Lua_Zone::*)(const std::string&,const std::string&))&Lua_Zone::SetBucket) + .def("SetBucket", (void(Lua_Zone::*)(const std::string&,const std::string&,const std::string&))&Lua_Zone::SetBucket) .def("SetEXPModifier", &Lua_Zone::SetEXPModifier) .def("SetEXPModifierByCharacterID", &Lua_Zone::SetEXPModifierByCharacterID) .def("SetInstanceTimer", &Lua_Zone::SetInstanceTimer) diff --git a/zone/lua_zone.h b/zone/lua_zone.h index a2dbb85b4..de670e1b0 100644 --- a/zone/lua_zone.h +++ b/zone/lua_zone.h @@ -141,6 +141,14 @@ public: void SetIsHotzone(bool is_hotzone); void ShowZoneGlobalLoot(Lua_Client c); + // data buckets + void SetBucket(const std::string& bucket_name, const std::string& bucket_value); + void SetBucket(const std::string& bucket_name, const std::string& bucket_value, const std::string& expiration = ""); + void DeleteBucket(const std::string& bucket_name); + std::string GetBucket(const std::string& bucket_name); + std::string GetBucketExpires(const std::string& bucket_name); + std::string GetBucketRemaining(const std::string& bucket_name); + }; #endif diff --git a/zone/perl_zone.cpp b/zone/perl_zone.cpp index 52e977651..d799e1240 100644 --- a/zone/perl_zone.cpp +++ b/zone/perl_zone.cpp @@ -526,6 +526,36 @@ void Perl_Zone_ShowZoneGlobalLoot(Zone* self, Client* c) self->ShowZoneGlobalLoot(c); } +void Perl_Zone_SetBucket(Zone* self, const std::string bucket_name, const std::string bucket_value) +{ + self->SetBucket(bucket_name, bucket_value); +} + +void Perl_Zone_SetBucket(Zone* self, const std::string bucket_name, const std::string bucket_value, const std::string expiration) +{ + self->SetBucket(bucket_name, bucket_value, expiration); +} + +void Perl_Zone_DeleteBucket(Zone* self, const std::string bucket_name) +{ + self->DeleteBucket(bucket_name); +} + +std::string Perl_Zone_GetBucket(Zone* self, const std::string bucket_name) +{ + return self->GetBucket(bucket_name); +} + +std::string Perl_Zone_GetBucketExpires(Zone* self, const std::string bucket_name) +{ + return self->GetBucketExpires(bucket_name); +} + +std::string Perl_Zone_GetBucketRemaining(Zone* self, const std::string bucket_name) +{ + return self->GetBucketRemaining(bucket_name); +} + void perl_register_zone() { perl::interpreter perl(PERL_GET_THX); @@ -538,11 +568,15 @@ void perl_register_zone() package.add("CanDoCombat", &Perl_Zone_CanDoCombat); package.add("CanLevitate", &Perl_Zone_CanLevitate); package.add("ClearSpawnTimers", &Perl_Zone_ClearSpawnTimers); + package.add("DeleteBucket", &Perl_Zone_DeleteBucket); package.add("Depop", (void(*)(Zone*))&Perl_Zone_Depop); package.add("Depop", (void(*)(Zone*, bool))&Perl_Zone_Depop); package.add("Despawn", &Perl_Zone_Despawn); package.add("GetAAEXPModifier", &Perl_Zone_GetAAEXPModifier); package.add("GetAAEXPModifierByCharacterID", &Perl_Zone_GetAAEXPModifierByCharacterID); + package.add("GetBucket", &Perl_Zone_GetBucket); + package.add("GetBucketExpires", &Perl_Zone_GetBucketExpires); + package.add("GetBucketRemaining", &Perl_Zone_GetBucketRemaining); package.add("GetContentFlags", &Perl_Zone_GetContentFlags); package.add("GetContentFlagsDisabled", &Perl_Zone_GetContentFlagsDisabled); package.add("GetExperienceMultiplier", &Perl_Zone_GetExperienceMultiplier); @@ -638,6 +672,8 @@ void perl_register_zone() package.add("Repop", (void(*)(Zone*, bool))&Perl_Zone_Repop); package.add("SetAAEXPModifier", &Perl_Zone_SetAAEXPModifier); package.add("SetAAEXPModifierByCharacterID", &Perl_Zone_SetAAEXPModifierByCharacterID); + package.add("SetBucket", (void(*)(Zone*, const std::string, const std::string))&Perl_Zone_SetBucket); + package.add("SetBucket", (void(*)(Zone*, const std::string, const std::string, const std::string))&Perl_Zone_SetBucket); package.add("SetEXPModifier", &Perl_Zone_SetEXPModifier); package.add("SetEXPModifierByCharacterID", &Perl_Zone_SetEXPModifierByCharacterID); package.add("SetInstanceTimer", &Perl_Zone_SetInstanceTimer); diff --git a/zone/zone.cpp b/zone/zone.cpp index c6d029189..8b1639ba4 100644 --- a/zone/zone.cpp +++ b/zone/zone.cpp @@ -172,6 +172,8 @@ bool Zone::Bootup(uint32 iZoneID, uint32 iInstanceID, bool is_static) { zone->RequestUCSServerStatus(); zone->StartShutdownTimer(); + DataBucket::LoadZoneCache(iZoneID, iInstanceID); + /* * Set Logging */ @@ -877,6 +879,8 @@ void Zone::Shutdown(bool quiet) return; } + DataBucket::DeleteCachedBuckets(DataBucketLoadType::Zone, zone->GetZoneID(), zone->GetInstanceID()); + entity_list.StopMobAI(); std::map::iterator itr; @@ -3187,4 +3191,56 @@ bool Zone::DoesAlternateCurrencyExist(uint32 currency_id) ); } +std::string Zone::GetBucket(const std::string& bucket_name) +{ + DataBucketKey k = {}; + k.zone_id = zoneid; + k.instance_id = instanceid; + k.key = bucket_name; + + return DataBucket::GetData(k).value; +} + +void Zone::SetBucket(const std::string& bucket_name, const std::string& bucket_value, const std::string& expiration) +{ + DataBucketKey k = {}; + k.zone_id = zoneid; + k.instance_id = instanceid; + k.key = bucket_name; + k.expires = expiration; + k.value = bucket_value; + + DataBucket::SetData(k); +} + +void Zone::DeleteBucket(const std::string& bucket_name) +{ + DataBucketKey k = {}; + k.zone_id = zoneid; + k.instance_id = instanceid; + k.key = bucket_name; + + DataBucket::DeleteData(k); +} + +std::string Zone::GetBucketExpires(const std::string& bucket_name) +{ + DataBucketKey k = {}; + k.zone_id = zoneid; + k.instance_id = instanceid; + k.key = bucket_name; + + return DataBucket::GetDataExpires(k); +} + +std::string Zone::GetBucketRemaining(const std::string& bucket_name) +{ + DataBucketKey k = {}; + k.zone_id = zoneid; + k.instance_id = instanceid; + k.key = bucket_name; + + return DataBucket::GetDataRemaining(k); +} + #include "zone_loot.cpp" diff --git a/zone/zone.h b/zone/zone.h index 967d87f79..4d78a5b2b 100755 --- a/zone/zone.h +++ b/zone/zone.h @@ -451,6 +451,12 @@ public: void LoadBaseData(); void ReloadBaseData(); + // data buckets + std::string GetBucket(const std::string& bucket_name); + void SetBucket(const std::string& bucket_name, const std::string& bucket_value, const std::string& expiration = ""); + void DeleteBucket(const std::string& bucket_name); + std::string GetBucketExpires(const std::string& bucket_name); + std::string GetBucketRemaining(const std::string& bucket_name); private: bool allow_mercs; diff --git a/zone/zone_cli.cpp b/zone/zone_cli.cpp index 77fab2d77..8bcb86e24 100644 --- a/zone/zone_cli.cpp +++ b/zone/zone_cli.cpp @@ -29,6 +29,7 @@ void ZoneCLI::CommandHandler(int argc, char **argv) auto function_map = EQEmuCommand::function_map; // Register commands + function_map["benchmark:databuckets"] = &ZoneCLI::BenchmarkDatabuckets; function_map["sidecar:serve-http"] = &ZoneCLI::SidecarServeHttp; function_map["tests:npc-handins"] = &ZoneCLI::NpcHandins; function_map["tests:npc-handins-multiquest"] = &ZoneCLI::NpcHandinsMultiQuest; @@ -36,6 +37,7 @@ void ZoneCLI::CommandHandler(int argc, char **argv) EQEmuCommand::HandleMenu(function_map, cmd, argc, argv); } +#include "cli/benchmark_databuckets.cpp" #include "cli/sidecar_serve_http.cpp" #include "cli/npc_handins.cpp" #include "cli/npc_handins_multiquest.cpp" diff --git a/zone/zone_cli.h b/zone/zone_cli.h index e2cf00a55..2f46d8297 100644 --- a/zone/zone_cli.h +++ b/zone/zone_cli.h @@ -6,6 +6,7 @@ class ZoneCLI { public: static void CommandHandler(int argc, char **argv); + static void BenchmarkDatabuckets(int argc, char **argv, argh::parser &cmd, std::string &description); static void SidecarServeHttp(int argc, char **argv, argh::parser &cmd, std::string &description); static bool RanConsoleCommand(int argc, char **argv); static bool RanSidecarCommand(int argc, char **argv);