[Databuckets] Add Zone Scoped Databuckets (#4690)

* [Databuckets] Add Zone Scoped Databuckets

* Add database indexes

* Update database_update_manifest.cpp

* Shutdown fix

* Testing

* Perf boost

* Revert "Perf boost"

This reverts commit 55d3e507d30719fc3f631806a4c5d09bc65430a1.

* Update data_bucket.cpp
This commit is contained in:
Chris Miles 2025-02-18 00:14:49 -06:00 committed by GitHub
parent 8315240b17
commit 49cf97ae9c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 640 additions and 51 deletions

View File

@ -6838,6 +6838,39 @@ DROP TABLE `expeditions`;
RENAME TABLE `expedition_lockouts` TO `dynamic_zone_lockouts`;
)"
},
ManifestEntry{
.version = 9306,
.description = "2025_02_16_data_buckets_zone_id_instance_id.sql",
.check = "SHOW COLUMNS FROM `data_buckets` LIKE 'zone_id'",
.condition = "empty",
.match = "",
.sql = R"(
-- Drop old indexes
DROP INDEX IF EXISTS `keys` ON `data_buckets`;
DROP INDEX IF EXISTS `idx_npc_expires` ON `data_buckets`;
DROP INDEX IF EXISTS `idx_bot_expires` ON `data_buckets`;
-- Add zone_id, instance_id
ALTER TABLE `data_buckets`
MODIFY COLUMN `npc_id` int(11) NOT NULL DEFAULT 0 AFTER `character_id`,
MODIFY COLUMN `bot_id` int(11) NOT NULL DEFAULT 0 AFTER `npc_id`,
ADD COLUMN `zone_id` smallint(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `bot_id`,
ADD COLUMN `instance_id` smallint(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `zone_id`;
ALTER TABLE `data_buckets`
MODIFY COLUMN `account_id` bigint(11) UNSIGNED NULL DEFAULT 0 AFTER `expires`,
MODIFY COLUMN `character_id` bigint(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `account_id`,
MODIFY COLUMN `npc_id` int(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `character_id`,
MODIFY COLUMN `bot_id` int(11) UNSIGNED NOT NULL DEFAULT 0 AFTER `npc_id`;
-- Create optimized unique index with `key` first
CREATE UNIQUE INDEX `keys` ON data_buckets (`key`, character_id, npc_id, bot_id, account_id, zone_id, instance_id);
-- Create indexes for just instance_id (instance deletion)
CREATE INDEX idx_instance_id ON data_buckets (instance_id);
)",
.content_schema_update = false
},
// -- template; copy/paste this when you need to create a new entry
// ManifestEntry{
// .version = 9228,

View File

@ -30,7 +30,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#include "../common/repositories/respawn_times_repository.h"
#include "../common/repositories/spawn_condition_values_repository.h"
#include "repositories/spawn2_disabled_repository.h"
#include "repositories/data_buckets_repository.h"
#include "database.h"
@ -479,6 +479,7 @@ void Database::DeleteInstance(uint16 instance_id)
DynamicZoneMembersRepository::DeleteByInstance(*this, instance_id);
DynamicZonesRepository::DeleteWhere(*this, fmt::format("instance_id = {}", instance_id));
CharacterCorpsesRepository::BuryInstance(*this, instance_id);
DataBucketsRepository::DeleteWhere(*this, fmt::format("instance_id = {}", instance_id));
}
void Database::FlagInstanceByGroupLeader(uint32 zone_id, int16 version, uint32 character_id, uint32 group_id)

View File

@ -23,10 +23,12 @@ public:
std::string key_;
std::string value;
uint32_t expires;
int64_t account_id;
int64_t character_id;
int64_t npc_id;
int64_t bot_id;
uint64_t account_id;
uint64_t character_id;
uint32_t npc_id;
uint32_t bot_id;
uint16_t zone_id;
uint16_t instance_id;
// cereal
template<class Archive>
@ -40,7 +42,9 @@ public:
CEREAL_NVP(account_id),
CEREAL_NVP(character_id),
CEREAL_NVP(npc_id),
CEREAL_NVP(bot_id)
CEREAL_NVP(bot_id),
CEREAL_NVP(zone_id),
CEREAL_NVP(instance_id)
);
}
};
@ -61,6 +65,8 @@ public:
"character_id",
"npc_id",
"bot_id",
"zone_id",
"instance_id",
};
}
@ -75,6 +81,8 @@ public:
"character_id",
"npc_id",
"bot_id",
"zone_id",
"instance_id",
};
}
@ -123,6 +131,8 @@ public:
e.character_id = 0;
e.npc_id = 0;
e.bot_id = 0;
e.zone_id = 0;
e.instance_id = 0;
return e;
}
@ -163,10 +173,12 @@ public:
e.key_ = row[1] ? row[1] : "";
e.value = row[2] ? row[2] : "";
e.expires = row[3] ? static_cast<uint32_t>(strtoul(row[3], nullptr, 10)) : 0;
e.account_id = row[4] ? strtoll(row[4], nullptr, 10) : 0;
e.character_id = row[5] ? strtoll(row[5], nullptr, 10) : 0;
e.npc_id = row[6] ? strtoll(row[6], nullptr, 10) : 0;
e.bot_id = row[7] ? strtoll(row[7], nullptr, 10) : 0;
e.account_id = row[4] ? strtoull(row[4], nullptr, 10) : 0;
e.character_id = row[5] ? strtoull(row[5], nullptr, 10) : 0;
e.npc_id = row[6] ? static_cast<uint32_t>(strtoul(row[6], nullptr, 10)) : 0;
e.bot_id = row[7] ? static_cast<uint32_t>(strtoul(row[7], nullptr, 10)) : 0;
e.zone_id = row[8] ? static_cast<uint16_t>(strtoul(row[8], nullptr, 10)) : 0;
e.instance_id = row[9] ? static_cast<uint16_t>(strtoul(row[9], nullptr, 10)) : 0;
return e;
}
@ -207,6 +219,8 @@ public:
v.push_back(columns[5] + " = " + std::to_string(e.character_id));
v.push_back(columns[6] + " = " + std::to_string(e.npc_id));
v.push_back(columns[7] + " = " + std::to_string(e.bot_id));
v.push_back(columns[8] + " = " + std::to_string(e.zone_id));
v.push_back(columns[9] + " = " + std::to_string(e.instance_id));
auto results = db.QueryDatabase(
fmt::format(
@ -236,6 +250,8 @@ public:
v.push_back(std::to_string(e.character_id));
v.push_back(std::to_string(e.npc_id));
v.push_back(std::to_string(e.bot_id));
v.push_back(std::to_string(e.zone_id));
v.push_back(std::to_string(e.instance_id));
auto results = db.QueryDatabase(
fmt::format(
@ -273,6 +289,8 @@ public:
v.push_back(std::to_string(e.character_id));
v.push_back(std::to_string(e.npc_id));
v.push_back(std::to_string(e.bot_id));
v.push_back(std::to_string(e.zone_id));
v.push_back(std::to_string(e.instance_id));
insert_chunks.push_back("(" + Strings::Implode(",", v) + ")");
}
@ -310,10 +328,12 @@ public:
e.key_ = row[1] ? row[1] : "";
e.value = row[2] ? row[2] : "";
e.expires = row[3] ? static_cast<uint32_t>(strtoul(row[3], nullptr, 10)) : 0;
e.account_id = row[4] ? strtoll(row[4], nullptr, 10) : 0;
e.character_id = row[5] ? strtoll(row[5], nullptr, 10) : 0;
e.npc_id = row[6] ? strtoll(row[6], nullptr, 10) : 0;
e.bot_id = row[7] ? strtoll(row[7], nullptr, 10) : 0;
e.account_id = row[4] ? strtoull(row[4], nullptr, 10) : 0;
e.character_id = row[5] ? strtoull(row[5], nullptr, 10) : 0;
e.npc_id = row[6] ? static_cast<uint32_t>(strtoul(row[6], nullptr, 10)) : 0;
e.bot_id = row[7] ? static_cast<uint32_t>(strtoul(row[7], nullptr, 10)) : 0;
e.zone_id = row[8] ? static_cast<uint16_t>(strtoul(row[8], nullptr, 10)) : 0;
e.instance_id = row[9] ? static_cast<uint16_t>(strtoul(row[9], nullptr, 10)) : 0;
all_entries.push_back(e);
}
@ -342,10 +362,12 @@ public:
e.key_ = row[1] ? row[1] : "";
e.value = row[2] ? row[2] : "";
e.expires = row[3] ? static_cast<uint32_t>(strtoul(row[3], nullptr, 10)) : 0;
e.account_id = row[4] ? strtoll(row[4], nullptr, 10) : 0;
e.character_id = row[5] ? strtoll(row[5], nullptr, 10) : 0;
e.npc_id = row[6] ? strtoll(row[6], nullptr, 10) : 0;
e.bot_id = row[7] ? strtoll(row[7], nullptr, 10) : 0;
e.account_id = row[4] ? strtoull(row[4], nullptr, 10) : 0;
e.character_id = row[5] ? strtoull(row[5], nullptr, 10) : 0;
e.npc_id = row[6] ? static_cast<uint32_t>(strtoul(row[6], nullptr, 10)) : 0;
e.bot_id = row[7] ? static_cast<uint32_t>(strtoul(row[7], nullptr, 10)) : 0;
e.zone_id = row[8] ? static_cast<uint16_t>(strtoul(row[8], nullptr, 10)) : 0;
e.instance_id = row[9] ? static_cast<uint16_t>(strtoul(row[9], nullptr, 10)) : 0;
all_entries.push_back(e);
}
@ -428,6 +450,8 @@ public:
v.push_back(std::to_string(e.character_id));
v.push_back(std::to_string(e.npc_id));
v.push_back(std::to_string(e.bot_id));
v.push_back(std::to_string(e.zone_id));
v.push_back(std::to_string(e.instance_id));
auto results = db.QueryDatabase(
fmt::format(
@ -458,6 +482,8 @@ public:
v.push_back(std::to_string(e.character_id));
v.push_back(std::to_string(e.npc_id));
v.push_back(std::to_string(e.bot_id));
v.push_back(std::to_string(e.zone_id));
v.push_back(std::to_string(e.instance_id));
insert_chunks.push_back("(" + Strings::Implode(",", v) + ")");
}

View File

@ -42,7 +42,7 @@
* Manifest: https://github.com/EQEmu/Server/blob/master/utils/sql/db_update_manifest.txt
*/
#define CURRENT_BINARY_DATABASE_VERSION 9305
#define CURRENT_BINARY_DATABASE_VERSION 9306
#define CURRENT_BINARY_BOTS_DATABASE_VERSION 9054
#endif

View File

@ -0,0 +1,277 @@
#include <chrono>
#include <iostream>
#include <random>
#include "../../common/http/httplib.h"
#include "../../common/eqemu_logsys.h"
#include "../sidecar_api/sidecar_api.h"
#include "../../common/platform.h"
#include "../data_bucket.h"
#include "../zonedb.h"
#include "../../common/repositories/data_buckets_repository.h"
void RunBenchmarkCycle(uint64_t target_rows)
{
const size_t OPERATIONS_PER_TEST = 5000;
const std::string test_key_prefix = "test_key_";
std::cout << Strings::Repeat("-", 70) << "\n";
std::cout << "📊 Running Benchmark at " << Strings::Commify(target_rows) << " Rows...\n";
std::cout << Strings::Repeat("-", 70) << "\n";
// 🧹 **Purge `test_key_*` Keys Before Each Run**
std::cout << "🧹 Purging test keys (`test_key_*`)...\n";
auto purge_start = std::chrono::high_resolution_clock::now();
DataBucketsRepository::DeleteWhere(database, "`key` LIKE '" + test_key_prefix + "%'");
auto purge_end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> purge_time = purge_end - purge_start;
std::cout << "✅ Purged test keys in " << purge_time.count() << " seconds.\n";
// 📊 **Ensure the Table Contains At Least `target_rows`**
auto populate_start = std::chrono::high_resolution_clock::now();
uint64_t current_count = DataBucketsRepository::Count(database);
if (current_count < target_rows) {
std::cout << "📌 Populating table to " << Strings::Commify(target_rows) << " rows...\n";
std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<int> entity_type(1, 5);
std::uniform_int_distribution<int> id_dist(1, 1000000);
std::uniform_int_distribution<int> expiry_dist(0, 86400 * 30); // Expiry up to 30 days
while (current_count < target_rows) {
std::vector<DataBucketsRepository::DataBuckets> batch;
for (size_t i = 0; i < 100000; ++i) {
if (i > target_rows - current_count) {
break;
}
int entity_choice = entity_type(rng);
int entity_id = id_dist(rng);
std::string key = "test_key_" + std::to_string(current_count + i);
std::string value = "value_" + std::to_string(current_count + i);
int expires = static_cast<int>(std::time(nullptr)) + expiry_dist(rng);
DataBucketsRepository::DataBuckets e{};
e.key_ = key;
e.value = value;
e.expires = expires;
e.account_id = (entity_choice == 1) ? entity_id : 0;
e.character_id = (entity_choice == 2) ? entity_id : 0;
e.npc_id = (entity_choice == 3) ? entity_id : 0;
e.bot_id = (entity_choice == 4) ? entity_id : 0;
e.zone_id = (entity_choice == 5) ? entity_id : 0;
e.instance_id = (entity_choice == 5) ? entity_id : 0;
batch.emplace_back(e);
}
DataBucketsRepository::InsertMany(database, batch);
current_count += batch.size();
}
}
else {
std::cout << "✅ Table already has " << current_count << " rows, proceeding with benchmark.\n";
}
auto populate_end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> populate_time = populate_end - populate_start;
std::cout << "✅ Populated table in " << populate_time.count() << " seconds.\n";
std::mt19937 rng(std::random_device{}());
std::uniform_int_distribution<int> id_dist(1, 1000);
// 🚀 **Measure Insert Performance**
std::vector<DataBucketKey> inserted_keys = {};
auto insert_start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < OPERATIONS_PER_TEST; ++i) {
std::string key = test_key_prefix + std::to_string(current_count + i);
std::string value = "value_" + std::to_string(current_count + i);
int expires = static_cast<int>(std::time(nullptr)) + 3600;
DataBucketKey e{
.key = key,
.value = value,
.expires = std::to_string(expires),
.account_id = 0,
.character_id = 0,
.npc_id = 0,
.bot_id = 0
};
// randomly set account_id, character_id, npc_id, or bot_id
switch (i % 4) {
case 0:
e.account_id = id_dist(rng);
break;
case 1:
e.character_id = id_dist(rng);
break;
case 2:
e.npc_id = id_dist(rng);
break;
case 3:
e.bot_id = id_dist(rng);
break;
case 4:
int entity_choice = id_dist(rng);
e.zone_id = entity_choice;
e.instance_id = entity_choice;
break;
}
DataBucket::SetData(e);
inserted_keys.emplace_back(e);
}
auto insert_end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> insert_time = insert_end - insert_start;
std::cout << "✅ Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " inserts in " << insert_time.count()
<< " seconds. (Individual Insert Performance)\n";
// ✏️ **Measure Update Performance Using DataBucket**
auto update_start = std::chrono::high_resolution_clock::now();
for (auto &key: inserted_keys) {
// 🔍 Retrieve existing bucket using scoped `GetData`
auto e = DataBucket::GetData(key);
if (e.id > 0) {
// create a new key object with the updated values
DataBucketKey bucket_entry_key{
.key = e.key_,
.value = "some_new_value",
.expires = std::to_string(e.expires),
.account_id = e.account_id,
.character_id = e.character_id,
.npc_id = e.npc_id,
.bot_id = e.bot_id,
.zone_id = e.zone_id,
.instance_id = e.instance_id
};
// 🔄 Update using DataBucket class
DataBucket::SetData(bucket_entry_key);
}
}
auto update_end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> update_time = update_end - update_start;
std::cout << "✅ Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " updates in " << update_time.count()
<< " seconds. (Scoped Update Performance)\n";
// 🔍 **Measure Cached Read Performance**
auto read_cached_start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < OPERATIONS_PER_TEST; ++i) {
std::string key = test_key_prefix + std::to_string(current_count + i);
DataBucketKey k{
.key = key,
.account_id = 0,
.character_id = 0,
.npc_id = 0,
.bot_id = 0,
.zone_id = 0,
.instance_id = 0
};
// randomly set account_id, character_id, npc_id, or bot_id
switch (i % 4) {
case 0:
k.account_id = id_dist(rng);
break;
case 1:
k.character_id = id_dist(rng);
break;
case 2:
k.npc_id = id_dist(rng);
break;
case 3:
k.bot_id = id_dist(rng);
break;
case 4:
int entity_choice = id_dist(rng);
k.zone_id = entity_choice;
k.instance_id = entity_choice;
}
DataBucket::GetData(key);
}
auto read_cached_end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> read_cached_time = read_cached_end - read_cached_start;
std::cout << "✅ Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " cached reads in "
<< read_cached_time.count() << " seconds. (DataBucket::GetData)\n";
// 🔍 **Measure Non-Cached Read Performance (Direct Query)**
auto read_uncached_start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < OPERATIONS_PER_TEST; ++i) {
std::string key = test_key_prefix + std::to_string(current_count + i);
DataBucketsRepository::GetWhere(database, "`key` = '" + key + "'");
}
auto read_uncached_end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> read_uncached_time = read_uncached_end - read_uncached_start;
std::cout << "✅ Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " non-cached reads in "
<< read_uncached_time.count() << " seconds. (DataBucketsRepository::GetWhere)\n";
// 🗑️ **Measure Delete Performance**
auto delete_start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < OPERATIONS_PER_TEST; ++i) {
std::string key = test_key_prefix + std::to_string(current_count + i);
DataBucketKey k{
.key = key,
.account_id = 0,
.character_id = 0,
.npc_id = 0,
.bot_id = 0,
.zone_id = 0,
.instance_id = 0
};
// randomly set account_id, character_id, npc_id, or bot_id
switch (i % 4) {
case 0:
k.account_id = id_dist(rng);
break;
case 1:
k.character_id = id_dist(rng);
break;
case 2:
k.npc_id = id_dist(rng);
break;
case 3:
k.bot_id = id_dist(rng);
break;
case 4:
int entity_choice = id_dist(rng);
k.zone_id = entity_choice;
k.instance_id = entity_choice;
}
DataBucket::DeleteData(k);
}
auto delete_end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> delete_time = delete_end - delete_start;
std::cout << "✅ Completed " << Strings::Commify(OPERATIONS_PER_TEST) << " deletes in " << delete_time.count()
<< " seconds.\n";
}
void ZoneCLI::BenchmarkDatabuckets(int argc, char **argv, argh::parser &cmd, std::string &description)
{
description = "Benchmark individual reads/writes/deletes in data_buckets at different table sizes.";
if (cmd[{"-h", "--help"}]) {
std::cout << "Usage: BenchmarkDatabuckets\n";
return;
}
if (std::getenv("DEBUG")) {
LogSys.SetDatabase(&database)->LoadLogDatabaseSettings();
}
auto start_time = std::chrono::high_resolution_clock::now();
std::vector<uint64_t> benchmark_sizes = {10000, 100000, 1000000};
for (auto size: benchmark_sizes) {
RunBenchmarkCycle(size);
}
// 🚀 **Total Benchmark Time**
auto end_time = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> total_elapsed = end_time - start_time;
std::cout << "\n🚀 Total Benchmark Time: " << total_elapsed.count() << " seconds\n";
}

View File

@ -19,10 +19,6 @@ void DataBucket::SetData(const std::string &bucket_key, const std::string &bucke
.key = bucket_key,
.value = bucket_value,
.expires = expires_time,
.account_id = 0,
.character_id = 0,
.npc_id = 0,
.bot_id = 0
};
DataBucket::SetData(k);
@ -54,6 +50,9 @@ void DataBucket::SetData(const DataBucketKey &k_)
}
else if (k.bot_id > 0) {
b.bot_id = k.bot_id;
} else if (k.zone_id > 0) {
b.zone_id = k.zone_id;
b.instance_id = k.instance_id;
}
const uint64 bucket_id = b.id;
@ -189,12 +188,14 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k_,
}
LogDataBuckets(
"Getting bucket key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}]",
"Getting bucket key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}] zone_id [{}] instance_id [{}]",
k.key,
k.bot_id,
k.account_id,
k.character_id,
k.npc_id
k.npc_id,
k.zone_id,
k.instance_id
);
bool can_cache = CanCache(k);
@ -244,17 +245,21 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k_,
.account_id = k.account_id,
.character_id = k.character_id,
.npc_id = k.npc_id,
.bot_id = k.bot_id
.bot_id = k.bot_id,
.zone_id = k.zone_id,
.instance_id = k.instance_id
}
);
LogDataBuckets(
"Key [{}] not found in database, adding to cache as a miss account_id [{}] character_id [{}] npc_id [{}] bot_id [{}] cache size before [{}] after [{}]",
"Key [{}] not found in database, adding to cache as a miss account_id [{}] character_id [{}] npc_id [{}] bot_id [{}] zone_id [{}] instance_id [{}] cache size before [{}] after [{}]",
k.key,
k.account_id,
k.character_id,
k.npc_id,
k.bot_id,
k.zone_id,
k.instance_id,
size_before,
g_data_bucket_cache.size()
);
@ -347,12 +352,15 @@ bool DataBucket::DeleteData(const DataBucketKey &k)
);
LogDataBuckets(
"Deleting bucket key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}] cache size before [{}] after [{}]",
"Deleting bucket key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}] bot_id [{}] zone_id [{}] instance_id [{}] cache size before [{}] after [{}]",
k.key,
k.bot_id,
k.account_id,
k.character_id,
k.npc_id,
k.bot_id,
k.zone_id,
k.instance_id,
size_before,
g_data_bucket_cache.size()
);
@ -390,12 +398,15 @@ std::string DataBucket::GetDataExpires(const DataBucketKey &k)
std::string DataBucket::GetDataRemaining(const DataBucketKey &k)
{
LogDataBuckets(
"Getting bucket remaining key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}]",
"Getting bucket remaining key [{}] bot_id [{}] account_id [{}] character_id [{}] npc_id [{}] bot_id [{}] zone_id [{}] instance_id [{}]",
k.key,
k.bot_id,
k.account_id,
k.character_id,
k.npc_id
k.npc_id,
k.bot_id,
k.zone_id,
k.instance_id
);
auto r = GetData(k);
@ -408,39 +419,46 @@ std::string DataBucket::GetDataRemaining(const DataBucketKey &k)
std::string DataBucket::GetScopedDbFilters(const DataBucketKey &k)
{
std::vector<std::string> query = {};
std::vector<std::string> q = {};
if (k.character_id > 0) {
query.emplace_back(fmt::format("character_id = {}", k.character_id));
q.emplace_back(fmt::format("character_id = {}", k.character_id));
}
else {
query.emplace_back("character_id = 0");
q.emplace_back("character_id = 0");
}
if (k.account_id > 0) {
query.emplace_back(fmt::format("account_id = {}", k.account_id));
q.emplace_back(fmt::format("account_id = {}", k.account_id));
}
else {
query.emplace_back("account_id = 0");
q.emplace_back("account_id = 0");
}
if (k.npc_id > 0) {
query.emplace_back(fmt::format("npc_id = {}", k.npc_id));
q.emplace_back(fmt::format("npc_id = {}", k.npc_id));
}
else {
query.emplace_back("npc_id = 0");
q.emplace_back("npc_id = 0");
}
if (k.bot_id > 0) {
query.emplace_back(fmt::format("bot_id = {}", k.bot_id));
q.emplace_back(fmt::format("bot_id = {}", k.bot_id));
}
else {
query.emplace_back("bot_id = 0");
q.emplace_back("bot_id = 0");
}
if (k.zone_id > 0) {
q.emplace_back(fmt::format("zone_id = {} AND instance_id = {}", k.zone_id, k.instance_id));
}
else {
q.emplace_back("zone_id = 0 AND instance_id = 0");
}
return fmt::format(
"{} {}",
Strings::Join(query, " AND "),
!query.empty() ? "AND" : ""
Strings::Join(q, " AND "),
!q.empty() ? "AND" : ""
);
}
@ -451,7 +469,52 @@ bool DataBucket::CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe,
dbe.bot_id == k.bot_id &&
dbe.account_id == k.account_id &&
dbe.character_id == k.character_id &&
dbe.npc_id == k.npc_id
dbe.npc_id == k.npc_id &&
dbe.zone_id == k.zone_id &&
dbe.instance_id == k.instance_id
);
}
void DataBucket::LoadZoneCache(uint16 zone_id, uint16 instance_id)
{
const auto &l = DataBucketsRepository::GetWhere(
database,
fmt::format(
"zone_id = {} AND instance_id = {} AND (`expires` > {} OR `expires` = 0)",
zone_id,
instance_id,
(long long) std::time(nullptr)
)
);
if (l.empty()) {
return;
}
LogDataBucketsDetail("cache size before [{}] l size [{}]", g_data_bucket_cache.size(), l.size());
uint32 added_count = 0;
for (const auto &e: l) {
if (!ExistsInCache(e)) {
added_count++;
}
}
for (const auto &e: l) {
if (!ExistsInCache(e)) {
LogDataBucketsDetail("bucket id [{}] bucket key [{}] bucket value [{}]", e.id, e.key_, e.value);
g_data_bucket_cache.emplace_back(e);
}
}
LogDataBucketsDetail("cache size after [{}]", g_data_bucket_cache.size());
LogDataBuckets(
"Loaded [{}] zone keys new cache size is [{}]",
l.size(),
g_data_bucket_cache.size()
);
}
@ -541,7 +604,7 @@ void DataBucket::BulkLoadEntitiesToCache(DataBucketLoadType::Type t, std::vector
);
}
void DataBucket::DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id)
void DataBucket::DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id, uint32 secondary_id)
{
size_t size_before = g_data_bucket_cache.size();
@ -553,7 +616,8 @@ void DataBucket::DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id)
return (
(type == DataBucketLoadType::Bot && e.bot_id == id) ||
(type == DataBucketLoadType::Account && e.account_id == id) ||
(type == DataBucketLoadType::Client && e.character_id == id)
(type == DataBucketLoadType::Client && e.character_id == id) ||
(type == DataBucketLoadType::Zone && e.zone_id == id && e.instance_id == secondary_id)
);
}
),
@ -595,7 +659,9 @@ void DataBucket::DeleteFromMissesCache(DataBucketsRepository::DataBuckets e)
ce.account_id == e.account_id &&
ce.character_id == e.character_id &&
ce.npc_id == e.npc_id &&
ce.bot_id == e.bot_id;
ce.bot_id == e.bot_id &&
ce.zone_id == e.zone_id &&
ce.instance_id == e.instance_id;
}
),
g_data_bucket_cache.end()
@ -647,13 +713,42 @@ void DataBucket::DeleteFromCache(uint64 id, DataBucketLoadType::Type type)
);
}
void DataBucket::DeleteZoneFromCache(uint16 zone_id, uint16 instance_id, DataBucketLoadType::Type type)
{
size_t size_before = g_data_bucket_cache.size();
g_data_bucket_cache.erase(
std::remove_if(
g_data_bucket_cache.begin(),
g_data_bucket_cache.end(),
[&](DataBucketsRepository::DataBuckets &e) {
switch (type) {
case DataBucketLoadType::Zone:
return e.zone_id == zone_id && e.instance_id == instance_id;
default:
return false;
}
}
),
g_data_bucket_cache.end()
);
LogDataBuckets(
"Deleted zone [{}] instance [{}] from cache size before [{}] after [{}]",
zone_id,
instance_id,
size_before,
g_data_bucket_cache.size()
);
}
// CanCache returns whether a bucket can be cached or not
// characters are only in one zone at a time so we can cache locally to the zone
// bots (not implemented) are only in one zone at a time so we can cache locally to the zone
// npcs (ids) can be in multiple zones so we can't cache locally to the zone
bool DataBucket::CanCache(const DataBucketKey &key)
{
if (key.character_id > 0 || key.account_id > 0 || key.bot_id > 0) {
if (key.character_id > 0 || key.account_id > 0 || key.bot_id > 0 || key.zone_id > 0) {
return true;
}

View File

@ -12,10 +12,12 @@ struct DataBucketKey {
std::string key;
std::string value;
std::string expires;
int64_t account_id = 0;
int64_t character_id = 0;
int64_t npc_id = 0;
int64_t bot_id = 0;
uint64_t account_id = 0;
uint64_t character_id = 0;
uint32_t npc_id = 0;
uint32_t bot_id = 0;
uint16_t zone_id = 0;
uint16_t instance_id = 0;
};
namespace DataBucketLoadType {
@ -23,6 +25,7 @@ namespace DataBucketLoadType {
Bot,
Account,
Client,
Zone,
MaxType
};
@ -30,6 +33,7 @@ namespace DataBucketLoadType {
"Bot",
"Account",
"Client",
"Zone"
};
}
@ -56,12 +60,14 @@ public:
static bool CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, const DataBucketKey &k);
static bool ExistsInCache(const DataBucketsRepository::DataBuckets &entry);
static void LoadZoneCache(uint16 zone_id, uint16 instance_id);
static void BulkLoadEntitiesToCache(DataBucketLoadType::Type t, std::vector<uint32> ids);
static void DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id);
static void DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id, uint32 secondary_id = 0);
static void DeleteFromMissesCache(DataBucketsRepository::DataBuckets e);
static void ClearCache();
static void DeleteFromCache(uint64 id, DataBucketLoadType::Type type);
static void DeleteZoneFromCache(uint16 zone_id, uint16 instance_id, DataBucketLoadType::Type type);
static bool CanCache(const DataBucketKey &key);
static DataBucketsRepository::DataBuckets
ExtractNestedValue(const DataBucketsRepository::DataBuckets &bucket, const std::string &full_key);

View File

@ -685,6 +685,42 @@ void Lua_Zone::ShowZoneGlobalLoot(Lua_Client c)
self->ShowZoneGlobalLoot(c);
}
void Lua_Zone::SetBucket(const std::string& bucket_name, const std::string& bucket_value)
{
Lua_Safe_Call_Void();
self->SetBucket(bucket_name, bucket_value);
}
void Lua_Zone::SetBucket(const std::string& bucket_name, const std::string& bucket_value, const std::string& expiration)
{
Lua_Safe_Call_Void();
self->SetBucket(bucket_name, bucket_value, expiration);
}
void Lua_Zone::DeleteBucket(const std::string& bucket_name)
{
Lua_Safe_Call_Void();
self->DeleteBucket(bucket_name);
}
std::string Lua_Zone::GetBucket(const std::string& bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucket(bucket_name);
}
std::string Lua_Zone::GetBucketExpires(const std::string& bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucketExpires(bucket_name);
}
std::string Lua_Zone::GetBucketRemaining(const std::string& bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucketRemaining(bucket_name);
}
luabind::scope lua_register_zone() {
return luabind::class_<Lua_Zone>("Zone")
.def(luabind::constructor<>())
@ -695,11 +731,15 @@ luabind::scope lua_register_zone() {
.def("CanDoCombat", &Lua_Zone::CanDoCombat)
.def("CanLevitate", &Lua_Zone::CanLevitate)
.def("ClearSpawnTimers", &Lua_Zone::ClearSpawnTimers)
.def("DeleteBucket", (void(Lua_Zone::*)(const std::string&))&Lua_Zone::DeleteBucket)
.def("Depop", (void(Lua_Zone::*)(void))&Lua_Zone::Depop)
.def("Depop", (void(Lua_Zone::*)(bool))&Lua_Zone::Depop)
.def("Despawn", &Lua_Zone::Despawn)
.def("GetAAEXPModifier", &Lua_Zone::GetAAEXPModifier)
.def("GetAAEXPModifierByCharacterID", &Lua_Zone::GetAAEXPModifierByCharacterID)
.def("GetBucket", (std::string(Lua_Zone::*)(const std::string&))&Lua_Zone::GetBucket)
.def("GetBucketExpires", (std::string(Lua_Zone::*)(const std::string&))&Lua_Zone::GetBucketExpires)
.def("GetBucketRemaining", (std::string(Lua_Zone::*)(const std::string&))&Lua_Zone::GetBucketRemaining)
.def("GetContentFlags", &Lua_Zone::GetContentFlags)
.def("GetContentFlagsDisabled", &Lua_Zone::GetContentFlagsDisabled)
.def("GetExperienceMultiplier", &Lua_Zone::GetExperienceMultiplier)
@ -795,6 +835,8 @@ luabind::scope lua_register_zone() {
.def("Repop", (void(Lua_Zone::*)(bool))&Lua_Zone::Repop)
.def("SetAAEXPModifier", &Lua_Zone::SetAAEXPModifier)
.def("SetAAEXPModifierByCharacterID", &Lua_Zone::SetAAEXPModifierByCharacterID)
.def("SetBucket", (void(Lua_Zone::*)(const std::string&,const std::string&))&Lua_Zone::SetBucket)
.def("SetBucket", (void(Lua_Zone::*)(const std::string&,const std::string&,const std::string&))&Lua_Zone::SetBucket)
.def("SetEXPModifier", &Lua_Zone::SetEXPModifier)
.def("SetEXPModifierByCharacterID", &Lua_Zone::SetEXPModifierByCharacterID)
.def("SetInstanceTimer", &Lua_Zone::SetInstanceTimer)

View File

@ -141,6 +141,14 @@ public:
void SetIsHotzone(bool is_hotzone);
void ShowZoneGlobalLoot(Lua_Client c);
// data buckets
void SetBucket(const std::string& bucket_name, const std::string& bucket_value);
void SetBucket(const std::string& bucket_name, const std::string& bucket_value, const std::string& expiration = "");
void DeleteBucket(const std::string& bucket_name);
std::string GetBucket(const std::string& bucket_name);
std::string GetBucketExpires(const std::string& bucket_name);
std::string GetBucketRemaining(const std::string& bucket_name);
};
#endif

View File

@ -526,6 +526,36 @@ void Perl_Zone_ShowZoneGlobalLoot(Zone* self, Client* c)
self->ShowZoneGlobalLoot(c);
}
void Perl_Zone_SetBucket(Zone* self, const std::string bucket_name, const std::string bucket_value)
{
self->SetBucket(bucket_name, bucket_value);
}
void Perl_Zone_SetBucket(Zone* self, const std::string bucket_name, const std::string bucket_value, const std::string expiration)
{
self->SetBucket(bucket_name, bucket_value, expiration);
}
void Perl_Zone_DeleteBucket(Zone* self, const std::string bucket_name)
{
self->DeleteBucket(bucket_name);
}
std::string Perl_Zone_GetBucket(Zone* self, const std::string bucket_name)
{
return self->GetBucket(bucket_name);
}
std::string Perl_Zone_GetBucketExpires(Zone* self, const std::string bucket_name)
{
return self->GetBucketExpires(bucket_name);
}
std::string Perl_Zone_GetBucketRemaining(Zone* self, const std::string bucket_name)
{
return self->GetBucketRemaining(bucket_name);
}
void perl_register_zone()
{
perl::interpreter perl(PERL_GET_THX);
@ -538,11 +568,15 @@ void perl_register_zone()
package.add("CanDoCombat", &Perl_Zone_CanDoCombat);
package.add("CanLevitate", &Perl_Zone_CanLevitate);
package.add("ClearSpawnTimers", &Perl_Zone_ClearSpawnTimers);
package.add("DeleteBucket", &Perl_Zone_DeleteBucket);
package.add("Depop", (void(*)(Zone*))&Perl_Zone_Depop);
package.add("Depop", (void(*)(Zone*, bool))&Perl_Zone_Depop);
package.add("Despawn", &Perl_Zone_Despawn);
package.add("GetAAEXPModifier", &Perl_Zone_GetAAEXPModifier);
package.add("GetAAEXPModifierByCharacterID", &Perl_Zone_GetAAEXPModifierByCharacterID);
package.add("GetBucket", &Perl_Zone_GetBucket);
package.add("GetBucketExpires", &Perl_Zone_GetBucketExpires);
package.add("GetBucketRemaining", &Perl_Zone_GetBucketRemaining);
package.add("GetContentFlags", &Perl_Zone_GetContentFlags);
package.add("GetContentFlagsDisabled", &Perl_Zone_GetContentFlagsDisabled);
package.add("GetExperienceMultiplier", &Perl_Zone_GetExperienceMultiplier);
@ -638,6 +672,8 @@ void perl_register_zone()
package.add("Repop", (void(*)(Zone*, bool))&Perl_Zone_Repop);
package.add("SetAAEXPModifier", &Perl_Zone_SetAAEXPModifier);
package.add("SetAAEXPModifierByCharacterID", &Perl_Zone_SetAAEXPModifierByCharacterID);
package.add("SetBucket", (void(*)(Zone*, const std::string, const std::string))&Perl_Zone_SetBucket);
package.add("SetBucket", (void(*)(Zone*, const std::string, const std::string, const std::string))&Perl_Zone_SetBucket);
package.add("SetEXPModifier", &Perl_Zone_SetEXPModifier);
package.add("SetEXPModifierByCharacterID", &Perl_Zone_SetEXPModifierByCharacterID);
package.add("SetInstanceTimer", &Perl_Zone_SetInstanceTimer);

View File

@ -172,6 +172,8 @@ bool Zone::Bootup(uint32 iZoneID, uint32 iInstanceID, bool is_static) {
zone->RequestUCSServerStatus();
zone->StartShutdownTimer();
DataBucket::LoadZoneCache(iZoneID, iInstanceID);
/*
* Set Logging
*/
@ -877,6 +879,8 @@ void Zone::Shutdown(bool quiet)
return;
}
DataBucket::DeleteCachedBuckets(DataBucketLoadType::Zone, zone->GetZoneID(), zone->GetInstanceID());
entity_list.StopMobAI();
std::map<uint32, NPCType *>::iterator itr;
@ -3187,4 +3191,56 @@ bool Zone::DoesAlternateCurrencyExist(uint32 currency_id)
);
}
std::string Zone::GetBucket(const std::string& bucket_name)
{
DataBucketKey k = {};
k.zone_id = zoneid;
k.instance_id = instanceid;
k.key = bucket_name;
return DataBucket::GetData(k).value;
}
void Zone::SetBucket(const std::string& bucket_name, const std::string& bucket_value, const std::string& expiration)
{
DataBucketKey k = {};
k.zone_id = zoneid;
k.instance_id = instanceid;
k.key = bucket_name;
k.expires = expiration;
k.value = bucket_value;
DataBucket::SetData(k);
}
void Zone::DeleteBucket(const std::string& bucket_name)
{
DataBucketKey k = {};
k.zone_id = zoneid;
k.instance_id = instanceid;
k.key = bucket_name;
DataBucket::DeleteData(k);
}
std::string Zone::GetBucketExpires(const std::string& bucket_name)
{
DataBucketKey k = {};
k.zone_id = zoneid;
k.instance_id = instanceid;
k.key = bucket_name;
return DataBucket::GetDataExpires(k);
}
std::string Zone::GetBucketRemaining(const std::string& bucket_name)
{
DataBucketKey k = {};
k.zone_id = zoneid;
k.instance_id = instanceid;
k.key = bucket_name;
return DataBucket::GetDataRemaining(k);
}
#include "zone_loot.cpp"

View File

@ -451,6 +451,12 @@ public:
void LoadBaseData();
void ReloadBaseData();
// data buckets
std::string GetBucket(const std::string& bucket_name);
void SetBucket(const std::string& bucket_name, const std::string& bucket_value, const std::string& expiration = "");
void DeleteBucket(const std::string& bucket_name);
std::string GetBucketExpires(const std::string& bucket_name);
std::string GetBucketRemaining(const std::string& bucket_name);
private:
bool allow_mercs;

View File

@ -29,6 +29,7 @@ void ZoneCLI::CommandHandler(int argc, char **argv)
auto function_map = EQEmuCommand::function_map;
// Register commands
function_map["benchmark:databuckets"] = &ZoneCLI::BenchmarkDatabuckets;
function_map["sidecar:serve-http"] = &ZoneCLI::SidecarServeHttp;
function_map["tests:npc-handins"] = &ZoneCLI::NpcHandins;
function_map["tests:npc-handins-multiquest"] = &ZoneCLI::NpcHandinsMultiQuest;
@ -36,6 +37,7 @@ void ZoneCLI::CommandHandler(int argc, char **argv)
EQEmuCommand::HandleMenu(function_map, cmd, argc, argv);
}
#include "cli/benchmark_databuckets.cpp"
#include "cli/sidecar_serve_http.cpp"
#include "cli/npc_handins.cpp"
#include "cli/npc_handins_multiquest.cpp"

View File

@ -6,6 +6,7 @@
class ZoneCLI {
public:
static void CommandHandler(int argc, char **argv);
static void BenchmarkDatabuckets(int argc, char **argv, argh::parser &cmd, std::string &description);
static void SidecarServeHttp(int argc, char **argv, argh::parser &cmd, std::string &description);
static bool RanConsoleCommand(int argc, char **argv);
static bool RanSidecarCommand(int argc, char **argv);