diff --git a/common/eqemu_logsys.h b/common/eqemu_logsys.h index 9e8a6811e..70e0c430b 100644 --- a/common/eqemu_logsys.h +++ b/common/eqemu_logsys.h @@ -137,6 +137,7 @@ namespace Logs { Bugs, QuestErrors, PlayerEvents, + DataBuckets, MaxCategoryID /* Don't Remove this */ }; @@ -233,6 +234,7 @@ namespace Logs { "Bugs", "QuestErrors", "PlayerEvents", + "DataBuckets", }; } diff --git a/common/eqemu_logsys_log_aliases.h b/common/eqemu_logsys_log_aliases.h index 3b4c0960d..faaa59bcc 100644 --- a/common/eqemu_logsys_log_aliases.h +++ b/common/eqemu_logsys_log_aliases.h @@ -794,6 +794,16 @@ OutF(LogSys, Logs::Detail, Logs::PlayerEvents, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\ } while (0) +#define LogDataBuckets(message, ...) do {\ + if (LogSys.IsLogEnabled(Logs::General, Logs::DataBuckets))\ + OutF(LogSys, Logs::General, Logs::DataBuckets, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\ +} while (0) + +#define LogDataBucketsDetail(message, ...) do {\ + if (LogSys.IsLogEnabled(Logs::Detail, Logs::DataBuckets))\ + OutF(LogSys, Logs::Detail, Logs::DataBuckets, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\ +} while (0) + #define Log(debug_level, log_category, message, ...) do {\ if (LogSys.IsLogEnabled(debug_level, log_category))\ LogSys.Out(debug_level, log_category, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\ diff --git a/common/repositories/base/base_data_buckets_repository.h b/common/repositories/base/base_data_buckets_repository.h index d90380652..94a08fce4 100644 --- a/common/repositories/base/base_data_buckets_repository.h +++ b/common/repositories/base/base_data_buckets_repository.h @@ -15,7 +15,7 @@ #include "../../database.h" #include "../../strings.h" #include - +#include class BaseDataBucketsRepository { public: @@ -27,6 +27,21 @@ public: int64_t character_id; int64_t npc_id; int64_t bot_id; + + // cereal + template + void serialize(Archive &ar) + { + ar( + CEREAL_NVP(id), + CEREAL_NVP(key_), + CEREAL_NVP(value), + CEREAL_NVP(expires), + CEREAL_NVP(character_id), + CEREAL_NVP(npc_id), + CEREAL_NVP(bot_id) + ); + } }; static std::string PrimaryKey() diff --git a/common/servertalk.h b/common/servertalk.h index 901e2c35b..7a4edca88 100644 --- a/common/servertalk.h +++ b/common/servertalk.h @@ -286,6 +286,8 @@ // player events #define ServerOP_PlayerEvent 0x5100 +#define ServerOP_DataBucketCacheUpdate 0x5200 + enum { CZUpdateType_Character, CZUpdateType_Group, @@ -1820,6 +1822,11 @@ struct ServerSendPlayerEvent_Struct { char cereal_data[0]; }; +struct ServerDataBucketCacheUpdate_Struct { + uint32_t cereal_size; + char cereal_data[0]; +}; + struct ServerFlagUpdate_Struct { uint32 account_id; int16 admin; diff --git a/utils/scripts/generators/repository-generator.pl b/utils/scripts/generators/repository-generator.pl index 507a3e05a..d71befd59 100644 --- a/utils/scripts/generators/repository-generator.pl +++ b/utils/scripts/generators/repository-generator.pl @@ -112,6 +112,7 @@ if ($requested_table_to_generate ne "all") { } my @cereal_enabled_tables = ( + "data_buckets", "player_event_logs" ); diff --git a/world/zoneserver.cpp b/world/zoneserver.cpp index 0136aa450..d1e1d45a1 100644 --- a/world/zoneserver.cpp +++ b/world/zoneserver.cpp @@ -46,6 +46,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #include "../common/repositories/player_event_logs_repository.h" #include "../common/events/player_event_logs.h" #include "../common/patches/patches.h" +#include "../zone/data_bucket.h" extern ClientList client_list; extern GroupLFPList LFPGroupList; @@ -1468,6 +1469,11 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) { DynamicZone::HandleZoneMessage(pack); break; } + case ServerOP_DataBucketCacheUpdate: { + zoneserver_list.SendPacket(pack); + + break; + } default: { LogInfo("Unknown ServerOPcode from zone {:#04x}, size [{}]", pack->opcode, pack->size); DumpPacket(pack->pBuffer, pack->size); diff --git a/zone/bonuses.cpp b/zone/bonuses.cpp index f811bc2bc..89cdc667f 100644 --- a/zone/bonuses.cpp +++ b/zone/bonuses.cpp @@ -6036,7 +6036,7 @@ float Mob::CheckHeroicBonusesDataBuckets(std::string bucket_name) DataBucketKey k = GetScopedBucketKeys(); k.key = bucket_name; if (IsOfClientBot()) { - bucket_value = DataBucket::CheckBucketKey(this, k); + bucket_value = DataBucket::GetData(k).value; } if (bucket_value.empty() || !Strings::IsNumber(bucket_value)) { diff --git a/zone/bot.cpp b/zone/bot.cpp index 9409b0f8a..5b3882c6c 100644 --- a/zone/bot.cpp +++ b/zone/bot.cpp @@ -427,6 +427,7 @@ Bot::Bot(uint32 botID, uint32 botOwnerCharacterID, uint32 botSpellsID, double to Bot::~Bot() { AI_Stop(); LeaveHealRotationMemberPool(); + DataBucket::DeleteCachedBuckets(DataBucketLoadType::Bot, GetBotID()); if (HasPet()) { GetPet()->Depop(); @@ -8197,18 +8198,18 @@ bool Bot::CheckDataBucket(std::string bucket_name, const std::string& bucket_val DataBucketKey k = GetScopedBucketKeys(); k.key = bucket_name; - auto player_value = DataBucket::CheckBucketKey(this, k); - if (player_value.empty() && GetBotOwner()) { + auto b = DataBucket::GetData(k); + if (b.value.empty() && GetBotOwner()) { // fetch from owner k = GetBotOwner()->GetScopedBucketKeys(); - player_value = DataBucket::CheckBucketKey(GetBotOwner(), k); - if (player_value.empty()) { + b = DataBucket::GetData(k); + if (b.value.empty()) { return false; } } - if (zone->CompareDataBucket(bucket_comparison, bucket_value, player_value)) { + if (zone->CompareDataBucket(bucket_comparison, bucket_value, b.value)) { return true; } } diff --git a/zone/client.cpp b/zone/client.cpp index 48f2e1584..9e128a164 100644 --- a/zone/client.cpp +++ b/zone/client.cpp @@ -381,6 +381,8 @@ Client::Client(EQStreamInterface *ieqs) : Mob( Client::~Client() { mMovementManager->RemoveClient(this); + DataBucket::DeleteCachedBuckets(DataBucketLoadType::Client, CharacterID()); + if (RuleB(Bots, Enabled)) { Bot::ProcessBotOwnerRefDelete(this); } diff --git a/zone/client_process.cpp b/zone/client_process.cpp index 2aa2dbc72..2acbae9f2 100644 --- a/zone/client_process.cpp +++ b/zone/client_process.cpp @@ -857,12 +857,12 @@ void Client::BulkSendMerchantInventory(int merchant_id, int npcid) { DataBucketKey k = GetScopedBucketKeys(); k.key = bucket_name; - auto const& player_value = DataBucket::CheckBucketKey(this, k); - if (player_value.empty()) { + auto b = DataBucket::GetData(k); + if (b.value.empty()) { continue; } - if (!zone->CompareDataBucket(ml.bucket_comparison, bucket_value, player_value)) { + if (!zone->CompareDataBucket(ml.bucket_comparison, bucket_value, b.value)) { continue; } } diff --git a/zone/data_bucket.cpp b/zone/data_bucket.cpp index 40f1d1431..c7e98acfd 100644 --- a/zone/data_bucket.cpp +++ b/zone/data_bucket.cpp @@ -1,9 +1,14 @@ #include "data_bucket.h" +#include "entity.h" #include "zonedb.h" #include "mob.h" +#include "worldserver.h" #include #include -#include "../common/repositories/data_buckets_repository.h" + +extern WorldServer worldserver; + +std::vector g_data_bucket_cache = {}; void DataBucket::SetData(const std::string &bucket_key, const std::string &bucket_value, std::string expires_time) { @@ -22,40 +27,67 @@ void DataBucket::SetData(const std::string &bucket_key, const std::string &bucke void DataBucket::SetData(const DataBucketKey &k) { auto b = DataBucketsRepository::NewEntity(); - auto r = GetData(k); + auto r = GetData(k, true); // if we have an entry, use it if (r.id > 0) { b = r; } + // add scoping to bucket if (k.character_id > 0) { b.character_id = k.character_id; - } else if (k.npc_id > 0) { + } + else if (k.npc_id > 0) { b.npc_id = k.npc_id; - } else if (k.bot_id > 0) { + } + else if (k.bot_id > 0) { b.bot_id = k.bot_id; } - uint64 bucket_id = b.id; - long long expires_time_unix = 0; + const uint64 bucket_id = b.id; + int64 expires_time_unix = 0; if (!k.expires.empty()) { - expires_time_unix = (long long) std::time(nullptr) + Strings::ToInt(k.expires); + expires_time_unix = static_cast(std::time(nullptr)) + Strings::ToInt(k.expires); if (isalpha(k.expires[0]) || isalpha(k.expires[k.expires.length() - 1])) { - expires_time_unix = (long long) std::time(nullptr) + Strings::TimeToSeconds(k.expires); + expires_time_unix = static_cast(std::time(nullptr)) + Strings::TimeToSeconds(k.expires); } } - if (bucket_id > 0) { - b.expires = expires_time_unix; - b.value = k.value; + b.expires = expires_time_unix; + b.value = k.value; + + if (bucket_id) { + // loop cache and update cache value and timestamp + for (auto &ce: g_data_bucket_cache) { + if (CheckBucketMatch(ce.e, k)) { + ce.e = b; + ce.updated_time = GetCurrentTimeUNIX(); + ce.update_action = DataBucketCacheUpdateAction::Upsert; + SendDataBucketCacheUpdate(ce); + break; + } + } + DataBucketsRepository::UpdateOne(database, b); } else { - b.expires = expires_time_unix; - b.key_ = k.key; - b.value = k.value; - DataBucketsRepository::InsertOne(database, b); + b.key_ = k.key; + b = DataBucketsRepository::InsertOne(database, b); + if (!ExistsInCache(b)) { + // add data bucket and timestamp to cache + auto ce = DataBucketCacheEntry{ + .e = b, + .updated_time = DataBucket::GetCurrentTimeUNIX(), + .update_action = DataBucketCacheUpdateAction::Upsert + }; + + g_data_bucket_cache.emplace_back(ce); + + SendDataBucketCacheUpdate(ce); + + DeleteFromMissesCache(b); + } } } @@ -66,8 +98,39 @@ std::string DataBucket::GetData(const std::string &bucket_key) return GetData(k).value; } -DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k) +// GetData fetches bucket data from the database or cache if it exists +// if the bucket doesn't exist, it will be added to the cache as a miss +// if ignore_misses_cache is true, the bucket will not be added to the cache as a miss +// the only place we should be ignoring the misses cache is on the initial read during SetData +DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k, bool ignore_misses_cache) { + LogDataBuckets( + "Getting bucket key [{}] bot_id [{}] character_id [{}] npc_id [{}]", + k.key, + k.bot_id, + k.character_id, + k.npc_id + ); + + for (const auto &ce: g_data_bucket_cache) { + if (CheckBucketMatch(ce.e, k)) { + if (ce.e.expires > 0 && ce.e.expires < std::time(nullptr)) { + LogDataBuckets("Attempted to read expired key [{}] removing from cache", ce.e.key_); + DeleteData(k); + return DataBucketsRepository::NewEntity(); + } + + // this is a bucket miss, return empty entity + // we still cache bucket misses, so we don't have to hit the database + if (ce.e.id == 0) { + return DataBucketsRepository::NewEntity(); + } + + LogDataBuckets("Returning key [{}] value [{}] from cache", ce.e.key_, ce.e.value); + return ce.e; + } + } + auto r = DataBucketsRepository::GetWhere( database, fmt::format( @@ -78,6 +141,40 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k) ); if (r.empty()) { + + // if we're ignoring the misses cache, don't add to the cache + // the only place this is ignored is during the initial read of SetData + if (!ignore_misses_cache) { + size_t size_before = g_data_bucket_cache.size(); + + // cache bucket misses, so we don't have to hit the database + // when scripts try to read a bucket that doesn't exist + g_data_bucket_cache.emplace_back( + DataBucketCacheEntry{ + .e = DataBucketsRepository::DataBuckets{ + .id = 0, + .key_ = k.key, + .value = "", + .expires = 0, + .character_id = k.character_id, + .npc_id = k.npc_id, + .bot_id = k.bot_id + }, + .updated_time = DataBucket::GetCurrentTimeUNIX() + } + ); + + LogDataBuckets( + "Key [{}] not found in database, adding to cache as a miss character_id [{}] npc_id [{}] bot_id [{}] cache size before [{}] after [{}]", + k.key, + k.character_id, + k.npc_id, + k.bot_id, + size_before, + g_data_bucket_cache.size() + ); + } + return {}; } @@ -87,6 +184,24 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k) return {}; } + bool has_cache = false; + for (auto &ce: g_data_bucket_cache) { + if (ce.e.id == r[0].id) { + has_cache = true; + break; + } + } + + if (!has_cache) { + // add data bucket and timestamp to cache + g_data_bucket_cache.emplace_back( + DataBucketCacheEntry{ + .e = r[0], + .updated_time = DataBucket::GetCurrentTimeUNIX() + } + ); + } + return r[0]; } @@ -107,57 +222,68 @@ std::string DataBucket::GetDataRemaining(const std::string &bucket_key) bool DataBucket::DeleteData(const std::string &bucket_key) { - DataBucketKey r = {}; - r.key = bucket_key; - return DeleteData(r); + DataBucketKey k = {}; + k.key = bucket_key; + return DeleteData(k); } +// GetDataBuckets bulk loads all data buckets for a mob bool DataBucket::GetDataBuckets(Mob *mob) { - DataBucketKey k = mob->GetScopedBucketKeys(); - auto l = BaseDataBucketsRepository::GetWhere( - database, - fmt::format( - "{} (`expires` > {} OR `expires` = 0)", - DataBucket::GetScopedDbFilters(k), - (long long) std::time(nullptr) - ) - ); + DataBucketLoadType::Type t; + const uint32 id = mob->GetMobTypeIdentifier(); - if (l.empty()) { + if (!id) { return false; } - mob->m_data_bucket_cache.clear(); - - DataBucketCache d; - - for (const auto &e: l) { - d.bucket_id = e.id; - d.bucket_key = e.key_; - d.bucket_value = e.value; - d.bucket_expires = e.expires; - - mob->m_data_bucket_cache.emplace_back(d); + if (mob->IsBot()) { + t = DataBucketLoadType::Bot; } + else if (mob->IsClient()) { + t = DataBucketLoadType::Client; + } + else if (mob->IsNPC()) { + t = DataBucketLoadType::NPC; + } + + BulkLoadEntities(t, {id}); return true; } -std::string DataBucket::CheckBucketKey(const Mob *mob, const DataBucketKey &k) -{ - std::string bucket_value; - for (const auto &d: mob->m_data_bucket_cache) { - if (d.bucket_key == k.key) { - bucket_value = d.bucket_value; - break; - } - } - return bucket_value; -} - bool DataBucket::DeleteData(const DataBucketKey &k) { + size_t size_before = g_data_bucket_cache.size(); + + // delete from cache where contents match + g_data_bucket_cache.erase( + std::remove_if( + g_data_bucket_cache.begin(), + g_data_bucket_cache.end(), + [&](DataBucketCacheEntry &ce) { + bool match = CheckBucketMatch(ce.e, k); + if (match) { + ce.update_action = DataBucketCacheUpdateAction::Delete; + SendDataBucketCacheUpdate(ce); + } + + return match; + } + ), + g_data_bucket_cache.end() + ); + + LogDataBuckets( + "Deleting bucket key [{}] bot_id [{}] character_id [{}] npc_id [{}] cache size before [{}] after [{}]", + k.key, + k.bot_id, + k.character_id, + k.npc_id, + size_before, + g_data_bucket_cache.size() + ); + return DataBucketsRepository::DeleteWhere( database, fmt::format( @@ -170,16 +296,32 @@ bool DataBucket::DeleteData(const DataBucketKey &k) std::string DataBucket::GetDataExpires(const DataBucketKey &k) { + LogDataBuckets( + "Getting bucket expiration key [{}] bot_id [{}] character_id [{}] npc_id [{}]", + k.key, + k.bot_id, + k.character_id, + k.npc_id + ); + auto r = GetData(k); if (r.id == 0) { return {}; } - return fmt::format("{}", r.expires); + return std::to_string(r.expires); } std::string DataBucket::GetDataRemaining(const DataBucketKey &k) { + LogDataBuckets( + "Getting bucket remaining key [{}] bot_id [{}] character_id [{}] npc_id [{}]", + k.key, + k.bot_id, + k.character_id, + k.npc_id + ); + auto r = GetData(k); if (r.id == 0) { return "0"; @@ -218,3 +360,306 @@ std::string DataBucket::GetScopedDbFilters(const DataBucketKey &k) !query.empty() ? "AND" : "" ); } + +bool DataBucket::CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, const DataBucketKey &k) +{ + return ( + dbe.key_ == k.key && + dbe.bot_id == k.bot_id && + dbe.character_id == k.character_id && + dbe.npc_id == k.npc_id + ); +} + +void DataBucket::BulkLoadEntities(DataBucketLoadType::Type t, std::vector ids) +{ + if (ids.empty()) { + return; + } + + if (ids.size() == 1) { + bool has_cache = false; + for (const auto &ce: g_data_bucket_cache) { + if (t == DataBucketLoadType::Bot) { + has_cache = ce.e.bot_id == ids[0]; + } + else if (t == DataBucketLoadType::Client) { + has_cache = ce.e.character_id == ids[0]; + } + else if (t == DataBucketLoadType::NPC) { + has_cache = ce.e.npc_id == ids[0]; + } + } + + if (has_cache) { + LogDataBucketsDetail("LoadType [{}] ID [{}] has cache", DataBucketLoadType::Name[t], ids[0]); + return; + } + } + + std::string column; + + switch (t) { + case DataBucketLoadType::Bot: + column = "bot_id"; + break; + case DataBucketLoadType::Client: + column = "character_id"; + break; + case DataBucketLoadType::NPC: + column = "npc_id"; + break; + default: + LogError("Incorrect LoadType [{}]", t); + break; + } + + const auto &l = DataBucketsRepository::GetWhere( + database, + fmt::format( + "{} IN ({}) AND (`expires` > {} OR `expires` = 0)", + column, + Strings::Join(ids, ", "), + (long long) std::time(nullptr) + ) + ); + + if (l.empty()) { + return; + } + + size_t size_before = g_data_bucket_cache.size(); + + LogDataBucketsDetail("cache size before [{}] l size [{}]", g_data_bucket_cache.size(), l.size()); + + uint32 added_count = 0; + + for (const auto &e: l) { + if (!ExistsInCache(e)) { + added_count++; + } + } + + g_data_bucket_cache.reserve(g_data_bucket_cache.size() + added_count); + + for (const auto &e: l) { + if (!ExistsInCache(e)) { + LogDataBucketsDetail("bucket id [{}] bucket key [{}] bucket value [{}]", e.id, e.key_, e.value); + + g_data_bucket_cache.emplace_back( + DataBucketCacheEntry{ + .e = e, + .updated_time = GetCurrentTimeUNIX() + } + ); + } + } + + LogDataBucketsDetail("cache size after [{}]", g_data_bucket_cache.size()); + + LogDataBuckets( + "Bulk Loaded ids [{}] column [{}] new cache size is [{}]", + ids.size(), + column, + g_data_bucket_cache.size() + ); +} + +void DataBucket::DeleteCachedBuckets(DataBucketLoadType::Type t, uint32 id) +{ + size_t size_before = g_data_bucket_cache.size(); + + g_data_bucket_cache.erase( + std::remove_if( + g_data_bucket_cache.begin(), + g_data_bucket_cache.end(), + [&](DataBucketCacheEntry &ce) { + return ( + (t == DataBucketLoadType::Bot && ce.e.bot_id == id) || + (t == DataBucketLoadType::Client && ce.e.character_id == id) || + (t == DataBucketLoadType::NPC && ce.e.npc_id == id) + ); + } + ), + g_data_bucket_cache.end() + ); + + LogDataBuckets( + "LoadType [{}] id [{}] cache size before [{}] after [{}]", + DataBucketLoadType::Name[t], + id, + size_before, + g_data_bucket_cache.size() + ); +} + +int64_t DataBucket::GetCurrentTimeUNIX() +{ + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() + ).count(); +} + +bool DataBucket::ExistsInCache(const DataBucketsRepository::DataBuckets &e) +{ + for (const auto &ce: g_data_bucket_cache) { + if (ce.e.id == e.id) { + return true; + } + } + + return false; +} + +bool DataBucket::SendDataBucketCacheUpdate(const DataBucketCacheEntry &e) +{ + if (!e.e.id) { + return false; + } + + EQ::Net::DynamicPacket p; + p.PutSerialize(0, e); + + auto pack_size = sizeof(ServerDataBucketCacheUpdate_Struct) + p.Length(); + auto pack = new ServerPacket(ServerOP_DataBucketCacheUpdate, static_cast(pack_size)); + auto buf = reinterpret_cast(pack->pBuffer); + + buf->cereal_size = static_cast(p.Length()); + + memcpy(buf->cereal_data, p.Data(), p.Length()); + + worldserver.SendPacket(pack); + + return true; +} + +void DataBucket::HandleWorldMessage(ServerPacket *p) +{ + DataBucketCacheEntry n; + auto s = (ServerDataBucketCacheUpdate_Struct *) p->pBuffer; + EQ::Util::MemoryStreamReader ss(s->cereal_data, s->cereal_size); + cereal::BinaryInputArchive archive(ss); + archive(n); + + LogDataBucketsDetail( + "Received cache packet for id [{}] key [{}] value [{}] action [{}]", + n.e.id, + n.e.key_, + n.e.value, + n.update_action + ); + + // delete + if (n.update_action == DataBucketCacheUpdateAction::Delete) { + g_data_bucket_cache.erase( + std::remove_if( + g_data_bucket_cache.begin(), + g_data_bucket_cache.end(), + [&](DataBucketCacheEntry &ce) { + bool match = n.e.id > 0 && ce.e.id == n.e.id; + if (match) { + LogDataBuckets( + "[delete] cache key [{}] id [{}] cache_size before [{}] after [{}]", + ce.e.key_, + ce.e.id, + g_data_bucket_cache.size(), + g_data_bucket_cache.size() - 1 + ); + } + return match; + } + ), + g_data_bucket_cache.end() + ); + + return; + } + + // update + bool has_key = false; + + for (auto &ce: g_data_bucket_cache) { + int64 time_delta = ce.updated_time - n.updated_time; + if (ce.updated_time >= n.updated_time) { + LogDataBuckets( + "Attempted to update older cache key [{}] rejecting old time [{}] new time [{}] delta [{}] cache_size [{}]", + ce.e.key_, + ce.updated_time, + n.updated_time, + time_delta, + g_data_bucket_cache.size() + ); + return; + } + + // update cache + if (ce.e.id == n.e.id) { + DeleteFromMissesCache(n.e); + + LogDataBuckets( + "[update] cache id [{}] key [{}] value [{}] old time [{}] new time [{}] delta [{}] cache_size [{}]", + ce.e.id, + ce.e.key_, + n.e.value, + ce.updated_time, + n.updated_time, + time_delta, + g_data_bucket_cache.size() + ); + ce.e = n.e; + ce.updated_time = n.updated_time; + has_key = true; + break; + } + } + + // create + if (!has_key) { + DeleteFromMissesCache(n.e); + + size_t size_before = g_data_bucket_cache.size(); + + g_data_bucket_cache.emplace_back( + DataBucketCacheEntry{ + .e = n.e, + .updated_time = GetCurrentTimeUNIX() + } + ); + + LogDataBuckets( + "[create] Adding new cache id [{}] key [{}] value [{}] cache size before [{}] after [{}]", + n.e.id, + n.e.key_, + n.e.value, + size_before, + g_data_bucket_cache.size() + ); + } +} + +void DataBucket::DeleteFromMissesCache(DataBucketsRepository::DataBuckets e) +{ + // delete from cache where there might have been a written bucket miss to the cache + // this is to prevent the cache from growing too large + size_t size_before = g_data_bucket_cache.size(); + + g_data_bucket_cache.erase( + std::remove_if( + g_data_bucket_cache.begin(), + g_data_bucket_cache.end(), + [&](DataBucketCacheEntry &ce) { + return ce.e.id == 0 && ce.e.key_ == e.key_ && + ce.e.character_id == e.character_id && + ce.e.npc_id == e.npc_id && + ce.e.bot_id == e.bot_id; + } + ), + g_data_bucket_cache.end() + ); + LogDataBucketsDetail( + "Deleted bucket misses from cache where key [{}] size before [{}] after [{}]", + e.key_, + size_before, + g_data_bucket_cache.size() + ); +} diff --git a/zone/data_bucket.h b/zone/data_bucket.h index e966c2b9d..74dba4df0 100644 --- a/zone/data_bucket.h +++ b/zone/data_bucket.h @@ -9,7 +9,29 @@ #include "../common/types.h" #include "../common/repositories/data_buckets_repository.h" #include "mob.h" +#include "../common/json/json_archive_single_line.h" +#include "../common/servertalk.h" +enum DataBucketCacheUpdateAction : uint8 { + Upsert, + Delete +}; + +struct DataBucketCacheEntry { + DataBucketsRepository::DataBuckets e; + int64_t updated_time{}; + DataBucketCacheUpdateAction update_action{}; + + template + void serialize(Archive &ar) + { + ar( + CEREAL_NVP(e), + CEREAL_NVP(updated_time), + CEREAL_NVP(update_action) + ); + } +}; struct DataBucketKey { std::string key; @@ -20,25 +42,52 @@ struct DataBucketKey { int64_t bot_id; }; +namespace DataBucketLoadType { + enum Type : uint8 { + Bot, + Client, + NPC, + MaxType + }; + + static const std::string Name[Type::MaxType] = { + "Bot", + "Client", + "NPC", + }; +} + class DataBucket { public: // non-scoped bucket methods (for global buckets) - static void SetData(const std::string& bucket_key, const std::string& bucket_value, std::string expires_time = ""); - static bool DeleteData(const std::string& bucket_key); - static std::string GetData(const std::string& bucket_key); - static std::string GetDataExpires(const std::string& bucket_key); - static std::string GetDataRemaining(const std::string& bucket_key); + static void SetData(const std::string &bucket_key, const std::string &bucket_value, std::string expires_time = ""); + static bool DeleteData(const std::string &bucket_key); + static std::string GetData(const std::string &bucket_key); + static std::string GetDataExpires(const std::string &bucket_key); + static std::string GetDataRemaining(const std::string &bucket_key); - static bool GetDataBuckets(Mob* mob); + static bool GetDataBuckets(Mob *mob); + + static int64_t GetCurrentTimeUNIX(); // scoped bucket methods - static void SetData(const DataBucketKey& k); - static bool DeleteData(const DataBucketKey& k); - static DataBucketsRepository::DataBuckets GetData(const DataBucketKey& k); - static std::string GetDataExpires(const DataBucketKey& k); - static std::string GetDataRemaining(const DataBucketKey& k); - static std::string CheckBucketKey(const Mob* mob, const DataBucketKey& k); - static std::string GetScopedDbFilters(const DataBucketKey& k); + static void SetData(const DataBucketKey &k); + static bool DeleteData(const DataBucketKey &k); + static DataBucketsRepository::DataBuckets GetData(const DataBucketKey &k, bool ignore_misses_cache = false); + static std::string GetDataExpires(const DataBucketKey &k); + static std::string GetDataRemaining(const DataBucketKey &k); + static std::string GetScopedDbFilters(const DataBucketKey &k); + + // bucket repository versus key matching + static bool CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, const DataBucketKey &k); + static bool ExistsInCache(const DataBucketsRepository::DataBuckets &e); + + static void BulkLoadEntities(DataBucketLoadType::Type t, std::vector ids); + static void DeleteCachedBuckets(DataBucketLoadType::Type t, uint32 id); + + static bool SendDataBucketCacheUpdate(const DataBucketCacheEntry &e); + static void HandleWorldMessage(ServerPacket *p); + static void DeleteFromMissesCache(DataBucketsRepository::DataBuckets e); }; #endif //EQEMU_DATABUCKET_H diff --git a/zone/lua_bot.cpp b/zone/lua_bot.cpp index 6d24896b5..cea430b6a 100644 --- a/zone/lua_bot.cpp +++ b/zone/lua_bot.cpp @@ -504,6 +504,42 @@ std::string Lua_Bot::GetRaceAbbreviation() { return GetPlayerRaceAbbreviation(self->GetBaseRace()); } +void Lua_Bot::DeleteBucket(std::string bucket_name) +{ + Lua_Safe_Call_Void(); + self->DeleteBucket(bucket_name); +} + +std::string Lua_Bot::GetBucket(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucket(bucket_name); +} + +std::string Lua_Bot::GetBucketExpires(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucketExpires(bucket_name); +} + +std::string Lua_Bot::GetBucketRemaining(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucketRemaining(bucket_name); +} + +void Lua_Bot::SetBucket(std::string bucket_name, std::string bucket_value) +{ + Lua_Safe_Call_Void(); + self->SetBucket(bucket_name, bucket_value); +} + +void Lua_Bot::SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration) +{ + Lua_Safe_Call_Void(); + self->SetBucket(bucket_name, bucket_value, expiration); +} + luabind::scope lua_register_bot() { return luabind::class_("Bot") .def(luabind::constructor<>()) @@ -531,6 +567,7 @@ luabind::scope lua_register_bot() { .def("Camp", (void(Lua_Bot::*)(bool))&Lua_Bot::Camp) .def("CountBotItem", (uint32(Lua_Bot::*)(uint32))&Lua_Bot::CountBotItem) .def("CountItemEquippedByID", (int(Lua_Bot::*)(uint32))&Lua_Bot::CountItemEquippedByID) + .def("DeleteBucket", (void(Lua_Bot::*)(std::string))&Lua_Bot::DeleteBucket) .def("Escape", (void(Lua_Bot::*)(void))&Lua_Bot::Escape) .def("Fling", (void(Lua_Bot::*)(float,float,float))&Lua_Bot::Fling) .def("Fling", (void(Lua_Bot::*)(float,float,float,bool))&Lua_Bot::Fling) @@ -551,6 +588,9 @@ luabind::scope lua_register_bot() { .def("GetBotID", (uint32(Lua_Bot::*)(void))&Lua_Bot::GetBotID) .def("GetBotItem", (Lua_ItemInst(Lua_Bot::*)(uint16))&Lua_Bot::GetBotItem) .def("GetBotItemIDBySlot", (uint32(Lua_Bot::*)(uint16))&Lua_Bot::GetBotItemIDBySlot) + .def("GetBucket", (std::string(Lua_Bot::*)(std::string))&Lua_Bot::GetBucket) + .def("GetBucketExpires", (std::string(Lua_Bot::*)(std::string))&Lua_Bot::GetBucketExpires) + .def("GetBucketRemaining", (std::string(Lua_Bot::*)(std::string))&Lua_Bot::GetBucketRemaining) .def("GetClassAbbreviation", (std::string(Lua_Bot::*)(void))&Lua_Bot::GetClassAbbreviation) .def("GetExpansionBitmask", (int(Lua_Bot::*)(void))&Lua_Bot::GetExpansionBitmask) .def("GetGroup", (Lua_Group(Lua_Bot::*)(void))&Lua_Bot::GetGroup) @@ -576,6 +616,8 @@ luabind::scope lua_register_bot() { .def("ReloadBotSpellSettings", (void(Lua_Bot::*)(void))&Lua_Bot::ReloadBotSpellSettings) .def("RemoveBotItem", (void(Lua_Bot::*)(uint32))&Lua_Bot::RemoveBotItem) .def("SendSpellAnim", (void(Lua_Bot::*)(uint16,uint16))&Lua_Bot::SendSpellAnim) + .def("SetBucket", (void(Lua_Bot::*)(std::string,std::string))&Lua_Bot::SetBucket) + .def("SetBucket", (void(Lua_Bot::*)(std::string,std::string,std::string))&Lua_Bot::SetBucket) .def("SetExpansionBitmask", (void(Lua_Bot::*)(int))&Lua_Bot::SetExpansionBitmask) .def("SetExpansionBitmask", (void(Lua_Bot::*)(int,bool))&Lua_Bot::SetExpansionBitmask) .def("SetSpellDuration", (void(Lua_Bot::*)(int))&Lua_Bot::SetSpellDuration) diff --git a/zone/lua_bot.h b/zone/lua_bot.h index c035d14b0..67910a177 100644 --- a/zone/lua_bot.h +++ b/zone/lua_bot.h @@ -68,6 +68,12 @@ public: void SendSpellAnim(uint16 target_id, uint16 spell_id); std::string GetClassAbbreviation(); std::string GetRaceAbbreviation(); + void DeleteBucket(std::string bucket_name); + std::string GetBucket(std::string bucket_name); + std::string GetBucketExpires(std::string bucket_name); + std::string GetBucketRemaining(std::string bucket_name); + void SetBucket(std::string bucket_name, std::string bucket_value); + void SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration); void ApplySpell(int spell_id); void ApplySpell(int spell_id, int duration); diff --git a/zone/lua_client.cpp b/zone/lua_client.cpp index c7e44fc5e..e5a4549ed 100644 --- a/zone/lua_client.cpp +++ b/zone/lua_client.cpp @@ -3098,6 +3098,42 @@ void Lua_Client::SetLDoNPoints(uint32 theme_id, uint32 points) self->SetLDoNPoints(theme_id, points); } +void Lua_Client::DeleteBucket(std::string bucket_name) +{ + Lua_Safe_Call_Void(); + self->DeleteBucket(bucket_name); +} + +std::string Lua_Client::GetBucket(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucket(bucket_name); +} + +std::string Lua_Client::GetBucketExpires(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucketExpires(bucket_name); +} + +std::string Lua_Client::GetBucketRemaining(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucketRemaining(bucket_name); +} + +void Lua_Client::SetBucket(std::string bucket_name, std::string bucket_value) +{ + Lua_Safe_Call_Void(); + self->SetBucket(bucket_name, bucket_value); +} + +void Lua_Client::SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration) +{ + Lua_Safe_Call_Void(); + self->SetBucket(bucket_name, bucket_value, expiration); +} + luabind::scope lua_register_client() { return luabind::class_("Client") .def(luabind::constructor<>()) @@ -3175,6 +3211,7 @@ luabind::scope lua_register_client() { .def("CreateExpeditionFromTemplate", &Lua_Client::CreateExpeditionFromTemplate) .def("CreateTaskDynamicZone", &Lua_Client::CreateTaskDynamicZone) .def("DecreaseByID", (bool(Lua_Client::*)(uint32,int))&Lua_Client::DecreaseByID) + .def("DeleteBucket", (void(Lua_Client::*)(std::string))&Lua_Client::DeleteBucket) .def("DeleteItemInInventory", (void(Lua_Client::*)(int,int))&Lua_Client::DeleteItemInInventory) .def("DeleteItemInInventory", (void(Lua_Client::*)(int,int,bool))&Lua_Client::DeleteItemInInventory) .def("DiaWind", (void(Lua_Client::*)(std::string))&Lua_Client::DialogueWindow) @@ -3249,6 +3286,9 @@ luabind::scope lua_register_client() { .def("GetBotRequiredLevel", (int(Lua_Client::*)(uint8))&Lua_Client::GetBotRequiredLevel) .def("GetBotSpawnLimit", (int(Lua_Client::*)(void))&Lua_Client::GetBotSpawnLimit) .def("GetBotSpawnLimit", (int(Lua_Client::*)(uint8))&Lua_Client::GetBotSpawnLimit) + .def("GetBucket", (std::string(Lua_Client::*)(std::string))&Lua_Client::GetBucket) + .def("GetBucketExpires", (std::string(Lua_Client::*)(std::string))&Lua_Client::GetBucketExpires) + .def("GetBucketRemaining", (std::string(Lua_Client::*)(std::string))&Lua_Client::GetBucketRemaining) .def("GetCarriedMoney", (uint64(Lua_Client::*)(void))&Lua_Client::GetCarriedMoney) .def("GetCarriedPlatinum", (uint32(Lua_Client::*)(void))&Lua_Client::GetCarriedPlatinum) .def("GetCharacterFactionLevel", (int(Lua_Client::*)(int))&Lua_Client::GetCharacterFactionLevel) @@ -3517,6 +3557,8 @@ luabind::scope lua_register_client() { .def("SetBotRequiredLevel", (void(Lua_Client::*)(int,uint8))&Lua_Client::SetBotRequiredLevel) .def("SetBotSpawnLimit", (void(Lua_Client::*)(int))&Lua_Client::SetBotSpawnLimit) .def("SetBotSpawnLimit", (void(Lua_Client::*)(int,uint8))&Lua_Client::SetBotSpawnLimit) + .def("SetBucket", (void(Lua_Client::*)(std::string,std::string))&Lua_Client::SetBucket) + .def("SetBucketExpires", (void(Lua_Client::*)(std::string,std::string,std::string))&Lua_Client::SetBucket) .def("SetClientMaxLevel", (void(Lua_Client::*)(int))&Lua_Client::SetClientMaxLevel) .def("SetConsumption", (void(Lua_Client::*)(int, int))&Lua_Client::SetConsumption) .def("SetDeity", (void(Lua_Client::*)(int))&Lua_Client::SetDeity) diff --git a/zone/lua_client.h b/zone/lua_client.h index c3d5f47a0..f2c7140ea 100644 --- a/zone/lua_client.h +++ b/zone/lua_client.h @@ -474,6 +474,12 @@ public: std::string GetClassAbbreviation(); std::string GetRaceAbbreviation(); void SetLDoNPoints(uint32 theme_id, uint32 points); + void DeleteBucket(std::string bucket_name); + std::string GetBucket(std::string bucket_name); + std::string GetBucketExpires(std::string bucket_name); + std::string GetBucketRemaining(std::string bucket_name); + void SetBucket(std::string bucket_name, std::string bucket_value); + void SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration); void ApplySpell(int spell_id); void ApplySpell(int spell_id, int duration); diff --git a/zone/lua_npc.cpp b/zone/lua_npc.cpp index 1be00ac38..e8a13a679 100644 --- a/zone/lua_npc.cpp +++ b/zone/lua_npc.cpp @@ -777,6 +777,42 @@ bool Lua_NPC::HasSpecialAbilities() { return self->HasSpecialAbilities(); } +void Lua_NPC::DeleteBucket(std::string bucket_name) +{ + Lua_Safe_Call_Void(); + self->DeleteBucket(bucket_name); +} + +std::string Lua_NPC::GetBucket(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucket(bucket_name); +} + +std::string Lua_NPC::GetBucketExpires(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucketExpires(bucket_name); +} + +std::string Lua_NPC::GetBucketRemaining(std::string bucket_name) +{ + Lua_Safe_Call_String(); + return self->GetBucketRemaining(bucket_name); +} + +void Lua_NPC::SetBucket(std::string bucket_name, std::string bucket_value) +{ + Lua_Safe_Call_Void(); + self->SetBucket(bucket_name, bucket_value); +} + +void Lua_NPC::SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration) +{ + Lua_Safe_Call_Void(); + self->SetBucket(bucket_name, bucket_value, expiration); +} + luabind::scope lua_register_npc() { return luabind::class_("NPC") .def(luabind::constructor<>()) @@ -804,12 +840,16 @@ luabind::scope lua_register_npc() { .def("ClearLastName", (void(Lua_NPC::*)(void))&Lua_NPC::ClearLastName) .def("CountItem", (uint16(Lua_NPC::*)(uint32))&Lua_NPC::CountItem) .def("CountLoot", (int(Lua_NPC::*)(void))&Lua_NPC::CountLoot) + .def("DeleteBucket", (void(Lua_NPC::*)(std::string))&Lua_NPC::DeleteBucket) .def("DisplayWaypointInfo", (void(Lua_NPC::*)(Lua_Client))&Lua_NPC::DisplayWaypointInfo) .def("DoClassAttacks", (void(Lua_NPC::*)(Lua_Mob))&Lua_NPC::DoClassAttacks) .def("GetAccuracyRating", (int(Lua_NPC::*)(void))&Lua_NPC::GetAccuracyRating) .def("GetAttackDelay", (int(Lua_NPC::*)(void))&Lua_NPC::GetAttackDelay) .def("GetAttackSpeed", (float(Lua_NPC::*)(void))&Lua_NPC::GetAttackSpeed) .def("GetAvoidanceRating", &Lua_NPC::GetAvoidanceRating) + .def("GetBucket", (std::string(Lua_NPC::*)(std::string))&Lua_NPC::GetBucket) + .def("GetBucketExpires", (std::string(Lua_NPC::*)(std::string))&Lua_NPC::GetBucketExpires) + .def("GetBucketRemaining", (std::string(Lua_NPC::*)(std::string))&Lua_NPC::GetBucketRemaining) .def("GetCopper", (uint32(Lua_NPC::*)(void))&Lua_NPC::GetCopper) .def("GetFirstSlotByItemID", (uint16(Lua_NPC::*)(uint32))&Lua_NPC::GetFirstSlotByItemID) .def("GetFollowCanRun", (bool(Lua_NPC::*)(void))&Lua_NPC::GetFollowCanRun) @@ -894,6 +934,8 @@ luabind::scope lua_register_npc() { .def("ScaleNPC", (void(Lua_NPC::*)(uint8,bool))&Lua_NPC::ScaleNPC) .def("SendPayload", (void(Lua_NPC::*)(int))&Lua_NPC::SendPayload) .def("SendPayload", (void(Lua_NPC::*)(int,std::string))&Lua_NPC::SendPayload) + .def("SetBucket", (void(Lua_NPC::*)(std::string,std::string))&Lua_NPC::SetBucket) + .def("SetBucket", (void(Lua_NPC::*)(std::string,std::string,std::string))&Lua_NPC::SetBucket) .def("SetCopper", (void(Lua_NPC::*)(uint32))&Lua_NPC::SetCopper) .def("SetFollowCanRun", (void(Lua_NPC::*)(bool))&Lua_NPC::SetFollowCanRun) .def("SetFollowDistance", (void(Lua_NPC::*)(int))&Lua_NPC::SetFollowDistance) diff --git a/zone/lua_npc.h b/zone/lua_npc.h index 9d0f959aa..f9d42bb47 100644 --- a/zone/lua_npc.h +++ b/zone/lua_npc.h @@ -176,6 +176,12 @@ public: void ScaleNPC(uint8 npc_level, bool override_special_abilities); bool IsUnderwaterOnly(); bool HasSpecialAbilities(); + void DeleteBucket(std::string bucket_name); + std::string GetBucket(std::string bucket_name); + std::string GetBucketExpires(std::string bucket_name); + std::string GetBucketRemaining(std::string bucket_name); + void SetBucket(std::string bucket_name, std::string bucket_value); + void SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration); }; #endif diff --git a/zone/mob.cpp b/zone/mob.cpp index 7ffaad109..d84633166 100644 --- a/zone/mob.cpp +++ b/zone/mob.cpp @@ -8352,3 +8352,16 @@ DataBucketKey Mob::GetScopedBucketKeys() return k; } + +uint32 Mob::GetMobTypeIdentifier() +{ + if (IsClient()) { + return CastToClient()->CharacterID(); + } else if (IsNPC()) { + return GetNPCTypeID(); + } else if (IsBot()) { + return CastToBot()->GetBotID(); + } + + return 0; +} diff --git a/zone/mob.h b/zone/mob.h index 8b2c73adb..b186c646e 100644 --- a/zone/mob.h +++ b/zone/mob.h @@ -1408,8 +1408,6 @@ public: /// this cures timing issues cuz dead animation isn't done but server side feigning is? inline bool GetFeigned() const { return(feigned); } - std::vector m_data_bucket_cache; - // Data Bucket Methods void DeleteBucket(std::string bucket_name); std::string GetBucket(std::string bucket_name); @@ -1417,6 +1415,8 @@ public: std::string GetBucketRemaining(std::string bucket_name); void SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration = ""); + uint32 GetMobTypeIdentifier(); + // Heroic Stat Benefits float CheckHeroicBonusesDataBuckets(std::string bucket_name); diff --git a/zone/spells.cpp b/zone/spells.cpp index 63eea41da..872b3698a 100644 --- a/zone/spells.cpp +++ b/zone/spells.cpp @@ -5886,7 +5886,7 @@ bool Client::SpellBucketCheck(uint16 spell_id, uint32 character_id) { spell_bucket_name ); - auto bucket_value = DataBucket::GetData(old_bucket_name); + std::string bucket_value = DataBucket::GetData(old_bucket_name); if (!bucket_value.empty()) { if (Strings::IsNumber(bucket_value) && Strings::IsNumber(spell_bucket_value)) { if (Strings::ToInt(bucket_value) >= Strings::ToInt(spell_bucket_value)) { diff --git a/zone/worldserver.cpp b/zone/worldserver.cpp index 8af0eb378..a512ffddf 100644 --- a/zone/worldserver.cpp +++ b/zone/worldserver.cpp @@ -3329,6 +3329,11 @@ void WorldServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) SharedTaskZoneMessaging::HandleWorldMessage(pack); break; } + case ServerOP_DataBucketCacheUpdate: + { + DataBucket::HandleWorldMessage(pack); + break; + } default: { LogInfo("Unknown ZS Opcode [{}] size [{}]", (int)pack->opcode, pack->size); break; diff --git a/zone/zonedb.cpp b/zone/zonedb.cpp index cf3542729..08d6950ee 100755 --- a/zone/zonedb.cpp +++ b/zone/zonedb.cpp @@ -1927,6 +1927,8 @@ const NPCType *ZoneDatabase::LoadNPCTypesData(uint32 npc_type_id, bool bulk_load ); } + std::vector npc_ids; + for (NpcTypesRepository::NpcTypes &n : NpcTypesRepository::GetWhere((Database &) content_db, filter)) { NPCType *t; t = new NPCType; @@ -2137,8 +2139,15 @@ const NPCType *ZoneDatabase::LoadNPCTypesData(uint32 npc_type_id, bool bulk_load zone->npctable[t->npc_id] = t; npc = t; + + // If NPC ID is not in npc_ids, add to vector + if (!std::count(npc_ids.begin(), npc_ids.end(), t->npc_id)) { + npc_ids.emplace_back(t->npc_id); + } } + DataBucket::BulkLoadEntities(DataBucketLoadType::NPC, npc_ids); + return npc; }