diff --git a/common/servertalk.h b/common/servertalk.h index 5761c1e79..62641ee58 100644 --- a/common/servertalk.h +++ b/common/servertalk.h @@ -319,8 +319,6 @@ // player events #define ServerOP_PlayerEvent 0x5100 -#define ServerOP_DataBucketCacheUpdate 0x5200 - enum { CZUpdateType_Character, CZUpdateType_Group, diff --git a/world/zoneserver.cpp b/world/zoneserver.cpp index b00afb4b7..25f6fdbd8 100644 --- a/world/zoneserver.cpp +++ b/world/zoneserver.cpp @@ -1564,11 +1564,6 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) { DynamicZone::HandleZoneMessage(pack); break; } - case ServerOP_DataBucketCacheUpdate: { - zoneserver_list.SendPacket(pack); - - break; - } case ServerOP_GuildTributeUpdate: { auto data = (GuildTributeUpdate *)pack->pBuffer; auto guild = guild_mgr.GetGuildByGuildID(data->guild_id); diff --git a/zone/bot.cpp b/zone/bot.cpp index 68e0f561e..aa746e767 100644 --- a/zone/bot.cpp +++ b/zone/bot.cpp @@ -3089,6 +3089,7 @@ bool Bot::Spawn(Client* botCharacterOwner) { m_targetable = true; entity_list.AddBot(this, true, true); + ClearDataBucketCache(); DataBucket::GetDataBuckets(this); LoadBotSpellSettings(); if (!AI_AddBotSpells(GetBotSpellID())) { diff --git a/zone/client_packet.cpp b/zone/client_packet.cpp index 94d6756dd..73a333f82 100644 --- a/zone/client_packet.cpp +++ b/zone/client_packet.cpp @@ -524,7 +524,6 @@ int Client::HandlePacket(const EQApplicationPacket *app) // Finish client connecting state void Client::CompleteConnect() { - UpdateWho(); client_state = CLIENT_CONNECTED; SendAllPackets(); @@ -1413,6 +1412,7 @@ void Client::Handle_Connect_OP_ZoneEntry(const EQApplicationPacket *app) drakkin_details = m_pp.drakkin_details; // Load Data Buckets + ClearDataBucketCache(); DataBucket::GetDataBuckets(this); // Max Level for Character:PerCharacterQglobalMaxLevel and Character:PerCharacterBucketMaxLevel diff --git a/zone/data_bucket.cpp b/zone/data_bucket.cpp index d022183bc..db2222348 100644 --- a/zone/data_bucket.cpp +++ b/zone/data_bucket.cpp @@ -8,7 +8,7 @@ extern WorldServer worldserver; -std::vector g_data_bucket_cache = {}; +std::vector g_data_bucket_cache = {}; void DataBucket::SetData(const std::string &bucket_key, const std::string &bucket_value, std::string expires_time) { @@ -58,14 +58,14 @@ void DataBucket::SetData(const DataBucketKey &k) b.value = k.value; if (bucket_id) { - // loop cache and update cache value and timestamp - for (auto &ce: g_data_bucket_cache) { - if (CheckBucketMatch(ce.e, k)) { - ce.e = b; - ce.updated_time = GetCurrentTimeUNIX(); - ce.update_action = DataBucketCacheUpdateAction::Upsert; - SendDataBucketCacheUpdate(ce); - break; + + // update the cache if it exists + if (CanCache(k)) { + for (auto &e: g_data_bucket_cache) { + if (CheckBucketMatch(e, k)) { + e = b; + break; + } } } @@ -74,28 +74,18 @@ void DataBucket::SetData(const DataBucketKey &k) else { b.key_ = k.key; b = DataBucketsRepository::InsertOne(database, b); - if (!ExistsInCache(b)) { - // add data bucket and timestamp to cache - auto ce = DataBucketCacheEntry{ - .e = b, - .updated_time = DataBucket::GetCurrentTimeUNIX(), - .update_action = DataBucketCacheUpdateAction::Upsert - }; - - g_data_bucket_cache.emplace_back(ce); - - SendDataBucketCacheUpdate(ce); + // add to cache if it doesn't exist + if (CanCache(k) && !ExistsInCache(b)) { DeleteFromMissesCache(b); + g_data_bucket_cache.emplace_back(b); } } } std::string DataBucket::GetData(const std::string &bucket_key) { - DataBucketKey k = {}; - k.key = bucket_key; - return GetData(k).value; + return GetData(DataBucketKey{.key = bucket_key}).value; } // GetData fetches bucket data from the database or cache if it exists @@ -112,22 +102,27 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k, b k.npc_id ); - for (const auto &ce: g_data_bucket_cache) { - if (CheckBucketMatch(ce.e, k)) { - if (ce.e.expires > 0 && ce.e.expires < std::time(nullptr)) { - LogDataBuckets("Attempted to read expired key [{}] removing from cache", ce.e.key_); - DeleteData(k); - return DataBucketsRepository::NewEntity(); - } + bool can_cache = CanCache(k); - // this is a bucket miss, return empty entity - // we still cache bucket misses, so we don't have to hit the database - if (ce.e.id == 0) { - return DataBucketsRepository::NewEntity(); - } + // check the cache first if we can cache + if (can_cache) { + for (const auto &e: g_data_bucket_cache) { + if (CheckBucketMatch(e, k)) { + if (e.expires > 0 && e.expires < std::time(nullptr)) { + LogDataBuckets("Attempted to read expired key [{}] removing from cache", e.key_); + DeleteData(k); + return DataBucketsRepository::NewEntity(); + } - LogDataBuckets("Returning key [{}] value [{}] from cache", ce.e.key_, ce.e.value); - return ce.e; + // this is a bucket miss, return empty entity + // we still cache bucket misses, so we don't have to hit the database + if (e.id == 0) { + return DataBucketsRepository::NewEntity(); + } + + LogDataBuckets("Returning key [{}] value [{}] from cache", e.key_, e.value); + return e; + } } } @@ -144,23 +139,21 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k, b // if we're ignoring the misses cache, don't add to the cache // the only place this is ignored is during the initial read of SetData - if (!ignore_misses_cache) { + bool add_to_misses_cache = !ignore_misses_cache && can_cache; + if (add_to_misses_cache) { size_t size_before = g_data_bucket_cache.size(); // cache bucket misses, so we don't have to hit the database // when scripts try to read a bucket that doesn't exist g_data_bucket_cache.emplace_back( - DataBucketCacheEntry{ - .e = DataBucketsRepository::DataBuckets{ - .id = 0, - .key_ = k.key, - .value = "", - .expires = 0, - .character_id = k.character_id, - .npc_id = k.npc_id, - .bot_id = k.bot_id - }, - .updated_time = DataBucket::GetCurrentTimeUNIX() + DataBucketsRepository::DataBuckets{ + .id = 0, + .key_ = k.key, + .value = "", + .expires = 0, + .character_id = k.character_id, + .npc_id = k.npc_id, + .bot_id = k.bot_id } ); @@ -178,60 +171,54 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k, b return {}; } + auto bucket = r.front(); + // if the entry has expired, delete it - if (r[0].expires > 0 && r[0].expires < (long long) std::time(nullptr)) { + if (bucket.expires > 0 && bucket.expires < (long long) std::time(nullptr)) { DeleteData(k); return {}; } - bool has_cache = false; - for (auto &ce: g_data_bucket_cache) { - if (ce.e.id == r[0].id) { - has_cache = true; - break; + // add to cache if it doesn't exist + if (can_cache) { + bool has_cache = false; + + for (auto &e: g_data_bucket_cache) { + if (e.id == bucket.id) { + has_cache = true; + break; + } + } + + if (!has_cache) { + g_data_bucket_cache.emplace_back(bucket); } } - if (!has_cache) { - // add data bucket and timestamp to cache - g_data_bucket_cache.emplace_back( - DataBucketCacheEntry{ - .e = r[0], - .updated_time = DataBucket::GetCurrentTimeUNIX() - } - ); - } - - return r[0]; + return bucket; } std::string DataBucket::GetDataExpires(const std::string &bucket_key) { - DataBucketKey k = {}; - k.key = bucket_key; - - return GetDataExpires(k); + return GetDataExpires(DataBucketKey{.key = bucket_key}); } std::string DataBucket::GetDataRemaining(const std::string &bucket_key) { - DataBucketKey k = {}; - k.key = bucket_key; - return GetDataRemaining(k); + return GetDataRemaining(DataBucketKey{.key = bucket_key}); } bool DataBucket::DeleteData(const std::string &bucket_key) { - DataBucketKey k = {}; - k.key = bucket_key; - return DeleteData(k); + return DeleteData(DataBucketKey{.key = bucket_key}); } // GetDataBuckets bulk loads all data buckets for a mob bool DataBucket::GetDataBuckets(Mob *mob) { - DataBucketLoadType::Type t; - const uint32 id = mob->GetMobTypeIdentifier(); + DataBucketLoadType::Type t{}; + + const uint32 id = mob->GetMobTypeIdentifier(); if (!id) { return false; @@ -243,46 +230,39 @@ bool DataBucket::GetDataBuckets(Mob *mob) else if (mob->IsClient()) { t = DataBucketLoadType::Client; } - else if (mob->IsNPC()) { - t = DataBucketLoadType::NPC; - } - BulkLoadEntities(t, {id}); + BulkLoadEntitiesToCache(t, {id}); return true; } bool DataBucket::DeleteData(const DataBucketKey &k) { - size_t size_before = g_data_bucket_cache.size(); + if (CanCache(k)) { + size_t size_before = g_data_bucket_cache.size(); - // delete from cache where contents match - g_data_bucket_cache.erase( - std::remove_if( - g_data_bucket_cache.begin(), - g_data_bucket_cache.end(), - [&](DataBucketCacheEntry &ce) { - bool match = CheckBucketMatch(ce.e, k); - if (match) { - ce.update_action = DataBucketCacheUpdateAction::Delete; - SendDataBucketCacheUpdate(ce); + // delete from cache where contents match + g_data_bucket_cache.erase( + std::remove_if( + g_data_bucket_cache.begin(), + g_data_bucket_cache.end(), + [&](DataBucketsRepository::DataBuckets &e) { + return CheckBucketMatch(e, k); } + ), + g_data_bucket_cache.end() + ); - return match; - } - ), - g_data_bucket_cache.end() - ); - - LogDataBuckets( - "Deleting bucket key [{}] bot_id [{}] character_id [{}] npc_id [{}] cache size before [{}] after [{}]", - k.key, - k.bot_id, - k.character_id, - k.npc_id, - size_before, - g_data_bucket_cache.size() - ); + LogDataBuckets( + "Deleting bucket key [{}] bot_id [{}] character_id [{}] npc_id [{}] cache size before [{}] after [{}]", + k.key, + k.bot_id, + k.character_id, + k.npc_id, + size_before, + g_data_bucket_cache.size() + ); + } return DataBucketsRepository::DeleteWhere( database, @@ -371,23 +351,21 @@ bool DataBucket::CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, ); } -void DataBucket::BulkLoadEntities(DataBucketLoadType::Type t, std::vector ids) +void DataBucket::BulkLoadEntitiesToCache(DataBucketLoadType::Type t, std::vector ids) { if (ids.empty()) { return; } if (ids.size() == 1) { - bool has_cache = false; - for (const auto &ce: g_data_bucket_cache) { + bool has_cache = false; + + for (const auto &e: g_data_bucket_cache) { if (t == DataBucketLoadType::Bot) { - has_cache = ce.e.bot_id == ids[0]; + has_cache = e.bot_id == ids[0]; } else if (t == DataBucketLoadType::Client) { - has_cache = ce.e.character_id == ids[0]; - } - else if (t == DataBucketLoadType::NPC) { - has_cache = ce.e.npc_id == ids[0]; + has_cache = e.character_id == ids[0]; } } @@ -406,9 +384,6 @@ void DataBucket::BulkLoadEntities(DataBucketLoadType::Type t, std::vector(t)); break; @@ -442,12 +417,7 @@ void DataBucket::BulkLoadEntities(DataBucketLoadType::Type t, std::vector( - std::chrono::system_clock::now().time_since_epoch() - ).count(); -} - -bool DataBucket::ExistsInCache(const DataBucketsRepository::DataBuckets &e) -{ - for (const auto &ce: g_data_bucket_cache) { - if (ce.e.id == e.id) { + for (const auto &e: g_data_bucket_cache) { + if (e.id == entry.id) { return true; } } @@ -507,134 +469,6 @@ bool DataBucket::ExistsInCache(const DataBucketsRepository::DataBuckets &e) return false; } -bool DataBucket::SendDataBucketCacheUpdate(const DataBucketCacheEntry &e) -{ - if (!e.e.id) { - return false; - } - - EQ::Net::DynamicPacket p; - p.PutSerialize(0, e); - - auto pack_size = sizeof(ServerDataBucketCacheUpdate_Struct) + p.Length(); - auto pack = new ServerPacket(ServerOP_DataBucketCacheUpdate, static_cast(pack_size)); - auto buf = reinterpret_cast(pack->pBuffer); - - buf->cereal_size = static_cast(p.Length()); - - memcpy(buf->cereal_data, p.Data(), p.Length()); - - worldserver.SendPacket(pack); - - return true; -} - -void DataBucket::HandleWorldMessage(ServerPacket *p) -{ - DataBucketCacheEntry n; - auto s = (ServerDataBucketCacheUpdate_Struct *) p->pBuffer; - EQ::Util::MemoryStreamReader ss(s->cereal_data, s->cereal_size); - cereal::BinaryInputArchive archive(ss); - archive(n); - - LogDataBucketsDetail( - "Received cache packet for id [{}] key [{}] value [{}] action [{}]", - n.e.id, - n.e.key_, - n.e.value, - static_cast(n.update_action) - ); - - // delete - if (n.update_action == DataBucketCacheUpdateAction::Delete) { - DeleteFromMissesCache(n.e); - - g_data_bucket_cache.erase( - std::remove_if( - g_data_bucket_cache.begin(), - g_data_bucket_cache.end(), - [&](DataBucketCacheEntry &ce) { - bool match = n.e.id > 0 && ce.e.id == n.e.id; - if (match) { - LogDataBuckets( - "[delete] cache key [{}] id [{}] cache_size before [{}] after [{}]", - ce.e.key_, - ce.e.id, - g_data_bucket_cache.size(), - g_data_bucket_cache.size() - 1 - ); - } - return match; - } - ), - g_data_bucket_cache.end() - ); - - return; - } - - // update - bool has_key = false; - for (auto &ce: g_data_bucket_cache) { - // update cache - if (ce.e.id == n.e.id) { - // reject old updates - int64 time_delta = ce.updated_time - n.updated_time; - if (ce.updated_time >= n.updated_time) { - LogDataBuckets( - "Attempted to update older cache key [{}] rejecting old time [{}] new time [{}] delta [{}] cache_size [{}]", - ce.e.key_, - ce.updated_time, - n.updated_time, - time_delta, - g_data_bucket_cache.size() - ); - return; - } - - DeleteFromMissesCache(n.e); - - LogDataBuckets( - "[update] cache id [{}] key [{}] value [{}] old time [{}] new time [{}] delta [{}] cache_size [{}]", - ce.e.id, - ce.e.key_, - n.e.value, - ce.updated_time, - n.updated_time, - time_delta, - g_data_bucket_cache.size() - ); - ce.e = n.e; - ce.updated_time = n.updated_time; - has_key = true; - break; - } - } - - // create - if (!has_key) { - DeleteFromMissesCache(n.e); - - size_t size_before = g_data_bucket_cache.size(); - - g_data_bucket_cache.emplace_back( - DataBucketCacheEntry{ - .e = n.e, - .updated_time = GetCurrentTimeUNIX() - } - ); - - LogDataBuckets( - "[create] Adding new cache id [{}] key [{}] value [{}] cache size before [{}] after [{}]", - n.e.id, - n.e.key_, - n.e.value, - size_before, - g_data_bucket_cache.size() - ); - } -} - void DataBucket::DeleteFromMissesCache(DataBucketsRepository::DataBuckets e) { // delete from cache where there might have been a written bucket miss to the cache @@ -645,11 +479,11 @@ void DataBucket::DeleteFromMissesCache(DataBucketsRepository::DataBuckets e) std::remove_if( g_data_bucket_cache.begin(), g_data_bucket_cache.end(), - [&](DataBucketCacheEntry &ce) { - return ce.e.id == 0 && ce.e.key_ == e.key_ && - ce.e.character_id == e.character_id && - ce.e.npc_id == e.npc_id && - ce.e.bot_id == e.bot_id; + [&](DataBucketsRepository::DataBuckets &ce) { + return ce.id == 0 && ce.key_ == e.key_ && + ce.character_id == e.character_id && + ce.npc_id == e.npc_id && + ce.bot_id == e.bot_id; } ), g_data_bucket_cache.end() @@ -667,3 +501,47 @@ void DataBucket::ClearCache() g_data_bucket_cache.clear(); LogInfo("Cleared data buckets cache"); } + +void DataBucket::DeleteFromCache(uint64 id, DataBucketLoadType::Type type) +{ + size_t size_before = g_data_bucket_cache.size(); + + g_data_bucket_cache.erase( + std::remove_if( + g_data_bucket_cache.begin(), + g_data_bucket_cache.end(), + [&](DataBucketsRepository::DataBuckets &e) { + switch (type) { + case DataBucketLoadType::Bot: + return e.bot_id == id; + case DataBucketLoadType::Client: + return e.character_id == id; + default: + return false; + } + } + ), + g_data_bucket_cache.end() + ); + + LogDataBuckets( + "Deleted [{}] id [{}] from cache size before [{}] after [{}]", + DataBucketLoadType::Name[type], + id, + size_before, + g_data_bucket_cache.size() + ); +} + +// CanCache returns whether a bucket can be cached or not +// characters are only in one zone at a time so we can cache locally to the zone +// bots (not implemented) are only in one zone at a time so we can cache locally to the zone +// npcs (ids) can be in multiple zones so we can't cache locally to the zone +bool DataBucket::CanCache(const DataBucketKey &key) +{ + if (key.character_id > 0 || key.bot_id > 0) { + return true; + } + + return false; +} diff --git a/zone/data_bucket.h b/zone/data_bucket.h index 4ae270e44..8503330f2 100644 --- a/zone/data_bucket.h +++ b/zone/data_bucket.h @@ -1,7 +1,3 @@ -// -// Created by Akkadius on 7/7/18. -// - #ifndef EQEMU_DATABUCKET_H #define EQEMU_DATABUCKET_H @@ -12,27 +8,6 @@ #include "../common/json/json_archive_single_line.h" #include "../common/servertalk.h" -enum DataBucketCacheUpdateAction : uint8 { - Upsert, - Delete -}; - -struct DataBucketCacheEntry { - DataBucketsRepository::DataBuckets e; - int64_t updated_time{}; - DataBucketCacheUpdateAction update_action{}; - - template - void serialize(Archive &ar) - { - ar( - CEREAL_NVP(e), - CEREAL_NVP(updated_time), - CEREAL_NVP(update_action) - ); - } -}; - struct DataBucketKey { std::string key; std::string value; @@ -46,14 +21,12 @@ namespace DataBucketLoadType { enum Type : uint8 { Bot, Client, - NPC, MaxType }; static const std::string Name[Type::MaxType] = { "Bot", "Client", - "NPC", }; } @@ -68,8 +41,6 @@ public: static bool GetDataBuckets(Mob *mob); - static int64_t GetCurrentTimeUNIX(); - // scoped bucket methods static void SetData(const DataBucketKey &k); static bool DeleteData(const DataBucketKey &k); @@ -80,15 +51,15 @@ public: // bucket repository versus key matching static bool CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, const DataBucketKey &k); - static bool ExistsInCache(const DataBucketsRepository::DataBuckets &e); + static bool ExistsInCache(const DataBucketsRepository::DataBuckets &entry); - static void BulkLoadEntities(DataBucketLoadType::Type t, std::vector ids); - static void DeleteCachedBuckets(DataBucketLoadType::Type t, uint32 id); + static void BulkLoadEntitiesToCache(DataBucketLoadType::Type t, std::vector ids); + static void DeleteCachedBuckets(DataBucketLoadType::Type type, uint32 id); - static bool SendDataBucketCacheUpdate(const DataBucketCacheEntry &e); - static void HandleWorldMessage(ServerPacket *p); static void DeleteFromMissesCache(DataBucketsRepository::DataBuckets e); static void ClearCache(); + static void DeleteFromCache(uint64 id, DataBucketLoadType::Type type); + static bool CanCache(const DataBucketKey &key); }; #endif //EQEMU_DATABUCKET_H diff --git a/zone/mob.cpp b/zone/mob.cpp index 8e4219319..dd62d9329 100644 --- a/zone/mob.cpp +++ b/zone/mob.cpp @@ -572,6 +572,8 @@ Mob::~Mob() m_close_mobs.clear(); + ClearDataBucketCache(); + LeaveHealRotationTargetPool(); } @@ -8606,3 +8608,21 @@ std::unordered_map &Mob::GetCloseMobList(float distance) { return entity_list.GetCloseMobList(this, distance); } + +void Mob::ClearDataBucketCache() +{ + if (IsOfClientBot()) { + uint64 id = 0; + DataBucketLoadType::Type t{}; + if (IsBot()) { + id = CastToBot()->GetBotID(); + t = DataBucketLoadType::Bot; + } + else if (IsClient()) { + id = CastToClient()->CharacterID(); + t = DataBucketLoadType::Client; + } + + DataBucket::DeleteFromCache(id, t); + } +} diff --git a/zone/mob.h b/zone/mob.h index d1bf386d6..297aa80cb 100644 --- a/zone/mob.h +++ b/zone/mob.h @@ -1491,6 +1491,8 @@ public: std::unordered_map &GetCloseMobList(float distance = 0.0f); void CheckScanCloseMobsMovingTimer(); + void ClearDataBucketCache(); + protected: void CommonDamage(Mob* other, int64 &damage, const uint16 spell_id, const EQ::skills::SkillType attack_skill, bool &avoidable, const int8 buffslot, const bool iBuffTic, eSpecialAttacks specal = eSpecialAttacks::None); static uint16 GetProcID(uint16 spell_id, uint8 effect_index); diff --git a/zone/worldserver.cpp b/zone/worldserver.cpp index afff2a534..7b6b870e2 100644 --- a/zone/worldserver.cpp +++ b/zone/worldserver.cpp @@ -3644,11 +3644,6 @@ void WorldServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) SharedTaskZoneMessaging::HandleWorldMessage(pack); break; } - case ServerOP_DataBucketCacheUpdate: - { - DataBucket::HandleWorldMessage(pack); - break; - } case ServerOP_GuildTributeUpdate: { GuildTributeUpdate* in = (GuildTributeUpdate*)pack->pBuffer; diff --git a/zone/zonedb.cpp b/zone/zonedb.cpp index 3f0ac20e8..f346fd57c 100644 --- a/zone/zonedb.cpp +++ b/zone/zonedb.cpp @@ -1923,8 +1923,6 @@ const NPCType *ZoneDatabase::LoadNPCTypesData(uint32 npc_type_id, bool bulk_load } } - DataBucket::BulkLoadEntities(DataBucketLoadType::NPC, npc_ids); - if (!npc_faction_ids.empty()) { zone->LoadNPCFactions(npc_faction_ids); zone->LoadNPCFactionAssociations(npc_faction_ids);