pa/eqemu-server
2
0
Fork 0
mirror of https://github.com/EQEmu/Server.git synced 2025-12-11 16:51:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[Data Buckets] Distributed Databucket Caching (#3500)

Browse Source 
* [Data Buckets] Zone-Based Data Bucket Caching

# Notes
- Adds a data bucket cache so we're not needlessly hitting the database every time we need to read a data bucket value.

* Cleanup and unify GetData access patterns

* Cache work

* Push

* Add to cache when we fetch and do a db hit

* Handle bucket misses in cache

* Formatting

* Logging

* [Data Buckets] Zone-Based Data Bucket Caching

- Adds a data bucket cache so we're not needlessly hitting the database every time we need to read a data bucket value.

* Cleanup and unify GetData access patterns

* Cache work

* Push

* Add to cache when we fetch and do a db hit

* Handle bucket misses in cache

* Formatting

* Remove redundant fetches from cache since GetData does the same thing

* Push progress

* Distributed cache work

* Logging

* Fix issue with scoping where same named keys could return overlapping results

* Misses cache tweak, logging, comments

* Add bot, client, and NPC bucket methods to Lua.

---------

Co-authored-by: Akkadius <akkadius1@gmail.com>
This commit is contained in:
Alex King 2023-07-24 13:22:50 -04:00 committed by GitHub
parent 6c2886a71d
commit a75648f73f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 788 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common/eqemu_logsys.h
View File

@ -137,6 +137,7 @@ namespace Logs {
Bugs,
QuestErrors,
PlayerEvents,
DataBuckets,
MaxCategoryID /* Don't Remove this */
};
@ -233,6 +234,7 @@ namespace Logs {
"Bugs",
"QuestErrors",
"PlayerEvents",
"DataBuckets",
};
}

10
common/eqemu_logsys_log_aliases.h
View File

@ -794,6 +794,16 @@
OutF(LogSys, Logs::Detail, Logs::PlayerEvents, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\
} while (0)
#define LogDataBuckets(message, ...) do {\
if (LogSys.IsLogEnabled(Logs::General, Logs::DataBuckets))\
OutF(LogSys, Logs::General, Logs::DataBuckets, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\
} while (0)
#define LogDataBucketsDetail(message, ...) do {\
if (LogSys.IsLogEnabled(Logs::Detail, Logs::DataBuckets))\
OutF(LogSys, Logs::Detail, Logs::DataBuckets, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\
} while (0)
#define Log(debug_level, log_category, message, ...) do {\
if (LogSys.IsLogEnabled(debug_level, log_category))\
LogSys.Out(debug_level, log_category, __FILE__, __func__, __LINE__, message, ##__VA_ARGS__);\

17
common/repositories/base/base_data_buckets_repository.h
View File

@ -15,7 +15,7 @@
#include "../../database.h"
#include "../../strings.h"
#include <ctime>
#include <cereal/cereal.hpp>
class BaseDataBucketsRepository {
public:
@ -27,6 +27,21 @@ public:
int64_t character_id;
int64_t npc_id;
int64_t bot_id;
// cereal
template<class Archive>
void serialize(Archive &ar)
{
ar(
CEREAL_NVP(id),
CEREAL_NVP(key_),
CEREAL_NVP(value),
CEREAL_NVP(expires),
CEREAL_NVP(character_id),
CEREAL_NVP(npc_id),
CEREAL_NVP(bot_id)
);
}
};
static std::string PrimaryKey()

7
common/servertalk.h
View File

@ -286,6 +286,8 @@
// player events
#define ServerOP_PlayerEvent 0x5100
#define ServerOP_DataBucketCacheUpdate 0x5200
enum {
CZUpdateType_Character,
CZUpdateType_Group,
@ -1820,6 +1822,11 @@ struct ServerSendPlayerEvent_Struct {
char cereal_data[0];
};
struct ServerDataBucketCacheUpdate_Struct {
uint32_t cereal_size;
char cereal_data[0];
};
struct ServerFlagUpdate_Struct {
uint32 account_id;
int16 admin;

1
utils/scripts/generators/repository-generator.pl
View File

@ -112,6 +112,7 @@ if ($requested_table_to_generate ne "all") {
}
my @cereal_enabled_tables = (
"data_buckets",
"player_event_logs"
);

6
world/zoneserver.cpp
View File

@ -46,6 +46,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#include "../common/repositories/player_event_logs_repository.h"
#include "../common/events/player_event_logs.h"
#include "../common/patches/patches.h"
#include "../zone/data_bucket.h"
extern ClientList client_list;
extern GroupLFPList LFPGroupList;
@ -1468,6 +1469,11 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
DynamicZone::HandleZoneMessage(pack);
break;
}
case ServerOP_DataBucketCacheUpdate: {
zoneserver_list.SendPacket(pack);
break;
}
default: {
LogInfo("Unknown ServerOPcode from zone {:#04x}, size [{}]", pack->opcode, pack->size);
DumpPacket(pack->pBuffer, pack->size);

2
zone/bonuses.cpp
View File

@ -6036,7 +6036,7 @@ float Mob::CheckHeroicBonusesDataBuckets(std::string bucket_name)
DataBucketKey k = GetScopedBucketKeys();
k.key = bucket_name;
if (IsOfClientBot()) {
bucket_value = DataBucket::CheckBucketKey(this, k);
bucket_value = DataBucket::GetData(k).value;
}
if (bucket_value.empty() || !Strings::IsNumber(bucket_value)) {

11
zone/bot.cpp
View File

@ -427,6 +427,7 @@ Bot::Bot(uint32 botID, uint32 botOwnerCharacterID, uint32 botSpellsID, double to
Bot::~Bot() {
AI_Stop();
LeaveHealRotationMemberPool();
DataBucket::DeleteCachedBuckets(DataBucketLoadType::Bot, GetBotID());
if (HasPet()) {
GetPet()->Depop();
@ -8197,18 +8198,18 @@ bool Bot::CheckDataBucket(std::string bucket_name, const std::string& bucket_val
DataBucketKey k = GetScopedBucketKeys();
k.key = bucket_name;
auto player_value = DataBucket::CheckBucketKey(this, k);
if (player_value.empty() && GetBotOwner()) {
auto b = DataBucket::GetData(k);
if (b.value.empty() && GetBotOwner()) {
// fetch from owner
k = GetBotOwner()->GetScopedBucketKeys();
player_value = DataBucket::CheckBucketKey(GetBotOwner(), k);
if (player_value.empty()) {
b = DataBucket::GetData(k);
if (b.value.empty()) {
return false;
}
}
if (zone->CompareDataBucket(bucket_comparison, bucket_value, player_value)) {
if (zone->CompareDataBucket(bucket_comparison, bucket_value, b.value)) {
return true;
}
}

2
zone/client.cpp
View File

@ -381,6 +381,8 @@ Client::Client(EQStreamInterface *ieqs) : Mob(
Client::~Client() {
mMovementManager->RemoveClient(this);
DataBucket::DeleteCachedBuckets(DataBucketLoadType::Client, CharacterID());
if (RuleB(Bots, Enabled)) {
Bot::ProcessBotOwnerRefDelete(this);
}

6
zone/client_process.cpp
View File

@ -857,12 +857,12 @@ void Client::BulkSendMerchantInventory(int merchant_id, int npcid) {
DataBucketKey k = GetScopedBucketKeys();
k.key = bucket_name;
auto const& player_value = DataBucket::CheckBucketKey(this, k);
if (player_value.empty()) {
auto b = DataBucket::GetData(k);
if (b.value.empty()) {
continue;
}
if (!zone->CompareDataBucket(ml.bucket_comparison, bucket_value, player_value)) {
if (!zone->CompareDataBucket(ml.bucket_comparison, bucket_value, b.value)) {
continue;
}
}

551
zone/data_bucket.cpp
View File

@ -1,9 +1,14 @@
#include "data_bucket.h"
#include "entity.h"
#include "zonedb.h"
#include "mob.h"
#include "worldserver.h"
#include <ctime>
#include <cctype>
#include "../common/repositories/data_buckets_repository.h"
extern WorldServer worldserver;
std::vector<DataBucketCacheEntry> g_data_bucket_cache = {};
void DataBucket::SetData(const std::string &bucket_key, const std::string &bucket_value, std::string expires_time)
{
@ -22,40 +27,67 @@ void DataBucket::SetData(const std::string &bucket_key, const std::string &bucke
void DataBucket::SetData(const DataBucketKey &k)
{
auto b = DataBucketsRepository::NewEntity();
auto r = GetData(k);
auto r = GetData(k, true);
// if we have an entry, use it
if (r.id > 0) {
b = r;
}
// add scoping to bucket
if (k.character_id > 0) {
b.character_id = k.character_id;
} else if (k.npc_id > 0) {
}
else if (k.npc_id > 0) {
b.npc_id = k.npc_id;
} else if (k.bot_id > 0) {
}
else if (k.bot_id > 0) {
b.bot_id = k.bot_id;
}
uint64 bucket_id = b.id;
long long expires_time_unix = 0;
const uint64 bucket_id = b.id;
int64 expires_time_unix = 0;
if (!k.expires.empty()) {
expires_time_unix = (long long) std::time(nullptr) + Strings::ToInt(k.expires);
expires_time_unix = static_cast<int64>(std::time(nullptr)) + Strings::ToInt(k.expires);
if (isalpha(k.expires[0]) || isalpha(k.expires[k.expires.length() - 1])) {
expires_time_unix = (long long) std::time(nullptr) + Strings::TimeToSeconds(k.expires);
expires_time_unix = static_cast<int64>(std::time(nullptr)) + Strings::TimeToSeconds(k.expires);
}
}
if (bucket_id > 0) {
b.expires = expires_time_unix;
b.value = k.value;
b.expires = expires_time_unix;
b.value = k.value;
if (bucket_id) {
// loop cache and update cache value and timestamp
for (auto &ce: g_data_bucket_cache) {
if (CheckBucketMatch(ce.e, k)) {
ce.e = b;
ce.updated_time = GetCurrentTimeUNIX();
ce.update_action = DataBucketCacheUpdateAction::Upsert;
SendDataBucketCacheUpdate(ce);
break;
}
}
DataBucketsRepository::UpdateOne(database, b);
}
else {
b.expires = expires_time_unix;
b.key_ = k.key;
b.value = k.value;
DataBucketsRepository::InsertOne(database, b);
b.key_ = k.key;
b = DataBucketsRepository::InsertOne(database, b);
if (!ExistsInCache(b)) {
// add data bucket and timestamp to cache
auto ce = DataBucketCacheEntry{
.e = b,
.updated_time = DataBucket::GetCurrentTimeUNIX(),
.update_action = DataBucketCacheUpdateAction::Upsert
};
g_data_bucket_cache.emplace_back(ce);
SendDataBucketCacheUpdate(ce);
DeleteFromMissesCache(b);
}
}
}
@ -66,8 +98,39 @@ std::string DataBucket::GetData(const std::string &bucket_key)
return GetData(k).value;
}
DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k)
// GetData fetches bucket data from the database or cache if it exists
// if the bucket doesn't exist, it will be added to the cache as a miss
// if ignore_misses_cache is true, the bucket will not be added to the cache as a miss
// the only place we should be ignoring the misses cache is on the initial read during SetData
DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k, bool ignore_misses_cache)
{
LogDataBuckets(
"Getting bucket key [{}] bot_id [{}] character_id [{}] npc_id [{}]",
k.key,
k.bot_id,
k.character_id,
k.npc_id
);
for (const auto &ce: g_data_bucket_cache) {
if (CheckBucketMatch(ce.e, k)) {
if (ce.e.expires > 0 && ce.e.expires < std::time(nullptr)) {
LogDataBuckets("Attempted to read expired key [{}] removing from cache", ce.e.key_);
DeleteData(k);
return DataBucketsRepository::NewEntity();
}
// this is a bucket miss, return empty entity
// we still cache bucket misses, so we don't have to hit the database
if (ce.e.id == 0) {
return DataBucketsRepository::NewEntity();
}
LogDataBuckets("Returning key [{}] value [{}] from cache", ce.e.key_, ce.e.value);
return ce.e;
}
}
auto r = DataBucketsRepository::GetWhere(
database,
fmt::format(
@ -78,6 +141,40 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k)
);
if (r.empty()) {
// if we're ignoring the misses cache, don't add to the cache
// the only place this is ignored is during the initial read of SetData
if (!ignore_misses_cache) {
size_t size_before = g_data_bucket_cache.size();
// cache bucket misses, so we don't have to hit the database
// when scripts try to read a bucket that doesn't exist
g_data_bucket_cache.emplace_back(
DataBucketCacheEntry{
.e = DataBucketsRepository::DataBuckets{
.id = 0,
.key_ = k.key,
.value = "",
.expires = 0,
.character_id = k.character_id,
.npc_id = k.npc_id,
.bot_id = k.bot_id
},
.updated_time = DataBucket::GetCurrentTimeUNIX()
}
);
LogDataBuckets(
"Key [{}] not found in database, adding to cache as a miss character_id [{}] npc_id [{}] bot_id [{}] cache size before [{}] after [{}]",
k.key,
k.character_id,
k.npc_id,
k.bot_id,
size_before,
g_data_bucket_cache.size()
);
}
return {};
}
@ -87,6 +184,24 @@ DataBucketsRepository::DataBuckets DataBucket::GetData(const DataBucketKey &k)
return {};
}
bool has_cache = false;
for (auto &ce: g_data_bucket_cache) {
if (ce.e.id == r[0].id) {
has_cache = true;
break;
}
}
if (!has_cache) {
// add data bucket and timestamp to cache
g_data_bucket_cache.emplace_back(
DataBucketCacheEntry{
.e = r[0],
.updated_time = DataBucket::GetCurrentTimeUNIX()
}
);
}
return r[0];
}
@ -107,57 +222,68 @@ std::string DataBucket::GetDataRemaining(const std::string &bucket_key)
bool DataBucket::DeleteData(const std::string &bucket_key)
{
DataBucketKey r = {};
r.key = bucket_key;
return DeleteData(r);
DataBucketKey k = {};
k.key = bucket_key;
return DeleteData(k);
}
// GetDataBuckets bulk loads all data buckets for a mob
bool DataBucket::GetDataBuckets(Mob *mob)
{
DataBucketKey k = mob->GetScopedBucketKeys();
auto l = BaseDataBucketsRepository::GetWhere(
database,
fmt::format(
"{} (`expires` > {} OR `expires` = 0)",
DataBucket::GetScopedDbFilters(k),
(long long) std::time(nullptr)
)
);
DataBucketLoadType::Type t;
const uint32 id = mob->GetMobTypeIdentifier();
if (l.empty()) {
if (!id) {
return false;
}
mob->m_data_bucket_cache.clear();
DataBucketCache d;
for (const auto &e: l) {
d.bucket_id = e.id;
d.bucket_key = e.key_;
d.bucket_value = e.value;
d.bucket_expires = e.expires;
mob->m_data_bucket_cache.emplace_back(d);
if (mob->IsBot()) {
t = DataBucketLoadType::Bot;
}
else if (mob->IsClient()) {
t = DataBucketLoadType::Client;
}
else if (mob->IsNPC()) {
t = DataBucketLoadType::NPC;
}
BulkLoadEntities(t, {id});
return true;
}
std::string DataBucket::CheckBucketKey(const Mob *mob, const DataBucketKey &k)
{
std::string bucket_value;
for (const auto &d: mob->m_data_bucket_cache) {
if (d.bucket_key == k.key) {
bucket_value = d.bucket_value;
break;
}
}
return bucket_value;
}
bool DataBucket::DeleteData(const DataBucketKey &k)
{
size_t size_before = g_data_bucket_cache.size();
// delete from cache where contents match
g_data_bucket_cache.erase(
std::remove_if(
g_data_bucket_cache.begin(),
g_data_bucket_cache.end(),
[&](DataBucketCacheEntry &ce) {
bool match = CheckBucketMatch(ce.e, k);
if (match) {
ce.update_action = DataBucketCacheUpdateAction::Delete;
SendDataBucketCacheUpdate(ce);
}
return match;
}
),
g_data_bucket_cache.end()
);
LogDataBuckets(
"Deleting bucket key [{}] bot_id [{}] character_id [{}] npc_id [{}] cache size before [{}] after [{}]",
k.key,
k.bot_id,
k.character_id,
k.npc_id,
size_before,
g_data_bucket_cache.size()
);
return DataBucketsRepository::DeleteWhere(
database,
fmt::format(
@ -170,16 +296,32 @@ bool DataBucket::DeleteData(const DataBucketKey &k)
std::string DataBucket::GetDataExpires(const DataBucketKey &k)
{
LogDataBuckets(
"Getting bucket expiration key [{}] bot_id [{}] character_id [{}] npc_id [{}]",
k.key,
k.bot_id,
k.character_id,
k.npc_id
);
auto r = GetData(k);
if (r.id == 0) {
return {};
}
return fmt::format("{}", r.expires);
return std::to_string(r.expires);
}
std::string DataBucket::GetDataRemaining(const DataBucketKey &k)
{
LogDataBuckets(
"Getting bucket remaining key [{}] bot_id [{}] character_id [{}] npc_id [{}]",
k.key,
k.bot_id,
k.character_id,
k.npc_id
);
auto r = GetData(k);
if (r.id == 0) {
return "0";
@ -218,3 +360,306 @@ std::string DataBucket::GetScopedDbFilters(const DataBucketKey &k)
!query.empty() ? "AND" : ""
);
}
bool DataBucket::CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, const DataBucketKey &k)
{
return (
dbe.key_ == k.key &&
dbe.bot_id == k.bot_id &&
dbe.character_id == k.character_id &&
dbe.npc_id == k.npc_id
);
}
void DataBucket::BulkLoadEntities(DataBucketLoadType::Type t, std::vector<uint32> ids)
{
if (ids.empty()) {
return;
}
if (ids.size() == 1) {
bool has_cache = false;
for (const auto &ce: g_data_bucket_cache) {
if (t == DataBucketLoadType::Bot) {
has_cache = ce.e.bot_id == ids[0];
}
else if (t == DataBucketLoadType::Client) {
has_cache = ce.e.character_id == ids[0];
}
else if (t == DataBucketLoadType::NPC) {
has_cache = ce.e.npc_id == ids[0];
}
}
if (has_cache) {
LogDataBucketsDetail("LoadType [{}] ID [{}] has cache", DataBucketLoadType::Name[t], ids[0]);
return;
}
}
std::string column;
switch (t) {
case DataBucketLoadType::Bot:
column = "bot_id";
break;
case DataBucketLoadType::Client:
column = "character_id";
break;
case DataBucketLoadType::NPC:
column = "npc_id";
break;
default:
LogError("Incorrect LoadType [{}]", t);
break;
}
const auto &l = DataBucketsRepository::GetWhere(
database,
fmt::format(
"{} IN ({}) AND (`expires` > {} OR `expires` = 0)",
column,
Strings::Join(ids, ", "),
(long long) std::time(nullptr)
)
);
if (l.empty()) {
return;
}
size_t size_before = g_data_bucket_cache.size();
LogDataBucketsDetail("cache size before [{}] l size [{}]", g_data_bucket_cache.size(), l.size());
uint32 added_count = 0;
for (const auto &e: l) {
if (!ExistsInCache(e)) {
added_count++;
}
}
g_data_bucket_cache.reserve(g_data_bucket_cache.size() + added_count);
for (const auto &e: l) {
if (!ExistsInCache(e)) {
LogDataBucketsDetail("bucket id [{}] bucket key [{}] bucket value [{}]", e.id, e.key_, e.value);
g_data_bucket_cache.emplace_back(
DataBucketCacheEntry{
.e = e,
.updated_time = GetCurrentTimeUNIX()
}
);
}
}
LogDataBucketsDetail("cache size after [{}]", g_data_bucket_cache.size());
LogDataBuckets(
"Bulk Loaded ids [{}] column [{}] new cache size is [{}]",
ids.size(),
column,
g_data_bucket_cache.size()
);
}
void DataBucket::DeleteCachedBuckets(DataBucketLoadType::Type t, uint32 id)
{
size_t size_before = g_data_bucket_cache.size();
g_data_bucket_cache.erase(
std::remove_if(
g_data_bucket_cache.begin(),
g_data_bucket_cache.end(),
[&](DataBucketCacheEntry &ce) {
return (
(t == DataBucketLoadType::Bot && ce.e.bot_id == id) ||
(t == DataBucketLoadType::Client && ce.e.character_id == id) ||
(t == DataBucketLoadType::NPC && ce.e.npc_id == id)
);
}
),
g_data_bucket_cache.end()
);
LogDataBuckets(
"LoadType [{}] id [{}] cache size before [{}] after [{}]",
DataBucketLoadType::Name[t],
id,
size_before,
g_data_bucket_cache.size()
);
}
int64_t DataBucket::GetCurrentTimeUNIX()
{
return std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::system_clock::now().time_since_epoch()
).count();
}
bool DataBucket::ExistsInCache(const DataBucketsRepository::DataBuckets &e)
{
for (const auto &ce: g_data_bucket_cache) {
if (ce.e.id == e.id) {
return true;
}
}
return false;
}
bool DataBucket::SendDataBucketCacheUpdate(const DataBucketCacheEntry &e)
{
if (!e.e.id) {
return false;
}
EQ::Net::DynamicPacket p;
p.PutSerialize(0, e);
auto pack_size = sizeof(ServerDataBucketCacheUpdate_Struct) + p.Length();
auto pack = new ServerPacket(ServerOP_DataBucketCacheUpdate, static_cast<uint32_t>(pack_size));
auto buf = reinterpret_cast<ServerDataBucketCacheUpdate_Struct *>(pack->pBuffer);
buf->cereal_size = static_cast<uint32_t>(p.Length());
memcpy(buf->cereal_data, p.Data(), p.Length());
worldserver.SendPacket(pack);
return true;
}
void DataBucket::HandleWorldMessage(ServerPacket *p)
{
DataBucketCacheEntry n;
auto s = (ServerDataBucketCacheUpdate_Struct *) p->pBuffer;
EQ::Util::MemoryStreamReader ss(s->cereal_data, s->cereal_size);
cereal::BinaryInputArchive archive(ss);
archive(n);
LogDataBucketsDetail(
"Received cache packet for id [{}] key [{}] value [{}] action [{}]",
n.e.id,
n.e.key_,
n.e.value,
n.update_action
);
// delete
if (n.update_action == DataBucketCacheUpdateAction::Delete) {
g_data_bucket_cache.erase(
std::remove_if(
g_data_bucket_cache.begin(),
g_data_bucket_cache.end(),
[&](DataBucketCacheEntry &ce) {
bool match = n.e.id > 0 && ce.e.id == n.e.id;
if (match) {
LogDataBuckets(
"[delete] cache key [{}] id [{}] cache_size before [{}] after [{}]",
ce.e.key_,
ce.e.id,
g_data_bucket_cache.size(),
g_data_bucket_cache.size() - 1
);
}
return match;
}
),
g_data_bucket_cache.end()
);
return;
}
// update
bool has_key = false;
for (auto &ce: g_data_bucket_cache) {
int64 time_delta = ce.updated_time - n.updated_time;
if (ce.updated_time >= n.updated_time) {
LogDataBuckets(
"Attempted to update older cache key [{}] rejecting old time [{}] new time [{}] delta [{}] cache_size [{}]",
ce.e.key_,
ce.updated_time,
n.updated_time,
time_delta,
g_data_bucket_cache.size()
);
return;
}
// update cache
if (ce.e.id == n.e.id) {
DeleteFromMissesCache(n.e);
LogDataBuckets(
"[update] cache id [{}] key [{}] value [{}] old time [{}] new time [{}] delta [{}] cache_size [{}]",
ce.e.id,
ce.e.key_,
n.e.value,
ce.updated_time,
n.updated_time,
time_delta,
g_data_bucket_cache.size()
);
ce.e = n.e;
ce.updated_time = n.updated_time;
has_key = true;
break;
}
}
// create
if (!has_key) {
DeleteFromMissesCache(n.e);
size_t size_before = g_data_bucket_cache.size();
g_data_bucket_cache.emplace_back(
DataBucketCacheEntry{
.e = n.e,
.updated_time = GetCurrentTimeUNIX()
}
);
LogDataBuckets(
"[create] Adding new cache id [{}] key [{}] value [{}] cache size before [{}] after [{}]",
n.e.id,
n.e.key_,
n.e.value,
size_before,
g_data_bucket_cache.size()
);
}
}
void DataBucket::DeleteFromMissesCache(DataBucketsRepository::DataBuckets e)
{
// delete from cache where there might have been a written bucket miss to the cache
// this is to prevent the cache from growing too large
size_t size_before = g_data_bucket_cache.size();
g_data_bucket_cache.erase(
std::remove_if(
g_data_bucket_cache.begin(),
g_data_bucket_cache.end(),
[&](DataBucketCacheEntry &ce) {
return ce.e.id == 0 && ce.e.key_ == e.key_ &&
ce.e.character_id == e.character_id &&
ce.e.npc_id == e.npc_id &&
ce.e.bot_id == e.bot_id;
}
),
g_data_bucket_cache.end()
);
LogDataBucketsDetail(
"Deleted bucket misses from cache where key [{}] size before [{}] after [{}]",
e.key_,
size_before,
g_data_bucket_cache.size()
);
}

75
zone/data_bucket.h
View File

@ -9,7 +9,29 @@
#include "../common/types.h"
#include "../common/repositories/data_buckets_repository.h"
#include "mob.h"
#include "../common/json/json_archive_single_line.h"
#include "../common/servertalk.h"
enum DataBucketCacheUpdateAction : uint8 {
Upsert,
Delete
};
struct DataBucketCacheEntry {
DataBucketsRepository::DataBuckets e;
int64_t updated_time{};
DataBucketCacheUpdateAction update_action{};
template<class Archive>
void serialize(Archive &ar)
{
ar(
CEREAL_NVP(e),
CEREAL_NVP(updated_time),
CEREAL_NVP(update_action)
);
}
};
struct DataBucketKey {
std::string key;
@ -20,25 +42,52 @@ struct DataBucketKey {
int64_t bot_id;
};
namespace DataBucketLoadType {
enum Type : uint8 {
Bot,
Client,
NPC,
MaxType
};
static const std::string Name[Type::MaxType] = {
"Bot",
"Client",
"NPC",
};
}
class DataBucket {
public:
// non-scoped bucket methods (for global buckets)
static void SetData(const std::string& bucket_key, const std::string& bucket_value, std::string expires_time = "");
static bool DeleteData(const std::string& bucket_key);
static std::string GetData(const std::string& bucket_key);
static std::string GetDataExpires(const std::string& bucket_key);
static std::string GetDataRemaining(const std::string& bucket_key);
static void SetData(const std::string &bucket_key, const std::string &bucket_value, std::string expires_time = "");
static bool DeleteData(const std::string &bucket_key);
static std::string GetData(const std::string &bucket_key);
static std::string GetDataExpires(const std::string &bucket_key);
static std::string GetDataRemaining(const std::string &bucket_key);
static bool GetDataBuckets(Mob* mob);
static bool GetDataBuckets(Mob *mob);
static int64_t GetCurrentTimeUNIX();
// scoped bucket methods
static void SetData(const DataBucketKey& k);
static bool DeleteData(const DataBucketKey& k);
static DataBucketsRepository::DataBuckets GetData(const DataBucketKey& k);
static std::string GetDataExpires(const DataBucketKey& k);
static std::string GetDataRemaining(const DataBucketKey& k);
static std::string CheckBucketKey(const Mob* mob, const DataBucketKey& k);
static std::string GetScopedDbFilters(const DataBucketKey& k);
static void SetData(const DataBucketKey &k);
static bool DeleteData(const DataBucketKey &k);
static DataBucketsRepository::DataBuckets GetData(const DataBucketKey &k, bool ignore_misses_cache = false);
static std::string GetDataExpires(const DataBucketKey &k);
static std::string GetDataRemaining(const DataBucketKey &k);
static std::string GetScopedDbFilters(const DataBucketKey &k);
// bucket repository versus key matching
static bool CheckBucketMatch(const DataBucketsRepository::DataBuckets &dbe, const DataBucketKey &k);
static bool ExistsInCache(const DataBucketsRepository::DataBuckets &e);
static void BulkLoadEntities(DataBucketLoadType::Type t, std::vector<uint32> ids);
static void DeleteCachedBuckets(DataBucketLoadType::Type t, uint32 id);
static bool SendDataBucketCacheUpdate(const DataBucketCacheEntry &e);
static void HandleWorldMessage(ServerPacket *p);
static void DeleteFromMissesCache(DataBucketsRepository::DataBuckets e);
};
#endif //EQEMU_DATABUCKET_H

42
zone/lua_bot.cpp
View File

@ -504,6 +504,42 @@ std::string Lua_Bot::GetRaceAbbreviation() {
return GetPlayerRaceAbbreviation(self->GetBaseRace());
}
void Lua_Bot::DeleteBucket(std::string bucket_name)
{
Lua_Safe_Call_Void();
self->DeleteBucket(bucket_name);
}
std::string Lua_Bot::GetBucket(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucket(bucket_name);
}
std::string Lua_Bot::GetBucketExpires(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucketExpires(bucket_name);
}
std::string Lua_Bot::GetBucketRemaining(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucketRemaining(bucket_name);
}
void Lua_Bot::SetBucket(std::string bucket_name, std::string bucket_value)
{
Lua_Safe_Call_Void();
self->SetBucket(bucket_name, bucket_value);
}
void Lua_Bot::SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration)
{
Lua_Safe_Call_Void();
self->SetBucket(bucket_name, bucket_value, expiration);
}
luabind::scope lua_register_bot() {
return luabind::class_<Lua_Bot, Lua_Mob>("Bot")
.def(luabind::constructor<>())
@ -531,6 +567,7 @@ luabind::scope lua_register_bot() {
.def("Camp", (void(Lua_Bot::*)(bool))&Lua_Bot::Camp)
.def("CountBotItem", (uint32(Lua_Bot::*)(uint32))&Lua_Bot::CountBotItem)
.def("CountItemEquippedByID", (int(Lua_Bot::*)(uint32))&Lua_Bot::CountItemEquippedByID)
.def("DeleteBucket", (void(Lua_Bot::*)(std::string))&Lua_Bot::DeleteBucket)
.def("Escape", (void(Lua_Bot::*)(void))&Lua_Bot::Escape)
.def("Fling", (void(Lua_Bot::*)(float,float,float))&Lua_Bot::Fling)
.def("Fling", (void(Lua_Bot::*)(float,float,float,bool))&Lua_Bot::Fling)
@ -551,6 +588,9 @@ luabind::scope lua_register_bot() {
.def("GetBotID", (uint32(Lua_Bot::*)(void))&Lua_Bot::GetBotID)
.def("GetBotItem", (Lua_ItemInst(Lua_Bot::*)(uint16))&Lua_Bot::GetBotItem)
.def("GetBotItemIDBySlot", (uint32(Lua_Bot::*)(uint16))&Lua_Bot::GetBotItemIDBySlot)
.def("GetBucket", (std::string(Lua_Bot::*)(std::string))&Lua_Bot::GetBucket)
.def("GetBucketExpires", (std::string(Lua_Bot::*)(std::string))&Lua_Bot::GetBucketExpires)
.def("GetBucketRemaining", (std::string(Lua_Bot::*)(std::string))&Lua_Bot::GetBucketRemaining)
.def("GetClassAbbreviation", (std::string(Lua_Bot::*)(void))&Lua_Bot::GetClassAbbreviation)
.def("GetExpansionBitmask", (int(Lua_Bot::*)(void))&Lua_Bot::GetExpansionBitmask)
.def("GetGroup", (Lua_Group(Lua_Bot::*)(void))&Lua_Bot::GetGroup)
@ -576,6 +616,8 @@ luabind::scope lua_register_bot() {
.def("ReloadBotSpellSettings", (void(Lua_Bot::*)(void))&Lua_Bot::ReloadBotSpellSettings)
.def("RemoveBotItem", (void(Lua_Bot::*)(uint32))&Lua_Bot::RemoveBotItem)
.def("SendSpellAnim", (void(Lua_Bot::*)(uint16,uint16))&Lua_Bot::SendSpellAnim)
.def("SetBucket", (void(Lua_Bot::*)(std::string,std::string))&Lua_Bot::SetBucket)
.def("SetBucket", (void(Lua_Bot::*)(std::string,std::string,std::string))&Lua_Bot::SetBucket)
.def("SetExpansionBitmask", (void(Lua_Bot::*)(int))&Lua_Bot::SetExpansionBitmask)
.def("SetExpansionBitmask", (void(Lua_Bot::*)(int,bool))&Lua_Bot::SetExpansionBitmask)
.def("SetSpellDuration", (void(Lua_Bot::*)(int))&Lua_Bot::SetSpellDuration)

6
zone/lua_bot.h
View File

@ -68,6 +68,12 @@ public:
void SendSpellAnim(uint16 target_id, uint16 spell_id);
std::string GetClassAbbreviation();
std::string GetRaceAbbreviation();
void DeleteBucket(std::string bucket_name);
std::string GetBucket(std::string bucket_name);
std::string GetBucketExpires(std::string bucket_name);
std::string GetBucketRemaining(std::string bucket_name);
void SetBucket(std::string bucket_name, std::string bucket_value);
void SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration);
void ApplySpell(int spell_id);
void ApplySpell(int spell_id, int duration);

42
zone/lua_client.cpp
View File

@ -3098,6 +3098,42 @@ void Lua_Client::SetLDoNPoints(uint32 theme_id, uint32 points)
self->SetLDoNPoints(theme_id, points);
}
void Lua_Client::DeleteBucket(std::string bucket_name)
{
Lua_Safe_Call_Void();
self->DeleteBucket(bucket_name);
}
std::string Lua_Client::GetBucket(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucket(bucket_name);
}
std::string Lua_Client::GetBucketExpires(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucketExpires(bucket_name);
}
std::string Lua_Client::GetBucketRemaining(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucketRemaining(bucket_name);
}
void Lua_Client::SetBucket(std::string bucket_name, std::string bucket_value)
{
Lua_Safe_Call_Void();
self->SetBucket(bucket_name, bucket_value);
}
void Lua_Client::SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration)
{
Lua_Safe_Call_Void();
self->SetBucket(bucket_name, bucket_value, expiration);
}
luabind::scope lua_register_client() {
return luabind::class_<Lua_Client, Lua_Mob>("Client")
.def(luabind::constructor<>())
@ -3175,6 +3211,7 @@ luabind::scope lua_register_client() {
.def("CreateExpeditionFromTemplate", &Lua_Client::CreateExpeditionFromTemplate)
.def("CreateTaskDynamicZone", &Lua_Client::CreateTaskDynamicZone)
.def("DecreaseByID", (bool(Lua_Client::*)(uint32,int))&Lua_Client::DecreaseByID)
.def("DeleteBucket", (void(Lua_Client::*)(std::string))&Lua_Client::DeleteBucket)
.def("DeleteItemInInventory", (void(Lua_Client::*)(int,int))&Lua_Client::DeleteItemInInventory)
.def("DeleteItemInInventory", (void(Lua_Client::*)(int,int,bool))&Lua_Client::DeleteItemInInventory)
.def("DiaWind", (void(Lua_Client::*)(std::string))&Lua_Client::DialogueWindow)
@ -3249,6 +3286,9 @@ luabind::scope lua_register_client() {
.def("GetBotRequiredLevel", (int(Lua_Client::*)(uint8))&Lua_Client::GetBotRequiredLevel)
.def("GetBotSpawnLimit", (int(Lua_Client::*)(void))&Lua_Client::GetBotSpawnLimit)
.def("GetBotSpawnLimit", (int(Lua_Client::*)(uint8))&Lua_Client::GetBotSpawnLimit)
.def("GetBucket", (std::string(Lua_Client::*)(std::string))&Lua_Client::GetBucket)
.def("GetBucketExpires", (std::string(Lua_Client::*)(std::string))&Lua_Client::GetBucketExpires)
.def("GetBucketRemaining", (std::string(Lua_Client::*)(std::string))&Lua_Client::GetBucketRemaining)
.def("GetCarriedMoney", (uint64(Lua_Client::*)(void))&Lua_Client::GetCarriedMoney)
.def("GetCarriedPlatinum", (uint32(Lua_Client::*)(void))&Lua_Client::GetCarriedPlatinum)
.def("GetCharacterFactionLevel", (int(Lua_Client::*)(int))&Lua_Client::GetCharacterFactionLevel)
@ -3517,6 +3557,8 @@ luabind::scope lua_register_client() {
.def("SetBotRequiredLevel", (void(Lua_Client::*)(int,uint8))&Lua_Client::SetBotRequiredLevel)
.def("SetBotSpawnLimit", (void(Lua_Client::*)(int))&Lua_Client::SetBotSpawnLimit)
.def("SetBotSpawnLimit", (void(Lua_Client::*)(int,uint8))&Lua_Client::SetBotSpawnLimit)
.def("SetBucket", (void(Lua_Client::*)(std::string,std::string))&Lua_Client::SetBucket)
.def("SetBucketExpires", (void(Lua_Client::*)(std::string,std::string,std::string))&Lua_Client::SetBucket)
.def("SetClientMaxLevel", (void(Lua_Client::*)(int))&Lua_Client::SetClientMaxLevel)
.def("SetConsumption", (void(Lua_Client::*)(int, int))&Lua_Client::SetConsumption)
.def("SetDeity", (void(Lua_Client::*)(int))&Lua_Client::SetDeity)

6
zone/lua_client.h
View File

@ -474,6 +474,12 @@ public:
std::string GetClassAbbreviation();
std::string GetRaceAbbreviation();
void SetLDoNPoints(uint32 theme_id, uint32 points);
void DeleteBucket(std::string bucket_name);
std::string GetBucket(std::string bucket_name);
std::string GetBucketExpires(std::string bucket_name);
std::string GetBucketRemaining(std::string bucket_name);
void SetBucket(std::string bucket_name, std::string bucket_value);
void SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration);
void ApplySpell(int spell_id);
void ApplySpell(int spell_id, int duration);

42
zone/lua_npc.cpp
View File

@ -777,6 +777,42 @@ bool Lua_NPC::HasSpecialAbilities() {
return self->HasSpecialAbilities();
}
void Lua_NPC::DeleteBucket(std::string bucket_name)
{
Lua_Safe_Call_Void();
self->DeleteBucket(bucket_name);
}
std::string Lua_NPC::GetBucket(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucket(bucket_name);
}
std::string Lua_NPC::GetBucketExpires(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucketExpires(bucket_name);
}
std::string Lua_NPC::GetBucketRemaining(std::string bucket_name)
{
Lua_Safe_Call_String();
return self->GetBucketRemaining(bucket_name);
}
void Lua_NPC::SetBucket(std::string bucket_name, std::string bucket_value)
{
Lua_Safe_Call_Void();
self->SetBucket(bucket_name, bucket_value);
}
void Lua_NPC::SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration)
{
Lua_Safe_Call_Void();
self->SetBucket(bucket_name, bucket_value, expiration);
}
luabind::scope lua_register_npc() {
return luabind::class_<Lua_NPC, Lua_Mob>("NPC")
.def(luabind::constructor<>())
@ -804,12 +840,16 @@ luabind::scope lua_register_npc() {
.def("ClearLastName", (void(Lua_NPC::*)(void))&Lua_NPC::ClearLastName)
.def("CountItem", (uint16(Lua_NPC::*)(uint32))&Lua_NPC::CountItem)
.def("CountLoot", (int(Lua_NPC::*)(void))&Lua_NPC::CountLoot)
.def("DeleteBucket", (void(Lua_NPC::*)(std::string))&Lua_NPC::DeleteBucket)
.def("DisplayWaypointInfo", (void(Lua_NPC::*)(Lua_Client))&Lua_NPC::DisplayWaypointInfo)
.def("DoClassAttacks", (void(Lua_NPC::*)(Lua_Mob))&Lua_NPC::DoClassAttacks)
.def("GetAccuracyRating", (int(Lua_NPC::*)(void))&Lua_NPC::GetAccuracyRating)
.def("GetAttackDelay", (int(Lua_NPC::*)(void))&Lua_NPC::GetAttackDelay)
.def("GetAttackSpeed", (float(Lua_NPC::*)(void))&Lua_NPC::GetAttackSpeed)
.def("GetAvoidanceRating", &Lua_NPC::GetAvoidanceRating)
.def("GetBucket", (std::string(Lua_NPC::*)(std::string))&Lua_NPC::GetBucket)
.def("GetBucketExpires", (std::string(Lua_NPC::*)(std::string))&Lua_NPC::GetBucketExpires)
.def("GetBucketRemaining", (std::string(Lua_NPC::*)(std::string))&Lua_NPC::GetBucketRemaining)
.def("GetCopper", (uint32(Lua_NPC::*)(void))&Lua_NPC::GetCopper)
.def("GetFirstSlotByItemID", (uint16(Lua_NPC::*)(uint32))&Lua_NPC::GetFirstSlotByItemID)
.def("GetFollowCanRun", (bool(Lua_NPC::*)(void))&Lua_NPC::GetFollowCanRun)
@ -894,6 +934,8 @@ luabind::scope lua_register_npc() {
.def("ScaleNPC", (void(Lua_NPC::*)(uint8,bool))&Lua_NPC::ScaleNPC)
.def("SendPayload", (void(Lua_NPC::*)(int))&Lua_NPC::SendPayload)
.def("SendPayload", (void(Lua_NPC::*)(int,std::string))&Lua_NPC::SendPayload)
.def("SetBucket", (void(Lua_NPC::*)(std::string,std::string))&Lua_NPC::SetBucket)
.def("SetBucket", (void(Lua_NPC::*)(std::string,std::string,std::string))&Lua_NPC::SetBucket)
.def("SetCopper", (void(Lua_NPC::*)(uint32))&Lua_NPC::SetCopper)
.def("SetFollowCanRun", (void(Lua_NPC::*)(bool))&Lua_NPC::SetFollowCanRun)
.def("SetFollowDistance", (void(Lua_NPC::*)(int))&Lua_NPC::SetFollowDistance)

6
zone/lua_npc.h
View File

@ -176,6 +176,12 @@ public:
void ScaleNPC(uint8 npc_level, bool override_special_abilities);
bool IsUnderwaterOnly();
bool HasSpecialAbilities();
void DeleteBucket(std::string bucket_name);
std::string GetBucket(std::string bucket_name);
std::string GetBucketExpires(std::string bucket_name);
std::string GetBucketRemaining(std::string bucket_name);
void SetBucket(std::string bucket_name, std::string bucket_value);
void SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration);
};
#endif

13
zone/mob.cpp
View File

@ -8352,3 +8352,16 @@ DataBucketKey Mob::GetScopedBucketKeys()
return k;
}
uint32 Mob::GetMobTypeIdentifier()
{
if (IsClient()) {
return CastToClient()->CharacterID();
} else if (IsNPC()) {
return GetNPCTypeID();
} else if (IsBot()) {
return CastToBot()->GetBotID();
}
return 0;
}

4
zone/mob.h
View File

@ -1408,8 +1408,6 @@ public:
/// this cures timing issues cuz dead animation isn't done but server side feigning is?
inline bool GetFeigned() const { return(feigned); }
std::vector<DataBucketCache> m_data_bucket_cache;
// Data Bucket Methods
void DeleteBucket(std::string bucket_name);
std::string GetBucket(std::string bucket_name);
@ -1417,6 +1415,8 @@ public:
std::string GetBucketRemaining(std::string bucket_name);
void SetBucket(std::string bucket_name, std::string bucket_value, std::string expiration = "");
uint32 GetMobTypeIdentifier();
// Heroic Stat Benefits
float CheckHeroicBonusesDataBuckets(std::string bucket_name);

2
zone/spells.cpp
View File

@ -5886,7 +5886,7 @@ bool Client::SpellBucketCheck(uint16 spell_id, uint32 character_id) {
spell_bucket_name
);
auto bucket_value = DataBucket::GetData(old_bucket_name);
std::string bucket_value = DataBucket::GetData(old_bucket_name);
if (!bucket_value.empty()) {
if (Strings::IsNumber(bucket_value) && Strings::IsNumber(spell_bucket_value)) {
if (Strings::ToInt(bucket_value) >= Strings::ToInt(spell_bucket_value)) {

5
zone/worldserver.cpp
View File

@ -3329,6 +3329,11 @@ void WorldServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p)
SharedTaskZoneMessaging::HandleWorldMessage(pack);
break;
}
case ServerOP_DataBucketCacheUpdate:
{
DataBucket::HandleWorldMessage(pack);
break;
}
default: {
LogInfo("Unknown ZS Opcode [{}] size [{}]", (int)pack->opcode, pack->size);
break;

9
zone/zonedb.cpp
View File

@ -1927,6 +1927,8 @@ const NPCType *ZoneDatabase::LoadNPCTypesData(uint32 npc_type_id, bool bulk_load
);
}
std::vector<uint32> npc_ids;
for (NpcTypesRepository::NpcTypes &n : NpcTypesRepository::GetWhere((Database &) content_db, filter)) {
NPCType *t;
t = new NPCType;
@ -2137,8 +2139,15 @@ const NPCType *ZoneDatabase::LoadNPCTypesData(uint32 npc_type_id, bool bulk_load
zone->npctable[t->npc_id] = t;
npc = t;
// If NPC ID is not in npc_ids, add to vector
if (!std::count(npc_ids.begin(), npc_ids.end(), t->npc_id)) {
npc_ids.emplace_back(t->npc_id);
}
}
DataBucket::BulkLoadEntities(DataBucketLoadType::NPC, npc_ids);
return npc;
}