From d1c7e45437ededcd70ff4ddd7737fa9c41449f55 Mon Sep 17 00:00:00 2001 From: Mitch Freeman <65987027+neckkola@users.noreply.github.com> Date: Thu, 9 May 2024 02:53:36 -0300 Subject: [PATCH] [FIX] Fix for world crash with over 1500 guilds (#4299) * Fix for world crash with over 1500 guilds There was an existing issue with certain clients (RoF2) if there were more than 1500 guilds. This also enhances the loading of guilds in both world and zone for performance if there are large number of guilds as RoF2 will support 1500+ guilds. * Safely access permissions map --------- Co-authored-by: Akkadius --- common/eq_packet_structs.h | 33 +++++++++ common/guild_base.cpp | 68 ++++++++++--------- common/guild_base.h | 2 +- common/patches/rof2.cpp | 63 ++++++----------- common/patches/rof2_limits.h | 1 + common/patches/titanium.cpp | 22 ++++++ common/patches/titanium_limits.h | 1 + common/patches/titanium_ops.h | 1 + common/patches/uf.cpp | 60 ++++++---------- common/patches/uf_limits.h | 1 + .../guild_permissions_repository.h | 25 +++++++ common/repositories/guild_ranks_repository.h | 24 +++++++ world/client.cpp | 21 +++--- zone/guild.cpp | 24 +++---- 14 files changed, 210 insertions(+), 136 deletions(-) diff --git a/common/eq_packet_structs.h b/common/eq_packet_structs.h index 51b6efea7..12e2c3d35 100644 --- a/common/eq_packet_structs.h +++ b/common/eq_packet_structs.h @@ -29,6 +29,7 @@ #include "textures.h" #include "../cereal/include/cereal/archives/binary.hpp" #include "../cereal/include/cereal/types/string.hpp" +#include "../cereal/include/cereal/types/vector.hpp" static const uint32 BUFF_COUNT = 42; @@ -1692,6 +1693,38 @@ struct GuildsList_Struct { GuildsListEntry_Struct Guilds[MAX_NUMBER_GUILDS]; }; +struct GuildsListMessagingEntry_Struct { + /*000*/ uint32 guild_id; + /*004*/ std::string guild_name; + + template + void serialize(Archive& archive) + { + archive( + CEREAL_NVP(guild_id), + CEREAL_NVP(guild_name) + ); + } +}; + +struct GuildsListMessaging_Struct { + /*000*/ char header[64]; + /*064*/ uint32 no_of_guilds; + /*068*/ uint32 string_length; + /*072*/ std::vector guild_detail; + + template + void serialize(Archive& archive) + { + archive( + CEREAL_NVP(header), + CEREAL_NVP(no_of_guilds), + CEREAL_NVP(string_length), + CEREAL_NVP(guild_detail) + ); + } +}; + struct GuildUpdate_Struct { uint32 guildID; GuildsListEntry_Struct entry; diff --git a/common/guild_base.cpp b/common/guild_base.cpp index 722e83e05..04275fa5b 100644 --- a/common/guild_base.cpp +++ b/common/guild_base.cpp @@ -92,10 +92,20 @@ BaseGuildManager::~BaseGuildManager() bool BaseGuildManager::LoadGuilds() { ClearGuilds(); - auto guilds = GuildsRepository::All(*m_db); - auto guilds_ranks = GuildRanksRepository::All(*m_db); - auto guilds_permissions = GuildPermissionsRepository::All(*m_db); - auto guilds_tributes = GuildTributesRepository::All(*m_db); + auto guilds = GuildsRepository::GetWhere( + *m_db, + fmt::format("`id` < '{}'", RoF2::constants::MAX_GUILD_ID) + ); + auto guilds_ranks = GuildRanksRepository::LoadAll(*m_db); + auto guilds_permissions = GuildPermissionsRepository::LoadAll(*m_db); + auto guilds_tributes = GuildTributesRepository::GetWhere( + *m_db, + fmt::format( + "`guild_id` < '{}'", + RoF2::constants::MAX_GUILD_ID + ) + ); + if (guilds.empty()) { LogGuilds("No Guilds found in database."); @@ -108,22 +118,27 @@ bool BaseGuildManager::LoadGuilds() _CreateGuild(g.id, g.name, g.leader, g.minstatus, g.motd, g.motd_setter, g.channel, g.url, g.favor); - for (auto const &r: guilds_ranks) { - if (r.guild_id == g.id) { - m_guilds[g.id]->rank_names[r.rank_] = r.title; + for (int i = 1; i <= GUILD_MAX_RANK; i++) { + auto key = fmt::format("{}-{}", g.id, i); + + if (guilds_ranks.contains(key)) { + m_guilds[g.id]->rank_names[i] = guilds_ranks.find(key)->second; } } auto count = 0; - for (auto const &p: guilds_permissions) { - if (p.guild_id == g.id) { + for (int i = 1; i <= GUILD_MAX_FUNCTIONS; i++) { + auto key = fmt::format("{}-{}", g.id, i); + if (guilds_permissions.contains(key)) { + auto p = guilds_permissions.find(key)->second; m_guilds[g.id]->functions[p.perm_id].id = p.id; m_guilds[g.id]->functions[p.perm_id].guild_id = p.guild_id; m_guilds[g.id]->functions[p.perm_id].perm_id = p.perm_id; m_guilds[g.id]->functions[p.perm_id].perm_value = p.permission; - count++; } + + count++; } if (count < GUILD_MAX_FUNCTIONS) { @@ -929,33 +944,24 @@ bool BaseGuildManager::GetCharInfo(uint32 char_id, CharGuildInfo &into) } -//returns ownership of the buffer. -uint8 *BaseGuildManager::MakeGuildList(const char *head_name, uint32 &length) const +GuildsListMessaging_Struct BaseGuildManager::MakeGuildList() { - //dynamic structs will make this a lot less painful. + GuildsListMessaging_Struct guild_list_messaging{}; + uint32 string_length = 0; - length = sizeof(GuildsList_Struct); - auto buffer = new uint8[length]; + for (auto const &g: m_guilds) { + GuildsListMessagingEntry_Struct guild_entry{}; - //a bit little better than memsetting the whole thing... - uint32 r, pos; - for (r = 0, pos = 0; r <= MAX_NUMBER_GUILDS; r++, pos += 64) { - //strcpy((char *) buffer+pos, "BAD GUILD"); - // These 'BAD GUILD' entries were showing in the drop-downs for selecting guilds in the LFP window, - // so just fill unused entries with an empty string instead. - buffer[pos] = '\0'; + guild_entry.guild_id = g.first; + guild_entry.guild_name = g.second->name; + string_length += g.second->name.length() + 1; + guild_list_messaging.guild_detail.push_back(guild_entry); } - strn0cpy((char *) buffer, head_name, 64); + guild_list_messaging.no_of_guilds = m_guilds.size(); + guild_list_messaging.string_length = string_length; - std::map::const_iterator cur, end; - cur = m_guilds.begin(); - end = m_guilds.end(); - for (; cur != end; ++cur) { - pos = 64 + (64 * cur->first); - strn0cpy((char *) buffer + pos, cur->second->name.c_str(), 64); - } - return (buffer); + return guild_list_messaging; } const char *BaseGuildManager::GetRankName(uint32 guild_id, uint8 rank) const diff --git a/common/guild_base.h b/common/guild_base.h index 3664ab7d2..0cd4fe5f8 100644 --- a/common/guild_base.h +++ b/common/guild_base.h @@ -134,7 +134,7 @@ class BaseGuildManager bool CheckGMStatus(uint32 guild_id, uint8 status) const; bool CheckPermission(uint32 guild_id, uint8 rank, GuildAction act) const; bool UpdateDbBankerFlag(uint32 charid, bool is_banker); - uint8* MakeGuildList(const char* head_name, uint32& length) const; //make a guild list packet, returns ownership of the buffer. + GuildsListMessaging_Struct MakeGuildList(); uint8 GetDisplayedRank(uint32 guild_id, uint8 rank, uint32 char_id) const; uint32 GetGuildIDByName(const char *GuildName); uint32 GetGuildIDByCharacterID(uint32 character_id); diff --git a/common/patches/rof2.cpp b/common/patches/rof2.cpp index 052e66108..ac5f61fa3 100644 --- a/common/patches/rof2.cpp +++ b/common/patches/rof2.cpp @@ -1401,58 +1401,35 @@ namespace RoF2 ENCODE(OP_GuildsList) { EQApplicationPacket *in = *p; - *p = nullptr; + *p = nullptr; - uint32 NumberOfGuilds = in->size / 64; - uint32 PacketSize = 68; // 64 x 0x00 + a uint32 that I am guessing is the highest guild ID in use. + GuildsListMessaging_Struct glms{}; + EQ::Util::MemoryStreamReader ss(reinterpret_cast(in->pBuffer), in->size); + cereal::BinaryInputArchive ar(ss); + ar(glms); - unsigned char *__emu_buffer = in->pBuffer; + auto packet_size = 64 + 4 + glms.guild_detail.size() * 4 + glms.string_length; + auto buffer = new uchar[packet_size]; + auto buf_pos = buffer; - char *InBuffer = (char *)__emu_buffer; + memset(buf_pos, 0, 64); + buf_pos += 64; - uint32 HighestGuildID = 0; + VARSTRUCT_ENCODE_TYPE(uint32, buf_pos, glms.no_of_guilds); - for (unsigned int i = 0; i < NumberOfGuilds; ++i) - { - if (InBuffer[0]) - { - PacketSize += (5 + strlen(InBuffer)); - HighestGuildID += 1; + for (auto const &g: glms.guild_detail) { + if (g.guild_id < RoF2::constants::MAX_GUILD_ID) { + VARSTRUCT_ENCODE_TYPE(uint32, buf_pos, g.guild_id); + strn0cpy((char *) buf_pos, g.guild_name.c_str(), g.guild_name.length() + 1); + buf_pos += g.guild_name.length() + 1; } - InBuffer += 64; } - PacketSize++; // Appears to be an extra 0x00 at the very end. + auto outapp = new EQApplicationPacket(OP_GuildsList); + outapp->size = packet_size; + outapp->pBuffer = buffer; - in->size = PacketSize; - in->pBuffer = new unsigned char[in->size]; - - InBuffer = (char *)__emu_buffer; - - char *OutBuffer = (char *)in->pBuffer; - - // Init the first 64 bytes to zero, as per live. - // - memset(OutBuffer, 0, 64); - - OutBuffer += 64; - - VARSTRUCT_ENCODE_TYPE(uint32, OutBuffer, HighestGuildID); - - for (unsigned int i = 0; i < NumberOfGuilds; ++i) - { - if (InBuffer[0]) - { - VARSTRUCT_ENCODE_TYPE(uint32, OutBuffer, i - 1); - VARSTRUCT_ENCODE_STRING(OutBuffer, InBuffer); - } - InBuffer += 64; - } - - VARSTRUCT_ENCODE_TYPE(uint8, OutBuffer, 0x00); - - delete[] __emu_buffer; - dest->FastQueuePacket(&in, ack_req); + dest->FastQueuePacket(&outapp); } ENCODE(OP_GuildTributeDonateItem) diff --git a/common/patches/rof2_limits.h b/common/patches/rof2_limits.h index fa5f551d1..7e7774de6 100644 --- a/common/patches/rof2_limits.h +++ b/common/patches/rof2_limits.h @@ -271,6 +271,7 @@ namespace RoF2 const size_t CHARACTER_CREATION_LIMIT = 12; const size_t SAY_LINK_BODY_SIZE = 56; + const uint32 MAX_GUILD_ID = 50000; } /*constants*/ diff --git a/common/patches/titanium.cpp b/common/patches/titanium.cpp index 01b8edbdd..568ab66c0 100644 --- a/common/patches/titanium.cpp +++ b/common/patches/titanium.cpp @@ -748,6 +748,28 @@ namespace Titanium FINISH_ENCODE(); } + ENCODE(OP_GuildsList) + { + EQApplicationPacket* in = *p; + *p = nullptr; + + GuildsListMessaging_Struct glms{}; + EQ::Util::MemoryStreamReader ss(reinterpret_cast(in->pBuffer), in->size); + cereal::BinaryInputArchive ar(ss); + ar(glms); + + auto outapp = new EQApplicationPacket(OP_GuildsList, sizeof(structs::GuildsList_Struct)); + auto out = (structs::GuildsList_Struct *) outapp->pBuffer; + + for (auto const& g : glms.guild_detail) { + if (g.guild_id < Titanium::constants::MAX_GUILD_ID) { + strn0cpy(out->Guilds[g.guild_id].name, g.guild_name.c_str(), sizeof(out->Guilds[g.guild_id].name)); + } + } + + dest->FastQueuePacket(&outapp); + } + ENCODE(OP_GuildMemberAdd) { ENCODE_LENGTH_EXACT(GuildMemberAdd_Struct) diff --git a/common/patches/titanium_limits.h b/common/patches/titanium_limits.h index 09d3d99ca..cdd3171f4 100644 --- a/common/patches/titanium_limits.h +++ b/common/patches/titanium_limits.h @@ -287,6 +287,7 @@ namespace Titanium const size_t CHARACTER_CREATION_LIMIT = 8; // Hard-coded in client - DO NOT ALTER const size_t SAY_LINK_BODY_SIZE = 45; + const uint32 MAX_GUILD_ID = 1500; } /*constants*/ diff --git a/common/patches/titanium_ops.h b/common/patches/titanium_ops.h index ce0de7a38..9f4867965 100644 --- a/common/patches/titanium_ops.h +++ b/common/patches/titanium_ops.h @@ -45,6 +45,7 @@ E(OP_Emote) E(OP_FormattedMessage) E(OP_GroundSpawn) E(OP_SetGuildRank) +E(OP_GuildsList) E(OP_GuildMemberLevelUpdate) E(OP_GuildMemberList) E(OP_GuildMemberAdd) diff --git a/common/patches/uf.cpp b/common/patches/uf.cpp index c77406500..290468cd3 100644 --- a/common/patches/uf.cpp +++ b/common/patches/uf.cpp @@ -1182,53 +1182,37 @@ namespace UF ENCODE(OP_GuildsList) { - EQApplicationPacket *in = *p; + EQApplicationPacket* in = *p; *p = nullptr; - uint32 NumberOfGuilds = in->size / 64; - uint32 PacketSize = 68; // 64 x 0x00 + a uint32 that I am guessing is the highest guild ID in use. + GuildsListMessaging_Struct glms{}; + EQ::Util::MemoryStreamReader ss(reinterpret_cast(in->pBuffer), in->size); + cereal::BinaryInputArchive ar(ss); + ar(glms); - unsigned char *__emu_buffer = in->pBuffer; - char *InBuffer = (char *)__emu_buffer; - uint32 actual_no_guilds = 0; + auto packet_size = 64 + 4 + glms.guild_detail.size() * 4 + glms.string_length; + auto buffer = new uchar[packet_size]; + auto buf_pos = buffer; - for (unsigned int i = 0; i < NumberOfGuilds; ++i) - { - if (InBuffer[0]) - { - PacketSize += (5 + strlen(InBuffer)); - actual_no_guilds++; + memset(buf_pos, 0, 64); + buf_pos += 64; + + VARSTRUCT_ENCODE_TYPE(uint32, buf_pos, glms.no_of_guilds); + + for (auto const& g : glms.guild_detail) { + if (g.guild_id < UF::constants::MAX_GUILD_ID) { + VARSTRUCT_ENCODE_TYPE(uint32, buf_pos, g.guild_id); + strn0cpy((char *) buf_pos, g.guild_name.c_str(), g.guild_name.length() + 1); + buf_pos += g.guild_name.length() + 1; } - InBuffer += 64; } - PacketSize++; // Appears to be an extra 0x00 at the very end. - in->size = PacketSize; - in->pBuffer = new unsigned char[in->size]; - InBuffer = (char *)__emu_buffer; - char *OutBuffer = (char *)in->pBuffer; + auto outapp = new EQApplicationPacket(OP_GuildsList); - // Init the first 64 bytes to zero, as per live. - // - memset(OutBuffer, 0, 64); - OutBuffer += 64; + outapp->size = packet_size; + outapp->pBuffer = buffer; - VARSTRUCT_ENCODE_TYPE(uint32, OutBuffer, actual_no_guilds); - - for (unsigned int i = 0; i < NumberOfGuilds; ++i) - { - if (InBuffer[0]) - { - VARSTRUCT_ENCODE_TYPE(uint32, OutBuffer, i - 1); - VARSTRUCT_ENCODE_STRING(OutBuffer, InBuffer); - } - InBuffer += 64; - } - - VARSTRUCT_ENCODE_TYPE(uint8, OutBuffer, 0x00); - - delete[] __emu_buffer; - dest->FastQueuePacket(&in, ack_req); + dest->FastQueuePacket(&outapp); } ENCODE(OP_GuildMemberAdd) diff --git a/common/patches/uf_limits.h b/common/patches/uf_limits.h index 978699ed2..29eb2495b 100644 --- a/common/patches/uf_limits.h +++ b/common/patches/uf_limits.h @@ -289,6 +289,7 @@ namespace UF const size_t CHARACTER_CREATION_LIMIT = 12; const size_t SAY_LINK_BODY_SIZE = 50; + const uint32 MAX_GUILD_ID = 50000; } /*constants*/ diff --git a/common/repositories/guild_permissions_repository.h b/common/repositories/guild_permissions_repository.h index 35c1c3da3..62d67621b 100644 --- a/common/repositories/guild_permissions_repository.h +++ b/common/repositories/guild_permissions_repository.h @@ -45,6 +45,31 @@ public: // Custom extended repository methods here + static std::map LoadAll(Database &db) + { + std::map all_entries; + + auto results = db.QueryDatabase( + fmt::format( + "{} WHERE `guild_id` < {}", + BaseSelect(), + RoF2::constants::MAX_GUILD_ID + )); + + for (auto row = results.begin(); row != results.end(); ++row) { + GuildPermissions e{}; + + e.id = static_cast(atoi(row[0])); + e.perm_id = static_cast(atoi(row[1])); + e.guild_id = static_cast(atoi(row[2])); + e.permission = static_cast(atoi(row[3])); + + auto key = fmt::format("{}-{}", e.guild_id, e.perm_id); + all_entries.emplace(key, e); + } + + return all_entries; + } }; #endif //EQEMU_GUILD_PERMISSIONS_REPOSITORY_H diff --git a/common/repositories/guild_ranks_repository.h b/common/repositories/guild_ranks_repository.h index 8cfe0118d..0bdd23bd5 100644 --- a/common/repositories/guild_ranks_repository.h +++ b/common/repositories/guild_ranks_repository.h @@ -60,6 +60,30 @@ public: return 1; } + + static std::map LoadAll(Database &db) + { + std::map all_entries; + + auto results = db.QueryDatabase(fmt::format( + "{} WHERE `guild_id` < {}", + BaseSelect(), + RoF2::constants::MAX_GUILD_ID) + ); + + for (auto row = results.begin(); row != results.end(); ++row) { + GuildRanks e{}; + + e.guild_id = row[0] ? static_cast(strtoul(row[0], nullptr, 10)) : 0; + e.rank_ = row[1] ? static_cast(strtoul(row[1], nullptr, 10)) : 0; + e.title = row[2] ? row[2] : ""; + + auto key = fmt::format("{}-{}", e.guild_id, e.rank_); + all_entries.emplace(key, e.title); + } + + return all_entries; + } }; #endif //EQEMU_GUILD_RANKS_REPOSITORY_H diff --git a/world/client.cpp b/world/client.cpp index 60962bc84..bcb9100a3 100644 --- a/world/client.cpp +++ b/world/client.cpp @@ -1577,19 +1577,20 @@ void Client::QueuePacket(const EQApplicationPacket* app, bool ack_req) { eqs->QueuePacket(app, ack_req); } -void Client::SendGuildList() { - EQApplicationPacket *outapp; - outapp = new EQApplicationPacket(OP_GuildsList); +void Client::SendGuildList() +{ + auto guilds_list = guild_mgr.MakeGuildList(); - //ask the guild manager to build us a nice guild list packet - outapp->pBuffer = guild_mgr.MakeGuildList("", outapp->size); - if(outapp->pBuffer == nullptr) { - safe_delete(outapp); - return; - } + std::stringstream ss; + cereal::BinaryOutputArchive ar(ss); + ar(guilds_list); + uint32 packet_size = ss.str().length(); - eqs->FastQueuePacket((EQApplicationPacket **)&outapp); + std::unique_ptr out(new EQApplicationPacket(OP_GuildsList, packet_size)); + memcpy(out->pBuffer, ss.str().data(), out->size); + + QueuePacket(out.get()); } // @merth: I have no idea what this struct is for, so it's hardcoded for now diff --git a/zone/guild.cpp b/zone/guild.cpp index ab47427e9..40bddc2e6 100644 --- a/zone/guild.cpp +++ b/zone/guild.cpp @@ -189,22 +189,20 @@ void Client::SendGuildSpawnAppearance() { UpdateWho(); } -void Client::SendGuildList() { - EQApplicationPacket *outapp; -// outapp = new EQApplicationPacket(OP_ZoneGuildList); - outapp = new EQApplicationPacket(OP_GuildsList); +void Client::SendGuildList() +{ + auto guilds_list = guild_mgr.MakeGuildList(); - //ask the guild manager to build us a nice guild list packet - outapp->pBuffer = guild_mgr.MakeGuildList(/*GetName()*/"", outapp->size); - if(outapp->pBuffer == nullptr) { - LogGuilds("Unable to make guild list!"); - safe_delete(outapp); - return; - } + std::stringstream ss; + cereal::BinaryOutputArchive ar(ss); + ar(guilds_list); - LogGuilds("Sending OP_ZoneGuildList of length [{}]", outapp->size); + uint32 packet_size = ss.str().length(); - FastQueuePacket(&outapp); + std::unique_ptr out(new EQApplicationPacket(OP_GuildsList, packet_size)); + memcpy(out->pBuffer, ss.str().data(), out->size); + + QueuePacket(out.get()); }