[Performance] Server Reload Overhaul (#4689)

* [Performance] Server Reload Overhaul

* Client::SendReloadCommandMessages

* Remove global buffs

world/console.cpp

@@ -911,11 +911,7 @@ void ConsoleReloadWorld(
 )
 {
 	connection->SendLine("Reloading World...");
-	auto               pack = new ServerPacket(ServerOP_ReloadWorld, sizeof(ReloadWorld_Struct));
-	ReloadWorld_Struct *RW  = (ReloadWorld_Struct *) pack->pBuffer;
-	RW->global_repop = ReloadWorld::Repop;
-	zoneserver_list.SendPacket(pack);
-	safe_delete(pack);
+	zoneserver_list.SendServerReload(ServerReload::Type::WorldRepop, nullptr);
 }
 
 auto debounce_reload = std::chrono::system_clock::now();

world/eqemu_api_world_data_service.cpp

@@ -5,6 +5,7 @@
 #include "zoneserver.h"
 #include "zonelist.h"
 #include "../common/database_schema.h"
+#include "../common/server_reload_types.h"
 #include "../common/zone_store.h"
 #include "worlddb.h"
 #include "wguild_mgr.h"
@@ -113,136 +114,47 @@ void callGetClientList(Json::Value &response)
 	client_list.GetClientList(response);
 }
 
-
-struct Reload {
-	std::string command{};
-	uint16      opcode;
-	std::string desc{};
-};
-
-std::vector<Reload> reload_types = {
-	Reload{.command = "aa", .opcode = ServerOP_ReloadAAData, .desc = "Alternate Advancement"},
-	Reload{.command = "alternate_currencies", .opcode = ServerOP_ReloadAlternateCurrencies, .desc = "Alternate Currencies"},
-	Reload{.command = "base_data", .opcode = ServerOP_ReloadBaseData, .desc = "Base Data"},
-	Reload{.command = "blocked_spells", .opcode = ServerOP_ReloadBlockedSpells, .desc = "Blocked Spells"},
-	Reload{.command = "commands", .opcode = ServerOP_ReloadCommands, .desc = "Commands"},
-	Reload{.command = "content_flags", .opcode = ServerOP_ReloadContentFlags, .desc = "Content Flags"},
-	Reload{.command = "data_buckets_cache", .opcode = ServerOP_ReloadDataBucketsCache, .desc = "Data Buckets Cache"},
-	Reload{.command = "doors", .opcode = ServerOP_ReloadDoors, .desc = "Doors"},
-	Reload{.command = "dztemplates", .opcode = ServerOP_ReloadDzTemplates, .desc = "Dynamic Zone Templates"},
-	Reload{.command = "ground_spawns", .opcode = ServerOP_ReloadGroundSpawns, .desc = "Ground Spawns"},
-	Reload{.command = "level_mods", .opcode = ServerOP_ReloadLevelEXPMods, .desc = "Level Mods"},
-	Reload{.command = "logs", .opcode = ServerOP_ReloadLogs, .desc = "Log Settings"},
-	Reload{.command = "loot", .opcode = ServerOP_ReloadLoot, .desc = "Loot"},
-	Reload{.command = "merchants", .opcode = ServerOP_ReloadMerchants, .desc = "Merchants"},
-	Reload{.command = "npc_emotes", .opcode = ServerOP_ReloadNPCEmotes, .desc = "NPC Emotes"},
-	Reload{.command = "npc_spells", .opcode = ServerOP_ReloadNPCSpells, .desc = "NPC Spells"},
-	Reload{.command = "objects", .opcode = ServerOP_ReloadObjects, .desc = "Objects"},
-	Reload{.command = "opcodes", .opcode = ServerOP_ReloadOpcodes, .desc = "Opcodes"},
-	Reload{.command = "perl_export", .opcode = ServerOP_ReloadPerlExportSettings, .desc = "Perl Event Export Settings"},
-	Reload{.command = "rules", .opcode = ServerOP_ReloadRules, .desc = "Rules"},
-	Reload{.command = "skill_caps", .opcode = ServerOP_ReloadSkillCaps, .desc = "Skill Caps"},
-	Reload{.command = "static", .opcode = ServerOP_ReloadStaticZoneData, .desc = "Static Zone Data"},
-	Reload{.command = "tasks", .opcode = ServerOP_ReloadTasks, .desc = "Tasks"},
-	Reload{.command = "titles", .opcode = ServerOP_ReloadTitles, .desc = "Titles"},
-	Reload{.command = "traps", .opcode = ServerOP_ReloadTraps, .desc = "Traps"},
-	Reload{.command = "variables", .opcode = ServerOP_ReloadVariables, .desc = "Variables"},
-	Reload{.command = "veteran_rewards", .opcode = ServerOP_ReloadVeteranRewards, .desc = "Veteran Rewards"},
-	Reload{.command = "world", .opcode = ServerOP_ReloadWorld, .desc = "World"},
-	Reload{.command = "zone_points", .opcode = ServerOP_ReloadZonePoints, .desc = "Zone Points"},
-};
-
 void getReloadTypes(Json::Value &response)
 {
-	for (auto &c: reload_types) {
+	for (auto &t: ServerReload::GetTypes()) {
 		Json::Value v;
 
-		v["command"]     = c.command;
-		v["opcode"]      = c.opcode;
-		v["description"] = c.desc;
+		v["command"]     = std::to_string(t);
+		v["description"] = ServerReload::GetName(t);
 		response.append(v);
 	}
 }
 
-
 void EQEmuApiWorldDataService::reload(Json::Value &r, const std::vector<std::string> &args)
 {
 	std::vector<std::string> commands{};
-	commands.reserve(reload_types.size());
-	for (auto &c: reload_types) {
-		commands.emplace_back(c.command);
+	commands.reserve(ServerReload::GetTypes().size());
+	for (auto &c: ServerReload::GetTypes()) {
+		commands.emplace_back(std::to_string(c));
 	}
 
 	std::string command = !args[1].empty() ? args[1] : "";
 	if (command.empty()) {
-		message(r, fmt::format("Need to provide a type to reload. Example(s) [{}]", Strings::Implode("|", commands)));
+		message(r, fmt::format("Need to provide a type ID to reload. Example(s) [{}]", Strings::Implode("|", commands)));
 		return;
 	}
 
 	ServerPacket *pack = nullptr;
 
-	bool      found_command = false;
-	for (auto &c: reload_types) {
-		if (command == c.command) {
-			if (c.command == "world") {
-				uint8 global_repop = ReloadWorld::NoRepop;
+	bool found_command = false;
 
-				if (Strings::IsNumber(args[2])) {
-					global_repop = static_cast<uint8>(Strings::ToUnsignedInt(args[2]));
-
-					if (global_repop > ReloadWorld::ForceRepop) {
-						global_repop = ReloadWorld::ForceRepop;
-					}
-				}
-
-				message(
-					r,
-					fmt::format(
-						"Attempting to reload Quests {}worldwide.",
-						(
-							global_repop ?
-								(
-									global_repop == ReloadWorld::Repop ?
-										"and repop NPCs " :
-										"and forcefully repop NPCs "
-								) :
-								""
-						)
-					)
-				);
-
-				pack = new ServerPacket(ServerOP_ReloadWorld, sizeof(ReloadWorld_Struct));
-				auto RW = (ReloadWorld_Struct *) pack->pBuffer;
-				RW->global_repop = global_repop;
-			}
-			else {
-				pack = new ServerPacket(c.opcode, 0);
-				message(r, fmt::format("Reloading [{}] globally", c.desc));
-
-				if (c.opcode == ServerOP_ReloadLogs) {
-					LogSys.LoadLogDatabaseSettings();
-					QSLink.SendPacket(pack);
-					UCSLink.SendPacket(pack);
-				}
-				else if (c.opcode == ServerOP_ReloadRules) {
-					RuleManager::Instance()->LoadRules(&database, RuleManager::Instance()->GetActiveRuleset(), true);
-				}
-			}
-
-			found_command = true;
+	for (auto &t: ServerReload::GetTypes()) {
+		if (std::to_string(t) == command || Strings::ToLower(ServerReload::GetName(t)) == command) {
+			message(r, fmt::format("Reloading [{}] globally", ServerReload::GetName(t)));
+			zoneserver_list.SendServerReload(t, nullptr);
 		}
+		found_command = true;
 	}
 
 	if (!found_command) {
 		message(r, fmt::format("Need to provide a type to reload. Example(s) [{}]", Strings::Implode("|", commands)));
 		return;
 	}
-
-	if (pack) {
-		zoneserver_list.SendPacket(pack);
-	}
-
-	safe_delete(pack);
 }
 
 void EQEmuApiWorldDataService::message(Json::Value &r, const std::string &message)

world/world_boot.h

@@ -4,6 +4,7 @@
 #include <string>
 #include "../common/types.h"
 #include "../common/discord/discord.h"
+#include "ucs.h"
 
 extern UCSConnection UCSLink;
 

world/world_event_scheduler.cpp

@@ -2,6 +2,7 @@
 #include "../common/servertalk.h"
 #include <ctime>
 #include "../common/rulesys.h"
+#include "../common/server_reload_types.h"
 
 void WorldEventScheduler::Process(ZSList *zs_list)
 {
@@ -55,11 +56,7 @@ void WorldEventScheduler::Process(ZSList *zs_list)
 				if (e.event_type == ServerEvents::EVENT_TYPE_RELOAD_WORLD) {
 					LogScheduler("Sending reload world event [{}]", e.event_data.c_str());
 
-					auto pack          = new ServerPacket(ServerOP_ReloadWorld, sizeof(ReloadWorld_Struct));
-					auto *reload_world = (ReloadWorld_Struct *) pack->pBuffer;
-					reload_world->global_repop = ReloadWorld::Repop;
-					zs_list->SendPacket(pack);
-					safe_delete(pack);
+					zs_list->SendServerReload(ServerReload::Type::WorldRepop, nullptr);
 				}
 			}
 		}

world/zonelist.cpp

@@ -28,10 +28,19 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 #include "../common/event_sub.h"
 #include "web_interface.h"
 #include "../common/zone_store.h"
+#include "../common/events/player_event_logs.h"
+#include "../common/patches/patches.h"
+#include "../common/skill_caps.h"
+#include "../common/content/world_content_service.h"
+#include "world_boot.h"
+#include "shared_task_manager.h"
+#include "dynamic_zone_manager.h"
+#include "ucs.h"
 
 extern uint32 numzones;
 extern EQ::Random emu_random;
 extern WebInterfaceList web_interface;
+extern SharedTaskManager shared_task_manager;
 volatile bool UCSServerAvailable_ = false;
 void CatchSignal(int sig_num);
 
@@ -861,3 +870,76 @@ bool ZSList::SendPacketToBootedZones(ServerPacket* pack)
 
 	return true;
 }
+
+void ZSList::SendServerReload(ServerReload::Type type, uchar *packet)
+{
+	static auto pack = ServerPacket(ServerOP_ServerReloadRequest, sizeof(ServerReload::Request));
+	auto        r    = (ServerReload::Request *) pack.pBuffer;
+
+	// Copy the packet data if it exists
+	if (packet) {
+		memcpy(pack.pBuffer, packet, sizeof(ServerReload::Request));
+	}
+
+	r->type                 = type;
+	r->requires_zone_booted = true;
+
+	LogInfo("Sending reload to all zones for type [{}]", ServerReload::GetName(type));
+
+	static const std::unordered_set<ServerReload::Type> no_zone_boot_required = {
+		ServerReload::Type::Opcodes,
+		ServerReload::Type::Rules,
+		ServerReload::Type::ContentFlags,
+		ServerReload::Type::Logs,
+		ServerReload::Type::Commands,
+		ServerReload::Type::PerlExportSettings,
+		ServerReload::Type::DataBucketsCache,
+		ServerReload::Type::WorldRepop
+	};
+
+	// Set requires_zone_booted flag before executing reload logic
+	if (no_zone_boot_required.contains(type)) {
+		r->requires_zone_booted = false;
+	}
+
+	// reload at the world level
+	if (type == ServerReload::Type::Opcodes) {
+		ReloadAllPatches();
+	} else if (type == ServerReload::Type::Rules) {
+		RuleManager::Instance()->LoadRules(&database, RuleManager::Instance()->GetActiveRuleset(), true);
+	} else if (type == ServerReload::Type::SkillCaps) {
+		skill_caps.ReloadSkillCaps();
+	} else if (type == ServerReload::Type::ContentFlags) {
+		content_service.SetExpansionContext()->ReloadContentFlags();
+	} else if (type == ServerReload::Type::Logs) {
+		LogSys.LoadLogDatabaseSettings();
+		player_event_logs.ReloadSettings();
+		UCSLink.SendPacket(&pack);
+	} else if (type == ServerReload::Type::Tasks) {
+		shared_task_manager.LoadTaskData();
+	} else if (type == ServerReload::Type::DzTemplates) {
+		dynamic_zone_manager.LoadTemplates();
+	}
+
+	// Send the packet to all zones with staggered delays
+	// to prevent all zones from reloading at the same time
+	// and causing a massive spike in CPU usage
+	// This is especially important for large servers
+	// with many zones
+	// we reload 10 zones every second
+	int counter = 0;
+
+	for (auto &z: zone_server_list) {
+		bool is_local = r->zone_server_id != 0;
+
+		// if the zone reload is local to a specific zone
+		if (r->zone_server_id != 0 && r->zone_server_id != z->GetID()) {
+			continue;
+		}
+
+		// if the reload is local, we don't need to stagger the reloads
+		r->reload_at_unix = is_local ? 0 : (std::time(nullptr) + 1) + (counter / 10);
+		z->SendPacket(&pack);
+		++counter;
+	}
+}

world/zonelist.h

@@ -5,6 +5,7 @@
 #include "../common/eqtime.h"
 #include "../common/timer.h"
 #include "../common/event/timer.h"
+#include "../common/server_reload_types.h"
 #include <vector>
 #include <memory>
 #include <deque>
@@ -69,6 +70,7 @@ public:
 	ZoneServer* FindByZoneID(uint32 ZoneID);
 
 	const std::list<std::unique_ptr<ZoneServer>> &getZoneServerList() const;
+	void SendServerReload(ServerReload::Type type, uchar *packet = nullptr);
 
 private:
 	void OnTick(EQ::Timer *t);

world/zoneserver.cpp

@@ -49,6 +49,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 #include "../zone/data_bucket.h"
 #include "../common/repositories/guild_tributes_repository.h"
 #include "../common/skill_caps.h"
+#include "../common/server_reload_types.h"
 
 extern ClientList client_list;
 extern GroupLFPList LFPGroupList;
@@ -1356,11 +1357,6 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 			QSLink.SendPacket(pack);
 			break;
 		}
-		case ServerOP_ReloadOpcodes: {
-			ReloadAllPatches();
-			zoneserver_list.SendPacket(pack);
-			break;
-		}
 		case ServerOP_CZDialogueWindow:
 		case ServerOP_CZLDoNUpdate:
 		case ServerOP_CZMarquee:
@@ -1384,30 +1380,6 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 		case ServerOP_RaidGroupSay:
 		case ServerOP_RaidSay:
 		case ServerOP_RefreshCensorship:
-		case ServerOP_ReloadAAData:
-		case ServerOP_ReloadAlternateCurrencies:
-		case ServerOP_ReloadBaseData:
-		case ServerOP_ReloadBlockedSpells:
-		case ServerOP_ReloadCommands:
-		case ServerOP_ReloadDoors:
-		case ServerOP_ReloadDataBucketsCache:
-		case ServerOP_ReloadFactions:
-		case ServerOP_ReloadGroundSpawns:
-		case ServerOP_ReloadLevelEXPMods:
-		case ServerOP_ReloadMerchants:
-		case ServerOP_ReloadNPCEmotes:
-		case ServerOP_ReloadNPCSpells:
-		case ServerOP_ReloadObjects:
-		case ServerOP_ReloadPerlExportSettings:
-		case ServerOP_ReloadStaticZoneData:
-		case ServerOP_ReloadTitles:
-		case ServerOP_ReloadTraps:
-		case ServerOP_ReloadVariables:
-		case ServerOP_ReloadVeteranRewards:
-		case ServerOP_ReloadWorld:
-		case ServerOP_ReloadZonePoints:
-		case ServerOP_ReloadZoneData:
-		case ServerOP_ReloadLoot:
 		case ServerOP_RezzPlayerAccept:
 		case ServerOP_SpawnStatusChange:
 		case ServerOP_TraderMessaging:
@@ -1425,14 +1397,9 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 			zoneserver_list.SendPacket(pack);
 			break;
 		}
-		case ServerOP_ReloadSkillCaps: {
-			zoneserver_list.SendPacket(pack);
-			skill_caps.ReloadSkillCaps();
-			break;
-		}
-		case ServerOP_ReloadRules: {
-			zoneserver_list.SendPacket(pack);
-			RuleManager::Instance()->LoadRules(&database, "default", true);
+		case ServerOP_ServerReloadRequest: {
+			auto o = (ServerReload::Request*) pack->pBuffer;
+			zoneserver_list.SendServerReload((ServerReload::Type) o->type, pack->pBuffer);
 			break;
 		}
 		case ServerOP_IsOwnerOnline: {
@@ -1460,29 +1427,6 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 			}
 			break;
 		}
-		case ServerOP_ReloadContentFlags: {
-			zoneserver_list.SendPacket(pack);
-			content_service.SetExpansionContext()->ReloadContentFlags();
-			break;
-		}
-		case ServerOP_ReloadLogs: {
-			zoneserver_list.SendPacket(pack);
-			QSLink.SendPacket(pack);
-			UCSLink.SendPacket(pack);
-			LogSys.LoadLogDatabaseSettings();
-			player_event_logs.ReloadSettings();
-			break;
-		}
-		case ServerOP_ReloadTasks: {
-			shared_task_manager.LoadTaskData();
-			zoneserver_list.SendPacket(pack);
-			break;
-		}
-		case ServerOP_ReloadDzTemplates: {
-			dynamic_zone_manager.LoadTemplates();
-			zoneserver_list.SendPacket(pack);
-			break;
-		}
 		case ServerOP_ChangeSharedMem: {
 			auto hotfix_name = std::string((char*) pack->pBuffer);