Implement world cache to monitor expeditions

This implements a small cache in world to track expedition states.

This fixes expired expeditions being left in zone caches unless the
expedition's dz instance was running to detect it (or unless an
expedition was deleted via a client using /kickplayers). This was also
leaving clients in a ghost expedition that no longer actually existed

common/ruletypes.h

@@ -791,6 +791,7 @@ RULE_BOOL(Expedition, UseDatabaseToVerifyLeaderCommands, false, "Use database in
 RULE_BOOL(Expedition, EmptyDzShutdownEnabled, true, "Enable early instance shutdown after last member of expedition removed")
 RULE_INT(Expedition, EmptyDzShutdownDelaySeconds, 900, "Seconds to set dynamic zone instance expiration if early shutdown enabled")
 RULE_INT(Expedition, RequestExpiredLockoutLeewaySeconds, 60, "Seconds remaining on lockout to count as expired for creation requests (client hides timers under 60s remaining)")
+RULE_INT(Expedition, WorldExpeditionProcessRateMS, 6000, "Timer interval (ms) that world checks expedition states")
 RULE_CATEGORY_END()
 
 RULE_CATEGORY(DynamicZone)

common/servertalk.h

@@ -159,6 +159,7 @@
 #define ServerOP_ExpeditionRequestInvite      0x040f
 #define ServerOP_ExpeditionReplayOnJoin       0x0410
 #define ServerOP_ExpeditionLockState          0x0411
+#define ServerOP_ExpeditionExpired            0x0412
 
 #define ServerOP_DzCharacterChange            0x0450
 #define ServerOP_DzRemoveAllCharacters        0x0451

world/expedition.cpp

@@ -27,10 +27,109 @@
 #include "../common/servertalk.h"
 #include "../common/string_util.h"
 
+ExpeditionCache expedition_cache;
+
 extern ClientList client_list;
 extern ZSList zoneserver_list;
 
-void Expedition::PurgeExpiredExpeditions()
+Expedition::Expedition(
+	uint32_t expedition_id, uint32_t instance_id, uint32_t dz_zone_id,
+	uint32_t start_time, uint32_t duration
+) :
+	m_expedition_id(expedition_id),
+	m_dz_instance_id(instance_id),
+	m_dz_zone_id(dz_zone_id),
+	m_start_time(start_time),
+	m_duration(duration)
+{
+	m_expire_time = std::chrono::system_clock::from_time_t(m_start_time + m_duration);
+}
+
+void Expedition::SendZonesExpeditionExpired()
+{
+	uint32_t pack_size = sizeof(ServerExpeditionID_Struct);
+	auto pack = std::unique_ptr<ServerPacket>(new ServerPacket(ServerOP_ExpeditionExpired, pack_size));
+	auto buf = reinterpret_cast<ServerExpeditionID_Struct*>(pack->pBuffer);
+	buf->expedition_id = GetID();
+	zoneserver_list.SendPacket(pack.get());
+}
+
+void ExpeditionCache::LoadActiveExpeditions()
+{
+	BenchTimer benchmark;
+
+	m_expeditions = ExpeditionDatabase::LoadExpeditions();
+
+	auto elapsed = benchmark.elapsed();
+	LogExpeditions("World caching [{}] expeditions took {}s", m_expeditions.size(), elapsed);
+}
+
+void ExpeditionCache::AddExpedition(uint32_t expedition_id)
+{
+	if (expedition_id == 0)
+	{
+		return;
+	}
+
+	auto expedition = ExpeditionDatabase::LoadExpedition(expedition_id);
+
+	if (expedition.GetID() == expedition_id)
+	{
+		auto it = std::find_if(m_expeditions.begin(), m_expeditions.end(), [&](const Expedition& expedition) {
+			return expedition.GetID() == expedition_id;
+		});
+
+		if (it == m_expeditions.end())
+		{
+			m_expeditions.emplace_back(expedition);
+		}
+	}
+}
+
+void ExpeditionCache::RemoveExpedition(uint32_t expedition_id)
+{
+	m_expeditions.erase(std::remove_if(m_expeditions.begin(), m_expeditions.end(),
+		[&](const Expedition& expedition) {
+			return expedition.GetID() == expedition_id;
+		}
+	), m_expeditions.end());
+}
+
+void ExpeditionCache::Process()
+{
+	if (!m_process_throttle_timer.Check())
+	{
+		return;
+	}
+
+	std::vector<uint32_t> expedition_ids;
+
+	// check for expired expeditions (using the dz instance expiration time)
+	for (auto it = m_expeditions.begin(); it != m_expeditions.end();)
+	{
+		if (!it->IsExpired())
+		{
+			++it;
+		}
+		else
+		{
+			// we need to delete expired expeditions from the database now instead
+			// of waiting for purge timer so members can request new expeditions.
+			// the dz should process this on its own to kick any clients inside it
+			LogExpeditions("Expedition [{}] expired, notifying zones and deleting", it->GetID());
+			expedition_ids.emplace_back(it->GetID());
+			it->SendZonesExpeditionExpired();
+			it = m_expeditions.erase(it);
+		}
+	}
+
+	if (!expedition_ids.empty())
+	{
+		ExpeditionDatabase::DeleteExpeditions(expedition_ids);
+	}
+}
+
+void ExpeditionDatabase::PurgeExpiredExpeditions()
 {
 	std::string query = SQL(
 		DELETE expedition FROM expedition_details expedition
@@ -54,7 +153,7 @@ void Expedition::PurgeExpiredExpeditions()
 	}
 }
 
-void Expedition::PurgeExpiredCharacterLockouts()
+void ExpeditionDatabase::PurgeExpiredCharacterLockouts()
 {
 	std::string query = SQL(
 		DELETE FROM expedition_character_lockouts
@@ -68,23 +167,139 @@ void Expedition::PurgeExpiredCharacterLockouts()
 	}
 }
 
-void Expedition::HandleZoneMessage(ServerPacket* pack)
+std::vector<Expedition> ExpeditionDatabase::LoadExpeditions()
+{
+	std::vector<Expedition> expeditions;
+
+	std::string query = SQL(
+		SELECT
+			expedition_details.id,
+			expedition_details.instance_id,
+			instance_list.zone,
+			instance_list.start_time,
+			instance_list.duration
+		FROM expedition_details
+			INNER JOIN instance_list ON expedition_details.instance_id = instance_list.id
+		ORDER BY expedition_details.id;
+	);
+
+	auto results = database.QueryDatabase(query);
+	if (!results.Success())
+	{
+		LogExpeditions("Failed to load expeditions for world cache");
+	}
+	else
+	{
+		for (auto row = results.begin(); row != results.end(); ++row)
+		{
+			expeditions.emplace_back(Expedition{
+				static_cast<uint32_t>(strtoul(row[0], nullptr, 10)), // expedition_id
+				static_cast<uint32_t>(strtoul(row[1], nullptr, 10)), // dz_instance_id
+				static_cast<uint32_t>(strtoul(row[2], nullptr, 10)), // dz_zone_id
+				static_cast<uint32_t>(strtoul(row[3], nullptr, 10)), // start_time
+				static_cast<uint32_t>(strtoul(row[4], nullptr, 10))  // duration
+			});
+		}
+	}
+
+	return expeditions;
+}
+
+Expedition ExpeditionDatabase::LoadExpedition(uint32_t expedition_id)
+{
+	std::string query = fmt::format(SQL(
+		SELECT
+			expedition_details.id,
+			expedition_details.instance_id,
+			instance_list.zone,
+			instance_list.start_time,
+			instance_list.duration
+		FROM expedition_details
+			INNER JOIN instance_list ON expedition_details.instance_id = instance_list.id
+		WHERE expedition_details.id = {};
+	), expedition_id);
+
+	auto results = database.QueryDatabase(query);
+	if (!results.Success())
+	{
+		LogExpeditions("Failed to load expedition [{}] for world cache", expedition_id);
+	}
+	else if (results.RowCount() > 0)
+	{
+		auto row = results.begin();
+		return Expedition{
+			static_cast<uint32_t>(strtoul(row[0], nullptr, 10)), // expedition_id
+			static_cast<uint32_t>(strtoul(row[1], nullptr, 10)), // dz_instance_id
+			static_cast<uint32_t>(strtoul(row[2], nullptr, 10)), // dz_zone_id
+			static_cast<uint32_t>(strtoul(row[3], nullptr, 10)), // start_time
+			static_cast<uint32_t>(strtoul(row[4], nullptr, 10))  // duration
+		};
+	}
+
+	return Expedition{};
+}
+
+void ExpeditionDatabase::DeleteExpeditions(const std::vector<uint32_t>& expedition_ids)
+{
+	std::string expedition_ids_query;
+	for (const auto& expedition_id : expedition_ids)
+	{
+		fmt::format_to(std::back_inserter(expedition_ids_query), "{},", expedition_id);
+	}
+
+	if (!expedition_ids_query.empty())
+	{
+		expedition_ids_query.pop_back(); // trailing comma
+
+		std::string query = fmt::format(
+			"DELETE FROM expedition_details WHERE id IN ({});", expedition_ids_query
+		);
+		database.QueryDatabase(query);
+
+		// todo: if not using foreign key constraints
+		//query = fmt::format(
+		//	"DELETE FROM expedition_members WHERE expedition_id IN ({});", expedition_ids_query
+		//);
+		//database.QueryDatabase(query);
+
+		//query = fmt::format(
+		//	"DELETE FROM expedition_lockouts WHERE expedition_id IN ({});", expedition_ids_query
+		//);
+		//database.QueryDatabase(query);
+	}
+}
+
+void ExpeditionMessage::HandleZoneMessage(ServerPacket* pack)
 {
 	switch (pack->opcode)
 	{
+	case ServerOP_ExpeditionCreate:
+	{
+		auto buf = reinterpret_cast<ServerExpeditionID_Struct*>(pack->pBuffer);
+		expedition_cache.AddExpedition(buf->expedition_id);
+		zoneserver_list.SendPacket(pack);
+		break;
+	}
+	case ServerOP_ExpeditionDeleted:
+	{
+		auto buf = reinterpret_cast<ServerExpeditionID_Struct*>(pack->pBuffer);
+		expedition_cache.RemoveExpedition(buf->expedition_id);
+		zoneserver_list.SendPacket(pack);
+		break;
+	}
 	case ServerOP_ExpeditionGetOnlineMembers:
 	{
-		Expedition::GetOnlineMembers(pack);
+		ExpeditionMessage::GetOnlineMembers(pack);
 		break;
 	}
 	case ServerOP_ExpeditionDzAddPlayer:
 	{
-		Expedition::AddPlayer(pack);
+		ExpeditionMessage::AddPlayer(pack);
 		break;
 	}
 	case ServerOP_ExpeditionDzMakeLeader:
 	{
-		Expedition::MakeLeader(pack);
+		ExpeditionMessage::MakeLeader(pack);
 		break;
 	}
 	case ServerOP_ExpeditionRemoveCharLockouts:
@@ -95,18 +310,18 @@ void Expedition::HandleZoneMessage(ServerPacket* pack)
 	}
 	case ServerOP_ExpeditionSaveInvite:
 	{
-		Expedition::SaveInvite(pack);
+		ExpeditionMessage::SaveInvite(pack);
 		break;
 	}
 	case ServerOP_ExpeditionRequestInvite:
 	{
-		Expedition::RequestInvite(pack);
+		ExpeditionMessage::RequestInvite(pack);
 		break;
 	}
 	}
 }
 
-void Expedition::AddPlayer(ServerPacket* pack)
+void ExpeditionMessage::AddPlayer(ServerPacket* pack)
 {
 	auto buf = reinterpret_cast<ServerDzCommand_Struct*>(pack->pBuffer);
 
@@ -128,7 +343,7 @@ void Expedition::AddPlayer(ServerPacket* pack)
 	}
 }
 
-void Expedition::MakeLeader(ServerPacket* pack)
+void ExpeditionMessage::MakeLeader(ServerPacket* pack)
 {
 	auto buf = reinterpret_cast<ServerDzCommand_Struct*>(pack->pBuffer);
 
@@ -150,7 +365,7 @@ void Expedition::MakeLeader(ServerPacket* pack)
 	}
 }
 
-void Expedition::GetOnlineMembers(ServerPacket* pack)
+void ExpeditionMessage::GetOnlineMembers(ServerPacket* pack)
 {
 	auto buf = reinterpret_cast<ServerExpeditionCharacters_Struct*>(pack->pBuffer);
 
@@ -177,7 +392,7 @@ void Expedition::GetOnlineMembers(ServerPacket* pack)
 	zoneserver_list.SendPacket(buf->sender_zone_id, buf->sender_instance_id, pack);
 }
 
-void Expedition::SaveInvite(ServerPacket* pack)
+void ExpeditionMessage::SaveInvite(ServerPacket* pack)
 {
 	auto buf = reinterpret_cast<ServerDzCommand_Struct*>(pack->pBuffer);
 
@@ -191,7 +406,7 @@ void Expedition::SaveInvite(ServerPacket* pack)
 	}
 }
 
-void Expedition::RequestInvite(ServerPacket* pack)
+void ExpeditionMessage::RequestInvite(ServerPacket* pack)
 {
 	auto buf = reinterpret_cast<ServerExpeditionCharacterID_Struct*>(pack->pBuffer);
 	ClientListEntry* cle = client_list.FindCLEByCharacterID(buf->character_id);

world/expedition.h

@@ -21,12 +21,27 @@
 #ifndef WORLD_EXPEDITION_H
 #define WORLD_EXPEDITION_H
 
+#include "../common/rulesys.h"
+#include "../common/timer.h"
+#include <chrono>
+#include <vector>
+
+extern class ExpeditionCache expedition_cache;
+
+class Expedition;
 class ServerPacket;
 
-namespace Expedition
+namespace ExpeditionDatabase
 {
 	void PurgeExpiredExpeditions();
 	void PurgeExpiredCharacterLockouts();
+	std::vector<Expedition> LoadExpeditions();
+	Expedition LoadExpedition(uint32_t expedition_id);
+	void DeleteExpeditions(const std::vector<uint32_t>& expedition_ids);
+};
+
+namespace ExpeditionMessage
+{
 	void HandleZoneMessage(ServerPacket* pack);
 	void AddPlayer(ServerPacket* pack);
 	void MakeLeader(ServerPacket* pack);
@@ -35,4 +50,38 @@ namespace Expedition
 	void RequestInvite(ServerPacket* pack);
 };
 
+class ExpeditionCache
+{
+public:
+	void AddExpedition(uint32_t expedition_id);
+	void RemoveExpedition(uint32_t expedition_id);
+	void LoadActiveExpeditions();
+	void Process();
+
+private:
+	std::vector<Expedition> m_expeditions;
+	Timer m_process_throttle_timer{static_cast<uint32_t>(RuleI(Expedition, WorldExpeditionProcessRateMS))};
+};
+
+class Expedition
+{
+public:
+	Expedition() = default;
+	Expedition(
+		uint32_t expedition_id, uint32_t instance_id, uint32_t dz_zone_id,
+		uint32_t expire_time, uint32_t duration);
+
+	uint32_t GetID() const { return m_expedition_id; }
+	bool IsExpired() const { return m_expire_time < std::chrono::system_clock::now(); }
+	void SendZonesExpeditionExpired();
+
+private:
+	uint32_t m_expedition_id  = 0;
+	uint32_t m_dz_instance_id = 0;
+	uint32_t m_dz_zone_id     = 0;
+	uint32_t m_start_time     = 0;
+	uint32_t m_duration       = 0;
+	std::chrono::time_point<std::chrono::system_clock> m_expire_time;
+};
+
 #endif

world/main.cpp

@@ -431,8 +431,11 @@ int main(int argc, char** argv) {
 	PurgeInstanceTimer.Start(450000);
 
 	LogInfo("Purging expired expeditions");
-	Expedition::PurgeExpiredExpeditions();
+	ExpeditionDatabase::PurgeExpiredExpeditions();
-	Expedition::PurgeExpiredCharacterLockouts();
+	ExpeditionDatabase::PurgeExpiredCharacterLockouts();
+
+	LogInfo("Loading active expeditions");
+	expedition_cache.LoadActiveExpeditions();
 
 	LogInfo("Loading char create info");
 	content_db.LoadCharacterCreateAllocations();
@@ -604,8 +607,8 @@ int main(int argc, char** argv) {
 		if (PurgeInstanceTimer.Check()) {
 			database.PurgeExpiredInstances();
 			database.PurgeAllDeletedDataBuckets();
-			Expedition::PurgeExpiredExpeditions();
+			ExpeditionDatabase::PurgeExpiredExpeditions();
-			Expedition::PurgeExpiredCharacterLockouts();
+			ExpeditionDatabase::PurgeExpiredCharacterLockouts();
 		}
 
 		if (EQTimeTimer.Check()) {
@@ -621,6 +624,7 @@ int main(int argc, char** argv) {
 		launcher_list.Process();
 		LFPGroupList.Process();
 		adventure_manager.Process();
+		expedition_cache.Process();
 
 		if (InterserverTimer.Check()) {
 			InterserverTimer.Start();

world/zoneserver.cpp

@@ -1368,8 +1368,6 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 		client_list.SendPacket(buf->character_name, pack);
 		break;
 	}
-	case ServerOP_ExpeditionCreate:
-	case ServerOP_ExpeditionDeleted:
 	case ServerOP_ExpeditionLeaderChanged:
 	case ServerOP_ExpeditionLockout:
 	case ServerOP_ExpeditionLockState:
@@ -1384,6 +1382,8 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 		zoneserver_list.SendPacket(pack);
 		break;
 	}
+	case ServerOP_ExpeditionCreate:
+	case ServerOP_ExpeditionDeleted:
 	case ServerOP_ExpeditionGetOnlineMembers:
 	case ServerOP_ExpeditionDzAddPlayer:
 	case ServerOP_ExpeditionDzMakeLeader:
@@ -1391,7 +1391,7 @@ void ZoneServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p) {
 	case ServerOP_ExpeditionSaveInvite:
 	case ServerOP_ExpeditionRequestInvite:
 	{
-		Expedition::HandleZoneMessage(pack);
+		ExpeditionMessage::HandleZoneMessage(pack);
 		break;
 	}
 	case ServerOP_DzCharacterChange:

zone/expedition.cpp

@@ -1272,7 +1272,7 @@ void Expedition::ProcessLockoutUpdate(
 	}
 }
 
-void Expedition::SendUpdatesToZoneMembers(bool clear)
+void Expedition::SendUpdatesToZoneMembers(bool clear, bool message_on_clear)
 {
 	if (!m_members.empty())
 	{
@@ -1289,7 +1289,7 @@ void Expedition::SendUpdatesToZoneMembers(bool clear)
 				member_client->QueuePacket(outapp_info.get());
 				member_client->QueuePacket(outapp_members.get());
 				member_client->SendExpeditionLockoutTimers();
-				if (clear)
+				if (clear && message_on_clear)
 				{
 					member_client->MessageString(
 						Chat::Yellow, EXPEDITION_REMOVED, member_client->GetName(), m_expedition_name.c_str()
@@ -1605,6 +1605,7 @@ void Expedition::HandleWorldMessage(ServerPacket* pack)
 		break;
 	}
 	case ServerOP_ExpeditionDeleted:
+	case ServerOP_ExpeditionExpired:
 	{
 		auto buf = reinterpret_cast<ServerExpeditionID_Struct*>(pack->pBuffer);
 		auto expedition = Expedition::FindCachedExpeditionByID(buf->expedition_id);
@@ -1612,7 +1613,9 @@ void Expedition::HandleWorldMessage(ServerPacket* pack)
 		{
 			if (!zone->IsZone(buf->sender_zone_id, buf->sender_instance_id))
 			{
-				expedition->SendUpdatesToZoneMembers(true);
+				// expired deletions should be silent
+				bool notify_members = (pack->opcode == ServerOP_ExpeditionDeleted);
+				expedition->SendUpdatesToZoneMembers(true, notify_members);
 			}
 
 			// remove even from sender zone

zone/expedition.h

@@ -154,7 +154,7 @@ private:
 	void SaveMembers(ExpeditionRequest& request);
 	void SendClientExpeditionInvite(Client* client, const std::string& inviter_name, const std::string& swap_remove_name);
 	void SendLeaderMessage(Client* leader_client, uint16_t chat_type, uint32_t string_id, const std::initializer_list<std::string>& parameters = {});
-	void SendUpdatesToZoneMembers(bool clear = false);
+	void SendUpdatesToZoneMembers(bool clear = false, bool message_on_clear = true);
 	void SendWorldDzLocationUpdate(uint16_t server_opcode, const DynamicZoneLocation& location);
 	void SendWorldExpeditionUpdate(bool destroyed = false);
 	void SendWorldGetOnlineMembers();

zone/worldserver.cpp

@@ -2901,6 +2901,7 @@ void WorldServer::HandleMessage(uint16 opcode, const EQ::Net::Packet &p)
 	}
 	case ServerOP_ExpeditionCreate:
 	case ServerOP_ExpeditionDeleted:
+	case ServerOP_ExpeditionExpired:
 	case ServerOP_ExpeditionLeaderChanged:
 	case ServerOP_ExpeditionLockout:
 	case ServerOP_ExpeditionLockState: