[Release] 22.62.2 (#4633)

* [Performance] Significantly Improve Client Network Resends

* Update daybreak_connection.cpp

* Improve resend algorithm to be exact about when to resend

CHANGELOG.md

@@ -1,3 +1,19 @@
+## [22.62.2] 2/1/2025
+
+### Fixes
+
+* Add price change check to the Bazaar Search Window purchase mechanics ([#4632](https://github.com/EQEmu/Server/pull/4632)) @neckkola 2025-02-01
+* NewBazaar Search Consumables ([#4631](https://github.com/EQEmu/Server/pull/4631)) @neckkola 2025-02-01
+* Update the shard bazaar search feature ([#4630](https://github.com/EQEmu/Server/pull/4630)) @neckkola 2025-02-01
+
+### Memory Leak
+
+* Revert " Change raw pointer to unique_ptr to avoid potential leak in dbg stream " ([#4616](https://github.com/EQEmu/Server/pull/4616)) @Akkadius 2025-01-27
+
+### Performance
+
+* Significantly Improve Client Network Resends ([#4629](https://github.com/EQEmu/Server/pull/4629)) @Akkadius 2025-02-01
+
 ## [22.62.1] 1/27/2025
 
 ### Memory Leak

common/bazaar.cpp

@@ -300,7 +300,7 @@ Bazaar::GetSearchResults(
 		r.serial_number           = t.trader.item_sn;
 		r.cost                    = t.trader.item_cost;
 		r.slot_id                 = t.trader.slot_id;
-		r.sum_charges             = t.trader.item_charges;
+		r.charges                 = t.trader.item_charges;
 		r.stackable               = item_results.at(t.trader.item_id).stackable;
 		r.icon_id                 = item_results.at(t.trader.item_id).icon;
 		r.trader_zone_id          = t.trader.char_zone_id;
@@ -312,9 +312,12 @@ Bazaar::GetSearchResults(
 		r.item_stat               = item_results.at(t.trader.item_id).stats;
 
 		if (RuleB(Bazaar, UseAlternateBazaarSearch)) {
-			if (convert || (r.trader_zone_id == Zones::BAZAAR && r.trader_zone_instance_id != char_zone_instance_id)) {
+			if (convert ||
+				char_zone_id != Zones::BAZAAR ||
+				(char_zone_id == Zones::BAZAAR && r.trader_zone_instance_id != char_zone_instance_id)
+				) {
 				r.trader_id = TraderRepository::TRADER_CONVERT_ID + r.trader_zone_instance_id;
-			}
+				}
 		}
 
 		all_entries.push_back(r);

common/database/database_update_manifest.cpp

@@ -6378,6 +6378,19 @@ CREATE INDEX idx_trader_active_transaction ON trader (active_transaction);
 )",
 		.content_schema_update = false
 	},
+	ManifestEntry{
+		.version = 9296,
+		.description = "2025_02_01_trader_table_listing_date.sql",
+		.check = "SHOW CREATE TABLE `trader`",
+		.condition = "missing",
+		.match = "listing_date",
+		.sql = R"(
+ALTER TABLE `trader`
+	ADD COLUMN `listing_date` DATETIME NULL DEFAULT NULL AFTER `active_transaction`,
+	ADD INDEX `idx_trader_listing_date` (`listing_date`);
+)",
+		.content_schema_update = false
+	}
 
 // -- template; copy/paste this when you need to create a new entry
 //	ManifestEntry{

common/net/daybreak_connection.cpp

@@ -1091,70 +1091,109 @@ void EQ::Net::DaybreakConnection::ProcessResend()
 	}
 }
 
+// observed client receive window is 300 packets, 140KB
+constexpr size_t MAX_CLIENT_RECV_PACKETS_PER_WINDOW = 300;
+constexpr size_t MAX_CLIENT_RECV_BYTES_PER_WINDOW   = 140 * 1024;
+
 void EQ::Net::DaybreakConnection::ProcessResend(int stream)
 {
 	if (m_status == DbProtocolStatus::StatusDisconnected) {
 		return;
 	}
 
-	auto resends = 0;
-	auto now = Clock::now();
-	auto s = &m_streams[stream];
-	for (auto &entry : s->sent_packets) {
-		auto time_since_last_send = std::chrono::duration_cast<std::chrono::milliseconds>(now - entry.second.last_sent);
-		if (entry.second.times_resent == 0) {
-			if ((size_t)time_since_last_send.count() > entry.second.resend_delay) {
-				auto &p = entry.second.packet;
-				if (p.Length() >= DaybreakHeader::size()) {
-					if (p.GetInt8(0) == 0 && p.GetInt8(1) >= OP_Fragment && p.GetInt8(1) <= OP_Fragment4) {
-						m_stats.resent_fragments++;
-					}
-					else {
-						m_stats.resent_full++;
-					}
-				}
-				else {
-					m_stats.resent_full++;
-				}
-				m_stats.resent_packets++;
+	if (m_streams[stream].sent_packets.empty()) {
+		return;
+	}
 
-				InternalBufferedSend(p);
-				entry.second.last_sent = now;
-				entry.second.times_resent++;
-				entry.second.resend_delay = EQ::Clamp(entry.second.resend_delay * 2, m_owner->m_options.resend_delay_min, m_owner->m_options.resend_delay_max);
-				resends++;
+	m_resend_packets_sent = 0;
+	m_resend_bytes_sent = 0;
+
+	auto now = Clock::now(); // Current time
+	auto s = &m_streams[stream];
+
+	// Get a reference resend delay (assume first packet represents the typical case)
+	if (!s->sent_packets.empty()) {
+		// Check if the first packet has timed out
+		auto &first_packet = s->sent_packets.begin()->second;
+		auto time_since_first_sent = std::chrono::duration_cast<std::chrono::milliseconds>(now - first_packet.first_sent).count();
+
+		// make sure that the first_packet in the list first_sent time is within the resend_delay and now
+		// if it is not, then we need to resend all packets in the list
+		if (time_since_first_sent <= first_packet.resend_delay && !m_acked_since_last_resend) {
+			LogNetcodeDetail(
+				"Not resending packets for stream [{}] time since first sent [{}] resend delay [{}] m_acked_since_last_resend [{}]",
+				stream,
+				time_since_first_sent,
+				first_packet.resend_delay,
+				m_acked_since_last_resend
+			);
+			return;
+		}
+
+		if (time_since_first_sent >= m_owner->m_options.resend_timeout) {
+			Close();
+			return;
+		}
+	}
+
+	if (LogSys.IsLogEnabled(Logs::Detail, Logs::Netcode)) {
+		size_t    total_size = 0;
+		for (auto &e: s->sent_packets) {
+			total_size += e.second.packet.Length();
+		}
+
+		LogNetcodeDetail(
+			"Resending packets for stream [{}] packet count [{}] total packet size [{}] m_acked_since_last_resend [{}]",
+			stream,
+			s->sent_packets.size(),
+			total_size,
+			m_acked_since_last_resend
+		);
+	}
+
+	for (auto &e: s->sent_packets) {
+		if (m_resend_packets_sent >= MAX_CLIENT_RECV_PACKETS_PER_WINDOW ||
+			m_resend_bytes_sent >= MAX_CLIENT_RECV_BYTES_PER_WINDOW) {
+			LogNetcodeDetail(
+				"Stopping resend because we hit thresholds m_resend_packets_sent [{}] max [{}] m_resend_bytes_sent [{}] max [{}]",
+				m_resend_packets_sent,
+				MAX_CLIENT_RECV_PACKETS_PER_WINDOW,
+				m_resend_bytes_sent,
+				MAX_CLIENT_RECV_BYTES_PER_WINDOW
+			);
+			break;
+		}
+
+		auto &sp = e.second;
+		auto &p  = sp.packet;
+		if (p.Length() >= DaybreakHeader::size()) {
+			if (p.GetInt8(0) == 0 && p.GetInt8(1) >= OP_Fragment && p.GetInt8(1) <= OP_Fragment4) {
+				m_stats.resent_fragments++;
+			}
+			else {
+				m_stats.resent_full++;
 			}
 		}
 		else {
-			auto time_since_first_sent = std::chrono::duration_cast<std::chrono::milliseconds>(now - entry.second.first_sent);
-			if (time_since_first_sent.count() >= m_owner->m_options.resend_timeout) {
-				Close();
-				return;
-			}
-
-			if ((size_t)time_since_last_send.count() > entry.second.resend_delay) {
-				auto &p = entry.second.packet;
-				if (p.Length() >= DaybreakHeader::size()) {
-					if (p.GetInt8(0) == 0 && p.GetInt8(1) >= OP_Fragment && p.GetInt8(1) <= OP_Fragment4) {
-						m_stats.resent_fragments++;
-					}
-					else {
-						m_stats.resent_full++;
-					}
-				}
-				else {
-					m_stats.resent_full++;
-				}
-				m_stats.resent_packets++;
-
-				InternalBufferedSend(p);
-				entry.second.last_sent = now;
-				entry.second.times_resent++;
-				entry.second.resend_delay = EQ::Clamp(entry.second.resend_delay * 2, m_owner->m_options.resend_delay_min, m_owner->m_options.resend_delay_max);
-				resends++;
-			}
+			m_stats.resent_full++;
 		}
+		m_stats.resent_packets++;
+
+		// Resend the packet
+		InternalBufferedSend(p);
+
+		m_resend_packets_sent++;
+		m_resend_bytes_sent += p.Length();
+		sp.last_sent = now;
+		sp.times_resent++;
+		sp.resend_delay = EQ::Clamp(
+			sp.resend_delay * 2,
+			m_owner->m_options.resend_delay_min,
+			m_owner->m_options.resend_delay_max
+		);
 	}
+
+	m_acked_since_last_resend = false;
 }
 
 void EQ::Net::DaybreakConnection::Ack(int stream, uint16_t seq)
@@ -1175,6 +1214,7 @@ void EQ::Net::DaybreakConnection::Ack(int stream, uint16_t seq)
 			m_rolling_ping = (m_rolling_ping * 2 + round_time) / 3;
 
 			iter = s->sent_packets.erase(iter);
+			m_acked_since_last_resend = true;
 		}
 		else {
 			++iter;

common/net/daybreak_connection.h

@@ -181,6 +181,11 @@ namespace EQ
 			Timestamp m_close_time;
 			double m_outgoing_budget;
 
+			// resend tracking
+			size_t m_resend_packets_sent = 0;
+			size_t m_resend_bytes_sent = 0;
+			bool m_acked_since_last_resend = false;
+
 			struct DaybreakSentPacket
 			{
 				DynamicPacket packet;

common/repositories/base/base_trader_repository.h

@@ -36,6 +36,7 @@ public:
 		uint32_t char_zone_id;
 		int32_t  char_zone_instance_id;
 		uint8_t  active_transaction;
+		time_t   listing_date;
 	};
 
 	static std::string PrimaryKey()
@@ -63,6 +64,7 @@ public:
 			"char_zone_id",
 			"char_zone_instance_id",
 			"active_transaction",
+			"listing_date",
 		};
 	}
 
@@ -86,6 +88,7 @@ public:
 			"char_zone_id",
 			"char_zone_instance_id",
 			"active_transaction",
+			"UNIX_TIMESTAMP(listing_date)",
 		};
 	}
 
@@ -143,6 +146,7 @@ public:
 		e.char_zone_id          = 0;
 		e.char_zone_instance_id = 0;
 		e.active_transaction    = 0;
+		e.listing_date          = 0;
 
 		return e;
 	}
@@ -196,6 +200,7 @@ public:
 			e.char_zone_id          = row[14] ? static_cast<uint32_t>(strtoul(row[14], nullptr, 10)) : 0;
 			e.char_zone_instance_id = row[15] ? static_cast<int32_t>(atoi(row[15])) : 0;
 			e.active_transaction    = row[16] ? static_cast<uint8_t>(strtoul(row[16], nullptr, 10)) : 0;
+			e.listing_date          = strtoll(row[17] ? row[17] : "-1", nullptr, 10);
 
 			return e;
 		}
@@ -245,6 +250,7 @@ public:
 		v.push_back(columns[14] + " = " + std::to_string(e.char_zone_id));
 		v.push_back(columns[15] + " = " + std::to_string(e.char_zone_instance_id));
 		v.push_back(columns[16] + " = " + std::to_string(e.active_transaction));
+		v.push_back(columns[17] + " = FROM_UNIXTIME(" + (e.listing_date > 0 ? std::to_string(e.listing_date) : "null") + ")");
 
 		auto results = db.QueryDatabase(
 			fmt::format(
@@ -283,6 +289,7 @@ public:
 		v.push_back(std::to_string(e.char_zone_id));
 		v.push_back(std::to_string(e.char_zone_instance_id));
 		v.push_back(std::to_string(e.active_transaction));
+		v.push_back("FROM_UNIXTIME(" + (e.listing_date > 0 ? std::to_string(e.listing_date) : "null") + ")");
 
 		auto results = db.QueryDatabase(
 			fmt::format(
@@ -329,6 +336,7 @@ public:
 			v.push_back(std::to_string(e.char_zone_id));
 			v.push_back(std::to_string(e.char_zone_instance_id));
 			v.push_back(std::to_string(e.active_transaction));
+			v.push_back("FROM_UNIXTIME(" + (e.listing_date > 0 ? std::to_string(e.listing_date) : "null") + ")");
 
 			insert_chunks.push_back("(" + Strings::Implode(",", v) + ")");
 		}
@@ -379,6 +387,7 @@ public:
 			e.char_zone_id          = row[14] ? static_cast<uint32_t>(strtoul(row[14], nullptr, 10)) : 0;
 			e.char_zone_instance_id = row[15] ? static_cast<int32_t>(atoi(row[15])) : 0;
 			e.active_transaction    = row[16] ? static_cast<uint8_t>(strtoul(row[16], nullptr, 10)) : 0;
+			e.listing_date          = strtoll(row[17] ? row[17] : "-1", nullptr, 10);
 
 			all_entries.push_back(e);
 		}
@@ -420,6 +429,7 @@ public:
 			e.char_zone_id          = row[14] ? static_cast<uint32_t>(strtoul(row[14], nullptr, 10)) : 0;
 			e.char_zone_instance_id = row[15] ? static_cast<int32_t>(atoi(row[15])) : 0;
 			e.active_transaction    = row[16] ? static_cast<uint8_t>(strtoul(row[16], nullptr, 10)) : 0;
+			e.listing_date          = strtoll(row[17] ? row[17] : "-1", nullptr, 10);
 
 			all_entries.push_back(e);
 		}
@@ -511,6 +521,7 @@ public:
 		v.push_back(std::to_string(e.char_zone_id));
 		v.push_back(std::to_string(e.char_zone_instance_id));
 		v.push_back(std::to_string(e.active_transaction));
+		v.push_back("FROM_UNIXTIME(" + (e.listing_date > 0 ? std::to_string(e.listing_date) : "null") + ")");
 
 		auto results = db.QueryDatabase(
 			fmt::format(
@@ -550,6 +561,7 @@ public:
 			v.push_back(std::to_string(e.char_zone_id));
 			v.push_back(std::to_string(e.char_zone_instance_id));
 			v.push_back(std::to_string(e.active_transaction));
+			v.push_back("FROM_UNIXTIME(" + (e.listing_date > 0 ? std::to_string(e.listing_date) : "null") + ")");
 
 			insert_chunks.push_back("(" + Strings::Implode(",", v) + ")");
 		}

common/repositories/trader_repository.h

@@ -130,7 +130,8 @@ public:
 		}
 
 		for (auto &i: items) {
-			i.item_cost = new_price;
+			i.item_cost    = new_price;
+			i.listing_date = time(nullptr);
 		}
 
 		return ReplaceMany(db, items);
@@ -178,6 +179,7 @@ public:
 
 		auto m = trader_item[0];
 		m.item_charges = quantity;
+		m.listing_date = time(nullptr);
 
 		return UpdateOne(db, m);
 	}
@@ -221,6 +223,7 @@ public:
 		}
 
 		e.active_transaction = status == true ? 1 : 0;
+		e.listing_date       = time(nullptr);
 
 		return UpdateOne(db, e);
 	}

common/version.h

@@ -25,7 +25,7 @@
 
 // Build variables
 // these get injected during the build pipeline
-#define CURRENT_VERSION "22.62.1-dev" // always append -dev to the current version for custom-builds
+#define CURRENT_VERSION "22.62.2-dev" // always append -dev to the current version for custom-builds
 #define LOGIN_VERSION "0.8.0"
 #define COMPILE_DATE    __DATE__
 #define COMPILE_TIME    __TIME__
@@ -42,7 +42,7 @@
  * Manifest: https://github.com/EQEmu/Server/blob/master/utils/sql/db_update_manifest.txt
  */
 
-#define CURRENT_BINARY_DATABASE_VERSION 9295
+#define CURRENT_BINARY_DATABASE_VERSION 9296
 #define CURRENT_BINARY_BOTS_DATABASE_VERSION 9045
 
 #endif

package.json

@@ -1,6 +1,6 @@
 {
   "name": "eqemu-server",
-  "version": "22.62.1",
+  "version": "22.62.2",
   "repository": {
     "type": "git",
     "url": "https://github.com/EQEmu/Server.git"

zone/client.h

@@ -1139,6 +1139,8 @@ public:
 	uint32 GetTraderCount() { return m_trader_count; }
 	void IncrementTraderCount() { m_trader_count += 1; }
 	void DecrementTraderCount() { m_trader_count > 0 ? m_trader_count -= 1 : m_trader_count = 0; }
+	void SetTraderTransactionDate() { m_trader_transaction_date = time(nullptr); }
+	time_t GetTraderTransactionDate() { return m_trader_transaction_date; }
 
 	eqFilterMode GetFilter(eqFilterType filter_id) const { return ClientFilters[filter_id]; }
 	void SetFilter(eqFilterType filter_id, eqFilterMode filter_mode) { ClientFilters[filter_id] = filter_mode; }
@@ -1976,6 +1978,7 @@ private:
 	uint8 firstlogon;
 	uint32 mercid; // current merc
 	uint8 mercSlot; // selected merc slot
+	time_t                                                         m_trader_transaction_date;
 	uint32                                                         m_trader_count{};
 	uint32                                                         m_buyer_id;
 	uint32                                                         m_barter_time;

zone/trading.cpp

@@ -1106,6 +1106,7 @@ void Client::TraderStartTrader(const EQApplicationPacket *app)
 			trader_item.item_id               = inst->GetID();
 			trader_item.item_sn               = in->serial_number[i];
 			trader_item.slot_id               = i;
+			trader_item.listing_date          = time(nullptr);
 			if (inst->IsAugmented()) {
 				auto augs              = inst->GetAugmentIDs();
 				trader_item.aug_slot_1 = augs.at(0);
@@ -1812,6 +1813,7 @@ void Client::DoBazaarSearch(BazaarSearchCriteria_Struct search_criteria)
 		return;
 	}
 
+	SetTraderTransactionDate();
 	std::stringstream           ss{};
 	cereal::BinaryOutputArchive ar(ss);
 	ar(results);
@@ -2798,6 +2800,7 @@ void Client::TraderPriceUpdate(const EQApplicationPacket *app)
 				trader_item.item_cost             = tpus->NewPrice;
 				trader_item.item_id               = newgis->items[i];
 				trader_item.item_sn               = newgis->serial_number[i];
+				trader_item.listing_date          = time(nullptr);
 				if (item_detail->IsAugmented()) {
 					auto augs              = item_detail->GetAugmentIDs();
 					trader_item.aug_slot_1 = augs.at(0);
@@ -3072,7 +3075,7 @@ void Client::BuyTraderItemOutsideBazaar(TraderBuy_Struct *tbs, const EQApplicati
 {
 	auto in          = (TraderBuy_Struct *) app->pBuffer;
 	auto trader_item = TraderRepository::GetItemBySerialNumber(database, tbs->serial_number, tbs->trader_id);
-	if (!trader_item.id) {
+	if (!trader_item.id || GetTraderTransactionDate() < trader_item.listing_date) {
 		LogTrading("Attempt to purchase an item outside of the Bazaar trader_id <red>[{}] item serial_number "
 				   "<red>[{}] The Traders data was outdated.",
 				   tbs->trader_id,