Updated the database scripts and re-synced the database bindings (#5077)

common/database/database_update.cpp

@@ -28,6 +28,7 @@
 #include "common/strings.h"
 
 #include <filesystem>
+#include <ranges>
 
 
 constexpr int BREAK_LENGTH = 70;
@@ -73,35 +74,38 @@ void DatabaseUpdate::CheckDbUpdates()
 		return;
 	}
 
-	if (UpdateManifest(manifest_entries, v.server_database_version, b.server_database_version)) {
+	int server_database_version = UpdateManifest(manifest_entries, v.server_database_version, b.server_database_version);
+	if (server_database_version > v.server_database_version) {
 		LogInfo(
 			"Updates ran successfully, setting database version to [{}] from [{}]",
-			b.server_database_version,
+			server_database_version,
 			v.server_database_version
 		);
-		m_database->QueryDatabase(fmt::format("UPDATE `db_version` SET `version` = {}", b.server_database_version));
+		m_database->QueryDatabase(fmt::format("UPDATE `db_version` SET `version` = {}", server_database_version));
 	}
 
-	if (UpdateManifest(manifest_entries_custom, v.custom_database_version, b.custom_database_version)) {
+	int custom_database_version = UpdateManifest(manifest_entries_custom, v.custom_database_version, b.custom_database_version);
+	if (custom_database_version > v.server_database_version) {
 		LogInfo(
 			"Updates ran successfully, setting database version to [{}] from [{}]",
-			b.custom_database_version,
+			custom_database_version,
 			v.custom_database_version
 		);
-		m_database->QueryDatabase(fmt::format("UPDATE `db_version` SET `custom_version` = {}", b.custom_database_version));
+		m_database->QueryDatabase(fmt::format("UPDATE `db_version` SET `custom_version` = {}", custom_database_version));
 	}
 
 	if (b.bots_database_version > 0) {
-		if (UpdateManifest(bot_manifest_entries, v.bots_database_version, b.bots_database_version)) {
+		int bots_database_version = UpdateManifest(bot_manifest_entries, v.bots_database_version, b.bots_database_version);
+		if (bots_database_version > v.bots_database_version) {
 			LogInfo(
 				"Updates ran successfully, setting database version to [{}] from [{}]",
-				b.bots_database_version,
+				bots_database_version,
 				v.bots_database_version
 			);
 			m_database->QueryDatabase(
 				fmt::format(
 					"UPDATE `db_version` SET `bots_version` = {}",
-					b.bots_database_version
+					bots_database_version
 				)
 			);
 		}
@@ -131,7 +135,7 @@ std::string DatabaseUpdate::GetQueryResult(const ManifestEntry& e)
 	return Strings::Join(result_lines, "\n");
 }
 
-bool DatabaseUpdate::ShouldRunMigration(ManifestEntry &e, std::string query_result)
+bool DatabaseUpdate::ShouldRunMigration(const ManifestEntry& e, std::string& query_result)
 {
 	std::string r = Strings::Trim(query_result);
 	if (e.condition == "contains") {
@@ -163,53 +167,52 @@ bool is_atty()
 #endif
 }
 
-// return true if we ran updates
-bool DatabaseUpdate::UpdateManifest(
-	std::vector<ManifestEntry> entries,
+std::string DisplayPrompt(const std::string& prompt)
+{
+	std::string input;
+	if (is_atty()) {
+		LogInfo("{} (Timeout 60s)", prompt);
+
+		// user input
+		bool gave_input = false;
+		time_t start_time = time(nullptr);
+		time_t wait_time_seconds = 60;
+
+		// spawn a concurrent thread that waits for input from std::cin
+		std::thread t1(
+			[&]() {
+				std::cin >> input;
+				gave_input = true;
+			}
+		);
+		t1.detach();
+
+		// check the inputReceived flag once every 50ms for 10 seconds
+		while (time(nullptr) < start_time + wait_time_seconds && !gave_input) {
+			std::this_thread::sleep_for(std::chrono::milliseconds(50));
+		}
+	}
+
+	return Strings::Trim(input);
+}
+
+// return last successful version updated
+int DatabaseUpdate::UpdateManifest(
+	std::vector<ManifestEntry>& entries,
 	int version_low,
 	int version_high
 )
 {
-	std::vector<int> missing_migrations = {};
+	int latest_version = version_low;
 	if (version_low != version_high) {
+		// assume at this point that we have a migration to do because there is a version number difference. If a
+		// migration for a specific manifest entry does not happen because of a missing test, then log it and
+		// continue (the assumption here is that the user has manually fixed the database at this point). If a force
+		// interactive flag is set, then stop for each query. Fail the migration if the user says no or it times out
+		// because it means the database isn't going to have a correct state to continue. Start with backing up the
+		// database as per user options.
 
-		EQEmuLogSys::Instance()->DisableMySQLErrorLogs();
-		bool force_interactive = false;
-		for (int version = version_low + 1; version <= version_high; ++version) {
-			for (auto &e: entries) {
-				if (e.version == version) {
-					bool        has_migration = true;
-					std::string r             = GetQueryResult(e);
-					if (ShouldRunMigration(e, r)) {
-						has_migration = false;
-						missing_migrations.emplace_back(e.version);
-					}
-
-					std::string prefix = fmt::format(
-						"[{}]",
-						has_migration ? "ok" : "missing"
-					);
-
-					LogInfo(
-						"[{}] {:>10} | [{}]",
-						e.version,
-						prefix,
-						e.description
-					);
-
-					if (!has_migration && e.force_interactive) {
-						force_interactive = true;
-					}
-				}
-			}
-		}
-		EQEmuLogSys::Instance()->EnableMySQLErrorLogs();
-		LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
-
-		if (!missing_migrations.empty() && m_skip_backup) {
-			LogInfo("Skipping database backup");
-		}
-		else if (!missing_migrations.empty()) {
+		if (!m_skip_backup) {
 			LogInfo("Automatically backing up database before applying updates");
 			LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
 			auto s = DatabaseDumpService();
@@ -217,120 +220,73 @@ bool DatabaseUpdate::UpdateManifest(
 			s.SetDumpWithCompression(true);
 			s.DatabaseDump();
 			LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
+		} else {
+			LogInfo("Skipping database backup");
 		}
 
-		if (!missing_migrations.empty()) {
-			LogInfo("Running database migrations. Please wait...");
-			LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
-		}
+		LogInfo("Running database migrations. Please wait...");
+		LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
 
-		if (force_interactive && !std::getenv("FORCE_INTERACTIVE")) {
-			LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
-			LogInfo("Some migrations require user input. Running interactively");
-			LogInfo("This is usually due to a major change that could cause data loss");
-			LogInfo("Your server is automatically backed up before these updates are applied");
-			LogInfo("but you should also make sure you take a backup prior to running this update");
-			LogInfo("Would you like to run this update? [y/n] (Timeout 60s)");
-			LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
+		auto filtered_entries = entries | std::views::filter(
+			[version_low, version_high](const ManifestEntry& entry)
+			{ return entry.version > version_low && entry.version <= version_high; });
 
-			// user input
-			std::string input;
-			bool        gave_input        = false;
-			time_t      start_time        = time(nullptr);
-			time_t      wait_time_seconds = 60;
+		std::vector<ManifestEntry> sorted_entries{filtered_entries.begin(), filtered_entries.end()};
+		std::ranges::sort(sorted_entries, {}, &ManifestEntry::version);
 
-			// spawn a concurrent thread that waits for input from std::cin
-			std::thread t1(
-				[&]() {
-					std::cin >> input;
-					gave_input = true;
-				}
-			);
-			t1.detach();
+		for (const auto& entry : sorted_entries) {
+			// this is the test to run this individual migration. If the test fails, then it is safe to assume
+			// that this migration has already happened manually or otherwise and it's safe to skip
+			// suppress error messages here, it's all tested in the following function
+			EQEmuLogSys::Instance()->DisableMySQLErrorLogs();
+			std::string result = GetQueryResult(entry);
+			EQEmuLogSys::Instance()->EnableMySQLErrorLogs();
 
-			// check the inputReceived flag once every 50ms for 10 seconds
-			while (time(nullptr) < start_time + wait_time_seconds && !gave_input) {
-				std::this_thread::sleep_for(std::chrono::milliseconds(50));
-			}
+			if (ShouldRunMigration(entry, result)) {
+				if (entry.force_interactive && !std::getenv("FORCE_INTERACTIVE")) {
+					LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
+					LogInfo("This migration requires user input. Running interactively");
+					LogInfo("This is usually due to a major change that could cause data loss");
+					LogInfo("Your server is automatically backed up before these updates are applied");
+					LogInfo("but you should also make sure you take a backup prior to running this update");
+					LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
 
-			// prompt for user skip
-			if (Strings::Trim(input) != "y") {
-				LogInfo("Exiting due to user input");
-				std::exit(1);
-			}
-		}
-
-		for (auto &m: missing_migrations) {
-			for (auto &e: entries) {
-				if (e.version == m) {
-					bool errored_migration = false;
-
-					auto r = (e.content_schema_update ? m_content_database : m_database)->QueryDatabaseMulti(e.sql);
-
-					// ignore empty query result "errors"
-					if (r.ErrorNumber() != 1065 && !r.ErrorMessage().empty()) {
-						LogError("(#{}) [{}]", r.ErrorNumber(), r.ErrorMessage());
-						errored_migration = true;
-
-						LogInfo("Required database update failed. This could be a problem");
-
-						// if terminal attached then prompt for skip
-						if (is_atty()) {
-							LogInfo("Would you like to skip this update? [y/n] (Timeout 60s)");
-
-							// user input
-							std::string input;
-							bool        gave_input        = false;
-							time_t      start_time        = time(nullptr);
-							time_t      wait_time_seconds = 60;
-
-							// spawn a concurrent thread that waits for input from std::cin
-							std::thread t1(
-								[&]() {
-									std::cin >> input;
-									gave_input = true;
-								}
-							);
-							t1.detach();
-
-							// check the inputReceived flag once every 50ms for 10 seconds
-							while (time(nullptr) < start_time + wait_time_seconds && !gave_input) {
-								std::this_thread::sleep_for(std::chrono::milliseconds(50));
-							}
-
-							// prompt for user skip
-							if (Strings::Trim(input) == "y") {
-								errored_migration = false;
-								LogInfo("Skipping update [{}] [{}]", e.version, e.description);
-							}
-						} else {
-							errored_migration = true;
-							LogInfo("Skipping update [{}] [{}]", e.version, e.description);
-						}
+					// prompt for user skip
+					if (DisplayPrompt("Would you like to run this update? [y/n]") != "y") {
+						LogInfo("Exiting due to user input");
+						return latest_version;
 					}
+				}
 
-					LogInfo(
-						"[{}] [{}] [{}]",
-						e.version,
-						e.description,
-						(errored_migration ? "error" : "ok")
-					);
+				auto r = (entry.content_schema_update ? m_content_database : m_database)->QueryDatabaseMulti(entry.sql);
 
-					if (errored_migration) {
-						LogError("Fatal | Database migration [{}] failed to run", e.description);
+				// ignore empty query result "errors"
+				if (r.ErrorNumber() != 1065 && !r.ErrorMessage().empty()) {
+					LogError("(#{}) [{}]", r.ErrorNumber(), r.ErrorMessage());
+					LogInfo("Required database update failed.");
+
+					// if terminal attached then prompt for skip
+					if (DisplayPrompt("Would you like to skip this update? [y/n]") == "y") {
+						LogInfo("Skipping update [{}] [{}]", entry.version, entry.description);
+					} else {
+						LogError("Fatal | Database migration [{}] failed to run", entry.description);
 						LogError("Fatal | Shutting down");
-						std::exit(1);
+						return latest_version;
 					}
+
+					LogInfo("[{}] [{}] [error]", entry.version, entry.description);
+				} else {
+					LogInfo("[{}] [{}] [ok]", entry.version, entry.description);
 				}
+
+				latest_version = entry.version;
 			}
 		}
 
 		LogInfo("{}", Strings::Repeat("-", BREAK_LENGTH));
-
-		return true;
 	}
 
-	return false;
+	return latest_version;
 }
 
 DatabaseUpdate *DatabaseUpdate::SetDatabase(Database *db)