From 8d252dfd9aac41699177b776e7f1636ec43b867f Mon Sep 17 00:00:00 2001 From: Chris Miles Date: Mon, 9 Mar 2020 03:00:07 -0500 Subject: [PATCH 01/12] Implement database dump service --- common/CMakeLists.txt | 4 + common/cli/eqemu_command_handler.cpp | 2 +- common/database/database_dump_service.h | 77 ++++++++++++++++ common/file_util.cpp | 67 ++++++++++++++ common/file_util.h | 32 +++++++ utils/sql/character_table_list.txt | 57 ------------ utils/sql/data_tables.txt | 6 -- utils/sql/system_tables.txt | 113 ------------------------ utils/sql/user_tables.txt | 94 -------------------- world/world_server_command_handler.cpp | 62 ++++++++++++- world/world_server_command_handler.h | 1 + 11 files changed, 243 insertions(+), 272 deletions(-) create mode 100644 common/database/database_dump_service.h create mode 100644 common/file_util.cpp create mode 100644 common/file_util.h delete mode 100644 utils/sql/character_table_list.txt delete mode 100644 utils/sql/data_tables.txt delete mode 100644 utils/sql/system_tables.txt delete mode 100644 utils/sql/user_tables.txt diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index a959d9093..e312c202d 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -9,6 +9,7 @@ SET(common_sources crash.cpp crc16.cpp crc32.cpp + database/database_dump_service.cpp database.cpp database_conversions.cpp database_instances.cpp @@ -31,6 +32,7 @@ SET(common_sources event_sub.cpp extprofile.cpp faction.cpp + file_util.cpp guild_base.cpp guilds.cpp inventory_profile.cpp @@ -120,6 +122,7 @@ SET(common_headers cli/argh.h cli/eqemu_command_handler.h cli/terminal_color.hpp + database/database_dump_service.h data_verification.h database.h database_schema.h @@ -150,6 +153,7 @@ SET(common_headers event_sub.h extprofile.h faction.h + file_util.h features.h fixed_memory_hash_set.h fixed_memory_variable_hash_set.h diff --git a/common/cli/eqemu_command_handler.cpp b/common/cli/eqemu_command_handler.cpp index bbcdb4612..96ed5a155 100644 --- a/common/cli/eqemu_command_handler.cpp +++ b/common/cli/eqemu_command_handler.cpp @@ -96,7 +96,7 @@ namespace EQEmuCommand { "\nCommand" << termcolor::reset << "\n\n" << termcolor::green << argv[1] << arguments_string << termcolor::reset << "\n" << - termcolor::yellow << (!options_string.empty() ? "\nOptions\n" : "") << + termcolor::yellow << (!options_string.empty() ? "\nOptions\n\n" : "") << termcolor::reset << termcolor::cyan << options_string << termcolor::reset; std::cout << command_string.str() << std::endl; diff --git a/common/database/database_dump_service.h b/common/database/database_dump_service.h new file mode 100644 index 000000000..096d4c5d3 --- /dev/null +++ b/common/database/database_dump_service.h @@ -0,0 +1,77 @@ +/** + * EQEmulator: Everquest Server Emulator + * Copyright (C) 2001-2020 EQEmulator Development Team (https://github.com/EQEmu/Server) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY except by those people which sell it, which + * are required to give you total support for your newly bought product; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * +*/ + +#ifndef EQEMU_DATABASE_DUMP_SERVICE_H +#define EQEMU_DATABASE_DUMP_SERVICE_H + + +class DatabaseDumpService { +public: + void Dump(); + bool IsDumpAllTables() const; + void SetDumpAllTables(bool dump_all_tables); + bool IsDumpWithNoData() const; + void SetDumpWithNoData(bool dump_with_no_data); + bool IsDumpSystemTables() const; + void SetDumpSystemTables(bool dump_system_tables); + bool IsDumpContentTables() const; + void SetDumpContentTables(bool dump_content_tables); + bool IsDumpPlayerTables() const; + void SetDumpPlayerTables(bool dump_player_tables); + bool IsDumpLoginServerTables() const; + void SetDumpLoginServerTables(bool dump_login_server_tables); + bool IsDumpNoTableLock() const; + void SetDumpNoTableLock(bool dump_no_table_lock); + bool IsDumpWithCompression() const; + void SetDumpWithCompression(bool dump_with_compression); + const std::string &GetDumpPath() const; + void SetDumpPath(const std::string &dump_path); + const std::string &GetDumpFileName() const; + void SetDumpFileName(const std::string &dump_file_name); + +private: + bool dump_all_tables; + bool dump_system_tables; + bool dump_content_tables; + bool dump_player_tables; + bool dump_login_server_tables; + bool dump_with_no_data; + bool dump_no_table_lock; + bool dump_with_compression; + std::string dump_path; + std::string dump_file_name; + + std::string execute(const std::string &cmd, bool return_result); + bool IsMySQLInstalled(); + std::string GetMySQLVersion(); + std::string GetBaseMySQLDumpCommand(); + std::string GetPlayerTablesList(); + std::string GetSystemTablesList(); + std::string GetContentTablesList(); + std::string GetLoginTableList(); + bool IsTarAvailable(); + bool IsRarAvailable(); + bool HasCompressionBinary(); + std::string GetDumpFileNameWithPath(); + std::string GetSetDumpPath(); +}; + + +#endif //EQEMU_DATABASE_DUMP_SERVICE_H diff --git a/common/file_util.cpp b/common/file_util.cpp new file mode 100644 index 000000000..8c9aca9a4 --- /dev/null +++ b/common/file_util.cpp @@ -0,0 +1,67 @@ +/** + * EQEmulator: Everquest Server Emulator + * Copyright (C) 2001-2020 EQEmulator Development Team (https://github.com/EQEmu/Server) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY except by those people which sell it, which + * are required to give you total support for your newly bought product; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * +*/ + +#include +#include "file_util.h" + +#ifdef _WINDOWS +#include +#include +#include +#include +#include +#include +#else + +#include +#include + +#endif + +/** + * @param name + * @return + */ +bool FileUtil::exists(const std::string &name) +{ + std::ifstream f(name.c_str()); + + return f.good(); +} + +/** + * @param directory_name + */ +void FileUtil::mkdir(const std::string& directory_name) +{ + +#ifdef _WINDOWS + struct _stat st; + if (_stat(directory_name.c_str(), &st) == 0) // exists + return; + _mkdir(directory_name.c_str()); +#else + struct stat st{}; + if (stat(directory_name.c_str(), &st) == 0) { // exists + return; + } + ::mkdir(directory_name.c_str(), 0755); +#endif +} \ No newline at end of file diff --git a/common/file_util.h b/common/file_util.h new file mode 100644 index 000000000..05869d303 --- /dev/null +++ b/common/file_util.h @@ -0,0 +1,32 @@ +/** + * EQEmulator: Everquest Server Emulator + * Copyright (C) 2001-2020 EQEmulator Development Team (https://github.com/EQEmu/Server) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY except by those people which sell it, which + * are required to give you total support for your newly bought product; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * +*/ + +#ifndef EQEMU_FILE_UTIL_H +#define EQEMU_FILE_UTIL_H + + +class FileUtil { +public: + static bool exists(const std::string &name); + static void mkdir(const std::string& directory_name); +}; + + +#endif //EQEMU_FILE_UTIL_H diff --git a/utils/sql/character_table_list.txt b/utils/sql/character_table_list.txt deleted file mode 100644 index ab6610dab..000000000 --- a/utils/sql/character_table_list.txt +++ /dev/null @@ -1,57 +0,0 @@ -account -account_ip -account_flags -account_rewards -adventure_details -adventure_stats -buyer -char_recipe_list -character_activities -character_alt_currency -character_alternate_abilities -character_auras -character_bandolier -character_bind -character_buffs -character_corpse_items -character_corpses -character_currency -character_data -character_disciplines -character_enabledtasks -character_inspect_messages -character_item_recast -character_languages -character_leadership_abilities -character_material -character_memmed_spells -character_pet_buffs -character_pet_info -character_pet_inventory -character_potionbelt -character_skills -character_spells -character_tasks -character_tribute -completed_tasks -data_buckets -faction_values -friends -guild_bank -guild_members -guild_ranks -guild_relations -guilds -instance_list_player -inventory -inventory_snapshots -keyring -mail -player_titlesets -quest_globals -sharedbank -timers -titles -trader -trader_audit -zone_flags" \ No newline at end of file diff --git a/utils/sql/data_tables.txt b/utils/sql/data_tables.txt deleted file mode 100644 index 8738c716b..000000000 --- a/utils/sql/data_tables.txt +++ /dev/null @@ -1,6 +0,0 @@ -command_settings -inventory_versions -launcher -rule_sets -rule_values -variables \ No newline at end of file diff --git a/utils/sql/system_tables.txt b/utils/sql/system_tables.txt deleted file mode 100644 index a33605130..000000000 --- a/utils/sql/system_tables.txt +++ /dev/null @@ -1,113 +0,0 @@ -aa_ability -aa_actions -aa_effects -aa_rank_effects -aa_rank_prereqs -aa_ranks -aa_required_level_cost -adventure_template -adventure_template_entry -adventure_template_entry_flavor -altadv_vars -alternate_currency -auras -base_data -blocked_spells -books -bug_reports -char_create_combinations -char_create_point_allocations -class_skill -damageshieldtypes -data_buckets -db_str -doors -eqtime -faction_base_data -faction_list -faction_list_mod -fear_hints -fishing -forage -global_loot -goallists -graveyard -grid -grid_entries -ground_spawns -horses -instance_list -items -ip_exemptions -ldon_trap_entries -ldon_trap_templates -level_exp_mods -logsys_categories -lootdrop -lootdrop_entries -loottable -loottable_entries -merc_armorinfo -merc_buffs -merc_inventory -merc_merchant_entries -merc_merchant_template_entries -merc_merchant_templates -merc_name_types -merc_npc_types -merc_spell_list_entries -merc_spell_lists -merc_stance_entries -merc_stats -merc_subtypes -merc_templates -merc_types -merc_weaponinfo -merchantlist -mercs -name_filter -npc_emotes -npc_faction -npc_faction_entries -npc_scale_global_base -npc_spells -npc_spells_effects -npc_spells_effects_entries -npc_spells_entries -npc_types -npc_types_metadata -npc_types_tint -object -perl_event_export_settings -pets -pets_equipmentset -pets_equipmentset_entries -profanity_list -proximities -races -saylink -skill_caps -spawn2 -spawn_condition_values -spawn_conditions -spawn_events -spawnentry -spawngroup -spells_new -start_zones -starting_items -task_activities -tasks -tasksets -titles -tradeskill_recipe -tradeskill_recipe_entries -traps -tribute_levels -tributes -veteran_reward_templates -zone -zone_points -zone_server -zone_state_dump -zoneserver_auth diff --git a/utils/sql/user_tables.txt b/utils/sql/user_tables.txt deleted file mode 100644 index ee19fe472..000000000 --- a/utils/sql/user_tables.txt +++ /dev/null @@ -1,94 +0,0 @@ -aa_timers -account -account_flags -account_ip -account_rewards -adventure_details -adventure_members -adventure_stats -banned_ips -bugs -buyer -char_recipe_list -character_activities -character_alt_currency -character_alternate_abilities -character_auras -character_bandolier -character_bind -character_buffs -character_corpse_items -character_corpses -character_currency -character_data -character_disciplines -character_enabledtasks -character_inspect_messages -character_item_recast -character_languages -character_leadership_abilities -character_material -character_memmed_spells -character_pet_buffs -character_pet_info -character_pet_inventory -character_potionbelt -character_skills -character_spells -character_tasks -character_tribute -chatchannels -completed_tasks -discovered_items -eventlog -faction_values -friends -gm_ips -group_id -group_leaders -guild_bank -guild_members -guild_ranks -guild_relations -guilds -hackers -instance_list_player -inventory -inventory_snapshots -item_tick -keyring -launcher_zones -lfguild -mail -merchantlist_temp -object_contents -petitions -player_titlesets -qs_merchant_transaction_record -qs_merchant_transaction_record_entries -qs_player_aa_rate_hourly -qs_player_delete_record -qs_player_delete_record_entries -qs_player_events -qs_player_handin_record -qs_player_handin_record_entries -qs_player_move_record -qs_player_move_record_entries -qs_player_npc_kill_record -qs_player_npc_kill_record_entries -qs_player_speech -qs_player_trade_record -qs_player_trade_record_entries -quest_globals -raid_details -raid_leaders -raid_members -reports -respawn_times -sharedbank -spell_buckets -spell_globals -timers -trader -trader_audit -zone_flags diff --git a/world/world_server_command_handler.cpp b/world/world_server_command_handler.cpp index 3a929d89f..565277f97 100644 --- a/world/world_server_command_handler.cpp +++ b/world/world_server_command_handler.cpp @@ -24,6 +24,7 @@ #include "../common/version.h" #include "worlddb.h" #include "../common/database_schema.h" +#include "../common/database/database_dump_service.h" namespace WorldserverCommandHandler { @@ -51,6 +52,7 @@ namespace WorldserverCommandHandler { function_map["database:version"] = &WorldserverCommandHandler::DatabaseVersion; function_map["database:set-account-status"] = &WorldserverCommandHandler::DatabaseSetAccountStatus; function_map["database:schema"] = &WorldserverCommandHandler::DatabaseGetSchema; + function_map["database:dump"] = &WorldserverCommandHandler::DatabaseDump; EQEmuCommand::HandleMenu(function_map, cmd, argc, argv); } @@ -145,7 +147,7 @@ namespace WorldserverCommandHandler { */ void DatabaseGetSchema(int argc, char **argv, argh::parser &cmd, std::string &description) { - description = "Displays server database schema"; + description = "Displays server database schema"; if (cmd[{"-h", "--help"}]) { return; @@ -202,4 +204,62 @@ namespace WorldserverCommandHandler { std::cout << payload.str() << std::endl; } + /** + * @param argc + * @param argv + * @param cmd + * @param description + */ + void DatabaseDump(int argc, char **argv, argh::parser &cmd, std::string &description) + { + description = "Dumps server database tables"; + + if (cmd[{"-h", "--help"}]) { + return; + } + + std::vector arguments = {}; + std::vector options = { + "--all", + "--content-tables", + "--login-tables", + "--player-tables", + "--system-tables", + "--table-structure-only", + "--no-table-lock", + "--dump-path=", + "--compress" + }; + + + if (argc < 3) { + EQEmuCommand::ValidateCmdInput(arguments, options, cmd, argc, argv); + return; + } + + auto database_dump_service = new DatabaseDumpService(); + bool dump_all = cmd[{"-a", "--all"}]; + + if (!cmd("--dump-path").str().empty()) { + database_dump_service->SetDumpPath(cmd("--dump-path").str()); + } + + /** + * Set Option + */ + database_dump_service->SetDumpContentTables(cmd[{"-c", "--content-tables"}] || dump_all); + database_dump_service->SetDumpLoginServerTables(cmd[{"-c", "--login-tables"}] || dump_all); + database_dump_service->SetDumpPlayerTables(cmd[{"-c", "--player-tables"}] || dump_all); + database_dump_service->SetDumpSystemTables(cmd[{"-c", "--system-tables"}] || dump_all); + database_dump_service->SetDumpWithNoData(cmd[{"-c", "--table-structure-only"}]); + database_dump_service->SetDumpAllTables(dump_all); + database_dump_service->SetDumpNoTableLock(cmd[{"--no-table-lock"}]); + database_dump_service->SetDumpWithCompression(cmd[{"--compress"}]); + + /** + * Dump + */ + database_dump_service->Dump(); + } + } \ No newline at end of file diff --git a/world/world_server_command_handler.h b/world/world_server_command_handler.h index f3d4317f2..a32a78f0f 100644 --- a/world/world_server_command_handler.h +++ b/world/world_server_command_handler.h @@ -30,6 +30,7 @@ namespace WorldserverCommandHandler { void DatabaseVersion(int argc, char **argv, argh::parser &cmd, std::string &description); void DatabaseSetAccountStatus(int argc, char **argv, argh::parser &cmd, std::string &description); void DatabaseGetSchema(int argc, char **argv, argh::parser &cmd, std::string &description); + void DatabaseDump(int argc, char **argv, argh::parser &cmd, std::string &description); }; From d4a1ea82dce70d3006c7c25c41d3001ffb0a263c Mon Sep 17 00:00:00 2001 From: Akkadius Date: Mon, 9 Mar 2020 03:01:59 -0500 Subject: [PATCH 02/12] Create database_dump_service.cpp --- common/database/database_dump_service.cpp | 434 ++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 common/database/database_dump_service.cpp diff --git a/common/database/database_dump_service.cpp b/common/database/database_dump_service.cpp new file mode 100644 index 000000000..68608bd67 --- /dev/null +++ b/common/database/database_dump_service.cpp @@ -0,0 +1,434 @@ +/** + * EQEmulator: Everquest Server Emulator + * Copyright (C) 2001-2020 EQEmulator Development Team (https://github.com/EQEmu/Server) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY except by those people which sell it, which + * are required to give you total support for your newly bought product; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR + * A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * +*/ + +#include +#include +#include +#include "database_dump_service.h" +#include "../eqemu_logsys.h" +#include "../string_util.h" +#include "../eqemu_config.h" +#include "../database_schema.h" +#include "../file_util.h" + +#define DATABASE_DUMP_PATH "backups/" + +/** + * @param cmd + * @param return_result + * @return + */ +std::string DatabaseDumpService::execute(const std::string &cmd, bool return_result = true) +{ + const char *file_name = "db-exec-result.txt"; + + if (return_result) { + std::system((cmd + " > " + file_name).c_str()); + } + else { + std::system((cmd).c_str()); + } + + std::string result; + + if (return_result) { + std::ifstream file(file_name); + result = {std::istreambuf_iterator(file), std::istreambuf_iterator()}; + std::remove(file_name); + + } + + return result; +} + +/** + * @return bool + */ +bool DatabaseDumpService::IsMySQLInstalled() +{ + std::string version_output = GetMySQLVersion(); + + return version_output.find("mysql") != std::string::npos && version_output.find("Ver") != std::string::npos; +} + +/** + * Linux + * @return bool + */ +bool DatabaseDumpService::IsTarAvailable() +{ + std::string version_output = execute("tar --version"); + + return version_output.find("GNU tar") != std::string::npos; +} + +/** + * Windows TODO + * @return bool + */ +bool DatabaseDumpService::IsRarAvailable() +{ + std::string version_output = execute("winrar --version"); + + return version_output.find("todo") != std::string::npos; +} + +/** + * @return + */ +bool DatabaseDumpService::HasCompressionBinary() +{ + return IsTarAvailable() || IsRarAvailable(); +} + +/** + * @return + */ +std::string DatabaseDumpService::GetMySQLVersion() +{ + std::string version_output = execute("mysql --version"); + + return trim(version_output); +} + +/** + * @return + */ +std::string DatabaseDumpService::GetBaseMySQLDumpCommand() +{ + auto config = EQEmuConfig::get(); + + return fmt::format( + "mysqldump -u {} -p{} -h {} {}", + config->DatabaseUsername, + config->DatabasePassword, + config->DatabaseHost, + config->DatabaseDB + ); +} + +/** + * @return + */ +std::string DatabaseDumpService::GetPlayerTablesList() +{ + std::string tables_list; + std::vector tables = DatabaseSchema::GetPlayerTables(); + for (const auto &table : tables) { + tables_list += table + " "; + } + + return trim(tables_list); +} + +/** + * @return + */ +std::string DatabaseDumpService::GetLoginTableList() +{ + std::string tables_list; + std::vector tables = DatabaseSchema::GetLoginTables(); + for (const auto &table : tables) { + tables_list += table + " "; + } + + return trim(tables_list); +} + +/** + * @return + */ +std::string DatabaseDumpService::GetSystemTablesList() +{ + std::string tables_list; + + std::vector tables = DatabaseSchema::GetServerTables(); + for (const auto &table : tables) { + tables_list += table + " "; + } + + tables = DatabaseSchema::GetStateTables(); + for (const auto &table : tables) { + tables_list += table + " "; + } + + tables = DatabaseSchema::GetVersionTables(); + for (const auto &table : tables) { + tables_list += table + " "; + } + + return trim(tables_list); +} + +/** + * @return + */ +std::string DatabaseDumpService::GetContentTablesList() +{ + std::string tables_list; + + std::vector tables = DatabaseSchema::GetContentTables(); + for (const auto &table : tables) { + tables_list += table + " "; + } + + return trim(tables_list); +} + +/** + * @return + */ +std::string GetDumpDate() +{ + + time_t now = time(nullptr); + struct tm time_struct{}; + char buf[80]; + time_struct = *localtime(&now); + strftime(buf, sizeof(buf), "%Y-%m-%d", &time_struct); + + std::string time = buf; + + return time; +} + +/** + * @return + */ +std::string DatabaseDumpService::GetSetDumpPath() +{ + return !GetDumpPath().empty() ? GetDumpPath() : DATABASE_DUMP_PATH; +} +/** + * @return + */ +std::string DatabaseDumpService::GetDumpFileNameWithPath() +{ + return GetSetDumpPath() + GetDumpFileName(); +} + +void DatabaseDumpService::Dump() +{ + if (!IsMySQLInstalled()) { + LogError("MySQL is not installed; Please check your PATH for a valid MySQL installation"); + return; + } + + LogInfo("MySQL installed [{}]", GetMySQLVersion()); + + SetDumpFileName(EQEmuConfig::get()->DatabaseDB + '-' + GetDumpDate()); + + auto config = EQEmuConfig::get(); + + LogInfo( + "Database [{}] Host [{}] Username [{}]", + config->DatabaseDB, + config->DatabaseHost, + config->DatabaseUsername + ); + + std::string options = "--compact --allow-keywords --extended-insert"; + + if (IsDumpWithNoData()) { + options += " --no-data"; + } + + if (IsDumpNoTableLock()) { + options += " --skip-lock-tables"; + } + + std::string tables_to_dump; + std::string dump_descriptor; + + if (!IsDumpAllTables()) { + if (IsDumpPlayerTables()) { + tables_to_dump += GetPlayerTablesList(); + dump_descriptor += "-player"; + } + + if (IsDumpSystemTables()) { + tables_to_dump += GetSystemTablesList(); + dump_descriptor += "-system"; + } + + if (IsDumpContentTables()) { + tables_to_dump += GetContentTablesList(); + dump_descriptor += "-content"; + } + + if (IsDumpLoginServerTables()) { + tables_to_dump += GetLoginTableList(); + dump_descriptor += "-login"; + } + } + + if (!dump_descriptor.empty()) { + SetDumpFileName(GetDumpFileName() + dump_descriptor); + } + + std::string execute_command = fmt::format( + "{} {} {} > {}.sql", + GetBaseMySQLDumpCommand(), + options, + tables_to_dump, + GetDumpFileNameWithPath() + ); + + if (!FileUtil::exists(GetSetDumpPath())) { + FileUtil::mkdir(GetSetDumpPath()); + } + + execute(execute_command, false); + + if (!tables_to_dump.empty()) { + LogInfo("Dumping Tables [{}]", tables_to_dump); + } + + LogInfo("Database dump created at [{}.sql]", GetDumpFileNameWithPath()); + + if (IsDumpWithCompression()) { + if (HasCompressionBinary()) { + LogInfo("Compression requested... Compressing dump [{}.sql]", GetDumpFileNameWithPath()); + + if (IsTarAvailable()) { + execute( + fmt::format( + "tar -zcvf {}.tar.gz -C {} {}.sql", + GetDumpFileNameWithPath(), + GetSetDumpPath(), + GetDumpFileName() + ) + ); + LogInfo("Compressed dump created at [{}.tar.gz]", GetDumpFileNameWithPath()); + } + } + else { + LogWarning("Compression requested but binary not found... Skipping..."); + } + } + +// LogDebug("[{}] dump-path", GetSetDumpPath()); +// LogDebug("[{}] compression", (IsDumpWithCompression() ? "true" : "false")); +// LogDebug("[{}] has-compression-binary", (HasCompressionBinary() ? "true" : "false")); +// LogDebug("[{}] content", (IsDumpContentTables() ? "true" : "false")); +// LogDebug("[{}] no-data", (IsDumpWithNoData() ? "true" : "false")); +// LogDebug("[{}] login", (IsDumpLoginServerTables() ? "true" : "false")); +// LogDebug("[{}] player", (IsDumpPlayerTables() ? "true" : "false")); +// LogDebug("[{}] system", (IsDumpSystemTables() ? "true" : "false")); +} + +bool DatabaseDumpService::IsDumpSystemTables() const +{ + return dump_system_tables; +} + +void DatabaseDumpService::SetDumpSystemTables(bool dump_system_tables) +{ + DatabaseDumpService::dump_system_tables = dump_system_tables; +} + +bool DatabaseDumpService::IsDumpContentTables() const +{ + return dump_content_tables; +} + +void DatabaseDumpService::SetDumpContentTables(bool dump_content_tables) +{ + DatabaseDumpService::dump_content_tables = dump_content_tables; +} + +bool DatabaseDumpService::IsDumpPlayerTables() const +{ + return dump_player_tables; +} + +void DatabaseDumpService::SetDumpPlayerTables(bool dump_player_tables) +{ + DatabaseDumpService::dump_player_tables = dump_player_tables; +} + +bool DatabaseDumpService::IsDumpLoginServerTables() const +{ + return dump_login_server_tables; +} + +void DatabaseDumpService::SetDumpLoginServerTables(bool dump_login_server_tables) +{ + DatabaseDumpService::dump_login_server_tables = dump_login_server_tables; +} + +bool DatabaseDumpService::IsDumpWithNoData() const +{ + return dump_with_no_data; +} + +void DatabaseDumpService::SetDumpWithNoData(bool dump_with_no_data) +{ + DatabaseDumpService::dump_with_no_data = dump_with_no_data; +} + +bool DatabaseDumpService::IsDumpAllTables() const +{ + return dump_all_tables; +} + +void DatabaseDumpService::SetDumpAllTables(bool dump_all_tables) +{ + DatabaseDumpService::dump_all_tables = dump_all_tables; +} +bool DatabaseDumpService::IsDumpNoTableLock() const +{ + return dump_no_table_lock; +} + +void DatabaseDumpService::SetDumpNoTableLock(bool dump_no_table_lock) +{ + DatabaseDumpService::dump_no_table_lock = dump_no_table_lock; +} + +bool DatabaseDumpService::IsDumpWithCompression() const +{ + return dump_with_compression; +} + +void DatabaseDumpService::SetDumpWithCompression(bool dump_with_compression) +{ + DatabaseDumpService::dump_with_compression = dump_with_compression; +} + +const std::string &DatabaseDumpService::GetDumpPath() const +{ + return dump_path; +} + +void DatabaseDumpService::SetDumpPath(const std::string &dump_path) +{ + DatabaseDumpService::dump_path = dump_path; +} + +void DatabaseDumpService::SetDumpFileName(const std::string &dump_file_name) +{ + DatabaseDumpService::dump_file_name = dump_file_name; +} + +const std::string &DatabaseDumpService::GetDumpFileName() const +{ + return dump_file_name; +} From dd1470892dc3eb6448852803b06bb2b6b50a1902 Mon Sep 17 00:00:00 2001 From: Akkadius Date: Mon, 9 Mar 2020 03:20:43 -0500 Subject: [PATCH 03/12] Update database_dump_service.cpp --- common/database/database_dump_service.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/database/database_dump_service.cpp b/common/database/database_dump_service.cpp index 68608bd67..4d0273bf6 100644 --- a/common/database/database_dump_service.cpp +++ b/common/database/database_dump_service.cpp @@ -28,6 +28,13 @@ #include "../database_schema.h" #include "../file_util.h" +#include +#if _WIN32 +#include +#else +#include +#endif + #define DATABASE_DUMP_PATH "backups/" /** From bfecd6ad14a8991e30292fe4a837242e1be341cf Mon Sep 17 00:00:00 2001 From: Akkadius Date: Mon, 9 Mar 2020 14:08:17 -0500 Subject: [PATCH 04/12] Add query-serv table dump option, add option to dump directly to console, add initializers for dump settings --- common/database/database_dump_service.cpp | 77 ++++++++++++++++++++--- common/database/database_dump_service.h | 23 ++++--- common/database_schema.h | 27 +++++++- world/world_server_command_handler.cpp | 4 ++ 4 files changed, 113 insertions(+), 18 deletions(-) diff --git a/common/database/database_dump_service.cpp b/common/database/database_dump_service.cpp index 4d0273bf6..f7a339c84 100644 --- a/common/database/database_dump_service.cpp +++ b/common/database/database_dump_service.cpp @@ -29,10 +29,13 @@ #include "../file_util.h" #include + #if _WIN32 #include #else + #include + #endif #define DATABASE_DUMP_PATH "backups/" @@ -159,6 +162,20 @@ std::string DatabaseDumpService::GetLoginTableList() return trim(tables_list); } +/** + * @return + */ +std::string DatabaseDumpService::GetQueryServTables() +{ + std::string tables_list; + std::vector tables = DatabaseSchema::GetQueryServerTables(); + for (const auto &table : tables) { + tables_list += table + " "; + } + + return trim(tables_list); +} + /** * @return */ @@ -238,6 +255,10 @@ void DatabaseDumpService::Dump() return; } + if (IsDumpOutputToConsole()) { + LogSys.SilenceConsoleLogging(); + } + LogInfo("MySQL installed [{}]", GetMySQLVersion()); SetDumpFileName(EQEmuConfig::get()->DatabaseDB + '-' + GetDumpDate()); @@ -266,43 +287,59 @@ void DatabaseDumpService::Dump() if (!IsDumpAllTables()) { if (IsDumpPlayerTables()) { - tables_to_dump += GetPlayerTablesList(); + tables_to_dump += GetPlayerTablesList() + " "; dump_descriptor += "-player"; } if (IsDumpSystemTables()) { - tables_to_dump += GetSystemTablesList(); + tables_to_dump += GetSystemTablesList() + " "; dump_descriptor += "-system"; } if (IsDumpContentTables()) { - tables_to_dump += GetContentTablesList(); + tables_to_dump += GetContentTablesList() + " "; dump_descriptor += "-content"; } if (IsDumpLoginServerTables()) { - tables_to_dump += GetLoginTableList(); + tables_to_dump += GetLoginTableList() + " "; dump_descriptor += "-login"; } + + if (IsDumpQueryServerTables()) { + tables_to_dump += GetQueryServTables(); + dump_descriptor += "-queryserv"; + } } if (!dump_descriptor.empty()) { SetDumpFileName(GetDumpFileName() + dump_descriptor); } + /** + * If we are dumping to stdout then we don't generate a file + */ + std::string pipe_file; + if (!IsDumpOutputToConsole()) { + pipe_file = fmt::format(" > {}.sql", GetDumpFileNameWithPath()); + } + std::string execute_command = fmt::format( - "{} {} {} > {}.sql", + "{} {} {} {}", GetBaseMySQLDumpCommand(), options, tables_to_dump, - GetDumpFileNameWithPath() + pipe_file ); - if (!FileUtil::exists(GetSetDumpPath())) { + if (!FileUtil::exists(GetSetDumpPath()) && !IsDumpOutputToConsole()) { FileUtil::mkdir(GetSetDumpPath()); } - execute(execute_command, false); + std::string execution_result = execute(execute_command, IsDumpOutputToConsole()); + if (!execution_result.empty()) { + std::cout << execution_result; + } if (!tables_to_dump.empty()) { LogInfo("Dumping Tables [{}]", tables_to_dump); @@ -310,7 +347,7 @@ void DatabaseDumpService::Dump() LogInfo("Database dump created at [{}.sql]", GetDumpFileNameWithPath()); - if (IsDumpWithCompression()) { + if (IsDumpWithCompression() && !IsDumpOutputToConsole()) { if (HasCompressionBinary()) { LogInfo("Compression requested... Compressing dump [{}.sql]", GetDumpFileNameWithPath()); @@ -331,8 +368,10 @@ void DatabaseDumpService::Dump() } } +// LogDebug("[{}] dump-to-console", IsDumpOutputToConsole()); // LogDebug("[{}] dump-path", GetSetDumpPath()); // LogDebug("[{}] compression", (IsDumpWithCompression() ? "true" : "false")); +// LogDebug("[{}] query-serv", (IsDumpQueryServerTables() ? "true" : "false")); // LogDebug("[{}] has-compression-binary", (HasCompressionBinary() ? "true" : "false")); // LogDebug("[{}] content", (IsDumpContentTables() ? "true" : "false")); // LogDebug("[{}] no-data", (IsDumpWithNoData() ? "true" : "false")); @@ -439,3 +478,23 @@ const std::string &DatabaseDumpService::GetDumpFileName() const { return dump_file_name; } + +bool DatabaseDumpService::IsDumpQueryServerTables() const +{ + return dump_query_server_tables; +} + +void DatabaseDumpService::SetDumpQueryServerTables(bool dump_query_server_tables) +{ + DatabaseDumpService::dump_query_server_tables = dump_query_server_tables; +} + +bool DatabaseDumpService::IsDumpOutputToConsole() const +{ + return dump_output_to_console; +} + +void DatabaseDumpService::SetDumpOutputToConsole(bool dump_output_to_console) +{ + DatabaseDumpService::dump_output_to_console = dump_output_to_console; +} \ No newline at end of file diff --git a/common/database/database_dump_service.h b/common/database/database_dump_service.h index 096d4c5d3..eb87520da 100644 --- a/common/database/database_dump_service.h +++ b/common/database/database_dump_service.h @@ -45,16 +45,22 @@ public: void SetDumpPath(const std::string &dump_path); const std::string &GetDumpFileName() const; void SetDumpFileName(const std::string &dump_file_name); + bool IsDumpQueryServerTables() const; + void SetDumpQueryServerTables(bool dump_query_server_tables); + bool IsDumpOutputToConsole() const; + void SetDumpOutputToConsole(bool dump_output_to_console); private: - bool dump_all_tables; - bool dump_system_tables; - bool dump_content_tables; - bool dump_player_tables; - bool dump_login_server_tables; - bool dump_with_no_data; - bool dump_no_table_lock; - bool dump_with_compression; + bool dump_all_tables = false; + bool dump_system_tables = false; + bool dump_content_tables = false; + bool dump_player_tables = false; + bool dump_query_server_tables = false; + bool dump_login_server_tables = false; + bool dump_with_no_data = false; + bool dump_no_table_lock = false; + bool dump_with_compression = false; + bool dump_output_to_console = false; std::string dump_path; std::string dump_file_name; @@ -71,6 +77,7 @@ private: bool HasCompressionBinary(); std::string GetDumpFileNameWithPath(); std::string GetSetDumpPath(); + std::string GetQueryServTables(); }; diff --git a/common/database_schema.h b/common/database_schema.h index b497ad474..5502441d0 100644 --- a/common/database_schema.h +++ b/common/database_schema.h @@ -233,7 +233,6 @@ namespace DatabaseSchema { "task_activities", "tasks", "tasksets", - "titles", "tradeskill_recipe", "tradeskill_recipe_entries", "traps", @@ -282,6 +281,32 @@ namespace DatabaseSchema { }; } + /** + * Gets QueryServer tables + * + * @return + */ + static std::vector GetQueryServerTables() + { + return { + "qs_merchant_transaction_record", + "qs_merchant_transaction_record_entries", + "qs_player_aa_rate_hourly", + "qs_player_delete_record", + "qs_player_delete_record_entries", + "qs_player_events", + "qs_player_handin_record", + "qs_player_handin_record_entries", + "qs_player_move_record", + "qs_player_move_record_entries", + "qs_player_npc_kill_record", + "qs_player_npc_kill_record_entries", + "qs_player_speech", + "qs_player_trade_record", + "qs_player_trade_record_entries", + }; + } + /** * Gets state tables * Tables that keep track of server state diff --git a/world/world_server_command_handler.cpp b/world/world_server_command_handler.cpp index 565277f97..32e794800 100644 --- a/world/world_server_command_handler.cpp +++ b/world/world_server_command_handler.cpp @@ -225,9 +225,11 @@ namespace WorldserverCommandHandler { "--login-tables", "--player-tables", "--system-tables", + "--query-serv-tables", "--table-structure-only", "--no-table-lock", "--dump-path=", + "--dump-output-to-console", "--compress" }; @@ -252,9 +254,11 @@ namespace WorldserverCommandHandler { database_dump_service->SetDumpPlayerTables(cmd[{"-c", "--player-tables"}] || dump_all); database_dump_service->SetDumpSystemTables(cmd[{"-c", "--system-tables"}] || dump_all); database_dump_service->SetDumpWithNoData(cmd[{"-c", "--table-structure-only"}]); + database_dump_service->SetDumpQueryServerTables(cmd[{"--query-serv-tables"}]); database_dump_service->SetDumpAllTables(dump_all); database_dump_service->SetDumpNoTableLock(cmd[{"--no-table-lock"}]); database_dump_service->SetDumpWithCompression(cmd[{"--compress"}]); + database_dump_service->SetDumpOutputToConsole(cmd[{"--dump-output-to-console"}]); /** * Dump From 53be04c39c54ea3aa481dceaa63d410e208e5a61 Mon Sep 17 00:00:00 2001 From: Akkadius Date: Mon, 9 Mar 2020 15:51:11 -0500 Subject: [PATCH 05/12] Don't lock tables by default; move tables around in schema; add peq-dump.sh script --- common/database/database_dump_service.cpp | 46 +++++++++++++++++----- common/database/database_dump_service.h | 27 +++++++------ common/database_schema.h | 17 ++++---- utils/sql/peq-dump/peq-dump.sh | 47 +++++++++++++++++++++++ world/world_server_command_handler.cpp | 6 ++- 5 files changed, 112 insertions(+), 31 deletions(-) create mode 100755 utils/sql/peq-dump/peq-dump.sh diff --git a/common/database/database_dump_service.cpp b/common/database/database_dump_service.cpp index f7a339c84..7610f3759 100644 --- a/common/database/database_dump_service.cpp +++ b/common/database/database_dump_service.cpp @@ -255,6 +255,10 @@ void DatabaseDumpService::Dump() return; } + if (IsDumpDropTableSyntaxOnly()) { + SetDumpOutputToConsole(true); + } + if (IsDumpOutputToConsole()) { LogSys.SilenceConsoleLogging(); } @@ -278,7 +282,7 @@ void DatabaseDumpService::Dump() options += " --no-data"; } - if (IsDumpNoTableLock()) { + if (!IsDumpTableLock()) { options += " --skip-lock-tables"; } @@ -336,9 +340,22 @@ void DatabaseDumpService::Dump() FileUtil::mkdir(GetSetDumpPath()); } - std::string execution_result = execute(execute_command, IsDumpOutputToConsole()); - if (!execution_result.empty()) { - std::cout << execution_result; + if (IsDumpDropTableSyntaxOnly()) { + std::vector tables = SplitString(tables_to_dump, ' '); + + for (auto &table : tables) { + std::cout << "DROP TABLE `" << table << "`;" << std::endl; + } + + if (tables_to_dump.empty()) { + std::cerr << "No tables were specified" << std::endl; + } + } + else { + std::string execution_result = execute(execute_command, IsDumpOutputToConsole()); + if (!execution_result.empty()) { + std::cout << execution_result; + } } if (!tables_to_dump.empty()) { @@ -439,14 +456,15 @@ void DatabaseDumpService::SetDumpAllTables(bool dump_all_tables) { DatabaseDumpService::dump_all_tables = dump_all_tables; } -bool DatabaseDumpService::IsDumpNoTableLock() const + +bool DatabaseDumpService::IsDumpTableLock() const { - return dump_no_table_lock; + return dump_table_lock; } -void DatabaseDumpService::SetDumpNoTableLock(bool dump_no_table_lock) +void DatabaseDumpService::SetDumpTableLock(bool dump_table_lock) { - DatabaseDumpService::dump_no_table_lock = dump_no_table_lock; + DatabaseDumpService::dump_table_lock = dump_table_lock; } bool DatabaseDumpService::IsDumpWithCompression() const @@ -497,4 +515,14 @@ bool DatabaseDumpService::IsDumpOutputToConsole() const void DatabaseDumpService::SetDumpOutputToConsole(bool dump_output_to_console) { DatabaseDumpService::dump_output_to_console = dump_output_to_console; -} \ No newline at end of file +} + +bool DatabaseDumpService::IsDumpDropTableSyntaxOnly() const +{ + return dump_drop_table_syntax_only; +} + +void DatabaseDumpService::SetDumpDropTableSyntaxOnly(bool dump_drop_table_syntax_only) +{ + DatabaseDumpService::dump_drop_table_syntax_only = dump_drop_table_syntax_only; +} diff --git a/common/database/database_dump_service.h b/common/database/database_dump_service.h index eb87520da..20f57e85c 100644 --- a/common/database/database_dump_service.h +++ b/common/database/database_dump_service.h @@ -37,8 +37,8 @@ public: void SetDumpPlayerTables(bool dump_player_tables); bool IsDumpLoginServerTables() const; void SetDumpLoginServerTables(bool dump_login_server_tables); - bool IsDumpNoTableLock() const; - void SetDumpNoTableLock(bool dump_no_table_lock); + bool IsDumpTableLock() const; + void SetDumpTableLock(bool dump_table_lock); bool IsDumpWithCompression() const; void SetDumpWithCompression(bool dump_with_compression); const std::string &GetDumpPath() const; @@ -49,18 +49,21 @@ public: void SetDumpQueryServerTables(bool dump_query_server_tables); bool IsDumpOutputToConsole() const; void SetDumpOutputToConsole(bool dump_output_to_console); + bool IsDumpDropTableSyntaxOnly() const; + void SetDumpDropTableSyntaxOnly(bool dump_drop_table_syntax_only); private: - bool dump_all_tables = false; - bool dump_system_tables = false; - bool dump_content_tables = false; - bool dump_player_tables = false; - bool dump_query_server_tables = false; - bool dump_login_server_tables = false; - bool dump_with_no_data = false; - bool dump_no_table_lock = false; - bool dump_with_compression = false; - bool dump_output_to_console = false; + bool dump_all_tables = false; + bool dump_system_tables = false; + bool dump_content_tables = false; + bool dump_player_tables = false; + bool dump_query_server_tables = false; + bool dump_login_server_tables = false; + bool dump_with_no_data = false; + bool dump_table_lock = false; + bool dump_with_compression = false; + bool dump_output_to_console = false; + bool dump_drop_table_syntax_only = false; std::string dump_path; std::string dump_file_name; diff --git a/common/database_schema.h b/common/database_schema.h index 5502441d0..40c9c39f1 100644 --- a/common/database_schema.h +++ b/common/database_schema.h @@ -85,7 +85,8 @@ namespace DatabaseSchema { } /** - * Gets all player and meta-data tables + * @description Gets all player and meta-data tables + * @note These tables have no content in the PEQ daily dump * * @return */ @@ -98,6 +99,8 @@ namespace DatabaseSchema { "account_rewards", "adventure_details", "adventure_stats", + "bugs", + "bug_reports", "buyer", "char_recipe_list", "character_activities", @@ -129,6 +132,8 @@ namespace DatabaseSchema { "character_tribute", "completed_tasks", "data_buckets", + "discovered_items", + "eventlog", "faction_values", "friends", "guild_bank", @@ -141,9 +146,12 @@ namespace DatabaseSchema { "inventory_snapshots", "keyring", "mail", + "petitions", "player_titlesets", "quest_globals", + "saylink", "sharedbank", + "reports", "timers", "titles", "trader", @@ -255,13 +263,9 @@ namespace DatabaseSchema { { return { "banned_ips", - "bugs", - "bug_reports", "command_settings", "db_str", - "discovered_items", "eqtime", - "eventlog", "gm_ips", "hackers", "ip_exemptions", @@ -271,12 +275,9 @@ namespace DatabaseSchema { "logsys_categories", "name_filter", "perl_event_export_settings", - "petitions", "profanity_list", - "reports", "rule_sets", "rule_values", - "saylink", "variables", }; } diff --git a/utils/sql/peq-dump/peq-dump.sh b/utils/sql/peq-dump/peq-dump.sh new file mode 100755 index 000000000..afcb63212 --- /dev/null +++ b/utils/sql/peq-dump/peq-dump.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +world_path="" + +if [ -d "bin" ] +then + world_path="bin/" +fi + +world_bin="${world_path}world" + +echo "World path is [$world_path] bin is [$world_bin]" + +# Run from the context of server directory +# +# --content-tables +# --login-tables +# --player-tables +# --system-tables +# --query-serv-tables + +############################################# +# dump +############################################# + +dump_path=/tmp/peq-dump/ + +mkdir -p ${dump_path} + +############################################# +# generate "drop_" table files +############################################# +bash -c "${world_bin} database:dump --content-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_content.sql" +bash -c "${world_bin} database:dump --login-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_login.sql" +bash -c "${world_bin} database:dump --player-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_player.sql" +bash -c "${world_bin} database:dump --system-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_system.sql" +bash -c "${world_bin} database:dump --query-serv-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_queryserv.sql" + +############################################# +# generate "create_" table files +############################################# +bash -c "${world_bin} database:dump --content-tables --dump-output-to-console > ${dump_path}create_tables_content.sql" +bash -c "${world_bin} database:dump --login-tables --table-structure-only --dump-output-to-console > ${dump_path}create_tables_login.sql" +bash -c "${world_bin} database:dump --player-tables --table-structure-only --dump-output-to-console > ${dump_path}create_tables_player.sql" +bash -c "${world_bin} database:dump --system-tables --dump-output-to-console > ${dump_path}create_tables_system.sql" +bash -c "${world_bin} database:dump --query-serv-tables --table-structure-only --dump-output-to-console > ${dump_path}create_tables_queryserv.sql" + diff --git a/world/world_server_command_handler.cpp b/world/world_server_command_handler.cpp index 32e794800..5ccbf167c 100644 --- a/world/world_server_command_handler.cpp +++ b/world/world_server_command_handler.cpp @@ -227,9 +227,10 @@ namespace WorldserverCommandHandler { "--system-tables", "--query-serv-tables", "--table-structure-only", - "--no-table-lock", + "--table-lock", "--dump-path=", "--dump-output-to-console", + "--drop-table-syntax-only", "--compress" }; @@ -256,9 +257,10 @@ namespace WorldserverCommandHandler { database_dump_service->SetDumpWithNoData(cmd[{"-c", "--table-structure-only"}]); database_dump_service->SetDumpQueryServerTables(cmd[{"--query-serv-tables"}]); database_dump_service->SetDumpAllTables(dump_all); - database_dump_service->SetDumpNoTableLock(cmd[{"--no-table-lock"}]); + database_dump_service->SetDumpTableLock(cmd[{"--table-lock"}]); database_dump_service->SetDumpWithCompression(cmd[{"--compress"}]); database_dump_service->SetDumpOutputToConsole(cmd[{"--dump-output-to-console"}]); + database_dump_service->SetDumpDropTableSyntaxOnly(cmd[{"--drop-table-syntax-only"}]); /** * Dump From 899e2d339793dad50107c30e695728cbd874cace Mon Sep 17 00:00:00 2001 From: Akkadius Date: Mon, 9 Mar 2020 16:31:30 -0500 Subject: [PATCH 06/12] Add innodb conversion script; will hook up in manifest later --- .../2020_03_09_convert_myisam_to_innodb.sql | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 utils/sql/git/required/2020_03_09_convert_myisam_to_innodb.sql diff --git a/utils/sql/git/required/2020_03_09_convert_myisam_to_innodb.sql b/utils/sql/git/required/2020_03_09_convert_myisam_to_innodb.sql new file mode 100644 index 000000000..ec9f6607b --- /dev/null +++ b/utils/sql/git/required/2020_03_09_convert_myisam_to_innodb.sql @@ -0,0 +1,99 @@ +ALTER TABLE `aa_actions` ENGINE=InnoDB; +ALTER TABLE `aa_effects` ENGINE=InnoDB; +ALTER TABLE `aa_required_level_cost` ENGINE=InnoDB; +ALTER TABLE `aa_timers` ENGINE=InnoDB; +ALTER TABLE `account_flags` ENGINE=InnoDB; +ALTER TABLE `account_ip` ENGINE=InnoDB; +ALTER TABLE `account` ENGINE=InnoDB; +ALTER TABLE `adventure_template_entry_flavor` ENGINE=InnoDB; +ALTER TABLE `adventure_template_entry` ENGINE=InnoDB; +ALTER TABLE `altadv_vars` ENGINE=InnoDB; +ALTER TABLE `alternate_currency` ENGINE=InnoDB; +ALTER TABLE `banned_ips` ENGINE=InnoDB; +ALTER TABLE `base_data` ENGINE=InnoDB; +ALTER TABLE `blocked_spells` ENGINE=InnoDB; +ALTER TABLE `buyer` ENGINE=InnoDB; +ALTER TABLE `char_create_combinations` ENGINE=InnoDB; +ALTER TABLE `char_create_point_allocations` ENGINE=InnoDB; +ALTER TABLE `character_activities` ENGINE=InnoDB; +ALTER TABLE `character_enabledtasks` ENGINE=InnoDB; +ALTER TABLE `character_tasks` ENGINE=InnoDB; +ALTER TABLE `chatchannels` ENGINE=InnoDB; +ALTER TABLE `completed_tasks` ENGINE=InnoDB; +ALTER TABLE `damageshieldtypes` ENGINE=InnoDB; +ALTER TABLE `discovered_items` ENGINE=InnoDB; +ALTER TABLE `eqtime` ENGINE=InnoDB; +ALTER TABLE `eventlog` ENGINE=InnoDB; +ALTER TABLE `faction_list_mod` ENGINE=InnoDB; +ALTER TABLE `faction_list` ENGINE=InnoDB; +ALTER TABLE `faction_values` ENGINE=InnoDB; +ALTER TABLE `friends` ENGINE=InnoDB; +ALTER TABLE `goallists` ENGINE=InnoDB; +ALTER TABLE `guild_bank` ENGINE=InnoDB; +ALTER TABLE `guild_members` ENGINE=InnoDB; +ALTER TABLE `guild_ranks` ENGINE=InnoDB; +ALTER TABLE `guild_relations` ENGINE=InnoDB; +ALTER TABLE `guilds` ENGINE=InnoDB; +ALTER TABLE `hackers` ENGINE=InnoDB; +ALTER TABLE `horses` ENGINE=InnoDB; +ALTER TABLE `inventory_versions` ENGINE=InnoDB; +ALTER TABLE `item_tick` ENGINE=InnoDB; +ALTER TABLE `items` ENGINE=InnoDB; +ALTER TABLE `keyring` ENGINE=InnoDB; +ALTER TABLE `launcher_zones` ENGINE=InnoDB; +ALTER TABLE `launcher` ENGINE=InnoDB; +ALTER TABLE `ldon_trap_entries` ENGINE=InnoDB; +ALTER TABLE `ldon_trap_templates` ENGINE=InnoDB; +ALTER TABLE `lfguild` ENGINE=InnoDB; +ALTER TABLE `lootdrop_entries` ENGINE=InnoDB; +ALTER TABLE `lootdrop` ENGINE=InnoDB; +ALTER TABLE `loottable_entries` ENGINE=InnoDB; +ALTER TABLE `loottable` ENGINE=InnoDB; +ALTER TABLE `mail` ENGINE=InnoDB; +ALTER TABLE `merc_armorinfo` ENGINE=InnoDB; +ALTER TABLE `merc_buffs` ENGINE=InnoDB; +ALTER TABLE `merc_inventory` ENGINE=InnoDB; +ALTER TABLE `merc_merchant_entries` ENGINE=InnoDB; +ALTER TABLE `merc_merchant_template_entries` ENGINE=InnoDB; +ALTER TABLE `merc_merchant_templates` ENGINE=InnoDB; +ALTER TABLE `merc_name_types` ENGINE=InnoDB; +ALTER TABLE `merc_npc_types` ENGINE=InnoDB; +ALTER TABLE `merc_spell_list_entries` ENGINE=InnoDB; +ALTER TABLE `merc_spell_lists` ENGINE=InnoDB; +ALTER TABLE `merc_stance_entries` ENGINE=InnoDB; +ALTER TABLE `merc_stats` ENGINE=InnoDB; +ALTER TABLE `merc_subtypes` ENGINE=InnoDB; +ALTER TABLE `merc_templates` ENGINE=InnoDB; +ALTER TABLE `merc_types` ENGINE=InnoDB; +ALTER TABLE `merc_weaponinfo` ENGINE=InnoDB; +ALTER TABLE `mercs` ENGINE=InnoDB; +ALTER TABLE `name_filter` ENGINE=InnoDB; +ALTER TABLE `npc_types` ENGINE=InnoDB; +ALTER TABLE `object_contents` ENGINE=InnoDB; +ALTER TABLE `petitions` ENGINE=InnoDB; +ALTER TABLE `pets_equipmentset_entries` ENGINE=InnoDB; +ALTER TABLE `pets_equipmentset` ENGINE=InnoDB; +ALTER TABLE `player_titlesets` ENGINE=InnoDB; +ALTER TABLE `proximities` ENGINE=InnoDB; +ALTER TABLE `races` ENGINE=InnoDB; +ALTER TABLE `raid_details` ENGINE=InnoDB; +ALTER TABLE `raid_leaders` ENGINE=InnoDB; +ALTER TABLE `raid_members` ENGINE=InnoDB; +ALTER TABLE `rule_sets` ENGINE=InnoDB; +ALTER TABLE `rule_values` ENGINE=InnoDB; +ALTER TABLE `saylink` ENGINE=InnoDB; +ALTER TABLE `sharedbank` ENGINE=InnoDB; +ALTER TABLE `skill_caps` ENGINE=InnoDB; +ALTER TABLE `spell_globals` ENGINE=InnoDB; +ALTER TABLE `spells_new` ENGINE=InnoDB; +ALTER TABLE `task_activities` ENGINE=InnoDB; +ALTER TABLE `tasks` ENGINE=InnoDB; +ALTER TABLE `tasksets` ENGINE=InnoDB; +ALTER TABLE `timers` ENGINE=InnoDB; +ALTER TABLE `titles` ENGINE=InnoDB; +ALTER TABLE `trader_audit` ENGINE=InnoDB; +ALTER TABLE `trader` ENGINE=InnoDB; +ALTER TABLE `tradeskill_recipe_entries` ENGINE=InnoDB; +ALTER TABLE `tradeskill_recipe` ENGINE=InnoDB; +ALTER TABLE `variables` ENGINE=InnoDB; +ALTER TABLE `veteran_reward_templates` ENGINE=InnoDB; \ No newline at end of file From 6438a37fb5869c362d08d4404da3b1f8dd2d983a Mon Sep 17 00:00:00 2001 From: Akkadius Date: Mon, 9 Mar 2020 16:31:43 -0500 Subject: [PATCH 07/12] Split up state table dump --- common/database/database_dump_service.cpp | 28 +++++++++++++++++++++-- common/database/database_dump_service.h | 26 ++++++++++++--------- common/database_schema.h | 17 +++++++------- utils/sql/peq-dump/peq-dump.sh | 12 ++++++---- world/world_server_command_handler.cpp | 15 +++++++----- 5 files changed, 66 insertions(+), 32 deletions(-) diff --git a/common/database/database_dump_service.cpp b/common/database/database_dump_service.cpp index 7610f3759..5d6da79e9 100644 --- a/common/database/database_dump_service.cpp +++ b/common/database/database_dump_service.cpp @@ -188,12 +188,21 @@ std::string DatabaseDumpService::GetSystemTablesList() tables_list += table + " "; } - tables = DatabaseSchema::GetStateTables(); + tables = DatabaseSchema::GetVersionTables(); for (const auto &table : tables) { tables_list += table + " "; } - tables = DatabaseSchema::GetVersionTables(); + return trim(tables_list); +} +/** + * @return + */ +std::string DatabaseDumpService::GetStateTablesList() +{ + std::string tables_list; + + std::vector tables = DatabaseSchema::GetStateTables(); for (const auto &table : tables) { tables_list += table + " "; } @@ -300,6 +309,11 @@ void DatabaseDumpService::Dump() dump_descriptor += "-system"; } + if (IsDumpStateTables()) { + tables_to_dump += GetStateTablesList() + " "; + dump_descriptor += "-state"; + } + if (IsDumpContentTables()) { tables_to_dump += GetContentTablesList() + " "; dump_descriptor += "-content"; @@ -526,3 +540,13 @@ void DatabaseDumpService::SetDumpDropTableSyntaxOnly(bool dump_drop_table_syntax { DatabaseDumpService::dump_drop_table_syntax_only = dump_drop_table_syntax_only; } + +bool DatabaseDumpService::IsDumpStateTables() const +{ + return dump_state_tables; +} + +void DatabaseDumpService::SetDumpStateTables(bool dump_state_tables) +{ + DatabaseDumpService::dump_state_tables = dump_state_tables; +} diff --git a/common/database/database_dump_service.h b/common/database/database_dump_service.h index 20f57e85c..439a380ba 100644 --- a/common/database/database_dump_service.h +++ b/common/database/database_dump_service.h @@ -51,19 +51,22 @@ public: void SetDumpOutputToConsole(bool dump_output_to_console); bool IsDumpDropTableSyntaxOnly() const; void SetDumpDropTableSyntaxOnly(bool dump_drop_table_syntax_only); + bool IsDumpStateTables() const; + void SetDumpStateTables(bool dump_state_tables); private: - bool dump_all_tables = false; - bool dump_system_tables = false; - bool dump_content_tables = false; - bool dump_player_tables = false; - bool dump_query_server_tables = false; - bool dump_login_server_tables = false; - bool dump_with_no_data = false; - bool dump_table_lock = false; - bool dump_with_compression = false; - bool dump_output_to_console = false; - bool dump_drop_table_syntax_only = false; + bool dump_all_tables = false; + bool dump_state_tables = false; + bool dump_system_tables = false; + bool dump_content_tables = false; + bool dump_player_tables = false; + bool dump_query_server_tables = false; + bool dump_login_server_tables = false; + bool dump_with_no_data = false; + bool dump_table_lock = false; + bool dump_with_compression = false; + bool dump_output_to_console = false; + bool dump_drop_table_syntax_only = false; std::string dump_path; std::string dump_file_name; @@ -73,6 +76,7 @@ private: std::string GetBaseMySQLDumpCommand(); std::string GetPlayerTablesList(); std::string GetSystemTablesList(); + std::string GetStateTablesList(); std::string GetContentTablesList(); std::string GetLoginTableList(); bool IsTarAvailable(); diff --git a/common/database_schema.h b/common/database_schema.h index 40c9c39f1..75856eb9f 100644 --- a/common/database_schema.h +++ b/common/database_schema.h @@ -99,8 +99,6 @@ namespace DatabaseSchema { "account_rewards", "adventure_details", "adventure_stats", - "bugs", - "bug_reports", "buyer", "char_recipe_list", "character_activities", @@ -133,7 +131,6 @@ namespace DatabaseSchema { "completed_tasks", "data_buckets", "discovered_items", - "eventlog", "faction_values", "friends", "guild_bank", @@ -149,9 +146,9 @@ namespace DatabaseSchema { "petitions", "player_titlesets", "quest_globals", - "saylink", "sharedbank", - "reports", + "spell_buckets", + "spell_globals", "timers", "titles", "trader", @@ -263,6 +260,7 @@ namespace DatabaseSchema { { return { "banned_ips", + "chatchannels", "command_settings", "db_str", "eqtime", @@ -318,7 +316,9 @@ namespace DatabaseSchema { { return { "adventure_members", - "chatchannels", + "bugs", + "bug_reports", + "eventlog", "group_id", "group_leaders", "item_tick", @@ -329,8 +329,9 @@ namespace DatabaseSchema { "raid_leaders", "raid_members", "respawn_times", - "spell_buckets", - "spell_globals", + "reports", + "saylink", + }; } diff --git a/utils/sql/peq-dump/peq-dump.sh b/utils/sql/peq-dump/peq-dump.sh index afcb63212..06300fbd7 100755 --- a/utils/sql/peq-dump/peq-dump.sh +++ b/utils/sql/peq-dump/peq-dump.sh @@ -34,14 +34,16 @@ bash -c "${world_bin} database:dump --content-tables --drop-table-syntax-only -- bash -c "${world_bin} database:dump --login-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_login.sql" bash -c "${world_bin} database:dump --player-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_player.sql" bash -c "${world_bin} database:dump --system-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_system.sql" +bash -c "${world_bin} database:dump --state-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_state.sql" bash -c "${world_bin} database:dump --query-serv-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_queryserv.sql" ############################################# # generate "create_" table files ############################################# -bash -c "${world_bin} database:dump --content-tables --dump-output-to-console > ${dump_path}create_tables_content.sql" -bash -c "${world_bin} database:dump --login-tables --table-structure-only --dump-output-to-console > ${dump_path}create_tables_login.sql" -bash -c "${world_bin} database:dump --player-tables --table-structure-only --dump-output-to-console > ${dump_path}create_tables_player.sql" -bash -c "${world_bin} database:dump --system-tables --dump-output-to-console > ${dump_path}create_tables_system.sql" -bash -c "${world_bin} database:dump --query-serv-tables --table-structure-only --dump-output-to-console > ${dump_path}create_tables_queryserv.sql" +bash -c "${world_bin} database:dump --login-tables --table-structure-only --dump-output-to-console | sed 's/ AUTO_INCREMENT=[0-9]*\b//g' > ${dump_path}create_tables_login.sql" +bash -c "${world_bin} database:dump --player-tables --table-structure-only --dump-output-to-console | sed 's/ AUTO_INCREMENT=[0-9]*\b//g' > ${dump_path}create_tables_player.sql" +bash -c "${world_bin} database:dump --state-tables --table-structure-only --dump-output-to-console | sed 's/ AUTO_INCREMENT=[0-9]*\b//g' > ${dump_path}create_tables_state.sql" +bash -c "${world_bin} database:dump --query-serv-tables --table-structure-only --dump-output-to-console | sed 's/ AUTO_INCREMENT=[0-9]*\b//g' > ${dump_path}create_tables_queryserv.sql" +bash -c "${world_bin} database:dump --content-tables --dump-output-to-console > ${dump_path}create_tables_content.sql" +bash -c "${world_bin} database:dump --system-tables --dump-output-to-console > ${dump_path}create_tables_system.sql" \ No newline at end of file diff --git a/world/world_server_command_handler.cpp b/world/world_server_command_handler.cpp index 5ccbf167c..9bad28541 100644 --- a/world/world_server_command_handler.cpp +++ b/world/world_server_command_handler.cpp @@ -224,6 +224,7 @@ namespace WorldserverCommandHandler { "--content-tables", "--login-tables", "--player-tables", + "--state-tables", "--system-tables", "--query-serv-tables", "--table-structure-only", @@ -250,13 +251,15 @@ namespace WorldserverCommandHandler { /** * Set Option */ - database_dump_service->SetDumpContentTables(cmd[{"-c", "--content-tables"}] || dump_all); - database_dump_service->SetDumpLoginServerTables(cmd[{"-c", "--login-tables"}] || dump_all); - database_dump_service->SetDumpPlayerTables(cmd[{"-c", "--player-tables"}] || dump_all); - database_dump_service->SetDumpSystemTables(cmd[{"-c", "--system-tables"}] || dump_all); - database_dump_service->SetDumpWithNoData(cmd[{"-c", "--table-structure-only"}]); - database_dump_service->SetDumpQueryServerTables(cmd[{"--query-serv-tables"}]); + database_dump_service->SetDumpContentTables(cmd[{"--content-tables"}] || dump_all); + database_dump_service->SetDumpLoginServerTables(cmd[{"--login-tables"}] || dump_all); + database_dump_service->SetDumpPlayerTables(cmd[{"--player-tables"}] || dump_all); + database_dump_service->SetDumpStateTables(cmd[{"--state-tables"}] || dump_all); + database_dump_service->SetDumpSystemTables(cmd[{"--system-tables"}] || dump_all); + database_dump_service->SetDumpQueryServerTables(cmd[{"--query-serv-tables"}] || dump_all); database_dump_service->SetDumpAllTables(dump_all); + + database_dump_service->SetDumpWithNoData(cmd[{"--table-structure-only"}]); database_dump_service->SetDumpTableLock(cmd[{"--table-lock"}]); database_dump_service->SetDumpWithCompression(cmd[{"--compress"}]); database_dump_service->SetDumpOutputToConsole(cmd[{"--dump-output-to-console"}]); From e7fab67d8ab0ab69a9be45a10e0000e341633ecb Mon Sep 17 00:00:00 2001 From: Akkadius Date: Mon, 9 Mar 2020 22:57:27 -0500 Subject: [PATCH 08/12] Finalize peq-dump.sh script [skip ci] --- utils/sql/peq-dump/peq-dump.sh | 36 +++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/utils/sql/peq-dump/peq-dump.sh b/utils/sql/peq-dump/peq-dump.sh index 06300fbd7..f60ed02d0 100755 --- a/utils/sql/peq-dump/peq-dump.sh +++ b/utils/sql/peq-dump/peq-dump.sh @@ -1,7 +1,11 @@ #!/usr/bin/env bash +# Run from the context of server directory world_path="" +############################################# +# world path +############################################# if [ -d "bin" ] then world_path="bin/" @@ -11,25 +15,18 @@ world_bin="${world_path}world" echo "World path is [$world_path] bin is [$world_bin]" -# Run from the context of server directory -# -# --content-tables -# --login-tables -# --player-tables -# --system-tables -# --query-serv-tables - ############################################# # dump ############################################# dump_path=/tmp/peq-dump/ - -mkdir -p ${dump_path} +echo "Generating dump path [${dump_path}]" +rm -rf ${dump_path} && mkdir -p ${dump_path} ############################################# # generate "drop_" table files ############################################# +echo "Generating [drop_*] table exports..." bash -c "${world_bin} database:dump --content-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_content.sql" bash -c "${world_bin} database:dump --login-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_login.sql" bash -c "${world_bin} database:dump --player-tables --drop-table-syntax-only --dump-output-to-console > ${dump_path}drop_tables_player.sql" @@ -40,10 +37,27 @@ bash -c "${world_bin} database:dump --query-serv-tables --drop-table-syntax-only ############################################# # generate "create_" table files ############################################# +echo "Generating [create_*] table exports..." + +# structure only bash -c "${world_bin} database:dump --login-tables --table-structure-only --dump-output-to-console | sed 's/ AUTO_INCREMENT=[0-9]*\b//g' > ${dump_path}create_tables_login.sql" bash -c "${world_bin} database:dump --player-tables --table-structure-only --dump-output-to-console | sed 's/ AUTO_INCREMENT=[0-9]*\b//g' > ${dump_path}create_tables_player.sql" bash -c "${world_bin} database:dump --state-tables --table-structure-only --dump-output-to-console | sed 's/ AUTO_INCREMENT=[0-9]*\b//g' > ${dump_path}create_tables_state.sql" bash -c "${world_bin} database:dump --query-serv-tables --table-structure-only --dump-output-to-console | sed 's/ AUTO_INCREMENT=[0-9]*\b//g' > ${dump_path}create_tables_queryserv.sql" +# with content bash -c "${world_bin} database:dump --content-tables --dump-output-to-console > ${dump_path}create_tables_content.sql" -bash -c "${world_bin} database:dump --system-tables --dump-output-to-console > ${dump_path}create_tables_system.sql" \ No newline at end of file +bash -c "${world_bin} database:dump --system-tables --dump-output-to-console > ${dump_path}create_tables_system.sql" + +############################################# +# zip +############################################# +human_date=$(date +"%B-%d-%Y" | tr '[:upper:]' '[:lower:]') + +echo "Compressing..." +bash -c "cd ${dump_path} && zip peq-latest.zip * && mv ${dump_path}peq-latest.zip /tmp/peq-latest.zip" + +echo "Cleaning up..." +rm -rf ${dump_path} + +echo "Dump located [/tmp/peq-latest.zip]" \ No newline at end of file From fa12b146a3e13fdbe11d09b7717bd57f57fc9156 Mon Sep 17 00:00:00 2001 From: Akkadius Date: Mon, 9 Mar 2020 23:05:01 -0500 Subject: [PATCH 09/12] Add "all" .sql files [skip ci] --- utils/sql/peq-dump/peq-dump.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/sql/peq-dump/peq-dump.sh b/utils/sql/peq-dump/peq-dump.sh index f60ed02d0..7fd9bf278 100755 --- a/utils/sql/peq-dump/peq-dump.sh +++ b/utils/sql/peq-dump/peq-dump.sh @@ -49,6 +49,13 @@ bash -c "${world_bin} database:dump --query-serv-tables --table-structure-only - bash -c "${world_bin} database:dump --content-tables --dump-output-to-console > ${dump_path}create_tables_content.sql" bash -c "${world_bin} database:dump --system-tables --dump-output-to-console > ${dump_path}create_tables_system.sql" +############################################# +# "all" exports +############################################# + +bash -c "cd ${dump_path} && ls * | grep create | sed 's/.*/source &;/' > create_all_tables.sql" +bash -c "cd ${dump_path} && ls * | grep drop | sed 's/.*/source &;/' > drop_all_tables.sql" + ############################################# # zip ############################################# From c42d6dcd1b0833fb8cf6fb57f6facb5b583b2dc7 Mon Sep 17 00:00:00 2001 From: Akkadius Date: Tue, 10 Mar 2020 00:14:28 -0500 Subject: [PATCH 10/12] Add 7zip compression; tweak dump settings --- common/database/database_dump_service.cpp | 26 +++++++++++++++++------ common/database/database_dump_service.h | 2 +- utils/sql/peq-dump/peq-dump.sh | 3 +-- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/common/database/database_dump_service.cpp b/common/database/database_dump_service.cpp index 5d6da79e9..e98a8d830 100644 --- a/common/database/database_dump_service.cpp +++ b/common/database/database_dump_service.cpp @@ -90,14 +90,14 @@ bool DatabaseDumpService::IsTarAvailable() } /** - * Windows TODO + * Windows * @return bool */ -bool DatabaseDumpService::IsRarAvailable() +bool DatabaseDumpService::Is7ZipAvailable() { std::string version_output = execute("winrar --version"); - return version_output.find("todo") != std::string::npos; + return version_output.find("7-Zip") != std::string::npos; } /** @@ -105,7 +105,7 @@ bool DatabaseDumpService::IsRarAvailable() */ bool DatabaseDumpService::HasCompressionBinary() { - return IsTarAvailable() || IsRarAvailable(); + return IsTarAvailable() || Is7ZipAvailable(); } /** @@ -285,7 +285,7 @@ void DatabaseDumpService::Dump() config->DatabaseUsername ); - std::string options = "--compact --allow-keywords --extended-insert"; + std::string options = "--allow-keywords --extended-insert"; if (IsDumpWithNoData()) { options += " --no-data"; @@ -358,7 +358,7 @@ void DatabaseDumpService::Dump() std::vector tables = SplitString(tables_to_dump, ' '); for (auto &table : tables) { - std::cout << "DROP TABLE `" << table << "`;" << std::endl; + std::cout << "DROP TABLE IF EXISTS `" << table << "`;" << std::endl; } if (tables_to_dump.empty()) { @@ -393,6 +393,20 @@ void DatabaseDumpService::Dump() ); LogInfo("Compressed dump created at [{}.tar.gz]", GetDumpFileNameWithPath()); } + else if (Is7ZipAvailable()) { + execute( + fmt::format( + "7z a -t7z {}.zip -C {} {}.sql", + GetDumpFileNameWithPath(), + GetSetDumpPath(), + GetDumpFileNameWithPath() + ) + ); + LogInfo("Compressed dump created at [{}.zip]", GetDumpFileNameWithPath()); + } + else { + LogInfo("Compression requested, but no available compression binary was found"); + } } else { LogWarning("Compression requested but binary not found... Skipping..."); diff --git a/common/database/database_dump_service.h b/common/database/database_dump_service.h index 439a380ba..f0614e85b 100644 --- a/common/database/database_dump_service.h +++ b/common/database/database_dump_service.h @@ -80,7 +80,7 @@ private: std::string GetContentTablesList(); std::string GetLoginTableList(); bool IsTarAvailable(); - bool IsRarAvailable(); + bool Is7ZipAvailable(); bool HasCompressionBinary(); std::string GetDumpFileNameWithPath(); std::string GetSetDumpPath(); diff --git a/utils/sql/peq-dump/peq-dump.sh b/utils/sql/peq-dump/peq-dump.sh index 7fd9bf278..911439c8f 100755 --- a/utils/sql/peq-dump/peq-dump.sh +++ b/utils/sql/peq-dump/peq-dump.sh @@ -52,7 +52,6 @@ bash -c "${world_bin} database:dump --system-tables --dump-output-to-console > $ ############################################# # "all" exports ############################################# - bash -c "cd ${dump_path} && ls * | grep create | sed 's/.*/source &;/' > create_all_tables.sql" bash -c "cd ${dump_path} && ls * | grep drop | sed 's/.*/source &;/' > drop_all_tables.sql" @@ -62,7 +61,7 @@ bash -c "cd ${dump_path} && ls * | grep drop | sed 's/.*/source &;/' > drop_all_ human_date=$(date +"%B-%d-%Y" | tr '[:upper:]' '[:lower:]') echo "Compressing..." -bash -c "cd ${dump_path} && zip peq-latest.zip * && mv ${dump_path}peq-latest.zip /tmp/peq-latest.zip" +bash -c "cd /tmp/ && rm peq-latest.zip && zip peq-latest.zip peq-dump/* && mv ${dump_path}peq-latest.zip /tmp/peq-latest.zip" echo "Cleaning up..." rm -rf ${dump_path} From 431a325414dc4702bc1d8c5eec0e97f60ebfce6d Mon Sep 17 00:00:00 2001 From: Akkadius Date: Tue, 10 Mar 2020 00:22:39 -0500 Subject: [PATCH 11/12] Update manifest [skip ci] --- common/version.h | 2 +- utils/sql/db_update_manifest.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/common/version.h b/common/version.h index 359c27576..99261fe83 100644 --- a/common/version.h +++ b/common/version.h @@ -34,7 +34,7 @@ * Manifest: https://github.com/EQEmu/Server/blob/master/utils/sql/db_update_manifest.txt */ -#define CURRENT_BINARY_DATABASE_VERSION 9151 +#define CURRENT_BINARY_DATABASE_VERSION 9152 #ifdef BOTS #define CURRENT_BINARY_BOTS_DATABASE_VERSION 9026 diff --git a/utils/sql/db_update_manifest.txt b/utils/sql/db_update_manifest.txt index 9be222856..4113a838a 100644 --- a/utils/sql/db_update_manifest.txt +++ b/utils/sql/db_update_manifest.txt @@ -405,6 +405,7 @@ 9149|2020_02_06_globalloot.sql|SHOW COLUMNS FROM `global_loot` LIKE 'hot_zone'|empty| 9150|2020_02_06_aa_reset_on_death.sql|SHOW COLUMNS FROM `aa_ability` LIKE 'reset_on_death'|empty| 9151|2020_03_05_npc_always_aggro.sql|SHOW COLUMNS FROM `npc_types` LIKE 'always_aggro'|empty| +9152|2020_03_09_convert_myisam_to_innodb.sql|SELECT * FROM db_version WHERE version >= 9151|empty| # Upgrade conditions: # This won't be needed after this system is implemented, but it is used database that are not From f3d82710669896e5360deb5216fb7c6c2b958db7 Mon Sep 17 00:00:00 2001 From: Akkadius Date: Tue, 10 Mar 2020 13:41:53 -0500 Subject: [PATCH 12/12] Slightly adjust manifest criteria [skip ci] --- utils/sql/db_update_manifest.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/sql/db_update_manifest.txt b/utils/sql/db_update_manifest.txt index 4113a838a..28b22028e 100644 --- a/utils/sql/db_update_manifest.txt +++ b/utils/sql/db_update_manifest.txt @@ -405,7 +405,7 @@ 9149|2020_02_06_globalloot.sql|SHOW COLUMNS FROM `global_loot` LIKE 'hot_zone'|empty| 9150|2020_02_06_aa_reset_on_death.sql|SHOW COLUMNS FROM `aa_ability` LIKE 'reset_on_death'|empty| 9151|2020_03_05_npc_always_aggro.sql|SHOW COLUMNS FROM `npc_types` LIKE 'always_aggro'|empty| -9152|2020_03_09_convert_myisam_to_innodb.sql|SELECT * FROM db_version WHERE version >= 9151|empty| +9152|2020_03_09_convert_myisam_to_innodb.sql|SELECT * FROM db_version WHERE version >= 9152|empty| # Upgrade conditions: # This won't be needed after this system is implemented, but it is used database that are not