From bcedfe7032531fbd4562596b1733c5fefba34aac Mon Sep 17 00:00:00 2001 From: hg <4683435+hgtw@users.noreply.github.com> Date: Tue, 12 Nov 2024 21:01:18 -0500 Subject: [PATCH] [Quest API] Add Native Database Querying Interface (#4531) * Add database quest API API functions are named to be similar to LuaSQL and perl DBI New connections are made for Database objects. These can either use credentials from the server eqemu_config or manual connections. * Add option to use zone db connections --- zone/CMakeLists.txt | 6 + zone/embparser.cpp | 2 + zone/embperl.h | 2 + zone/lua_database.cpp | 214 ++++++++++++++++++++++++++++++++++ zone/lua_database.h | 51 +++++++++ zone/lua_parser.cpp | 4 +- zone/perl_database.cpp | 255 +++++++++++++++++++++++++++++++++++++++++ zone/perl_database.h | 50 ++++++++ zone/quest_db.cpp | 57 +++++++++ zone/quest_db.h | 30 +++++ 10 files changed, 670 insertions(+), 1 deletion(-) create mode 100644 zone/lua_database.cpp create mode 100644 zone/lua_database.h create mode 100644 zone/perl_database.cpp create mode 100644 zone/perl_database.h create mode 100644 zone/quest_db.cpp create mode 100644 zone/quest_db.h diff --git a/zone/CMakeLists.txt b/zone/CMakeLists.txt index 238dcf9f0..1ead62e1b 100644 --- a/zone/CMakeLists.txt +++ b/zone/CMakeLists.txt @@ -54,6 +54,7 @@ SET(zone_sources lua_buff.cpp lua_corpse.cpp lua_client.cpp + lua_database.cpp lua_door.cpp lua_encounter.cpp lua_entity.cpp @@ -110,6 +111,7 @@ SET(zone_sources perl_bot.cpp perl_buff.cpp perl_client.cpp + perl_database.cpp perl_doors.cpp perl_entity.cpp perl_expedition.cpp @@ -135,6 +137,7 @@ SET(zone_sources qglobals.cpp queryserv.cpp questmgr.cpp + quest_db.cpp quest_parser_collection.cpp raids.cpp raycast_mesh.cpp @@ -215,6 +218,7 @@ SET(zone_headers lua_buff.h lua_client.h lua_corpse.h + lua_database.h lua_door.h lua_encounter.h lua_entity.h @@ -251,6 +255,7 @@ SET(zone_headers pathfinder_interface.h pathfinder_nav_mesh.h pathfinder_null.h + perl_database.h perlpacket.h petitions.h pets.h @@ -260,6 +265,7 @@ SET(zone_headers queryserv.h quest_interface.h questmgr.h + quest_db.h quest_parser_collection.h raids.h raycast_mesh.h diff --git a/zone/embparser.cpp b/zone/embparser.cpp index 8d4145c67..113374e81 100644 --- a/zone/embparser.cpp +++ b/zone/embparser.cpp @@ -58,6 +58,7 @@ void perl_register_expedition_lock_messages(); void perl_register_bot(); void perl_register_buff(); void perl_register_merc(); +void perl_register_database(); #endif // EMBPERL_XS_CLASSES #endif // EMBPERL_XS @@ -1185,6 +1186,7 @@ void PerlembParser::MapFunctions() perl_register_bot(); perl_register_buff(); perl_register_merc(); + perl_register_database(); #endif // EMBPERL_XS_CLASSES } diff --git a/zone/embperl.h b/zone/embperl.h index 9fe757184..022971059 100644 --- a/zone/embperl.h +++ b/zone/embperl.h @@ -21,6 +21,8 @@ Eglin #include namespace perl = perlbind; +#undef connect +#undef bind #undef Null #ifdef WIN32 diff --git a/zone/lua_database.cpp b/zone/lua_database.cpp new file mode 100644 index 000000000..aa2a5bea4 --- /dev/null +++ b/zone/lua_database.cpp @@ -0,0 +1,214 @@ +#ifdef LUA_EQEMU + +#include "lua_database.h" +#include "zonedb.h" +#include +#include + +// Luabind adopts the PreparedStmt wrapper object allocated with new and deletes it via GC +// Lua GC is non-deterministic so handles should be closed explicitly to free db resources +// Script errors/exceptions will hold resources until GC deletes the wrapper object + +Lua_MySQLPreparedStmt* Lua_Database::Prepare(lua_State* L, std::string query) +{ + return m_db ? new Lua_MySQLPreparedStmt(L, m_db->Prepare(std::move(query))) : nullptr; +} + +void Lua_Database::Close() +{ + m_db.reset(); +} + +// --------------------------------------------------------------------------- + +void Lua_MySQLPreparedStmt::Close() +{ + m_stmt.reset(); +} + +void Lua_MySQLPreparedStmt::Execute(lua_State* L) +{ + if (m_stmt) + { + m_res = m_stmt->Execute(); + } +} + +void Lua_MySQLPreparedStmt::Execute(lua_State* L, luabind::object args) +{ + if (m_stmt) + { + std::vector inputs; + + // iterate table until nil like ipairs to guarantee traversal order + for (int i = 1, type; (type = luabind::type(args[i])) != LUA_TNIL; ++i) + { + switch (type) + { + case LUA_TBOOLEAN: + inputs.emplace_back(luabind::object_cast(args[i])); + break; + case LUA_TNUMBER: // all numbers are doubles in lua before 5.3 + inputs.emplace_back(luabind::object_cast(args[i])); + break; + case LUA_TSTRING: + inputs.emplace_back(luabind::object_cast(args[i])); + break; + case LUA_TTABLE: // let tables substitute for null since nils can't exist + inputs.emplace_back(nullptr); + break; + default: + break; + } + } + + m_res = m_stmt->Execute(inputs); + } +} + +void Lua_MySQLPreparedStmt::SetOptions(luabind::object table) +{ + if (m_stmt) + { + mysql::StmtOptions opts = m_stmt->GetOptions(); + if (luabind::type(table["buffer_results"]) == LUA_TBOOLEAN) + { + opts.buffer_results = luabind::object_cast(table["buffer_results"]); + } + if (luabind::type(table["use_max_length"]) == LUA_TBOOLEAN) + { + opts.use_max_length = luabind::object_cast(table["use_max_length"]); + } + m_stmt->SetOptions(opts); + } +} + +static void PushValue(lua_State* L, const mysql::StmtColumn& col) +{ + if (col.IsNull()) + { + lua_pushnil(L); // clear entry in cache from any previous row + return; + } + + // 64-bit ints are pushed as strings since lua 5.1 only has 53-bit precision + switch (col.Type()) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + lua_pushnumber(L, col.Get().value()); + break; + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_BIT: + case MYSQL_TYPE_TIME: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_TIMESTAMP: + { + std::string str = col.GetStr().value(); + lua_pushlstring(L, str.data(), str.size()); + } + break; + default: // string types, push raw buffer to avoid copy + { + std::string_view str = col.GetStrView().value(); + lua_pushlstring(L, str.data(), str.size()); + } + break; + } +} + +luabind::object Lua_MySQLPreparedStmt::FetchArray(lua_State* L) +{ + auto row = m_stmt ? m_stmt->Fetch() : mysql::StmtRow(); + if (!row) + { + return luabind::object(); + } + + // perf: bypass luabind operator[] + m_row_array.push(L); + for (const mysql::StmtColumn& col : row) + { + PushValue(L, col); + lua_rawseti(L, -2, col.Index() + 1); + } + lua_pop(L, 1); + + return m_row_array; +} + +luabind::object Lua_MySQLPreparedStmt::FetchHash(lua_State* L) +{ + auto row = m_stmt ? m_stmt->Fetch() : mysql::StmtRow(); + if (!row) + { + return luabind::object(); + } + + // perf: bypass luabind operator[] + m_row_hash.push(L); + for (const mysql::StmtColumn& col : row) + { + PushValue(L, col); + lua_setfield(L, -2, col.Name().c_str()); + } + lua_pop(L, 1); + + return m_row_hash; +} + +int Lua_MySQLPreparedStmt::ColumnCount() +{ + return m_res.ColumnCount(); +} + +uint64_t Lua_MySQLPreparedStmt::LastInsertID() +{ + return m_res.LastInsertID(); +} + +uint64_t Lua_MySQLPreparedStmt::RowCount() +{ + return m_res.RowCount(); +} + +uint64_t Lua_MySQLPreparedStmt::RowsAffected() +{ + return m_res.RowsAffected(); +} + +luabind::scope lua_register_database() +{ + return luabind::class_("Database") + .enum_("constants") + [( + luabind::value("Default", static_cast(QuestDB::Connection::Default)), + luabind::value("Content", static_cast(QuestDB::Connection::Content)) + )] + .def(luabind::constructor<>()) + .def(luabind::constructor()) + .def(luabind::constructor()) + .def(luabind::constructor()) + .def("close", &Lua_Database::Close) + .def("prepare", &Lua_Database::Prepare, luabind::adopt(luabind::result)), + + luabind::class_("MySQLPreparedStmt") + .def("close", &Lua_MySQLPreparedStmt::Close) + .def("execute", static_cast(&Lua_MySQLPreparedStmt::Execute)) + .def("execute", static_cast(&Lua_MySQLPreparedStmt::Execute)) + .def("fetch", &Lua_MySQLPreparedStmt::FetchArray) + .def("fetch_array", &Lua_MySQLPreparedStmt::FetchArray) + .def("fetch_hash", &Lua_MySQLPreparedStmt::FetchHash) + .def("insert_id", &Lua_MySQLPreparedStmt::LastInsertID) + .def("num_fields", &Lua_MySQLPreparedStmt::ColumnCount) + .def("num_rows", &Lua_MySQLPreparedStmt::RowCount) + .def("rows_affected", &Lua_MySQLPreparedStmt::RowsAffected) + .def("set_options", &Lua_MySQLPreparedStmt::SetOptions); +} + +#endif // LUA_EQEMU diff --git a/zone/lua_database.h b/zone/lua_database.h new file mode 100644 index 000000000..ca81f67cd --- /dev/null +++ b/zone/lua_database.h @@ -0,0 +1,51 @@ +#pragma once + +#ifdef LUA_EQEMU + +#include "quest_db.h" +#include "../common/mysql_stmt.h" +#include + +namespace luabind { struct scope; } +luabind::scope lua_register_database(); + +class Lua_MySQLPreparedStmt; + +class Lua_Database : public QuestDB +{ +public: + using QuestDB::QuestDB; + + void Close(); + Lua_MySQLPreparedStmt* Prepare(lua_State*, std::string query); +}; + +class Lua_MySQLPreparedStmt +{ +public: + Lua_MySQLPreparedStmt(lua_State* L, mysql::PreparedStmt&& stmt) + : m_stmt(std::make_unique(std::move(stmt))) + , m_row_array(luabind::newtable(L)) + , m_row_hash(luabind::newtable(L)) {} + + void Close(); + void Execute(lua_State*); + void Execute(lua_State*, luabind::object args); + void SetOptions(luabind::object table_opts); + luabind::object FetchArray(lua_State*); + luabind::object FetchHash(lua_State*); + + // StmtResult functions accessible through this class to simplify api + int ColumnCount(); + uint64_t LastInsertID(); + uint64_t RowCount(); + uint64_t RowsAffected(); + +private: + std::unique_ptr m_stmt; + mysql::StmtResult m_res = {}; + luabind::object m_row_array; // perf: table cache for fetches + luabind::object m_row_hash; +}; + +#endif // LUA_EQEMU diff --git a/zone/lua_parser.cpp b/zone/lua_parser.cpp index 0b3e4913e..6f624f149 100644 --- a/zone/lua_parser.cpp +++ b/zone/lua_parser.cpp @@ -42,6 +42,7 @@ #include "lua_spawn.h" #include "lua_spell.h" #include "lua_stat_bonuses.h" +#include "lua_database.h" const char *LuaEvents[_LargestEventID] = { "event_say", @@ -1318,7 +1319,8 @@ void LuaParser::MapFunctions(lua_State *L) { lua_register_expedition(), lua_register_expedition_lock_messages(), lua_register_buff(), - lua_register_exp_source() + lua_register_exp_source(), + lua_register_database() )]; } catch(std::exception &ex) { diff --git a/zone/perl_database.cpp b/zone/perl_database.cpp new file mode 100644 index 000000000..d0e7c1609 --- /dev/null +++ b/zone/perl_database.cpp @@ -0,0 +1,255 @@ +#include "../common/features.h" + +#ifdef EMBPERL_XS_CLASSES + +#include "embperl.h" +#include "perl_database.h" +#include "zonedb.h" + +// Perl takes ownership of returned objects allocated with new and deletes +// them via the DESTROY method when the last perl reference goes out of scope + +void Perl_Database::Destroy(Perl_Database* ptr) +{ + delete ptr; +} + +Perl_Database* Perl_Database::Connect() +{ + return new Perl_Database(); +} + +Perl_Database* Perl_Database::Connect(Connection type) +{ + return new Perl_Database(type); +} + +Perl_Database* Perl_Database::Connect(Connection type, bool connect) +{ + return new Perl_Database(type, connect); +} + +Perl_Database* Perl_Database::Connect(const char* host, const char* user, const char* pass, const char* db, uint32_t port) +{ + return new Perl_Database(host, user, pass, db, port); +} + +Perl_MySQLPreparedStmt* Perl_Database::Prepare(std::string query) +{ + return m_db ? new Perl_MySQLPreparedStmt(m_db->Prepare(std::move(query))) : nullptr; +} + +void Perl_Database::Close() +{ + m_db.reset(); +} + +// --------------------------------------------------------------------------- + +void Perl_MySQLPreparedStmt::Destroy(Perl_MySQLPreparedStmt* ptr) +{ + delete ptr; +} + +void Perl_MySQLPreparedStmt::Close() +{ + m_stmt.reset(); +} + +void Perl_MySQLPreparedStmt::Execute() +{ + if (m_stmt) + { + m_res = m_stmt->Execute(); + } +} + +void Perl_MySQLPreparedStmt::Execute(perl::array args) +{ + // passes all script args as strings + if (m_stmt) + { + std::vector inputs; + for (const perl::scalar& arg : args) + { + if (arg.is_null()) + { + inputs.emplace_back(nullptr); + } + else + { + inputs.emplace_back(arg.c_str()); + } + } + m_res = m_stmt->Execute(inputs); + } +} + +void Perl_MySQLPreparedStmt::SetOptions(perl::hash hash) +{ + if (m_stmt) + { + mysql::StmtOptions opts = m_stmt->GetOptions(); + if (hash.exists("buffer_results")) + { + opts.buffer_results = hash["buffer_results"].as(); + } + if (hash.exists("use_max_length")) + { + opts.use_max_length = hash["use_max_length"].as(); + } + m_stmt->SetOptions(opts); + } +} + +static void PushValue(PerlInterpreter* my_perl, SV* sv, const mysql::StmtColumn& col) +{ + if (col.IsNull()) + { + sv_setsv(sv, &PL_sv_undef); + return; + } + + switch (col.Type()) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_BIT: + if (col.IsUnsigned()) + { + sv_setuv(sv, col.Get().value()); + } + else + { + sv_setiv(sv, col.Get().value()); + } + break; + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + sv_setnv(sv, col.Get().value()); + break; + case MYSQL_TYPE_TIME: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_TIMESTAMP: + { + std::string str = col.GetStr().value(); + sv_setpvn(sv, str.data(), str.size()); + } + break; + default: // string types, push raw buffer to avoid copy + { + std::string_view str = col.GetStrView().value(); + sv_setpvn(sv, str.data(), str.size()); + } + break; + } +} + +perl::array Perl_MySQLPreparedStmt::FetchArray() +{ + auto row = m_stmt ? m_stmt->Fetch() : mysql::StmtRow(); + if (!row) + { + return perl::array(); + } + + // perf: bypass perlbind operator[]/push and use cache to limit SV allocs + dTHX; + AV* av = static_cast(m_row_array); + for (const mysql::StmtColumn& col : row) + { + SV** sv = av_fetch(av, col.Index(), true); + PushValue(my_perl, *sv, col); + } + + SvREFCNT_inc(av); // return a ref to our cache (no copy) + return perl::array(std::move(av)); +} + +perl::reference Perl_MySQLPreparedStmt::FetchArrayRef() +{ + perl::array array = FetchArray(); + return array.size() == 0 ? perl::reference() : perl::reference(array); +} + +perl::reference Perl_MySQLPreparedStmt::FetchHashRef() +{ + auto row = m_stmt ? m_stmt->Fetch() : mysql::StmtRow(); + if (!row) + { + return perl::reference(); + } + + // perf: bypass perlbind operator[] and use cache to limit SV allocs + dTHX; + HV* hv = static_cast(m_row_hash); + for (const mysql::StmtColumn& col : row) + { + SV** sv = hv_fetch(hv, col.Name().c_str(), static_cast(col.Name().size()), true); + PushValue(my_perl, *sv, col); + } + + SvREFCNT_inc(hv); // return a ref to our cache (no copy) + return perl::reference(std::move(hv)); +} + +int Perl_MySQLPreparedStmt::ColumnCount() +{ + return m_res.ColumnCount(); +} + +uint64_t Perl_MySQLPreparedStmt::LastInsertID() +{ + return m_res.LastInsertID(); +} + +uint64_t Perl_MySQLPreparedStmt::RowCount() +{ + return m_res.RowCount(); +} + +uint64_t Perl_MySQLPreparedStmt::RowsAffected() +{ + return m_res.RowsAffected(); +} + +void perl_register_database() +{ + perl::interpreter perl(PERL_GET_THX); + + { + auto package = perl.new_class("Database"); + package.add_const("Default", static_cast(QuestDB::Connection::Default)); + package.add_const("Content", static_cast(QuestDB::Connection::Content)); + package.add("DESTROY", &Perl_Database::Destroy); + package.add("new", static_cast(&Perl_Database::Connect)); + package.add("new", static_cast(&Perl_Database::Connect)); + package.add("new", static_cast(&Perl_Database::Connect)); + package.add("new", static_cast(&Perl_Database::Connect)); + package.add("close", &Perl_Database::Close); + package.add("prepare", &Perl_Database::Prepare); + } + + { + auto package = perl.new_class("MySQLPreparedStmt"); + package.add("DESTROY", &Perl_MySQLPreparedStmt::Destroy); + package.add("close", &Perl_MySQLPreparedStmt::Close); + package.add("execute", static_cast(&Perl_MySQLPreparedStmt::Execute)); + package.add("execute", static_cast(&Perl_MySQLPreparedStmt::Execute)); + package.add("fetch", &Perl_MySQLPreparedStmt::FetchArray); + package.add("fetch_array", &Perl_MySQLPreparedStmt::FetchArray); + package.add("fetch_arrayref", &Perl_MySQLPreparedStmt::FetchArrayRef); + package.add("fetch_hashref", &Perl_MySQLPreparedStmt::FetchHashRef); + package.add("insert_id", &Perl_MySQLPreparedStmt::LastInsertID); + package.add("num_fields", &Perl_MySQLPreparedStmt::ColumnCount); + package.add("num_rows", &Perl_MySQLPreparedStmt::RowCount); + package.add("rows_affected", &Perl_MySQLPreparedStmt::RowsAffected); + package.add("set_options", &Perl_MySQLPreparedStmt::SetOptions); + } +} + +#endif // EMBPERL_XS_CLASSES diff --git a/zone/perl_database.h b/zone/perl_database.h new file mode 100644 index 000000000..2c1922bb5 --- /dev/null +++ b/zone/perl_database.h @@ -0,0 +1,50 @@ +#pragma once + +#include "quest_db.h" +#include "../common/mysql_stmt.h" + +class Perl_MySQLPreparedStmt; + +class Perl_Database : public QuestDB +{ +public: + using QuestDB::QuestDB; + + static void Destroy(Perl_Database* ptr); + static Perl_Database* Connect(); + static Perl_Database* Connect(Connection type); + static Perl_Database* Connect(Connection type, bool connect); + static Perl_Database* Connect(const char* host, const char* user, const char* pass, const char* db, uint32_t port); + + void Close(); + Perl_MySQLPreparedStmt* Prepare(std::string query); +}; + +class Perl_MySQLPreparedStmt +{ +public: + Perl_MySQLPreparedStmt(mysql::PreparedStmt&& stmt) + : m_stmt(std::make_unique(std::move(stmt))) {} + + static void Destroy(Perl_MySQLPreparedStmt* ptr); + + void Close(); + void Execute(); + void Execute(perl::array args); + void SetOptions(perl::hash hash_opts); + perl::array FetchArray(); + perl::reference FetchArrayRef(); + perl::reference FetchHashRef(); + + // StmtResult functions accessible through this class to simplify api + int ColumnCount(); + uint64_t LastInsertID(); + uint64_t RowCount(); + uint64_t RowsAffected(); + +private: + std::unique_ptr m_stmt; + mysql::StmtResult m_res = {}; + perl::array m_row_array; // perf: cache for fetches + perl::hash m_row_hash; +}; diff --git a/zone/quest_db.cpp b/zone/quest_db.cpp new file mode 100644 index 000000000..5eda523d4 --- /dev/null +++ b/zone/quest_db.cpp @@ -0,0 +1,57 @@ +#include "quest_db.h" +#include "zonedb.h" +#include "zone_config.h" + +// New connections avoid concurrency issues and allow use of unbuffered results +// with prepared statements. Using zone connections w/o buffering would cause +// "Commands out of sync" errors if any queries occur before results consumed. +QuestDB::QuestDB(Connection type, bool connect) +{ + if (connect) + { + m_db = std::unique_ptr(new Database(), Deleter(true)); + + const auto config = EQEmuConfig::get(); + + if (type == Connection::Default || type == Connection::Content && config->ContentDbHost.empty()) + { + m_db->Connect(config->DatabaseHost, config->DatabaseUsername, config->DatabasePassword, + config->DatabaseDB, config->DatabasePort, "questdb"); + } + else if (type == Connection::Content) + { + m_db->Connect(config->ContentDbHost, config->ContentDbUsername, config->ContentDbPassword, + config->ContentDbName, config->ContentDbPort, "questdb"); + } + } + else if (type == Connection::Default) + { + m_db = std::unique_ptr(&database, Deleter(false)); + } + else if (type == Connection::Content) + { + m_db = std::unique_ptr(&content_db, Deleter(false)); + } + + if (!m_db || (connect && m_db->GetStatus() != DBcore::Connected)) + { + throw std::runtime_error(fmt::format("Failed to connect to db type [{}]", static_cast(type))); + } +} + +QuestDB::QuestDB(const char* host, const char* user, const char* pass, const char* db, uint32_t port) + : m_db(new Database(), Deleter(true)) +{ + if (!m_db->Connect(host, user, pass, db, port, "questdb")) + { + throw std::runtime_error(fmt::format("Failed to connect to db [{}:{}]", host, port)); + } +} + +void QuestDB::Deleter::operator()(Database* ptr) noexcept +{ + if (owner) + { + delete ptr; + } +}; diff --git a/zone/quest_db.h b/zone/quest_db.h new file mode 100644 index 000000000..c1d897cc4 --- /dev/null +++ b/zone/quest_db.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +class Database; + +// Base class for quest apis to manage connection to a MySQL database +class QuestDB +{ +public: + enum class Connection { Default = 0, Content }; + + // Throws std::runtime_error on connection failure + QuestDB() : QuestDB(Connection::Default) {} + QuestDB(Connection type) : QuestDB(type, false) {} + QuestDB(Connection type, bool connect); + QuestDB(const char* host, const char* user, const char* pass, const char* db, uint32_t port); + +protected: + // allow optional ownership of pointer to support using zone db connections + struct Deleter + { + Deleter() : owner(true) {} + Deleter(bool owner_) : owner(owner_) {} + bool owner = true; + void operator()(Database* ptr) noexcept; + }; + + std::unique_ptr m_db; +};