diff --git a/zone/CMakeLists.txt b/zone/CMakeLists.txt index 238dcf9f06..1ead62e1b0 100644 --- a/zone/CMakeLists.txt +++ b/zone/CMakeLists.txt @@ -54,6 +54,7 @@ SET(zone_sources lua_buff.cpp lua_corpse.cpp lua_client.cpp + lua_database.cpp lua_door.cpp lua_encounter.cpp lua_entity.cpp @@ -110,6 +111,7 @@ SET(zone_sources perl_bot.cpp perl_buff.cpp perl_client.cpp + perl_database.cpp perl_doors.cpp perl_entity.cpp perl_expedition.cpp @@ -135,6 +137,7 @@ SET(zone_sources qglobals.cpp queryserv.cpp questmgr.cpp + quest_db.cpp quest_parser_collection.cpp raids.cpp raycast_mesh.cpp @@ -215,6 +218,7 @@ SET(zone_headers lua_buff.h lua_client.h lua_corpse.h + lua_database.h lua_door.h lua_encounter.h lua_entity.h @@ -251,6 +255,7 @@ SET(zone_headers pathfinder_interface.h pathfinder_nav_mesh.h pathfinder_null.h + perl_database.h perlpacket.h petitions.h pets.h @@ -260,6 +265,7 @@ SET(zone_headers queryserv.h quest_interface.h questmgr.h + quest_db.h quest_parser_collection.h raids.h raycast_mesh.h diff --git a/zone/embparser.cpp b/zone/embparser.cpp index 8d4145c671..113374e814 100644 --- a/zone/embparser.cpp +++ b/zone/embparser.cpp @@ -58,6 +58,7 @@ void perl_register_expedition_lock_messages(); void perl_register_bot(); void perl_register_buff(); void perl_register_merc(); +void perl_register_database(); #endif // EMBPERL_XS_CLASSES #endif // EMBPERL_XS @@ -1185,6 +1186,7 @@ void PerlembParser::MapFunctions() perl_register_bot(); perl_register_buff(); perl_register_merc(); + perl_register_database(); #endif // EMBPERL_XS_CLASSES } diff --git a/zone/embperl.h b/zone/embperl.h index 9fe7571847..0229710594 100644 --- a/zone/embperl.h +++ b/zone/embperl.h @@ -21,6 +21,8 @@ Eglin #include namespace perl = perlbind; +#undef connect +#undef bind #undef Null #ifdef WIN32 diff --git a/zone/lua_database.cpp b/zone/lua_database.cpp new file mode 100644 index 0000000000..aa2a5bea4b --- /dev/null +++ b/zone/lua_database.cpp @@ -0,0 +1,214 @@ +#ifdef LUA_EQEMU + +#include "lua_database.h" +#include "zonedb.h" +#include +#include + +// Luabind adopts the PreparedStmt wrapper object allocated with new and deletes it via GC +// Lua GC is non-deterministic so handles should be closed explicitly to free db resources +// Script errors/exceptions will hold resources until GC deletes the wrapper object + +Lua_MySQLPreparedStmt* Lua_Database::Prepare(lua_State* L, std::string query) +{ + return m_db ? new Lua_MySQLPreparedStmt(L, m_db->Prepare(std::move(query))) : nullptr; +} + +void Lua_Database::Close() +{ + m_db.reset(); +} + +// --------------------------------------------------------------------------- + +void Lua_MySQLPreparedStmt::Close() +{ + m_stmt.reset(); +} + +void Lua_MySQLPreparedStmt::Execute(lua_State* L) +{ + if (m_stmt) + { + m_res = m_stmt->Execute(); + } +} + +void Lua_MySQLPreparedStmt::Execute(lua_State* L, luabind::object args) +{ + if (m_stmt) + { + std::vector inputs; + + // iterate table until nil like ipairs to guarantee traversal order + for (int i = 1, type; (type = luabind::type(args[i])) != LUA_TNIL; ++i) + { + switch (type) + { + case LUA_TBOOLEAN: + inputs.emplace_back(luabind::object_cast(args[i])); + break; + case LUA_TNUMBER: // all numbers are doubles in lua before 5.3 + inputs.emplace_back(luabind::object_cast(args[i])); + break; + case LUA_TSTRING: + inputs.emplace_back(luabind::object_cast(args[i])); + break; + case LUA_TTABLE: // let tables substitute for null since nils can't exist + inputs.emplace_back(nullptr); + break; + default: + break; + } + } + + m_res = m_stmt->Execute(inputs); + } +} + +void Lua_MySQLPreparedStmt::SetOptions(luabind::object table) +{ + if (m_stmt) + { + mysql::StmtOptions opts = m_stmt->GetOptions(); + if (luabind::type(table["buffer_results"]) == LUA_TBOOLEAN) + { + opts.buffer_results = luabind::object_cast(table["buffer_results"]); + } + if (luabind::type(table["use_max_length"]) == LUA_TBOOLEAN) + { + opts.use_max_length = luabind::object_cast(table["use_max_length"]); + } + m_stmt->SetOptions(opts); + } +} + +static void PushValue(lua_State* L, const mysql::StmtColumn& col) +{ + if (col.IsNull()) + { + lua_pushnil(L); // clear entry in cache from any previous row + return; + } + + // 64-bit ints are pushed as strings since lua 5.1 only has 53-bit precision + switch (col.Type()) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + lua_pushnumber(L, col.Get().value()); + break; + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_BIT: + case MYSQL_TYPE_TIME: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_TIMESTAMP: + { + std::string str = col.GetStr().value(); + lua_pushlstring(L, str.data(), str.size()); + } + break; + default: // string types, push raw buffer to avoid copy + { + std::string_view str = col.GetStrView().value(); + lua_pushlstring(L, str.data(), str.size()); + } + break; + } +} + +luabind::object Lua_MySQLPreparedStmt::FetchArray(lua_State* L) +{ + auto row = m_stmt ? m_stmt->Fetch() : mysql::StmtRow(); + if (!row) + { + return luabind::object(); + } + + // perf: bypass luabind operator[] + m_row_array.push(L); + for (const mysql::StmtColumn& col : row) + { + PushValue(L, col); + lua_rawseti(L, -2, col.Index() + 1); + } + lua_pop(L, 1); + + return m_row_array; +} + +luabind::object Lua_MySQLPreparedStmt::FetchHash(lua_State* L) +{ + auto row = m_stmt ? m_stmt->Fetch() : mysql::StmtRow(); + if (!row) + { + return luabind::object(); + } + + // perf: bypass luabind operator[] + m_row_hash.push(L); + for (const mysql::StmtColumn& col : row) + { + PushValue(L, col); + lua_setfield(L, -2, col.Name().c_str()); + } + lua_pop(L, 1); + + return m_row_hash; +} + +int Lua_MySQLPreparedStmt::ColumnCount() +{ + return m_res.ColumnCount(); +} + +uint64_t Lua_MySQLPreparedStmt::LastInsertID() +{ + return m_res.LastInsertID(); +} + +uint64_t Lua_MySQLPreparedStmt::RowCount() +{ + return m_res.RowCount(); +} + +uint64_t Lua_MySQLPreparedStmt::RowsAffected() +{ + return m_res.RowsAffected(); +} + +luabind::scope lua_register_database() +{ + return luabind::class_("Database") + .enum_("constants") + [( + luabind::value("Default", static_cast(QuestDB::Connection::Default)), + luabind::value("Content", static_cast(QuestDB::Connection::Content)) + )] + .def(luabind::constructor<>()) + .def(luabind::constructor()) + .def(luabind::constructor()) + .def(luabind::constructor()) + .def("close", &Lua_Database::Close) + .def("prepare", &Lua_Database::Prepare, luabind::adopt(luabind::result)), + + luabind::class_("MySQLPreparedStmt") + .def("close", &Lua_MySQLPreparedStmt::Close) + .def("execute", static_cast(&Lua_MySQLPreparedStmt::Execute)) + .def("execute", static_cast(&Lua_MySQLPreparedStmt::Execute)) + .def("fetch", &Lua_MySQLPreparedStmt::FetchArray) + .def("fetch_array", &Lua_MySQLPreparedStmt::FetchArray) + .def("fetch_hash", &Lua_MySQLPreparedStmt::FetchHash) + .def("insert_id", &Lua_MySQLPreparedStmt::LastInsertID) + .def("num_fields", &Lua_MySQLPreparedStmt::ColumnCount) + .def("num_rows", &Lua_MySQLPreparedStmt::RowCount) + .def("rows_affected", &Lua_MySQLPreparedStmt::RowsAffected) + .def("set_options", &Lua_MySQLPreparedStmt::SetOptions); +} + +#endif // LUA_EQEMU diff --git a/zone/lua_database.h b/zone/lua_database.h new file mode 100644 index 0000000000..ca81f67cdc --- /dev/null +++ b/zone/lua_database.h @@ -0,0 +1,51 @@ +#pragma once + +#ifdef LUA_EQEMU + +#include "quest_db.h" +#include "../common/mysql_stmt.h" +#include + +namespace luabind { struct scope; } +luabind::scope lua_register_database(); + +class Lua_MySQLPreparedStmt; + +class Lua_Database : public QuestDB +{ +public: + using QuestDB::QuestDB; + + void Close(); + Lua_MySQLPreparedStmt* Prepare(lua_State*, std::string query); +}; + +class Lua_MySQLPreparedStmt +{ +public: + Lua_MySQLPreparedStmt(lua_State* L, mysql::PreparedStmt&& stmt) + : m_stmt(std::make_unique(std::move(stmt))) + , m_row_array(luabind::newtable(L)) + , m_row_hash(luabind::newtable(L)) {} + + void Close(); + void Execute(lua_State*); + void Execute(lua_State*, luabind::object args); + void SetOptions(luabind::object table_opts); + luabind::object FetchArray(lua_State*); + luabind::object FetchHash(lua_State*); + + // StmtResult functions accessible through this class to simplify api + int ColumnCount(); + uint64_t LastInsertID(); + uint64_t RowCount(); + uint64_t RowsAffected(); + +private: + std::unique_ptr m_stmt; + mysql::StmtResult m_res = {}; + luabind::object m_row_array; // perf: table cache for fetches + luabind::object m_row_hash; +}; + +#endif // LUA_EQEMU diff --git a/zone/lua_parser.cpp b/zone/lua_parser.cpp index 0b3e4913e9..6f624f1498 100644 --- a/zone/lua_parser.cpp +++ b/zone/lua_parser.cpp @@ -42,6 +42,7 @@ #include "lua_spawn.h" #include "lua_spell.h" #include "lua_stat_bonuses.h" +#include "lua_database.h" const char *LuaEvents[_LargestEventID] = { "event_say", @@ -1318,7 +1319,8 @@ void LuaParser::MapFunctions(lua_State *L) { lua_register_expedition(), lua_register_expedition_lock_messages(), lua_register_buff(), - lua_register_exp_source() + lua_register_exp_source(), + lua_register_database() )]; } catch(std::exception &ex) { diff --git a/zone/perl_database.cpp b/zone/perl_database.cpp new file mode 100644 index 0000000000..d0e7c1609e --- /dev/null +++ b/zone/perl_database.cpp @@ -0,0 +1,255 @@ +#include "../common/features.h" + +#ifdef EMBPERL_XS_CLASSES + +#include "embperl.h" +#include "perl_database.h" +#include "zonedb.h" + +// Perl takes ownership of returned objects allocated with new and deletes +// them via the DESTROY method when the last perl reference goes out of scope + +void Perl_Database::Destroy(Perl_Database* ptr) +{ + delete ptr; +} + +Perl_Database* Perl_Database::Connect() +{ + return new Perl_Database(); +} + +Perl_Database* Perl_Database::Connect(Connection type) +{ + return new Perl_Database(type); +} + +Perl_Database* Perl_Database::Connect(Connection type, bool connect) +{ + return new Perl_Database(type, connect); +} + +Perl_Database* Perl_Database::Connect(const char* host, const char* user, const char* pass, const char* db, uint32_t port) +{ + return new Perl_Database(host, user, pass, db, port); +} + +Perl_MySQLPreparedStmt* Perl_Database::Prepare(std::string query) +{ + return m_db ? new Perl_MySQLPreparedStmt(m_db->Prepare(std::move(query))) : nullptr; +} + +void Perl_Database::Close() +{ + m_db.reset(); +} + +// --------------------------------------------------------------------------- + +void Perl_MySQLPreparedStmt::Destroy(Perl_MySQLPreparedStmt* ptr) +{ + delete ptr; +} + +void Perl_MySQLPreparedStmt::Close() +{ + m_stmt.reset(); +} + +void Perl_MySQLPreparedStmt::Execute() +{ + if (m_stmt) + { + m_res = m_stmt->Execute(); + } +} + +void Perl_MySQLPreparedStmt::Execute(perl::array args) +{ + // passes all script args as strings + if (m_stmt) + { + std::vector inputs; + for (const perl::scalar& arg : args) + { + if (arg.is_null()) + { + inputs.emplace_back(nullptr); + } + else + { + inputs.emplace_back(arg.c_str()); + } + } + m_res = m_stmt->Execute(inputs); + } +} + +void Perl_MySQLPreparedStmt::SetOptions(perl::hash hash) +{ + if (m_stmt) + { + mysql::StmtOptions opts = m_stmt->GetOptions(); + if (hash.exists("buffer_results")) + { + opts.buffer_results = hash["buffer_results"].as(); + } + if (hash.exists("use_max_length")) + { + opts.use_max_length = hash["use_max_length"].as(); + } + m_stmt->SetOptions(opts); + } +} + +static void PushValue(PerlInterpreter* my_perl, SV* sv, const mysql::StmtColumn& col) +{ + if (col.IsNull()) + { + sv_setsv(sv, &PL_sv_undef); + return; + } + + switch (col.Type()) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_BIT: + if (col.IsUnsigned()) + { + sv_setuv(sv, col.Get().value()); + } + else + { + sv_setiv(sv, col.Get().value()); + } + break; + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + sv_setnv(sv, col.Get().value()); + break; + case MYSQL_TYPE_TIME: + case MYSQL_TYPE_DATE: + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_TIMESTAMP: + { + std::string str = col.GetStr().value(); + sv_setpvn(sv, str.data(), str.size()); + } + break; + default: // string types, push raw buffer to avoid copy + { + std::string_view str = col.GetStrView().value(); + sv_setpvn(sv, str.data(), str.size()); + } + break; + } +} + +perl::array Perl_MySQLPreparedStmt::FetchArray() +{ + auto row = m_stmt ? m_stmt->Fetch() : mysql::StmtRow(); + if (!row) + { + return perl::array(); + } + + // perf: bypass perlbind operator[]/push and use cache to limit SV allocs + dTHX; + AV* av = static_cast(m_row_array); + for (const mysql::StmtColumn& col : row) + { + SV** sv = av_fetch(av, col.Index(), true); + PushValue(my_perl, *sv, col); + } + + SvREFCNT_inc(av); // return a ref to our cache (no copy) + return perl::array(std::move(av)); +} + +perl::reference Perl_MySQLPreparedStmt::FetchArrayRef() +{ + perl::array array = FetchArray(); + return array.size() == 0 ? perl::reference() : perl::reference(array); +} + +perl::reference Perl_MySQLPreparedStmt::FetchHashRef() +{ + auto row = m_stmt ? m_stmt->Fetch() : mysql::StmtRow(); + if (!row) + { + return perl::reference(); + } + + // perf: bypass perlbind operator[] and use cache to limit SV allocs + dTHX; + HV* hv = static_cast(m_row_hash); + for (const mysql::StmtColumn& col : row) + { + SV** sv = hv_fetch(hv, col.Name().c_str(), static_cast(col.Name().size()), true); + PushValue(my_perl, *sv, col); + } + + SvREFCNT_inc(hv); // return a ref to our cache (no copy) + return perl::reference(std::move(hv)); +} + +int Perl_MySQLPreparedStmt::ColumnCount() +{ + return m_res.ColumnCount(); +} + +uint64_t Perl_MySQLPreparedStmt::LastInsertID() +{ + return m_res.LastInsertID(); +} + +uint64_t Perl_MySQLPreparedStmt::RowCount() +{ + return m_res.RowCount(); +} + +uint64_t Perl_MySQLPreparedStmt::RowsAffected() +{ + return m_res.RowsAffected(); +} + +void perl_register_database() +{ + perl::interpreter perl(PERL_GET_THX); + + { + auto package = perl.new_class("Database"); + package.add_const("Default", static_cast(QuestDB::Connection::Default)); + package.add_const("Content", static_cast(QuestDB::Connection::Content)); + package.add("DESTROY", &Perl_Database::Destroy); + package.add("new", static_cast(&Perl_Database::Connect)); + package.add("new", static_cast(&Perl_Database::Connect)); + package.add("new", static_cast(&Perl_Database::Connect)); + package.add("new", static_cast(&Perl_Database::Connect)); + package.add("close", &Perl_Database::Close); + package.add("prepare", &Perl_Database::Prepare); + } + + { + auto package = perl.new_class("MySQLPreparedStmt"); + package.add("DESTROY", &Perl_MySQLPreparedStmt::Destroy); + package.add("close", &Perl_MySQLPreparedStmt::Close); + package.add("execute", static_cast(&Perl_MySQLPreparedStmt::Execute)); + package.add("execute", static_cast(&Perl_MySQLPreparedStmt::Execute)); + package.add("fetch", &Perl_MySQLPreparedStmt::FetchArray); + package.add("fetch_array", &Perl_MySQLPreparedStmt::FetchArray); + package.add("fetch_arrayref", &Perl_MySQLPreparedStmt::FetchArrayRef); + package.add("fetch_hashref", &Perl_MySQLPreparedStmt::FetchHashRef); + package.add("insert_id", &Perl_MySQLPreparedStmt::LastInsertID); + package.add("num_fields", &Perl_MySQLPreparedStmt::ColumnCount); + package.add("num_rows", &Perl_MySQLPreparedStmt::RowCount); + package.add("rows_affected", &Perl_MySQLPreparedStmt::RowsAffected); + package.add("set_options", &Perl_MySQLPreparedStmt::SetOptions); + } +} + +#endif // EMBPERL_XS_CLASSES diff --git a/zone/perl_database.h b/zone/perl_database.h new file mode 100644 index 0000000000..2c1922bb51 --- /dev/null +++ b/zone/perl_database.h @@ -0,0 +1,50 @@ +#pragma once + +#include "quest_db.h" +#include "../common/mysql_stmt.h" + +class Perl_MySQLPreparedStmt; + +class Perl_Database : public QuestDB +{ +public: + using QuestDB::QuestDB; + + static void Destroy(Perl_Database* ptr); + static Perl_Database* Connect(); + static Perl_Database* Connect(Connection type); + static Perl_Database* Connect(Connection type, bool connect); + static Perl_Database* Connect(const char* host, const char* user, const char* pass, const char* db, uint32_t port); + + void Close(); + Perl_MySQLPreparedStmt* Prepare(std::string query); +}; + +class Perl_MySQLPreparedStmt +{ +public: + Perl_MySQLPreparedStmt(mysql::PreparedStmt&& stmt) + : m_stmt(std::make_unique(std::move(stmt))) {} + + static void Destroy(Perl_MySQLPreparedStmt* ptr); + + void Close(); + void Execute(); + void Execute(perl::array args); + void SetOptions(perl::hash hash_opts); + perl::array FetchArray(); + perl::reference FetchArrayRef(); + perl::reference FetchHashRef(); + + // StmtResult functions accessible through this class to simplify api + int ColumnCount(); + uint64_t LastInsertID(); + uint64_t RowCount(); + uint64_t RowsAffected(); + +private: + std::unique_ptr m_stmt; + mysql::StmtResult m_res = {}; + perl::array m_row_array; // perf: cache for fetches + perl::hash m_row_hash; +}; diff --git a/zone/quest_db.cpp b/zone/quest_db.cpp new file mode 100644 index 0000000000..5eda523d41 --- /dev/null +++ b/zone/quest_db.cpp @@ -0,0 +1,57 @@ +#include "quest_db.h" +#include "zonedb.h" +#include "zone_config.h" + +// New connections avoid concurrency issues and allow use of unbuffered results +// with prepared statements. Using zone connections w/o buffering would cause +// "Commands out of sync" errors if any queries occur before results consumed. +QuestDB::QuestDB(Connection type, bool connect) +{ + if (connect) + { + m_db = std::unique_ptr(new Database(), Deleter(true)); + + const auto config = EQEmuConfig::get(); + + if (type == Connection::Default || type == Connection::Content && config->ContentDbHost.empty()) + { + m_db->Connect(config->DatabaseHost, config->DatabaseUsername, config->DatabasePassword, + config->DatabaseDB, config->DatabasePort, "questdb"); + } + else if (type == Connection::Content) + { + m_db->Connect(config->ContentDbHost, config->ContentDbUsername, config->ContentDbPassword, + config->ContentDbName, config->ContentDbPort, "questdb"); + } + } + else if (type == Connection::Default) + { + m_db = std::unique_ptr(&database, Deleter(false)); + } + else if (type == Connection::Content) + { + m_db = std::unique_ptr(&content_db, Deleter(false)); + } + + if (!m_db || (connect && m_db->GetStatus() != DBcore::Connected)) + { + throw std::runtime_error(fmt::format("Failed to connect to db type [{}]", static_cast(type))); + } +} + +QuestDB::QuestDB(const char* host, const char* user, const char* pass, const char* db, uint32_t port) + : m_db(new Database(), Deleter(true)) +{ + if (!m_db->Connect(host, user, pass, db, port, "questdb")) + { + throw std::runtime_error(fmt::format("Failed to connect to db [{}:{}]", host, port)); + } +} + +void QuestDB::Deleter::operator()(Database* ptr) noexcept +{ + if (owner) + { + delete ptr; + } +}; diff --git a/zone/quest_db.h b/zone/quest_db.h new file mode 100644 index 0000000000..c1d897cc47 --- /dev/null +++ b/zone/quest_db.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +class Database; + +// Base class for quest apis to manage connection to a MySQL database +class QuestDB +{ +public: + enum class Connection { Default = 0, Content }; + + // Throws std::runtime_error on connection failure + QuestDB() : QuestDB(Connection::Default) {} + QuestDB(Connection type) : QuestDB(type, false) {} + QuestDB(Connection type, bool connect); + QuestDB(const char* host, const char* user, const char* pass, const char* db, uint32_t port); + +protected: + // allow optional ownership of pointer to support using zone db connections + struct Deleter + { + Deleter() : owner(true) {} + Deleter(bool owner_) : owner(owner_) {} + bool owner = true; + void operator()(Database* ptr) noexcept; + }; + + std::unique_ptr m_db; +};