diff --git a/bin/conf/PseuWoW.conf.default b/bin/conf/PseuWoW.conf.default index b487d4e..d44b216 100644 --- a/bin/conf/PseuWoW.conf.default +++ b/bin/conf/PseuWoW.conf.default @@ -141,4 +141,13 @@ SkipAddonChat=1 // (doesn't matter if they have scripts attached or not, they will be dumped always) DumpPackets=1 +// Specify how many threads should be used for loading data files +// 0 - Do not use any multithreading to load files (will pause execution everytime a file is loaded). + Use this setting if there are threading problems or similar. +// 1 - Allows to load files in background, but serial, one after another +// 2 or more - Optimize for multicore processors; parallel file loading. +// Note: Using too many threads may result in overall slower loading times due to harddisk seek overhead. +// Default: 2 +DataLoaderThreads=2 + diff --git a/src/Client/DefScriptInterface.cpp b/src/Client/DefScriptInterface.cpp index f5cf0d2..45b0af4 100644 --- a/src/Client/DefScriptInterface.cpp +++ b/src/Client/DefScriptInterface.cpp @@ -10,6 +10,7 @@ #include "Channel.h" #include "CacheHandler.h" #include "SCPDatabase.h" +#include "MemoryDataHolder.h" void DefScriptPackage::_InitDefScriptInterface(void) @@ -60,6 +61,7 @@ void DefScriptPackage::_InitDefScriptInterface(void) AddFunc("spoofworldpacket",&DefScriptPackage::SCSpoofWorldPacket); AddFunc("loaddb",&DefScriptPackage::SCLoadDB); AddFunc("adddbpath",&DefScriptPackage::SCAddDBPath); + AddFunc("preloadfile",&DefScriptPackage::SCPreloadFile); } DefReturnResult DefScriptPackage::SCshdn(CmdSet& Set) @@ -1227,6 +1229,12 @@ DefReturnResult DefScriptPackage::SCGetPos(CmdSet &Set) return ""; } +DefReturnResult DefScriptPackage::SCPreloadFile(CmdSet& Set) +{ + MemoryDataHolder::BackgroundLoadFile(Set.defaultarg); + return true; +} + void DefScriptPackage::My_LoadUserPermissions(VarSet &vs) { static char *prefix = "USERS::"; diff --git a/src/Client/DefScriptInterfaceInclude.h b/src/Client/DefScriptInterfaceInclude.h index db661e5..ebeba69 100644 --- a/src/Client/DefScriptInterfaceInclude.h +++ b/src/Client/DefScriptInterfaceInclude.h @@ -48,9 +48,10 @@ DefReturnResult SCGetObjectDistance(CmdSet&); DefReturnResult SCSwitchOpcodeHandler(CmdSet&); DefReturnResult SCOpcodeDisabled(CmdSet&); DefReturnResult SCSpoofWorldPacket(CmdSet&); -DefReturnResult SCLoadDB(CmdSet &Set); -DefReturnResult SCAddDBPath(CmdSet &Set); -DefReturnResult SCGetPos(CmdSet &Set); +DefReturnResult SCLoadDB(CmdSet&); +DefReturnResult SCAddDBPath(CmdSet&); +DefReturnResult SCGetPos(CmdSet&); +DefReturnResult SCPreloadFile(CmdSet&); void my_print(const char *fmt, ...); diff --git a/src/Client/GUI/GUIEventReceiver.h b/src/Client/GUI/GUIEventReceiver.h index 4abdb41..ab4f462 100644 --- a/src/Client/GUI/GUIEventReceiver.h +++ b/src/Client/GUI/GUIEventReceiver.h @@ -41,7 +41,7 @@ public: { s32 id = event.GUIEvent.Caller->getID(); - printf("event type %u ID %u\n",event.GUIEvent.EventType,id); + DEBUG(logdev("GUIEventReceiver: event type %u ID %u",event.GUIEvent.EventType,id)); switch(event.GUIEvent.EventType) { diff --git a/src/Client/MemoryDataHolder.cpp b/src/Client/MemoryDataHolder.cpp index fdaed8b..99b0c26 100644 --- a/src/Client/MemoryDataHolder.cpp +++ b/src/Client/MemoryDataHolder.cpp @@ -1,78 +1,108 @@ #include #include "MemoryDataHolder.h" #include "DefScript/TypeStorage.h" +#include "zthread/Condition.h" +#include "zthread/Task.h" +#include "zthread/PoolExecutor.h" namespace MemoryDataHolder { class DataLoaderRunnable; + ZThread::PoolExecutor *executor = NULL; ZThread::FastMutex mutex; - TypeStorage storage; + TypeStorage storage; TypeStorage loaders; + TypeStorage refs; + bool alwaysSingleThreaded = false; + + void Init(void) + { + if(!executor) + executor = new ZThread::PoolExecutor(1); // TODO: fix memleak on shutdown? + } + + void SetThreadCount(uint32 t) + { + // 0 threads used means we use no threading at all + if(!t) + { + logdetail("MemoryDataHolder: Single-threaded mode."); + alwaysSingleThreaded = true; + executor->size(1); + } + else + { + logdetail("MemoryDataHolder: Using %u threads.", t); + alwaysSingleThreaded = false; + executor->size(t); + } + } - // instances of this class MUST be created with new-operator, or Destroy() will cause a crash! class DataLoaderRunnable : public ZThread::Runnable { public: DataLoaderRunnable() { - _buf = NULL; _threaded = false; } + ~DataLoaderRunnable() + { + logdev("~DataLoaderRunnable(%s)", _name.c_str()); + } // the threaded part void run() { - uint32 size = GetFileSize(_name.c_str()); + const char *name = _name.c_str(); + memblock *mb = new memblock(); + + mb->size = GetFileSize(name); // couldnt open file if size is 0 - if(!size) + if(!mb->size) { - logerror("DataLoaderRunnable: Error opening file: '%s'",_name.c_str()); - DoCallbacks(false); // call callback func, 'false' to indicate file coulsnt be loaded - Destroy(); + logerror("DataLoaderRunnable: Error opening file: '%s'", name); + loaders.Unlink(name); + DoCallbacks(name, MDH_FILE_ERROR); // call callback func, 'false' to indicate file couldnt be loaded + delete mb; return; } - _buf = new uint8[size]; + mb->alloc(mb->size); std::ifstream fh; - fh.open(_name.c_str(), std::ios_base::in | std::ios_base::binary); + fh.open(name, std::ios_base::in | std::ios_base::binary); if(!fh.is_open()) { - logerror("DataLoaderRunnable: Error opening file: '%s'",_name.c_str()); - delete _buf; - _buf = NULL; - DoCallbacks(false); - Destroy(); + logerror("DataLoaderRunnable: Error opening file: '%s'", name); + loaders.Unlink(name); + mb->free(); + delete mb; + DoCallbacks(name, MDH_FILE_ERROR); return; } - fh.read((char*)_buf,size); + logdev("DataLoaderRunnable: Reading '%s'... (%s)", name, FilesizeFormat(mb->size).c_str()); + fh.read((char*)mb->ptr, mb->size); fh.close(); - storage.Assign(_name,&_buf); - loaders.UnlinkByPtr(this); // must be unlinked after the file is fully loaded, but before the callbacks are processed! - DoCallbacks(true); - Destroy(); + storage.Assign(name, mb); + loaders.Unlink(name); // must be unlinked after the file is fully loaded, but before the callbacks are processed! + logdev("DataLoaderRunnable: Done with '%s' (%s)", name, FilesizeFormat(mb->size).c_str()); + DoCallbacks(name, MDH_FILE_OK | MDH_FILE_JUST_LOADED); } - inline void AddCallback(callback_func func, void *ptr = NULL) + inline void AddCallback(callback_func func, void *ptr = NULL, ZThread::Condition *cond = NULL) { - _callbacks[func] = ptr; + callback_struct cbs; + cbs.func = func; + cbs.ptr = ptr; + cbs.cond = cond; + _callbacks.push_back(cbs); } - inline void SetName(std::string name) + inline void DoCallbacks(std::string fn, uint32 flags) { - _name = name; - } - // if this class has done its work, delete self - inline void Destroy(void) - { - delete this; - } - inline uint8 *GetBuf(void) - { - return _buf; - } - inline void DoCallbacks(bool success = true) - { - for(std::map::iterator it = _callbacks.begin(); it != _callbacks.end(); it++) + for(CallbackStore::iterator it = _callbacks.begin(); it != _callbacks.end(); it++) { - (*(it->first))(it->second,success); + if(it->cond) + it->cond->broadcast(); + if(it->func) + (*(it->func))(it->ptr, fn, flags); } } inline void SetThreaded(bool t) @@ -83,28 +113,51 @@ namespace MemoryDataHolder { return _threaded; } - inline bool HasCallbackFunc(callback_func f) + inline void SetName(std::string n) { - return _callbacks.find(f) != _callbacks.end(); + _name = n; } - - private: - std::string _name; - std::map _callbacks; - uint8 *_buf; + CallbackStore _callbacks; bool _threaded; + std::string _name; }; - uint8 *GetFile(std::string s, bool threaded = false, callback_func func = NULL,void *ptr = NULL) + memblock GetFile(std::string s, bool threaded, callback_func func, void *ptr, ZThread::Condition *cond, bool ref_counted) { - mutex.acquire(); // we need excusive access, other threads might unload the requested file during checking - if(uint8 **buf = storage.GetNoCreate(s)) + mutex.acquire(); // we need exclusive access, other threads might unload the requested file during checking + + if(alwaysSingleThreaded) + threaded = false; + + // manage reference counter + uint32 *refcount = refs.GetNoCreate(s); + if(!refcount || !*refcount) { - // the file was requested some other time, is still present in memory and the pointer can simply be returned - mutex.release(); // everything ok, mutex can be unloaded safely before returning - return *buf; + refcount = new uint32; + *refcount = ref_counted ? 1 : 0; + refs.Assign(s,refcount); + } + else + { + if(ref_counted) + { + (*refcount)++; + } + } + + if(memblock *mb = storage.GetNoCreate(s)) + { + // the file was requested some other time, is still present in memory and the pointer can simply be returned... + mutex.release(); // everything ok, mutex can be unloaded safely + // execute callback and broadcast condition (must check for MDH_FILE_ALREADY_EXIST in callback func) + if(func) + (*func)(ptr, s, MDH_FILE_OK | MDH_FILE_ALREADY_EXIST); + if(cond) + cond->broadcast(); + + return *mb; } else { @@ -114,30 +167,82 @@ namespace MemoryDataHolder // no loader thread is working on that file... r = new DataLoaderRunnable(); loaders.Assign(s,r); - r->AddCallback(func,ptr); // not threadsafe! - // after assigning/registering a new loader to the file, the mutex can be released safely - mutex.release(); - r->SetName(s); + r->AddCallback(func,ptr,cond); // not threadsafe! r->SetThreaded(threaded); + r->SetName(s); // here we set the filename the thread should load + // the mutex can be released safely now + mutex.release(); if(threaded) { - ZThread::Thread t(r); // start thread + ZThread::Task task(r); + executor->execute(task); } else { - r->run(); // will exit after the whole file is loaded and the (one) callback is run - return r->GetBuf(); + r->run(); // will exit after the whole file is loaded and the callbacks were run + memblock *mb = storage.GetNoCreate(s); + delete r; + return *mb; } } else // if a loader is already existing, add callbacks to that loader. { - r->AddCallback(func,ptr); + r->AddCallback(func,ptr,cond); mutex.release(); } } - return NULL; + return memblock(); + } + + bool IsLoaded(std::string s) + { + ZThread::Guard g(mutex); + return storage.Exists(s); + } + + // ensure the file is present in memory, but do not touch the reference counter + void BackgroundLoadFile(std::string s) + { + GetFile(s, true, NULL, NULL, NULL, false); + } + + + bool Delete(std::string s) + { + ZThread::Guard g(mutex); + uint32 *refcount = refs.GetNoCreate(s); + if(!refcount) + { + logerror("MemoryDataHolder:Delete(\"%s\"): no refcount", s.c_str()); + return false; + } + else + { + if(*refcount) + (*refcount)--; + logdev("MemoryDataHolder::Delete(\"%s\"): refcount dropped to %u", s.c_str(), *refcount); + } + if(!*refcount) + { + refs.Delete(s); + if(memblock *mb = storage.GetNoCreate(s)) + { + logdev("MemoryDataHolder:: deleting 0x%X (size %u)", mb->ptr, mb->size); + mb->free(); + storage.Delete(s); + return true; + } + else + { + logerror("MemoryDataHolder::Delete(\"%s\"): no buf existing",s.c_str()); + return false; + } + } + return true; } + + }; diff --git a/src/Client/MemoryDataHolder.h b/src/Client/MemoryDataHolder.h index b609340..08cc164 100644 --- a/src/Client/MemoryDataHolder.h +++ b/src/Client/MemoryDataHolder.h @@ -3,12 +3,48 @@ #include "common.h" +namespace ZThread +{ + class Condition; +}; + namespace MemoryDataHolder { - typedef void (*callback_func)(void*,bool); + enum ResultFlags + { + MDH_FILE_ERROR = 0, // file doesnt exist, cant be loaded, etc + MDH_FILE_OK = 1, // file was loaded properly or already present in memory. point is: we have good data + MDH_FILE_ALREADY_EXIST = 2, // file was loaded before + MDH_FILE_JUST_LOADED = 4, // file was freshly loaded + }; + typedef void (*callback_func)(void *ptr,std::string filename, uint32 flags); + struct callback_struct + { + callback_func func; + void *ptr; + ZThread::Condition *cond; + }; + typedef std::deque CallbackStore; - uint8 *GetFile(std::string&,bool,callback_func,void*); + struct memblock + { + memblock() : ptr(NULL), size(0) {} + memblock(uint8 *p, uint32 s) : ptr(p), size(s) {} + void alloc(uint32 s) { size = s; ptr = new uint8[s]; } + void free(void) { delete [] ptr; } + uint8 *ptr; + uint32 size; + }; + + void Init(void); + void SetThreadCount(uint32); + + memblock GetFile(std::string s, bool threaded = false, callback_func func = NULL,void *ptr = NULL, ZThread::Condition *cond = NULL, bool ref_counted = true); + inline memblock GetFileBasic(std::string s) { return GetFile(s, false, NULL, NULL, NULL, false); } + bool IsLoaded(std::string); + void BackgroundLoadFile(std::string); + bool Delete(std::string); }; #endif diff --git a/src/Client/PseuWoW.cpp b/src/Client/PseuWoW.cpp index ceae02f..347e657 100644 --- a/src/Client/PseuWoW.cpp +++ b/src/Client/PseuWoW.cpp @@ -15,6 +15,7 @@ #include "RemoteController.h" #include "Cli.h" #include "GUI/SceneData.h" +#include "MemoryDataHolder.h" //###### Start of program code ####### @@ -490,6 +491,7 @@ void PseuInstanceConf::ApplyFromVarSet(VarSet &v) skipaddonchat=(bool)atoi(v.Get("SKIPADDONCHAT").c_str()); dumpPackets=(uint8)atoi(v.Get("DUMPPACKETS").c_str()); softquit=(bool)atoi(v.Get("SOFTQUIT").c_str()); + dataLoaderThreads=atoi(v.Get("DATALOADERTHREADS").c_str()); // clientversion is a bit more complicated to add { @@ -521,7 +523,9 @@ void PseuInstanceConf::ApplyFromVarSet(VarSet &v) fov = atof(v.Get("GUI::FOV").c_str()); masterSoundVolume = atof(v.Get("GUI::MASTERSOUNDVOLUME").c_str()); + // cleanups, internal settings, etc. log_setloglevel(debug); + MemoryDataHolder::SetThreadCount(dataLoaderThreads); } diff --git a/src/Client/PseuWoW.h b/src/Client/PseuWoW.h index 387b94f..81a4c7b 100644 --- a/src/Client/PseuWoW.h +++ b/src/Client/PseuWoW.h @@ -18,6 +18,18 @@ class PseuInstanceRunnable; class CliRunnable; class RemoteController; +// possible conditions threads can wait for. used for thread synchronisation. extend if needed. +enum InstanceConditions +{ + COND_GUI_INITIALIZED, + COND_GUI_SCENE_CHANGED, + COND_GUI_CLOSED, + COND_MAP_LOADED, + + COND_MAX +}; + + class PseuInstanceConf { public: @@ -59,6 +71,7 @@ class PseuInstanceConf bool skipaddonchat; uint8 dumpPackets; bool softquit; + uint8 dataLoaderThreads; // gui related bool enablegui; diff --git a/src/Client/World/WorldSession.cpp b/src/Client/World/WorldSession.cpp index 4fe8da9..008dce9 100644 --- a/src/Client/World/WorldSession.cpp +++ b/src/Client/World/WorldSession.cpp @@ -534,12 +534,7 @@ void WorldSession::_HandleAuthChallengeOpcode(WorldPacket& recvPacket) WorldPacket auth; auth<<(uint32)(GetInstance()->GetConf()->clientbuild)< instanceList; // TODO: move this to a "Master" class later @@ -77,8 +78,9 @@ void _new_handler(void) throw; } -int main(int argc, char* argv[]) { - try +int main(int argc, char* argv[]) +{ + try { std::set_new_handler(_new_handler); log_prepare("logfile.txt","a"); @@ -91,6 +93,7 @@ int main(int argc, char* argv[]) { logcustom(0,GREEN,"Compiled: %s %s",__DATE__,__TIME__); _HookSignals(); + MemoryDataHolder::Init(); // 1 instance is enough for now PseuInstanceRunnable *r=new PseuInstanceRunnable(); diff --git a/src/PseuWoW.vcproj b/src/PseuWoW.vcproj index d609ed6..186a70c 100644 --- a/src/PseuWoW.vcproj +++ b/src/PseuWoW.vcproj @@ -186,14 +186,12 @@ + Name="Release|Win32"> + Name="Debug|Win32">