* finally implemented MemoryDataHolder functions. They are not yet used (this will be next step), but tested and works.

bin/conf/PseuWoW.conf.default

@@ -141,4 +141,13 @@ SkipAddonChat=1
 //     (doesn't matter if they have scripts attached or not, they will be dumped always)
 DumpPackets=1
 
+// Specify how many threads should be used for loading data files
+// 0 - Do not use any multithreading to load files (will pause execution everytime a file is loaded).
+       Use this setting if there are threading problems or similar.
+// 1 - Allows to load files in background, but serial, one after another
+// 2 or more - Optimize for multicore processors; parallel file loading.
+// Note: Using too many threads may result in overall slower loading times due to harddisk seek overhead.
+// Default: 2
+DataLoaderThreads=2
+
 

src/Client/DefScriptInterface.cpp

@@ -10,6 +10,7 @@
 #include "Channel.h"
 #include "CacheHandler.h"
 #include "SCPDatabase.h"
+#include "MemoryDataHolder.h"
 
 
 void DefScriptPackage::_InitDefScriptInterface(void)
@@ -60,6 +61,7 @@ void DefScriptPackage::_InitDefScriptInterface(void)
     AddFunc("spoofworldpacket",&DefScriptPackage::SCSpoofWorldPacket);
     AddFunc("loaddb",&DefScriptPackage::SCLoadDB);
     AddFunc("adddbpath",&DefScriptPackage::SCAddDBPath);
+    AddFunc("preloadfile",&DefScriptPackage::SCPreloadFile);
 }
 
 DefReturnResult DefScriptPackage::SCshdn(CmdSet& Set)
@@ -1227,6 +1229,12 @@ DefReturnResult DefScriptPackage::SCGetPos(CmdSet &Set)
     return "";
 }
 
+DefReturnResult DefScriptPackage::SCPreloadFile(CmdSet& Set)
+{
+    MemoryDataHolder::BackgroundLoadFile(Set.defaultarg);
+    return true;
+}
+
 void DefScriptPackage::My_LoadUserPermissions(VarSet &vs)
 {
     static char *prefix = "USERS::";

src/Client/DefScriptInterfaceInclude.h

@@ -48,9 +48,10 @@ DefReturnResult SCGetObjectDistance(CmdSet&);
 DefReturnResult SCSwitchOpcodeHandler(CmdSet&);
 DefReturnResult SCOpcodeDisabled(CmdSet&);
 DefReturnResult SCSpoofWorldPacket(CmdSet&);
-DefReturnResult SCLoadDB(CmdSet &Set);
+DefReturnResult SCLoadDB(CmdSet&);
-DefReturnResult SCAddDBPath(CmdSet &Set);
+DefReturnResult SCAddDBPath(CmdSet&);
-DefReturnResult SCGetPos(CmdSet &Set);
+DefReturnResult SCGetPos(CmdSet&);
+DefReturnResult SCPreloadFile(CmdSet&);
 
 
 void my_print(const char *fmt, ...);

src/Client/GUI/GUIEventReceiver.h

@@ -41,7 +41,7 @@ public:
         {
             s32 id = event.GUIEvent.Caller->getID();
 
-            printf("event type %u ID %u\n",event.GUIEvent.EventType,id);
+            DEBUG(logdev("GUIEventReceiver: event type %u ID %u",event.GUIEvent.EventType,id));
 
             switch(event.GUIEvent.EventType)
             {

src/Client/MemoryDataHolder.cpp

@@ -1,78 +1,108 @@
 #include <fstream>
 #include "MemoryDataHolder.h"
 #include "DefScript/TypeStorage.h"
+#include "zthread/Condition.h"
+#include "zthread/Task.h"
+#include "zthread/PoolExecutor.h"
 
 namespace MemoryDataHolder
 {
     class DataLoaderRunnable;
+    ZThread::PoolExecutor *executor = NULL;
 
     ZThread::FastMutex mutex;
-    TypeStorage<uint8*> storage;
+    TypeStorage<memblock> storage;
     TypeStorage<DataLoaderRunnable> loaders;
+    TypeStorage<uint32> refs;
+    bool alwaysSingleThreaded = false;
+
+    void Init(void)
+    {
+        if(!executor)
+            executor = new ZThread::PoolExecutor(1); // TODO: fix memleak on shutdown?
+    }
+    
+    void SetThreadCount(uint32 t)
+    {
+        // 0 threads used means we use no threading at all
+        if(!t)
+        {
+            logdetail("MemoryDataHolder: Single-threaded mode.");
+            alwaysSingleThreaded = true;
+            executor->size(1);
+        }
+        else
+        {
+            logdetail("MemoryDataHolder: Using %u threads.", t);
+            alwaysSingleThreaded = false;
+            executor->size(t);
+        }
+    }
     
-    // instances of this class MUST be created with new-operator, or Destroy() will cause a crash!
     class DataLoaderRunnable : public ZThread::Runnable
     {
     public:
         DataLoaderRunnable()
         {
-            _buf = NULL;
             _threaded = false;
         }
+        ~DataLoaderRunnable()
+        {
+            logdev("~DataLoaderRunnable(%s)", _name.c_str());
+        }
         // the threaded part
         void run()
         {
-            uint32 size = GetFileSize(_name.c_str());
+            const char *name = _name.c_str();
+            memblock *mb = new memblock();
+
+            mb->size = GetFileSize(name);
             // couldnt open file if size is 0
-            if(!size)
+            if(!mb->size)
             {
-                logerror("DataLoaderRunnable: Error opening file: '%s'",_name.c_str());
+                logerror("DataLoaderRunnable: Error opening file: '%s'", name);
-                DoCallbacks(false); // call callback func, 'false' to indicate file coulsnt be loaded
+                loaders.Unlink(name);
-                Destroy();
+                DoCallbacks(name, MDH_FILE_ERROR); // call callback func, 'false' to indicate file couldnt be loaded
+                delete mb;
                 return;
             }
-            _buf = new uint8[size];
+            mb->alloc(mb->size);
             std::ifstream fh;
-            fh.open(_name.c_str(), std::ios_base::in | std::ios_base::binary);
+            fh.open(name, std::ios_base::in | std::ios_base::binary);
             if(!fh.is_open())
             {
-                logerror("DataLoaderRunnable: Error opening file: '%s'",_name.c_str());
+                logerror("DataLoaderRunnable: Error opening file: '%s'", name);
-                delete _buf;
+                loaders.Unlink(name);
-                _buf = NULL;
+                mb->free();
-                DoCallbacks(false);
+                delete mb;
-                Destroy();
+                DoCallbacks(name, MDH_FILE_ERROR);
                 return;
             }
-            fh.read((char*)_buf,size);
+            logdev("DataLoaderRunnable: Reading '%s'... (%s)", name, FilesizeFormat(mb->size).c_str());
+            fh.read((char*)mb->ptr, mb->size);
             fh.close();
-            storage.Assign(_name,&_buf);
+            storage.Assign(name, mb);
-            loaders.UnlinkByPtr(this); // must be unlinked after the file is fully loaded, but before the callbacks are processed!
+            loaders.Unlink(name); // must be unlinked after the file is fully loaded, but before the callbacks are processed!
-            DoCallbacks(true);
+            logdev("DataLoaderRunnable: Done with '%s' (%s)", name, FilesizeFormat(mb->size).c_str());
-            Destroy();
+            DoCallbacks(name, MDH_FILE_OK | MDH_FILE_JUST_LOADED);
         }
 
-        inline void AddCallback(callback_func func, void *ptr = NULL)
+        inline void AddCallback(callback_func func, void *ptr = NULL, ZThread::Condition *cond = NULL)
         {
-            _callbacks[func] = ptr;
+            callback_struct cbs;
+            cbs.func = func;
+            cbs.ptr = ptr;
+            cbs.cond = cond;
+            _callbacks.push_back(cbs);
         }
-        inline void SetName(std::string name)
+        inline void DoCallbacks(std::string fn, uint32 flags)
         {
-            _name = name;
+            for(CallbackStore::iterator it = _callbacks.begin(); it != _callbacks.end(); it++)
-        }
-        // if this class has done its work, delete self
-        inline void Destroy(void)
             {
-            delete this;
+                if(it->cond)
-        }
+                    it->cond->broadcast();
-        inline uint8 *GetBuf(void)
+                if(it->func)
-        {
+                    (*(it->func))(it->ptr, fn, flags);
-            return _buf;
-        }
-        inline void DoCallbacks(bool success = true)
-        {
-            for(std::map<callback_func,void*>::iterator it = _callbacks.begin(); it != _callbacks.end(); it++)
-            {
-                (*(it->first))(it->second,success);
             }
         }
         inline void SetThreaded(bool t)
@@ -83,28 +113,51 @@ namespace MemoryDataHolder
         {
             return _threaded;
         }
-        inline bool HasCallbackFunc(callback_func f)
+        inline void SetName(std::string n)
         {
-            return _callbacks.find(f) != _callbacks.end();
+            _name = n;
         }
 
-
+       CallbackStore _callbacks;
-   private:
-       std::string _name;
-       std::map<callback_func, void*> _callbacks;
-       uint8 *_buf;
        bool _threaded;
+       std::string _name;
     };
 
 
-    uint8 *GetFile(std::string s, bool threaded = false, callback_func func = NULL,void *ptr = NULL)
+    memblock GetFile(std::string s, bool threaded, callback_func func, void *ptr, ZThread::Condition *cond, bool ref_counted)
     {
-        mutex.acquire(); // we need excusive access, other threads might unload the requested file during checking
+        mutex.acquire(); // we need exclusive access, other threads might unload the requested file during checking
-        if(uint8 **buf = storage.GetNoCreate(s))
+
+        if(alwaysSingleThreaded)
+            threaded = false;
+
+        // manage reference counter
+        uint32 *refcount = refs.GetNoCreate(s);
+        if(!refcount || !*refcount)
         {
-            // the file was requested some other time, is still present in memory and the pointer can simply be returned
+            refcount = new uint32;
-            mutex.release(); // everything ok, mutex can be unloaded safely before returning
+            *refcount = ref_counted ? 1 : 0;
-            return *buf;
+            refs.Assign(s,refcount);
+        }
+        else
+        {
+            if(ref_counted)
+            {
+                (*refcount)++;
+            }
+        }
+
+        if(memblock *mb = storage.GetNoCreate(s))
+        {
+            // the file was requested some other time, is still present in memory and the pointer can simply be returned...
+            mutex.release(); // everything ok, mutex can be unloaded safely
+            // execute callback and broadcast condition (must check for MDH_FILE_ALREADY_EXIST in callback func)
+            if(func)
+                (*func)(ptr, s, MDH_FILE_OK | MDH_FILE_ALREADY_EXIST);
+            if(cond)
+                cond->broadcast();
+
+            return *mb;
         }
         else
         {
@@ -114,30 +167,82 @@ namespace MemoryDataHolder
                 // no loader thread is working on that file...
                 r = new DataLoaderRunnable();
                 loaders.Assign(s,r);
-                r->AddCallback(func,ptr); // not threadsafe!
+                r->AddCallback(func,ptr,cond); // not threadsafe!
-                // after assigning/registering a new loader to the file, the mutex can be released safely
-                mutex.release();
-                r->SetName(s);
                 r->SetThreaded(threaded);
+                r->SetName(s); // here we set the filename the thread should load
+                // the mutex can be released safely now
+                mutex.release();
                 if(threaded)
                 {
-                    ZThread::Thread t(r); // start thread
+                    ZThread::Task task(r);
+                    executor->execute(task);
                 }
                 else
                 {
-                    r->run(); // will exit after the whole file is loaded and the (one) callback is run
+                    r->run(); // will exit after the whole file is loaded and the callbacks were run
-                    return r->GetBuf();
+                    memblock *mb = storage.GetNoCreate(s);
+                    delete r;
+                    return *mb;
                 }
             }
             else // if a loader is already existing, add callbacks to that loader.
             {  
-                r->AddCallback(func,ptr);
+                r->AddCallback(func,ptr,cond);
                 mutex.release();
             }
         }
-        return NULL;
+        return memblock();
     }
 
+    bool IsLoaded(std::string s)
+    {
+        ZThread::Guard<ZThread::FastMutex> g(mutex);
+        return storage.Exists(s);
+    }
+
+    // ensure the file is present in memory, but do not touch the reference counter
+    void BackgroundLoadFile(std::string s)
+    {
+        GetFile(s, true, NULL, NULL, NULL, false);
+    }
+
+
+    bool Delete(std::string s)
+    {
+        ZThread::Guard<ZThread::FastMutex> g(mutex);
+        uint32 *refcount = refs.GetNoCreate(s);
+        if(!refcount)
+        {
+            logerror("MemoryDataHolder:Delete(\"%s\"): no refcount", s.c_str());
+            return false;
+        }
+        else
+        {
+            if(*refcount)
+                (*refcount)--;
+            logdev("MemoryDataHolder::Delete(\"%s\"): refcount dropped to %u", s.c_str(), *refcount);
+        }
+        if(!*refcount)
+        {
+            refs.Delete(s);
+            if(memblock *mb = storage.GetNoCreate(s))
+            {
+                logdev("MemoryDataHolder:: deleting 0x%X (size %u)", mb->ptr, mb->size);
+                mb->free();
+                storage.Delete(s);
+                return true;
+            }
+            else
+            {
+                logerror("MemoryDataHolder::Delete(\"%s\"): no buf existing",s.c_str());
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+
 
 
 };

src/Client/MemoryDataHolder.h

@@ -3,12 +3,48 @@
 
 #include "common.h"
 
+namespace ZThread
+{
+    class Condition;
+};
+
 namespace MemoryDataHolder
 {
-    typedef void (*callback_func)(void*,bool);
+    enum ResultFlags
+    {
+        MDH_FILE_ERROR = 0, // file doesnt exist, cant be loaded, etc
+        MDH_FILE_OK = 1, // file was loaded properly or already present in memory. point is: we have good data
+        MDH_FILE_ALREADY_EXIST = 2, // file was loaded before
+        MDH_FILE_JUST_LOADED = 4, // file was freshly loaded
+    };
 
+    typedef void (*callback_func)(void *ptr,std::string filename, uint32 flags);
+    struct callback_struct
+    {
+        callback_func func;
+        void *ptr;
+        ZThread::Condition *cond;
+    };
+    typedef std::deque<callback_struct> CallbackStore;
 
-    uint8 *GetFile(std::string&,bool,callback_func,void*);
+    struct memblock
+    {
+        memblock() : ptr(NULL), size(0) {}
+        memblock(uint8 *p, uint32 s) : ptr(p), size(s) {}
+        void alloc(uint32 s) { size = s; ptr = new uint8[s]; }
+        void free(void) { delete [] ptr; }
+        uint8 *ptr;
+        uint32 size;
+    };
+
+    void Init(void);
+    void SetThreadCount(uint32);
+
+    memblock GetFile(std::string s, bool threaded = false, callback_func func = NULL,void *ptr = NULL, ZThread::Condition *cond = NULL, bool ref_counted = true);
+    inline memblock GetFileBasic(std::string s) { return GetFile(s, false, NULL, NULL, NULL, false); }
+    bool IsLoaded(std::string);
+    void BackgroundLoadFile(std::string);
+    bool Delete(std::string);
 };
 
 #endif

src/Client/PseuWoW.cpp

@@ -15,6 +15,7 @@
 #include "RemoteController.h"
 #include "Cli.h"
 #include "GUI/SceneData.h"
+#include "MemoryDataHolder.h"
 
 
 //###### Start of program code #######
@@ -490,6 +491,7 @@ void PseuInstanceConf::ApplyFromVarSet(VarSet &v)
     skipaddonchat=(bool)atoi(v.Get("SKIPADDONCHAT").c_str());
     dumpPackets=(uint8)atoi(v.Get("DUMPPACKETS").c_str());
     softquit=(bool)atoi(v.Get("SOFTQUIT").c_str());
+    dataLoaderThreads=atoi(v.Get("DATALOADERTHREADS").c_str());
 
     // clientversion is a bit more complicated to add
     {
@@ -521,7 +523,9 @@ void PseuInstanceConf::ApplyFromVarSet(VarSet &v)
     fov = atof(v.Get("GUI::FOV").c_str());
     masterSoundVolume = atof(v.Get("GUI::MASTERSOUNDVOLUME").c_str());
 
+    // cleanups, internal settings, etc.
     log_setloglevel(debug);
+    MemoryDataHolder::SetThreadCount(dataLoaderThreads);
 }
 
 

src/Client/PseuWoW.h

@@ -18,6 +18,18 @@ class PseuInstanceRunnable;
 class CliRunnable;
 class RemoteController;
 
+// possible conditions threads can wait for. used for thread synchronisation. extend if needed.
+enum InstanceConditions
+{
+    COND_GUI_INITIALIZED,
+    COND_GUI_SCENE_CHANGED,
+    COND_GUI_CLOSED,
+    COND_MAP_LOADED,
+
+    COND_MAX
+};
+
+
 class PseuInstanceConf
 {
     public:
@@ -59,6 +71,7 @@ class PseuInstanceConf
     bool skipaddonchat;
     uint8 dumpPackets;
     bool softquit;
+    uint8 dataLoaderThreads;
 
     // gui related
     bool enablegui;

src/Client/World/WorldSession.cpp

@@ -534,12 +534,7 @@ void WorldSession::_HandleAuthChallengeOpcode(WorldPacket& recvPacket)
         WorldPacket auth;
         auth<<(uint32)(GetInstance()->GetConf()->clientbuild)<<unk<<acc<<unk<<clientseed_uint32;
         auth.append(digest.GetDigest(),20);
-        auth << (uint32)0;
+        auth << (uint32)0; // TODO: this is not correct value, expected: 160 bytes of addon_data
-        //uint8 *addon_info = MemoryDataHolder::GetFileBasic("data/packet/addon_info.bin");
-        //if(addon_info)
-        //{
-        //    logdebug("Auth: Using custom addon info, %s", FilesizeFormat(
-        //    auth.append(addon_info, 160);
 
         auth.SetOpcode(CMSG_AUTH_SESSION);
 

src/Client/main.cpp

@@ -3,6 +3,7 @@
 #include "common.h"
 #include "main.h"
 #include "PseuWoW.h"
+#include "MemoryDataHolder.h"
 
 
 std::list<PseuInstanceRunnable*> instanceList; // TODO: move this to a "Master" class later
@@ -77,7 +78,8 @@ void _new_handler(void)
     throw;
 }
 
-int main(int argc, char* argv[]) {
+int main(int argc, char* argv[])
+{
     try
     {
         std::set_new_handler(_new_handler);
@@ -91,6 +93,7 @@ int main(int argc, char* argv[]) {
         logcustom(0,GREEN,"Compiled: %s  %s",__DATE__,__TIME__);
         
         _HookSignals();
+        MemoryDataHolder::Init();
 
         // 1 instance is enough for now
         PseuInstanceRunnable *r=new PseuInstanceRunnable();

src/PseuWoW.vcproj

@@ -186,14 +186,12 @@
 			<File
 				RelativePath=".\Client\MemoryDataHolder.cpp">
 				<FileConfiguration
-					Name="Release|Win32"
+					Name="Release|Win32">
-					ExcludedFromBuild="TRUE">
 					<Tool
 						Name="VCCLCompilerTool"/>
 				</FileConfiguration>
 				<FileConfiguration
-					Name="Debug|Win32"
+					Name="Debug|Win32">
-					ExcludedFromBuild="TRUE">
 					<Tool
 						Name="VCCLCompilerTool"/>
 				</FileConfiguration>