* implemented automatic detection of added SCP files (not only changed or deleted) which will trigger recompacting then.

* optimized the way SCP files are loaded and evaluated if they belong to a specific database or not (maybe little speedup too)
* added zlib compression for compacted SCP files. this will save a lot of space later when handling larger scp files (for ex. spell.scp)
* fixed memory leak in ZCompressor::Deflate()
* fixed crash/exit/stuck when too small data failed to compress
* fixed possible crash(?) in GetFileSize()
* fixed compiler warning

src/Client/GUI/CImageLoaderBLP.cpp

@@ -5,7 +5,7 @@
 #include "SImage.h"
 #include "CImageLoaderBLP.h"
 
-#ifdef _DEBUG
+#if defined(_DEBUG) && !defined(DEBUG)
 #define DEBUG(code) code;
 #else
 #define DEBUG(code) ;

src/Client/PseuWoW.cpp

@@ -119,6 +119,7 @@ bool PseuInstance::Init(void)
 
     dbmgr.AddSearchPath("./cache");
     dbmgr.AddSearchPath("./data/scp");
+    dbmgr.SetCompression(6);
 
     _scp->variables.Set("@version_short",_ver_short);
     _scp->variables.Set("@version",_ver);

src/Client/SCPDatabase.cpp

@@ -19,7 +19,8 @@ struct memblock
     uint8 *ptr;
     uint32 size;
 };
-TypeStorage<memblock> Pointers;
+TypeStorage<memblock> Pointers; // stores filename -> file content
+std::map<std::string,std::string> FileRelation; // stores filename -> DB name
 
 SCPDatabase::~SCPDatabase()
 {
@@ -215,6 +216,7 @@ uint32 SCPDatabaseMgr::AutoLoadFile(char *fn)
                 {
                     dbname = value;
                     db = GetDB(dbname,true); // create db if not existing
+                    FileRelation[fn] = dbname;
                 }
                 else if(db)
                         db->fields[id][entry] = value;
@@ -255,7 +257,7 @@ uint32 SCPDatabaseMgr::GetDataTypeFromString(char *s)
     return isint ? SCP_TYPE_INT : SCP_TYPE_FLOAT;
 }
 
-bool SCPDatabaseMgr::Compact(char *dbname, char *outfile)
+bool SCPDatabaseMgr::Compact(char *dbname, char *outfile, uint32 compression)
 {
     logdebug("Compacting database '%s' into file '%s'", dbname, outfile);
     SCPDatabase *db = GetDB(dbname);
@@ -280,7 +282,7 @@ bool SCPDatabaseMgr::Compact(char *dbname, char *outfile)
     stringdata << ""; // write an empty string so that offset 0 will always return empty string
 
     // the whole process is divided into 2 passes:
-    // - pass 0 autodetects the datatypes stored in the dirfferent entries (SCPFieldDef)
+    // - pass 0 autodetects the datatypes stored in the different entries (SCPFieldDef)
     // - pass 1 creates the data field holding the values and string offsets, and also stuffs string values into the ByteBuffer
     while(pass < 2)
     {
@@ -383,7 +385,7 @@ bool SCPDatabaseMgr::Compact(char *dbname, char *outfile)
     uint32 offsMD5, nMD5, sizeMD5;
     uint32 offsIndexes, nIndexes, sizeIndexes;
     uint32 offsFields, sizeFields;
-    uint32 offsData;
+    uint32 offsData, sizeData;
     uint32 offsStrings, sizeStrings;
 
     // MD5 hashes of source files
@@ -434,37 +436,61 @@ bool SCPDatabaseMgr::Compact(char *dbname, char *outfile)
     // -- most of it is handled somewhere above
     sizeStrings = stringdata.size();
 
-    // precalc absolute offsets, header is 18 bytes large
+    // calc data field size in bytes
-    offsMD5 = HEADER_SIZE;
+    sizeData = blocksize * sizeof(uint32);
+
+    // precalc relative offsets, after header (min 21 bytes, maybe some extra bytes)
+    offsMD5 = 0; // directly after header
     offsIndexes = offsMD5 + sizeMD5;
     offsFields = offsIndexes + sizeIndexes;
     offsData = offsFields + sizeFields;
-    offsStrings = offsData + blocksize * sizeof(uint32);
+    offsStrings = offsData + sizeData;
 
     // buffer the file header
-    ByteBuffer hbuf;
+    ByteBuffer hbuf(HEADER_SIZE);
     hbuf.append("SCPC",4); // identifier
-    hbuf << (uint32)0; // flags, not yet used
+
+    uint32 flags = 0;
+
+    hbuf << flags; // flags, placeholder; real value is put below
     hbuf << (uint32)0 << (uint32)0 << (uint32)0 << (uint32)0; // padding, not yet used
     hbuf << offsMD5 << nMD5 << sizeMD5;
     hbuf << offsIndexes << nIndexes << sizeIndexes;
     hbuf << offsFields << nFields << sizeFields;
-    hbuf << offsData << section << blocksize * sizeof(uint32);
+    hbuf << offsData << section << sizeData;
     hbuf << offsStrings << nStrings << sizeStrings;
 
+    ZCompressor z;
+    z.reserve(sizeMD5 + sizeIndexes + sizeFields + sizeData + sizeStrings);
+    z.append(md5buf);
+    z.append(indexbuf);
+    z.append(fieldbuf);
+    z.append((uint8*)membuf, sizeData);
+    z.append(stringdata);
+
+    if(compression)
+    {
+        z.Deflate(compression);
+        if(z.Compressed())
+        {
+            hbuf << z.RealSize();
+            flags |= SCP_FLAG_COMPRESSED;
+        }
+        else
+        {
+            logdebug("SCP Compact: Unable to compress '%s' (too small?)",outfile);
+            return false;
+        }
+    }
+
+    hbuf.put<uint32>(4,flags); // first 4 bytes are 'SCPC', then flags...
+
     FILE *fh = fopen(outfile,"wb");
     if(!fh)
         return false;
-    if(fh)
+
-    {
+    fwrite(hbuf.contents(), hbuf.size(), 1, fh);
-        fwrite(hbuf.contents(),hbuf.size(),1,fh);
+    fwrite(z.contents(), z.size(), 1, fh);
-        if(sizeMD5) // just in case no md5sums are stored
-            fwrite(md5buf.contents(),sizeMD5,1,fh);
-        fwrite(indexbuf.contents(),sizeIndexes,1,fh);
-        fwrite(fieldbuf.contents(),sizeFields,1,fh);
-        fwrite(membuf,sizeof(uint32),blocksize,fh);
-        fwrite(stringdata.contents(),sizeStrings,1,fh);
-    }
     fclose(fh);
 
     if(!db)
@@ -492,51 +518,27 @@ bool SCPDatabaseMgr::Compact(char *dbname, char *outfile)
     return true;
 }
 
-uint32 SCPDatabaseMgr::SearchAndLoad(char *dbname, bool no_compiled)
+void SCPDatabaseMgr::_FilterFiles(std::deque<std::string>& files, std::string dbname)
 {
-    uint32 count = 0;
+    for(std::deque<std::string>::iterator it = files.begin(); it != files.end(); )
-    std::deque<std::string> goodfiles;
-
-    for(std::deque<std::string>::iterator it = _paths.begin(); it != _paths.end(); it++)
-    {
-        std::deque<std::string> files = GetFileList(*it);
-        sort(files.begin(),files.end()); // rough alphabetical sort
-        for(std::deque<std::string>::iterator itf = files.begin(); itf != files.end(); itf++)
-        {
-            std::string& fn = *itf;
-            if(fn.length() < 5)
-                continue;
-            std::string filepath = *it + fn;
-            // check for special case: <dbname>.ccp in this directory? load it and skip the rest
-            if(!no_compiled && !stricmp(std::string(dbname).append(".ccp").c_str(), fn.c_str()))
-            {
-                logdebug("Loading pre-compacted database '%s%s'", it->c_str(), fn.c_str());
-                DropDB(dbname); // if sth got loaded before, remove that
-                // load SCC database file and skip rest
-                if(LoadCompactSCP((char*)filepath.c_str(), dbname))
-                {
-                    logdebug("Loaded '%s' -> %s, skipping scp files",filepath.c_str(),dbname);
-                    return 1;
-                }
-            }
-            else if(!stricmp(fn.c_str() + fn.length() - 4, ".scp"))
-            {
-                // skip 0-byte files
-                if(GetFileSize(filepath.c_str()))
-                    goodfiles.push_back(filepath);                
-            }
-        }
-    }
-
-    logdetail("Pre-compacted SCC file for '%s' invalid, creating from SCP (%u files total)",dbname,goodfiles.size());
-
-    for(std::deque<std::string>::iterator it = goodfiles.begin(); it != goodfiles.end(); it++)
     {
+        std::map<std::string,std::string>::iterator w;
         bool load_it = false;
+        // first check if the file was already loaded once, in this case use cached data
+        if( (w = FileRelation.find(*it)) != FileRelation.end() )
+        {
+            if(w->second == dbname)
+                load_it = true;
+        }
+        else // if not previously loaded, load now and cache
+        {
             std::fstream fh;
             fh.open( it->c_str() , std::ios_base::in | std::ios_base::binary);
             if( !fh.is_open() )
+            {
+                logerror("SCP: Can't open file '%s'", it->c_str());
                 continue;
+            }
 
             uint32 size = 1000; // search for #dbname tag in first 1000 bytes
             char *buf = new char[size];
@@ -556,30 +558,108 @@ uint32 SCPDatabaseMgr::SearchAndLoad(char *dbname, bool no_compiled)
                         line.erase(0,1);
                     if(line[0] == '#')
                     {
-                    if(!strnicmp(line.c_str(),"#dbname=",8) && !stricmp(line.c_str()+8, dbname))
+                        if(!strnicmp(line.c_str(),"#dbname=",8))
+                        {
+                            std::string t = line.c_str() + 8; // current db name
+                            FileRelation[*it] = t;
+                            if(!stricmp(t.c_str(), dbname.c_str()))
                             {
                                 load_it = true;
                                 break;
                             }
                         }
+                    }
                     line.clear();
                 }
                 else
                     line += buf[pos];
             }
             delete [] buf;
+        }
 
         if(load_it)
+            it++;
+        else
+        {
+            //DEBUG(logdebug("SCP: '%s' not used for [%s]", it->c_str(), dbname.c_str()));
+            it = files.erase(it);
+        }
+    }
+    DEBUG(logdebug("-> %u files belong to this DB",files.size()));
+}
+
+uint32 SCPDatabaseMgr::SearchAndLoad(char *dbname, bool no_compiled)
+{
+    uint32 count = 0;
+    std::deque<std::string> goodfiles;
+    std::string ccpFile;
+
+    for(std::deque<std::string>::iterator it = _paths.begin(); it != _paths.end(); it++)
+    {
+        std::deque<std::string> files = GetFileList(*it);
+        sort(files.begin(),files.end()); // rough alphabetical sort
+        for(std::deque<std::string>::iterator itf = files.begin(); itf != files.end(); itf++)
+        {
+            std::string& fn = *itf;
+            if(fn.length() < 5)
+                continue;
+            std::string filepath = *it + fn;
+            // check for special case: <dbname>.ccp in this directory? load it!
+            // others must be checked only for MD5-match and if new files are there not yet recorded in MD5
+            if(!no_compiled && !stricmp(std::string(dbname).append(".ccp").c_str(), fn.c_str()))
+            {
+                ccpFile = filepath;
+            }
+            else if(!stricmp(fn.c_str() + fn.length() - 4, ".scp"))
+            {
+                // skip 0-byte files
+                if(GetFileSize(filepath.c_str()))
+                    goodfiles.push_back(filepath);
+                else
+                    FileRelation[filepath] = ""; // empty files cant belong to a DB
+            }
+        }
+    }
+
+    // goodfiles stores a list of all scp files found, we need to remove those that are not required for this DB
+    _FilterFiles(goodfiles,dbname);
+
+    if(!goodfiles.size())
+    {
+        logerror("SCP: No files found that contain database [%u]", dbname);
+        return 0;
+    }
+
+    // string only exists if CCP file was found and if it should no be skipped
+    if(ccpFile.size())
+    {
+        logdebug("Loading pre-compacted database '%s'", ccpFile.c_str());
+        DropDB(dbname); // if sth got loaded before, remove that
+        // load SCC database file
+        if(LoadCompactSCP((char*)ccpFile.c_str(), dbname, goodfiles.size()))
+        {
+            logdebug("Loaded '%s' -> %s",ccpFile.c_str(),dbname);
+            return goodfiles.size();
+        }
+        else
+        {
+            logdetail("Pre-compacted SCC file for '%s' outdated, creating from SCP (%u files total)",dbname,goodfiles.size());
+        }
+    }
+
+    for(std::deque<std::string>::iterator it = goodfiles.begin(); it != goodfiles.end(); it++)
     {
         logdebug("File '%s' matching database '%s', loading", it->c_str(), dbname);
         count++;
         uint32 sections = AutoLoadFile((char*)it->c_str());
-        }
+        logdebug("%u sections loaded", sections);
     }
 
     char fn[100];
     sprintf(fn,"./cache/%s.ccp",dbname);
-    Compact(dbname, fn);
+    Compact(dbname, fn, _compr);
+
+    logdetail("Database '%s' loaded from source and compacted with compression %u", dbname, _compr);
 
     return count;
 }
@@ -616,7 +696,7 @@ void SCPDatabaseMgr::AddSearchPath(char *path)
     _paths.push_back(p);
 }
 
-bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
+bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname, uint32 nSourcefiles)
 {
     uint32 filesize = GetFileSize(fn);
     if(filesize < HEADER_SIZE)
@@ -632,10 +712,9 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
         return false;
     }
 
-    ByteBuffer bb;
+    ByteBuffer hbuf;
-    bb.resize(filesize);
+    hbuf.resize(HEADER_SIZE);
-    fh.read((char*)bb.contents(), filesize);
+    fh.read((char*)hbuf.contents(), HEADER_SIZE);
-    fh.close();
 
     char tag[4];
     uint32 flags, padding[4];
@@ -645,19 +724,44 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
     uint32 offsData, nRows, sizeData;
     uint32 offsStrings, nStrings, sizeStrings;
 
-    bb.read((uint8*)&tag[0],4);
+    uint32 realsize; // used when compressed
+
+    hbuf.read((uint8*)&tag[0],4);
     if(memcmp(tag,"SCPC",4))
     {
         logerror("'%s' is not a compact database file!",fn);
         return false;
     }
-    bb >> flags;
+    hbuf >> flags;
-    bb >> padding[0] >> padding[1] >> padding[2] >> padding[3];
+    hbuf >> padding[0] >> padding[1] >> padding[2] >> padding[3];
-    bb >> offsMD5 >> nMD5 >> sizeMD5;
+    hbuf >> offsMD5 >> nMD5 >> sizeMD5;
-    bb >> offsIndexes >> nIndexes >> sizeIndexes;
+    hbuf >> offsIndexes >> nIndexes >> sizeIndexes;
-    bb >> offsFields >> nFields >> sizeFields;
+    hbuf >> offsFields >> nFields >> sizeFields;
-    bb >> offsData >> nRows >> sizeData;
+    hbuf >> offsData >> nRows >> sizeData;
-    bb >> offsStrings >> nStrings >> sizeStrings;
+    hbuf >> offsStrings >> nStrings >> sizeStrings;
+
+    // read some extra bytes depending on flags
+    if(flags & SCP_FLAG_COMPRESSED)
+        fh.read((char*)&realsize, sizeof(uint32));
+
+    ZCompressor z;
+    uint32 remain = sizeMD5 + sizeIndexes + sizeFields + sizeData + sizeStrings;
+    z.resize(remain);
+    fh.read((char*)z.contents(), remain);
+
+    fh.close(); // All data are read from the file now, it can safely be closed
+
+    if(flags & SCP_FLAG_COMPRESSED)
+    {
+        z.Compressed(true);
+        z.RealSize(realsize);
+        z.Inflate();
+        if(z.Compressed())
+        {
+            logerror("LoadCompactSCP: Unable to uncompress '%s'",fn);
+            return false;
+        }
+    }
 
     SCPDatabase *db = GetDB(dbname,true);
     db->_name = dbname;
@@ -666,9 +770,11 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
     ByteBuffer md5buf(sizeMD5);
     md5buf.resize(sizeMD5);
     // read MD5 block
-    bb.rpos(offsMD5);
+    z.rpos(offsMD5);
-    if(bb.rpos() == offsMD5)
+    if(z.rpos() == offsMD5)
-        bb.read((uint8*)md5buf.contents(),sizeMD5);
+    {
+        z.read((uint8*)md5buf.contents(),sizeMD5);
+    }
     else
     {
         logerror("'%s' has wrong MD5 offset, can't load",fn);
@@ -710,6 +816,15 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
         }
     }
 
+    // check if there are any new files matching this database, that are not yet compacted and hashed.
+    // if the size differs now, and no changes were detected so far, there are probably new files added
+    if(nSourcefiles > nMD5)
+    {
+        logdebug("There are more source files existing then hashed in the CCP file, must recompact.");
+        return false;
+    }
+    ASSERT(nMD5 == nSourcefiles); // if we didnt return until now, something isnt good
+
     // everything good so far? we reached this point? then its likely that the rest of the file is ok, alloc remaining buffers
     ByteBuffer indexbuf(sizeIndexes);
     ByteBuffer fieldsbuf(sizeFields);
@@ -717,9 +832,9 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
     fieldsbuf.resize(sizeFields);
 
     // read indexes block
-    bb.rpos(offsIndexes);
+    z.rpos(offsIndexes);
-    if(bb.rpos() == offsIndexes)
+    if(z.rpos() == offsIndexes)
-        bb.read((uint8*)indexbuf.contents(),sizeIndexes);
+        z.read((uint8*)indexbuf.contents(),sizeIndexes);
     else
     {
         logerror("'%s' has wrong indexes offset, can't load",fn);
@@ -727,9 +842,9 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
     }
 
     // read field definitions buf
-    bb.rpos(offsFields);
+    z.rpos(offsFields);
-    if(bb.rpos() == offsFields)
+    if(z.rpos() == offsFields)
-        bb.read((uint8*)fieldsbuf.contents(),sizeFields);
+        z.read((uint8*)fieldsbuf.contents(),sizeFields);
     else
     {
         logerror("'%s' has wrong field defs offset, can't load",fn);
@@ -756,11 +871,11 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
 
     // read main data block
     ASSERT(nRows * nFields == sizeData / sizeof(uint32));
-    bb.rpos(offsData);
+    z.rpos(offsData);
-    if(bb.rpos() == offsData)
+    if(z.rpos() == offsData)
     {
         db->_intbuf = new uint32[nRows * nFields];
-        bb.read((uint8*)db->_intbuf, sizeData); // load this somewhat fast and without a for loop
+        z.read((uint8*)db->_intbuf, sizeData); // load this somewhat fast and without a for loop
     }
     else
     {
@@ -769,11 +884,11 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
     }
 
     // read strings
-    bb.rpos(offsStrings);
+    z.rpos(offsStrings);
-    if(bb.rpos() == offsStrings)
+    if(z.rpos() == offsStrings)
     {
         db->_stringbuf = new char[sizeStrings];
-        bb.read((uint8*)db->_stringbuf,sizeStrings);
+        z.read((uint8*)db->_stringbuf,sizeStrings);
     }
     else
     {
@@ -786,7 +901,7 @@ bool SCPDatabaseMgr::LoadCompactSCP(char *fn, char *dbname)
 
     db->DropTextData(); // delete pointers to file content created at md5 comparison
 
-    logdebug("'%s' loaded successfully",fn);
+    // all fine, DB loaded
 
     return true;
 }

src/Client/SCPDatabase.h

@@ -2,6 +2,7 @@
 #define _SCPDATABASE_H
 
 #include "DefScript/TypeStorage.h"
+#include "ZCompressor.h"
 #include <set>
 
 enum SCPFieldTypes
@@ -11,6 +12,11 @@ enum SCPFieldTypes
     SCP_TYPE_STRING = 2
 };
 
+enum SCPFlags
+{
+    SCP_FLAG_COMPRESSED = 1
+};
+
 struct SCPFieldDef
 {
     uint32 id;
@@ -89,18 +95,23 @@ class SCPDatabaseMgr
 {
     friend class SCPDatabase;
 public:
+    SCPDatabaseMgr() : _compr(0) {}
     SCPDatabase *GetDB(std::string n, bool create = false);
     uint32 AutoLoadFile(char *fn);
     inline void DropDB(std::string s) { _map.Delete(stringToLower(s)); }
-    bool Compact(char *dbname, char *outfile);
+    bool Compact(char *dbname, char *outfile, uint32 compression = 0);
     static uint32 GetDataTypeFromString(char *s);
     uint32 SearchAndLoad(char*,bool);
     void AddSearchPath(char*);
-    bool LoadCompactSCP(char*, char*);
+    bool LoadCompactSCP(char*, char*, uint32);
+    void SetCompression(uint32 c) { _compr = c; } // min=0, max=9
+    uint32 GetCompression(void) { return _compr; }
 
 private:
+    void _FilterFiles(std::deque<std::string>& files, std::string dbname);
     SCPDatabaseMap _map;
     std::deque<std::string> _paths;
+    uint32 _compr; // zlib compression level
 };
 
 

src/shared/ZCompressor.cpp

@@ -51,8 +51,7 @@ void ZCompressor::_compress(void* dst, uint32 *dst_size, void* src, uint32 src_s
     {
         //printf("Can't compress (zlib: deflate should report Z_STREAM_END)\n");
         *dst_size = 0;
-        //return;
+        return;
-        exit(1);
     }
 
     if (Z_OK != deflateEnd(&c_stream))
@@ -86,6 +85,7 @@ void ZCompressor::Deflate(uint8 level)
     rpos(0);
     wpos(0);
     append(buf,newsize);
+    delete [] buf;
 
     _iscompressed=true;
 
@@ -105,7 +105,7 @@ void ZCompressor::Inflate(void)
     result = uncompress(target, &origsize, (uint8*)contents(), size());
     if( result!=Z_OK || origsize!=_real_size)
     {
-        //printf("ZCompressor: Inflate error! result=%d cursize=%u origsize=%u realsize=%u\n",result,size(),origsize,_real_size);
+        logerror("ZCompressor: Inflate error! result=%d cursize=%u origsize=%u realsize=%u\n",result,size(),origsize,_real_size);
         delete [] target;
         return;
     }

src/shared/tools.cpp

@@ -201,6 +201,8 @@ uint32 getMSTime(void)
 
 uint32 GetFileSize(const char* sFileName)
 {
+    if(!sFileName || !*sFileName)
+        return 0;
     std::ifstream f;
     f.open(sFileName, std::ios_base::binary | std::ios_base::in);
     if (!f.good() || f.eof() || !f.is_open()) { return 0; }