Optimize minimap save/load

BUGS

@@ -1,5 +1,6 @@
 == CRASHS
 modules recursivity makes client crash, it should generate a warning
+boost::asio makes the client crash in rarely conditions when trying to connect but internet is offline
 
 == P1 BUGS (affects game play)
 in some situations creatures may disappears while walking

src/framework/application.cpp

@@ -116,11 +116,11 @@ void Application::init(const std::string& compactName, const std::vector<std::st
 
 void Application::deinit()
 {
-    g_lua.callGlobalField("g_app", "onTerminate");
-
     // hide the window because there is no render anymore
     g_window.hide();
 
+    g_lua.callGlobalField("g_app", "onTerminate");
+
     // run modules unload events
     g_modules.unloadModules();
     g_modules.clear();

src/framework/core/filestream.cpp

@@ -25,11 +25,13 @@
 
 #include <physfs.h>
 
-FileStream::FileStream(const std::string& name, PHYSFS_File *fileHandle)
+FileStream::FileStream(const std::string& name, PHYSFS_File *fileHandle, bool writeable) :
+    m_name(name),
+    m_fileHandle(fileHandle),
+    m_pos(0),
+    m_writeable(writeable),
+    m_caching(false)
 {
-    m_name = name;
-    m_pos = 0;
-    m_fileHandle = fileHandle;
 }
 
 FileStream::~FileStream()
@@ -40,18 +42,22 @@ FileStream::~FileStream()
 
 void FileStream::cache()
 {
-    if(!m_fileHandle)
+    m_caching = true;
-        return;
 
-    // cache entire file into data buffer
+    if(!m_writeable) {
-    m_pos = PHYSFS_tell(m_fileHandle);
+        if(!m_fileHandle)
-    PHYSFS_seek(m_fileHandle, 0);
+            return;
-    int size = PHYSFS_fileLength(m_fileHandle);
-    m_data.resize(size);
-    if(PHYSFS_read(m_fileHandle, &m_data[0], size, 1) == -1)
-        throwError("unable to read file data", true);
 
-    close();
+        // cache entire file into data buffer
+        m_pos = PHYSFS_tell(m_fileHandle);
+        PHYSFS_seek(m_fileHandle, 0);
+        int size = PHYSFS_fileLength(m_fileHandle);
+        m_data.resize(size);
+        if(PHYSFS_read(m_fileHandle, m_data.data(), size, 1) == -1)
+            throwError("unable to read file data", true);
+        PHYSFS_close(m_fileHandle);
+        m_fileHandle = nullptr;
+    }
 }
 
 void FileStream::close()
@@ -60,21 +66,34 @@ void FileStream::close()
         if(!PHYSFS_close(m_fileHandle))
             throwError("close failed", true);
         m_fileHandle = nullptr;
-    } else {
-        m_data.clear();
-        m_pos = 0;
     }
+
+    m_data.clear();
+    m_pos = 0;
 }
 
 void FileStream::flush()
 {
-    if(m_fileHandle && PHYSFS_flush(m_fileHandle) == 0)
+    if(!m_writeable)
-        throwError("flush failed", true);
+        throwError("filestream is not writeable");
+
+    if(m_fileHandle) {
+        if(m_caching) {
+            if(!PHYSFS_seek(m_fileHandle, 0))
+                throwError("flush seek failed", true);
+            uint len = m_data.size();
+            if(PHYSFS_write(m_fileHandle, m_data.data(), 1, len) != len)
+                throwError("flush write failed", true);
+        }
+
+        if(PHYSFS_flush(m_fileHandle) == 0)
+            throwError("flush failed", true);
+    }
 }
 
 int FileStream::read(void *buffer, uint32 size, uint32 nmemb)
 {
-    if(m_fileHandle) {
+    if(!m_caching) {
         int res = PHYSFS_read(m_fileHandle, buffer, size, nmemb);
         if(res == -1)
             throwError("read failed", true);
@@ -96,13 +115,19 @@ int FileStream::read(void *buffer, uint32 size, uint32 nmemb)
 
 void FileStream::write(const void *buffer, uint32 count)
 {
-    if(PHYSFS_write(m_fileHandle, buffer, 1, count) != count)
+    if(!m_caching) {
-        throwError("write failed", true);
+        if(PHYSFS_write(m_fileHandle, buffer, 1, count) != count)
+            throwError("write failed", true);
+    } else {
+        m_data.grow(m_pos + count);
+        memcpy(&m_data[m_pos], buffer, count);
+        m_pos += count;
+    }
 }
 
 void FileStream::seek(uint32 pos)
 {
-    if(m_fileHandle) {
+    if(!m_caching) {
         if(!PHYSFS_seek(m_fileHandle, pos))
             throwError("seek failed", true);
     } else {
@@ -119,7 +144,7 @@ void FileStream::skip(uint len)
 
 int FileStream::size()
 {
-    if(m_fileHandle)
+    if(!m_caching)
         return PHYSFS_fileLength(m_fileHandle);
     else
         return m_data.size();
@@ -127,7 +152,7 @@ int FileStream::size()
 
 int FileStream::tell()
 {
-    if(m_fileHandle)
+    if(!m_caching)
         return PHYSFS_tell(m_fileHandle);
     else
         return m_pos;
@@ -136,7 +161,7 @@ int FileStream::tell()
 uint8 FileStream::getU8()
 {
     uint8 v = 0;
-    if(m_fileHandle) {
+    if(!m_caching) {
         if(PHYSFS_read(m_fileHandle, &v, 1, 1) != 1)
             throwError("read failed", true);
     } else {
@@ -152,7 +177,7 @@ uint8 FileStream::getU8()
 uint16 FileStream::getU16()
 {
     uint16 v = 0;
-    if(m_fileHandle) {
+    if(!m_caching) {
         if(PHYSFS_readULE16(m_fileHandle, &v) == 0)
             throwError("read failed", true);
     } else {
@@ -168,7 +193,7 @@ uint16 FileStream::getU16()
 uint32 FileStream::getU32()
 {
     uint32 v = 0;
-    if(m_fileHandle) {
+    if(!m_caching) {
         if(PHYSFS_readULE32(m_fileHandle, &v) == 0)
             throwError("read failed", true);
     } else {
@@ -184,7 +209,7 @@ uint32 FileStream::getU32()
 uint64 FileStream::getU64()
 {
     uint64 v = 0;
-    if(m_fileHandle) {
+    if(!m_caching) {
         if(PHYSFS_readULE64(m_fileHandle, (PHYSFS_uint64*)&v) == 0)
             throwError("read failed", true);
     } else {
@@ -223,26 +248,49 @@ std::string FileStream::getString()
 
 void FileStream::addU8(uint8 v)
 {
-    if(PHYSFS_write(m_fileHandle, &v, 1, 1) != 1)
+    if(!m_caching) {
-        throwError("write failed", true);
+        if(PHYSFS_write(m_fileHandle, &v, 1, 1) != 1)
+            throwError("write failed", true);
+    } else {
+        m_data.add(v);
+        m_pos++;
+    }
 }
 
 void FileStream::addU16(uint16 v)
 {
-    if(PHYSFS_writeULE16(m_fileHandle, v) == 0)
+    if(!m_caching) {
-        throwError("write failed", true);
+        if(PHYSFS_writeULE16(m_fileHandle, v) == 0)
+            throwError("write failed", true);
+    } else {
+        m_data.grow(m_pos + 2);
+        stdext::writeLE16(&m_data[m_pos], v);
+        m_pos += 2;
+    }
 }
 
 void FileStream::addU32(uint32 v)
 {
-    if(PHYSFS_writeULE32(m_fileHandle, v) == 0)
+    if(!m_caching) {
-        throwError("write failed", true);
+        if(PHYSFS_writeULE32(m_fileHandle, v) == 0)
+            throwError("write failed", true);
+    } else {
+        m_data.grow(m_pos + 4);
+        stdext::writeLE32(&m_data[m_pos], v);
+        m_pos += 4;
+    }
 }
 
 void FileStream::addU64(uint64 v)
 {
-    if(PHYSFS_writeULE64(m_fileHandle, v) == 0)
+    if(!m_caching) {
-        throwError("write failed", true);
+        if(PHYSFS_writeULE64(m_fileHandle, v) == 0)
+            throwError("write failed", true);
+    } else {
+        m_data.grow(m_pos + 8);
+        stdext::writeLE64(&m_data[m_pos], v);
+        m_pos += 8;
+    }
 }
 
 void FileStream::addString(const std::string& v)

src/framework/core/filestream.h

@@ -25,6 +25,7 @@
 
 #include "declarations.h"
 #include <framework/luascript/luaobject.h>
+#include <framework/util/databuffer.h>
 
 struct PHYSFS_File;
 
@@ -32,7 +33,7 @@ struct PHYSFS_File;
 class FileStream : public LuaObject
 {
 public:
-    FileStream(const std::string& name, PHYSFS_File *fileHandle);
+    FileStream(const std::string& name, PHYSFS_File *fileHandle, bool writeable);
     ~FileStream();
 
     void cache();
@@ -64,9 +65,11 @@ private:
 
     std::string m_name;
     PHYSFS_File *m_fileHandle;
-
-    std::vector<uint8_t> m_data;
     uint m_pos;
+    bool m_writeable;
+    bool m_caching;
+
+    DataBuffer<uint8_t> m_data;
 };
 
 #endif

src/framework/core/resourcemanager.cpp

@@ -209,7 +209,7 @@ FileStreamPtr ResourceManager::openFile(const std::string& fileName)
         g_logger.error(stdext::format("unable to open file '%s': %s", fullPath, PHYSFS_getLastError()));
         return nullptr;
     }
-    return FileStreamPtr(new FileStream(fullPath, file));
+    return FileStreamPtr(new FileStream(fullPath, file, false));
 }
 
 FileStreamPtr ResourceManager::appendFile(const std::string& fileName)
@@ -219,7 +219,7 @@ FileStreamPtr ResourceManager::appendFile(const std::string& fileName)
         g_logger.error(stdext::format("failed to append file '%s': %s", fileName, PHYSFS_getLastError()));
         return nullptr;
     }
-    return FileStreamPtr(new FileStream(fileName, file));
+    return FileStreamPtr(new FileStream(fileName, file, true));
 }
 
 FileStreamPtr ResourceManager::createFile(const std::string& fileName)
@@ -229,7 +229,7 @@ FileStreamPtr ResourceManager::createFile(const std::string& fileName)
         g_logger.error(stdext::format("failed to create file '%s': %s", fileName, PHYSFS_getLastError()));
         return nullptr;
     }
-    return FileStreamPtr(new FileStream(fileName, file));
+    return FileStreamPtr(new FileStream(fileName, file, true));
 }
 
 bool ResourceManager::deleteFile(const std::string& fileName)

src/framework/luascript/luaobject.cpp

@@ -25,7 +25,8 @@
 
 #include <framework/application.h>
 
-LuaObject::LuaObject() : m_fieldsTableRef(-1)
+LuaObject::LuaObject() :
+    m_fieldsTableRef(-1)
 {
 }
 

src/framework/util/databuffer.h

@@ -27,42 +27,75 @@ template<class T>
 class DataBuffer
 {
 public:
-    DataBuffer(int res = 64) {
+    DataBuffer(uint res = 64) :
-        m_capacity = res;
+        m_size(0),
-        m_buffer = new T[m_capacity];
+        m_capacity(res),
-        m_size = 0;
+        m_buffer(new T[m_capacity]) { }
+    ~DataBuffer() {
+        if(m_buffer)
+            delete[] m_buffer;
     }
-    ~DataBuffer() { delete[] m_buffer; }
 
     inline void reset() { m_size = 0; }
-    inline bool isEmpty() const { return m_size == 0; }
+    inline void clear() {
+        m_size = 0;
+        m_capacity = 0;
+        delete[] m_buffer;
+        m_buffer = nullptr;
+    }
 
-    inline int size() const { return m_size; }
+    inline bool empty() const { return m_size == 0; }
+    inline uint size() const { return m_size; }
     inline T *data() const { return m_buffer; }
 
-    inline const T& at(int i) const { return m_buffer[i]; }
+    inline const T& at(uint i) const { return m_buffer[i]; }
     inline const T& last() const { return m_buffer[m_size-1]; }
     inline const T& first() const { return m_buffer[0]; }
-    inline const T& operator[](int i) const { return m_buffer[i]; }
+    inline const T& operator[](uint i) const { return m_buffer[i]; }
-    inline T& operator[](int i) { return m_buffer[i]; }
+    inline T& operator[](uint i) { return m_buffer[i]; }
 
-    inline void add(const T &t) {
+    inline void reserve(uint n) {
-        if(m_size >= m_capacity) {
+        if(n > m_capacity) {
-            m_capacity *= 2;
+            T *buffer = new T[n];
-            T *buffer = new T[m_capacity];
+            for(uint i=0;i<m_size;++i)
-            for(int i=0;i<m_size;++i)
                 buffer[i] = m_buffer[i];
-            delete[] m_buffer;
+            if(m_buffer)
+                delete[] m_buffer;
             m_buffer = buffer;
+            m_capacity = n;
+        }
+    }
+
+    inline void resize(uint n, T def = T()) {
+        if(n == m_size)
+            return;
+        reserve(n);
+        for(uint i=m_size;i<n;++i)
+            m_buffer[i] = def;
+        m_size = n;
+    }
+
+    inline void grow(uint n) {
+        if(n <= m_size)
+            return;
+        if(n > m_capacity) {
+            uint newcapacity = m_capacity;
+            do { newcapacity *= 2; } while(newcapacity < n);
+            reserve(newcapacity);
         }
-        m_buffer[m_size++] = t;
+        m_size = n;
+    }
+
+    inline void add(const T& v) {
+        grow(m_size + 1);
+        m_buffer[m_size-1] = v;
     }
 
     inline DataBuffer &operator<<(const T &t) { add(t); return *this; }
 
 private:
-    int m_size;
+    uint m_size;
-    int m_capacity;
+    uint m_capacity;
     T *m_buffer;
 };
 

src/otclient/core/item.cpp

@@ -32,12 +32,14 @@
 #include <framework/graphics/graphics.h>
 #include <framework/core/filestream.h>
 
-Item::Item() : Thing()
+Item::Item() :
+    m_id(0),
+    m_countOrSubType(1),
+    m_actionId(0),
+    m_uniqueId(0),
+    m_shaderProgram(g_shaders.getDefaultItemShader()),
+    m_otbType(g_things.getNullOtbType())
 {
-    m_id = 0;
-    m_countOrSubType = 1;
-    m_shaderProgram = g_shaders.getDefaultItemShader();
-    m_otbType = g_things.getNullOtbType();
 }
 
 ItemPtr Item::create(int id)

src/otclient/core/map.cpp

@@ -286,12 +286,16 @@ bool Map::loadOtcm(const std::string& fileName)
 {
     try {
         FileStreamPtr fin = g_resources.openFile(fileName);
+        fin->cache();
 
         uint32 signature = fin->getU32();
         if(signature != OTCM_SIGNATURE)
             stdext::throw_exception("invalid otcm file");
 
+        uint16 start = fin->getU16();
         uint16 version = fin->getU16();
+        fin->getU32(); // flags
+
         switch(version) {
             case 1: {
                 fin->getString(); // description
@@ -309,8 +313,11 @@ bool Map::loadOtcm(const std::string& fileName)
                 break;
         }
 
+        fin->seek(start);
+
         while(true) {
             Position pos;
+
             pos.x = fin->getU16();
             pos.y = fin->getU16();
             pos.z = fin->getU8();
@@ -338,6 +345,7 @@ bool Map::loadOtcm(const std::string& fileName)
         }
 
         fin->close();
+
         return true;
     } catch(stdext::exception& e) {
         g_logger.error(stdext::format("failed to load OTCM map: %s", e.what()));
@@ -348,17 +356,34 @@ bool Map::loadOtcm(const std::string& fileName)
 void Map::saveOtcm(const std::string& fileName)
 {
     try {
+        g_clock.update();
+
         FileStreamPtr fin = g_resources.createFile(fileName);
+        fin->cache();
+
+        //TODO: compression flag with zlib
+        uint32 flags = 0;
 
+        // header
         fin->addU32(OTCM_SIGNATURE);
+        fin->addU16(0); // data start, will be overwritten later
         fin->addU16(OTCM_VERSION);
-        fin->addString("OTCM 1.0");
+        fin->addU32(flags);
+
+        // version 1 header
+        fin->addString("OTCM 1.0"); // map description
         fin->addU32(g_things.getDatSignature());
         fin->addU16(g_game.getProtocolVersion());
         fin->addString(g_game.getWorldName());
 
+        // go back and rewrite where the map data starts
+        uint32 start = fin->tell();
+        fin->seek(4);
+        fin->addU16(start);
+        fin->seek(start);
+
         for(auto& pair : m_tiles) {
-            TilePtr tile = pair.second;
+            const TilePtr& tile = pair.second;
             if(!tile || tile->isEmpty())
                 continue;
 
@@ -424,14 +449,18 @@ void Map::addThing(const ThingPtr& thing, const Position& pos, int stackPos)
     if(!thing)
         return;
 
-    Position oldPos = thing->getPosition();
     TilePtr tile = getOrCreateTile(pos);
 
-    if(MissilePtr missile = thing->asMissile()) {
+    Position oldPos = thing->getPosition();
-        m_floorMissiles[pos.z].push_back(missile);
+
-    } else if(AnimatedTextPtr animatedText = thing->asAnimatedText()) {
+    if(thing->isItem() || thing->isCreature() || thing->isEffect()) {
-        m_animatedTexts.push_back(animatedText);
+        tile->addThing(thing, stackPos);
-    } else if(StaticTextPtr staticText = thing->asStaticText()) {
+    } else if(thing->isMissile()) {
+        m_floorMissiles[pos.z].push_back(thing->asMissile());
+    } else if(thing->isAnimatedText()) {
+        m_animatedTexts.push_back(thing->asAnimatedText());
+    } else if(thing->isStaticText()) {
+        StaticTextPtr staticText = thing->asStaticText();
         bool mustAdd = true;
         for(auto it = m_staticTexts.begin(), end = m_staticTexts.end(); it != end; ++it) {
             StaticTextPtr cStaticText = *it;
@@ -449,14 +478,13 @@ void Map::addThing(const ThingPtr& thing, const Position& pos, int stackPos)
 
         if(mustAdd)
             m_staticTexts.push_back(staticText);
-    } else {
-        tile->addThing(thing, stackPos);
     }
 
     thing->startAnimation();
     thing->setPosition(pos);
 
-    if(CreaturePtr creature = thing->asCreature()) {
+    if(thing->isCreature()) {
+        CreaturePtr creature = thing->asCreature();
         if(oldPos != pos) {
             if(oldPos.isInRange(pos,1,1))
                 g_game.processCreatureMove(creature, oldPos, pos);

src/otclient/core/map.h

@@ -76,7 +76,7 @@ enum OTBM_NodeTypes_t
 };
 
 enum {
-    OTCM_SIGNATURE = 0x12345678,
+    OTCM_SIGNATURE = 0x4D43544F,
     OTCM_VERSION = 1,
 };
 

src/otclient/core/shadermanager.h

@@ -44,8 +44,8 @@ public:
     PainterShaderProgramPtr createItemShader(const std::string& name, const std::string& file);
     PainterShaderProgramPtr createMapShader(const std::string& name, const std::string& file) { return createFragmentShader(name, file); }
 
-    PainterShaderProgramPtr getDefaultItemShader() { return m_defaultItemShader; }
+    const PainterShaderProgramPtr& getDefaultItemShader() { return m_defaultItemShader; }
-    PainterShaderProgramPtr getDefaultMapShader() { return m_defaultMapShader; }
+    const PainterShaderProgramPtr& getDefaultMapShader() { return m_defaultMapShader; }
 
     PainterShaderProgramPtr getShader(const std::string& name);
 

src/otclient/core/thing.cpp

@@ -28,9 +28,9 @@
 #include "tile.h"
 #include "game.h"
 
-Thing::Thing()
+Thing::Thing() :
+    m_datType(g_things.getNullDatType())
 {
-    m_datType = g_things.getNullDatType();
 }
 
 int Thing::getStackPriority()

src/otclient/core/thingtypemanager.cpp

@@ -247,16 +247,20 @@ void ThingTypeManager::addOtbType(const ThingTypeOtbPtr& otbType)
     m_otbTypes[id] = otbType;
 }
 
-ThingTypeOtbPtr ThingTypeManager::findOtbForClientId(uint16 id)
+const ThingTypeOtbPtr& ThingTypeManager::findOtbForClientId(uint16 id)
 {
+    if(m_otbTypes.empty())
+        return m_nullOtbType;
+
     for(const ThingTypeOtbPtr& otbType : m_otbTypes) {
         if(otbType->getClientId() == id)
             return otbType;
     }
+
     return m_nullOtbType;
 }
 
-ThingTypeDatPtr ThingTypeManager::getDatType(uint16 id, DatCategory category)
+const ThingTypeDatPtr& ThingTypeManager::getDatType(uint16 id, DatCategory category)
 {
     if(category >= DatLastCategory || id >= m_datTypes[category].size()) {
         g_logger.error(stdext::format("invalid thing type client id %d in category %d", id, category));
@@ -265,7 +269,7 @@ ThingTypeDatPtr ThingTypeManager::getDatType(uint16 id, DatCategory category)
     return m_datTypes[category][id];
 }
 
-ThingTypeOtbPtr ThingTypeManager::getOtbType(uint16 id)
+const ThingTypeOtbPtr& ThingTypeManager::getOtbType(uint16 id)
 {
     if(id >= m_otbTypes.size()) {
         g_logger.error(stdext::format("invalid thing type server id %d", id));

src/otclient/core/thingtypemanager.h

@@ -40,13 +40,13 @@ public:
     bool loadXml(const std::string& file);
 
     void addOtbType(const ThingTypeOtbPtr& otbType);
-    ThingTypeOtbPtr findOtbForClientId(uint16 id);
+    const ThingTypeOtbPtr& findOtbForClientId(uint16 id);
 
-    ThingTypeDatPtr& getNullDatType() { return m_nullDatType; }
+    const ThingTypeDatPtr& getNullDatType() { return m_nullDatType; }
-    ThingTypeOtbPtr& getNullOtbType() { return m_nullOtbType; }
+    const ThingTypeOtbPtr& getNullOtbType() { return m_nullOtbType; }
 
-    ThingTypeDatPtr getDatType(uint16 id, DatCategory category);
+    const ThingTypeDatPtr& getDatType(uint16 id, DatCategory category);
-    ThingTypeOtbPtr getOtbType(uint16 id);
+    const ThingTypeOtbPtr& getOtbType(uint16 id);
 
     uint32 getDatSignature() { return m_datSignature; }
     uint32 getOtbVersion() { return m_otbVersion; }

src/otclient/core/tile.cpp

@@ -30,11 +30,12 @@
 #include <otclient/net/protocolgame.h>
 #include <framework/graphics/fontmanager.h>
 
-Tile::Tile(const Position& position)
+Tile::Tile(const Position& position) :
+    m_position(position),
+    m_drawElevation(0),
+    m_flags(0),
+    m_minimapColorByte(0)
 {
-    m_drawElevation = 0;
-    m_position = position;
-    m_minimapColorByte = 0;
 }
 
 void Tile::draw(const Point& dest, float scaleFactor, int drawFlags)