Optimize overall memory usage

* Fixes in otbm loader
* Rework BinaryTree

src/framework/CMakeLists.txt

@@ -33,6 +33,7 @@ set(framework_SOURCES ${framework_SOURCES}
     ${CMAKE_CURRENT_LIST_DIR}/util/rsa.cpp
     ${CMAKE_CURRENT_LIST_DIR}/util/rsa.h
     ${CMAKE_CURRENT_LIST_DIR}/util/size.h
+    ${CMAKE_CURRENT_LIST_DIR}/util/attribstorage.h
 
     # stdext
     ${CMAKE_CURRENT_LIST_DIR}/stdext/cast.h

src/framework/core/binarytree.cpp

@@ -23,21 +23,54 @@
 #include "binarytree.h"
 #include "filestream.h"
 
-void BinaryTree::unserialize(const FileStreamPtr& fin)
+BinaryTree::BinaryTree(const FileStreamPtr& fin) :
+    m_fin(fin), m_pos(0xFFFFFFFF)
+{
+    m_startPos = fin->tell();
+}
+
+BinaryTree::~BinaryTree()
+{
+}
+
+void BinaryTree::skipNodes()
 {
     while(true) {
-        uint8 byte = fin->getU8();
+        uint8 byte = m_fin->getU8();
         switch(byte) {
             case BINARYTREE_NODE_START: {
-                BinaryTreePtr node(new BinaryTree(shared_from_this()));
+                skipNodes();
-                m_children.push_back(node);
-                node->unserialize(fin);
                 break;
             }
             case BINARYTREE_NODE_END:
                 return;
             case BINARYTREE_ESCAPE_CHAR:
-                m_buffer.add(fin->getU8());
+                m_fin->getU8();
+                break;
+            default:
+                break;
+        }
+    }
+}
+
+void BinaryTree::unserialize()
+{
+    if(m_pos != 0xFFFFFFFF)
+        return;
+    m_pos = 0;
+
+    m_fin->seek(m_startPos);
+    while(true) {
+        uint8 byte = m_fin->getU8();
+        switch(byte) {
+            case BINARYTREE_NODE_START: {
+                skipNodes();
+                break;
+            }
+            case BINARYTREE_NODE_END:
+                return;
+            case BINARYTREE_ESCAPE_CHAR:
+                m_buffer.add(m_fin->getU8());
                 break;
             default:
                 m_buffer.add(byte);
@@ -46,13 +79,33 @@ void BinaryTree::unserialize(const FileStreamPtr& fin)
     }
 }
 
-void BinaryTree::serialize(const FileStreamPtr& fin)
+BinaryTreeVec BinaryTree::getChildren()
 {
-
+    BinaryTreeVec children;
+    m_fin->seek(m_startPos);
+    while(true) {
+        uint8 byte = m_fin->getU8();
+        switch(byte) {
+            case BINARYTREE_NODE_START: {
+                BinaryTreePtr node(new BinaryTree(m_fin));
+                children.push_back(node);
+                node->skipNodes();
+                break;
+            }
+            case BINARYTREE_NODE_END:
+                return children;
+            case BINARYTREE_ESCAPE_CHAR:
+                m_fin->getU8();
+                break;
+            default:
+                break;
+        }
+    }
 }
 
 void BinaryTree::seek(uint pos)
 {
+    unserialize();
     if(pos > m_buffer.size())
         stdext::throw_exception("BinaryTree: seek failed");
     m_pos = pos;
@@ -60,6 +113,7 @@ void BinaryTree::seek(uint pos)
 
 uint8 BinaryTree::getU8()
 {
+    unserialize();
     if(m_pos+1 > m_buffer.size())
         stdext::throw_exception("BinaryTree: getU8 failed");
     uint8 v = m_buffer[m_pos];
@@ -69,6 +123,7 @@ uint8 BinaryTree::getU8()
 
 uint16 BinaryTree::getU16()
 {
+    unserialize();
     if(m_pos+2 > m_buffer.size())
         stdext::throw_exception("BinaryTree: getU16 failed");
     uint16 v = stdext::readLE16(&m_buffer[m_pos]);
@@ -78,6 +133,7 @@ uint16 BinaryTree::getU16()
 
 uint32 BinaryTree::getU32()
 {
+    unserialize();
     if(m_pos+4 > m_buffer.size())
         stdext::throw_exception("BinaryTree: getU32 failed");
     uint32 v = stdext::readLE32(&m_buffer[m_pos]);
@@ -87,6 +143,7 @@ uint32 BinaryTree::getU32()
 
 uint64 BinaryTree::getU64()
 {
+    unserialize();
     if(m_pos+8 > m_buffer.size())
         stdext::throw_exception("BinaryTree: getU64 failed");
     uint64 v = stdext::readLE64(&m_buffer[m_pos]);
@@ -96,10 +153,8 @@ uint64 BinaryTree::getU64()
 
 std::string BinaryTree::getString()
 {
+    unserialize();
     uint16 len = getU16();
-    if(len == 0 || len > 8192)
-        stdext::throw_exception("BinaryTree: getString failed: invalid or too large string length");
-
     if(m_pos+len > m_buffer.size())
         stdext::throw_exception("BinaryTree: getString failed: string length exceeded buffer size.");
 

src/framework/core/binarytree.h

@@ -32,18 +32,16 @@ enum {
     BINARYTREE_NODE_END = 0xFF
 };
 
-class BinaryTree : public std::enable_shared_from_this<BinaryTree>
+class BinaryTree
 {
 public:
-    BinaryTree(const BinaryTreePtr& parent = nullptr) : m_pos(0), m_parent(parent) { }
+    BinaryTree(const FileStreamPtr& fin);
-
+    ~BinaryTree();
-    void unserialize(const FileStreamPtr& fin);
-    void serialize(const FileStreamPtr& fin);
 
     void seek(uint pos);
     void skip(uint len) { seek(tell() + len); }
     uint tell() { return m_pos; }
-    uint size() { return m_buffer.size(); }
+    uint size() { unserialize(); return m_buffer.size(); }
 
     uint8 getU8();
     uint16 getU16();
@@ -51,16 +49,17 @@ public:
     uint64 getU64();
     std::string getString();
 
-    BinaryTreeVec getChildren() { return m_children; }
+    BinaryTreeVec getChildren();
-    BinaryTreePtr getParent() { return m_parent.lock(); }
+    bool canRead() { unserialize(); return m_pos < m_buffer.size(); }
-    bool canRead() { return m_pos < m_buffer.size(); }
 
 private:
-    uint m_pos;
+    void unserialize();
+    void skipNodes();
 
-    BinaryTreeVec m_children;
+    FileStreamPtr m_fin;
-    BinaryTreeWeakPtr m_parent;
     DataBuffer<uint8> m_buffer;
+    uint m_pos;
+    uint m_startPos;
 };
 
 #endif

src/framework/core/filestream.cpp

@@ -143,7 +143,7 @@ void FileStream::skip(uint len)
     seek(tell() + len);
 }
 
-int FileStream::size()
+uint FileStream::size()
 {
     if(!m_caching)
         return PHYSFS_fileLength(m_fileHandle);
@@ -151,7 +151,7 @@ int FileStream::size()
         return m_data.size();
 }
 
-int FileStream::tell()
+uint FileStream::tell()
 {
     if(!m_caching)
         return PHYSFS_tell(m_fileHandle);
@@ -249,13 +249,11 @@ std::string FileStream::getString()
 
 BinaryTreePtr FileStream::getBinaryTree()
 {
-    BinaryTreePtr root = BinaryTreePtr(new BinaryTree);
     uint8 byte = getU8();
-    if(byte == BINARYTREE_NODE_START)
+    if(byte != BINARYTREE_NODE_START)
-        root->unserialize(asFileStream());
-    else
         stdext::throw_exception(stdext::format("failed to read node start (getBinaryTree): %d", byte));
-    return root;
+
+    return BinaryTreePtr(new BinaryTree(asFileStream()));
 }
 
 void FileStream::addU8(uint8 v)

src/framework/core/filestream.h

@@ -43,8 +43,8 @@ public:
     int read(void *buffer, uint size, uint nmemb = 1);
     void seek(uint pos);
     void skip(uint len);
-    int size();
+    uint size();
-    int tell();
+    uint tell();
     std::string name() { return m_name; }
 
     uint8 getU8();

src/framework/luaengine/luainterface.cpp

@@ -721,7 +721,10 @@ void LuaInterface::error()
 
 int LuaInterface::ref()
 {
-    return luaL_ref(L, LUA_REGISTRYINDEX);
+    int ref = luaL_ref(L, LUA_REGISTRYINDEX);
+    assert(ref != LUA_NOREF);
+    assert(ref < 2147483647);
+    return ref;
 }
 
 int LuaInterface::weakRef()

src/framework/luaengine/luaobject.h

@@ -27,6 +27,7 @@
 
 /// LuaObject, all script-able classes have it as base
 // @bindclass
+#pragma pack(push,1) // disable memory alignment
 class LuaObject : public std::enable_shared_from_this<LuaObject>
 {
 public:
@@ -88,6 +89,7 @@ private:
     int m_fieldsTableRef;
     int m_metatableRef;
 };
+#pragma pack(pop)
 
 #include "luainterface.h"
 

src/framework/util/attribstorage.h

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2010-2012 OTClient <https://github.com/edubart/otclient>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef ATTRIBSTORAGE_H
+#define ATTRIBSTORAGE_H
+
+#include "../stdext/types.h"
+#include <tuple>
+#include <boost/any.hpp>
+
+
+#pragma pack(push,1) // disable memory alignment
+
+// this class was designed to use less memory as possible
+class AttribStorage {
+public:
+    AttribStorage() : m_attribs(nullptr), m_size(0) { }
+    ~AttribStorage() { if(m_attribs) delete[] m_attribs; }
+
+    template<typename T>
+    void set(uint8 id, T value) {
+        bool done = false;
+        for(int i=0;i<m_size;++i) {
+            if(std::get<0>(m_attribs[i]) == id) {
+                std::get<1>(m_attribs[i]) = value;
+                done = true;
+                break;
+            }
+        }
+        if(!done) {
+            auto attribs = new std::tuple<uint8, boost::any>[m_size+1];
+            if(m_size > 0) {
+                for(int i=0;i<m_size;++i)
+                    attribs[i] = m_attribs[i];
+                delete[] m_attribs;
+            }
+            m_attribs = attribs;
+            m_attribs[m_size++] = std::make_tuple(id, value);
+        }
+    }
+
+    template<typename T>
+    T get(uint8 id) const {
+        for(int i=0;i<m_size;++i)
+            if(std::get<0>(m_attribs[i]) == id)
+                return boost::any_cast<T>(std::get<1>(m_attribs[i]));
+        return T();
+    }
+
+    bool has(uint8 id) const {
+        for(int i=0;i<m_size;++i)
+            if(std::get<0>(m_attribs[i]) == id)
+                return true;
+        return false;
+    }
+private:
+    std::tuple<uint8, boost::any>* m_attribs;
+    uint8 m_size;
+};
+#pragma pack(pop)
+
+#endif

src/main.cpp

@@ -25,6 +25,11 @@
 #include <framework/luaengine/luainterface.h>
 #include <otclient/otclient.h>
 
+#include <framework/core/binarytree.h>
+#include <otclient/item.h>
+#include <otclient/tile.h>
+#include <boost/any.hpp>
+
 int main(int argc, const char* argv[])
 {
     std::vector<std::string> args(argv, argv + argc);

src/otclient/creature.cpp

@@ -113,20 +113,21 @@ void Creature::internalDrawOutfit(const Point& dest, float scaleFactor, bool ani
             if(yPattern > 0 && !(m_outfit.getAddons() & (1 << (yPattern-1))))
                 continue;
 
-            m_datType->draw(dest, scaleFactor, 0, xPattern, yPattern, 0, animationPhase);
+            auto datType = rawGetDatType();
+            datType->draw(dest, scaleFactor, 0, xPattern, yPattern, 0, animationPhase);
 
             if(getLayers() > 1) {
                 Color oldColor = g_painter->getColor();
                 Painter::CompositionMode oldComposition = g_painter->getCompositionMode();
                 g_painter->setCompositionMode(Painter::CompositionMode_Multiply);
                 g_painter->setColor(m_outfit.getHeadColor());
-                m_datType->draw(dest, scaleFactor, DatYellowMask, xPattern, yPattern, 0, animationPhase);
+                datType->draw(dest, scaleFactor, DatYellowMask, xPattern, yPattern, 0, animationPhase);
                 g_painter->setColor(m_outfit.getBodyColor());
-                m_datType->draw(dest, scaleFactor, DatRedMask, xPattern, yPattern, 0, animationPhase);
+                datType->draw(dest, scaleFactor, DatRedMask, xPattern, yPattern, 0, animationPhase);
                 g_painter->setColor(m_outfit.getLegsColor());
-                m_datType->draw(dest, scaleFactor, DatGreenMask, xPattern, yPattern, 0, animationPhase);
+                datType->draw(dest, scaleFactor, DatGreenMask, xPattern, yPattern, 0, animationPhase);
                 g_painter->setColor(m_outfit.getFeetColor());
-                m_datType->draw(dest, scaleFactor, DatBlueMask, xPattern, yPattern, 0, animationPhase);
+                datType->draw(dest, scaleFactor, DatBlueMask, xPattern, yPattern, 0, animationPhase);
                 g_painter->setColor(oldColor);
                 g_painter->setCompositionMode(oldComposition);
             }
@@ -154,7 +155,7 @@ void Creature::internalDrawOutfit(const Point& dest, float scaleFactor, bool ani
         if(m_outfit.getCategory() == DatEffectCategory)
             animationPhase = std::min(animationPhase+1, getAnimationPhases());
 
-        m_datType->draw(dest, scaleFactor, 0, 0, 0, 0, animationPhase);
+        rawGetDatType()->draw(dest, scaleFactor, 0, 0, 0, 0, animationPhase);
     }
 }
 
@@ -462,7 +463,6 @@ void Creature::setOutfit(const Outfit& outfit)
 {
     m_walkAnimationPhase = 0; // might happen when player is walking and outfit is changed.
     m_outfit = outfit;
-    m_datType = g_things.getDatType(outfit.getId(), outfit.getCategory());
 }
 
 void Creature::setSkull(uint8 skull)
@@ -548,3 +548,13 @@ Point Creature::getDrawOffset()
     }
     return drawOffset;
 }
+
+const ThingTypeDatPtr& Creature::getDatType()
+{
+    return g_things.getDatType(m_outfit.getId(), m_outfit.getCategory());
+}
+
+ThingTypeDat* Creature::rawGetDatType()
+{
+    return g_things.rawGetDatType(m_outfit.getId(), m_outfit.getCategory());
+}

src/otclient/creature.h

@@ -96,6 +96,9 @@ public:
     CreaturePtr asCreature() { return std::static_pointer_cast<Creature>(shared_from_this()); }
     bool isCreature() { return true; }
 
+    const ThingTypeDatPtr& getDatType();
+    ThingTypeDat *rawGetDatType();
+
 protected:
     virtual void updateWalkAnimation(int totalPixelsWalked);
     virtual void updateWalkOffset(int totalPixelsWalked);

src/otclient/effect.cpp

@@ -32,7 +32,7 @@ void Effect::draw(const Point& dest, float scaleFactor, bool animate)
     int animationPhase = 0;
     if(animate)
         animationPhase = std::min((int)(m_animationTimer.ticksElapsed() / Otc::EFFECT_TICKS_PER_FRAME), getAnimationPhases() - 1);
-    m_datType->draw(dest, scaleFactor, 0, 0, 0, 0, animationPhase);
+    rawGetDatType()->draw(dest, scaleFactor, 0, 0, 0, 0, animationPhase);
 }
 
 void Effect::startAnimation()
@@ -47,5 +47,14 @@ void Effect::startAnimation()
 void Effect::setId(uint32 id)
 {
     m_id = id;
-    m_datType = g_things.getDatType(m_id, DatEffectCategory);
+}
+
+const ThingTypeDatPtr& Effect::getDatType()
+{
+    return g_things.getDatType(m_id, DatEffectCategory);
+}
+
+ThingTypeDat *Effect::rawGetDatType()
+{
+    return g_things.rawGetDatType(m_id, DatEffectCategory);
 }

src/otclient/effect.h

@@ -41,6 +41,9 @@ public:
     EffectPtr asEffect() { return std::static_pointer_cast<Effect>(shared_from_this()); }
     bool isEffect() { return true; }
 
+    const ThingTypeDatPtr& getDatType();
+    ThingTypeDat *rawGetDatType();
+
 private:
     Timer m_animationTimer;
     uint16 m_id;

src/otclient/item.cpp

@@ -38,11 +38,8 @@
 
 Item::Item() :
     m_id(0),
-    m_countOrSubType(1),
+    m_otbId(0),
-    m_actionId(0),
+    m_countOrSubType(1)
-    m_uniqueId(0),
-    m_shaderProgram(g_shaders.getDefaultItemShader()),
-    m_otbType(g_things.getNullOtbType())
 {
 }
 
@@ -174,32 +171,20 @@ void Item::draw(const Point& dest, float scaleFactor, bool animate)
         zPattern = m_position.z % getNumPatternZ();
     }
 
-    bool useShader = g_painter->hasShaders() && m_shaderProgram;
+    rawGetDatType()->draw(dest, scaleFactor, 0, xPattern, yPattern, zPattern, animationPhase);
-    if(useShader) {
-        m_shaderProgram->bind();
-        m_shaderProgram->setUniformValue(ShaderManager::ITEM_ID_UNIFORM, (int)m_id);
-
-        g_painter->setShaderProgram(m_shaderProgram);
-    }
-
-    m_datType->draw(dest, scaleFactor, 0, xPattern, yPattern, zPattern, animationPhase);
-
-    if(useShader)
-        g_painter->resetShaderProgram();
 }
 
 void Item::setId(uint32 id)
 {
-    m_datType = g_things.getDatType(id, DatItemCategory);
+    m_otbId = g_things.findOtbForClientId(id)->getServerId();
-    m_otbType = g_things.findOtbForClientId(id);
+    m_id = id;
-    m_id = m_datType->getId();
 }
 
 void Item::setOtbId(uint16 id)
 {
-    m_otbType = g_things.getOtbType(id);
+    auto otbType = g_things.getOtbType(id);
-    m_datType = g_things.getDatType(m_otbType->getClientId(), DatItemCategory);
+    m_otbId = id;
-    m_id = m_datType->getId();
+    m_id = otbType->getClientId();
 }
 
 bool Item::isValid()
@@ -229,29 +214,29 @@ void Item::unserializeItem(const BinaryTreePtr &in)
                 setUniqueId(in->getU16());
                 break;
             case ATTR_NAME:
-                setName(in->getString());
+                m_attribs.set(ATTR_NAME, in->getString());
                 break;
             case ATTR_TEXT:
-                setText(in->getString());
+                m_attribs.set(ATTR_TEXT, in->getString());
                 break;
             case ATTR_DESC:
-                m_description = in->getString();
+                m_attribs.set(ATTR_DESC, in->getString());
                 break;
             case ATTR_CONTAINER_ITEMS:
-                m_isContainer = true;
+                m_attribs.set(ATTR_CONTAINER_ITEMS, in->getU32());
-                in->skip(4);
                 break;
             case ATTR_HOUSEDOORID:
-                m_isDoor = true;
+                m_attribs.set(ATTR_HOUSEDOORID, in->getU8());
-                m_doorId = in->getU8();
                 break;
             case ATTR_DEPOT_ID:
-                m_depotId = in->getU16();
+                m_attribs.set(ATTR_DEPOT_ID, in->getU16());
                 break;
             case ATTR_TELE_DEST: {
-                m_teleportDestination.x = in->getU16();
+                Position teleportDestination;
-                m_teleportDestination.y = in->getU16();
+                teleportDestination.x = in->getU16();
-                m_teleportDestination.z = in->getU8();
+                teleportDestination.y = in->getU16();
+                teleportDestination.z = in->getU8();
+                m_attribs.set(ATTR_TELE_DEST, teleportDestination);
                 break;
             }
             case ATTR_ARTICLE:
@@ -286,12 +271,7 @@ void Item::unserializeItem(const BinaryTreePtr &in)
 
 bool Item::isMoveable()
 {
-    if(m_datType)
+    return !rawGetDatType()->isNotMoveable();
-        return !m_datType->isNotMoveable();
-
-    g_logger.warning(stdext::format(
-                         "Invalid dat type for item %d", m_id));
-    return false;
 }
 
 ItemPtr Item::clone()
@@ -300,3 +280,13 @@ ItemPtr Item::clone()
     *(item.get()) = *this;
     return item;
 }
+
+const ThingTypeDatPtr& Item::getDatType()
+{
+    return g_things.getDatType(m_id, DatItemCategory);
+}
+
+ThingTypeDat* Item::rawGetDatType()
+{
+    return g_things.rawGetDatType(m_id, DatItemCategory);
+}

src/otclient/item.h

@@ -24,6 +24,7 @@
 #define ITEM_H
 
 #include <framework/global.h>
+#include <framework/util/attribstorage.h>
 #include "thing.h"
 #include "thingtypeotb.h"
 
@@ -66,10 +67,12 @@ enum AttrTypes_t
 };
 
 // @bindclass
+#pragma pack(push,1) // disable memory alignment
 class Item : public Thing
 {
 public:
     Item();
+    virtual ~Item() { }
 
     static ItemPtr create(int id);
     static ItemPtr createFromOtb(int id);
@@ -81,43 +84,45 @@ public:
     void setCountOrSubType(int value) { m_countOrSubType = value; }
     void setCount(int count) { m_countOrSubType = count; }
     void setSubType(int subType) { m_countOrSubType = subType; }
-    void setActionId(int actionId) { m_actionId = actionId; }
-    void setUniqueId(int uniqueId) { m_uniqueId = uniqueId; }
-    void setDoorId(int doorId) { m_doorId = doorId; }
-    void setName(const std::string &name) { m_name = name; }
-    void setText(const std::string &text) { m_text = text; }
-    void setDescription(const std::string &description) { m_description = description; }
 
     int getCountOrSubType() { return m_countOrSubType; }
     int getSubType() { return m_countOrSubType; }
     int getCount() { return m_countOrSubType; }
     uint32 getId() { return m_id; }
-    std::string getName() { return m_name; }
-    uint8 getDoorId() { return m_doorId; }
     bool isValid();
 
     ItemPtr clone();
-    ItemPtr asItem() { return std::static_pointer_cast<Item>(shared_from_this()); }
-    bool isItem() { return true; }
 
     void unserializeItem(const BinaryTreePtr &in);
     bool isMoveable();
-    bool isContainer() { return m_isContainer; }
+
-    bool isDoor() { return m_isDoor; }
+    void setDepotId(uint16 depotId) { m_attribs.set(ATTR_DEPOT_ID, depotId); }
+    uint16 getDepotId() { return m_attribs.get<uint16>(ATTR_DEPOT_ID); }
+
+    void setDoorId(uint8 doorId) { m_attribs.set(ATTR_HOUSEDOORID, doorId); }
+    uint8 getDoorId() { return m_attribs.get<uint8>(ATTR_HOUSEDOORID); }
+
+    void setActionId(uint16 actionId) { m_attribs.set(ATTR_ACTION_ID, actionId); }
+    void setUniqueId(uint16 uniqueId) { m_attribs.set(ATTR_UNIQUE_ID, uniqueId); }
+
+    bool isDepot() { return m_attribs.has(ATTR_DEPOT_ID); }
+    bool isContainer() { return m_attribs.has(ATTR_CONTAINER_ITEMS); }
+    bool isDoor() { return m_attribs.has(ATTR_HOUSEDOORID); }
+    bool isTeleport() { return m_attribs.has(ATTR_TELE_DEST); }
+
+    ItemPtr asItem() { return std::static_pointer_cast<Item>(shared_from_this()); }
+    bool isItem() { return true; }
+
+    const ThingTypeDatPtr& getDatType();
+    ThingTypeDat *rawGetDatType();
 
 private:
     uint16 m_id;
+    uint16 m_otbId;
     uint8 m_countOrSubType;
-    uint32 m_actionId, m_uniqueId;
+    AttribStorage m_attribs;
-    uint16 m_depotId;
-    uint8 m_doorId;
-    Boolean<false> m_isContainer;
-    Boolean<false> m_isDoor;
-
-    std::string m_name, m_text, m_description;
-    PainterShaderProgramPtr m_shaderProgram;
-    ThingTypeOtbPtr m_otbType;
-    Position m_teleportDestination;
 };
+#pragma pack(pop)
+
 
 #endif

src/otclient/map.cpp

@@ -136,7 +136,6 @@ void Map::loadOtbm(const std::string& fileName)
                 if(type != OTBM_TILE && type != OTBM_HOUSETILE)
                     stdext::throw_exception(stdext::format("invalid node tile type %d", (int)type));
 
-                TilePtr tile = nullptr;
                 ItemPtr ground = nullptr;
                 HousePtr house = nullptr;
                 uint32 flags = TILESTATE_NONE;
@@ -146,7 +145,7 @@ void Map::loadOtbm(const std::string& fileName)
 
                 if(type ==  OTBM_HOUSETILE) {
                     uint32 hId = nodeTile->getU32();
-                    tile = createTile(pos);
+                    TilePtr tile = getOrCreateTile(pos);
                     if(!(house = m_houses.getHouse(hId))) {
                         house = HousePtr(new House(hId));
                         m_houses.addHouse(house);
@@ -175,13 +174,7 @@ void Map::loadOtbm(const std::string& fileName)
                             break;
                         }
                         case OTBM_ATTR_ITEM: {
-                            ItemPtr item = Item::createFromOtb(nodeTile->getU16());
+                            addThing(Item::createFromOtb(nodeTile->getU16()), pos);
-                            if(tile)
-                                addThing(item, pos, 255);
-                            else if(item->isGround())
-                                ground = item;
-                            else
-                                tile = createTileEx(pos, ground, item);
                             break;
                         }
                         default:
@@ -196,6 +189,7 @@ void Map::loadOtbm(const std::string& fileName)
 
                     ItemPtr item = Item::createFromOtb(nodeItem->getU16());
                     item->unserializeItem(nodeItem);
+
                     if(item->isContainer()) {
                         // This is a temporary way for reading container items.
                         MapContainerPtr mapContainer(new MapContainer);
@@ -217,17 +211,12 @@ void Map::loadOtbm(const std::string& fileName)
                             item = nullptr;
                         } else if(item->isDoor())
                             house->addDoor(item->getDoorId(), pos);
-                    } else if(tile)
-                        addThing(item, pos);
-                    else if(item->isGround())
-                        ground = item;
-                    else
-                        tile = createTileEx(pos, ground, item);
                     }
 
-                if(!tile)
+                    addThing(item, pos);
-                    tile = createTileEx(pos, ground);
+                }
 
+                if(const TilePtr& tile = getTile(pos))
                     tile->setFlags((tileflags_t)flags);
             }
         } else if(mapDataType == OTBM_TOWNS) {
@@ -263,6 +252,13 @@ void Map::loadOtbm(const std::string& fileName)
             stdext::throw_exception("Unknown map data node");
     }
 
+
+    int numItems = 0;
+    for(const auto& it : m_tiles)
+        numItems += it.second->getThingCount();
+
+    g_logger.debug(stdext::format("Total items: %d", numItems));
+    g_logger.debug(stdext::format("Total tiles: %d", m_tiles.size()));
     g_logger.debug("OTBM read successfully.");
     fin->close();
     /// TODO read XML Stuff (houses & spawns).

src/otclient/map.h

@@ -122,6 +122,7 @@ public:
     // thing related
     void addThing(const ThingPtr& thing, const Position& pos, int stackPos = -1);
     ThingPtr getThing(const Position& pos, int stackPos);
+    bool removeThing(const ThingPtr& thing);
     bool removeThingByPos(const Position& pos, int stackPos);
 
     // tile related
@@ -131,7 +132,6 @@ public:
     const TilePtr& getTile(const Position& pos);
     TilePtr getOrCreateTile(const Position& pos);
     void cleanTile(const Position& pos);
-    bool removeThing(const ThingPtr& thing);
 
     // known creature related
     void addCreature(const CreaturePtr& creature);

src/otclient/missile.cpp

@@ -63,7 +63,7 @@ void Missile::draw(const Point& dest, float scaleFactor, bool animate)
     }
 
     float fraction = m_animationTimer.ticksElapsed() / m_duration;
-    m_datType->draw(dest + m_delta * fraction * scaleFactor, scaleFactor, 0, xPattern, yPattern, 0, 0);
+    rawGetDatType()->draw(dest + m_delta * fraction * scaleFactor, scaleFactor, 0, xPattern, yPattern, 0, 0);
 }
 
 void Missile::setPath(const Position& fromPosition, const Position& toPosition)
@@ -84,5 +84,14 @@ void Missile::setPath(const Position& fromPosition, const Position& toPosition)
 void Missile::setId(uint32 id)
 {
     m_id = id;
-    m_datType = g_things.getDatType(m_id, DatMissileCategory);
+}
+
+const ThingTypeDatPtr& Missile::getDatType()
+{
+    return g_things.getDatType(m_id, DatMissileCategory);
+}
+
+ThingTypeDat* Missile::rawGetDatType()
+{
+    return g_things.rawGetDatType(m_id, DatMissileCategory);
 }

src/otclient/missile.h

@@ -37,8 +37,6 @@ class Missile : public Thing
 public:
     void draw(const Point& dest, float scaleFactor, bool animate);
 
-    void updateAnimation();
-
     void setId(uint32 id);
     void setPath(const Position& fromPosition, const Position& toPosition);
 
@@ -47,6 +45,9 @@ public:
     MissilePtr asMissile() { return std::static_pointer_cast<Missile>(shared_from_this()); }
     bool isMissile() { return true; }
 
+    const ThingTypeDatPtr& getDatType();
+    ThingTypeDat *rawGetDatType();
+
 private:
     Timer m_animationTimer;
     Point m_delta;

src/otclient/position.h

@@ -214,7 +214,7 @@ public:
 
 struct PositionHasher : std::unary_function<Position, std::size_t> {
     std::size_t operator()(const Position& pos) const {
-        return ((((pos.x * 2048) + pos.y) * 16) + pos.z) % (2048*2048);
+        return (((((pos.x * 2048) + pos.y) * 16) + pos.z) % (2048*2048)) % 1000000;
     }
 };
 

src/otclient/thing.cpp

@@ -29,7 +29,7 @@
 #include "game.h"
 
 Thing::Thing() :
-    m_datType(g_things.getNullDatType())
+    m_datId(0)
 {
 }
 
@@ -74,3 +74,13 @@ int Thing::getStackpos()
         return -1;
     }
 }
+
+const ThingTypeDatPtr& Thing::getDatType()
+{
+    return g_things.getNullDatType();
+}
+
+ThingTypeDat* Thing::rawGetDatType()
+{
+    return g_things.getNullDatType().get();
+}

src/otclient/thing.h

@@ -25,6 +25,7 @@
 
 #include "declarations.h"
 #include "thingtypedat.h"
+#include "thingtypemanager.h"
 #include <framework/luaengine/luaobject.h>
 
 struct Light
@@ -34,6 +35,7 @@ struct Light
 };
 
 // @bindclass
+#pragma pack(push,1) // disable memory alignment
 class Thing : public LuaObject
 {
 public:
@@ -77,65 +79,67 @@ public:
     virtual bool isStaticText() { return false; }
 
     // type shortcuts
-    ThingTypeDatPtr getType() { return m_datType; }
+    virtual const ThingTypeDatPtr& getDatType();
-    Size getSize() { return m_datType->getSize(); }
+    virtual ThingTypeDat *rawGetDatType();
-    int getWidth() { return m_datType->getWidth(); }
+    Size getSize() { return rawGetDatType()->getSize(); }
-    int getHeight() { return m_datType->getHeight(); }
+    int getWidth() { return rawGetDatType()->getWidth(); }
-    Point getDisplacement() { return m_datType->getDisplacement(); }
+    int getHeight() { return rawGetDatType()->getHeight(); }
-    int getDisplacementX() { return m_datType->getDisplacementX(); }
+    Point getDisplacement() { return rawGetDatType()->getDisplacement(); }
-    int getDisplacementY() { return m_datType->getDisplacementY(); }
+    int getDisplacementX() { return rawGetDatType()->getDisplacementX(); }
-    int getExactSize() { return m_datType->getExactSize(); }
+    int getDisplacementY() { return rawGetDatType()->getDisplacementY(); }
-    int getLayers() { return m_datType->getLayers(); }
+    int getExactSize() { return rawGetDatType()->getExactSize(); }
-    int getNumPatternX() { return m_datType->getNumPatternX(); }
+    int getLayers() { return rawGetDatType()->getLayers(); }
-    int getNumPatternY() { return m_datType->getNumPatternY(); }
+    int getNumPatternX() { return rawGetDatType()->getNumPatternX(); }
-    int getNumPatternZ() { return m_datType->getNumPatternZ(); }
+    int getNumPatternY() { return rawGetDatType()->getNumPatternY(); }
-    int getAnimationPhases() { return m_datType->getAnimationPhases(); }
+    int getNumPatternZ() { return rawGetDatType()->getNumPatternZ(); }
-    int getGroundSpeed() { return m_datType->getGroundSpeed(); }
+    int getAnimationPhases() { return rawGetDatType()->getAnimationPhases(); }
-    int getMaxTextLength() { return m_datType->getMaxTextLength(); }
+    int getGroundSpeed() { return rawGetDatType()->getGroundSpeed(); }
-    int getLightLevel() { return m_datType->getLightLevel(); }
+    int getMaxTextLength() { return rawGetDatType()->getMaxTextLength(); }
-    int getLightColor() { return m_datType->getLightColor(); }
+    int getLightLevel() { return rawGetDatType()->getLightLevel(); }
-    int getMinimapColor() { return m_datType->getMinimapColor(); }
+    int getLightColor() { return rawGetDatType()->getLightColor(); }
-    int getLensHelp() { return m_datType->getLensHelp(); }
+    int getMinimapColor() { return rawGetDatType()->getMinimapColor(); }
-    int getClothSlot() { return m_datType->getClothSlot(); }
+    int getLensHelp() { return rawGetDatType()->getLensHelp(); }
-    int getElevation() { return m_datType->getElevation(); }
+    int getClothSlot() { return rawGetDatType()->getClothSlot(); }
-    bool isGround() { return m_datType->isGround(); }
+    int getElevation() { return rawGetDatType()->getElevation(); }
-    bool isGroundBorder() { return m_datType->isGroundBorder(); }
+    bool isGround() { return rawGetDatType()->isGround(); }
-    bool isOnBottom() { return m_datType->isOnBottom(); }
+    bool isGroundBorder() { return rawGetDatType()->isGroundBorder(); }
-    bool isOnTop() { return m_datType->isOnTop(); }
+    bool isOnBottom() { return rawGetDatType()->isOnBottom(); }
-    bool isContainer() { return m_datType->isContainer(); }
+    bool isOnTop() { return rawGetDatType()->isOnTop(); }
-    bool isStackable() { return m_datType->isStackable(); }
+    bool isContainer() { return rawGetDatType()->isContainer(); }
-    bool isForceUse() { return m_datType->isForceUse(); }
+    bool isStackable() { return rawGetDatType()->isStackable(); }
-    bool isMultiUse() { return m_datType->isMultiUse(); }
+    bool isForceUse() { return rawGetDatType()->isForceUse(); }
-    bool isWritable() { return m_datType->isWritable(); }
+    bool isMultiUse() { return rawGetDatType()->isMultiUse(); }
-    bool isWritableOnce() { return m_datType->isWritableOnce(); }
+    bool isWritable() { return rawGetDatType()->isWritable(); }
-    bool isFluidContainer() { return m_datType->isFluidContainer(); }
+    bool isWritableOnce() { return rawGetDatType()->isWritableOnce(); }
-    bool isFluid() { return m_datType->isFluid(); }
+    bool isFluidContainer() { return rawGetDatType()->isFluidContainer(); }
-    bool isNotWalkable() { return m_datType->isNotWalkable(); }
+    bool isFluid() { return rawGetDatType()->isFluid(); }
-    bool isNotMoveable() { return m_datType->isNotMoveable(); }
+    bool isNotWalkable() { return rawGetDatType()->isNotWalkable(); }
-    bool blockProjectile() { return m_datType->blockProjectile(); }
+    bool isNotMoveable() { return rawGetDatType()->isNotMoveable(); }
-    bool isNotPathable() { return m_datType->isNotPathable(); }
+    bool blockProjectile() { return rawGetDatType()->blockProjectile(); }
-    bool isPickupable() { return m_datType->isPickupable(); }
+    bool isNotPathable() { return rawGetDatType()->isNotPathable(); }
-    bool isHangable() { return m_datType->isHangable(); }
+    bool isPickupable() { return rawGetDatType()->isPickupable(); }
-    bool isHookSouth() { return m_datType->isHookSouth(); }
+    bool isHangable() { return rawGetDatType()->isHangable(); }
-    bool isHookEast() { return m_datType->isHookEast(); }
+    bool isHookSouth() { return rawGetDatType()->isHookSouth(); }
-    bool isRotateable() { return m_datType->isRotateable(); }
+    bool isHookEast() { return rawGetDatType()->isHookEast(); }
-    bool hasLight() { return m_datType->hasLight(); }
+    bool isRotateable() { return rawGetDatType()->isRotateable(); }
-    bool isDontHide() { return m_datType->isDontHide(); }
+    bool hasLight() { return rawGetDatType()->hasLight(); }
-    bool isTranslucent() { return m_datType->isTranslucent(); }
+    bool isDontHide() { return rawGetDatType()->isDontHide(); }
-    bool hasDisplacement() { return m_datType->hasDisplacement(); }
+    bool isTranslucent() { return rawGetDatType()->isTranslucent(); }
-    bool hasElevation() { return m_datType->hasElevation(); }
+    bool hasDisplacement() { return rawGetDatType()->hasDisplacement(); }
-    bool isLyingCorpse() { return m_datType->isLyingCorpse(); }
+    bool hasElevation() { return rawGetDatType()->hasElevation(); }
-    bool isAnimateAlways() { return m_datType->isAnimateAlways(); }
+    bool isLyingCorpse() { return rawGetDatType()->isLyingCorpse(); }
-    bool hasMiniMapColor() { return m_datType->hasMiniMapColor(); }
+    bool isAnimateAlways() { return rawGetDatType()->isAnimateAlways(); }
-    bool hasLensHelp() { return m_datType->hasLensHelp(); }
+    bool hasMiniMapColor() { return rawGetDatType()->hasMiniMapColor(); }
-    bool isFullGround() { return m_datType->isFullGround(); }
+    bool hasLensHelp() { return rawGetDatType()->hasLensHelp(); }
-    bool isIgnoreLook() { return m_datType->isIgnoreLook(); }
+    bool isFullGround() { return rawGetDatType()->isFullGround(); }
-    bool isCloth() { return m_datType->isCloth(); }
+    bool isIgnoreLook() { return rawGetDatType()->isIgnoreLook(); }
+    bool isCloth() { return rawGetDatType()->isCloth(); }
 
 protected:
     Position m_position;
-    ThingTypeDatPtr m_datType;
+    uint16 m_datId;
 };
+#pragma pack(pop)
 
 #endif
 

src/otclient/thingtypemanager.cpp

@@ -211,3 +211,4 @@ const ThingTypeOtbPtr& ThingTypeManager::getOtbType(uint16 id)
     }
     return m_otbTypes[id];
 }
+

src/otclient/thingtypemanager.h

@@ -47,6 +47,8 @@ public:
 
     const ThingTypeDatPtr& getDatType(uint16 id, DatCategory category);
     const ThingTypeOtbPtr& getOtbType(uint16 id);
+    ThingTypeDat *rawGetDatType(uint16 id, DatCategory category) { return m_datTypes[category][id].get(); }
+    ThingTypeOtb *rawGetOtbType(uint16 id) { return m_otbTypes[id].get(); }
 
     uint32 getDatSignature() { return m_datSignature; }
     uint32 getOtbMajorVersion() { return m_otbMajorVersion; }

src/otclient/tile.h

@@ -46,6 +46,7 @@ enum tileflags_t
     TILESTATE_DEPOT = 1 << 22
 };
 
+#pragma pack(push,1) // disable memory alignment
 class Tile : public LuaObject
 {
 public:
@@ -99,10 +100,11 @@ public:
     bool limitsFloorsView();
     bool canErase();
 
-    TilePtr asTile() { return std::static_pointer_cast<Tile>(shared_from_this()); }
     void setFlags(tileflags_t flags) { m_flags |= (uint32)flags; }
     uint32 flags() { return m_flags; }
 
+    TilePtr asTile() { return std::static_pointer_cast<Tile>(shared_from_this()); }
+
 private:
     void update();
 
@@ -114,5 +116,6 @@ private:
     uint32 m_flags;
     uint8 m_minimapColorByte;
 };
+#pragma pack(pop)
 
 #endif