MMFs + a scaled down hash set to use them with

common/CMakeLists.txt

@@ -35,6 +35,7 @@ SET(common_sources
     logsys.cpp
     logsys_eqemu.cpp
     md5.cpp
+    memory_mapped_file.cpp
     misc.cpp
     MiscFunctions.cpp
     moremath.cpp
@@ -129,6 +130,7 @@ SET(common_headers
     eqtime.h
     errmsg.h
     extprofile.h
+    fixed_memory_hash_set.h
     guild_base.h
     guilds.h
     ipc_mutex.h
@@ -141,6 +143,7 @@ SET(common_headers
     logtypes.h
     mail_oplist.h
     md5.h
+    memory_mapped_file.h
     misc.h
     MiscFunctions.h
     moremath.h

common/fixed_memory_hash_set.h

@@ -0,0 +1,226 @@
+#ifndef _EQEMU_FIXED_MEMORY_HASHSET_H
+#define _EQEMU_FIXED_MEMORY_HASHSET_H
+
+#include <string.h>
+#include "eqemu_exception.h"
+#include "types.h"
+
+namespace EQEmu {
+
+    /*! Simple HashSet designed to be used in fixed memory that may be difficult to use an 
+    allocator for (shared memory), we assume all keys are unsigned int
+    */
+    template<class T>
+    class FixedMemoryHashSet {
+        typedef uint32 key_type;
+        typedef T value_type;
+        typedef uint8 byte;
+        typedef value_type& reference;
+        typedef const value_type& const_reference;
+        typedef size_t size_type;
+    public:
+        /*!
+            Constructor which initializes the hash set
+        \param data Raw data
+        \param size Raw data size
+        \param element_count Max number of possible unique elements that can be inserted. 
+        \param max_element_id Number of offsets to store: eg highest "key" that will be used.
+        */
+        FixedMemoryHashSet(byte *data, size_type size, key_type element_count, key_type max_element_id) {
+            data_ = data;
+            size_ = size;
+
+            byte *ptr = data;
+            *reinterpret_cast<key_type*>(ptr) = max_element_id;
+            offset_count_ = max_element_id;
+            ptr += sizeof(key_type);
+
+            *reinterpret_cast<key_type*>(ptr) = element_count;
+            max_elements_ = element_count;
+            ptr += sizeof(key_type);
+
+            *reinterpret_cast<key_type*>(ptr) = 0;
+            current_elements_ = 0;
+            ptr += sizeof(key_type);
+
+            offsets_ = reinterpret_cast<key_type*>(ptr);
+            memset(ptr, 0xFFFFFFFFU, sizeof(key_type) * max_element_id);
+            ptr += sizeof(key_type) * max_element_id;
+
+            elements_ = reinterpret_cast<value_type*>(ptr);
+        }
+
+        /*!
+            Constructor which does not initialize the hash set.  Builds the hash set from what data is 
+            stored in the data pointer passed.
+        \param data Raw data
+        \param size Raw data size
+        */
+        FixedMemoryHashSet(byte *data, size_type size) {
+            data_ = data;
+            size_ = size;
+
+            byte *ptr = data;
+
+            offset_count_ = *reinterpret_cast<key_type*>(ptr);
+            ptr += sizeof(key_type);
+
+            max_elements_ = *reinterpret_cast<key_type*>(ptr);
+            ptr += sizeof(key_type);
+
+            current_elements_ = *reinterpret_cast<key_type*>(ptr);
+            ptr += sizeof(key_type);
+
+            offsets_ = reinterpret_cast<key_type*>(ptr);
+            ptr += sizeof(key_type) * offset_count_;
+
+            elements_ = reinterpret_cast<value_type*>(ptr);
+        }
+
+        //! Copy Constructor
+        FixedMemoryHashSet(const FixedMemoryHashSet& other) :
+            data_(other.data_),
+            size_(other.size_),
+            offset_count_(other.offset_count_),
+            max_elements_(other.max_elements_),
+            current_elements_(other.current_elements_),
+            offsets_(other.offsets_),
+            elements_(other.elements_)
+        {
+        }
+
+        //! RValue-Move Constructor
+        FixedMemoryHashSet(FixedMemoryHashSet&& other) :
+            data_(other.data_),
+            size_(other.size_),
+            offset_count_(other.offset_count_),
+            max_elements_(other.max_elements_),
+            current_elements_(other.current_elements_),
+            offsets_(other.offsets_),
+            elements_(other.elements_)
+        {
+        }
+
+        //! Destructor
+        ~FixedMemoryHashSet() {
+        }
+
+        //! Assignment operator
+        const FixedMemoryHashSet& operator=(const FixedMemoryHashSet& other) {
+            data_ = other.data_;
+            size_ = other.size_;
+            offset_count_ = other.offset_count_;
+            max_elements_ = other.max_elements_;
+            current_elements_ = other.current_elements_;
+            offsets_ = other.offsets_;
+            elements_ = other.elements_;
+            return *this;
+        }
+    
+        //! Returns whether the set is empty (has 0 elements) or not
+        bool empty() const {
+            return current_elements_ == 0;
+        }
+
+        //! Returns the number of unique elements in the set currently
+        size_type size() const {
+            return current_elements_;
+        }
+
+        //! Returns the maximum number of elements one can insert into the set.
+        size_type max_size() const {
+            return max_elements_;
+        }
+        
+        /*!
+            Retrieve value operator
+        \param i Index to retrieve the value from
+        */
+        reference operator[](const key_type& i) {
+            if(i >= offset_count_) {
+                EQ_EXCEPT("Fixed Memory Hash Set", "Index out of range");
+            }
+
+            if(offsets_[i] == 0xFFFFFFFFU) {
+                EQ_EXCEPT("Fixed Memory Hash Set", "Element not found.");
+            }
+
+            return elements_[offsets_[i]];
+        }
+
+        /*!
+            Retrieve value function
+        \param i Index to retrieve the value from
+        */
+        reference at(const key_type& i) {
+            if(i >= offset_count_) {
+                EQ_EXCEPT("Fixed Memory Hash Set", "Index out of range.");
+            }
+
+            if(offsets_[i] == 0xFFFFFFFFU) {
+                EQ_EXCEPT("Fixed Memory Hash Set", "Element not found.");
+            }
+
+            return elements_[offsets_[i]];
+        }
+
+        /*!
+            Checks if there is a value at a certain index
+        \param i Index to check for a value
+        */
+        bool exists(const key_type& i) const {
+            if(i >= offset_count_) {
+                return false;
+            }
+
+            if(offsets_[i] == 0xFFFFFFFFU) {
+                return false;
+            }
+
+            return true;
+        }
+
+        /*!
+            Inserts a value into the set at a specific index
+        \param i Index to insert the value at
+        \param v Value to insert
+        */
+        void insert(const key_type& i, const_reference v) {
+            if(i >= offset_count_) {
+                EQ_EXCEPT("Fixed Memory Hash Set", "Index out of range.");
+            }
+
+            if(offsets_[i] != 0xFFFFFFFFU) {
+                elements_[offsets_[i]] = v;
+            } else {
+                if(current_elements_ >= max_elements_) {
+                    EQ_EXCEPT("Fixed Memory Hash Set", "Insert pointer out of range.");
+                }
+
+                offsets_[i] = current_elements_;
+                memcpy(&elements_[current_elements_], &v, sizeof(value_type));
+                ++current_elements_;
+                *reinterpret_cast<key_type*>(data_ + (sizeof(key_type) * 2)) = current_elements_;
+            }
+        }
+
+        //! Calculates how much memory we should allocate based on element size and count 
+        static size_type estimated_size(key_type element_count, key_type max_elements) {
+            size_type total_size = 3 * sizeof(key_type);
+            total_size += sizeof(key_type) * max_elements;
+            total_size += sizeof(T) * element_count;
+            return total_size;
+        }
+
+    private:
+        unsigned char *data_;
+        size_type size_;
+        key_type offset_count_;
+        key_type max_elements_;
+        key_type current_elements_;
+        key_type *offsets_;
+        value_type *elements_;
+    };
+} // EQEmu
+
+#endif

common/memory_mapped_file.cpp

@@ -0,0 +1,193 @@
+#include "memory_mapped_file.h"
+#ifdef _WINDOWS
+#include <windows.h>
+#else
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <err.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdio.h>
+#endif
+#include "eqemu_exception.h"
+#include "ipc_mutex.h"
+
+namespace EQEmu {
+
+    struct MemoryMappedFile::Implementation {
+#ifdef _WINDOWS
+        HANDLE mapped_object_;
+#else
+        int fd_;
+#endif
+    };
+
+    MemoryMappedFile::MemoryMappedFile(std::string filename, uint32 size) 
+        : filename_(filename), size_(size) {
+        imp_ = new Implementation;
+        IPCMutex mut(filename + "Internal");
+    
+        if(!mut.Lock()) {
+            EQ_EXCEPT("Shared Memory", "Could not lock shared mutex.");
+        }
+    
+#ifdef _WINDOWS
+        DWORD total_size = size + sizeof(shared_memory_struct);
+        HANDLE file = CreateFile(filename.c_str(), 
+            GENERIC_READ | GENERIC_WRITE, 
+            FILE_SHARE_READ | FILE_SHARE_WRITE, 
+            NULL,
+            OPEN_ALWAYS,
+            0,
+            NULL);
+    
+        if(file == INVALID_HANDLE_VALUE) {
+            EQ_EXCEPT("Shared Memory", "Could not open a file for this shared memory segment.");
+        }
+    
+        imp_->mapped_object_ = CreateFileMapping(file,
+            NULL,
+            PAGE_READWRITE, 
+            0,
+            total_size,
+            filename.c_str());
+    
+        if(!imp_->mapped_object_) {
+            mut.Unlock();
+            EQ_EXCEPT("Shared Memory", "Could not create a file mapping for this shared memory file.");
+        }
+    
+        memory_ = reinterpret_cast<shared_memory_struct*>(MapViewOfFile(imp_->mapped_object_,
+            FILE_MAP_ALL_ACCESS,
+            0,
+            0, 
+            total_size));
+    
+        if(!memory_) {
+            mut.Unlock();
+            EQ_EXCEPT("Shared Memory", "Could not map a view of the shared memory file.");
+        }
+    
+#else
+        size_t total_size = size + sizeof(shared_memory_struct);
+        imp_->fd_ = open(filename.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
+        if(imp_->fd_ == -1) {
+            mut.Unlock();
+            EQ_EXCEPT("Shared Memory", "Could not open a file for this shared memory segment.");
+        }
+
+        if(ftruncate(imp_->fd_, total_size) == -1) {
+            EQ_EXCEPT("Shared Memory", "Could not set file size for this shared memory segment.");
+        }
+
+        memory_ = reinterpret_cast<shared_memory_struct*>(
+            mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_FILE | MAP_SHARED, imp_->fd_, 0));
+
+        if(memory_ == MAP_FAILED) {
+            mut.Unlock();
+            EQ_EXCEPT("Shared Memory", "Could not create a file mapping for this shared memory file.");
+        }
+#endif
+        mut.Unlock();
+    }
+    
+    MemoryMappedFile::MemoryMappedFile(std::string filename) 
+        : filename_(filename) {
+        imp_ = new Implementation;
+        IPCMutex mut(filename + "Internal");
+
+        if(!mut.Lock()) {
+            EQ_EXCEPT("Shared Memory", "Could not lock shared mutex.");
+        }
+    
+        //get existing size
+        FILE *f = fopen(filename.c_str(), "rb");
+        if(!f) {
+            EQ_EXCEPT("Shared Memory", "Could not open the file to find the existing file size.");
+        }
+        fseek(f, 0U, SEEK_END);
+        uint32 size = static_cast<uint32>(ftell(f)) - sizeof(shared_memory_struct);
+        fclose(f);
+
+#ifdef _WINDOWS
+        DWORD total_size = size + sizeof(shared_memory_struct);
+        HANDLE file = CreateFile(filename.c_str(), 
+            GENERIC_READ | GENERIC_WRITE, 
+            FILE_SHARE_READ | FILE_SHARE_WRITE, 
+            NULL,
+            OPEN_ALWAYS,
+            0,
+            NULL);
+    
+        if(file == INVALID_HANDLE_VALUE) {
+            EQ_EXCEPT("Shared Memory", "Could not open a file for this shared memory segment.");
+        }
+    
+        imp_->mapped_object_ = CreateFileMapping(file,
+            NULL,
+            PAGE_READWRITE, 
+            0,
+            total_size,
+            filename.c_str());
+    
+        if(!imp_->mapped_object_) {
+            mut.Unlock();
+            EQ_EXCEPT("Shared Memory", "Could not create a file mapping for this shared memory file.");
+        }
+    
+        memory_ = reinterpret_cast<shared_memory_struct*>(MapViewOfFile(imp_->mapped_object_,
+            FILE_MAP_ALL_ACCESS,
+            0,
+            0, 
+            total_size));
+    
+        if(!memory_) {
+            mut.Unlock();
+            EQ_EXCEPT("Shared Memory", "Could not map a view of the shared memory file.");
+        }
+    
+#else
+        size_t total_size = size + sizeof(shared_memory_struct);
+        imp_->fd_ = open(filename.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
+        if(imp_->fd_ == -1) {
+            mut.Unlock();
+            EQ_EXCEPT("Shared Memory", "Could not open a file for this shared memory segment.");
+        }
+
+        if(ftruncate(imp_->fd_, total_size) == -1) {
+            EQ_EXCEPT("Shared Memory", "Could not set file size for this shared memory segment.");
+        }
+
+        memory_ = reinterpret_cast<shared_memory_struct*>(
+            mmap(NULL, total_size, PROT_READ | PROT_WRITE, MAP_FILE | MAP_SHARED, imp_->fd_, 0));
+
+        if(memory_ == MAP_FAILED) {
+            mut.Unlock();
+            EQ_EXCEPT("Shared Memory", "Could not create a file mapping for this shared memory file.");
+        }
+#endif
+        mut.Unlock();
+    }
+
+    MemoryMappedFile::~MemoryMappedFile() {
+#ifdef _WINDOWS
+        if(imp_->mapped_object_) {
+            CloseHandle(imp_->mapped_object_);
+        }
+#else
+        if(memory_) {
+            size_t total_size = size_ + sizeof(shared_memory_struct);
+            munmap(reinterpret_cast<void*>(memory_), total_size);
+            close(imp_->fd_);
+        }
+#endif
+        delete imp_;
+    }
+    
+    void MemoryMappedFile::ZeroFile() {
+        memset(reinterpret_cast<void*>(memory_), 0, sizeof(shared_memory_struct));
+        memory_->loaded = false;
+        memory_->size = size_;
+    }
+} // EQEmu

common/memory_mapped_file.h

@@ -0,0 +1,75 @@
+#ifndef _EQEMU_MEMORYMAPPEDFILE_H_
+#define _EQEMU_MEMORYMAPPEDFILE_H_
+
+#include <string>
+#include "types.h"
+
+namespace EQEmu {
+
+    //! Memory Backed Shared Memory
+    /*!
+        Allows us to create shared memory that is backed by a file on both windows and unix platforms that 
+        works in a consistent manner.  Non-copyable.
+    */
+    class MemoryMappedFile {
+        struct Implementation;
+        struct shared_memory_struct;
+
+        //! Underlying data structure.
+        struct shared_memory_struct {
+            bool loaded;
+            uint32 size;
+            unsigned char data[1];
+        };
+    public:
+        //! Constructor
+        /*! 
+            Creates a mmf for the given filename and of size.
+        \param filename Actual filename of the mmf.
+        \param size Size in bytes of the mmf.
+        */
+        MemoryMappedFile(std::string filename, uint32 size);
+        
+        //! Constructor
+        /*! 
+            Creates a mmf for the given filename and gets the size based on the existing size.
+        \param filename Actual filename of the mmf.
+        */
+        MemoryMappedFile(std::string filename);
+
+        //! Destructor
+        ~MemoryMappedFile();
+
+        //! Get Data Operator
+        inline void *operator->() const { return memory_->data; }
+        
+        //! Get Data Function
+        inline void *Get() const { return memory_->data; }
+        
+        //! Get Size Function
+        inline uint32 Size() const { return memory_->size; }
+        
+        //! Returns whether this memory is loaded or not
+        inline bool Loaded() const { return memory_->loaded; }
+        
+        //! Sets the memory to be loaded
+        inline void SetLoaded() { memory_->loaded = true; }
+        
+        //! Zeros all the memory in the file, and set it to be unloaded
+        void ZeroFile();
+    private:
+        //! Copy Constructor
+        MemoryMappedFile(const MemoryMappedFile&);
+        
+        //! Assignment Operator
+        const MemoryMappedFile& operator=(const MemoryMappedFile&);
+
+        std::string filename_; //!< Filename of this shared memory object
+        uint32 size_; //!< Size in bytes of this shared memory object
+        shared_memory_struct *memory_; //!< Underlying data of the shared memory object.
+
+        Implementation *imp_; //!< Underlying implementation.
+    };
+} // EQEmu
+
+#endif