forked from simongog/sdsl-lite
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added little memory management system.
And example how the new feature can be used is given in examples/hugepages.cpp.
- Loading branch information
Simon Gog
committed
Aug 30, 2012
1 parent
fca2cbb
commit 2e58639
Showing
4 changed files
with
251 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| #include <sdsl/int_vector.hpp> | ||
| #include <sdsl/suffixtrees.hpp> | ||
| #include <sdsl/wt_rlmn.hpp> | ||
| #include <iostream> | ||
| #include <string> | ||
|
|
||
| using namespace sdsl; | ||
| using namespace std; | ||
|
|
||
| template<class tCsa> | ||
| void do_something(const tCsa &csa){ | ||
| uint64_t sum=0; | ||
| for(size_t i=0; i<csa.size();++i){ | ||
| sum+=csa[i]; | ||
| } | ||
| cout<<"sum="<<sum<<endl; | ||
| } | ||
|
|
||
| int main(int argc, char** argv){ | ||
| // util::verbose = true; | ||
| { | ||
| bit_vector b(32,1); | ||
| cout << b << endl; | ||
| } | ||
| { | ||
| bit_vector b(32,0); | ||
| bool mapped = mm::map_hp(); | ||
| if ( mapped ) mm::unmap_hp(); | ||
| cout << b << endl; | ||
| } | ||
| bool mapped; | ||
| // bool mapped = mm::map_hp(); | ||
| // if( mapped ) mm::unmap_hp(); | ||
|
|
||
| csa_wt<wt_rlmn<> > csa; | ||
| construct_csa(string(argv[1]), csa); | ||
| do_something(csa); // before it is mapped | ||
| mapped = mm::map_hp(); | ||
| do_something(csa); // while it is mapped | ||
| if ( mapped ) mm::unmap_hp(); | ||
| do_something(csa); // after it is unmapped | ||
| util::clear(csa); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| /*!\file memory_management.hpp | ||
| \brief memory_management.hpp contains two function for allocating and deallocating memory | ||
| \author Your name | ||
| */ | ||
| #ifndef INCLUDED_MEMORY_MANAGEMENT | ||
| #define INCLUDED_MEMORY_MANAGEMENT | ||
|
|
||
| #include "uintx_t.hpp" | ||
| #include "util.hpp" | ||
| #include <map> | ||
| #include <iostream> | ||
| using std::cout; | ||
| using std::endl; | ||
|
|
||
| namespace sdsl{ | ||
|
|
||
| class mm_item_base{ | ||
| public: | ||
| mm_item_base(){}; | ||
| virtual bool map_hp(uint64_t*&){return false;};// const = 0; | ||
| virtual bool unmap_hp(){return false;};// const = 0; | ||
| virtual ~mm_item_base(){}; | ||
| virtual uint64_t size(){return 0;}; | ||
| }; | ||
|
|
||
| template<class int_vector_type> | ||
| class mm_item : public mm_item_base{ | ||
| private: | ||
| int_vector_type *m_v; | ||
| public: | ||
| explicit mm_item(int_vector_type *v):m_v(v){} | ||
| ~mm_item(){ } | ||
|
|
||
| //! Map content of int_vector to a hugepage starting at address addr | ||
| /*! | ||
| * Details: The content of the corresponding int_vector of mm_item | ||
| * is copied into a hugepage starting at address addr. | ||
| * The string position in the hugepage is then increased | ||
| * by the number of bytes used by the int_vector. | ||
| * So addr is the new starting address for the next | ||
| * mm_item which have to be mapped. | ||
| */ | ||
| bool map_hp(uint64_t*& addr){ | ||
| uint64_t len = size(); | ||
| if ( m_v->m_data != NULL ){ | ||
| memcpy((char*)addr, m_v->m_data, len); // copy old data | ||
| free(m_v->m_data); | ||
| m_v->m_data = addr; | ||
| addr += (len/8); | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| //! | ||
| bool unmap_hp(){ | ||
| uint64_t len = size(); | ||
| if ( util::verbose ){ | ||
| std::cerr<<"unmap int_vector of size "<< len <<std::endl; | ||
| std::cerr<<"m_data="<<m_v->m_data<<std::endl; | ||
| } | ||
| uint64_t* tmp_data = (uint64_t*)malloc(len); // allocate memory for m_data | ||
| memcpy(tmp_data, m_v->m_data, len); // copy data from the mmapped region | ||
| m_v->m_data = tmp_data; | ||
| return true; | ||
| } | ||
|
|
||
| uint64_t size(){ | ||
| return ((m_v->bit_size()+63)>>6)<<3; | ||
| } | ||
| }; | ||
|
|
||
| class mm_initializer; // forward declaration of initialization helper | ||
|
|
||
| // memory management class | ||
| class mm{ | ||
| friend class mm_initializer; | ||
| typedef std::map<uint64_t, mm_item_base*> tMVecItem; | ||
| static tMVecItem m_items; | ||
| static uint64_t m_total_memory; | ||
| static uint64_t *m_data; | ||
| public: | ||
| mm(); | ||
|
|
||
| template<class int_vector_type> | ||
| static void add(int_vector_type *v){ | ||
| if( mm::m_items.find((uint64_t)v) == mm::m_items.end() ){ | ||
| mm_item_base* item = new mm_item<int_vector_type>(v); | ||
| if(util::verbose) cout << "mm::add: add vector " << v << endl; | ||
| mm::m_items[(uint64_t)v] = item; | ||
| }else{ | ||
| if(util::verbose) cout << "mm::add: mm_item is already in the set" << endl; | ||
| } | ||
| } | ||
|
|
||
| template<class int_vector_type> | ||
| static void remove(int_vector_type *v){ | ||
| if( mm::m_items.find((uint64_t)v) != mm::m_items.end() ){ | ||
| if( util::verbose ){ cout << "mm:remove: remove vector " << v << endl; }; | ||
| mm_item_base* item = m_items[(uint64_t)v]; | ||
| mm::m_items.erase((uint64_t)v); | ||
| delete item; | ||
| }else{ | ||
| if( util::verbose ){ cout << "mm:remove: mm_item is not in the set" << endl; }; | ||
| } | ||
| } | ||
|
|
||
| static bool map_hp(); | ||
| static bool unmap_hp(); | ||
| }; | ||
|
|
||
| static class mm_initializer{ | ||
| public: | ||
| mm_initializer (); | ||
| ~mm_initializer (); | ||
| } initializer; | ||
|
|
||
| } // end namespace | ||
|
|
||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| #include "sdsl/memory_management.hpp" | ||
|
|
||
| #include <cstdlib> // for malloc and free | ||
| #include <sys/mman.h> | ||
|
|
||
| #define HUGE_LEN 1073741824 | ||
| #define HUGE_PROTECTION (PROT_READ | PROT_WRITE) | ||
| #define HUGE_FLAGS (MAP_HUGETLB | MAP_ANONYMOUS | MAP_PRIVATE) | ||
|
|
||
| //! Namespace for the succinct data structure library | ||
| namespace sdsl | ||
| { | ||
| static int nifty_counter; | ||
| std::map<uint64_t, mm_item_base*> mm::m_items; | ||
| uint64_t mm::m_total_memory; | ||
| uint64_t *mm::m_data; | ||
|
|
||
| mm_initializer::mm_initializer(){ | ||
| if ( 0 == nifty_counter++ ){ | ||
| // initialize static members object here | ||
| // mm::m_items.clear(); | ||
| mm::m_total_memory = 0; | ||
| mm::m_data = NULL; | ||
| } | ||
| } | ||
| mm_initializer::~mm_initializer(){ | ||
| if ( 0 == --nifty_counter ){ | ||
| // clean up | ||
| } | ||
| } | ||
|
|
||
| bool mm::map_hp(){ | ||
| m_total_memory = 0; // memory of all int_vectors | ||
| for(tMVecItem::const_iterator it=m_items.begin(); it!=m_items.end(); ++it){ | ||
| m_total_memory += it->second->size(); | ||
| } | ||
| if(util::verbose){ | ||
| std::cout<<"m_total_memory"<<m_total_memory<<std::endl; | ||
| } | ||
| size_t hpgs= (m_total_memory+HUGE_LEN-1)/HUGE_LEN; // number of huge pages required to store the int_vectors | ||
| m_data = (uint64_t*)mmap(NULL, hpgs*HUGE_LEN, HUGE_PROTECTION, HUGE_FLAGS, 0, 0); | ||
| if (m_data == MAP_FAILED) { | ||
| std::cout << "mmap was not successful" << std::endl; | ||
| return false; | ||
| }else{ | ||
| if( util::verbose ){ | ||
| std::cerr<<"map " << m_total_memory << " bytes" << std::endl; | ||
| } | ||
| } | ||
| // map int_vectors | ||
| uint64_t *addr = m_data; | ||
| bool success = true; | ||
| for(tMVecItem::const_iterator it=m_items.begin(); it!=m_items.end(); ++it){ | ||
| // std::cerr<<"addr = "<< addr << std::endl; | ||
| success = success && it->second->map_hp( addr ); | ||
| } | ||
| return success; | ||
| } | ||
|
|
||
| bool mm::unmap_hp(){ | ||
| size_t hpgs= (m_total_memory+HUGE_LEN-1)/HUGE_LEN; // number of huge pages | ||
| if ( util::verbose ){ | ||
| std::cerr<<"unmap "<< m_total_memory << " bytes" <<std::endl; | ||
| std::cerr.flush(); | ||
| } | ||
| bool success = true; | ||
| for(tMVecItem::const_iterator it=m_items.begin(); it!=m_items.end(); ++it){ | ||
| success = success && it->second->unmap_hp(); | ||
| } | ||
| // uint64_t* tmp_data = (uint64_t*)malloc(m_total_memory); // allocate memory for int_vectors | ||
| // memcpy(tmp_data, m_data, len); // copy data from the mmapped region | ||
| int ret = munmap((void*)m_data, hpgs*HUGE_LEN ); | ||
| if ( ret == -1 ){ | ||
| perror("Unmap failed"); | ||
| return false; | ||
| } | ||
| return success; | ||
| } | ||
|
|
||
| } // end namespace |