Skip to content

Commit

Permalink
Merge pull request simongog#81 from mpetri/new-memory-management
Browse files Browse the repository at this point in the history 
New memory management system with better logging and hugepage support
  • Loading branch information
@simongog
simongog committed Sep 11, 2013
2 parents 2bc7e2f + 4f12fdb commit f215cd7
Show file tree
Hide file tree
Showing 11 changed files with 691 additions and 352 deletions.
26 changes: 17 additions & 9 deletions examples/hugepages.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ void do_something(const tCsa& csa)
sum+=csa.psi(i);
}
auto stop = timer::now();
cout << duration_cast<microseconds>(stop-start).count() << endl;
cout << "runtime in s: " << duration_cast<seconds>(stop-start).count() << endl;
cout <<"sum="<<sum<<endl;
}

Expand All @@ -28,15 +28,23 @@ int main(int argc, char** argv)
cout << " (2) Runs a benchmark with enabled/disabled 1GB=hugepages." << endl;
return 1;
}

if (argc==3) {
memory_manager::use_hugepages(500*1024*1024);
}

memory_monitor::start();

csa_wt<> csa;
auto start = timer::now();
construct(csa, argv[1], 1);
auto stop = timer::now();
cout << "construction in s: " << duration_cast<seconds>(stop-start).count() << endl;
do_something(csa); // before it is mapped
if (mm::map_hp()) {
cout << "Now the memory is mapped to hugepages " << endl;
do_something(csa); // while it is mapped
mm::unmap_hp();
} else {
cout << "Not able to map the memory to hugepages" << endl;
}
do_something(csa); // after it is unmapped

memory_monitor::stop();

std::ofstream ofs("cst-construction.json");
memory_monitor::write_memory_log<JSON>(ofs);
ofs.close();
}
20 changes: 10 additions & 10 deletions include/sdsl/construct.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ void construct(t_index& idx, const std::string& file, cache_config& config, uint
template<class t_index>
void construct(t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, wt_tag)
{
mm::log("wt-begin");
memory_monitor::event("wt-begin");
int_vector<t_index::alphabet_category::WIDTH> text;
load_vector_from_file(text, file, num_bytes);
std::string tmp_key = util::to_string(util::pid())+"_"+util::to_string(util::id());
Expand All @@ -107,7 +107,7 @@ void construct(t_index& idx, const std::string& file, cache_config& config, uint
idx.swap(tmp);
}
sdsl::remove(tmp_file_name);
mm::log("wt-end");
memory_monitor::event("wt-end");
}

// Specialization for CSAs
Expand All @@ -121,23 +121,23 @@ void construct(t_index& idx, const std::string& file, cache_config& config, uint
// (1) check, if the text is cached
if (!cache_file_exists(KEY_TEXT, config)) {
text_type text;
mm::log("text-begin");
memory_monitor::event("text-begin");
load_vector_from_file(text, file, num_bytes);
if (contains_no_zero_symbol(text, file)) {
append_zero_symbol(text);
store_to_cache(text, KEY_TEXT, config);
}
load_from_cache(text, KEY_TEXT, config);
mm::log("text-end");
memory_monitor::event("text-end");
}
register_cache_file(KEY_TEXT, config);
}
{
// (2) check, if the suffix array is cached
if (!cache_file_exists(constants::KEY_SA, config)) {
mm::log("sa-begin");
memory_monitor::event("sa-begin");
construct_sa<t_index::alphabet_category::WIDTH>(config);
mm::log("sa-end");
memory_monitor::event("sa-end");
}
register_cache_file(constants::KEY_SA, config);
int_vector<> sa;
Expand All @@ -146,9 +146,9 @@ void construct(t_index& idx, const std::string& file, cache_config& config, uint
{
// (3) construct BWT
if (!cache_file_exists(KEY_BWT, config)) {
mm::log("bwt-begin");
memory_monitor::event("bwt-begin");
construct_bwt<t_index::alphabet_category::WIDTH>(config);
mm::log("bwt-end");
memory_monitor::event("bwt-end");
}
register_cache_file(constants::KEY_BWT, config);
int_vector<t_index::alphabet_category::WIDTH> bwt;
Expand Down Expand Up @@ -187,13 +187,13 @@ void construct(t_index& idx, const std::string& file, cache_config& config, uint
register_cache_file(KEY_BWT, config);
register_cache_file(constants::KEY_SA, config);
if (!cache_file_exists(constants::KEY_LCP, config)) {
mm::log("lcp-begin");
memory_monitor::event("lcp-begin");
if (t_index::alphabet_category::WIDTH==8) {
construct_lcp_semi_extern_PHI(config);
} else {
construct_lcp_PHI<t_index::alphabet_category::WIDTH>(config);
}
mm::log("lcp-end");
memory_monitor::event("lcp-end");
}
register_cache_file(constants::KEY_LCP, config);
}
Expand Down
20 changes: 10 additions & 10 deletions include/sdsl/csa_sada.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,18 +362,18 @@ csa_sada<t_enc_vec, t_dens, t_inv_dens, t_sa_sample_strat, t_isa, t_alphabet_str
}
int_vector_buffer<alphabet_type::int_width> bwt_buf(cache_file_name(key_trait<alphabet_type::int_width>::KEY_BWT,config));
size_type n = bwt_buf.size();
mm::log("csa-alphabet-construct-begin");
memory_monitor::event("csa-alphabet-construct-begin");
{
alphabet_type tmp_alphabet(bwt_buf, n);
m_alphabet.swap(tmp_alphabet);
}
mm::log("csa-alphabet-construct-end");
memory_monitor::event("csa-alphabet-construct-end");

int_vector<> cnt_chr(sigma, 0, bits::hi(n)+1);
for (typename alphabet_type::sigma_type i=0; i < sigma; ++i) {
cnt_chr[i] = C[i];
}
mm::log("csa-psi-begin");
memory_monitor::event("csa-psi-begin");
// calculate psi
{
// TODO: move PSI construct into construct_PSI.hpp
Expand All @@ -386,25 +386,25 @@ csa_sada<t_enc_vec, t_dens, t_inv_dens, t_sa_sample_strat, t_isa, t_alphabet_str
return;
}
}
mm::log("csa-psi-end");
memory_monitor::event("csa-psi-end");
int_vector_buffer<> psi_buf(cache_file_name(constants::KEY_PSI, config));
mm::log("csa-psi-encode-begin");
memory_monitor::event("csa-psi-encode-begin");
{
t_enc_vec tmp_psi(psi_buf);
m_psi.swap(tmp_psi);
}
mm::log("csa-psi-encode-end");
memory_monitor::event("csa-psi-encode-end");
int_vector_buffer<> sa_buf(cache_file_name(constants::KEY_SA, config));
mm::log("sa-sample-begin");
memory_monitor::event("sa-sample-begin");
{
sa_sample_type tmp_sa_sample(config);
m_sa_sample.swap(tmp_sa_sample);
}
mm::log("sa-sample-end");
memory_monitor::event("sa-sample-end");

mm::log("isa-sample-begin");
memory_monitor::event("isa-sample-begin");
set_isa_samples<csa_sada>(sa_buf, m_isa_sample);
mm::log("isa-sample-end");
memory_monitor::event("isa-sample-end");
}

template<class t_enc_vec, uint32_t t_dens, uint32_t t_inv_dens, class t_sa_sample_strat, class t_isa, class t_alphabet_strat>
Expand Down
16 changes: 8 additions & 8 deletions include/sdsl/csa_wt.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,30 +276,30 @@ csa_wt<t_wt, t_dens, t_inv_dens, t_sa_sample_strat, t_isa, t_alphabet_strat>::cs
int_vector_buffer<alphabet_type::int_width> bwt_buf(cache_file_name(key_trait<alphabet_type::int_width>::KEY_BWT,config));
int_vector_buffer<> sa_buf(cache_file_name(constants::KEY_SA, config));
size_type n = bwt_buf.size();
mm::log("csa-alphabet-construct-begin");
memory_monitor::event("csa-alphabet-construct-begin");
{
alphabet_type tmp_alphabet(bwt_buf, n);
m_alphabet.swap(tmp_alphabet);
}
mm::log("csa-alphabet-construct-end");
memory_monitor::event("csa-alphabet-construct-end");

mm::log("wt-begin");
memory_monitor::event("wt-begin");
{
wavelet_tree_type tmp_wt(bwt_buf, n);
m_wavelet_tree.swap(tmp_wt);
}
mm::log("wt-end");
memory_monitor::event("wt-end");

mm::log("sa-sample-begin");
memory_monitor::event("sa-sample-begin");
{
sa_sample_type tmp_sa_sample(config);
m_sa_sample.swap(tmp_sa_sample);
}
mm::log("sa-sample-end");
memory_monitor::event("sa-sample-end");

mm::log("isa-sample-begin");
memory_monitor::event("isa-sample-begin");
set_isa_samples<csa_wt>(sa_buf, m_isa_sample);
mm::log("isa-sample-end");
memory_monitor::event("isa-sample-end");
}


Expand Down
12 changes: 6 additions & 6 deletions include/sdsl/cst_sada.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ class cst_sada
//! Construct CST from file_map
cst_sada(cache_config& config) {
{
mm::log("bps-dfs-begin");
memory_monitor::event("bps-dfs-begin");
cst_sct3<> temp_cst(config, true);
m_bp.resize(4*(temp_cst.bp.size()/2));
util::set_to_value(m_bp, 0);
Expand All @@ -158,19 +158,19 @@ class cst_sada
++idx;
}
m_bp.resize(idx);
mm::log("bps-dfs-end");
memory_monitor::event("bps-dfs-end");
}
mm::log("bpss-dfs-begin");
memory_monitor::event("bpss-dfs-begin");
util::assign(m_bp_support, bp_support_type(&m_bp));
util::init_support(m_bp_rank10, &m_bp);
util::init_support(m_bp_select10, &m_bp);
mm::log("bpss-dfs-end");
memory_monitor::event("bpss-dfs-end");

mm::log("bpss-clcp-begin");
memory_monitor::event("bpss-clcp-begin");
cache_config tmp_config(false, config.dir, config.id, config.file_map);
construct_lcp(m_lcp, *this, tmp_config);
config.file_map = tmp_config.file_map;
mm::log("bpss-clcp-end");
memory_monitor::event("bpss-clcp-end");

load_from_cache(m_csa, util::class_to_hash(m_csa), config);
}
Expand Down
12 changes: 6 additions & 6 deletions include/sdsl/cst_sct3.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1076,24 +1076,24 @@ class cst_sct3
template<class t_csa, class t_lcp, class t_bp_support, class t_rank>
cst_sct3<t_csa, t_lcp, t_bp_support, t_rank>::cst_sct3(cache_config& config, bool build_only_bps)
{
mm::log("bps-sct-begin");
memory_monitor::event("bps-sct-begin");
int_vector_buffer<> lcp_buf(cache_file_name(constants::KEY_LCP, config));
m_nodes = construct_supercartesian_tree_bp_succinct_and_first_child(lcp_buf, m_bp, m_first_child) + m_bp.size()/2;
if (m_bp.size() == 2) { // handle special case, when the tree consists only of the root node
m_nodes = 1;
}
mm::log("bps-sct-end");
mm::log("bpss-sct-begin");
memory_monitor::event("bps-sct-end");
memory_monitor::event("bpss-sct-begin");
util::init_support(m_bp_support, &m_bp);
util::init_support(m_first_child_rank, &m_first_child);
mm::log("bpss-sct-end");
memory_monitor::event("bpss-sct-end");

if (!build_only_bps) {
mm::log("clcp-begin");
memory_monitor::event("clcp-begin");
cache_config tmp_config(false, config.dir, config.id, config.file_map);
construct_lcp(m_lcp, *this, tmp_config);
config.file_map = tmp_config.file_map;
mm::log("clcp-end");
memory_monitor::event("clcp-end");
}
if (!build_only_bps) {
load_from_cache(m_csa, util::class_to_hash(m_csa), config);
Expand Down
22 changes: 5 additions & 17 deletions include/sdsl/int_vector.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,6 @@
#ifndef INCLUDED_SDSL_INT_VECTOR
#define INCLUDED_SDSL_INT_VECTOR

#include <sys/mman.h>

#define HUGE_LEN 1073741824
#define HUGE_PROTECTION (PROT_READ | PROT_WRITE)
#define HUGE_FLAGS (MAP_HUGETLB | MAP_ANONYMOUS | MAP_PRIVATE)

#include "compatibility.hpp"
#include "bits.hpp"
#include "structure_tree.hpp"
Expand Down Expand Up @@ -54,7 +48,6 @@
#include <string>
#include <initializer_list>


//! Namespace for the succinct data structure library.
namespace sdsl
{
Expand Down Expand Up @@ -277,8 +270,7 @@ class int_vector
friend class int_vector_const_iterator<int_vector>;
friend class coder::elias_delta;
friend class coder::fibonacci;
friend class mm_item<int_vector>;
friend class mm;
friend class memory_manager;

friend void util::set_random_bits<int_vector>(int_vector& v, int);
friend void util::_set_zero_bits<int_vector>(int_vector&);
Expand Down Expand Up @@ -312,7 +304,7 @@ class int_vector
int_vector(std::initializer_list<t_T> il) : int_vector() {
resize(il.size());
size_type idx = 0;
for (auto x : il) {
for (auto x : il) {
(*this)[idx++] = x;
}
}
Expand Down Expand Up @@ -1118,7 +1110,6 @@ template<uint8_t t_width>
inline int_vector<t_width>::int_vector(size_type size, value_type default_value, uint8_t intWidth):
m_size(0), m_data(nullptr), m_width(t_width)
{
mm::add(this);
width(intWidth);
resize(size);
util::set_to_value(*this, default_value); // new initialization
Expand All @@ -1128,16 +1119,14 @@ template<uint8_t t_width>
inline int_vector<t_width>::int_vector(int_vector&& v) :
m_size(v.m_size), m_data(v.m_data), m_width(v.m_width)
{
v.m_size = 0; // has to be set for mm::remove
v.m_data = nullptr; // ownership of v.m_data now transfered
mm::add(this, true);
v.m_size = 0;
}

template<uint8_t t_width>
inline int_vector<t_width>::int_vector(const int_vector& v):
m_size(0), m_data(nullptr), m_width(v.m_width)
{
mm::add(this);
bit_resize(v.bit_size());
if (v.capacity() > 0) {
if (memcpy(m_data, v.data() ,v.capacity()/8)==nullptr) {
Expand Down Expand Up @@ -1173,8 +1162,7 @@ int_vector<t_width>& int_vector<t_width>::operator=(int_vector&& v)
template<uint8_t t_width>
int_vector<t_width>::~int_vector()
{
mm::remove(this);
free(m_data);
memory_manager::clear(*this);
}

template<uint8_t t_width>
Expand All @@ -1196,7 +1184,7 @@ void int_vector<t_width>::swap(int_vector& v)
template<uint8_t t_width>
void int_vector<t_width>::bit_resize(const size_type size)
{
mm::realloc(*this, size);
memory_manager::resize(*this, size);
}

template<uint8_t t_width>
Expand Down
Loading

0 comments on commit f215cd7

Please sign in to comment.