From fdf74cedc118cd7e6dd2c6ac5b4c12e10ba33f5e Mon Sep 17 00:00:00 2001 From: zhenyatian Date: Tue, 23 Jul 2019 12:34:19 +0800 Subject: [PATCH] insert/remove update --- Database/Database.cpp | 368 ++++++++++++++++++++++++-- Database/Database.h | 2 + KVstore/IVArray/IVArray.cpp | 111 +++++++- KVstore/IVArray/IVArray.h | 1 + KVstore/IVArray/IVBlockManager.cpp | 301 +++++++++++++++++----- KVstore/IVArray/IVBlockManager.h | 1 + KVstore/KVstore.cpp | 401 ++++++++++++++++++++++++++--- KVstore/KVstore.h | 2 +- 8 files changed, 1048 insertions(+), 139 deletions(-) diff --git a/Database/Database.cpp b/Database/Database.cpp index 6954fa46..0832415d 100644 --- a/Database/Database.cpp +++ b/Database/Database.cpp @@ -860,17 +860,17 @@ Database::load_cache() cerr << "Get in" << endl; { pid_t p = getpid(); - char file[64] = { 0 };//ÎļþÃû - FILE *fd; //¶¨ÒåÎļþÖ¸Õëfd - char line_buff[256] = { 0 }; //¶ÁÈ¡ÐеĻº³åÇø + char file[64] = { 0 };//�ļ��� + FILE *fd; //�����ļ�ָ��fd + char line_buff[256] = { 0 }; //��ȡ�еĻ����� sprintf(file, "/proc/%d/status", p); fprintf(stderr, "current pid:%d\n", p); - fd = fopen(file, "r"); //ÒÔR¶ÁµÄ·½Ê½´ò¿ªÎļþÔÙ¸³¸øÖ¸Õëfd + fd = fopen(file, "r"); //��R���ķ�ʽ���ļ��ٸ���ָ��fd - int i; //»ñÈ¡vmrss:ʵ¼ÊÎïÀíÄÚ´æÕ¼Óà - char name[32];//´æ·ÅÏîÄ¿Ãû³Æ - int vmrss;//´æ·ÅÄÚ´æ - for (i = 0; i<17; i++) //¶ÁÈ¡VmRSSÕâÒ»ÐеÄÊý¾Ý + int i; //��ȡvmrss:ʵ�������ڴ�ռ�� + char name[32];//�����Ŀ���� + int vmrss;//����ڴ� + for (i = 0; i<17; i++) //��ȡVmRSS��һ�е����� { char* ret = fgets(line_buff, sizeof(line_buff), fd); if (i == 11 || i == 12 || i == 15 || i == 16) @@ -879,7 +879,7 @@ Database::load_cache() fprintf(stderr, "%s\t%d kb\n", name, vmrss); } } - fclose(fd); //¹Ø±ÕÎļþfd + fclose(fd); //�ر��ļ�fd }*/ } @@ -1384,7 +1384,7 @@ bool Database::save() this->stringindex->flush(); this->clear_update_log(); - //cerr<<"database checkpoint: "<getName()<getName()<Addstring(t); } } - cout<<"Add triples to Trie to prepare for BuildPrefix"<BuildPrefix(); cout << "BuildPrefix done. used" <* _vert //return updateLen; } + +unsigned +Database::insertTripleByGroup(const TripleWithObjType* _triple, TYPE_TRIPLE_NUM _triple_num, vector* _vertices, vector* _predicates) +{ + long tv_kv_store_begin = Util::get_cur_time(); + + TYPE_TRIPLE_NUM valid_num = 0; + TYPE_TRIPLE_NUM tuple_num = 0; + TYPE_TRIPLE_NUM repeat_num = 0; + TYPE_ENTITY_LITERAL_ID** id_tuples = new TYPE_ENTITY_LITERAL_ID*[_triple_num]; + //struct ID_TUPLE { + // TYPE_ENTITY_LITERAL_ID _sub_id; + // TYPE_ENTITY_LITERAL_ID _pre_id; + // unsigned len = 0; + // TYPE_ENTITY_LITERAL_ID* _obj_id = new TYPE_ENTITY_LITERAL_ID[5]; + //}id_tuple[1000000]; + //unsigned max_len = 1; + //unsigned len = 1; + for (int i = 0; i < _triple_num; ++i) + { + //cout << "insert:" << _triple[i].subject << "-" << _triple[i].predicate << "-" << _triple[i].object << endl; + TYPE_ENTITY_LITERAL_ID _sub_id = (this->kvstore)->getIDByEntity(_triple[i].subject); + bool _is_new_sub = false; + if (_sub_id == INVALID_ENTITY_LITERAL_ID) + { + _is_new_sub = true; + _sub_id = this->allocEntityID(); + this->sub_num++; + (this->kvstore)->setIDByEntity(_triple[i].subject, _sub_id); + (this->kvstore)->setEntityByID(_sub_id, _triple[i].subject); + if (_vertices != NULL) + _vertices->push_back(_sub_id); + } + TYPE_PREDICATE_ID _pre_id = (this->kvstore)->getIDByPredicate(_triple[i].predicate); + bool _is_new_pre = false; + if (_pre_id == INVALID_PREDICATE_ID) + { + _is_new_pre = true; + _pre_id = this->allocPredicateID(); + (this->kvstore)->setIDByPredicate(_triple[i].predicate, _pre_id); + (this->kvstore)->setPredicateByID(_pre_id, _triple[i].predicate); + if (_predicates != NULL) + _predicates->push_back(_pre_id); + } + TYPE_ENTITY_LITERAL_ID _obj_id = INVALID_ENTITY_LITERAL_ID; + bool _is_new_obj = false; + bool is_obj_entity = _triple[i].isObjEntity(); + if (is_obj_entity) + { + _obj_id = (this->kvstore)->getIDByEntity(_triple[i].object); + if (_obj_id == INVALID_ENTITY_LITERAL_ID) + { + _is_new_obj = true; + _obj_id = this->allocEntityID(); + (this->kvstore)->setIDByEntity(_triple[i].object, _obj_id); + (this->kvstore)->setEntityByID(_obj_id, _triple[i].object); + if (_vertices != NULL) + _vertices->push_back(_obj_id); + } + } + else + { + _obj_id = (this->kvstore)->getIDByLiteral(_triple[i].object); + if (_obj_id == INVALID_ENTITY_LITERAL_ID) + { + _is_new_obj = true; + _obj_id = this->allocLiteralID(); + (this->kvstore)->setIDByLiteral(_triple[i].object, _obj_id); + (this->kvstore)->setLiteralByID(_obj_id, _triple[i].object); + if (_vertices != NULL) + _vertices->push_back(_obj_id); + } + } + bool _triple_exist = false; + if (!_is_new_sub && !_is_new_pre && !_is_new_obj) + { + _triple_exist = this->exist_triple(_sub_id, _pre_id, _obj_id); + } + if (_triple_exist) + { + cout << "this triple already exist" << endl; + continue; + } + else + { + this->triples_num++; + id_tuples[valid_num] = new unsigned[3]; + id_tuples[valid_num][0] = _sub_id; + id_tuples[valid_num][1] = _pre_id; + id_tuples[valid_num][2] = _obj_id; + valid_num++; + } + } + if (valid_num == 0) + { + return 0; + } + TYPE_ENTITY_LITERAL_ID** id_tuple = new TYPE_ENTITY_LITERAL_ID*[valid_num]; + qsort(id_tuples, valid_num, sizeof(int*), Util::_pso_cmp); + id_tuple[0] = new unsigned[valid_num * 2 + 2]; + id_tuple[0][0] = id_tuples[0][1]; + id_tuple[0][2] = id_tuples[0][0]; + id_tuple[0][3] = id_tuples[0][2]; + id_tuple[0][1] = 4; + for (int i = 1; i < valid_num; ++i) + { + if (id_tuples[i][0] == id_tuples[i - 1][0] && id_tuples[i][1] == id_tuples[i - 1][1] && id_tuples[i][2] == id_tuples[i - 1][2]) + { + repeat_num++; + cout << "this triple already exist" << endl; + continue; + } + if (id_tuples[i][1] == id_tuples[i - 1][1]) + { + id_tuple[tuple_num][id_tuple[tuple_num][1]] = id_tuples[i][0]; + id_tuple[tuple_num][id_tuple[tuple_num][1] + 1] = id_tuples[i][2]; + id_tuple[tuple_num][1]+=2; + } + else + { + tuple_num++; + id_tuple[tuple_num] = new unsigned[valid_num * 2 + 2]; + id_tuple[tuple_num][0] = id_tuples[i][1]; + id_tuple[tuple_num][2] = id_tuples[i][0]; + id_tuple[tuple_num][3] = id_tuples[i][2]; + id_tuple[tuple_num][1] = 4; + } + delete[] id_tuples[i - 1]; + } + delete[] id_tuples[valid_num - 1]; + delete[] id_tuples; + tuple_num++; + long after_processtriple = Util::get_cur_time(); + cout << "process triple used: "<< (after_processtriple - tv_kv_store_begin) << "ms" << endl; + + for (int i = 0; i < tuple_num; ++i) + { + for (int j = 2; j < id_tuple[i][1]; j+=2) + { + (this->kvstore)->updateInsert_s2values(id_tuple[i][j], id_tuple[i][0], id_tuple[i][j+1]); + (this->kvstore)->updateInsert_o2values(id_tuple[i][j], id_tuple[i][0], id_tuple[i][j+1]); + (this->kvstore)->updateInsert_p2values(id_tuple[i][j], id_tuple[i][0], id_tuple[i][j+1]); + } + delete[] id_tuple[i]; + } + delete[] id_tuple; + return valid_num - repeat_num; +} + bool Database::removeTriple(const TripleWithObjType& _triple, vector* _vertices, vector* _predicates) { @@ -3125,6 +3274,167 @@ Database::removeTriple(const TripleWithObjType& _triple, vector* _vert return true; } +unsigned +Database::removeTripleByGroup(const TripleWithObjType* _triple, TYPE_TRIPLE_NUM _triple_num, vector* _vertices, vector* _predicates) +{ + long tv_kv_store_begin = Util::get_cur_time(); + long timecost = 0; + + TYPE_TRIPLE_NUM valid_num = 0; + TYPE_TRIPLE_NUM tuple_num = 0; + TYPE_TRIPLE_NUM repeat_num = 0; + TYPE_ENTITY_LITERAL_ID** id_tuples = new TYPE_ENTITY_LITERAL_ID*[_triple_num]; + + for (int i = 0; i < _triple_num; ++i) + { + long tv_kv_store_begin = Util::get_cur_time(); + + TYPE_ENTITY_LITERAL_ID _sub_id = (this->kvstore)->getIDByEntity(_triple[i].subject); + TYPE_PREDICATE_ID _pre_id = (this->kvstore)->getIDByPredicate(_triple[i].predicate); + TYPE_ENTITY_LITERAL_ID _obj_id = INVALID_ENTITY_LITERAL_ID; + if(_triple[i].isObjEntity()) + { + _obj_id = (this->kvstore)->getIDByEntity(_triple[i].object); + } + else + { + _obj_id = (this->kvstore)->getIDByLiteral(_triple[i].object); + } + if (_sub_id == INVALID_ENTITY_LITERAL_ID || _pre_id == INVALID_PREDICATE_ID || _obj_id == INVALID_ENTITY_LITERAL_ID) + { + continue; + } + + long tmp__1 = Util::get_cur_time(); + bool _exist_triple = this->exist_triple(_sub_id, _pre_id, _obj_id); + long tmp__2 = Util::get_cur_time(); + timecost += tmp__2 - tmp__1; + + if (!_exist_triple) + { + continue; + } + else + { + this->triples_num--; + } + long tv_kv_store_end = Util::get_cur_time(); + + id_tuples[valid_num] = new unsigned[3]; + id_tuples[valid_num][0] = _sub_id; + id_tuples[valid_num][1] = _pre_id; + id_tuples[valid_num][2] = _obj_id; + valid_num++; + } + if (valid_num == 0) + { + return 0; + } + TYPE_ENTITY_LITERAL_ID** id_tuple = new TYPE_ENTITY_LITERAL_ID*[valid_num]; + qsort(id_tuples, valid_num, sizeof(int*), Util::_pso_cmp); + id_tuple[0] = new unsigned[valid_num * 2 + 2]; + id_tuple[0][0] = id_tuples[0][1]; + id_tuple[0][2] = id_tuples[0][0]; + id_tuple[0][3] = id_tuples[0][2]; + id_tuple[0][1] = 4; + for (int i = 1; i < valid_num; ++i) + { + if (id_tuples[i][0] == id_tuples[i - 1][0] && id_tuples[i][1] == id_tuples[i - 1][1] && id_tuples[i][2] == id_tuples[i - 1][2]) + { + repeat_num++; + cout << "this triple already exist" << endl; + continue; + } + if (id_tuples[i][1] == id_tuples[i - 1][1]) + { + id_tuple[tuple_num][id_tuple[tuple_num][1]] = id_tuples[i][0]; + id_tuple[tuple_num][id_tuple[tuple_num][1] + 1] = id_tuples[i][2]; + id_tuple[tuple_num][1]+=2; + } + else + { + tuple_num++; + id_tuple[tuple_num] = new unsigned[valid_num * 2 + 2]; + id_tuple[tuple_num][0] = id_tuples[i][1]; + id_tuple[tuple_num][2] = id_tuples[i][0]; + id_tuple[tuple_num][3] = id_tuples[i][2]; + id_tuple[tuple_num][1] = 4; + } + delete[] id_tuples[i - 1]; + } + delete[] id_tuples[valid_num - 1]; + delete[] id_tuples; + tuple_num++; + long after_processtriple = Util::get_cur_time(); + cout << "exist used: " << timecost << "ms" << endl; + cout << "process triple used: "<< (after_processtriple - tv_kv_store_begin) << "ms" << endl; + for (int i = 0; i < tuple_num; ++i) + { + for (int j = 2; j < id_tuple[i][1]; j+=2) + { + (this->kvstore)->updateRemove_s2values(id_tuple[i][j], id_tuple[i][0], id_tuple[i][j+1]); + (this->kvstore)->updateRemove_o2values(id_tuple[i][j], id_tuple[i][0], id_tuple[i][j+1]); + (this->kvstore)->updateRemove_p2values(id_tuple[i][j], id_tuple[i][0], id_tuple[i][j+1]); + + int sub_degree = (this->kvstore)->getEntityDegree(id_tuple[i][j]); + if (sub_degree == 0) + { + string _sub = this->kvstore->getEntityByID(id_tuple[i][j]); + this->kvstore->subEntityByID(id_tuple[i][j]); + this->kvstore->subIDByEntity(_sub); + this->freeEntityID(id_tuple[i][j]); + this->sub_num--; + if (_vertices != NULL) + _vertices->push_back(id_tuple[i][j]); + } + bool is_obj_entity = Util::is_entity_ele(id_tuple[i][j+1]); + int obj_degree; + if (is_obj_entity) + { + obj_degree = this->kvstore->getEntityDegree(id_tuple[i][j+1]); + if (obj_degree == 0) + { + string _obj = this->kvstore->getEntityByID(id_tuple[i][j+1]); + this->kvstore->subEntityByID(id_tuple[i][j+1]); + this->kvstore->subIDByEntity(_obj); + this->freeEntityID(id_tuple[i][j+1]); + if (_vertices != NULL) + _vertices->push_back(id_tuple[i][j+1]); + } + } + else + { + obj_degree = this->kvstore->getLiteralDegree(id_tuple[i][j+1]); + if (obj_degree == 0) + { + string _obj = this->kvstore->getLiteralByID(id_tuple[i][j+1]); + this->kvstore->subLiteralByID(id_tuple[i][j+1]); + this->kvstore->subIDByLiteral(_obj); + this->freeLiteralID(id_tuple[i][j+1]); + if (_vertices != NULL) + _vertices->push_back(id_tuple[i][j+1]); + } + } + int pre_degree = this->kvstore->getPredicateDegree(id_tuple[i][0]); + if (pre_degree == 0) + { + string _pre = this->kvstore->getPredicateByID(id_tuple[i][0]); + this->kvstore->subPredicateByID(id_tuple[i][0]); + this->kvstore->subIDByPredicate(_pre); + this->freePredicateID(id_tuple[i][0]); + if (_predicates != NULL) + _predicates->push_back(id_tuple[i][0]); + } + } + } + for (int i = 0; i < tuple_num; ++i) + { + delete[] id_tuple[i]; + } + delete id_tuple; + return valid_num - repeat_num; +} + bool Database::insert(std::string _rdf_file, bool _is_restore) { @@ -3819,21 +4129,22 @@ Database::insert(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, #else //USE_GROUP_INSERT //NOTICE:we deal with insertions one by one here //Callers should save the vstree(node and info) after calling this function - for (TYPE_TRIPLE_NUM i = 0; i < _triple_num; ++i) - { - bool ret = this->insertTriple(_triples[i], &vertices, &predicates); - if(ret) - { - valid_num++; - } - } + //for (TYPE_TRIPLE_NUM i = 0; i < _triple_num; ++i) + //{ + //bool ret = this->insertTriple(_triples[i], &vertices, &predicates); + //if(ret) + //{ + //valid_num++; + //} + //} + valid_num = insertTripleByGroup(_triples, _triple_num, &vertices, &predicates); #endif //USE_GROUP_INSERT this->stringindex->SetTrie(kvstore->getTrie()); //update string index this->stringindex->change(vertices, *this->kvstore, true); this->stringindex->change(predicates, *this->kvstore, false); - + return valid_num; } @@ -4227,14 +4538,15 @@ Database::remove(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, #else //NOTICE:we deal with deletions one by one here //Callers should save the vstree(node and info) after calling this function - for (TYPE_TRIPLE_NUM i = 0; i < _triple_num; ++i) - { - bool ret = this->removeTriple(_triples[i], &vertices, &predicates); - if(ret) - { - valid_num++; - } - } + //for (TYPE_TRIPLE_NUM i = 0; i < _triple_num; ++i) + //{ + // bool ret = this->removeTriple(_triples[i], &vertices, &predicates); + // if(ret) + // { + // valid_num++; + // } + //} + valid_num = removeTripleByGroup(_triples, _triple_num, &vertices, &predicates); #endif this->stringindex->SetTrie(kvstore->getTrie()); //update string index diff --git a/Database/Database.h b/Database/Database.h index b6d0fe8a..0a851a35 100644 --- a/Database/Database.h +++ b/Database/Database.h @@ -274,7 +274,9 @@ class Database //insert and delete, notice that modify is not needed here //we can read from file or use sparql syntax bool insertTriple(const TripleWithObjType& _triple, vector* _vertices = NULL, vector* _predicates = NULL); + unsigned insertTripleByGroup(const TripleWithObjType* _triple, TYPE_TRIPLE_NUM _triple_num, vector* _vertices = NULL, vector* _predicates = NULL); bool removeTriple(const TripleWithObjType& _triple, vector* _vertices = NULL, vector* _predicates = NULL); + unsigned removeTripleByGroup(const TripleWithObjType* _triple, TYPE_TRIPLE_NUM _triple_num, vector* _vertices = NULL, vector* _predicates = NULL); //NOTICE:one by one is too costly, sort and insert/delete at a time will be better unsigned insert(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, bool _is_restore=false); //bool insert(const vector& _triples, vector& _vertices, vector& _predicates); diff --git a/KVstore/IVArray/IVArray.cpp b/KVstore/IVArray/IVArray.cpp index 7067ca59..f36df8ce 100644 --- a/KVstore/IVArray/IVArray.cpp +++ b/KVstore/IVArray/IVArray.cpp @@ -50,7 +50,7 @@ IVArray::IVArray(string _dir_path, string _filename, string mode, unsigned long //index_time_map.clear(); //time_index_map.clear(); MAX_CACHE_SIZE = buffer_size; -// MAX_CACHE_SIZE = 10 * (1 << 30); +// MAX_CACHE_SIZE = 10 * (1 << 18); cache_head = new IVEntry; cache_tail_id = -1; @@ -112,8 +112,7 @@ IVArray::IVArray(string _dir_path, string _filename, string mode, unsigned long pread(fd, &_store, 1 * sizeof(unsigned), offset); // if (i % 1000000 == 0) -// cout << _filename << ": Key " << i << " stored in block " << _store << endl; - +//cout << _filename << ": Key " << i << " stored in block " << _store << endl; array[i].setStore(_store); array[i].setDirtyFlag(false); @@ -151,6 +150,8 @@ IVArray::save() { //TODO Recycle free block _store = BM->WriteValue(str, len); + + //cout << ": Key " << i << " stored in block " << _store << endl; array[i].setStore(_store); } @@ -179,6 +180,7 @@ IVArray::SwapOut() int targetID; if ((targetID = cache_head->getNext()) == -1) // cache is empty { + cout << "cache is empty" << endl; return false; } @@ -201,8 +203,12 @@ IVArray::SwapOut() { //TODO recycle free blocks unsigned store = BM->WriteValue(str, len); + // cout << "SwapOut:" << len << endl; if (store == 0) + { + cout<< "writeValue return wrong store"<CacheLock.unlock(); + + //_str=new char[_len-24]; + //memcpy(_str,tmpStr,_len-24); + _len -= 24; + //delete[] tmpStr; + return ret; } // read in disk unsigned store = array[_key].getStore(); + //cout << "search:" << store << endl; if (!BM->ReadValue(store, _str, _len)) { this->CacheLock.unlock(); return false; } + //cout << "search:" << _key << "---" << _len << endl; if(!VList::isLongList(_len)) { // if (array[_key].Lock.try_lock()) @@ -317,13 +335,81 @@ IVArray::search(unsigned _key, char *&_str, unsigned &_len) // if (array[_key].inCache()) // return true; AddInCache(_key, _str, _len); - char *debug = new char [_len]; - memcpy(debug, _str, _len); - _str = debug; + char *debug = new char[_len]; + memcpy(debug, _str, _len); + //_len -= 24; + _str = debug; // array[_key].Lock.unlock(); // } } + //char *debug = new char[_len-24]; + //memcpy(debug, tmpStr, _len-24); + //_str = debug; + _len -= 24; + //delete[] tmpStr; + + this->CacheLock.unlock(); + return true; +} + +bool +IVArray::search(unsigned _key, char *&_str, unsigned &_len, unsigned &_index,int &_in_cache) +{ + this->CacheLock.lock(); + if (_key >= CurEntryNum ||!array[_key].isUsed()) + { + _str = NULL; + _len = 0; + this->CacheLock.unlock(); + return false; + } + + //char *tmpStr=NULL; + + if (array[_key].inCache()) + { + + _in_cache=1; + + UpdateTime(_key); + bool ret = array[_key].getBstr(_str, _len); + _index = *((unsigned*)_str + _len / 4 - 2); + this->CacheLock.unlock(); + + //_str=new char[_len-24]; + //memcpy(_str,tmpStr,_len-24); + _len -= 24; + //delete[] tmpStr; + return ret; + } + + _in_cache=0; + + unsigned store = array[_key].getStore(); + + //cout << "new search,store:" << store<<",_key:" <<_key<< endl; + if (!BM->ReadValue(store, _str, _len)) + { + this->CacheLock.unlock(); + return false; + } + //cout << "new search:" << _key << "---" << _len << endl; + if(!VList::isLongList(_len)) + { + AddInCache(_key, _str, _len); + char *debug = new char[_len]; + memcpy(debug, _str, _len); + _str = debug; + //_len -= 24; + //_index = store; + } + //char *debug = new char[_len-24]; + //memcpy(debug, tmpStr, _len-24); + //_str = debug; + _len -= 24; + _index = store; + //delete[] tmpStr; this->CacheLock.unlock(); return true; @@ -339,7 +425,6 @@ IVArray::insert(unsigned _key, char *_str, unsigned _len) if (_key >= IVArray::MAX_KEY_NUM) { - cout << _key << ' ' << MAX_KEY_NUM << endl; cout << "IVArray insert error: Key is bigger than MAX_KEY_NUM" << endl; return false; } @@ -375,6 +460,7 @@ IVArray::insert(unsigned _key, char *_str, unsigned _len) unsigned store = BM->WriteValue(_str, _len); if (store == 0) { +//cout<<"writevalue return wrong store"<FreeBlocks(store); + //BM->FreeBlocks(store); + *((unsigned*)_str + _len / 4 - 2) = store; AddInCache(_key, _str, _len); } @@ -490,7 +581,7 @@ IVArray::PinCache(unsigned _key) { return; } - + array[_key].setBstr(_str, _len); array[_key].setCacheFlag(true); array[_key].setCachePinFlag(true); diff --git a/KVstore/IVArray/IVArray.h b/KVstore/IVArray/IVArray.h index 2cfe636b..eb1500a2 100644 --- a/KVstore/IVArray/IVArray.h +++ b/KVstore/IVArray/IVArray.h @@ -57,6 +57,7 @@ class IVArray ~IVArray(); bool search(unsigned _key, char *& _str, unsigned& _len); + bool search(unsigned _key, char *&_str, unsigned &_len, unsigned &_index,int &_in_cache); bool modify(unsigned _key, char *_str, unsigned _len); bool remove(unsigned _key); bool insert(unsigned _key, char *_str, unsigned _len); diff --git a/KVstore/IVArray/IVBlockManager.cpp b/KVstore/IVArray/IVBlockManager.cpp index 212e4ece..ac449917 100644 --- a/KVstore/IVArray/IVBlockManager.cpp +++ b/KVstore/IVArray/IVBlockManager.cpp @@ -132,45 +132,67 @@ IVBlockManager::ReadValue(unsigned _blk_index, char *&_str, unsigned &_len) } unsigned next_blk = _blk_index; // index of next block - unsigned Len_left; // how many bits left to read + unsigned pre_left; // how many bits left to read + unsigned obj_left; int fd = fileno(ValueFile); - //fseek(ValueFile, (off_t)BLOCK_SIZE * (_blk_index - 1) + sizeof(unsigned), SEEK_SET); - off_t offset = (off_t)BLOCK_SIZE * (_blk_index - 1) + sizeof(unsigned); - pread(fd, &Len_left, 1 * sizeof(unsigned), offset); -// fread(&Len_left, sizeof(unsigned), 1, ValueFile); - - _str = new char [Len_left]; - _len = Len_left; - //cout << "Read Value: _len = " << _len << endl; - //char *pstr = _str; // pointer to where data in next reading is stored - - if (_str == NULL) + // read first x list + off_t offset = (off_t)(BLOCK_SIZE) * (_blk_index - 1); + offset += 1 * sizeof(unsigned); + pread(fd, &pre_left, 1 * sizeof(unsigned), offset); + pread(fd, &obj_left, 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + obj_left = obj_left * 4 + 24; + _str = new char[pre_left + obj_left]; + unsigned len1 = pre_left; + + while(pre_left > 0 && next_blk > 0) { - cout << "IVBlockManager Error: fail when new" << endl; - return false; + unsigned Bits2read = BLOCK_DATA_SIZE < pre_left ? BLOCK_DATA_SIZE : pre_left; + offset = (off_t)(BLOCK_SIZE) * (next_blk - 1); + pread(fd, &next_blk, 1 * sizeof(unsigned), offset); + offset += sizeof(unsigned) + sizeof(unsigned); + pread(fd, _str + (len1 - pre_left), Bits2read * sizeof(char), offset); + pre_left -= Bits2read; } - do - { - unsigned Bits2read = BLOCK_DATA_SIZE < Len_left ? BLOCK_DATA_SIZE : Len_left; + // read second x list + //offset = (off_t)(BLOCK_SIZE) * (next_blk - 1); + //offset += 1 * sizeof(unsigned); + //pread(fd, &obj_left, 1 * sizeof(unsigned), offset); + //char *_str2 = new char[obj_left]; + unsigned len2 = obj_left; - offset = (off_t)BLOCK_SIZE * (next_blk - 1); + while(obj_left > 0 && next_blk > 0) + { + unsigned Bits2read = BLOCK_DATA_SIZE < obj_left ? BLOCK_DATA_SIZE : obj_left; + offset = (off_t)(BLOCK_SIZE) * (next_blk - 1); pread(fd, &next_blk, 1 * sizeof(unsigned), offset); -// fseek(ValueFile, (off_t)BLOCK_SIZE * (next_blk - 1), SEEK_SET); -// fread(&next_blk, sizeof(unsigned), 1, ValueFile); offset += sizeof(unsigned) + sizeof(unsigned); - pread(fd, _str + (_len - Len_left), Bits2read * sizeof(char), offset); -/// pread(fd, pstr, Bits2read * sizeof(char), offset); -// fseek(ValueFile, sizeof(unsigned), SEEK_CUR); -// fread(_str + (_len - Len_left), sizeof(char), Bits2read, ValueFile); + pread(fd, _str + (len2 - obj_left) + len1, Bits2read * sizeof(char), offset); + obj_left -= Bits2read; + } -// pstr += Bits2read; - Len_left -= Bits2read; + //char buff[1024]; + //char filepath[1024]; + //memset(buff, 0, 1024); + //memset(filepath, 0, 1024); + //sprintf(buff, "/proc/%d/fd/%d", getpid(), fd); + //readlink(buff, filepath, 1024); - }while(next_blk > 0 && Len_left > 0); - //cout << "Read Value Check: _len = " << _len << endl; + //_str = new char[len1 + len2]; + _len = len1 + len2; + //memcpy(_str, _str1, len1); + //memcpy(_str + len1, _str2, len2); + + if (_str == NULL) + { + cout << "IVBlockManager Error: fail when new" << endl; + return false; + } + + //delete[] _str1; + //delete[] _str2; return true; } @@ -254,50 +276,211 @@ IVBlockManager::getWhereToWrite(unsigned _len) unsigned IVBlockManager::WriteValue(const char *_str, const unsigned _len) { - if (!getWhereToWrite(_len)) + unsigned _pre_Size = *((unsigned*)_str + _len / 4 - 6); + unsigned _pre_Pos = *((unsigned*)_str + _len / 4 - 5); + unsigned _obj_Size = *((unsigned*)_str + _len / 4 - 4); + unsigned _obj_Pos = *((unsigned*)_str + _len / 4 - 3); + unsigned _add_type = *((unsigned*)_str + _len / 4 - 1); + + int fd = fileno(ValueFile); + + unsigned blockToSkip = _pre_Pos / BLOCK_DATA_SIZE; + unsigned pre_left = _len - _obj_Size * 4 - 24; + + char *_str1 = new char[pre_left]; + char *_bak1 = _str1; + memcpy(_str1, _str, pre_left); + unsigned obj_left = _obj_Size * 4 + 24; + char *_str2 = new char[obj_left]; + char *_bak2 = _str2; + memcpy(_str2, _str + pre_left, obj_left); + + char buff[1024]; + char filepath[1024]; + memset(buff, 0, 1024); + memset(filepath, 0, 1024); + sprintf(buff, "/proc/%d/fd/%d", getpid(), fd); + readlink(buff, filepath, 1024); + +//cout << "writevalue:" << _add_type << endl; +//cout << "writevalue" << filepath << "-" << _add_type << "-" << pre_left << "-" << obj_left << endl; + if (_add_type == 0) { - return false; + unsigned add_block = pre_left / BLOCK_DATA_SIZE + obj_left / BLOCK_DATA_SIZE + (pre_left % BLOCK_DATA_SIZE != 0) + (obj_left % BLOCK_DATA_SIZE != 0); + if (!getWhereToWrite(add_block * BLOCK_DATA_SIZE)) + { + return false; + } + int fd = fileno(ValueFile); + + BlockInfo *p = BlockToWrite; + *((unsigned*)_str + _len / 4 - 2) = p->num; + *((unsigned*)_str2 + obj_left / 4 - 2) = p->num; + //cout << "write - add" << p->num << endl; + while (p != NULL && pre_left > 0) + { + BlockInfo *nextp = p->next; + unsigned Bits2Write = BLOCK_DATA_SIZE < pre_left ? BLOCK_DATA_SIZE:pre_left; + off_t offset = (off_t)(BLOCK_SIZE) * (p->num - 1); + unsigned NextIndex = 0; + if (nextp != NULL) + { + NextIndex = nextp->num; + } + pwrite(fd, &NextIndex, 1 * sizeof(unsigned), offset); + offset += sizeof(unsigned); + pwrite(fd, &pre_left, 1 * sizeof(unsigned), offset); + offset += sizeof(unsigned); + pwrite(fd, _str1, Bits2Write * sizeof(char), offset); + pre_left -= Bits2Write; + _str1 += Bits2Write; + p = nextp; + } + while (p != NULL && obj_left > 0) + { + BlockInfo *nextp = p->next; + unsigned Bits2Write = BLOCK_DATA_SIZE < obj_left ? BLOCK_DATA_SIZE:obj_left; + off_t offset = (off_t)(BLOCK_SIZE) * (p->num - 1); + unsigned NextIndex = 0; + if (nextp != NULL) + { + NextIndex = nextp->num; + } + pwrite(fd, &NextIndex, 1 * sizeof(unsigned), offset); + offset += sizeof(unsigned); + pwrite(fd, &obj_left, 1 * sizeof(unsigned), offset); + offset += sizeof(unsigned); + pwrite(fd, _str2, Bits2Write * sizeof(char), offset); + obj_left -= Bits2Write; + _str2 += Bits2Write; + p = nextp; + } + + if (pre_left > 0 || obj_left > 0) + { + cout << "Get Where To Write error: space is not enough" << endl; + return 0; + } + + delete[] _bak1; + delete[] _bak2; + + return BlockToWrite->num; } + + _add_type = _add_type & 0x7fffffff; + int free_num = int((pre_left + _add_type * 4) / BLOCK_DATA_SIZE) - int(pre_left / BLOCK_DATA_SIZE); + + unsigned _blk_Index = *((unsigned*)_str + _len / 4 - 2); + unsigned next_blk = _blk_Index; + off_t offset = (off_t)(BLOCK_SIZE) * (next_blk - 1); + if (_pre_Pos >= 0) + { + offset = (off_t)BLOCK_SIZE * (next_blk - 1); + pwrite(fd, _str1, 12 * sizeof(char), offset + 2 * sizeof(unsigned)); + + while (blockToSkip > 0 && next_blk > 0) + { + offset = (off_t)BLOCK_SIZE * (next_blk - 1); + pread(fd, &next_blk, 1 * sizeof(unsigned), offset); + pwrite(fd, &pre_left, 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + pre_left -= BLOCK_DATA_SIZE; + _str1 += BLOCK_DATA_SIZE; + blockToSkip--; + } - // write _str - int fd = fileno(ValueFile); - BlockInfo *p = BlockToWrite; - char *pstr = (char *)_str; // pointer to buffer of where to write next - unsigned len_left = _len; // how many bytes left to write + while (pre_left > 0 && next_blk > 0) + { + unsigned Bits2Write = BLOCK_DATA_SIZE < pre_left ? BLOCK_DATA_SIZE:pre_left; + offset = (off_t)BLOCK_SIZE * (next_blk - 1); + pread(fd, &next_blk, 1 * sizeof(unsigned), offset); + pwrite(fd, &pre_left, 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + pwrite(fd, _str1, Bits2Write * sizeof(char), offset + 2 * sizeof(unsigned)); + pre_left -= Bits2Write; + _str1 += Bits2Write; + } + + if (pre_left > 0) + { + if (!getWhereToWrite(pre_left)) + { + return false; + } + BlockInfo *p = BlockToWrite; + pwrite(fd, &(p->num), 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + unsigned NextIndex = next_blk; + offset = (off_t)(BLOCK_SIZE) * (p->num - 1); + pwrite(fd, &NextIndex, 1 * sizeof(unsigned), offset); + pwrite(fd, &pre_left, 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + pwrite(fd, _str1, pre_left, offset + 2 * sizeof(unsigned)); + } + } - while (p != NULL) + // cout << "free:" << free_num << endl; + if (free_num > 0) { - BlockInfo *nextp = p->next; - unsigned Bits2Write = BLOCK_DATA_SIZE < len_left ? BLOCK_DATA_SIZE:len_left; - off_t offset = (off_t)(BLOCK_SIZE) * (p->num - 1); - unsigned NextIndex = 0; - if (nextp != NULL) + unsigned tmp_blk, tmp_off; + for (int i = 0; i < free_num; ++i) { - NextIndex = nextp->num; + tmp_blk = next_blk; + tmp_off = (off_t)BLOCK_SIZE * (tmp_blk - 1); + pread(fd, &tmp_blk, 1 * sizeof(unsigned), tmp_off); } - // write down next block's index - pwrite(fd, &NextIndex, 1 * sizeof(unsigned), offset); - offset += sizeof(unsigned); - // write down how many bits left - pwrite(fd, &len_left, 1 * sizeof(unsigned), offset); - offset += sizeof(unsigned); - // write down value - pwrite(fd, pstr, Bits2Write * sizeof(char), offset); - - len_left -= Bits2Write; - pstr += Bits2Write; - p = nextp; - + pwrite(fd, 0, 1 * sizeof(unsigned), tmp_off); + FreeBlocks(next_blk); + pwrite(fd, &tmp_blk, 1 * sizeof(unsigned), offset); + } + + + offset = (off_t)(BLOCK_SIZE) * (next_blk - 1); + blockToSkip = _obj_Pos / BLOCK_DATA_SIZE; + + while (blockToSkip > 0 && next_blk > 0) + { + offset = (off_t)BLOCK_SIZE * (next_blk - 1); + pread(fd, &next_blk, 1 * sizeof(unsigned), offset); + pwrite(fd, &obj_left, 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + obj_left -= BLOCK_DATA_SIZE; + _str2 += BLOCK_DATA_SIZE; + blockToSkip--; + } + + while (obj_left > 0 && next_blk > 0) + { + unsigned Bits2Write = BLOCK_DATA_SIZE < obj_left ? BLOCK_DATA_SIZE:obj_left; + offset = (off_t)BLOCK_SIZE * (next_blk - 1); + pread(fd, &next_blk, 1 * sizeof(unsigned), offset); + pwrite(fd, &obj_left, 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + pwrite(fd, _str2, Bits2Write * sizeof(char), offset + 2 * sizeof(unsigned)); + obj_left -= Bits2Write; + _str2 += Bits2Write; + } + + if (obj_left > 0) + { + if (!getWhereToWrite(obj_left)) + { + return false; + } + BlockInfo *p = BlockToWrite; + pwrite(fd, &(p->num), 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + unsigned NextIndex = 0; + offset = (off_t)(BLOCK_SIZE) * (p->num - 1); + pwrite(fd, &NextIndex, 1 * sizeof(unsigned), offset); + pwrite(fd, &obj_left, 1 * sizeof(unsigned), offset + 1 * sizeof(unsigned)); + pwrite(fd, _str2, obj_left, offset + 2 * sizeof(unsigned)); } - if (len_left > 0) + if (next_blk > 0) { - cout << "Get Where To Write error: space is not enough" << endl; - return 0; + pwrite(fd, 0, 1 * sizeof(unsigned), offset); + FreeBlocks(next_blk); } - return BlockToWrite->num; + delete[] _bak1; + delete[] _bak2; + return _blk_Index; } // the key is deleted, so free series of blocks, maybe useless if using SSD diff --git a/KVstore/IVArray/IVBlockManager.h b/KVstore/IVArray/IVBlockManager.h index d92832e1..0c4a9f57 100644 --- a/KVstore/IVArray/IVBlockManager.h +++ b/KVstore/IVArray/IVBlockManager.h @@ -60,6 +60,7 @@ class IVBlockManager bool ReadValue(unsigned _blk_index, char *&_str, unsigned &_len); + void SaveFreeBlockList(); bool FreeBlocks(const unsigned index); diff --git a/KVstore/KVstore.cpp b/KVstore/KVstore.cpp index cf6faa7c..32ff6e80 100644 --- a/KVstore/KVstore.cpp +++ b/KVstore/KVstore.cpp @@ -48,13 +48,12 @@ KVstore::flush() { this->flush(this->entity2id); this->flush(this->id2entity); - + this->flush(this->literal2id); this->flush(this->id2literal); - + this->flush(this->predicate2id); this->flush(this->id2predicate); - this->flush(this->subID2values); this->flush(this->preID2values); this->flush(this->objID2values); @@ -533,24 +532,33 @@ KVstore::updateTupleslist_remove(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ bool KVstore::updateInsert_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id) { + //cout << "updateInsert_s(spo)" <<_sub_id << "-" << _pre_id << "-" << _obj_id << endl; unsigned* _tmp = NULL; unsigned _len = 0; - bool _get = this->getValueByKey(this->subID2values, _sub_id, (char*&)_tmp, _len); + unsigned _blk_index = 0; + int _in_cache=0; + bool _get = this->getValueAndIndexByKey(this->subID2values, _sub_id, (char*&)_tmp, _len, _blk_index,_in_cache); bool _is_entity = Util::is_entity_ele(_obj_id); - + //subID doesn't exist if (!_get) { //unsigned _values[6]; - unsigned* _values = new unsigned[6]; + unsigned* _values = new unsigned[12]; _values[0] = 1; _values[1] = 1; _values[2] = _is_entity ? 1 : 0; _values[3] = _pre_id; _values[4] = 5; _values[5] = _obj_id; + _values[6] = 1; + _values[7] = 3; + _values[8] = 1; + _values[9] = 0; + _values[10] = 0; + _values[11] = 0; //NOTICE: not use array in stack here, otherwise it will be freed, and data in B+Tree, too - this->addValueByKey(this->subID2values, _sub_id, (char*)_values, sizeof(int) * 6); + this->addValueByKey(this->subID2values, _sub_id, (char*)_values, sizeof(int) * 12); } //subID exists @@ -565,7 +573,7 @@ KVstore::updateInsert_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID //if (_position == -1) { _values_len = _len / sizeof(unsigned) + 3; - _values = new unsigned[_values_len]; + _values = new unsigned[_values_len + 6]; _values[0] = _tmp[0] + 1; _values[1] = _tmp[1] + 1; _values[2] = _tmp[2] + (_is_entity ? 1 : 0); @@ -582,6 +590,24 @@ KVstore::updateInsert_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID else { _offset_old = _tmp[4 + 2 * i]; } + + _values[_values_len] = _tmp[1] + 1; + //_values[_values_len + 1] = j; + _values[_values_len + 2] = _tmp[0] + 1; + //_values[_values_len + 3] = _offset_old - (3 + _tmp[1] * 2); + _values[_values_len + 4] = _blk_index; + _values[_values_len + 1] = (_in_cache == 0 ? j : min(j, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? _offset_old - (3 + _tmp[1] * 2) : min(_offset_old - (3 + _tmp[1] * 2), *((unsigned *)_tmp + _len / 4 + 3))); + + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + } + else + { + _values[_values_len + 5] = (1 << 31); + } + //cout << "updateInsert_s" << "-" << _sub_id << "-" << _blk_index << endl; _values[j + 1] = _offset_old + 2; j += 2; for (; i < _tmp[1]; i++, j += 2) { @@ -604,7 +630,7 @@ KVstore::updateInsert_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID else { _values_len = _len / sizeof(unsigned) + 1; - _values = new unsigned[_values_len]; + _values = new unsigned[_values_len + 6]; memcpy(_values, _tmp, sizeof(unsigned) * _tmp[4 + 2 * _position]); _values[0]++; if (_is_entity) { @@ -624,13 +650,32 @@ KVstore::updateInsert_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID for (i = _tmp[4 + 2 * _position], j = _tmp[4 + 2 * _position]; i < right && _tmp[i] < _obj_id; i++, j++) { _values[j] = _tmp[i]; } + + _values[_values_len] = _tmp[1]; + //_values[_values_len + 1] = 4 + 2 * _position; + _values[_values_len + 2] = _tmp[0] + 1; + //_values[_values_len + 3] = j; + _values[_values_len + 4] = _blk_index; + + _values[_values_len + 1] = (_in_cache == 0 ? 4 + 2 * _position : min(4 + 2 * _position, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? j : min(j, *((unsigned *)_tmp + _len / 4 + 3))); + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + + } + else + { + _values[_values_len + 5] = (1 << 31); + } + _values[j] = _obj_id; j++; for (; i < 3 + 2 * _tmp[1] + _tmp[0]; i++, j++) { _values[j] = _tmp[i]; } } - this->setValueByKey(this->subID2values, _sub_id, (char*)_values, sizeof(unsigned) * _values_len); + this->setValueByKey(this->subID2values, _sub_id, (char*)_values, sizeof(unsigned) * (_values_len + 6)); //delete[] _values; } delete []_tmp; @@ -642,7 +687,9 @@ KVstore::updateRemove_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID { unsigned* _tmp = NULL; unsigned _len = 0; - bool _get = this->getValueByKey(this->subID2values, _sub_id, (char*&)_tmp, _len); + unsigned _blk_index = 0; + int _in_cache=0; + bool _get = this->getValueAndIndexByKey(this->subID2values, _sub_id, (char*&)_tmp, _len, _blk_index,_in_cache); bool _is_entity = Util::is_entity_ele(_obj_id); if (!_get) @@ -675,7 +722,7 @@ KVstore::updateRemove_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID //preID doesn't exist after removal if (_oidlen_sp == 1) { _values_len = _len / sizeof(unsigned) - 3; - _values = new unsigned[_values_len]; + _values = new unsigned[_values_len + 6]; memcpy(_values, _tmp, sizeof(unsigned) * (3 + 2 * _position)); _values[0]--; _values[1]--; @@ -690,6 +737,25 @@ KVstore::updateRemove_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _values[j] = _tmp[i]; _values[j + 1] = _tmp[i + 1] - 3; } + + _values[_values_len] = _tmp[1] - 1; + // _values[_values_len + 1] = _position * 2 + 3; + _values[_values_len + 2] = _tmp[0] - 1; + // _values[_values_len + 3] = _tmp[_position * 2 + 4] - _tmp[1] * 2 - 3; + _values[_values_len + 4] = _blk_index; + + _values[_values_len+1] = (_in_cache == 0 ? _position * 2 + 3 : min(_position * 2 + 3, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? _tmp[_position * 2 + 4] - _tmp[1] * 2 - 3 : min(_tmp[_position * 2 + 4] - _tmp[1] * 2 - 3, *((unsigned *)_tmp + _len / 4 + 3))); + + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + } + else + { + _values[_values_len + 5] = ((*((unsigned *)_tmp + _len / 4 + 5) & 0x7fffffff) + 2) | 0x80000000; + } + for (; i < 3 + 2 * _tmp[1] + _tmp[0]; i++, j++) { if (i == _tmp[4 + 2 * _position]) { j--; @@ -702,7 +768,7 @@ KVstore::updateRemove_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID //preID still exists after removal else { _values_len = _len / sizeof(unsigned) - 1; - _values = new unsigned[_values_len]; + _values = new unsigned[_values_len + 6]; memcpy(_values, _tmp, sizeof(unsigned) * _tmp[4 + 2 * _position]); _values[0]--; if (_is_entity) { @@ -716,13 +782,32 @@ KVstore::updateRemove_s2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID i < 3 + 2 * _tmp[1] + _tmp[0] && _tmp[i] < _obj_id; i++, j++) { _values[j] = _tmp[i]; } + + _values[_values_len] = _tmp[1]; + //_values[_values_len + 1] = _position * 2 + 6; + _values[_values_len + 2] = _tmp[0] - 1; + // _values[_values_len + 3] = _tmp[_position * 2 + 4] - _tmp[1] * 2 - 3; + _values[_values_len + 4] = _blk_index; + + _values[_values_len + 1] = (_in_cache == 0 ? _position * 2 + 6 : min(_position * 2 + 6, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? _tmp[_position * 2 + 4] - _tmp[1] * 2 - 3 : min(_tmp[_position * 2 + 4] - _tmp[1] * 2 - 3, *((unsigned *)_tmp + _len / 4 + 3))); + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + + } + else + { + _values[_values_len + 5] = (*((unsigned *)_tmp + _len / 4 + 5) & 0x7fffffff) | 0x80000000; + } + i++; for (; i < 3 + 2 * _tmp[1] + _tmp[0]; i++, j++) { _values[j] = _tmp[i]; } } - this->setValueByKey(this->subID2values, _sub_id, (char*)_values, sizeof(unsigned) * _values_len); + this->setValueByKey(this->subID2values, _sub_id, (char*)_values, sizeof(unsigned) * (_values_len + 6)); //delete[] _values; } delete []_tmp; @@ -762,20 +847,29 @@ KVstore::updateRemove_s2values(TYPE_ENTITY_LITERAL_ID _subid, const std::vector< bool KVstore::updateInsert_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id) { + //cout << "updateInsert_o(spo)" <<_sub_id << "-" << _pre_id << "-" << _obj_id << endl; unsigned* _tmp = NULL; unsigned _len = 0; - bool _get = this->getValueByKey(this->objID2values, _obj_id, (char*&)_tmp, _len); + unsigned _blk_index = 0; + int _in_cache=0; + bool _get = this->getValueAndIndexByKey(this->objID2values, _obj_id, (char*&)_tmp, _len, _blk_index,_in_cache); //objID doesn't exist if (!_get) { //unsigned _values[5]; - unsigned* _values = new unsigned[5]; + unsigned* _values = new unsigned[11]; _values[0] = 1; _values[1] = 1; _values[2] = _pre_id; _values[3] = 4; _values[4] = _sub_id; - this->addValueByKey(this->objID2values, _obj_id, (char*)_values, sizeof(unsigned) * 5); + _values[5] = 1; + _values[6] = 2; + _values[7] = 1; + _values[8] = 0; + _values[9] = 0; + _values[10] = 0; + this->addValueByKey(this->objID2values, _obj_id, (char*)_values, sizeof(unsigned) * 11); } //objID exists @@ -788,7 +882,7 @@ KVstore::updateInsert_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID if (_position == INVALID) { _values_len = _len / sizeof(unsigned) + 3; - _values = new unsigned[_values_len]; + _values = new unsigned[_values_len + 6]; _values[0] = _tmp[0] + 1; _values[1] = _tmp[1] + 1; unsigned i, j; @@ -804,6 +898,27 @@ KVstore::updateInsert_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID else { _offset_old = _tmp[3 + 2 * i]; } + + _values[_values_len] = _tmp[1] + 1; + // _values[_values_len + 1] = j; + _values[_values_len + 2] = _tmp[0] + 1; + // _values[_values_len + 3] = _offset_old - (2 + _tmp[1] * 2); + _values[_values_len + 4] = _blk_index; + + _values[_values_len + 1] = (_in_cache == 0 ? j : min(j, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? _offset_old - (2 + _tmp[1] * 2) : min(_offset_old - (2 + _tmp[1] * 2), *((unsigned *)_tmp + _len / 4 + 3))); + + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + + } + else + { + _values[_values_len + 5] = (1 << 31); + } + + _values[j + 1] = _offset_old + 2; j += 2; for (; i < _tmp[1]; i++, j += 2) { @@ -826,7 +941,7 @@ KVstore::updateInsert_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID //preID exists else { _values_len = _len / sizeof(unsigned) + 1; - _values = new unsigned[_values_len]; + _values = new unsigned[_values_len + 6]; memcpy(_values, _tmp, sizeof(unsigned) * _tmp[3 + 2 * _position]); _values[0]++; for (unsigned i = _position + 1; i < _tmp[1]; i++) { @@ -843,6 +958,26 @@ KVstore::updateInsert_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID for (i = _tmp[3 + 2 * _position], j = _tmp[3 + 2 * _position]; i < right && _tmp[i] < _sub_id; i++, j++) { _values[j] = _tmp[i]; } + + _values[_values_len] = _tmp[1]; + //_values[_values_len + 1] = 3 + 2 * _position; + _values[_values_len + 2] = _tmp[0] + 1; + //_values[_values_len + 3] = j; + _values[_values_len + 4] = _blk_index; + + _values[_values_len + 1] = (_in_cache == 0 ? 3 + 2 * _position : min(3 + 2 * _position, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? j : min(j, *((unsigned *)_tmp + _len / 4 + 3))); + + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + + } + else + { + _values[_values_len + 5] = (1 << 31); + } + _values[j] = _sub_id; j++; for (; i < 2 + 2 * _tmp[1] + _tmp[0]; i++, j++) { @@ -850,7 +985,7 @@ KVstore::updateInsert_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID } } - this->setValueByKey(this->objID2values, _obj_id, (char*)_values, sizeof(unsigned) * _values_len); + this->setValueByKey(this->objID2values, _obj_id, (char*)_values, sizeof(unsigned) * (_values_len + 6)); //delete[] _values; } delete []_tmp; @@ -862,7 +997,9 @@ KVstore::updateRemove_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID { unsigned* _tmp = NULL; unsigned _len = 0; - bool _get = this->getValueByKey(this->objID2values, _obj_id, (char*&)_tmp, _len); + unsigned _blk_index = 0; + int _in_cache=0; + bool _get = this->getValueAndIndexByKey(this->objID2values, _obj_id, (char*&)_tmp, _len, _blk_index,_in_cache); if (!_get) { return false; @@ -889,7 +1026,7 @@ KVstore::updateRemove_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID //preID doesn't exist after removal if (_sidlen_op == 1) { _values_len = _len / sizeof(unsigned) - 3; - _values = new unsigned[_values_len]; + _values = new unsigned[_values_len + 6]; memcpy(_values, _tmp, sizeof(unsigned) * (2 + 2 * _position)); _values[0]--; _values[1]--; @@ -901,6 +1038,26 @@ KVstore::updateRemove_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _values[j] = _tmp[i]; _values[j + 1] = _tmp[i + 1] - 3; } + + _values[_values_len] = _tmp[1] - 1; + // _values[_values_len + 1] = _position * 2 + 2; + _values[_values_len + 2] = _tmp[0] - 1; + // _values[_values_len + 3] = _tmp[_position * 2 + 3] - _tmp[1] * 2 - 2; + _values[_values_len + 4] = _blk_index; + + _values[_values_len + 1] = (_in_cache == 0 ? _position * 2 + 2 : min(_position * 2 + 2, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? _tmp[_position * 2 + 3] - _tmp[1] * 2 - 2 : min(_tmp[_position * 2 + 3] - _tmp[1] * 2 - 2, *((unsigned *)_tmp + _len / 4 + 3))); + + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + + } + else + { + _values[_values_len + 5] = ((*((unsigned *)_tmp + _len / 4 + 5) & 0x7fffffff) + 2) | 0x80000000; + } + for (; i < 2 + 2 * _tmp[1] + _tmp[0]; i++, j++) { if (i == _tmp[3 + 2 * _position]) { j--; @@ -913,7 +1070,7 @@ KVstore::updateRemove_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID //preID still exists after removal else { _values_len = _len / sizeof(unsigned) - 1; - _values = new unsigned[_values_len]; + _values = new unsigned[_values_len + 6]; memcpy(_values, _tmp, sizeof(unsigned) * _tmp[3 + 2 * _position]); _values[0]--; for (unsigned i = _position + 1; i < _tmp[1]; i++) { @@ -924,13 +1081,33 @@ KVstore::updateRemove_o2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID i < 2 + 2 * _tmp[1] + _tmp[0] && _tmp[i] < _sub_id; i++, j++) { _values[j] = _tmp[i]; } + + _values[_values_len] = _tmp[1]; + // _values[_values_len + 1] = _position * 2 + 5; + _values[_values_len + 2] = _tmp[0] - 1; + // _values[_values_len + 3] = _tmp[_position * 2 + 3] - _tmp[1] * 2 - 2; + _values[_values_len + 4] = _blk_index; + + _values[_values_len + 1] = (_in_cache == 0 ? _position * 2 + 5 : min(_position * 2 + 5, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? _tmp[_position * 2 + 3] - _tmp[1] * 2 - 2 : min(_tmp[_position * 2 + 3] - _tmp[1] * 2 - 2, *((unsigned *)_tmp + _len / 4 + 3))); + + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + + } + else + { + _values[_values_len + 5] = (*((unsigned *)_tmp + _len / 4 + 5) & 0x7fffffff) | 0x80000000; + } + i++; for (; i < 2 + 2 * _tmp[1] + _tmp[0]; i++, j++) { _values[j] = _tmp[i]; } } - this->setValueByKey(this->objID2values, _obj_id, (char*)_values, sizeof(unsigned) * _values_len); + this->setValueByKey(this->objID2values, _obj_id, (char*)_values, sizeof(unsigned) * (_values_len + 6)); //delete[] _values; } delete []_tmp; @@ -969,24 +1146,33 @@ KVstore::updateRemove_o2values(TYPE_ENTITY_LITERAL_ID _objid, const std::vector< bool KVstore::updateInsert_p2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _pre_id, TYPE_ENTITY_LITERAL_ID _obj_id) { + // cout << "updateInsert_p(spo)" <<_sub_id << "-" << _pre_id << "-" << _obj_id << endl; unsigned* _tmp = NULL; unsigned _len = 0; - bool _get = this->getValueByKey(this->preID2values, _pre_id, (char*&)_tmp, _len); + unsigned _blk_index = 0; + int _in_cache=0; + bool _get = this->getValueAndIndexByKey(this->preID2values, _pre_id, (char*&)_tmp, _len, _blk_index,_in_cache); //preid doesn't exist if (!_get) { //unsigned _values[3]; - unsigned* _values = new unsigned[3]; + unsigned* _values = new unsigned[9]; _values[0] = 1; _values[1] = _sub_id; _values[2] = _obj_id; - this->addValueByKey(this->preID2values, _pre_id, (char*)_values, sizeof(unsigned) * 3); + _values[3] = 1; + _values[4] = 2; + _values[5] = 1; + _values[6] = 0; + _values[7] = 0; + _values[8] = 0; + this->addValueByKey(this->preID2values, _pre_id, (char*)_values, sizeof(unsigned) * 9); } //preid exists else { unsigned _values_len = _len / sizeof(unsigned) + 2; - unsigned* _values = new unsigned[_values_len]; + unsigned* _values = new unsigned[_values_len + 6]; unsigned i, j; _values[0] = _tmp[0] + 1; for (i = 1, j = 1; @@ -995,6 +1181,27 @@ KVstore::updateInsert_p2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _values[j] = _tmp[i]; _values[j + _tmp[0] + 1] = _tmp[i + _tmp[0]]; } + + _values[_values_len] = _tmp[0] + 1; + //_values[_values_len + 1] = j; + _values[_values_len + 2] = _tmp[0] + 1; + //_values[_values_len + 3] = j - 1; + _values[_values_len + 4] = _blk_index; + + + _values[_values_len + 1] = (_in_cache == 0 ? j : min(j, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = ((_in_cache == 0) ? j - 1 : min(j - 1, *((unsigned *)_tmp + _len / 4 + 1 + 3))); + + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + } + else + { + _values[_values_len + 5] = (1 << 31); + } + + _values[j] = _sub_id; _values[j + _tmp[0] + 1] = _obj_id; j++; @@ -1002,7 +1209,7 @@ KVstore::updateInsert_p2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _values[j] = _tmp[i]; _values[j + _tmp[0] + 1] = _tmp[i + _tmp[0]]; } - this->setValueByKey(this->preID2values, _pre_id, (char*)_values, sizeof(unsigned) * _values_len); + this->setValueByKey(this->preID2values, _pre_id, (char*)_values, sizeof(unsigned) * (_values_len + 6)); //delete[] _values; } delete []_tmp; @@ -1014,7 +1221,9 @@ KVstore::updateRemove_p2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID { unsigned* _tmp = NULL; unsigned _len = 0; - bool _get = this->getValueByKey(this->preID2values, _pre_id, (char*&)_tmp, _len); + unsigned _blk_index = 0; + int _in_cache=0; + bool _get = this->getValueAndIndexByKey(this->preID2values, _pre_id, (char*&)_tmp, _len, _blk_index,_in_cache); if (!_get) { return false; @@ -1028,7 +1237,7 @@ KVstore::updateRemove_p2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID //preid still exists after removal else { unsigned _values_len = _len / sizeof(unsigned) - 2; - unsigned* _values = new unsigned[_values_len]; + unsigned* _values = new unsigned[_values_len + 6]; unsigned i, j; _values[0] = _tmp[0] - 1; for (i = 1, j = 1; @@ -1037,12 +1246,31 @@ KVstore::updateRemove_p2values(TYPE_ENTITY_LITERAL_ID _sub_id, TYPE_PREDICATE_ID _values[j] = _tmp[i]; _values[j + _tmp[0] - 1] = _tmp[i + _tmp[0]]; } + + _values[_values_len] = _tmp[0] - 1; + //_values[_values_len + 1] = i; + _values[_values_len + 2] = _tmp[0] - 1; + //_values[_values_len + 3] = i - 1; + _values[_values_len + 4] = _blk_index; + + _values[_values_len + 1] = (_in_cache == 0 ? i : min(i, *((unsigned *)_tmp + _len / 4 + 1))); + _values[_values_len + 3] = (_in_cache == 0 ? (i - 1) : min(i - 1, *((unsigned *)_tmp + _len / 4 + 3))); + + if (_blk_index == 0) + { + _values[_values_len + 5] = 0; + } + else + { + _values[_values_len + 5] = ((*((unsigned *)_tmp + _len / 4 + 5) & 0x7fffffff) + 1) | 0x80000000; + } + i++; for (; i < 1 + _tmp[0]; i++, j++) { _values[j] = _tmp[i]; _values[j + _tmp[0] - 1] = _tmp[i + _tmp[0]]; } - this->setValueByKey(this->preID2values, _pre_id, (char*)_values, sizeof(unsigned) * _values_len); + this->setValueByKey(this->preID2values, _pre_id, (char*)_values, sizeof(unsigned) * (_values_len + 6)); //delete[] _values; } delete []_tmp; @@ -1653,7 +1881,7 @@ KVstore::build_subID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num { _pidoffsetlist_s[j] += 3 + _pidoffsetlist_s.size(); } - unsigned* _entrylist_s = new unsigned[3 + _pidoffsetlist_s.size() + _oidlist_s.size()]; + unsigned* _entrylist_s = new unsigned[9 + _pidoffsetlist_s.size() + _oidlist_s.size() + 6]; //triples number _entrylist_s[0] = _oidlist_s.size(); //pre number @@ -1671,8 +1899,15 @@ KVstore::build_subID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num { _entrylist_s[j] = _oidlist_s[k]; } + + _entrylist_s[j] = _entrylist_s[1]; + _entrylist_s[j + 1] = 0; + _entrylist_s[j + 2] = _entrylist_s[0]; + _entrylist_s[j + 3] = 0; + _entrylist_s[j + 5] = 0; - this->addValueByKey(this->subID2values, _sub_id, (char*)_entrylist_s, sizeof(unsigned) * j); + this->addValueByKey(this->subID2values, _sub_id, (char*)_entrylist_s, sizeof(unsigned) * (j + 6)); + //delete[] _entrylist_s; } } @@ -1953,7 +2188,7 @@ KVstore::build_objID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num for (unsigned j = 1; j < _pidoffsetlist_o.size(); j += 2) { _pidoffsetlist_o[j] += 2 + _pidoffsetlist_o.size(); } - unsigned* _entrylist_o = new unsigned[2 + _pidoffsetlist_o.size() + _sidlist_o.size()]; + unsigned* _entrylist_o = new unsigned[8 + _pidoffsetlist_o.size() + _sidlist_o.size()]; //triples number _entrylist_o[0] = _sidlist_o.size(); //pre number @@ -1967,9 +2202,16 @@ KVstore::build_objID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num for (k = 0; k < _sidlist_o.size(); j++, k++) { _entrylist_o[j] = _sidlist_o[k]; } - - this->addValueByKey(this->objID2values, _obj_id, (char*)_entrylist_o, sizeof(unsigned) * j); - + + _entrylist_o[j] = _entrylist_o[1]; + _entrylist_o[j + 1] = 0; + _entrylist_o[j + 2] = _entrylist_o[0]; + _entrylist_o[j + 3] = 0; + _entrylist_o[j + 5] = 0; + + + this->addValueByKey(this->objID2values, _obj_id, (char*)_entrylist_o, sizeof(unsigned) * (j + 6)); + //delete[] _entrylist_o; } } @@ -2187,7 +2429,7 @@ KVstore::build_preID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num _pre_change = (i + 1 == _triples_num) || (_p_id_tuples[i].preid != _p_id_tuples[i+1].preid); if (_pre_change) { - unsigned* _entrylist_p = new unsigned[1 + _sidlist_p.size() * 2]; + unsigned* _entrylist_p = new unsigned[7 + _sidlist_p.size() * 2]; //triples number _entrylist_p[0] = _sidlist_p.size(); unsigned j, k; @@ -2199,7 +2441,15 @@ KVstore::build_preID2values(ID_TUPLE* _p_id_tuples, TYPE_TRIPLE_NUM _triples_num for (k = 0; k < _oidlist_p.size(); j++, k++) { _entrylist_p[j] = _oidlist_p[k]; } - this->addValueByKey(this->preID2values, _pre_id, (char*)_entrylist_p, sizeof(unsigned) * j); + + _entrylist_p[j] = _entrylist_p[0]; + _entrylist_p[j + 1] = 0; + _entrylist_p[j + 2] = _entrylist_p[0]; + _entrylist_p[j + 3] = 0; + _entrylist_p[j + 5] = 0; + + this->addValueByKey(this->preID2values, _pre_id, (char*)_entrylist_p, sizeof(unsigned) * (j + 6)); + //delete[] _entrylist_p; } } @@ -2222,14 +2472,17 @@ KVstore::getsubIDlistBypreID(TYPE_PREDICATE_ID _preid, unsigned*& _subidlist, un _list_len = 0; return false; } - + _list_len = _tmp[0]; _subidlist = new unsigned[_list_len]; + memcpy(_subidlist, _tmp + 1, sizeof(unsigned) * _list_len); + if (_no_duplicate) { _list_len = Util::removeDuplicate(_subidlist, _list_len); } + //cout << _list_len << "-----------------------------" << endl; //if this is a long list, then we should remove itself after copying //otherwise, we should not free the list memory // if(VList::listNeedDelete(_len)) @@ -2673,6 +2926,60 @@ KVstore::getValueByKey(IVTree* _p_btree, unsigned _key, char*& _val, unsigned& _ return _p_btree->search(_key, _val, _vlen); }*/ +/*bool +KVstore::getValueByKey(IVArray* _array, unsigned _key, char*& _val, unsigned& _vlen) const +{ + if (Util::is_literal_ele(_key) && _array == objID2values) + { + unsigned key = _key - Util::LITERAL_FIRST_ID; + bool get = objID2values_literal->search(_key, _val, _vlen); + if (get) + { + _vlen -= 24; + char *debug = new char [_vlen]; + memcpy(debug, _val, _vlen); + _val = debug; + } + return get; + } + bool get = _array->search(_key, _val, _vlen); + if (get) + { + cout << _vlen << endl; + _vlen -= 24; + char *debug = new char [_vlen]; + memcpy(debug, _val, _vlen); + _val = debug; + } + return get; +} + +bool +KVstore::getValueAndIndexByKey(IVArray* _array, unsigned _key, char*& _val, unsigned& _vlen, unsigned& _vindex) const +{ + if (Util::is_literal_ele(_key) && _array == objID2values) + { + unsigned key = _key - Util::LITERAL_FIRST_ID; + bool get = objID2values_literal->search(_key, _val, _vlen, _vindex); + if (get) + { + _vlen -= 24; + char *debug = new char [_vlen]; + memcpy(debug, _val, _vlen); + _val = debug; + } + return get; + } + bool get = _array->search(_key, _val, _vlen, _vindex); + if (get) + { + _vlen -= 24; + char *debug = new char [_vlen]; + memcpy(debug, _val, _vlen); + _val = debug; + } + return get; +}*/ bool KVstore::getValueByKey(IVArray* _array, unsigned _key, char*& _val, unsigned& _vlen) const { @@ -2684,6 +2991,18 @@ KVstore::getValueByKey(IVArray* _array, unsigned _key, char*& _val, unsigned& _v return _array->search(_key, _val, _vlen); } +bool +KVstore::getValueAndIndexByKey(IVArray* _array, unsigned _key, char*& _val, unsigned& _vlen, unsigned& _vindex,int &_in_cache) const +{ + if (Util::is_literal_ele(_key) && _array == objID2values) + { + unsigned key = _key - Util::LITERAL_FIRST_ID; + return objID2values_literal->search(key, _val, _vlen, _vindex,_in_cache); + } + return _array->search(_key, _val, _vlen, _vindex,_in_cache); +} + + TYPE_ENTITY_LITERAL_ID KVstore::getIDByStr(SITree* _p_btree, const char* _key, unsigned _klen) const { diff --git a/KVstore/KVstore.h b/KVstore/KVstore.h index bfef16eb..e3b1d041 100644 --- a/KVstore/KVstore.h +++ b/KVstore/KVstore.h @@ -259,7 +259,7 @@ class KVstore bool getValueByKey(ISArray* _array, unsigned _key, char*& _val, unsigned& _vlen) const; // bool getValueByKey(IVTree* _p_btree, unsigned _key, char*& _val, unsigned& _vlen) const; bool getValueByKey(IVArray* _array, unsigned _key, char*& _val, unsigned& _vlen) const; - + bool getValueAndIndexByKey(IVArray* _array, unsigned _key, char*& _val, unsigned& _vlen, unsigned& _vindex,int &_in_cache) const; TYPE_ENTITY_LITERAL_ID getIDByStr(SITree* _p_btree, const char* _key, unsigned _klen) const;