Update to version v2.11.0

.travis.yml

@@ -29,7 +29,7 @@ git:
     - mkdir -p ${GOPATH}/src/github.com/restream && ln -s `pwd` ${GOPATH}/src/github.com/restream/reindexer
   install:
     - sudo apt-get -qq update > /dev/null && sudo apt-get -qq install -y -f build-essential binutils cmake curl git google-perftools libgoogle-perftools-dev libsnappy-dev libleveldb-dev flex python3-pip
-    - curl https://raw.githubusercontent.com/travis-ci/gimme/master/gimme > ~/gimme && chmod +x ~/gimme && ~/gimme 1.9
+    - curl https://raw.githubusercontent.com/travis-ci/gimme/master/gimme > ~/gimme && chmod +x ~/gimme && eval "$(~/gimme 1.13.1)"
   before_script:
     - git clone https://github.com/google/googletest.git /tmp/dep_googletest && cd /tmp/dep_googletest && cmake -DBUILD_GMOCK=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-std=c++11" . && make -j4 && sudo make install
     - git clone https://github.com/google/benchmark.git /tmp/dep_googlebench && cd /tmp/dep_googlebench && git checkout tags/v1.4.0 -b v.1.4.0 && cmake -DBENCHMARK_ENABLE_TESTING=Off -DCMAKE_BUILD_TYPE=Release . && make -j4 && sudo make install

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v2.10.0"
+const ReindexerVersion = "v2.11.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,3 +1,12 @@
+# Version 2.11.0 (29.07.2020)
+
+# Core
+- [fix] Crash with SEGV or assert on ordered queries after copying tx
+- [fix] Correct normalization of SELECT COUNT(*) SQL queries
+- [fea] More efficient replication startup on active master updates
+- [fix] Namespace indexes optimization after load was not started
+- [fix] Centos 7 build fixed
+
 # Version 2.10.0 (05.07.2020)
 
 # Core

cpp_src/CMakeLists.txt

@@ -21,7 +21,7 @@ else()
   option (LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "2.10.0")
+set (REINDEXER_VERSION_DEFAULT "2.11.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")

cpp_src/cmd/reindexer_server/test/end_to_end_test.sh

@@ -29,9 +29,9 @@ while ((1==1)); do
 done
 
 if [ -z "$1" ]; then
-  python3 .
+  pytest . --alluredir=/builds/itv-backend/reindexer/allure-results
 else
-  python3 . "$1"
+  pytest . -k "$1" --alluredir=/builds/itv-backend/reindexer/allure-results
 fi
 test_exit_code=$?
 

cpp_src/core/lsn.h

@@ -60,8 +60,8 @@ struct lsn_t {
 	int64_t Counter() const {
 		int64_t server = payload_ / digitCountLSNMult;
 		return payload_ - server * digitCountLSNMult;
-	};
-	short Server() const { return payload_ / digitCountLSNMult; };
+	}
+	short Server() const { return payload_ / digitCountLSNMult; }
 	bool isEmpty() const { return Counter() == digitCountLSNMult - 1ll; }
 
 	int compare(lsn_t o) {

cpp_src/core/namespace/namespaceimpl.cc

@@ -102,7 +102,7 @@ void NamespaceImpl::copyContentsFrom(const NamespaceImpl &src) {
 	storage_ = src.storage_;
 	updates_ = src.updates_;
 	unflushedCount_.store(src.unflushedCount_.load(std::memory_order_acquire), std::memory_order_release);	// 0
-	sortOrdersBuilt_ = src.sortOrdersBuilt_.load();															// false
+	sortOrdersBuilt_ = false;
 	meta_ = src.meta_;
 	dbpath_ = src.dbpath_;
 	queryCache_ = src.queryCache_;
@@ -121,7 +121,7 @@ void NamespaceImpl::copyContentsFrom(const NamespaceImpl &src) {
 	serverId_ = src.serverId_;
 	storageOpts_ = src.storageOpts_;
 	for (auto &idxIt : src.indexes_) indexes_.push_back(unique_ptr<Index>(idxIt->Clone()));
-
+	markUpdated();
 	logPrintf(LogTrace, "Namespace::CopyContentsFrom (%s)", name_);
 }
 

cpp_src/core/nsselecter/nsselecter.cc

@@ -63,8 +63,8 @@ void NsSelecter::operator()(QueryResults &result, SelectCtx &ctx, const RdxConte
 	// DO NOT use deducted sort order in the following cases:
 	// - query contains explicity specified sort order
 	// - query contains FullText query.
-	bool disableOptimizeSortOrder = !ctx.query.sortingEntries_.empty() || ctx.preResult;
-	SortingEntries sortBy = (isFt || disableOptimizeSortOrder) ? ctx.query.sortingEntries_ : qPreproc.DetectOptimalSortOrder();
+	const bool disableOptimizeSortOrder = isFt || !ctx.query.sortingEntries_.empty() || ctx.preResult;
+	SortingEntries sortBy = disableOptimizeSortOrder ? ctx.query.sortingEntries_ : qPreproc.DetectOptimalSortOrder();
 
 	if (ctx.preResult) {
 		if (ctx.preResult->executionMode == JoinPreResult::ModeBuild) {

cpp_src/core/nsselecter/selectiteratorcontainer.cc

@@ -194,7 +194,7 @@ SelectKeyResults SelectIteratorContainer::processQueryEntry(const QueryEntry &qe
 		opts.forceComparator = 1;
 	}
 	if (ctx_->sortingContext.isOptimizationEnabled()) {
-		if (ctx_->sortingContext.uncommitedIndex == qe.idxNo && enableSortIndexOptimize) {
+		if (enableSortIndexOptimize) {
 			opts.unbuiltSortOrders = 1;
 		} else {
 			opts.forceComparator = 1;
@@ -310,6 +310,7 @@ void SelectIteratorContainer::PrepareIteratorsForSelectLoop(const QueryEntries &
 															bool isQueryFt, const NamespaceImpl &ns, SelectFunction::Ptr selectFnc,
 															FtCtx::Ptr &ftCtx, const RdxContext &rdxCtx) {
 	size_t next = 0;
+	bool sortIndexCreated = false;
 	for (size_t i = begin; i < end; i = queries.Next(i)) {
 		next = queries.Next(i);
 		auto op = queries.GetOperation(i);
@@ -332,12 +333,13 @@ void SelectIteratorContainer::PrepareIteratorsForSelectLoop(const QueryEntries &
 					}
 					selectResults = processQueryEntry(qe, ns, strictMode);
 				} else {
-					bool enableSortIndexOptimize =
-						!this->Size() && (op != OpNot) && !qe.distinct && (next == end || queries.GetOperation(next) != OpOr);
+					const bool enableSortIndexOptimize = !sortIndexCreated && (op == OpAnd) && !qe.distinct && (begin == 0) &&
+														 (ctx_->sortingContext.uncommitedIndex == qe.idxNo) &&
+														 (next == end || queries.GetOperation(next) != OpOr);
 					selectResults = processQueryEntry(qe, enableSortIndexOptimize, ns, sortId, isQueryFt, selectFnc, isIndexFt,
 													  isIndexSparse, ftCtx, rdxCtx);
+					if (enableSortIndexOptimize) sortIndexCreated = true;
 				}
-
 				processQueryEntryResults(selectResults, op, ns, qe, isIndexFt, isIndexSparse, nonIndexField);
 			} else {
 				processJoinEntry(qe, op);

cpp_src/core/query/sql/sqlencoder.cc

@@ -134,7 +134,15 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const {
 					assert(query_.aggregations_[0].fields_.size() == 1);
 					distinctIndex = query_.aggregations_[0].fields_[0];
 				}
-				if (!query_.selectFilter_.empty()) {
+				if (query_.selectFilter_.empty()) {
+					if (query_.count != 0) {
+						if (needComma) ser << ", ";
+						ser << '*';
+						if (query_.calcTotal != ModeNoTotal) {
+							needComma = true;
+						}
+					}
+				} else {
 					for (const auto &filter : query_.selectFilter_) {
 						if (filter == distinctIndex) continue;
 						if (needComma) {
@@ -144,13 +152,13 @@ WrSerializer &SQLEncoder::GetSQL(WrSerializer &ser, bool stripArgs) const {
 						}
 						ser << filter;
 					}
-				} else {
-					if (needComma) ser << ", ";
-					ser << '*';
 				}
 			}
-			if (query_.calcTotal == ModeAccurateTotal) ser << ", COUNT(*)";
-			if (query_.calcTotal == ModeCachedTotal) ser << ", COUNT_CACHED(*)";
+			if (query_.calcTotal != ModeNoTotal) {
+				if (needComma) ser << ", ";
+				if (query_.calcTotal == ModeAccurateTotal) ser << "COUNT(*)";
+				if (query_.calcTotal == ModeCachedTotal) ser << "COUNT_CACHED(*)";
+			}
 			ser << " FROM " << query_._namespace;
 		} break;
 		case QueryDelete:

cpp_src/core/query/sql/sqlparser.cc

@@ -251,6 +251,9 @@ int SQLParser::parseOrderBy(tokenizer &parser, SortingEntries &sortingEntries, h
 			throw Error(errParseSQL, "Expected name, but found '%s' in query, %s", tok.text(), parser.where());
 		SortingEntry sortingEntry;
 		sortingEntry.expression = string(tok.text());
+		if (sortingEntry.expression.empty()) {
+			throw Error(errParseSQL, "Order by expression should not be empty, %s", parser.where());
+		}
 		tok = peekSqlToken(parser, SortDirectionSqlToken);
 		if (tok.text() == "("_sv && nameWithCase.text() == "field"_sv) {
 			parser.next_token();

cpp_src/core/querystat.cc

@@ -9,14 +9,14 @@ namespace reindexer {
 template <void (PerfStatCounterST::*hitFunc)(std::chrono::microseconds)>
 void QueriesStatTracer::hit(const QuerySQL &sql, std::chrono::microseconds time) {
 	std::unique_lock<std::mutex> lck(mtx_);
-	const auto it = stat_.find(sql.normolized);
+	const auto it = stat_.find(sql.normalized);
 	if (it == stat_.end()) {
-		(stat_.emplace(string(sql.normolized), Stat(sql.nonNormolized)).first->second.*hitFunc)(time);
+		(stat_.emplace(string(sql.normalized), Stat(sql.nonNormalized)).first->second.*hitFunc)(time);
 	} else {
 		const auto maxTime = it->second.MaxTime();
 		(it->second.*hitFunc)(time);
 		if (it->second.MaxTime() > maxTime) {
-			it->second.longestQuery = string(sql.nonNormolized);
+			it->second.longestQuery = string(sql.nonNormalized);
 		}
 	}
 }

cpp_src/core/querystat.h

@@ -21,8 +21,8 @@ struct QueryPerfStat {
 class QueriesStatTracer {
 public:
 	struct QuerySQL {
-		string_view normolized;
-		string_view nonNormolized;
+		string_view normalized;
+		string_view nonNormalized;
 	};
 
 	void Hit(const QuerySQL& sql, std::chrono::microseconds time) { hit<&PerfStatCounterST::Hit>(sql, time); }

cpp_src/core/reindexerimpl.cc

@@ -676,9 +676,9 @@ struct ItemRefLess {
 
 Error ReindexerImpl::Select(const Query& q, QueryResults& result, const InternalRdxContext& ctx) {
 	try {
-		WrSerializer normolizedSQL, nonNormolizedSQL;
-		if (ctx.NeedTraceActivity()) q.GetSQL(nonNormolizedSQL, false);
-		const auto rdxCtx = ctx.CreateRdxContext(ctx.NeedTraceActivity() ? nonNormolizedSQL.Slice() : "", activities_, result);
+		WrSerializer normalizedSQL, nonNormalizedSQL;
+		if (ctx.NeedTraceActivity()) q.GetSQL(nonNormalizedSQL, false);
+		const auto rdxCtx = ctx.CreateRdxContext(ctx.NeedTraceActivity() ? nonNormalizedSQL.Slice() : "", activities_, result);
 		NsLocker<const RdxContext> locks(rdxCtx);
 
 		auto mainNsWrp = getNamespace(q._namespace, rdxCtx);
@@ -688,10 +688,10 @@ Error ReindexerImpl::Select(const Query& q, QueryResults& result, const Internal
 		PerfStatCalculatorMT calc(mainNs->selectPerfCounter_, mainNs->enablePerfCounters_);	 // todo more accurate detect joined queries
 		auto& tracker = queriesStatTracker_;
 		if (profilingCfg.queriesPerfStats) {
-			q.GetSQL(normolizedSQL, true);
-			if (!ctx.NeedTraceActivity()) q.GetSQL(nonNormolizedSQL, false);
+			q.GetSQL(normalizedSQL, true);
+			if (!ctx.NeedTraceActivity()) q.GetSQL(nonNormalizedSQL, false);
 		}
-		const QueriesStatTracer::QuerySQL sql{normolizedSQL.Slice(), nonNormolizedSQL.Slice()};
+		const QueriesStatTracer::QuerySQL sql{normalizedSQL.Slice(), nonNormalizedSQL.Slice()};
 		QueryStatCalculator statCalculator(
 			[&sql, &tracker](bool lockHit, std::chrono::microseconds time) {
 				if (lockHit)
@@ -1200,7 +1200,7 @@ std::vector<string> defDBConfig = {
 				"start_copy_policy_tx_size":10000
 				"copy_policy_multiplier":5
 				"tx_size_to_always_copy":100000,
-				"optimization_timeout_ms":800
+				"optimization_timeout_ms":800,
 			}
     	]
 	})json",

cpp_src/gtests/tests/fixtures/replication_api.cc

@@ -63,7 +63,7 @@ void ReplicationApi::WaitSync(const std::string& ns) {
 	ReplicationStateApi state{lsn_t(), lsn_t(), 0, 0, false};
 	while (state.lsn.isEmpty()) {
 		counter++;
-		EXPECT_TRUE(counter / 100 < kMaxSyncTimeSec);
+		ASSERT_TRUE(counter / 100 < kMaxSyncTimeSec);
 		ReplicationStateApi xstate = GetSrv(masterId_)->GetState(ns);  // get an reference state and then compare all with it
 		for (size_t i = 0; i < svc_.size(); i++) {
 			if (i != masterId_) {
@@ -94,7 +94,7 @@ void ReplicationApi::SwitchMaster(size_t id) {
 	GetSrv(masterId_)->MakeMaster(config);
 	for (size_t i = 0; i < svc_.size(); i++) {
 		std::string masterDsn = "cproto://127.0.0.1:" + std::to_string(kDefaultRpcPort + masterId_) + "/node" + std::to_string(masterId_);
-		ReplicationConfigTest config("slave", false, true, i, masterDsn);
+		ReplicationConfigTest config("slave", false, true, i, masterDsn, "server_" + std::to_string(i));
 		if (i != masterId_) GetSrv(i)->MakeSlave(masterId_, config);
 	}
 }

cpp_src/gtests/tests/fixtures/replication_api.h

@@ -45,7 +45,6 @@ class ReplicationApi : public ::testing::Test {
 	void ForceSync();
 	// Switch master
 	void SwitchMaster(size_t id);
-	//
 
 	size_t masterId_ = 0;
 

cpp_src/gtests/tests/fixtures/rpcclient_api.h

@@ -74,7 +74,7 @@ class RPCClientTestApi : public ::testing::Test {
 	void TearDown() {}
 
 	void AddServer(const string& addr = kDefaultRPCServerAddr, const RPCServerConfig& conf = RPCServerConfig()) {
-		servers_.emplace(addr, new TestServer(conf));
+		servers_.emplace(addr, std::unique_ptr<TestServer>(new TestServer(conf)));
 	}
 	void StartServer(const string& addr = kDefaultRPCServerAddr, Error errOnLogin = Error()) {
 		auto it = servers_.find(addr);