Merge branch 'master' into test/clt_select_1439

.github/workflows/build_template.yml

@@ -63,7 +63,7 @@ jobs:
         shell: bash
     timeout-minutes: 30
     container:
-      image: manticoresearch/external_toolchain:clang15_cmake3263
+      image: manticoresearch/external_toolchain:clang16_cmake3263
       env:
         CACHEB: "../cache"
         DIAGNOSTIC: 1

deps.txt

@@ -1,6 +1,6 @@
 backup 1.3.5 24020911 2ffbd02
-buddy 2.1.7 24021308 f975242
-mcl 2.2.5 240115 214ce90
+buddy 2.1.7 24021618 903f71c
+mcl 2.2.5 240217 8c90c1f
 executor 1.0.13 23122917 fa7313e
 executor-win 0.3.5 221125 a40554c
 tzdata 0.9.9 230720 daeb8a8

dist/build_dockers/cross/external_toolchain/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:focal
 ENV DEBIAN_FRONTEND noninteractive
 
 # 1-st layer, basic toolchain stuff
-ENV llvm=15
+ENV llvm=16
 RUN apt-get update && apt-get install -y \
     make \
     ninja-build \
@@ -14,7 +14,7 @@ RUN apt-get update && apt-get install -y \
 && rm -rf /var/lib/apt/lists/*
 
 # 2-nd layer, llvm
-ENV llvm=15
+ENV llvm=16
 RUN curl -L https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
 && add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-$llvm main" \
 && apt-get install -y \
@@ -58,7 +58,7 @@ CMD []
 
 # docker buildx create --name manticore_build --platform linux/amd64,linux/arm64
 # docker buildx use manticore_build
-# docker buildx build --platform linux/amd64,linux/arm64 --push -t manticoresearch/external_toolchain:clang15_cmake3263 .
+# docker buildx build --platform linux/amd64,linux/arm64 --push -t manticoresearch/external_toolchain:clang16_cmake3263 .
 
 # deprecated:
 # on amd:

manual/Securing_and_compacting_a_table/Freezing_a_table.md

@@ -7,7 +7,7 @@ FREEZE tbl1[, tbl2, ...]
 ```
 
 `FREEZE` readies a real-time/plain table for a secure [backup](../Securing_and_compacting_a_table/Backup_and_restore.md). Specifically, it:
-1. Deactivates table compaction. If the table is currently being compacted, `FREEZE` will wait for completion.
+1. Deactivates table compaction. If the table is currently being compacted, `FREEZE` will interrupt it.
 2. Transfers the current RAM chunk to a disk chunk.
 3. Flushes attributes.
 4. Disables implicit operations that could modify the disk files.

src/replication/api_command_cluster.cpp

@@ -147,6 +147,10 @@ void HandleAPICommandCluster ( ISphOutputBuffer & tOut, WORD uCommandVer, InputB
 		ReceiveDistIndex ( tOut, tBuf, sCluster );
 		break;
 
+	case E_CLUSTER::GET_NODE_STATE:
+		ReceiveClusterGetState ( tOut, tBuf, sCluster );
+		break;
+
 	default:
 		TlsMsg::Err ( "INTERNAL ERROR: unhandled command %d", (int) eClusterCmd );
 		break;

src/replication/api_command_cluster.h

@@ -32,6 +32,7 @@ enum class E_CLUSTER : WORD
 	GET_NODES			= 6,
 	UPDATE_NODES		= 7,
 	INDEX_ADD_DIST		= 8,
+	GET_NODE_STATE		= 9,
 };
 
 inline constexpr const char* szClusterCmd ( E_CLUSTER eCmd )
@@ -46,6 +47,7 @@ inline constexpr const char* szClusterCmd ( E_CLUSTER eCmd )
 	case E_CLUSTER::GET_NODES: return "get_nodes";
 	case E_CLUSTER::UPDATE_NODES: return "update_nodes";
 	case E_CLUSTER::INDEX_ADD_DIST: return "index_add_distributed";
+	case E_CLUSTER::GET_NODE_STATE: return "get_node_state";
 	default: return "unknown";
 	}
 }

src/replication/cluster_commands.h

@@ -20,6 +20,7 @@ void ReceiveClusterGetNodes ( ISphOutputBuffer& tOut, InputBuffer_c& tBuf, CSphS
 void ReceiveClusterIndexAddLocal ( ISphOutputBuffer& tOut, InputBuffer_c& tBuf, CSphString& sCluster );
 void ReceiveClusterSynced ( ISphOutputBuffer& tOut, InputBuffer_c& tBuf, CSphString& sCluster );
 void ReceiveClusterUpdateNodes ( ISphOutputBuffer& tOut, InputBuffer_c& tBuf, CSphString& sCluster );
+void ReceiveClusterGetState ( ISphOutputBuffer & tOut, InputBuffer_c & tBuf, CSphString & sCluster );
 
 // sending doesn't include cluster name
 void ReceiveClusterFileSend ( ISphOutputBuffer& tOutb, InputBuffer_c& tBuf );

src/replication/cluster_delete.cpp

@@ -38,6 +38,6 @@ void SendClusterDeleteToNodes ( const VecTraits_T<CSphString>& dNodes, const CSp
 
 	ClusterRequest_t tData { sCluster };
 	ClusterDelete_c tReq;
-	auto dAgents = tReq.MakeAgents ( GetDescAPINodes ( dNodes, Resolve_e::QUICK ), ReplicationTimeoutQuery(), tData );
+	auto dAgents = tReq.MakeAgents ( GetDescAPINodes ( dNodes, Resolve_e::SLOW ), ReplicationTimeoutQuery(), tData );
 	PerformRemoteTasksWrap ( dAgents, tReq, tReq, true );
 }

src/replication/cluster_get_nodes.cpp

@@ -112,4 +112,74 @@ StrVec_t GetNodeListFromRemotes ( const ClusterDesc_t& tDesc )
 	else
 		TlsMsg::Err ( "cluster '%s', invalid nodes '%s'(%s), error: %s", tDesc.m_sName.cstr(), StrVec2Str ( dNodes ).cstr(), StrVec2Str ( tDesc.m_dClusterNodes ).cstr(), TlsMsg::szError() );
 	return dNodes;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// cluster get nodes state
+/////////////////////////////////////////////////////////////////////////////
+
+struct ClusterNodesStatesReply_t : public ClusterRequest_t
+{
+	ClusterNodesStatesVec_t m_dStates;
+	RemoteNodeClusterState_t m_tState;
+};
+
+// API command to remote node to get node state
+using ClusterNodeState_c = ClusterCommand_T<E_CLUSTER::GET_NODE_STATE, ClusterRequest_t, ClusterNodesStatesReply_t>;
+
+void operator<< ( ISphOutputBuffer & tOut, const ClusterNodesStatesReply_t & tReq )
+{
+	tOut << (const ClusterRequest_t&)tReq;
+	tOut.SendByte ( (BYTE)tReq.m_tState.m_eState );
+	tOut.SendString ( tReq.m_tState.m_sNode.cstr() );
+	tOut.SendString ( tReq.m_tState.m_sHash.cstr() );
+}
+
+void operator>> ( InputBuffer_c & tIn, ClusterNodesStatesReply_t & tReq )
+{
+	tIn >> (ClusterRequest_t&)tReq;
+
+	RemoteNodeClusterState_t & tState = tReq.m_dStates.Add();
+	tState.m_eState = (ClusterState_e)tIn.GetByte();
+	tState.m_sNode = tIn.GetString();
+	tState.m_sHash = tIn.GetString();
+}
+
+static bool SendClusterNodesStates ( const CSphString & sCluster, const VecTraits_T<CSphString> & dNodes, ClusterNodesStatesVec_t & dStates )
+{
+	ClusterNodeState_c::REQUEST_T tReq;
+	tReq.m_sCluster = sCluster;
+
+	auto dAgents = ClusterNodeState_c::MakeAgents ( GetDescAPINodes ( dNodes, Resolve_e::SLOW ), ReplicationTimeoutQuery(), tReq );
+	// no nodes left seems a valid case
+	if ( dAgents.IsEmpty() )
+		return true;
+
+	ClusterNodeState_c tReply;
+	// FIXME!!! handle errors
+	PerformRemoteTasksWrap ( dAgents, tReply, tReply, true );
+
+	for ( const AgentConn_t * pAgent : dAgents )
+	{
+		if ( pAgent->m_bSuccess )
+			dStates.Append ( ClusterNodeState_c::GetRes ( *pAgent ).m_dStates );
+	}
+
+	return true;
+}
+
+ClusterNodesStatesVec_t GetStatesFromRemotes ( const ClusterDesc_t & tDesc )
+{
+	ClusterNodesStatesVec_t dStates;
+	SendClusterNodesStates ( tDesc.m_sName, tDesc.m_dClusterNodes, dStates );
+	return dStates;
+}
+
+void ReceiveClusterGetState ( ISphOutputBuffer & tOut, InputBuffer_c & tBuf, CSphString & sCluster )
+{
+	ClusterNodesStatesReply_t tRequest;
+	ClusterNodeState_c::ParseRequest ( tBuf, tRequest );
+
+	ClusterGetState ( tRequest.m_sCluster, tRequest.m_tState );
+	ClusterNodeState_c::BuildReply ( tOut, tRequest );
 }

src/replication/cluster_get_nodes.h

@@ -12,9 +12,20 @@
 
 #include "std/string.h"
 #include "searchdaemon.h"
+#include "common.h"
 
 // command to all remote nodes at cluster to get actual nodes list
 StrVec_t GetNodeListFromRemotes ( const ClusterDesc_t& tDesc );
 
+struct RemoteNodeClusterState_t
+{
+	CSphString m_sNode;
+	ClusterState_e m_eState;
+	CSphString m_sHash;
+};
 
+using ClusterNodesStatesVec_t = CSphVector<RemoteNodeClusterState_t>;
 
+// command to all remote nodes at cluster to get actual nodes states
+ClusterNodesStatesVec_t GetStatesFromRemotes ( const ClusterDesc_t & tDesc );
+bool ClusterGetState ( const CSphString & sCluster, RemoteNodeClusterState_t & tState );

src/replication/cluster_index_add_local.cpp

@@ -163,7 +163,7 @@ static bool RotateFiles ( const std::unique_ptr<MergeState_t>& pState, FilesTrai
 }
 
 // load index from disk files for cluster use
-static ServedIndexRefPtr_c LoadNewIndex ( const CSphString& sIndexPath, IndexType_e eIndexType, const char* szIndexName, const CSphString& sCluster, FilesTrait_t& tIndexFiles )
+static ServedIndexRefPtr_c LoadNewIndex ( const CSphString & sIndexPath, IndexType_e eIndexType, const char * szIndexName, FilesTrait_t & tIndexFiles )
 {
 	CSphConfigSection hIndex;
 	hIndex.Add ( CSphVariant ( sIndexPath.cstr() ), "path" );
@@ -181,7 +181,6 @@ static ServedIndexRefPtr_c LoadNewIndex ( const CSphString& sIndexPath, IndexTyp
 		return pResult;
 
 	assert ( pNewServed );
-	pNewServed->m_sCluster = sCluster;
 
 	StrVec_t dWarnings;
 	bool bPrealloc = PreallocNewIndex ( *pNewServed, &hIndex, szIndexName, dWarnings, sError );
@@ -197,9 +196,9 @@ static ServedIndexRefPtr_c LoadNewIndex ( const CSphString& sIndexPath, IndexTyp
 }
 
 // load index into daemon
-static bool LoadAndReplaceIndex ( const CSphString& sIndexPath, IndexType_e eIndexType, const CSphString& sIndexName, const CSphString& sCluster, FilesTrait_t& tIndexFiles )
+static bool LoadAndReplaceIndex ( const CSphString& sIndexPath, IndexType_e eIndexType, const CSphString & sIndexName, FilesTrait_t & tIndexFiles )
 {
-	auto pNewIndex = LoadNewIndex ( sIndexPath, eIndexType, sIndexName.cstr(), sCluster, tIndexFiles );
+	auto pNewIndex = LoadNewIndex ( sIndexPath, eIndexType, sIndexName.cstr(), tIndexFiles );
 	if ( !pNewIndex )
 		return false;
 
@@ -209,16 +208,16 @@ static bool LoadAndReplaceIndex ( const CSphString& sIndexPath, IndexType_e eInd
 
 // load index into daemon
 // in case index already exists prohibit it to save on index delete as disk files has fresh data received from remote node
-static bool LoadIndex ( const CSphString& sIndexPath, IndexType_e eIndexType, const CSphString& sIndexName, const CSphString& sCluster, FilesTrait_t& tIndexFiles )
+static bool LoadIndex ( const CSphString & sIndexPath, IndexType_e eIndexType, const CSphString & sIndexName, FilesTrait_t & tIndexFiles )
 {
 	cServedIndexRefPtr_c pOldIndex = GetServed ( sIndexName );
 	if ( !ServedDesc_t::IsMutable ( pOldIndex ) )
-		return LoadAndReplaceIndex ( sIndexPath, eIndexType, sIndexName.cstr(), sCluster, tIndexFiles );
+		return LoadAndReplaceIndex ( sIndexPath, eIndexType, sIndexName.cstr(), tIndexFiles );
 
 	WIdx_T<RtIndex_i*> pIndex { pOldIndex };
 	pIndex->ProhibitSave();
 	pIndex->GetIndexFiles ( tIndexFiles.m_dOld, tIndexFiles.m_dOld );
-	return LoadAndReplaceIndex ( sIndexPath, eIndexType, sIndexName.cstr(), sCluster, tIndexFiles );
+	return LoadAndReplaceIndex ( sIndexPath, eIndexType, sIndexName.cstr(), tIndexFiles );
 }
 
 
@@ -260,9 +259,12 @@ static bool AddReceivedIndex ( const ClusterIndexAddLocalRequest_t& tAddCmd )
 
 	FilesTrait_t tIndexFiles;
 
-	if ( !LoadIndex ( pMerge->m_sIndexPath, tAddCmd.m_eIndex, tAddCmd.m_sIndex, tAddCmd.m_sCluster, tIndexFiles ) )
+	if ( !LoadIndex ( pMerge->m_sIndexPath, tAddCmd.m_eIndex, tAddCmd.m_sIndex, tIndexFiles ) )
 		return TlsMsg::Err ( "failed to load index '%s'", tAddCmd.m_sIndex.cstr() );
 
+	if ( !AddLoadedIndexIntoCluster ( tAddCmd.m_sCluster, tAddCmd.m_sIndex ) )
+		return false;
+
 	// keep: only files that donor has
 	// remove:
 	// - index files renamed to .old

src/replication/cluster_index_add_local.h

@@ -29,4 +29,4 @@ void operator>> ( MemInputBuffer_c& tOut, ClusterIndexAddLocalRequest_t& tReq );
 
 using ClusterIndexAddLocal_c = ClusterCommand_T<E_CLUSTER::INDEX_ADD_LOCAL, ClusterIndexAddLocalRequest_t>;
 
-
+bool AddLoadedIndexIntoCluster ( const CSphString & sCluster, const CSphString & sIndex );

src/replication/cluster_update_nodes.cpp

@@ -37,7 +37,7 @@ bool SendClusterUpdateNodes ( const CSphString& sCluster, NODES_E eNodes, const
 	tRequest.m_sCluster = sCluster;
 	tRequest.m_eKindNodes = eNodes;
 
-	auto dAgents = ClusterUpdateNodes_c::MakeAgents ( GetDescAPINodes ( dNodes, Resolve_e::QUICK ), ReplicationTimeoutQuery(), tRequest );
+	auto dAgents = ClusterUpdateNodes_c::MakeAgents ( GetDescAPINodes ( dNodes, Resolve_e::SLOW ), ReplicationTimeoutQuery(), tRequest );
 	// no nodes left seems a valid case
 	if ( dAgents.IsEmpty() )
 		return true;

src/replication/common.h

@@ -23,3 +23,16 @@ bool ClusterDelete ( const CSphString& sCluster );
 // command at remote node for CLUSTER_SYNCED to pick up received indexes and notify Galera that sst received
 struct ClusterSyncedRequest_t;
 bool ClusterSynced ( const ClusterSyncedRequest_t& tCmd );
+
+// cluster state this node sees
+enum class ClusterState_e
+{
+	// stop terminal states
+	CLOSED,				// node shut well or not started
+	DESTROYED,			// node closed with error
+	// transaction states
+	JOINING,			// node joining cluster
+	DONOR,				// node is donor for another node joining cluster
+	// ready terminal state 
+	SYNCED				// node works as usual
+};

src/searchdaemon.h

@@ -181,7 +181,7 @@ enum SearchdCommandV_e : WORD
 	VER_COMMAND_PING		= 0x100,
 	VER_COMMAND_UVAR		= 0x100,
 	VER_COMMAND_CALLPQ		= 0x100,
-	VER_COMMAND_CLUSTER		= 0x107,
+	VER_COMMAND_CLUSTER		= 0x108,
 	VER_COMMAND_GETFIELD	= 0x100,
 
 	VER_COMMAND_WRONG = 0,