diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml index eca579806c..1ea6a27176 100644 --- a/.github/workflows/scala.yml +++ b/.github/workflows/scala.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [ develop ] + branches: [ develop, rfairfax/build_test ] pull_request: branches: [ develop, integration, main, feat-index-rust ] @@ -19,6 +19,11 @@ jobs: with: java-version: '11' distribution: 'adopt' + - name: Install Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt, clippy + target: x86_64-apple-darwin, aarch64-apple-darwin, aarch64-unknown-linux-gnu - name: Run tests run: .github/workflows/runtests.sh - name: Coverage Reports diff --git a/.gitignore b/.gitignore index fef830a629..a2dcb6e69c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,7 @@ metastore_db/ **/kafka/src/test/scala/filodb/kafka/shard* *lib* +# Allow Rust's lib.rs since we're otherwise blocking *lib* above +!lib.rs + coordinator/src/test/resources/ diff --git a/core/src/main/resources/filodb-defaults.conf b/core/src/main/resources/filodb-defaults.conf index c81be31415..1afa875f2a 100644 --- a/core/src/main/resources/filodb-defaults.conf +++ b/core/src/main/resources/filodb-defaults.conf @@ -800,6 +800,11 @@ filodb { # Whether caching on index is disabled underlying Lucene index uses LRUCache enabled by default, the flag lets us #disable this feature disable-index-caching = false + + # The Part Key index implementation to use. Supported values: + # lucene - Lucene based index (default) + # tantivy - Tantivy based index + part-key-index-type = lucene } # for standalone worker cluster configuration, see akka-bootstrapper diff --git a/core/src/main/scala/filodb.core/memstore/PartKeyIndex.scala b/core/src/main/scala/filodb.core/memstore/PartKeyIndex.scala index 6386f71216..1493157978 100644 --- a/core/src/main/scala/filodb.core/memstore/PartKeyIndex.scala +++ b/core/src/main/scala/filodb.core/memstore/PartKeyIndex.scala @@ -64,6 +64,11 @@ abstract class PartKeyIndexRaw(ref: DatasetRef, protected val lifecycleManager: Option[IndexMetadataStore] = None) extends StrictLogging { + protected val startTimeLookupLatency = Kamon.histogram("index-startTimes-for-odp-lookup-latency", + MeasurementUnit.time.nanoseconds) + .withTag("dataset", ref.dataset) + .withTag("shard", shardNum) + protected val queryIndexLookupLatency = Kamon.histogram("index-partition-lookup-latency", MeasurementUnit.time.nanoseconds) .withTag("dataset", ref.dataset) diff --git a/core/src/main/scala/filodb.core/memstore/PartKeyLuceneIndex.scala b/core/src/main/scala/filodb.core/memstore/PartKeyLuceneIndex.scala index f9056a7aa2..053be92acf 100644 --- a/core/src/main/scala/filodb.core/memstore/PartKeyLuceneIndex.scala +++ b/core/src/main/scala/filodb.core/memstore/PartKeyLuceneIndex.scala @@ -79,11 +79,6 @@ class PartKeyLuceneIndex(ref: DatasetRef, import PartKeyLuceneIndex._ import PartKeyIndexRaw._ - val startTimeLookupLatency = Kamon.histogram("index-startTimes-for-odp-lookup-latency", - MeasurementUnit.time.nanoseconds) - .withTag("dataset", ref.dataset) - .withTag("shard", shardNum) - val partIdFromPartKeyLookupLatency = Kamon.histogram("index-ingestion-partId-lookup-latency", MeasurementUnit.time.nanoseconds) .withTag("dataset", ref.dataset) diff --git a/core/src/main/scala/filodb.core/memstore/PartKeyTantivyIndex.scala b/core/src/main/scala/filodb.core/memstore/PartKeyTantivyIndex.scala new file mode 100644 index 0000000000..63e8c1361c --- /dev/null +++ b/core/src/main/scala/filodb.core/memstore/PartKeyTantivyIndex.scala @@ -0,0 +1,213 @@ +package filodb.core.memstore + +import java.io.File +import java.nio.file.Files + +import debox.Buffer +import org.apache.commons.lang3.SystemUtils +import org.apache.lucene.util.BytesRef + +import filodb.core.DatasetRef +import filodb.core.metadata.Column.ColumnType.{MapColumn, StringColumn} +import filodb.core.metadata.PartitionSchema +import filodb.core.query.ColumnFilter + +class PartKeyTantivyIndex(ref: DatasetRef, + schema: PartitionSchema, + shardNum: Int, + retentionMillis: Long, // only used to calculate fallback startTime + diskLocation: Option[File] = None, + lifecycleManager: Option[IndexMetadataStore] = None + ) extends PartKeyIndexRaw(ref, shardNum, schema, diskLocation, lifecycleManager) { + + // Compute field names for native schema code + private val schemaFields = schema.columns.filter { c => + c.columnType == StringColumn + }.map { c => + c.name + }.toArray + + private val schemaMapFields = schema.columns.filter { c => + c.columnType == MapColumn + }.map { c => + c.name + }.toArray + + private val schemaMultiColumnFacets = schema.options.multiColumnFacets.keys.toArray + + // Native handle for cross JNI operations + private var indexHandle: Long = loadIndexData(() => TantivyNativeMethods.newIndexHandle(indexDiskLocation.toString, + schemaFields, schemaMapFields, schemaMultiColumnFacets)) + + logger.info(s"Created tantivy index for dataset=$ref shard=$shardNum at $indexDiskLocation") + + override def reset(): Unit = { + TantivyNativeMethods.reset(indexHandle) + } + + override def startFlushThread(flushDelayMinSeconds: Int, flushDelayMaxSeconds: Int): Unit = { + ??? + } + + override def partIdsEndedBefore(endedBefore: Long): Buffer[Int] = { + ??? + } + + override def removePartitionsEndedBefore(endedBefore: Long, returnApproxDeletedCount: Boolean): Int = { + ??? + } + + override def removePartKeys(partIds: Buffer[Int]): Unit = { + ??? + } + + override def indexRamBytes: Long = { + ??? + } + + override def indexNumEntries: Long = { + ??? + } + + override def closeIndex(): Unit = { + logger.info(s"Closing index on dataset=$ref shard=$shardNum") + + commit() + TantivyNativeMethods.freeIndexHandle(indexHandle) + indexHandle = 0 + } + + override def indexNames(limit: Int): Seq[String] = { + ??? + } + + override def indexValues(fieldName: String, topK: Int): Seq[TermInfo] = { + ??? + } + + override def labelNamesEfficient(colFilters: Seq[ColumnFilter], startTime: Long, endTime: Long): Seq[String] = { + ??? + } + + override def labelValuesEfficient(colFilters: Seq[ColumnFilter], startTime: Long, endTime: Long, + colName: String, limit: Int): Seq[String] = { + ??? + } + + override def addPartKey(partKeyOnHeapBytes: Array[Byte], partId: Int, startTime: Long, endTime: Long, + partKeyBytesRefOffset: Int)(partKeyNumBytes: Int, documentId: String): Unit = { + ??? + } + + override def upsertPartKey(partKeyOnHeapBytes: Array[Byte], partId: Int, startTime: Long, endTime: Long, + partKeyBytesRefOffset: Int)(partKeyNumBytes: Int, documentId: String): Unit = { + ??? + } + + override def partKeyFromPartId(partId: Int): Option[BytesRef] = { + ??? + } + + override def startTimeFromPartId(partId: Int): Long = { + ??? + } + + override def endTimeFromPartId(partId: Int): Long = { + ??? + } + + override def startTimeFromPartIds(partIds: Iterator[Int]): debox.Map[Int, Long] = { + ??? + } + + override def commit(): Unit = { + TantivyNativeMethods.commit(indexHandle) + } + + override def updatePartKeyWithEndTime(partKeyOnHeapBytes: Array[Byte], partId: Int, endTime: Long, + partKeyBytesRefOffset: Int)(partKeyNumBytes: Int, documentId: String): Unit = { + ??? + } + + override def refreshReadersBlocking(): Unit = { + ??? + } + + override def partIdsFromFilters(columnFilters: Seq[ColumnFilter], startTime: Long, endTime: Long, + limit: Int): Buffer[Int] = { + ??? + } + + override def partKeyRecordsFromFilters(columnFilters: Seq[ColumnFilter], startTime: Long, endTime: Long, + limit: Int): Seq[PartKeyLuceneIndexRecord] = { + ??? + } + + override def partIdFromPartKeySlow(partKeyBase: Any, partKeyOffset: Long): Option[Int] = { + ??? + } + + override def singlePartKeyFromFilters(columnFilters: Seq[ColumnFilter], startTime: Long, + endTime: Long): Option[Array[Byte]] = { + ??? + } + + override protected def addIndexedField(key: String, value: String): Unit = { + ??? + } + + protected def addIndexedMapField(mapColumn: String, key: String, value: String): Unit = { + ??? + } + + protected override def addMultiColumnFacet(key: String, value: String): Unit = { + ??? + } +} + +// JNI methods +protected object TantivyNativeMethods { + // Load native library from jar + private def loadLibrary(): Unit = { + val tempDir = Files.createTempDirectory("filodb-native-") + + val lib = System.mapLibraryName("filodb_core") + + val arch = SystemUtils.OS_ARCH + val kernel = if (SystemUtils.IS_OS_LINUX) { + "linux" + } else if (SystemUtils.IS_OS_MAC) { + "darwin" + } else if (SystemUtils.IS_OS_WINDOWS) { + "windows" + } else { + sys.error(s"Unhandled platform ${SystemUtils.OS_NAME}") + } + + val resourcePath: String = "/native/" + kernel + "/" + arch + "/" + lib + val resourceStream = Option(TantivyNativeMethods.getClass.getResourceAsStream(resourcePath)).get + + val finalPath = tempDir.resolve(lib) + Files.copy(resourceStream, finalPath) + + System.load(finalPath.toAbsolutePath.toString) + } + + loadLibrary() + + @native + def newIndexHandle(diskLocation: String, schemaFields: Array[String], + schemaMapFields: Array[String], schemaMultiColumnFacets: Array[String]): Long + + // Free memory used by an index handle + @native + def freeIndexHandle(handle: Long): Unit + + // Reset index data (delete all docs) + @native + def reset(handle: Long): Unit + + // Commit changes to the index + @native + def commit(handle: Long): Unit +} \ No newline at end of file diff --git a/core/src/main/scala/filodb.core/memstore/TimeSeriesShard.scala b/core/src/main/scala/filodb.core/memstore/TimeSeriesShard.scala index 50fc6c8b5a..85b1547716 100644 --- a/core/src/main/scala/filodb.core/memstore/TimeSeriesShard.scala +++ b/core/src/main/scala/filodb.core/memstore/TimeSeriesShard.scala @@ -285,6 +285,7 @@ class TimeSeriesShard(val ref: DatasetRef, private val indexFacetingEnabledAllLabels = filodbConfig.getBoolean("memstore.index-faceting-enabled-for-all-labels") private val numParallelFlushes = filodbConfig.getInt("memstore.flush-task-parallelism") private val disableIndexCaching = filodbConfig.getBoolean("memstore.disable-index-caching") + private val partKeyIndexType = filodbConfig.getString("memstore.part-key-index-type") /////// END CONFIGURATION FIELDS /////////////////// @@ -311,9 +312,14 @@ class TimeSeriesShard(val ref: DatasetRef, * Used to answer queries not involving the full partition key. * Maintained using a high-performance bitmap index. */ - private[memstore] final val partKeyIndex: PartKeyIndexRaw = new PartKeyLuceneIndex(ref, schemas.part, - indexFacetingEnabledAllLabels, indexFacetingEnabledShardKeyLabels, shardNum, - storeConfig.diskTTLSeconds * 1000, disableIndexCaching = disableIndexCaching) + private[memstore] final val partKeyIndex: PartKeyIndexRaw = partKeyIndexType match { + case "lucene" => new PartKeyLuceneIndex(ref, schemas.part, + indexFacetingEnabledAllLabels, indexFacetingEnabledShardKeyLabels, shardNum, + storeConfig.diskTTLSeconds * 1000, disableIndexCaching = disableIndexCaching) + case "tantivy" => new PartKeyTantivyIndex(ref, schemas.part, + shardNum, storeConfig.diskTTLSeconds * 1000) + case x => sys.error(s"Unsupported part key index type: '$x'") + } private val cardTracker: CardinalityTracker = initCardTracker() diff --git a/core/src/rust/Cargo.lock b/core/src/rust/Cargo.lock new file mode 100644 index 0000000000..d00932f87e --- /dev/null +++ b/core/src/rust/Cargo.lock @@ -0,0 +1,1266 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + +[[package]] +name = "async-trait" +version = "0.1.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "bitpacking" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" +dependencies = [ + "crunchy", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" + +[[package]] +name = "cc" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastdivide" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b" + +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "filodb_core" +version = "0.1.0" +dependencies = [ + "bytes", + "jni", + "nom", + "num-derive", + "num-traits", + "regex", + "tantivy", + "thiserror", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "lru" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" + +[[package]] +name = "measure_time" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +dependencies = [ + "instant", + "log", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memmap2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "oneshot" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e296cf87e61c9cfc1a61c3c63a0f7f286ed4554e0e22be84e8a38e1d264a2a29" + +[[package]] +name = "ownedbytes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.204" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.204" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.120" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +dependencies = [ + "serde", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "syn" +version = "2.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tantivy" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "num_cpus", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror", + "time", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +dependencies = [ + "nom", +] + +[[package]] +name = "tantivy-sstable" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +dependencies = [ + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +dependencies = [ + "murmurhash32", + "rand_distr", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2675633b1499176c2dff06b0856a27976a8f9d436737b4cf4f312d4d91d8bbb" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.12+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/core/src/rust/Cargo.toml b/core/src/rust/Cargo.toml new file mode 100644 index 0000000000..edc187292d --- /dev/null +++ b/core/src/rust/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "filodb_core" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +jni = "0.21.1" +nom = "7.1.3" +num-derive = "0.4.2" +num-traits = "0.2.19" +regex = "1.10.5" +tantivy = "0.22.0" +thiserror = "1.0.62" + +[dev-dependencies] +bytes = "1.6.1" diff --git a/core/src/rust/Clippy.toml b/core/src/rust/Clippy.toml new file mode 100644 index 0000000000..0358cdb508 --- /dev/null +++ b/core/src/rust/Clippy.toml @@ -0,0 +1,2 @@ +allow-unwrap-in-tests = true +allow-expect-in-tests = true diff --git a/core/src/rust/src/errors.rs b/core/src/rust/src/errors.rs new file mode 100644 index 0000000000..f52539b627 --- /dev/null +++ b/core/src/rust/src/errors.rs @@ -0,0 +1,43 @@ +//! Error types to translate to exceptions for the JVM + +use std::borrow::Cow; + +use jni::JNIEnv; + +const RUNTIME_EXCEPTION_CLASS: &str = "java/lang/RuntimeException"; + +/// Result type for java exception methods +pub type JavaResult = Result; + +/// Error type that can be thrown as an exception +pub struct JavaException { + class: &'static str, + message: Cow<'static, str>, +} + +impl JavaException { + /// Create a new java.lang.RuntimeException + pub fn new_runtime_exception(message: impl Into>) -> Self { + Self::new(RUNTIME_EXCEPTION_CLASS, message) + } + + /// Create a new exception with a specified class and message + pub fn new(class: &'static str, message: impl Into>) -> Self { + Self { + class, + message: message.into(), + } + } + + /// Throw the generated exception on a JNIEnv + pub fn set_exception_details(&self, env: &mut JNIEnv) { + let _ = env.throw_new(self.class, &self.message); + } +} + +// Default conversion for Rust std errors - throw RuntimeException +impl From for JavaException { + fn from(value: T) -> Self { + Self::new_runtime_exception(format!("{value}")) + } +} diff --git a/core/src/rust/src/exec.rs b/core/src/rust/src/exec.rs new file mode 100644 index 0000000000..b4f18a5ff0 --- /dev/null +++ b/core/src/rust/src/exec.rs @@ -0,0 +1,56 @@ +//! Helpers for executing code in a JNI method + +use jni::{sys::jobject, JNIEnv}; + +use crate::errors::JavaResult; + +/// Execs a function in a JNI context, supplying an environment +/// and translating any errors to exceptions +/// +/// All JNI functions should use this to ensure error handling +/// is properly done +/// +/// Do *not* panic in any calls - avoid unwrap, expect, etc. +pub fn jni_exec(env: &mut JNIEnv, func: F) -> T +where + F: FnOnce(&mut JNIEnv) -> JavaResult, + T: EarlyReturn, +{ + let ret = func(env); + match ret { + Ok(r) => r, + Err(e) => { + // An error occurred, throw an exception + e.set_exception_details(env); + + T::abort_value() + } + } +} + +/// Trait for early return values when an exception is being thrown +pub trait EarlyReturn { + fn abort_value() -> Self; +} + +impl EarlyReturn for jobject { + fn abort_value() -> Self { + std::ptr::null_mut() + } +} + +impl EarlyReturn for i32 { + fn abort_value() -> Self { + 0 + } +} + +impl EarlyReturn for i64 { + fn abort_value() -> Self { + 0 + } +} + +impl EarlyReturn for () { + fn abort_value() -> Self {} +} diff --git a/core/src/rust/src/index.rs b/core/src/rust/src/index.rs new file mode 100644 index 0000000000..2fdc7b1555 --- /dev/null +++ b/core/src/rust/src/index.rs @@ -0,0 +1,138 @@ +//! Methods to create / destroy the index + +use jni::{ + objects::{JClass, JObjectArray, JString}, + sys::jlong, + JNIEnv, +}; +use tantivy::{ + directory::MmapDirectory, + schema::{ + FacetOptions, Field, JsonObjectOptions, Schema, SchemaBuilder, TextFieldIndexing, FAST, + INDEXED, STORED, STRING, + }, + Index, TantivyDocument, +}; + +use crate::{ + errors::{JavaException, JavaResult}, + exec::jni_exec, + jnienv::JNIEnvExt, + state::{FieldConstants, IndexHandle}, +}; + +pub const WRITER_MEM_BUDGET: usize = 50 * 1024 * 1024; + +/// Create a new index state object by loading and configuring schema +#[no_mangle] +pub extern "system" fn Java_filodb_core_memstore_TantivyNativeMethods_00024_newIndexHandle( + mut env: JNIEnv, + _class: JClass, + disk_location: JString, + schema_fields: JObjectArray, + map_fields: JObjectArray, + multi_column_facet_fields: JObjectArray, +) -> jlong { + jni_exec(&mut env, |env| { + let disk_location: String = env.get_string(&disk_location)?.into(); + let directory = MmapDirectory::open(disk_location)?; + + // Build the schema for documents + let fields = FieldConstants::load(env)?; + let (schema, default_field) = build_schema( + env, + &schema_fields, + &map_fields, + &multi_column_facet_fields, + &fields, + )?; + + // Open index + let index = Index::open_or_create(directory, schema.clone())?; + let writer = index.writer::(WRITER_MEM_BUDGET)?; + let reader = index.reader()?; + + Ok(IndexHandle::new_handle( + schema, + default_field, + writer, + reader, + fields, + )) + }) +} + +#[no_mangle] +pub extern "system" fn Java_filodb_core_memstore_TantivyNativeMethods_00024_freeIndexHandle( + mut env: JNIEnv, + _class: JClass, + handle: jlong, +) { + jni_exec(&mut env, |_| { + unsafe { + drop(Box::from_raw(handle as *mut IndexHandle)); + } + + Ok(()) + }); +} + +fn build_schema( + env: &mut JNIEnv, + schema_fields: &JObjectArray, + map_fields: &JObjectArray, + multi_column_facet_fields: &JObjectArray, + constants: &FieldConstants, +) -> JavaResult<(Schema, Option)> { + let mut builder = SchemaBuilder::new(); + + builder.add_text_field(&constants.part_id, STRING); + builder.add_i64_field(&constants.part_id_dv, INDEXED | STORED | FAST); + builder.add_bytes_field(&constants.part_key, INDEXED | STORED | FAST); + builder.add_i64_field(&constants.start_time, INDEXED | STORED | FAST); + builder.add_i64_field(&constants.end_time, INDEXED | STORED | FAST); + + // Fields from input schema + env.foreach_string_in_array(schema_fields, |name| { + builder.add_text_field(&name, STRING); + + Ok(()) + })?; + + // Map fields - only one supported + let len = env.get_array_length(map_fields)?; + if len > 1 { + return Err(JavaException::new_runtime_exception( + "More than one map field specified", + )); + } + + let default_field = if len == 1 { + let name = env.get_object_array_element(map_fields, 0)?.into(); + let name = env.get_rust_string(&name)?; + + let field = builder.add_json_field( + &name, + JsonObjectOptions::default() + .set_indexing_options(TextFieldIndexing::default().set_tokenizer("raw")), + ); + + Some(field) + } else { + None + }; + + env.foreach_string_in_array(multi_column_facet_fields, |name| { + builder.add_facet_field(&constants.facet_field_name(&name), FacetOptions::default()); + + Ok(()) + })?; + + // Default facet for label list, always added + builder.add_facet_field( + &constants.facet_field_name(&constants.label_list), + FacetOptions::default(), + ); + + Ok((builder.build(), default_field)) +} diff --git a/core/src/rust/src/jnienv.rs b/core/src/rust/src/jnienv.rs new file mode 100644 index 0000000000..3a6f87eb08 --- /dev/null +++ b/core/src/rust/src/jnienv.rs @@ -0,0 +1,82 @@ +//! Extensions to JNIEnv + +use jni::{ + objects::{JObject, JObjectArray, JString}, + JNIEnv, +}; + +use crate::errors::JavaResult; + +/// Helper extensions for working with JVM types +#[allow(dead_code)] +pub trait JNIEnvExt<'a> { + /// Get a rust string from Java String + fn get_rust_string(&mut self, obj: &JString) -> JavaResult; + + /// Get a class name and return it as a string + /// This is equivilant to Java code `obj.class.name` + fn get_object_class_name(&mut self, obj: &JObject) -> JavaResult; + + /// Get a scala object instance + fn get_scala_object(&mut self, name: &str) -> JavaResult>; + + /// Get a scala String val from an object + fn get_scala_text_constant(&mut self, obj: &JObject, name: &str) -> JavaResult; + + /// Run a closure over every String in a String[] + fn foreach_string_in_array(&mut self, array: &JObjectArray, func: F) -> JavaResult<()> + where + F: FnMut(String) -> JavaResult<()>; +} + +impl<'a> JNIEnvExt<'a> for JNIEnv<'a> { + fn get_rust_string(&mut self, obj: &JString) -> JavaResult { + let ret = self.get_string(obj)?.into(); + Ok(ret) + } + + fn get_object_class_name(&mut self, obj: &JObject) -> JavaResult { + let class = self.get_object_class(obj)?; + let name = self + .get_field(&class, "name", "Ljava/lang/String;")? + .l()? + .into(); + + let ret = self.get_string(&name)?.into(); + Ok(ret) + } + + fn get_scala_object(&mut self, name: &str) -> JavaResult> { + // objects are static fields on the class name named MODULE$ + let obj = self + .get_static_field(format!("{name}$"), "MODULE$", format!("L{name}$;"))? + .l()?; + + Ok(obj) + } + + fn get_scala_text_constant(&mut self, obj: &JObject, name: &str) -> JavaResult { + // private val foo = ""; is modeled in scala as a member method named foo that returns String + let val: JString = self + .call_method(obj, name, "()Ljava/lang/String;", &[])? + .l()? + .into(); + + let ret = self.get_string(&val)?.into(); + Ok(ret) + } + + fn foreach_string_in_array(&mut self, array: &JObjectArray, mut func: F) -> JavaResult<()> + where + F: FnMut(String) -> JavaResult<()>, + { + let len = self.get_array_length(array)?; + for idx in 0..len { + let s = self.get_object_array_element(array, idx)?.into(); + let s = self.get_rust_string(&s)?; + func(s)?; + } + + Ok(()) + } +} diff --git a/core/src/rust/src/lib.rs b/core/src/rust/src/lib.rs new file mode 100644 index 0000000000..483adaf29b --- /dev/null +++ b/core/src/rust/src/lib.rs @@ -0,0 +1,25 @@ +//! Native methods for FiloDB core +//! +//! This library extensively uses JNI to interop with JVM code. +//! +//! Any new code should do the following to ensure consistency: +//! +//! * All JNI methods should be wrapped in jni_exec. This turns any +//! Rust errors into RuntimeExceptions and allows for cleaner Rust +//! error handling. +//! * No panic/unwrap/expect calls should be used. Panicing will destroy +//! the JVM process. +//! * Try to use primitive types when possible. Getting fields on JVM +//! objects requires reflection like overhead that can't be optimized +//! as well +//! * Minimize the calls back into the JVM. Perfer to get passed in +//! needed information as arguments vs calling object methods. +//! + +#![deny(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +mod errors; +mod exec; +mod index; +mod jnienv; +mod state; diff --git a/core/src/rust/src/state.rs b/core/src/rust/src/state.rs new file mode 100644 index 0000000000..899d793d04 --- /dev/null +++ b/core/src/rust/src/state.rs @@ -0,0 +1,124 @@ +//! State objects shared with Java + +// Temporary until all logic is checked in +#![allow(dead_code)] + +use std::{ + collections::{BTreeMap, HashMap}, + sync::{atomic::AtomicBool, Mutex, RwLock}, +}; + +use jni::{sys::jlong, JNIEnv}; +use tantivy::{ + schema::{Field, OwnedValue, Schema}, + IndexReader, IndexWriter, TantivyDocument, +}; + +use crate::{errors::JavaResult, jnienv::JNIEnvExt}; + +pub struct IndexHandle { + // Immutable fields that don't need synchronization + // + // + // Schema for this nidex + pub schema: Schema, + // Default field for JSON searches + pub default_field: Option, + // Field data + pub fields: FieldConstants, + + // Fields that need synchronization + // + // + pub changes_pending: AtomicBool, + // Active writer + pub writer: RwLock, + // Active reader + pub reader: Mutex, + // Actively ingesting doc + pub ingesting_doc: Mutex, +} + +impl IndexHandle { + pub fn new_handle( + schema: Schema, + default_field: Option, + writer: IndexWriter, + reader: IndexReader, + fields: FieldConstants, + ) -> jlong { + let obj = Box::new(Self { + schema, + default_field, + writer: RwLock::new(writer), + reader: Mutex::new(reader), + changes_pending: AtomicBool::new(false), + fields, + ingesting_doc: Mutex::new(IngestingDocument::default()), + }); + + Box::into_raw(obj) as jlong + } + + /// Decode handle back into a reference + pub fn get_ref_from_handle<'a>(handle: jlong) -> &'a Self { + let ptr = handle as *const IndexHandle; + + unsafe { &*ptr } + } +} + +/// A document that is actively being built up for ingesting +#[derive(Default)] +pub struct IngestingDocument { + // List of map entries we're building up to store in the document + pub map_values: HashMap>, + // List of field names in the document being ingested + pub field_names: Vec, + // Document state for ingestion + pub doc: TantivyDocument, +} + +const PART_KEY_INDEX_RAW_CLASS: &str = "filodb/core/memstore/PartKeyIndexRaw"; + +// Field constants loaded from Scala classes +pub struct FieldConstants { + pub part_id: String, + pub part_id_dv: String, + pub part_key: String, + pub start_time: String, + pub end_time: String, + pub label_list: String, + pub facet_field_prefix: String, +} + +impl FieldConstants { + /// Get a computed facet field name + pub fn facet_field_name(&self, name: &str) -> String { + format!("{}{}", self.facet_field_prefix, name) + } + + /// Load constants from Scala runtime classes + pub fn load(env: &mut JNIEnv) -> JavaResult { + let constants_object = env.get_scala_object(PART_KEY_INDEX_RAW_CLASS)?; + + let part_id = env.get_scala_text_constant(&constants_object, "PART_ID_FIELD")?; + let part_id_dv = env.get_scala_text_constant(&constants_object, "PART_ID_DV")?; + let part_key = env.get_scala_text_constant(&constants_object, "PART_KEY")?; + let start_time = env.get_scala_text_constant(&constants_object, "START_TIME")?; + let end_time = env.get_scala_text_constant(&constants_object, "END_TIME")?; + let label_list = env.get_scala_text_constant(&constants_object, "LABEL_LIST")?; + let facet_field_prefix = + env.get_scala_text_constant(&constants_object, "FACET_FIELD_PREFIX")?; + + Ok(Self { + part_id, + part_id_dv, + part_key, + start_time, + end_time, + label_list, + facet_field_prefix, + }) + } +} diff --git a/core/src/test/resources/application_test.conf b/core/src/test/resources/application_test.conf index a2fb216240..eab2a584f9 100644 --- a/core/src/test/resources/application_test.conf +++ b/core/src/test/resources/application_test.conf @@ -109,7 +109,7 @@ filodb { index-faceting-enabled-shard-key-labels = true index-faceting-enabled-for-all-labels = true disable-index-caching = false - + part-key-index-type = lucene } tasks { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 0426db04eb..ed1ec11a85 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -80,7 +80,8 @@ object Dependencies { "com.esotericsoftware" % "kryo" % "4.0.0" excludeAll(excludeMinlog), "com.dorkbox" % "MinLog-SLF4J" % "1.12", "com.github.ben-manes.caffeine" % "caffeine" % "3.0.5", - "com.twitter" %% "chill" % "0.9.3" + "com.twitter" %% "chill" % "0.9.3", + "org.apache.commons" % "commons-lang3" % "3.14.0" ) lazy val sparkJobsDeps = commonDeps ++ Seq( diff --git a/project/FiloBuild.scala b/project/FiloBuild.scala index 24cb7bb26e..fd5ca08f0f 100644 --- a/project/FiloBuild.scala +++ b/project/FiloBuild.scala @@ -1,5 +1,6 @@ import Dependencies._ import FiloSettings._ +import RustPlugin._ import com.typesafe.sbt.SbtMultiJvm.MultiJvmKeys.MultiJvm import io.gatling.sbt.GatlingPlugin import pl.project13.scala.sbt.JmhPlugin @@ -45,6 +46,7 @@ object Submodules { ) lazy val core = (project in file("core")) + .enablePlugins(RustPlugin) .dependsOn(memory % "compile->compile; test->test") .settings( commonSettings,