diff --git a/.gitignore b/.gitignore index 7514d55cc3c9a..291a63cdeef92 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,5 @@ testfixtures_shared/ .ci/jobs/ # build files generated -doc-tools/missing-doclet/bin/ \ No newline at end of file +doc-tools/missing-doclet/bin/ +server/src/main/java/org/opensearch/indices/KLSPerformanceTest.java diff --git a/server/build.gradle b/server/build.gradle index f6db3d53a0dcc..426719f53632e 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -158,6 +158,10 @@ dependencies { api "com.google.protobuf:protobuf-java:${versions.protobuf}" api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}" + // roaring bitmaps + api 'org.roaringbitmap:RoaringBitmap:0.9.49' + runtimeOnly 'org.roaringbitmap:shims:0.9.49' // might fix complaining about ArraysShims? + testImplementation(project(":test:framework")) { // tests use the locally compiled version of server exclude group: 'org.opensearch', module: 'server' diff --git a/server/licenses/RoaringBitmap-0.9.49.jar.sha1 b/server/licenses/RoaringBitmap-0.9.49.jar.sha1 new file mode 100644 index 0000000000000..919a73c074b6a --- /dev/null +++ b/server/licenses/RoaringBitmap-0.9.49.jar.sha1 @@ -0,0 +1 @@ +b45b49c1ec5c5fc48580412d0ca635e1833110ea \ No newline at end of file diff --git a/server/licenses/RoaringBitmap-LICENSE.txt b/server/licenses/RoaringBitmap-LICENSE.txt new file mode 100644 index 0000000000000..a890d4a062fad --- /dev/null +++ b/server/licenses/RoaringBitmap-LICENSE.txt @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright 2013-2016 the RoaringBitmap authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/RoaringBitmap-NOTICE.txt b/server/licenses/RoaringBitmap-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/server/licenses/shims-0.9.49.jar.sha1 b/server/licenses/shims-0.9.49.jar.sha1 new file mode 100644 index 0000000000000..9e76614ca5207 --- /dev/null +++ b/server/licenses/shims-0.9.49.jar.sha1 @@ -0,0 +1 @@ +8bd7794fbdaa9536354dd2d8d961d9503beb9460 \ No newline at end of file diff --git a/server/licenses/shims-LICENSE.txt b/server/licenses/shims-LICENSE.txt new file mode 100644 index 0000000000000..a890d4a062fad --- /dev/null +++ b/server/licenses/shims-LICENSE.txt @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright 2013-2016 the RoaringBitmap authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/shims-NOTICE.txt b/server/licenses/shims-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/server/src/main/java/org/opensearch/indices/KeyLookupStore.java b/server/src/main/java/org/opensearch/indices/KeyLookupStore.java new file mode 100644 index 0000000000000..60e1386a460ec --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/KeyLookupStore.java @@ -0,0 +1,133 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.indices; + +/** + * An interface for objects that hold an in-memory record of hashes of keys in the disk cache. + * These objects have some internal data structure which stores some transformation of added + * int values. The internal representations may have collisions. Example transformations include a modulo + * or -abs(value), or some combination. + */ +public interface KeyLookupStore { + + /** + * Transforms the input value into the internal representation for this keystore + * and adds it to the internal data structure. + * @param value The value to add. + * @return true if the value was added, false if it wasn't added because of a + * collision or if it was already present. + */ + boolean add(T value) throws Exception; + + /** + * Checks if the transformation of the value is in the keystore. + * @param value The value to check. + * @return true if the value was found, false otherwise. Due to collisions, false positives are + * possible, but there should be no false negatives unless forceRemove() is called. + */ + boolean contains(T value) throws Exception; + + /** + * Returns the transformed version of the input value, that would be used to stored it in the keystore. + * This transformation should be always be the same for a given instance. + * @param value The value to transform. + * @return The transformed value. + */ + T getInternalRepresentation(T value); + + /** + * Attempts to safely remove a value from the internal structure, maintaining the property that contains(value) + * will never return a false negative. If removing would lead to a false negative, the value won't be removed. + * Classes may not implement safe removal. + * @param value The value to attempt to remove. + * @return true if the value was removed, false if it wasn't. + */ + boolean remove(T value) throws Exception; + + /** + * Returns the number of distinct values stored in the internal data structure. + * Does not count values which weren't successfully added due to collisions. + * @return The number of values + */ + int getSize(); + + /** + * Returns the number of times add() has been run, including unsuccessful attempts. + * @return The number of adding attempts. + */ + int getTotalAdds(); + + /** + * Returns the number of times add() has returned false due to a collision. + * @return The number of collisions. + */ + int getCollisions(); + + + /** + * Checks if two values would collide after being transformed by this store's transformation. + * @param value1 The first value to compare. + * @param value2 The second value to compare. + * @return true if the transformations are equal, false otherwise. + */ + boolean isCollision(T value1, T value2); + + /** + * Returns an estimate of the store's memory usage. + * @return The memory usage, in MB + */ + long getMemorySizeInBytes(); + + /** + * Returns the cap for the store's memory usage. + * @return The cap, in bytes + */ + long getMemorySizeCapInBytes(); + + /** + * Returns whether the store is at memory capacity and can't accept more entries + */ + boolean isFull(); + + /** + * Deletes the internal data structure and regenerates it from the values passed in. + * Also resets all stats related to adding. + * @param newValues The keys that should be in the reset structure. + */ + void regenerateStore(T[] newValues) throws Exception; + + /** + * Deletes all keys and resets all stats related to adding. + */ + void clear() throws Exception; +} diff --git a/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java new file mode 100644 index 0000000000000..3789989b5eaf1 --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/RBMIntKeyLookupStore.java @@ -0,0 +1,264 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.indices; + +import org.opensearch.common.metrics.CounterMetric; +import org.roaringbitmap.RoaringBitmap; + +import java.util.HashSet; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * This class implements KeyLookupStore using a roaring bitmap with a modulo applied to values. + * The modulo increases the density of values, which makes RBMs more memory-efficient. The recommended modulo is ~2^28. + * It also maintains a hash set of values which have had collisions. Values which haven't had collisions can be + * safely removed from the store. The fraction of collided values should be low, + * about 0.5% for a store with 10^7 values and a modulo of 2^28. + * The store estimates its memory footprint and will stop adding more values once it reaches its memory cap. + */ +public class RBMIntKeyLookupStore implements KeyLookupStore { + protected final int modulo; + protected class KeyStoreStats { + protected int size; + protected long memSizeCapInBytes; + protected CounterMetric numAddAttempts; + protected CounterMetric numCollisions; + protected boolean guaranteesNoFalseNegatives; + protected int maxNumEntries; + protected boolean atCapacity; + protected CounterMetric numRemovalAttempts; + protected CounterMetric numSuccessfulRemovals; + protected KeyStoreStats(long memSizeCapInBytes, int maxNumEntries) { + this.size = 0; + this.numAddAttempts = new CounterMetric(); + this.numCollisions = new CounterMetric(); + this.memSizeCapInBytes = memSizeCapInBytes; + this.maxNumEntries = maxNumEntries; + this.atCapacity = false; + this.numRemovalAttempts = new CounterMetric(); + this.numSuccessfulRemovals = new CounterMetric(); + } + } + + protected KeyStoreStats stats; + protected RoaringBitmap rbm; + private HashSet collidedInts; + protected RBMSizeEstimator sizeEstimator; + protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + protected final Lock readLock = lock.readLock(); + protected final Lock writeLock = lock.writeLock(); + + RBMIntKeyLookupStore(int modulo, long memSizeCapInBytes) { + this.modulo = modulo; + sizeEstimator = new RBMSizeEstimator(modulo); + this.stats = new KeyStoreStats(memSizeCapInBytes, calculateMaxNumEntries(memSizeCapInBytes)); + this.rbm = new RoaringBitmap(); + collidedInts = new HashSet<>(); + } + + protected int calculateMaxNumEntries(long memSizeCapInBytes) { + if (memSizeCapInBytes == 0) { + return Integer.MAX_VALUE; + } + return sizeEstimator.getNumEntriesFromSizeInBytes(memSizeCapInBytes); + } + + protected final int transform(int value) { + return modulo == 0 ? value : value % modulo; + } + + protected void handleCollisions(int transformedValue) { + stats.numCollisions.inc(); + collidedInts.add(transformedValue); + } + + @Override + public boolean add(Integer value) throws Exception { + if (value == null) { + return false; + } + writeLock.lock(); + stats.numAddAttempts.inc(); + try { + if (stats.size == stats.maxNumEntries) { + stats.atCapacity = true; + return false; + } + int transformedValue = transform(value); + boolean alreadyContained = contains(value); + if (!alreadyContained) { + rbm.add(transformedValue); + stats.size++; + return true; + } + handleCollisions(transformedValue); + return false; + } finally { + writeLock.unlock(); + } + } + + @Override + public boolean contains(Integer value) throws Exception { + if (value == null) { + return false; + } + int transformedValue = transform(value); + readLock.lock(); + try { + return rbm.contains(transformedValue); + } finally { + readLock.unlock(); + } + } + + @Override + public Integer getInternalRepresentation(Integer value) { + if (value == null) { + return 0; + } + return Integer.valueOf(transform(value)); + } + + @Override + public boolean remove(Integer value) throws Exception { + if (value == null) { + return false; + } + int transformedValue = transform(value); + readLock.lock(); + try { + if (!contains(value)) { + return false; + } + stats.numRemovalAttempts.inc(); + if (collidedInts.contains(transformedValue)) { + return false; + } + } finally { + readLock.unlock(); + } + writeLock.lock(); + try { + rbm.remove(transformedValue); + stats.size--; + stats.numSuccessfulRemovals.inc(); + return true; + } finally { + writeLock.unlock(); + } + } + + @Override + public int getSize() { + readLock.lock(); + try { + return stats.size; + } finally { + readLock.unlock(); + } + } + + @Override + public int getTotalAdds() { + return (int) stats.numAddAttempts.count(); + } + + @Override + public int getCollisions() { + return (int) stats.numCollisions.count(); + } + + + @Override + public boolean isCollision(Integer value1, Integer value2) { + if (value1 == null || value2 == null) { + return false; + } + return transform(value1) == transform(value2); + } + + @Override + public long getMemorySizeInBytes() { + return sizeEstimator.getSizeInBytes(stats.size) + RBMSizeEstimator.getHashsetMemSizeInBytes(collidedInts.size()); + } + + @Override + public long getMemorySizeCapInBytes() { + return stats.memSizeCapInBytes; + } + + @Override + public boolean isFull() { + return stats.atCapacity; + } + + @Override + public void regenerateStore(Integer[] newValues) throws Exception { + rbm.clear(); + collidedInts = new HashSet<>(); + stats.size = 0; + stats.numAddAttempts = new CounterMetric(); + stats.numCollisions = new CounterMetric(); + stats.guaranteesNoFalseNegatives = true; + stats.numRemovalAttempts = new CounterMetric(); + stats.numSuccessfulRemovals = new CounterMetric(); + for (int i = 0; i < newValues.length; i++) { + if (newValues[i] != null) { + add(newValues[i]); + } + } + } + + + + @Override + public void clear() throws Exception { + regenerateStore(new Integer[]{}); + } + public int getNumRemovalAttempts() { + return (int) stats.numRemovalAttempts.count(); + } + + public int getNumSuccessfulRemovals() { + return (int) stats.numSuccessfulRemovals.count(); + } + + public boolean valueHasHadCollision(Integer value) { + if (value == null) { + return false; + } + return collidedInts.contains(transform(value)); + } +} diff --git a/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java b/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java new file mode 100644 index 0000000000000..6e3b8e581ba9f --- /dev/null +++ b/server/src/main/java/org/opensearch/indices/RBMSizeEstimator.java @@ -0,0 +1,131 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.indices; + +/** + * A class used to estimate roaring bitmap memory sizes (and hash set sizes). + * Values based on experiments with adding randomly distributed integers, which matches the use case for KeyLookupStore. + * In this use case, true values are much higher than an RBM's self-reported size, especially for small RBMs: see + * https://github.com/RoaringBitmap/RoaringBitmap/issues/257 + */ +public class RBMSizeEstimator { + public static final int BYTES_IN_MB = 1048576; + public static final double HASHSET_MEM_SLOPE = 6.46 * Math.pow(10, -5); + protected final double slope; + protected final double bufferMultiplier; + protected final double intercept; + + RBMSizeEstimator(int modulo) { + double[] memValues = calculateMemoryCoefficients(modulo); + this.bufferMultiplier = memValues[0]; + this.slope = memValues[1]; + this.intercept = memValues[2]; + } + + public static double[] calculateMemoryCoefficients(int modulo) { + // Sets up values to help estimate RBM size given a modulo + // Returns an array of {bufferMultiplier, slope, intercept} + + double modifiedModulo; + if (modulo == 0) { + modifiedModulo = 32.0; + } else { + modifiedModulo = Math.log(modulo) / Math.log(2); + } + // we "round up" the modulo to the nearest tested value + double highCutoff = 29.001; // Floating point makes 29 not work + double mediumCutoff = 28.0; + double lowCutoff = 26.0; + double bufferMultiplier = 1.0; + double slope; + double intercept; + if (modifiedModulo > highCutoff) { + // modulo > 2^29 + bufferMultiplier = 1.2; + slope = 0.637; + intercept = 3.091; + } else if (modifiedModulo > mediumCutoff) { + // 2^29 >= modulo > 2^28 + slope = 0.619; + intercept = 2.993; + } else if (modifiedModulo > lowCutoff) { + // 2^28 >= modulo > 2^26 + slope = 0.614; + intercept = 2.905; + } else { + slope = 0.628; + intercept = 2.603; + } + return new double[] { bufferMultiplier, slope, intercept }; + } + + public long getSizeInBytes(int numEntries) { + // Based on a linear fit in log-log space, so that we minimize the error as a proportion rather than as + // an absolute value. Should be within ~50% of the true value at worst, and should overestimate rather + // than underestimate the memory usage + return (long) ((long) Math.pow(numEntries, slope) * (long) Math.pow(10, intercept) * bufferMultiplier); + } + + public int getNumEntriesFromSizeInBytes(long sizeInBytes) { + // This function has some precision issues especially when composed with its inverse: + // numEntries = getNumEntriesFromSizeInBytes(getSizeInBytes(numEntries)) + // In this case the result can be off by up to a couple percent + // However, this shouldn't really matter as both functions are based on memory estimates with higher errors than a couple percent + // and this composition won't happen outside of tests + return (int) Math.pow(sizeInBytes / (bufferMultiplier * Math.pow(10, intercept)), 1 / slope); + + } + + public static long getSizeInBytesWithModulo(int numEntries, int modulo) { + double[] memValues = calculateMemoryCoefficients(modulo); + return (long) ((long) Math.pow(numEntries, memValues[1]) * (long) Math.pow(10, memValues[2]) * memValues[0]); + } + + public static int getNumEntriesFromSizeInBytesWithModulo(long sizeInBytes, int modulo) { + double[] memValues = calculateMemoryCoefficients(modulo); + return (int) Math.pow(sizeInBytes / (memValues[0] * Math.pow(10, memValues[2])), 1 / memValues[1]); + } + + + protected static long convertMBToBytes(double valMB) { + return (long) (valMB * BYTES_IN_MB); + } + + protected static double convertBytesToMB(long valBytes) { + return (double) valBytes / BYTES_IN_MB; + } + + protected static long getHashsetMemSizeInBytes(int numEntries) { + return convertMBToBytes(HASHSET_MEM_SLOPE * numEntries); + } +} diff --git a/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java b/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java new file mode 100644 index 0000000000000..c857c1ecab768 --- /dev/null +++ b/server/src/test/java/org/opensearch/indices/RBMIntKeyLookupStoreTests.java @@ -0,0 +1,295 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.indices; + +import org.opensearch.common.Randomness; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Random; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; + +public class RBMIntKeyLookupStoreTests extends OpenSearchTestCase { + public void testInit() { + long memCap = 100 * RBMSizeEstimator.BYTES_IN_MB; + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), memCap); + assertEquals(0, kls.getSize()); + assertEquals(memCap, kls.getMemorySizeCapInBytes()); + } + public void testTransformationLogic() throws Exception { + int modulo = (int) Math.pow(2, 29); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + int offset = 3; + for (int i = 0; i < 4; i++) { // after this we run into max value, but thats not a flaw with the class design + int posValue = i * modulo + offset; + kls.add(posValue); + int negValue = -(i * modulo + offset); + kls.add(negValue); + } + assertEquals(2, kls.getSize()); + int[] testVals = new int[]{0, 1, -1, -23495, 23058, modulo, -modulo, Integer.MAX_VALUE, Integer.MIN_VALUE}; + for (int value : testVals) { + assertTrue(kls.getInternalRepresentation(value) < modulo); + assertTrue(kls.getInternalRepresentation(value) > -modulo); + } + } + + public void testContains() throws Exception { + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + for (int i = 0; i < 2000; i++) { + kls.add(i); + assertTrue(kls.contains(i)); + } + } + + public void testAddingStatsGetters() throws Exception { + int modulo = (int) Math.pow(2, 15); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, 0L); + kls.add(15); + kls.add(-15); + assertEquals(2, kls.getTotalAdds()); + assertEquals(0, kls.getCollisions()); + + int offset = 1; + for (int i = 0; i < 10; i++) { + kls.add(i * modulo + offset); + } + assertEquals(12, kls.getTotalAdds()); + assertEquals(9, kls.getCollisions()); + } + + public void testRegenerateStore() throws Exception { + int numToAdd = 10000000; + Random rand = Randomness.get(); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + for (int i = 0; i < numToAdd; i++) { + kls.add(i); + } + assertEquals(numToAdd, kls.getSize()); + Integer[] newVals = new Integer[1000]; // margin accounts for collisions + for (int j = 0; j < newVals.length; j++) { + newVals[j] = rand.nextInt(); + } + kls.regenerateStore(newVals); + assertTrue(Math.abs(kls.getSize() - newVals.length) < 3); // inexact due to collisions + + // test clear() + kls.clear(); + assertEquals(0, kls.getSize()); + } + + public void testAddingDuplicates() throws Exception { + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + int numToAdd = 4820411; + for (int i = 0; i < numToAdd; i++) { + kls.add(i); + kls.add(i); + } + for (int j = 0; j < 1000; j++) { + kls.add(577); + } + assertEquals(numToAdd, kls.getSize()); + } + + public void testMemoryCapBlocksAdd() throws Exception { + int modulo = (int) Math.pow(2, 29); + for (int maxEntries: new int[]{2342000, 1000, 100000}) { + long memSizeCapInBytes = RBMSizeEstimator.getSizeInBytesWithModulo(maxEntries, modulo); + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes); + for (int j = 0; j < maxEntries + 1000; j++) { + kls.add(j); + } + assertTrue(Math.abs(maxEntries - kls.getSize()) < (double) maxEntries / 25); + // exact cap varies a small amount bc of floating point, especially when we use bytes instead of MB for calculations + // precision gets much worse when we compose the two functions, as we do here, but this wouldn't happen in an actual use case + } + } + + public void testConcurrency() throws Exception { + Random rand = Randomness.get(); + for (int j = 0; j < 5; j++) { // test with different numbers of threads + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + int numThreads = rand.nextInt(50) + 1; + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads); + // In this test we want to add the first 200K numbers and check they're all correctly there. + // We do some duplicates too to ensure those aren't incorrectly added. + int amountToAdd = 200000; + ArrayList> wasAdded = new ArrayList<>(amountToAdd); + ArrayList> duplicatesWasAdded = new ArrayList<>(); + for (int i = 0; i < amountToAdd; i++) { + wasAdded.add(null); + } + for (int i = 0; i < amountToAdd; i++) { + final int val = i; + Future fut = executor.submit(() -> { + boolean didAdd; + try { + didAdd = kls.add(val); + } catch (Exception e) { + throw new RuntimeException(e); + } + return didAdd; + }); + wasAdded.set(val, fut); + if (val % 1000 == 0) { + // do a duplicate add + Future duplicateFut = executor.submit(() -> { + boolean didAdd; + try { + didAdd = kls.add(val); + } catch (Exception e) { + throw new RuntimeException(e); + } + return didAdd; + }); + duplicatesWasAdded.add(duplicateFut); + } + } + int originalAdds = 0; + int duplicateAdds = 0; + for (Future fut : wasAdded) { + if (fut.get()) { + originalAdds++; + } + } + for (Future duplicateFut : duplicatesWasAdded) { + if (duplicateFut.get()) { + duplicateAdds++; + } + } + for (int i = 0; i < amountToAdd; i++) { + assertTrue(kls.contains(i)); + } + assertEquals(amountToAdd, originalAdds + duplicateAdds); + assertEquals(amountToAdd, kls.getSize()); + assertEquals(amountToAdd / 1000, kls.getCollisions()); + executor.shutdown(); + } + } + + public void testRemoveNoCollisions() throws Exception { + long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB; + int numToAdd = 195000; + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(0, memCap); + // there should be no collisions for sequential positive numbers up to modulo + for (int i = 0; i < numToAdd; i++) { + kls.add(i); + } + for (int i = 0; i < 1000; i++) { + assertTrue(kls.remove(i)); + assertFalse(kls.contains(i)); + assertFalse(kls.valueHasHadCollision(i)); + } + assertEquals(numToAdd - 1000, kls.getSize()); + } + + public void testRemoveWithCollisions() throws Exception { + int modulo = (int) Math.pow(2, 26); + long memCap = 100L * RBMSizeEstimator.BYTES_IN_MB; + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memCap); + for (int i = 0; i < 10; i++) { + kls.add(i); + if (i % 2 == 1) { + kls.add(-i); + assertFalse(kls.valueHasHadCollision(i)); + kls.add(i + modulo); + assertTrue(kls.valueHasHadCollision(i)); + } else { + assertFalse(kls.valueHasHadCollision(i)); + } + } + assertEquals(15, kls.getSize()); + for (int i = 0; i < 10; i++) { + boolean didRemove = kls.remove(i); + if (i % 2 == 1) { + // we expect a collision with i + modulo, so we can't remove + assertFalse(didRemove); + assertTrue(kls.contains(i)); + // but we should be able to remove -i + boolean didRemoveNegative = kls.remove(-i); + assertTrue(didRemoveNegative); + assertFalse(kls.contains(-i)); + } else { + // we expect no collision + assertTrue(didRemove); + assertFalse(kls.contains(i)); + assertFalse(kls.valueHasHadCollision(i)); + } + } + assertEquals(5, kls.getSize()); + int offset = 12; + kls.add(offset); + for (int j = 1; j < 5; j++) { + kls.add(offset + j * modulo); + } + assertEquals(6, kls.getSize()); + assertFalse(kls.remove(offset + modulo)); + assertTrue(kls.valueHasHadCollision(offset + 15 * modulo)); + assertTrue(kls.contains(offset + 17 * modulo)); + } + + public void testNullInputs() throws Exception { + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore((int) Math.pow(2, 29), 0L); + assertFalse(kls.add(null)); + assertFalse(kls.contains(null)); + assertEquals(0, (int) kls.getInternalRepresentation(null)); + assertFalse(kls.remove(null)); + assertFalse(kls.isCollision(null, null)); + assertEquals(0, kls.getTotalAdds()); + Integer[] newVals = new Integer[]{1, 17, -2, null, -4, null}; + kls.regenerateStore(newVals); + assertEquals(4, kls.getSize()); + } + + public void testMemoryCapValueInitialization() { + double[] logModulos = new double[] { 0.0, 31.2, 30, 29, 28, 13 }; + double[] expectedMultipliers = new double[] { 1.2, 1.2, 1.2, 1, 1, 1 }; + double[] expectedSlopes = new double[] { 0.637, 0.637, 0.637, 0.619, 0.614, 0.629 }; + double[] expectedIntercepts = new double[] { 3.091, 3.091, 3.091, 2.993, 2.905, 2.603 }; + long memSizeCapInBytes = (long) 100.0 * RBMSizeEstimator.BYTES_IN_MB; + double delta = 0.01; + for (int i = 0; i < logModulos.length; i++) { + int modulo = 0; + if (logModulos[i] != 0) { + modulo = (int) Math.pow(2, logModulos[i]); + } + RBMIntKeyLookupStore kls = new RBMIntKeyLookupStore(modulo, memSizeCapInBytes); + assertEquals(kls.stats.memSizeCapInBytes, kls.getMemorySizeCapInBytes(), 1.0); + assertEquals(expectedMultipliers[i], kls.sizeEstimator.bufferMultiplier, delta); + assertEquals(expectedSlopes[i], kls.sizeEstimator.slope, delta); + assertEquals(expectedIntercepts[i], kls.sizeEstimator.intercept, delta); + } + + } +}