From 8614d45497fa07353d0c650551a2e8b91418b2f3 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 23 Jan 2024 22:36:30 -0800 Subject: [PATCH] Implement weighted updates to KllFloatsSketch --- .../datasketches/kll/KllFloatsHelper.java | 73 ++++--- .../datasketches/kll/KllFloatsSketch.java | 25 +++ .../kll/KllFloatsSketchSortedView.java | 4 +- .../datasketches/kll/KllHeapFloatsSketch.java | 21 +- .../kll/KllDoublesValidationTest.java | 2 +- .../datasketches/kll/KllFloatsSketchTest.java | 4 +- .../datasketches/kll/KllMiscDoublesTest.java | 26 +-- .../datasketches/kll/KllMiscFloatsTest.java | 191 +++++++++++++++++- 8 files changed, 296 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java index d2a604a08..8c35fc66a 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsHelper.java @@ -24,6 +24,7 @@ import static org.apache.datasketches.common.Util.isEven; import static org.apache.datasketches.common.Util.isOdd; import static org.apache.datasketches.kll.KllHelper.findLevelToCompact; +import static org.apache.datasketches.kll.KllSketch.DEFAULT_M; import java.util.Arrays; import java.util.Random; @@ -39,6 +40,20 @@ // final class KllFloatsHelper { + /** + * Create Items Array from given item and weight. + * Used with weighted update only. + * @param item the given item + * @param weight the given weight + * @return the Items Array. + */ + static float[] createItemsArray(final float item, final int weight) { + final int itemsArrLen = Integer.bitCount(weight); + final float[] itemsArr = new float[itemsArrLen]; + Arrays.fill(itemsArr, item); + return itemsArr; + } + /** * The following code is only valid in the special case of exactly reaching capacity while updating. * It cannot be used while merging, while reducing k, or anything else. @@ -107,7 +122,7 @@ private static void compressWhileUpdatingSketch(final KllFloatsSketch fltSk) { fltSk.setFloatItemsArray(myFloatItemsArr); } - //assumes readOnly = false, and UPDATABLE, called from KllFloatsSketch::merge + //assumes readOnly = false and UPDATABLE, called from KllFloatsSketch::merge static void mergeFloatImpl(final KllFloatsSketch mySketch, final KllFloatsSketch otherFltSk) { if (otherFltSk.isEmpty()) { return; } @@ -140,7 +155,7 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final int[] myCurLevelsArr = mySketch.levelsArr; final float[] myCurFloatItemsArr = mySketch.getFloatItemsArray(); - // then rename them and initialize in case there are no higher levels + // create aliases in case there are no higher levels int myNewNumLevels = myCurNumLevels; int[] myNewLevelsArr = myCurLevelsArr; float[] myNewFloatItemsArr = myCurFloatItemsArr; @@ -150,12 +165,13 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, final int tmpSpaceNeeded = mySketch.getNumRetained() + KllHelper.getNumRetainedAboveLevelZero(otherNumLevels, otherLevelsArr); final float[] workbuf = new float[tmpSpaceNeeded]; - final int ub = KllHelper.ubOnNumLevels(finalN); - final int[] worklevels = new int[ub + 2]; // ub+1 does not work - final int[] outlevels = new int[ub + 2]; final int provisionalNumLevels = max(myCurNumLevels, otherNumLevels); + final int ub = max(KllHelper.ubOnNumLevels(finalN), provisionalNumLevels); + final int[] worklevels = new int[ub + 2]; // ub+1 does not work + final int[] outlevels = new int[ub + 2]; + populateFloatWorkArrays(workbuf, worklevels, provisionalNumLevels, myCurNumLevels, myCurLevelsArr, myCurFloatItemsArr, otherNumLevels, otherLevelsArr, otherFloatItemsArr); @@ -199,7 +215,7 @@ static void mergeFloatImpl(final KllFloatsSketch mySketch, KllHelper.memorySpaceMgmt(mySketch, myNewLevelsArr.length, myNewFloatItemsArr.length); mySketch.setWritableMemory(wmem); } - } + } //end of updating levels above level 0 //Update Preamble: mySketch.setN(finalN); @@ -298,31 +314,34 @@ private static void randomlyHalveUpFloats(final float[] buf, final int start, fi } } - //Called from KllFloatsSketch::update and this - static void updateFloat(final KllFloatsSketch fltSk, - final float item) { - if (Float.isNaN(item)) { return; } //ignore - if (fltSk.isEmpty()) { - fltSk.setMinItem(item); - fltSk.setMaxItem(item); - } else { - fltSk.setMinItem(min(fltSk.getMinItem(), item)); - fltSk.setMaxItem(max(fltSk.getMaxItem(), item)); - } - int level0space = fltSk.levelsArr[0]; - assert level0space >= 0; - if (level0space == 0) { + //Called from KllFloatsSketch::update and merge + static void updateFloat(final KllFloatsSketch fltSk, final float item) { + fltSk.updateMinMax(item); + int freeSpace = fltSk.levelsArr[0]; + assert freeSpace >= 0; + if (freeSpace == 0) { compressWhileUpdatingSketch(fltSk); - level0space = fltSk.levelsArr[0]; - assert (level0space > 0); + freeSpace = fltSk.levelsArr[0]; + assert (freeSpace > 0); } fltSk.incN(); fltSk.setLevelZeroSorted(false); - final int nextPos = level0space - 1; + final int nextPos = freeSpace - 1; fltSk.setLevelsArrayAt(0, nextPos); fltSk.setFloatItemsArrayAt(nextPos, item); } + //Called from KllFloatsSketch::update with weight + static void updateFloat(final KllFloatsSketch fltSk, final float item, final int weight) { + if (weight < fltSk.levelsArr[0]) { + for (int i = 0; i < weight; i++) { updateFloat(fltSk, item); } + } else { + fltSk.updateMinMax(item); + final KllHeapFloatsSketch tmpSk = new KllHeapFloatsSketch(fltSk.getK(), DEFAULT_M, item, weight); + fltSk.merge(tmpSk); + } + } + /** * Compression algorithm used to merge higher levels. *

Here is what we do for each level:

@@ -453,6 +472,7 @@ private static void populateFloatWorkArrays( worklevels[0] = 0; // Note: the level zero data from "other" was already inserted into "self" + // This copies into workbuf. final int selfPopZero = KllHelper.currentLevelSizeItems(0, myCurNumLevels, myCurLevelsArr); System.arraycopy( myCurFloatItemsArr, myCurLevelsArr[0], workbuf, worklevels[0], selfPopZero); worklevels[1] = worklevels[0] + selfPopZero; @@ -462,11 +482,14 @@ private static void populateFloatWorkArrays( final int otherPop = KllHelper.currentLevelSizeItems(lvl, otherNumLevels, otherLevelsArr); worklevels[lvl + 1] = worklevels[lvl] + selfPop + otherPop; + if (selfPop == 0 && otherPop == 0) { continue; } if (selfPop > 0 && otherPop == 0) { System.arraycopy(myCurFloatItemsArr, myCurLevelsArr[lvl], workbuf, worklevels[lvl], selfPop); - } else if (selfPop == 0 && otherPop > 0) { + } + else if (selfPop == 0 && otherPop > 0) { System.arraycopy(otherFloatItemsArr, otherLevelsArr[lvl], workbuf, worklevels[lvl], otherPop); - } else if (selfPop > 0 && otherPop > 0) { + } + else if (selfPop > 0 && otherPop > 0) { mergeSortedFloatArrays( myCurFloatItemsArr, myCurLevelsArr[lvl], selfPop, otherFloatItemsArr, otherLevelsArr[lvl], otherPop, diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java index 59dc183e6..bde606ea2 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketch.java @@ -319,11 +319,26 @@ public String toString(final boolean withLevels, final boolean withLevelsAndItem @Override public void update(final float item) { + if (Float.isNaN(item)) { return; } //ignore if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } KllFloatsHelper.updateFloat(this, item); kllFloatsSV = null; } + /** + * Weighted update. Updates this sketch with the given item the number of times specified by the given integer weight. + * @param item the item to be repeated. NaNs are ignored. + * @param weight the number of times the update of item is to be repeated. It must be ≥ one. + */ + public void update(final float item, final int weight) { + if (Float.isNaN(item)) { return; } //ignore + if (readOnly) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } + if (weight < 1) { throw new SketchesArgumentException("Weight is less than one."); } + if (weight == 1) { KllFloatsHelper.updateFloat(this, item); } + else { KllFloatsHelper.updateFloat(this, item, weight); } + kllFloatsSV = null; + } + //restricted /** @@ -390,4 +405,14 @@ private final void refreshSortedView() { abstract void setMinItem(float item); + void updateMinMax(final float item) { + if (isEmpty()) { + setMinItem(item); + setMaxItem(item); + } else { + setMinItem(min(getMinItem(), item)); + setMaxItem(max(getMaxItem(), item)); + } + } + } diff --git a/src/main/java/org/apache/datasketches/kll/KllFloatsSketchSortedView.java b/src/main/java/org/apache/datasketches/kll/KllFloatsSketchSortedView.java index db13bc19e..fc64c0d1b 100644 --- a/src/main/java/org/apache/datasketches/kll/KllFloatsSketchSortedView.java +++ b/src/main/java/org/apache/datasketches/kll/KllFloatsSketchSortedView.java @@ -83,8 +83,6 @@ public KllFloatsSketchSortedView(final KllFloatsSketch sketch) { populateFromSketch(srcQuantiles, srcLevels, srcNumLevels, numQuantiles); } - //end of constructors - @Override public long[] getCumulativeWeights() { return cumWeights.clone(); @@ -114,7 +112,7 @@ public float getQuantile(final double rank, final QuantileSearchCriteria searchC final InequalitySearch crit = (searchCrit == INCLUSIVE) ? InequalitySearch.GE : InequalitySearch.GT; final int index = InequalitySearch.find(cumWeights, 0, len - 1, naturalRank, crit); if (index == -1) { - return quantiles[quantiles.length - 1]; //EXCLUSIVE (GT) case: normRank == 1.0; + return quantiles[len - 1]; //EXCLUSIVE (GT) case: normRank == 1.0; } return quantiles[index]; } diff --git a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java index 9b2595b96..4388354e2 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllHeapFloatsSketch.java @@ -81,6 +81,25 @@ final class KllHeapFloatsSketch extends KllFloatsSketch { this.floatItems = new float[k]; } + /** + * Used for creating a temporary sketch for use with weighted updates. + */ + KllHeapFloatsSketch(final int k, final int m, final float item, final int weight) { + super(UPDATABLE); + KllHelper.checkM(m); + KllHelper.checkK(k, m); + this.levelsArr = KllHelper.createLevelsArray(weight); + this.readOnly = false; + this.k = k; + this.m = m; + this.n = weight; + this.minK = k; + this.isLevelZeroSorted = false; + this.minFloatItem = item; + this.maxFloatItem = item; + this.floatItems = KllFloatsHelper.createItemsArray(item, weight); + } + /** * Heapify constructor. * @param srcMem Memory object that contains data serialized by this sketch. @@ -280,6 +299,6 @@ float[] getFloatRetainedItemsArray() { } @Override - void setWritableMemory(final WritableMemory wmem) { } //inheritance dummy + void setWritableMemory(final WritableMemory wmem) { } } diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesValidationTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesValidationTest.java index 41530f55b..42802ba65 100644 --- a/src/test/java/org/apache/datasketches/kll/KllDoublesValidationTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllDoublesValidationTest.java @@ -160,7 +160,7 @@ public class KllDoublesValidationTest { private static int[] makeInputArray(int n, int stride) { assert isOdd(stride); - int mask = (1 << 23) - 1; + int mask = (1 << 23) - 1; // because library items are single-precision floats int cur = 0; int[] arr = new int[n]; for (int i = 0; i < n; i++) { diff --git a/src/test/java/org/apache/datasketches/kll/KllFloatsSketchTest.java b/src/test/java/org/apache/datasketches/kll/KllFloatsSketchTest.java index 846965cb8..243dd8327 100644 --- a/src/test/java/org/apache/datasketches/kll/KllFloatsSketchTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllFloatsSketchTest.java @@ -163,8 +163,8 @@ public void manyValuesEstimationMode() { assertEquals(pmf[0], 0.5, PMF_EPS_FOR_K_256); assertEquals(pmf[1], 0.5, PMF_EPS_FOR_K_256); - assertEquals(sketch.getMinItem(), 0f); - assertEquals(sketch.getMaxItem(), n - 1f); + assertEquals(sketch.getMinItem(), 0f); // min value is exact + assertEquals(sketch.getMaxItem(), n - 1f); // max value is exact // check at every 0.1 percentage point final double[] fractions = new double[1001]; diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java index e84d4ecb5..c331c02af 100644 --- a/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllMiscDoublesTest.java @@ -173,23 +173,23 @@ public void visualCheckToString() { public void viewHeapCompactions() { int k = 20; int n = 108; - boolean withSummary = false; - boolean withDetail = true; + boolean withLevels = false; + boolean withLevelsAndItems = true; int compaction = 0; KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(k); for (int i = 1; i <= n; i++) { sk.update(i); if (sk.levelsArr[0] == 0) { println(LS + "#<<< BEFORE COMPACTION # " + (++compaction) + " >>>"); - println(sk.toString(withSummary, withDetail)); + println(sk.toString(withLevels, withLevelsAndItems)); sk.update(++i); println(LS + "#<<< AFTER COMPACTION # " + (compaction) + " >>>"); - println(sk.toString(withSummary, withDetail)); + println(sk.toString(withLevels, withLevelsAndItems)); assertEquals(sk.getDoubleItemsArray()[sk.levelsArr[0]], i); } } println(LS + "#<<< END STATE # >>>"); - println(sk.toString(withSummary, withDetail)); + println(sk.toString(withLevels, withLevelsAndItems)); println(""); } @@ -197,8 +197,8 @@ public void viewHeapCompactions() { public void viewDirectCompactions() { int k = 20; int n = 108; - boolean withSummary = false; - boolean withDetail = true; + boolean withLevels = false; + boolean withLevelsAndItems = true; int compaction = 0; int sizeBytes = KllSketch.getMaxSerializedSizeBytes(k, n, DOUBLES_SKETCH, true); WritableMemory wmem = WritableMemory.allocate(sizeBytes); @@ -207,15 +207,15 @@ public void viewDirectCompactions() { sk.update(i); if (sk.levelsArr[0] == 0) { println(LS + "#<<< BEFORE COMPACTION # " + (++compaction) + " >>>"); - println(sk.toString(withSummary, withDetail)); + println(sk.toString(withLevels, withLevelsAndItems)); sk.update(++i); println(LS + "#<<< AFTER COMPACTION # " + (compaction) + " >>>"); - println(sk.toString(withSummary, withDetail)); + println(sk.toString(withLevels, withLevelsAndItems)); assertEquals(sk.getDoubleItemsArray()[sk.levelsArr[0]], i); } } println(LS + "#<<< END STATE # >>>"); - println(sk.toString(withSummary, withDetail)); + println(sk.toString(withLevels, withLevelsAndItems)); println(""); } @@ -341,14 +341,14 @@ private static void outputLevels(int weight, int[] levelsArr) { public void viewMemorySketchData() { int k = 20; int n = 109; - boolean withSummary = true; - boolean withDetail = true; + boolean withLevels = true; + boolean withLevelsAndItems = true; KllDoublesSketch sk = KllDoublesSketch.newHeapInstance(k); for (int i = 1; i <= n; i++) { sk.update(i); } byte[] byteArr = sk.toByteArray(); Memory mem = Memory.wrap(byteArr); KllDoublesSketch ddSk = KllDoublesSketch.wrap(mem); - println(ddSk.toString(withSummary, withDetail)); + println(ddSk.toString(withLevels, withLevelsAndItems)); assertEquals(ddSk.getN(), n); } diff --git a/src/test/java/org/apache/datasketches/kll/KllMiscFloatsTest.java b/src/test/java/org/apache/datasketches/kll/KllMiscFloatsTest.java index 6cc495575..202f2c794 100644 --- a/src/test/java/org/apache/datasketches/kll/KllMiscFloatsTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllMiscFloatsTest.java @@ -19,7 +19,10 @@ package org.apache.datasketches.kll; +import static org.apache.datasketches.common.Util.bitAt; +import static org.apache.datasketches.kll.KllHelper.getGrowthSchemeForGivenN; import static org.apache.datasketches.kll.KllSketch.SketchType.FLOATS_SKETCH; +import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -27,6 +30,7 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.kll.KllDirectFloatsSketch.KllDirectCompactFloatsSketch; +import org.apache.datasketches.kll.KllSketch.SketchType; import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.memory.MemoryRequestServer; @@ -38,6 +42,7 @@ /** * @author Lee Rhodes */ +@SuppressWarnings("unused") public class KllMiscFloatsTest { static final String LS = System.getProperty("line.separator"); private final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); @@ -168,37 +173,50 @@ public void visualCheckToString() { public void viewHeapCompactions() { int k = 20; int n = 108; + boolean withLevels = false; + boolean withLevelsAndItems = true; int compaction = 0; KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(k); for (int i = 1; i <= n; i++) { sk.update(i); if (sk.levelsArr[0] == 0) { println(LS + "#<<< BEFORE COMPACTION # " + (++compaction) + " >>>"); - println(sk.toString(true, true)); + println(sk.toString(withLevels, withLevelsAndItems)); sk.update(++i); println(LS + "#<<< AFTER COMPACTION # " + (compaction) + " >>>"); - println(sk.toString(true, true)); + println(sk.toString(withLevels, withLevelsAndItems)); assertEquals(sk.getFloatItemsArray()[sk.levelsArr[0]], i); } } + println(LS + "#<<< END STATE # >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + println(""); } @Test //set static enablePrinting = true for visual checking public void viewDirectCompactions() { int k = 20; int n = 108; + boolean withLevels = false; + boolean withLevelsAndItems = true; + int compaction = 0; int sizeBytes = KllSketch.getMaxSerializedSizeBytes(k, n, FLOATS_SKETCH, true); WritableMemory wmem = WritableMemory.allocate(sizeBytes); KllFloatsSketch sk = KllFloatsSketch.newDirectInstance(k, wmem, memReqSvr); for (int i = 1; i <= n; i++) { sk.update(i); if (sk.levelsArr[0] == 0) { - println(sk.toString(true, true)); + println(LS + "#<<< BEFORE COMPACTION # " + (++compaction) + " >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); sk.update(++i); - println(sk.toString(true, true)); + println(LS + "#<<< AFTER COMPACTION # " + (compaction) + " >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); assertEquals(sk.getFloatItemsArray()[sk.levelsArr[0]], i); } } + println(LS + "#<<< END STATE # >>>"); + println(sk.toString(withLevels, withLevelsAndItems)); + println(""); } @Test //set static enablePrinting = true for visual checking @@ -210,14 +228,177 @@ public void viewCompactionAndSortedView() { FloatsSortedView sv = sk.getSortedView(); FloatsSortedViewIterator itr = sv.iterator(); println("### SORTED VIEW"); - printf("%12s%12s\n", "Value", "CumWeight"); + printf("%12s%12s\n", "Value", "Weight"); + long[] correct = {2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}; + int i = 0; while (itr.next()) { float v = itr.getQuantile(); long wt = itr.getWeight(); printf("%12.1f%12d\n", v, wt); + assertEquals(wt, correct[i++]); + } + } + + @Test //set static enablePrinting = true for visual checking + public void checkWeightedUpdates1() { + int k = 20; + int weight = 127; + float item = 10.0F; + KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(k); + println(sk.toString(true, true)); + sk.update(item, weight); + println(sk.toString(true, true)); + assertEquals(sk.getNumRetained(), 7); + assertEquals(sk.getN(), weight); + sk.update(item, weight); + println(sk.toString(true, true)); + assertEquals(sk.getNumRetained(), 14); + assertEquals(sk.getN(), 254); + } + + @Test //set static enablePrinting = true for visual checking + public void checkWeightedUpdates2() { + int k = 20; + int initial = 1000; + int weight = 127; + float item = 10.0F; + KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(k); + for (int i = 1; i <= initial; i++) { sk.update(i + 1000); } + println(sk.toString(true, true)); + sk.update(item, weight); + println(sk.toString(true, true)); + assertEquals(sk.getNumRetained(), 65); + assertEquals(sk.getN(), 1127); + + FloatsSortedViewIterator itr = sk.getSortedView().iterator(); + println("### SORTED VIEW"); + printf("%12s %12s %12s\n", "Value", "Weight", "NaturalRank"); + long cumWt = 0; + while (itr.next()) { + double v = itr.getQuantile(); + long wt = itr.getWeight(); + long natRank = itr.getNaturalRank(INCLUSIVE); + cumWt += wt; + assertEquals(cumWt, natRank); + printf("%12.1f %12d %12d\n", v, wt, natRank); + } + assertEquals(cumWt, sk.getN()); + } + + @Test //set static enablePrinting = true for visual checking + public void checkCreateItemsArray() { //used with weighted updates + float item = 10.0F; + int weight = 108; + float[] itemsArr = KllFloatsHelper.createItemsArray(item, weight); + assertEquals(itemsArr.length, 4); + for (int i = 0; i < itemsArr.length; i++) { itemsArr[i] = item; } + outputItems(itemsArr); + } + + private static void outputItems(float[] itemsArr) { + String[] hdr2 = {"Index", "Value"}; + String hdr2fmt = "%6s %15s\n"; + String d2fmt = "%6d %15f\n"; + println("ItemsArr"); + printf(hdr2fmt, (Object[]) hdr2); + for (int i = 0; i < itemsArr.length; i++) { + printf(d2fmt, i, itemsArr[i]); } + println(""); } + @Test //set static enablePrinting = true for visual checking + public void checkCreateLevelsArray() { //used with weighted updates + int weight = 108; + int[] levelsArr = KllHelper.createLevelsArray(weight); + assertEquals(levelsArr.length, 8); + int[] correct = {0,0,0,1,2,2,3,4}; + for (int i = 0; i < levelsArr.length; i++) { + assertEquals(levelsArr[i], correct[i]); + } + outputLevels(weight, levelsArr); + } + + private static void outputLevels(int weight, int[] levelsArr) { + String[] hdr = {"Lvl", "StartAdr", "BitPattern", "Weight"}; + String hdrfmt = "%3s %9s %10s %s\n"; + String dfmt = "%3d %9d %10d %d\n"; + String dfmt_2 = "%3d %9d %s\n"; + println("Count = " + weight + " => " + (Integer.toBinaryString(weight))); + println("LevelsArr"); + printf(hdrfmt, (Object[]) hdr); + for (int i = 0; i < levelsArr.length; i++) { + if (i == levelsArr.length - 1) { printf(dfmt_2, i, levelsArr[i], "ItemsArr.length"); } + else { + int j = bitAt(weight, i); + printf(dfmt, i, levelsArr[i], j, 1 << (i)); + } + } + println(""); + } + + @Test + public void viewMemorySketchData() { + int k = 20; + int n = 109; + boolean withLevels = true; + boolean withLevelsAndItems = true; + KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(k); + for (int i = 1; i <= n; i++) { sk.update(i); } + byte[] byteArr = sk.toByteArray(); + Memory mem = Memory.wrap(byteArr); + KllFloatsSketch fltSk = KllFloatsSketch.wrap(mem); + println(fltSk.toString(withLevels, withLevelsAndItems)); + assertEquals(fltSk.getN(), n); + } + + @Test //set static enablePrinting = true for visual checking + public void checkIntCapAux() { + String[] hdr = {"level", "depth", "wt", "cap", "(end)", "MaxN"}; + String hdrFmt = "%6s %6s %28s %10s %10s %34s\n"; + String dataFmt = "%6d %6d %,28d %,10d %,10d %,34.0f\n"; + int k = 1000; + int m = 8; + int numLevels = 20; + println("k=" + k + ", m=" + m + ", numLevels=" + numLevels); + printf(hdrFmt, (Object[]) hdr); + double maxN = 0; + double[] correct = {0,1,1,2,2,3,5,8,12,17,26,39,59,88,132,198,296,444,667,1000}; + for (int i = 0; i < numLevels; i++) { + int depth = numLevels - i - 1; + long cap = KllHelper.intCapAux(k, depth); + long end = Math.max(m, cap); + long wt = 1L << i; + maxN += (double)wt * (double)end; + printf(dataFmt, i, depth, wt, cap, end, maxN); + assertEquals(cap, correct[i]); + } + } + + @Test //set static enablePrinting = true for visual checking + public void checkIntCapAuxAux() { + String[] hdr = {"d","twoK","2k*2^d","3^d","tmp=2k*2^d/3^d","(tmp + 1)/2", "(end)"}; + String hdrFmt = "%6s %10s %20s %20s %15s %12s %10s\n"; + String dataFmt = "%6d %10d %,20d %,20d %15d %12d %10d\n"; + long k = (1L << 16) - 1L; + long m = 8; + println("k = " + k + ", m = " + m); + printf(hdrFmt, (Object[]) hdr); + long[] correct = + {65535,43690,29127,19418,12945,8630,5753,3836,2557,1705,1136,758,505,337,224,150,100,67,44,30,20,13,9,6,4,3,2,1,1,1,0}; + for (int i = 0; i < 31; i++) { + long twoK = k << 1; + long twoKxtwoD = twoK << i; + long threeToD = KllHelper.powersOfThree[i]; + long tmp = twoKxtwoD / threeToD; + long result = (tmp + 1L) >>> 1; + long end = Math.max(m, result); //performed later + printf(dataFmt, i, twoK, twoKxtwoD, threeToD, tmp, result, end); + assertEquals(result,correct[i]); + assertEquals(result, KllHelper.intCapAuxAux(k, i)); + } + } + @Test public void checkGrowLevels() { KllFloatsSketch sk = KllFloatsSketch.newHeapInstance(20);