Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

These tests were attempting to #602

Merged
merged 1 commit into from
Sep 27, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -43,21 +43,24 @@
public class DirectAuxHashMapTest {

@Test
public void checkGrow() {
public void checkGrow() { //It is very rare, but this forces an HLL_4 to exceed its computed memory size.
int lgConfigK = 4;
TgtHllType tgtHllType = TgtHllType.HLL_4;
int n = 8; //put lgConfigK == 4 into HLL mode
int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
HllSketch hllSketch;
WritableMemory wmem = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
WritableMemory wmemCopy = wmem; //copy of wmem
assertTrue(wmemCopy.isDirect()); //original copy of wmem is off-heap
assertTrue(wmemCopy.isAlive()); //original copy of wmem is Alive
hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
for (int i = 0; i < n; i++) {
hllSketch.update(i);
}
hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
hllSketch.couponUpdate(HllUtil.pair(8, 15));
hllSketch.couponUpdate(HllUtil.pair(9, 15));
//println(hllSketch.toString(true, true, true, true));
//println(hllSketch.toString(true, true, true, true)); //
DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
assertTrue(hllSketch.isMemory());
@@ -75,19 +78,21 @@ public void checkGrow() {
byteArray = hllSketch.toUpdatableByteArray();
WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
hllSketch2 = HllSketch.writableWrap(wmem2);
//println(hllSketch2.toString(true, true, true, true));
//println(hllSketch2.toString(true, true, true, true)); //
DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);

//Check grow to on-heap
hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
//println(hllSketch.toString(true, true, true, true));
//println(hllSketch.toString(true, true, true, true)); //
dha = (DirectHllArray) hllSketch.hllSketchImpl;
assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
assertTrue(hllSketch.isMemory());
assertFalse(hllSketch.isOffHeap());
assertTrue(wmemCopy.isDirect()); //original copy of wmem was off-heap and still is
assertFalse(wmemCopy.isAlive()); //original copy of wmem has been closed
}

@Test
Original file line number Diff line number Diff line change
@@ -48,6 +48,7 @@ public void checkLimitedMemoryScenarios() { //Requesting application
//########## Owning Implementation
// This part would actually be part of the Memory owning implementation so it is faked here
WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());

//########## Receiving Application
@@ -70,6 +71,8 @@ public void checkLimitedMemoryScenarios() { //Requesting application
// so the the wmem reference is invalid. Use the sketch to get the last memory reference.
WritableMemory lastMem = usk1.getMemory();
println("Final mem size: " + usk1.getMemory().getCapacity());
assertTrue(wmemCopy.isDirect());
assertFalse(wmemCopy.isAlive());
}

@Test
@@ -79,6 +82,7 @@ public void checkGrowBaseBuf() {
final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything

WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());
final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
for (int i = 1; i <= u; i++) {
@@ -88,6 +92,8 @@ public void checkGrowBaseBuf() {
println("curCombBufItemCap: " + currentSpace);
assertEquals(currentSpace, 2 * k);
println("last Mem Cap: " + usk1.getMemory().getCapacity());
assertTrue(wmemCopy.isDirect());
assertFalse(wmemCopy.isAlive());
}

@Test
@@ -97,6 +103,7 @@ public void checkGrowCombBuf() {
final int initBytes = ((2 * k) + 4) << 3; //just room for BB

WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
WritableMemory wmemCopy = wmem;
println("Initial mem size: " + wmem.getCapacity());
final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
for (int i = 1; i <= u; i++) {
@@ -108,6 +115,8 @@ public void checkGrowCombBuf() {
final int newSpace = usk1.getCombinedBufferItemCapacity();
println("newCombBurItemCap: " + newSpace);
assertEquals(newCB.length, 3 * k);
assertTrue(wmemCopy.isDirect());
assertFalse(wmemCopy.isAlive());
}

@Test
@@ -119,6 +128,7 @@ public void checkGrowFromWrappedEmptySketch() {
final Memory origSketchMem = Memory.wrap(usk1.toByteArray());

WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
WritableMemory wmemCopy = wmem;
origSketchMem.copyTo(0, wmem, 0, initBytes);
UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem);
assertTrue(wmem.isSameResource(usk2.getMemory()));
@@ -135,6 +145,8 @@ public void checkGrowFromWrappedEmptySketch() {

final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
assertEquals(mem2.getCapacity(), expectedSize);
assertTrue(wmemCopy.isDirect());
assertFalse(wmemCopy.isAlive());
}

@Test
Original file line number Diff line number Diff line change
@@ -140,18 +140,22 @@ public void checkEmptyExceptions() {
@Test
public void directSketchShouldMoveOntoHeapEventually() {
WritableMemory wmem = WritableMemory.allocateDirect(1000, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
WritableMemory wmemCopy = wmem;
UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
Assert.assertTrue(sketch.isSameResource(wmem));
for (int i = 0; i < 1000; i++) {
sketch.update(i);
}
println(sketch.toString());
assertTrue(wmemCopy.isDirect());
assertFalse(wmemCopy.isAlive());
}

@Test
public void directSketchShouldMoveOntoHeapEventually2() {
int i = 0;
WritableMemory wmem = WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer());
WritableMemory wmemCopy = wmem;
UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
Assert.assertTrue(sketch.isSameResource(wmem));
for (; i < 1000; i++) {
@@ -163,6 +167,8 @@ public void directSketchShouldMoveOntoHeapEventually2() {
}
}
assertFalse(wmem.isAlive());
assertTrue(wmemCopy.isDirect());
assertFalse(wmemCopy.isAlive());
}

@Test
Original file line number Diff line number Diff line change
@@ -781,13 +781,16 @@ public void checkMoveAndResize() {
int u = 2 * k;
int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory
WritableMemory wmemCopy = wmem;
UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
assertTrue(sketch.isSameResource(wmem));
for (int i = 0; i < u; i++) { sketch.update(i); }
Memory mem = sketch.getMemory();
assertTrue(mem.isAlive());
assertFalse(mem.isDirect()); //now on heap.
assertTrue(wmemCopy.isDirect()); //original copy
assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer
assertFalse(wmemCopy.isAlive()); //original copy closed
}

@Test
@@ -796,6 +799,7 @@ public void checkReadOnlyRebuildResize() {
int u = 2 * k;
int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory
WritableMemory wmemCopy = wmem;
UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
for (int i = 0; i < u; i++) { sketch.update(i); }
double est1 = sketch.getEstimate();
@@ -808,6 +812,8 @@ public void checkReadOnlyRebuildResize() {
assertTrue(mem2.isAlive());
assertFalse(mem2.isDirect()); //now on heap
assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer
assertTrue(wmemCopy.isDirect());
assertFalse(wmemCopy.isAlive());
try {
roSketch.rebuild();
fail();
Original file line number Diff line number Diff line change
@@ -193,6 +193,7 @@ public void checkMoveAndResizeOffHeap() {
final int bytes = Sketches.getMaxUpdateSketchBytes(k);
WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2); //not really used, except as a reference.
WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2); //too small, forces new allocation on heap
WritableMemory wmem2Copy = wmem2;
final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
assertTrue(sketch.isSameResource(wmem)); //also testing the isSameResource function

@@ -206,6 +207,8 @@ public void checkMoveAndResizeOffHeap() {
assertFalse(union2.isSameResource(wmem2)); //obviously not
wmem.close(); //empty, but we must close it anyway.
assertFalse(wmem2.isAlive());//previously closed via the DefaultMemoryRequestServer.
assertTrue(wmem2Copy.isDirect());
assertFalse(wmem2Copy.isAlive());
}

@Test
Loading