Merge pull request #602 from apache/fix_direct_overfill_tests

These tests were attempting to

src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java

@@ -43,21 +43,24 @@
 public class DirectAuxHashMapTest {
 
   @Test
-  public void checkGrow() {
+  public void checkGrow() { //It is very rare, but this forces an HLL_4 to exceed its computed memory size.
     int lgConfigK = 4;
     TgtHllType tgtHllType = TgtHllType.HLL_4;
     int n = 8; //put lgConfigK == 4 into HLL mode
     int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
     HllSketch hllSketch;
     WritableMemory wmem = WritableMemory.allocateDirect(bytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmemCopy = wmem;  //copy of wmem
+    assertTrue(wmemCopy.isDirect()); //original copy of wmem is off-heap
+    assertTrue(wmemCopy.isAlive());  //original copy of wmem is Alive
     hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
     for (int i = 0; i < n; i++) {
       hllSketch.update(i);
     }
     hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values
     hllSketch.couponUpdate(HllUtil.pair(8, 15));
     hllSketch.couponUpdate(HllUtil.pair(9, 15));
-    //println(hllSketch.toString(true, true, true, true));
+    //println(hllSketch.toString(true, true, true, true)); //
     DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl;
     assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2);
     assertTrue(hllSketch.isMemory());
@@ -75,19 +78,21 @@ public void checkGrow() {
     byteArray = hllSketch.toUpdatableByteArray();
     WritableMemory wmem2 = WritableMemory.writableWrap(byteArray);
     hllSketch2 = HllSketch.writableWrap(wmem2);
-    //println(hllSketch2.toString(true, true, true, true));
+    //println(hllSketch2.toString(true, true, true, true)); //
     DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl;
     assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2);
     assertEquals(dha2.getAuxHashMap().getAuxCount(), 3);
 
     //Check grow to on-heap
     hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow
-    //println(hllSketch.toString(true, true, true, true));
+    //println(hllSketch.toString(true, true, true, true)); //
     dha = (DirectHllArray) hllSketch.hllSketchImpl;
     assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3);
     assertEquals(dha.getAuxHashMap().getAuxCount(), 4);
     assertTrue(hllSketch.isMemory());
     assertFalse(hllSketch.isOffHeap());
+    assertTrue(wmemCopy.isDirect()); //original copy of wmem was off-heap and still is
+    assertFalse(wmemCopy.isAlive()); //original copy of wmem has been closed
   }
 
   @Test

src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java

@@ -48,6 +48,7 @@ public void checkLimitedMemoryScenarios() { //Requesting application
     //########## Owning Implementation
     // This part would actually be part of the Memory owning implementation so it is faked here
     WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmemCopy = wmem;
     println("Initial mem size: " + wmem.getCapacity());
 
     //########## Receiving Application
@@ -70,6 +71,8 @@ public void checkLimitedMemoryScenarios() { //Requesting application
     // so the the wmem reference is invalid. Use the sketch to get the last memory reference.
     WritableMemory lastMem = usk1.getMemory();
     println("Final mem size: " + usk1.getMemory().getCapacity());
+    assertTrue(wmemCopy.isDirect());
+    assertFalse(wmemCopy.isAlive());
   }
 
   @Test
@@ -79,6 +82,7 @@ public void checkGrowBaseBuf() {
     final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything
 
     WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmemCopy = wmem;
     println("Initial mem size: " + wmem.getCapacity());
     final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
     for (int i = 1; i <= u; i++) {
@@ -88,6 +92,8 @@ public void checkGrowBaseBuf() {
     println("curCombBufItemCap: " + currentSpace);
     assertEquals(currentSpace, 2 * k);
     println("last Mem Cap: " + usk1.getMemory().getCapacity());
+    assertTrue(wmemCopy.isDirect());
+    assertFalse(wmemCopy.isAlive());
   }
 
   @Test
@@ -97,6 +103,7 @@ public void checkGrowCombBuf() {
     final int initBytes = ((2 * k) + 4) << 3; //just room for BB
 
     WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmemCopy = wmem;
     println("Initial mem size: " + wmem.getCapacity());
     final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
     for (int i = 1; i <= u; i++) {
@@ -108,6 +115,8 @@ public void checkGrowCombBuf() {
     final int newSpace = usk1.getCombinedBufferItemCapacity();
     println("newCombBurItemCap: " + newSpace);
     assertEquals(newCB.length, 3 * k);
+    assertTrue(wmemCopy.isDirect());
+    assertFalse(wmemCopy.isAlive());
   }
 
   @Test
@@ -119,6 +128,7 @@ public void checkGrowFromWrappedEmptySketch() {
     final Memory origSketchMem = Memory.wrap(usk1.toByteArray());
 
     WritableMemory wmem = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmemCopy = wmem;
     origSketchMem.copyTo(0, wmem, 0, initBytes);
     UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem);
     assertTrue(wmem.isSameResource(usk2.getMemory()));
@@ -135,6 +145,8 @@ public void checkGrowFromWrappedEmptySketch() {
 
     final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
     assertEquals(mem2.getCapacity(), expectedSize);
+    assertTrue(wmemCopy.isDirect());
+    assertFalse(wmemCopy.isAlive());
   }
 
   @Test

src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java

@@ -140,18 +140,22 @@ public void checkEmptyExceptions() {
   @Test
   public void directSketchShouldMoveOntoHeapEventually() {
     WritableMemory wmem = WritableMemory.allocateDirect(1000, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer());
+    WritableMemory wmemCopy = wmem;
     UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
     Assert.assertTrue(sketch.isSameResource(wmem));
     for (int i = 0; i < 1000; i++) {
       sketch.update(i);
     }
     println(sketch.toString());
+    assertTrue(wmemCopy.isDirect());
+    assertFalse(wmemCopy.isAlive());
   }
 
   @Test
   public void directSketchShouldMoveOntoHeapEventually2() {
     int i = 0;
     WritableMemory wmem = WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer());
+    WritableMemory wmemCopy = wmem;
     UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem);
     Assert.assertTrue(sketch.isSameResource(wmem));
     for (; i < 1000; i++) {
@@ -163,6 +167,8 @@ public void directSketchShouldMoveOntoHeapEventually2() {
       }
     }
     assertFalse(wmem.isAlive());
+    assertTrue(wmemCopy.isDirect());
+    assertFalse(wmemCopy.isAlive());
   }
 
   @Test

src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java

@@ -781,13 +781,16 @@ public void checkMoveAndResize() {
     int u = 2 * k;
     int bytes = Sketches.getMaxUpdateSketchBytes(k);
     WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory
+    WritableMemory wmemCopy = wmem;
     UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
     assertTrue(sketch.isSameResource(wmem));
     for (int i = 0; i < u; i++) { sketch.update(i); }
     Memory mem = sketch.getMemory();
     assertTrue(mem.isAlive());
     assertFalse(mem.isDirect()); //now on heap.
+    assertTrue(wmemCopy.isDirect()); //original copy
     assertFalse(wmem.isAlive()); //wmem closed by MemoryRequestServer
+    assertFalse(wmemCopy.isAlive()); //original copy closed
   }
 
   @Test
@@ -796,6 +799,7 @@ public void checkReadOnlyRebuildResize() {
     int u = 2 * k;
     int bytes = Sketches.getMaxUpdateSketchBytes(k);
     WritableMemory wmem = WritableMemory.allocateDirect(bytes/2); //will request more memory
+    WritableMemory wmemCopy = wmem;
     UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
     for (int i = 0; i < u; i++) { sketch.update(i); }
     double est1 = sketch.getEstimate();
@@ -808,6 +812,8 @@ public void checkReadOnlyRebuildResize() {
     assertTrue(mem2.isAlive());
     assertFalse(mem2.isDirect()); //now on heap
     assertFalse(wmem.isAlive());  //wmem closed by MemoryRequestServer
+    assertTrue(wmemCopy.isDirect());
+    assertFalse(wmemCopy.isAlive());
     try {
       roSketch.rebuild();
       fail();

src/test/java/org/apache/datasketches/theta/UnionImplTest.java

@@ -193,6 +193,7 @@ public void checkMoveAndResizeOffHeap() {
     final int bytes = Sketches.getMaxUpdateSketchBytes(k);
     WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2); //not really used, except as a reference.
     WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2); //too small, forces new allocation on heap
+    WritableMemory wmem2Copy = wmem2;
     final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
     assertTrue(sketch.isSameResource(wmem)); //also testing the isSameResource function
 
@@ -206,6 +207,8 @@ public void checkMoveAndResizeOffHeap() {
     assertFalse(union2.isSameResource(wmem2));  //obviously not
     wmem.close(); //empty, but we must close it anyway.
     assertFalse(wmem2.isAlive());//previously closed via the DefaultMemoryRequestServer.
+    assertTrue(wmem2Copy.isDirect());
+    assertFalse(wmem2Copy.isAlive());
   }
 
   @Test