Examples with WritableHandle org.apache.datasketches.memory.WritableHandle used on opensource projects

Example 1 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class DirectCouponListTest method promotions.

private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, boolean compact, CurMode tgtMode) {
    int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
    HllSketch hllSketch;
    // println("DIRECT");
    byte[] barr1;
    WritableMemory wmem = null;
    try (WritableHandle hand = WritableMemory.allocateDirect(bytes)) {
        wmem = hand.getWritable();
        // byte[] byteArr = new byte[bytes];
        // WritableMemory wmem = WritableMemory.wrap(byteArr);
        hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem);
        assertTrue(hllSketch.isEmpty());
        for (int i = 0; i < n; i++) {
            hllSketch.update(i);
        }
        // println(hllSketch.toString(true, true, false, false));
        assertFalse(hllSketch.isEmpty());
        assertEquals(hllSketch.getCurMode(), tgtMode);
        assertTrue(hllSketch.isMemory());
        assertTrue(hllSketch.isOffHeap());
        assertTrue(hllSketch.isSameResource(wmem));
        // convert direct sketch to byte[]
        barr1 = (compact) ? hllSketch.toCompactByteArray() : hllSketch.toUpdatableByteArray();
        // println(PreambleUtil.toString(barr1));
        hllSketch.reset();
        assertTrue(hllSketch.isEmpty());
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
    // println("HEAP");
    HllSketch hllSketch2 = new HllSketch(lgConfigK, tgtHllType);
    for (int i = 0; i < n; i++) {
        hllSketch2.update(i);
    }
    // println(hllSketch2.toString(true, true, false, false));
    // println(PreambleUtil.toString(barr2));
    assertEquals(hllSketch2.getCurMode(), tgtMode);
    assertFalse(hllSketch2.isMemory());
    assertFalse(hllSketch2.isOffHeap());
    assertFalse(hllSketch2.isSameResource(wmem));
    byte[] barr2 = (compact) ? hllSketch2.toCompactByteArray() : hllSketch2.toUpdatableByteArray();
    assertEquals(barr1.length, barr2.length, barr1.length + ", " + barr2.length);
    // printDiffs(barr1, barr2);
    assertEquals(barr1, barr2);
}

Example 2 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class DirectQuantilesMemoryRequestTest method checkLimitedMemoryScenarios.

@Test
public void checkLimitedMemoryScenarios() {
    // Requesting application
    final int k = 128;
    final int u = 40 * k;
    // just the BB
    final int initBytes = ((2 * k) + 4) << 3;
    // This part would actually be part of the Memory owning implemention so it is faked here
    try (WritableHandle wdh = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
        final WritableMemory wmem = wdh.getWritable();
        println("Initial mem size: " + wmem.getCapacity());
        // ########## Receiving Application
        // The receiving application has been given wmem to use for a sketch,
        // but alas, it is not ultimately large enough.
        final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem);
        assertTrue(usk1.isEmpty());
        // Load the sketch
        for (int i = 0; i < u; i++) {
            // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as
            // needed, and requests via the MemoryRequest to free the old allocations.
            usk1.update(i);
        }
        final double result = usk1.getQuantile(0.5);
        println("Result: " + result);
        // Success
        assertEquals(result, u / 2.0, 0.05 * u);
        // ########## Owning Implementation
        // The actual Memory has been re-allocated several times,
        // so the above wmem reference is invalid.
        println("\nFinal mem size: " + wmem.getCapacity());
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 3 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class UnionImplTest method checkUnionCompactOrderedSource.

@Test
public void checkUnionCompactOrderedSource() {
    final int k = 1 << 12;
    final UpdateSketch sk = Sketches.updateSketchBuilder().build();
    for (int i = 0; i < k; i++) {
        sk.update(i);
    }
    final double est1 = sk.getEstimate();
    final int bytes = Sketches.getMaxCompactSketchBytes(sk.getRetainedEntries(true));
    try (WritableHandle h = WritableMemory.allocateDirect(bytes)) {
        final WritableMemory wmem = h.getWritable();
        // ordered, direct
        final CompactSketch csk = sk.compact(true, wmem);
        final Union union = Sketches.setOperationBuilder().buildUnion();
        union.union(csk);
        final double est2 = union.getResult().getEstimate();
        assertEquals(est2, est1);
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}

Example 4 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class UnionImplTest method checkMoveAndResize.

@Test
public void checkMoveAndResize() {
    final int k = 1 << 12;
    final int u = 2 * k;
    final int bytes = Sketches.getMaxUpdateSketchBytes(k);
    try (WritableHandle wh = WritableMemory.allocateDirect(bytes / 2);
        WritableHandle wh2 = WritableMemory.allocateDirect(bytes / 2)) {
        final WritableMemory wmem = wh.getWritable();
        final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
        assertTrue(sketch.isSameResource(wmem));
        final WritableMemory wmem2 = wh2.getWritable();
        final Union union = SetOperation.builder().buildUnion(wmem2);
        assertTrue(union.isSameResource(wmem2));
        for (int i = 0; i < u; i++) {
            union.update(i);
        }
        assertFalse(union.isSameResource(wmem));
        // on-heap union
        final Union union2 = SetOperation.builder().buildUnion();
        // obviously not
        assertFalse(union2.isSameResource(wmem2));
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}

Example 5 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class HeapifyWrapSerVer1and2Test method checkWrapCompactSketchAssumedDefaultSeed.

@Test
public void checkWrapCompactSketchAssumedDefaultSeed() {
    final int k = 64;
    final long seed = DEFAULT_UPDATE_SEED;
    final short seedHash = Util.computeSeedHash(seed);
    UpdateSketch sv3usk = UpdateSketch.builder().setNominalEntries(k).setSeed(seed).build();
    for (int i = 0; i < k; i++) {
        sv3usk.update(i);
    }
    CompactSketch sv3cskResult;
    WritableHandle wh;
    CompactSketch sv3csk = sv3usk.compact();
    // SV3 test
    wh = putOffHeap(Memory.wrap(sv3csk.toByteArray()));
    sv3cskResult = Sketches.wrapCompactSketch(wh.getWritable());
    assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate());
    assertEquals(sv3cskResult.getSeedHash(), seedHash);
    assertTrue(sv3cskResult.isDirect());
    try {
        wh.close();
    } catch (Exception e) {
    }
    // SV2 test
    wh = putOffHeap(BackwardConversions.convertSerVer3toSerVer2(sv3csk, seed));
    sv3cskResult = Sketches.wrapCompactSketch(wh.getWritable());
    assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate());
    assertEquals(sv3cskResult.getSeedHash(), seedHash);
    assertFalse(sv3cskResult.isDirect());
    try {
        wh.close();
    } catch (Exception e) {
    }
    // SV1 test
    wh = putOffHeap(BackwardConversions.convertSerVer3toSerVer1(sv3csk));
    sv3cskResult = Sketches.wrapCompactSketch(wh.getWritable());
    assertEquals(sv3cskResult.getEstimate(), sv3usk.getEstimate());
    assertEquals(sv3cskResult.getSeedHash(), seedHash);
    assertFalse(sv3cskResult.isDirect());
    try {
        wh.close();
    } catch (Exception e) {
    }
}