Examples with WritableHandle org.apache.datasketches.memory.WritableHandle used on opensource projects

Example 36 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class DebugUnionTest method test.

@Test
public void test() {
    final int n = 70_000;
    final int valueLimit = 1000;
    final int numSketches = 3;
    final int sketchK = 8;
    final int unionK = 8;
    UpdateDoublesSketch[] sketchArr = new UpdateDoublesSketch[numSketches];
    // builds the input sketches, all on heap
    // make deterministic for test
    DoublesSketch.setRandom(1);
    // holds input values
    final HashSet<Double> set = new HashSet<>();
    for (int s = 0; s < numSketches; s++) {
        sketchArr[s] = buildHeapSketch(sketchK, n, valueLimit, set);
    }
    // loads the on heap union
    // make deterministic for test
    DoublesSketch.setRandom(1);
    DoublesUnion hUnion = DoublesUnion.builder().setMaxK(unionK).build();
    for (int s = 0; s < numSketches; s++) {
        hUnion.update(sketchArr[s]);
    }
    DoublesSketch hSketch = hUnion.getResult();
    // loads the direct union
    // make deterministic for test
    DoublesSketch.setRandom(1);
    DoublesUnion dUnion;
    DoublesSketch dSketch;
    try (WritableHandle wdh = WritableMemory.allocateDirect(10_000_000)) {
        WritableMemory wmem = wdh.getWritable();
        dUnion = DoublesUnion.builder().setMaxK(8).build(wmem);
        for (int s = 0; s < numSketches; s++) {
            dUnion.update(sketchArr[s]);
        }
        // result is on heap
        dSketch = dUnion.getResult();
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
    // iterates and counts errors
    int hCount = hSketch.getRetainedItems();
    int dCount = dSketch.getRetainedItems();
    // Retained items must be the same
    assertEquals(hCount, dCount);
    int hErrors = 0;
    int dErrors = 0;
    DoublesSketchIterator hit = hSketch.iterator();
    DoublesSketchIterator dit = dSketch.iterator();
    while (hit.next() && dit.next()) {
        double v = hit.getValue();
        if (!set.contains(v)) {
            hErrors++;
        }
        double w = dit.getValue();
        if (!set.contains(w)) {
            dErrors++;
        }
        // Items must be returned in same order and be equal
        assertEquals(v, w, 0);
    }
    assertTrue(hErrors == 0);
    assertTrue(dErrors == 0);
    println("HeapUnion  : Values: " + hCount + ", errors: " + hErrors);
    // println(hSketch.toString(true, true));
    println("DirectUnion: Values: " + dCount + ", errors: " + dErrors);
// println(dSketch.toString(true, true));
}

Example 37 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class DirectQuantilesMemoryRequestTest method checkGrowBaseBuf.

@Test
public void checkGrowBaseBuf() {
    final int k = 128;
    // don't need the BB to fill here
    final int u = 32;
    // not enough to hold everything
    final int initBytes = (4 + (u / 2)) << 3;
    try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
        // final MemoryManager memMgr = new MemoryManager();
        // final WritableMemory mem1 = memMgr.request(initBytes);
        final WritableMemory mem1 = memHandler.getWritable();
        println("Initial mem size: " + mem1.getCapacity());
        final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1);
        for (int i = 1; i <= u; i++) {
            usk1.update(i);
        }
        final int currentSpace = usk1.getCombinedBufferItemCapacity();
        println("curCombBufItemCap: " + currentSpace);
        assertEquals(currentSpace, 2 * k);
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}

Example 38 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class DirectQuantilesMemoryRequestTest method checkGrowFromWrappedEmptySketch.

@Test
public void checkGrowFromWrappedEmptySketch() {
    final int k = 16;
    final int n = 0;
    // 8 bytes
    final int initBytes = DoublesSketch.getUpdatableStorageBytes(k, n);
    final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build();
    final Memory origSketchMem = Memory.wrap(usk1.toByteArray());
    try (WritableHandle memHandle = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
        WritableMemory mem = memHandle.getWritable();
        origSketchMem.copyTo(0, mem, 0, initBytes);
        UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem);
        assertTrue(mem.isSameResource(usk2.getMemory()));
        assertEquals(mem.getCapacity(), initBytes);
        assertTrue(mem.isDirect());
        assertTrue(usk2.isEmpty());
        // update the sketch forcing it to grow on-heap
        for (int i = 1; i <= 5; i++) {
            usk2.update(i);
        }
        assertEquals(usk2.getN(), 5);
        WritableMemory mem2 = usk2.getMemory();
        assertFalse(mem.isSameResource(mem2));
        // should now be on-heap
        assertFalse(mem2.isDirect());
        final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
        assertEquals(mem2.getCapacity(), expectedSize);
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}

Example 39 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class DirectQuantilesMemoryRequestTest method checkGrowCombBuf.

@Test
public void checkGrowCombBuf() {
    final int k = 128;
    // just to fill the BB
    final int u = (2 * k) - 1;
    // just room for BB
    final int initBytes = ((2 * k) + 4) << 3;
    try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) {
        // final MemoryManager memMgr = new MemoryManager();
        // final WritableMemory mem1 = memMgr.request(initBytes);
        final WritableMemory mem1 = memHandler.getWritable();
        println("Initial mem size: " + mem1.getCapacity());
        final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1);
        for (int i = 1; i <= u; i++) {
            usk1.update(i);
        }
        final int currentSpace = usk1.getCombinedBufferItemCapacity();
        println("curCombBufItemCap: " + currentSpace);
        final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k);
        final int newSpace = usk1.getCombinedBufferItemCapacity();
        println("newCombBurItemCap: " + newSpace);
        assertEquals(newCB.length, 3 * k);
    // memMgr.free(mem1);
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}

Example 40 with WritableHandle

use of org.apache.datasketches.memory.WritableHandle in project sketches-core by DataSketches.

the class DoublesSketchTest method checkEmptyDirect.

@Test
public void checkEmptyDirect() {
    try (WritableHandle wdh = WritableMemory.allocateDirect(1000)) {
        WritableMemory mem = wdh.getWritable();
        UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem);
        // exercises a specific path
        sketch.toByteArray();
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}