Examples with WritableMemory org.apache.datasketches.memory.WritableMemory used on opensource projects

Example 66 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class SketchesTest method checkSetOpMethods.

@Test
public void checkSetOpMethods() {
    final int k = 1024;
    final Memory mem1 = getCompactSketchMemory(k, 0, k);
    final Memory mem2 = getCompactSketchMemory(k, k / 2, 3 * k / 2);
    final SetOperationBuilder bldr = setOperationBuilder();
    final Union union = bldr.setNominalEntries(2 * k).buildUnion();
    union.union(mem1);
    CompactSketch cSk = union.getResult(true, null);
    assertEquals((int) cSk.getEstimate(), k);
    union.union(mem2);
    cSk = union.getResult(true, null);
    assertEquals((int) cSk.getEstimate(), 3 * k / 2);
    final byte[] ubytes = union.toByteArray();
    final WritableMemory uMem = WritableMemory.writableWrap(ubytes);
    Union union2 = (Union) heapifySetOperation(uMem);
    cSk = union2.getResult(true, null);
    assertEquals((int) cSk.getEstimate(), 3 * k / 2);
    union2 = (Union) heapifySetOperation(uMem, Util.DEFAULT_UPDATE_SEED);
    cSk = union2.getResult(true, null);
    assertEquals((int) cSk.getEstimate(), 3 * k / 2);
    union2 = (Union) wrapSetOperation(uMem);
    cSk = union2.getResult(true, null);
    assertEquals((int) cSk.getEstimate(), 3 * k / 2);
    union2 = (Union) wrapSetOperation(uMem, Util.DEFAULT_UPDATE_SEED);
    cSk = union2.getResult(true, null);
    assertEquals((int) cSk.getEstimate(), 3 * k / 2);
    final int serVer = getSerializationVersion(uMem);
    assertEquals(serVer, 3);
}

Example 67 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class UpdateSketchTest method checkIncompatibleFamily.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkIncompatibleFamily() {
    UpdateSketch sk = Sketches.updateSketchBuilder().build();
    sk.update(1);
    WritableMemory wmem = WritableMemory.writableWrap(sk.compact().toByteArray());
    UpdateSketch.wrap(wmem, DEFAULT_UPDATE_SEED);
}

Example 68 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class UpdateSketchTest method checkCompactOpsMemoryToCompact.

@SuppressWarnings("unused")
@Test
public void checkCompactOpsMemoryToCompact() {
    WritableMemory skwmem, cskwmem1, cskwmem2, cskwmem3;
    CompactSketch csk1, csk2, csk3;
    int lgK = 6;
    UpdateSketch sk = Sketches.updateSketchBuilder().setLogNominalEntries(lgK).build();
    int n = 1 << (lgK + 1);
    for (int i = 2; i < n; i++) {
        sk.update(i);
    }
    int cbytes = sk.getCompactBytes();
    byte[] byteArr = sk.toByteArray();
    skwmem = WritableMemory.writableWrap(byteArr);
    cskwmem1 = WritableMemory.allocate(cbytes);
    cskwmem2 = WritableMemory.allocate(cbytes);
    cskwmem3 = WritableMemory.allocate(cbytes);
    csk1 = sk.compact(true, cskwmem1);
    csk2 = CompactOperations.memoryToCompact(skwmem, true, cskwmem2);
    csk3 = CompactOperations.memoryToCompact(cskwmem1, true, cskwmem3);
    assertTrue(cskwmem1.equals(cskwmem2));
    assertTrue(cskwmem1.equals(cskwmem3));
}

Example 69 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectUnionTest method checkForDruidBug2.

@Test
public void checkForDruidBug2() {
    // update union with just sketch memory reference
    final int k = 16384;
    final UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build();
    for (int i = 0; i < 100000; i++) {
        usk.update(Integer.toString(i));
    }
    // optional but created the symptom
    usk.rebuild();
    final WritableMemory memIn = WritableMemory.allocate(usk.getCompactBytes());
    // side effect of loading the memIn
    usk.compact(true, memIn);
    // create empty target union in off-heap mem
    final WritableMemory mem = WritableMemory.writableWrap(new byte[getMaxUnionBytes(k)]);
    final Union union1 = SetOperation.builder().setNominalEntries(k).buildUnion(mem);
    union1.union(memIn);
    final CompactSketch csk = union1.getResult();
    assertTrue(csk.getTheta() < 0.2);
    assertEquals(csk.getRetainedEntries(true), 16384);
    final double est = csk.getEstimate();
    assertTrue(est > 98663.0);
    assertTrue(est < 101530.0);
}

Example 70 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectUnionTest method checkMultiUnion.

@Test
public void checkMultiUnion() {
    // 8192
    final int lgK = 13;
    final int k = 1 << lgK;
    final UpdateSketch usk1 = UpdateSketch.builder().setNominalEntries(k).build();
    final UpdateSketch usk2 = UpdateSketch.builder().setNominalEntries(k).build();
    final UpdateSketch usk3 = UpdateSketch.builder().setNominalEntries(k).build();
    final UpdateSketch usk4 = UpdateSketch.builder().setNominalEntries(k).build();
    int v = 0;
    int u = 1000000;
    for (int i = 0; i < u; i++) {
        usk1.update(i + v);
    }
    v += u;
    u = 26797;
    for (int i = 0; i < u; i++) {
        usk2.update(i + v);
    }
    v += u;
    for (int i = 0; i < u; i++) {
        usk3.update(i + v);
    }
    v += u;
    for (int i = 0; i < u; i++) {
        usk4.update(i + v);
    }
    v += u;
    // union memory
    final WritableMemory uMem = WritableMemory.writableWrap(new byte[getMaxUnionBytes(k)]);
    final Union union = SetOperation.builder().setNominalEntries(k).buildUnion(uMem);
    // updates with heap UpdateSketch
    union.union(usk1);
    // updates with heap UpdateSketch
    union.union(usk2);
    // updates with heap UpdateSketch
    union.union(usk3);
    // updates with heap UpdateSketch
    union.union(usk4);
    final CompactSketch csk = union.getResult(true, null);
    final double est = csk.getEstimate();
    assertEquals(est, v, .01 * v);
}