Examples with WritableMemory org.apache.datasketches.memory.WritableMemory used on opensource projects

Example 86 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class UnionImplTest method checkVer1FamilyException.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkVer1FamilyException() {
    final int k = 16;
    final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build();
    for (int i = 0; i < k; i++) {
        sketch.update(i);
    }
    final CompactSketch csk = sketch.compact(true, null);
    final WritableMemory v1mem = (WritableMemory) convertSerVer3toSerVer1(csk);
    // corrupt family
    v1mem.putByte(PreambleUtil.FAMILY_BYTE, (byte) 0);
    final Union union = Sketches.setOperationBuilder().setNominalEntries(k).buildUnion();
    union.union(v1mem);
}

Example 87 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class UnionImplTest method checkMoveAndResize.

@Test
public void checkMoveAndResize() {
    final int k = 1 << 12;
    final int u = 2 * k;
    final int bytes = Sketches.getMaxUpdateSketchBytes(k);
    try (WritableHandle wh = WritableMemory.allocateDirect(bytes / 2);
        WritableHandle wh2 = WritableMemory.allocateDirect(bytes / 2)) {
        final WritableMemory wmem = wh.getWritable();
        final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem);
        assertTrue(sketch.isSameResource(wmem));
        final WritableMemory wmem2 = wh2.getWritable();
        final Union union = SetOperation.builder().buildUnion(wmem2);
        assertTrue(union.isSameResource(wmem2));
        for (int i = 0; i < u; i++) {
            union.update(i);
        }
        assertFalse(union.isSameResource(wmem));
        // on-heap union
        final Union union2 = SetOperation.builder().buildUnion();
        // obviously not
        assertFalse(union2.isSameResource(wmem2));
    } catch (final Exception e) {
        throw new RuntimeException(e);
    }
}

Example 88 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectUnionTest method checkWrapEstNoOverlap.

@Test
public void checkWrapEstNoOverlap() {
    // 4096
    final int lgK = 12;
    final int k = 1 << lgK;
    final int u = 4 * k;
    // 2k estimating
    final UpdateSketch usk1 = UpdateSketch.builder().setNominalEntries(k).build();
    // 2k exact
    final UpdateSketch usk2 = UpdateSketch.builder().setNominalEntries(2 * k).build();
    for (int i = 0; i < u / 2; i++) {
        // 2k
        usk1.update(i);
    }
    for (int i = u / 2; i < u; i++) {
        // 2k no overlap, exact
        usk2.update(i);
    }
    final WritableMemory uMem = WritableMemory.writableWrap(new byte[getMaxUnionBytes(k)]);
    final Union union = SetOperation.builder().setNominalEntries(k).buildUnion(uMem);
    // update with heap UpdateSketch
    union.union(usk1);
    // update with heap UpdateSketch, early stop not possible
    union.union(usk2);
    testAllCompactForms(union, u, 0.05);
    final Union union2 = Sketches.wrapUnion(WritableMemory.writableWrap(union.toByteArray()));
    testAllCompactForms(union2, u, 0.05);
}

Example 89 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectUnionTest method checkDirectMemoryIn.

@Test
public void checkDirectMemoryIn() {
    // 4096
    final int lgK = 12;
    final int k = 1 << lgK;
    final int u1 = 2 * k;
    // smaller exact sketch forces early stop
    final int u2 = 1024;
    final int totU = u1 + u2;
    final UpdateSketch usk1 = UpdateSketch.builder().setNominalEntries(k).build();
    final UpdateSketch usk2 = UpdateSketch.builder().setNominalEntries(k).build();
    for (int i = 0; i < u1; i++) {
        // 2*k
        usk1.update(i);
    }
    for (int i = u1; i < totU; i++) {
        // 2*k + 1024 no overlap
        usk2.update(i);
    }
    final Memory skMem1 = Memory.wrap(usk1.compact(false, null).toByteArray());
    final Memory skMem2 = Memory.wrap(usk2.compact(true, null).toByteArray());
    final CompactSketch csk1 = (CompactSketch) Sketch.wrap(skMem1);
    final CompactSketch csk2 = (CompactSketch) Sketch.wrap(skMem2);
    // union memory
    final WritableMemory uMem = WritableMemory.writableWrap(new byte[getMaxUnionBytes(k)]);
    final Union union = SetOperation.builder().setNominalEntries(k).buildUnion(uMem);
    union.union(csk1);
    union.union(csk2);
    final CompactSketch cOut = union.getResult(true, null);
    assertEquals(cOut.getEstimate(), totU, .05 * k);
}

Example 90 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectUnionTest method checkForDruidBug.

@Test
public void checkForDruidBug() {
    final int k = 16384;
    final UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build();
    for (int i = 0; i < 100000; i++) {
        usk.update(Integer.toString(i));
    }
    // optional but created the symptom
    usk.rebuild();
    final Sketch s = usk.compact();
    // create empty target union in off-heap mem
    final WritableMemory mem = WritableMemory.writableWrap(new byte[getMaxUnionBytes(k)]);
    final Union union1 = SetOperation.builder().setNominalEntries(k).buildUnion(mem);
    union1.union(s);
    final CompactSketch csk = union1.getResult();
    assertTrue(csk.getTheta() < 0.2);
    assertEquals(csk.getRetainedEntries(true), 16384);
    final double est = csk.getEstimate();
    assertTrue(est > 98663.0);
    assertTrue(est < 101530.0);
}