Examples with WritableMemory org.apache.datasketches.memory.WritableMemory used on opensource projects

Example 11 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class ArrayOfBooleansSerDe method serializeToByteArray.

@Override
public byte[] serializeToByteArray(final Boolean[] items) {
    final int bytesNeeded = computeBytesNeeded(items.length);
    final byte[] bytes = new byte[bytesNeeded];
    final WritableMemory mem = WritableMemory.writableWrap(bytes);
    byte val = 0;
    for (int i = 0; i < items.length; ++i) {
        if (items[i]) {
            val |= 0x1 << (i & 0x7);
        }
        if ((i & 0x7) == 0x7) {
            mem.putByte(i >>> 3, val);
            val = 0;
        }
    }
    // write out any remaining values (if val=0, still good to be explicit)
    if ((items.length & 0x7) > 0) {
        mem.putByte(bytesNeeded - 1, val);
    }
    return bytes;
}

Example 12 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class ArrayOfDoublesSerDe method serializeToByteArray.

@Override
public byte[] serializeToByteArray(final Double[] items) {
    final byte[] bytes = new byte[Double.BYTES * items.length];
    final WritableMemory mem = WritableMemory.writableWrap(bytes);
    long offsetBytes = 0;
    for (int i = 0; i < items.length; i++) {
        mem.putDouble(offsetBytes, items[i]);
        offsetBytes += Double.BYTES;
    }
    return bytes;
}

Example 13 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class CouponListTest method checkDuplicatesAndMisc.

private static void checkDuplicatesAndMisc(boolean direct) {
    int lgConfigK = 8;
    TgtHllType tgtHllType = TgtHllType.HLL_4;
    int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
    WritableMemory wmem = WritableMemory.allocate(bytes);
    HllSketch sk = (direct) ? new HllSketch(lgConfigK, tgtHllType, wmem) : new HllSketch(8);
    for (int i = 1; i <= 7; i++) {
        sk.update(i);
        sk.update(i);
    }
    assertEquals(sk.getCurMode(), CurMode.LIST);
    assertEquals(sk.getCompositeEstimate(), 7.0, 7 * .01);
    assertEquals(sk.getHipEstimate(), 7.0, 7 * .01);
    // dummy
    sk.hllSketchImpl.putEmptyFlag(false);
    // dummy
    sk.hllSketchImpl.putRebuildCurMinNumKxQFlag(false);
    if (direct) {
        assertNotNull(sk.getWritableMemory());
        assertNotNull(sk.getMemory());
    } else {
        assertNull(sk.getWritableMemory());
        assertNull(sk.getMemory());
    }
    sk.update(8);
    sk.update(8);
    assertEquals(sk.getCurMode(), CurMode.SET);
    assertEquals(sk.getCompositeEstimate(), 8.0, 8 * .01);
    assertEquals(sk.getHipEstimate(), 8.0, 8 * .01);
    if (direct) {
        assertNotNull(sk.getWritableMemory());
        assertNotNull(sk.getMemory());
    } else {
        assertNull(sk.getWritableMemory());
        assertNull(sk.getMemory());
    }
    for (int i = 9; i <= 25; i++) {
        sk.update(i);
        sk.update(i);
    }
    assertEquals(sk.getCurMode(), CurMode.HLL);
    assertEquals(sk.getCompositeEstimate(), 25.0, 25 * .1);
    if (direct) {
        assertNotNull(sk.getWritableMemory());
        assertNotNull(sk.getMemory());
    } else {
        assertNull(sk.getWritableMemory());
        assertNull(sk.getMemory());
    }
    double re = sk.getRelErr(true, true, 4, 1);
    assertTrue(re < 0.0);
}

Example 14 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectAuxHashMapTest method initSketchAndMap.

static void initSketchAndMap(boolean direct, boolean compact) {
    // this combination should create an Aux with ~18 exceptions
    int lgK = 15;
    int lgU = 20;
    println("HLL_4, lgK: " + lgK + ", lgU: " + lgU);
    HashMap<Integer, Integer> map = new HashMap<>();
    // create sketch
    HllSketch sketch;
    if (direct) {
        int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, TgtHllType.HLL_4);
        WritableMemory wmem = WritableMemory.allocate(bytes);
        sketch = new HllSketch(lgK, TgtHllType.HLL_4, wmem);
    } else {
        sketch = new HllSketch(lgK, TgtHllType.HLL_4);
    }
    for (int i = 0; i < (1 << lgU); i++) {
        sketch.update(i);
    }
    // check Ser Bytes
    assertEquals(sketch.getUpdatableSerializationBytes(), 40 + (1 << (lgK - 1)) + (4 << LG_AUX_ARR_INTS[lgK]));
    // extract the auxHashMap entries into a HashMap for easy checking
    // extract direct aux iterator
    AbstractHllArray absDirectHllArr = (AbstractHllArray) sketch.hllSketchImpl;
    // the auxHashMap must exist for this test
    AuxHashMap auxMap = absDirectHllArr.getAuxHashMap();
    int auxCount = auxMap.getAuxCount();
    assertEquals(auxMap.getCompactSizeBytes(), auxCount << 2);
    int auxArrInts = 1 << auxMap.getLgAuxArrInts();
    assertEquals(auxMap.getUpdatableSizeBytes(), auxArrInts << 2);
    PairIterator itr = absDirectHllArr.getAuxIterator();
    println("Source Aux Array.");
    println(itr.getHeader());
    while (itr.nextValid()) {
        // create the aux reference map
        map.put(itr.getSlot(), itr.getValue());
        println(itr.getString());
    }
    double est = sketch.getEstimate();
    println("\nHLL Array of original sketch: should match Source Aux Array.");
    // check HLL arr consistencies
    checkHllArr(sketch, map);
    // serialize the direct sk as compact
    byte[] byteArr = (compact) ? sketch.toCompactByteArray() : sketch.toUpdatableByteArray();
    // Heapify the byteArr image & check estimate
    HllSketch heapSk = HllSketch.heapify(Memory.wrap(byteArr));
    assertEquals(heapSk.getEstimate(), est, 0.0);
    println("\nAux Array of heapified serialized sketch.");
    // check Aux consistencies
    checkAux(heapSk, map);
    println("\nHLL Array of heapified serialized sketch.");
    // check HLL arr consistencies
    checkHllArr(heapSk, map);
    // Wrap the image as read-only & check estimate
    HllSketch wrapSk = HllSketch.wrap(Memory.wrap(byteArr));
    assertEquals(wrapSk.getEstimate(), est, 0.0);
    println("\nAux Array of wrapped RO serialized sketch.");
    checkAux(wrapSk, map);
    println("\nHLL Array of wrapped RO serialized sketch.");
    checkHllArr(wrapSk, map);
    println(wrapSk.toString(false, false, true, true));
}

Example 15 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectAuxHashMapTest method checkMustReplace.

@Test
public void checkMustReplace() {
    int lgK = 7;
    int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, TgtHllType.HLL_4);
    WritableMemory wmem = WritableMemory.allocate(bytes);
    HllSketch sk = new HllSketch(lgK, TgtHllType.HLL_4, wmem);
    for (int i = 0; i < 25; i++) {
        sk.update(i);
    }
    DirectHllArray dHllArr = (DirectHllArray) sk.hllSketchImpl;
    AuxHashMap map = dHllArr.getNewAuxHashMap();
    map.mustAdd(100, 5);
    int val = map.mustFindValueFor(100);
    assertEquals(val, 5);
    map.mustReplace(100, 10);
    val = map.mustFindValueFor(100);
    assertEquals(val, 10);
    assertTrue(map.isMemory());
    assertFalse(map.isOffHeap());
    assertNull(map.copy());
    assertNull(map.getAuxIntArr());
    try {
        map.mustAdd(100, 12);
        fail();
    } catch (SketchesStateException e) {
    // expected
    }
    try {
        map.mustFindValueFor(101);
        fail();
    } catch (SketchesStateException e) {
    // expected
    }
    try {
        map.mustReplace(101, 5);
        fail();
    } catch (SketchesStateException e) {
    // expected
    }
}