Examples with WritableMemory org.apache.datasketches.memory.WritableMemory used on opensource projects

Example 16 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectAuxHashMapTest method checkDiffToByteArr.

@Test
public void checkDiffToByteArr() {
    // this combination should create an Aux with ~18 exceptions
    int lgK = 12;
    int lgU = 19;
    TgtHllType type = TgtHllType.HLL_4;
    int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, type);
    byte[] memByteArr = new byte[bytes];
    WritableMemory wmem = WritableMemory.writableWrap(memByteArr);
    HllSketch heapSk = new HllSketch(lgK, type);
    HllSketch dirSk = new HllSketch(lgK, type, wmem);
    for (int i = 0; i < (1 << lgU); i++) {
        heapSk.update(i);
        // problem starts here.
        dirSk.update(i);
    }
    AbstractHllArray heapHllArr = (AbstractHllArray) heapSk.hllSketchImpl;
    AbstractHllArray dirHllArr = (AbstractHllArray) dirSk.hllSketchImpl;
    assert dirHllArr instanceof DirectHllArray;
    AuxHashMap heapAux = heapHllArr.getAuxHashMap();
    assert heapAux instanceof HeapAuxHashMap;
    AuxHashMap dirAux = dirHllArr.getAuxHashMap();
    // TOOD FAILS!
    assert dirAux instanceof DirectAuxHashMap;
    println("HeapAuxCount: " + heapAux.getAuxCount());
    println("DirAuxCount: " + dirAux.getAuxCount());
    int heapCurMin = heapHllArr.getCurMin();
    int dirCurMin = dirHllArr.getCurMin();
    println("HeapCurMin: " + heapCurMin);
    println("DirCurMin: " + dirCurMin);
    PairIterator auxItr;
    auxItr = heapHllArr.getAuxIterator();
    println("\nHeap Pairs");
    // println(itr.getHeader());
    while (auxItr.nextValid()) {
        println("" + auxItr.getPair());
    }
    auxItr = dirHllArr.getAuxIterator();
    println("\nDirect Pairs");
    // println(itr.getHeader());
    while (auxItr.nextValid()) {
        println("" + auxItr.getPair());
    }
    PairIterator hllItr;
    hllItr = heapSk.iterator();
    println("Heap HLL arr");
    println(hllItr.getHeader());
    while (hllItr.nextValid()) {
        if ((hllItr.getValue() - heapCurMin) > 14) {
            println(hllItr.getString() + ", " + hllItr.getPair());
        }
    }
    hllItr = dirSk.iterator();
    println("Direct HLL arr");
    println(hllItr.getHeader());
    while (hllItr.nextValid()) {
        if ((hllItr.getValue() - dirCurMin) > 14) {
            println(hllItr.getString() + ", " + hllItr.getPair());
        }
    }
    byte[] heapImg = heapSk.toUpdatableByteArray();
    Memory heapImgMem = Memory.wrap(heapImg);
    byte[] dirImg = dirSk.toUpdatableByteArray();
    Memory dirImgMem = Memory.wrap(dirImg);
    println("heapLen: " + heapImg.length + ", dirLen: " + dirImg.length + ", memObjLen: " + memByteArr.length);
    int auxStart = 40 + (1 << (lgK - 1));
    println("AuxStart: " + auxStart);
    println(String.format("%14s%14s%14s", "dir wmem", "heap to b[]", "direct to b[]"));
    for (int i = auxStart; i < heapImg.length; i += 4) {
        println(String.format("%14d%14d%14d", wmem.getInt(i), heapImgMem.getInt(i), dirImgMem.getInt(i)));
        assert memByteArr[i] == heapImg[i];
        assert heapImg[i] == dirImg[i] : "i: " + i;
    }
    assertEquals(heapImg, dirImg);
}

Example 17 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectHllSketchTest method checkToCompactByteArr.

@Test
public void checkToCompactByteArr() {
    int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4);
    WritableMemory wmem = WritableMemory.allocate(bytes);
    HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem);
    for (int i = 0; i < 8; i++) {
        sk.update(i);
    }
    byte[] compByteArr = sk.toCompactByteArray();
    Memory compMem = Memory.wrap(compByteArr);
    HllSketch sk2 = HllSketch.wrap(compMem);
    byte[] compByteArr2 = sk2.toCompactByteArray();
    assertEquals(compByteArr2, compByteArr);
}

Example 18 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectHllSketchTest method checkPutKxQ1_Misc.

@Test
public void checkPutKxQ1_Misc() {
    int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4);
    WritableMemory wmem = WritableMemory.allocate(bytes);
    HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem);
    for (int i = 0; i < 8; i++) {
        sk.update(i);
    }
    assertTrue(sk.getCurMode() == CurMode.HLL);
    AbstractHllArray absArr = (AbstractHllArray) sk.hllSketchImpl;
    absArr.putKxQ1(1.0);
    assertEquals(absArr.getKxQ1(), 1.0);
    absArr.putKxQ1(0.0);
    Memory mem = wmem;
    HllSketch sk2 = HllSketch.wrap(mem);
    try {
        sk2.reset();
        fail();
    } catch (SketchesArgumentException e) {
    // expected
    }
}

Example 19 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectHllSketchTest method noWriteAccess.

private static void noWriteAccess(TgtHllType tgtHllType, int n) {
    int lgConfigK = 8;
    int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
    WritableMemory wmem = WritableMemory.allocate(bytes);
    HllSketch sk = new HllSketch(lgConfigK, tgtHllType, wmem);
    for (int i = 0; i < n; i++) {
        sk.update(i);
    }
    HllSketch sk2 = HllSketch.wrap(wmem);
    try {
        sk2.update(1);
        fail();
    } catch (SketchesReadOnlyException e) {
    // expected
    }
}

Example 20 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class HllArrayTest method toByteArrayHeapify.

private static void toByteArrayHeapify(int lgK, TgtHllType tgtHllType, int u, boolean direct) {
    HllSketch sk1;
    WritableMemory wmem = null;
    if (direct) {
        int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
        wmem = WritableMemory.allocate(bytes);
        sk1 = new HllSketch(lgK, tgtHllType, wmem);
    } else {
        sk1 = new HllSketch(lgK, tgtHllType);
    }
    for (int i = 0; i < u; i++) {
        sk1.update(i);
    }
    assert sk1.hllSketchImpl instanceof AbstractHllArray;
    if (sk1.hllSketchImpl instanceof HllArray) {
        assertFalse(sk1.hllSketchImpl.isMemory());
        assertFalse(sk1.isSameResource(wmem));
    } else {
        // DirectHllArray
        assertTrue(sk1.hllSketchImpl.isMemory());
        assertTrue(sk1.isSameResource(wmem));
    }
    // sk1.update(u);
    double est1 = sk1.getEstimate();
    assertEquals(est1, u, u * .03);
    assertEquals(sk1.getHipEstimate(), est1, 0.0);
    // misc calls
    sk1.hllSketchImpl.putEmptyFlag(false);
    sk1.hllSketchImpl.putRebuildCurMinNumKxQFlag(true);
    sk1.hllSketchImpl.putRebuildCurMinNumKxQFlag(false);
    byte[] byteArray = sk1.toCompactByteArray();
    HllSketch sk2 = HllSketch.heapify(byteArray);
    double est2 = sk2.getEstimate();
    assertEquals(est2, est1, 0.0);
    byteArray = sk1.toUpdatableByteArray();
    sk2 = HllSketch.heapify(byteArray);
    est2 = sk2.getEstimate();
    assertEquals(est2, est1, 0.0);
    sk1.reset();
    assertEquals(sk1.getEstimate(), 0.0, 0.0);
}