Examples with WritableMemory org.apache.datasketches.memory.WritableMemory used on opensource projects

Example 36 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class HllSketchTest method checkSerSizes.

private static void checkSerSizes(int lgConfigK, TgtHllType tgtHllType, boolean direct) {
    int bytes = getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
    WritableMemory wmem = WritableMemory.allocate(bytes);
    HllSketch sk = (direct) ? new HllSketch(lgConfigK, tgtHllType, wmem) : new HllSketch(lgConfigK, tgtHllType);
    int i;
    // LIST
    for (i = 0; i < 7; i++) {
        sk.update(i);
    }
    int expected = LIST_INT_ARR_START + (i << 2);
    assertEquals(sk.getCompactSerializationBytes(), expected);
    expected = LIST_INT_ARR_START + (4 << LG_INIT_LIST_SIZE);
    assertEquals(sk.getUpdatableSerializationBytes(), expected);
    // SET
    for (i = 7; i < 24; i++) {
        sk.update(i);
    }
    expected = HASH_SET_INT_ARR_START + (i << 2);
    assertEquals(sk.getCompactSerializationBytes(), expected);
    expected = HASH_SET_INT_ARR_START + (4 << LG_INIT_SET_SIZE);
    assertEquals(sk.getUpdatableSerializationBytes(), expected);
    // HLL
    sk.update(i);
    assertEquals(sk.getCurMode(), CurMode.HLL);
    AbstractHllArray absHll = (AbstractHllArray) sk.hllSketchImpl;
    int auxCountBytes = 0;
    int auxArrBytes = 0;
    if (absHll.tgtHllType == HLL_4) {
        AuxHashMap auxMap = absHll.getAuxHashMap();
        if (auxMap != null) {
            auxCountBytes = auxMap.getAuxCount() << 2;
            auxArrBytes = 4 << auxMap.getLgAuxArrInts();
        } else {
            auxArrBytes = 4 << LG_AUX_ARR_INTS[lgConfigK];
        }
    }
    int hllArrBytes = absHll.getHllByteArrBytes();
    expected = HLL_BYTE_ARR_START + hllArrBytes + auxCountBytes;
    assertEquals(sk.getCompactSerializationBytes(), expected);
    expected = HLL_BYTE_ARR_START + hllArrBytes + auxArrBytes;
    assertEquals(sk.getUpdatableSerializationBytes(), expected);
    int fullAuxBytes = (tgtHllType == TgtHllType.HLL_4) ? (4 << LG_AUX_ARR_INTS[lgConfigK]) : 0;
    expected = HLL_BYTE_ARR_START + hllArrBytes + fullAuxBytes;
    assertEquals(getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType), expected);
}

Example 37 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class HllSketchTest method checkWritableWrapOfCompact.

@SuppressWarnings("unused")
@Test(expectedExceptions = SketchesArgumentException.class)
public void checkWritableWrapOfCompact() {
    HllSketch sk = new HllSketch();
    byte[] byteArr = sk.toCompactByteArray();
    WritableMemory wmem = WritableMemory.writableWrap(byteArr);
    HllSketch sk2 = HllSketch.writableWrap(wmem);
}

Example 38 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class HllSketchTest method copyAs.

private static void copyAs(TgtHllType srcType, TgtHllType dstType, boolean direct) {
    int lgK = 8;
    int n1 = 7;
    int n2 = 24;
    int n3 = 1000;
    int base = 0;
    int bytes = getMaxUpdatableSerializationBytes(lgK, srcType);
    WritableMemory wmem = WritableMemory.allocate(bytes);
    HllSketch src = (direct) ? new HllSketch(lgK, srcType, wmem) : new HllSketch(lgK, srcType);
    for (int i = 0; i < n1; i++) {
        src.update(i + base);
    }
    HllSketch dst = src.copyAs(dstType);
    assertEquals(dst.getEstimate(), src.getEstimate(), 0.0);
    for (int i = n1; i < n2; i++) {
        src.update(i);
    }
    dst = src.copyAs(dstType);
    assertEquals(dst.getEstimate(), src.getEstimate(), 0.0);
    for (int i = n2; i < n3; i++) {
        src.update(i);
    }
    dst = src.copyAs(dstType);
    assertEquals(dst.getEstimate(), src.getEstimate(), 0.0);
}

Example 39 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class ItemsSketchTest method checkMemExceptions.

@Test
public void checkMemExceptions() {
    ItemsSketch<Long> sk1 = new ItemsSketch<>(1 << LG_MIN_MAP_SIZE);
    sk1.update(Long.valueOf(1), 1);
    ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    byte[] byteArr = sk1.toByteArray(serDe);
    WritableMemory mem = WritableMemory.writableWrap(byteArr);
    // FrequentItemsSketch<Long> sk2 = FrequentItemsSketch.getInstance(mem, serDe);
    // println(sk2.toString());
    // The correct first 8 bytes.
    long pre0 = mem.getLong(0);
    // Now start corrupting
    // Corrupt
    tryBadMem(mem, PREAMBLE_LONGS_BYTE, 2);
    // restore
    mem.putLong(0, pre0);
    // Corrupt
    tryBadMem(mem, SER_VER_BYTE, 2);
    // restore
    mem.putLong(0, pre0);
    // Corrupt
    tryBadMem(mem, FAMILY_BYTE, 2);
    // restore
    mem.putLong(0, pre0);
    // Corrupt to true
    tryBadMem(mem, FLAGS_BYTE, 4);
    // restore
    mem.putLong(0, pre0);
}

Example 40 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class MurmurHash3v2Test method byteArrChecks.

@Test
public void byteArrChecks() {
    long seed = 0;
    int offset = 0;
    int bytes = 1024;
    long[] hash2 = new long[2];
    for (int j = 1; j < bytes; j++) {
        byte[] in = new byte[bytes];
        WritableMemory wmem = WritableMemory.writableWrap(in);
        for (int i = 0; i < j; i++) {
            wmem.putByte(i, (byte) (-128 + i));
        }
        long[] hash1 = MurmurHash3.hash(in, 0);
        hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2);
        long[] hash3 = MurmurHash3v2.hash(in, seed);
        assertEquals(hash1, hash2);
        assertEquals(hash1, hash3);
    }
}