Examples with WritableMemory com.yahoo.memory.WritableMemory used on opensource projects

Example 76 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class PreambleUtilTest method checkInsertsAndExtracts.

@Test
public void checkInsertsAndExtracts() {
    final int bytes = 32;
    try (WritableDirectHandle offHeapMemHandler = WritableMemory.allocateDirect(bytes)) {
        final WritableMemory offHeapMem = offHeapMemHandler.get();
        final WritableMemory onHeapMem = WritableMemory.wrap(new byte[bytes]);
        final Object onHeapMemObj = onHeapMem.getArray();
        final Object offHeapMemObj = offHeapMem.getArray();
        final long onHeapCumOffset = onHeapMem.getCumulativeOffset(0L);
        final long offHeapCumOffset = offHeapMem.getCumulativeOffset(0L);
        onHeapMem.clear();
        offHeapMem.clear();
        //BYTES
        int v = 0XFF;
        int onH, offH;
        //PREAMBLE_LONGS_BYTE;
        insertPreLongs(onHeapMemObj, onHeapCumOffset, v);
        onH = extractPreLongs(onHeapMem);
        assertEquals(onH, v);
        insertPreLongs(offHeapMemObj, offHeapCumOffset, v);
        offH = extractPreLongs(offHeapMem);
        assertEquals(offH, v);
        onHeapMem.clear();
        offHeapMem.clear();
        //SER_VER_BYTE;
        insertSerVer(onHeapMemObj, onHeapCumOffset, v);
        onH = extractSerVer(onHeapMem);
        assertEquals(onH, v);
        insertSerVer(offHeapMemObj, offHeapCumOffset, v);
        offH = extractSerVer(offHeapMem);
        assertEquals(offH, v);
        onHeapMem.clear();
        offHeapMem.clear();
        //FAMILY_BYTE;
        insertFamilyID(onHeapMemObj, onHeapCumOffset, v);
        onH = extractFamilyID(onHeapMem);
        assertEquals(onH, v);
        insertFamilyID(offHeapMemObj, offHeapCumOffset, v);
        offH = extractFamilyID(offHeapMem);
        assertEquals(offH, v);
        onHeapMem.clear();
        offHeapMem.clear();
        //FLAGS_BYTE;
        insertFlags(onHeapMemObj, onHeapCumOffset, v);
        onH = extractFlags(onHeapMem);
        assertEquals(onH, v);
        insertFlags(offHeapMemObj, offHeapCumOffset, v);
        offH = extractFlags(offHeapMem);
        assertEquals(offH, v);
        onHeapMem.clear();
        offHeapMem.clear();
        //SHORTS
        v = 0XFFFF;
        //K_SHORT;
        insertK(onHeapMemObj, onHeapCumOffset, v);
        onH = extractK(onHeapMem);
        assertEquals(onH, v);
        insertK(offHeapMemObj, offHeapCumOffset, v);
        offH = extractK(offHeapMem);
        assertEquals(offH, v);
        onHeapMem.clear();
        offHeapMem.clear();
        //LONGS
        //N_LONG;
        long onHL, offHL, vL = 1L << 30;
        insertN(onHeapMemObj, onHeapCumOffset, vL);
        onHL = extractN(onHeapMem);
        assertEquals(onHL, vL);
        insertN(offHeapMemObj, offHeapCumOffset, vL);
        offHL = extractN(offHeapMem);
        assertEquals(offHL, vL);
        onHeapMem.clear();
        offHeapMem.clear();
        //DOUBLES
        //MIN_DOUBLE;
        double onHD, offHD, vD = 1L << 40;
        insertMinDouble(onHeapMemObj, onHeapCumOffset, vD);
        onHD = extractMinDouble(onHeapMem);
        assertEquals(onHD, vD);
        insertMinDouble(offHeapMemObj, offHeapCumOffset, vD);
        offHD = extractMinDouble(offHeapMem);
        assertEquals(offHD, vD);
        onHeapMem.clear();
        offHeapMem.clear();
        //MAX_DOUBLE;
        insertMaxDouble(onHeapMemObj, onHeapCumOffset, vD);
        onHD = extractMaxDouble(onHeapMem);
        assertEquals(onHD, vD);
        insertMaxDouble(offHeapMemObj, offHeapCumOffset, vD);
        offHD = extractMaxDouble(offHeapMem);
        assertEquals(offHD, vD);
        onHeapMem.clear();
        offHeapMem.clear();
    }
}

Example 77 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkBadFamily.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkBadFamily() {
    final WritableMemory mem = getBasicSerializedLongsRIS();
    // corrupt the family ID
    mem.putByte(FAMILY_BYTE, (byte) Family.ALPHA.getID());
    try {
        PreambleUtil.preambleToString(mem);
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().startsWith("Inspecting preamble with Sampling family"));
    }
    ReservoirItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
    fail();
}

Example 78 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class ReservoirLongsSketchTest method checkBadFamily.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkBadFamily() {
    final WritableMemory mem = getBasicSerializedRLS();
    // corrupt the family ID
    mem.putByte(FAMILY_BYTE, (byte) 0);
    ReservoirLongsSketch.heapify(mem);
    fail();
}

Example 79 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class ReservoirLongsUnionTest method checkVersionConversionWithGadget.

@Test
public void checkVersionConversionWithGadget() {
    final long n = 32;
    final int k = 256;
    final short encK = ReservoirSize.computeSize(k);
    final ReservoirLongsUnion rlu = ReservoirLongsUnion.newInstance(k);
    for (long i = 0; i < n; ++i) {
        rlu.update(i);
    }
    final byte[] unionBytesOrig = rlu.toByteArray();
    // get a new byte[], manually revert to v1, then reconstruct
    final byte[] unionBytes = rlu.toByteArray();
    final WritableMemory unionMem = WritableMemory.wrap(unionBytes);
    unionMem.putByte(SER_VER_BYTE, (byte) 1);
    // zero out all 4 bytes
    unionMem.putInt(RESERVOIR_SIZE_INT, 0);
    unionMem.putShort(RESERVOIR_SIZE_SHORT, encK);
    // force gadget header to v1, too
    final int offset = Family.RESERVOIR_UNION.getMaxPreLongs() << 3;
    unionMem.putByte(offset + SER_VER_BYTE, (byte) 1);
    // zero out all 4 bytes
    unionMem.putInt(offset + RESERVOIR_SIZE_INT, 0);
    unionMem.putShort(offset + RESERVOIR_SIZE_SHORT, encK);
    final ReservoirLongsUnion rebuilt = ReservoirLongsUnion.heapify(unionMem);
    final byte[] rebuiltBytes = rebuilt.toByteArray();
    assertEquals(unionBytesOrig.length, rebuiltBytes.length);
    for (int i = 0; i < unionBytesOrig.length; ++i) {
        assertEquals(unionBytesOrig[i], rebuiltBytes[i]);
    }
}

Example 80 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class DirectIntersectionTest method checkExactIntersectionFullOverlap.

@Test
public void checkExactIntersectionFullOverlap() {
    int lgK = 9;
    int k = 1 << lgK;
    Intersection inter;
    UpdateSketch usk1 = UpdateSketch.builder().setNominalEntries(k).build();
    UpdateSketch usk2 = UpdateSketch.builder().setNominalEntries(k).build();
    for (int i = 0; i < k; i++) usk1.update(i);
    for (int i = 0; i < k; i++) usk2.update(i);
    int memBytes = getMaxIntersectionBytes(k);
    byte[] memArr = new byte[memBytes];
    WritableMemory iMem = WritableMemory.wrap(memArr);
    inter = SetOperation.builder().buildIntersection(iMem);
    inter.update(usk1);
    inter.update(usk2);
    CompactSketch rsk1;
    boolean ordered = true;
    rsk1 = inter.getResult(!ordered, null);
    assertEquals(rsk1.getEstimate(), (double) k);
    rsk1 = inter.getResult(ordered, null);
    assertEquals(rsk1.getEstimate(), (double) k);
    boolean compact = true;
    int bytes = rsk1.getCurrentBytes(compact);
    byte[] byteArray = new byte[bytes];
    WritableMemory mem = WritableMemory.wrap(byteArray);
    //executed twice to fully exercise the internal state machine
    rsk1 = inter.getResult(!ordered, mem);
    assertEquals(rsk1.getEstimate(), (double) k);
    rsk1 = inter.getResult(ordered, mem);
    assertEquals(rsk1.getEstimate(), (double) k);
}