Examples with WritableMemory com.yahoo.memory.WritableMemory used on opensource projects

Example 61 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class VarOptItemsSketchTest method checkBadFamily.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkBadFamily() {
    final VarOptItemsSketch<Long> sketch = getUnweightedLongsVIS(32, 16);
    final byte[] bytes = sketch.toByteArray(new ArrayOfLongsSerDe());
    final WritableMemory mem = WritableMemory.wrap(bytes);
    // corrupt the family ID
    mem.putByte(FAMILY_BYTE, (byte) 0);
    VarOptItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
    fail();
}

Example 62 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class VarOptItemsSketchTest method checkCorruptSerializedWeight.

@Test
public void checkCorruptSerializedWeight() {
    final VarOptItemsSketch<String> vis = VarOptItemsSketch.newInstance(24);
    for (int i = 1; i < 10; ++i) {
        vis.update(Integer.toString(i), i);
    }
    final byte[] sketchBytes = vis.toByteArray(new ArrayOfStringsSerDe(), String.class);
    final WritableMemory mem = WritableMemory.wrap(sketchBytes);
    // weights will be stored in the first double after the preamble
    final int numPreLongs = PreambleUtil.extractPreLongs(mem);
    final int weightOffset = numPreLongs << 3;
    // inject a negative weight
    mem.putDouble(weightOffset, -1.25);
    try {
        VarOptItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
        fail();
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().equals("Possible Corruption: Non-positive weight in " + "heapify(): -1.25"));
    }
}

Example 63 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkVersionConversion.

/*
  @Test
  public void checkReadOnlyHeapify() {
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    Memory sketchMem = getBasicSerializedLongsRIS();

    // Load from read-only and writable memory to ensure they deserialize identically
    ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.heapify(sketchMem.asReadOnlyMemory(),
            serDe);
    ReservoirItemsSketch<Long> fromWritable = ReservoirItemsSketch.heapify(sketchMem, serDe);
    validateReservoirEquality(ris, fromWritable);

    // Same with an empty sketch
    final byte[] sketchBytes = ReservoirItemsSketch.<Long>newInstance(32).toByteArray(serDe);
    sketchMem = new NativeMemory(sketchBytes);

    ris = ReservoirItemsSketch.heapify(sketchMem.asReadOnlyMemory(), serDe);
    fromWritable = ReservoirItemsSketch.heapify(sketchMem, serDe);
    validateReservoirEquality(ris, fromWritable);
  }
  */
@Test
public void checkVersionConversion() {
    // version change from 1 to 2 only impact first preamble long, so empty sketch is sufficient
    final int k = 32768;
    final short encK = ReservoirSize.computeSize(k);
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.newInstance(k);
    final byte[] sketchBytesOrig = ris.toByteArray(serDe);
    // get a new byte[], manually revert to v1, then reconstruct
    final byte[] sketchBytes = ris.toByteArray(serDe);
    final WritableMemory sketchMem = WritableMemory.wrap(sketchBytes);
    sketchMem.putByte(SER_VER_BYTE, (byte) 1);
    // zero out all 4 bytes
    sketchMem.putInt(RESERVOIR_SIZE_INT, 0);
    sketchMem.putShort(RESERVOIR_SIZE_SHORT, encK);
    println(PreambleUtil.preambleToString(sketchMem));
    final ReservoirItemsSketch<Long> rebuilt = ReservoirItemsSketch.heapify(sketchMem, serDe);
    final byte[] rebuiltBytes = rebuilt.toByteArray(serDe);
    assertEquals(sketchBytesOrig.length, rebuiltBytes.length);
    for (int i = 0; i < sketchBytesOrig.length; ++i) {
        assertEquals(sketchBytesOrig[i], rebuiltBytes[i]);
    }
}

Example 64 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkVersionConversionWithEmptyGadget.

@Test
public void checkVersionConversionWithEmptyGadget() {
    final int k = 32768;
    final short encK = ReservoirSize.computeSize(k);
    final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    final ReservoirItemsUnion<String> riu = ReservoirItemsUnion.newInstance(k);
    final byte[] unionBytesOrig = riu.toByteArray(serDe);
    // get a new byte[], manually revert to v1, then reconstruct
    final byte[] unionBytes = riu.toByteArray(serDe);
    final WritableMemory unionMem = WritableMemory.wrap(unionBytes);
    unionMem.putByte(SER_VER_BYTE, (byte) 1);
    // zero out all 4 bytes
    unionMem.putInt(RESERVOIR_SIZE_INT, 0);
    unionMem.putShort(RESERVOIR_SIZE_SHORT, encK);
    println(PreambleUtil.preambleToString(unionMem));
    final ReservoirItemsUnion<String> rebuilt = ReservoirItemsUnion.heapify(unionMem, serDe);
    final byte[] rebuiltBytes = rebuilt.toByteArray(serDe);
    assertEquals(unionBytesOrig.length, rebuiltBytes.length);
    for (int i = 0; i < unionBytesOrig.length; ++i) {
        assertEquals(unionBytesOrig[i], rebuiltBytes[i]);
    }
}

Example 65 with WritableMemory

use of com.yahoo.memory.WritableMemory in project sketches-core by DataSketches.

the class HeapCompactDoublesSketchTest method checkHeapifyUnsortedCompactV2.

@Test
public void checkHeapifyUnsortedCompactV2() {
    final int k = 64;
    final UpdateDoublesSketch qs = DoublesSketch.builder().setK(64).build();
    for (int i = 0; i < 3 * k; ++i) {
        qs.update(i);
    }
    assertEquals(qs.getBaseBufferCount(), k);
    final byte[] sketchBytes = qs.toByteArray(true);
    final WritableMemory mem = WritableMemory.wrap(sketchBytes);
    // modify to make v2, clear compact flag, and insert a -1 in the middle of the base buffer
    PreambleUtil.insertSerVer(mem.getArray(), mem.getCumulativeOffset(0), 2);
    PreambleUtil.insertFlags(mem.getArray(), mem.getCumulativeOffset(0), 0);
    final long tgtAddr = COMBINED_BUFFER + (Double.BYTES * k / 2);
    mem.putDouble(tgtAddr, -1.0);
    assert mem.getDouble(tgtAddr - Double.BYTES) > mem.getDouble(tgtAddr);
    // ensure the heapified base buffer is sorted
    final HeapCompactDoublesSketch qs2 = HeapCompactDoublesSketch.heapifyInstance(mem);
    checkBaseBufferIsSorted(qs2);
}