Examples with Memory com.yahoo.memory.Memory used on opensource projects

Example 51 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkNewGadget.

@Test
public void checkNewGadget() {
    final int maxK = 1024;
    final int bigK = 1536;
    final int smallK = 128;
    // downsample input sketch, use as gadget (exact mode, but irrelevant here)
    final ReservoirItemsSketch<Long> bigKSketch = getBasicSketch(maxK / 2, bigK);
    final byte[] bigKBytes = bigKSketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory bigKMem = Memory.wrap(bigKBytes);
    ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(maxK);
    riu.update(bigKMem, new ArrayOfLongsSerDe());
    assertNotNull(riu.getResult());
    assertEquals(riu.getResult().getK(), maxK);
    assertEquals(riu.getResult().getN(), maxK / 2);
    // sketch k < maxK but in sampling mode
    final ReservoirItemsSketch<Long> smallKSketch = getBasicSketch(maxK, smallK);
    final byte[] smallKBytes = smallKSketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory smallKMem = Memory.wrap(smallKBytes);
    riu = ReservoirItemsUnion.newInstance(maxK);
    riu.update(smallKMem, new ArrayOfLongsSerDe());
    assertNotNull(riu.getResult());
    assertTrue(riu.getResult().getK() < maxK);
    assertEquals(riu.getResult().getK(), smallK);
    assertEquals(riu.getResult().getN(), maxK);
    // sketch k < maxK and in exact mode
    final ReservoirItemsSketch<Long> smallKExactSketch = getBasicSketch(smallK, smallK);
    final byte[] smallKExactBytes = smallKExactSketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory smallKExactMem = Memory.wrap(smallKExactBytes);
    riu = ReservoirItemsUnion.newInstance(maxK);
    riu.update(smallKExactMem, new ArrayOfLongsSerDe());
    assertNotNull(riu.getResult());
    assertEquals(riu.getResult().getK(), maxK);
    assertEquals(riu.getResult().getN(), smallK);
}

Example 52 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirLongsSketchTest method checkEmptySketch.

@Test
public void checkEmptySketch() {
    final ReservoirLongsSketch rls = ReservoirLongsSketch.newInstance(5);
    assertTrue(rls.getSamples() == null);
    final byte[] sketchBytes = rls.toByteArray();
    final Memory mem = Memory.wrap(sketchBytes);
    // only minPreLongs bytes and should deserialize to empty
    assertEquals(sketchBytes.length, Family.RESERVOIR.getMinPreLongs() << 3);
    final ReservoirLongsSketch loadedRls = ReservoirLongsSketch.heapify(mem);
    assertEquals(loadedRls.getNumSamples(), 0);
    println("Empty sketch:");
    println(rls.toString());
}

Example 53 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirLongsSketchTest method validateSerializeAndDeserialize.

private static void validateSerializeAndDeserialize(final ReservoirLongsSketch rls) {
    final byte[] sketchBytes = rls.toByteArray();
    assertEquals(sketchBytes.length, (Family.RESERVOIR.getMaxPreLongs() + rls.getNumSamples()) << 3);
    // ensure full reservoir rebuilds correctly
    final Memory mem = Memory.wrap(sketchBytes);
    final ReservoirLongsSketch loadedRls = ReservoirLongsSketch.heapify(mem);
    validateReservoirEquality(rls, loadedRls);
}

Example 54 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirLongsUnionTest method checkSerialization.

@Test
public void checkSerialization() {
    final int n = 100;
    final int k = 25;
    final ReservoirLongsUnion rlu = ReservoirLongsUnion.newInstance(k);
    for (int i = 0; i < n; ++i) {
        rlu.update(i);
    }
    final byte[] unionBytes = rlu.toByteArray();
    final Memory mem = Memory.wrap(unionBytes);
    final ReservoirLongsUnion rebuiltUnion = ReservoirLongsUnion.heapify(mem);
    validateUnionEquality(rlu, rebuiltUnion);
}

Example 55 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirLongsUnionTest method checkStandardMergeNoCopy.

@Test
public void checkStandardMergeNoCopy() {
    final int k = 1024;
    final int n1 = 256;
    final int n2 = 256;
    final ReservoirLongsSketch sketch1 = getBasicSketch(n1, k);
    final ReservoirLongsSketch sketch2 = getBasicSketch(n2, k);
    final ReservoirLongsUnion rlu = ReservoirLongsUnion.newInstance(k);
    rlu.update(sketch1);
    rlu.update(sketch2);
    assertNotNull(rlu.getResult());
    assertEquals(rlu.getResult().getK(), k);
    assertEquals(rlu.getResult().getN(), n1 + n2);
    assertEquals(rlu.getResult().getNumSamples(), n1 + n2);
    // creating from Memory should avoid a copy
    final int n3 = 2048;
    final ReservoirLongsSketch sketch3 = getBasicSketch(n3, k);
    final byte[] sketch3Bytes = sketch3.toByteArray();
    final Memory mem = Memory.wrap(sketch3Bytes);
    rlu.update(mem);
    assertEquals(rlu.getResult().getK(), k);
    assertEquals(rlu.getResult().getN(), n1 + n2 + n3);
    assertEquals(rlu.getResult().getNumSamples(), k);
}