Examples with Memory com.yahoo.memory.Memory used on opensource projects

Example 31 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkPolymorphicType.

@Test
public void checkPolymorphicType() {
    final ReservoirItemsSketch<Number> ris = ReservoirItemsSketch.newInstance(6);
    assertNull(ris.getSamples());
    assertNull(ris.getSamples(Number.class));
    // using mixed types
    ris.update(1);
    ris.update(2L);
    ris.update(3.0);
    ris.update((short) (44023 & 0xFFFF));
    ris.update((byte) (68 & 0xFF));
    ris.update(4.0F);
    final Number[] data = ris.getSamples(Number.class);
    assertNotNull(data);
    assertEquals(data.length, 6);
    // copying samples without specifying Number.class should fail
    try {
        ris.getSamples();
        fail();
    } catch (final ArrayStoreException e) {
    // expected
    }
    // likewise for toByteArray() (which uses getDataSamples() internally for type handling)
    final ArrayOfNumbersSerDe serDe = new ArrayOfNumbersSerDe();
    try {
        ris.toByteArray(serDe);
        fail();
    } catch (final ArrayStoreException e) {
    // expected
    }
    final byte[] sketchBytes = ris.toByteArray(serDe, Number.class);
    assertEquals(sketchBytes.length, 49);
    final Memory mem = Memory.wrap(sketchBytes);
    final ReservoirItemsSketch<Number> loadedRis = ReservoirItemsSketch.heapify(mem, serDe);
    assertEquals(ris.getNumSamples(), loadedRis.getNumSamples());
    final Number[] samples1 = ris.getSamples(Number.class);
    final Number[] samples2 = loadedRis.getSamples(Number.class);
    assertNotNull(samples1);
    assertNotNull(samples2);
    assertEquals(samples1.length, samples2.length);
    for (int i = 0; i < samples1.length; ++i) {
        assertEquals(samples1[i], samples2[i]);
    }
}

Example 32 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkEmptySketch.

@Test
public void checkEmptySketch() {
    final ReservoirItemsSketch<String> ris = ReservoirItemsSketch.newInstance(5);
    assertTrue(ris.getSamples() == null);
    final byte[] sketchBytes = ris.toByteArray(new ArrayOfStringsSerDe());
    final Memory mem = Memory.wrap(sketchBytes);
    // only minPreLongs bytes and should deserialize to empty
    assertEquals(sketchBytes.length, Family.RESERVOIR.getMinPreLongs() << 3);
    final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    final ReservoirItemsSketch<String> loadedRis = ReservoirItemsSketch.heapify(mem, serDe);
    assertEquals(loadedRis.getNumSamples(), 0);
    println("Empty sketch:");
    println("  Preamble:");
    println(PreambleUtil.preambleToString(mem));
    println("  Sketch:");
    println(ris.toString());
}

Example 33 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkUnderFullReservoir.

@Test
public void checkUnderFullReservoir() {
    final int k = 128;
    final int n = 64;
    final ReservoirItemsSketch<String> ris = ReservoirItemsSketch.newInstance(k);
    int expectedLength = 0;
    for (int i = 0; i < n; ++i) {
        final String intStr = Integer.toString(i);
        expectedLength += intStr.length() + Integer.BYTES;
        ris.update(intStr);
    }
    assertEquals(ris.getNumSamples(), n);
    final String[] data = ris.getSamples();
    assertNotNull(data);
    assertEquals(ris.getNumSamples(), ris.getN());
    assertEquals(data.length, n);
    // items in submit order until reservoir at capacity so check
    for (int i = 0; i < n; ++i) {
        assertEquals(data[i], Integer.toString(i));
    }
    // not using validateSerializeAndDeserialize() to check with a non-Long
    ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    expectedLength += Family.RESERVOIR.getMaxPreLongs() << 3;
    final byte[] sketchBytes = ris.toByteArray(serDe);
    assertEquals(sketchBytes.length, expectedLength);
    // ensure reservoir rebuilds correctly
    final Memory mem = Memory.wrap(sketchBytes);
    final ReservoirItemsSketch<String> loadedRis = ReservoirItemsSketch.heapify(mem, serDe);
    validateReservoirEquality(ris, loadedRis);
    println("Under-full reservoir:");
    println("  Preamble:");
    println(PreambleUtil.preambleToString(mem));
    println("  Sketch:");
    println(ris.toString());
}

Example 34 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkSerialization.

@Test
public void checkSerialization() {
    final int n = 100;
    final int k = 25;
    final ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(k);
    for (long i = 0; i < n; ++i) {
        riu.update(i);
    }
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final byte[] unionBytes = riu.toByteArray(serDe);
    final Memory mem = Memory.wrap(unionBytes);
    println(PreambleUtil.preambleToString(mem));
    final ReservoirItemsUnion<Long> rebuiltUnion = ReservoirItemsUnion.heapify(mem, serDe);
    assertEquals(riu.getMaxK(), rebuiltUnion.getMaxK());
    ReservoirItemsSketchTest.validateReservoirEquality(riu.getResult(), rebuiltUnion.getResult());
}

Example 35 with Memory

use of com.yahoo.memory.Memory in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkInstantiation.

@Test
public void checkInstantiation() {
    final int n = 100;
    final int k = 25;
    // create empty unions
    ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(k);
    assertNull(riu.getResult());
    riu.update(5L);
    assertNotNull(riu.getResult());
    // pass in a sketch, as both an object and memory
    final ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.newInstance(k);
    for (long i = 0; i < n; ++i) {
        ris.update(i);
    }
    riu.reset();
    assertEquals(riu.getResult().getN(), 0);
    riu.update(ris);
    assertEquals(riu.getResult().getN(), ris.getN());
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    // only the gadget is serialized
    final byte[] sketchBytes = ris.toByteArray(serDe);
    final Memory mem = Memory.wrap(sketchBytes);
    riu = ReservoirItemsUnion.newInstance(ris.getK());
    riu.update(mem, serDe);
    assertNotNull(riu.getResult());
    println(riu.toString());
}