Examples with ArrayOfStringsSerDe com.yahoo.sketches.ArrayOfStringsSerDe used on opensource projects

Example 1 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class VarOptItemsSketchTest method checkCorruptSerializedWeight.

@Test
public void checkCorruptSerializedWeight() {
    final VarOptItemsSketch<String> vis = VarOptItemsSketch.newInstance(24);
    for (int i = 1; i < 10; ++i) {
        vis.update(Integer.toString(i), i);
    }
    final byte[] sketchBytes = vis.toByteArray(new ArrayOfStringsSerDe(), String.class);
    final WritableMemory mem = WritableMemory.wrap(sketchBytes);
    // weights will be stored in the first double after the preamble
    final int numPreLongs = PreambleUtil.extractPreLongs(mem);
    final int weightOffset = numPreLongs << 3;
    // inject a negative weight
    mem.putDouble(weightOffset, -1.25);
    try {
        VarOptItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
        fail();
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().equals("Possible Corruption: Non-positive weight in " + "heapify(): -1.25"));
    }
}

Example 2 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkEmptySketch.

@Test
public void checkEmptySketch() {
    final ReservoirItemsSketch<String> ris = ReservoirItemsSketch.newInstance(5);
    assertTrue(ris.getSamples() == null);
    final byte[] sketchBytes = ris.toByteArray(new ArrayOfStringsSerDe());
    final Memory mem = Memory.wrap(sketchBytes);
    // only minPreLongs bytes and should deserialize to empty
    assertEquals(sketchBytes.length, Family.RESERVOIR.getMinPreLongs() << 3);
    final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    final ReservoirItemsSketch<String> loadedRis = ReservoirItemsSketch.heapify(mem, serDe);
    assertEquals(loadedRis.getNumSamples(), 0);
    println("Empty sketch:");
    println("  Preamble:");
    println(PreambleUtil.preambleToString(mem));
    println("  Sketch:");
    println(ris.toString());
}

Example 3 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkUnderFullReservoir.

@Test
public void checkUnderFullReservoir() {
    final int k = 128;
    final int n = 64;
    final ReservoirItemsSketch<String> ris = ReservoirItemsSketch.newInstance(k);
    int expectedLength = 0;
    for (int i = 0; i < n; ++i) {
        final String intStr = Integer.toString(i);
        expectedLength += intStr.length() + Integer.BYTES;
        ris.update(intStr);
    }
    assertEquals(ris.getNumSamples(), n);
    final String[] data = ris.getSamples();
    assertNotNull(data);
    assertEquals(ris.getNumSamples(), ris.getN());
    assertEquals(data.length, n);
    // items in submit order until reservoir at capacity so check
    for (int i = 0; i < n; ++i) {
        assertEquals(data[i], Integer.toString(i));
    }
    // not using validateSerializeAndDeserialize() to check with a non-Long
    ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    expectedLength += Family.RESERVOIR.getMaxPreLongs() << 3;
    final byte[] sketchBytes = ris.toByteArray(serDe);
    assertEquals(sketchBytes.length, expectedLength);
    // ensure reservoir rebuilds correctly
    final Memory mem = Memory.wrap(sketchBytes);
    final ReservoirItemsSketch<String> loadedRis = ReservoirItemsSketch.heapify(mem, serDe);
    validateReservoirEquality(ris, loadedRis);
    println("Under-full reservoir:");
    println("  Preamble:");
    println(PreambleUtil.preambleToString(mem));
    println("  Sketch:");
    println(ris.toString());
}

Example 4 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkVersionConversionWithEmptyGadget.

@Test
public void checkVersionConversionWithEmptyGadget() {
    final int k = 32768;
    final short encK = ReservoirSize.computeSize(k);
    final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    final ReservoirItemsUnion<String> riu = ReservoirItemsUnion.newInstance(k);
    final byte[] unionBytesOrig = riu.toByteArray(serDe);
    // get a new byte[], manually revert to v1, then reconstruct
    final byte[] unionBytes = riu.toByteArray(serDe);
    final WritableMemory unionMem = WritableMemory.wrap(unionBytes);
    unionMem.putByte(SER_VER_BYTE, (byte) 1);
    // zero out all 4 bytes
    unionMem.putInt(RESERVOIR_SIZE_INT, 0);
    unionMem.putShort(RESERVOIR_SIZE_SHORT, encK);
    println(PreambleUtil.preambleToString(unionMem));
    final ReservoirItemsUnion<String> rebuilt = ReservoirItemsUnion.heapify(unionMem, serDe);
    final byte[] rebuiltBytes = rebuilt.toByteArray(serDe);
    assertEquals(unionBytesOrig.length, rebuiltBytes.length);
    for (int i = 0; i < unionBytesOrig.length; ++i) {
        assertEquals(unionBytesOrig[i], rebuiltBytes[i]);
    }
}

Example 5 with ArrayOfStringsSerDe

use of com.yahoo.sketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class ItemsSketchTest method checkGoodSerDeId.

@Test
public void checkGoodSerDeId() {
    ItemsSketch<String> sketch = ItemsSketch.getInstance(Comparator.naturalOrder());
    byte[] byteArr = sketch.toByteArray(new ArrayOfStringsSerDe());
    Memory mem = Memory.wrap(byteArr);
    //println(PreambleUtil.toString(mem));
    ItemsSketch.getInstance(mem, Comparator.naturalOrder(), new ArrayOfStringsSerDe());
}