Examples with ArrayOfStringsSerDe org.apache.datasketches.ArrayOfStringsSerDe used on opensource projects

Example 6 with ArrayOfStringsSerDe

use of org.apache.datasketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class ItemsSketchTest method checkPutMemoryException.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkPutMemoryException() {
    final ItemsSketch<String> sketch = ItemsSketch.getInstance(16, Comparator.naturalOrder());
    for (int i = 0; i < 40; i++) {
        sketch.update(Integer.toString(i));
    }
    final byte[] byteArr = new byte[100];
    final WritableMemory mem = WritableMemory.writableWrap(byteArr);
    sketch.putMemory(mem, new ArrayOfStringsSerDe());
}

Example 7 with ArrayOfStringsSerDe

use of org.apache.datasketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkVersionConversionWithEmptyGadget.

@Test
public void checkVersionConversionWithEmptyGadget() {
    final int k = 32768;
    final short encK = ReservoirSize.computeSize(k);
    final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    final ReservoirItemsUnion<String> riu = ReservoirItemsUnion.newInstance(k);
    final byte[] unionBytesOrig = riu.toByteArray(serDe);
    // get a new byte[], manually revert to v1, then reconstruct
    final byte[] unionBytes = riu.toByteArray(serDe);
    final WritableMemory unionMem = WritableMemory.writableWrap(unionBytes);
    unionMem.putByte(SER_VER_BYTE, (byte) 1);
    // zero out all 4 bytes
    unionMem.putInt(RESERVOIR_SIZE_INT, 0);
    unionMem.putShort(RESERVOIR_SIZE_SHORT, encK);
    println(PreambleUtil.preambleToString(unionMem));
    final ReservoirItemsUnion<String> rebuilt = ReservoirItemsUnion.heapify(unionMem, serDe);
    final byte[] rebuiltBytes = rebuilt.toByteArray(serDe);
    assertEquals(unionBytesOrig.length, rebuiltBytes.length);
    for (int i = 0; i < unionBytesOrig.length; ++i) {
        assertEquals(unionBytesOrig[i], rebuiltBytes[i]);
    }
}

Example 8 with ArrayOfStringsSerDe

use of org.apache.datasketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkUnderFullReservoir.

@Test
public void checkUnderFullReservoir() {
    final int k = 128;
    final int n = 64;
    final ReservoirItemsSketch<String> ris = ReservoirItemsSketch.newInstance(k);
    int expectedLength = 0;
    for (int i = 0; i < n; ++i) {
        final String intStr = Integer.toString(i);
        expectedLength += intStr.length() + Integer.BYTES;
        ris.update(intStr);
    }
    assertEquals(ris.getNumSamples(), n);
    final String[] data = ris.getSamples();
    assertNotNull(data);
    assertEquals(ris.getNumSamples(), ris.getN());
    assertEquals(data.length, n);
    // items in submit order until reservoir at capacity so check
    for (int i = 0; i < n; ++i) {
        assertEquals(data[i], Integer.toString(i));
    }
    // not using validateSerializeAndDeserialize() to check with a non-Long
    ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
    expectedLength += Family.RESERVOIR.getMaxPreLongs() << 3;
    final byte[] sketchBytes = ris.toByteArray(serDe);
    assertEquals(sketchBytes.length, expectedLength);
    // ensure reservoir rebuilds correctly
    final Memory mem = Memory.wrap(sketchBytes);
    final ReservoirItemsSketch<String> loadedRis = ReservoirItemsSketch.heapify(mem, serDe);
    validateReservoirEquality(ris, loadedRis);
    println("Under-full reservoir:");
    println("  Preamble:");
    println(PreambleUtil.preambleToString(mem));
    println("  Sketch:");
    println(ris.toString());
}

Example 9 with ArrayOfStringsSerDe

use of org.apache.datasketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class VarOptItemsSketchTest method checkCorruptSerializedWeight.

@Test
public void checkCorruptSerializedWeight() {
    final VarOptItemsSketch<String> vis = VarOptItemsSketch.newInstance(24);
    for (int i = 1; i < 10; ++i) {
        vis.update(Integer.toString(i), i);
    }
    final byte[] sketchBytes = vis.toByteArray(new ArrayOfStringsSerDe(), String.class);
    final WritableMemory mem = WritableMemory.writableWrap(sketchBytes);
    // weights will be stored in the first double after the preamble
    final int numPreLongs = PreambleUtil.extractPreLongs(mem);
    final int weightOffset = numPreLongs << 3;
    // inject a negative weight
    mem.putDouble(weightOffset, -1.25);
    try {
        VarOptItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
        fail();
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().equals("Possible Corruption: Non-positive weight in " + "heapify(): -1.25"));
    }
}

Example 10 with ArrayOfStringsSerDe

use of org.apache.datasketches.ArrayOfStringsSerDe in project sketches-core by DataSketches.

the class VarOptItemsSketchTest method checkNonEmptyDegenerateSketch.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkNonEmptyDegenerateSketch() {
    // make an empty serialized sketch, then copy the items into a
    // PreambleUtil.VO_WARMUP_PRELONGS-sized byte array
    // so there'll be no items, then clear the empty flag so it will try to load
    // the rest.
    final VarOptItemsSketch<String> vis = VarOptItemsSketch.newInstance(12, ResizeFactor.X2);
    final byte[] sketchBytes = vis.toByteArray(new ArrayOfStringsSerDe());
    final byte[] dstByteArr = new byte[PreambleUtil.VO_PRELONGS_WARMUP << 3];
    final WritableMemory mem = WritableMemory.writableWrap(dstByteArr);
    mem.putByteArray(0, sketchBytes, 0, sketchBytes.length);
    // ensure non-empty but with H and R region sizes set to 0
    // set not-empty
    PreambleUtil.insertFlags(mem, 0);
    PreambleUtil.insertHRegionItemCount(mem, 0);
    PreambleUtil.insertRRegionItemCount(mem, 0);
    VarOptItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
}