Examples with ArrayOfLongsSerDe org.apache.datasketches.ArrayOfLongsSerDe used on opensource projects

Example 21 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsSketchTest method checkEndOfWarmupSketchSerialization.

@Test
public void checkEndOfWarmupSketchSerialization() {
    final int k = 2048;
    final VarOptItemsSketch<Long> sketch = getUnweightedLongsVIS(k, k);
    final byte[] bytes = sketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory mem = Memory.wrap(bytes);
    // ensure still only 2 preLongs
    assertEquals(PreambleUtil.extractPreLongs(mem), PreambleUtil.VO_PRELONGS_WARMUP);
    final VarOptItemsSketch<Long> rebuilt = VarOptItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
    checkIfEqual(rebuilt, sketch);
}

Example 22 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsSketchTest method checkFullSketchSerialization.

@Test
public void checkFullSketchSerialization() {
    final VarOptItemsSketch<Long> sketch = VarOptItemsSketch.newInstance(32);
    for (long i = 0; i < 32; ++i) {
        sketch.update(i, 1.0);
    }
    sketch.update(100L, 100.0);
    sketch.update(101L, 101.0);
    assertEquals(sketch.getNumSamples(), 32);
    // first 2 entries should be heavy and in heap order (smallest at root)
    final VarOptItemsSamples<Long> samples = sketch.getSketchSamples();
    final Long[] data = samples.items();
    final double[] weights = samples.weights();
    assertEquals(weights[0], 100.0);
    assertEquals(weights[1], 101.0);
    assertEquals((long) data[0], 100L);
    assertEquals((long) data[1], 101L);
    final byte[] bytes = sketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory mem = Memory.wrap(bytes);
    // ensure 3 preLongs
    assertEquals(PreambleUtil.extractPreLongs(mem), Family.VAROPT.getMaxPreLongs());
    final VarOptItemsSketch<Long> rebuilt = VarOptItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
    checkIfEqual(rebuilt, sketch);
}

Example 23 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsSketchTest method checkMalformedPreamble.

@Test
public void checkMalformedPreamble() {
    final int k = 50;
    final VarOptItemsSketch<Long> sketch = getUnweightedLongsVIS(k, k);
    final byte[] sketchBytes = sketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory srcMem = Memory.wrap(sketchBytes);
    // we'll use the same initial sketch a few times, so grab a copy of it
    final byte[] copyBytes = new byte[sketchBytes.length];
    final WritableMemory mem = WritableMemory.writableWrap(copyBytes);
    // copy the bytes
    srcMem.copyTo(0, mem, 0, sketchBytes.length);
    assertEquals(PreambleUtil.extractPreLongs(mem), PreambleUtil.VO_PRELONGS_WARMUP);
    // no items in R but max preLongs
    try {
        PreambleUtil.insertPreLongs(mem, Family.VAROPT.getMaxPreLongs());
        VarOptItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
        fail();
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().startsWith("Possible Corruption: " + Family.VAROPT.getMaxPreLongs() + " preLongs but"));
    }
    // refresh the copy
    srcMem.copyTo(0, mem, 0, sketchBytes.length);
    assertEquals(PreambleUtil.extractPreLongs(mem), PreambleUtil.VO_PRELONGS_WARMUP);
    // negative H region count
    try {
        PreambleUtil.insertHRegionItemCount(mem, -1);
        VarOptItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
        fail();
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().equals("Possible Corruption: H region count cannot be negative: -1"));
    }
    // refresh the copy
    srcMem.copyTo(0, mem, 0, sketchBytes.length);
    assertEquals(PreambleUtil.extractHRegionItemCount(mem), k);
    // negative R region count
    try {
        PreambleUtil.insertRRegionItemCount(mem, -128);
        VarOptItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
        fail();
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().equals("Possible Corruption: R region count cannot be negative: -128"));
    }
    // refresh the copy
    srcMem.copyTo(0, mem, 0, sketchBytes.length);
    assertEquals(PreambleUtil.extractRRegionItemCount(mem), 0);
    // invalid k < 1
    try {
        PreambleUtil.insertK(mem, 0);
        VarOptItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
        fail();
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().equals("Possible Corruption: k must be at least 1: 0"));
    }
    // refresh the copy
    srcMem.copyTo(0, mem, 0, sketchBytes.length);
    assertEquals(PreambleUtil.extractK(mem), k);
    // invalid n < 0
    try {
        PreambleUtil.insertN(mem, -1024);
        VarOptItemsSketch.heapify(mem, new ArrayOfLongsSerDe());
        fail();
    } catch (final SketchesArgumentException e) {
        assertTrue(e.getMessage().equals("Possible Corruption: n cannot be negative: -1024"));
    }
}

Example 24 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsUnionTest method serializeSamplingUnion.

@Test
public void serializeSamplingUnion() {
    final int n = 256;
    final int k = 128;
    final VarOptItemsSketch<Long> sketch = getUnweightedLongsVIS(k, n);
    sketch.update(n + 1L, 1000.0);
    sketch.update(n + 2L, 1001.0);
    sketch.update(n + 3L, 1002.0);
    sketch.update(n + 4L, 1003.0);
    sketch.update(n + 5L, 1004.0);
    sketch.update(n + 6L, 1005.0);
    sketch.update(n + 7L, 1006.0);
    sketch.update(n + 8L, 1007.0);
    final VarOptItemsUnion<Long> union = VarOptItemsUnion.newInstance(k);
    union.update(sketch);
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final byte[] unionBytes = union.toByteArray(serDe);
    final Memory mem = Memory.wrap(unionBytes);
    final VarOptItemsUnion<Long> rebuilt = VarOptItemsUnion.heapify(mem, serDe);
    compareUnionsExact(rebuilt, union);
    assertEquals(rebuilt.toString(), union.toString());
}

Example 25 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsUnionTest method checkBadSerVer.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkBadSerVer() {
    final int k = 25;
    final int n = 30;
    final VarOptItemsUnion<Long> union = VarOptItemsUnion.newInstance(k);
    union.update(getUnweightedLongsVIS(k, n));
    final byte[] bytes = union.toByteArray(new ArrayOfLongsSerDe());
    final WritableMemory mem = WritableMemory.writableWrap(bytes);
    // corrupt the serialization version
    mem.putByte(SER_VER_BYTE, (byte) 0);
    VarOptItemsUnion.heapify(mem, new ArrayOfLongsSerDe());
    fail();
}