Examples with ArrayOfLongsSerDe org.apache.datasketches.ArrayOfLongsSerDe used on opensource projects

Example 26 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkInstantiation.

@Test
public void checkInstantiation() {
    final int n = 100;
    final int k = 25;
    // create empty unions
    ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(k);
    assertNull(riu.getResult());
    riu.update(5L);
    assertNotNull(riu.getResult());
    // pass in a sketch, as both an object and memory
    final ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.newInstance(k);
    for (long i = 0; i < n; ++i) {
        ris.update(i);
    }
    riu.reset();
    assertEquals(riu.getResult().getN(), 0);
    riu.update(ris);
    assertEquals(riu.getResult().getN(), ris.getN());
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    // only the gadget is serialized
    final byte[] sketchBytes = ris.toByteArray(serDe);
    final Memory mem = Memory.wrap(sketchBytes);
    riu = ReservoirItemsUnion.newInstance(ris.getK());
    riu.update(mem, serDe);
    assertNotNull(riu.getResult());
    println(riu.toString());
}

Example 27 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkStandardMergeNoCopy.

@Test
public void checkStandardMergeNoCopy() {
    final int k = 1024;
    final int n1 = 256;
    final int n2 = 256;
    final ReservoirItemsSketch<Long> sketch1 = getBasicSketch(n1, k);
    final ReservoirItemsSketch<Long> sketch2 = getBasicSketch(n2, k);
    final ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(k);
    riu.update(sketch1);
    riu.update(sketch2);
    assertNotNull(riu.getResult());
    assertEquals(riu.getResult().getK(), k);
    assertEquals(riu.getResult().getN(), n1 + n2);
    assertEquals(riu.getResult().getNumSamples(), n1 + n2);
    // creating from Memory should avoid a copy
    final int n3 = 2048;
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final ReservoirItemsSketch<Long> sketch3 = getBasicSketch(n3, k);
    final byte[] sketch3Bytes = sketch3.toByteArray(serDe);
    final Memory mem = Memory.wrap(sketch3Bytes);
    riu.update(mem, serDe);
    assertEquals(riu.getResult().getK(), k);
    assertEquals(riu.getResult().getN(), n1 + n2 + n3);
    assertEquals(riu.getResult().getNumSamples(), k);
}

Example 28 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsUnionTest method unionExactReservoirSketch.

@Test
public void unionExactReservoirSketch() {
    // build a varopt union which contains both heavy and light items, then copy it and
    // compare unioning:
    // 1. A varopt sketch of items with weight 1.0
    // 2. A reservoir sample made of the same input items as above
    // and we should find that the resulting unions are equivalent.
    final int k = 20;
    final long n = 2 * k;
    final VarOptItemsSketch<Long> baseVis = VarOptItemsSketch.newInstance(k);
    for (long i = 1; i <= n; ++i) {
        baseVis.update(-i, i);
    }
    baseVis.update(-n - 1L, n * n);
    baseVis.update(-n - 2L, n * n);
    baseVis.update(-n - 3L, n * n);
    final VarOptItemsUnion<Long> union1 = VarOptItemsUnion.newInstance(k);
    union1.update(baseVis);
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final Memory unionImg = Memory.wrap(union1.toByteArray(serDe));
    final VarOptItemsUnion<Long> union2 = VarOptItemsUnion.heapify(unionImg, serDe);
    // sanity check
    compareUnionsExact(union1, union2);
    final VarOptItemsSketch<Long> vis = VarOptItemsSketch.newInstance(k);
    final ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.newInstance(k);
    union2.update((ReservoirItemsSketch<Long>) null);
    // empty
    union2.update(ris);
    // union2 should be unchanged
    compareUnionsExact(union1, union2);
    for (long i = 1; i < (k - 1); ++i) {
        ris.update(i);
        vis.update(i, 1.0);
    }
    union1.update(vis);
    union2.update(ris);
    compareUnionsEquivalent(union1, union2);
}

Example 29 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsUnionTest method serializeExactUnion.

@Test
public void serializeExactUnion() {
    final int n1 = 32;
    final int n2 = 64;
    final int k = 128;
    final VarOptItemsSketch<Long> sketch1 = getUnweightedLongsVIS(k, n1);
    final VarOptItemsSketch<Long> sketch2 = getUnweightedLongsVIS(k, n2);
    final VarOptItemsUnion<Long> union = VarOptItemsUnion.newInstance(k);
    union.update(sketch1);
    union.update(sketch2);
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final byte[] unionBytes = union.toByteArray(serDe);
    final Memory mem = Memory.wrap(unionBytes);
    final VarOptItemsUnion<Long> rebuilt = VarOptItemsUnion.heapify(mem, serDe);
    compareUnionsExact(rebuilt, union);
    assertEquals(rebuilt.toString(), union.toString());
}

Example 30 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkSerialization.

@Test
public void checkSerialization() {
    final int n = 100;
    final int k = 25;
    final ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(k);
    for (long i = 0; i < n; ++i) {
        riu.update(i);
    }
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final byte[] unionBytes = riu.toByteArray(serDe);
    final Memory mem = Memory.wrap(unionBytes);
    println(PreambleUtil.preambleToString(mem));
    final ReservoirItemsUnion<Long> rebuiltUnion = ReservoirItemsUnion.heapify(mem, serDe);
    assertEquals(riu.getMaxK(), rebuiltUnion.getMaxK());
    ReservoirItemsSketchTest.validateReservoirEquality(riu.getResult(), rebuiltUnion.getResult());
}