Examples with ArrayOfLongsSerDe org.apache.datasketches.ArrayOfLongsSerDe used on opensource projects

Example 11 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkEmptyUnion.

@Test
public void checkEmptyUnion() {
    final ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(1024);
    final byte[] unionBytes = riu.toByteArray(new ArrayOfLongsSerDe());
    // will intentionally break if changing empty union serialization
    assertEquals(unionBytes.length, 8);
    println(riu.toString());
}

Example 12 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkNewGadget.

@Test
public void checkNewGadget() {
    final int maxK = 1024;
    final int bigK = 1536;
    final int smallK = 128;
    // downsample input sketch, use as gadget (exact mode, but irrelevant here)
    final ReservoirItemsSketch<Long> bigKSketch = getBasicSketch(maxK / 2, bigK);
    final byte[] bigKBytes = bigKSketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory bigKMem = Memory.wrap(bigKBytes);
    ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(maxK);
    riu.update(bigKMem, new ArrayOfLongsSerDe());
    assertNotNull(riu.getResult());
    assertEquals(riu.getResult().getK(), maxK);
    assertEquals(riu.getResult().getN(), maxK / 2);
    // sketch k < maxK but in sampling mode
    final ReservoirItemsSketch<Long> smallKSketch = getBasicSketch(maxK, smallK);
    final byte[] smallKBytes = smallKSketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory smallKMem = Memory.wrap(smallKBytes);
    riu = ReservoirItemsUnion.newInstance(maxK);
    riu.update(smallKMem, new ArrayOfLongsSerDe());
    assertNotNull(riu.getResult());
    assertTrue(riu.getResult().getK() < maxK);
    assertEquals(riu.getResult().getK(), smallK);
    assertEquals(riu.getResult().getN(), maxK);
    // sketch k < maxK and in exact mode
    final ReservoirItemsSketch<Long> smallKExactSketch = getBasicSketch(smallK, smallK);
    final byte[] smallKExactBytes = smallKExactSketch.toByteArray(new ArrayOfLongsSerDe());
    final Memory smallKExactMem = Memory.wrap(smallKExactBytes);
    riu = ReservoirItemsUnion.newInstance(maxK);
    riu.update(smallKExactMem, new ArrayOfLongsSerDe());
    assertNotNull(riu.getResult());
    assertEquals(riu.getResult().getK(), maxK);
    assertEquals(riu.getResult().getN(), smallK);
}

Example 13 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkNullUpdate.

/*
  @Test
  public void checkReadOnlyInstantiation() {
    final int k = 100;
    final ReservoirItemsUnion<Long> union = ReservoirItemsUnion.newInstance(k);
    for (long i = 0; i < 2 * k; ++i) {
      union.update(i);
    }

    final byte[] unionBytes = union.toByteArray(new ArrayOfLongsSerDe());
    final Memory mem = Memory.wrap(unionBytes);

    final ReservoirItemsUnion<Long> riu;
    riu = ReservoirItemsUnion.heapify(mem, new ArrayOfLongsSerDe());

    assertNotNull(riu);
    assertEquals(riu.getMaxK(), k);
    ReservoirItemsSketchTest.validateReservoirEquality(riu.getResult(), union.getResult());
  }
  */
@Test
public void checkNullUpdate() {
    final ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(1024);
    assertNull(riu.getResult());
    // null sketch
    final ReservoirItemsSketch<Long> nullSketch = null;
    riu.update(nullSketch);
    assertNull(riu.getResult());
    // null memory
    riu.update(null, new ArrayOfLongsSerDe());
    assertNull(riu.getResult());
    // null item
    riu.update((Long) null);
    assertNull(riu.getResult());
    // valid input
    riu.update(5L);
    assertNotNull(riu.getResult());
}

Example 14 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsUnionTest method checkBadPreLongs.

@Test(expectedExceptions = SketchesArgumentException.class)
public void checkBadPreLongs() {
    final int k = 25;
    final int n = 30;
    final VarOptItemsUnion<Long> union = VarOptItemsUnion.newInstance(k);
    union.update(getUnweightedLongsVIS(k, n));
    final byte[] bytes = union.toByteArray(new ArrayOfLongsSerDe());
    final WritableMemory mem = WritableMemory.writableWrap(bytes);
    // corrupt the preLongs count to 0
    mem.putByte(PREAMBLE_LONGS_BYTE, (byte) (Family.VAROPT.getMinPreLongs() - 1));
    VarOptItemsUnion.heapify(mem, new ArrayOfLongsSerDe());
    fail();
}

Example 15 with ArrayOfLongsSerDe

use of org.apache.datasketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class VarOptItemsUnionTest method unionReservoirVariousTauValues.

@Test
public void unionReservoirVariousTauValues() {
    final int k = 20;
    final long n = 2 * k;
    final VarOptItemsSketch<Long> baseVis = VarOptItemsSketch.newInstance(k);
    for (long i = 1; i <= n; ++i) {
        baseVis.update(-i, 1.0);
    }
    final VarOptItemsUnion<Long> union1 = VarOptItemsUnion.newInstance(k);
    union1.update(baseVis);
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final Memory unionImg = Memory.wrap(union1.toByteArray(serDe));
    final VarOptItemsUnion<Long> union2 = VarOptItemsUnion.heapify(unionImg, serDe);
    // sanity check
    compareUnionsExact(union1, union2);
    // reservoir tau will be greater than gadget's tau
    VarOptItemsSketch<Long> vis = VarOptItemsSketch.newInstance(k);
    ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.newInstance(k);
    for (long i = 1; i < (2 * n); ++i) {
        ris.update(i);
        vis.update(i, 1.0);
    }
    union1.update(vis);
    union2.update(ris);
    compareUnionsEquivalent(union1, union2);
    // reservoir tau will be smaller than gadget's tau
    vis = VarOptItemsSketch.newInstance(k);
    ris = ReservoirItemsSketch.newInstance(k);
    for (long i = 1; i <= (k + 1); ++i) {
        ris.update(i);
        vis.update(i, 1.0);
    }
    union1.update(vis);
    union2.update(ris);
    compareUnionsEquivalent(union1, union2);
}