Examples with ArrayOfLongsSerDe com.yahoo.sketches.ArrayOfLongsSerDe used on opensource projects

Example 6 with ArrayOfLongsSerDe

use of com.yahoo.sketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method checkVersionConversion.

/*
  @Test
  public void checkReadOnlyHeapify() {
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    Memory sketchMem = getBasicSerializedLongsRIS();

    // Load from read-only and writable memory to ensure they deserialize identically
    ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.heapify(sketchMem.asReadOnlyMemory(),
            serDe);
    ReservoirItemsSketch<Long> fromWritable = ReservoirItemsSketch.heapify(sketchMem, serDe);
    validateReservoirEquality(ris, fromWritable);

    // Same with an empty sketch
    final byte[] sketchBytes = ReservoirItemsSketch.<Long>newInstance(32).toByteArray(serDe);
    sketchMem = new NativeMemory(sketchBytes);

    ris = ReservoirItemsSketch.heapify(sketchMem.asReadOnlyMemory(), serDe);
    fromWritable = ReservoirItemsSketch.heapify(sketchMem, serDe);
    validateReservoirEquality(ris, fromWritable);
  }
  */
@Test
public void checkVersionConversion() {
    // version change from 1 to 2 only impact first preamble long, so empty sketch is sufficient
    final int k = 32768;
    final short encK = ReservoirSize.computeSize(k);
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.newInstance(k);
    final byte[] sketchBytesOrig = ris.toByteArray(serDe);
    // get a new byte[], manually revert to v1, then reconstruct
    final byte[] sketchBytes = ris.toByteArray(serDe);
    final WritableMemory sketchMem = WritableMemory.wrap(sketchBytes);
    sketchMem.putByte(SER_VER_BYTE, (byte) 1);
    // zero out all 4 bytes
    sketchMem.putInt(RESERVOIR_SIZE_INT, 0);
    sketchMem.putShort(RESERVOIR_SIZE_SHORT, encK);
    println(PreambleUtil.preambleToString(sketchMem));
    final ReservoirItemsSketch<Long> rebuilt = ReservoirItemsSketch.heapify(sketchMem, serDe);
    final byte[] rebuiltBytes = rebuilt.toByteArray(serDe);
    assertEquals(sketchBytesOrig.length, rebuiltBytes.length);
    for (int i = 0; i < sketchBytesOrig.length; ++i) {
        assertEquals(sketchBytesOrig[i], rebuiltBytes[i]);
    }
}

Example 7 with ArrayOfLongsSerDe

use of com.yahoo.sketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsSketchTest method getBasicSerializedLongsRIS.

private static WritableMemory getBasicSerializedLongsRIS() {
    final int k = 10;
    final int n = 20;
    final ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.newInstance(k);
    assertEquals(ris.getNumSamples(), 0);
    for (int i = 0; i < n; ++i) {
        ris.update((long) i);
    }
    assertEquals(ris.getNumSamples(), Math.min(n, k));
    assertEquals(ris.getN(), n);
    assertEquals(ris.getK(), k);
    final byte[] sketchBytes = ris.toByteArray(new ArrayOfLongsSerDe());
    return WritableMemory.wrap(sketchBytes);
}

Example 8 with ArrayOfLongsSerDe

use of com.yahoo.sketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkSerialization.

@Test
public void checkSerialization() {
    final int n = 100;
    final int k = 25;
    final ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(k);
    for (long i = 0; i < n; ++i) {
        riu.update(i);
    }
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    final byte[] unionBytes = riu.toByteArray(serDe);
    final Memory mem = Memory.wrap(unionBytes);
    println(PreambleUtil.preambleToString(mem));
    final ReservoirItemsUnion<Long> rebuiltUnion = ReservoirItemsUnion.heapify(mem, serDe);
    assertEquals(riu.getMaxK(), rebuiltUnion.getMaxK());
    ReservoirItemsSketchTest.validateReservoirEquality(riu.getResult(), rebuiltUnion.getResult());
}

Example 9 with ArrayOfLongsSerDe

use of com.yahoo.sketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkInstantiation.

@Test
public void checkInstantiation() {
    final int n = 100;
    final int k = 25;
    // create empty unions
    ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(k);
    assertNull(riu.getResult());
    riu.update(5L);
    assertNotNull(riu.getResult());
    // pass in a sketch, as both an object and memory
    final ReservoirItemsSketch<Long> ris = ReservoirItemsSketch.newInstance(k);
    for (long i = 0; i < n; ++i) {
        ris.update(i);
    }
    riu.reset();
    assertEquals(riu.getResult().getN(), 0);
    riu.update(ris);
    assertEquals(riu.getResult().getN(), ris.getN());
    final ArrayOfLongsSerDe serDe = new ArrayOfLongsSerDe();
    // only the gadget is serialized
    final byte[] sketchBytes = ris.toByteArray(serDe);
    final Memory mem = Memory.wrap(sketchBytes);
    riu = ReservoirItemsUnion.newInstance(ris.getK());
    riu.update(mem, serDe);
    assertNotNull(riu.getResult());
    println(riu.toString());
}

Example 10 with ArrayOfLongsSerDe

use of com.yahoo.sketches.ArrayOfLongsSerDe in project sketches-core by DataSketches.

the class ReservoirItemsUnionTest method checkNullUpdate.

/*
  @Test
  public void checkReadOnlyInstantiation() {
    final int k = 100;
    final ReservoirItemsUnion<Long> union = ReservoirItemsUnion.newInstance(k);
    for (long i = 0; i < 2 * k; ++i) {
      union.update(i);
    }

    final byte[] unionBytes = union.toByteArray(new ArrayOfLongsSerDe());
    final Memory mem = Memory.wrap(unionBytes);

    final ReservoirItemsUnion<Long> riu;
    riu = ReservoirItemsUnion.heapify(mem, new ArrayOfLongsSerDe());

    assertNotNull(riu);
    assertEquals(riu.getMaxK(), k);
    ReservoirItemsSketchTest.validateReservoirEquality(riu.getResult(), union.getResult());
  }
  */
@Test
public void checkNullUpdate() {
    final ReservoirItemsUnion<Long> riu = ReservoirItemsUnion.newInstance(1024);
    assertNull(riu.getResult());
    // null sketch
    final ReservoirItemsSketch<Long> nullSketch = null;
    riu.update(nullSketch);
    assertNull(riu.getResult());
    // null memory
    riu.update(null, new ArrayOfLongsSerDe());
    assertNull(riu.getResult());
    // null item
    riu.update((Long) null);
    assertNull(riu.getResult());
    // valid input
    riu.update(5L);
    assertNotNull(riu.getResult());
}