Examples with WritableMemory org.apache.datasketches.memory.WritableMemory used on opensource projects

Example 51 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class UnionCaseTest method buildMemoryUnion.

private Union buildMemoryUnion(int lgMaxK, int n) {
    final int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgMaxK, TgtHllType.HLL_8);
    WritableMemory wmem = WritableMemory.allocate(bytes);
    Union u = new Union(lgMaxK, wmem);
    for (int i = 0; i < n; i++) {
        u.update(i + v);
    }
    v += n;
    return u;
}

Example 52 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class BaseHllSketchTest method misc.

@Test
public void misc() {
    HllSketch sk = new HllSketch(10, TgtHllType.HLL_4);
    assertTrue(sk.isEstimationMode());
    sk.reset();
    assertEquals(BaseHllSketch.getSerializationVersion(), PreambleUtil.SER_VER);
    WritableMemory wmem = WritableMemory.writableWrap(sk.toCompactByteArray());
    assertEquals(BaseHllSketch.getSerializationVersion(wmem), PreambleUtil.SER_VER);
}

Example 53 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class ArrayOfDoublesUnionTest method directDruidUsageTwoSketches.

@Test
public void directDruidUsageTwoSketches() {
    final WritableMemory mem = WritableMemory.writableWrap(new byte[1000000]);
    // just set up memory to wrap later
    new ArrayOfDoublesSetOperationBuilder().buildUnion(mem);
    int key = 0;
    // estimation mode
    final int n1 = 100000;
    final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build();
    for (int i = 0; i < n1; i++) {
        sketch1.update(key++, new double[] { 1.0 });
    }
    // as Druid wraps memory
    ArrayOfDoublesSketches.wrapUnion(mem).union(sketch1.compact(WritableMemory.writableWrap(new byte[1000000])));
    // estimation mode
    final int n2 = 1000000;
    final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build();
    for (int i = 0; i < n2; i++) {
        sketch2.update(key++, new double[] { 1.0 });
    }
    // as Druid wraps memory
    ArrayOfDoublesSketches.wrapUnion(mem).union(sketch2.compact(WritableMemory.writableWrap(new byte[1000000])));
    // build one sketch that must be the same as union
    // reset to have the same keys
    key = 0;
    final int n = n1 + n2;
    final ArrayOfDoublesUpdatableSketch expected = new ArrayOfDoublesUpdatableSketchBuilder().build();
    for (int i = 0; i < n; i++) {
        expected.update(key++, new double[] { 1.0 });
    }
    // union result is trimmed, so we need to trim this sketch for valid comparison
    expected.trim();
    final ArrayOfDoublesSketch result = ArrayOfDoublesUnion.wrap(mem).getResult();
    Assert.assertEquals(result.getEstimate(), expected.getEstimate());
    Assert.assertEquals(result.isEstimationMode(), expected.isEstimationMode());
    Assert.assertEquals(result.getUpperBound(1), expected.getUpperBound(1));
    Assert.assertEquals(result.getLowerBound(1), expected.getLowerBound(1));
    Assert.assertEquals(result.getRetainedEntries(), expected.getRetainedEntries());
    Assert.assertEquals(result.getNumValues(), expected.getNumValues());
}

Example 54 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class ArrayOfDoublesUnionTest method heapExactMode.

@Test
public void heapExactMode() {
    final ArrayOfDoublesUpdatableSketch sketch1 = new ArrayOfDoublesUpdatableSketchBuilder().build();
    sketch1.update(1, new double[] { 1.0 });
    sketch1.update(1, new double[] { 1.0 });
    sketch1.update(1, new double[] { 1.0 });
    sketch1.update(2, new double[] { 1.0 });
    final ArrayOfDoublesUpdatableSketch sketch2 = new ArrayOfDoublesUpdatableSketchBuilder().build();
    sketch2.update(2, new double[] { 1.0 });
    sketch2.update(2, new double[] { 1.0 });
    sketch2.update(3, new double[] { 1.0 });
    sketch2.update(3, new double[] { 1.0 });
    sketch2.update(3, new double[] { 1.0 });
    final ArrayOfDoublesUnion union = new ArrayOfDoublesSetOperationBuilder().buildUnion();
    union.union(sketch1);
    union.union(sketch2);
    final int maxBytes = ArrayOfDoublesUnion.getMaxBytes(ArrayOfDoublesSetOperationBuilder.DEFAULT_NOMINAL_ENTRIES, ArrayOfDoublesSetOperationBuilder.DEFAULT_NUMBER_OF_VALUES);
    // 48 bytes preamble + 2 * nominal entries * (key size + value size)
    Assert.assertEquals(maxBytes, 131120);
    ArrayOfDoublesCompactSketch result = union.getResult();
    Assert.assertEquals(result.getEstimate(), 3.0);
    double[][] values = result.getValues();
    Assert.assertEquals(values[0][0], 3.0);
    Assert.assertEquals(values[1][0], 3.0);
    Assert.assertEquals(values[2][0], 3.0);
    final WritableMemory wmem = WritableMemory.writableWrap(union.toByteArray());
    final ArrayOfDoublesUnion wrappedUnion = ArrayOfDoublesSketches.wrapUnion(wmem);
    result = wrappedUnion.getResult();
    Assert.assertEquals(result.getEstimate(), 3.0);
    values = result.getValues();
    Assert.assertEquals(values[0][0], 3.0);
    Assert.assertEquals(values[1][0], 3.0);
    Assert.assertEquals(values[2][0], 3.0);
    union.reset();
    result = union.getResult();
    Assert.assertTrue(result.isEmpty());
    Assert.assertFalse(result.isEstimationMode());
    Assert.assertEquals(result.getEstimate(), 0.0);
    Assert.assertEquals(result.getUpperBound(1), 0.0);
    Assert.assertEquals(result.getLowerBound(1), 0.0);
    Assert.assertEquals(result.getTheta(), 1.0);
}

Example 55 with WritableMemory

use of org.apache.datasketches.memory.WritableMemory in project sketches-core by DataSketches.

the class ReservoirLongsSketch method toByteArray.

/**
 * Returns a byte array representation of this sketch
 *
 * @return a byte array representation of this sketch
 */
public byte[] toByteArray() {
    final int preLongs, outBytes;
    final boolean empty = itemsSeen_ == 0;
    final int numItems = (int) Math.min(reservoirSize_, itemsSeen_);
    if (empty) {
        preLongs = 1;
        outBytes = 8;
    } else {
        preLongs = Family.RESERVOIR.getMaxPreLongs();
        // for longs, we know the size
        outBytes = (preLongs + numItems) << 3;
    }
    final byte[] outArr = new byte[outBytes];
    final WritableMemory mem = WritableMemory.writableWrap(outArr);
    // build first preLong
    // Byte 0
    PreambleUtil.insertPreLongs(mem, preLongs);
    PreambleUtil.insertLgResizeFactor(mem, rf_.lg());
    // Byte 1
    PreambleUtil.insertSerVer(mem, SER_VER);
    // Byte 2
    PreambleUtil.insertFamilyID(mem, Family.RESERVOIR.getID());
    if (empty) {
        // Byte 3
        PreambleUtil.insertFlags(mem, EMPTY_FLAG_MASK);
    } else {
        PreambleUtil.insertFlags(mem, 0);
    }
    // Bytes 4-7
    PreambleUtil.insertK(mem, reservoirSize_);
    if (!empty) {
        // second preLong, only if non-empty
        PreambleUtil.insertN(mem, itemsSeen_);
        // insert the serialized samples, offset by the preamble size
        final int preBytes = preLongs << 3;
        mem.putLongArray(preBytes, data_, 0, numItems);
    }
    return outArr;
}