Examples with Memory org.apache.datasketches.memory.Memory used on opensource projects

Example 6 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class LongsSketchTest method frequentItemsByteSerialTest.

@Test
public void frequentItemsByteSerialTest() {
    // Empty Sketch
    LongsSketch sketch = new LongsSketch(16);
    byte[] bytearray0 = sketch.toByteArray();
    WritableMemory mem0 = WritableMemory.writableWrap(bytearray0);
    LongsSketch new_sketch0 = LongsSketch.getInstance(mem0);
    String str0 = LongsSketch.toString(mem0);
    println(str0);
    String string0 = sketch.serializeToString();
    String new_string0 = new_sketch0.serializeToString();
    Assert.assertTrue(string0.equals(new_string0));
    LongsSketch sketch2 = new LongsSketch(128);
    sketch.update(10, 100);
    sketch.update(10, 100);
    sketch.update(15, 3443);
    sketch.update(1000001, 1010230);
    sketch.update(1000002, 1010230);
    byte[] bytearray1 = sketch.toByteArray();
    Memory mem1 = Memory.wrap(bytearray1);
    LongsSketch new_sketch1 = LongsSketch.getInstance(mem1);
    String str1 = LongsSketch.toString(bytearray1);
    println(str1);
    String string1 = sketch.serializeToString();
    String new_string1 = new_sketch1.serializeToString();
    Assert.assertTrue(string1.equals(new_string1));
    Assert.assertTrue(new_sketch1.getMaximumMapCapacity() == sketch.getMaximumMapCapacity());
    Assert.assertTrue(new_sketch1.getCurrentMapCapacity() == sketch.getCurrentMapCapacity());
    sketch2.update(190, 12902390);
    sketch2.update(191, 12902390);
    sketch2.update(192, 12902390);
    sketch2.update(193, 12902390);
    sketch2.update(194, 12902390);
    sketch2.update(195, 12902390);
    sketch2.update(196, 12902390);
    sketch2.update(197, 12902390);
    sketch2.update(198, 12902390);
    sketch2.update(199, 12902390);
    sketch2.update(200, 12902390);
    sketch2.update(201, 12902390);
    sketch2.update(202, 12902390);
    sketch2.update(203, 12902390);
    sketch2.update(204, 12902390);
    sketch2.update(205, 12902390);
    sketch2.update(206, 12902390);
    sketch2.update(207, 12902390);
    sketch2.update(208, 12902390);
    byte[] bytearray2 = sketch2.toByteArray();
    Memory mem2 = Memory.wrap(bytearray2);
    LongsSketch new_sketch2 = LongsSketch.getInstance(mem2);
    String string2 = sketch2.serializeToString();
    String new_string2 = new_sketch2.serializeToString();
    Assert.assertTrue(string2.equals(new_string2));
    Assert.assertTrue(new_sketch2.getMaximumMapCapacity() == sketch2.getMaximumMapCapacity());
    Assert.assertTrue(new_sketch2.getCurrentMapCapacity() == sketch2.getCurrentMapCapacity());
    Assert.assertTrue(new_sketch2.getStreamLength() == sketch2.getStreamLength());
    LongsSketch merged_sketch = sketch.merge(sketch2);
    byte[] bytearray = sketch.toByteArray();
    Memory mem = Memory.wrap(bytearray);
    LongsSketch new_sketch = LongsSketch.getInstance(mem);
    String string = sketch.serializeToString();
    String new_string = new_sketch.serializeToString();
    Assert.assertTrue(string.equals(new_string));
    Assert.assertTrue(new_sketch.getMaximumMapCapacity() == merged_sketch.getMaximumMapCapacity());
    Assert.assertTrue(new_sketch.getCurrentMapCapacity() == merged_sketch.getCurrentMapCapacity());
    Assert.assertTrue(new_sketch.getStreamLength() == merged_sketch.getStreamLength());
}

Example 7 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class DoublesUnionImplTest method checkUnionQuantiles.

@Test
public void checkUnionQuantiles() {
    final int k = 128;
    final int n1 = k * 13;
    final int n2 = (k * 8) + (k / 2);
    final int n = n1 + n2;
    // assuming k = 128
    final double errorTolerance = 0.0175 * n;
    final UpdateDoublesSketch sketch1 = buildAndLoadQS(k, n1);
    final CompactDoublesSketch sketch2 = buildAndLoadQS(k, n2, n1).compact();
    // virgin 256
    final DoublesUnion union = DoublesUnion.builder().setMaxK(256).build();
    union.update(sketch2);
    union.update(sketch1);
    final Memory mem = Memory.wrap(union.getResult().toByteArray(true));
    final DoublesSketch result = DoublesSketch.wrap(mem);
    assertEquals(result.getN(), n1 + n2);
    assertEquals(result.getK(), k);
    for (double fraction = 0.05; fraction < 1.0; fraction += 0.05) {
        assertEquals(result.getQuantile(fraction), fraction * n, errorTolerance);
    }
}

Example 8 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class ForwardCompatibilityTest method getAndCheck.

private static void getAndCheck(String ver, int n, double quantile) {
    // make deterministic
    DoublesSketch.rand.setSeed(131);
    // create fileName
    int k = 128;
    double nf = 0.5;
    String fileName = String.format("Qk%d_n%d_v%s.sk", k, n, ver);
    println("fullName: " + fileName);
    println("Old Median: " + quantile);
    // Read File bytes
    byte[] byteArr2 = getResourceBytes(fileName);
    Memory srcMem = Memory.wrap(byteArr2);
    // heapify as update sketch
    DoublesSketch qs2 = UpdateDoublesSketch.heapify(srcMem);
    // Test the quantile
    double q2 = qs2.getQuantile(nf);
    println("New Median: " + q2);
    Assert.assertEquals(q2, quantile, 0.0);
    // same thing with compact sketch
    qs2 = CompactDoublesSketch.heapify(srcMem);
    // Test the quantile
    q2 = qs2.getQuantile(nf);
    println("New Median: " + q2);
    Assert.assertEquals(q2, quantile, 0.0);
}

Example 9 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class HeapUpdateDoublesSketchTest method checkPutMemory.

@Test
public void checkPutMemory() {
    // k = 128
    UpdateDoublesSketch qs1 = DoublesSketch.builder().build();
    for (int i = 0; i < 1000; i++) {
        qs1.update(i);
    }
    int bytes = qs1.getUpdatableStorageBytes();
    WritableMemory dstMem = WritableMemory.writableWrap(new byte[bytes]);
    qs1.putMemory(dstMem, false);
    Memory srcMem = dstMem;
    DoublesSketch qs2 = DoublesSketch.heapify(srcMem);
    assertEquals(qs1.getMinValue(), qs2.getMinValue(), 0.0);
    assertEquals(qs1.getMaxValue(), qs2.getMaxValue(), 0.0);
}

Example 10 with Memory

use of org.apache.datasketches.memory.Memory in project sketches-core by DataSketches.

the class HeapUpdateDoublesSketchTest method serializeDeserializeEmptyNonCompact.

@Test
public void serializeDeserializeEmptyNonCompact() {
    UpdateDoublesSketch sketch1 = DoublesSketch.builder().build();
    // Ordered, Not Compact, Empty
    byte[] byteArr = sketch1.toByteArray(false);
    assertEquals(byteArr.length, sketch1.getStorageBytes());
    Memory mem = Memory.wrap(byteArr);
    UpdateDoublesSketch sketch2 = (UpdateDoublesSketch) DoublesSketch.heapify(mem);
    for (int i = 0; i < 1000; i++) {
        sketch2.update(i);
    }
    assertEquals(sketch2.getMinValue(), 0.0);
    assertEquals(sketch2.getMaxValue(), 999.0);
    assertEquals(sketch2.getQuantile(0.5), 500.0, 4.0);
}