Examples with UpdateDoublesSketch com.yahoo.sketches.quantiles.UpdateDoublesSketch used on opensource projects

Example 6 with UpdateDoublesSketch

use of com.yahoo.sketches.quantiles.UpdateDoublesSketch in project sketches-pig by DataSketches.

the class UnionDoublesSketchTest method accumulator.

@Test
public void accumulator() throws Exception {
    Accumulator<Tuple> func = new UnionDoublesSketch();
    // no input yet
    Tuple resultTuple = func.getValue();
    DoublesSketch sketch = getSketch(resultTuple);
    Assert.assertTrue(sketch.isEmpty());
    // null input tuple
    func.accumulate(null);
    resultTuple = func.getValue();
    sketch = getSketch(resultTuple);
    Assert.assertTrue(sketch.isEmpty());
    // empty input tuple
    func.accumulate(tupleFactory.newTuple());
    resultTuple = func.getValue();
    sketch = getSketch(resultTuple);
    Assert.assertTrue(sketch.isEmpty());
    // empty bag
    func.accumulate(tupleFactory.newTuple(bagFactory.newDefaultBag()));
    resultTuple = func.getValue();
    sketch = getSketch(resultTuple);
    Assert.assertTrue(sketch.isEmpty());
    // normal case
    DataBag bag = bagFactory.newDefaultBag();
    UpdateDoublesSketch inputSketch = DoublesSketch.builder().build();
    inputSketch.update(1.0);
    bag.add(tupleFactory.newTuple(new DataByteArray(inputSketch.toByteArray())));
    func.accumulate(tupleFactory.newTuple(bag));
    func.accumulate(tupleFactory.newTuple(bag));
    resultTuple = func.getValue();
    sketch = getSketch(resultTuple);
    Assert.assertFalse(sketch.isEmpty());
    Assert.assertEquals(sketch.getN(), 2);
    // cleanup
    func.cleanup();
    resultTuple = func.getValue();
    sketch = getSketch(resultTuple);
    Assert.assertTrue(sketch.isEmpty());
}

Example 7 with UpdateDoublesSketch

use of com.yahoo.sketches.quantiles.UpdateDoublesSketch in project sketches-pig by DataSketches.

the class ArrayOfDoublesSketchToQuantilesSketch method exec.

@Override
public DataByteArray exec(final Tuple input) throws IOException {
    if ((input == null) || (input.size() == 0)) {
        return null;
    }
    final DataByteArray dba = (DataByteArray) input.get(0);
    final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(dba.get()));
    int column = 1;
    if (input.size() > 1) {
        column = (int) input.get(1);
        if ((column < 1) || (column > sketch.getNumValues())) {
            throw new IllegalArgumentException("Column number out of range. The given sketch has " + sketch.getNumValues() + " columns");
        }
    }
    final DoublesSketchBuilder builder = DoublesSketch.builder();
    if (k > 0) {
        builder.setK(k);
    }
    final UpdateDoublesSketch qs = builder.build();
    final ArrayOfDoublesSketchIterator it = sketch.iterator();
    while (it.next()) {
        qs.update(it.getValues()[column - 1]);
    }
    return new DataByteArray(qs.compact().toByteArray());
}

Example 8 with UpdateDoublesSketch

use of com.yahoo.sketches.quantiles.UpdateDoublesSketch in project sketches-pig by DataSketches.

the class DoubleSummarySketchToPercentile method exec.

@Override
public Double exec(final Tuple input) throws IOException {
    if (input.size() != 2) {
        throw new IllegalArgumentException("expected two inputs: sketch and pecentile");
    }
    final DataByteArray dba = (DataByteArray) input.get(0);
    final Sketch<DoubleSummary> sketch = Sketches.heapifySketch(Memory.wrap(dba.get()), SUMMARY_DESERIALIZER);
    final double percentile = (double) input.get(1);
    if ((percentile < 0) || (percentile > 100)) {
        throw new IllegalArgumentException("percentile must be between 0 and 100");
    }
    final UpdateDoublesSketch qs = DoublesSketch.builder().setK(QUANTILES_SKETCH_SIZE).build();
    final SketchIterator<DoubleSummary> it = sketch.iterator();
    while (it.next()) {
        qs.update(it.getSummary().getValue());
    }
    return qs.getQuantile(percentile / 100);
}

Example 9 with UpdateDoublesSketch

use of com.yahoo.sketches.quantiles.UpdateDoublesSketch in project gaffer-doc by gchq.

the class DoublesSketchElementGenerator method _apply.

@Override
public Iterable<Element> _apply(final String line) {
    final List<Element> elements = new ArrayList<>();
    for (int i = 0; i < 1000; i++) {
        final UpdateDoublesSketch doublesSketch = DoublesSketch.builder().build();
        doublesSketch.update(RANDOM.nextGaussian());
        final Edge edge = new Edge.Builder().group("red").source("A").dest("B").property("doublesSketch", doublesSketch).build();
        elements.add(edge);
    }
    return elements;
}

Example 10 with UpdateDoublesSketch

use of com.yahoo.sketches.quantiles.UpdateDoublesSketch in project Gaffer by gchq.

the class DoublesSketchSerialiserTest method testSerialiseAndDeserialise.

@Test
public void testSerialiseAndDeserialise() {
    final UpdateDoublesSketch sketch = DoublesSketch.builder().build();
    sketch.update(1.0D);
    sketch.update(2.0D);
    sketch.update(3.0D);
    testSerialiser(sketch);
    final DoublesSketch emptySketch = DoublesSketch.builder().build();
    testSerialiser(emptySketch);
}