Examples with MomentSketchWrapper org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper used on opensource projects

Example 6 with MomentSketchWrapper

use of org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper in project druid by druid-io.

the class MomentsSketchAggregatorTest method buildingSketchesAtIngestionTime.

@Test
public void buildingSketchesAtIngestionTime() throws Exception {
    Sequence<ResultRow> seq = helper.createIndexAndRunQueryOnSegment(new File(this.getClass().getClassLoader().getResource("doubles_build_data.tsv").getFile()), String.join("\n", "{", "  \"type\": \"string\",", "  \"parseSpec\": {", "    \"format\": \"tsv\",", "    \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", "    \"dimensionsSpec\": {", "      \"dimensions\": [\"product\"],", "      \"dimensionExclusions\": [ \"sequenceNumber\"],", "      \"spatialDimensions\": []", "    },", "    \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\", \"valueWithNulls\"]", "  }", "}"), "[" + "{\"type\": \"momentSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 10, \"compress\": true}," + "{\"type\": \"momentSketch\", \"name\": \"sketchWithNulls\", \"fieldName\": \"valueWithNulls\", \"k\": 10, \"compress\": true}" + "]", 0, // minTimestamp
    Granularities.NONE, 10, // maxRowCount
    String.join("\n", "{", "  \"queryType\": \"groupBy\",", "  \"dataSource\": \"test_datasource\",", "  \"granularity\": \"ALL\",", "  \"dimensions\": [],", "  \"aggregations\": [", "    {\"type\": \"momentSketchMerge\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 10, \"compress\": true},", "    {\"type\": \"momentSketchMerge\", \"name\": \"sketchWithNulls\", \"fieldName\": \"sketchWithNulls\", \"k\": 10, \"compress\": true}", "  ],", "  \"postAggregations\": [", "    {\"type\": \"momentSketchSolveQuantiles\", \"name\": \"quantiles\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", "    {\"type\": \"momentSketchMin\", \"name\": \"min\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", "    {\"type\": \"momentSketchMax\", \"name\": \"max\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", "    {\"type\": \"momentSketchSolveQuantiles\", \"name\": \"quantilesWithNulls\", \"fractions\": [0, 0.5, 1], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketchWithNulls\"}},", "    {\"type\": \"momentSketchMin\", \"name\": \"minWithNulls\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketchWithNulls\"}},", "    {\"type\": \"momentSketchMax\", \"name\": \"maxWithNulls\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketchWithNulls\"}}", "  ],", "  \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", "}"));
    List<ResultRow> results = seq.toList();
    Assert.assertEquals(1, results.size());
    ResultRow row = results.get(0);
    // "sketch"
    MomentSketchWrapper sketchObject = (MomentSketchWrapper) row.get(0);
    // 400 total products since this is pre-rollup
    Assert.assertEquals(400.0, sketchObject.getPowerSums()[0], 1e-10);
    // "sketchWithNulls"
    MomentSketchWrapper sketchObjectWithNulls = (MomentSketchWrapper) row.get(1);
    // 23 null values (377 when nulls are not replaced with default)
    Assert.assertEquals(NullHandling.replaceWithDefault() ? 400.0 : 377.0, sketchObjectWithNulls.getPowerSums()[0], 1e-10);
    // "quantiles"
    double[] quantilesArray = (double[]) row.get(2);
    Assert.assertEquals(0, quantilesArray[0], 0.05);
    Assert.assertEquals(.5, quantilesArray[1], 0.05);
    Assert.assertEquals(1.0, quantilesArray[2], 0.05);
    // "min"
    Double minValue = (Double) row.get(3);
    Assert.assertEquals(0.0011, minValue, 0.0001);
    // "max"
    Double maxValue = (Double) row.get(4);
    Assert.assertEquals(0.9969, maxValue, 0.0001);
    // "quantilesWithNulls"
    double[] quantilesArrayWithNulls = (double[]) row.get(5);
    Assert.assertEquals(NullHandling.replaceWithDefault() ? 0.0 : 5.0, quantilesArrayWithNulls[0], 0.05);
    Assert.assertEquals(NullHandling.replaceWithDefault() ? 7.721400294818661d : 7.57, quantilesArrayWithNulls[1], 0.05);
    Assert.assertEquals(10.0, quantilesArrayWithNulls[2], 0.05);
    // "minWithNulls"
    Double minValueWithNulls = (Double) row.get(6);
    Assert.assertEquals(NullHandling.replaceWithDefault() ? 0.0 : 5.0164, minValueWithNulls, 0.0001);
    // "maxWithNulls"
    Double maxValueWithNulls = (Double) row.get(7);
    Assert.assertEquals(9.9788, maxValueWithNulls, 0.0001);
}

Example 7 with MomentSketchWrapper

use of org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper in project druid by druid-io.

the class MomentSketchBuildBufferAggregator method aggregate.

@Override
public synchronized void aggregate(final ByteBuffer buffer, final int position) {
    if (selector.isNull()) {
        return;
    }
    ByteBuffer mutationBuffer = buffer.duplicate();
    mutationBuffer.position(position);
    MomentSketchWrapper ms0 = MomentSketchWrapper.fromBytes(mutationBuffer);
    double x = selector.getDouble();
    ms0.add(x);
    mutationBuffer.position(position);
    ms0.toBytes(mutationBuffer);
}

Example 8 with MomentSketchWrapper

use of org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper in project druid by druid-io.

the class MomentSketchMergeAggregator method aggregate.

@Override
public void aggregate() {
    final MomentSketchWrapper sketch = selector.getObject();
    if (sketch == null) {
        return;
    }
    this.momentsSketch.merge(sketch);
}

Example 9 with MomentSketchWrapper

use of org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper in project druid by druid-io.

the class MomentSketchMergeBufferAggregator method aggregate.

@Override
public void aggregate(ByteBuffer buf, int position) {
    MomentSketchWrapper msNext = selector.getObject();
    if (msNext == null) {
        return;
    }
    ByteBuffer mutationBuffer = buf.duplicate();
    mutationBuffer.position(position);
    MomentSketchWrapper ms0 = MomentSketchWrapper.fromBytes(mutationBuffer);
    ms0.merge(msNext);
    mutationBuffer.position(position);
    ms0.toBytes(mutationBuffer);
}