use of org.apache.druid.query.groupby.ResultRow in project druid by druid-io.
the class SketchAggregationWithSimpleDataTest method testSimpleDataIngestAndGpByQuery.
@Test
public void testSimpleDataIngestAndGpByQuery() throws Exception {
try (final AggregationTestHelper gpByQueryAggregationTestHelper = AggregationTestHelper.createGroupByQueryAggregationTestHelper(sm.getJacksonModules(), config, tempFolder)) {
final GroupByQuery groupByQuery = SketchAggregationTest.readQueryFromClasspath("simple_test_data_group_by_query.json", gpByQueryAggregationTestHelper.getObjectMapper(), vectorize);
Sequence<ResultRow> seq = gpByQueryAggregationTestHelper.runQueryOnSegments(ImmutableList.of(s1, s2), groupByQuery);
List<MapBasedRow> results = seq.map(row -> row.toMapBasedRow(groupByQuery)).toList();
Assert.assertEquals(5, results.size());
Assert.assertEquals(ImmutableList.of(new MapBasedRow(DateTimes.of("2014-10-19T00:00:00.000Z"), ImmutableMap.<String, Object>builder().put("product", "product_3").put("sketch_count", 38.0).put("sketchEstimatePostAgg", 38.0).put("sketchUnionPostAggEstimate", 38.0).put("sketchIntersectionPostAggEstimate", 38.0).put("sketchAnotBPostAggEstimate", 0.0).put("non_existing_col_validation", 0.0).build()), new MapBasedRow(DateTimes.of("2014-10-19T00:00:00.000Z"), ImmutableMap.<String, Object>builder().put("product", "product_1").put("sketch_count", 42.0).put("sketchEstimatePostAgg", 42.0).put("sketchUnionPostAggEstimate", 42.0).put("sketchIntersectionPostAggEstimate", 42.0).put("sketchAnotBPostAggEstimate", 0.0).put("non_existing_col_validation", 0.0).build()), new MapBasedRow(DateTimes.of("2014-10-19T00:00:00.000Z"), ImmutableMap.<String, Object>builder().put("product", "product_2").put("sketch_count", 42.0).put("sketchEstimatePostAgg", 42.0).put("sketchUnionPostAggEstimate", 42.0).put("sketchIntersectionPostAggEstimate", 42.0).put("sketchAnotBPostAggEstimate", 0.0).put("non_existing_col_validation", 0.0).build()), new MapBasedRow(DateTimes.of("2014-10-19T00:00:00.000Z"), ImmutableMap.<String, Object>builder().put("product", "product_4").put("sketch_count", 42.0).put("sketchEstimatePostAgg", 42.0).put("sketchUnionPostAggEstimate", 42.0).put("sketchIntersectionPostAggEstimate", 42.0).put("sketchAnotBPostAggEstimate", 0.0).put("non_existing_col_validation", 0.0).build()), new MapBasedRow(DateTimes.of("2014-10-19T00:00:00.000Z"), ImmutableMap.<String, Object>builder().put("product", "product_5").put("sketch_count", 42.0).put("sketchEstimatePostAgg", 42.0).put("sketchUnionPostAggEstimate", 42.0).put("sketchIntersectionPostAggEstimate", 42.0).put("sketchAnotBPostAggEstimate", 0.0).put("non_existing_col_validation", 0.0).build())), results);
}
}
use of org.apache.druid.query.groupby.ResultRow in project druid by druid-io.
the class ArrayOfDoublesSketchAggregationTest method buildingSketchesAtQueryTime.
@Test
public void buildingSketchesAtQueryTime() throws Exception {
Sequence<ResultRow> seq = helper.createIndexAndRunQueryOnSegment(new File(this.getClass().getClassLoader().getResource("tuple/array_of_doubles_build_data.tsv").getFile()), String.join("\n", "{", " \"type\": \"string\",", " \"parseSpec\": {", " \"format\": \"tsv\",", " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", " \"dimensionsSpec\": {", " \"dimensions\": [\"product\", \"key\", {\"type\": \"long\", \"name\": \"key_num\"}],", " \"dimensionExclusions\": [],", " \"spatialDimensions\": []", " },", " \"columns\": [\"timestamp\", \"product\", \"key\", \"key_num\", \"value\"]", " }", "}"), String.join("\n", "[", " {\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}", "]"), // minTimestamp
0, Granularities.NONE, // maxRowCount
40, String.join("\n", "{", " \"queryType\": \"groupBy\",", " \"dataSource\": \"test_datasource\",", " \"granularity\": \"ALL\",", " \"dimensions\": [],", " \"aggregations\": [", " {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"key\", \"metricColumns\": [\"value\"], \"nominalEntries\": 1024},", " {\"type\": \"count\", \"name\":\"cnt\"}", " ],", " \"postAggregations\": [", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"estimate\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", " {\"type\": \"arrayOfDoublesSketchToQuantilesSketch\", \"name\": \"quantiles-sketch\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"union\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"union\",", " \"operation\": \"UNION\",", " \"nominalEntries\": 1024,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"intersection\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"intersection\",", " \"operation\": \"INTERSECT\",", " \"nominalEntries\": 1024,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"anotb\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"anotb\",", " \"operation\": \"NOT\",", " \"nominalEntries\": 1024,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }}", " ],", " \"intervals\": [\"2015-01-01T00:00:00.000Z/2015-01-31T00:00:00.000Z\"]", "}"));
List<ResultRow> results = seq.toList();
Assert.assertEquals(1, results.size());
ResultRow row = results.get(0);
Assert.assertEquals("cnt", 40.0, new Double(row.get(1).toString()), 0);
Assert.assertEquals("sketch", 40.0, (double) row.get(0), 0);
Assert.assertEquals("estimate", 40.0, new Double(row.get(2).toString()), 0);
Assert.assertEquals("union", 40.0, new Double(row.get(4).toString()), 0);
Assert.assertEquals("intersection", 40.0, new Double(row.get(5).toString()), 0);
Assert.assertEquals("anotb", 0, new Double(row.get(6).toString()), 0);
// quantiles-sketch
Object obj = row.get(3);
Assert.assertTrue(obj instanceof DoublesSketch);
DoublesSketch ds = (DoublesSketch) obj;
Assert.assertEquals(40, ds.getN());
Assert.assertEquals(1.0, ds.getMinValue(), 0);
Assert.assertEquals(1.0, ds.getMaxValue(), 0);
}
use of org.apache.druid.query.groupby.ResultRow in project druid by druid-io.
the class ArrayOfDoublesSketchAggregationTest method buildingSketchesAtIngestionTimeTwoValues.
@Test
public void buildingSketchesAtIngestionTimeTwoValues() throws Exception {
Sequence<ResultRow> seq = helper.createIndexAndRunQueryOnSegment(new File(this.getClass().getClassLoader().getResource("tuple/array_of_doubles_build_data_two_values.tsv").getFile()), String.join("\n", "{", " \"type\": \"string\",", " \"parseSpec\": {", " \"format\": \"tsv\",", " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", " \"dimensionsSpec\": {", " \"dimensions\": [\"product\"],", " \"dimensionExclusions\": [],", " \"spatialDimensions\": []", " },", " \"columns\": [\"timestamp\", \"product\", \"key\", \"value1\", \"value2\"]", " }", "}"), String.join("\n", "[", " {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"key\", \"metricColumns\": [ \"value1\", \"value2\" ], \"nominalEntries\": 1024}", "]"), // minTimestamp
0, Granularities.NONE, // maxRowCount
10, String.join("\n", "{", " \"queryType\": \"groupBy\",", " \"dataSource\": \"test_datasource\",", " \"granularity\": \"ALL\",", " \"dimensions\": [],", " \"aggregations\": [", " {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"nominalEntries\": 1024, \"numberOfValues\": 2}", " ],", " \"postAggregations\": [", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"estimate\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", " {\"type\": \"arrayOfDoublesSketchToQuantilesSketch\", \"name\": \"quantiles-sketch\", \"column\": 2, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"union\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"union\",", " \"operation\": \"UNION\",", " \"nominalEntries\": 1024,", " \"numberOfValues\": 2,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"intersection\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"intersection\",", " \"operation\": \"INTERSECT\",", " \"nominalEntries\": 1024,", " \"numberOfValues\": 2,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"anotb\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"anotb\",", " \"operation\": \"NOT\",", " \"nominalEntries\": 1024,", " \"numberOfValues\": 2,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {", " \"type\": \"arrayOfDoublesSketchToMeans\",", " \"name\": \"means\",", " \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}", " }", " ],", " \"intervals\": [\"2015-01-01T00:00:00.000Z/2015-01-31T00:00:00.000Z\"]", "}"));
List<ResultRow> results = seq.toList();
Assert.assertEquals(1, results.size());
ResultRow row = results.get(0);
Assert.assertEquals("sketch", 40.0, (double) row.get(0), 0);
Assert.assertEquals("estimate", 40.0, (double) row.get(1), 0);
Assert.assertEquals("union", 40.0, (double) row.get(3), 0);
Assert.assertEquals("intersection", 40.0, (double) row.get(4), 0);
Assert.assertEquals("anotb", 0, (double) row.get(5), 0);
// means
Object meansObj = row.get(6);
Assert.assertTrue(meansObj instanceof double[]);
double[] means = (double[]) meansObj;
Assert.assertEquals(2, means.length);
Assert.assertEquals(1.0, means[0], 0);
Assert.assertEquals(2.0, means[1], 0);
// quantiles-sketch
Object obj = row.get(2);
Assert.assertTrue(obj instanceof DoublesSketch);
DoublesSketch ds = (DoublesSketch) obj;
Assert.assertEquals(40, ds.getN());
Assert.assertEquals(2.0, ds.getMinValue(), 0);
Assert.assertEquals(2.0, ds.getMaxValue(), 0);
}
use of org.apache.druid.query.groupby.ResultRow in project druid by druid-io.
the class ArrayOfDoublesSketchAggregationTest method ingestingSketchesTwoValues.
@Test
public void ingestingSketchesTwoValues() throws Exception {
Sequence<ResultRow> seq = helper.createIndexAndRunQueryOnSegment(new File(this.getClass().getClassLoader().getResource("tuple/array_of_doubles_sketch_data_two_values.tsv").getFile()), String.join("\n", "{", " \"type\": \"string\",", " \"parseSpec\": {", " \"format\": \"tsv\",", " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", " \"dimensionsSpec\": {", " \"dimensions\": [\"product\"],", " \"dimensionExclusions\": [],", " \"spatialDimensions\": []", " },", " \"columns\": [\"timestamp\", \"product\", \"sketch\"]", " }", "}"), String.join("\n", "[", " {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"nominalEntries\": 1024, \"numberOfValues\": 2}", "]"), // minTimestamp
0, Granularities.NONE, // maxRowCount
10, String.join("\n", "{", " \"queryType\": \"groupBy\",", " \"dataSource\": \"test_datasource\",", " \"granularity\": \"ALL\",", " \"dimensions\": [],", " \"aggregations\": [", " {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"nominalEntries\": 1024, \"numberOfValues\": 2}", " ],", " \"postAggregations\": [", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"estimate\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", " {\"type\": \"arrayOfDoublesSketchToQuantilesSketch\", \"name\": \"quantiles-sketch\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"union\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"union\",", " \"operation\": \"UNION\",", " \"nominalEntries\": 1024,", " \"numberOfValues\": 2,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"intersection\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"intersection\",", " \"operation\": \"INTERSECT\",", " \"nominalEntries\": 1024,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"anotb\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"anotb\",", " \"operation\": \"NOT\",", " \"nominalEntries\": 1024,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {", " \"type\": \"arrayOfDoublesSketchToMeans\",", " \"name\": \"means\",", " \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}", " }", " ],", " \"intervals\": [\"2015-01-01T00:00:00.000Z/2015-01-31T00:00:00.000Z\"]", "}"));
List<ResultRow> results = seq.toList();
Assert.assertEquals(1, results.size());
ResultRow row = results.get(0);
Assert.assertEquals("sketch", 40.0, (double) row.get(0), 0);
Assert.assertEquals("estimate", 40.0, (double) row.get(1), 0);
Assert.assertEquals("union", 40.0, (double) row.get(3), 0);
Assert.assertEquals("intersection", 40.0, (double) row.get(4), 0);
Assert.assertEquals("anotb", 0, (double) row.get(5), 0);
// means
Object meansObj = row.get(6);
Assert.assertTrue(meansObj instanceof double[]);
double[] means = (double[]) meansObj;
Assert.assertEquals(2, means.length);
Assert.assertEquals(1.0, means[0], 0);
Assert.assertEquals(2.0, means[1], 0);
// quantiles-sketch
Object quantilesObj = row.get(2);
Assert.assertTrue(quantilesObj instanceof DoublesSketch);
DoublesSketch ds = (DoublesSketch) quantilesObj;
Assert.assertEquals(40, ds.getN());
Assert.assertEquals(1.0, ds.getMinValue(), 0);
Assert.assertEquals(1.0, ds.getMaxValue(), 0);
}
use of org.apache.druid.query.groupby.ResultRow in project druid by druid-io.
the class ArrayOfDoublesSketchAggregationTest method buildingSketchesAtIngestionTimeThreeValuesAndNulls.
@Test
public void buildingSketchesAtIngestionTimeThreeValuesAndNulls() throws Exception {
Sequence<ResultRow> seq = helper.createIndexAndRunQueryOnSegment(new File(this.getClass().getClassLoader().getResource("tuple/array_of_doubles_build_data_three_values_and_nulls.tsv").getFile()), String.join("\n", "{", " \"type\": \"string\",", " \"parseSpec\": {", " \"format\": \"tsv\",", " \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", " \"dimensionsSpec\": {", " \"dimensions\": [\"product\"],", " \"dimensionExclusions\": [],", " \"spatialDimensions\": []", " },", " \"columns\": [\"timestamp\", \"product\", \"key\", \"value1\", \"value2\", \"value3\"]", " }", "}"), String.join("\n", "[", " {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"key\", \"metricColumns\": [ \"value1\", \"value2\", \"value3\" ], \"nominalEntries\": 1024}", "]"), // minTimestamp
0, Granularities.NONE, // maxRowCount
10, String.join("\n", "{", " \"queryType\": \"groupBy\",", " \"dataSource\": \"test_datasource\",", " \"granularity\": \"ALL\",", " \"dimensions\": [],", " \"aggregations\": [", " {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"nominalEntries\": 1024, \"numberOfValues\": 3}", " ],", " \"postAggregations\": [", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"estimate\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", " {\"type\": \"arrayOfDoublesSketchToQuantilesSketch\", \"name\": \"quantiles-sketch\", \"column\": 2, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"union\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"union\",", " \"operation\": \"UNION\",", " \"nominalEntries\": 1024,", " \"numberOfValues\": 3,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"intersection\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"intersection\",", " \"operation\": \"INTERSECT\",", " \"nominalEntries\": 1024,", " \"numberOfValues\": 3,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"anotb\", \"field\": {", " \"type\": \"arrayOfDoublesSketchSetOp\",", " \"name\": \"anotb\",", " \"operation\": \"NOT\",", " \"nominalEntries\": 1024,", " \"numberOfValues\": 3,", " \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", " }},", " {", " \"type\": \"arrayOfDoublesSketchToMeans\",", " \"name\": \"means\",", " \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}", " },", " {\"type\": \"arrayOfDoublesSketchToQuantilesSketch\", \"name\": \"quantiles-sketch-with-nulls\", \"column\": 3, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", " ],", " \"intervals\": [\"2015-01-01T00:00:00.000Z/2015-01-31T00:00:00.000Z\"]", "}"));
List<ResultRow> results = seq.toList();
Assert.assertEquals(1, results.size());
ResultRow row = results.get(0);
Assert.assertEquals("sketch", NullHandling.replaceWithDefault() ? 40.0 : 30.0, (double) row.get(0), 0);
Assert.assertEquals("estimate", NullHandling.replaceWithDefault() ? 40.0 : 30.0, (double) row.get(1), 0);
Assert.assertEquals("union", NullHandling.replaceWithDefault() ? 40.0 : 30.0, (double) row.get(3), 0);
Assert.assertEquals("intersection", NullHandling.replaceWithDefault() ? 40.0 : 30.0, (double) row.get(4), 0);
Assert.assertEquals("anotb", 0, (double) row.get(5), 0);
// means
Object meansObj = row.get(6);
Assert.assertTrue(meansObj instanceof double[]);
double[] means = (double[]) meansObj;
Assert.assertEquals(3, means.length);
Assert.assertEquals(1.0, means[0], 0);
Assert.assertEquals(2.0, means[1], 0);
Assert.assertEquals(NullHandling.replaceWithDefault() ? 2.25 : 3.0, means[2], 0.1);
// quantiles-sketch
Object obj = row.get(2);
Assert.assertTrue(obj instanceof DoublesSketch);
DoublesSketch ds = (DoublesSketch) obj;
Assert.assertEquals(NullHandling.replaceWithDefault() ? 40 : 30, ds.getN());
Assert.assertEquals(2.0, ds.getMinValue(), 0);
Assert.assertEquals(2.0, ds.getMaxValue(), 0);
// quantiles-sketch-with-nulls
Object objSketch2 = row.get(7);
Assert.assertTrue(objSketch2 instanceof DoublesSketch);
DoublesSketch ds2 = (DoublesSketch) objSketch2;
Assert.assertEquals(NullHandling.replaceWithDefault() ? 40 : 30, ds2.getN());
Assert.assertEquals(NullHandling.replaceWithDefault() ? 0.0 : 3.0, ds2.getMinValue(), 0);
Assert.assertEquals(3.0, ds2.getMaxValue(), 0);
}
Aggregations