Examples with DoublesSketch org.apache.datasketches.quantiles.DoublesSketch used on opensource projects

Example 6 with DoublesSketch

use of org.apache.datasketches.quantiles.DoublesSketch in project druid by druid-io.

the class DoublesSketchComplexMetricSerdeTest method testExtractorOnDecimalNumber.

@Test
public void testExtractorOnDecimalNumber() {
    final DoublesSketchComplexMetricSerde serde = new DoublesSketchComplexMetricSerde();
    final ComplexMetricExtractor extractor = serde.getExtractor();
    final DoublesSketch sketch = (DoublesSketch) extractor.extractValue(new MapBasedInputRow(0L, ImmutableList.of(), ImmutableMap.of("foo", "3.1")), "foo");
    Assert.assertEquals(1, sketch.getRetainedItems());
    Assert.assertEquals(3.1d, sketch.getMaxValue(), 0.01d);
}

Example 7 with DoublesSketch

use of org.apache.datasketches.quantiles.DoublesSketch in project druid by druid-io.

the class DoublesSketchComplexMetricSerdeTest method testExtractorOnNegativeNumber.

@Test
public void testExtractorOnNegativeNumber() {
    final DoublesSketchComplexMetricSerde serde = new DoublesSketchComplexMetricSerde();
    final ComplexMetricExtractor extractor = serde.getExtractor();
    final DoublesSketch sketch = (DoublesSketch) extractor.extractValue(new MapBasedInputRow(0L, ImmutableList.of(), ImmutableMap.of("foo", "-133")), "foo");
    Assert.assertEquals(1, sketch.getRetainedItems());
    Assert.assertEquals(-133d, sketch.getMaxValue(), 0.01d);
}

Example 8 with DoublesSketch

use of org.apache.datasketches.quantiles.DoublesSketch in project druid by druid-io.

the class DoublesSketchComplexMetricSerdeTest method testExtractorOnEmptyString.

@Test
public void testExtractorOnEmptyString() {
    final DoublesSketchComplexMetricSerde serde = new DoublesSketchComplexMetricSerde();
    final ComplexMetricExtractor extractor = serde.getExtractor();
    final DoublesSketch sketch = (DoublesSketch) extractor.extractValue(new MapBasedInputRow(0L, ImmutableList.of(), ImmutableMap.of("foo", "")), "foo");
    Assert.assertEquals(0, sketch.getRetainedItems());
}

Example 9 with DoublesSketch

use of org.apache.datasketches.quantiles.DoublesSketch in project druid by druid-io.

the class ArrayOfDoublesSketchAggregationTest method buildingSketchesAtQueryTime.

@Test
public void buildingSketchesAtQueryTime() throws Exception {
    Sequence<ResultRow> seq = helper.createIndexAndRunQueryOnSegment(new File(this.getClass().getClassLoader().getResource("tuple/array_of_doubles_build_data.tsv").getFile()), String.join("\n", "{", "  \"type\": \"string\",", "  \"parseSpec\": {", "    \"format\": \"tsv\",", "    \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", "    \"dimensionsSpec\": {", "      \"dimensions\": [\"product\", \"key\", {\"type\": \"long\", \"name\": \"key_num\"}],", "      \"dimensionExclusions\": [],", "      \"spatialDimensions\": []", "    },", "    \"columns\": [\"timestamp\", \"product\", \"key\", \"key_num\", \"value\"]", "  }", "}"), String.join("\n", "[", "  {\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}", "]"), // minTimestamp
    0, Granularities.NONE, // maxRowCount
    40, String.join("\n", "{", "  \"queryType\": \"groupBy\",", "  \"dataSource\": \"test_datasource\",", "  \"granularity\": \"ALL\",", "  \"dimensions\": [],", "  \"aggregations\": [", "    {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"key\", \"metricColumns\": [\"value\"], \"nominalEntries\": 1024},", "    {\"type\": \"count\", \"name\":\"cnt\"}", "  ],", "  \"postAggregations\": [", "    {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"estimate\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", "    {\"type\": \"arrayOfDoublesSketchToQuantilesSketch\", \"name\": \"quantiles-sketch\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", "    {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"union\", \"field\": {", "      \"type\": \"arrayOfDoublesSketchSetOp\",", "      \"name\": \"union\",", "      \"operation\": \"UNION\",", "      \"nominalEntries\": 1024,", "      \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", "    }},", "    {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"intersection\", \"field\": {", "      \"type\": \"arrayOfDoublesSketchSetOp\",", "      \"name\": \"intersection\",", "      \"operation\": \"INTERSECT\",", "      \"nominalEntries\": 1024,", "      \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", "    }},", "    {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"anotb\", \"field\": {", "      \"type\": \"arrayOfDoublesSketchSetOp\",", "      \"name\": \"anotb\",", "      \"operation\": \"NOT\",", "      \"nominalEntries\": 1024,", "      \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", "    }}", "  ],", "  \"intervals\": [\"2015-01-01T00:00:00.000Z/2015-01-31T00:00:00.000Z\"]", "}"));
    List<ResultRow> results = seq.toList();
    Assert.assertEquals(1, results.size());
    ResultRow row = results.get(0);
    Assert.assertEquals("cnt", 40.0, new Double(row.get(1).toString()), 0);
    Assert.assertEquals("sketch", 40.0, (double) row.get(0), 0);
    Assert.assertEquals("estimate", 40.0, new Double(row.get(2).toString()), 0);
    Assert.assertEquals("union", 40.0, new Double(row.get(4).toString()), 0);
    Assert.assertEquals("intersection", 40.0, new Double(row.get(5).toString()), 0);
    Assert.assertEquals("anotb", 0, new Double(row.get(6).toString()), 0);
    // quantiles-sketch
    Object obj = row.get(3);
    Assert.assertTrue(obj instanceof DoublesSketch);
    DoublesSketch ds = (DoublesSketch) obj;
    Assert.assertEquals(40, ds.getN());
    Assert.assertEquals(1.0, ds.getMinValue(), 0);
    Assert.assertEquals(1.0, ds.getMaxValue(), 0);
}

Example 10 with DoublesSketch

use of org.apache.datasketches.quantiles.DoublesSketch in project druid by druid-io.

the class ArrayOfDoublesSketchAggregationTest method buildingSketchesAtIngestionTimeTwoValues.

@Test
public void buildingSketchesAtIngestionTimeTwoValues() throws Exception {
    Sequence<ResultRow> seq = helper.createIndexAndRunQueryOnSegment(new File(this.getClass().getClassLoader().getResource("tuple/array_of_doubles_build_data_two_values.tsv").getFile()), String.join("\n", "{", "  \"type\": \"string\",", "  \"parseSpec\": {", "    \"format\": \"tsv\",", "    \"timestampSpec\": {\"column\": \"timestamp\", \"format\": \"yyyyMMddHH\"},", "    \"dimensionsSpec\": {", "      \"dimensions\": [\"product\"],", "      \"dimensionExclusions\": [],", "      \"spatialDimensions\": []", "    },", "    \"columns\": [\"timestamp\", \"product\", \"key\", \"value1\", \"value2\"]", "  }", "}"), String.join("\n", "[", "  {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"key\", \"metricColumns\": [ \"value1\", \"value2\" ], \"nominalEntries\": 1024}", "]"), // minTimestamp
    0, Granularities.NONE, // maxRowCount
    10, String.join("\n", "{", "  \"queryType\": \"groupBy\",", "  \"dataSource\": \"test_datasource\",", "  \"granularity\": \"ALL\",", "  \"dimensions\": [],", "  \"aggregations\": [", "    {\"type\": \"arrayOfDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"nominalEntries\": 1024, \"numberOfValues\": 2}", "  ],", "  \"postAggregations\": [", "    {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"estimate\", \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", "    {\"type\": \"arrayOfDoublesSketchToQuantilesSketch\", \"name\": \"quantiles-sketch\", \"column\": 2, \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}},", "    {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"union\", \"field\": {", "      \"type\": \"arrayOfDoublesSketchSetOp\",", "      \"name\": \"union\",", "      \"operation\": \"UNION\",", "      \"nominalEntries\": 1024,", "      \"numberOfValues\": 2,", "      \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", "    }},", "    {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"intersection\", \"field\": {", "      \"type\": \"arrayOfDoublesSketchSetOp\",", "      \"name\": \"intersection\",", "      \"operation\": \"INTERSECT\",", "      \"nominalEntries\": 1024,", "      \"numberOfValues\": 2,", "      \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", "    }},", "    {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"anotb\", \"field\": {", "      \"type\": \"arrayOfDoublesSketchSetOp\",", "      \"name\": \"anotb\",", "      \"operation\": \"NOT\",", "      \"nominalEntries\": 1024,", "      \"numberOfValues\": 2,", "      \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]", "    }},", "    {", "      \"type\": \"arrayOfDoublesSketchToMeans\",", "      \"name\": \"means\",", "      \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}", "    }", "  ],", "  \"intervals\": [\"2015-01-01T00:00:00.000Z/2015-01-31T00:00:00.000Z\"]", "}"));
    List<ResultRow> results = seq.toList();
    Assert.assertEquals(1, results.size());
    ResultRow row = results.get(0);
    Assert.assertEquals("sketch", 40.0, (double) row.get(0), 0);
    Assert.assertEquals("estimate", 40.0, (double) row.get(1), 0);
    Assert.assertEquals("union", 40.0, (double) row.get(3), 0);
    Assert.assertEquals("intersection", 40.0, (double) row.get(4), 0);
    Assert.assertEquals("anotb", 0, (double) row.get(5), 0);
    // means
    Object meansObj = row.get(6);
    Assert.assertTrue(meansObj instanceof double[]);
    double[] means = (double[]) meansObj;
    Assert.assertEquals(2, means.length);
    Assert.assertEquals(1.0, means[0], 0);
    Assert.assertEquals(2.0, means[1], 0);
    // quantiles-sketch
    Object obj = row.get(2);
    Assert.assertTrue(obj instanceof DoublesSketch);
    DoublesSketch ds = (DoublesSketch) obj;
    Assert.assertEquals(40, ds.getN());
    Assert.assertEquals(2.0, ds.getMinValue(), 0);
    Assert.assertEquals(2.0, ds.getMaxValue(), 0);
}