Examples with ExpressionVirtualColumn org.apache.druid.segment.virtual.ExpressionVirtualColumn used on opensource projects

Example 36 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class ThetaSketchSqlAggregatorTest method testThetaSketchPostAggs.

@Test
public void testThetaSketchPostAggs() throws Exception {
    final List<Object[]> expectedResults;
    if (NullHandling.replaceWithDefault()) {
        expectedResults = ImmutableList.of(new Object[] { 6L, 2.0d, 3.0d, "{\"estimate\":2.0,\"highBound\":2.0,\"lowBound\":2.0,\"numStdDev\":10}", "\"AQMDAAA6zJOQxkPsNomrZQ==\"", "\"AgMDAAAazJMGAAAAAACAP1XTBztMIcMJ+HOoBBne1zKQxkPsNomrZUeWbJt3n+VpF8EdUoUHAXvxsLkOSE0lfQ==\"", "\"AQMDAAA6zJMXwR1ShQcBew==\"", "\"AQMDAAA6zJOQxkPsNomrZQ==\"", 1.0d });
    } else {
        expectedResults = ImmutableList.of(new Object[] { 6L, 2.0d, 3.0d, "{\"estimate\":2.0,\"highBound\":2.0,\"lowBound\":2.0,\"numStdDev\":10}", "\"AQMDAAA6zJOQxkPsNomrZQ==\"", "\"AgMDAAAazJMGAAAAAACAP1XTBztMIcMJ+HOoBBne1zKQxkPsNomrZUeWbJt3n+VpF8EdUoUHAXvxsLkOSE0lfQ==\"", "\"AQMDAAA6zJMXwR1ShQcBew==\"", "\"AQMDAAA6zJOQxkPsNomrZQ==\"", 1.0d });
    }
    testQuery("SELECT\n" + "  SUM(cnt),\n" + "  theta_sketch_estimate(DS_THETA(dim2)),\n" + "  theta_sketch_estimate(DS_THETA(CONCAT(dim2, 'hello'))),\n" + "  theta_sketch_estimate_with_error_bounds(DS_THETA(dim2), 10),\n" + "  THETA_SKETCH_INTERSECT(DS_THETA(dim2), DS_THETA(dim1)),\n" + "  THETA_SKETCH_UNION(DS_THETA(dim2), DS_THETA(dim1)),\n" + "  THETA_SKETCH_NOT(DS_THETA(dim2), DS_THETA(dim1)),\n" + "  THETA_SKETCH_INTERSECT(32768, DS_THETA(dim2), DS_THETA(dim1)),\n" + "  theta_sketch_estimate(THETA_SKETCH_INTERSECT(THETA_SKETCH_INTERSECT(DS_THETA(dim2), DS_THETA(dim1)), DS_THETA(dim2)))\n" + "FROM druid.foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "concat(\"dim2\",'hello')", ColumnType.STRING, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new LongSumAggregatorFactory("a0", "cnt"), new SketchMergeAggregatorFactory("a1", "dim2", null, null, null, null), new SketchMergeAggregatorFactory("a2", "v0", null, null, null, null), new SketchMergeAggregatorFactory("a3", "dim1", null, null, null, null))).postAggregators(new SketchEstimatePostAggregator("p1", new FieldAccessPostAggregator("p0", "a1"), null), new SketchEstimatePostAggregator("p3", new FieldAccessPostAggregator("p2", "a2"), null), new SketchEstimatePostAggregator("p5", new FieldAccessPostAggregator("p4", "a1"), 10), new SketchSetPostAggregator("p8", "INTERSECT", null, ImmutableList.of(new FieldAccessPostAggregator("p6", "a1"), new FieldAccessPostAggregator("p7", "a3"))), new SketchSetPostAggregator("p11", "UNION", null, ImmutableList.of(new FieldAccessPostAggregator("p9", "a1"), new FieldAccessPostAggregator("p10", "a3"))), new SketchSetPostAggregator("p14", "NOT", null, ImmutableList.of(new FieldAccessPostAggregator("p12", "a1"), new FieldAccessPostAggregator("p13", "a3"))), new SketchSetPostAggregator("p17", "INTERSECT", 32768, ImmutableList.of(new FieldAccessPostAggregator("p15", "a1"), new FieldAccessPostAggregator("p16", "a3"))), new SketchEstimatePostAggregator("p23", new SketchSetPostAggregator("p22", "INTERSECT", null, ImmutableList.of(new SketchSetPostAggregator("p20", "INTERSECT", null, ImmutableList.of(new FieldAccessPostAggregator("p18", "a1"), new FieldAccessPostAggregator("p19", "a3"))), new FieldAccessPostAggregator("p21", "a1"))), null)).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 37 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class ThetaSketchSqlAggregatorTest method testApproxCountDistinctThetaSketch.

@Test
public void testApproxCountDistinctThetaSketch() throws Exception {
    // Cannot vectorize due to SUBSTRING.
    cannotVectorize();
    final String sql = "SELECT\n" + "  SUM(cnt),\n" + "  APPROX_COUNT_DISTINCT_DS_THETA(dim2),\n" + // uppercase
    "  APPROX_COUNT_DISTINCT_DS_THETA(dim2) FILTER(WHERE dim2 <> ''),\n" + // lowercase; also, filtered
    "  APPROX_COUNT_DISTINCT(SUBSTRING(dim2, 1, 1)),\n" + // on extractionFn, using A.C.D.
    "  COUNT(DISTINCT SUBSTRING(dim2, 1, 1) || 'x'),\n" + // on expression, using COUNT DISTINCT
    "  APPROX_COUNT_DISTINCT_DS_THETA(thetasketch_dim1, 32768),\n" + // on native theta sketch column
    "  APPROX_COUNT_DISTINCT_DS_THETA(thetasketch_dim1)\n" + // on native theta sketch column
    "FROM druid.foo";
    final List<Object[]> expectedResults;
    if (NullHandling.replaceWithDefault()) {
        expectedResults = ImmutableList.of(new Object[] { 6L, 2L, 2L, 1L, 2L, 5L, 5L });
    } else {
        expectedResults = ImmutableList.of(new Object[] { 6L, 2L, 2L, 1L, 1L, 5L, 5L });
    }
    testQuery(sql, ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "substring(\"dim2\", 0, 1)", ColumnType.STRING, TestExprMacroTable.INSTANCE), new ExpressionVirtualColumn("v1", "concat(substring(\"dim2\", 0, 1),'x')", ColumnType.STRING, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new LongSumAggregatorFactory("a0", "cnt"), new SketchMergeAggregatorFactory("a1", "dim2", null, null, null, null), new FilteredAggregatorFactory(new SketchMergeAggregatorFactory("a2", "dim2", null, null, null, null), BaseCalciteQueryTest.not(BaseCalciteQueryTest.selector("dim2", "", null))), new SketchMergeAggregatorFactory("a3", "v0", null, null, null, null), new SketchMergeAggregatorFactory("a4", "v1", null, null, null, null), new SketchMergeAggregatorFactory("a5", "thetasketch_dim1", 32768, null, null, null), new SketchMergeAggregatorFactory("a6", "thetasketch_dim1", null, null, null, null))).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 38 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class BloomFilterSqlAggregatorTest method testBloomFilterAggLongVirtualColumn.

@Test
public void testBloomFilterAggLongVirtualColumn() throws Exception {
    cannotVectorize();
    BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
    for (InputRow row : CalciteTests.ROWS1_WITH_NUMERIC_DIMS) {
        Object raw = row.getRaw("l1");
        if (raw == null) {
            if (NullHandling.replaceWithDefault()) {
                expected1.addLong(NullHandling.defaultLongValue());
            } else {
                expected1.addBytes(null, 0, 0);
            }
        } else {
            expected1.addLong(2 * ((Number) raw).longValue());
        }
    }
    testQuery("SELECT\n" + "BLOOM_FILTER(l1 * 2, 1000)\n" + "FROM numfoo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE3).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "(\"l1\" * 2)", ColumnType.LONG, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new BloomFilterAggregatorFactory("a0:agg", new DefaultDimensionSpec("v0", "a0:v0"), TEST_NUM_ENTRIES))).context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { CalciteTests.getJsonMapper().writeValueAsString(expected1) }));
}

Example 39 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class FixedBucketsHistogramQuantileSqlAggregatorTest method testQuantileOnFloatAndLongs.

@Test
public void testQuantileOnFloatAndLongs() throws Exception {
    final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.0299999713897705, 3.5, 6.440000057220459, 6.470000267028809, 12.40999984741211, 6.494999885559082, 5.497499942779541, 6.499499797821045, 1.25 });
    testQuery("SELECT\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.01, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.5, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.98, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.99, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1 * 2, 0.97, 40, 0.0, 20.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.99, 20, 0.0, 10.0) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.999, 20, 0.0, 10.0) FILTER(WHERE dim1 <> 'abc'),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.999, 20, 0.0, 10.0) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(cnt, 0.5, 20, 0.0, 10.0)\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "(\"m1\" * 2)", ColumnType.FLOAT, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new FixedBucketsHistogramAggregatorFactory("a0:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new FixedBucketsHistogramAggregatorFactory("a4:agg", "v0", 40, 0.0d, 20.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new FilteredAggregatorFactory(new FixedBucketsHistogramAggregatorFactory("a5:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new SelectorDimFilter("dim1", "abc", null)), new FilteredAggregatorFactory(new FixedBucketsHistogramAggregatorFactory("a6:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))), new FixedBucketsHistogramAggregatorFactory("a8:agg", "cnt", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false))).postAggregators(new QuantilePostAggregator("a0", "a0:agg", 0.01f), new QuantilePostAggregator("a1", "a0:agg", 0.50f), new QuantilePostAggregator("a2", "a0:agg", 0.98f), new QuantilePostAggregator("a3", "a0:agg", 0.99f), new QuantilePostAggregator("a4", "a4:agg", 0.97f), new QuantilePostAggregator("a5", "a5:agg", 0.99f), new QuantilePostAggregator("a6", "a6:agg", 0.999f), new QuantilePostAggregator("a7", "a5:agg", 0.999f), new QuantilePostAggregator("a8", "a8:agg", 0.50f)).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 40 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class QuantileSqlAggregatorTest method testQuantileOnCastedString.

@Test
public void testQuantileOnCastedString() throws Exception {
    cannotVectorize();
    final List<Object[]> expectedResults;
    if (NullHandling.replaceWithDefault()) {
        expectedResults = ImmutableList.of(new Object[] { "", 0.0d }, new Object[] { "a", 0.0d }, new Object[] { "b", 0.0d }, new Object[] { "c", 10.100000381469727d }, new Object[] { "d", 2.0d });
    } else {
        expectedResults = ImmutableList.of(new Object[] { null, Double.NaN }, new Object[] { "", 1.0d }, new Object[] { "a", Double.NaN }, new Object[] { "b", 10.100000381469727d }, new Object[] { "c", 10.100000381469727d }, new Object[] { "d", 2.0d });
    }
    testQuery("SELECT dim3, APPROX_QUANTILE(CAST(dim1 as DOUBLE), 0.5) from foo group by dim3", ImmutableList.of(GroupByQuery.builder().setDataSource(CalciteTests.DATASOURCE1).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setVirtualColumns(new ExpressionVirtualColumn("v0", "CAST(\"dim1\", 'DOUBLE')", ColumnType.FLOAT, ExprMacroTable.nil())).setDimensions(new DefaultDimensionSpec("dim3", "d0")).setAggregatorSpecs(new ApproximateHistogramAggregatorFactory("a0:agg", "v0", 50, 7, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, false)).setPostAggregatorSpecs(ImmutableList.of(new QuantilePostAggregator("a0", "a0:agg", 0.5f))).setContext(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}