Examples with ExpressionVirtualColumn org.apache.druid.segment.virtual.ExpressionVirtualColumn used on opensource projects

Example 76 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class TimeseriesQueryRunnerTest method testTimeseriesWithVirtualColumn.

@Test
public void testTimeseriesWithVirtualColumn() {
    TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(QueryRunnerTestHelper.DAY_GRAN).intervals(QueryRunnerTestHelper.FIRST_TO_THIRD).aggregators(Arrays.asList(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "expr"), QueryRunnerTestHelper.QUALITY_UNIQUES)).descending(descending).virtualColumns(new ExpressionVirtualColumn("expr", "index", ColumnType.FLOAT, TestExprMacroTable.INSTANCE)).context(makeContext()).build();
    List<Result<TimeseriesResultValue>> expectedResults = Arrays.asList(new Result<>(DateTimes.of("2011-04-01"), new TimeseriesResultValue(ImmutableMap.of("rows", 13L, "idx", 6619L, "uniques", QueryRunnerTestHelper.UNIQUES_9))), new Result<>(DateTimes.of("2011-04-02"), new TimeseriesResultValue(ImmutableMap.of("rows", 13L, "idx", 5827L, "uniques", QueryRunnerTestHelper.UNIQUES_9))));
    Iterable<Result<TimeseriesResultValue>> results = runner.run(QueryPlus.wrap(query)).toList();
    assertExpectedResults(expectedResults, results);
}

Example 77 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class BloomFilterSqlAggregatorTest method testBloomFilterAggFloatVirtualColumn.

@Test
public void testBloomFilterAggFloatVirtualColumn() throws Exception {
    cannotVectorize();
    BloomKFilter expected1 = new BloomKFilter(TEST_NUM_ENTRIES);
    for (InputRow row : CalciteTests.ROWS1_WITH_NUMERIC_DIMS) {
        Object raw = row.getRaw("f1");
        if (raw == null) {
            if (NullHandling.replaceWithDefault()) {
                expected1.addFloat(NullHandling.defaultFloatValue());
            } else {
                expected1.addBytes(null, 0, 0);
            }
        } else {
            expected1.addFloat(2 * ((Number) raw).floatValue());
        }
    }
    testQuery("SELECT\n" + "BLOOM_FILTER(f1 * 2, 1000)\n" + "FROM numfoo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE3).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "(\"f1\" * 2)", ColumnType.FLOAT, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new BloomFilterAggregatorFactory("a0:agg", new DefaultDimensionSpec("v0", "a0:v0"), TEST_NUM_ENTRIES))).context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { CalciteTests.getJsonMapper().writeValueAsString(expected1) }));
}

Example 78 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class JoinFilterAnalyzer method rewriteSelectorFilter.

/**
 * Rewrites a selector filter on a join table into an IN filter on the base table.
 *
 * @param selectorFilter                    SelectorFilter to be rewritten
 * @param joinFilterPreAnalysis             The pre-analysis computed by {@link #computeJoinFilterPreAnalysis)}
 * @param pushDownVirtualColumnsForLhsExprs See comments on {@link #analyzeJoinFilterClause}
 *
 * @return A JoinFilterAnalysis that indicates how to handle the potentially rewritten filter
 */
private static JoinFilterAnalysis rewriteSelectorFilter(SelectorFilter selectorFilter, JoinFilterPreAnalysis joinFilterPreAnalysis, Map<Expr, VirtualColumn> pushDownVirtualColumnsForLhsExprs) {
    List<Filter> newFilters = new ArrayList<>();
    String filteringColumn = selectorFilter.getDimension();
    String filteringValue = selectorFilter.getValue();
    if (areSomeColumnsFromPostJoinVirtualColumns(joinFilterPreAnalysis.getPostJoinVirtualColumns(), selectorFilter.getRequiredColumns())) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    if (!joinFilterPreAnalysis.getJoinableClauses().areSomeColumnsFromJoin(selectorFilter.getRequiredColumns())) {
        return new JoinFilterAnalysis(false, selectorFilter, selectorFilter);
    }
    List<JoinFilterColumnCorrelationAnalysis> correlationAnalyses = joinFilterPreAnalysis.getCorrelationsByFilteringColumn().get(filteringColumn);
    if (correlationAnalyses == null) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    for (JoinFilterColumnCorrelationAnalysis correlationAnalysis : correlationAnalyses) {
        if (correlationAnalysis.supportsPushDown()) {
            Optional<Set<String>> correlatedValues = correlationAnalysis.getCorrelatedValuesMap().get(Pair.of(filteringColumn, filteringValue));
            if (!correlatedValues.isPresent()) {
                return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
            }
            Set<String> newFilterValues = correlatedValues.get();
            // in nothing => match nothing
            if (newFilterValues.isEmpty()) {
                return new JoinFilterAnalysis(true, selectorFilter, FalseFilter.instance());
            }
            for (String correlatedBaseColumn : correlationAnalysis.getBaseColumns()) {
                Filter rewrittenFilter = new InDimFilter(correlatedBaseColumn, newFilterValues).toFilter();
                newFilters.add(rewrittenFilter);
            }
            for (Expr correlatedBaseExpr : correlationAnalysis.getBaseExpressions()) {
                // We need to create a virtual column for the expressions when pushing down
                VirtualColumn pushDownVirtualColumn = pushDownVirtualColumnsForLhsExprs.computeIfAbsent(correlatedBaseExpr, (expr) -> {
                    String vcName = getCorrelatedBaseExprVirtualColumnName(pushDownVirtualColumnsForLhsExprs.size());
                    return new ExpressionVirtualColumn(vcName, correlatedBaseExpr, ColumnType.STRING);
                });
                Filter rewrittenFilter = new InDimFilter(pushDownVirtualColumn.getOutputName(), newFilterValues).toFilter();
                newFilters.add(rewrittenFilter);
            }
        }
    }
    if (newFilters.isEmpty()) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    return new JoinFilterAnalysis(true, selectorFilter, Filters.maybeAnd(newFilters).orElse(null));
}

Example 79 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class TopNQueryRunnerTest method testFullOnTopNLongVirtualColumn.

@Test
public void testFullOnTopNLongVirtualColumn() {
    TopNQuery query = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(QueryRunnerTestHelper.ALL_GRAN).dimension(new DefaultDimensionSpec("ql_expr", "ql_alias", ColumnType.LONG)).metric("maxIndex").threshold(4).intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC).aggregators(Lists.newArrayList(Iterables.concat(commonAggregators, Lists.newArrayList(new DoubleMaxAggregatorFactory("maxIndex", "index"), new DoubleMinAggregatorFactory("minIndex", "index"))))).postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT).virtualColumns(new ExpressionVirtualColumn("ql_expr", "qualityLong", ColumnType.LONG, ExprMacroTable.nil())).build();
    List<Result<TopNResultValue>> expectedResults = Collections.singletonList(new Result<>(DateTimes.of("2011-01-12T00:00:00.000Z"), new TopNResultValue(Arrays.<Map<String, Object>>asList(ImmutableMap.<String, Object>builder().put("ql_alias", 1400L).put(QueryRunnerTestHelper.INDEX_METRIC, 217725.41940800005D).put("rows", 279L).put("addRowsIndexConstant", 218005.41940800005D).put("uniques", QueryRunnerTestHelper.UNIQUES_1).put("maxIndex", 1870.061029D).put("minIndex", 91.270553D).build(), ImmutableMap.<String, Object>builder().put("ql_alias", 1600L).put(QueryRunnerTestHelper.INDEX_METRIC, 210865.67977600006D).put("rows", 279L).put("addRowsIndexConstant", 211145.67977600006D).put("uniques", QueryRunnerTestHelper.UNIQUES_1).put("maxIndex", 1862.737933D).put("minIndex", 99.284525D).build(), ImmutableMap.<String, Object>builder().put("ql_alias", 1000L).put(QueryRunnerTestHelper.INDEX_METRIC, 12270.807093D).put("rows", 93L).put("addRowsIndexConstant", 12364.807093D).put("uniques", QueryRunnerTestHelper.UNIQUES_1).put("maxIndex", 277.273533D).put("minIndex", 71.315931D).build(), ImmutableMap.<String, Object>builder().put("ql_alias", 1200L).put(QueryRunnerTestHelper.INDEX_METRIC, 12086.472791D).put("rows", 93L).put("addRowsIndexConstant", 12180.472791D).put("uniques", QueryRunnerTestHelper.UNIQUES_1).put("maxIndex", 193.787574D).put("minIndex", 84.710523D).build()))));
    assertExpectedResults(expectedResults, query);
}

Example 80 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class TopNQueryTest method testGetRequiredColumns.

@Test
public void testGetRequiredColumns() {
    final TopNQuery query = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).intervals(QueryRunnerTestHelper.FIRST_TO_THIRD).virtualColumns(new ExpressionVirtualColumn("v", "\"other\"", ColumnType.STRING, ExprMacroTable.nil())).dimension(DefaultDimensionSpec.of("v")).aggregators(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index")).granularity(QueryRunnerTestHelper.DAY_GRAN).postAggregators(ImmutableList.of(new FieldAccessPostAggregator("x", "idx"))).metric(new NumericTopNMetricSpec("idx")).threshold(100).build();
    Assert.assertEquals(ImmutableSet.of("__time", "other", "index"), query.getRequiredColumns());
}