Examples with VirtualColumn org.apache.druid.segment.VirtualColumn used on opensource projects

Example 16 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project druid by druid-io.

the class JoinFilterAnalyzer method rewriteSelectorFilter.

/**
 * Rewrites a selector filter on a join table into an IN filter on the base table.
 *
 * @param selectorFilter                    SelectorFilter to be rewritten
 * @param joinFilterPreAnalysis             The pre-analysis computed by {@link #computeJoinFilterPreAnalysis)}
 * @param pushDownVirtualColumnsForLhsExprs See comments on {@link #analyzeJoinFilterClause}
 *
 * @return A JoinFilterAnalysis that indicates how to handle the potentially rewritten filter
 */
private static JoinFilterAnalysis rewriteSelectorFilter(SelectorFilter selectorFilter, JoinFilterPreAnalysis joinFilterPreAnalysis, Map<Expr, VirtualColumn> pushDownVirtualColumnsForLhsExprs) {
    List<Filter> newFilters = new ArrayList<>();
    String filteringColumn = selectorFilter.getDimension();
    String filteringValue = selectorFilter.getValue();
    if (areSomeColumnsFromPostJoinVirtualColumns(joinFilterPreAnalysis.getPostJoinVirtualColumns(), selectorFilter.getRequiredColumns())) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    if (!joinFilterPreAnalysis.getJoinableClauses().areSomeColumnsFromJoin(selectorFilter.getRequiredColumns())) {
        return new JoinFilterAnalysis(false, selectorFilter, selectorFilter);
    }
    List<JoinFilterColumnCorrelationAnalysis> correlationAnalyses = joinFilterPreAnalysis.getCorrelationsByFilteringColumn().get(filteringColumn);
    if (correlationAnalyses == null) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    for (JoinFilterColumnCorrelationAnalysis correlationAnalysis : correlationAnalyses) {
        if (correlationAnalysis.supportsPushDown()) {
            Optional<Set<String>> correlatedValues = correlationAnalysis.getCorrelatedValuesMap().get(Pair.of(filteringColumn, filteringValue));
            if (!correlatedValues.isPresent()) {
                return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
            }
            Set<String> newFilterValues = correlatedValues.get();
            // in nothing => match nothing
            if (newFilterValues.isEmpty()) {
                return new JoinFilterAnalysis(true, selectorFilter, FalseFilter.instance());
            }
            for (String correlatedBaseColumn : correlationAnalysis.getBaseColumns()) {
                Filter rewrittenFilter = new InDimFilter(correlatedBaseColumn, newFilterValues).toFilter();
                newFilters.add(rewrittenFilter);
            }
            for (Expr correlatedBaseExpr : correlationAnalysis.getBaseExpressions()) {
                // We need to create a virtual column for the expressions when pushing down
                VirtualColumn pushDownVirtualColumn = pushDownVirtualColumnsForLhsExprs.computeIfAbsent(correlatedBaseExpr, (expr) -> {
                    String vcName = getCorrelatedBaseExprVirtualColumnName(pushDownVirtualColumnsForLhsExprs.size());
                    return new ExpressionVirtualColumn(vcName, correlatedBaseExpr, ColumnType.STRING);
                });
                Filter rewrittenFilter = new InDimFilter(pushDownVirtualColumn.getOutputName(), newFilterValues).toFilter();
                newFilters.add(rewrittenFilter);
            }
        }
    }
    if (newFilters.isEmpty()) {
        return JoinFilterAnalysis.createNoPushdownFilterAnalysis(selectorFilter);
    }
    return new JoinFilterAnalysis(true, selectorFilter, Filters.maybeAnd(newFilters).orElse(null));
}

Example 17 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project druid by druid-io.

the class LookupSerdeModuleTest method testExpressionVirtualColumnSerde.

@Test
public void testExpressionVirtualColumnSerde() throws Exception {
    final ExpressionVirtualColumn virtualColumn = new ExpressionVirtualColumn("v", "lookup(xxx, 'beep')", ColumnType.STRING, injector.getInstance(ExprMacroTable.class));
    Assert.assertEquals(virtualColumn, objectMapper.readValue(objectMapper.writeValueAsBytes(virtualColumn), VirtualColumn.class));
}

Example 18 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project druid by druid-io.

the class DruidQuery method getVirtualColumns.

private VirtualColumns getVirtualColumns(final boolean includeDimensions) {
    // 'sourceRowSignature' could provide a list of all defined virtual columns while constructing a query, but we
    // still want to collect the set of VirtualColumns this way to ensure we only add what is still being used after
    // the various transforms and optimizations
    Set<VirtualColumn> virtualColumns = new HashSet<>();
    // rewrite any "specialized" virtual column expressions as top level virtual columns so that their native
    // implementation can be used instead of being composed as part of some expression tree in an expresson virtual
    // column
    Set<String> specialized = new HashSet<>();
    virtualColumnRegistry.visitAllSubExpressions((expression) -> {
        switch(expression.getType()) {
            case SPECIALIZED:
                // add the expression to the top level of the registry as a standalone virtual column
                final String name = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(expression, expression.getDruidType());
                specialized.add(name);
                // replace with an identifier expression of the new virtual column name
                return DruidExpression.ofColumn(expression.getDruidType(), name);
            default:
                // do nothing
                return expression;
        }
    });
    // we always want to add any virtual columns used by the query level DimFilter
    if (filter != null) {
        for (String columnName : filter.getRequiredColumns()) {
            if (virtualColumnRegistry.isVirtualColumnDefined(columnName)) {
                virtualColumns.add(virtualColumnRegistry.getVirtualColumn(columnName));
            }
        }
    }
    if (selectProjection != null) {
        for (String columnName : selectProjection.getVirtualColumns()) {
            if (virtualColumnRegistry.isVirtualColumnDefined(columnName)) {
                virtualColumns.add(virtualColumnRegistry.getVirtualColumn(columnName));
            }
        }
    }
    if (grouping != null) {
        if (includeDimensions) {
            for (DimensionExpression expression : grouping.getDimensions()) {
                if (virtualColumnRegistry.isVirtualColumnDefined(expression.getVirtualColumn())) {
                    virtualColumns.add(virtualColumnRegistry.getVirtualColumn(expression.getVirtualColumn()));
                }
            }
        }
        for (Aggregation aggregation : grouping.getAggregations()) {
            virtualColumns.addAll(virtualColumnRegistry.getAllVirtualColumns(aggregation.getRequiredColumns()));
        }
    }
    if (sorting != null && sorting.getProjection() != null && grouping == null) {
        for (String columnName : sorting.getProjection().getVirtualColumns()) {
            if (virtualColumnRegistry.isVirtualColumnDefined(columnName)) {
                virtualColumns.add(virtualColumnRegistry.getVirtualColumn(columnName));
            }
        }
    }
    if (dataSource instanceof JoinDataSource) {
        for (String expression : ((JoinDataSource) dataSource).getVirtualColumnCandidates()) {
            if (virtualColumnRegistry.isVirtualColumnDefined(expression)) {
                virtualColumns.add(virtualColumnRegistry.getVirtualColumn(expression));
            }
        }
    }
    for (String columnName : specialized) {
        if (virtualColumnRegistry.isVirtualColumnDefined(columnName)) {
            virtualColumns.add(virtualColumnRegistry.getVirtualColumn(columnName));
        }
    }
    // sort for predictable output
    List<VirtualColumn> columns = new ArrayList<>(virtualColumns);
    columns.sort(Comparator.comparing(VirtualColumn::getOutputName));
    return VirtualColumns.create(columns);
}

Example 19 with VirtualColumn

use of org.apache.druid.segment.VirtualColumn in project druid by druid-io.

the class ExpressionTestHelper method testFilter.

void testFilter(final SqlOperator op, final List<? extends RexNode> exprs, final List<VirtualColumn> expectedVirtualColumns, final DimFilter expectedFilter, final boolean expectedResult) {
    final RexNode rexNode = rexBuilder.makeCall(op, exprs);
    final VirtualColumnRegistry virtualColumnRegistry = VirtualColumnRegistry.create(rowSignature, TestExprMacroTable.INSTANCE);
    final DimFilter filter = Expressions.toFilter(PLANNER_CONTEXT, rowSignature, virtualColumnRegistry, rexNode);
    Assert.assertEquals("Filter for: " + rexNode, expectedFilter, filter);
    final List<VirtualColumn> virtualColumns = filter.getRequiredColumns().stream().map(virtualColumnRegistry::getVirtualColumn).filter(Objects::nonNull).sorted(Comparator.comparing(VirtualColumn::getOutputName)).collect(Collectors.toList());
    Assert.assertEquals("Virtual columns for: " + rexNode, expectedVirtualColumns.stream().sorted(Comparator.comparing(VirtualColumn::getOutputName)).collect(Collectors.toList()), virtualColumns);
    final ValueMatcher matcher = expectedFilter.toFilter().makeMatcher(new VirtualizedColumnSelectorFactory(RowBasedColumnSelectorFactory.create(RowAdapters.standardRow(), () -> new MapBasedRow(0L, bindings), rowSignature, false), VirtualColumns.create(virtualColumns)));
    Assert.assertEquals("Result for: " + rexNode, expectedResult, matcher.matches());
}