Search in sources :

Example 56 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class GroupByRules method translateAggregateCall.

/**
 * Translate an AggregateCall to Druid equivalents.
 *
 * @return translated aggregation, or null if translation failed.
 */
public static Aggregation translateAggregateCall(final PlannerContext plannerContext, final RowSignature rowSignature, final VirtualColumnRegistry virtualColumnRegistry, final RexBuilder rexBuilder, final Project project, final List<Aggregation> existingAggregations, final String name, final AggregateCall call, final boolean finalizeAggregations) {
    final DimFilter filter;
    if (call.filterArg >= 0) {
        // AGG(xxx) FILTER(WHERE yyy)
        if (project == null) {
            // We need some kind of projection to support filtered aggregations.
            return null;
        }
        final RexNode expression = project.getChildExps().get(call.filterArg);
        final DimFilter nonOptimizedFilter = Expressions.toFilter(plannerContext, rowSignature, virtualColumnRegistry, expression);
        if (nonOptimizedFilter == null) {
            return null;
        } else {
            filter = Filtration.create(nonOptimizedFilter).optimizeFilterOnly(virtualColumnRegistry.getFullRowSignature()).getDimFilter();
        }
    } else {
        filter = null;
    }
    final SqlAggregator sqlAggregator = plannerContext.getOperatorTable().lookupAggregator(call.getAggregation());
    if (sqlAggregator == null) {
        return null;
    }
    // Compute existingAggregations for SqlAggregator impls that want it.
    final List<Aggregation> existingAggregationsWithSameFilter = new ArrayList<>();
    for (Aggregation existingAggregation : existingAggregations) {
        if (filter == null) {
            final boolean doesMatch = existingAggregation.getAggregatorFactories().stream().noneMatch(factory -> factory instanceof FilteredAggregatorFactory);
            if (doesMatch) {
                existingAggregationsWithSameFilter.add(existingAggregation);
            }
        } else {
            final boolean doesMatch = existingAggregation.getAggregatorFactories().stream().allMatch(factory -> factory instanceof FilteredAggregatorFactory && ((FilteredAggregatorFactory) factory).getFilter().equals(filter));
            if (doesMatch) {
                existingAggregationsWithSameFilter.add(Aggregation.create(existingAggregation.getAggregatorFactories().stream().map(factory -> ((FilteredAggregatorFactory) factory).getAggregator()).collect(Collectors.toList()), existingAggregation.getPostAggregator()));
            }
        }
    }
    final Aggregation retVal = sqlAggregator.toDruidAggregation(plannerContext, rowSignature, virtualColumnRegistry, rexBuilder, name, call, project, existingAggregationsWithSameFilter, finalizeAggregations);
    if (retVal == null) {
        return null;
    } else {
        // Check if this refers to the existingAggregationsWithSameFilter. If so, no need to apply the filter.
        if (isUsingExistingAggregation(retVal, existingAggregationsWithSameFilter)) {
            return retVal;
        } else {
            return retVal.filter(rowSignature, virtualColumnRegistry, filter);
        }
    }
}
Also used : Aggregation(org.apache.druid.sql.calcite.aggregation.Aggregation) FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) Project(org.apache.calcite.rel.core.Project) RexBuilder(org.apache.calcite.rex.RexBuilder) SqlAggregator(org.apache.druid.sql.calcite.aggregation.SqlAggregator) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) Set(java.util.Set) Aggregation(org.apache.druid.sql.calcite.aggregation.Aggregation) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) List(java.util.List) DimFilter(org.apache.druid.query.filter.DimFilter) RexNode(org.apache.calcite.rex.RexNode) RowSignature(org.apache.druid.segment.column.RowSignature) VirtualColumnRegistry(org.apache.druid.sql.calcite.rel.VirtualColumnRegistry) PlannerContext(org.apache.druid.sql.calcite.planner.PlannerContext) AggregateCall(org.apache.calcite.rel.core.AggregateCall) Expressions(org.apache.druid.sql.calcite.expression.Expressions) Filtration(org.apache.druid.sql.calcite.filtration.Filtration) SqlAggregator(org.apache.druid.sql.calcite.aggregation.SqlAggregator) ArrayList(java.util.ArrayList) DimFilter(org.apache.druid.query.filter.DimFilter) RexNode(org.apache.calcite.rex.RexNode)

Example 57 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project hive by apache.

the class DruidStorageHandlerUtils method toDruidFilter.

@Nullable
private static DimFilter toDruidFilter(ExprNodeDesc filterExpr, Configuration configuration, List<VirtualColumn> virtualColumns, boolean resolveDynamicValues) {
    if (filterExpr == null) {
        return null;
    }
    Class<? extends GenericUDF> genericUDFClass = getGenericUDFClassFromExprDesc(filterExpr);
    if (FunctionRegistry.isOpAnd(filterExpr)) {
        Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
        List<DimFilter> delegates = Lists.newArrayList();
        while (iterator.hasNext()) {
            DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
            if (filter != null) {
                delegates.add(filter);
            }
        }
        if (!delegates.isEmpty()) {
            return new AndDimFilter(delegates);
        }
    }
    if (FunctionRegistry.isOpOr(filterExpr)) {
        Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
        List<DimFilter> delegates = Lists.newArrayList();
        while (iterator.hasNext()) {
            DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
            if (filter != null) {
                delegates.add(filter);
            }
        }
        if (!delegates.isEmpty()) {
            return new OrDimFilter(delegates);
        }
    } else if (GenericUDFBetween.class == genericUDFClass) {
        List<ExprNodeDesc> child = filterExpr.getChildren();
        String col = extractColName(child.get(1), virtualColumns);
        if (col != null) {
            try {
                StringComparator comparator = stringTypeInfos.contains(child.get(1).getTypeInfo()) ? StringComparators.LEXICOGRAPHIC : StringComparators.NUMERIC;
                String lower = evaluate(child.get(2), configuration, resolveDynamicValues);
                String upper = evaluate(child.get(3), configuration, resolveDynamicValues);
                return new BoundDimFilter(col, lower, upper, false, false, null, null, comparator);
            } catch (HiveException e) {
                throw new RuntimeException(e);
            }
        }
    } else if (GenericUDFInBloomFilter.class == genericUDFClass) {
        List<ExprNodeDesc> child = filterExpr.getChildren();
        String col = extractColName(child.get(0), virtualColumns);
        if (col != null) {
            try {
                BloomKFilter bloomFilter = evaluateBloomFilter(child.get(1), configuration, resolveDynamicValues);
                return new BloomDimFilter(col, BloomKFilterHolder.fromBloomKFilter(bloomFilter), null);
            } catch (HiveException | IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
    return null;
}
Also used : GenericUDFBetween(org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween) BoundDimFilter(org.apache.druid.query.filter.BoundDimFilter) HiveException(org.apache.hadoop.hive.ql.metadata.HiveException) AndDimFilter(org.apache.druid.query.filter.AndDimFilter) GenericUDFToString(org.apache.hadoop.hive.ql.udf.generic.GenericUDFToString) IOException(java.io.IOException) StringComparator(org.apache.druid.query.ordering.StringComparator) BloomKFilter(org.apache.druid.query.filter.BloomKFilter) OrDimFilter(org.apache.druid.query.filter.OrDimFilter) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) BloomDimFilter(org.apache.druid.query.filter.BloomDimFilter) ExprNodeDesc(org.apache.hadoop.hive.ql.plan.ExprNodeDesc) AndDimFilter(org.apache.druid.query.filter.AndDimFilter) DimFilter(org.apache.druid.query.filter.DimFilter) BoundDimFilter(org.apache.druid.query.filter.BoundDimFilter) BloomDimFilter(org.apache.druid.query.filter.BloomDimFilter) OrDimFilter(org.apache.druid.query.filter.OrDimFilter) Nullable(javax.annotation.Nullable)

Aggregations

DimFilter (org.apache.druid.query.filter.DimFilter)57 AndDimFilter (org.apache.druid.query.filter.AndDimFilter)31 SelectorDimFilter (org.apache.druid.query.filter.SelectorDimFilter)30 ArrayList (java.util.ArrayList)29 BoundDimFilter (org.apache.druid.query.filter.BoundDimFilter)29 OrDimFilter (org.apache.druid.query.filter.OrDimFilter)27 Test (org.junit.Test)21 InDimFilter (org.apache.druid.query.filter.InDimFilter)18 List (java.util.List)14 HashMap (java.util.HashMap)12 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)12 ExtractionDimFilter (org.apache.druid.query.filter.ExtractionDimFilter)11 NotDimFilter (org.apache.druid.query.filter.NotDimFilter)11 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)9 RegexDimFilter (org.apache.druid.query.filter.RegexDimFilter)9 ISE (org.apache.druid.java.util.common.ISE)7 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)7 JavaScriptDimFilter (org.apache.druid.query.filter.JavaScriptDimFilter)7 SearchQueryDimFilter (org.apache.druid.query.filter.SearchQueryDimFilter)7 Interval (org.joda.time.Interval)7