Examples with DimFilter org.apache.druid.query.filter.DimFilter used on opensource projects

Example 21 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class Aggregation method filter.

public Aggregation filter(final RowSignature rowSignature, final VirtualColumnRegistry virtualColumnRegistry, final DimFilter filter) {
    if (filter == null) {
        return this;
    }
    if (postAggregator != null) {
        // Verify that this Aggregation contains all input to its postAggregator.
        // If not, this "filter" call won't work right.
        final Set<String> dependentFields = postAggregator.getDependentFields();
        final Set<String> aggregatorNames = new HashSet<>();
        for (AggregatorFactory aggregatorFactory : aggregatorFactories) {
            aggregatorNames.add(aggregatorFactory.getName());
        }
        for (String field : dependentFields) {
            if (!aggregatorNames.contains(field)) {
                throw new ISE("Cannot filter an Aggregation that does not contain its inputs: %s", this);
            }
        }
    }
    final DimFilter baseOptimizedFilter = Filtration.create(filter).optimizeFilterOnly(virtualColumnRegistry.getFullRowSignature()).getDimFilter();
    final List<AggregatorFactory> newAggregators = new ArrayList<>();
    for (AggregatorFactory agg : aggregatorFactories) {
        if (agg instanceof FilteredAggregatorFactory) {
            final FilteredAggregatorFactory filteredAgg = (FilteredAggregatorFactory) agg;
            newAggregators.add(new FilteredAggregatorFactory(filteredAgg.getAggregator(), Filtration.create(new AndDimFilter(ImmutableList.of(filteredAgg.getFilter(), baseOptimizedFilter))).optimizeFilterOnly(virtualColumnRegistry.getFullRowSignature()).getDimFilter()));
        } else {
            newAggregators.add(new FilteredAggregatorFactory(agg, baseOptimizedFilter));
        }
    }
    return new Aggregation(newAggregators, postAggregator);
}

Example 22 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class CombineAndSimplifyBounds method doSimplify.

/**
 * Simplify BoundDimFilters that are children of an OR or an AND.
 *
 * @param children    the filters
 * @param disjunction true for OR, false for AND
 *
 * @return simplified filters
 */
private static DimFilter doSimplify(final List<DimFilter> children, boolean disjunction) {
    // Copy the list of child filters. We'll modify the copy and eventually return it.
    final List<DimFilter> newChildren = Lists.newArrayList(children);
    // Group Bound filters by dimension, extractionFn, and comparator and compute a RangeSet for each one.
    final Map<BoundRefKey, List<BoundDimFilter>> bounds = new HashMap<>();
    // all and/or filters have at least 1 child
    boolean allFalse = true;
    for (final DimFilter child : newChildren) {
        if (child instanceof BoundDimFilter) {
            final BoundDimFilter bound = (BoundDimFilter) child;
            final BoundRefKey boundRefKey = BoundRefKey.from(bound);
            final List<BoundDimFilter> filterList = bounds.computeIfAbsent(boundRefKey, k -> new ArrayList<>());
            filterList.add(bound);
            allFalse = false;
        } else {
            allFalse &= child instanceof FalseDimFilter;
        }
    }
    // short circuit if can never be true
    if (allFalse) {
        return Filtration.matchNothing();
    }
    // Try to simplify filters within each group.
    for (Map.Entry<BoundRefKey, List<BoundDimFilter>> entry : bounds.entrySet()) {
        final BoundRefKey boundRefKey = entry.getKey();
        final List<BoundDimFilter> filterList = entry.getValue();
        // Create a RangeSet for this group.
        final RangeSet<BoundValue> rangeSet = disjunction ? RangeSets.unionRanges(Bounds.toRanges(filterList)) : RangeSets.intersectRanges(Bounds.toRanges(filterList));
        if (rangeSet.asRanges().size() < filterList.size()) {
            // We found a simplification. Remove the old filters and add new ones.
            for (final BoundDimFilter bound : filterList) {
                if (!newChildren.remove(bound)) {
                    // Don't expect this to happen, but include it as a sanity check.
                    throw new ISE("Tried to remove bound, but couldn't");
                }
            }
            if (rangeSet.asRanges().isEmpty()) {
                // range set matches nothing, equivalent to FALSE
                newChildren.add(Filtration.matchNothing());
            }
            for (final Range<BoundValue> range : rangeSet.asRanges()) {
                if (!range.hasLowerBound() && !range.hasUpperBound()) {
                    // range matches all, equivalent to TRUE
                    newChildren.add(Filtration.matchEverything());
                } else {
                    newChildren.add(Bounds.toFilter(boundRefKey, range));
                }
            }
        }
    }
    // Finally: Go through newChildren, removing or potentially exiting early based on TRUE / FALSE marker filters.
    Preconditions.checkState(newChildren.size() > 0, "newChildren.size > 0");
    final Iterator<DimFilter> iterator = newChildren.iterator();
    while (iterator.hasNext()) {
        final DimFilter newChild = iterator.next();
        if (Filtration.matchNothing().equals(newChild)) {
            // AND with FALSE => always false, short circuit
            if (disjunction) {
                iterator.remove();
            } else {
                return Filtration.matchNothing();
            }
        } else if (Filtration.matchEverything().equals(newChild)) {
            // AND with TRUE => ignore
            if (disjunction) {
                return Filtration.matchEverything();
            } else {
                iterator.remove();
            }
        }
    }
    if (newChildren.isEmpty()) {
        // If "newChildren" is empty at this point, it must have consisted entirely of TRUE / FALSE marker filters.
        if (disjunction) {
            // Must have been all FALSE filters (the only kind we would have removed above).
            return Filtration.matchNothing();
        } else {
            // Must have been all TRUE filters (the only kind we would have removed above).
            return Filtration.matchEverything();
        }
    } else if (newChildren.size() == 1) {
        return newChildren.get(0);
    } else {
        return disjunction ? new OrDimFilter(newChildren) : new AndDimFilter(newChildren);
    }
}

Example 23 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class BottomUpTransform method apply0.

private DimFilter apply0(final DimFilter filter) {
    if (filter instanceof AndDimFilter) {
        final List<DimFilter> oldFilters = ((AndDimFilter) filter).getFields();
        final List<DimFilter> newFilters = new ArrayList<>();
        for (DimFilter oldFilter : oldFilters) {
            final DimFilter newFilter = apply0(oldFilter);
            if (newFilter != null) {
                newFilters.add(newFilter);
            }
        }
        if (!newFilters.equals(oldFilters)) {
            return checkedProcess(new AndDimFilter(newFilters));
        } else {
            return checkedProcess(filter);
        }
    } else if (filter instanceof OrDimFilter) {
        final List<DimFilter> oldFilters = ((OrDimFilter) filter).getFields();
        final List<DimFilter> newFilters = new ArrayList<>();
        for (DimFilter oldFilter : oldFilters) {
            final DimFilter newFilter = apply0(oldFilter);
            if (newFilter != null) {
                newFilters.add(newFilter);
            }
        }
        if (!newFilters.equals(oldFilters)) {
            return checkedProcess(new OrDimFilter(newFilters));
        } else {
            return checkedProcess(filter);
        }
    } else if (filter instanceof NotDimFilter) {
        final DimFilter oldFilter = ((NotDimFilter) filter).getField();
        final DimFilter newFilter = apply0(oldFilter);
        if (!oldFilter.equals(newFilter)) {
            return checkedProcess(new NotDimFilter(newFilter));
        } else {
            return checkedProcess(filter);
        }
    } else {
        return checkedProcess(filter);
    }
}

Example 24 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project druid by druid-io.

the class Queries method withBaseDataSource.

/**
 * Rewrite "query" to refer to some specific base datasource, instead of the one it currently refers to.
 *
 * Unlike the seemingly-similar {@link Query#withDataSource}, this will walk down the datasource tree and replace
 * only the base datasource (in the sense defined in {@link DataSourceAnalysis}).
 */
public static <T> Query<T> withBaseDataSource(final Query<T> query, final DataSource newBaseDataSource) {
    final Query<T> retVal;
    if (query.getDataSource() instanceof QueryDataSource) {
        final Query<?> subQuery = ((QueryDataSource) query.getDataSource()).getQuery();
        retVal = query.withDataSource(new QueryDataSource(withBaseDataSource(subQuery, newBaseDataSource)));
    } else {
        final DataSourceAnalysis analysis = DataSourceAnalysis.forDataSource(query.getDataSource());
        DataSource current = newBaseDataSource;
        DimFilter joinBaseFilter = analysis.getJoinBaseTableFilter().orElse(null);
        for (final PreJoinableClause clause : analysis.getPreJoinableClauses()) {
            current = JoinDataSource.create(current, clause.getDataSource(), clause.getPrefix(), clause.getCondition(), clause.getJoinType(), joinBaseFilter);
            joinBaseFilter = null;
        }
        retVal = query.withDataSource(current);
    }
    // Verify postconditions, just in case.
    final DataSourceAnalysis analysis = DataSourceAnalysis.forDataSource(retVal.getDataSource());
    if (!newBaseDataSource.equals(analysis.getBaseDataSource())) {
        throw new ISE("Unable to replace base dataSource");
    }
    return retVal;
}

Example 25 with DimFilter

use of org.apache.druid.query.filter.DimFilter in project hive by apache.

the class DruidStorageHandlerUtils method addDynamicFilters.

public static org.apache.druid.query.Query addDynamicFilters(org.apache.druid.query.Query query, ExprNodeGenericFuncDesc filterExpr, Configuration conf, boolean resolveDynamicValues) {
    List<VirtualColumn> virtualColumns = Arrays.asList(getVirtualColumns(query).getVirtualColumns());
    org.apache.druid.query.Query rv = query;
    DimFilter joinReductionFilter = toDruidFilter(filterExpr, conf, virtualColumns, resolveDynamicValues);
    if (joinReductionFilter != null) {
        String type = query.getType();
        DimFilter filter = new AndDimFilter(joinReductionFilter, query.getFilter());
        switch(type) {
            case org.apache.druid.query.Query.TIMESERIES:
                rv = Druids.TimeseriesQueryBuilder.copy((TimeseriesQuery) query).filters(filter).virtualColumns(VirtualColumns.create(virtualColumns)).build();
                break;
            case org.apache.druid.query.Query.TOPN:
                rv = new TopNQueryBuilder((TopNQuery) query).filters(filter).virtualColumns(VirtualColumns.create(virtualColumns)).build();
                break;
            case org.apache.druid.query.Query.GROUP_BY:
                rv = new GroupByQuery.Builder((GroupByQuery) query).setDimFilter(filter).setVirtualColumns(VirtualColumns.create(virtualColumns)).build();
                break;
            case org.apache.druid.query.Query.SCAN:
                rv = Druids.ScanQueryBuilder.copy((ScanQuery) query).filters(filter).virtualColumns(VirtualColumns.create(virtualColumns)).build();
                break;
            default:
                throw new UnsupportedOperationException("Unsupported Query type " + type);
        }
    }
    return rv;
}