Examples with DimFilter io.druid.query.filter.DimFilter used on opensource projects

Example 51 with DimFilter

use of io.druid.query.filter.DimFilter in project druid by druid-io.

the class GroupByRules method applyHaving.

/**
   * Applies a filter to an aggregating druidRel, as a HavingSpec. Do not call this method unless
   * {@link #canApplyHaving(DruidRel)} returns true.
   *
   * @return new rel, or null if the filter cannot be applied
   */
private static DruidRel applyHaving(final DruidOperatorTable operatorTable, final DruidRel druidRel, final Filter postFilter) {
    Preconditions.checkState(canApplyHaving(druidRel), "Cannot applyHaving.");
    final DimFilter dimFilter = Expressions.toFilter(operatorTable, druidRel.getPlannerContext(), druidRel.getOutputRowSignature(), postFilter.getCondition());
    if (dimFilter != null) {
        return druidRel.withQueryBuilder(druidRel.getQueryBuilder().withHaving(dimFilter));
    } else {
        return null;
    }
}

Example 52 with DimFilter

use of io.druid.query.filter.DimFilter in project druid by druid-io.

the class Expressions method toLeafFilter.

/**
   * Translates "condition" to a Druid filter, assuming it does not contain any boolean expressions. Returns null
   * if we cannot translate the condition.
   *
   * @param plannerContext planner context
   * @param rowSignature   row signature of the dataSource to be filtered
   * @param expression     Calcite row expression
   */
private static DimFilter toLeafFilter(final DruidOperatorTable operatorTable, final PlannerContext plannerContext, final RowSignature rowSignature, final RexNode expression) {
    if (expression.isAlwaysTrue()) {
        return Filtration.matchEverything();
    } else if (expression.isAlwaysFalse()) {
        return Filtration.matchNothing();
    }
    final SqlKind kind = expression.getKind();
    if (kind == SqlKind.LIKE) {
        final List<RexNode> operands = ((RexCall) expression).getOperands();
        final RowExtraction rex = toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), operands.get(0));
        if (rex == null || !rex.isFilterable(rowSignature)) {
            return null;
        }
        return new LikeDimFilter(rex.getColumn(), RexLiteral.stringValue(operands.get(1)), operands.size() > 2 ? RexLiteral.stringValue(operands.get(2)) : null, rex.getExtractionFn());
    } else if (kind == SqlKind.EQUALS || kind == SqlKind.NOT_EQUALS || kind == SqlKind.GREATER_THAN || kind == SqlKind.GREATER_THAN_OR_EQUAL || kind == SqlKind.LESS_THAN || kind == SqlKind.LESS_THAN_OR_EQUAL) {
        final List<RexNode> operands = ((RexCall) expression).getOperands();
        Preconditions.checkState(operands.size() == 2, "WTF?! Expected 2 operands, got[%,d]", operands.size());
        boolean flip = false;
        RexNode lhs = operands.get(0);
        RexNode rhs = operands.get(1);
        if (lhs.getKind() == SqlKind.LITERAL && rhs.getKind() != SqlKind.LITERAL) {
            // swap lhs, rhs
            RexNode x = lhs;
            lhs = rhs;
            rhs = x;
            flip = true;
        }
        // rhs must be a literal
        if (rhs.getKind() != SqlKind.LITERAL) {
            return null;
        }
        // lhs must be translatable to a RowExtraction to be filterable
        final RowExtraction rex = toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), lhs);
        if (rex == null || !rex.isFilterable(rowSignature)) {
            return null;
        }
        final String column = rex.getColumn();
        final ExtractionFn extractionFn = rex.getExtractionFn();
        if (column.equals(Column.TIME_COLUMN_NAME) && extractionFn instanceof TimeFormatExtractionFn) {
            // Check if we can strip the extractionFn and convert the filter to a direct filter on __time.
            // This allows potential conversion to query-level "intervals" later on, which is ideal for Druid queries.
            final Granularity granularity = ExtractionFns.toQueryGranularity(extractionFn);
            if (granularity != null) {
                // lhs is FLOOR(__time TO granularity); rhs must be a timestamp
                final long rhsMillis = toMillisLiteral(rhs, plannerContext.getTimeZone());
                final Interval rhsInterval = granularity.bucket(new DateTime(rhsMillis));
                // Is rhs aligned on granularity boundaries?
                final boolean rhsAligned = rhsInterval.getStartMillis() == rhsMillis;
                // Create a BoundRefKey that strips the extractionFn and compares __time as a number.
                final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC);
                if (kind == SqlKind.EQUALS) {
                    return rhsAligned ? Bounds.interval(boundRefKey, rhsInterval) : Filtration.matchNothing();
                } else if (kind == SqlKind.NOT_EQUALS) {
                    return rhsAligned ? new NotDimFilter(Bounds.interval(boundRefKey, rhsInterval)) : Filtration.matchEverything();
                } else if ((!flip && kind == SqlKind.GREATER_THAN) || (flip && kind == SqlKind.LESS_THAN)) {
                    return Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.GREATER_THAN_OR_EQUAL) || (flip && kind == SqlKind.LESS_THAN_OR_EQUAL)) {
                    return rhsAligned ? Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) : Bounds.greaterThanOrEqualTo(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.LESS_THAN) || (flip && kind == SqlKind.GREATER_THAN)) {
                    return rhsAligned ? Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getStartMillis())) : Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else if ((!flip && kind == SqlKind.LESS_THAN_OR_EQUAL) || (flip && kind == SqlKind.GREATER_THAN_OR_EQUAL)) {
                    return Bounds.lessThan(boundRefKey, String.valueOf(rhsInterval.getEndMillis()));
                } else {
                    throw new IllegalStateException("WTF?! Shouldn't have got here...");
                }
            }
        }
        final String val;
        final RexLiteral rhsLiteral = (RexLiteral) rhs;
        if (SqlTypeName.NUMERIC_TYPES.contains(rhsLiteral.getTypeName())) {
            val = String.valueOf(RexLiteral.value(rhsLiteral));
        } else if (SqlTypeName.CHAR_TYPES.contains(rhsLiteral.getTypeName())) {
            val = String.valueOf(RexLiteral.stringValue(rhsLiteral));
        } else if (SqlTypeName.TIMESTAMP == rhsLiteral.getTypeName() || SqlTypeName.DATE == rhsLiteral.getTypeName()) {
            val = String.valueOf(toMillisLiteral(rhsLiteral, plannerContext.getTimeZone()));
        } else {
            // Don't know how to filter on this kind of literal.
            return null;
        }
        // Numeric lhs needs a numeric comparison.
        final boolean lhsIsNumeric = SqlTypeName.NUMERIC_TYPES.contains(lhs.getType().getSqlTypeName()) || SqlTypeName.TIMESTAMP == lhs.getType().getSqlTypeName() || SqlTypeName.DATE == lhs.getType().getSqlTypeName();
        final StringComparator comparator = lhsIsNumeric ? StringComparators.NUMERIC : StringComparators.LEXICOGRAPHIC;
        final BoundRefKey boundRefKey = new BoundRefKey(column, extractionFn, comparator);
        final DimFilter filter;
        // Always use BoundDimFilters, to simplify filter optimization later (it helps to remember the comparator).
        if (kind == SqlKind.EQUALS) {
            filter = Bounds.equalTo(boundRefKey, val);
        } else if (kind == SqlKind.NOT_EQUALS) {
            filter = new NotDimFilter(Bounds.equalTo(boundRefKey, val));
        } else if ((!flip && kind == SqlKind.GREATER_THAN) || (flip && kind == SqlKind.LESS_THAN)) {
            filter = Bounds.greaterThan(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.GREATER_THAN_OR_EQUAL) || (flip && kind == SqlKind.LESS_THAN_OR_EQUAL)) {
            filter = Bounds.greaterThanOrEqualTo(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.LESS_THAN) || (flip && kind == SqlKind.GREATER_THAN)) {
            filter = Bounds.lessThan(boundRefKey, val);
        } else if ((!flip && kind == SqlKind.LESS_THAN_OR_EQUAL) || (flip && kind == SqlKind.GREATER_THAN_OR_EQUAL)) {
            filter = Bounds.lessThanOrEqualTo(boundRefKey, val);
        } else {
            throw new IllegalStateException("WTF?! Shouldn't have got here...");
        }
        return filter;
    } else {
        return null;
    }
}

Example 53 with DimFilter

use of io.druid.query.filter.DimFilter in project druid by druid-io.

the class BottomUpTransform method apply0.

private DimFilter apply0(final DimFilter filter) {
    if (filter instanceof AndDimFilter) {
        final List<DimFilter> oldFilters = ((AndDimFilter) filter).getFields();
        final List<DimFilter> newFilters = Lists.newArrayList();
        for (DimFilter oldFilter : oldFilters) {
            final DimFilter newFilter = apply0(oldFilter);
            if (newFilter != null) {
                newFilters.add(newFilter);
            }
        }
        if (!newFilters.equals(oldFilters)) {
            return checkedProcess(new AndDimFilter(newFilters));
        } else {
            return checkedProcess(filter);
        }
    } else if (filter instanceof OrDimFilter) {
        final List<DimFilter> oldFilters = ((OrDimFilter) filter).getFields();
        final List<DimFilter> newFilters = Lists.newArrayList();
        for (DimFilter oldFilter : oldFilters) {
            final DimFilter newFilter = apply0(oldFilter);
            if (newFilter != null) {
                newFilters.add(newFilter);
            }
        }
        if (!newFilters.equals(oldFilters)) {
            return checkedProcess(new OrDimFilter(newFilters));
        } else {
            return checkedProcess(filter);
        }
    } else if (filter instanceof NotDimFilter) {
        final DimFilter oldFilter = ((NotDimFilter) filter).getField();
        final DimFilter newFilter = apply0(oldFilter);
        if (!oldFilter.equals(newFilter)) {
            return checkedProcess(new NotDimFilter(newFilter));
        } else {
            return checkedProcess(filter);
        }
    } else {
        return checkedProcess(filter);
    }
}

Example 54 with DimFilter

use of io.druid.query.filter.DimFilter in project druid by druid-io.

the class ConvertSelectorsToIns method process.

@Override
public DimFilter process(DimFilter filter) {
    if (filter instanceof OrDimFilter) {
        // Copy children list
        final List<DimFilter> children = Lists.newArrayList(((OrDimFilter) filter).getFields());
        // Group filters by dimension and extractionFn.
        final Map<BoundRefKey, List<SelectorDimFilter>> selectors = Maps.newHashMap();
        for (DimFilter child : children) {
            if (child instanceof SelectorDimFilter) {
                final SelectorDimFilter selector = (SelectorDimFilter) child;
                final BoundRefKey boundRefKey = BoundRefKey.from(selector, sourceRowSignature.naturalStringComparator(RowExtraction.of(selector.getDimension(), selector.getExtractionFn())));
                List<SelectorDimFilter> filterList = selectors.get(boundRefKey);
                if (filterList == null) {
                    filterList = Lists.newArrayList();
                    selectors.put(boundRefKey, filterList);
                }
                filterList.add(selector);
            }
        }
        // Emit IN filters for each group of size > 1.
        for (Map.Entry<BoundRefKey, List<SelectorDimFilter>> entry : selectors.entrySet()) {
            final List<SelectorDimFilter> filterList = entry.getValue();
            if (filterList.size() > 1) {
                // We found a simplification. Remove the old filters and add new ones.
                final List<String> values = Lists.newArrayList();
                for (final SelectorDimFilter selector : filterList) {
                    values.add(selector.getValue());
                    if (!children.remove(selector)) {
                        throw new ISE("WTF?! Tried to remove selector but couldn't?");
                    }
                }
                children.add(new InDimFilter(entry.getKey().getDimension(), values, entry.getKey().getExtractionFn()));
            }
        }
        if (!children.equals(((OrDimFilter) filter).getFields())) {
            return children.size() == 1 ? children.get(0) : new OrDimFilter(children);
        } else {
            return filter;
        }
    } else {
        return filter;
    }
}

Example 55 with DimFilter

use of io.druid.query.filter.DimFilter in project druid by druid-io.

the class MoveTimeFiltersToIntervals method extractConvertibleTimeBounds.

/**
   * Extract bound filters on __time that can be converted to query-level "intervals".
   *
   * @return pair of new dimFilter + RangeSet of __time that should be ANDed together. Either can be null but not both.
   */
private static Pair<DimFilter, RangeSet<Long>> extractConvertibleTimeBounds(final DimFilter filter) {
    if (filter instanceof AndDimFilter) {
        final List<DimFilter> children = ((AndDimFilter) filter).getFields();
        final List<DimFilter> newChildren = Lists.newArrayList();
        final List<RangeSet<Long>> rangeSets = Lists.newArrayList();
        for (DimFilter child : children) {
            final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
            if (pair.lhs != null) {
                newChildren.add(pair.lhs);
            }
            if (pair.rhs != null) {
                rangeSets.add(pair.rhs);
            }
        }
        final DimFilter newFilter;
        if (newChildren.size() == 0) {
            newFilter = null;
        } else if (newChildren.size() == 1) {
            newFilter = newChildren.get(0);
        } else {
            newFilter = new AndDimFilter(newChildren);
        }
        return Pair.of(newFilter, rangeSets.isEmpty() ? null : RangeSets.intersectRangeSets(rangeSets));
    } else if (filter instanceof OrDimFilter) {
        final List<DimFilter> children = ((OrDimFilter) filter).getFields();
        final List<RangeSet<Long>> rangeSets = Lists.newArrayList();
        boolean allCompletelyConverted = true;
        boolean allHadIntervals = true;
        for (DimFilter child : children) {
            final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
            if (pair.lhs != null) {
                allCompletelyConverted = false;
            }
            if (pair.rhs != null) {
                rangeSets.add(pair.rhs);
            } else {
                allHadIntervals = false;
            }
        }
        if (allCompletelyConverted) {
            return Pair.of(null, RangeSets.unionRangeSets(rangeSets));
        } else {
            return Pair.of(filter, allHadIntervals ? RangeSets.unionRangeSets(rangeSets) : null);
        }
    } else if (filter instanceof NotDimFilter) {
        final DimFilter child = ((NotDimFilter) filter).getField();
        final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
        if (pair.rhs != null && pair.lhs == null) {
            return Pair.of(null, pair.rhs.complement());
        } else {
            return Pair.of(filter, null);
        }
    } else if (filter instanceof BoundDimFilter) {
        final BoundDimFilter bound = (BoundDimFilter) filter;
        if (BoundRefKey.from(bound).equals(TIME_BOUND_REF_KEY)) {
            return Pair.of(null, RangeSets.of(toLongRange(Bounds.toRange(bound))));
        } else {
            return Pair.of(filter, null);
        }
    } else {
        return Pair.of(filter, null);
    }
}