Examples with NotDimFilter org.apache.druid.query.filter.NotDimFilter used on opensource projects

Example 16 with NotDimFilter

use of org.apache.druid.query.filter.NotDimFilter in project druid by druid-io.

the class MoveTimeFiltersToIntervals method extractConvertibleTimeBounds.

/**
 * Extract bound filters on __time that can be converted to query-level "intervals".
 *
 * @return pair of new dimFilter + RangeSet of __time that should be ANDed together. Either can be null but not both.
 */
private static Pair<DimFilter, RangeSet<Long>> extractConvertibleTimeBounds(final DimFilter filter) {
    if (filter instanceof AndDimFilter) {
        final List<DimFilter> children = ((AndDimFilter) filter).getFields();
        final List<DimFilter> newChildren = new ArrayList<>();
        final List<RangeSet<Long>> rangeSets = new ArrayList<>();
        for (DimFilter child : children) {
            final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
            if (pair.lhs != null) {
                newChildren.add(pair.lhs);
            }
            if (pair.rhs != null) {
                rangeSets.add(pair.rhs);
            }
        }
        final DimFilter newFilter;
        if (newChildren.size() == 0) {
            newFilter = null;
        } else if (newChildren.size() == 1) {
            newFilter = newChildren.get(0);
        } else {
            newFilter = new AndDimFilter(newChildren);
        }
        return Pair.of(newFilter, rangeSets.isEmpty() ? null : RangeSets.intersectRangeSets(rangeSets));
    } else if (filter instanceof OrDimFilter) {
        final List<DimFilter> children = ((OrDimFilter) filter).getFields();
        final List<RangeSet<Long>> rangeSets = new ArrayList<>();
        boolean allCompletelyConverted = true;
        boolean allHadIntervals = true;
        for (DimFilter child : children) {
            final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
            if (pair.lhs != null) {
                allCompletelyConverted = false;
            }
            if (pair.rhs != null) {
                rangeSets.add(pair.rhs);
            } else {
                allHadIntervals = false;
            }
        }
        if (allCompletelyConverted) {
            return Pair.of(null, RangeSets.unionRangeSets(rangeSets));
        } else {
            return Pair.of(filter, allHadIntervals ? RangeSets.unionRangeSets(rangeSets) : null);
        }
    } else if (filter instanceof NotDimFilter) {
        final DimFilter child = ((NotDimFilter) filter).getField();
        final Pair<DimFilter, RangeSet<Long>> pair = extractConvertibleTimeBounds(child);
        if (pair.rhs != null && pair.lhs == null) {
            return Pair.of(null, pair.rhs.complement());
        } else {
            return Pair.of(filter, null);
        }
    } else if (filter instanceof BoundDimFilter) {
        final BoundDimFilter bound = (BoundDimFilter) filter;
        if (BoundRefKey.from(bound).equals(TIME_BOUND_REF_KEY)) {
            return Pair.of(null, RangeSets.of(toLongRange(Bounds.toRange(bound))));
        } else {
            return Pair.of(filter, null);
        }
    } else {
        return Pair.of(filter, null);
    }
}

Example 17 with NotDimFilter

use of org.apache.druid.query.filter.NotDimFilter in project druid by druid-io.

the class CombineAndSimplifyBounds method process.

@Override
public DimFilter process(DimFilter filter) {
    if (filter instanceof FalseDimFilter) {
        // we might sometimes come into here with just a false from optimizing impossible conditions
        return filter;
    } else if (filter instanceof AndDimFilter) {
        final List<DimFilter> children = getAndFilterChildren((AndDimFilter) filter);
        final DimFilter one = doSimplifyAnd(children);
        final DimFilter two = negate(doSimplifyOr(negateAll(children)));
        return computeCost(one) <= computeCost(two) ? one : two;
    } else if (filter instanceof OrDimFilter) {
        final List<DimFilter> children = getOrFilterChildren((OrDimFilter) filter);
        final DimFilter one = doSimplifyOr(children);
        final DimFilter two = negate(doSimplifyAnd(negateAll(children)));
        return computeCost(one) <= computeCost(two) ? one : two;
    } else if (filter instanceof NotDimFilter) {
        final DimFilter field = ((NotDimFilter) filter).getField();
        final DimFilter candidate;
        if (field instanceof OrDimFilter) {
            candidate = doSimplifyAnd(negateAll(getOrFilterChildren((OrDimFilter) field)));
        } else if (field instanceof AndDimFilter) {
            candidate = doSimplifyOr(negateAll(getAndFilterChildren((AndDimFilter) field)));
        } else {
            candidate = negate(field);
        }
        return computeCost(filter) <= computeCost(candidate) ? filter : candidate;
    } else {
        return filter;
    }
}

Example 18 with NotDimFilter

use of org.apache.druid.query.filter.NotDimFilter in project druid by druid-io.

the class StringSqlAggregator method toDruidAggregation.

@Nullable
@Override
public Aggregation toDruidAggregation(PlannerContext plannerContext, RowSignature rowSignature, VirtualColumnRegistry virtualColumnRegistry, RexBuilder rexBuilder, String name, AggregateCall aggregateCall, Project project, List<Aggregation> existingAggregations, boolean finalizeAggregations) {
    final List<DruidExpression> arguments = aggregateCall.getArgList().stream().map(i -> Expressions.fromFieldAccess(rowSignature, project, i)).map(rexNode -> Expressions.toDruidExpression(plannerContext, rowSignature, rexNode)).collect(Collectors.toList());
    if (arguments.stream().anyMatch(Objects::isNull)) {
        return null;
    }
    RexNode separatorNode = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(1));
    if (!separatorNode.isA(SqlKind.LITERAL)) {
        // separator must be a literal
        return null;
    }
    String separator = RexLiteral.stringValue(separatorNode);
    if (separator == null) {
        // separator must not be null
        return null;
    }
    Integer maxSizeBytes = null;
    if (arguments.size() > 2) {
        RexNode maxBytes = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(2));
        if (!maxBytes.isA(SqlKind.LITERAL)) {
            // maxBytes must be a literal
            return null;
        }
        maxSizeBytes = ((Number) RexLiteral.value(maxBytes)).intValue();
    }
    final DruidExpression arg = arguments.get(0);
    final ExprMacroTable macroTable = plannerContext.getExprMacroTable();
    final String initialvalue = "[]";
    final ColumnType elementType = ColumnType.STRING;
    final String fieldName;
    if (arg.isDirectColumnAccess()) {
        fieldName = arg.getDirectColumn();
    } else {
        fieldName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(arg, elementType);
    }
    final String finalizer = StringUtils.format("if(array_length(o) == 0, null, array_to_string(o, '%s'))", separator);
    final NotDimFilter dimFilter = new NotDimFilter(new SelectorDimFilter(fieldName, null, null));
    if (aggregateCall.isDistinct()) {
        return Aggregation.create(// string_agg ignores nulls
        new FilteredAggregatorFactory(new ExpressionLambdaAggregatorFactory(name, ImmutableSet.of(fieldName), null, initialvalue, null, true, false, false, StringUtils.format("array_set_add(\"__acc\", \"%s\")", fieldName), StringUtils.format("array_set_add_all(\"__acc\", \"%s\")", name), null, finalizer, maxSizeBytes != null ? new HumanReadableBytes(maxSizeBytes) : null, macroTable), dimFilter));
    } else {
        return Aggregation.create(// string_agg ignores nulls
        new FilteredAggregatorFactory(new ExpressionLambdaAggregatorFactory(name, ImmutableSet.of(fieldName), null, initialvalue, null, true, false, false, StringUtils.format("array_append(\"__acc\", \"%s\")", fieldName), StringUtils.format("array_concat(\"__acc\", \"%s\")", name), null, finalizer, maxSizeBytes != null ? new HumanReadableBytes(maxSizeBytes) : null, macroTable), dimFilter));
    }
}

Example 19 with NotDimFilter

use of org.apache.druid.query.filter.NotDimFilter in project druid by druid-io.

the class Expressions method toSimpleLeafFilter.

/**
 * Translates to a simple leaf filter, i.e. not an "expression" type filter. Note that the filter may still
 * reference expression virtual columns, if and only if "virtualColumnRegistry" is defined.
 *
 * @param plannerContext        planner context
 * @param rowSignature          input row signature
 * @param virtualColumnRegistry re-usable virtual column references, may be null if virtual columns aren't allowed
 * @param rexNode               Calcite row expression
 */
@Nullable
private static DimFilter toSimpleLeafFilter(final PlannerContext plannerContext, final RowSignature rowSignature, @Nullable final VirtualColumnRegistry virtualColumnRegistry, final RexNode rexNode) {
    final SqlKind kind = rexNode.getKind();
    if (kind == SqlKind.IS_TRUE || kind == SqlKind.IS_NOT_FALSE) {
        return toSimpleLeafFilter(plannerContext, rowSignature, virtualColumnRegistry, Iterables.getOnlyElement(((RexCall) rexNode).getOperands()));
    } else if (kind == SqlKind.IS_FALSE || kind == SqlKind.IS_NOT_TRUE) {
        return new NotDimFilter(toSimpleLeafFilter(plannerContext, rowSignature, virtualColumnRegistry, Iterables.getOnlyElement(((RexCall) rexNode).getOperands())));
    } else if (kind == SqlKind.IS_NULL || kind == SqlKind.IS_NOT_NULL) {
        final RexNode operand = Iterables.getOnlyElement(((RexCall) rexNode).getOperands());
        final DruidExpression druidExpression = toDruidExpression(plannerContext, rowSignature, operand);
        if (druidExpression == null) {
            return null;
        }
        final DimFilter equalFilter;
        if (druidExpression.isSimpleExtraction()) {
            equalFilter = new SelectorDimFilter(druidExpression.getSimpleExtraction().getColumn(), NullHandling.defaultStringValue(), druidExpression.getSimpleExtraction().getExtractionFn());
        } else if (virtualColumnRegistry != null) {
            final String virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(druidExpression, operand.getType());
            equalFilter = new SelectorDimFilter(virtualColumn, NullHandling.defaultStringValue(), null);
        } else {
            return null;
        }
        return kind == SqlKind.IS_NOT_NULL ? new NotDimFilter(equalFilter) : equalFilter;
    } else if (kind == SqlKind.EQUALS || kind == SqlKind.NOT_EQUALS || kind == SqlKind.GREATER_THAN || kind == SqlKind.GREATER_THAN_OR_EQUAL || kind == SqlKind.LESS_THAN || kind == SqlKind.LESS_THAN_OR_EQUAL) {
        final List<RexNode> operands = ((RexCall) rexNode).getOperands();
        Preconditions.checkState(operands.size() == 2, "Expected 2 operands, got[%,d]", operands.size());
        boolean flip = false;
        RexNode lhs = operands.get(0);
        RexNode rhs = operands.get(1);
        if (lhs.getKind() == SqlKind.LITERAL && rhs.getKind() != SqlKind.LITERAL) {
            // swap lhs, rhs
            RexNode x = lhs;
            lhs = rhs;
            rhs = x;
            flip = true;
        }
        // Flip operator, maybe.
        final SqlKind flippedKind;
        if (flip) {
            switch(kind) {
                case EQUALS:
                case NOT_EQUALS:
                    flippedKind = kind;
                    break;
                case GREATER_THAN:
                    flippedKind = SqlKind.LESS_THAN;
                    break;
                case GREATER_THAN_OR_EQUAL:
                    flippedKind = SqlKind.LESS_THAN_OR_EQUAL;
                    break;
                case LESS_THAN:
                    flippedKind = SqlKind.GREATER_THAN;
                    break;
                case LESS_THAN_OR_EQUAL:
                    flippedKind = SqlKind.GREATER_THAN_OR_EQUAL;
                    break;
                default:
                    throw new ISE("Kind[%s] not expected here", kind);
            }
        } else {
            flippedKind = kind;
        }
        // rhs must be a literal
        if (rhs.getKind() != SqlKind.LITERAL) {
            return null;
        }
        // Translate lhs to a DruidExpression.
        final DruidExpression lhsExpression = toDruidExpression(plannerContext, rowSignature, lhs);
        if (lhsExpression == null) {
            return null;
        }
        // Special handling for filters on FLOOR(__time TO granularity).
        final Granularity queryGranularity = toQueryGranularity(lhsExpression, plannerContext.getExprMacroTable());
        if (queryGranularity != null) {
            // lhs is FLOOR(__time TO granularity); rhs must be a timestamp
            final long rhsMillis = Calcites.calciteDateTimeLiteralToJoda(rhs, plannerContext.getTimeZone()).getMillis();
            return buildTimeFloorFilter(ColumnHolder.TIME_COLUMN_NAME, queryGranularity, flippedKind, rhsMillis);
        }
        final String column;
        final ExtractionFn extractionFn;
        if (lhsExpression.isSimpleExtraction()) {
            column = lhsExpression.getSimpleExtraction().getColumn();
            extractionFn = lhsExpression.getSimpleExtraction().getExtractionFn();
        } else if (virtualColumnRegistry != null) {
            column = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(lhsExpression, lhs.getType());
            extractionFn = null;
        } else {
            return null;
        }
        if (column.equals(ColumnHolder.TIME_COLUMN_NAME) && extractionFn instanceof TimeFormatExtractionFn) {
            // Check if we can strip the extractionFn and convert the filter to a direct filter on __time.
            // This allows potential conversion to query-level "intervals" later on, which is ideal for Druid queries.
            final Granularity granularity = ExtractionFns.toQueryGranularity(extractionFn);
            if (granularity != null) {
                // lhs is FLOOR(__time TO granularity); rhs must be a timestamp
                final long rhsMillis = Calcites.calciteDateTimeLiteralToJoda(rhs, plannerContext.getTimeZone()).getMillis();
                final Interval rhsInterval = granularity.bucket(DateTimes.utc(rhsMillis));
                // Is rhs aligned on granularity boundaries?
                final boolean rhsAligned = rhsInterval.getStartMillis() == rhsMillis;
                // Create a BoundRefKey that strips the extractionFn and compares __time as a number.
                final BoundRefKey boundRefKey = new BoundRefKey(column, null, StringComparators.NUMERIC);
                return getBoundTimeDimFilter(flippedKind, boundRefKey, rhsInterval, rhsAligned);
            }
        }
        final String val;
        final RexLiteral rhsLiteral = (RexLiteral) rhs;
        if (SqlTypeName.NUMERIC_TYPES.contains(rhsLiteral.getTypeName())) {
            val = String.valueOf(RexLiteral.value(rhsLiteral));
        } else if (SqlTypeName.CHAR_TYPES.contains(rhsLiteral.getTypeName())) {
            val = String.valueOf(RexLiteral.stringValue(rhsLiteral));
        } else if (SqlTypeName.TIMESTAMP == rhsLiteral.getTypeName() || SqlTypeName.DATE == rhsLiteral.getTypeName()) {
            val = String.valueOf(Calcites.calciteDateTimeLiteralToJoda(rhsLiteral, plannerContext.getTimeZone()).getMillis());
        } else {
            // Don't know how to filter on this kind of literal.
            return null;
        }
        // Numeric lhs needs a numeric comparison.
        final StringComparator comparator = Calcites.getStringComparatorForRelDataType(lhs.getType());
        final BoundRefKey boundRefKey = new BoundRefKey(column, extractionFn, comparator);
        final DimFilter filter;
        // Always use BoundDimFilters, to simplify filter optimization later (it helps to remember the comparator).
        switch(flippedKind) {
            case EQUALS:
                filter = Bounds.equalTo(boundRefKey, val);
                break;
            case NOT_EQUALS:
                filter = new NotDimFilter(Bounds.equalTo(boundRefKey, val));
                break;
            case GREATER_THAN:
                filter = Bounds.greaterThan(boundRefKey, val);
                break;
            case GREATER_THAN_OR_EQUAL:
                filter = Bounds.greaterThanOrEqualTo(boundRefKey, val);
                break;
            case LESS_THAN:
                filter = Bounds.lessThan(boundRefKey, val);
                break;
            case LESS_THAN_OR_EQUAL:
                filter = Bounds.lessThanOrEqualTo(boundRefKey, val);
                break;
            default:
                throw new IllegalStateException("Shouldn't have got here");
        }
        return filter;
    } else if (rexNode instanceof RexCall) {
        final SqlOperator operator = ((RexCall) rexNode).getOperator();
        final SqlOperatorConversion conversion = plannerContext.getOperatorTable().lookupOperatorConversion(operator);
        if (conversion == null) {
            return null;
        } else {
            return conversion.toDruidFilter(plannerContext, rowSignature, virtualColumnRegistry, rexNode);
        }
    } else {
        return null;
    }
}

Example 20 with NotDimFilter

use of org.apache.druid.query.filter.NotDimFilter in project druid by druid-io.

the class BitwiseSqlAggregator method toDruidAggregation.

@Nullable
@Override
public Aggregation toDruidAggregation(PlannerContext plannerContext, RowSignature rowSignature, VirtualColumnRegistry virtualColumnRegistry, RexBuilder rexBuilder, String name, AggregateCall aggregateCall, Project project, List<Aggregation> existingAggregations, boolean finalizeAggregations) {
    final List<DruidExpression> arguments = aggregateCall.getArgList().stream().map(i -> Expressions.fromFieldAccess(rowSignature, project, i)).map(rexNode -> Expressions.toDruidExpression(plannerContext, rowSignature, rexNode)).collect(Collectors.toList());
    if (arguments.stream().anyMatch(Objects::isNull)) {
        return null;
    }
    final DruidExpression arg = arguments.get(0);
    final ExprMacroTable macroTable = plannerContext.getExprMacroTable();
    final String fieldName;
    if (arg.isDirectColumnAccess()) {
        fieldName = arg.getDirectColumn();
    } else {
        fieldName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(arg, ColumnType.LONG);
    }
    return Aggregation.create(new FilteredAggregatorFactory(new ExpressionLambdaAggregatorFactory(name, ImmutableSet.of(fieldName), null, "0", null, null, false, false, StringUtils.format("%s(\"__acc\", \"%s\")", op.getDruidFunction(), fieldName), null, null, null, null, macroTable), new NotDimFilter(new SelectorDimFilter(fieldName, null, null))));
}