Examples with InExpression org.apache.carbondata.core.scan.expression.conditional.InExpression used on opensource projects

Example 6 with InExpression

use of org.apache.carbondata.core.scan.expression.conditional.InExpression in project carbondata by apache.

the class BloomCoarseGrainIndex method createQueryModel.

private List<BloomQueryModel> createQueryModel(Expression expression) {
    List<BloomQueryModel> queryModels = new ArrayList<BloomQueryModel>();
    // bloom index only support equalTo and In operators now
    if (expression instanceof EqualToExpression) {
        Expression left = ((EqualToExpression) expression).getLeft();
        Expression right = ((EqualToExpression) expression).getRight();
        String column;
        if (left instanceof ColumnExpression && right instanceof LiteralExpression) {
            column = ((ColumnExpression) left).getColumnName();
            if (this.name2Col.containsKey(column)) {
                BloomQueryModel bloomQueryModel = buildQueryModelForEqual((ColumnExpression) left, (LiteralExpression) right);
                queryModels.add(bloomQueryModel);
            }
            return queryModels;
        } else if (left instanceof LiteralExpression && right instanceof ColumnExpression) {
            column = ((ColumnExpression) right).getColumnName();
            if (this.name2Col.containsKey(column)) {
                BloomQueryModel bloomQueryModel = buildQueryModelForEqual((ColumnExpression) right, (LiteralExpression) left);
                queryModels.add(bloomQueryModel);
            }
            return queryModels;
        } else {
            String errorMsg = "BloomFilter can only support the 'equal' filter like 'Col = PlainValue'";
            LOGGER.warn(errorMsg);
            throw new RuntimeException(errorMsg);
        }
    } else if (expression instanceof InExpression) {
        Expression left = ((InExpression) expression).getLeft();
        Expression right = ((InExpression) expression).getRight();
        String column;
        if (left instanceof ColumnExpression && right instanceof ListExpression) {
            column = ((ColumnExpression) left).getColumnName();
            if (this.name2Col.containsKey(column)) {
                BloomQueryModel bloomQueryModel = buildQueryModelForIn((ColumnExpression) left, (ListExpression) right);
                queryModels.add(bloomQueryModel);
            }
            return queryModels;
        } else if (left instanceof ListExpression && right instanceof ColumnExpression) {
            column = ((ColumnExpression) right).getColumnName();
            if (this.name2Col.containsKey(column)) {
                BloomQueryModel bloomQueryModel = buildQueryModelForIn((ColumnExpression) right, (ListExpression) left);
                queryModels.add(bloomQueryModel);
            }
            return queryModels;
        } else {
            String errorMsg = "BloomFilter can only support the 'in' filter like 'Col in PlainValue'";
            LOGGER.warn(errorMsg);
            throw new RuntimeException(errorMsg);
        }
    } else if (expression instanceof AndExpression) {
        queryModels.addAll(createQueryModel(((AndExpression) expression).getLeft()));
        queryModels.addAll(createQueryModel(((AndExpression) expression).getRight()));
        return queryModels;
    }
    return queryModels;
}

Example 7 with InExpression

use of org.apache.carbondata.core.scan.expression.conditional.InExpression in project carbondata by apache.

the class FilterUtil method removeInExpressionNodeWithPositionIdColumn.

/**
 * This method will check for ColumnExpression with column name positionID and if found will
 * replace the InExpression with true expression. This is done to stop serialization of List
 * expression which is right children of InExpression as it can impact the query performance
 * as the size of list grows bigger.
 *
 * @param expression
 */
public static void removeInExpressionNodeWithPositionIdColumn(Expression expression) {
    ExpressionType filterExpressionType = expression.getFilterExpressionType();
    if (ExpressionType.AND == filterExpressionType) {
        Expression rightExpression = ((AndExpression) expression).getRight();
        if (rightExpression instanceof InExpression) {
            List<Expression> children = rightExpression.getChildren();
            if (null != children && !children.isEmpty()) {
                Expression childExpression = children.get(0);
                // check for the positionId as the column name in ColumnExpression
                if (childExpression instanceof ColumnExpression && ((ColumnExpression) childExpression).getColumnName().equalsIgnoreCase(CarbonCommonConstants.POSITION_ID)) {
                    // Remove the right expression node and point the expression to left node expression
                    expression.findAndSetChild(((AndExpression) expression).getRight(), new TrueExpression(null));
                    LOGGER.info("In expression removed from the filter expression list to prevent it from" + " serializing on executor");
                }
            }
        }
    }
}

Example 8 with InExpression

use of org.apache.carbondata.core.scan.expression.conditional.InExpression in project carbondata by apache.

the class PrestoFilterUtil method parseFilterExpression.

/**
 * Convert presto-TupleDomain predication into Carbon scan express condition
 *
 * @param originalConstraint presto-TupleDomain
 * @return
 */
static Expression parseFilterExpression(TupleDomain<ColumnHandle> originalConstraint) {
    ImmutableList.Builder<Expression> filters = ImmutableList.builder();
    Domain domain;
    for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) {
        // Build ColumnExpression for Expression(Carbondata)
        CarbondataColumnHandle cdch = (CarbondataColumnHandle) c;
        Type type = cdch.getColumnType();
        DataType coltype = Spi2CarbondataTypeMapper(cdch);
        Expression colExpression = new ColumnExpression(cdch.getColumnName(), coltype);
        domain = originalConstraint.getDomains().get().get(c);
        checkArgument(domain.getType().isOrderable(), "Domain type must be orderable");
        List<Object> singleValues = new ArrayList<>();
        Map<Object, List<Expression>> valueExpressionMap = new HashMap<>();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            if (range.isSingleValue()) {
                Object value = ConvertDataByType(range.getLow().getValue(), type);
                singleValues.add(value);
            } else {
                if (!range.getLow().isLowerUnbounded()) {
                    Object value = ConvertDataByType(range.getLow().getValue(), type);
                    switch(range.getLow().getBound()) {
                        case ABOVE:
                            if (type == TimestampType.TIMESTAMP) {
                            // todo not now
                            } else {
                                GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype));
                                valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(greater);
                            }
                            break;
                        case EXACTLY:
                            GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(greater);
                            break;
                        case BELOW:
                            throw new IllegalArgumentException("Low marker should never use BELOW bound");
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
                    }
                }
                if (!range.getHigh().isUpperUnbounded()) {
                    Object value = ConvertDataByType(range.getHigh().getValue(), type);
                    switch(range.getHigh().getBound()) {
                        case ABOVE:
                            throw new IllegalArgumentException("High marker should never use ABOVE bound");
                        case EXACTLY:
                            LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(less);
                            break;
                        case BELOW:
                            LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
                            valueExpressionMap.computeIfAbsent(value, key -> new ArrayList<>()).add(less2);
                            break;
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
                    }
                }
            }
        }
        if (singleValues.size() == 1) {
            Expression ex;
            if (coltype.equals(DataTypes.STRING)) {
                ex = new EqualToExpression(colExpression, new LiteralExpression(singleValues.get(0), coltype));
            } else if (coltype.equals(DataTypes.TIMESTAMP) || coltype.equals(DataTypes.DATE)) {
                Long value = (Long) singleValues.get(0);
                ex = new EqualToExpression(colExpression, new LiteralExpression(value, coltype));
            } else
                ex = new EqualToExpression(colExpression, new LiteralExpression(singleValues.get(0), coltype));
            filters.add(ex);
        } else if (singleValues.size() > 1) {
            ListExpression candidates = null;
            List<Expression> exs = singleValues.stream().map((a) -> new LiteralExpression(a, coltype)).collect(Collectors.toList());
            candidates = new ListExpression(exs);
            filters.add(new InExpression(colExpression, candidates));
        } else if (valueExpressionMap.size() > 0) {
            List<Expression> valuefilters = new ArrayList<>();
            Expression finalFilters = null;
            List<Expression> expressions;
            for (Map.Entry<Object, List<Expression>> entry : valueExpressionMap.entrySet()) {
                expressions = valueExpressionMap.get(entry.getKey());
                if (expressions.size() == 1) {
                    finalFilters = expressions.get(0);
                } else if (expressions.size() >= 2) {
                    finalFilters = new OrExpression(expressions.get(0), expressions.get(1));
                    for (int i = 2; i < expressions.size(); i++) {
                        finalFilters = new OrExpression(finalFilters, expressions.get(i));
                    }
                }
                valuefilters.add(finalFilters);
            }
            if (valuefilters.size() == 1) {
                finalFilters = valuefilters.get(0);
            } else if (valuefilters.size() >= 2) {
                finalFilters = new AndExpression(valuefilters.get(0), valuefilters.get(1));
                for (int i = 2; i < valuefilters.size(); i++) {
                    finalFilters = new AndExpression(finalFilters, valuefilters.get(i));
                }
            }
            filters.add(finalFilters);
        }
    }
    Expression finalFilters;
    List<Expression> tmp = filters.build();
    if (tmp.size() > 1) {
        finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
        if (tmp.size() > 2) {
            for (int i = 2; i < tmp.size(); i++) {
                finalFilters = new AndExpression(finalFilters, tmp.get(i));
            }
        }
    } else if (tmp.size() == 1)
        finalFilters = tmp.get(0);
    else
        return null;
    return finalFilters;
}

Example 9 with InExpression

use of org.apache.carbondata.core.scan.expression.conditional.InExpression in project carbondata by apache.

the class FilterExpressionProcessor method getFilterResolverBasedOnExpressionType.

/**
 * Factory method which will return the resolver instance based on filter expression
 * expressions.
 */
private FilterResolverIntf getFilterResolverBasedOnExpressionType(ExpressionType filterExpressionType, boolean isExpressionResolve, Expression expression, AbsoluteTableIdentifier tableIdentifier, Expression expressionTree) {
    BinaryConditionalExpression currentCondExpression = null;
    ConditionalExpression condExpression = null;
    switch(filterExpressionType) {
        case FALSE:
            return new FalseConditionalResolverImpl(expression, false, false);
        case TRUE:
            return new TrueConditionalResolverImpl(expression, false, false);
        case EQUALS:
            currentCondExpression = (BinaryConditionalExpression) expression;
            // check for implicit column in the expression
            if (currentCondExpression instanceof InExpression) {
                CarbonColumn carbonColumn = currentCondExpression.getColumnList().get(0).getCarbonColumn();
                if (carbonColumn.hasEncoding(Encoding.IMPLICIT)) {
                    return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true, currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
                }
            }
            CarbonColumn column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
            if (currentCondExpression.isSingleColumn() && !column.getDataType().isComplexType()) {
                // expression as it is already resolved and has the surrogates in the filter value
                if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft()) && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight()) || FilterUtil.checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
                    return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true, tableIdentifier);
                }
                if (currentCondExpression.getFilterExpressionType() == ExpressionType.GREATERTHAN || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN || currentCondExpression.getFilterExpressionType() == ExpressionType.GREATERTHAN_EQUALTO || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) {
                    return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, true, tableIdentifier);
                }
                return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true, column.isMeasure());
            }
            break;
        case RANGE:
            return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true, false);
        case NOT_EQUALS:
            currentCondExpression = (BinaryConditionalExpression) expression;
            column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
            if (currentCondExpression.isSingleColumn() && !column.getDataType().isComplexType()) {
                if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft()) && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight()) || FilterUtil.checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
                    return new RowLevelFilterResolverImpl(expression, isExpressionResolve, false, tableIdentifier);
                }
                if (expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN || expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN_EQUALTO || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) {
                    return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false, tableIdentifier);
                }
                return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false, column.isMeasure());
            }
            break;
        default:
            if (expression instanceof ConditionalExpression) {
                condExpression = (ConditionalExpression) expression;
                column = condExpression.getColumnList().get(0).getCarbonColumn();
                if (condExpression.isSingleColumn() && !column.isComplex()) {
                    condExpression = (ConditionalExpression) expression;
                    if (condExpression.getColumnList().get(0).getCarbonColumn().isMeasure()) {
                        return new ConditionalFilterResolverImpl(expression, true, true, condExpression.getColumnList().get(0).getCarbonColumn().isMeasure());
                    }
                }
            }
    }
    return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
}

Example 10 with InExpression

use of org.apache.carbondata.core.scan.expression.conditional.InExpression in project carbondata by apache.

the class FilterExpressionProcessorTest method testGetFilterResolverBasedOnExpressionType.

@Test
public void testGetFilterResolverBasedOnExpressionType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
    CarbonColumn carbonColumn = new CarbonColumn(columnSchema, 0, 0);
    ColumnExpression columnExpression = new ColumnExpression("IMEI", DataTypes.STRING);
    columnExpression.setCarbonColumn(carbonColumn);
    LiteralExpression literalExpression = new LiteralExpression("ImeiValue", DataTypes.STRING);
    InExpression equalToExpression = new InExpression(columnExpression, literalExpression);
    FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
    Method method = FilterExpressionProcessor.class.getDeclaredMethod("getFilterResolverBasedOnExpressionType", ExpressionType.class, boolean.class, Expression.class, AbsoluteTableIdentifier.class, Expression.class);
    method.setAccessible(true);
    Object result = method.invoke(filterExpressionProcessor, ExpressionType.EQUALS, false, equalToExpression, null, null);
    Assert.assertTrue(result.getClass().getName().equals("org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolverImpl"));
}