Examples with ColumnFilterInfo org.apache.carbondata.core.scan.filter.ColumnFilterInfo used on opensource projects

Example 1 with ColumnFilterInfo

use of org.apache.carbondata.core.scan.filter.ColumnFilterInfo in project carbondata by apache.

the class RestructureEvaluatorImpl method isDimensionDefaultValuePresentInFilterValues.

/**
 * This method will check whether a default value for the non-existing column is present
 * in the filter values list
 *
 * @param dimColumnEvaluatorInfo
 * @return
 */
protected boolean isDimensionDefaultValuePresentInFilterValues(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
    boolean isDefaultValuePresentInFilterValues = false;
    ColumnFilterInfo filterValues = dimColumnEvaluatorInfo.getFilterValues();
    CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension();
    byte[] defaultValue = dimension.getDefaultValue();
    if (!dimension.hasEncoding(Encoding.DICTIONARY)) {
        // 3 cases: is NUll, is Not Null and filter on default value of newly added column
        if (null == defaultValue && dimension.getDataType() == DataTypes.STRING) {
            // default value for case where user gives is Null condition
            defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
        } else if (null == defaultValue) {
            defaultValue = CarbonCommonConstants.EMPTY_BYTE_ARRAY;
        }
        List<byte[]> noDictionaryFilterValuesList = filterValues.getNoDictionaryFilterValuesList();
        for (byte[] filterValue : noDictionaryFilterValuesList) {
            int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(defaultValue, filterValue);
            if (compare == 0) {
                isDefaultValuePresentInFilterValues = true;
                break;
            }
        }
    } else {
        // for dictionary and direct dictionary cases
        // 3 cases: is NUll, is Not Null and filter on default value of newly added column
        int defaultSurrogateValueToCompare = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
        if (null != defaultValue) {
            defaultSurrogateValueToCompare++;
        }
        List<Integer> filterList = null;
        if (filterValues.isIncludeFilter() && !filterValues.isOptimized()) {
            filterList = filterValues.getFilterList();
        } else {
            filterList = filterValues.getExcludeFilterList();
        }
        for (Integer filterValue : filterList) {
            if (defaultSurrogateValueToCompare == filterValue) {
                isDefaultValuePresentInFilterValues = true;
                break;
            }
        }
    }
    return isDefaultValuePresentInFilterValues;
}

Example 2 with ColumnFilterInfo

use of org.apache.carbondata.core.scan.filter.ColumnFilterInfo in project carbondata by apache.

the class NoDictionaryTypeVisitor method populateFilterResolvedInfo.

/**
 * Visitor Method will update the filter related details in visitableObj, For no dictionary
 * type columns the filter members will resolved directly, no need to look up in dictionary
 * since it will not be part of dictionary, directly the actual data can be converted as
 * byte[] and can be set. this type of encoding is effective when the particular column
 * is having very high cardinality.
 *
 * @param visitableObj
 * @param metadata
 * @throws FilterUnsupportedException,if exception occurs while evaluating
 * filter models.
 */
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException {
    if (visitableObj instanceof DimColumnResolvedFilterInfo) {
        DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
        ColumnFilterInfo resolvedFilterObject = null;
        List<String> evaluateResultListFinal = new ArrayList<>(1);
        try {
            // handling for is null case scenarios
            if (metadata.getExpression() instanceof EqualToExpression) {
                EqualToExpression expression = (EqualToExpression) metadata.getExpression();
                if (expression.isNull) {
                    evaluateResultListFinal = new ArrayList<>(1);
                    evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
                }
            } else {
                evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
            }
            // displaying the report as per hive compatibility.
            if (!metadata.isIncludeFilter() && !evaluateResultListFinal.contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
                evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
            }
        } catch (FilterIllegalMemberException e) {
            throw new FilterUnsupportedException(e);
        }
        resolvedFilterObject = FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getColumnExpression().getDataType());
        resolveDimension.setFilterValues(resolvedFilterObject);
    }
}

Example 3 with ColumnFilterInfo

use of org.apache.carbondata.core.scan.filter.ColumnFilterInfo in project carbondata by apache.

the class RangeNoDictionaryTypeVisitor method populateFilterResolvedInfo.

/**
 * Visitor Method will update the filter related details in visitableObj, For no dictionary
 * type columns the filter members will resolved directly, no need to look up in dictionary
 * since it will not be part of dictionary, directly the actual data can be converted as
 * byte[] and can be set. this type of encoding is effective when the particular column
 * is having very high cardinality.
 *
 * @param visitableObj
 * @param metadata
 * @throws FilterUnsupportedException,if exception occurs while evaluating
 * filter models.
 */
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException {
    if (visitableObj instanceof DimColumnResolvedFilterInfo) {
        DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
        ColumnFilterInfo resolvedFilterObject = null;
        List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
        List<String> evaluateResultListFinal = new ArrayList<String>();
        try {
            // Add The Range Filter Values.
            if (metadata.getExpression() instanceof RangeExpression) {
                listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals();
            }
            for (ExpressionResult result : listOfExpressionResults) {
                if (result.getString() == null) {
                    evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
                    continue;
                }
                evaluateResultListFinal.add(result.getString());
            }
            // evaluateResultListFinal.add(metadata.getExpression().evaluate().getListAsString());
            if (!metadata.isIncludeFilter() && !evaluateResultListFinal.contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
                evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
            }
        } catch (FilterIllegalMemberException e) {
            throw new FilterUnsupportedException(e);
        }
        resolvedFilterObject = FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getColumnExpression().getDataType());
        resolveDimension.setFilterValues(resolvedFilterObject);
    }
}

Example 4 with ColumnFilterInfo

use of org.apache.carbondata.core.scan.filter.ColumnFilterInfo in project carbondata by apache.

the class DictionaryColumnVisitor method populateFilterResolvedInfo.

/**
 * This Visitor method is used to populate the visitableObj with direct dictionary filter details
 * where the filters values will be resolve using dictionary cache.
 *
 * @param visitableObj
 * @param metadata
 * @throws FilterUnsupportedException,if exception occurs while evaluating
 *                                       filter models.
 * @throws IOException
 * @throws FilterUnsupportedException
 */
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
    if (visitableObj instanceof DimColumnResolvedFilterInfo) {
        DimColumnResolvedFilterInfo resolveDimension = (DimColumnResolvedFilterInfo) visitableObj;
        ColumnFilterInfo resolvedFilterObject = null;
        List<String> evaluateResultListFinal;
        try {
            evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
        } catch (FilterIllegalMemberException e) {
            throw new FilterUnsupportedException(e);
        }
        try {
            resolvedFilterObject = FilterUtil.getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getTableProvider());
            if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
                // this is because two times it will flip the same bit
                if (!resolvedFilterObject.getExcludeFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
                    resolvedFilterObject.getExcludeFilterList().add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
                }
                Collections.sort(resolvedFilterObject.getExcludeFilterList());
            }
        } catch (QueryExecutionException e) {
            throw new FilterUnsupportedException(e);
        }
        resolveDimension.setFilterValues(resolvedFilterObject);
    }
}

Example 5 with ColumnFilterInfo

use of org.apache.carbondata.core.scan.filter.ColumnFilterInfo in project carbondata by apache.

the class MeasureColumnVisitor method populateFilterResolvedInfo.

/**
 * Visitor Method will update the filter related details in visitableObj, For no dictionary
 * type columns the filter members will resolved directly, no need to look up in dictionary
 * since it will not be part of dictionary, directly the actual data can be converted as
 * byte[] and can be set. this type of encoding is effective when the particular column
 * is having very high cardinality.
 *
 * @param visitableObj
 * @param metadata
 * @throws FilterUnsupportedException,if exception occurs while evaluating
 *                                       filter models.
 */
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException {
    if (visitableObj instanceof MeasureColumnResolvedFilterInfo) {
        MeasureColumnResolvedFilterInfo resolveDimension = (MeasureColumnResolvedFilterInfo) visitableObj;
        ColumnFilterInfo resolvedFilterObject = null;
        List<String> evaluateResultListFinal = new ArrayList<>(1);
        try {
            // handling for is null case scenarios
            if (metadata.getExpression() instanceof EqualToExpression) {
                EqualToExpression expression = (EqualToExpression) metadata.getExpression();
                if (expression.isNull) {
                    evaluateResultListFinal = new ArrayList<>(1);
                    evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
                }
            } else {
                evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
            }
            // displaying the report as per hive compatibility.
            if (!metadata.isIncludeFilter() && !evaluateResultListFinal.contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
                evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
            }
        } catch (FilterIllegalMemberException e) {
            throw new FilterUnsupportedException(e);
        }
        resolvedFilterObject = FilterUtil.getMeasureValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getColumnExpression().getDataType(), resolveDimension.getMeasure());
        resolveDimension.setFilterValues(resolvedFilterObject);
    }
}