Examples with FilterIllegalMemberException org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException used on opensource projects

Example 21 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class DictionaryColumnVisitor method populateFilterResolvedInfo.

/**
   * This Visitor method is used to populate the visitableObj with direct dictionary filter details
   * where the filters values will be resolve using dictionary cache.
   *
   * @param visitableObj
   * @param metadata
   * @throws FilterUnsupportedException,if exception occurs while evaluating
   *                                       filter models.
   * @throws IOException
   * @throws FilterUnsupportedException
   */
public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
    DimColumnFilterInfo resolvedFilterObject = null;
    List<String> evaluateResultListFinal;
    try {
        evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
    } catch (FilterIllegalMemberException e) {
        throw new FilterUnsupportedException(e);
    }
    resolvedFilterObject = FilterUtil.getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter());
    if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
        // this is because two times it will flip the same bit
        if (!resolvedFilterObject.getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
            resolvedFilterObject.getFilterList().add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
        }
        Collections.sort(resolvedFilterObject.getFilterList());
    }
    visitableObj.setFilterValues(resolvedFilterObject);
}

Example 22 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class NoDictionaryTypeVisitor method populateFilterResolvedInfo.

/**
   * Visitor Method will update the filter related details in visitableObj, For no dictionary
   * type columns the filter members will resolved directly, no need to look up in dictionary
   * since it will not be part of dictionary, directly the actual data can be converted as
   * byte[] and can be set. this type of encoding is effective when the particular column
   * is having very high cardinality.
   *
   * @param visitableObj
   * @param metadata
   * @throws FilterUnsupportedException,if exception occurs while evaluating
   * filter models.
   */
public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException {
    DimColumnFilterInfo resolvedFilterObject = null;
    List<String> evaluateResultListFinal = null;
    try {
        // handling for is null case scenarios
        if (metadata.getExpression() instanceof EqualToExpression) {
            EqualToExpression expression = (EqualToExpression) metadata.getExpression();
            if (expression.isNull) {
                evaluateResultListFinal = new ArrayList<>(1);
                evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
            }
        } else {
            evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
        }
        // displaying the report as per hive compatibility.
        if (!metadata.isIncludeFilter() && !evaluateResultListFinal.contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
            evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
        }
    } catch (FilterIllegalMemberException e) {
        throw new FilterUnsupportedException(e);
    }
    resolvedFilterObject = FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getColumnExpression().getDataType());
    visitableObj.setFilterValues(resolvedFilterObject);
}

Example 23 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class FilterUtil method getFilterListForAllMembersRS.

/**
 * This method will get the member based on filter expression evaluation from the
 * forward dictionary cache, this method will be basically used in restructure.
 *
 * @param expression
 * @param columnExpression
 * @param defaultValues
 * @param defaultSurrogate
 * @param isIncludeFilter
 * @return
 * @throws FilterUnsupportedException
 */
public static ColumnFilterInfo getFilterListForAllMembersRS(Expression expression, ColumnExpression columnExpression, String defaultValues, int defaultSurrogate, boolean isIncludeFilter) throws FilterUnsupportedException {
    List<Integer> filterValuesList = new ArrayList<Integer>(20);
    List<String> evaluateResultListFinal = new ArrayList<String>(20);
    ColumnFilterInfo columnFilterInfo = null;
    try {
        RowIntf row = new RowImpl();
        if (defaultValues.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
            defaultValues = null;
        }
        row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(defaultValues, columnExpression.getCarbonColumn().getDataType()) });
        Boolean rslt = expression.evaluate(row).getBoolean();
        if (null != rslt && rslt == isIncludeFilter) {
            if (null == defaultValues) {
                evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
            } else {
                evaluateResultListFinal.add(defaultValues);
            }
        }
    } catch (FilterIllegalMemberException e) {
        LOGGER.error(e.getMessage());
    }
    if (null == defaultValues) {
        defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
    }
    columnFilterInfo = new ColumnFilterInfo();
    for (int i = 0; i < evaluateResultListFinal.size(); i++) {
        if (evaluateResultListFinal.get(i).equals(defaultValues)) {
            filterValuesList.add(defaultSurrogate);
            break;
        }
    }
    columnFilterInfo.setFilterList(filterValuesList);
    return columnFilterInfo;
}

Example 24 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class FilterUtil method prepareIncludeFilterMembers.

private static void prepareIncludeFilterMembers(Expression expression, final ColumnExpression columnExpression, boolean isIncludeFilter, Dictionary forwardDictionary, List<Integer> surrogates) throws FilterUnsupportedException {
    DictionaryChunksWrapper dictionaryWrapper;
    dictionaryWrapper = forwardDictionary.getDictionaryChunks();
    int surrogateCount = 0;
    while (dictionaryWrapper.hasNext()) {
        byte[] columnVal = dictionaryWrapper.next();
        ++surrogateCount;
        try {
            RowIntf row = new RowImpl();
            String stringValue = new String(columnVal, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
            if (stringValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
                stringValue = null;
            }
            row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue, columnExpression.getCarbonColumn().getDataType()) });
            Boolean rslt = expression.evaluate(row).getBoolean();
            if (null != rslt) {
                if (rslt) {
                    if (null == stringValue) {
                        // this is for query like select name from table unknowexpr(name,1)
                        // != 'value' -> for null dictionary value
                        surrogates.add(CarbonCommonConstants.DICT_VALUE_NULL);
                    } else if (isIncludeFilter) {
                        // this is for query like select ** from * where unknwonexpr(*) == 'value'
                        surrogates.add(surrogateCount);
                    }
                } else if (null != stringValue && !isIncludeFilter) {
                    // this is for isNot null or not in query( e.x select ** from t where name is not null
                    surrogates.add(surrogateCount);
                }
            }
        } catch (FilterIllegalMemberException e) {
            LOGGER.debug(e.getMessage());
        }
    }
}

Example 25 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class RowLevelFilterExecutorImpl method applyFilter.

@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
    if (exp instanceof MatchExpression) {
        BitSetGroup bitSetGroup = rawBlockletColumnChunks.getBitSetGroup();
        if (bitSetGroup == null) {
            // It means there are no index created on this table
            throw new FilterUnsupportedException(String.format("%s is not supported on table %s", exp.getFilterExpressionType().name(), tableIdentifier.getTableName()));
        }
        return bitSetGroup;
    }
    readColumnChunks(rawBlockletColumnChunks);
    // CHECKSTYLE:ON
    int[] numberOfRows = null;
    int pageNumbers = 0;
    if (dimColEvaluatorInfoList.size() > 0) {
        if (isDimensionPresentInCurrentBlock[0]) {
            pageNumbers = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getPagesCount();
            numberOfRows = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getRowCount();
        } else {
            // specific for restructure case where default values need to be filled
            pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
            numberOfRows = new int[pageNumbers];
            for (int i = 0; i < pageNumbers; i++) {
                numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
            }
        }
    }
    if (msrColEvalutorInfoList.size() > 0) {
        if (isMeasurePresentInCurrentBlock[0]) {
            pageNumbers = rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0).getColumnIndex()].getPagesCount();
            numberOfRows = rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0).getColumnIndex()].getRowCount();
        } else {
            // specific for restructure case where default values need to be filled
            pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
            numberOfRows = new int[pageNumbers];
            for (int i = 0; i < pageNumbers; i++) {
                numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
            }
        }
    }
    BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
    if (isDimensionPresentInCurrentBlock.length == 1 && isDimensionPresentInCurrentBlock[0] && dimColEvaluatorInfoList.get(0).getDimension().getDataType().isComplexType() && exp instanceof EqualToExpression) {
        LiteralExpression literalExp = (LiteralExpression) (((EqualToExpression) exp).getRight());
        // convert filter value to byte[] to compare with byte[] data from columnPage
        Object literalExpValue = literalExp.getLiteralExpValue();
        DataType literalExpDataType = literalExp.getLiteralExpDataType();
        if (literalExpDataType == DataTypes.TIMESTAMP) {
            if ((long) literalExpValue == 0) {
                literalExpValue = null;
            } else {
                literalExpValue = (long) literalExpValue / TimeStampGranularityTypeValue.MILLIS_SECONDS.getValue();
            }
        } else if (literalExpDataType == DataTypes.DATE) {
            // change data type to int to get the byte[] filter value as it is direct dictionary
            literalExpDataType = DataTypes.INT;
            if (literalExpValue == null) {
                literalExpValue = CarbonCommonConstants.DIRECT_DICT_VALUE_NULL;
            } else {
                literalExpValue = (int) literalExpValue + DateDirectDictionaryGenerator.cutOffDate;
            }
        }
        byte[] filterValueInBytes = DataTypeUtil.getBytesDataDataTypeForNoDictionaryColumn(literalExpValue, literalExpDataType);
        ArrayQueryType complexType = (ArrayQueryType) complexDimensionInfoMap.get(dimensionChunkIndex[0]);
        int totalCount = 0;
        // check all the pages
        for (int i = 0; i < pageNumbers; i++) {
            if (limit != -1 && totalCount >= limit) {
                break;
            }
            BitSet set = new BitSet(numberOfRows[i]);
            int[][] numberOfChild = complexType.getNumberOfChild(rawBlockletColumnChunks.getDimensionRawColumnChunks(), null, numberOfRows[i], i);
            DimensionColumnPage page = complexType.parseBlockAndReturnChildData(rawBlockletColumnChunks.getDimensionRawColumnChunks(), null, i);
            // check every row
            for (int index = 0; index < numberOfRows[i]; index++) {
                if (limit != -1 && totalCount >= limit) {
                    break;
                }
                int dataOffset = numberOfChild[index][1];
                // loop the children
                for (int j = 0; j < numberOfChild[index][0]; j++) {
                    byte[] obj = page.getChunkData(dataOffset++);
                    if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(obj, filterValueInBytes) == 0) {
                        set.set(index);
                        totalCount++;
                        break;
                    }
                }
            }
            bitSetGroup.setBitSet(set, i);
        }
    } else {
        for (int i = 0; i < pageNumbers; i++) {
            BitSet set = new BitSet(numberOfRows[i]);
            RowIntf row = new RowImpl();
            BitSet prvBitset = null;
            // otherwise use older flow
            if (!useBitsetPipeLine || null == rawBlockletColumnChunks.getBitSetGroup() || null == bitSetGroup.getBitSet(i) || rawBlockletColumnChunks.getBitSetGroup().getBitSet(i).isEmpty()) {
                for (int index = 0; index < numberOfRows[i]; index++) {
                    createRow(rawBlockletColumnChunks, row, i, index);
                    Boolean result = false;
                    try {
                        result = exp.evaluate(row).getBoolean();
                    }// too much log inforation only once the log will be printed.
                     catch (FilterIllegalMemberException e) {
                        FilterUtil.logError(e, false);
                    }
                    if (null != result && result) {
                        set.set(index);
                    }
                }
            } else {
                prvBitset = rawBlockletColumnChunks.getBitSetGroup().getBitSet(i);
                for (int index = prvBitset.nextSetBit(0); index >= 0; index = prvBitset.nextSetBit(index + 1)) {
                    createRow(rawBlockletColumnChunks, row, i, index);
                    Boolean rslt = false;
                    try {
                        rslt = exp.evaluate(row).getBoolean();
                    } catch (FilterIllegalMemberException e) {
                        FilterUtil.logError(e, false);
                    }
                    if (null != rslt && rslt) {
                        set.set(index);
                    }
                }
            }
            bitSetGroup.setBitSet(set, i);
        }
    }
    return bitSetGroup;
}