Examples with FilterIllegalMemberException org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException used on opensource projects

Example 11 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class RowLevelFilterExecuterImpl method applyFilter.

@Override
public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws FilterUnsupportedException, IOException {
    readBlocks(blockChunkHolder);
    // CHECKSTYLE:ON
    int[] numberOfRows = null;
    int pageNumbers = 0;
    if (dimColEvaluatorInfoList.size() > 0) {
        if (isDimensionPresentInCurrentBlock[0]) {
            pageNumbers = blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getPagesCount();
            numberOfRows = blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getRowCount();
        } else {
            // specific for restructure case where default values need to be filled
            pageNumbers = blockChunkHolder.getDataBlock().numberOfPages();
            numberOfRows = new int[] { blockChunkHolder.getDataBlock().nodeSize() };
        }
    }
    if (msrColEvalutorInfoList.size() > 0) {
        if (isMeasurePresentInCurrentBlock[0]) {
            pageNumbers = blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getPagesCount();
            numberOfRows = blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getRowCount();
        } else {
            // specific for restructure case where default values need to be filled
            pageNumbers = blockChunkHolder.getDataBlock().numberOfPages();
            numberOfRows = new int[] { blockChunkHolder.getDataBlock().nodeSize() };
        }
    }
    BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
    for (int i = 0; i < pageNumbers; i++) {
        BitSet set = new BitSet(numberOfRows[i]);
        RowIntf row = new RowImpl();
        boolean invalidRowsPresent = false;
        for (int index = 0; index < numberOfRows[i]; index++) {
            createRow(blockChunkHolder, row, i, index);
            Boolean rslt = false;
            try {
                rslt = exp.evaluate(row).getBoolean();
            }// too much log inforation only once the log will be printed.
             catch (FilterIllegalMemberException e) {
                FilterUtil.logError(e, invalidRowsPresent);
            }
            if (null != rslt && rslt) {
                set.set(index);
            }
        }
        bitSetGroup.setBitSet(set, i);
    }
    return bitSetGroup;
}

Example 12 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class RowLevelRangeFilterResolverImpl method getNoDictionaryRangeValues.

private List<byte[]> getNoDictionaryRangeValues() {
    List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
    if (this.getFilterExpression() instanceof BinaryConditionalExpression) {
        listOfExpressionResults = ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals();
    }
    List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
    boolean invalidRowsPresent = false;
    for (ExpressionResult result : listOfExpressionResults) {
        try {
            if (result.getString() == null) {
                filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
                continue;
            }
            filterValuesList.add(DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result.getString(), result.getDataType()));
        } catch (FilterIllegalMemberException e) {
            // Any invalid member while evaluation shall be ignored, system will log the
            // error only once since all rows the evaluation happens so inorder to avoid
            // too much log inforation only once the log will be printed.
            FilterUtil.logError(e, invalidRowsPresent);
        }
    }
    Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {

        @Override
        public int compare(byte[] filterMember1, byte[] filterMember2) {
            return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2);
        }
    };
    Collections.sort(filterValuesList, filterNoDictValueComaparator);
    return filterValuesList;
}

Example 13 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class DictionaryColumnVisitor method populateFilterResolvedInfo.

/**
   * This Visitor method is used to populate the visitableObj with direct dictionary filter details
   * where the filters values will be resolve using dictionary cache.
   *
   * @param visitableObj
   * @param metadata
   * @throws FilterUnsupportedException,if exception occurs while evaluating
   *                                       filter models.
   * @throws IOException
   * @throws FilterUnsupportedException
   */
public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
    DimColumnFilterInfo resolvedFilterObject = null;
    List<String> evaluateResultListFinal;
    try {
        evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
    } catch (FilterIllegalMemberException e) {
        throw new FilterUnsupportedException(e);
    }
    resolvedFilterObject = FilterUtil.getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter());
    if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
        // this is because two times it will flip the same bit
        if (!resolvedFilterObject.getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
            resolvedFilterObject.getFilterList().add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
        }
        Collections.sort(resolvedFilterObject.getFilterList());
    }
    visitableObj.setFilterValues(resolvedFilterObject);
}

Example 14 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class NoDictionaryTypeVisitor method populateFilterResolvedInfo.

/**
   * Visitor Method will update the filter related details in visitableObj, For no dictionary
   * type columns the filter members will resolved directly, no need to look up in dictionary
   * since it will not be part of dictionary, directly the actual data can be converted as
   * byte[] and can be set. this type of encoding is effective when the particular column
   * is having very high cardinality.
   *
   * @param visitableObj
   * @param metadata
   * @throws FilterUnsupportedException,if exception occurs while evaluating
   * filter models.
   */
public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException {
    DimColumnFilterInfo resolvedFilterObject = null;
    List<String> evaluateResultListFinal = null;
    try {
        // handling for is null case scenarios
        if (metadata.getExpression() instanceof EqualToExpression) {
            EqualToExpression expression = (EqualToExpression) metadata.getExpression();
            if (expression.isNull) {
                evaluateResultListFinal = new ArrayList<>(1);
                evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
            }
        } else {
            evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
        }
        // displaying the report as per hive compatibility.
        if (!metadata.isIncludeFilter() && !evaluateResultListFinal.contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
            evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
        }
    } catch (FilterIllegalMemberException e) {
        throw new FilterUnsupportedException(e);
    }
    resolvedFilterObject = FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getColumnExpression().getDataType());
    visitableObj.setFilterValues(resolvedFilterObject);
}

Example 15 with FilterIllegalMemberException

use of org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException in project carbondata by apache.

the class ExpressionResultTest method getTime.

public Long getTime(String value) throws FilterIllegalMemberException {
    if (value == null) {
        return null;
    }
    SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
    Date dateToStr;
    try {
        dateToStr = parser.parse(value.toString());
        return dateToStr.getTime();
    } catch (ParseException e) {
        throw new FilterIllegalMemberException("Cannot convert value to Time/Long type value");
    }
}