Examples with MeasureColumnResolvedFilterInfo org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo used on opensource projects

Example 1 with MeasureColumnResolvedFilterInfo

use of org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo in project carbondata by apache.

the class RowLevelFilterExecuterImpl method createRow.

/**
   * Method will read the members of particular dimension block and create
   * a row instance for further processing of the filters
   *
   * @param blockChunkHolder
   * @param row
   * @param index
   * @throws IOException
   */
private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int pageIndex, int index) throws IOException {
    Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
    String memberString;
    for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
        DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
        // if filter dimension is not present in the current add its default value
        if (!isDimensionPresentInCurrentBlock[i]) {
            // fill default value here
            record[dimColumnEvaluatorInfo.getRowIndex()] = getDimensionDefaultValue(dimColumnEvaluatorInfo);
            continue;
        }
        if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
            if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
                record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDimension().getDefaultValue();
            }
            DimensionColumnDataChunk columnDataChunk = blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]].convertToDimColDataChunk(pageIndex);
            if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && columnDataChunk instanceof VariableLengthDimensionDataChunk) {
                VariableLengthDimensionDataChunk dimensionColumnDataChunk = (VariableLengthDimensionDataChunk) columnDataChunk;
                byte[] memberBytes = dimensionColumnDataChunk.getChunkData(index);
                if (null != memberBytes) {
                    if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
                        memberBytes = null;
                    }
                    record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes, dimColumnEvaluatorInfo.getDimension().getDataType());
                } else {
                    continue;
                }
            } else {
                int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex, dimColumnEvaluatorInfo, dimensionBlocksIndex[i]);
                if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    memberString = getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
                    record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil.getDataBasedOnDataType(memberString, dimColumnEvaluatorInfo.getDimension().getDataType());
                } else if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    Object member = getFilterActualValueFromDirectDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
                    record[dimColumnEvaluatorInfo.getRowIndex()] = member;
                }
            }
        } else {
            try {
                GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
                ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
                DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
                complexType.parseBlocksAndReturnComplexColumnByteArray(blockChunkHolder.getDimensionRawDataChunk(), index, pageIndex, dataOutputStream);
                record[dimColumnEvaluatorInfo.getRowIndex()] = complexType.getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
                byteStream.close();
            } catch (IOException e) {
                LOGGER.info(e.getMessage());
            }
        }
    }
    DataType msrType;
    for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
        MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
        switch(msrColumnEvalutorInfo.getType()) {
            case SHORT:
                msrType = DataType.SHORT;
                break;
            case INT:
                msrType = DataType.INT;
                break;
            case LONG:
                msrType = DataType.LONG;
                break;
            case DECIMAL:
                msrType = DataType.DECIMAL;
                break;
            default:
                msrType = DataType.DOUBLE;
        }
        // in the current block measure list
        if (!isMeasurePresentInCurrentBlock[i]) {
            byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue();
            record[msrColumnEvalutorInfo.getRowIndex()] = RestructureUtil.getMeasureDefaultValue(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(), defaultValue);
            continue;
        }
        Object msrValue;
        MeasureColumnDataChunk measureColumnDataChunk = blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].convertToMeasureColDataChunk(pageIndex);
        switch(msrType) {
            case SHORT:
                msrValue = (short) measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case INT:
                msrValue = (int) measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case LONG:
                msrValue = measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
                break;
            case DECIMAL:
                BigDecimal bigDecimalValue = measureColumnDataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
                if (null != bigDecimalValue && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale() > bigDecimalValue.scale()) {
                    bigDecimalValue = bigDecimalValue.setScale(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale(), RoundingMode.HALF_UP);
                }
                msrValue = bigDecimalValue;
                break;
            default:
                msrValue = measureColumnDataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
        }
        record[msrColumnEvalutorInfo.getRowIndex()] = measureColumnDataChunk.getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
    }
    row.setValues(record);
}

Example 2 with MeasureColumnResolvedFilterInfo

use of org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo in project carbondata by apache.

the class RowLevelFilterExecuterImpl method readBlocks.

@Override
public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
    for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
        DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
        if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
            if (null == blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]) {
                blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] = blockChunkHolder.getDataBlock().getDimensionChunk(blockChunkHolder.getFileReader(), dimensionBlocksIndex[i]);
            }
        } else {
            GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
            complexType.fillRequiredBlockData(blockChunkHolder);
        }
    }
    if (null != msrColEvalutorInfoList) {
        for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
            if (null == blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]) {
                blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]] = blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(), measureBlocksIndex[0]);
            }
        }
    }
}

Example 3 with MeasureColumnResolvedFilterInfo

use of org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo in project carbondata by apache.

the class FilterUtil method getIncludeFilterExecuter.

/**
 * It gives filter executer based on columnar or column group
 *
 * @param dimColResolvedFilterInfo
 * @param segmentProperties
 * @return
 */
private static FilterExecuter getIncludeFilterExecuter(DimColumnResolvedFilterInfo dimColResolvedFilterInfo, MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo, SegmentProperties segmentProperties) {
    if (null != msrColResolvedFilterInfo && msrColResolvedFilterInfo.getMeasure().isColumnar()) {
        CarbonMeasure measuresFromCurrentBlock = segmentProperties.getMeasureFromCurrentBlock(msrColResolvedFilterInfo.getMeasure().getColumnId());
        if (null != measuresFromCurrentBlock) {
            // update dimension and column index according to the dimension position in current block
            MeasureColumnResolvedFilterInfo msrColResolvedFilterInfoCopyObject = msrColResolvedFilterInfo.getCopyObject();
            msrColResolvedFilterInfoCopyObject.setMeasure(measuresFromCurrentBlock);
            msrColResolvedFilterInfoCopyObject.setColumnIndex(measuresFromCurrentBlock.getOrdinal());
            msrColResolvedFilterInfoCopyObject.setType(measuresFromCurrentBlock.getDataType());
            return new IncludeFilterExecuterImpl(null, msrColResolvedFilterInfoCopyObject, segmentProperties, true);
        } else {
            return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo, msrColResolvedFilterInfo, true);
        }
    }
    if (null != dimColResolvedFilterInfo) {
        CarbonDimension dimension = dimColResolvedFilterInfo.getDimension();
        if (dimension.hasEncoding(Encoding.IMPLICIT)) {
            return new ImplicitIncludeFilterExecutorImpl(dimColResolvedFilterInfo);
        } else if (dimension.isColumnar()) {
            CarbonDimension dimensionFromCurrentBlock = segmentProperties.getDimensionFromCurrentBlock(dimColResolvedFilterInfo.getDimension());
            if (null != dimensionFromCurrentBlock) {
                // update dimension and column index according to the dimension position in current block
                DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject = dimColResolvedFilterInfo.getCopyObject();
                dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
                dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
                return new IncludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, null, segmentProperties, false);
            } else {
                return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo, msrColResolvedFilterInfo, false);
            }
        } else {
            return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
        }
    } else {
        return new IncludeColGroupFilterExecuterImpl(null, segmentProperties);
    }
}

Example 4 with MeasureColumnResolvedFilterInfo

use of org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo in project carbondata by apache.

the class RowLevelFilterExecuterImpl method convertRow.

/**
 * convert encoded row to actual value row for filter to evaluate expression
 * @param value this row will be converted to actual value
 * @param dimOrdinalMax for measure column, its index in row = dimOrdinalMax + its ordinal
 * @return actual value row
 * @throws IOException
 */
private RowIntf convertRow(RowIntf value, int dimOrdinalMax) throws IOException {
    Object[] record = new Object[value.size()];
    String memberString;
    for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
        DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
        int index = dimColumnEvaluatorInfo.getDimension().getOrdinal();
        // if filter dimension is not present in the current add its default value
        if (!isDimensionPresentInCurrentBlock[i]) {
            // fill default value here
            record[index] = getDimensionDefaultValue(dimColumnEvaluatorInfo);
            // already set value, so continue to set next dimension
            continue;
        }
        if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
            if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
                record[index] = dimColumnEvaluatorInfo.getDimension().getDefaultValue();
            }
            byte[] memberBytes = (byte[]) value.getVal(index);
            if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
                // no dictionary
                if (null != memberBytes) {
                    if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
                        memberBytes = null;
                    } else if (memberBytes.length == 0) {
                        memberBytes = null;
                    }
                    record[index] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes, dimColumnEvaluatorInfo.getDimension().getDataType());
                }
            } else {
                // dictionary
                int dictionaryValue = ByteUtil.toInt(memberBytes, 0);
                if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    memberString = getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
                    record[index] = DataTypeUtil.getDataBasedOnDataType(memberString, dimColumnEvaluatorInfo.getDimension().getDataType());
                } else if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    Object member = getFilterActualValueFromDirectDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
                    record[index] = member;
                }
            }
        } else {
            // complex
            record[index] = value.getVal(index);
        }
    }
    for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
        MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
        int index = msrColumnEvalutorInfo.getMeasure().getOrdinal() + dimOrdinalMax;
        // in the current block measure list
        if (!isMeasurePresentInCurrentBlock[i]) {
            byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue();
            record[index] = RestructureUtil.getMeasureDefaultValue(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(), defaultValue);
            // already set value, so continue to set next measure
            continue;
        }
        // measure
        record[index] = value.getVal(index);
    }
    RowIntf row = new RowImpl();
    row.setValues(record);
    return row;
}

Example 5 with MeasureColumnResolvedFilterInfo

use of org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo in project carbondata by apache.

the class MeasureColumnVisitor method populateFilterResolvedInfo.

/**
 * Visitor Method will update the filter related details in visitableObj, For no dictionary
 * type columns the filter members will resolved directly, no need to look up in dictionary
 * since it will not be part of dictionary, directly the actual data can be converted as
 * byte[] and can be set. this type of encoding is effective when the particular column
 * is having very high cardinality.
 *
 * @param visitableObj
 * @param metadata
 * @throws FilterUnsupportedException,if exception occurs while evaluating
 *                                       filter models.
 */
public void populateFilterResolvedInfo(ColumnResolvedFilterInfo visitableObj, FilterResolverMetadata metadata) throws FilterUnsupportedException {
    if (visitableObj instanceof MeasureColumnResolvedFilterInfo) {
        MeasureColumnResolvedFilterInfo resolveDimension = (MeasureColumnResolvedFilterInfo) visitableObj;
        ColumnFilterInfo resolvedFilterObject = null;
        List<String> evaluateResultListFinal = new ArrayList<>(1);
        try {
            // handling for is null case scenarios
            if (metadata.getExpression() instanceof EqualToExpression) {
                EqualToExpression expression = (EqualToExpression) metadata.getExpression();
                if (expression.isNull) {
                    evaluateResultListFinal = new ArrayList<>(1);
                    evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
                }
            } else {
                evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
            }
            // displaying the report as per hive compatibility.
            if (!metadata.isIncludeFilter() && !evaluateResultListFinal.contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
                evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
            }
        } catch (FilterIllegalMemberException e) {
            throw new FilterUnsupportedException(e);
        }
        resolvedFilterObject = FilterUtil.getMeasureValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getColumnExpression().getDataType(), resolveDimension.getMeasure());
        resolveDimension.setFilterValues(resolvedFilterObject);
    }
}