Search in sources :

Example 1 with MatchExpression

use of org.apache.carbondata.core.scan.expression.MatchExpression in project carbondata by apache.

the class RowLevelFilterExecuterImpl method applyFilter.

@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
    if (exp instanceof MatchExpression) {
        return rawBlockletColumnChunks.getBitSetGroup();
    }
    readColumnChunks(rawBlockletColumnChunks);
    // CHECKSTYLE:ON
    int[] numberOfRows = null;
    int pageNumbers = 0;
    if (dimColEvaluatorInfoList.size() > 0) {
        if (isDimensionPresentInCurrentBlock[0]) {
            pageNumbers = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getPagesCount();
            numberOfRows = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getRowCount();
        } else {
            // specific for restructure case where default values need to be filled
            pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
            numberOfRows = new int[pageNumbers];
            for (int i = 0; i < pageNumbers; i++) {
                numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
            }
        }
    }
    if (msrColEvalutorInfoList.size() > 0) {
        if (isMeasurePresentInCurrentBlock[0]) {
            pageNumbers = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]].getPagesCount();
            numberOfRows = rawBlockletColumnChunks.getMeasureRawColumnChunks()[measureChunkIndex[0]].getRowCount();
        } else {
            // specific for restructure case where default values need to be filled
            pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
            numberOfRows = new int[pageNumbers];
            for (int i = 0; i < pageNumbers; i++) {
                numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
            }
        }
    }
    BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
    for (int i = 0; i < pageNumbers; i++) {
        BitSet set = new BitSet(numberOfRows[i]);
        RowIntf row = new RowImpl();
        BitSet prvBitset = null;
        // otherwise use older flow
        if (!useBitsetPipeLine || null == rawBlockletColumnChunks.getBitSetGroup() || null == bitSetGroup.getBitSet(i) || rawBlockletColumnChunks.getBitSetGroup().getBitSet(i).isEmpty()) {
            for (int index = 0; index < numberOfRows[i]; index++) {
                createRow(rawBlockletColumnChunks, row, i, index);
                Boolean rslt = false;
                try {
                    rslt = exp.evaluate(row).getBoolean();
                }// too much log inforation only once the log will be printed.
                 catch (FilterIllegalMemberException e) {
                    FilterUtil.logError(e, false);
                }
                if (null != rslt && rslt) {
                    set.set(index);
                }
            }
        } else {
            prvBitset = rawBlockletColumnChunks.getBitSetGroup().getBitSet(i);
            for (int index = prvBitset.nextSetBit(0); index >= 0; index = prvBitset.nextSetBit(index + 1)) {
                createRow(rawBlockletColumnChunks, row, i, index);
                Boolean rslt = false;
                try {
                    rslt = exp.evaluate(row).getBoolean();
                } catch (FilterIllegalMemberException e) {
                    FilterUtil.logError(e, false);
                }
                if (null != rslt && rslt) {
                    set.set(index);
                }
            }
        }
        bitSetGroup.setBitSet(set, i);
    }
    return bitSetGroup;
}
Also used : RowImpl(org.apache.carbondata.core.scan.filter.intf.RowImpl) BitSetGroup(org.apache.carbondata.core.util.BitSetGroup) BitSet(java.util.BitSet) RowIntf(org.apache.carbondata.core.scan.filter.intf.RowIntf) MatchExpression(org.apache.carbondata.core.scan.expression.MatchExpression) FilterIllegalMemberException(org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException)

Example 2 with MatchExpression

use of org.apache.carbondata.core.scan.expression.MatchExpression in project carbondata by apache.

the class RowLevelFilterExecutorImpl method applyFilter.

@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
    if (exp instanceof MatchExpression) {
        BitSetGroup bitSetGroup = rawBlockletColumnChunks.getBitSetGroup();
        if (bitSetGroup == null) {
            // It means there are no index created on this table
            throw new FilterUnsupportedException(String.format("%s is not supported on table %s", exp.getFilterExpressionType().name(), tableIdentifier.getTableName()));
        }
        return bitSetGroup;
    }
    readColumnChunks(rawBlockletColumnChunks);
    // CHECKSTYLE:ON
    int[] numberOfRows = null;
    int pageNumbers = 0;
    if (dimColEvaluatorInfoList.size() > 0) {
        if (isDimensionPresentInCurrentBlock[0]) {
            pageNumbers = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getPagesCount();
            numberOfRows = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]].getRowCount();
        } else {
            // specific for restructure case where default values need to be filled
            pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
            numberOfRows = new int[pageNumbers];
            for (int i = 0; i < pageNumbers; i++) {
                numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
            }
        }
    }
    if (msrColEvalutorInfoList.size() > 0) {
        if (isMeasurePresentInCurrentBlock[0]) {
            pageNumbers = rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0).getColumnIndex()].getPagesCount();
            numberOfRows = rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0).getColumnIndex()].getRowCount();
        } else {
            // specific for restructure case where default values need to be filled
            pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
            numberOfRows = new int[pageNumbers];
            for (int i = 0; i < pageNumbers; i++) {
                numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
            }
        }
    }
    BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
    if (isDimensionPresentInCurrentBlock.length == 1 && isDimensionPresentInCurrentBlock[0] && dimColEvaluatorInfoList.get(0).getDimension().getDataType().isComplexType() && exp instanceof EqualToExpression) {
        LiteralExpression literalExp = (LiteralExpression) (((EqualToExpression) exp).getRight());
        // convert filter value to byte[] to compare with byte[] data from columnPage
        Object literalExpValue = literalExp.getLiteralExpValue();
        DataType literalExpDataType = literalExp.getLiteralExpDataType();
        if (literalExpDataType == DataTypes.TIMESTAMP) {
            if ((long) literalExpValue == 0) {
                literalExpValue = null;
            } else {
                literalExpValue = (long) literalExpValue / TimeStampGranularityTypeValue.MILLIS_SECONDS.getValue();
            }
        } else if (literalExpDataType == DataTypes.DATE) {
            // change data type to int to get the byte[] filter value as it is direct dictionary
            literalExpDataType = DataTypes.INT;
            if (literalExpValue == null) {
                literalExpValue = CarbonCommonConstants.DIRECT_DICT_VALUE_NULL;
            } else {
                literalExpValue = (int) literalExpValue + DateDirectDictionaryGenerator.cutOffDate;
            }
        }
        byte[] filterValueInBytes = DataTypeUtil.getBytesDataDataTypeForNoDictionaryColumn(literalExpValue, literalExpDataType);
        ArrayQueryType complexType = (ArrayQueryType) complexDimensionInfoMap.get(dimensionChunkIndex[0]);
        int totalCount = 0;
        // check all the pages
        for (int i = 0; i < pageNumbers; i++) {
            if (limit != -1 && totalCount >= limit) {
                break;
            }
            BitSet set = new BitSet(numberOfRows[i]);
            int[][] numberOfChild = complexType.getNumberOfChild(rawBlockletColumnChunks.getDimensionRawColumnChunks(), null, numberOfRows[i], i);
            DimensionColumnPage page = complexType.parseBlockAndReturnChildData(rawBlockletColumnChunks.getDimensionRawColumnChunks(), null, i);
            // check every row
            for (int index = 0; index < numberOfRows[i]; index++) {
                if (limit != -1 && totalCount >= limit) {
                    break;
                }
                int dataOffset = numberOfChild[index][1];
                // loop the children
                for (int j = 0; j < numberOfChild[index][0]; j++) {
                    byte[] obj = page.getChunkData(dataOffset++);
                    if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(obj, filterValueInBytes) == 0) {
                        set.set(index);
                        totalCount++;
                        break;
                    }
                }
            }
            bitSetGroup.setBitSet(set, i);
        }
    } else {
        for (int i = 0; i < pageNumbers; i++) {
            BitSet set = new BitSet(numberOfRows[i]);
            RowIntf row = new RowImpl();
            BitSet prvBitset = null;
            // otherwise use older flow
            if (!useBitsetPipeLine || null == rawBlockletColumnChunks.getBitSetGroup() || null == bitSetGroup.getBitSet(i) || rawBlockletColumnChunks.getBitSetGroup().getBitSet(i).isEmpty()) {
                for (int index = 0; index < numberOfRows[i]; index++) {
                    createRow(rawBlockletColumnChunks, row, i, index);
                    Boolean result = false;
                    try {
                        result = exp.evaluate(row).getBoolean();
                    }// too much log inforation only once the log will be printed.
                     catch (FilterIllegalMemberException e) {
                        FilterUtil.logError(e, false);
                    }
                    if (null != result && result) {
                        set.set(index);
                    }
                }
            } else {
                prvBitset = rawBlockletColumnChunks.getBitSetGroup().getBitSet(i);
                for (int index = prvBitset.nextSetBit(0); index >= 0; index = prvBitset.nextSetBit(index + 1)) {
                    createRow(rawBlockletColumnChunks, row, i, index);
                    Boolean rslt = false;
                    try {
                        rslt = exp.evaluate(row).getBoolean();
                    } catch (FilterIllegalMemberException e) {
                        FilterUtil.logError(e, false);
                    }
                    if (null != rslt && rslt) {
                        set.set(index);
                    }
                }
            }
            bitSetGroup.setBitSet(set, i);
        }
    }
    return bitSetGroup;
}
Also used : RowImpl(org.apache.carbondata.core.scan.filter.intf.RowImpl) EqualToExpression(org.apache.carbondata.core.scan.expression.conditional.EqualToExpression) LiteralExpression(org.apache.carbondata.core.scan.expression.LiteralExpression) BitSet(java.util.BitSet) RowIntf(org.apache.carbondata.core.scan.filter.intf.RowIntf) FilterUnsupportedException(org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException) ArrayQueryType(org.apache.carbondata.core.scan.complextypes.ArrayQueryType) MatchExpression(org.apache.carbondata.core.scan.expression.MatchExpression) VariableLengthDimensionColumnPage(org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage) DimensionColumnPage(org.apache.carbondata.core.datastore.chunk.DimensionColumnPage) BitSetGroup(org.apache.carbondata.core.util.BitSetGroup) DataType(org.apache.carbondata.core.metadata.datatype.DataType) FilterIllegalMemberException(org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException)

Aggregations

BitSet (java.util.BitSet)2 MatchExpression (org.apache.carbondata.core.scan.expression.MatchExpression)2 FilterIllegalMemberException (org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException)2 RowImpl (org.apache.carbondata.core.scan.filter.intf.RowImpl)2 RowIntf (org.apache.carbondata.core.scan.filter.intf.RowIntf)2 BitSetGroup (org.apache.carbondata.core.util.BitSetGroup)2 DimensionColumnPage (org.apache.carbondata.core.datastore.chunk.DimensionColumnPage)1 VariableLengthDimensionColumnPage (org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage)1 DataType (org.apache.carbondata.core.metadata.datatype.DataType)1 ArrayQueryType (org.apache.carbondata.core.scan.complextypes.ArrayQueryType)1 LiteralExpression (org.apache.carbondata.core.scan.expression.LiteralExpression)1 EqualToExpression (org.apache.carbondata.core.scan.expression.conditional.EqualToExpression)1 FilterUnsupportedException (org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException)1