Examples with ImplicitColumnFilterExecutor org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor used on opensource projects

Example 1 with ImplicitColumnFilterExecutor

use of org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor in project carbondata by apache.

the class BlockIndex method addBlockBasedOnMinMaxValue.

/**
 * select the blocks based on column min and max value
 *
 * @param filterExecutor
 * @param maxValue
 * @param minValue
 * @param minMaxFlag
 * @param filePath
 * @param blockletId
 * @return
 */
private boolean addBlockBasedOnMinMaxValue(FilterExecutor filterExecutor, byte[][] maxValue, byte[][] minValue, boolean[] minMaxFlag, String filePath, int blockletId) {
    BitSet bitSet = null;
    if (filterExecutor instanceof ImplicitColumnFilterExecutor) {
        String uniqueBlockPath;
        CarbonTable carbonTable = segmentPropertiesWrapper.getCarbonTable();
        if (carbonTable.isHivePartitionTable()) {
            // While data loading to SI created on Partition table, on partition directory, '/' will be
            // replaced with '#', to support multi level partitioning. For example, BlockId will be
            // look like `part1=1#part2=2/xxxxxxxxx`. During query also, blockId should be
            // replaced by '#' in place of '/', to match and prune data on SI table.
            uniqueBlockPath = CarbonUtil.getBlockId(carbonTable.getAbsoluteTableIdentifier(), filePath, "", true, false, true);
        } else {
            uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1);
        }
        // blocklet information
        if (blockletId != -1) {
            uniqueBlockPath = uniqueBlockPath + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
        }
        bitSet = ((ImplicitColumnFilterExecutor) filterExecutor).isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath, minMaxFlag);
    } else {
        bitSet = filterExecutor.isScanRequired(maxValue, minValue, minMaxFlag);
    }
    return !bitSet.isEmpty();
}

Example 2 with ImplicitColumnFilterExecutor

use of org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor in project carbondata by apache.

the class BlockletDataMap method addBlockBasedOnMinMaxValue.

/**
 * select the blocks based on column min and max value
 *
 * @param filterExecuter
 * @param maxValue
 * @param minValue
 * @param filePath
 * @param blockletId
 * @return
 */
private boolean addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter, byte[][] maxValue, byte[][] minValue, String filePath, int blockletId) {
    BitSet bitSet = null;
    if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
        String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1);
        // blocklet information
        if (blockletId != -1) {
            uniqueBlockPath = uniqueBlockPath + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
        }
        bitSet = ((ImplicitColumnFilterExecutor) filterExecuter).isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
    } else {
        bitSet = filterExecuter.isScanRequired(maxValue, minValue);
    }
    if (!bitSet.isEmpty()) {
        return true;
    } else {
        return false;
    }
}

Example 3 with ImplicitColumnFilterExecutor

use of org.apache.carbondata.core.scan.filter.executer.ImplicitColumnFilterExecutor in project carbondata by apache.

the class BlockletFilterScanner method isScanRequired.

@Override
public boolean isScanRequired(DataRefNode dataBlock) {
    // adding statistics for number of pages
    QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
    totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalPagesScanned.getCount() + dataBlock.numberOfPages());
    QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
    totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, totalBlockletStatistic.getCount() + 1);
    // apply min max
    if (isMinMaxEnabled) {
        if (null == dataBlock.getColumnsMaxValue() || null == dataBlock.getColumnsMinValue()) {
            return true;
        }
        BitSet bitSet = null;
        // check for implicit include filter instance
        if (filterExecutor instanceof ImplicitColumnFilterExecutor) {
            String blockletId = blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR + dataBlock.blockletIndex();
            bitSet = ((ImplicitColumnFilterExecutor) filterExecutor).isFilterValuesPresentInBlockOrBlocklet(dataBlock.getColumnsMaxValue(), dataBlock.getColumnsMinValue(), blockletId, dataBlock.minMaxFlagArray());
        } else {
            bitSet = this.filterExecutor.isScanRequired(dataBlock.getColumnsMaxValue(), dataBlock.getColumnsMinValue(), dataBlock.minMaxFlagArray());
        }
        return !bitSet.isEmpty();
    }
    return true;
}