Examples with DimensionRawColumnChunk org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk used on opensource projects

Example 1 with DimensionRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk in project carbondata by apache.

the class AbstractBlockletScanner method scanBlocklet.

@Override
public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException, FilterUnsupportedException {
    long startTime = System.currentTimeMillis();
    AbstractScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
    QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
    totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, totalBlockletStatistic.getCount() + 1);
    QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
    validScannedBlockletStatistic.addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, validScannedBlockletStatistic.getCount() + 1);
    // adding statistics for valid number of pages
    QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
    validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    // adding statistics for number of pages
    QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
    totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    scannedResult.setBlockletId(blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder.getDataBlock().nodeNumber());
    DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDimensionRawDataChunk();
    DimensionColumnDataChunk[][] dimensionColumnDataChunks = new DimensionColumnDataChunk[dimensionRawColumnChunks.length][];
    for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
        if (dimensionRawColumnChunks[i] != null) {
            dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
        }
    }
    scannedResult.setDimensionChunks(dimensionColumnDataChunks);
    MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getMeasureRawDataChunk();
    MeasureColumnDataChunk[][] measureColumnDataChunks = new MeasureColumnDataChunk[measureRawColumnChunks.length][];
    for (int i = 0; i < measureRawColumnChunks.length; i++) {
        if (measureRawColumnChunks[i] != null) {
            measureColumnDataChunks[i] = measureRawColumnChunks[i].convertToMeasureColDataChunks();
        }
    }
    scannedResult.setMeasureChunks(measureColumnDataChunks);
    int[] numberOfRows = new int[] { blocksChunkHolder.getDataBlock().nodeSize() };
    if (blockExecutionInfo.getAllSelectedDimensionBlocksIndexes().length > 0) {
        for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
            if (dimensionRawColumnChunks[i] != null) {
                numberOfRows = dimensionRawColumnChunks[i].getRowCount();
                break;
            }
        }
    } else if (blockExecutionInfo.getAllSelectedMeasureBlocksIndexes().length > 0) {
        for (int i = 0; i < measureRawColumnChunks.length; i++) {
            if (measureRawColumnChunks[i] != null) {
                numberOfRows = measureRawColumnChunks[i].getRowCount();
                break;
            }
        }
    }
    scannedResult.setNumberOfRows(numberOfRows);
    // loading delete data cache in blockexecutioninfo instance
    DeleteDeltaCacheLoaderIntf deleteCacheLoader = new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(), blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
    deleteCacheLoader.loadDeleteDeltaFileDataToCache();
    scannedResult.setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
    scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
    // adding statistics for carbon scan time
    QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
    scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, scanTime.getCount() + (System.currentTimeMillis() - startTime));
    return scannedResult;
}

Example 2 with DimensionRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk in project carbondata by apache.

the class AbstractBlockletScanner method readBlocklet.

@Override
public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
    long startTime = System.currentTimeMillis();
    DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDataBlock().getDimensionChunks(blocksChunkHolder.getFileReader(), blockExecutionInfo.getAllSelectedDimensionBlocksIndexes());
    blocksChunkHolder.setDimensionRawDataChunk(dimensionRawColumnChunks);
    MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getDataBlock().getMeasureChunks(blocksChunkHolder.getFileReader(), blockExecutionInfo.getAllSelectedMeasureBlocksIndexes());
    blocksChunkHolder.setMeasureRawDataChunk(measureRawColumnChunks);
    // adding statistics for carbon read time
    QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
    readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME, readTime.getCount() + (System.currentTimeMillis() - startTime));
}

Example 3 with DimensionRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk in project carbondata by apache.

the class RangeValueFilterExecuterImpl method applyNoAndDirectFilter.

/**
   * Method to apply the Range Filter.
   * @param blockChunkHolder
   * @return
   * @throws FilterUnsupportedException
   * @throws IOException
   */
public BitSetGroup applyNoAndDirectFilter(BlocksChunkHolder blockChunkHolder) throws FilterUnsupportedException, IOException {
    // select all rows if dimension does not exists in the current block
    if (!isDimensionPresentInCurrentBlock) {
        int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
        return FilterUtil.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(), numberOfRows, true);
    }
    int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(dimColEvaluatorInfo.getColumnIndex());
    if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
        blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock().getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
    }
    DimensionRawColumnChunk rawColumnChunk = blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
    BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
    for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
        if (rawColumnChunk.getMaxValues() != null) {
            if (isScanRequired(rawColumnChunk.getMinValues()[i], rawColumnChunk.getMaxValues()[i], this.filterRangesValues)) {
                if (isRangeFullyCoverBlock == true) {
                    // Set all the bits in this case as filter Min Max values cover the whole block.
                    BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
                    bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
                    bitSetGroup.setBitSet(bitSet, i);
                } else {
                    BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i), rawColumnChunk.getRowCount()[i]);
                    bitSetGroup.setBitSet(bitSet, i);
                }
            }
        } else {
            BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i), rawColumnChunk.getRowCount()[i]);
            bitSetGroup.setBitSet(bitSet, i);
        }
    }
    return bitSetGroup;
}

Example 4 with DimensionRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk in project carbondata by apache.

the class ExcludeFilterExecuterImpl method applyFilter.

@Override
public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
    int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(dimColEvaluatorInfo.getColumnIndex());
    if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
        blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock().getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
    }
    DimensionRawColumnChunk dimensionRawColumnChunk = blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
    DimensionColumnDataChunk[] dimensionColumnDataChunks = dimensionRawColumnChunk.convertToDimColDataChunks();
    BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
    for (int i = 0; i < dimensionColumnDataChunks.length; i++) {
        BitSet bitSet = getFilteredIndexes(dimensionColumnDataChunks[i], dimensionRawColumnChunk.getRowCount()[i]);
        bitSetGroup.setBitSet(bitSet, i);
    }
    return bitSetGroup;
}

Example 5 with DimensionRawColumnChunk

use of org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk in project carbondata by apache.

the class RowLevelRangeGrtrThanEquaToFilterExecuterImpl method applyFilter.

@Override
public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws FilterUnsupportedException, IOException {
    // select all rows if dimension does not exists in the current block
    if (!isDimensionPresentInCurrentBlock[0]) {
        int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
        return FilterUtil.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(), numberOfRows, true);
    }
    int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
    if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
        blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock().getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
    }
    DimensionRawColumnChunk rawColumnChunk = blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
    BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
    for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
        if (rawColumnChunk.getMaxValues() != null) {
            if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
                int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
                if (compare <= 0) {
                    BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
                    bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
                    bitSetGroup.setBitSet(bitSet, i);
                } else {
                    BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i), rawColumnChunk.getRowCount()[i]);
                    bitSetGroup.setBitSet(bitSet, i);
                }
            }
        } else {
            BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i), rawColumnChunk.getRowCount()[i]);
            bitSetGroup.setBitSet(bitSet, i);
        }
    }
    return bitSetGroup;
}