Examples with QueryStatistic org.apache.carbondata.core.stats.QueryStatistic used on opensource projects

Example 1 with QueryStatistic

use of org.apache.carbondata.core.stats.QueryStatistic in project carbondata by apache.

the class AbstractBlockletScanner method scanBlocklet.

@Override
public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException, FilterUnsupportedException {
    long startTime = System.currentTimeMillis();
    AbstractScannedResult scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
    QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
    totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, totalBlockletStatistic.getCount() + 1);
    QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
    validScannedBlockletStatistic.addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, validScannedBlockletStatistic.getCount() + 1);
    // adding statistics for valid number of pages
    QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
    validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED, validPages.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    // adding statistics for number of pages
    QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
    totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    scannedResult.setBlockletId(blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder.getDataBlock().nodeNumber());
    DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDimensionRawDataChunk();
    DimensionColumnDataChunk[][] dimensionColumnDataChunks = new DimensionColumnDataChunk[dimensionRawColumnChunks.length][];
    for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
        if (dimensionRawColumnChunks[i] != null) {
            dimensionColumnDataChunks[i] = dimensionRawColumnChunks[i].convertToDimColDataChunks();
        }
    }
    scannedResult.setDimensionChunks(dimensionColumnDataChunks);
    MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getMeasureRawDataChunk();
    MeasureColumnDataChunk[][] measureColumnDataChunks = new MeasureColumnDataChunk[measureRawColumnChunks.length][];
    for (int i = 0; i < measureRawColumnChunks.length; i++) {
        if (measureRawColumnChunks[i] != null) {
            measureColumnDataChunks[i] = measureRawColumnChunks[i].convertToMeasureColDataChunks();
        }
    }
    scannedResult.setMeasureChunks(measureColumnDataChunks);
    int[] numberOfRows = new int[] { blocksChunkHolder.getDataBlock().nodeSize() };
    if (blockExecutionInfo.getAllSelectedDimensionBlocksIndexes().length > 0) {
        for (int i = 0; i < dimensionRawColumnChunks.length; i++) {
            if (dimensionRawColumnChunks[i] != null) {
                numberOfRows = dimensionRawColumnChunks[i].getRowCount();
                break;
            }
        }
    } else if (blockExecutionInfo.getAllSelectedMeasureBlocksIndexes().length > 0) {
        for (int i = 0; i < measureRawColumnChunks.length; i++) {
            if (measureRawColumnChunks[i] != null) {
                numberOfRows = measureRawColumnChunks[i].getRowCount();
                break;
            }
        }
    }
    scannedResult.setNumberOfRows(numberOfRows);
    // loading delete data cache in blockexecutioninfo instance
    DeleteDeltaCacheLoaderIntf deleteCacheLoader = new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(), blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
    deleteCacheLoader.loadDeleteDeltaFileDataToCache();
    scannedResult.setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
    scannedResult.setRawColumnChunks(dimensionRawColumnChunks);
    // adding statistics for carbon scan time
    QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
    scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME, scanTime.getCount() + (System.currentTimeMillis() - startTime));
    return scannedResult;
}

Example 2 with QueryStatistic

use of org.apache.carbondata.core.stats.QueryStatistic in project carbondata by apache.

the class AbstractBlockletScanner method readBlocklet.

@Override
public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
    long startTime = System.currentTimeMillis();
    DimensionRawColumnChunk[] dimensionRawColumnChunks = blocksChunkHolder.getDataBlock().getDimensionChunks(blocksChunkHolder.getFileReader(), blockExecutionInfo.getAllSelectedDimensionBlocksIndexes());
    blocksChunkHolder.setDimensionRawDataChunk(dimensionRawColumnChunks);
    MeasureRawColumnChunk[] measureRawColumnChunks = blocksChunkHolder.getDataBlock().getMeasureChunks(blocksChunkHolder.getFileReader(), blockExecutionInfo.getAllSelectedMeasureBlocksIndexes());
    blocksChunkHolder.setMeasureRawDataChunk(measureRawColumnChunks);
    // adding statistics for carbon read time
    QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
    readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME, readTime.getCount() + (System.currentTimeMillis() - startTime));
}

Example 3 with QueryStatistic

use of org.apache.carbondata.core.stats.QueryStatistic in project carbondata by apache.

the class FilterScanner method isScanRequired.

@Override
public boolean isScanRequired(BlocksChunkHolder blocksChunkHolder) throws IOException {
    // adding statistics for number of pages
    QueryStatistic totalPagesScanned = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
    totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalPagesScanned.getCount() + blocksChunkHolder.getDataBlock().numberOfPages());
    // apply min max
    if (isMinMaxEnabled) {
        BitSet bitSet = this.filterExecuter.isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(), blocksChunkHolder.getDataBlock().getColumnsMinValue());
        if (bitSet.isEmpty()) {
            CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(), blocksChunkHolder.getMeasureRawDataChunk());
            return false;
        }
    }
    return true;
}

Example 4 with QueryStatistic

use of org.apache.carbondata.core.stats.QueryStatistic in project carbondata by apache.

the class FilterScanner method readBlocklet.

@Override
public void readBlocklet(BlocksChunkHolder blocksChunkHolder) throws IOException {
    long startTime = System.currentTimeMillis();
    this.filterExecuter.readBlocks(blocksChunkHolder);
    // adding statistics for carbon read time
    QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
    readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME, readTime.getCount() + (System.currentTimeMillis() - startTime));
}

Example 5 with QueryStatistic

use of org.apache.carbondata.core.stats.QueryStatistic in project carbondata by apache.

the class AbstractQueryExecutor method getBlockExecutionInfos.

protected List<BlockExecutionInfo> getBlockExecutionInfos(QueryModel queryModel) throws IOException, QueryExecutionException {
    initQuery(queryModel);
    List<BlockExecutionInfo> blockExecutionInfoList = new ArrayList<BlockExecutionInfo>();
    // and query will be executed based on that infos
    for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
        blockExecutionInfoList.add(getBlockExecutionInfoForBlock(queryModel, queryProperties.dataBlocks.get(i), queryModel.getTableBlockInfos().get(i).getBlockletInfos().getStartBlockletNumber(), queryModel.getTableBlockInfos().get(i).getBlockletInfos().getNumberOfBlockletToScan(), queryModel.getTableBlockInfos().get(i).getFilePath()));
    }
    if (null != queryModel.getStatisticsRecorder()) {
        QueryStatistic queryStatistic = new QueryStatistic();
        queryStatistic.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM, blockExecutionInfoList.size());
        queryModel.getStatisticsRecorder().recordStatistics(queryStatistic);
    }
    return blockExecutionInfoList;
}