Examples with QueryStatisticsRecorder org.apache.carbondata.core.stats.QueryStatisticsRecorder used on opensource projects

Example 1 with QueryStatisticsRecorder

use of org.apache.carbondata.core.stats.QueryStatisticsRecorder in project carbondata by apache.

the class CarbonInputFormat method getDataBlocksOfSegment.

/**
   * get data blocks of given segment
   */
private List<DataRefNode> getDataBlocksOfSegment(JobContext job, FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier absoluteTableIdentifier, FilterResolverIntf resolver, BitSet matchedPartitions, String segmentId, CacheClient cacheClient, SegmentUpdateStatusManager updateStatusManager) throws IOException {
    Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = null;
    try {
        QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
        QueryStatistic statistic = new QueryStatistic();
        segmentIndexMap = getSegmentAbstractIndexs(job, absoluteTableIdentifier, segmentId, cacheClient, updateStatusManager);
        List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
        if (null != segmentIndexMap) {
            for (Map.Entry<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> entry : segmentIndexMap.entrySet()) {
                SegmentTaskIndexStore.TaskBucketHolder taskHolder = entry.getKey();
                int taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskHolder.taskNo);
                // if this partition is not required, here will skip it.
                if (matchedPartitions == null || matchedPartitions.get(taskId)) {
                    AbstractIndex abstractIndex = entry.getValue();
                    List<DataRefNode> filterredBlocks;
                    // if no filter is given get all blocks from Btree Index
                    if (null == resolver) {
                        filterredBlocks = getDataBlocksOfIndex(abstractIndex);
                    } else {
                        // apply filter and get matching blocks
                        filterredBlocks = filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex, absoluteTableIdentifier);
                    }
                    resultFilterredBlocks.addAll(filterredBlocks);
                }
            }
        }
        statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
        recorder.recordStatisticsForDriver(statistic, job.getConfiguration().get("query.id"));
        return resultFilterredBlocks;
    } finally {
        // low memory systems the same memory can be utilized efficiently
        if (null != segmentIndexMap) {
            List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers = new ArrayList<>(1);
            tableSegmentUniqueIdentifiers.add(new TableSegmentUniqueIdentifier(absoluteTableIdentifier, segmentId));
            cacheClient.getSegmentAccessClient().clearAccessCount(tableSegmentUniqueIdentifiers);
        }
    }
}

Example 2 with QueryStatisticsRecorder

use of org.apache.carbondata.core.stats.QueryStatisticsRecorder in project carbondata by apache.

the class InMemoryBTreeIndex method getDataBlocksOfSegment.

/**
   * get data blocks of given segment
   */
private List<DataRefNode> getDataBlocksOfSegment(JobContext job, FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier identifier, FilterResolverIntf resolver) throws IOException {
    QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
    QueryStatistic statistic = new QueryStatistic();
    Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = getSegmentAbstractIndexs(job, identifier);
    List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
    // build result
    for (AbstractIndex abstractIndex : segmentIndexMap.values()) {
        List<DataRefNode> filterredBlocks = null;
        // if no filter is given get all blocks from Btree Index
        if (null == resolver) {
            filterredBlocks = getDataBlocksOfIndex(abstractIndex);
        } else {
            // apply filter and get matching blocks
            filterredBlocks = filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex, identifier);
        }
        resultFilterredBlocks.addAll(filterredBlocks);
    }
    statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
    recorder.recordStatistics(statistic);
    recorder.logStatistics();
    return resultFilterredBlocks;
}