Examples with DataRefNode org.apache.carbondata.core.datastore.DataRefNode used on opensource projects

Example 11 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class InMemoryBTreeIndex method filter.

@Override
public List<Block> filter(JobContext job, FilterResolverIntf filter) throws IOException {
    List<Block> result = new LinkedList<>();
    FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
    AbsoluteTableIdentifier identifier = null;
    //for this segment fetch blocks matching filter in BTree
    List<DataRefNode> dataRefNodes = getDataBlocksOfSegment(job, filterExpressionProcessor, identifier, filter);
    for (DataRefNode dataRefNode : dataRefNodes) {
        BlockBTreeLeafNode leafNode = (BlockBTreeLeafNode) dataRefNode;
        TableBlockInfo tableBlockInfo = leafNode.getTableBlockInfo();
        result.add(new CarbonInputSplit(segment.getId(), new Path(tableBlockInfo.getFilePath()), tableBlockInfo.getBlockOffset(), tableBlockInfo.getBlockLength(), tableBlockInfo.getLocations(), tableBlockInfo.getBlockletInfos().getNoOfBlockLets(), tableBlockInfo.getVersion()));
    }
    return result;
}

Example 12 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class InMemoryBTreeIndex method getDataBlocksOfSegment.

/**
   * get data blocks of given segment
   */
private List<DataRefNode> getDataBlocksOfSegment(JobContext job, FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier identifier, FilterResolverIntf resolver) throws IOException {
    QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
    QueryStatistic statistic = new QueryStatistic();
    Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = getSegmentAbstractIndexs(job, identifier);
    List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
    // build result
    for (AbstractIndex abstractIndex : segmentIndexMap.values()) {
        List<DataRefNode> filterredBlocks = null;
        // if no filter is given get all blocks from Btree Index
        if (null == resolver) {
            filterredBlocks = getDataBlocksOfIndex(abstractIndex);
        } else {
            // apply filter and get matching blocks
            filterredBlocks = filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex, identifier);
        }
        resultFilterredBlocks.addAll(filterredBlocks);
    }
    statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
    recorder.recordStatistics(statistic);
    recorder.logStatistics();
    return resultFilterredBlocks;
}

Example 13 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class BlockLevelTraverser method getBlockRowMapping.

/**
   *
   * @param abstractIndex
   * @param blockRowMap
   * @param segId
   * @param updateStatusManager
   * @throws KeyGenException
   */
public long getBlockRowMapping(AbstractIndex abstractIndex, Map<String, Long> blockRowMap, String segId, SegmentUpdateStatusManager updateStatusManager) throws KeyGenException {
    IndexKey searchStartKey = FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
    DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(abstractIndex.getSegmentProperties().getEachDimColumnValueSize(), abstractIndex.getSegmentProperties().getNumberOfSortColumns(), abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
    DataRefNode currentBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
    long count = 0;
    while (currentBlock != null) {
        String blockName = ((BlockBTreeLeafNode) currentBlock).getTableBlockInfo().getFilePath();
        blockName = CarbonTablePath.getCarbonDataFileName(blockName);
        blockName = blockName + CarbonTablePath.getCarbonDataExtension();
        long rowCount = currentBlock.nodeSize();
        String key = CarbonUpdateUtil.getSegmentBlockNameKey(segId, blockName);
        // if block is invalid then dont add the count
        SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key);
        if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getStatus())) {
            blockRowMap.put(key, rowCount);
            count++;
        }
        currentBlock = currentBlock.getNextDataRefNode();
    }
    return count;
}

Example 14 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class CarbonInputFormat method getSplits.

/**
   * {@inheritDoc}
   * Configurations FileInputFormat.INPUT_DIR, CarbonInputFormat.INPUT_SEGMENT_NUMBERS
   * are used to get table path to read.
   *
   * @return
   * @throws IOException
   */
private List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver, BitSet matchedPartitions, CacheClient cacheClient) throws IOException {
    List<InputSplit> result = new LinkedList<InputSplit>();
    FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
    AbsoluteTableIdentifier absoluteTableIdentifier = getCarbonTable(job.getConfiguration()).getAbsoluteTableIdentifier();
    SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(absoluteTableIdentifier);
    //for each segment fetch blocks matching filter in Driver BTree
    for (String segmentNo : getSegmentsToAccess(job)) {
        List<DataRefNode> dataRefNodes = getDataBlocksOfSegment(job, filterExpressionProcessor, absoluteTableIdentifier, filterResolver, matchedPartitions, segmentNo, cacheClient, updateStatusManager);
        for (DataRefNode dataRefNode : dataRefNodes) {
            BlockBTreeLeafNode leafNode = (BlockBTreeLeafNode) dataRefNode;
            TableBlockInfo tableBlockInfo = leafNode.getTableBlockInfo();
            if (CarbonUtil.isInvalidTableBlock(tableBlockInfo, updateStatusManager.getInvalidTimestampRange(tableBlockInfo.getSegmentId()), updateStatusManager)) {
                continue;
            }
            result.add(new CarbonInputSplit(segmentNo, new Path(tableBlockInfo.getFilePath()), tableBlockInfo.getBlockOffset(), tableBlockInfo.getBlockLength(), tableBlockInfo.getLocations(), tableBlockInfo.getBlockletInfos().getNoOfBlockLets(), tableBlockInfo.getVersion()));
        }
    }
    return result;
}

Example 15 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class CarbonInputFormat method getDataBlocksOfIndex.

/**
   * get data blocks of given btree
   */
private List<DataRefNode> getDataBlocksOfIndex(AbstractIndex abstractIndex) {
    List<DataRefNode> blocks = new LinkedList<DataRefNode>();
    SegmentProperties segmentProperties = abstractIndex.getSegmentProperties();
    try {
        IndexKey startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
        IndexKey endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
        // Add all blocks of btree into result
        DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(), segmentProperties.getNumberOfSortColumns(), segmentProperties.getNumberOfNoDictSortColumns());
        DataRefNode startBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
        DataRefNode endBlock = blockFinder.findLastDataBlock(abstractIndex.getDataRefNode(), endIndexKey);
        while (startBlock != endBlock) {
            blocks.add(startBlock);
            startBlock = startBlock.getNextDataRefNode();
        }
        blocks.add(endBlock);
    } catch (KeyGenException e) {
        LOG.error("Could not generate start key", e);
    }
    return blocks;
}