Examples with DataRefNodeFinder org.apache.carbondata.core.datastore.DataRefNodeFinder used on opensource projects

Example 6 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary() {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
    ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
    buffer.rewind();
    buffer.put((byte) 1);
    buffer.putInt(0);
    buffer.array();
    IndexKey key = new IndexKey(null, buffer.array());
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(0, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(0, findLastBlock.nodeNumber());
}

Example 7 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1() {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
    ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
    buffer.rewind();
    buffer.putShort((short) 1);
    buffer.putInt(12);
    buffer.array();
    IndexKey key = new IndexKey(null, buffer.array());
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(1, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(1, findLastBlock.nodeNumber());
}

Example 8 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class BlockLevelTraverser method getBlockRowMapping.

/**
   *
   * @param abstractIndex
   * @param blockRowMap
   * @param segId
   * @param updateStatusManager
   * @throws KeyGenException
   */
public long getBlockRowMapping(AbstractIndex abstractIndex, Map<String, Long> blockRowMap, String segId, SegmentUpdateStatusManager updateStatusManager) throws KeyGenException {
    IndexKey searchStartKey = FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
    DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(abstractIndex.getSegmentProperties().getEachDimColumnValueSize(), abstractIndex.getSegmentProperties().getNumberOfSortColumns(), abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
    DataRefNode currentBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
    long count = 0;
    while (currentBlock != null) {
        String blockName = ((BlockBTreeLeafNode) currentBlock).getTableBlockInfo().getFilePath();
        blockName = CarbonTablePath.getCarbonDataFileName(blockName);
        blockName = blockName + CarbonTablePath.getCarbonDataExtension();
        long rowCount = currentBlock.nodeSize();
        String key = CarbonUpdateUtil.getSegmentBlockNameKey(segId, blockName);
        // if block is invalid then dont add the count
        SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key);
        if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getStatus())) {
            blockRowMap.put(key, rowCount);
            count++;
        }
        currentBlock = currentBlock.getNextDataRefNode();
    }
    return count;
}

Example 9 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class CarbonInputFormat method getDataBlocksOfIndex.

/**
   * get data blocks of given btree
   */
private List<DataRefNode> getDataBlocksOfIndex(AbstractIndex abstractIndex) {
    List<DataRefNode> blocks = new LinkedList<DataRefNode>();
    SegmentProperties segmentProperties = abstractIndex.getSegmentProperties();
    try {
        IndexKey startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
        IndexKey endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
        // Add all blocks of btree into result
        DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(), segmentProperties.getNumberOfSortColumns(), segmentProperties.getNumberOfNoDictSortColumns());
        DataRefNode startBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
        DataRefNode endBlock = blockFinder.findLastDataBlock(abstractIndex.getDataRefNode(), endIndexKey);
        while (startBlock != endBlock) {
            blocks.add(startBlock);
            startBlock = startBlock.getNextDataRefNode();
        }
        blocks.add(endBlock);
    } catch (KeyGenException e) {
        LOG.error("Could not generate start key", e);
    }
    return blocks;
}

Example 10 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class InMemoryBTreeIndex method getDataBlocksOfIndex.

/**
   * get data blocks of given btree
   */
private List<DataRefNode> getDataBlocksOfIndex(AbstractIndex abstractIndex) {
    List<DataRefNode> blocks = new LinkedList<DataRefNode>();
    SegmentProperties segmentProperties = abstractIndex.getSegmentProperties();
    try {
        IndexKey startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
        IndexKey endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
        // Add all blocks of btree into result
        DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(), segmentProperties.getNumberOfSortColumns(), segmentProperties.getNumberOfNoDictSortColumns());
        DataRefNode startBlock = blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
        DataRefNode endBlock = blockFinder.findLastDataBlock(abstractIndex.getDataRefNode(), endIndexKey);
        while (startBlock != endBlock) {
            blocks.add(startBlock);
            startBlock = startBlock.getNextDataRefNode();
        }
        blocks.add(endBlock);
    } catch (KeyGenException e) {
        LOG.error("Could not generate start key", e);
    }
    return blocks;
}