Examples with DataRefNodeFinder org.apache.carbondata.core.datastore.DataRefNodeFinder used on opensource projects

Example 1 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class AbstractDetailQueryResultIterator method intialiseInfos.

private void intialiseInfos() {
    for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
        DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize(), blockInfo.getDataBlock().getSegmentProperties().getNumberOfSortColumns(), blockInfo.getDataBlock().getSegmentProperties().getNumberOfNoDictSortColumns());
        DataRefNode startDataBlock = finder.findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
        while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
            startDataBlock = startDataBlock.getNextDataRefNode();
        }
        long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
        //if number of block is less than 0 then take end block.
        if (numberOfBlockToScan <= 0) {
            DataRefNode endDataBlock = finder.findLastDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getEndKey());
            numberOfBlockToScan = endDataBlock.nodeNumber() - startDataBlock.nodeNumber() + 1;
        }
        blockInfo.setFirstDataBlock(startDataBlock);
        blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
    }
}

Example 2 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class FilterExpressionProcessor method getFilterredBlocks.

/**
   * This API will scan the Segment level all btrees and selects the required
   * block reference  nodes inorder to push the same to executer for applying filters
   * on the respective data reference node.
   * Following Algorithm is followed in below API
   * Step:1 Get the start end key based on the filter tree resolver information
   * Step:2 Prepare the IndexKeys inorder to scan the tree and get the start and end reference
   * node(block)
   * Step:3 Once data reference node ranges retrieved traverse the node within this range
   * and select the node based on the block min and max value and the filter value.
   * Step:4 The selected blocks will be send to executers for applying the filters with the help
   * of Filter executers.
   *
   */
public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode, FilterResolverIntf filterResolver, AbstractIndex tableSegment, AbsoluteTableIdentifier tableIdentifier) {
    // Need to get the current dimension tables
    List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
    // getting the start and end index key based on filter for hitting the
    // selected block reference nodes based on filter resolver tree.
    LOGGER.debug("preparing the start and end key for finding" + "start and end block as per filter resolver");
    List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
    FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(), filterResolver, listOfStartEndKeys);
    // reading the first value from list which has start key
    IndexKey searchStartKey = listOfStartEndKeys.get(0);
    // reading the last value from list which has end key
    IndexKey searchEndKey = listOfStartEndKeys.get(1);
    if (null == searchStartKey && null == searchEndKey) {
        try {
            // TODO need to handle for no dictionary dimensions
            searchStartKey = FilterUtil.prepareDefaultStartIndexKey(tableSegment.getSegmentProperties());
            // TODO need to handle for no dictionary dimensions
            searchEndKey = FilterUtil.prepareDefaultEndIndexKey(tableSegment.getSegmentProperties());
        } catch (KeyGenException e) {
            return listOfDataBlocksToScan;
        }
    }
    LOGGER.debug("Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey.getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey.getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys());
    long startTimeInMillis = System.currentTimeMillis();
    DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(tableSegment.getSegmentProperties().getEachDimColumnValueSize(), tableSegment.getSegmentProperties().getNumberOfSortColumns(), tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns());
    DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
    DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
    FilterExecuter filterExecuter = FilterUtil.getFilterExecuterTree(filterResolver, tableSegment.getSegmentProperties(), null);
    while (startBlock != endBlock) {
        addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, startBlock);
        startBlock = startBlock.getNextDataRefNode();
    }
    addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, endBlock);
    LOGGER.info("Total Time in retrieving the data reference node" + "after scanning the btree " + (System.currentTimeMillis() - startTimeInMillis) + " Total number of data reference node for executing filter(s) " + listOfDataBlocksToScan.size());
    return listOfDataBlocksToScan;
}

Example 3 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1() throws KeyGenException {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
    int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
    KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 1, 1 }), null);
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(0, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(0, findLastBlock.nodeNumber());
}

Example 4 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey.

/**
   * Below method will test when key which is not present and key which is
   * more than
   * last node key is passes for searching it should give first block
   */
@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey() throws KeyGenException {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
    int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
    KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 10001, 10001 }), null);
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(99, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(99, findLastBlock.nodeNumber());
}

Example 5 with DataRefNodeFinder

use of org.apache.carbondata.core.datastore.DataRefNodeFinder in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2() throws KeyGenException {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
    int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
    KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 0, 0 }), null);
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(0, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(0, findLastBlock.nodeNumber());
}