Search in sources :

Example 6 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeBuilderGetMethodIsGivingNotNullRootNode.

@Test
public void testBtreeBuilderGetMethodIsGivingNotNullRootNode() {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getDataFileFooterList();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
}
Also used : DataFileFooter(org.apache.carbondata.core.metadata.blocklet.DataFileFooter) BTreeBuilderInfo(org.apache.carbondata.core.datastore.BTreeBuilderInfo) DataRefNode(org.apache.carbondata.core.datastore.DataRefNode) BtreeBuilder(org.apache.carbondata.core.datastore.BtreeBuilder) Test(org.junit.Test)

Example 7 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2() throws KeyGenException {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
    int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
    KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 0, 0 }), null);
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(0, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(0, findLastBlock.nodeNumber());
}
Also used : IndexKey(org.apache.carbondata.core.datastore.IndexKey) MultiDimKeyVarLengthGenerator(org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator) DataFileFooter(org.apache.carbondata.core.metadata.blocklet.DataFileFooter) BTreeBuilderInfo(org.apache.carbondata.core.datastore.BTreeBuilderInfo) DataRefNode(org.apache.carbondata.core.datastore.DataRefNode) DataRefNodeFinder(org.apache.carbondata.core.datastore.DataRefNodeFinder) KeyGenerator(org.apache.carbondata.core.keygenerator.KeyGenerator) BtreeBuilder(org.apache.carbondata.core.datastore.BtreeBuilder) Test(org.junit.Test)

Example 8 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary() {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
    ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
    buffer.rewind();
    buffer.put((byte) 1);
    buffer.putInt(0);
    buffer.array();
    IndexKey key = new IndexKey(null, buffer.array());
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(0, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(0, findLastBlock.nodeNumber());
}
Also used : IndexKey(org.apache.carbondata.core.datastore.IndexKey) DataFileFooter(org.apache.carbondata.core.metadata.blocklet.DataFileFooter) BTreeBuilderInfo(org.apache.carbondata.core.datastore.BTreeBuilderInfo) DataRefNode(org.apache.carbondata.core.datastore.DataRefNode) DataRefNodeFinder(org.apache.carbondata.core.datastore.DataRefNodeFinder) ByteBuffer(java.nio.ByteBuffer) BtreeBuilder(org.apache.carbondata.core.datastore.BtreeBuilder) Test(org.junit.Test)

Example 9 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class CarbonInputFormat method getDataBlocksOfSegment.

/**
   * get data blocks of given segment
   */
private List<DataRefNode> getDataBlocksOfSegment(JobContext job, FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier absoluteTableIdentifier, FilterResolverIntf resolver, BitSet matchedPartitions, String segmentId, CacheClient cacheClient, SegmentUpdateStatusManager updateStatusManager) throws IOException {
    Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = null;
    try {
        QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
        QueryStatistic statistic = new QueryStatistic();
        segmentIndexMap = getSegmentAbstractIndexs(job, absoluteTableIdentifier, segmentId, cacheClient, updateStatusManager);
        List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
        if (null != segmentIndexMap) {
            for (Map.Entry<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> entry : segmentIndexMap.entrySet()) {
                SegmentTaskIndexStore.TaskBucketHolder taskHolder = entry.getKey();
                int taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskHolder.taskNo);
                // if this partition is not required, here will skip it.
                if (matchedPartitions == null || matchedPartitions.get(taskId)) {
                    AbstractIndex abstractIndex = entry.getValue();
                    List<DataRefNode> filterredBlocks;
                    // if no filter is given get all blocks from Btree Index
                    if (null == resolver) {
                        filterredBlocks = getDataBlocksOfIndex(abstractIndex);
                    } else {
                        // apply filter and get matching blocks
                        filterredBlocks = filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex, absoluteTableIdentifier);
                    }
                    resultFilterredBlocks.addAll(filterredBlocks);
                }
            }
        }
        statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
        recorder.recordStatisticsForDriver(statistic, job.getConfiguration().get("query.id"));
        return resultFilterredBlocks;
    } finally {
        // low memory systems the same memory can be utilized efficiently
        if (null != segmentIndexMap) {
            List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers = new ArrayList<>(1);
            tableSegmentUniqueIdentifiers.add(new TableSegmentUniqueIdentifier(absoluteTableIdentifier, segmentId));
            cacheClient.getSegmentAccessClient().clearAccessCount(tableSegmentUniqueIdentifiers);
        }
    }
}
Also used : DataRefNode(org.apache.carbondata.core.datastore.DataRefNode) TableSegmentUniqueIdentifier(org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier) AbstractIndex(org.apache.carbondata.core.datastore.block.AbstractIndex) QueryStatisticsRecorder(org.apache.carbondata.core.stats.QueryStatisticsRecorder) SegmentTaskIndexStore(org.apache.carbondata.core.datastore.SegmentTaskIndexStore) QueryStatistic(org.apache.carbondata.core.stats.QueryStatistic)

Example 10 with DataRefNode

use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1() {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
    ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
    buffer.rewind();
    buffer.putShort((short) 1);
    buffer.putInt(12);
    buffer.array();
    IndexKey key = new IndexKey(null, buffer.array());
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(1, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(1, findLastBlock.nodeNumber());
}
Also used : IndexKey(org.apache.carbondata.core.datastore.IndexKey) DataFileFooter(org.apache.carbondata.core.metadata.blocklet.DataFileFooter) BTreeBuilderInfo(org.apache.carbondata.core.datastore.BTreeBuilderInfo) DataRefNode(org.apache.carbondata.core.datastore.DataRefNode) DataRefNodeFinder(org.apache.carbondata.core.datastore.DataRefNodeFinder) ByteBuffer(java.nio.ByteBuffer) BtreeBuilder(org.apache.carbondata.core.datastore.BtreeBuilder) Test(org.junit.Test)

Aggregations

DataRefNode (org.apache.carbondata.core.datastore.DataRefNode)16 DataRefNodeFinder (org.apache.carbondata.core.datastore.DataRefNodeFinder)10 IndexKey (org.apache.carbondata.core.datastore.IndexKey)9 BTreeBuilderInfo (org.apache.carbondata.core.datastore.BTreeBuilderInfo)6 BtreeBuilder (org.apache.carbondata.core.datastore.BtreeBuilder)6 DataFileFooter (org.apache.carbondata.core.metadata.blocklet.DataFileFooter)6 Test (org.junit.Test)6 BTreeDataRefNodeFinder (org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder)5 LinkedList (java.util.LinkedList)3 KeyGenException (org.apache.carbondata.core.keygenerator.KeyGenException)3 KeyGenerator (org.apache.carbondata.core.keygenerator.KeyGenerator)3 MultiDimKeyVarLengthGenerator (org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator)3 ByteBuffer (java.nio.ByteBuffer)2 AbstractIndex (org.apache.carbondata.core.datastore.block.AbstractIndex)2 SegmentProperties (org.apache.carbondata.core.datastore.block.SegmentProperties)2 TableBlockInfo (org.apache.carbondata.core.datastore.block.TableBlockInfo)2 BlockBTreeLeafNode (org.apache.carbondata.core.datastore.impl.btree.BlockBTreeLeafNode)2 AbsoluteTableIdentifier (org.apache.carbondata.core.metadata.AbsoluteTableIdentifier)2 FilterExpressionProcessor (org.apache.carbondata.core.scan.filter.FilterExpressionProcessor)2 QueryStatistic (org.apache.carbondata.core.stats.QueryStatistic)2