use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.
the class BTreeBlockFinderTest method testBtreeBuilderGetMethodIsGivingNotNullRootNode.
@Test
public void testBtreeBuilderGetMethodIsGivingNotNullRootNode() {
BtreeBuilder builder = new BlockBTreeBuilder();
List<DataFileFooter> footerList = getDataFileFooterList();
BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
}
use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.
the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2.
@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey2() throws KeyGenException {
BtreeBuilder builder = new BlockBTreeBuilder();
List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 0, 0 }), null);
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
assertEquals(0, findFirstBlock.nodeNumber());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
assertEquals(0, findLastBlock.nodeNumber());
}
use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.
the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary.
@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary() {
BtreeBuilder builder = new BlockBTreeBuilder();
List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey();
BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
buffer.rewind();
buffer.put((byte) 1);
buffer.putInt(0);
buffer.array();
IndexKey key = new IndexKey(null, buffer.array());
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
assertEquals(0, findFirstBlock.nodeNumber());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
assertEquals(0, findLastBlock.nodeNumber());
}
use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.
the class CarbonInputFormat method getDataBlocksOfSegment.
/**
* get data blocks of given segment
*/
private List<DataRefNode> getDataBlocksOfSegment(JobContext job, FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier absoluteTableIdentifier, FilterResolverIntf resolver, BitSet matchedPartitions, String segmentId, CacheClient cacheClient, SegmentUpdateStatusManager updateStatusManager) throws IOException {
Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = null;
try {
QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
QueryStatistic statistic = new QueryStatistic();
segmentIndexMap = getSegmentAbstractIndexs(job, absoluteTableIdentifier, segmentId, cacheClient, updateStatusManager);
List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
if (null != segmentIndexMap) {
for (Map.Entry<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> entry : segmentIndexMap.entrySet()) {
SegmentTaskIndexStore.TaskBucketHolder taskHolder = entry.getKey();
int taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskHolder.taskNo);
// if this partition is not required, here will skip it.
if (matchedPartitions == null || matchedPartitions.get(taskId)) {
AbstractIndex abstractIndex = entry.getValue();
List<DataRefNode> filterredBlocks;
// if no filter is given get all blocks from Btree Index
if (null == resolver) {
filterredBlocks = getDataBlocksOfIndex(abstractIndex);
} else {
// apply filter and get matching blocks
filterredBlocks = filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex, absoluteTableIdentifier);
}
resultFilterredBlocks.addAll(filterredBlocks);
}
}
}
statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
recorder.recordStatisticsForDriver(statistic, job.getConfiguration().get("query.id"));
return resultFilterredBlocks;
} finally {
// low memory systems the same memory can be utilized efficiently
if (null != segmentIndexMap) {
List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers = new ArrayList<>(1);
tableSegmentUniqueIdentifiers.add(new TableSegmentUniqueIdentifier(absoluteTableIdentifier, segmentId));
cacheClient.getSegmentAccessClient().clearAccessCount(tableSegmentUniqueIdentifiers);
}
}
}
use of org.apache.carbondata.core.datastore.DataRefNode in project carbondata by apache.
the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1.
@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1() {
BtreeBuilder builder = new BlockBTreeBuilder();
List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey();
BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
buffer.rewind();
buffer.putShort((short) 1);
buffer.putInt(12);
buffer.array();
IndexKey key = new IndexKey(null, buffer.array());
DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
assertEquals(1, findFirstBlock.nodeNumber());
DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
assertEquals(1, findLastBlock.nodeNumber());
}
Aggregations