Examples with AbstractIndex org.apache.carbondata.core.datastore.block.AbstractIndex used on opensource projects

Example 6 with AbstractIndex

use of org.apache.carbondata.core.datastore.block.AbstractIndex in project carbondata by apache.

the class BlockIndexStore method getAll.

/**
   * The method takes list of tableblocks as input and load them in btree lru cache
   * and returns the list of data blocks meta
   *
   * @param tableBlocksInfos List of unique table blocks
   * @return List<AbstractIndex>
   * @throws IndexBuilderException
   */
@Override
public List<AbstractIndex> getAll(List<TableBlockUniqueIdentifier> tableBlocksInfos) throws IndexBuilderException {
    AbstractIndex[] loadedBlock = new AbstractIndex[tableBlocksInfos.size()];
    int numberOfCores = 1;
    try {
        numberOfCores = Integer.parseInt(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.NUM_CORES, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
    } catch (NumberFormatException e) {
        numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
    }
    ExecutorService executor = Executors.newFixedThreadPool(numberOfCores);
    List<Future<AbstractIndex>> blocksList = new ArrayList<Future<AbstractIndex>>();
    for (TableBlockUniqueIdentifier tableBlockUniqueIdentifier : tableBlocksInfos) {
        blocksList.add(executor.submit(new BlockLoaderThread(tableBlockUniqueIdentifier)));
    }
    // shutdown the executor gracefully and wait until all the task is finished
    executor.shutdown();
    try {
        executor.awaitTermination(1, TimeUnit.HOURS);
    } catch (InterruptedException e) {
        throw new IndexBuilderException(e);
    }
    // fill the block which were not loaded before to loaded blocks array
    fillLoadedBlocks(loadedBlock, blocksList);
    return Arrays.asList(loadedBlock);
}

Example 7 with AbstractIndex

use of org.apache.carbondata.core.datastore.block.AbstractIndex in project carbondata by apache.

the class BlockIndexStore method loadBlock.

private AbstractIndex loadBlock(TableBlockUniqueIdentifier tableBlockUniqueIdentifier) throws IOException {
    AbstractIndex tableBlock = new BlockIndex();
    BlockInfo blockInfo = new BlockInfo(tableBlockUniqueIdentifier.getTableBlockInfo());
    String lruCacheKey = getLruCacheKey(tableBlockUniqueIdentifier.getAbsoluteTableIdentifier(), blockInfo);
    checkAndLoadTableBlocks(tableBlock, tableBlockUniqueIdentifier, lruCacheKey);
    // finally remove the lock object from block info lock as once block is loaded
    // it will not come inside this if condition
    blockInfoLock.remove(blockInfo);
    return tableBlock;
}

Example 8 with AbstractIndex

use of org.apache.carbondata.core.datastore.block.AbstractIndex in project carbondata by apache.

the class BlockIndexStoreTest method testLoadAndGetTaskIdToSegmentsMapForSingleSegment.

@Test
public void testLoadAndGetTaskIdToSegmentsMapForSingleSegment() throws IOException {
    File file = getPartFile();
    TableBlockInfo info = new TableBlockInfo(file.getAbsolutePath(), 0, "0", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "t3", "1");
    AbsoluteTableIdentifier absoluteTableIdentifier = new AbsoluteTableIdentifier("/src/test/resources", carbonTableIdentifier);
    try {
        List<TableBlockUniqueIdentifier> tableBlockInfoList = getTableBlockUniqueIdentifierList(Arrays.asList(new TableBlockInfo[] { info }), absoluteTableIdentifier);
        List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockInfoList);
        assertTrue(loadAndGetBlocks.size() == 1);
    } catch (Exception e) {
        assertTrue(false);
    }
    List<String> segmentIds = new ArrayList<>();
    segmentIds.add(info.getSegmentId());
    cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
}

Example 9 with AbstractIndex

use of org.apache.carbondata.core.datastore.block.AbstractIndex in project carbondata by apache.

the class CarbonInputFormat method getDataBlocksOfSegment.

/**
   * get data blocks of given segment
   */
private List<DataRefNode> getDataBlocksOfSegment(JobContext job, FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier absoluteTableIdentifier, FilterResolverIntf resolver, BitSet matchedPartitions, String segmentId, CacheClient cacheClient, SegmentUpdateStatusManager updateStatusManager) throws IOException {
    Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = null;
    try {
        QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
        QueryStatistic statistic = new QueryStatistic();
        segmentIndexMap = getSegmentAbstractIndexs(job, absoluteTableIdentifier, segmentId, cacheClient, updateStatusManager);
        List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
        if (null != segmentIndexMap) {
            for (Map.Entry<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> entry : segmentIndexMap.entrySet()) {
                SegmentTaskIndexStore.TaskBucketHolder taskHolder = entry.getKey();
                int taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(taskHolder.taskNo);
                // if this partition is not required, here will skip it.
                if (matchedPartitions == null || matchedPartitions.get(taskId)) {
                    AbstractIndex abstractIndex = entry.getValue();
                    List<DataRefNode> filterredBlocks;
                    // if no filter is given get all blocks from Btree Index
                    if (null == resolver) {
                        filterredBlocks = getDataBlocksOfIndex(abstractIndex);
                    } else {
                        // apply filter and get matching blocks
                        filterredBlocks = filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex, absoluteTableIdentifier);
                    }
                    resultFilterredBlocks.addAll(filterredBlocks);
                }
            }
        }
        statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
        recorder.recordStatisticsForDriver(statistic, job.getConfiguration().get("query.id"));
        return resultFilterredBlocks;
    } finally {
        // low memory systems the same memory can be utilized efficiently
        if (null != segmentIndexMap) {
            List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers = new ArrayList<>(1);
            tableSegmentUniqueIdentifiers.add(new TableSegmentUniqueIdentifier(absoluteTableIdentifier, segmentId));
            cacheClient.getSegmentAccessClient().clearAccessCount(tableSegmentUniqueIdentifiers);
        }
    }
}

Example 10 with AbstractIndex

use of org.apache.carbondata.core.datastore.block.AbstractIndex in project carbondata by apache.

the class InMemoryBTreeIndex method getDataBlocksOfSegment.

/**
   * get data blocks of given segment
   */
private List<DataRefNode> getDataBlocksOfSegment(JobContext job, FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier identifier, FilterResolverIntf resolver) throws IOException {
    QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
    QueryStatistic statistic = new QueryStatistic();
    Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = getSegmentAbstractIndexs(job, identifier);
    List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
    // build result
    for (AbstractIndex abstractIndex : segmentIndexMap.values()) {
        List<DataRefNode> filterredBlocks = null;
        // if no filter is given get all blocks from Btree Index
        if (null == resolver) {
            filterredBlocks = getDataBlocksOfIndex(abstractIndex);
        } else {
            // apply filter and get matching blocks
            filterredBlocks = filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(), resolver, abstractIndex, identifier);
        }
        resultFilterredBlocks.addAll(filterredBlocks);
    }
    statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
    recorder.recordStatistics(statistic);
    recorder.logStatistics();
    return resultFilterredBlocks;
}