Examples with IndexBuilderException org.apache.carbondata.core.datastore.exception.IndexBuilderException used on opensource projects

Example 1 with IndexBuilderException

use of org.apache.carbondata.core.datastore.exception.IndexBuilderException in project carbondata by apache.

the class SegmentTaskIndexStore method loadAndGetTaskIdToSegmentsMap.

/**
   * Below method will be used to load the segment of segments
   * One segment may have multiple task , so  table segment will be loaded
   * based on task id and will return the map of taksId to table segment
   * map
   *
   * @param segmentToTableBlocksInfos segment id to block info
   * @param absoluteTableIdentifier   absolute table identifier
   * @return map of taks id to segment mapping
   * @throws IOException
   */
private SegmentTaskIndexWrapper loadAndGetTaskIdToSegmentsMap(Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos, AbsoluteTableIdentifier absoluteTableIdentifier, TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier) throws IOException {
    // task id to segment map
    Iterator<Map.Entry<String, List<TableBlockInfo>>> iteratorOverSegmentBlocksInfos = segmentToTableBlocksInfos.entrySet().iterator();
    Map<TaskBucketHolder, AbstractIndex> taskIdToSegmentIndexMap = null;
    SegmentTaskIndexWrapper segmentTaskIndexWrapper = null;
    SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(absoluteTableIdentifier);
    String segmentId = null;
    TaskBucketHolder taskBucketHolder = null;
    try {
        while (iteratorOverSegmentBlocksInfos.hasNext()) {
            // segment id to table block mapping
            Map.Entry<String, List<TableBlockInfo>> next = iteratorOverSegmentBlocksInfos.next();
            // group task id to table block info mapping for the segment
            Map<TaskBucketHolder, List<TableBlockInfo>> taskIdToTableBlockInfoMap = mappedAndGetTaskIdToTableBlockInfo(segmentToTableBlocksInfos);
            segmentId = next.getKey();
            // get the existing map of task id to table segment map
            UpdateVO updateVO = updateStatusManager.getInvalidTimestampRange(segmentId);
            // check if segment is already loaded, if segment is already loaded
            //no need to load the segment block
            String lruCacheKey = tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier();
            segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache.get(lruCacheKey);
            if (segmentTaskIndexWrapper == null || tableSegmentUniqueIdentifier.isSegmentUpdated()) {
                // get the segment loader lock object this is to avoid
                // same segment is getting loaded multiple times
                // in case of concurrent query
                Object segmentLoderLockObject = segmentLockMap.get(lruCacheKey);
                if (null == segmentLoderLockObject) {
                    segmentLoderLockObject = addAndGetSegmentLock(lruCacheKey);
                }
                // acquire lock to lod the segment
                synchronized (segmentLoderLockObject) {
                    segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache.get(lruCacheKey);
                    if (null == segmentTaskIndexWrapper || tableSegmentUniqueIdentifier.isSegmentUpdated()) {
                        // so that the same can be updated after loading the btree.
                        if (tableSegmentUniqueIdentifier.isSegmentUpdated() && null != segmentTaskIndexWrapper) {
                            taskIdToSegmentIndexMap = segmentTaskIndexWrapper.getTaskIdToTableSegmentMap();
                        } else {
                            // creating a map of take if to table segment
                            taskIdToSegmentIndexMap = new HashMap<TaskBucketHolder, AbstractIndex>();
                            segmentTaskIndexWrapper = new SegmentTaskIndexWrapper(taskIdToSegmentIndexMap);
                            segmentTaskIndexWrapper.incrementAccessCount();
                        }
                        Iterator<Map.Entry<TaskBucketHolder, List<TableBlockInfo>>> iterator = taskIdToTableBlockInfoMap.entrySet().iterator();
                        long requiredSize = calculateRequiredSize(taskIdToTableBlockInfoMap, absoluteTableIdentifier);
                        segmentTaskIndexWrapper.setMemorySize(requiredSize + segmentTaskIndexWrapper.getMemorySize());
                        boolean isAddedToLruCache = lruCache.put(lruCacheKey, segmentTaskIndexWrapper, requiredSize);
                        if (isAddedToLruCache) {
                            while (iterator.hasNext()) {
                                Map.Entry<TaskBucketHolder, List<TableBlockInfo>> taskToBlockInfoList = iterator.next();
                                taskBucketHolder = taskToBlockInfoList.getKey();
                                taskIdToSegmentIndexMap.put(taskBucketHolder, loadBlocks(taskBucketHolder, taskToBlockInfoList.getValue(), absoluteTableIdentifier));
                            }
                        } else {
                            throw new IndexBuilderException("Can not load the segment. No Enough space available.");
                        }
                        // set the latest timestamp.
                        segmentTaskIndexWrapper.setRefreshedTimeStamp(updateVO.getCreatedOrUpdatedTimeStamp());
                        // tableSegmentMapTemp.put(next.getKey(), taskIdToSegmentIndexMap);
                        // removing from segment lock map as once segment is loaded
                        // if concurrent query is coming for same segment
                        // it will wait on the lock so after this segment will be already
                        // loaded so lock is not required, that is why removing the
                        // the lock object as it wont be useful
                        segmentLockMap.remove(lruCacheKey);
                    } else {
                        segmentTaskIndexWrapper.incrementAccessCount();
                    }
                }
            } else {
                segmentTaskIndexWrapper.incrementAccessCount();
            }
        }
    } catch (IndexBuilderException e) {
        LOGGER.error("Problem while loading the segment");
        throw e;
    }
    return segmentTaskIndexWrapper;
}

Example 2 with IndexBuilderException

use of org.apache.carbondata.core.datastore.exception.IndexBuilderException in project carbondata by apache.

the class BlockIndexStore method getAll.

/**
 * The method takes list of tableblocks as input and load them in btree lru cache
 * and returns the list of data blocks meta
 *
 * @param tableBlocksInfos List of unique table blocks
 * @return List<AbstractIndex>
 * @throws IndexBuilderException
 */
@Override
public List<AbstractIndex> getAll(List<TableBlockUniqueIdentifier> tableBlocksInfos) throws IndexBuilderException {
    AbstractIndex[] loadedBlock = new AbstractIndex[tableBlocksInfos.size()];
    int numberOfCores = 1;
    try {
        numberOfCores = Integer.parseInt(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.NUM_CORES, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
    } catch (NumberFormatException e) {
        numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
    }
    ExecutorService executor = Executors.newFixedThreadPool(numberOfCores);
    List<Future<AbstractIndex>> blocksList = new ArrayList<Future<AbstractIndex>>();
    for (TableBlockUniqueIdentifier tableBlockUniqueIdentifier : tableBlocksInfos) {
        blocksList.add(executor.submit(new BlockLoaderThread(tableBlockUniqueIdentifier)));
    }
    // shutdown the executor gracefully and wait until all the task is finished
    executor.shutdown();
    try {
        executor.awaitTermination(1, TimeUnit.HOURS);
    } catch (InterruptedException e) {
        throw new IndexBuilderException(e);
    }
    // fill the block which were not loaded before to loaded blocks array
    fillLoadedBlocks(loadedBlock, blocksList);
    return Arrays.asList(loadedBlock);
}

Example 3 with IndexBuilderException

use of org.apache.carbondata.core.datastore.exception.IndexBuilderException in project carbondata by apache.

the class SegmentTaskIndexStore method loadAndGetTaskIdToSegmentsMap.

/**
 * Below method will be used to load the segment of segments
 * One segment may have multiple task , so  table segment will be loaded
 * based on task id and will return the map of taksId to table segment
 * map
 *
 * @param segmentToTableBlocksInfos segment id to block info
 * @param table   table handle
 * @return map of taks id to segment mapping
 * @throws IOException
 */
private SegmentTaskIndexWrapper loadAndGetTaskIdToSegmentsMap(Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos, CarbonTable table, TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier) throws IOException {
    // task id to segment map
    Iterator<Map.Entry<String, List<TableBlockInfo>>> iteratorOverSegmentBlocksInfos = segmentToTableBlocksInfos.entrySet().iterator();
    Map<TaskBucketHolder, AbstractIndex> taskIdToSegmentIndexMap = null;
    SegmentTaskIndexWrapper segmentTaskIndexWrapper = null;
    SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(table);
    String segmentId = null;
    TaskBucketHolder taskBucketHolder = null;
    try {
        while (iteratorOverSegmentBlocksInfos.hasNext()) {
            // Initialize the UpdateVO to Null for each segment.
            UpdateVO updateVO = null;
            // segment id to table block mapping
            Map.Entry<String, List<TableBlockInfo>> next = iteratorOverSegmentBlocksInfos.next();
            // group task id to table block info mapping for the segment
            Map<TaskBucketHolder, List<TableBlockInfo>> taskIdToTableBlockInfoMap = mappedAndGetTaskIdToTableBlockInfo(segmentToTableBlocksInfos);
            segmentId = next.getKey();
            // updateVO is only required when Updates Or Delete performed on the Table.
            if (updateStatusManager.getUpdateStatusDetails().length != 0) {
                // get the existing map of task id to table segment map
                updateVO = updateStatusManager.getInvalidTimestampRange(segmentId);
            }
            // check if segment is already loaded, if segment is already loaded
            // no need to load the segment block
            String lruCacheKey = tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier();
            segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache.get(lruCacheKey);
            if ((segmentTaskIndexWrapper == null) || ((null != updateVO) && (tableSegmentUniqueIdentifier.isSegmentUpdated()))) {
                // get the segment loader lock object this is to avoid
                // same segment is getting loaded multiple times
                // in case of concurrent query
                Object segmentLoderLockObject = segmentLockMap.get(lruCacheKey);
                if (null == segmentLoderLockObject) {
                    segmentLoderLockObject = addAndGetSegmentLock(lruCacheKey);
                }
                // acquire lock to lod the segment
                synchronized (segmentLoderLockObject) {
                    segmentTaskIndexWrapper = (SegmentTaskIndexWrapper) lruCache.get(lruCacheKey);
                    if ((null == segmentTaskIndexWrapper) || ((null != updateVO) && (tableSegmentUniqueIdentifier.isSegmentUpdated()))) {
                        // so that the same can be updated after loading the btree.
                        if (tableSegmentUniqueIdentifier.isSegmentUpdated() && null != segmentTaskIndexWrapper) {
                            taskIdToSegmentIndexMap = segmentTaskIndexWrapper.getTaskIdToTableSegmentMap();
                        } else {
                            // creating a map of take if to table segment
                            taskIdToSegmentIndexMap = new HashMap<TaskBucketHolder, AbstractIndex>();
                            segmentTaskIndexWrapper = new SegmentTaskIndexWrapper(taskIdToSegmentIndexMap);
                            segmentTaskIndexWrapper.incrementAccessCount();
                        }
                        Iterator<Map.Entry<TaskBucketHolder, List<TableBlockInfo>>> iterator = taskIdToTableBlockInfoMap.entrySet().iterator();
                        AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
                        long requiredSize = calculateRequiredSize(taskIdToTableBlockInfoMap, absoluteTableIdentifier);
                        segmentTaskIndexWrapper.setMemorySize(requiredSize);
                        boolean canAddToLruCache = lruCache.tryPut(lruCacheKey, requiredSize);
                        if (canAddToLruCache) {
                            while (iterator.hasNext()) {
                                Map.Entry<TaskBucketHolder, List<TableBlockInfo>> taskToBlockInfoList = iterator.next();
                                taskBucketHolder = taskToBlockInfoList.getKey();
                                taskIdToSegmentIndexMap.put(taskBucketHolder, loadBlocks(taskBucketHolder, taskToBlockInfoList.getValue(), absoluteTableIdentifier));
                            }
                            long updatedRequiredSize = ObjectSizeCalculator.estimate(segmentTaskIndexWrapper, requiredSize);
                            // update the actual size of object
                            segmentTaskIndexWrapper.setMemorySize(updatedRequiredSize);
                            if (!lruCache.put(lruCacheKey, segmentTaskIndexWrapper, updatedRequiredSize)) {
                                throw new IndexBuilderException("Can not load the segment. No Enough space available.");
                            }
                        } else {
                            throw new IndexBuilderException("Can not load the segment. No Enough space available.");
                        }
                        // Refresh the Timestamp for those tables which underwent through IUD Operations.
                        if (null != updateVO) {
                            // set the latest timestamp.
                            segmentTaskIndexWrapper.setRefreshedTimeStamp(updateVO.getCreatedOrUpdatedTimeStamp());
                        } else {
                            segmentTaskIndexWrapper.setRefreshedTimeStamp(0L);
                        }
                        // tableSegmentMapTemp.put(next.getKey(), taskIdToSegmentIndexMap);
                        // removing from segment lock map as once segment is loaded
                        // if concurrent query is coming for same segment
                        // it will wait on the lock so after this segment will be already
                        // loaded so lock is not required, that is why removing the
                        // the lock object as it wont be useful
                        segmentLockMap.remove(lruCacheKey);
                    } else {
                        segmentTaskIndexWrapper.incrementAccessCount();
                    }
                }
            } else {
                segmentTaskIndexWrapper.incrementAccessCount();
            }
        }
    } catch (IndexBuilderException e) {
        LOGGER.error("Problem while loading the segment");
        throw e;
    }
    return segmentTaskIndexWrapper;
}

Example 4 with IndexBuilderException

use of org.apache.carbondata.core.datastore.exception.IndexBuilderException in project carbondata by apache.

the class AbstractBlockIndexStoreCache method checkAndLoadTableBlocks.

/**
 * This method will get the value for the given key. If value does not exist
 * for the given key, it will check and load the value.
 *
 * @param tableBlock
 * @param tableBlockUniqueIdentifier
 * @param lruCacheKey
 */
protected void checkAndLoadTableBlocks(AbstractIndex tableBlock, TableBlockUniqueIdentifier tableBlockUniqueIdentifier, String lruCacheKey) throws IOException {
    // calculate the required size is
    TableBlockInfo blockInfo = tableBlockUniqueIdentifier.getTableBlockInfo();
    long requiredMetaSize = CarbonUtil.calculateMetaSize(blockInfo);
    if (requiredMetaSize > 0) {
        // load table blocks data
        // getting the data file meta data of the block
        DataFileFooter footer = CarbonUtil.readMetadatFile(blockInfo);
        footer.setBlockInfo(new BlockInfo(blockInfo));
        // building the block
        tableBlock.buildIndex(Collections.singletonList(footer));
        requiredMetaSize = ObjectSizeCalculator.estimate(blockInfo, requiredMetaSize);
        tableBlock.setMemorySize(requiredMetaSize);
        tableBlock.incrementAccessCount();
        boolean isTableBlockAddedToLruCache = lruCache.put(lruCacheKey, tableBlock, requiredMetaSize);
        if (!isTableBlockAddedToLruCache) {
            throw new IndexBuilderException("Cannot load table blocks into memory. Not enough memory available");
        }
    } else {
        throw new IndexBuilderException("Invalid carbon data file: " + blockInfo.getFilePath());
    }
}