Examples with BlockIndex org.apache.carbondata.core.indexstore.blockletindex.BlockIndex used on opensource projects

Example 1 with BlockIndex

use of org.apache.carbondata.core.indexstore.blockletindex.BlockIndex in project carbondata by apache.

the class BlockletIndexStore method put.

@Override
public void put(TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper, BlockletIndexWrapper wrapper) throws IOException {
    // and then use the put interface
    if (null == getIfPresent(tableBlockIndexUniqueIdentifierWrapper)) {
        List<BlockIndex> indexes = wrapper.getIndexes();
        try {
            for (BlockIndex blockIndex : indexes) {
                blockIndex.convertToUnsafeDMStore();
            }
            // get cacheExpirationTime for table from tableProperties
            long expirationTime = CarbonUtil.getExpiration_time(tableBlockIndexUniqueIdentifierWrapper.getCarbonTable());
            // Locking is not required here because in LRU cache map add method is synchronized to add
            // only one entry at a time and if a key already exists it will not overwrite the entry
            lruCache.put(tableBlockIndexUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier().getUniqueTableSegmentIdentifier(), wrapper, wrapper.getMemorySize(), expirationTime);
        } catch (Throwable e) {
            // clear all the memory acquired by index in case of any failure
            for (Index blockletIndex : indexes) {
                blockletIndex.clear();
            }
            throw new IOException("Problem in adding index to cache.", e);
        }
    }
}

Example 2 with BlockIndex

use of org.apache.carbondata.core.indexstore.blockletindex.BlockIndex in project carbondata by apache.

the class BlockletIndexStore method get.

public BlockletIndexWrapper get(TableBlockIndexUniqueIdentifierWrapper identifierWrapper, Map<String, Map<String, BlockMetaInfo>> segInfoCache) {
    TableBlockIndexUniqueIdentifier identifier = identifierWrapper.getTableBlockIndexUniqueIdentifier();
    String lruCacheKey = identifier.getUniqueTableSegmentIdentifier();
    BlockletIndexWrapper blockletIndexWrapper = (BlockletIndexWrapper) lruCache.get(lruCacheKey);
    List<BlockIndex> indexes = new ArrayList<>();
    if (blockletIndexWrapper == null) {
        try {
            SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore(identifierWrapper.getConfiguration());
            Set<String> filesRead = new HashSet<>();
            String segmentFilePath = identifier.getIndexFilePath();
            if (segInfoCache == null) {
                segInfoCache = new HashMap<>();
            }
            Map<String, BlockMetaInfo> carbonDataFileBlockMetaInfoMapping = segInfoCache.get(segmentFilePath);
            if (carbonDataFileBlockMetaInfoMapping == null) {
                carbonDataFileBlockMetaInfoMapping = BlockletIndexUtil.createCarbonDataFileBlockMetaInfoMapping(segmentFilePath, identifierWrapper.getConfiguration());
                segInfoCache.put(segmentFilePath, carbonDataFileBlockMetaInfoMapping);
            }
            // if the identifier is not a merge file we can directly load the indexes
            if (identifier.getMergeIndexFileName() == null) {
                List<DataFileFooter> indexInfos = new ArrayList<>();
                Map<String, BlockMetaInfo> blockMetaInfoMap = BlockletIndexUtil.getBlockMetaInfoMap(identifierWrapper, indexFileStore, filesRead, carbonDataFileBlockMetaInfoMapping, indexInfos);
                BlockIndex blockIndex = loadAndGetIndex(identifier, indexFileStore, blockMetaInfoMap, identifierWrapper.getCarbonTable(), identifierWrapper.isAddToUnsafe(), identifierWrapper.getConfiguration(), identifierWrapper.isSerializeDmStore(), indexInfos);
                indexes.add(blockIndex);
                blockletIndexWrapper = new BlockletIndexWrapper(identifier.getSegmentId(), indexes);
            } else {
                // if the identifier is a merge file then collect the index files and load the indexes
                List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = BlockletIndexUtil.getIndexFileIdentifiersFromMergeFile(identifier, indexFileStore);
                for (TableBlockIndexUniqueIdentifier blockIndexUniqueIdentifier : tableBlockIndexUniqueIdentifiers) {
                    List<DataFileFooter> indexInfos = new ArrayList<>();
                    Map<String, BlockMetaInfo> blockMetaInfoMap = BlockletIndexUtil.getBlockMetaInfoMap(new TableBlockIndexUniqueIdentifierWrapper(blockIndexUniqueIdentifier, identifierWrapper.getCarbonTable()), indexFileStore, filesRead, carbonDataFileBlockMetaInfoMapping, indexInfos);
                    if (!blockMetaInfoMap.isEmpty()) {
                        BlockIndex blockIndex = loadAndGetIndex(blockIndexUniqueIdentifier, indexFileStore, blockMetaInfoMap, identifierWrapper.getCarbonTable(), identifierWrapper.isAddToUnsafe(), identifierWrapper.getConfiguration(), identifierWrapper.isSerializeDmStore(), indexInfos);
                        indexes.add(blockIndex);
                    }
                }
                blockletIndexWrapper = new BlockletIndexWrapper(identifier.getSegmentId(), indexes);
            }
            if (identifierWrapper.isAddTableBlockToUnsafeAndLRUCache()) {
                long expiration_time = CarbonUtil.getExpiration_time(identifierWrapper.getCarbonTable());
                lruCache.put(identifier.getUniqueTableSegmentIdentifier(), blockletIndexWrapper, blockletIndexWrapper.getMemorySize(), expiration_time);
            }
        } catch (Throwable e) {
            // clear all the memory used by indexes loaded
            for (Index index : indexes) {
                index.clear();
            }
            LOGGER.error("memory exception when loading index: " + e.getMessage(), e);
            throw new RuntimeException(e);
        }
    }
    return blockletIndexWrapper;
}

Example 3 with BlockIndex

use of org.apache.carbondata.core.indexstore.blockletindex.BlockIndex in project carbondata by apache.

the class BlockletIndexStore method getAll.

@Override
public List<BlockletIndexWrapper> getAll(List<TableBlockIndexUniqueIdentifierWrapper> tableSegmentUniqueIdentifiers) throws IOException {
    Map<String, Map<String, BlockMetaInfo>> segInfoCache = new HashMap<String, Map<String, BlockMetaInfo>>();
    List<BlockletIndexWrapper> blockletIndexWrappers = new ArrayList<>(tableSegmentUniqueIdentifiers.size());
    List<TableBlockIndexUniqueIdentifierWrapper> missedIdentifiersWrapper = new ArrayList<>();
    BlockletIndexWrapper blockletIndexWrapper = null;
    // Get the indexes for each index file from cache.
    try {
        for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : tableSegmentUniqueIdentifiers) {
            BlockletIndexWrapper indexWrapper = getIfPresent(identifierWrapper);
            if (indexWrapper != null) {
                blockletIndexWrappers.add(indexWrapper);
            } else {
                missedIdentifiersWrapper.add(identifierWrapper);
            }
        }
        if (missedIdentifiersWrapper.size() > 0) {
            for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : missedIdentifiersWrapper) {
                blockletIndexWrapper = get(identifierWrapper, segInfoCache);
                blockletIndexWrappers.add(blockletIndexWrapper);
            }
        }
    } catch (Throwable e) {
        if (null != blockletIndexWrapper) {
            List<BlockIndex> indexes = blockletIndexWrapper.getIndexes();
            for (Index index : indexes) {
                index.clear();
            }
        }
        throw new IOException("Problem in loading segment blocks: " + e.getMessage(), e);
    }
    return blockletIndexWrappers;
}

Example 4 with BlockIndex

use of org.apache.carbondata.core.indexstore.blockletindex.BlockIndex in project carbondata by apache.

the class TableIndex method pruneWithFilter.

private List<ExtendedBlocklet> pruneWithFilter(List<Segment> segments, IndexFilter filter, Set<Path> partitionLocations, List<ExtendedBlocklet> blocklets, Map<Segment, List<Index>> indexes) throws IOException {
    Set<String> missingSISegments = filter.getMissingSISegments();
    for (Segment segment : segments) {
        List<Index> segmentIndices = indexes.get(segment);
        if (segment == null || segmentIndices == null || segmentIndices.isEmpty()) {
            continue;
        }
        boolean isExternalOrMissingSISegment = segment.isExternalSegment() || (missingSISegments != null && missingSISegments.contains(segment.getSegmentNo()));
        List<Blocklet> pruneBlocklets = new ArrayList<>();
        SegmentProperties segmentProperties;
        if (segmentIndices.get(0) instanceof BlockIndex) {
            segmentProperties = segmentPropertiesFetcher.getSegmentPropertiesFromIndex(segmentIndices.get(0));
        } else {
            segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment, partitionLocations);
        }
        if (filter.isResolvedOnSegment(segmentProperties)) {
            FilterExecutor filterExecutor;
            if (!isExternalOrMissingSISegment) {
                filterExecutor = FilterUtil.getFilterExecutorTree(filter.getResolver(), segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
            } else {
                filterExecutor = FilterUtil.getFilterExecutorTree(filter.getExternalSegmentResolver(), segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
            }
            for (Index index : segmentIndices) {
                if (!isExternalOrMissingSISegment) {
                    pruneBlocklets.addAll(index.prune(filter.getResolver(), segmentProperties, filterExecutor, this.table));
                } else {
                    pruneBlocklets.addAll(index.prune(filter.getExternalSegmentResolver(), segmentProperties, filterExecutor, this.table));
                }
            }
        } else {
            FilterExecutor filterExecutor;
            Expression expression = filter.getExpression();
            if (!isExternalOrMissingSISegment) {
                filterExecutor = FilterUtil.getFilterExecutorTree(new IndexFilter(segmentProperties, table, expression).getResolver(), segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
            } else {
                filterExecutor = FilterUtil.getFilterExecutorTree(new IndexFilter(segmentProperties, table, expression).getExternalSegmentResolver(), segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
            }
            for (Index index : segmentIndices) {
                if (!isExternalOrMissingSISegment) {
                    pruneBlocklets.addAll(index.prune(filter.getExpression(), segmentProperties, table, filterExecutor));
                } else {
                    pruneBlocklets.addAll(index.prune(filter.getExternalSegmentFilter(), segmentProperties, table, filterExecutor));
                }
            }
        }
        blocklets.addAll(addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment));
    }
    return blocklets;
}

Example 5 with BlockIndex

use of org.apache.carbondata.core.indexstore.blockletindex.BlockIndex in project carbondata by apache.

the class BlockletIndexStore method loadAndGetIndex.

/**
 * Below method will be used to load the segment of segments
 * One segment may have multiple task , so  table segment will be loaded
 * based on task id and will return the map of taskId to table segment
 * map
 *
 * @return map of task id to segment mapping
 * @throws IOException
 */
private BlockIndex loadAndGetIndex(TableBlockIndexUniqueIdentifier identifier, SegmentIndexFileStore indexFileStore, Map<String, BlockMetaInfo> blockMetaInfoMap, CarbonTable carbonTable, boolean addTableBlockToUnsafe, Configuration configuration, boolean serializeDmStore, List<DataFileFooter> indexInfos) throws IOException {
    BlockIndex blockIndex = (BlockIndex) BlockletIndexFactory.createIndex(carbonTable);
    final BlockletIndexModel blockletIndexModel = new BlockletIndexModel(carbonTable, identifier.getIndexFilePath() + CarbonCommonConstants.FILE_SEPARATOR + identifier.getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()), blockMetaInfoMap, identifier.getSegmentId(), addTableBlockToUnsafe, configuration, serializeDmStore);
    blockletIndexModel.setIndexInfos(indexInfos);
    blockIndex.init(blockletIndexModel);
    return blockIndex;
}