Examples with BlockletIndexWrapper org.apache.carbondata.core.indexstore.BlockletIndexWrapper used on opensource projects

Example 6 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class BlockletIndexFactory method getCacheSize.

@Override
public String getCacheSize() {
    long sum = 0L;
    int numOfIndexFiles = 0;
    for (Map.Entry<String, SegmentBlockIndexInfo> entry : segmentMap.entrySet()) {
        for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : entry.getValue().getTableBlockIndexUniqueIdentifiers()) {
            BlockletIndexWrapper blockletIndexWrapper = cache.getIfPresent(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, getCarbonTable()));
            if (blockletIndexWrapper != null) {
                sum += blockletIndexWrapper.getMemorySize();
                numOfIndexFiles++;
            }
        }
    }
    return numOfIndexFiles + ":" + sum;
}

Example 7 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class BlockletIndexFactory method getAllUncached.

@Override
public List<IndexInputSplit> getAllUncached(List<Segment> validSegments, IndexExprWrapper indexExprWrapper) throws IOException {
    List<IndexInputSplit> distributableToBeLoaded = new ArrayList<>();
    for (Segment segment : validSegments) {
        IndexInputSplitWrapper indexInputSplitWrappers = indexExprWrapper.toDistributableSegment(segment);
        Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = getTableSegmentUniqueIdentifiers(segment);
        for (TableBlockIndexUniqueIdentifier identifier : tableBlockIndexUniqueIdentifiers) {
            BlockletIndexWrapper blockletIndexWrapper = cache.getIfPresent(new TableBlockIndexUniqueIdentifierWrapper(identifier, this.getCarbonTable()));
            if (identifier.getIndexFilePath() == null || blockletIndexWrapper == null) {
                ((BlockletIndexInputSplit) indexInputSplitWrappers.getDistributable()).setTableBlockIndexUniqueIdentifier(identifier);
                distributableToBeLoaded.add(indexInputSplitWrappers.getDistributable());
            }
        }
    }
    return distributableToBeLoaded;
}

Example 8 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

/**
 * Get the index for all segments
 */
public Map<Segment, List<CoarseGrainIndex>> getIndexes(List<Segment> segments, Set<Path> partitionLocations, IndexFilter filter) throws IOException {
    List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>();
    Map<Segment, List<CoarseGrainIndex>> indexMap = new HashMap<>();
    Map<String, Segment> segmentMap = new HashMap<>();
    for (Segment segment : segments) {
        segmentMap.put(segment.getSegmentNo(), segment);
        Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
        if (!partitionLocations.isEmpty()) {
            // get tableBlockIndexUniqueIdentifierWrappers from segment file info
            getTableBlockUniqueIdentifierWrappers(partitionLocations, tableBlockIndexUniqueIdentifierWrappers, identifiers);
        } else {
            SegmentMetaDataInfo segmentMetaDataInfo = segment.getSegmentMetaDataInfo();
            boolean isLoadAllIndex = Boolean.parseBoolean(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_LOAD_ALL_SEGMENT_INDEXES_TO_CACHE, CarbonCommonConstants.CARBON_LOAD_ALL_SEGMENT_INDEXES_TO_CACHE_DEFAULT));
            if (!isLoadAllIndex && null != segmentMetaDataInfo && null != filter && !filter.isEmpty() && null != filter.getExpression() && null == FilterUtil.getImplicitFilterExpression(filter.getExpression())) {
                getTableBlockIndexUniqueIdentifierUsingSegmentMinMax(segment, segmentMetaDataInfo, filter, identifiers, tableBlockIndexUniqueIdentifierWrappers);
            } else {
                for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
                    tableBlockIndexUniqueIdentifierWrappers.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
                }
            }
        }
    }
    List<BlockletIndexWrapper> blockletIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers);
    for (BlockletIndexWrapper wrapper : blockletIndexWrappers) {
        Segment segment = segmentMap.get(wrapper.getSegmentId());
        List<CoarseGrainIndex> indexes = indexMap.get(segment);
        if (null == indexes) {
            indexes = new ArrayList<CoarseGrainIndex>();
        }
        indexes.addAll(wrapper.getIndexes());
        indexMap.put(segment, indexes);
    }
    return indexMap;
}

Example 9 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

@Override
public List<CoarseGrainIndex> getIndexes(IndexInputSplit distributable) throws IOException {
    BlockletIndexInputSplit mapDistributable = (BlockletIndexInputSplit) distributable;
    List<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper;
    String segmentNo = mapDistributable.getSegment().getSegmentNo();
    if (mapDistributable.getSegmentPath() != null) {
        identifiersWrapper = getTableBlockIndexUniqueIdentifier(distributable);
    } else {
        identifiersWrapper = getTableBlockIndexUniqueIdentifier(mapDistributable.getFilePath(), segmentNo);
    }
    List<CoarseGrainIndex> indexes = new ArrayList<>();
    try {
        List<BlockletIndexWrapper> wrappers = cache.getAll(identifiersWrapper);
        for (BlockletIndexWrapper wrapper : wrappers) {
            indexes.addAll(wrapper.getIndexes());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return indexes;
}