Examples with CoarseGrainIndex org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex used on opensource projects

Example 6 with CoarseGrainIndex

use of org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex in project carbondata by apache.

the class TableIndex method getRowCount.

/**
 * Prune the index of the given segments and return the Map of blocklet path and row count
 *
 * @param allSegments
 * @param partitions
 * @return
 * @throws IOException
 */
public long getRowCount(List<Segment> allSegments, final List<PartitionSpec> partitions, TableIndex defaultIndex) throws IOException {
    List<Segment> segments = getCarbonSegments(allSegments);
    long totalRowCount = 0L;
    for (Segment segment : segments) {
        List<CoarseGrainIndex> indexes = defaultIndex.getIndexFactory().getIndexes(segment);
        for (CoarseGrainIndex index : indexes) {
            totalRowCount += index.getRowCount(segment, partitions);
        }
    }
    return totalRowCount;
}

Example 7 with CoarseGrainIndex

use of org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

/**
 * Get the index for all segments
 */
public Map<Segment, List<CoarseGrainIndex>> getIndexes(List<Segment> segments, Set<Path> partitionLocations, IndexFilter filter) throws IOException {
    List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>();
    Map<Segment, List<CoarseGrainIndex>> indexMap = new HashMap<>();
    Map<String, Segment> segmentMap = new HashMap<>();
    for (Segment segment : segments) {
        segmentMap.put(segment.getSegmentNo(), segment);
        Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
        if (!partitionLocations.isEmpty()) {
            // get tableBlockIndexUniqueIdentifierWrappers from segment file info
            getTableBlockUniqueIdentifierWrappers(partitionLocations, tableBlockIndexUniqueIdentifierWrappers, identifiers);
        } else {
            SegmentMetaDataInfo segmentMetaDataInfo = segment.getSegmentMetaDataInfo();
            boolean isLoadAllIndex = Boolean.parseBoolean(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_LOAD_ALL_SEGMENT_INDEXES_TO_CACHE, CarbonCommonConstants.CARBON_LOAD_ALL_SEGMENT_INDEXES_TO_CACHE_DEFAULT));
            if (!isLoadAllIndex && null != segmentMetaDataInfo && null != filter && !filter.isEmpty() && null != filter.getExpression() && null == FilterUtil.getImplicitFilterExpression(filter.getExpression())) {
                getTableBlockIndexUniqueIdentifierUsingSegmentMinMax(segment, segmentMetaDataInfo, filter, identifiers, tableBlockIndexUniqueIdentifierWrappers);
            } else {
                for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
                    tableBlockIndexUniqueIdentifierWrappers.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
                }
            }
        }
    }
    List<BlockletIndexWrapper> blockletIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers);
    for (BlockletIndexWrapper wrapper : blockletIndexWrappers) {
        Segment segment = segmentMap.get(wrapper.getSegmentId());
        List<CoarseGrainIndex> indexes = indexMap.get(segment);
        if (null == indexes) {
            indexes = new ArrayList<CoarseGrainIndex>();
        }
        indexes.addAll(wrapper.getIndexes());
        indexMap.put(segment, indexes);
    }
    return indexMap;
}

Example 8 with CoarseGrainIndex

use of org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

@Override
public List<CoarseGrainIndex> getIndexes(IndexInputSplit distributable) throws IOException {
    BlockletIndexInputSplit mapDistributable = (BlockletIndexInputSplit) distributable;
    List<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper;
    String segmentNo = mapDistributable.getSegment().getSegmentNo();
    if (mapDistributable.getSegmentPath() != null) {
        identifiersWrapper = getTableBlockIndexUniqueIdentifier(distributable);
    } else {
        identifiersWrapper = getTableBlockIndexUniqueIdentifier(mapDistributable.getFilePath(), segmentNo);
    }
    List<CoarseGrainIndex> indexes = new ArrayList<>();
    try {
        List<BlockletIndexWrapper> wrappers = cache.getAll(identifiersWrapper);
        for (BlockletIndexWrapper wrapper : wrappers) {
            indexes.addAll(wrapper.getIndexes());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return indexes;
}