Examples with IndexInputSplit org.apache.carbondata.core.index.IndexInputSplit used on opensource projects

Example 6 with IndexInputSplit

use of org.apache.carbondata.core.index.IndexInputSplit in project carbondata by apache.

the class BlockletIndexFactory method getAllUncached.

@Override
public List<IndexInputSplit> getAllUncached(List<Segment> validSegments, IndexExprWrapper indexExprWrapper) throws IOException {
    List<IndexInputSplit> distributableToBeLoaded = new ArrayList<>();
    for (Segment segment : validSegments) {
        IndexInputSplitWrapper indexInputSplitWrappers = indexExprWrapper.toDistributableSegment(segment);
        Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = getTableSegmentUniqueIdentifiers(segment);
        for (TableBlockIndexUniqueIdentifier identifier : tableBlockIndexUniqueIdentifiers) {
            BlockletIndexWrapper blockletIndexWrapper = cache.getIfPresent(new TableBlockIndexUniqueIdentifierWrapper(identifier, this.getCarbonTable()));
            if (identifier.getIndexFilePath() == null || blockletIndexWrapper == null) {
                ((BlockletIndexInputSplit) indexInputSplitWrappers.getDistributable()).setTableBlockIndexUniqueIdentifier(identifier);
                distributableToBeLoaded.add(indexInputSplitWrappers.getDistributable());
            }
        }
    }
    return distributableToBeLoaded;
}

Example 7 with IndexInputSplit

use of org.apache.carbondata.core.index.IndexInputSplit in project carbondata by apache.

the class LuceneIndexFactoryBase method toDistributable.

/**
 * Get all distributable objects of a segmentId
 */
@Override
public List<IndexInputSplit> toDistributable(Segment segment) {
    List<IndexInputSplit> splits = new ArrayList<>();
    CarbonFile[] indexDirs = getAllIndexDirs(tableIdentifier.getTablePath(), segment.getSegmentNo());
    if (segment.getFilteredIndexShardNames().size() == 0) {
        for (CarbonFile indexDir : indexDirs) {
            IndexInputSplit luceneIndexInputSplit = new LuceneIndexInputSplit(tableIdentifier.getTablePath(), indexDir.getAbsolutePath());
            luceneIndexInputSplit.setSegment(segment);
            luceneIndexInputSplit.setIndexSchema(getIndexSchema());
            splits.add(luceneIndexInputSplit);
        }
        return splits;
    }
    for (CarbonFile indexDir : indexDirs) {
        // Filter out the tasks which are filtered through CG index.
        if (getIndexLevel() != IndexLevel.FG && !segment.getFilteredIndexShardNames().contains(indexDir.getName())) {
            continue;
        }
        IndexInputSplit luceneIndexInputSplit = new LuceneIndexInputSplit(CarbonTablePath.getSegmentPath(tableIdentifier.getTablePath(), segment.getSegmentNo()), indexDir.getAbsolutePath());
        luceneIndexInputSplit.setSegment(segment);
        luceneIndexInputSplit.setIndexSchema(getIndexSchema());
        splits.add(luceneIndexInputSplit);
    }
    return splits;
}