Examples with CacheableIndex org.apache.carbondata.core.index.dev.CacheableIndex used on opensource projects

Example 1 with CacheableIndex

use of org.apache.carbondata.core.index.dev.CacheableIndex in project carbondata by apache.

the class BlockletIndexInputFormat method getSplits.

@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
    IndexFactory indexFactory = IndexStoreManager.getInstance().getDefaultIndex(table).getIndexFactory();
    CacheableIndex factory = (CacheableIndex) indexFactory;
    List<IndexInputSplit> validDistributables = factory.getAllUncached(validSegments, indexExprWrapper);
    if (!validSegments.isEmpty()) {
        this.readCommittedScope = validSegments.get(0).getReadCommittedScope();
    }
    CarbonBlockLoaderHelper instance = CarbonBlockLoaderHelper.getInstance();
    int distributableSize = validDistributables.size();
    List<InputSplit> inputSplits = new ArrayList<>(distributableSize);
    keys = new HashSet<>();
    Iterator<IndexInputSplit> iterator = validDistributables.iterator();
    while (iterator.hasNext()) {
        BlockletIndexInputSplit next = (BlockletIndexInputSplit) iterator.next();
        String key = next.getSegmentPath();
        if (instance.checkAlreadySubmittedBlock(table.getAbsoluteTableIdentifier(), key)) {
            inputSplits.add(next);
            keys.add(key);
        }
    }
    int sizeOfDistToBeLoaded = inputSplits.size();
    LOGGER.info("Submitted blocks " + sizeOfDistToBeLoaded + ", " + distributableSize + " . Rest already considered for load in other job.");
    return inputSplits;
}