Examples with IndexInputSplitWrapper org.apache.carbondata.core.index.dev.expr.IndexInputSplitWrapper used on opensource projects

Example 1 with IndexInputSplitWrapper

use of org.apache.carbondata.core.index.dev.expr.IndexInputSplitWrapper in project carbondata by apache.

the class IndexInputFormat method createRecordReader.

@Override
public RecordReader<Void, ExtendedBlocklet> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) {
    return new RecordReader<Void, ExtendedBlocklet>() {

        private Iterator<ExtendedBlocklet> blockletIterator;

        private ExtendedBlocklet currBlocklet;

        @Override
        public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException {
            IndexInputSplitWrapper distributable = (IndexInputSplitWrapper) inputSplit;
            distributable.getDistributable().getSegment().setReadCommittedScope(readCommittedScope);
            List<Segment> segmentsToLoad = new ArrayList<>();
            segmentsToLoad.add(distributable.getDistributable().getSegment());
            List<ExtendedBlocklet> blocklets = new ArrayList<>();
            if (indexLevel == null) {
                TableIndex defaultIndex = IndexStoreManager.getInstance().getIndex(table, distributable.getDistributable().getIndexSchema());
                IndexFilter filter = new IndexFilter(filterResolverIntf);
                filter.setTable(table);
                filter.setMissingSISegments(missingSISegments);
                if (filterResolverIntf != null) {
                    filter.setExpression(filterResolverIntf.getFilterExpression());
                }
                blocklets = defaultIndex.prune(segmentsToLoad, filter, partitions);
                blocklets = IndexUtil.pruneIndexes(table, filterResolverIntf, segmentsToLoad, partitions, blocklets, indexChooser);
            } else {
                blocklets = IndexUtil.pruneIndexes(table, filterResolverIntf, segmentsToLoad, partitions, blocklets, indexLevel, indexChooser);
            }
            blockletIterator = blocklets.iterator();
        }

        @Override
        public boolean nextKeyValue() {
            boolean hasNext = blockletIterator.hasNext();
            if (hasNext) {
                currBlocklet = blockletIterator.next();
            } else {
                // close all resources when all the results are returned
                close();
            }
            return hasNext;
        }

        @Override
        public Void getCurrentKey() {
            return null;
        }

        @Override
        public ExtendedBlocklet getCurrentValue() {
            return currBlocklet;
        }

        @Override
        public float getProgress() {
            return 0;
        }

        @Override
        public void close() {
            // Clear the Indexes from executor
            if (isFallbackJob) {
                IndexStoreManager.getInstance().clearIndexCache(table.getAbsoluteTableIdentifier(), false);
            }
        }
    };
}

Example 2 with IndexInputSplitWrapper

use of org.apache.carbondata.core.index.dev.expr.IndexInputSplitWrapper in project carbondata by apache.

the class IndexUtil method pruneIndexes.

static List<ExtendedBlocklet> pruneIndexes(CarbonTable table, FilterResolverIntf filterResolverIntf, List<Segment> segmentsToLoad, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets, IndexLevel indexLevel, IndexChooser indexChooser) throws IOException {
    IndexExprWrapper indexExprWrapper = indexChooser.chooseIndex(indexLevel, filterResolverIntf);
    if (indexExprWrapper != null) {
        List<ExtendedBlocklet> extendedBlocklets = new ArrayList<>();
        // Prune segments from already pruned blocklets
        for (IndexInputSplitWrapper wrapper : indexExprWrapper.toDistributable(segmentsToLoad)) {
            TableIndex index = IndexStoreManager.getInstance().getIndex(table, wrapper.getDistributable().getIndexSchema());
            List<Index> indices = index.getTableIndexes(wrapper.getDistributable());
            List<ExtendedBlocklet> prunedBlocklet = new ArrayList<>();
            if (table.isTransactionalTable()) {
                prunedBlocklet.addAll(index.prune(indices, wrapper.getDistributable(), indexExprWrapper.getFilterResolverIntf(wrapper.getUniqueId()), partitions));
            } else {
                prunedBlocklet.addAll(index.prune(segmentsToLoad, new IndexFilter(filterResolverIntf), partitions));
            }
            // For all blocklets initialize the detail info so that it can be serialized to the driver.
            for (ExtendedBlocklet blocklet : prunedBlocklet) {
                blocklet.getDetailInfo();
                blocklet.setIndexUniqueId(wrapper.getUniqueId());
                blocklet.setCgIndexPresent(true);
            }
            extendedBlocklets.addAll(prunedBlocklet);
        }
        return indexExprWrapper.pruneBlocklets(extendedBlocklets);
    }
    return blocklets;
}

Example 3 with IndexInputSplitWrapper

use of org.apache.carbondata.core.index.dev.expr.IndexInputSplitWrapper in project carbondata by apache.

the class BlockletIndexFactory method getAllUncached.

@Override
public List<IndexInputSplit> getAllUncached(List<Segment> validSegments, IndexExprWrapper indexExprWrapper) throws IOException {
    List<IndexInputSplit> distributableToBeLoaded = new ArrayList<>();
    for (Segment segment : validSegments) {
        IndexInputSplitWrapper indexInputSplitWrappers = indexExprWrapper.toDistributableSegment(segment);
        Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = getTableSegmentUniqueIdentifiers(segment);
        for (TableBlockIndexUniqueIdentifier identifier : tableBlockIndexUniqueIdentifiers) {
            BlockletIndexWrapper blockletIndexWrapper = cache.getIfPresent(new TableBlockIndexUniqueIdentifierWrapper(identifier, this.getCarbonTable()));
            if (identifier.getIndexFilePath() == null || blockletIndexWrapper == null) {
                ((BlockletIndexInputSplit) indexInputSplitWrappers.getDistributable()).setTableBlockIndexUniqueIdentifier(identifier);
                distributableToBeLoaded.add(indexInputSplitWrappers.getDistributable());
            }
        }
    }
    return distributableToBeLoaded;
}