Examples with Index org.apache.carbondata.core.index.dev.Index used on opensource projects

Example 6 with Index

use of org.apache.carbondata.core.index.dev.Index in project carbondata by apache.

the class TableIndex method prune.

/**
 * This method is used from any machine after it is distributed. It takes the distributable object
 * to prune the filters.
 *
 * @param distributable
 * @param filterExp
 * @return
 */
public List<ExtendedBlocklet> prune(List<Index> indices, IndexInputSplit distributable, FilterResolverIntf filterExp, List<PartitionSpec> partitions) throws IOException {
    List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
    List<Blocklet> blocklets = new ArrayList<>();
    Set<Path> partitionsToPrune = getPartitionLocations(partitions);
    SegmentProperties segmentProperties = segmentPropertiesFetcher.getSegmentProperties(distributable.getSegment(), partitionsToPrune);
    FilterExecutor filterExecutor = FilterUtil.getFilterExecutorTree(filterExp, segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
    for (Index index : indices) {
        blocklets.addAll(index.prune(filterExp, segmentProperties, filterExecutor, table));
    }
    BlockletSerializer serializer = new BlockletSerializer();
    String writePath = identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + indexSchema.getIndexName();
    if (indexFactory.getIndexLevel() == IndexLevel.FG) {
        FileFactory.mkdirs(writePath);
    }
    for (Blocklet blocklet : blocklets) {
        ExtendedBlocklet detailedBlocklet = blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegment());
        if (indexFactory.getIndexLevel() == IndexLevel.FG) {
            String blockletWritePath = writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime();
            detailedBlocklet.setIndexWriterPath(blockletWritePath);
            serializer.serializeBlocklet((FineGrainBlocklet) blocklet, blockletWritePath);
        }
        detailedBlocklet.setSegment(distributable.getSegment());
        detailedBlocklets.add(detailedBlocklet);
    }
    return detailedBlocklets;
}

Example 7 with Index

use of org.apache.carbondata.core.index.dev.Index in project carbondata by apache.

the class TableIndex method pruneWithFilter.

private List<ExtendedBlocklet> pruneWithFilter(List<Segment> segments, IndexFilter filter, Set<Path> partitionLocations, List<ExtendedBlocklet> blocklets, Map<Segment, List<Index>> indexes) throws IOException {
    Set<String> missingSISegments = filter.getMissingSISegments();
    for (Segment segment : segments) {
        List<Index> segmentIndices = indexes.get(segment);
        if (segment == null || segmentIndices == null || segmentIndices.isEmpty()) {
            continue;
        }
        boolean isExternalOrMissingSISegment = segment.isExternalSegment() || (missingSISegments != null && missingSISegments.contains(segment.getSegmentNo()));
        List<Blocklet> pruneBlocklets = new ArrayList<>();
        SegmentProperties segmentProperties;
        if (segmentIndices.get(0) instanceof BlockIndex) {
            segmentProperties = segmentPropertiesFetcher.getSegmentPropertiesFromIndex(segmentIndices.get(0));
        } else {
            segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment, partitionLocations);
        }
        if (filter.isResolvedOnSegment(segmentProperties)) {
            FilterExecutor filterExecutor;
            if (!isExternalOrMissingSISegment) {
                filterExecutor = FilterUtil.getFilterExecutorTree(filter.getResolver(), segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
            } else {
                filterExecutor = FilterUtil.getFilterExecutorTree(filter.getExternalSegmentResolver(), segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
            }
            for (Index index : segmentIndices) {
                if (!isExternalOrMissingSISegment) {
                    pruneBlocklets.addAll(index.prune(filter.getResolver(), segmentProperties, filterExecutor, this.table));
                } else {
                    pruneBlocklets.addAll(index.prune(filter.getExternalSegmentResolver(), segmentProperties, filterExecutor, this.table));
                }
            }
        } else {
            FilterExecutor filterExecutor;
            Expression expression = filter.getExpression();
            if (!isExternalOrMissingSISegment) {
                filterExecutor = FilterUtil.getFilterExecutorTree(new IndexFilter(segmentProperties, table, expression).getResolver(), segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
            } else {
                filterExecutor = FilterUtil.getFilterExecutorTree(new IndexFilter(segmentProperties, table, expression).getExternalSegmentResolver(), segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
            }
            for (Index index : segmentIndices) {
                if (!isExternalOrMissingSISegment) {
                    pruneBlocklets.addAll(index.prune(filter.getExpression(), segmentProperties, table, filterExecutor));
                } else {
                    pruneBlocklets.addAll(index.prune(filter.getExternalSegmentFilter(), segmentProperties, table, filterExecutor));
                }
            }
        }
        blocklets.addAll(addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment));
    }
    return blocklets;
}

Example 8 with Index

use of org.apache.carbondata.core.index.dev.Index in project carbondata by apache.

the class IndexUtil method pruneIndexes.

static List<ExtendedBlocklet> pruneIndexes(CarbonTable table, FilterResolverIntf filterResolverIntf, List<Segment> segmentsToLoad, List<PartitionSpec> partitions, List<ExtendedBlocklet> blocklets, IndexLevel indexLevel, IndexChooser indexChooser) throws IOException {
    IndexExprWrapper indexExprWrapper = indexChooser.chooseIndex(indexLevel, filterResolverIntf);
    if (indexExprWrapper != null) {
        List<ExtendedBlocklet> extendedBlocklets = new ArrayList<>();
        // Prune segments from already pruned blocklets
        for (IndexInputSplitWrapper wrapper : indexExprWrapper.toDistributable(segmentsToLoad)) {
            TableIndex index = IndexStoreManager.getInstance().getIndex(table, wrapper.getDistributable().getIndexSchema());
            List<Index> indices = index.getTableIndexes(wrapper.getDistributable());
            List<ExtendedBlocklet> prunedBlocklet = new ArrayList<>();
            if (table.isTransactionalTable()) {
                prunedBlocklet.addAll(index.prune(indices, wrapper.getDistributable(), indexExprWrapper.getFilterResolverIntf(wrapper.getUniqueId()), partitions));
            } else {
                prunedBlocklet.addAll(index.prune(segmentsToLoad, new IndexFilter(filterResolverIntf), partitions));
            }
            // For all blocklets initialize the detail info so that it can be serialized to the driver.
            for (ExtendedBlocklet blocklet : prunedBlocklet) {
                blocklet.getDetailInfo();
                blocklet.setIndexUniqueId(wrapper.getUniqueId());
                blocklet.setCgIndexPresent(true);
            }
            extendedBlocklets.addAll(prunedBlocklet);
        }
        return indexExprWrapper.pruneBlocklets(extendedBlocklets);
    }
    return blocklets;
}

Example 9 with Index

use of org.apache.carbondata.core.index.dev.Index in project carbondata by apache.

the class BlockletIndexFactory method clear.

@Override
public void clear(String segment) {
    SegmentBlockIndexInfo segmentBlockIndexInfo = segmentMap.remove(segment);
    Set<TableBlockIndexUniqueIdentifier> blockIndexes = null;
    if (null != segmentBlockIndexInfo) {
        blockIndexes = segmentBlockIndexInfo.getTableBlockIndexUniqueIdentifiers();
    }
    if (blockIndexes != null) {
        for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
            TableBlockIndexUniqueIdentifierWrapper blockIndexWrapper = new TableBlockIndexUniqueIdentifierWrapper(blockIndex, this.getCarbonTable());
            BlockletIndexWrapper wrapper = cache.getIfPresent(blockIndexWrapper);
            if (null != wrapper) {
                List<BlockIndex> indexes = wrapper.getIndexes();
                for (Index index : indexes) {
                    if (index != null) {
                        cache.invalidate(blockIndexWrapper);
                        index.clear();
                    }
                }
            }
        }
    }
}

Example 10 with Index

use of org.apache.carbondata.core.index.dev.Index in project carbondata by apache.

the class BlockletIndexFactory method getExtendedBlocklet.

private ExtendedBlocklet getExtendedBlocklet(Set<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper, Blocklet blocklet) throws IOException {
    for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : identifiersWrapper) {
        BlockletIndexWrapper wrapper = cache.get(identifierWrapper);
        List<BlockIndex> indexes = wrapper.getIndexes();
        for (Index index : indexes) {
            if (((BlockIndex) index).getTableTaskInfo(BlockletIndexRowIndexes.SUMMARY_INDEX_FILE_NAME).startsWith(blocklet.getFilePath())) {
                return ((BlockIndex) index).getDetailedBlocklet(blocklet.getBlockletId());
            }
        }
    }
    throw new IOException("Blocklet not found: " + blocklet.toString());
}