Examples with Segment org.apache.carbondata.core.index.Segment used on opensource projects

Example 26 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class StreamPruner method listAllStreamFiles.

// TODO optimize and move the code to StreamSegment , but it's in the streaming module.
private List<StreamFile> listAllStreamFiles(List<Segment> segments, boolean withMinMax) throws IOException {
    List<StreamFile> streamFileList = new ArrayList<>();
    for (Segment segment : segments) {
        String segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getAbsoluteTableIdentifier().getTablePath(), segment.getSegmentNo());
        String indexFile = CarbonTablePath.getCarbonStreamIndexFilePath(segmentDir);
        if (FileFactory.isFileExist(indexFile)) {
            CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
            indexReader.openThriftReader(indexFile);
            try {
                while (indexReader.hasNext()) {
                    BlockIndex blockIndex = indexReader.readBlockIndexInfo();
                    String filePath = segmentDir + File.separator + blockIndex.getFile_name();
                    long length = blockIndex.getFile_size();
                    StreamFile streamFile = new StreamFile(segment.getSegmentNo(), filePath, length);
                    streamFileList.add(streamFile);
                    if (withMinMax) {
                        if (blockIndex.getBlock_index() != null && blockIndex.getBlock_index().getMin_max_index() != null) {
                            streamFile.setMinMaxIndex(CarbonMetadataUtil.convertExternalMinMaxIndex(blockIndex.getBlock_index().getMin_max_index()));
                        }
                    }
                }
            } finally {
                indexReader.closeThriftReader();
            }
        }
    }
    totalFileNums = streamFileList.size();
    return streamFileList;
}

Example 27 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class SecondaryIndexFactory method getIndexes.

private Map<Segment, List<CoarseGrainIndex>> getIndexes(List<Segment> segments, PositionReferenceInfo positionReferenceInfo) throws IOException {
    Map<Segment, List<CoarseGrainIndex>> indexes = new HashMap<>();
    List<String> allSegmentIds = segments.stream().map(Segment::getSegmentNo).collect(Collectors.toList());
    for (Segment segment : segments) {
        indexes.put(segment, this.getIndexes(segment, allSegmentIds, positionReferenceInfo));
    }
    return indexes;
}

Example 28 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class BlockletIndexFactory method getAllUncached.

@Override
public List<IndexInputSplit> getAllUncached(List<IndexInputSplit> distributableList) throws IOException {
    List<IndexInputSplit> distributableToBeLoaded = new ArrayList<>(distributableList.size());
    for (IndexInputSplit distributable : distributableList) {
        Segment segment = distributable.getSegment();
        Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = getTableBlockIndexUniqueIdentifiers(segment);
        // filter out the tableBlockIndexUniqueIdentifiers based on distributable
        TableBlockIndexUniqueIdentifier validIdentifier = BlockletIndexUtil.filterIdentifiersBasedOnDistributable(tableBlockIndexUniqueIdentifiers, (BlockletIndexInputSplit) distributable);
        if (null == cache.getIfPresent(new TableBlockIndexUniqueIdentifierWrapper(validIdentifier, this.getCarbonTable()))) {
            ((BlockletIndexInputSplit) distributable).setTableBlockIndexUniqueIdentifier(validIdentifier);
            distributableToBeLoaded.add(distributable);
        }
    }
    return distributableToBeLoaded;
}

Example 29 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class BlockletIndexFactory method getAllUncached.

@Override
public List<IndexInputSplit> getAllUncached(List<Segment> validSegments, IndexExprWrapper indexExprWrapper) throws IOException {
    List<IndexInputSplit> distributableToBeLoaded = new ArrayList<>();
    for (Segment segment : validSegments) {
        IndexInputSplitWrapper indexInputSplitWrappers = indexExprWrapper.toDistributableSegment(segment);
        Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = getTableSegmentUniqueIdentifiers(segment);
        for (TableBlockIndexUniqueIdentifier identifier : tableBlockIndexUniqueIdentifiers) {
            BlockletIndexWrapper blockletIndexWrapper = cache.getIfPresent(new TableBlockIndexUniqueIdentifierWrapper(identifier, this.getCarbonTable()));
            if (identifier.getIndexFilePath() == null || blockletIndexWrapper == null) {
                ((BlockletIndexInputSplit) indexInputSplitWrappers.getDistributable()).setTableBlockIndexUniqueIdentifier(identifier);
                distributableToBeLoaded.add(indexInputSplitWrappers.getDistributable());
            }
        }
    }
    return distributableToBeLoaded;
}

Example 30 with Segment

use of org.apache.carbondata.core.index.Segment in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

/**
 * Get the index for all segments
 */
public Map<Segment, List<CoarseGrainIndex>> getIndexes(List<Segment> segments, Set<Path> partitionLocations, IndexFilter filter) throws IOException {
    List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>();
    Map<Segment, List<CoarseGrainIndex>> indexMap = new HashMap<>();
    Map<String, Segment> segmentMap = new HashMap<>();
    for (Segment segment : segments) {
        segmentMap.put(segment.getSegmentNo(), segment);
        Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
        if (!partitionLocations.isEmpty()) {
            // get tableBlockIndexUniqueIdentifierWrappers from segment file info
            getTableBlockUniqueIdentifierWrappers(partitionLocations, tableBlockIndexUniqueIdentifierWrappers, identifiers);
        } else {
            SegmentMetaDataInfo segmentMetaDataInfo = segment.getSegmentMetaDataInfo();
            boolean isLoadAllIndex = Boolean.parseBoolean(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_LOAD_ALL_SEGMENT_INDEXES_TO_CACHE, CarbonCommonConstants.CARBON_LOAD_ALL_SEGMENT_INDEXES_TO_CACHE_DEFAULT));
            if (!isLoadAllIndex && null != segmentMetaDataInfo && null != filter && !filter.isEmpty() && null != filter.getExpression() && null == FilterUtil.getImplicitFilterExpression(filter.getExpression())) {
                getTableBlockIndexUniqueIdentifierUsingSegmentMinMax(segment, segmentMetaDataInfo, filter, identifiers, tableBlockIndexUniqueIdentifierWrappers);
            } else {
                for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
                    tableBlockIndexUniqueIdentifierWrappers.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
                }
            }
        }
    }
    List<BlockletIndexWrapper> blockletIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers);
    for (BlockletIndexWrapper wrapper : blockletIndexWrappers) {
        Segment segment = segmentMap.get(wrapper.getSegmentId());
        List<CoarseGrainIndex> indexes = indexMap.get(segment);
        if (null == indexes) {
            indexes = new ArrayList<CoarseGrainIndex>();
        }
        indexes.addAll(wrapper.getIndexes());
        indexMap.put(segment, indexes);
    }
    return indexMap;
}