Examples with CoarseGrainIndex org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex used on opensource projects

Example 1 with CoarseGrainIndex

use of org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

@Override
public List<CoarseGrainIndex> getIndexes(Segment segment, Set<Path> partitionLocations) throws IOException {
    List<CoarseGrainIndex> indexes = new ArrayList<>();
    Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
    List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>(identifiers.size());
    getTableBlockUniqueIdentifierWrappers(partitionLocations, tableBlockIndexUniqueIdentifierWrappers, identifiers);
    List<BlockletIndexWrapper> blockletIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers);
    for (BlockletIndexWrapper wrapper : blockletIndexWrappers) {
        indexes.addAll(wrapper.getIndexes());
    }
    return indexes;
}

Example 2 with CoarseGrainIndex

use of org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex in project carbondata by apache.

the class BlockletIndexFactory method getSegmentProperties.

@Override
public SegmentProperties getSegmentProperties(Segment segment, Set<Path> partitionLocations) throws IOException {
    List<CoarseGrainIndex> indexes = getIndexes(segment, partitionLocations);
    assert (indexes.size() > 0);
    CoarseGrainIndex coarseGrainIndex = indexes.get(0);
    assert (coarseGrainIndex instanceof BlockIndex);
    BlockIndex index = (BlockIndex) coarseGrainIndex;
    return index.getSegmentProperties();
}

Example 3 with CoarseGrainIndex

use of org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex in project carbondata by apache.

the class BlockletIndexFactory method getAllBlocklets.

@Override
public List<Blocklet> getAllBlocklets(Segment segment, Set<Path> partitionLocations) throws IOException {
    List<Blocklet> blocklets = new ArrayList<>();
    List<CoarseGrainIndex> indexes = getIndexes(segment, partitionLocations);
    if (indexes.size() == 0) {
        return blocklets;
    }
    SegmentProperties segmentProperties = getSegmentPropertiesFromIndex(indexes.get(0));
    for (CoarseGrainIndex index : indexes) {
        blocklets.addAll(index.prune(null, segmentProperties, null, this.getCarbonTable()));
    }
    return blocklets;
}

Example 4 with CoarseGrainIndex

use of org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex in project carbondata by apache.

the class BloomCoarseGrainIndexFactory method getIndexes.

@Override
public List<CoarseGrainIndex> getIndexes(Segment segment) throws IOException {
    List<CoarseGrainIndex> indexes = new ArrayList<>();
    try {
        Set<String> shardPaths = segmentMap.get(segment.getSegmentNo());
        if (shardPaths == null) {
            shardPaths = getAllShardPaths(getCarbonTable().getTablePath(), segment.getSegmentNo(), indexName);
            segmentMap.put(segment.getSegmentNo(), shardPaths);
        }
        Set<String> filteredShards = segment.getFilteredIndexShardNames();
        for (String shard : shardPaths) {
            if (shard.endsWith(BloomIndexFileStore.MERGE_BLOOM_INDEX_SHARD_NAME) || filteredShards.contains(new File(shard).getName())) {
                // Filter out the tasks which are filtered through Main index.
                // for merge shard, shard pruning delay to be done before pruning blocklet
                BloomCoarseGrainIndex bloomDM = new BloomCoarseGrainIndex();
                bloomDM.init(new BloomIndexModel(shard, cache, segment.getConfiguration()));
                bloomDM.initIndexColumnConverters(getCarbonTable(), indexMeta.getIndexedColumns());
                bloomDM.setFilteredShard(filteredShards);
                indexes.add(bloomDM);
            }
        }
    } catch (Exception e) {
        throw new IOException("Error occurs while init Bloom Index", e);
    }
    return indexes;
}

Example 5 with CoarseGrainIndex

use of org.apache.carbondata.core.index.dev.cgindex.CoarseGrainIndex in project carbondata by apache.

the class TableIndex method getBlockRowCount.

/**
 * Prune the index of the given segments and return the Map of blocklet path and row count
 *
 * @param allSegments
 * @param partitions
 * @return
 * @throws IOException
 */
public Map<String, Long> getBlockRowCount(List<Segment> allSegments, final List<PartitionSpec> partitions, TableIndex defaultIndex) throws IOException {
    List<Segment> segments = getCarbonSegments(allSegments);
    Map<String, Long> blockletToRowCountMap = new HashMap<>();
    for (Segment segment : segments) {
        List<CoarseGrainIndex> indexes = defaultIndex.getIndexFactory().getIndexes(segment);
        for (CoarseGrainIndex index : indexes) {
            if (null != partitions && !partitions.isEmpty()) {
                // partitions are dropped so return empty list.
                if (index.validatePartitionInfo(partitions)) {
                    return new HashMap<>();
                }
            }
            index.getRowCountForEachBlock(segment, partitions, blockletToRowCountMap);
        }
    }
    return blockletToRowCountMap;
}