Examples with Blocklet org.apache.carbondata.core.indexstore.Blocklet used on opensource projects

Example 1 with Blocklet

use of org.apache.carbondata.core.indexstore.Blocklet in project carbondata by apache.

the class MinMaxIndexDataMap method prune.

/**
 * Block Prunning logic for Min Max DataMap.
 *
 * @param filterExp
 * @param segmentProperties
 * @return
 */
@Override
public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties, List<PartitionSpec> partitions) {
    List<Blocklet> blocklets = new ArrayList<>();
    if (filterExp == null) {
        for (int i = 0; i < readMinMaxDataMap.length; i++) {
            blocklets.add(new Blocklet(filePath, String.valueOf(readMinMaxDataMap[i].getBlockletId())));
        }
    } else {
        FilterExecuter filterExecuter = FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
        int startIndex = 0;
        while (startIndex < readMinMaxDataMap.length) {
            BitSet bitSet = filterExecuter.isScanRequired(readMinMaxDataMap[startIndex].getMaxValues(), readMinMaxDataMap[startIndex].getMinValues());
            if (!bitSet.isEmpty()) {
                blocklets.add(new Blocklet(filePath, String.valueOf(readMinMaxDataMap[startIndex].getBlockletId())));
            }
            startIndex++;
        }
    }
    return blocklets;
}

Example 2 with Blocklet

use of org.apache.carbondata.core.indexstore.Blocklet in project carbondata by apache.

the class BlockletDataMap method prune.

private List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties) {
    if (unsafeMemoryDMStore.getRowCount() == 0) {
        return new ArrayList<>();
    }
    List<Blocklet> blocklets = new ArrayList<>();
    if (filterExp == null) {
        int rowCount = unsafeMemoryDMStore.getRowCount();
        for (int i = 0; i < rowCount; i++) {
            DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(i).convertToSafeRow();
            blocklets.add(createBlocklet(safeRow, safeRow.getShort(BLOCKLET_ID_INDEX)));
        }
    } else {
        // Remove B-tree jump logic as start and end key prepared is not
        // correct for old store scenarios
        int startIndex = 0;
        int endIndex = unsafeMemoryDMStore.getRowCount();
        FilterExecuter filterExecuter = FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
        while (startIndex < endIndex) {
            DataMapRow safeRow = unsafeMemoryDMStore.getUnsafeRow(startIndex).convertToSafeRow();
            int blockletId = safeRow.getShort(BLOCKLET_ID_INDEX);
            String filePath = new String(safeRow.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
            boolean isValid = addBlockBasedOnMinMaxValue(filterExecuter, getMinMaxValue(safeRow, MAX_VALUES_INDEX), getMinMaxValue(safeRow, MIN_VALUES_INDEX), filePath, blockletId);
            if (isValid) {
                blocklets.add(createBlocklet(safeRow, blockletId));
            }
            startIndex++;
        }
    }
    return blocklets;
}

Example 3 with Blocklet

use of org.apache.carbondata.core.indexstore.Blocklet in project carbondata by apache.

the class BlockletDataMapFactory method getAllBlocklets.

@Override
public List<Blocklet> getAllBlocklets(Segment segment, List<PartitionSpec> partitions) throws IOException {
    List<Blocklet> blocklets = new ArrayList<>();
    List<CoarseGrainDataMap> dataMaps = getDataMaps(segment);
    for (CoarseGrainDataMap dataMap : dataMaps) {
        blocklets.addAll(dataMap.prune(null, getSegmentProperties(segment), partitions));
    }
    return blocklets;
}

Example 4 with Blocklet

use of org.apache.carbondata.core.indexstore.Blocklet in project carbondata by apache.

the class TableDataMap method prune.

/**
 * Pass the valid segments and prune the datamap using filter expression
 *
 * @param segments
 * @param filterExp
 * @return
 */
public List<ExtendedBlocklet> prune(List<Segment> segments, FilterResolverIntf filterExp, List<PartitionSpec> partitions) throws IOException {
    List<ExtendedBlocklet> blocklets = new ArrayList<>();
    SegmentProperties segmentProperties;
    for (Segment segment : segments) {
        List<Blocklet> pruneBlocklets = new ArrayList<>();
        // if filter is not passed then return all the blocklets
        if (filterExp == null) {
            pruneBlocklets = blockletDetailsFetcher.getAllBlocklets(segment, partitions);
        } else {
            List<DataMap> dataMaps = dataMapFactory.getDataMaps(segment);
            segmentProperties = segmentPropertiesFetcher.getSegmentProperties(segment);
            for (DataMap dataMap : dataMaps) {
                pruneBlocklets.addAll(dataMap.prune(filterExp, segmentProperties, partitions));
            }
        }
        blocklets.addAll(addSegmentId(blockletDetailsFetcher.getExtendedBlocklets(pruneBlocklets, segment), segment.getSegmentNo()));
    }
    return blocklets;
}

Example 5 with Blocklet

use of org.apache.carbondata.core.indexstore.Blocklet in project carbondata by apache.

the class TableDataMap method prune.

/**
 * This method is used from any machine after it is distributed. It takes the distributable object
 * to prune the filters.
 *
 * @param distributable
 * @param filterExp
 * @return
 */
public List<ExtendedBlocklet> prune(DataMapDistributable distributable, FilterResolverIntf filterExp, List<PartitionSpec> partitions) throws IOException {
    List<ExtendedBlocklet> detailedBlocklets = new ArrayList<>();
    List<Blocklet> blocklets = new ArrayList<>();
    List<DataMap> dataMaps = dataMapFactory.getDataMaps(distributable);
    for (DataMap dataMap : dataMaps) {
        blocklets.addAll(dataMap.prune(filterExp, segmentPropertiesFetcher.getSegmentProperties(distributable.getSegment()), partitions));
    }
    BlockletSerializer serializer = new BlockletSerializer();
    String writePath = identifier.getTablePath() + CarbonCommonConstants.FILE_SEPARATOR + dataMapSchema.getDataMapName();
    if (dataMapFactory.getDataMapType() == DataMapLevel.FG) {
        FileFactory.mkdirs(writePath, FileFactory.getFileType(writePath));
    }
    for (Blocklet blocklet : blocklets) {
        ExtendedBlocklet detailedBlocklet = blockletDetailsFetcher.getExtendedBlocklet(blocklet, distributable.getSegment());
        if (dataMapFactory.getDataMapType() == DataMapLevel.FG) {
            String blockletwritePath = writePath + CarbonCommonConstants.FILE_SEPARATOR + System.nanoTime();
            detailedBlocklet.setDataMapWriterPath(blockletwritePath);
            serializer.serializeBlocklet((FineGrainBlocklet) blocklet, blockletwritePath);
        }
        detailedBlocklet.setSegmentId(distributable.getSegment().getSegmentNo());
        detailedBlocklets.add(detailedBlocklet);
    }
    return detailedBlocklets;
}