Examples with TableBlockIndexUniqueIdentifierWrapper org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper used on opensource projects

Example 11 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexFactory method getCacheSize.

@Override
public String getCacheSize() {
    long sum = 0L;
    int numOfIndexFiles = 0;
    for (Map.Entry<String, SegmentBlockIndexInfo> entry : segmentMap.entrySet()) {
        for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : entry.getValue().getTableBlockIndexUniqueIdentifiers()) {
            BlockletIndexWrapper blockletIndexWrapper = cache.getIfPresent(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, getCarbonTable()));
            if (blockletIndexWrapper != null) {
                sum += blockletIndexWrapper.getMemorySize();
                numOfIndexFiles++;
            }
        }
    }
    return numOfIndexFiles + ":" + sum;
}

Example 12 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexFactory method getAllUncached.

@Override
public List<IndexInputSplit> getAllUncached(List<Segment> validSegments, IndexExprWrapper indexExprWrapper) throws IOException {
    List<IndexInputSplit> distributableToBeLoaded = new ArrayList<>();
    for (Segment segment : validSegments) {
        IndexInputSplitWrapper indexInputSplitWrappers = indexExprWrapper.toDistributableSegment(segment);
        Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = getTableSegmentUniqueIdentifiers(segment);
        for (TableBlockIndexUniqueIdentifier identifier : tableBlockIndexUniqueIdentifiers) {
            BlockletIndexWrapper blockletIndexWrapper = cache.getIfPresent(new TableBlockIndexUniqueIdentifierWrapper(identifier, this.getCarbonTable()));
            if (identifier.getIndexFilePath() == null || blockletIndexWrapper == null) {
                ((BlockletIndexInputSplit) indexInputSplitWrappers.getDistributable()).setTableBlockIndexUniqueIdentifier(identifier);
                distributableToBeLoaded.add(indexInputSplitWrappers.getDistributable());
            }
        }
    }
    return distributableToBeLoaded;
}

Example 13 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexFactory method getTableBlockIndexUniqueIdentifierUsingSegmentMinMax.

/**
 * Using blockLevel minmax values, identify if segment has to be added for further pruning and to
 * load segment index info to cache
 * @param segment to be identified if needed for loading block indexes
 * @param segmentMetaDataInfo list of block level min max values
 * @param filter filter expression
 * @param identifiers tableBlockIndexUniqueIdentifiers
 * @param tableBlockIndexUniqueIdentifierWrappers to add tableBlockIndexUniqueIdentifiers
 */
private void getTableBlockIndexUniqueIdentifierUsingSegmentMinMax(Segment segment, SegmentMetaDataInfo segmentMetaDataInfo, IndexFilter filter, Set<TableBlockIndexUniqueIdentifier> identifiers, List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers) {
    boolean isScanRequired = false;
    Map<String, SegmentColumnMetaDataInfo> segmentColumnMetaDataInfoMap = segmentMetaDataInfo.getSegmentColumnMetaDataInfoMap();
    int length = segmentColumnMetaDataInfoMap.size();
    // Add columnSchemas based on the columns present in segment
    List<ColumnSchema> columnSchemas = new ArrayList<>();
    byte[][] min = new byte[length][];
    byte[][] max = new byte[length][];
    boolean[] minMaxFlag = new boolean[length];
    int i = 0;
    // get current columnSchema list for the table
    Map<String, ColumnSchema> tableColumnSchemas = this.getCarbonTable().getTableInfo().getFactTable().getListOfColumns().stream().collect(Collectors.toMap(ColumnSchema::getColumnUniqueId, ColumnSchema::clone));
    // fill min,max and columnSchema values
    for (Map.Entry<String, SegmentColumnMetaDataInfo> columnMetaData : segmentColumnMetaDataInfoMap.entrySet()) {
        ColumnSchema columnSchema = tableColumnSchemas.get(columnMetaData.getKey());
        if (null != columnSchema) {
            // get segment sort column and column drift info
            boolean isSortColumnInSegment = columnMetaData.getValue().isSortColumn();
            boolean isColumnDriftInSegment = columnMetaData.getValue().isColumnDrift();
            if (null != columnSchema.getColumnProperties()) {
                // get current sort column and column drift info from current columnSchema
                String isSortColumn = columnSchema.getColumnProperties().get(CarbonCommonConstants.SORT_COLUMNS);
                String isColumnDrift = columnSchema.getColumnProperties().get(CarbonCommonConstants.COLUMN_DRIFT);
                if (null != isSortColumn) {
                    if (isSortColumn.equalsIgnoreCase("true") && !isSortColumnInSegment) {
                        // Unset current column schema column properties
                        modifyColumnSchemaForSortColumn(columnSchema, isColumnDriftInSegment, isColumnDrift, false);
                    } else if (isSortColumn.equalsIgnoreCase("false") && isSortColumnInSegment) {
                        // set sort column to true in current column schema column properties
                        modifyColumnSchemaForSortColumn(columnSchema, isColumnDriftInSegment, isColumnDrift, true);
                    }
                } else {
                    modifyColumnSchemaForSortColumn(columnSchema, isColumnDriftInSegment, isColumnDrift, false);
                }
            }
            columnSchemas.add(columnSchema);
            min[i] = columnMetaData.getValue().getColumnMinValue();
            max[i] = columnMetaData.getValue().getColumnMaxValue();
            minMaxFlag[i] = min[i].length != 0 && max[i].length != 0;
            i++;
        }
    }
    // get segmentProperties using created columnSchemas list
    SegmentProperties segmentProperties = SegmentPropertiesAndSchemaHolder.getInstance().addSegmentProperties(this.getCarbonTable(), columnSchemas, segment.getSegmentNo()).getSegmentProperties();
    FilterResolverIntf resolver = new IndexFilter(segmentProperties, this.getCarbonTable(), filter.getExpression()).getResolver();
    // prepare filter executor using IndexFilter resolver
    FilterExecutor filterExecutor = FilterUtil.getFilterExecutorTree(resolver, segmentProperties, null, null, false);
    // check if block has to be pruned based on segment minmax
    BitSet scanRequired = filterExecutor.isScanRequired(max, min, minMaxFlag);
    if (!scanRequired.isEmpty()) {
        isScanRequired = true;
    }
    if (isScanRequired) {
        for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
            tableBlockIndexUniqueIdentifierWrappers.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
        }
    }
}

Example 14 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

/**
 * Get the index for all segments
 */
public Map<Segment, List<CoarseGrainIndex>> getIndexes(List<Segment> segments, Set<Path> partitionLocations, IndexFilter filter) throws IOException {
    List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>();
    Map<Segment, List<CoarseGrainIndex>> indexMap = new HashMap<>();
    Map<String, Segment> segmentMap = new HashMap<>();
    for (Segment segment : segments) {
        segmentMap.put(segment.getSegmentNo(), segment);
        Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
        if (!partitionLocations.isEmpty()) {
            // get tableBlockIndexUniqueIdentifierWrappers from segment file info
            getTableBlockUniqueIdentifierWrappers(partitionLocations, tableBlockIndexUniqueIdentifierWrappers, identifiers);
        } else {
            SegmentMetaDataInfo segmentMetaDataInfo = segment.getSegmentMetaDataInfo();
            boolean isLoadAllIndex = Boolean.parseBoolean(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_LOAD_ALL_SEGMENT_INDEXES_TO_CACHE, CarbonCommonConstants.CARBON_LOAD_ALL_SEGMENT_INDEXES_TO_CACHE_DEFAULT));
            if (!isLoadAllIndex && null != segmentMetaDataInfo && null != filter && !filter.isEmpty() && null != filter.getExpression() && null == FilterUtil.getImplicitFilterExpression(filter.getExpression())) {
                getTableBlockIndexUniqueIdentifierUsingSegmentMinMax(segment, segmentMetaDataInfo, filter, identifiers, tableBlockIndexUniqueIdentifierWrappers);
            } else {
                for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
                    tableBlockIndexUniqueIdentifierWrappers.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
                }
            }
        }
    }
    List<BlockletIndexWrapper> blockletIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers);
    for (BlockletIndexWrapper wrapper : blockletIndexWrappers) {
        Segment segment = segmentMap.get(wrapper.getSegmentId());
        List<CoarseGrainIndex> indexes = indexMap.get(segment);
        if (null == indexes) {
            indexes = new ArrayList<CoarseGrainIndex>();
        }
        indexes.addAll(wrapper.getIndexes());
        indexMap.put(segment, indexes);
    }
    return indexMap;
}

Example 15 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

@Override
public List<CoarseGrainIndex> getIndexes(IndexInputSplit distributable) throws IOException {
    BlockletIndexInputSplit mapDistributable = (BlockletIndexInputSplit) distributable;
    List<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper;
    String segmentNo = mapDistributable.getSegment().getSegmentNo();
    if (mapDistributable.getSegmentPath() != null) {
        identifiersWrapper = getTableBlockIndexUniqueIdentifier(distributable);
    } else {
        identifiersWrapper = getTableBlockIndexUniqueIdentifier(mapDistributable.getFilePath(), segmentNo);
    }
    List<CoarseGrainIndex> indexes = new ArrayList<>();
    try {
        List<BlockletIndexWrapper> wrappers = cache.getAll(identifiersWrapper);
        for (BlockletIndexWrapper wrapper : wrappers) {
            indexes.addAll(wrapper.getIndexes());
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return indexes;
}