Examples with SegmentColumnMetaDataInfo org.apache.carbondata.core.segmentmeta.SegmentColumnMetaDataInfo used on opensource projects

Example 1 with SegmentColumnMetaDataInfo

use of org.apache.carbondata.core.segmentmeta.SegmentColumnMetaDataInfo in project carbondata by apache.

the class BlockletIndexFactory method getTableBlockIndexUniqueIdentifierUsingSegmentMinMax.

/**
 * Using blockLevel minmax values, identify if segment has to be added for further pruning and to
 * load segment index info to cache
 * @param segment to be identified if needed for loading block indexes
 * @param segmentMetaDataInfo list of block level min max values
 * @param filter filter expression
 * @param identifiers tableBlockIndexUniqueIdentifiers
 * @param tableBlockIndexUniqueIdentifierWrappers to add tableBlockIndexUniqueIdentifiers
 */
private void getTableBlockIndexUniqueIdentifierUsingSegmentMinMax(Segment segment, SegmentMetaDataInfo segmentMetaDataInfo, IndexFilter filter, Set<TableBlockIndexUniqueIdentifier> identifiers, List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers) {
    boolean isScanRequired = false;
    Map<String, SegmentColumnMetaDataInfo> segmentColumnMetaDataInfoMap = segmentMetaDataInfo.getSegmentColumnMetaDataInfoMap();
    int length = segmentColumnMetaDataInfoMap.size();
    // Add columnSchemas based on the columns present in segment
    List<ColumnSchema> columnSchemas = new ArrayList<>();
    byte[][] min = new byte[length][];
    byte[][] max = new byte[length][];
    boolean[] minMaxFlag = new boolean[length];
    int i = 0;
    // get current columnSchema list for the table
    Map<String, ColumnSchema> tableColumnSchemas = this.getCarbonTable().getTableInfo().getFactTable().getListOfColumns().stream().collect(Collectors.toMap(ColumnSchema::getColumnUniqueId, ColumnSchema::clone));
    // fill min,max and columnSchema values
    for (Map.Entry<String, SegmentColumnMetaDataInfo> columnMetaData : segmentColumnMetaDataInfoMap.entrySet()) {
        ColumnSchema columnSchema = tableColumnSchemas.get(columnMetaData.getKey());
        if (null != columnSchema) {
            // get segment sort column and column drift info
            boolean isSortColumnInSegment = columnMetaData.getValue().isSortColumn();
            boolean isColumnDriftInSegment = columnMetaData.getValue().isColumnDrift();
            if (null != columnSchema.getColumnProperties()) {
                // get current sort column and column drift info from current columnSchema
                String isSortColumn = columnSchema.getColumnProperties().get(CarbonCommonConstants.SORT_COLUMNS);
                String isColumnDrift = columnSchema.getColumnProperties().get(CarbonCommonConstants.COLUMN_DRIFT);
                if (null != isSortColumn) {
                    if (isSortColumn.equalsIgnoreCase("true") && !isSortColumnInSegment) {
                        // Unset current column schema column properties
                        modifyColumnSchemaForSortColumn(columnSchema, isColumnDriftInSegment, isColumnDrift, false);
                    } else if (isSortColumn.equalsIgnoreCase("false") && isSortColumnInSegment) {
                        // set sort column to true in current column schema column properties
                        modifyColumnSchemaForSortColumn(columnSchema, isColumnDriftInSegment, isColumnDrift, true);
                    }
                } else {
                    modifyColumnSchemaForSortColumn(columnSchema, isColumnDriftInSegment, isColumnDrift, false);
                }
            }
            columnSchemas.add(columnSchema);
            min[i] = columnMetaData.getValue().getColumnMinValue();
            max[i] = columnMetaData.getValue().getColumnMaxValue();
            minMaxFlag[i] = min[i].length != 0 && max[i].length != 0;
            i++;
        }
    }
    // get segmentProperties using created columnSchemas list
    SegmentProperties segmentProperties = SegmentPropertiesAndSchemaHolder.getInstance().addSegmentProperties(this.getCarbonTable(), columnSchemas, segment.getSegmentNo()).getSegmentProperties();
    FilterResolverIntf resolver = new IndexFilter(segmentProperties, this.getCarbonTable(), filter.getExpression()).getResolver();
    // prepare filter executor using IndexFilter resolver
    FilterExecutor filterExecutor = FilterUtil.getFilterExecutorTree(resolver, segmentProperties, null, null, false);
    // check if block has to be pruned based on segment minmax
    BitSet scanRequired = filterExecutor.isScanRequired(max, min, minMaxFlag);
    if (!scanRequired.isEmpty()) {
        isScanRequired = true;
    }
    if (isScanRequired) {
        for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
            tableBlockIndexUniqueIdentifierWrappers.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
        }
    }
}