Examples with CarbonColumn org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn used on opensource projects

Example 11 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class SchemaGenerator method getColumnOrdinalsToAccess.

/**
 * Method to fill the column ordinals to access based on the columns to be cached
 *
 * @param segmentProperties
 * @param minMaxCacheColumns
 * @return
 */
private static int[] getColumnOrdinalsToAccess(SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns) {
    int[] columnOrdinalsTOAccess = null;
    if (null != minMaxCacheColumns) {
        columnOrdinalsTOAccess = new int[minMaxCacheColumns.size()];
        int counter = 0;
        for (CarbonColumn column : minMaxCacheColumns) {
            columnOrdinalsTOAccess[counter++] = BlockletIndexUtil.getColumnOrdinal(segmentProperties, column);
        }
    } else {
        // when columns to cache is not specified then column access order will be same as the array
        // index of min max length
        columnOrdinalsTOAccess = new int[segmentProperties.getNumberOfColumns()];
        for (int i = 0; i < columnOrdinalsTOAccess.length; i++) {
            columnOrdinalsTOAccess[i] = i;
        }
    }
    return columnOrdinalsTOAccess;
}

Example 12 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class FilterUtil method updateIndexOfColumnExpression.

public static void updateIndexOfColumnExpression(Expression exp, int dimOrdinalMax) {
    // if expression is null, not require to update index.
    if (exp == null) {
        return;
    }
    if (exp.getChildren() == null || exp.getChildren().size() == 0) {
        if (exp instanceof ColumnExpression) {
            ColumnExpression ce = (ColumnExpression) exp;
            CarbonColumn column = ce.getCarbonColumn();
            if (column.isDimension()) {
                ce.setColIndex(column.getOrdinal());
            } else {
                ce.setColIndex(dimOrdinalMax + column.getOrdinal());
            }
        }
    } else {
        if (exp.getChildren().size() > 0) {
            List<Expression> children = exp.getChildren();
            for (int i = 0; i < children.size(); i++) {
                updateIndexOfColumnExpression(children.get(i), dimOrdinalMax);
            }
        }
    }
}

Example 13 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class FilterUtil method checkIfFilterColumnIsCachedInDriver.

/**
 * Method to check whether current node needs to be replaced with true filter to avoid pruning
 * for case when filter column is not cached in the min/max cached dimension
 *
 * @param columnResolvedFilterInfo
 * @param segmentProperties
 * @param minMaxCacheColumns
 * @param isMeasure
 * @return
 */
private static boolean checkIfFilterColumnIsCachedInDriver(ColumnResolvedFilterInfo columnResolvedFilterInfo, SegmentProperties segmentProperties, List<CarbonColumn> minMaxCacheColumns, boolean isMeasure, boolean isStreamDataFile) {
    boolean replaceCurrentNodeWithTrueFilter = false;
    CarbonColumn columnFromCurrentBlock = null;
    if (isMeasure) {
        columnFromCurrentBlock = segmentProperties.getMeasureFromCurrentBlock(columnResolvedFilterInfo.getMeasure());
    } else {
        columnFromCurrentBlock = segmentProperties.getDimensionFromCurrentBlock(columnResolvedFilterInfo.getDimension());
    }
    if (null != columnFromCurrentBlock) {
        // check for filter dimension in the cached column list
        if (null != minMaxCacheColumns) {
            int columnIndexInMinMaxByteArray = getFilterColumnIndexInCachedColumns(minMaxCacheColumns, columnFromCurrentBlock);
            if (columnIndexInMinMaxByteArray != -1) {
                columnResolvedFilterInfo.setColumnIndexInMinMaxByteArray(columnIndexInMinMaxByteArray);
            } else {
                // will be true only if column caching is enabled and current filter column is not cached
                replaceCurrentNodeWithTrueFilter = true;
            }
        } else {
            // and  then the ordinal of column will be its index in the min/max byte array
            if (isMeasure) {
                // last dimension ordinal.
                if (isStreamDataFile) {
                    columnResolvedFilterInfo.setColumnIndexInMinMaxByteArray(segmentProperties.getDimensions().size() + columnFromCurrentBlock.getOrdinal());
                } else {
                    columnResolvedFilterInfo.setColumnIndexInMinMaxByteArray(segmentProperties.getLastDimensionColOrdinal() + columnFromCurrentBlock.getOrdinal());
                }
            } else {
                columnResolvedFilterInfo.setColumnIndexInMinMaxByteArray(columnFromCurrentBlock.getOrdinal());
            }
        }
    }
    return replaceCurrentNodeWithTrueFilter;
}

Example 14 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method initializeNoReArrangeIndexes.

private void initializeNoReArrangeIndexes() {
    // Data might have partition columns in the end in new insert into flow.
    // But when convert to 3 parts, just keep in internal order. so derive index for that.
    List<CarbonColumn> listOfColumns = new ArrayList<>();
    listOfColumns.addAll(configuration.getTableSpec().getCarbonTable().getVisibleDimensions());
    listOfColumns.addAll(configuration.getTableSpec().getCarbonTable().getVisibleMeasures());
    // In case of partition, partition data will be at the end. So, need to keep data position
    Map<String, Integer> dataPositionMap = new HashMap<>();
    int dataPosition = 0;
    for (DataField field : configuration.getDataFields()) {
        dataPositionMap.put(field.getColumn().getColName(), dataPosition++);
    }
    // get the index of each type and to be used in 3 parts conversion
    for (CarbonColumn column : listOfColumns) {
        if (column.hasEncoding(Encoding.DICTIONARY)) {
            directDictionaryDimensionIndex.add(dataPositionMap.get(column.getColName()));
        } else {
            if (column.getDataType().isComplexType()) {
                complexTypeIndex.add(dataPositionMap.get(column.getColName()));
            } else if (column.isMeasure()) {
                measureIndex.add(dataPositionMap.get(column.getColName()));
            } else {
                // other dimensions
                otherDimensionIndex.add(dataPositionMap.get(column.getColName()));
            }
        }
    }
}

Example 15 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class CarbonFactDataHandlerModel method convertComplexDimensionToComplexIndexMap.

/**
 * This routine takes the Complex Dimension and convert into generic DataType.
 *
 * @param segmentProperties
 * @param nullFormat
 * @return
 */
private static Map<Integer, GenericDataType> convertComplexDimensionToComplexIndexMap(SegmentProperties segmentProperties, String nullFormat) {
    List<CarbonDimension> complexDimensions = segmentProperties.getComplexDimensions();
    int simpleDimsCount = segmentProperties.getNumberOfPrimitiveDimensions();
    DataField[] dataFields = new DataField[complexDimensions.size()];
    int i = 0;
    for (CarbonColumn complexDimension : complexDimensions) {
        dataFields[i++] = new DataField(complexDimension);
    }
    return getComplexMap(nullFormat, simpleDimsCount, dataFields);
}