Examples with CarbonColumn org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn used on opensource projects

Example 31 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class StreamRecordReader method initialize.

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
    // input
    if (split instanceof CarbonInputSplit) {
        fileSplit = (CarbonInputSplit) split;
    } else if (split instanceof CarbonMultiBlockSplit) {
        fileSplit = ((CarbonMultiBlockSplit) split).getAllSplits().get(0);
    } else {
        fileSplit = (FileSplit) split;
    }
    // metadata
    hadoopConf = context.getConfiguration();
    if (model == null) {
        CarbonTableInputFormat<Object> format = new CarbonTableInputFormat<>();
        model = format.createQueryModel(split, context);
    }
    carbonTable = model.getTable();
    List<CarbonDimension> dimensions = carbonTable.getVisibleDimensions();
    dimensionCount = dimensions.size();
    List<CarbonMeasure> measures = carbonTable.getVisibleMeasures();
    measureCount = measures.size();
    List<CarbonColumn> carbonColumnList = carbonTable.getStreamStorageOrderColumn();
    storageColumns = carbonColumnList.toArray(new CarbonColumn[carbonColumnList.size()]);
    isNoDictColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(storageColumns);
    directDictionaryGenerators = new DirectDictionaryGenerator[storageColumns.length];
    for (int i = 0; i < storageColumns.length; i++) {
        if (storageColumns[i].getDataType() == DataTypes.DATE) {
            directDictionaryGenerators[i] = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(storageColumns[i].getDataType());
        }
    }
    dimensionsIsVarcharTypeMap = new boolean[dimensionCount];
    for (int i = 0; i < dimensionCount; i++) {
        dimensionsIsVarcharTypeMap[i] = storageColumns[i].getDataType() == DataTypes.VARCHAR;
    }
    measureDataTypes = new DataType[measureCount];
    for (int i = 0; i < measureCount; i++) {
        measureDataTypes[i] = storageColumns[dimensionCount + i].getDataType();
    }
    // decode data
    allNonNull = new BitSet(storageColumns.length);
    projection = model.getProjectionColumns();
    isRequired = new boolean[storageColumns.length];
    boolean[] isFilterDimensions = model.getIsFilterDimensions();
    boolean[] isFilterMeasures = model.getIsFilterMeasures();
    isFilterRequired = new boolean[storageColumns.length];
    filterMap = new int[storageColumns.length];
    for (int i = 0; i < storageColumns.length; i++) {
        if (storageColumns[i].isDimension()) {
            if (isFilterDimensions[storageColumns[i].getOrdinal()]) {
                isRequired[i] = true;
                isFilterRequired[i] = true;
                filterMap[i] = storageColumns[i].getOrdinal();
            }
        } else {
            if (isFilterMeasures[storageColumns[i].getOrdinal()]) {
                isRequired[i] = true;
                isFilterRequired[i] = true;
                filterMap[i] = carbonTable.getDimensionOrdinalMax() + storageColumns[i].getOrdinal();
            }
        }
    }
    isProjectionRequired = new boolean[storageColumns.length];
    projectionMap = new int[storageColumns.length];
    for (int j = 0; j < projection.length; j++) {
        for (int i = 0; i < storageColumns.length; i++) {
            if (storageColumns[i].getColName().equals(projection[j].getColName())) {
                isRequired[i] = true;
                isProjectionRequired[i] = true;
                projectionMap[i] = j;
                break;
            }
        }
    }
    // initialize filter
    if (null != model.getIndexFilter()) {
        initializeFilter();
    } else if (projection.length == 0) {
        skipScanData = true;
    }
}

Example 32 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class BloomCoarseGrainIndex method initIndexColumnConverters.

/**
 * init field converters for index columns
 */
public void initIndexColumnConverters(CarbonTable carbonTable, List<CarbonColumn> indexedColumn) {
    this.name2Col = new HashMap<>(indexedColumn.size());
    for (CarbonColumn col : indexedColumn) {
        this.name2Col.put(col.getColName(), col);
    }
    this.name2Converters = new HashMap<>(indexedColumn.size());
    String nullFormat = "\\N";
    for (int i = 0; i < indexedColumn.size(); i++) {
        DataField dataField = new DataField(indexedColumn.get(i));
        String dateFormat = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
        dataField.setDateFormat(dateFormat);
        String tsFormat = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
        dataField.setTimestampFormat(tsFormat);
        FieldConverter fieldConverter = FieldEncoderFactory.getInstance().createFieldEncoder(dataField, i, nullFormat, false, false, carbonTable.getTablePath(), null);
        this.name2Converters.put(indexedColumn.get(i).getColName(), fieldConverter);
    }
    this.badRecordLogHolder = new BadRecordLogHolder();
    this.badRecordLogHolder.setLogged(false);
}

Example 33 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class Util method convertToSparkSchema.

public static StructType convertToSparkSchema(CarbonTable table) {
    List<CarbonColumn> columns = table.getCreateOrderColumn();
    ColumnSchema[] schema = new ColumnSchema[columns.size()];
    int i = 0;
    for (CarbonColumn column : columns) {
        schema[i++] = column.getColumnSchema();
    }
    return convertToSparkSchema(table, schema, false);
}

Example 34 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class CarbonTable method getStreamStorageOrderColumn.

/**
 * This method will give storage order column list
 */
public List<CarbonColumn> getStreamStorageOrderColumn() {
    List<CarbonDimension> dimensions = visibleDimensions;
    List<CarbonMeasure> measures = visibleMeasures;
    List<CarbonColumn> columnList = new ArrayList<>(dimensions.size() + measures.size());
    List<CarbonColumn> complexDimensionList = new ArrayList<>(dimensions.size());
    for (CarbonColumn column : dimensions) {
        if (column.isComplex()) {
            complexDimensionList.add(column);
        } else {
            columnList.add(column);
        }
    }
    columnList.addAll(complexDimensionList);
    for (CarbonColumn column : measures) {
        if (!(column.getColName().equals("default_dummy_measure"))) {
            columnList.add(column);
        }
    }
    return columnList;
}

Example 35 with CarbonColumn

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn in project carbondata by apache.

the class FilterExpressionProcessorTest method testGetFilterResolverBasedOnExpressionType.

@Test
public void testGetFilterResolverBasedOnExpressionType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
    CarbonColumn carbonColumn = new CarbonColumn(columnSchema, 0, 0);
    ColumnExpression columnExpression = new ColumnExpression("IMEI", DataTypes.STRING);
    columnExpression.setCarbonColumn(carbonColumn);
    LiteralExpression literalExpression = new LiteralExpression("ImeiValue", DataTypes.STRING);
    InExpression equalToExpression = new InExpression(columnExpression, literalExpression);
    FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
    Method method = FilterExpressionProcessor.class.getDeclaredMethod("getFilterResolverBasedOnExpressionType", ExpressionType.class, boolean.class, Expression.class, AbsoluteTableIdentifier.class, Expression.class);
    method.setAccessible(true);
    Object result = method.invoke(filterExpressionProcessor, ExpressionType.EQUALS, false, equalToExpression, null, null);
    Assert.assertTrue(result.getClass().getName().equals("org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolverImpl"));
}