Examples with StructField org.apache.carbondata.core.metadata.datatype.StructField used on opensource projects

Example 1 with StructField

use of org.apache.carbondata.core.metadata.datatype.StructField in project carbondata by apache.

the class CarbonWriterBuilder method buildCarbonTable.

/**
 * Build a {@link CarbonTable}
 */
private CarbonTable buildCarbonTable() {
    TableSchemaBuilder tableSchemaBuilder = TableSchema.builder();
    if (blockletSize > 0) {
        tableSchemaBuilder = tableSchemaBuilder.blockSize(blockSize);
    }
    List<String> sortColumnsList;
    if (sortColumns != null) {
        sortColumnsList = Arrays.asList(sortColumns);
    } else {
        sortColumnsList = new LinkedList<>();
    }
    for (Field field : schema.getFields()) {
        tableSchemaBuilder.addColumn(new StructField(field.getFieldName(), field.getDataType()), sortColumnsList.contains(field.getFieldName()));
    }
    String tableName = "_tempTable";
    String dbName = "_tempDB";
    TableSchema schema = tableSchemaBuilder.build();
    schema.setTableName(tableName);
    CarbonTable table = CarbonTable.builder().tableName(schema.getTableName()).databaseName(dbName).tablePath(path).tableSchema(schema).build();
    return table;
}

Example 2 with StructField

use of org.apache.carbondata.core.metadata.datatype.StructField in project carbondata by apache.

the class CarbonTableOutputFormat method setFileHeader.

private static void setFileHeader(Configuration configuration, CarbonLoadModel model) throws IOException {
    StructType inputSchema = getInputSchema(configuration);
    if (inputSchema == null || inputSchema.getFields().size() == 0) {
        throw new UnsupportedOperationException("Input schema must be set");
    }
    List<StructField> fields = inputSchema.getFields();
    StringBuilder builder = new StringBuilder();
    String[] columns = new String[fields.size()];
    int i = 0;
    for (StructField field : fields) {
        builder.append(field.getFieldName());
        builder.append(",");
        columns[i++] = field.getFieldName();
    }
    String header = builder.toString();
    model.setCsvHeader(header.substring(0, header.length() - 1));
    model.setCsvHeaderColumns(columns);
}

Example 3 with StructField

use of org.apache.carbondata.core.metadata.datatype.StructField in project carbondata by apache.

the class CarbonColumnVectorWrapper method convertSparkToCarbonDataType.

// TODO: this is copied from carbondata-spark-common module, use presto type instead of this
private org.apache.carbondata.core.metadata.datatype.DataType convertSparkToCarbonDataType(org.apache.spark.sql.types.DataType dataType) {
    if (dataType instanceof StringType) {
        return DataTypes.STRING;
    } else if (dataType instanceof ShortType) {
        return DataTypes.SHORT;
    } else if (dataType instanceof IntegerType) {
        return DataTypes.INT;
    } else if (dataType instanceof LongType) {
        return DataTypes.LONG;
    } else if (dataType instanceof DoubleType) {
        return DataTypes.DOUBLE;
    } else if (dataType instanceof FloatType) {
        return DataTypes.FLOAT;
    } else if (dataType instanceof DateType) {
        return DataTypes.DATE;
    } else if (dataType instanceof BooleanType) {
        return DataTypes.BOOLEAN;
    } else if (dataType instanceof TimestampType) {
        return DataTypes.TIMESTAMP;
    } else if (dataType instanceof NullType) {
        return DataTypes.NULL;
    } else if (dataType instanceof DecimalType) {
        DecimalType decimal = (DecimalType) dataType;
        return DataTypes.createDecimalType(decimal.precision(), decimal.scale());
    } else if (dataType instanceof ArrayType) {
        org.apache.spark.sql.types.DataType elementType = ((ArrayType) dataType).elementType();
        return DataTypes.createArrayType(convertSparkToCarbonDataType(elementType));
    } else if (dataType instanceof StructType) {
        StructType structType = (StructType) dataType;
        org.apache.spark.sql.types.StructField[] fields = structType.fields();
        List<StructField> carbonFields = new ArrayList<>();
        for (org.apache.spark.sql.types.StructField field : fields) {
            carbonFields.add(new StructField(field.name(), convertSparkToCarbonDataType(field.dataType())));
        }
        return DataTypes.createStructType(carbonFields);
    } else {
        throw new UnsupportedOperationException("getting " + dataType + " from presto");
    }
}

Example 4 with StructField

use of org.apache.carbondata.core.metadata.datatype.StructField in project carbondata by apache.

the class TableSchemaBuilderSuite method testBuilder.

@Test
public void testBuilder() {
    TableSchemaBuilder builder = TableSchema.builder();
    builder.addColumn(new StructField("a", DataTypes.INT), true);
    builder.addColumn(new StructField("b", DataTypes.DOUBLE), false);
    TableSchema schema = builder.build();
    Assert.assertEquals(2, schema.getListOfColumns().size());
    List<ColumnSchema> columns = schema.getListOfColumns();
    Assert.assertEquals("a", columns.get(0).getColumnName());
    Assert.assertEquals("b", columns.get(1).getColumnName());
}

Example 5 with StructField

use of org.apache.carbondata.core.metadata.datatype.StructField in project carbondata by apache.

the class TableSchemaBuilderSuite method testRepeatedColumn.

@Test(expected = IllegalArgumentException.class)
public void testRepeatedColumn() {
    TableSchemaBuilder builder = TableSchema.builder();
    builder.addColumn(new StructField("a", DataTypes.INT), true);
    builder.addColumn(new StructField("a", DataTypes.DOUBLE), false);
    TableSchema schema = builder.build();
}