Examples with DataType org.apache.spark.sql.types.DataType used on opensource projects

Example 6 with DataType

use of org.apache.spark.sql.types.DataType in project incubator-systemml by apache.

the class FrameRDDConverterUtils method convertFrameSchemaToDFSchema.

/**
 * This function will convert Frame schema into DataFrame schema
 *
 * @param fschema frame schema
 * @param containsID true if contains ID column
 * @return Spark StructType of StructFields representing schema
 */
public static StructType convertFrameSchemaToDFSchema(ValueType[] fschema, boolean containsID) {
    // generate the schema based on the string of schema
    List<StructField> fields = new ArrayList<>();
    // add id column type
    if (containsID)
        fields.add(DataTypes.createStructField(RDDConverterUtils.DF_ID_COLUMN, DataTypes.DoubleType, true));
    // add remaining types
    int col = 1;
    for (ValueType schema : fschema) {
        DataType dt = null;
        switch(schema) {
            case STRING:
                dt = DataTypes.StringType;
                break;
            case DOUBLE:
                dt = DataTypes.DoubleType;
                break;
            case INT:
                dt = DataTypes.LongType;
                break;
            case BOOLEAN:
                dt = DataTypes.BooleanType;
                break;
            default:
                dt = DataTypes.StringType;
                LOG.warn("Using default type String for " + schema.toString());
        }
        fields.add(DataTypes.createStructField("C" + col++, dt, true));
    }
    return DataTypes.createStructType(fields);
}