Examples with MinorType org.apache.drill.common.types.TypeProtos.MinorType used on opensource projects

Example 81 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class SyslogBatchReader method buildSchema.

public TupleMetadata buildSchema() {
    SchemaBuilder builder = new SchemaBuilder();
    for (Map.Entry<String, MinorType> entry : mappedColumns.entrySet()) {
        builder.addNullable(entry.getKey(), entry.getValue());
    }
    if (!config.flattenStructuredData()) {
        ColumnMetadata structuredDataMap = MetadataUtils.newMap(STRUCTURED_DATA_MAP_NAME);
        builder.add(structuredDataMap);
    }
    builder.addNullable("message", MinorType.VARCHAR);
    // Add _raw column
    ColumnMetadata colSchema = MetadataUtils.newScalar(RAW_COLUMN_NAME, MinorType.VARCHAR, TypeProtos.DataMode.OPTIONAL);
    colSchema.setBooleanProperty(ColumnMetadata.EXCLUDE_FROM_WILDCARD, true);
    builder.add(colSchema);
    return builder.buildSchema();
}

Example 82 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class JdbcRecordWriter method updateSchema.

@Override
public void updateSchema(VectorAccessible batch) {
    BatchSchema schema = batch.getSchema();
    String columnName;
    MinorType type;
    String sql;
    boolean nullable = false;
    CreateTableStmtBuilder queryBuilder = new CreateTableStmtBuilder(tableName, dialect);
    for (MaterializedField field : schema) {
        columnName = JdbcDDLQueryUtils.addBackTicksToField(field.getName());
        type = field.getType().getMinorType();
        logger.debug("Adding column {} of type {}.", columnName, type);
        if (field.getType().getMode() == DataMode.REPEATED) {
            throw UserException.dataWriteError().message("Drill does not yet support writing arrays to JDBC. " + columnName + " is an array.").build(logger);
        }
        if (field.getType().getMode() == DataMode.OPTIONAL) {
            nullable = true;
        }
        int precision = field.getPrecision();
        int scale = field.getScale();
        queryBuilder.addColumn(columnName, field.getType().getMinorType(), nullable, precision, scale);
    }
    sql = queryBuilder.build().getCreateTableQuery();
    sql = JdbcDDLQueryUtils.cleanDDLQuery(sql, dialect);
    logger.debug("Final query: {}", sql);
    // Execute the query to build the schema
    try (Statement statement = connection.createStatement()) {
        logger.debug("Executing CREATE query: {}", sql);
        statement.execute(sql);
    } catch (SQLException e) {
        throw UserException.dataReadError(e).message("The JDBC storage plugin failed while trying to create the schema. ").addContext("Sql", sql).build(logger);
    }
}

Example 83 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class OpenTSDBRecordReader method initCols.

private void initCols(Schema schema) throws SchemaChangeException {
    ImmutableList.Builder<ProjectedColumnInfo> pciBuilder = ImmutableList.builder();
    for (int i = 0; i < schema.getColumnCount(); i++) {
        ColumnDTO column = schema.getColumnByIndex(i);
        final String name = column.getColumnName();
        final OpenTSDBTypes type = column.getColumnType();
        TypeProtos.MinorType minorType = TYPES.get(type);
        if (isMinorTypeNull(minorType)) {
            String message = String.format("A column you queried has a data type that is not currently supported by the OpenTSDB storage plugin. " + "The column's name was %s and its OpenTSDB data type was %s. ", name, type.toString());
            throw UserException.unsupportedError().message(message).build(logger);
        }
        ProjectedColumnInfo pci = getProjectedColumnInfo(column, name, minorType);
        pciBuilder.add(pci);
    }
    projectedCols = pciBuilder.build();
}

Example 84 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class ListState method addColumn.

/**
 * Add a new column representing a type within the list. This is where the list
 * strangeness occurs. The list starts with no type, then evolves to have a single
 * type (held by the list vector). Upon the second type, the list vector is modified
 * to hold a union vector, which then holds the existing type and the new type.
 * After that, the third and later types simply are added to the union.
 * Very, very ugly, but it is how the list vector works until we improve it...
 * <p>
 * We must make three parallel changes:
 * <ul>
 * <li>Modify the list vector structure.</li>
 * <li>Modify the union writer structure. (If a list type can evolve, then the
 * writer structure is an array of unions. But, since the union itself does
 * not exist in the 0 and 1 type cases, we use "shims" to model these odd
 * cases.</li>
 * <li>Modify the vector state for the list. If the list is "promoted" to a
 * union, then add the union to the list vector's state for management in
 * vector events.</li>
 * </ul>
 */
@Override
protected void addColumn(ColumnState colState) {
    final MinorType type = colState.schema().type();
    assert !columns.containsKey(type);
    // Add the new column (type) to metadata.
    final int prevColCount = columns.size();
    columns.put(type, colState);
    if (prevColCount == 0) {
        addFirstType(colState);
    } else if (prevColCount == 1) {
        addSecondType(colState);
    } else {
        // Already have a union; just add another type.
        unionVector().addType(colState.vector());
    }
}

Example 85 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class ParquetRecordWriter method getType.

private Type getType(MaterializedField field) {
    MinorType minorType = field.getType().getMinorType();
    DataMode dataMode = field.getType().getMode();
    switch(minorType) {
        case MAP:
            List<Type> types = getChildrenTypes(field);
            return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getName(), types);
        case DICT:
            // RepeatedDictVector has DictVector as data vector hence the need to get the first child
            // for REPEATED case to be able to access map's key and value fields
            MaterializedField dictField = dataMode != DataMode.REPEATED ? field : ((List<MaterializedField>) field.getChildren()).get(0);
            List<Type> keyValueTypes = getChildrenTypes(dictField);
            GroupType keyValueGroup = new GroupType(Repetition.REPEATED, GROUP_KEY_VALUE_NAME, keyValueTypes);
            if (dataMode == DataMode.REPEATED) {
                // Parquet's MAP repetition must be either optional or required, so nest it inside Parquet's LIST type
                GroupType elementType = org.apache.parquet.schema.Types.buildGroup(Repetition.OPTIONAL).as(OriginalType.MAP).addField(keyValueGroup).named(LIST);
                GroupType listGroup = new GroupType(Repetition.REPEATED, LIST, elementType);
                return org.apache.parquet.schema.Types.buildGroup(Repetition.OPTIONAL).as(OriginalType.LIST).addField(listGroup).named(field.getName());
            } else {
                return org.apache.parquet.schema.Types.buildGroup(Repetition.OPTIONAL).as(OriginalType.MAP).addField(keyValueGroup).named(field.getName());
            }
        case LIST:
            MaterializedField elementField = getDataField(field);
            ListBuilder<GroupType> listBuilder = org.apache.parquet.schema.Types.list(dataMode == DataMode.OPTIONAL ? Repetition.OPTIONAL : Repetition.REQUIRED);
            addElementType(listBuilder, elementField);
            GroupType listType = listBuilder.named(field.getName());
            return listType;
        case NULL:
            MaterializedField newField = field.withType(TypeProtos.MajorType.newBuilder().setMinorType(MinorType.INT).setMode(DataMode.OPTIONAL).build());
            return getPrimitiveType(newField);
        default:
            return getPrimitiveType(field);
    }
}