Search in sources :

Example 1 with Type

use of org.apache.avro.Schema.Type in project pinot by linkedin.

the class AvroQueryGenerator method getColumnType.

public static DataType getColumnType(Field field) {
    org.apache.avro.Schema fieldSchema = field.schema();
    fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);
    final Type type = fieldSchema.getType();
    if (type == Type.ARRAY) {
        org.apache.avro.Schema elementSchema = extractSchemaFromUnionIfNeeded(fieldSchema.getElementType());
        if (elementSchema.getType() == Type.RECORD) {
            if (elementSchema.getFields().size() == 1) {
                elementSchema = elementSchema.getFields().get(0).schema();
            } else {
                throw new RuntimeException("More than one schema in Multi-value column!");
            }
            elementSchema = extractSchemaFromUnionIfNeeded(elementSchema);
        }
        return DataType.valueOf(elementSchema.getType());
    } else {
        return DataType.valueOf(type);
    }
}
Also used : DataType(com.linkedin.pinot.common.data.FieldSpec.DataType) Type(org.apache.avro.Schema.Type) Schema(org.apache.avro.Schema)

Example 2 with Type

use of org.apache.avro.Schema.Type in project pinot by linkedin.

the class AvroRecordReader method isSingleValueField.

public static boolean isSingleValueField(Field field) {
    org.apache.avro.Schema fieldSchema = field.schema();
    fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);
    final Type type = fieldSchema.getType();
    if (type == Type.ARRAY) {
        return false;
    }
    return true;
}
Also used : Type(org.apache.avro.Schema.Type) DataType(com.linkedin.pinot.common.data.FieldSpec.DataType)

Example 3 with Type

use of org.apache.avro.Schema.Type in project pinot by linkedin.

the class AvroRecordReader method getColumnType.

public static DataType getColumnType(Field field) {
    org.apache.avro.Schema fieldSchema = field.schema();
    fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);
    final Type type = fieldSchema.getType();
    if (type == Type.ARRAY) {
        org.apache.avro.Schema elementSchema = extractSchemaFromUnionIfNeeded(fieldSchema.getElementType());
        if (elementSchema.getType() == Type.RECORD) {
            if (elementSchema.getFields().size() == 1) {
                elementSchema = elementSchema.getFields().get(0).schema();
            } else {
                throw new RuntimeException("More than one schema in Multi-value column!");
            }
            elementSchema = extractSchemaFromUnionIfNeeded(elementSchema);
        }
        return DataType.valueOf(elementSchema.getType());
    } else {
        return DataType.valueOf(type);
    }
}
Also used : Type(org.apache.avro.Schema.Type) DataType(com.linkedin.pinot.common.data.FieldSpec.DataType)

Example 4 with Type

use of org.apache.avro.Schema.Type in project pinot by linkedin.

the class SegmentTestUtils method isSingleValueField.

private static boolean isSingleValueField(Field field) {
    org.apache.avro.Schema fieldSchema = field.schema();
    fieldSchema = extractSchemaFromUnionIfNeeded(fieldSchema);
    final Type type = fieldSchema.getType();
    if (type == Type.ARRAY) {
        return false;
    }
    return true;
}
Also used : Type(org.apache.avro.Schema.Type) DataType(com.linkedin.pinot.common.data.FieldSpec.DataType) FieldType(com.linkedin.pinot.common.data.FieldSpec.FieldType)

Example 5 with Type

use of org.apache.avro.Schema.Type in project beam by apache.

the class BigQueryAvroUtils method convertRequiredField.

private static Object convertRequiredField(Type avroType, TableFieldSchema fieldSchema, Object v) {
    // REQUIRED fields are represented as the corresponding Avro types. For example, a BigQuery
    // INTEGER type maps to an Avro LONG type.
    checkNotNull(v, "REQUIRED field %s should not be null", fieldSchema.getName());
    // Per https://cloud.google.com/bigquery/docs/reference/v2/tables#schema, the type field
    // is required, so it may not be null.
    String bqType = fieldSchema.getType();
    Type expectedAvroType = BIG_QUERY_TO_AVRO_TYPES.get(bqType);
    verifyNotNull(expectedAvroType, "Unsupported BigQuery type: %s", bqType);
    verify(avroType == expectedAvroType, "Expected Avro schema type %s, not %s, for BigQuery %s field %s", expectedAvroType, avroType, bqType, fieldSchema.getName());
    switch(fieldSchema.getType()) {
        case "STRING":
        case "DATE":
        case "DATETIME":
        case "TIME":
            // Avro will use a CharSequence to represent String objects, but it may not always use
            // java.lang.String; for example, it may prefer org.apache.avro.util.Utf8.
            verify(v instanceof CharSequence, "Expected CharSequence (String), got %s", v.getClass());
            return v.toString();
        case "INTEGER":
            verify(v instanceof Long, "Expected Long, got %s", v.getClass());
            return ((Long) v).toString();
        case "FLOAT":
            verify(v instanceof Double, "Expected Double, got %s", v.getClass());
            return v;
        case "BOOLEAN":
            verify(v instanceof Boolean, "Expected Boolean, got %s", v.getClass());
            return v;
        case "TIMESTAMP":
            // TIMESTAMP data types are represented as Avro LONG types. They are converted back to
            // Strings with variable-precision (up to six digits) to match the JSON files export
            // by BigQuery.
            verify(v instanceof Long, "Expected Long, got %s", v.getClass());
            Double doubleValue = ((Long) v) / 1000000.0;
            return formatTimestamp(doubleValue.toString());
        case "RECORD":
            verify(v instanceof GenericRecord, "Expected GenericRecord, got %s", v.getClass());
            return convertGenericRecordToTableRow((GenericRecord) v, fieldSchema.getFields());
        case "BYTES":
            verify(v instanceof ByteBuffer, "Expected ByteBuffer, got %s", v.getClass());
            ByteBuffer byteBuffer = (ByteBuffer) v;
            byte[] bytes = new byte[byteBuffer.limit()];
            byteBuffer.get(bytes);
            return BaseEncoding.base64().encode(bytes);
        default:
            throw new UnsupportedOperationException(String.format("Unexpected BigQuery field schema type %s for field named %s", fieldSchema.getType(), fieldSchema.getName()));
    }
}
Also used : Type(org.apache.avro.Schema.Type) GenericRecord(org.apache.avro.generic.GenericRecord) ByteBuffer(java.nio.ByteBuffer)

Aggregations

Type (org.apache.avro.Schema.Type)57 Schema (org.apache.avro.Schema)39 Field (org.apache.avro.Schema.Field)20 HashMap (java.util.HashMap)10 ArrayList (java.util.ArrayList)9 ByteBuffer (java.nio.ByteBuffer)8 Map (java.util.Map)8 LogicalType (org.apache.avro.LogicalType)7 DataType (com.linkedin.pinot.common.data.FieldSpec.DataType)6 List (java.util.List)6 IOException (java.io.IOException)5 LinkedHashMap (java.util.LinkedHashMap)5 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)5 PersistentBase (org.apache.gora.persistency.impl.PersistentBase)5 SQLException (java.sql.SQLException)4 GenericArray (org.apache.avro.generic.GenericArray)4 GenericRecord (org.apache.avro.generic.GenericRecord)4 Utf8 (org.apache.avro.util.Utf8)4 EventCreationException (com.linkedin.databus2.producers.EventCreationException)3 SourceType (com.linkedin.databus2.relay.config.ReplicationBitSetterStaticConfig.SourceType)3