Examples with Type org.apache.avro.Schema.Type used on opensource projects

Example 71 with Type

use of org.apache.avro.Schema.Type in project incubator-gobblin by apache.

the class AvroToJdbcEntryConverter method convertSchema.

/**
 * Converts Avro schema to JdbcEntrySchema.
 *
 * Few precondition to the Avro schema
 * 1. Avro schema should have one entry type record at first depth.
 * 2. Avro schema can recurse by having record inside record.
 * 3. Supported Avro primitive types and conversion
 *  boolean --> java.lang.Boolean
 *  int --> java.lang.Integer
 *  long --> java.lang.Long or java.sql.Date , java.sql.Time , java.sql.Timestamp
 *  float --> java.lang.Float
 *  double --> java.lang.Double
 *  bytes --> byte[]
 *  string --> java.lang.String
 *  null: only allowed if it's within union (see complex types for more details)
 * 4. Supported Avro complex types
 *  Records: Supports nested record type as well.
 *  Enum --> java.lang.String
 *  Unions --> Only allowed if it have one primitive type in it, along with Record type, or null type with one primitive type where null will be ignored.
 *  Once Union is narrowed down to one primitive type, it will follow conversion of primitive type above.
 * {@inheritDoc}
 *
 * 5. In order to make conversion from Avro long type to java.sql.Date or java.sql.Time or java.sql.Timestamp,
 * converter will get table metadata from JDBC.
 * 6. As it needs JDBC connection from condition 5, it also assumes that it will use JDBC publisher where it will get connection information from.
 * 7. Conversion assumes that both schema, Avro and JDBC, uses same column name where name space in Avro is ignored.
 *    For case sensitivity, Avro is case sensitive where it differs in JDBC based on underlying database. As Avro is case sensitive, column name equality also take case sensitive in to account.
 *
 * @see org.apache.gobblin.converter.Converter#convertSchema(java.lang.Object, org.apache.gobblin.configuration.WorkUnitState)
 */
@Override
public JdbcEntrySchema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException {
    LOG.info("Converting schema " + inputSchema);
    Preconditions.checkArgument(Type.RECORD.equals(inputSchema.getType()), "%s is expected for the first level element in Avro schema %s", Type.RECORD, inputSchema);
    Map<String, Type> avroColumnType = flatten(inputSchema);
    String jsonStr = Preconditions.checkNotNull(workUnit.getProp(CONVERTER_AVRO_JDBC_DATE_FIELDS));
    java.lang.reflect.Type typeOfMap = new TypeToken<Map<String, JdbcType>>() {
    }.getType();
    Map<String, JdbcType> dateColumnMapping = new Gson().fromJson(jsonStr, typeOfMap);
    LOG.info("Date column mapping: " + dateColumnMapping);
    List<JdbcEntryMetaDatum> jdbcEntryMetaData = Lists.newArrayList();
    for (Map.Entry<String, Type> avroEntry : avroColumnType.entrySet()) {
        String colName = tryConvertAvroColNameToJdbcColName(avroEntry.getKey());
        JdbcType JdbcType = dateColumnMapping.get(colName);
        if (JdbcType == null) {
            JdbcType = AVRO_TYPE_JDBC_TYPE_MAPPING.get(avroEntry.getValue());
        }
        Preconditions.checkNotNull(JdbcType, "Failed to convert " + avroEntry + " AVRO_TYPE_JDBC_TYPE_MAPPING: " + AVRO_TYPE_JDBC_TYPE_MAPPING + " , dateColumnMapping: " + dateColumnMapping);
        jdbcEntryMetaData.add(new JdbcEntryMetaDatum(colName, JdbcType));
    }
    JdbcEntrySchema converted = new JdbcEntrySchema(jdbcEntryMetaData);
    LOG.info("Converted schema into " + converted);
    return converted;
}

Example 72 with Type

use of org.apache.avro.Schema.Type in project avro-util by linkedin.

the class RecordBuilderBase method isValidValue.

protected static boolean isValidValue(Field f, Object value) {
    if (value != null) {
        return true;
    } else {
        Schema schema = f.schema();
        Type type = schema.getType();
        if (type == Type.NULL) {
            return true;
        } else {
            if (type == Type.UNION) {
                Iterator i$ = schema.getTypes().iterator();
                while (i$.hasNext()) {
                    Schema s = (Schema) i$.next();
                    if (s.getType() == Type.NULL) {
                        return true;
                    }
                }
            }
            return false;
        }
    }
}

Example 73 with Type

use of org.apache.avro.Schema.Type in project akhq by tchiotludo.

the class AvroDeserializer method objectDeserializer.

@SuppressWarnings("unchecked")
private static Object objectDeserializer(Object value, Schema schema) {
    LogicalType logicalType = schema.getLogicalType();
    Type primitiveType = schema.getType();
    if (logicalType != null) {
        switch(logicalType.getName()) {
            case DATE:
                return AvroDeserializer.dateDeserializer(value, schema, primitiveType, logicalType);
            case DECIMAL:
                return AvroDeserializer.decimalDeserializer(value, schema, primitiveType, logicalType);
            case TIME_MICROS:
                return AvroDeserializer.timeMicrosDeserializer(value, schema, primitiveType, logicalType);
            case TIME_MILLIS:
                return AvroDeserializer.timeMillisDeserializer(value, schema, primitiveType, logicalType);
            case TIMESTAMP_MICROS:
                return AvroDeserializer.timestampMicrosDeserializer(value, schema, primitiveType, logicalType);
            case TIMESTAMP_MILLIS:
                return AvroDeserializer.timestampMillisDeserializer(value, schema, primitiveType, logicalType);
            case UUID:
                return AvroDeserializer.uuidDeserializer(value, schema, primitiveType, logicalType);
            default:
                throw new IllegalStateException("Unexpected value: " + logicalType);
        }
    } else {
        switch(primitiveType) {
            case UNION:
                return AvroDeserializer.unionDeserializer(value, schema);
            case MAP:
                return AvroDeserializer.mapDeserializer((Map<String, ?>) value, schema);
            case RECORD:
                return AvroDeserializer.recordDeserializer((GenericRecord) value);
            case ENUM:
                return value.toString();
            case ARRAY:
                return arrayDeserializer((Collection<?>) value, schema);
            case FIXED:
                return ((GenericFixed) value).bytes();
            case STRING:
                return ((CharSequence) value).toString();
            case BYTES:
                return ((ByteBuffer) value).array();
            case INT:
            case LONG:
            case FLOAT:
            case DOUBLE:
            case BOOLEAN:
            case NULL:
                return value;
            default:
                throw new IllegalStateException("Unexpected value: " + primitiveType);
        }
    }
}

Example 74 with Type

use of org.apache.avro.Schema.Type in project avro by a0x8o.

the class SpecificData method createSchema.

/**
 * Create the schema for a Java type.
 */
@SuppressWarnings(value = "unchecked")
protected Schema createSchema(java.lang.reflect.Type type, Map<String, Schema> names) {
    if (type instanceof Class && CharSequence.class.isAssignableFrom((Class) type))
        return Schema.create(Type.STRING);
    else if (type == ByteBuffer.class)
        return Schema.create(Type.BYTES);
    else if ((type == Integer.class) || (type == Integer.TYPE))
        return Schema.create(Type.INT);
    else if ((type == Long.class) || (type == Long.TYPE))
        return Schema.create(Type.LONG);
    else if ((type == Float.class) || (type == Float.TYPE))
        return Schema.create(Type.FLOAT);
    else if ((type == Double.class) || (type == Double.TYPE))
        return Schema.create(Type.DOUBLE);
    else if ((type == Boolean.class) || (type == Boolean.TYPE))
        return Schema.create(Type.BOOLEAN);
    else if ((type == Void.class) || (type == Void.TYPE))
        return Schema.create(Type.NULL);
    else if (type instanceof ParameterizedType) {
        ParameterizedType ptype = (ParameterizedType) type;
        Class raw = (Class) ptype.getRawType();
        java.lang.reflect.Type[] params = ptype.getActualTypeArguments();
        if (Collection.class.isAssignableFrom(raw)) {
            // array
            if (params.length != 1)
                throw new AvroTypeException("No array type specified.");
            return Schema.createArray(createSchema(params[0], names));
        } else if (Map.class.isAssignableFrom(raw)) {
            // map
            java.lang.reflect.Type key = params[0];
            java.lang.reflect.Type value = params[1];
            if (!(key instanceof Class && CharSequence.class.isAssignableFrom((Class<?>) key)))
                throw new AvroTypeException("Map key class not CharSequence: " + SchemaUtil.describe(key));
            return Schema.createMap(createSchema(value, names));
        } else {
            return createSchema(raw, names);
        }
    } else if (type instanceof Class) {
        // class
        Class c = (Class) type;
        String fullName = c.getName();
        Schema schema = names.get(fullName);
        if (schema == null)
            try {
                schema = (Schema) (c.getDeclaredField("SCHEMA$").get(null));
                if (!fullName.equals(getClassName(schema)))
                    // HACK: schema mismatches class. maven shade plugin? try replacing.
                    schema = new Schema.Parser().parse(schema.toString().replace(schema.getNamespace(), c.getPackage().getName()));
            } catch (NoSuchFieldException e) {
                throw new AvroRuntimeException("Not a Specific class: " + c);
            } catch (IllegalAccessException e) {
                throw new AvroRuntimeException(e);
            }
        names.put(fullName, schema);
        return schema;
    }
    throw new AvroTypeException("Unknown type: " + type);
}

Example 75 with Type

use of org.apache.avro.Schema.Type in project knime-cloud by knime.

the class AbstractAmazonPersonalizeDataUploadNodeModel method createSchema.

private String createSchema(final AmazonPersonalize personalizeClient, final DataTableSpec spec) {
    final StringBuilder schemaNameBuilder = new StringBuilder(getSchemaNamePrefix());
    FieldAssembler<Schema> fieldAssembler = createFieldAssembler(SCHEMA_NAMESPACE);
    for (final String colName : spec.getColumnNames()) {
        if (!colName.startsWith(PREFIX_METADATA_FIELD)) {
            continue;
        }
        final DataColumnSpec colSpec = spec.getColumnSpec(colName);
        final boolean isCategorical;
        final Type type;
        if (colSpec.getType().isCompatible(StringValue.class)) {
            isCategorical = true;
            type = Type.STRING;
        } else if (colSpec.getType().isCompatible(IntValue.class)) {
            isCategorical = false;
            type = Type.INT;
        } else if (colSpec.getType().isCompatible(LongValue.class)) {
            isCategorical = false;
            type = Type.LONG;
        } else {
            isCategorical = false;
            type = Type.DOUBLE;
        }
        schemaNameBuilder.append("-" + type);
        // 'categorical' must be set for metadata
        fieldAssembler = fieldAssembler.name(colName).prop("categorical", isCategorical).type(Schema.create(type)).noDefault();
    }
    final String schemaName = schemaNameBuilder.toString();
    // check if the same schema has been created before
    final List<DatasetSchemaSummary> existingSchemas = AmazonPersonalizeUtils.listAllSchemas(personalizeClient);
    final Optional<DatasetSchemaSummary> schemaSummary = existingSchemas.stream().filter(e -> e.getName().equals(schemaName)).findAny();
    // if so, use this one again
    if (schemaSummary.isPresent()) {
        return schemaSummary.get().getSchemaArn();
    }
    // otherwise create new one
    final Schema schema = fieldAssembler.endRecord();
    final CreateSchemaRequest createSchemaRequest = new CreateSchemaRequest().withName(schemaName).withSchema(schema.toString());
    return personalizeClient.createSchema(createSchemaRequest).getSchemaArn();
}