Examples with RecordDataSchema com.linkedin.data.schema.RecordDataSchema used on opensource projects

Example 91 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestDataTranslator method testMissingDefaultFieldsOnDataMap.

@Test
public void testMissingDefaultFieldsOnDataMap() throws IOException {
    final String SCHEMA = "{" + "   \"type\":\"record\"," + "   \"name\":\"Foo\"," + "   \"fields\":[" + "      {" + "         \"name\":\"field1\"," + "         \"type\":\"string\"" + "      }," + "      {" + "         \"name\":\"field2\"," + "         \"type\":{" + "            \"type\":\"array\"," + "            \"items\":\"string\"" + "         }," + "         \"default\":[ ]" + "      }" + "   ]" + "}";
    RecordDataSchema pegasusSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(SCHEMA);
    Schema avroShema = Schema.parse(SCHEMA);
    DataMap dataMap = new DataMap();
    dataMap.put("field1", "test");
    GenericRecord record = DataTranslator.dataMapToGenericRecord(dataMap, pegasusSchema, avroShema);
    assertEquals(record.get("field2"), new GenericData.Array<>(0, Schema.createArray(Schema.create(Schema.Type.STRING))));
}

Example 92 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class PegasusUnionToAvroRecordConvertCallback method callback.

@SuppressWarnings("unchecked")
@Override
public void callback(List<String> path, DataSchema schema) {
    if (schema.getType() != DataSchema.Type.RECORD) {
        return;
    }
    // These are handled in AvroOverrideFactory#createFromDataSchema() while encoding the Avro schema.
    if (schema.getProperties().get("avro") != null) {
        return;
    }
    RecordDataSchema recordSchema = (RecordDataSchema) schema;
    for (RecordDataSchema.Field field : recordSchema.getFields()) {
        DataSchema fieldSchema = field.getType().getDereferencedDataSchema();
        Map<String, Object> propagatedProperties = (Map<String, Object>) SchemaToAvroJsonEncoder.produceFieldProperties(field, _options);
        // The conversion from Pegasus union type to an Avro record is performed when the union appears as either the
        // field's direct type or when the field's type is an array or a map whose (nested) elements is of union type.
        // This conversion ignores the default value when specified on an array or map typed field. Since the elements in
        // the default value collection can have conflicting union members, it will be hard to figure out the optionality
        // property of the fields in the generated Avro record.
        DataMap modifiedDefaultValue = modifyFieldDefaultValue(field, path);
        DataSchema modifiedSchema = modifyFieldSchema(recordSchema, field, fieldSchema, modifiedDefaultValue);
        if (modifiedSchema != null) {
            overrideUnionFieldSchemaAndDefault(field, modifiedSchema, modifiedDefaultValue, propagatedProperties);
        }
    }
}

Example 93 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class PegasusUnionToAvroRecordConvertCallback method modifyFieldSchema.

/**
 * Modify the schema for the specified field. The schema modification happens only for fields whose type is either a
 * union that uses aliases for its members or an array or map that has a similar union as its element type (either
 * direct or nested).
 *
 * @param recordSchema An instance of {@link RecordDataSchema} for the record that contains this field
 * @param field An instance of {@link com.linkedin.data.schema.RecordDataSchema.Field} whose schema needs to be translated
 * @param dataSchema An instance of {@link DataSchema} that is being translated. The initial call will have the field's
 *                   schema but when called recursively for arrays and maps, the schema will be of it's elements.
 * @param modifiedDefaultValue An instance of {@link DataMap} which is the modified default value for the field
 * @return An instance of {@link DataSchema} which is the translated schema for the field or null, if the schema
 * doesn't have to be translated
 */
private DataSchema modifyFieldSchema(RecordDataSchema recordSchema, RecordDataSchema.Field field, DataSchema dataSchema, DataMap modifiedDefaultValue) {
    DataSchema modifiedSchema = null;
    switch(dataSchema.getType()) {
        case ARRAY:
            DataSchema itemsSchema = ((ArrayDataSchema) dataSchema).getItems().getDereferencedDataSchema();
            // Stop propagating the default value if the field type is an array
            DataSchema modifiedItemsSchema = modifyFieldSchema(recordSchema, field, itemsSchema, null);
            if (modifiedItemsSchema != null) {
                modifiedSchema = new ArrayDataSchema(modifiedItemsSchema);
            }
            break;
        case MAP:
            DataSchema valuesSchema = ((MapDataSchema) dataSchema).getValues().getDereferencedDataSchema();
            // Stop propagating the default value if the field type is a map
            DataSchema modifiedValuesSchema = modifyFieldSchema(recordSchema, field, valuesSchema, null);
            if (modifiedValuesSchema != null) {
                modifiedSchema = new MapDataSchema(modifiedValuesSchema);
            }
            break;
        case UNION:
            UnionDataSchema unionDataSchema = (UnionDataSchema) dataSchema;
            if (unionDataSchema.areMembersAliased()) {
                modifiedSchema = buildContainerRecordFromUnion(unionDataSchema, field.getName(), recordSchema.getFullName(), modifiedDefaultValue);
            }
            break;
    }
    return modifiedSchema;
}

Example 94 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestCustomAvroSchema method translate.

private void translate(String dataSchemaFieldsJson, String avroSchemaFieldsJson, String dataJson, String avroDataJson) throws IOException {
    boolean debug = false;
    String fullSchemaJson = DATA_SCHEMA_JSON_TEMPLATE.replace("##FIELDS", dataSchemaFieldsJson);
    String avroSchemaFieldsJsonAfterVariableExpansion = avroSchemaFieldsJson.replace("##ANYRECORD_FULL_JSON", ANYRECORD_AVRO_SCHEMA_JSON).replace("##ANYRECORD_NAME", ANYRECORD_AVRO_FULL_NAME);
    String fullAvroSchemaJson = AVRO_SCHEMA_JSON_TEMPLATE.replace("##FIELDS", avroSchemaFieldsJsonAfterVariableExpansion);
    PegasusSchemaParser parser = TestUtil.schemaParserFromString(fullSchemaJson);
    assertFalse(parser.hasError(), parser.errorMessage());
    RecordDataSchema schema = (RecordDataSchema) parser.topLevelDataSchemas().get(2);
    String avroJsonOutput = SchemaTranslator.dataToAvroSchemaJson(schema);
    assertEquals(TestUtil.dataMapFromString(avroJsonOutput), TestUtil.dataMapFromString(fullAvroSchemaJson));
    Schema avroSchema = Schema.parse(avroJsonOutput);
    Schema avroSchema2 = SchemaTranslator.dataToAvroSchema(schema);
    assertEquals(avroSchema, avroSchema2);
    String avroSchemaToString = avroSchema.toString();
    assertEquals(Schema.parse(avroSchemaToString), Schema.parse(fullAvroSchemaJson));
    if (debug) {
        TestUtil.out.println(schema);
        TestUtil.out.println(avroSchema);
    }
    // translate from Data to Avro generic
    DataMap inputDataMap = TestUtil.dataMapFromString(dataJson);
    GenericRecord genericRecord = DataTranslator.dataMapToGenericRecord(inputDataMap, schema);
    GenericRecord expectedGenericRecord = AvroUtil.genericRecordFromJson(avroDataJson, avroSchema);
    assertEquals(genericRecord.toString(), expectedGenericRecord.toString());
    // translate form Avro generic back to Data
    Object data = DataTranslator.genericRecordToDataMap(genericRecord, schema, avroSchema);
    assertEquals(data, inputDataMap);
}

Example 95 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestSchemaSampleDataGenerator method testRecordSchema.

@Test
public void testRecordSchema() {
    final RecordDataSchema schema = (RecordDataSchema) DataTemplateUtil.getSchema(Certification.class);
    final DataMap value = SchemaSampleDataGenerator.buildRecordData(schema, _spec);
    final ValidationResult result = ValidateDataAgainstSchema.validate(value, schema, new ValidationOptions());
    Assert.assertTrue(result.isValid(), Arrays.toString(result.getMessages().toArray()));
}