Examples with RecordDataSchema com.linkedin.data.schema.RecordDataSchema used on opensource projects

Example 41 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestTypeRefRecordTemplate method testArraySchema.

@Test
public void testArraySchema() {
    TyperefTest record = new TyperefTest();
    RecordDataSchema recordDataSchema = record.schema();
    DoubleArray doubleArray = new DoubleArray();
    record.setDoubleRefArray(doubleArray);
    doubleArray = record.getDoubleRefArray();
    assertEquals(doubleArray.schema(), DataTemplateUtil.getSchema(DoubleArray.class));
    assertNotEquals(recordDataSchema.getField("doubleRefArray").getType(), doubleArray.schema());
    IntegerArray intArray = new IntegerArray();
    record.setIntArray(intArray);
    intArray = record.getIntArray();
    assertEquals(intArray.schema(), DataTemplateUtil.getSchema(IntegerArray.class));
    assertNotEquals(recordDataSchema.getField("intArray").getType(), intArray.schema());
    record.setIntRefArray(intArray);
    intArray = record.getIntRefArray();
    assertEquals(intArray.schema(), DataTemplateUtil.getSchema(IntegerArray.class));
    assertNotEquals(recordDataSchema.getField("intRefArray").getType(), intArray.schema());
    assertNotEquals(recordDataSchema.getField("intArray").getType(), recordDataSchema.getField("intRefArray").getType());
}

Example 42 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestCustomAvroSchema method translate.

private void translate(String dataSchemaFieldsJson, String avroSchemaFieldsJson, String dataJson, String avroDataJson) throws IOException {
    boolean debug = false;
    String fullSchemaJson = DATA_SCHEMA_JSON_TEMPLATE.replace("##FIELDS", dataSchemaFieldsJson);
    String avroSchemaFieldsJsonAfterVariableExpansion = avroSchemaFieldsJson.replace("##ANYRECORD_FULL_JSON", ANYRECORD_AVRO_SCHEMA_JSON).replace("##ANYRECORD_NAME", ANYRECORD_AVRO_FULL_NAME);
    String fullAvroSchemaJson = AVRO_SCHEMA_JSON_TEMPLATE.replace("##FIELDS", avroSchemaFieldsJsonAfterVariableExpansion);
    PegasusSchemaParser parser = TestUtil.schemaParserFromString(fullSchemaJson);
    assertFalse(parser.hasError(), parser.errorMessage());
    RecordDataSchema schema = (RecordDataSchema) parser.topLevelDataSchemas().get(2);
    String avroJsonOutput = SchemaTranslator.dataToAvroSchemaJson(schema);
    assertEquals(TestUtil.dataMapFromString(avroJsonOutput), TestUtil.dataMapFromString(fullAvroSchemaJson));
    Schema avroSchema = Schema.parse(avroJsonOutput);
    Schema avroSchema2 = SchemaTranslator.dataToAvroSchema(schema);
    assertEquals(avroSchema, avroSchema2);
    String avroSchemaToString = avroSchema.toString();
    assertEquals(Schema.parse(avroSchemaToString), Schema.parse(fullAvroSchemaJson));
    if (debug) {
        TestUtil.out.println(schema);
        TestUtil.out.println(avroSchema);
    }
    // translate from Data to Avro generic
    DataMap inputDataMap = TestUtil.dataMapFromString(dataJson);
    GenericRecord genericRecord = DataTranslator.dataMapToGenericRecord(inputDataMap, schema);
    GenericRecord expectedGenericRecord = AvroUtil.genericRecordFromJson(avroDataJson, avroSchema);
    assertEquals(genericRecord.toString(), expectedGenericRecord.toString());
    // translate form Avro generic back to Data
    Object data = DataTranslator.genericRecordToDataMap(genericRecord, schema, avroSchema);
    assertEquals(data, inputDataMap);
}

Example 43 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestDataTranslator method testAvroSchemaMissingFields.

@Test
public void testAvroSchemaMissingFields() throws IOException {
    final String P_SCHEMA = "{" + "  \"type\" : \"record\",\n" + "  \"name\" : \"Foo\",\n" + "  \"fields\" : [\n" + "{ \"name\": \"field1\", \"type\": \"int\" }," + "{ \"name\": \"field2\", \"type\": \"int\", \"optional\": true }," + "{ \"name\": \"field3\", \"type\": \"int\", \"optional\": true, \"default\": 42 }," + "{ \"name\": \"field4\", \"type\": \"int\", \"default\": 42 }," + "{ \"name\": \"field5\", \"type\": \"null\" }" + "] }";
    Schema avroSchema = Schema.parse("{ \"name\": \"foo\", \"type\": \"record\", \"fields\":[]}");
    DataMap map = DataTranslator.genericRecordToDataMap(new GenericData.Record(avroSchema), (RecordDataSchema) TestUtil.dataSchemaFromString(P_SCHEMA), avroSchema);
    assertEquals(map.size(), 0);
}

Example 44 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestDataTranslator method testDataTranslation.

private void testDataTranslation(String schemaText, String[][] row) throws IOException {
    boolean debug = false;
    if (debug)
        out.print(schemaText);
    RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(schemaText);
    Schema avroSchema = SchemaTranslator.dataToAvroSchema(recordDataSchema);
    if (debug)
        out.println(avroSchema);
    // translate data
    for (int col = 1; col < row.length; col++) {
        String result;
        GenericRecord avroRecord = null;
        Exception exc = null;
        if (debug)
            out.println(col + " DataMap: " + row[col][0]);
        DataMap dataMap = TestUtil.dataMapFromString(row[col][0]);
        // translate from Pegasus to Avro
        try {
            avroRecord = DataTranslator.dataMapToGenericRecord(dataMap, recordDataSchema, avroSchema);
            String avroJson = AvroUtil.jsonFromGenericRecord(avroRecord);
            if (debug)
                out.println(col + " GenericRecord: " + avroJson);
            result = avroJson;
        } catch (Exception e) {
            exc = e;
            result = TestUtil.stringFromException(e);
            if (debug)
                out.println(col + " Exception: " + result);
        }
        int start = 1;
        boolean oneWay = false;
        if (start < row[col].length && row[col][start] == ONE_WAY) {
            oneWay = true;
            start++;
        }
        // verify
        for (int i = start; i < row[col].length; i++) {
            if (debug)
                out.println(col + " Test:" + row[col][i]);
            if (debug && exc != null && result.contains(row[col][i]) == false)
                exc.printStackTrace(out);
            String expectedBeforeNamespaceProcessor = row[col][i];
            String expected = TestAvroUtil.namespaceProcessor(expectedBeforeNamespaceProcessor);
            if (debug && expected != expectedBeforeNamespaceProcessor)
                out.println(" Expected:" + expected);
            assertTrue(result.contains(expected));
        }
        if (avroRecord != null) {
            // translate from Avro back to Pegasus
            DataMap dataMapResult = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema);
            ValidationResult vr = ValidateDataAgainstSchema.validate(dataMap, recordDataSchema, new ValidationOptions(RequiredMode.MUST_BE_PRESENT, CoercionMode.NORMAL));
            DataMap fixedInputDataMap = (DataMap) vr.getFixed();
            assertTrue(vr.isValid());
            if (oneWay == false) {
                assertEquals(dataMapResult, fixedInputDataMap);
            }
            // serialize avroRecord to binary and back
            byte[] avroBytes = AvroUtil.bytesFromGenericRecord(avroRecord);
            GenericRecord avroRecordFromBytes = AvroUtil.genericRecordFromBytes(avroBytes, avroRecord.getSchema());
            byte[] avroBytesAgain = AvroUtil.bytesFromGenericRecord(avroRecordFromBytes);
            assertEquals(avroBytes, avroBytesAgain);
            // check result of roundtrip binary serialization
            DataMap dataMapFromBinaryResult = DataTranslator.genericRecordToDataMap(avroRecordFromBytes, recordDataSchema, avroSchema);
            vr = ValidateDataAgainstSchema.validate(dataMapFromBinaryResult, recordDataSchema, new ValidationOptions(RequiredMode.MUST_BE_PRESENT, CoercionMode.NORMAL));
            fixedInputDataMap = (DataMap) vr.getFixed();
            assertTrue(vr.isValid());
            if (oneWay == false) {
                assertEquals(dataMapResult, fixedInputDataMap);
            }
        }
    }
}

Example 45 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestDataTranslator method testMissingDefaultFieldsOnDataMap.

@Test
public void testMissingDefaultFieldsOnDataMap() throws IOException {
    final String SCHEMA = "{" + "   \"type\":\"record\"," + "   \"name\":\"Foo\"," + "   \"fields\":[" + "      {" + "         \"name\":\"field1\"," + "         \"type\":\"string\"" + "      }," + "      {" + "         \"name\":\"field2\"," + "         \"type\":{" + "            \"type\":\"array\"," + "            \"items\":\"string\"" + "         }," + "         \"default\":[ ]" + "      }" + "   ]" + "}";
    RecordDataSchema pegasusSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(SCHEMA);
    Schema avroShema = Schema.parse(SCHEMA);
    DataMap dataMap = new DataMap();
    dataMap.put("field1", "test");
    GenericRecord record = DataTranslator.dataMapToGenericRecord(dataMap, pegasusSchema, avroShema);
    assertEquals(record.get("field2"), new GenericData.Array<>(0, Schema.createArray(Schema.create(Schema.Type.STRING))));
}