Examples with RecordDataSchema com.linkedin.data.schema.RecordDataSchema used on opensource projects

Example 86 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class ActionRequestBuilder method build.

@SuppressWarnings("unchecked")
@Override
public ActionRequest<V> build() {
    if (_name == null) {
        throw new IllegalStateException("name required to build action request");
    }
    RecordDataSchema requestDataSchema;
    RecordDataSchema actionResponseDataSchema;
    FieldDef<V> responseFieldDef;
    if (// old builder code in use
    _resourceSpec.getRequestMetadata(_name) == null) {
        requestDataSchema = DynamicRecordMetadata.buildSchema(_name, _actionParams.keySet());
        if (_elementType == null) {
            _elementType = new TypeSpec<>(_elementClass);
        }
        Collection<FieldDef<?>> responseFieldDefCollection;
        if (_elementType.getType() == Void.class) {
            responseFieldDef = null;
            responseFieldDefCollection = Collections.emptyList();
        } else {
            responseFieldDef = new FieldDef<>(ActionResponse.VALUE_NAME, _elementType.getType(), _elementType.getSchema());
            responseFieldDefCollection = Collections.<FieldDef<?>>singleton(responseFieldDef);
        }
        actionResponseDataSchema = DynamicRecordMetadata.buildSchema(_name, responseFieldDefCollection);
    } else {
        requestDataSchema = _resourceSpec.getRequestMetadata(_name).getRecordDataSchema();
        actionResponseDataSchema = _resourceSpec.getActionResponseMetadata(_name).getRecordDataSchema();
        responseFieldDef = (FieldDef<V>) _resourceSpec.getActionResponseMetadata(_name).getFieldDef(ActionResponse.VALUE_NAME);
    }
    @SuppressWarnings("unchecked") ActionResponseDecoder<V> actionResponseDecoder = new ActionResponseDecoder<>(responseFieldDef, actionResponseDataSchema);
    DynamicRecordTemplate inputParameters = new DynamicRecordTemplate(requestDataSchema, buildReadOnlyActionParameters());
    inputParameters.data().setReadOnly();
    return new ActionRequest<>(inputParameters, buildReadOnlyHeaders(), buildReadOnlyCookies(), actionResponseDecoder, _resourceSpec, buildReadOnlyQueryParameters(), getQueryParamClasses(), _name, getBaseUriTemplate(), buildReadOnlyPathKeys(), getRequestOptions(), buildReadOnlyId(), _streamingAttachments == null ? null : Collections.unmodifiableList(_streamingAttachments));
}

Example 87 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestDataTranslator method testNamespaceOverrideWithOptionalField.

/**
 * This test is for testing data translates correctly from Avro to Pegasus and Pegasus to Avro
 * in namespaces mismatching case (e.g. overridden namespace) for schemas with option fields use case.
 *
 * To enable the namespaces overridden support, we introduce a new Object - DataTranslationOptions to DataTranslator
 * and add a field - namespaceOverrideMapping, which enables customers to pass the Avro - Pegasus overridden namespaces map,
 * when the namespace for one of these schemas is overridden.
 * So that DataTranslator is able to find the corresponding schema between Avro and Pegasus.
 */
@Test
public void testNamespaceOverrideWithOptionalField() throws IOException {
    String schemaText = "{\n" + "  \"type\" : \"record\",\n" + "  \"name\" : \"Foo\",\n" + "  \"namespace\" : \"a.b.c\",\n" + "  \"fields\" : [\n" + "    { \"name\" : \"a\", \"type\" : { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] }, \"optional\": true }\n" + "  ]\n" + "}\n";
    RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(schemaText);
    String avroSchemaText = "{\n" + "  \"type\" : \"record\",\n" + "  \"name\" : \"Foo\",\n" + "  \"namespace\" : \"avro.a.b.c\",\n" + "  \"fields\" : [\n" + "    { \"name\" : \"a\", \"type\" : [ \"null\", { \"type\" : \"record\", \"name\" : \"FooFoo\", \"fields\" : [ { \"name\" : \"b\", \"type\" : \"int\" } ] } ] }\n" + "  ]\n" + "}\n";
    Schema avroSchema = Schema.parse(avroSchemaText);
    GenericRecord avroRecord = AvroUtil.genericRecordFromJson(TestAvroUtil.namespaceProcessor("{ \"a\" : { \"##NS(avro.a.b.c.)FooFoo\": { \"b\" : 1 } } }"), avroSchema);
    // Test Avro-to-Pegasus
    AvroRecordToDataMapTranslationOptions avroRecordToDataMapTranslationOptions = new AvroRecordToDataMapTranslationOptions();
    avroRecordToDataMapTranslationOptions.setAvroToDataSchemaNamespaceMapping(Collections.singletonMap("avro.a.b.c", "a.b.c"));
    DataMap pegasusDataMap = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema, avroRecordToDataMapTranslationOptions);
    Assert.assertEquals(pegasusDataMap.getDataMap("a").get("b"), 1);
    // Test Pegasus-to-Avro
    DataMapToAvroRecordTranslationOptions dataMapToAvroRecordTranslationOptions = new DataMapToAvroRecordTranslationOptions();
    dataMapToAvroRecordTranslationOptions.setAvroToDataSchemaNamespaceMapping(Collections.singletonMap("avro.a.b.c", "a.b.c"));
    GenericRecord reconvertedAvroRecord = DataTranslator.dataMapToGenericRecord(pegasusDataMap, recordDataSchema, avroSchema, dataMapToAvroRecordTranslationOptions);
    Assert.assertEquals(((GenericRecord) reconvertedAvroRecord.get("a")).get("b"), 1);
}

Example 88 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestDataTranslator method testPegasusDefaultToAvroOptionalTranslation.

@Test(dataProvider = "defaultToAvroOptionalTranslationProvider", description = "generic record to data map should not care about the specific list implementation")
public void testPegasusDefaultToAvroOptionalTranslation(Object... testSchemaTextAndDataMap) throws IOException {
    // Test if the pegasus default field has been correctly translated
    // i.e. if value present, translate it
    // if no value present, don't translate it
    // Create pegasus schema text
    String rawPegasusTestSchemaText;
    PegasusToAvroDefaultFieldTranslationMode schemaTranslationMode = null;
    String expectedAvroSchemaString;
    String dataMapString;
    PegasusToAvroDefaultFieldTranslationMode dataTranslationMode = null;
    String expectedAvroRecordJsonString;
    rawPegasusTestSchemaText = (String) testSchemaTextAndDataMap[0];
    schemaTranslationMode = (PegasusToAvroDefaultFieldTranslationMode) testSchemaTextAndDataMap[1];
    expectedAvroSchemaString = (String) testSchemaTextAndDataMap[2];
    dataMapString = (String) testSchemaTextAndDataMap[3];
    dataTranslationMode = (PegasusToAvroDefaultFieldTranslationMode) testSchemaTextAndDataMap[4];
    expectedAvroRecordJsonString = (String) testSchemaTextAndDataMap[5];
    boolean isError = (boolean) testSchemaTextAndDataMap[6];
    String errorMsg = (String) testSchemaTextAndDataMap[7];
    List<String> schemaTextForTesting = null;
    try {
        // Test this also works for TypeRef
        if (rawPegasusTestSchemaText.contains("##T_START")) {
            String noTyperefSchemaText = rawPegasusTestSchemaText.replace("##T_START", "").replace("##T_END", "");
            String typerefSchemaText = rawPegasusTestSchemaText.replace("##T_START", "{ \"type\" : \"typeref\", \"name\" : \"Ref\", \"ref\" : ").replace("##T_END", "}");
            schemaTextForTesting = Arrays.asList(noTyperefSchemaText, typerefSchemaText);
        } else {
            schemaTextForTesting = Arrays.asList(rawPegasusTestSchemaText);
        }
        for (String pegasusSchemaText : schemaTextForTesting) {
            // Create pegasus schema
            RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(pegasusSchemaText);
            // Translate to Avro Schema so can create the GenericRecord data holder
            Schema avroSchema = SchemaTranslator.dataToAvroSchema(recordDataSchema, new DataToAvroSchemaTranslationOptions(schemaTranslationMode));
            // AvroSchema translated needs to be as expected
            Schema expectedAvroSchema = Schema.parse(expectedAvroSchemaString);
            assertEquals(avroSchema, expectedAvroSchema);
            // Have a DataMap from pegasus schema
            DataMap dataMap = TestUtil.dataMapFromString(dataMapString);
            // Create option, pass into data translator
            DataMapToAvroRecordTranslationOptions options = new DataMapToAvroRecordTranslationOptionsBuilder().defaultFieldDataTranslationMode(dataTranslationMode).build();
            // Translate to generic record
            GenericRecord avroRecord = DataTranslator.dataMapToGenericRecord(dataMap, recordDataSchema, avroSchema, options);
            String avroJson = AvroUtil.jsonFromGenericRecord(avroRecord);
            // avroJson compare
            assertEquals(avroJson, expectedAvroRecordJsonString);
            // validation result test
            DataMap dataMapResult = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema);
            ValidationResult vr = ValidateDataAgainstSchema.validate(dataMapResult, recordDataSchema, new // Not filling back Default value
            ValidationOptions(// Not filling back Default value
            RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT, CoercionMode.NORMAL));
            DataMap fixedInputDataMap = (DataMap) vr.getFixed();
            assertTrue(vr.isValid());
            assertEquals(dataMapResult, fixedInputDataMap);
            // serialize avroRecord to binary and back
            byte[] avroBytes = AvroUtil.bytesFromGenericRecord(avroRecord);
            GenericRecord avroRecordFromBytes = AvroUtil.genericRecordFromBytes(avroBytes, avroRecord.getSchema());
            byte[] avroBytesAgain = AvroUtil.bytesFromGenericRecord(avroRecordFromBytes);
            assertEquals(avroBytes, avroBytesAgain);
            // check result of roundtrip binary serialization
            DataMap dataMapFromBinaryResult = DataTranslator.genericRecordToDataMap(avroRecordFromBytes, recordDataSchema, avroSchema);
            vr = ValidateDataAgainstSchema.validate(dataMapFromBinaryResult, recordDataSchema, new // Not filling back Default value
            ValidationOptions(// Not filling back Default value
            RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT, CoercionMode.NORMAL));
            fixedInputDataMap = (DataMap) vr.getFixed();
            assertTrue(vr.isValid());
            assertEquals(dataMapResult, fixedInputDataMap);
        }
    } catch (Exception e) {
        assertTrue(isError);
        assertEquals(e.getMessage(), errorMsg);
    }
}

Example 89 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestDataTranslator method testDataTranslation.

private void testDataTranslation(String schemaText, String[][] row) throws IOException {
    boolean debug = false;
    if (debug)
        out.print(schemaText);
    RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(schemaText);
    Schema avroSchema = SchemaTranslator.dataToAvroSchema(recordDataSchema);
    if (debug)
        out.println(avroSchema);
    // translate data
    for (int col = 1; col < row.length; col++) {
        String result;
        GenericRecord avroRecord = null;
        Exception exc = null;
        if (debug)
            out.println(col + " DataMap: " + row[col][0]);
        DataMap dataMap = TestUtil.dataMapFromString(row[col][0]);
        // translate from Pegasus to Avro
        try {
            avroRecord = DataTranslator.dataMapToGenericRecord(dataMap, recordDataSchema, avroSchema);
            String avroJson = AvroUtil.jsonFromGenericRecord(avroRecord);
            if (debug)
                out.println(col + " GenericRecord: " + avroJson);
            result = avroJson;
        } catch (Exception e) {
            exc = e;
            result = TestUtil.stringFromException(e);
            if (debug)
                out.println(col + " Exception: " + result);
        }
        int start = 1;
        boolean oneWay = false;
        if (start < row[col].length && row[col][start] == ONE_WAY) {
            oneWay = true;
            start++;
        }
        // verify
        for (int i = start; i < row[col].length; i++) {
            if (debug)
                out.println(col + " Test:" + row[col][i]);
            if (debug && exc != null && result.contains(row[col][i]) == false)
                exc.printStackTrace(out);
            String expectedBeforeNamespaceProcessor = row[col][i];
            String expected = TestAvroUtil.namespaceProcessor(expectedBeforeNamespaceProcessor);
            if (debug && expected != expectedBeforeNamespaceProcessor)
                out.println(" Expected:" + expected);
            assertTrue(result.contains(expected));
        }
        if (avroRecord != null) {
            // translate from Avro back to Pegasus
            DataMap dataMapResult = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema);
            ValidationResult vr = ValidateDataAgainstSchema.validate(dataMap, recordDataSchema, new ValidationOptions(RequiredMode.MUST_BE_PRESENT, CoercionMode.NORMAL));
            DataMap fixedInputDataMap = (DataMap) vr.getFixed();
            assertTrue(vr.isValid());
            if (oneWay == false) {
                assertEquals(dataMapResult, fixedInputDataMap);
            }
            // serialize avroRecord to binary and back
            byte[] avroBytes = AvroUtil.bytesFromGenericRecord(avroRecord);
            GenericRecord avroRecordFromBytes = AvroUtil.genericRecordFromBytes(avroBytes, avroRecord.getSchema());
            byte[] avroBytesAgain = AvroUtil.bytesFromGenericRecord(avroRecordFromBytes);
            assertEquals(avroBytes, avroBytesAgain);
            // check result of roundtrip binary serialization
            DataMap dataMapFromBinaryResult = DataTranslator.genericRecordToDataMap(avroRecordFromBytes, recordDataSchema, avroSchema);
            vr = ValidateDataAgainstSchema.validate(dataMapFromBinaryResult, recordDataSchema, new ValidationOptions(RequiredMode.MUST_BE_PRESENT, CoercionMode.NORMAL));
            fixedInputDataMap = (DataMap) vr.getFixed();
            assertTrue(vr.isValid());
            if (oneWay == false) {
                assertEquals(dataMapResult, fixedInputDataMap);
            }
        }
    }
}

Example 90 with RecordDataSchema

use of com.linkedin.data.schema.RecordDataSchema in project rest.li by linkedin.

the class TestDataTranslator method testAvroSchemaMissingFields.

@Test
public void testAvroSchemaMissingFields() throws IOException {
    final String P_SCHEMA = "{" + "  \"type\" : \"record\",\n" + "  \"name\" : \"Foo\",\n" + "  \"fields\" : [\n" + "{ \"name\": \"field1\", \"type\": \"int\" }," + "{ \"name\": \"field2\", \"type\": \"int\", \"optional\": true }," + "{ \"name\": \"field3\", \"type\": \"int\", \"optional\": true, \"default\": 42 }," + "{ \"name\": \"field4\", \"type\": \"int\", \"default\": 42 }," + "{ \"name\": \"field5\", \"type\": \"null\" }" + "] }";
    Schema avroSchema = Schema.parse("{ \"name\": \"foo\", \"type\": \"record\", \"fields\":[]}");
    DataMap map = DataTranslator.genericRecordToDataMap(new GenericData.Record(avroSchema), (RecordDataSchema) TestUtil.dataSchemaFromString(P_SCHEMA), avroSchema);
    assertEquals(map.size(), 0);
}