Examples with ValidationOptions com.linkedin.data.schema.validation.ValidationOptions used on opensource projects

Example 16 with ValidationOptions

use of com.linkedin.data.schema.validation.ValidationOptions in project rest.li by linkedin.

the class ActionArgumentBuilder method extractRequestData.

@Override
public RestLiRequestData extractRequestData(RoutingResult routingResult, RestRequest request) {
    ResourceMethodDescriptor resourceMethodDescriptor = routingResult.getResourceMethod();
    final DataMap data;
    if (request.getEntity() == null || request.getEntity().length() == 0) {
        data = new DataMap();
    } else {
        data = DataMapUtils.readMap(request);
    }
    DynamicRecordTemplate template = new DynamicRecordTemplate(data, resourceMethodDescriptor.getRequestDataSchema());
    ValidationResult result = ValidateDataAgainstSchema.validate(data, template.schema(), new ValidationOptions(RequiredMode.IGNORE, CoercionMode.NORMAL));
    if (!result.isValid()) {
        throw new RoutingException("Parameters of method '" + resourceMethodDescriptor.getActionName() + "' failed validation with error '" + result.getMessages() + "'", HttpStatus.S_400_BAD_REQUEST.getCode());
    }
    return new RestLiRequestDataImpl.Builder().entity(template).build();
}

Example 17 with ValidationOptions

use of com.linkedin.data.schema.validation.ValidationOptions in project rest.li by linkedin.

the class ArgumentBuilder method buildDataTemplateArgument.

private static DataTemplate<?> buildDataTemplateArgument(final ResourceContext context, final Parameter<?> param) {
    Object paramValue = context.getStructuredParameter(param.getName());
    DataTemplate<?> paramRecordTemplate;
    if (paramValue == null) {
        return null;
    } else {
        @SuppressWarnings("unchecked") final Class<? extends RecordTemplate> paramType = (Class<? extends RecordTemplate>) param.getType();
        /**
       * It is possible for the paramValue provided by ResourceContext to be coerced to the wrong type.
       * If a query param is a single value param for example www.domain.com/resource?foo=1.
       * Then ResourceContext will parse foo as a String with value = 1.
       * However if a query param contains many values for example www.domain.com/resource?foo=1&foo=2&foo=3
       * Then ResourceContext will parse foo as an DataList with value [1,2,3]
       *
       * So this means if the 'final' type of a query param is an Array and the paramValue we received from
       * ResourceContext is not a DataList we will have to wrap the paramValue inside a DataList
       */
        if (AbstractArrayTemplate.class.isAssignableFrom(paramType) && paramValue.getClass() != DataList.class) {
            paramRecordTemplate = DataTemplateUtil.wrap(new DataList(Arrays.asList(paramValue)), paramType);
        } else {
            paramRecordTemplate = DataTemplateUtil.wrap(paramValue, paramType);
        }
        // Validate against the class schema with FixupMode.STRING_TO_PRIMITIVE to parse the
        // strings into the corresponding primitive types.
        ValidateDataAgainstSchema.validate(paramRecordTemplate.data(), paramRecordTemplate.schema(), new ValidationOptions(RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT, CoercionMode.STRING_TO_PRIMITIVE));
        return paramRecordTemplate;
    }
}

Example 18 with ValidationOptions

use of com.linkedin.data.schema.validation.ValidationOptions in project rest.li by linkedin.

the class TestAnyRecordValidator method testAnyRecordValidation.

@Test
public void testAnyRecordValidation() throws IOException {
    Object[][] inputs = { { // DataMap is empty
    ANYRECORD_SCHEMA, "{" + "}", new AnyRecordValidator.Parameter(false, null), ResultFlag.NOT_VALID, new String[] { "expects data to be a DataMap with one entry" } }, { // DataMap has more than one entry
    ANYRECORD_SCHEMA, "{" + "  \"a\" : 1," + "  \"b\" : 2" + "}", new AnyRecordValidator.Parameter(false, null), ResultFlag.NOT_VALID, new String[] { "expects data to be a DataMap with one entry" } }, { // no resolver, any type schema need not be valid
    ANYRECORD_SCHEMA, "{" + "  \"abc\" : { }\n" + "}", new AnyRecordValidator.Parameter(false, null), ResultFlag.VALID, new String[] { "INFO", "cannot obtain schema for \"abc\", no resolver" } }, { // no resolver, any type schema must be valid
    ANYRECORD_SCHEMA, "{" + "  \"abc\" : { }\n" + "}", new AnyRecordValidator.Parameter(true, null), ResultFlag.NOT_VALID, new String[] { "ERROR", "cannot obtain schema for \"abc\", no resolver" } }, { // no resolver but schema exists, any type schema must be valid
    ANYRECORD_SCHEMA, "{" + "  \"com.linkedin.Foo\" : { }\n" + "}", new AnyRecordValidator.Parameter(true, null), ResultFlag.NOT_VALID, new String[] { "ERROR", "cannot obtain schema for \"com.linkedin.Foo\", no resolver" } }, { // schema exists, any type schema must be valid
    ANYRECORD_SCHEMA, "{" + "  \"com.linkedin.Foo\" : { }\n" + "}", new AnyRecordValidator.Parameter(true, _resolver), ResultFlag.VALID, new String[] {} }, { // resolver cannot resolve name to schema, any type schema must be valid
    ANYRECORD_SCHEMA, "{" + "  \"com.linkedin.DoesNotExist\" : { }\n" + "}", new AnyRecordValidator.Parameter(true, _resolver), ResultFlag.NOT_VALID, new String[] { "ERROR", "cannot obtain schema for \"com.linkedin.DoesNotExist\" (" + RESOLVER_ERROR_MESSAGE + ")" } }, { // resolver cannot resolve name to schema, any type schema need not be valid
    ANYRECORD_SCHEMA, "{" + "  \"com.linkedin.DoesNotExist\" : { }\n" + "}", new AnyRecordValidator.Parameter(false, _resolver), ResultFlag.VALID, new String[] { "INFO", "cannot obtain schema for \"com.linkedin.DoesNotExist\" (" + RESOLVER_ERROR_MESSAGE + ")" } }, { // type schema is valid and any data is valid, any type schema must be valid
    ANYRECORD_SCHEMA, "{" + "  \"com.linkedin.Bar\" : { \"b\" : \"hello\" }\n" + "}", new AnyRecordValidator.Parameter(true, _resolver), ResultFlag.VALID, new String[] {} }, { // type schema is valid and any data is not valid, any type schema must be valid
    ANYRECORD_SCHEMA, "{" + "  \"com.linkedin.Bar\" : { \"b\" : 1 }\n" + "}", new AnyRecordValidator.Parameter(true, _resolver), ResultFlag.NOT_VALID, new String[] { "ERROR", "1 cannot be coerced to String" } }, { // type schema is valid and any data is not valid, any type schema must be valid
    ANYRECORD_SCHEMA, "{" + "  \"com.linkedin.Bar\" : { \"b\" : 1 }\n" + "}", new AnyRecordValidator.Parameter(false, _resolver), ResultFlag.NOT_VALID, new String[] { "ERROR", "1 cannot be coerced to String" } }, { // AnyRecord is field, must sure that the field is being validated
    ANYRECORDCLIENT_SCHEMA, "{" + "  \"required\" : {\n" + "    \"com.linkedin.Bar\" : { \"b\" : 1 }\n" + "  }\n" + "}", new AnyRecordValidator.Parameter(true, _resolver), ResultFlag.NOT_VALID, new String[] { "ERROR", "/required/com.linkedin.Bar/b", "1 cannot be coerced to String" } }, { // AnyRecord within AnyRecord, make sure nested AnyRecord is validated
    ANYRECORDCLIENT_SCHEMA, "{" + "  \"required\" : {\n" + "    \"com.linkedin.data.schema.validator.AnyRecord\" : {\n" + "      \"com.linkedin.Bar\" : { \"b\" : 1 }\n" + "    }\n" + "  }\n" + "}", new AnyRecordValidator.Parameter(true, _resolver), ResultFlag.NOT_VALID, new String[] { "ERROR", "/required/com.linkedin.data.schema.validator.AnyRecord/com.linkedin.Bar/b", "1 cannot be coerced to String" } } };
    final boolean debug = false;
    ValidationOptions options = new ValidationOptions();
    for (Object[] row : inputs) {
        int i = 0;
        DataSchema schema = (DataSchema) row[i++];
        Object object = TestUtil.dataMapFromString((String) row[i++]);
        AnyRecordValidator.Parameter anyRecordValidatorParameter = (AnyRecordValidator.Parameter) row[i++];
        AnyRecordValidator.setParameter(options, anyRecordValidatorParameter);
        DataSchemaAnnotationValidator validator = new DataSchemaAnnotationValidator(schema);
        if (debug)
            TestUtil.out.println(validator);
        ValidationResult result = ValidateDataAgainstSchema.validate(object, schema, options, validator);
        checkValidationResult(result, row, i, debug);
    }
}

Example 19 with ValidationOptions

use of com.linkedin.data.schema.validation.ValidationOptions in project rest.li by linkedin.

the class TestDataTranslator method testDataTranslation.

private void testDataTranslation(String schemaText, String[][] row) throws IOException {
    boolean debug = false;
    if (debug)
        out.print(schemaText);
    RecordDataSchema recordDataSchema = (RecordDataSchema) TestUtil.dataSchemaFromString(schemaText);
    Schema avroSchema = SchemaTranslator.dataToAvroSchema(recordDataSchema);
    if (debug)
        out.println(avroSchema);
    // translate data
    for (int col = 1; col < row.length; col++) {
        String result;
        GenericRecord avroRecord = null;
        Exception exc = null;
        if (debug)
            out.println(col + " DataMap: " + row[col][0]);
        DataMap dataMap = TestUtil.dataMapFromString(row[col][0]);
        // translate from Pegasus to Avro
        try {
            avroRecord = DataTranslator.dataMapToGenericRecord(dataMap, recordDataSchema, avroSchema);
            String avroJson = AvroUtil.jsonFromGenericRecord(avroRecord);
            if (debug)
                out.println(col + " GenericRecord: " + avroJson);
            result = avroJson;
        } catch (Exception e) {
            exc = e;
            result = TestUtil.stringFromException(e);
            if (debug)
                out.println(col + " Exception: " + result);
        }
        int start = 1;
        boolean oneWay = false;
        if (start < row[col].length && row[col][start] == ONE_WAY) {
            oneWay = true;
            start++;
        }
        // verify
        for (int i = start; i < row[col].length; i++) {
            if (debug)
                out.println(col + " Test:" + row[col][i]);
            if (debug && exc != null && result.contains(row[col][i]) == false)
                exc.printStackTrace(out);
            String expectedBeforeNamespaceProcessor = row[col][i];
            String expected = TestAvroUtil.namespaceProcessor(expectedBeforeNamespaceProcessor);
            if (debug && expected != expectedBeforeNamespaceProcessor)
                out.println(" Expected:" + expected);
            assertTrue(result.contains(expected));
        }
        if (avroRecord != null) {
            // translate from Avro back to Pegasus
            DataMap dataMapResult = DataTranslator.genericRecordToDataMap(avroRecord, recordDataSchema, avroSchema);
            ValidationResult vr = ValidateDataAgainstSchema.validate(dataMap, recordDataSchema, new ValidationOptions(RequiredMode.MUST_BE_PRESENT, CoercionMode.NORMAL));
            DataMap fixedInputDataMap = (DataMap) vr.getFixed();
            assertTrue(vr.isValid());
            if (oneWay == false) {
                assertEquals(dataMapResult, fixedInputDataMap);
            }
            // serialize avroRecord to binary and back
            byte[] avroBytes = AvroUtil.bytesFromGenericRecord(avroRecord);
            GenericRecord avroRecordFromBytes = AvroUtil.genericRecordFromBytes(avroBytes, avroRecord.getSchema());
            byte[] avroBytesAgain = AvroUtil.bytesFromGenericRecord(avroRecordFromBytes);
            assertEquals(avroBytes, avroBytesAgain);
            // check result of roundtrip binary serialization
            DataMap dataMapFromBinaryResult = DataTranslator.genericRecordToDataMap(avroRecordFromBytes, recordDataSchema, avroSchema);
            vr = ValidateDataAgainstSchema.validate(dataMapFromBinaryResult, recordDataSchema, new ValidationOptions(RequiredMode.MUST_BE_PRESENT, CoercionMode.NORMAL));
            fixedInputDataMap = (DataMap) vr.getFixed();
            assertTrue(vr.isValid());
            if (oneWay == false) {
                assertEquals(dataMapResult, fixedInputDataMap);
            }
        }
    }
}

Example 20 with ValidationOptions

use of com.linkedin.data.schema.validation.ValidationOptions in project rest.li by linkedin.

the class SchemaTranslator method avroToDataSchema.

/**
   * Translate an Avro {@link Schema} to a {@link DataSchema}.
   * <p>
   * If the translation mode is {@link AvroToDataSchemaTranslationMode#RETURN_EMBEDDED_SCHEMA}
   * and a {@link DataSchema} is embedded in the Avro schema, then return the embedded schema.
   * An embedded schema is present if the Avro {@link Schema} has a "com.linkedin.data" property and the
   * "com.linkedin.data" property contains both "schema" and "optionalDefaultMode" properties.
   * The "schema" property provides the embedded {@link DataSchema}.
   * The "optionalDefaultMode" property provides how optional default values were translated.
   * <p>
   * If the translation mode is {@link AvroToDataSchemaTranslationMode#VERIFY_EMBEDDED_SCHEMA}
   * and a {@link DataSchema} is embedded in the Avro schema, then verify that the embedded schema
   * translates to the input Avro schema. If the translated and embedded schema is the same,
   * then return the embedded schema, else throw {@link IllegalArgumentException}.
   * <p>
   * If the translation mode is {@link com.linkedin.data.avro.AvroToDataSchemaTranslationMode#TRANSLATE}
   * or no embedded {@link DataSchema} is present, then this method
   * translates the provided Avro {@link Schema} to a {@link DataSchema}
   * as described follows:
   * <p>
   * This method translates union with null record fields in Avro {@link Schema}
   * to optional fields in {@link DataSchema}. Record fields
   * whose type is a union with null will be translated to a new type, and the field becomes optional.
   * If the Avro union has two types (one of them is the null type), then the new type of the
   * field is the non-null member type of the union. If the Avro union does not have two types
   * (one of them is the null type) then the new type of the field is a union type with the null type
   * removed from the original union.
   * <p>
   * This method also translates default values. If the field's type is a union with null
   * and has a default value, then this method also translates the default value of the field
   * to comply with the new type of the field. If the default value is null,
   * then remove the default value. If new type is not a union and the default value
   * is of the non-null member type, then assign the default value to the
   * non-null value within the union value (i.e. the value of the only entry within the
   * JSON object.) If the new type is a union and the default value is of the
   * non-null member type, then assign the default value to a JSON object
   * containing a single entry with the key being the member type discriminator of
   * the first union member and the value being the actual member value.
   * <p>
   * Both the schema and default value translation takes into account that default value
   * representation for Avro unions does not include the member type discriminator and
   * the type of the default value is always the 1st member of the union.
   *
   * @param avroSchemaInJson provides the JSON representation of the Avro {@link Schema}.
   * @param options specifies the {@link AvroToDataSchemaTranslationOptions}.
   * @return the translated {@link DataSchema}.
   * @throws IllegalArgumentException if the Avro {@link Schema} cannot be translated.
   */
public static DataSchema avroToDataSchema(String avroSchemaInJson, AvroToDataSchemaTranslationOptions options) throws IllegalArgumentException {
    ValidationOptions validationOptions = SchemaParser.getDefaultSchemaParserValidationOptions();
    validationOptions.setAvroUnionMode(true);
    SchemaParserFactory parserFactory = SchemaParserFactory.instance(validationOptions);
    DataSchemaResolver resolver = getResolver(parserFactory, options);
    PegasusSchemaParser parser = parserFactory.create(resolver);
    parser.parse(avroSchemaInJson);
    if (parser.hasError()) {
        throw new IllegalArgumentException(parser.errorMessage());
    }
    assert (parser.topLevelDataSchemas().size() == 1);
    DataSchema dataSchema = parser.topLevelDataSchemas().get(0);
    DataSchema resultDataSchema = null;
    AvroToDataSchemaTranslationMode translationMode = options.getTranslationMode();
    if (translationMode == AvroToDataSchemaTranslationMode.RETURN_EMBEDDED_SCHEMA || translationMode == AvroToDataSchemaTranslationMode.VERIFY_EMBEDDED_SCHEMA) {
        // check for embedded schema
        Object dataProperty = dataSchema.getProperties().get(SchemaTranslator.DATA_PROPERTY);
        if (dataProperty != null && dataProperty.getClass() == DataMap.class) {
            Object schemaProperty = ((DataMap) dataProperty).get(SchemaTranslator.SCHEMA_PROPERTY);
            if (schemaProperty.getClass() == DataMap.class) {
                SchemaParser embeddedSchemaParser = SchemaParserFactory.instance().create(null);
                embeddedSchemaParser.parse(Arrays.asList(schemaProperty));
                if (embeddedSchemaParser.hasError()) {
                    throw new IllegalArgumentException("Embedded schema is invalid\n" + embeddedSchemaParser.errorMessage());
                }
                assert (embeddedSchemaParser.topLevelDataSchemas().size() == 1);
                resultDataSchema = embeddedSchemaParser.topLevelDataSchemas().get(0);
                if (translationMode == AvroToDataSchemaTranslationMode.VERIFY_EMBEDDED_SCHEMA) {
                    // additional verification to make sure that embedded schema translates to Avro schema
                    DataToAvroSchemaTranslationOptions dataToAvdoSchemaOptions = new DataToAvroSchemaTranslationOptions();
                    Object optionalDefaultModeProperty = ((DataMap) dataProperty).get(SchemaTranslator.OPTIONAL_DEFAULT_MODE_PROPERTY);
                    dataToAvdoSchemaOptions.setOptionalDefaultMode(OptionalDefaultMode.valueOf(optionalDefaultModeProperty.toString()));
                    Schema avroSchemaFromEmbedded = dataToAvroSchema(resultDataSchema, dataToAvdoSchemaOptions);
                    Schema avroSchemaFromJson = Schema.parse(avroSchemaInJson);
                    if (avroSchemaFromEmbedded.equals(avroSchemaFromJson) == false) {
                        throw new IllegalArgumentException("Embedded schema does not translate to input Avro schema: " + avroSchemaInJson);
                    }
                }
            }
        }
    }
    if (resultDataSchema == null) {
        // translationMode == TRANSLATE or no embedded schema
        DataSchemaTraverse traverse = new DataSchemaTraverse();
        traverse.traverse(dataSchema, AvroToDataSchemaConvertCallback.INSTANCE);
        // convert default values
        traverse.traverse(dataSchema, DefaultAvroToDataConvertCallback.INSTANCE);
        // make sure it can round-trip
        String dataSchemaJson = dataSchema.toString();
        resultDataSchema = DataTemplateUtil.parseSchema(dataSchemaJson);
    }
    return resultDataSchema;
}