Examples with DataType org.apache.nifi.serialization.record.DataType used on opensource projects

Example 26 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class StandardSchemaValidator method getCanonicalDataType.

private DataType getCanonicalDataType(final DataType dataType, final Object rawValue, final StandardSchemaValidationResult result, final String fieldPrefix, final RecordField field) {
    final RecordFieldType fieldType = dataType.getFieldType();
    final DataType canonicalDataType;
    if (fieldType == RecordFieldType.CHOICE) {
        canonicalDataType = DataTypeUtils.chooseDataType(rawValue, (ChoiceDataType) dataType);
        if (canonicalDataType == null) {
            result.addValidationError(new StandardValidationError(concat(fieldPrefix, field), rawValue, ValidationErrorType.INVALID_FIELD, "Value is of type " + rawValue.getClass().getName() + " but was expected to be of type " + dataType));
            return null;
        }
    } else {
        canonicalDataType = dataType;
    }
    return canonicalDataType;
}

Example 27 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class StandardSchemaValidator method isTypeCorrect.

private boolean isTypeCorrect(final Object value, final DataType dataType) {
    switch(dataType.getFieldType()) {
        case ARRAY:
            if (!(value instanceof Object[])) {
                return false;
            }
            final ArrayDataType arrayDataType = (ArrayDataType) dataType;
            final DataType elementType = arrayDataType.getElementType();
            final Object[] array = (Object[]) value;
            for (final Object arrayVal : array) {
                if (!isTypeCorrect(arrayVal, elementType)) {
                    return false;
                }
            }
            return true;
        case MAP:
            if (!(value instanceof Map)) {
                return false;
            }
            final MapDataType mapDataType = (MapDataType) dataType;
            final DataType valueDataType = mapDataType.getValueType();
            final Map<?, ?> map = (Map<?, ?>) value;
            for (final Object mapValue : map.values()) {
                if (!isTypeCorrect(mapValue, valueDataType)) {
                    return false;
                }
            }
            return true;
        case RECORD:
            return value instanceof Record;
        case CHOICE:
            final ChoiceDataType choiceDataType = (ChoiceDataType) dataType;
            for (final DataType choice : choiceDataType.getPossibleSubTypes()) {
                if (isTypeCorrect(value, choice)) {
                    return true;
                }
            }
            return false;
        case BIGINT:
            return value instanceof BigInteger;
        case BOOLEAN:
            return value instanceof Boolean;
        case BYTE:
            return value instanceof Byte;
        case CHAR:
            return value instanceof Character;
        case DATE:
            return value instanceof java.sql.Date;
        case DOUBLE:
            return value instanceof Double;
        case FLOAT:
            // Same goes for Short/Integer
            return value instanceof Float;
        case INT:
            return value instanceof Integer;
        case LONG:
            return value instanceof Long;
        case SHORT:
            return value instanceof Short;
        case STRING:
            return value instanceof String;
        case TIME:
            return value instanceof java.sql.Time;
        case TIMESTAMP:
            return value instanceof java.sql.Timestamp;
    }
    return false;
}

Example 28 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class StandardSchemaValidator method verifyComplexType.

private void verifyComplexType(final DataType dataType, final Object rawValue, final StandardSchemaValidationResult result, final String fieldPrefix, final RecordField field) {
    // If the field type is RECORD, or if the field type is a CHOICE that allows for a RECORD and the value is a RECORD, then we
    // need to dig into each of the sub-fields. To do this, we first need to determine the 'canonical data type'.
    final DataType canonicalDataType = getCanonicalDataType(dataType, rawValue, result, fieldPrefix, field);
    if (canonicalDataType == null) {
        return;
    }
    // Now that we have the 'canonical data type', we check if it is a Record. If so, we need to validate each sub-field.
    if (canonicalDataType.getFieldType() == RecordFieldType.RECORD) {
        verifyChildRecord(canonicalDataType, rawValue, dataType, result, field, fieldPrefix);
    }
    if (canonicalDataType.getFieldType() == RecordFieldType.ARRAY) {
        final ArrayDataType arrayDataType = (ArrayDataType) canonicalDataType;
        final DataType elementType = arrayDataType.getElementType();
        final Object[] arrayObject = (Object[]) rawValue;
        int i = 0;
        for (final Object arrayValue : arrayObject) {
            verifyComplexType(elementType, arrayValue, result, fieldPrefix + "[" + i + "]", field);
            i++;
        }
    }
}

Example 29 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class StandardSchemaValidator method validate.

private SchemaValidationResult validate(final Record record, final RecordSchema schema, final String fieldPrefix) {
    // Ensure that for every field in the schema, the type is correct (if we care) and that
    // a value is present (unless it is nullable).
    final StandardSchemaValidationResult result = new StandardSchemaValidationResult();
    for (final RecordField field : schema.getFields()) {
        final Object rawValue = record.getValue(field);
        // If there is no value, then it is always valid unless the field is required.
        if (rawValue == null) {
            if (!field.isNullable() && field.getDefaultValue() == null) {
                result.addValidationError(new StandardValidationError(concat(fieldPrefix, field), ValidationErrorType.MISSING_FIELD, "Field is required"));
            }
            continue;
        }
        // Check that the type is correct.
        final DataType dataType = field.getDataType();
        if (validationContext.isStrictTypeChecking()) {
            if (!isTypeCorrect(rawValue, dataType)) {
                result.addValidationError(new StandardValidationError(concat(fieldPrefix, field), rawValue, ValidationErrorType.INVALID_FIELD, "Value is of type " + rawValue.getClass().getName() + " but was expected to be of type " + dataType));
                continue;
            }
        } else {
            // but will be false if the value is "123" and should be an Array or Record.
            if (!DataTypeUtils.isCompatibleDataType(rawValue, dataType)) {
                result.addValidationError(new StandardValidationError(concat(fieldPrefix, field), rawValue, ValidationErrorType.INVALID_FIELD, "Value is of type " + rawValue.getClass().getName() + " but was expected to be of type " + dataType));
                continue;
            }
        }
        // If the field type is RECORD, or if the field type is a CHOICE that allows for a RECORD and the value is a RECORD, then we
        // need to dig into each of the sub-fields. To do this, we first need to determine the 'canonical data type'.
        final DataType canonicalDataType = getCanonicalDataType(dataType, rawValue, result, fieldPrefix, field);
        if (canonicalDataType == null) {
            continue;
        }
        // Now that we have the 'canonical data type', we check if it is a Record. If so, we need to validate each sub-field.
        verifyComplexType(dataType, rawValue, result, fieldPrefix, field);
    }
    if (!validationContext.isExtraFieldAllowed()) {
        for (final String fieldName : record.getRawFieldNames()) {
            if (!schema.getDataType(fieldName).isPresent()) {
                result.addValidationError(new StandardValidationError(fieldPrefix + "/" + fieldName, ValidationErrorType.EXTRA_FIELD, "Field is not present in the schema"));
            }
        }
    }
    return result;
}

Example 30 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class TestStandardSchemaValidator method testValidateCorrectSimpleTypesStrictValidation.

@Test
public void testValidateCorrectSimpleTypesStrictValidation() throws ParseException {
    final List<RecordField> fields = new ArrayList<>();
    for (final RecordFieldType fieldType : RecordFieldType.values()) {
        if (fieldType == RecordFieldType.CHOICE) {
            final List<DataType> possibleTypes = new ArrayList<>();
            possibleTypes.add(RecordFieldType.INT.getDataType());
            possibleTypes.add(RecordFieldType.LONG.getDataType());
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getChoiceDataType(possibleTypes)));
        } else if (fieldType == RecordFieldType.MAP) {
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getMapDataType(RecordFieldType.INT.getDataType())));
        } else {
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getDataType()));
        }
    }
    final DateFormat df = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS");
    df.setTimeZone(TimeZone.getTimeZone("gmt"));
    final long time = df.parse("2017/01/01 17:00:00.000").getTime();
    final Map<String, Object> intMap = new LinkedHashMap<>();
    intMap.put("height", 48);
    intMap.put("width", 96);
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> valueMap = new LinkedHashMap<>();
    valueMap.put("string", "string");
    valueMap.put("boolean", true);
    valueMap.put("byte", (byte) 1);
    valueMap.put("char", 'c');
    valueMap.put("short", (short) 8);
    valueMap.put("int", 9);
    valueMap.put("bigint", BigInteger.valueOf(8L));
    valueMap.put("long", 8L);
    valueMap.put("float", 8.0F);
    valueMap.put("double", 8.0D);
    valueMap.put("date", new Date(time));
    valueMap.put("time", new Time(time));
    valueMap.put("timestamp", new Timestamp(time));
    valueMap.put("record", null);
    valueMap.put("array", null);
    valueMap.put("choice", 48L);
    valueMap.put("map", intMap);
    final Record record = new MapRecord(schema, valueMap);
    final SchemaValidationContext validationContext = new SchemaValidationContext(schema, false, true);
    final StandardSchemaValidator validator = new StandardSchemaValidator(validationContext);
    final SchemaValidationResult result = validator.validate(record);
    assertTrue(result.isValid());
    assertNotNull(result.getValidationErrors());
    assertTrue(result.getValidationErrors().isEmpty());
}