Examples with RecordFieldType org.apache.nifi.serialization.record.RecordFieldType used on opensource projects

Example 11 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class TestJsonTreeRowRecordReader method testElementWithNestedData.

@Test
public void testElementWithNestedData() throws IOException, MalformedRecordException {
    final DataType accountType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema());
    final List<RecordField> fields = getDefaultFields();
    fields.add(new RecordField("account", accountType));
    fields.remove(new RecordField("balance", RecordFieldType.DOUBLE.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream in = new FileInputStream(new File("src/test/resources/json/single-element-nested.json"));
        final JsonTreeRowRecordReader reader = new JsonTreeRowRecordReader(in, Mockito.mock(ComponentLog.class), schema, dateFormat, timeFormat, timestampFormat)) {
        final List<RecordFieldType> dataTypes = schema.getDataTypes().stream().map(dt -> dt.getFieldType()).collect(Collectors.toList());
        final List<RecordFieldType> expectedTypes = Arrays.asList(new RecordFieldType[] { RecordFieldType.INT, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.STRING, RecordFieldType.RECORD });
        assertEquals(expectedTypes, dataTypes);
        final Object[] firstRecordValues = reader.nextRecord().getValues();
        final Object[] allButLast = Arrays.copyOfRange(firstRecordValues, 0, firstRecordValues.length - 1);
        Assert.assertArrayEquals(new Object[] { 1, "John Doe", "123 My Street", "My City", "MS", "11111", "USA" }, allButLast);
        final Object last = firstRecordValues[firstRecordValues.length - 1];
        assertTrue(Record.class.isAssignableFrom(last.getClass()));
        final Record record = (Record) last;
        assertEquals(42, record.getValue("id"));
        assertEquals(4750.89, record.getValue("balance"));
        assertNull(reader.nextRecord());
    }
}

Example 12 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class DataTypeUtils method isScalarValue.

public static boolean isScalarValue(final DataType dataType, final Object value) {
    final RecordFieldType fieldType = dataType.getFieldType();
    final RecordFieldType chosenType;
    if (fieldType == RecordFieldType.CHOICE) {
        final ChoiceDataType choiceDataType = (ChoiceDataType) dataType;
        final DataType chosenDataType = chooseDataType(value, choiceDataType);
        if (chosenDataType == null) {
            return false;
        }
        chosenType = chosenDataType.getFieldType();
    } else {
        chosenType = fieldType;
    }
    switch(chosenType) {
        case ARRAY:
        case MAP:
        case RECORD:
            return false;
    }
    return true;
}

Example 13 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class StandardSchemaValidator method getCanonicalDataType.

private DataType getCanonicalDataType(final DataType dataType, final Object rawValue, final StandardSchemaValidationResult result, final String fieldPrefix, final RecordField field) {
    final RecordFieldType fieldType = dataType.getFieldType();
    final DataType canonicalDataType;
    if (fieldType == RecordFieldType.CHOICE) {
        canonicalDataType = DataTypeUtils.chooseDataType(rawValue, (ChoiceDataType) dataType);
        if (canonicalDataType == null) {
            result.addValidationError(new StandardValidationError(concat(fieldPrefix, field), rawValue, ValidationErrorType.INVALID_FIELD, "Value is of type " + rawValue.getClass().getName() + " but was expected to be of type " + dataType));
            return null;
        }
    } else {
        canonicalDataType = dataType;
    }
    return canonicalDataType;
}

Example 14 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class TestStandardSchemaValidator method testValidateCorrectSimpleTypesStrictValidation.

@Test
public void testValidateCorrectSimpleTypesStrictValidation() throws ParseException {
    final List<RecordField> fields = new ArrayList<>();
    for (final RecordFieldType fieldType : RecordFieldType.values()) {
        if (fieldType == RecordFieldType.CHOICE) {
            final List<DataType> possibleTypes = new ArrayList<>();
            possibleTypes.add(RecordFieldType.INT.getDataType());
            possibleTypes.add(RecordFieldType.LONG.getDataType());
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getChoiceDataType(possibleTypes)));
        } else if (fieldType == RecordFieldType.MAP) {
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getMapDataType(RecordFieldType.INT.getDataType())));
        } else {
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getDataType()));
        }
    }
    final DateFormat df = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss.SSS");
    df.setTimeZone(TimeZone.getTimeZone("gmt"));
    final long time = df.parse("2017/01/01 17:00:00.000").getTime();
    final Map<String, Object> intMap = new LinkedHashMap<>();
    intMap.put("height", 48);
    intMap.put("width", 96);
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> valueMap = new LinkedHashMap<>();
    valueMap.put("string", "string");
    valueMap.put("boolean", true);
    valueMap.put("byte", (byte) 1);
    valueMap.put("char", 'c');
    valueMap.put("short", (short) 8);
    valueMap.put("int", 9);
    valueMap.put("bigint", BigInteger.valueOf(8L));
    valueMap.put("long", 8L);
    valueMap.put("float", 8.0F);
    valueMap.put("double", 8.0D);
    valueMap.put("date", new Date(time));
    valueMap.put("time", new Time(time));
    valueMap.put("timestamp", new Timestamp(time));
    valueMap.put("record", null);
    valueMap.put("array", null);
    valueMap.put("choice", 48L);
    valueMap.put("map", intMap);
    final Record record = new MapRecord(schema, valueMap);
    final SchemaValidationContext validationContext = new SchemaValidationContext(schema, false, true);
    final StandardSchemaValidator validator = new StandardSchemaValidator(validationContext);
    final SchemaValidationResult result = validator.validate(record);
    assertTrue(result.isValid());
    assertNotNull(result.getValidationErrors());
    assertTrue(result.getValidationErrors().isEmpty());
}

Example 15 with RecordFieldType

use of org.apache.nifi.serialization.record.RecordFieldType in project nifi by apache.

the class WildcardIndexPath method evaluate.

@Override
@SuppressWarnings("unchecked")
public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) {
    final Stream<FieldValue> parentResult = getParentPath().evaluate(context);
    return parentResult.filter(Filters.fieldTypeFilter(RecordFieldType.MAP, RecordFieldType.ARRAY)).flatMap(fieldValue -> {
        final RecordFieldType fieldType = fieldValue.getField().getDataType().getFieldType();
        final Object value = fieldValue.getValue();
        if (value == null) {
            return Stream.empty();
        }
        if (fieldType == RecordFieldType.MAP) {
            final Map<String, ?> map = (Map<String, ?>) value;
            return map.entrySet().stream().map(entry -> {
                final DataType valueType = ((MapDataType) fieldValue.getField().getDataType()).getValueType();
                final RecordField elementField = new RecordField(fieldValue.getField().getFieldName(), valueType);
                return new MapEntryFieldValue(entry.getValue(), elementField, fieldValue, entry.getKey());
            });
        } else {
            final Object[] array = (Object[]) value;
            return IntStream.range(0, array.length).mapToObj(index -> {
                final DataType elementDataType = ((ArrayDataType) fieldValue.getField().getDataType()).getElementType();
                final RecordField elementField = new RecordField(fieldValue.getField().getFieldName(), elementDataType);
                return new ArrayIndexFieldValue(array[index], elementField, fieldValue, index);
            });
        }
    });
}