Examples with DataType org.apache.nifi.serialization.record.DataType used on opensource projects

Example 36 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class DataTypeUtils method convertType.

public static Object convertType(final Object value, final DataType dataType, final Supplier<DateFormat> dateFormat, final Supplier<DateFormat> timeFormat, final Supplier<DateFormat> timestampFormat, final String fieldName) {
    if (value == null) {
        return null;
    }
    switch(dataType.getFieldType()) {
        case BIGINT:
            return toBigInt(value, fieldName);
        case BOOLEAN:
            return toBoolean(value, fieldName);
        case BYTE:
            return toByte(value, fieldName);
        case CHAR:
            return toCharacter(value, fieldName);
        case DATE:
            return toDate(value, dateFormat, fieldName);
        case DOUBLE:
            return toDouble(value, fieldName);
        case FLOAT:
            return toFloat(value, fieldName);
        case INT:
            return toInteger(value, fieldName);
        case LONG:
            return toLong(value, fieldName);
        case SHORT:
            return toShort(value, fieldName);
        case STRING:
            return toString(value, () -> getDateFormat(dataType.getFieldType(), dateFormat, timeFormat, timestampFormat));
        case TIME:
            return toTime(value, timeFormat, fieldName);
        case TIMESTAMP:
            return toTimestamp(value, timestampFormat, fieldName);
        case ARRAY:
            return toArray(value, fieldName);
        case MAP:
            return toMap(value, fieldName);
        case RECORD:
            final RecordDataType recordType = (RecordDataType) dataType;
            final RecordSchema childSchema = recordType.getChildSchema();
            return toRecord(value, childSchema, fieldName);
        case CHOICE:
            {
                final ChoiceDataType choiceDataType = (ChoiceDataType) dataType;
                final DataType chosenDataType = chooseDataType(value, choiceDataType);
                if (chosenDataType == null) {
                    throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type " + value.getClass() + " for field " + fieldName + " to any of the following available Sub-Types for a Choice: " + choiceDataType.getPossibleSubTypes());
                }
                return convertType(value, chosenDataType, fieldName);
            }
    }
    return null;
}

Example 37 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class JacksonCSVRecordReader method nextRecord.

@Override
public Record nextRecord(final boolean coerceTypes, final boolean dropUnknownFields) throws IOException, MalformedRecordException {
    final RecordSchema schema = getSchema();
    if (recordStream.hasNext()) {
        String[] csvRecord = recordStream.next();
        // If the first record is the header names (and we're using them), store those off for use in creating the value map on the next iterations
        if (rawFieldNames == null) {
            if (!hasHeader || ignoreHeader) {
                rawFieldNames = schema.getFieldNames();
            } else {
                rawFieldNames = Arrays.asList(csvRecord);
                // Advance the stream to keep the record count correct
                if (recordStream.hasNext()) {
                    csvRecord = recordStream.next();
                } else {
                    return null;
                }
            }
        }
        // Check for empty lines and ignore them
        boolean foundRecord = true;
        if (csvRecord == null || (csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) {
            foundRecord = false;
            while (recordStream.hasNext()) {
                csvRecord = recordStream.next();
                if (csvRecord != null && !(csvRecord.length == 1 && StringUtils.isEmpty(csvRecord[0]))) {
                    // This is a non-empty record/row, so continue processing
                    foundRecord = true;
                    break;
                }
            }
        }
        // If we didn't find a record, then the end of the file was comprised of empty lines, so we have no record to return
        if (!foundRecord) {
            return null;
        }
        final Map<String, Object> values = new HashMap<>(rawFieldNames.size() * 2);
        final int numFieldNames = rawFieldNames.size();
        for (int i = 0; i < csvRecord.length; i++) {
            final String rawFieldName = numFieldNames <= i ? "unknown_field_index_" + i : rawFieldNames.get(i);
            String rawValue = (i >= csvRecord.length) ? null : csvRecord[i];
            final Optional<DataType> dataTypeOption = schema.getDataType(rawFieldName);
            if (!dataTypeOption.isPresent() && dropUnknownFields) {
                continue;
            }
            final Object value;
            if (coerceTypes && dataTypeOption.isPresent()) {
                value = convert(rawValue, dataTypeOption.get(), rawFieldName);
            } else if (dataTypeOption.isPresent()) {
                // The CSV Reader is going to return all fields as Strings, because CSV doesn't have any way to
                // dictate a field type. As a result, we will use the schema that we have to attempt to convert
                // the value into the desired type if it's a simple type.
                value = convertSimpleIfPossible(rawValue, dataTypeOption.get(), rawFieldName);
            } else {
                value = rawValue;
            }
            values.put(rawFieldName, value);
        }
        return new MapRecord(schema, values, coerceTypes, dropUnknownFields);
    }
    return null;
}

Example 38 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class JsonPathRowRecordReader method convert.

@SuppressWarnings("unchecked")
protected Object convert(final Object value, final DataType dataType, final String fieldName, final Object defaultValue) {
    if (value == null) {
        return defaultValue;
    }
    if (value instanceof List) {
        if (dataType.getFieldType() != RecordFieldType.ARRAY) {
            throw new IllegalTypeConversionException("Cannot convert value [" + value + "] of type Array to " + dataType);
        }
        final ArrayDataType arrayType = (ArrayDataType) dataType;
        final List<?> list = (List<?>) value;
        final Object[] coercedValues = new Object[list.size()];
        int i = 0;
        for (final Object rawValue : list) {
            coercedValues[i++] = convert(rawValue, arrayType.getElementType(), fieldName, null);
        }
        return coercedValues;
    }
    if (dataType.getFieldType() == RecordFieldType.RECORD && value instanceof Map) {
        final RecordDataType recordDataType = (RecordDataType) dataType;
        final RecordSchema childSchema = recordDataType.getChildSchema();
        final Map<String, Object> rawValues = (Map<String, Object>) value;
        final Map<String, Object> coercedValues = new HashMap<>();
        for (final Map.Entry<String, Object> entry : rawValues.entrySet()) {
            final String key = entry.getKey();
            final Optional<DataType> desiredTypeOption = childSchema.getDataType(key);
            if (desiredTypeOption.isPresent()) {
                final Optional<RecordField> field = childSchema.getField(key);
                final Object defaultFieldValue = field.isPresent() ? field.get().getDefaultValue() : null;
                final Object coercedValue = convert(entry.getValue(), desiredTypeOption.get(), fieldName + "." + key, defaultFieldValue);
                coercedValues.put(key, coercedValue);
            }
        }
        return new MapRecord(childSchema, coercedValues);
    } else {
        return DataTypeUtils.convertType(value, dataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, fieldName);
    }
}

Example 39 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class JsonPathRowRecordReader method convertJsonNodeToRecord.

@Override
protected Record convertJsonNodeToRecord(final JsonNode jsonNode, final RecordSchema schema, final boolean coerceTypes, final boolean dropUnknownFields) throws IOException {
    if (jsonNode == null) {
        return null;
    }
    final DocumentContext ctx = JsonPath.using(STRICT_PROVIDER_CONFIGURATION).parse(jsonNode.toString());
    final Map<String, Object> values = new HashMap<>(schema.getFieldCount());
    for (final Map.Entry<String, JsonPath> entry : jsonPaths.entrySet()) {
        final String fieldName = entry.getKey();
        final DataType desiredType = schema.getDataType(fieldName).orElse(null);
        if (desiredType == null && dropUnknownFields) {
            continue;
        }
        final JsonPath jsonPath = entry.getValue();
        Object value;
        try {
            value = ctx.read(jsonPath);
        } catch (final PathNotFoundException pnfe) {
            logger.debug("Evaluated JSONPath Expression {} but the path was not found; will use a null value", new Object[] { entry.getValue() });
            value = null;
        }
        final Optional<RecordField> field = schema.getField(fieldName);
        final Object defaultValue = field.isPresent() ? field.get().getDefaultValue() : null;
        if (coerceTypes && desiredType != null) {
            value = convert(value, desiredType, fieldName, defaultValue);
        } else {
            final DataType dataType = field.isPresent() ? field.get().getDataType() : null;
            value = convert(value, dataType);
        }
        values.put(fieldName, value);
    }
    return new MapRecord(schema, values);
}

Example 40 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class JsonTreeRowRecordReader method convertJsonNodeToRecord.

private Record convertJsonNodeToRecord(final JsonNode jsonNode, final RecordSchema schema, final String fieldNamePrefix, final boolean coerceTypes, final boolean dropUnknown) throws IOException, MalformedRecordException {
    final Map<String, Object> values = new HashMap<>(schema.getFieldCount() * 2);
    if (dropUnknown) {
        for (final RecordField recordField : schema.getFields()) {
            final JsonNode childNode = getChildNode(jsonNode, recordField);
            if (childNode == null) {
                continue;
            }
            final String fieldName = recordField.getFieldName();
            final Object value;
            if (coerceTypes) {
                final DataType desiredType = recordField.getDataType();
                final String fullFieldName = fieldNamePrefix == null ? fieldName : fieldNamePrefix + fieldName;
                value = convertField(childNode, fullFieldName, desiredType, dropUnknown);
            } else {
                value = getRawNodeValue(childNode, recordField == null ? null : recordField.getDataType());
            }
            values.put(fieldName, value);
        }
    } else {
        final Iterator<String> fieldNames = jsonNode.getFieldNames();
        while (fieldNames.hasNext()) {
            final String fieldName = fieldNames.next();
            final JsonNode childNode = jsonNode.get(fieldName);
            final RecordField recordField = schema.getField(fieldName).orElse(null);
            final Object value;
            if (coerceTypes && recordField != null) {
                final DataType desiredType = recordField.getDataType();
                final String fullFieldName = fieldNamePrefix == null ? fieldName : fieldNamePrefix + fieldName;
                value = convertField(childNode, fullFieldName, desiredType, dropUnknown);
            } else {
                value = getRawNodeValue(childNode, recordField == null ? null : recordField.getDataType());
            }
            values.put(fieldName, value);
        }
    }
    final Supplier<String> supplier = () -> jsonNode.toString();
    return new MapRecord(schema, values, SerializedForm.of(supplier, "application/json"), false, dropUnknown);
}