Search in sources :

Example 6 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class AbstractJsonRowRecordReader method getRawNodeValue.

protected Object getRawNodeValue(final JsonNode fieldNode, final DataType dataType) throws IOException {
    if (fieldNode == null || fieldNode.isNull()) {
        return null;
    }
    if (fieldNode.isNumber()) {
        return fieldNode.getNumberValue();
    }
    if (fieldNode.isBinary()) {
        return fieldNode.getBinaryValue();
    }
    if (fieldNode.isBoolean()) {
        return fieldNode.getBooleanValue();
    }
    if (fieldNode.isTextual()) {
        return fieldNode.getTextValue();
    }
    if (fieldNode.isArray()) {
        final ArrayNode arrayNode = (ArrayNode) fieldNode;
        final int numElements = arrayNode.size();
        final Object[] arrayElements = new Object[numElements];
        int count = 0;
        final DataType elementDataType;
        if (dataType != null && dataType.getFieldType() == RecordFieldType.ARRAY) {
            final ArrayDataType arrayDataType = (ArrayDataType) dataType;
            elementDataType = arrayDataType.getElementType();
        } else {
            elementDataType = null;
        }
        for (final JsonNode node : arrayNode) {
            final Object value = getRawNodeValue(node, elementDataType);
            arrayElements[count++] = value;
        }
        return arrayElements;
    }
    if (fieldNode.isObject()) {
        RecordSchema childSchema;
        if (dataType != null && RecordFieldType.RECORD == dataType.getFieldType()) {
            final RecordDataType recordDataType = (RecordDataType) dataType;
            childSchema = recordDataType.getChildSchema();
        } else {
            childSchema = null;
        }
        if (childSchema == null) {
            childSchema = new SimpleRecordSchema(Collections.emptyList());
        }
        final Iterator<String> fieldNames = fieldNode.getFieldNames();
        final Map<String, Object> childValues = new HashMap<>();
        while (fieldNames.hasNext()) {
            final String childFieldName = fieldNames.next();
            final Object childValue = getRawNodeValue(fieldNode.get(childFieldName), dataType);
            childValues.put(childFieldName, childValue);
        }
        final MapRecord record = new MapRecord(childSchema, childValues);
        return record;
    }
    return null;
}
Also used : SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) MapRecord(org.apache.nifi.serialization.record.MapRecord) HashMap(java.util.HashMap) JsonNode(org.codehaus.jackson.JsonNode) RecordDataType(org.apache.nifi.serialization.record.type.RecordDataType) DataType(org.apache.nifi.serialization.record.DataType) RecordDataType(org.apache.nifi.serialization.record.type.RecordDataType) ArrayDataType(org.apache.nifi.serialization.record.type.ArrayDataType) ArrayNode(org.codehaus.jackson.node.ArrayNode) ArrayDataType(org.apache.nifi.serialization.record.type.ArrayDataType) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema)

Example 7 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class JsonPathRowRecordReader method convert.

@SuppressWarnings("unchecked")
protected Object convert(final Object value, final DataType dataType) {
    if (value == null) {
        return null;
    }
    if (value instanceof List) {
        final List<?> list = (List<?>) value;
        final Object[] array = new Object[list.size()];
        final DataType elementDataType;
        if (dataType != null && dataType.getFieldType() == RecordFieldType.ARRAY) {
            elementDataType = ((ArrayDataType) dataType).getElementType();
        } else {
            elementDataType = null;
        }
        int i = 0;
        for (final Object val : list) {
            array[i++] = convert(val, elementDataType);
        }
        return array;
    }
    if (value instanceof Map) {
        final Map<String, ?> map = (Map<String, ?>) value;
        boolean record = false;
        for (final Object obj : map.values()) {
            if (obj instanceof JsonNode) {
                record = true;
            }
        }
        if (!record) {
            return value;
        }
        RecordSchema childSchema = null;
        if (dataType != null && dataType.getFieldType() == RecordFieldType.RECORD) {
            childSchema = ((RecordDataType) dataType).getChildSchema();
        }
        if (childSchema == null) {
            childSchema = new SimpleRecordSchema(Collections.emptyList());
        }
        final Map<String, Object> values = new HashMap<>();
        for (final Map.Entry<String, ?> entry : map.entrySet()) {
            final String key = entry.getKey();
            final Object childValue = entry.getValue();
            final RecordField recordField = childSchema.getField(key).orElse(null);
            final DataType childDataType = recordField == null ? null : recordField.getDataType();
            values.put(key, convert(childValue, childDataType));
        }
        return new MapRecord(childSchema, values);
    }
    return value;
}
Also used : SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) MapRecord(org.apache.nifi.serialization.record.MapRecord) RecordField(org.apache.nifi.serialization.record.RecordField) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) JsonNode(org.codehaus.jackson.JsonNode) DataType(org.apache.nifi.serialization.record.DataType) RecordDataType(org.apache.nifi.serialization.record.type.RecordDataType) ArrayDataType(org.apache.nifi.serialization.record.type.ArrayDataType) List(java.util.List) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema)

Example 8 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class WriteJsonResult method writeRecord.

private void writeRecord(final Record record, final RecordSchema writeSchema, final JsonGenerator generator, final GeneratorTask startTask, final GeneratorTask endTask, final boolean schemaAware) throws JsonGenerationException, IOException {
    final Optional<SerializedForm> serializedForm = record.getSerializedForm();
    if (serializedForm.isPresent()) {
        final SerializedForm form = serializedForm.get();
        if (form.getMimeType().equals(getMimeType()) && record.getSchema().equals(writeSchema)) {
            final Object serialized = form.getSerialized();
            if (serialized instanceof String) {
                generator.writeRawValue((String) serialized);
                return;
            }
        }
    }
    try {
        startTask.apply(generator);
        if (schemaAware) {
            for (final RecordField field : writeSchema.getFields()) {
                final String fieldName = field.getFieldName();
                final Object value = record.getValue(field);
                if (value == null) {
                    if (nullSuppression == NullSuppression.NEVER_SUPPRESS || (nullSuppression == NullSuppression.SUPPRESS_MISSING) && isFieldPresent(field, record)) {
                        generator.writeNullField(fieldName);
                    }
                    continue;
                }
                generator.writeFieldName(fieldName);
                final DataType dataType = writeSchema.getDataType(fieldName).get();
                writeValue(generator, value, fieldName, dataType);
            }
        } else {
            for (final String fieldName : record.getRawFieldNames()) {
                final Object value = record.getValue(fieldName);
                if (value == null) {
                    if (nullSuppression == NullSuppression.NEVER_SUPPRESS || (nullSuppression == NullSuppression.SUPPRESS_MISSING) && record.getRawFieldNames().contains(fieldName)) {
                        generator.writeNullField(fieldName);
                    }
                    continue;
                }
                generator.writeFieldName(fieldName);
                writeRawValue(generator, value, fieldName);
            }
        }
        endTask.apply(generator);
    } catch (final Exception e) {
        logger.error("Failed to write {} with schema {} as a JSON Object due to {}", new Object[] { record, record.getSchema(), e.toString(), e });
        throw e;
    }
}
Also used : RecordField(org.apache.nifi.serialization.record.RecordField) DataType(org.apache.nifi.serialization.record.DataType) ChoiceDataType(org.apache.nifi.serialization.record.type.ChoiceDataType) RecordDataType(org.apache.nifi.serialization.record.type.RecordDataType) MapDataType(org.apache.nifi.serialization.record.type.MapDataType) ArrayDataType(org.apache.nifi.serialization.record.type.ArrayDataType) SerializedForm(org.apache.nifi.serialization.record.SerializedForm) IOException(java.io.IOException) JsonGenerationException(org.codehaus.jackson.JsonGenerationException)

Example 9 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class WriteJsonResult method writeValue.

@SuppressWarnings("unchecked")
private void writeValue(final JsonGenerator generator, final Object value, final String fieldName, final DataType dataType) throws JsonGenerationException, IOException {
    if (value == null) {
        generator.writeNull();
        return;
    }
    final DataType chosenDataType = dataType.getFieldType() == RecordFieldType.CHOICE ? DataTypeUtils.chooseDataType(value, (ChoiceDataType) dataType) : dataType;
    final Object coercedValue = DataTypeUtils.convertType(value, chosenDataType, LAZY_DATE_FORMAT, LAZY_TIME_FORMAT, LAZY_TIMESTAMP_FORMAT, fieldName);
    if (coercedValue == null) {
        generator.writeNull();
        return;
    }
    switch(chosenDataType.getFieldType()) {
        case DATE:
            {
                final String stringValue = DataTypeUtils.toString(coercedValue, LAZY_DATE_FORMAT);
                if (DataTypeUtils.isLongTypeCompatible(stringValue)) {
                    generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName));
                } else {
                    generator.writeString(stringValue);
                }
                break;
            }
        case TIME:
            {
                final String stringValue = DataTypeUtils.toString(coercedValue, LAZY_TIME_FORMAT);
                if (DataTypeUtils.isLongTypeCompatible(stringValue)) {
                    generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName));
                } else {
                    generator.writeString(stringValue);
                }
                break;
            }
        case TIMESTAMP:
            {
                final String stringValue = DataTypeUtils.toString(coercedValue, LAZY_TIMESTAMP_FORMAT);
                if (DataTypeUtils.isLongTypeCompatible(stringValue)) {
                    generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName));
                } else {
                    generator.writeString(stringValue);
                }
                break;
            }
        case DOUBLE:
            generator.writeNumber(DataTypeUtils.toDouble(coercedValue, fieldName));
            break;
        case FLOAT:
            generator.writeNumber(DataTypeUtils.toFloat(coercedValue, fieldName));
            break;
        case LONG:
            generator.writeNumber(DataTypeUtils.toLong(coercedValue, fieldName));
            break;
        case INT:
        case BYTE:
        case SHORT:
            generator.writeNumber(DataTypeUtils.toInteger(coercedValue, fieldName));
            break;
        case CHAR:
        case STRING:
            generator.writeString(coercedValue.toString());
            break;
        case BIGINT:
            if (coercedValue instanceof Long) {
                generator.writeNumber(((Long) coercedValue).longValue());
            } else {
                generator.writeNumber((BigInteger) coercedValue);
            }
            break;
        case BOOLEAN:
            final String stringValue = coercedValue.toString();
            if ("true".equalsIgnoreCase(stringValue)) {
                generator.writeBoolean(true);
            } else if ("false".equalsIgnoreCase(stringValue)) {
                generator.writeBoolean(false);
            } else {
                generator.writeString(stringValue);
            }
            break;
        case RECORD:
            {
                final Record record = (Record) coercedValue;
                final RecordDataType recordDataType = (RecordDataType) chosenDataType;
                final RecordSchema childSchema = recordDataType.getChildSchema();
                writeRecord(record, childSchema, generator, gen -> gen.writeStartObject(), gen -> gen.writeEndObject(), true);
                break;
            }
        case MAP:
            {
                final MapDataType mapDataType = (MapDataType) chosenDataType;
                final DataType valueDataType = mapDataType.getValueType();
                final Map<String, ?> map = (Map<String, ?>) coercedValue;
                generator.writeStartObject();
                for (final Map.Entry<String, ?> entry : map.entrySet()) {
                    final String mapKey = entry.getKey();
                    final Object mapValue = entry.getValue();
                    generator.writeFieldName(mapKey);
                    writeValue(generator, mapValue, fieldName + "." + mapKey, valueDataType);
                }
                generator.writeEndObject();
                break;
            }
        case ARRAY:
        default:
            if (coercedValue instanceof Object[]) {
                final Object[] values = (Object[]) coercedValue;
                final ArrayDataType arrayDataType = (ArrayDataType) dataType;
                final DataType elementType = arrayDataType.getElementType();
                writeArray(values, fieldName, generator, elementType);
            } else {
                generator.writeString(coercedValue.toString());
            }
            break;
    }
}
Also used : AbstractRecordSetWriter(org.apache.nifi.serialization.AbstractRecordSetWriter) SchemaAccessWriter(org.apache.nifi.schema.access.SchemaAccessWriter) DataType(org.apache.nifi.serialization.record.DataType) ComponentLog(org.apache.nifi.logging.ComponentLog) Supplier(java.util.function.Supplier) ChoiceDataType(org.apache.nifi.serialization.record.type.ChoiceDataType) RawRecordWriter(org.apache.nifi.serialization.record.RawRecordWriter) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) JsonGenerator(org.codehaus.jackson.JsonGenerator) Map(java.util.Map) BigInteger(java.math.BigInteger) DateFormat(java.text.DateFormat) Record(org.apache.nifi.serialization.record.Record) OutputStream(java.io.OutputStream) RecordField(org.apache.nifi.serialization.record.RecordField) RecordDataType(org.apache.nifi.serialization.record.type.RecordDataType) WriteResult(org.apache.nifi.serialization.WriteResult) DataTypeUtils(org.apache.nifi.serialization.record.util.DataTypeUtils) Set(java.util.Set) IOException(java.io.IOException) JsonGenerationException(org.codehaus.jackson.JsonGenerationException) JsonFactory(org.codehaus.jackson.JsonFactory) SerializedForm(org.apache.nifi.serialization.record.SerializedForm) MapDataType(org.apache.nifi.serialization.record.type.MapDataType) Optional(java.util.Optional) RecordSetWriter(org.apache.nifi.serialization.RecordSetWriter) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) ArrayDataType(org.apache.nifi.serialization.record.type.ArrayDataType) MapDataType(org.apache.nifi.serialization.record.type.MapDataType) RecordDataType(org.apache.nifi.serialization.record.type.RecordDataType) DataType(org.apache.nifi.serialization.record.DataType) ChoiceDataType(org.apache.nifi.serialization.record.type.ChoiceDataType) RecordDataType(org.apache.nifi.serialization.record.type.RecordDataType) MapDataType(org.apache.nifi.serialization.record.type.MapDataType) ArrayDataType(org.apache.nifi.serialization.record.type.ArrayDataType) Record(org.apache.nifi.serialization.record.Record) ChoiceDataType(org.apache.nifi.serialization.record.type.ChoiceDataType) ArrayDataType(org.apache.nifi.serialization.record.type.ArrayDataType) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) Map(java.util.Map)

Example 10 with DataType

use of org.apache.nifi.serialization.record.DataType in project nifi by apache.

the class TestWriteCSVResult method testDataTypes.

@Test
public void testDataTypes() throws IOException {
    final CSVFormat csvFormat = CSVFormat.DEFAULT.withQuoteMode(QuoteMode.ALL).withRecordSeparator("\n");
    final StringBuilder headerBuilder = new StringBuilder();
    final List<RecordField> fields = new ArrayList<>();
    for (final RecordFieldType fieldType : RecordFieldType.values()) {
        if (fieldType == RecordFieldType.CHOICE) {
            final List<DataType> possibleTypes = new ArrayList<>();
            possibleTypes.add(RecordFieldType.INT.getDataType());
            possibleTypes.add(RecordFieldType.LONG.getDataType());
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getChoiceDataType(possibleTypes)));
        } else {
            fields.add(new RecordField(fieldType.name().toLowerCase(), fieldType.getDataType()));
        }
        headerBuilder.append('"').append(fieldType.name().toLowerCase()).append('"').append(",");
    }
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final long now = System.currentTimeMillis();
    try (final WriteCSVResult result = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "UTF-8")) {
        final Map<String, Object> valueMap = new HashMap<>();
        valueMap.put("string", "a孟bc李12儒3");
        valueMap.put("boolean", true);
        valueMap.put("byte", (byte) 1);
        valueMap.put("char", 'c');
        valueMap.put("short", (short) 8);
        valueMap.put("int", 9);
        valueMap.put("bigint", BigInteger.valueOf(8L));
        valueMap.put("long", 8L);
        valueMap.put("float", 8.0F);
        valueMap.put("double", 8.0D);
        valueMap.put("date", new Date(now));
        valueMap.put("time", new Time(now));
        valueMap.put("timestamp", new Timestamp(now));
        valueMap.put("record", null);
        valueMap.put("choice", 48L);
        valueMap.put("array", null);
        final Record record = new MapRecord(schema, valueMap);
        final RecordSet rs = RecordSet.of(schema, record);
        result.write(rs);
    }
    final String output = new String(baos.toByteArray(), StandardCharsets.UTF_8);
    headerBuilder.deleteCharAt(headerBuilder.length() - 1);
    final String headerLine = headerBuilder.toString();
    final String[] splits = output.split("\n");
    assertEquals(2, splits.length);
    assertEquals(headerLine, splits[0]);
    final String values = splits[1];
    final StringBuilder expectedBuilder = new StringBuilder();
    expectedBuilder.append("\"a孟bc李12儒3\",\"true\",\"1\",\"c\",\"8\",\"9\",\"8\",\"8\",\"8.0\",\"8.0\",");
    final String dateValue = getDateFormat(RecordFieldType.DATE.getDefaultFormat()).format(now);
    final String timeValue = getDateFormat(RecordFieldType.TIME.getDefaultFormat()).format(now);
    final String timestampValue = getDateFormat(RecordFieldType.TIMESTAMP.getDefaultFormat()).format(now);
    expectedBuilder.append('"').append(dateValue).append('"').append(',');
    expectedBuilder.append('"').append(timeValue).append('"').append(',');
    expectedBuilder.append('"').append(timestampValue).append('"').append(',');
    expectedBuilder.append(",\"48\",,");
    final String expectedValues = expectedBuilder.toString();
    assertEquals(expectedValues, values);
}
Also used : SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) RecordField(org.apache.nifi.serialization.record.RecordField) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ArrayList(java.util.ArrayList) Time(java.sql.Time) Timestamp(java.sql.Timestamp) DataType(org.apache.nifi.serialization.record.DataType) Record(org.apache.nifi.serialization.record.Record) MapRecord(org.apache.nifi.serialization.record.MapRecord) RecordSet(org.apache.nifi.serialization.record.RecordSet) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SimpleRecordSchema(org.apache.nifi.serialization.SimpleRecordSchema) MapRecord(org.apache.nifi.serialization.record.MapRecord) SchemaNameAsAttribute(org.apache.nifi.schema.access.SchemaNameAsAttribute) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Date(java.sql.Date) CSVFormat(org.apache.commons.csv.CSVFormat) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Test(org.junit.Test)

Aggregations

DataType (org.apache.nifi.serialization.record.DataType)45 RecordField (org.apache.nifi.serialization.record.RecordField)36 RecordSchema (org.apache.nifi.serialization.record.RecordSchema)27 ArrayDataType (org.apache.nifi.serialization.record.type.ArrayDataType)24 SimpleRecordSchema (org.apache.nifi.serialization.SimpleRecordSchema)22 RecordDataType (org.apache.nifi.serialization.record.type.RecordDataType)22 ChoiceDataType (org.apache.nifi.serialization.record.type.ChoiceDataType)21 MapDataType (org.apache.nifi.serialization.record.type.MapDataType)20 ArrayList (java.util.ArrayList)17 RecordFieldType (org.apache.nifi.serialization.record.RecordFieldType)17 HashMap (java.util.HashMap)15 Record (org.apache.nifi.serialization.record.Record)14 Map (java.util.Map)13 MapRecord (org.apache.nifi.serialization.record.MapRecord)13 Test (org.junit.Test)13 LinkedHashMap (java.util.LinkedHashMap)11 List (java.util.List)11 ComponentLog (org.apache.nifi.logging.ComponentLog)10 File (java.io.File)9 IOException (java.io.IOException)9