Examples with RecordField org.apache.nifi.serialization.record.RecordField used on opensource projects

Example 16 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class JsonPathRowRecordReader method convert.

@SuppressWarnings("unchecked")
protected Object convert(final Object value, final DataType dataType) {
    if (value == null) {
        return null;
    }
    if (value instanceof List) {
        final List<?> list = (List<?>) value;
        final Object[] array = new Object[list.size()];
        final DataType elementDataType;
        if (dataType != null && dataType.getFieldType() == RecordFieldType.ARRAY) {
            elementDataType = ((ArrayDataType) dataType).getElementType();
        } else {
            elementDataType = null;
        }
        int i = 0;
        for (final Object val : list) {
            array[i++] = convert(val, elementDataType);
        }
        return array;
    }
    if (value instanceof Map) {
        final Map<String, ?> map = (Map<String, ?>) value;
        boolean record = false;
        for (final Object obj : map.values()) {
            if (obj instanceof JsonNode) {
                record = true;
            }
        }
        if (!record) {
            return value;
        }
        RecordSchema childSchema = null;
        if (dataType != null && dataType.getFieldType() == RecordFieldType.RECORD) {
            childSchema = ((RecordDataType) dataType).getChildSchema();
        }
        if (childSchema == null) {
            childSchema = new SimpleRecordSchema(Collections.emptyList());
        }
        final Map<String, Object> values = new HashMap<>();
        for (final Map.Entry<String, ?> entry : map.entrySet()) {
            final String key = entry.getKey();
            final Object childValue = entry.getValue();
            final RecordField recordField = childSchema.getField(key).orElse(null);
            final DataType childDataType = recordField == null ? null : recordField.getDataType();
            values.put(key, convert(childValue, childDataType));
        }
        return new MapRecord(childSchema, values);
    }
    return value;
}

Example 17 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class WriteJsonResult method writeRecord.

private void writeRecord(final Record record, final RecordSchema writeSchema, final JsonGenerator generator, final GeneratorTask startTask, final GeneratorTask endTask, final boolean schemaAware) throws JsonGenerationException, IOException {
    final Optional<SerializedForm> serializedForm = record.getSerializedForm();
    if (serializedForm.isPresent()) {
        final SerializedForm form = serializedForm.get();
        if (form.getMimeType().equals(getMimeType()) && record.getSchema().equals(writeSchema)) {
            final Object serialized = form.getSerialized();
            if (serialized instanceof String) {
                generator.writeRawValue((String) serialized);
                return;
            }
        }
    }
    try {
        startTask.apply(generator);
        if (schemaAware) {
            for (final RecordField field : writeSchema.getFields()) {
                final String fieldName = field.getFieldName();
                final Object value = record.getValue(field);
                if (value == null) {
                    if (nullSuppression == NullSuppression.NEVER_SUPPRESS || (nullSuppression == NullSuppression.SUPPRESS_MISSING) && isFieldPresent(field, record)) {
                        generator.writeNullField(fieldName);
                    }
                    continue;
                }
                generator.writeFieldName(fieldName);
                final DataType dataType = writeSchema.getDataType(fieldName).get();
                writeValue(generator, value, fieldName, dataType);
            }
        } else {
            for (final String fieldName : record.getRawFieldNames()) {
                final Object value = record.getValue(fieldName);
                if (value == null) {
                    if (nullSuppression == NullSuppression.NEVER_SUPPRESS || (nullSuppression == NullSuppression.SUPPRESS_MISSING) && record.getRawFieldNames().contains(fieldName)) {
                        generator.writeNullField(fieldName);
                    }
                    continue;
                }
                generator.writeFieldName(fieldName);
                writeRawValue(generator, value, fieldName);
            }
        }
        endTask.apply(generator);
    } catch (final Exception e) {
        logger.error("Failed to write {} with schema {} as a JSON Object due to {}", new Object[] { record, record.getSchema(), e.toString(), e });
        throw e;
    }
}

Example 18 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestCSVRecordReader method testExtraFieldNotInHeader.

@Test
public void testExtraFieldNotInHeader() throws IOException, MalformedRecordException {
    final List<RecordField> fields = getDefaultFields();
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final String headerLine = "id, name, balance, address, city, state, zipCode, country";
    final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111, USA, North America";
    final String csvData = headerLine + "\n" + inputRecord;
    final byte[] inputData = csvData.getBytes();
    // test nextRecord does not contain a 'continent' field
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final CSVRecordReader reader = createReader(bais, schema, format)) {
        final Record record = reader.nextRecord(false, false);
        assertNotNull(record);
        assertEquals("1", record.getValue("id"));
        assertEquals("John", record.getValue("name"));
        assertEquals("40.80", record.getValue("balance"));
        assertEquals("123 My Street", record.getValue("address"));
        assertEquals("My City", record.getValue("city"));
        assertEquals("MS", record.getValue("state"));
        assertEquals("11111", record.getValue("zipCode"));
        assertEquals("USA", record.getValue("country"));
        assertEquals("North America", record.getValue("unknown_field_index_8"));
        assertNull(reader.nextRecord(false, false));
    }
}

Example 19 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestCSVRecordReader method testDateNullFormat.

@Test
public void testDateNullFormat() throws IOException, MalformedRecordException {
    final String text = "date\n1983-01-01";
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("date", RecordFieldType.DATE.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
        final CSVRecordReader reader = new CSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false, null, RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {
        final Record record = reader.nextRecord(false, false);
        assertEquals("1983-01-01", (String) record.getValue("date"));
    }
}

Example 20 with RecordField

use of org.apache.nifi.serialization.record.RecordField in project nifi by apache.

the class TestCSVRecordReader method testSimpleParse.

@Test
public void testSimpleParse() throws IOException, MalformedRecordException {
    final List<RecordField> fields = getDefaultFields();
    fields.replaceAll(f -> f.getFieldName().equals("balance") ? new RecordField("balance", doubleDataType) : f);
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream fis = new FileInputStream(new File("src/test/resources/csv/single-bank-account.csv"));
        final CSVRecordReader reader = createReader(fis, schema, format)) {
        final Object[] record = reader.nextRecord().getValues();
        final Object[] expectedValues = new Object[] { "1", "John Doe", 4750.89D, "123 My Street", "My City", "MS", "11111", "USA" };
        Assert.assertArrayEquals(expectedValues, record);
        assertNull(reader.nextRecord());
    }
}