Examples with SimpleRecordSchema org.apache.nifi.serialization.SimpleRecordSchema used on opensource projects

Example 21 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestJacksonCSVRecordReader method testFieldInSchemaButNotHeader.

@Test
public void testFieldInSchemaButNotHeader() throws IOException, MalformedRecordException {
    final List<RecordField> fields = getDefaultFields();
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final String headerLine = "id, name, balance, address, city, state, zipCode";
    final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111, USA";
    final String csvData = headerLine + "\n" + inputRecord;
    final byte[] inputData = csvData.getBytes();
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final JacksonCSVRecordReader reader = createReader(bais, schema, format)) {
        final Record record = reader.nextRecord();
        assertNotNull(record);
        assertEquals("1", record.getValue("id"));
        assertEquals("John", record.getValue("name"));
        assertEquals("40.80", record.getValue("balance"));
        assertEquals("123 My Street", record.getValue("address"));
        assertEquals("My City", record.getValue("city"));
        assertEquals("MS", record.getValue("state"));
        assertEquals("11111", record.getValue("zipCode"));
        // If schema says that there are fields a, b, c
        // and the CSV has a header line that says field names are a, b
        // and then the data has values 1,2,3
        // then a=1, b=2, c=null
        assertNull(record.getValue("country"));
        assertNull(reader.nextRecord());
    }
    // our schema to be the definitive list of what fields exist.
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final JacksonCSVRecordReader reader = new JacksonCSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, true, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {
        final Record record = reader.nextRecord();
        assertNotNull(record);
        assertEquals("1", record.getValue("id"));
        assertEquals("John", record.getValue("name"));
        assertEquals("40.80", record.getValue("balance"));
        assertEquals("123 My Street", record.getValue("address"));
        assertEquals("My City", record.getValue("city"));
        assertEquals("MS", record.getValue("state"));
        assertEquals("11111", record.getValue("zipCode"));
        // If schema says that there are fields a, b, c
        // and the CSV has a header line that says field names are a, b
        // and then the data has values 1,2,3
        // then a=1, b=2, c=null
        // But if we configure the reader to Ignore the header, then this will not occur!
        assertEquals("USA", record.getValue("country"));
        assertNull(reader.nextRecord());
    }
}

Example 22 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestJacksonCSVRecordReader method testExtraFieldNotInHeader.

@Test
public void testExtraFieldNotInHeader() throws IOException, MalformedRecordException {
    final List<RecordField> fields = getDefaultFields();
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final String headerLine = "id, name, balance, address, city, state, zipCode, country";
    final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111, USA, North America";
    final String csvData = headerLine + "\n" + inputRecord;
    final byte[] inputData = csvData.getBytes();
    // test nextRecord does not contain a 'continent' field
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final JacksonCSVRecordReader reader = createReader(bais, schema, format)) {
        final Record record = reader.nextRecord(false, false);
        assertNotNull(record);
        assertEquals("1", record.getValue("id"));
        assertEquals("John", record.getValue("name"));
        assertEquals("40.80", record.getValue("balance"));
        assertEquals("123 My Street", record.getValue("address"));
        assertEquals("My City", record.getValue("city"));
        assertEquals("MS", record.getValue("state"));
        assertEquals("11111", record.getValue("zipCode"));
        assertEquals("USA", record.getValue("country"));
        assertEquals("North America", record.getValue("unknown_field_index_8"));
        assertNull(reader.nextRecord(false, false));
    }
}

Example 23 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestJacksonCSVRecordReader method testMissingField.

@Test
public void testMissingField() throws IOException, MalformedRecordException {
    final List<RecordField> fields = getDefaultFields();
    fields.replaceAll(f -> f.getFieldName().equals("balance") ? new RecordField("balance", doubleDataType) : f);
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final String headerLine = "id, name, balance, address, city, state, zipCode, country";
    final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111";
    final String csvData = headerLine + "\n" + inputRecord;
    final byte[] inputData = csvData.getBytes();
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final JacksonCSVRecordReader reader = createReader(bais, schema, format)) {
        final Record record = reader.nextRecord();
        assertNotNull(record);
        assertEquals("1", record.getValue("id"));
        assertEquals("John", record.getValue("name"));
        assertEquals(40.8D, record.getValue("balance"));
        assertEquals("123 My Street", record.getValue("address"));
        assertEquals("My City", record.getValue("city"));
        assertEquals("MS", record.getValue("state"));
        assertEquals("11111", record.getValue("zipCode"));
        assertNull(record.getValue("country"));
        assertNull(reader.nextRecord());
    }
}

Example 24 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestJacksonCSVRecordReader method testExcelFormat.

@Test
public void testExcelFormat() throws IOException, MalformedRecordException {
    final List<RecordField> fields = new ArrayList<RecordField>();
    fields.add(new RecordField("fieldA", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("fieldB", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final String headerLine = "fieldA,fieldB";
    final String inputRecord = "valueA,valueB";
    final String csvData = headerLine + "\n" + inputRecord;
    final byte[] inputData = csvData.getBytes();
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final JacksonCSVRecordReader reader = createReader(bais, schema, CSVFormat.EXCEL)) {
        final Object[] record = reader.nextRecord().getValues();
        final Object[] expectedValues = new Object[] { "valueA", "valueB" };
        Assert.assertArrayEquals(expectedValues, record);
        assertNull(reader.nextRecord());
    }
}

Example 25 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestJacksonCSVRecordReader method testMultipleRecordsEscapedWithSpecialChar.

@Test
public void testMultipleRecordsEscapedWithSpecialChar() throws IOException, MalformedRecordException {
    char delimiter = StringEscapeUtils.unescapeJava("\u0001").charAt(0);
    final CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withTrim().withQuote('"').withDelimiter(delimiter);
    final List<RecordField> fields = getDefaultFields();
    fields.replaceAll(f -> f.getFieldName().equals("balance") ? new RecordField("balance", doubleDataType) : f);
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream fis = new FileInputStream(new File("src/test/resources/csv/multi-bank-account_escapedchar.csv"));
        final JacksonCSVRecordReader reader = createReader(fis, schema, format)) {
        final Object[] firstRecord = reader.nextRecord().getValues();
        final Object[] firstExpectedValues = new Object[] { "1", "John Doe", 4750.89D, "123 My Street", "My City", "MS", "11111", "USA" };
        Assert.assertArrayEquals(firstExpectedValues, firstRecord);
        final Object[] secondRecord = reader.nextRecord().getValues();
        final Object[] secondExpectedValues = new Object[] { "2", "Jane Doe", 4820.09D, "321 Your Street", "Your City", "NY", "33333", "USA" };
        Assert.assertArrayEquals(secondExpectedValues, secondRecord);
        assertNull(reader.nextRecord());
    }
}