Examples with SimpleRecordSchema org.apache.nifi.serialization.SimpleRecordSchema used on opensource projects

Example 26 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestJacksonCSVRecordReader method testUTF8.

@Test
public void testUTF8() throws IOException, MalformedRecordException {
    final String text = "name\n黃凱揚";
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    try (final InputStream bais = new ByteArrayInputStream(text.getBytes());
        final JacksonCSVRecordReader reader = new JacksonCSVRecordReader(bais, Mockito.mock(ComponentLog.class), schema, format, true, false, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), "UTF-8")) {
        final Record record = reader.nextRecord();
        final String name = (String) record.getValue("name");
        assertEquals("黃凱揚", name);
    }
}

Example 27 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestJacksonCSVRecordReader method testReadRawWithDifferentFieldName.

@Test
public void testReadRawWithDifferentFieldName() throws IOException, MalformedRecordException {
    final List<RecordField> fields = getDefaultFields();
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final String headerLine = "id, name, balance, address, city, state, zipCode, continent";
    final String inputRecord = "1, John, 40.80, 123 My Street, My City, MS, 11111, North America";
    final String csvData = headerLine + "\n" + inputRecord;
    final byte[] inputData = csvData.getBytes();
    // test nextRecord does not contain a 'continent' field
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final JacksonCSVRecordReader reader = createReader(bais, schema, format)) {
        final Record record = reader.nextRecord();
        assertNotNull(record);
        assertEquals("1", record.getValue("id"));
        assertEquals("John", record.getValue("name"));
        assertEquals("40.80", record.getValue("balance"));
        assertEquals("123 My Street", record.getValue("address"));
        assertEquals("My City", record.getValue("city"));
        assertEquals("MS", record.getValue("state"));
        assertEquals("11111", record.getValue("zipCode"));
        assertNull(record.getValue("country"));
        assertNull(record.getValue("continent"));
        assertNull(reader.nextRecord());
    }
    // test nextRawRecord does contain 'continent' field
    try (final InputStream bais = new ByteArrayInputStream(inputData);
        final JacksonCSVRecordReader reader = createReader(bais, schema, format)) {
        final Record record = reader.nextRecord(false, false);
        assertNotNull(record);
        assertEquals("1", record.getValue("id"));
        assertEquals("John", record.getValue("name"));
        assertEquals("40.80", record.getValue("balance"));
        assertEquals("123 My Street", record.getValue("address"));
        assertEquals("My City", record.getValue("city"));
        assertEquals("MS", record.getValue("state"));
        assertEquals("11111", record.getValue("zipCode"));
        assertNull(record.getValue("country"));
        assertEquals("North America", record.getValue("continent"));
        assertNull(reader.nextRecord(false, false));
    }
}

Example 28 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestWriteCSVResult method testMissingAndExtraFieldWriteRecord.

@Test
public void testMissingAndExtraFieldWriteRecord() throws IOException {
    final CSVFormat csvFormat = CSVFormat.DEFAULT.withEscape('\\').withQuoteMode(QuoteMode.NONE).withRecordSeparator("\n");
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new LinkedHashMap<>();
    values.put("id", "1");
    values.put("dob", "1/1/1970");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final String output;
    try (final WriteCSVResult writer = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "ASCII")) {
        writer.beginRecordSet();
        writer.writeRecord(record);
        writer.finishRecordSet();
        writer.flush();
        output = baos.toString();
    }
    assertEquals("id,name\n1,\n", output);
}

Example 29 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestWriteCSVResult method testMissingFieldWriteRecord.

@Test
public void testMissingFieldWriteRecord() throws IOException {
    final CSVFormat csvFormat = CSVFormat.DEFAULT.withEscape('\\').withQuoteMode(QuoteMode.NONE).withRecordSeparator("\n");
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new LinkedHashMap<>();
    values.put("id", "1");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final String output;
    try (final WriteCSVResult writer = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "ASCII")) {
        writer.beginRecordSet();
        writer.writeRecord(record);
        writer.finishRecordSet();
        writer.flush();
        output = baos.toString();
    }
    assertEquals("id,name\n1,\n", output);
}

Example 30 with SimpleRecordSchema

use of org.apache.nifi.serialization.SimpleRecordSchema in project nifi by apache.

the class TestWriteCSVResult method testMissingAndExtraFieldWriteRawRecord.

@Test
public void testMissingAndExtraFieldWriteRawRecord() throws IOException {
    final CSVFormat csvFormat = CSVFormat.DEFAULT.withEscape('\\').withQuoteMode(QuoteMode.NONE).withRecordSeparator("\n");
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("id", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
    final RecordSchema schema = new SimpleRecordSchema(fields);
    final Map<String, Object> values = new LinkedHashMap<>();
    values.put("id", "1");
    values.put("dob", "1/1/1970");
    final Record record = new MapRecord(schema, values);
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final String output;
    try (final WriteCSVResult writer = new WriteCSVResult(csvFormat, schema, new SchemaNameAsAttribute(), baos, RecordFieldType.DATE.getDefaultFormat(), RecordFieldType.TIME.getDefaultFormat(), RecordFieldType.TIMESTAMP.getDefaultFormat(), true, "ASCII")) {
        writer.beginRecordSet();
        writer.writeRawRecord(record);
        writer.finishRecordSet();
        writer.flush();
        output = baos.toString();
    }
    assertEquals("id,dob,name\n1,1/1/1970,\n", output);
}