Examples with PojoTypeInfo org.apache.flink.api.java.typeutils.PojoTypeInfo used on opensource projects

Example 6 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class CsvInputFormatTest method testPojoSubclassType.

/**
	 * Tests that the CSV input format can deal with POJOs which are subclasses.
	 *
	 * @throws Exception
	 */
@Test
public void testPojoSubclassType() throws Exception {
    final String fileContent = "t1,foobar,tweet2\nt2,barfoo,tweet2";
    final File tempFile = File.createTempFile("CsvReaderPOJOSubclass", "tmp");
    tempFile.deleteOnExit();
    OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(tempFile));
    writer.write(fileContent);
    writer.close();
    @SuppressWarnings("unchecked") PojoTypeInfo<TwitterPOJO> typeInfo = (PojoTypeInfo<TwitterPOJO>) TypeExtractor.createTypeInfo(TwitterPOJO.class);
    CsvInputFormat<TwitterPOJO> inputFormat = new PojoCsvInputFormat<>(new Path(tempFile.toURI().toString()), typeInfo);
    inputFormat.configure(new Configuration());
    FileInputSplit[] splits = inputFormat.createInputSplits(1);
    inputFormat.open(splits[0]);
    List<TwitterPOJO> expected = new ArrayList<>();
    for (String line : fileContent.split("\n")) {
        String[] elements = line.split(",");
        expected.add(new TwitterPOJO(elements[0], elements[1], elements[2]));
    }
    List<TwitterPOJO> actual = new ArrayList<>();
    TwitterPOJO pojo;
    while ((pojo = inputFormat.nextRecord(new TwitterPOJO())) != null) {
        actual.add(pojo);
    }
    assertEquals(expected, actual);
}

Example 7 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class CsvInputFormatTest method testPojoType.

@Test
public void testPojoType() throws Exception {
    File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
    tempFile.deleteOnExit();
    tempFile.setWritable(true);
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile));
    wrt.write("123,AAA,3.123,BBB\n");
    wrt.write("456,BBB,1.123,AAA\n");
    wrt.close();
    @SuppressWarnings("unchecked") PojoTypeInfo<PojoItem> typeInfo = (PojoTypeInfo<PojoItem>) TypeExtractor.createTypeInfo(PojoItem.class);
    CsvInputFormat<PojoItem> inputFormat = new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo);
    inputFormat.configure(new Configuration());
    FileInputSplit[] splits = inputFormat.createInputSplits(1);
    inputFormat.open(splits[0]);
    validatePojoItem(inputFormat);
}

Example 8 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class CsvInputFormatTest method testPojoTypeWithTrailingEmptyFields.

@Test
public void testPojoTypeWithTrailingEmptyFields() throws Exception {
    final String fileContent = "123,,3.123,,\n456,BBB,3.23,,";
    final FileInputSplit split = createTempFile(fileContent);
    @SuppressWarnings("unchecked") PojoTypeInfo<PrivatePojoItem> typeInfo = (PojoTypeInfo<PrivatePojoItem>) TypeExtractor.createTypeInfo(PrivatePojoItem.class);
    CsvInputFormat<PrivatePojoItem> inputFormat = new PojoCsvInputFormat<PrivatePojoItem>(PATH, typeInfo);
    inputFormat.configure(new Configuration());
    inputFormat.open(split);
    PrivatePojoItem item = new PrivatePojoItem();
    inputFormat.nextRecord(item);
    assertEquals(123, item.field1);
    assertEquals("", item.field2);
    assertEquals(Double.valueOf(3.123), item.field3);
    assertEquals("", item.field4);
    inputFormat.nextRecord(item);
    assertEquals(456, item.field1);
    assertEquals("BBB", item.field2);
    assertEquals(Double.valueOf(3.23), item.field3);
    assertEquals("", item.field4);
}

Example 9 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class CsvInputFormatTest method testPojoTypeWithMappingInformation.

@Test
public void testPojoTypeWithMappingInformation() throws Exception {
    File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
    tempFile.deleteOnExit();
    tempFile.setWritable(true);
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile));
    wrt.write("123,3.123,AAA,BBB\n");
    wrt.write("456,1.123,BBB,AAA\n");
    wrt.close();
    @SuppressWarnings("unchecked") PojoTypeInfo<PojoItem> typeInfo = (PojoTypeInfo<PojoItem>) TypeExtractor.createTypeInfo(PojoItem.class);
    CsvInputFormat<PojoItem> inputFormat = new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new String[] { "field1", "field3", "field2", "field4" });
    inputFormat.configure(new Configuration());
    FileInputSplit[] splits = inputFormat.createInputSplits(1);
    inputFormat.open(splits[0]);
    validatePojoItem(inputFormat);
}

Example 10 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class CsvInputFormatTest method testPojoTypeWithPrivateField.

@Test
public void testPojoTypeWithPrivateField() throws Exception {
    File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
    tempFile.deleteOnExit();
    tempFile.setWritable(true);
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile));
    wrt.write("123,AAA,3.123,BBB\n");
    wrt.write("456,BBB,1.123,AAA\n");
    wrt.close();
    @SuppressWarnings("unchecked") PojoTypeInfo<PrivatePojoItem> typeInfo = (PojoTypeInfo<PrivatePojoItem>) TypeExtractor.createTypeInfo(PrivatePojoItem.class);
    CsvInputFormat<PrivatePojoItem> inputFormat = new PojoCsvInputFormat<PrivatePojoItem>(new Path(tempFile.toURI().toString()), typeInfo);
    inputFormat.configure(new Configuration());
    FileInputSplit[] splits = inputFormat.createInputSplits(1);
    inputFormat.open(splits[0]);
    PrivatePojoItem item = new PrivatePojoItem();
    inputFormat.nextRecord(item);
    assertEquals(123, item.field1);
    assertEquals("AAA", item.field2);
    assertEquals(Double.valueOf(3.123), item.field3);
    assertEquals("BBB", item.field4);
    inputFormat.nextRecord(item);
    assertEquals(456, item.field1);
    assertEquals("BBB", item.field2);
    assertEquals(Double.valueOf(1.123), item.field3);
    assertEquals("AAA", item.field4);
}