Examples with FileInputSplit org.apache.flink.core.fs.FileInputSplit used on opensource projects

Example 31 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class PrimitiveInputFormatTest method testIntegerInput.

@Test
public void testIntegerInput() throws IOException {
    try {
        final String fileContent = "111|222|";
        final FileInputSplit split = createInputSplit(fileContent);
        final PrimitiveInputFormat<Integer> format = new PrimitiveInputFormat<Integer>(PATH, "|", Integer.class);
        format.configure(new Configuration());
        format.open(split);
        Integer result = null;
        result = format.nextRecord(result);
        assertEquals(Integer.valueOf(111), result);
        result = format.nextRecord(result);
        assertEquals(Integer.valueOf(222), result);
        result = format.nextRecord(result);
        assertNull(result);
        assertTrue(format.reachedEnd());
    } catch (Exception ex) {
        fail("Test failed due to a " + ex.getClass().getName() + ": " + ex.getMessage());
    }
}

Example 32 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class PrimitiveInputFormatTest method testDoubleInputLinewise.

@Test
public void testDoubleInputLinewise() throws IOException {
    try {
        final String fileContent = "1.21\n2.23\n";
        final FileInputSplit split = createInputSplit(fileContent);
        final PrimitiveInputFormat<Double> format = new PrimitiveInputFormat<Double>(PATH, Double.class);
        format.configure(new Configuration());
        format.open(split);
        Double result = null;
        result = format.nextRecord(result);
        assertEquals(Double.valueOf(1.21), result);
        result = format.nextRecord(result);
        assertEquals(Double.valueOf(2.23), result);
        result = format.nextRecord(result);
        assertNull(result);
        assertTrue(format.reachedEnd());
    } catch (Exception ex) {
        fail("Test failed due to a " + ex.getClass().getName() + ": " + ex.getMessage());
    }
}

Example 33 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class PrimitiveInputFormatTest method testRemovingTrailingCR.

@Test
public void testRemovingTrailingCR() {
    try {
        String first = "First line";
        String second = "Second line";
        String fileContent = first + "\r\n" + second + "\r\n";
        final FileInputSplit split = createInputSplit(fileContent);
        final PrimitiveInputFormat<String> format = new PrimitiveInputFormat<String>(PATH, String.class);
        format.configure(new Configuration());
        format.open(split);
        String result = null;
        result = format.nextRecord(result);
        assertEquals(first, result);
        result = format.nextRecord(result);
        assertEquals(second, result);
    } catch (Exception ex) {
        fail("Test failed due to a " + ex.getClass().getName() + ": " + ex.getMessage());
    }
}

Example 34 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class PrimitiveInputFormatTest method testFailingInput.

@Test(expected = IOException.class)
public void testFailingInput() throws IOException {
    final String fileContent = "111|222|asdf|17";
    final FileInputSplit split = createInputSplit(fileContent);
    final PrimitiveInputFormat<Integer> format = new PrimitiveInputFormat<Integer>(PATH, "|", Integer.class);
    format.configure(new Configuration());
    format.open(split);
    Integer result = null;
    result = format.nextRecord(result);
    assertEquals(Integer.valueOf(111), result);
    result = format.nextRecord(result);
    assertEquals(Integer.valueOf(222), result);
    result = format.nextRecord(result);
}

Example 35 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class PrimitiveInputFormatTest method testStringInput.

@Test
public void testStringInput() {
    try {
        final String fileContent = "abc||def||||";
        final FileInputSplit split = createInputSplit(fileContent);
        final PrimitiveInputFormat<String> format = new PrimitiveInputFormat<String>(PATH, "||", String.class);
        final Configuration parameters = new Configuration();
        format.configure(parameters);
        format.open(split);
        String result = null;
        result = format.nextRecord(result);
        assertEquals("abc", result);
        result = format.nextRecord(result);
        assertEquals("def", result);
        result = format.nextRecord(result);
        assertEquals("", result);
        result = format.nextRecord(result);
        assertNull(result);
        assertTrue(format.reachedEnd());
    } catch (Exception ex) {
        ex.printStackTrace();
        fail("Test failed due to a " + ex.getClass().getName() + ": " + ex.getMessage());
    }
}