Examples with FileInputSplit org.apache.flink.core.fs.FileInputSplit used on opensource projects

Example 11 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class PrimitiveInputFormatTest method testFailingInput.

@Test(expected = IOException.class)
public void testFailingInput() throws IOException {
    final String fileContent = "111|222|asdf|17";
    final FileInputSplit split = createInputSplit(fileContent);
    final PrimitiveInputFormat<Integer> format = new PrimitiveInputFormat<Integer>(PATH, "|", Integer.class);
    format.configure(new Configuration());
    format.open(split);
    Integer result = null;
    result = format.nextRecord(result);
    assertEquals(Integer.valueOf(111), result);
    result = format.nextRecord(result);
    assertEquals(Integer.valueOf(222), result);
    result = format.nextRecord(result);
}

Example 12 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class PrimitiveInputFormatTest method testStringInput.

@Test
public void testStringInput() {
    try {
        final String fileContent = "abc||def||||";
        final FileInputSplit split = createInputSplit(fileContent);
        final PrimitiveInputFormat<String> format = new PrimitiveInputFormat<String>(PATH, "||", String.class);
        final Configuration parameters = new Configuration();
        format.configure(parameters);
        format.open(split);
        String result = null;
        result = format.nextRecord(result);
        assertEquals("abc", result);
        result = format.nextRecord(result);
        assertEquals("def", result);
        result = format.nextRecord(result);
        assertEquals("", result);
        result = format.nextRecord(result);
        assertNull(result);
        assertTrue(format.reachedEnd());
    } catch (Exception ex) {
        ex.printStackTrace();
        fail("Test failed due to a " + ex.getClass().getName() + ": " + ex.getMessage());
    }
}

Example 13 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class RowCsvInputFormatTest method createTempFile.

private static FileInputSplit createTempFile(String content) throws IOException {
    File tempFile = File.createTempFile("test_contents", "tmp");
    tempFile.deleteOnExit();
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile), StandardCharsets.UTF_8);
    wrt.write(content);
    wrt.close();
    return new FileInputSplit(0, new Path(tempFile.toURI().toString()), 0, tempFile.length(), new String[] { "localhost" });
}

Example 14 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class RowCsvInputFormatTest method testReadSparseWithNullFieldsForTypes.

@Test
public void testReadSparseWithNullFieldsForTypes() throws Exception {
    String fileContent = "111|x|222|x|333|x|444|x|555|x|666|x|777|x|888|x|999|x|000|x|\n" + "000|x|999|x|888|x|777|x|666|x|555|x|444|x|333|x|222|x|111|x|";
    FileInputSplit split = createTempFile(fileContent);
    TypeInformation[] fieldTypes = new TypeInformation[] { BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO };
    RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, new int[] { 0, 3, 7 });
    format.setFieldDelimiter("|x|");
    format.configure(new Configuration());
    format.open(split);
    Row result = new Row(3);
    result = format.nextRecord(result);
    assertNotNull(result);
    assertEquals(111, result.getField(0));
    assertEquals(444, result.getField(1));
    assertEquals(888, result.getField(2));
    result = format.nextRecord(result);
    assertNotNull(result);
    assertEquals(0, result.getField(0));
    assertEquals(777, result.getField(1));
    assertEquals(333, result.getField(2));
    result = format.nextRecord(result);
    assertNull(result);
    assertTrue(format.reachedEnd());
}

Example 15 with FileInputSplit

use of org.apache.flink.core.fs.FileInputSplit in project flink by apache.

the class RowCsvInputFormatTest method readStringFields.

@Test
public void readStringFields() throws Exception {
    String fileContent = "abc|def|ghijk\nabc||hhg\n|||\n||";
    FileInputSplit split = createTempFile(fileContent);
    TypeInformation[] fieldTypes = new TypeInformation[] { BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO };
    RowCsvInputFormat format = new RowCsvInputFormat(PATH, fieldTypes, "\n", "|");
    format.configure(new Configuration());
    format.open(split);
    Row result = new Row(3);
    result = format.nextRecord(result);
    assertNotNull(result);
    assertEquals("abc", result.getField(0));
    assertEquals("def", result.getField(1));
    assertEquals("ghijk", result.getField(2));
    result = format.nextRecord(result);
    assertNotNull(result);
    assertEquals("abc", result.getField(0));
    assertEquals("", result.getField(1));
    assertEquals("hhg", result.getField(2));
    result = format.nextRecord(result);
    assertNotNull(result);
    assertEquals("", result.getField(0));
    assertEquals("", result.getField(1));
    assertEquals("", result.getField(2));
    result = format.nextRecord(result);
    assertNotNull(result);
    assertEquals("", result.getField(0));
    assertEquals("", result.getField(1));
    assertEquals("", result.getField(2));
    result = format.nextRecord(result);
    assertNull(result);
    assertTrue(format.reachedEnd());
}