Examples with FileInput org.embulk.spi.FileInput used on opensource projects

Example 1 with FileInput

use of org.embulk.spi.FileInput in project embulk by embulk.

the class TestCsvTokenizer method recoverFromQuotedSizeLimitExceededException.

@Test
public void recoverFromQuotedSizeLimitExceededException() throws Exception {
    config.set("max_quoted_size_limit", 12);
    reloadPluginTask();
    String[] lines = new String[] { "v1,v2", // this is a broken line and should be skipped
    "v3,\"0123", // this line should be not be skiped
    "v4,v5", // this line should be not be skiped
    "v6,v7" };
    FileInput input = newFileInputFromLines(task, lines);
    LineDecoder decoder = new LineDecoder(input, task);
    CsvTokenizer tokenizer = new CsvTokenizer(decoder, task);
    Schema schema = task.getSchemaConfig().toSchema();
    tokenizer.nextFile();
    assertTrue(tokenizer.nextRecord());
    assertEquals("v1", tokenizer.nextColumn());
    assertEquals("v2", tokenizer.nextColumn());
    assertTrue(tokenizer.nextRecord());
    assertEquals("v3", tokenizer.nextColumn());
    try {
        tokenizer.nextColumn();
        fail();
    } catch (Exception e) {
        assertTrue(e instanceof CsvTokenizer.QuotedSizeLimitExceededException);
    }
    assertEquals("v3,\"0123", tokenizer.skipCurrentLine());
    assertTrue(tokenizer.nextRecord());
    assertEquals("v4", tokenizer.nextColumn());
    assertEquals("v5", tokenizer.nextColumn());
    assertTrue(tokenizer.nextRecord());
    assertEquals("v6", tokenizer.nextColumn());
    assertEquals("v7", tokenizer.nextColumn());
}

Example 2 with FileInput

use of org.embulk.spi.FileInput in project embulk by embulk.

the class Decoders method open.

public static FileInput open(List<DecoderPlugin> plugins, List<TaskSource> taskSources, FileInput input) {
    FileInput in = input;
    int pos = 0;
    while (pos < plugins.size()) {
        in = plugins.get(pos).open(taskSources.get(pos), in);
        pos++;
    }
    return in;
}