Examples with ConfigSource org.embulk.config.ConfigSource used on opensource projects

Example 1 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestLineDecoder method testLoadConfig.

@Test
public void testLoadConfig() {
    ConfigSource config = Exec.newConfigSource().set("charset", "utf-16").set("newline", "CRLF");
    LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
    assertEquals(StandardCharsets.UTF_16, task.getCharset());
    assertEquals(Newline.CRLF, task.getNewline());
}

Example 2 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestFileInputRunner method testTransactionAborted.

@Test
public void testTransactionAborted() {
    Buffer[] buffers = new Buffer[] { runtime.getBufferAllocator().allocate(), runtime.getBufferAllocator().allocate() };
    MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin(new LinkedList<Buffer>(Arrays.asList(buffers)));
    final FileInputRunner runner = new FileInputRunner(fileInputPlugin);
    ConfigSource config = Exec.newConfigSource().set("parser", ImmutableMap.of("type", "mock", "columns", ImmutableList.of(ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()), ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()), ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()), ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()), ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()), ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
    final MockPageOutput output = new MockPageOutput();
    MockParserPlugin.raiseException = true;
    try {
        runner.transaction(config, new InputPlugin.Control() {

            public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
                List<TaskReport> reports = new ArrayList<>();
                reports.add(runner.run(inputTaskSource, schema, 0, output));
                return reports;
            }
        });
    } catch (RuntimeException re) {
    // Just passing through.
    }
    assertEquals(false, fileInputPlugin.transactionCompleted);
    assertEquals(0, output.pages.size());
}

Example 3 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestFileInputRunner method testMockParserIteration.

@Test
public void testMockParserIteration() {
    Buffer[] buffers = new Buffer[] { runtime.getBufferAllocator().allocate(), runtime.getBufferAllocator().allocate() };
    MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin(new LinkedList<Buffer>(Arrays.asList(buffers)));
    final FileInputRunner runner = new FileInputRunner(fileInputPlugin);
    ConfigSource config = Exec.newConfigSource().set("parser", ImmutableMap.of("type", "mock", "columns", ImmutableList.of(ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()), ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()), ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()), ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()), ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()), ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
    final MockPageOutput output = new MockPageOutput();
    runner.transaction(config, new InputPlugin.Control() {

        public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
            List<TaskReport> reports = new ArrayList<>();
            reports.add(runner.run(inputTaskSource, schema, 0, output));
            return reports;
        }
    });
    assertEquals(true, fileInputPlugin.transactionCompleted);
    assertEquals(1, output.pages.size());
    Schema schema = config.getNested("parser").loadConfig(MockParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
    List<Object[]> records = Pages.toObjects(schema, output.pages);
    assertEquals(2, records.size());
    for (Object[] record : records) {
        assertEquals(6, record.length);
        assertEquals(true, record[0]);
        assertEquals(2L, record[1]);
        assertEquals(3.0D, (Double) record[2], 0.01D);
        assertEquals("45", record[3]);
        assertEquals(678L, ((Timestamp) record[4]).toEpochMilli());
        assertEquals("{\"_c2\":10,\"_c1\":true,\"_c4\":{\"k\":\"v\"},\"_c3\":\"embulk\"}", record[5].toString());
    }
}

Example 4 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestCsvFormatterPlugin method checkLoadConfig.

@Test
public void checkLoadConfig() {
    ConfigSource config = Exec.newConfigSource().set("charset", "utf-16").set("newline", "LF").set("header_line", false).set("delimiter", "\t").set("quote", "\\").set("quote_policy", "ALL").set("escape", "\"").set("null_string", "\\N").set("newline_in_field", "CRLF");
    CsvFormatterPlugin.PluginTask task = config.loadConfig(CsvFormatterPlugin.PluginTask.class);
    assertEquals(Charset.forName("utf-16"), task.getCharset());
    assertEquals(Newline.LF, task.getNewline());
    assertEquals(false, task.getHeaderLine());
    assertEquals('\t', task.getDelimiterChar());
    assertEquals('\\', task.getQuoteChar());
    assertEquals(CsvFormatterPlugin.QuotePolicy.ALL, task.getQuotePolicy());
    assertEquals('\"', (char) task.getEscapeChar().get());
    assertEquals("\\N", task.getNullString());
    assertEquals(Newline.CRLF, task.getNewlineInField());
}

Example 5 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestCsvFormatterPlugin method checkDefaultValues.

@Test
public void checkDefaultValues() {
    ConfigSource config = Exec.newConfigSource();
    CsvFormatterPlugin.PluginTask task = config.loadConfig(CsvFormatterPlugin.PluginTask.class);
    assertEquals(Charset.forName("utf-8"), task.getCharset());
    assertEquals(Newline.CRLF, task.getNewline());
    assertEquals(true, task.getHeaderLine());
    assertEquals(',', task.getDelimiterChar());
    assertEquals('\"', task.getQuoteChar());
    assertEquals(CsvFormatterPlugin.QuotePolicy.MINIMAL, task.getQuotePolicy());
    assertEquals(false, task.getEscapeChar().isPresent());
    assertEquals("", task.getNullString());
    assertEquals(org.joda.time.DateTimeZone.UTC, task.getDefaultTimeZone());
    assertEquals("%Y-%m-%d %H:%M:%S.%6N %z", task.getDefaultTimestampFormat());
    assertEquals(Newline.LF, task.getNewlineInField());
}