Examples with TaskSource org.embulk.config.TaskSource used on opensource projects

Example 1 with TaskSource

use of org.embulk.config.TaskSource in project embulk by embulk.

the class TestFileInputRunner method testTransactionAborted.

@Test
public void testTransactionAborted() {
    Buffer[] buffers = new Buffer[] { runtime.getBufferAllocator().allocate(), runtime.getBufferAllocator().allocate() };
    MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin(new LinkedList<Buffer>(Arrays.asList(buffers)));
    final FileInputRunner runner = new FileInputRunner(fileInputPlugin);
    ConfigSource config = Exec.newConfigSource().set("parser", ImmutableMap.of("type", "mock", "columns", ImmutableList.of(ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()), ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()), ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()), ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()), ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()), ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
    final MockPageOutput output = new MockPageOutput();
    MockParserPlugin.raiseException = true;
    try {
        runner.transaction(config, new InputPlugin.Control() {

            public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
                List<TaskReport> reports = new ArrayList<>();
                reports.add(runner.run(inputTaskSource, schema, 0, output));
                return reports;
            }
        });
    } catch (RuntimeException re) {
    // Just passing through.
    }
    assertEquals(false, fileInputPlugin.transactionCompleted);
    assertEquals(0, output.pages.size());
}

Example 2 with TaskSource

use of org.embulk.config.TaskSource in project embulk by embulk.

the class TestFileInputRunner method testMockParserIteration.

@Test
public void testMockParserIteration() {
    Buffer[] buffers = new Buffer[] { runtime.getBufferAllocator().allocate(), runtime.getBufferAllocator().allocate() };
    MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin(new LinkedList<Buffer>(Arrays.asList(buffers)));
    final FileInputRunner runner = new FileInputRunner(fileInputPlugin);
    ConfigSource config = Exec.newConfigSource().set("parser", ImmutableMap.of("type", "mock", "columns", ImmutableList.of(ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()), ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()), ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()), ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()), ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()), ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
    final MockPageOutput output = new MockPageOutput();
    runner.transaction(config, new InputPlugin.Control() {

        public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
            List<TaskReport> reports = new ArrayList<>();
            reports.add(runner.run(inputTaskSource, schema, 0, output));
            return reports;
        }
    });
    assertEquals(true, fileInputPlugin.transactionCompleted);
    assertEquals(1, output.pages.size());
    Schema schema = config.getNested("parser").loadConfig(MockParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
    List<Object[]> records = Pages.toObjects(schema, output.pages);
    assertEquals(2, records.size());
    for (Object[] record : records) {
        assertEquals(6, record.length);
        assertEquals(true, record[0]);
        assertEquals(2L, record[1]);
        assertEquals(3.0D, (Double) record[2], 0.01D);
        assertEquals("45", record[3]);
        assertEquals(678L, ((Timestamp) record[4]).toEpochMilli());
        assertEquals("{\"_c2\":10,\"_c1\":true,\"_c4\":{\"k\":\"v\"},\"_c3\":\"embulk\"}", record[5].toString());
    }
}

Example 3 with TaskSource

use of org.embulk.config.TaskSource in project embulk by embulk.

the class TestRenameFilterPlugin method checkConfigExceptionIfUnknownRenamingOperatorName.

@Test
public void checkConfigExceptionIfUnknownRenamingOperatorName() {
    ConfigSource pluginConfig = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.of("rule", "some_unknown_renaming_operator")));
    try {
        filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {

            public void run(TaskSource task, Schema schema) {
            }
        });
        fail();
    } catch (Throwable t) {
        assertTrue(t instanceof ConfigException);
    }
}

Example 4 with TaskSource

use of org.embulk.config.TaskSource in project embulk by embulk.

the class TestRenameFilterPlugin method throwSchemaConfigExceptionIfColumnNotFound.

@Test
public void throwSchemaConfigExceptionIfColumnNotFound() {
    ConfigSource pluginConfig = Exec.newConfigSource().set("columns", ImmutableMap.of("not_found", "any_name"));
    try {
        filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {

            public void run(TaskSource task, Schema schema) {
            }
        });
        fail();
    } catch (Throwable t) {
        assertTrue(t instanceof SchemaConfigException);
    }
}

Example 5 with TaskSource

use of org.embulk.config.TaskSource in project embulk by embulk.

the class TestRenameFilterPlugin method checkConfigExceptionIfUnknownListTypeOfRenamingOperator.

@Test
public void checkConfigExceptionIfUnknownListTypeOfRenamingOperator() {
    // A list [] shouldn't come as a renaming rule.
    ConfigSource pluginConfig = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableList.of("listed_operator1", "listed_operator2")));
    try {
        filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {

            public void run(TaskSource task, Schema schema) {
            }
        });
        fail();
    } catch (Throwable t) {
        assertTrue(t instanceof ConfigException);
    }
}