Examples with ConfigSource org.embulk.config.ConfigSource used on opensource projects

Example 36 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestCsvGuessPlugin method assertGuessByResource.

static void assertGuessByResource(TestingEmbulk embulk, String seedYamlResourceName, String sourceCsvResourceName, String resultResourceName) throws IOException {
    ConfigSource seed = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + seedYamlResourceName);
    ConfigDiff guessed = embulk.parserBuilder().parser(seed).exec(embulk.newConfig().set("exclude_guess_plugins", ImmutableList.of("json"))).inputResource(RESOURCE_NAME_PREFIX + sourceCsvResourceName).guess();
    assertThat(guessed, is((DataSource) embulk.loadYamlResource(RESOURCE_NAME_PREFIX + resultResourceName)));
}

Example 37 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestRemoveColumnsFilterPlugin method assertRecordsByResource.

static void assertRecordsByResource(TestingEmbulk embulk, String inConfigYamlResourceName, String filterConfigYamlResourceName, String sourceCsvResourceName, String resultCsvResourceName) throws IOException {
    Path inputPath = embulk.createTempFile("csv");
    Path outputPath = embulk.createTempFile("csv");
    // in: config
    copyResource(RESOURCE_NAME_PREFIX + sourceCsvResourceName, inputPath);
    ConfigSource inConfig = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + inConfigYamlResourceName).set("path_prefix", inputPath.toAbsolutePath().toString());
    // remove_columns filter config
    ConfigSource filterConfig = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + filterConfigYamlResourceName);
    TestingEmbulk.RunResult result = embulk.inputBuilder().in(inConfig).filters(ImmutableList.of(filterConfig)).outputPath(outputPath).run();
    assertThat(readSortedFile(outputPath), is(readResource(RESOURCE_NAME_PREFIX + resultCsvResourceName)));
}

Example 38 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestCsvAllStringsGuessPlugin method testSimple.

@Test
public void testSimple() throws Exception {
    ConfigSource exec = embulk.newConfig().set("guess_plugins", ImmutableList.of("csv_all_strings")).set("exclude_guess_plugins", ImmutableList.of("csv"));
    ConfigDiff guessed = embulk.parserBuilder().exec(exec).inputResource(RESOURCE_NAME_PREFIX + "test_simple.csv").guess();
    assertThat(guessed, is((DataSource) embulk.loadYamlResource(RESOURCE_NAME_PREFIX + "test_simple_guessed.yml")));
}

Example 39 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class RenameFilterPlugin method transaction.

@Override
public void transaction(ConfigSource config, Schema inputSchema, FilterPlugin.Control control) {
    PluginTask task = config.loadConfig(PluginTask.class);
    Map<String, String> renameMap = task.getRenameMap();
    List<ConfigSource> rulesList = task.getRulesList();
    // Check if the given column in "columns" exists or not.
    for (String columnName : renameMap.keySet()) {
        // throws SchemaConfigException
        inputSchema.lookupColumn(columnName);
    }
    // Rename by "columns": to be applied before "rules".
    Schema.Builder builder = Schema.builder();
    for (Column column : inputSchema.getColumns()) {
        String name = column.getName();
        if (renameMap.containsKey(name)) {
            name = renameMap.get(name);
        }
        builder.add(name, column.getType());
    }
    Schema intermediateSchema = builder.build();
    // Rename by "rules".
    Schema outputSchema = intermediateSchema;
    for (ConfigSource rule : rulesList) {
        outputSchema = applyRule(rule, intermediateSchema);
        intermediateSchema = outputSchema;
    }
    control.run(task.dump(), outputSchema);
}

Example 40 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestFileOutputRunner method testMockFormatterIteration.

@Test
public void testMockFormatterIteration() {
    MockFileOutputPlugin fileOutputPlugin = new MockFileOutputPlugin();
    final FileOutputRunner runner = new FileOutputRunner(fileOutputPlugin);
    ImmutableList<ImmutableMap<String, Object>> columns = ImmutableList.of(ImmutableMap.<String, Object>of("name", "col1", "type", "boolean", "option", ImmutableMap.of()), ImmutableMap.<String, Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()), ImmutableMap.<String, Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()), ImmutableMap.<String, Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()), ImmutableMap.<String, Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()), ImmutableMap.<String, Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
    ConfigSource config = Exec.newConfigSource().set("type", "unused?").set("formatter", ImmutableMap.of("type", "mock", "columns", columns));
    final Schema schema = config.getNested("formatter").loadConfig(MockParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
    runner.transaction(config, schema, 1, new OutputPlugin.Control() {

        public List<TaskReport> run(final TaskSource outputTask) {
            TransactionalPageOutput tran = runner.open(outputTask, schema, 1);
            boolean committed = false;
            try {
                ImmutableMapValue jsonValue = newMap(newString("_c1"), newBoolean(true), newString("_c2"), newInteger(10), newString("_c3"), newString("embulk"), newString("_c4"), newMap(newString("k"), newString("v")));
                for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue, true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue)) {
                    tran.add(page);
                }
                tran.commit();
                committed = true;
            } finally {
                if (!committed) {
                    tran.abort();
                }
                tran.close();
            }
            return new ArrayList<TaskReport>();
        }
    });
    assertEquals(true, fileOutputPlugin.transactionCompleted);
    assertEquals(2, MockFormatterPlugin.records.size());
    for (List<Object> record : MockFormatterPlugin.records) {
        assertEquals(Boolean.TRUE, record.get(0));
        assertEquals(2L, record.get(1));
        assertEquals(3.0D, (Double) record.get(2), 0.1D);
        assertEquals("45", record.get(3));
        assertEquals(678L, ((Timestamp) record.get(4)).toEpochMilli());
        assertEquals("{\"_c1\":true,\"_c2\":10,\"_c3\":\"embulk\",\"_c4\":{\"k\":\"v\"}}", record.get(5).toString());
    }
}