Examples with ConfigSource org.embulk.config.ConfigSource used on opensource projects

Example 26 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestJsonParserPlugin method useSkipInvalidEscapeString.

@Test
public void useSkipInvalidEscapeString() throws Exception {
    ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "SKIP");
    transaction(config, fileInput("{\"\\a\":\"b\"}\\"));
    List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages);
    assertEquals(1, records.size());
    Object[] record = records.get(0);
    Map<Value, Value> map = ((Value) record[0]).asMapValue().map();
    assertEquals(newString("b"), map.get(newString("")));
}

Example 27 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestRenameFilterPlugin method checkRegexReplaceRuleInternal.

private void checkRegexReplaceRuleInternal(final String[] original, final String[] expected, final String match, final String replace) {
    HashMap<String, Object> parameters = new HashMap<>();
    parameters.put("rule", "regex_replace");
    parameters.put("match", match);
    parameters.put("replace", replace);
    ConfigSource config = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.copyOf(parameters)));
    renameAndCheckSchema(config, original, expected);
}

Example 28 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestRenameFilterPlugin method checkConfigExceptionIfUnknownStringTypeOfRenamingOperator.

@Test
public void checkConfigExceptionIfUnknownStringTypeOfRenamingOperator() {
    // A simple string shouldn't come as a renaming rule.
    ConfigSource pluginConfig = Exec.newConfigSource().set("rules", ImmutableList.of("string_rule"));
    try {
        filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {

            public void run(TaskSource task, Schema schema) {
            }
        });
        fail();
    } catch (Throwable t) {
        assertTrue(t instanceof ConfigException);
    }
}

Example 29 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestRenameFilterPlugin method checkUniqueNumberSuffixRuleInternal.

private void checkUniqueNumberSuffixRuleInternal(final String[] originalColumnNames, final String[] expectedColumnNames, String delimiter, int digits, int max_length) {
    Schema.Builder originalSchemaBuilder = Schema.builder();
    for (String originalColumnName : originalColumnNames) {
        originalSchemaBuilder.add(originalColumnName, STRING);
    }
    final Schema originalSchema = originalSchemaBuilder.build();
    HashMap<String, Object> parameters = new HashMap<>();
    parameters.put("rule", "unique_number_suffix");
    if (!delimiter.equals(DEFAULT)) {
        parameters.put("delimiter", delimiter);
    }
    if (digits >= 0) {
        parameters.put("digits", digits);
    }
    if (max_length != -1) {
        parameters.put("max_length", max_length);
    }
    ConfigSource pluginConfig = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.copyOf(parameters)));
    filter.transaction(pluginConfig, originalSchema, new FilterPlugin.Control() {

        @Override
        public void run(TaskSource task, Schema newSchema) {
            ArrayList<String> resolvedColumnNamesList = new ArrayList<>(newSchema.size());
            for (Column resolvedColumn : newSchema.getColumns()) {
                resolvedColumnNamesList.add(resolvedColumn.getName());
            }
            String[] resolvedColumnNames = Iterables.toArray(resolvedColumnNamesList, String.class);
            assertEquals(expectedColumnNames, resolvedColumnNames);
            for (int i = 0; i < expectedColumnNames.length; ++i) {
                Column original = originalSchema.getColumn(i);
                Column resolved = newSchema.getColumn(i);
                assertEquals(original.getType(), resolved.getType());
            }
        }
    });
}

Example 30 with ConfigSource

use of org.embulk.config.ConfigSource in project embulk by embulk.

the class TestRenameFilterPlugin method checkRuleLowerToUpperRule.

@Test
public void checkRuleLowerToUpperRule() {
    final String[] original = { "_C0", "_C1", "_c2" };
    final String[] expected = { "_C0", "_C1", "_C2" };
    ConfigSource config = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.of("rule", "lower_to_upper")));
    renameAndCheckSchema(config, original, expected);
}