Examples with Schema org.embulk.spi.Schema used on opensource projects

Example 11 with Schema

use of org.embulk.spi.Schema in project embulk by embulk.

the class TestRenameFilterPlugin method checkConfigExceptionIfUnknownStringTypeOfRenamingOperator.

@Test
public void checkConfigExceptionIfUnknownStringTypeOfRenamingOperator() {
    // A simple string shouldn't come as a renaming rule.
    ConfigSource pluginConfig = Exec.newConfigSource().set("rules", ImmutableList.of("string_rule"));
    try {
        filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {

            public void run(TaskSource task, Schema schema) {
            }
        });
        fail();
    } catch (Throwable t) {
        assertTrue(t instanceof ConfigException);
    }
}

Example 12 with Schema

use of org.embulk.spi.Schema in project embulk by embulk.

the class TestRenameFilterPlugin method checkUniqueNumberSuffixRuleInternal.

private void checkUniqueNumberSuffixRuleInternal(final String[] originalColumnNames, final String[] expectedColumnNames, String delimiter, int digits, int max_length) {
    Schema.Builder originalSchemaBuilder = Schema.builder();
    for (String originalColumnName : originalColumnNames) {
        originalSchemaBuilder.add(originalColumnName, STRING);
    }
    final Schema originalSchema = originalSchemaBuilder.build();
    HashMap<String, Object> parameters = new HashMap<>();
    parameters.put("rule", "unique_number_suffix");
    if (!delimiter.equals(DEFAULT)) {
        parameters.put("delimiter", delimiter);
    }
    if (digits >= 0) {
        parameters.put("digits", digits);
    }
    if (max_length != -1) {
        parameters.put("max_length", max_length);
    }
    ConfigSource pluginConfig = Exec.newConfigSource().set("rules", ImmutableList.of(ImmutableMap.copyOf(parameters)));
    filter.transaction(pluginConfig, originalSchema, new FilterPlugin.Control() {

        @Override
        public void run(TaskSource task, Schema newSchema) {
            ArrayList<String> resolvedColumnNamesList = new ArrayList<>(newSchema.size());
            for (Column resolvedColumn : newSchema.getColumns()) {
                resolvedColumnNamesList.add(resolvedColumn.getName());
            }
            String[] resolvedColumnNames = Iterables.toArray(resolvedColumnNamesList, String.class);
            assertEquals(expectedColumnNames, resolvedColumnNames);
            for (int i = 0; i < expectedColumnNames.length; ++i) {
                Column original = originalSchema.getColumn(i);
                Column resolved = newSchema.getColumn(i);
                assertEquals(original.getType(), resolved.getType());
            }
        }
    });
}

Example 13 with Schema

use of org.embulk.spi.Schema in project embulk by embulk.

the class TestRenameFilterPlugin method renameAndCheckSchema.

private void renameAndCheckSchema(ConfigSource config, final String[] original, final String[] expected) {
    final Schema originalSchema = makeSchema(original);
    filter.transaction(config, originalSchema, new FilterPlugin.Control() {

        @Override
        public void run(TaskSource task, Schema renamedSchema) {
            assertEquals(originalSchema.getColumnCount(), renamedSchema.getColumnCount());
            assertEquals(expected.length, renamedSchema.getColumnCount());
            for (int i = 0; i < renamedSchema.getColumnCount(); ++i) {
                assertEquals(originalSchema.getColumnType(i), renamedSchema.getColumnType(i));
                assertEquals(expected[i], renamedSchema.getColumnName(i));
            }
        }
    });
}

Example 14 with Schema

use of org.embulk.spi.Schema in project embulk by embulk.

the class TestRenameFilterPlugin method checkRenaming.

@Test
public void checkRenaming() {
    ConfigSource pluginConfig = Exec.newConfigSource().set("columns", ImmutableMap.of("_c0", "_c0_new"));
    filter.transaction(pluginConfig, SCHEMA, new FilterPlugin.Control() {

        @Override
        public void run(TaskSource task, Schema newSchema) {
            // _c0 -> _c0_new
            Column old0 = SCHEMA.getColumn(0);
            Column new0 = newSchema.getColumn(0);
            assertEquals("_c0_new", new0.getName());
            assertEquals(old0.getType(), new0.getType());
            // _c1 is not changed
            Column old1 = SCHEMA.getColumn(1);
            Column new1 = newSchema.getColumn(1);
            assertEquals("_c1", new1.getName());
            assertEquals(old1.getType(), new1.getType());
        }
    });
}

Example 15 with Schema

use of org.embulk.spi.Schema in project embulk by embulk.

the class RemoveColumnsFilterPlugin method transaction.

@Override
public void transaction(ConfigSource config, Schema inputSchema, FilterPlugin.Control control) {
    PluginTask task = config.loadConfig(PluginTask.class);
    // validate remove: and keep:
    if (task.getRemove().isPresent() && task.getKeep().isPresent()) {
        throw new ConfigException("remove: and keep: must not be multi-select");
    }
    if (!task.getRemove().isPresent() && !task.getKeep().isPresent()) {
        throw new ConfigException("Must require remove: or keep:");
    }
    boolean acceptUnmatchedColumns = task.getAcceptUnmatchedColumns();
    ImmutableList.Builder<Column> outputColumns = ImmutableList.builder();
    int index = 0;
    int[] indexMapping = new int[inputSchema.size()];
    for (int i = 0; i < indexMapping.length; i++) {
        indexMapping[i] = -1;
    }
    if (task.getRemove().isPresent()) {
        // specify remove:
        List<String> removeColumns = getExistentColumns(inputSchema, task.getRemove().get(), acceptUnmatchedColumns);
        for (Column column : inputSchema.getColumns()) {
            if (!removeColumns.contains(column.getName())) {
                outputColumns.add(new Column(index, column.getName(), column.getType()));
                indexMapping[column.getIndex()] = index;
                index++;
            }
        }
    } else {
        // specify keep:
        List<String> keepColumns = getExistentColumns(inputSchema, task.getKeep().get(), acceptUnmatchedColumns);
        for (Column column : inputSchema.getColumns()) {
            if (keepColumns.contains(column.getName())) {
                outputColumns.add(new Column(index, column.getName(), column.getType()));
                indexMapping[column.getIndex()] = index;
                index++;
            }
        }
    }
    task.setIndexMapping(indexMapping);
    control.run(task.dump(), new Schema(outputColumns.build()));
}