Examples with InputPlugin org.embulk.spi.InputPlugin used on opensource projects

Example 1 with InputPlugin

use of org.embulk.spi.InputPlugin in project embulk by embulk.

the class GuessExecutor method doGuess.

private ConfigDiff doGuess(ConfigSource config) {
    ConfigSource inputConfig = config.getNested("in");
    ConfigSource execConfig = config.getNestedOrGetEmpty("exec");
    InputPlugin input = newInputPlugin(inputConfig);
    ConfigDiff inputGuessed;
    if (input instanceof ConfigurableGuessInputPlugin) {
        inputGuessed = ((ConfigurableGuessInputPlugin) input).guess(execConfig, inputConfig);
    } else {
        try {
            inputGuessed = input.guess(inputConfig);
        } catch (AbstractMethodError ex) {
            // for backward compatibility with embulk v0.4 interface
            throw new UnsupportedOperationException(input.getClass().getSimpleName() + ".guess(ConfigSource) is not implemented. This input plugin does not support guessing.");
        }
    }
    ConfigDiff wrapped = Exec.newConfigDiff();
    wrapped.getNestedOrSetEmpty("in").merge(inputGuessed);
    return wrapped;
}

Example 2 with InputPlugin

use of org.embulk.spi.InputPlugin in project embulk by embulk.

the class Executors method process.

public static void process(ExecSession exec, ProcessTask task, int taskIndex, ProcessStateCallback callback) {
    InputPlugin inputPlugin = exec.newPlugin(InputPlugin.class, task.getInputPluginType());
    List<FilterPlugin> filterPlugins = Filters.newFilterPlugins(exec, task.getFilterPluginTypes());
    OutputPlugin outputPlugin = exec.newPlugin(OutputPlugin.class, task.getOutputPluginType());
    // TODO assert task.getExecutorSchema().equals task.getOutputSchema()
    process(exec, taskIndex, inputPlugin, task.getInputSchema(), task.getInputTaskSource(), filterPlugins, task.getFilterSchemas(), task.getFilterTaskSources(), outputPlugin, task.getOutputSchema(), task.getOutputTaskSource(), callback);
}

Example 3 with InputPlugin

use of org.embulk.spi.InputPlugin in project embulk by embulk.

the class PreviewExecutor method doPreview.

@SuppressWarnings("checkstyle:OverloadMethodsDeclarationOrder")
private PreviewResult doPreview(final PreviewTask task, final InputPlugin input, final List<FilterPlugin> filterPlugins) {
    try {
        input.transaction(task.getInputConfig(), new InputPlugin.Control() {

            public List<TaskReport> run(final TaskSource inputTask, Schema inputSchema, final int taskCount) {
                Filters.transaction(filterPlugins, task.getFilterConfigs(), inputSchema, new Filters.Control() {

                    public void run(final List<TaskSource> filterTasks, final List<Schema> filterSchemas) {
                        Schema inputSchema = filterSchemas.get(0);
                        Schema outputSchema = filterSchemas.get(filterSchemas.size() - 1);
                        PageOutput out = new SamplingPageOutput(task.getSampleRows(), outputSchema);
                        try {
                            for (int taskIndex = 0; taskIndex < taskCount; taskIndex++) {
                                try {
                                    out = Filters.open(filterPlugins, filterTasks, filterSchemas, out);
                                    input.run(inputTask, inputSchema, taskIndex, out);
                                } catch (NoSampleException ex) {
                                    if (taskIndex == taskCount - 1) {
                                        throw ex;
                                    }
                                }
                            }
                        } finally {
                            out.close();
                        }
                    }
                });
                // program never reaches here because SamplingPageOutput.finish throws an error.
                throw new NoSampleException("No input records to preview");
            }
        });
        throw new AssertionError("PreviewExecutor executor must throw PreviewedNoticeError");
    } catch (PreviewedNoticeError previewed) {
        return previewed.getPreviewResult();
    }
}

Example 4 with InputPlugin

use of org.embulk.spi.InputPlugin in project embulk by embulk.

the class PreviewExecutor method doPreview.

private PreviewResult doPreview(ConfigSource config) {
    PreviewTask task = config.loadConfig(PreviewTask.class);
    InputPlugin inputPlugin = newInputPlugin(task);
    List<FilterPlugin> filterPlugins = newFilterPlugins(task);
    if (inputPlugin instanceof FileInputRunner) {
        // file input runner
        Buffer sample = SamplingParserPlugin.runFileInputSampling((FileInputRunner) inputPlugin, config.getNested("in"), createSampleBufferConfigFromExecConfig(task.getExecConfig()));
        FileInputRunner previewRunner = new FileInputRunner(new BufferFileInputPlugin(sample));
        return doPreview(task, previewRunner, filterPlugins);
    } else {
        return doPreview(task, inputPlugin, filterPlugins);
    }
}