Examples with FileInputRunner org.embulk.spi.FileInputRunner used on opensource projects

Example 1 with FileInputRunner

use of org.embulk.spi.FileInputRunner in project embulk by embulk.

the class BulkLoader method doCleanup.

public void doCleanup(ConfigSource config, ResumeState resume) {
    BulkLoaderTask task = config.loadConfig(BulkLoaderTask.class);
    // TODO don't create filter plugins
    ProcessPluginSet plugins = new ProcessPluginSet(task);
    ImmutableList.Builder<TaskReport> successfulInputTaskReports = ImmutableList.builder();
    ImmutableList.Builder<TaskReport> successfulOutputTaskReports = ImmutableList.builder();
    for (Optional<TaskReport> inputTaskReport : resume.getInputTaskReports()) {
        if (inputTaskReport.isPresent()) {
            successfulInputTaskReports.add(inputTaskReport.get());
        }
    }
    for (Optional<TaskReport> outputTaskReport : resume.getOutputTaskReports()) {
        if (outputTaskReport.isPresent()) {
            successfulOutputTaskReports.add(outputTaskReport.get());
        }
    }
    final TaskSource inputTaskSource;
    if (plugins.getInputPlugin() instanceof FileInputRunner) {
        inputTaskSource = FileInputRunner.getFileInputTaskSource(resume.getInputTaskSource());
    } else {
        inputTaskSource = resume.getInputTaskSource();
    }
    plugins.getInputPlugin().cleanup(inputTaskSource, resume.getInputSchema(), resume.getInputTaskReports().size(), successfulInputTaskReports.build());
    final TaskSource outputTaskSource;
    if (plugins.getOutputPlugin() instanceof FileOutputRunner) {
        outputTaskSource = FileOutputRunner.getFileOutputTaskSource(resume.getOutputTaskSource());
    } else {
        outputTaskSource = resume.getOutputTaskSource();
    }
    plugins.getOutputPlugin().cleanup(outputTaskSource, resume.getOutputSchema(), resume.getOutputTaskReports().size(), successfulOutputTaskReports.build());
}

Example 2 with FileInputRunner

use of org.embulk.spi.FileInputRunner in project embulk by embulk.

the class GuessExecutor method guessParserConfig.

private ConfigDiff guessParserConfig(Buffer sample, ConfigSource config, List<PluginType> guessPlugins, final int guessParserSampleBufferBytes) {
    // repeat guessing upto 10 times
    ConfigDiff lastGuessed = Exec.newConfigDiff();
    for (int i = 0; i < 10; i++) {
        // include last-guessed config to run guess input
        ConfigSource originalConfig = config.deepCopy().merge(lastGuessed);
        ConfigSource guessInputConfig = originalConfig.deepCopy();
        guessInputConfig.getNestedOrSetEmpty("parser").set("type", // override in.parser.type so that FileInputRunner.run uses GuessParserPlugin
        "system_guess").set("guess_plugins", guessPlugins).set("orig_config", originalConfig).set("guess_parser_sample_buffer_bytes", guessParserSampleBufferBytes);
        // run FileInputPlugin
        final FileInputRunner input = new FileInputRunner(new BufferFileInputPlugin(sample));
        ConfigDiff guessed;
        try {
            input.transaction(guessInputConfig, new InputPlugin.Control() {

                public List<TaskReport> run(TaskSource inputTaskSource, Schema schema, int taskCount) {
                    if (taskCount == 0) {
                        throw new NoSampleException("No input files to guess");
                    }
                    input.run(inputTaskSource, null, 0, new PageOutput() {

                        @Override
                        public void add(Page page) {
                            // TODO exception class
                            throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");
                        }

                        @Override
                        public void finish() {
                        }

                        @Override
                        public void close() {
                        }
                    });
                    throw new AssertionError("Guess executor must throw GuessedNoticeError");
                }
            });
            throw new AssertionError("Guess executor must throw GuessedNoticeError");
        } catch (GuessedNoticeError error) {
            guessed = lastGuessed.deepCopy().merge(error.getGuessedConfig());
        }
        // merge to the last-guessed config
        if (lastGuessed.equals(guessed)) {
            // not changed
            return lastGuessed;
        }
        lastGuessed = guessed;
    }
    return lastGuessed;
}

Example 3 with FileInputRunner

use of org.embulk.spi.FileInputRunner in project embulk by embulk.

the class MavenPluginSource method newPlugin.

@Override
public <T> T newPlugin(Class<T> pluginInterface, PluginType pluginType) throws PluginSourceNotMatchException {
    final String category;
    if (InputPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "input";
    } else if (OutputPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "output";
    } else if (ParserPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "parser";
    } else if (FormatterPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "formatter";
    } else if (DecoderPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "decoder";
    } else if (EncoderPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "encoder";
    } else if (FilterPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "filter";
    } else if (GuessPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "guess";
    } else if (ExecutorPlugin.class.isAssignableFrom(pluginInterface)) {
        category = "executor";
    } else {
        // unsupported plugin category
        throw new PluginSourceNotMatchException("Plugin interface " + pluginInterface + " is not supported.");
    }
    if (pluginType.getSourceType() != PluginSource.Type.MAVEN) {
        throw new PluginSourceNotMatchException();
    }
    final MavenPluginType mavenPluginType = (MavenPluginType) pluginType;
    final PluginClassLoaderFactory pluginClassLoaderFactory = this.injector.getInstance(PluginClassLoaderFactory.class);
    final MavenArtifactFinder mavenArtifactFinder;
    try {
        mavenArtifactFinder = MavenArtifactFinder.create(getLocalMavenRepository());
    } catch (MavenRepositoryNotFoundException ex) {
        throw new PluginSourceNotMatchException(ex);
    }
    final Path jarPath;
    try {
        jarPath = mavenArtifactFinder.findMavenArtifactJar(mavenPluginType.getGroup(), "embulk-" + category + "-" + mavenPluginType.getName(), mavenPluginType.getClassifier(), mavenPluginType.getVersion());
    } catch (MavenArtifactNotFoundException ex) {
        throw new PluginSourceNotMatchException(ex);
    }
    final Class<?> pluginMainClass;
    try (JarPluginLoader loader = JarPluginLoader.load(jarPath, pluginClassLoaderFactory)) {
        pluginMainClass = loader.getPluginMainClass();
    } catch (InvalidJarPluginException ex) {
        throw new PluginSourceNotMatchException(ex);
    }
    final Object pluginMainObject;
    try {
        // FileInputPlugin and FileOutputPlugin are wrapped with FileInputRunner and FileOutputRunner here.
        if (FileInputPlugin.class.isAssignableFrom(pluginMainClass)) {
            final FileInputPlugin fileInputPluginMainObject;
            try {
                fileInputPluginMainObject = (FileInputPlugin) this.injector.getInstance(pluginMainClass);
            } catch (ClassCastException ex) {
                throw new PluginSourceNotMatchException("[FATAL/INTERNAL] Plugin class \"" + pluginMainClass.getName() + "\" is not file-input.", ex);
            }
            pluginMainObject = new FileInputRunner(fileInputPluginMainObject);
        } else if (FileOutputPlugin.class.isAssignableFrom(pluginMainClass)) {
            final FileOutputPlugin fileOutputPluginMainObject;
            try {
                fileOutputPluginMainObject = (FileOutputPlugin) this.injector.getInstance(pluginMainClass);
            } catch (ClassCastException ex) {
                throw new PluginSourceNotMatchException("[FATAL/INTERNAL] Plugin class \"" + pluginMainClass.getName() + "\" is not file-output.", ex);
            }
            pluginMainObject = new FileOutputRunner(fileOutputPluginMainObject);
        } else {
            if (!pluginInterface.isAssignableFrom(pluginMainClass)) {
                throw new PluginSourceNotMatchException("Plugin class \"" + pluginMainClass.getName() + "\" is not a valid " + category + " plugin.");
            }
            pluginMainObject = this.injector.getInstance(pluginMainClass);
        }
    } catch (ExceptionInInitializerError ex) {
        throw new PluginSourceNotMatchException("Plugin class \"" + pluginMainClass.getName() + "\" is not instantiatable due to exception in initialization.", ex);
    } catch (SecurityException ex) {
        throw new PluginSourceNotMatchException("Plugin class \"" + pluginMainClass.getName() + "\" is not instantiatable due to security manager.", ex);
    }
    try {
        return pluginInterface.cast(pluginMainObject);
    } catch (ClassCastException ex) {
        throw new PluginSourceNotMatchException("[FATAL/INTERNAL] Plugin class \"" + pluginMainClass.getName() + "\" is not " + category + " actually.", ex);
    }
}

Example 4 with FileInputRunner

use of org.embulk.spi.FileInputRunner in project embulk by embulk.

the class PreviewExecutor method doPreview.

private PreviewResult doPreview(ConfigSource config) {
    PreviewTask task = config.loadConfig(PreviewTask.class);
    InputPlugin inputPlugin = newInputPlugin(task);
    List<FilterPlugin> filterPlugins = newFilterPlugins(task);
    if (inputPlugin instanceof FileInputRunner) {
        // file input runner
        Buffer sample = SamplingParserPlugin.runFileInputSampling((FileInputRunner) inputPlugin, config.getNested("in"), createSampleBufferConfigFromExecConfig(task.getExecConfig()));
        FileInputRunner previewRunner = new FileInputRunner(new BufferFileInputPlugin(sample));
        return doPreview(task, previewRunner, filterPlugins);
    } else {
        return doPreview(task, inputPlugin, filterPlugins);
    }
}

Example 5 with FileInputRunner

use of org.embulk.spi.FileInputRunner in project embulk by embulk.

the class InjectedPluginSource method registerPluginTo.

public static <T> void registerPluginTo(Binder binder, Class<T> iface, String name, final Class<?> impl) {
    PluginFactory<T> factory;
    if (FileInputPlugin.class.isAssignableFrom(impl)) {
        Preconditions.checkArgument(InputPlugin.class.equals(iface));
        factory = new PluginFactory<T>() {

            @SuppressWarnings("unchecked")
            public T newPlugin(Injector injector) {
                return (T) new FileInputRunner((FileInputPlugin) injector.getInstance(impl));
            }
        };
    } else if (FileOutputPlugin.class.isAssignableFrom(impl)) {
        Preconditions.checkArgument(OutputPlugin.class.equals(iface));
        factory = new PluginFactory<T>() {

            @SuppressWarnings("unchecked")
            public T newPlugin(Injector injector) {
                return (T) new FileOutputRunner((FileOutputPlugin) injector.getInstance(impl));
            }
        };
    } else {
        Preconditions.checkArgument(iface.isAssignableFrom(impl));
        factory = new PluginFactory<T>() {

            @SuppressWarnings("unchecked")
            public T newPlugin(Injector injector) {
                return (T) injector.getInstance(impl);
            }
        };
    }
    binder.bind(PluginFactory.class).annotatedWith(pluginFactoryName(iface, name)).toInstance(factory);
}