Examples with ConfigDiff org.embulk.config.ConfigDiff used on opensource projects

Example 6 with ConfigDiff

use of org.embulk.config.ConfigDiff in project embulk by embulk.

the class LocalFileInputPlugin method resume.

@Override
public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control) {
    PluginTask task = taskSource.loadTask(PluginTask.class);
    control.run(taskSource, taskCount);
    // build next config
    ConfigDiff configDiff = Exec.newConfigDiff();
    // last_path
    if (task.getFiles().isEmpty()) {
        // keep the last value
        if (task.getLastPath().isPresent()) {
            configDiff.set("last_path", task.getLastPath().get());
        }
    } else {
        List<String> files = new ArrayList<String>(task.getFiles());
        Collections.sort(files);
        configDiff.set("last_path", files.get(files.size() - 1));
    }
    return configDiff;
}

Example 7 with ConfigDiff

use of org.embulk.config.ConfigDiff in project embulk by embulk.

the class EmbulkRunner method runInternal.

private void runInternal(final ConfigSource originalConfigSource, final Path configDiffPath, // deprecated
final Path outputPath, final Path resumeStatePath) throws IOException {
    try {
        checkFileWritable(outputPath);
    } catch (IOException ex) {
        throw new RuntimeException("Not writable: " + outputPath.toString());
    }
    try {
        checkFileWritable(configDiffPath);
    } catch (IOException ex) {
        throw new RuntimeException("Not writable: " + configDiffPath.toString());
    }
    try {
        checkFileWritable(resumeStatePath);
    } catch (IOException ex) {
        throw new RuntimeException("Not writable: " + resumeStatePath.toString());
    }
    final ConfigSource configSource;
    if (configDiffPath != null && Files.size(configDiffPath) > 0L) {
        configSource = originalConfigSource.merge(readConfig(configDiffPath, Collections.<String, Object>emptyMap(), null));
    } else {
        configSource = originalConfigSource;
    }
    final ConfigSource resumeConfig;
    if (resumeStatePath != null) {
        ConfigSource resumeConfigTemp = null;
        try {
            resumeConfigTemp = readYamlConfigFile(resumeStatePath);
        } catch (Throwable ex) {
            // TODO log?
            resumeConfigTemp = null;
        }
        if (resumeConfigTemp == null || resumeConfigTemp.isEmpty()) {
            resumeConfig = null;
        } else {
            resumeConfig = resumeConfigTemp;
        }
    } else {
        resumeConfig = null;
    }
    final EmbulkEmbed.ResumableResult resumableResult;
    final ExecutionResult executionResultTemp;
    if (resumeConfig != null) {
        resumableResult = this.embed.resumeState(configSource, resumeConfig).resume();
        executionResultTemp = null;
    } else if (resumeStatePath != null) {
        resumableResult = this.embed.runResumable(configSource);
        executionResultTemp = null;
    } else {
        resumableResult = null;
        executionResultTemp = this.embed.run(configSource);
    }
    final ExecutionResult executionResult;
    if (executionResultTemp == null) {
        if (!resumableResult.isSuccessful()) {
            if (resumableResult.getTransactionStage().isBefore(TransactionStage.RUN)) {
                // delete resume file
                if (resumeStatePath != null) {
                    try {
                        Files.deleteIfExists(resumeStatePath);
                    } catch (Throwable ex) {
                        System.err.println("Failed to delete: " + resumeStatePath.toString());
                    }
                }
            } else {
                rootLogger.info("Writing resume state to '" + resumeStatePath.toString() + "'");
                try {
                    writeResumeState(resumeStatePath, resumableResult.getResumeState());
                } catch (IOException ex) {
                    throw new RuntimeException(ex);
                }
                rootLogger.info("Resume state is written. Run the transaction again with -r option to resume or use \"cleanup\" subcommand to delete intermediate data.");
            }
            throw new RuntimeException(resumableResult.getCause());
        }
        executionResult = resumableResult.getSuccessfulResult();
    } else {
        executionResult = executionResultTemp;
    }
    // delete resume file
    if (resumeStatePath != null) {
        try {
            Files.deleteIfExists(resumeStatePath);
        } catch (Throwable ex) {
            System.err.println("Failed to delete: " + resumeStatePath.toString());
        }
    }
    final ConfigDiff configDiff = executionResult.getConfigDiff();
    rootLogger.info("Committed.");
    rootLogger.info("Next config diff: " + configDiff.toString());
    writeConfig(configDiffPath, configDiff);
    // deprecated
    writeConfig(outputPath, configSource.merge(configDiff));
}

Example 8 with ConfigDiff

use of org.embulk.config.ConfigDiff in project embulk by embulk.

the class EmbulkRunner method guessInternal.

private void guessInternal(final ConfigSource configSource, final Path outputPath) throws IOException {
    try {
        checkFileWritable(outputPath);
    } catch (IOException ex) {
        throw new RuntimeException("Not writable: " + outputPath.toString());
    }
    final ConfigDiff configDiff = this.embed.guess(configSource);
    final ConfigSource guessedConfigSource = configSource.merge(configDiff);
    final String yaml = writeConfig(outputPath, guessedConfigSource);
    System.err.println(yaml);
    if (outputPath != null) {
        System.out.println("Created '" + outputPath + "' file.");
    } else {
        System.out.println("Use -o PATH option to write the guessed config file to a file.");
    }
}

Example 9 with ConfigDiff

use of org.embulk.config.ConfigDiff in project embulk by embulk.

the class BulkLoader method doResume.

private ExecutionResult doResume(ConfigSource config, final ResumeState resume) {
    final BulkLoaderTask task = config.loadConfig(BulkLoaderTask.class);
    final ExecutorPlugin exec = newExecutorPlugin(task);
    final ProcessPluginSet plugins = new ProcessPluginSet(task);
    final LoaderState state = newLoaderState(Exec.getLogger(BulkLoader.class), plugins);
    state.setTransactionStage(TransactionStage.INPUT_BEGIN);
    try {
        @SuppressWarnings("checkstyle:LineLength") ConfigDiff inputConfigDiff = plugins.getInputPlugin().resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputTaskReports().size(), new InputPlugin.Control() {

            public List<TaskReport> run(final TaskSource inputTask, final Schema inputSchema, final int inputTaskCount) {
                // TODO validate inputTask?
                // TODO validate inputSchema
                state.setInputTaskSource(inputTask);
                state.setTransactionStage(TransactionStage.FILTER_BEGIN);
                Filters.transaction(plugins.getFilterPlugins(), task.getFilterConfigs(), inputSchema, new Filters.Control() {

                    public void run(final List<TaskSource> filterTasks, final List<Schema> schemas) {
                        state.setSchemas(schemas);
                        state.setFilterTaskSources(filterTasks);
                        state.setTransactionStage(TransactionStage.EXECUTOR_BEGIN);
                        exec.transaction(task.getExecConfig(), last(schemas), inputTaskCount, new ExecutorPlugin.Control() {

                            public void transaction(final Schema executorSchema, final int outputTaskCount, final ExecutorPlugin.Executor executor) {
                                // TODO validate executorSchema
                                state.setExecutorSchema(executorSchema);
                                state.setTransactionStage(TransactionStage.OUTPUT_BEGIN);
                                @SuppressWarnings("checkstyle:LineLength") ConfigDiff outputConfigDiff = plugins.getOutputPlugin().resume(resume.getOutputTaskSource(), executorSchema, outputTaskCount, new OutputPlugin.Control() {

                                    public List<TaskReport> run(final TaskSource outputTask) {
                                        // TODO validate outputTask?
                                        state.setOutputTaskSource(outputTask);
                                        restoreResumedTaskReports(resume, state);
                                        state.setTransactionStage(TransactionStage.RUN);
                                        if (!state.isAllTasksCommitted()) {
                                            execute(task, executor, state);
                                        }
                                        if (!state.isAllTasksCommitted()) {
                                            throw new RuntimeException(String.format("%d input tasks and %d output tasks failed", state.countUncommittedInputTasks(), state.countUncommittedOutputTasks()));
                                        }
                                        state.setTransactionStage(TransactionStage.OUTPUT_COMMIT);
                                        return state.getAllOutputTaskReports();
                                    }
                                });
                                state.setOutputConfigDiff(outputConfigDiff);
                                state.setTransactionStage(TransactionStage.EXECUTOR_COMMIT);
                            }
                        });
                        state.setTransactionStage(TransactionStage.FILTER_COMMIT);
                    }
                });
                state.setTransactionStage(TransactionStage.INPUT_COMMIT);
                return state.getAllInputTaskReports();
            }
        });
        state.setInputConfigDiff(inputConfigDiff);
        state.setTransactionStage(TransactionStage.CLEANUP);
        cleanupCommittedTransaction(config, state);
        return state.buildExecuteResult();
    } catch (Throwable ex) {
        if (isSkippedTransaction(ex)) {
            ConfigDiff configDiff = ((SkipTransactionException) ex).getConfigDiff();
            return state.buildExecuteResultOfSkippedExecution(configDiff);
        } else if (state.isAllTasksCommitted() && state.isAllTransactionsCommitted()) {
            // ignore the exception
            return state.buildExecuteResultWithWarningException(ex);
        }
        throw state.buildPartialExecuteException(ex, Exec.session());
    }
}