Examples with TaskReport org.embulk.config.TaskReport used on opensource projects

Example 6 with TaskReport

use of org.embulk.config.TaskReport in project embulk by embulk.

the class Executors method process.

public static void process(ExecSession exec, int taskIndex, InputPlugin inputPlugin, Schema inputSchema, TaskSource inputTaskSource, List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources, OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource, ProcessStateCallback callback) {
    TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput(outputPlugin.open(outputTaskSource, outputSchema, taskIndex));
    callback.started();
    // essential exception.
    try (CloseResource closer = new CloseResource(tran)) {
        try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) {
            PageOutput filtered = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran);
            closer.closeThis(filtered);
            TaskReport inputTaskReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered);
            if (inputTaskReport == null) {
                inputTaskReport = exec.newTaskReport();
            }
            callback.inputCommitted(inputTaskReport);
            TaskReport outputTaskReport = tran.commit();
            aborter.dontAbort();
            if (outputTaskReport == null) {
                outputTaskReport = exec.newTaskReport();
            }
            // TODO check output.finish() is called. wrap or abstract
            callback.outputCommitted(outputTaskReport);
        }
    }
}

Example 7 with TaskReport

use of org.embulk.config.TaskReport in project embulk by embulk.

the class BulkLoader method restoreResumedTaskReports.

private static void restoreResumedTaskReports(ResumeState resume, LoaderState state) {
    int inputTaskCount = resume.getInputTaskReports().size();
    int outputTaskCount = resume.getOutputTaskReports().size();
    state.initialize(inputTaskCount, outputTaskCount);
    for (int i = 0; i < inputTaskCount; i++) {
        Optional<TaskReport> report = resume.getInputTaskReports().get(i);
        if (report.isPresent()) {
            TaskState task = state.getInputTaskState(i);
            task.start();
            task.setTaskReport(report.get());
            task.finish();
        }
    }
    for (int i = 0; i < outputTaskCount; i++) {
        Optional<TaskReport> report = resume.getOutputTaskReports().get(i);
        if (report.isPresent()) {
            TaskState task = state.getOutputTaskState(i);
            task.start();
            task.setTaskReport(report.get());
            task.finish();
        }
    }
}

Example 8 with TaskReport

use of org.embulk.config.TaskReport in project embulk by embulk.

the class BulkLoader method doResume.

private ExecutionResult doResume(ConfigSource config, final ResumeState resume) {
    final BulkLoaderTask task = config.loadConfig(BulkLoaderTask.class);
    final ExecutorPlugin exec = newExecutorPlugin(task);
    final ProcessPluginSet plugins = new ProcessPluginSet(task);
    final LoaderState state = newLoaderState(Exec.getLogger(BulkLoader.class), plugins);
    state.setTransactionStage(TransactionStage.INPUT_BEGIN);
    try {
        @SuppressWarnings("checkstyle:LineLength") ConfigDiff inputConfigDiff = plugins.getInputPlugin().resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputTaskReports().size(), new InputPlugin.Control() {

            public List<TaskReport> run(final TaskSource inputTask, final Schema inputSchema, final int inputTaskCount) {
                // TODO validate inputTask?
                // TODO validate inputSchema
                state.setInputTaskSource(inputTask);
                state.setTransactionStage(TransactionStage.FILTER_BEGIN);
                Filters.transaction(plugins.getFilterPlugins(), task.getFilterConfigs(), inputSchema, new Filters.Control() {

                    public void run(final List<TaskSource> filterTasks, final List<Schema> schemas) {
                        state.setSchemas(schemas);
                        state.setFilterTaskSources(filterTasks);
                        state.setTransactionStage(TransactionStage.EXECUTOR_BEGIN);
                        exec.transaction(task.getExecConfig(), last(schemas), inputTaskCount, new ExecutorPlugin.Control() {

                            public void transaction(final Schema executorSchema, final int outputTaskCount, final ExecutorPlugin.Executor executor) {
                                // TODO validate executorSchema
                                state.setExecutorSchema(executorSchema);
                                state.setTransactionStage(TransactionStage.OUTPUT_BEGIN);
                                @SuppressWarnings("checkstyle:LineLength") ConfigDiff outputConfigDiff = plugins.getOutputPlugin().resume(resume.getOutputTaskSource(), executorSchema, outputTaskCount, new OutputPlugin.Control() {

                                    public List<TaskReport> run(final TaskSource outputTask) {
                                        // TODO validate outputTask?
                                        state.setOutputTaskSource(outputTask);
                                        restoreResumedTaskReports(resume, state);
                                        state.setTransactionStage(TransactionStage.RUN);
                                        if (!state.isAllTasksCommitted()) {
                                            execute(task, executor, state);
                                        }
                                        if (!state.isAllTasksCommitted()) {
                                            throw new RuntimeException(String.format("%d input tasks and %d output tasks failed", state.countUncommittedInputTasks(), state.countUncommittedOutputTasks()));
                                        }
                                        state.setTransactionStage(TransactionStage.OUTPUT_COMMIT);
                                        return state.getAllOutputTaskReports();
                                    }
                                });
                                state.setOutputConfigDiff(outputConfigDiff);
                                state.setTransactionStage(TransactionStage.EXECUTOR_COMMIT);
                            }
                        });
                        state.setTransactionStage(TransactionStage.FILTER_COMMIT);
                    }
                });
                state.setTransactionStage(TransactionStage.INPUT_COMMIT);
                return state.getAllInputTaskReports();
            }
        });
        state.setInputConfigDiff(inputConfigDiff);
        state.setTransactionStage(TransactionStage.CLEANUP);
        cleanupCommittedTransaction(config, state);
        return state.buildExecuteResult();
    } catch (Throwable ex) {
        if (isSkippedTransaction(ex)) {
            ConfigDiff configDiff = ((SkipTransactionException) ex).getConfigDiff();
            return state.buildExecuteResultOfSkippedExecution(configDiff);
        } else if (state.isAllTasksCommitted() && state.isAllTransactionsCommitted()) {
            // ignore the exception
            return state.buildExecuteResultWithWarningException(ex);
        }
        throw state.buildPartialExecuteException(ex, Exec.session());
    }
}

Example 9 with TaskReport

use of org.embulk.config.TaskReport in project embulk by embulk.

the class FileInputRunner method run.

@Override
public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) {
    final RunnerTask task = taskSource.loadTask(RunnerTask.class);
    List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task);
    ParserPlugin parserPlugin = newParserPlugin(task);
    TransactionalFileInput tran = PluginWrappers.transactionalFileInput(fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex));
    try (CloseResource closer = new CloseResource(tran)) {
        try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) {
            FileInput fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), tran);
            closer.closeThis(fileInput);
            parserPlugin.run(task.getParserTaskSource(), schema, fileInput, output);
            // TODO check output.finish() is called. wrap
            TaskReport report = tran.commit();
            aborter.dontAbort();
            return report;
        }
    }
}