Examples with SubscopedBrokerBuilder org.apache.gobblin.broker.iface.SubscopedBrokerBuilder used on opensource projects

Example 1 with SubscopedBrokerBuilder

use of org.apache.gobblin.broker.iface.SubscopedBrokerBuilder in project incubator-gobblin by apache.

the class GobblinMultiTaskAttempt method runWorkUnits.

/**
 * Run a given list of {@link WorkUnit}s of a job.
 *
 * <p>
 *   This method assumes that the given list of {@link WorkUnit}s have already been flattened and
 *   each {@link WorkUnit} contains the task ID in the property {@link ConfigurationKeys#TASK_ID_KEY}.
 * </p>
 *
 * @param countDownLatch a {@link java.util.concurrent.CountDownLatch} waited on for job completion
 * @return a list of {@link Task}s from the {@link WorkUnit}s
 */
private List<Task> runWorkUnits(CountUpAndDownLatch countDownLatch) {
    List<Task> tasks = Lists.newArrayList();
    while (this.workUnits.hasNext()) {
        WorkUnit workUnit = this.workUnits.next();
        String taskId = workUnit.getProp(ConfigurationKeys.TASK_ID_KEY);
        // skip tasks that executed successsfully in a prior attempt
        if (taskSuccessfulInPriorAttempt(taskId)) {
            continue;
        }
        countDownLatch.countUp();
        SubscopedBrokerBuilder<GobblinScopeTypes, ?> taskBrokerBuilder = this.jobBroker.newSubscopedBuilder(new TaskScopeInstance(taskId));
        WorkUnitState workUnitState = new WorkUnitState(workUnit, this.jobState, taskBrokerBuilder);
        workUnitState.setId(taskId);
        workUnitState.setProp(ConfigurationKeys.JOB_ID_KEY, this.jobId);
        workUnitState.setProp(ConfigurationKeys.TASK_ID_KEY, taskId);
        if (this.containerIdOptional.isPresent()) {
            workUnitState.setProp(ConfigurationKeys.TASK_ATTEMPT_ID_KEY, this.containerIdOptional.get());
        }
        // Create a new task from the work unit and submit the task to run
        Task task = createTaskRunnable(workUnitState, countDownLatch);
        this.taskStateTracker.registerNewTask(task);
        task.setTaskFuture(this.taskExecutor.submit(task));
        tasks.add(task);
    }
    new EventSubmitter.Builder(JobMetrics.get(this.jobId).getMetricContext(), "gobblin.runtime").build().submit(JobEvent.TASKS_SUBMITTED, "tasksCount", Long.toString(countDownLatch.getRegisteredParties()));
    return tasks;
}