Examples with JobConfig alluxio.job.JobConfig used on opensource projects

Example 6 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class WorkflowTrackerTest method testBasic.

@Test
public void testBasic() throws Exception {
    ArrayList<JobConfig> jobs = Lists.newArrayList();
    TestPlanConfig child1 = new TestPlanConfig("1");
    TestPlanConfig child2 = new TestPlanConfig("2");
    jobs.add(child1);
    jobs.add(child2);
    CompositeConfig config = new CompositeConfig(jobs, true);
    mWorkflowTracker.run(config, 0);
    verify(mMockJobMaster).run(child1, 100);
    WorkflowInfo info = mWorkflowTracker.getStatus(0, true);
    assertEquals(Status.RUNNING, info.getStatus());
    verify(mMockJobMaster, never()).run(child2, 101);
    PlanInfo plan100 = new PlanInfo(100, child1, null);
    plan100.setStatus(Status.COMPLETED);
    mWorkflowTracker.onPlanStatusChange(plan100);
    verify(mMockJobMaster).run(child2, 101);
    assertEquals(Status.RUNNING, mWorkflowTracker.getStatus(0, true).getStatus());
    PlanInfo plan101 = new PlanInfo(101, child2, null);
    plan101.setStatus(Status.COMPLETED);
    mWorkflowTracker.onPlanStatusChange(plan101);
    assertEquals(Status.COMPLETED, mWorkflowTracker.getStatus(0, true).getStatus());
}

Example 7 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class TaskExecutor method run.

@Override
public void run() {
    JobConfig jobConfig = null;
    Serializable taskArgs = null;
    try {
        jobConfig = (JobConfig) SerializationUtils.deserialize(mRunTaskCommand.getJobConfig().toByteArray());
        if (mRunTaskCommand.hasTaskArgs()) {
            taskArgs = SerializationUtils.deserialize(mRunTaskCommand.getTaskArgs().toByteArray());
        }
    } catch (IOException | ClassNotFoundException e) {
        fail(e, jobConfig, null);
    }
    PlanDefinition<JobConfig, Serializable, Serializable> definition;
    try {
        definition = PlanDefinitionRegistry.INSTANCE.getJobDefinition(jobConfig);
    } catch (JobDoesNotExistException e) {
        LOG.error("The job definition for config {} does not exist.", jobConfig.getName());
        fail(e, jobConfig, taskArgs);
        return;
    }
    mTaskExecutorManager.notifyTaskRunning(mJobId, mTaskId);
    Serializable result;
    try {
        result = definition.runTask(jobConfig, taskArgs, mContext);
    } catch (InterruptedException e) {
        // Cleanup around the interruption should already have been handled by a different thread
        Thread.currentThread().interrupt();
        return;
    } catch (Throwable t) {
        fail(t, jobConfig, taskArgs);
        return;
    }
    mTaskExecutorManager.notifyTaskCompletion(mJobId, mTaskId, result);
}

Example 8 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class PlanCoordinator method start.

private synchronized void start() throws JobDoesNotExistException {
    // get the job definition
    LOG.info("Starting job Id={} Config={}", mPlanInfo.getId(), mPlanInfo.getJobConfig());
    PlanDefinition<JobConfig, ?, ?> definition;
    try {
        definition = PlanDefinitionRegistry.INSTANCE.getJobDefinition(mPlanInfo.getJobConfig());
    } catch (JobDoesNotExistException e) {
        LOG.info("Exception when getting jobDefinition from jobConfig: ", e);
        mPlanInfo.setErrorType(ErrorUtils.getErrorType(e));
        mPlanInfo.setErrorMessage(e.getMessage());
        DistributedCmdMetrics.incrementForAllConfigsFailStatus(mPlanInfo.getJobConfig());
        mPlanInfo.setStatus(Status.FAILED);
        throw e;
    }
    SelectExecutorsContext context = new SelectExecutorsContext(mPlanInfo.getId(), mJobServerContext);
    Set<? extends Pair<WorkerInfo, ?>> taskAddressToArgs;
    ArrayList<WorkerInfo> workersInfoListCopy = Lists.newArrayList(mWorkersInfoList);
    Collections.shuffle(workersInfoListCopy);
    try {
        taskAddressToArgs = definition.selectExecutors(mPlanInfo.getJobConfig(), workersInfoListCopy, context);
    } catch (Exception e) {
        LOG.warn("Failed to select executor. {})", e.toString());
        LOG.info("Exception: ", e);
        setJobAsFailed(ErrorUtils.getErrorType(e), e.getMessage());
        return;
    }
    if (taskAddressToArgs.isEmpty()) {
        LOG.warn("No executor was selected.");
        updateStatus();
    }
    for (Pair<WorkerInfo, ?> pair : taskAddressToArgs) {
        LOG.debug("Selected executor {} with parameters {}.", pair.getFirst(), pair.getSecond());
        int taskId = mTaskIdToWorkerInfo.size();
        // create task
        mPlanInfo.addTask(taskId, pair.getFirst(), pair.getSecond());
        // submit commands
        JobConfig config;
        if (mPlanInfo.getJobConfig() instanceof BatchedJobConfig) {
            BatchedJobConfig planConfig = (BatchedJobConfig) mPlanInfo.getJobConfig();
            config = new BatchedJobConfig(planConfig.getJobType(), new HashSet<>());
        } else {
            config = mPlanInfo.getJobConfig();
        }
        mCommandManager.submitRunTaskCommand(mPlanInfo.getId(), taskId, config, pair.getSecond(), pair.getFirst().getId());
        mTaskIdToWorkerInfo.put((long) taskId, pair.getFirst());
        mWorkerIdToTaskIds.putIfAbsent(pair.getFirst().getId(), Lists.newArrayList());
        mWorkerIdToTaskIds.get(pair.getFirst().getId()).add((long) taskId);
    }
}

Example 9 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class PlanCoordinator method join.

/**
 * Joins the task results and produces a final result.
 *
 * @param taskInfoList the list of task information
 * @return the aggregated result as a String
 * @throws Exception if any error occurs
 */
private String join(List<TaskInfo> taskInfoList) throws Exception {
    // get the job definition
    PlanDefinition<JobConfig, Serializable, Serializable> definition = PlanDefinitionRegistry.INSTANCE.getJobDefinition(mPlanInfo.getJobConfig());
    Map<WorkerInfo, Serializable> taskResults = Maps.newHashMap();
    for (TaskInfo taskInfo : taskInfoList) {
        taskResults.put(mTaskIdToWorkerInfo.get(taskInfo.getTaskId()), taskInfo.getResult());
    }
    return definition.join(mPlanInfo.getJobConfig(), taskResults);
}

Example 10 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class CommandManagerTest method submitRunTaskCommand.

@Test
public void submitRunTaskCommand() throws Exception {
    long jobId = 0L;
    int taskId = 1;
    JobConfig jobConfig = new TestPlanConfig("/test");
    long workerId = 2L;
    List<Integer> args = Lists.newArrayList(1);
    mManager.submitRunTaskCommand(jobId, taskId, jobConfig, args, workerId);
    List<JobCommand> commands = mManager.pollAllPendingCommands(workerId);
    Assert.assertEquals(1, commands.size());
    JobCommand command = commands.get(0);
    Assert.assertEquals(jobId, command.getRunTaskCommand().getJobId());
    Assert.assertEquals(taskId, command.getRunTaskCommand().getTaskId());
    Assert.assertEquals(jobConfig, SerializationUtils.deserialize(command.getRunTaskCommand().getJobConfig().toByteArray()));
    Assert.assertEquals(args, SerializationUtils.deserialize(command.getRunTaskCommand().getTaskArgs().toByteArray()));
}