Examples with CompositeConfig alluxio.job.workflow.composite.CompositeConfig used on opensource projects

Example 1 with CompositeConfig

use of alluxio.job.workflow.composite.CompositeConfig in project alluxio by Alluxio.

the class JobMasterTest method runNestedNonExistingJobConfig.

@Test
public void runNestedNonExistingJobConfig() throws Exception {
    JobConfig innerJobConfig = new CompositeConfig(Lists.newArrayList(new DummyPlanConfig()), true);
    CompositeConfig jobConfig = new CompositeConfig(Lists.newArrayList(innerJobConfig), true);
    long jobId = mJobMaster.run(jobConfig);
    JobInfo status = mJobMaster.getStatus(jobId);
    Assert.assertEquals(Status.FAILED, status.getStatus());
    List<JobInfo> children = status.getChildren();
    Assert.assertEquals(1, children.size());
    JobInfo child = children.get(0);
    Assert.assertEquals(Status.FAILED, child.getStatus());
    Assert.assertEquals(0, child.getChildren().size());
}

Example 2 with CompositeConfig

use of alluxio.job.workflow.composite.CompositeConfig in project alluxio by Alluxio.

the class WorkflowTrackerTest method testCleanup.

@Test
public void testCleanup() throws Exception {
    SleepJobConfig jobConfig = new SleepJobConfig(1);
    mPlanTracker.run(jobConfig, mCommandManager, mMockJobServerContext, mWorkers, 1);
    jobConfig = new SleepJobConfig(1);
    mPlanTracker.run(jobConfig, mCommandManager, mMockJobServerContext, mWorkers, 2);
    jobConfig = new SleepJobConfig(1);
    mPlanTracker.run(jobConfig, mCommandManager, mMockJobServerContext, mWorkers, 3);
    doAnswer(invocation -> {
        PlanConfig config = invocation.getArgument(0, PlanConfig.class);
        long jobId = invocation.getArgument(1, Long.class);
        mPlanTracker.run(config, mCommandManager, mMockJobServerContext, mWorkers, jobId);
        return null;
    }).when(mMockJobMaster).run(any(PlanConfig.class), any(Long.class));
    ArrayList<JobConfig> jobs = Lists.newArrayList();
    SleepJobConfig child1 = new SleepJobConfig(1);
    SleepJobConfig child2 = new SleepJobConfig(2);
    jobs.add(child1);
    jobs.add(child2);
    CompositeConfig config = new CompositeConfig(jobs, false);
    mWorkflowTracker.run(config, 0);
    try {
        mPlanTracker.run(new SleepJobConfig(1), mCommandManager, mMockJobServerContext, mWorkers, 4);
        fail();
    } catch (ResourceExhaustedException e) {
    // Should fail
    }
    mPlanTracker.coordinators().stream().filter(coordinator -> coordinator.getJobId() == 100).findFirst().get().setJobAsFailed("TestError", "failed");
    mPlanTracker.run(new SleepJobConfig(1), mCommandManager, mMockJobServerContext, mWorkers, 4);
    assertNotNull(mWorkflowTracker.getStatus(0, true));
    try {
        mPlanTracker.run(new SleepJobConfig(1), mCommandManager, mMockJobServerContext, mWorkers, 5);
        fail();
    } catch (ResourceExhaustedException e) {
    // Should fail
    }
    mPlanTracker.coordinators().stream().filter(coordinator -> coordinator.getJobId() == 101).findFirst().get().setJobAsFailed("TestError", "failed");
    mPlanTracker.run(new SleepJobConfig(1), mCommandManager, mMockJobServerContext, mWorkers, 5);
    assertNull(mWorkflowTracker.getStatus(100, true));
}

Example 3 with CompositeConfig

use of alluxio.job.workflow.composite.CompositeConfig in project alluxio by Alluxio.

the class WorkflowTrackerTest method testBasic.

@Test
public void testBasic() throws Exception {
    ArrayList<JobConfig> jobs = Lists.newArrayList();
    TestPlanConfig child1 = new TestPlanConfig("1");
    TestPlanConfig child2 = new TestPlanConfig("2");
    jobs.add(child1);
    jobs.add(child2);
    CompositeConfig config = new CompositeConfig(jobs, true);
    mWorkflowTracker.run(config, 0);
    verify(mMockJobMaster).run(child1, 100);
    WorkflowInfo info = mWorkflowTracker.getStatus(0, true);
    assertEquals(Status.RUNNING, info.getStatus());
    verify(mMockJobMaster, never()).run(child2, 101);
    PlanInfo plan100 = new PlanInfo(100, child1, null);
    plan100.setStatus(Status.COMPLETED);
    mWorkflowTracker.onPlanStatusChange(plan100);
    verify(mMockJobMaster).run(child2, 101);
    assertEquals(Status.RUNNING, mWorkflowTracker.getStatus(0, true).getStatus());
    PlanInfo plan101 = new PlanInfo(101, child2, null);
    plan101.setStatus(Status.COMPLETED);
    mWorkflowTracker.onPlanStatusChange(plan101);
    assertEquals(Status.COMPLETED, mWorkflowTracker.getStatus(0, true).getStatus());
}

Example 4 with CompositeConfig

use of alluxio.job.workflow.composite.CompositeConfig in project alluxio by Alluxio.

the class JobMasterIntegrationTest method cancel.

@Test
public void cancel() throws Exception {
    SleepJobConfig childJob1 = new SleepJobConfig(50000);
    SleepJobConfig childJob2 = new SleepJobConfig(45000);
    SleepJobConfig childJob3 = new SleepJobConfig(40000);
    CompositeConfig jobConfig = new CompositeConfig(Lists.newArrayList(childJob1, childJob2, childJob3), false);
    long jobId = mJobMaster.run(jobConfig);
    JobInfo status = mJobMaster.getStatus(jobId);
    List<JobInfo> children = status.getChildren();
    assertEquals(3, children.size());
    long child0 = children.get(0).getId();
    long child1 = children.get(1).getId();
    long child2 = children.get(2).getId();
    mJobMaster.cancel(jobId);
    JobTestUtils.waitForJobStatus(mJobMaster, jobId, Status.CANCELED);
    JobTestUtils.waitForJobStatus(mJobMaster, child0, Status.CANCELED);
    JobTestUtils.waitForJobStatus(mJobMaster, child1, Status.CANCELED);
    JobTestUtils.waitForJobStatus(mJobMaster, child2, Status.CANCELED);
}

Example 5 with CompositeConfig

use of alluxio.job.workflow.composite.CompositeConfig in project alluxio by Alluxio.

the class WorkflowTrackerTest method testEmpty.

@Test
public void testEmpty() throws Exception {
    ArrayList<JobConfig> jobs = Lists.newArrayList();
    CompositeConfig config = new CompositeConfig(jobs, true);
    mWorkflowTracker.run(config, 0);
    WorkflowInfo info = mWorkflowTracker.getStatus(0, true);
    assertEquals(Status.COMPLETED, info.getStatus());
}