Examples with JobConfig alluxio.job.JobConfig used on opensource projects

Example 1 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class JobMasterTest method flowControl.

@Test
public void flowControl() throws Exception {
    PlanCoordinator coordinator = PowerMockito.mock(PlanCoordinator.class);
    mockStatic(PlanCoordinator.class);
    when(PlanCoordinator.create(any(CommandManager.class), any(JobServerContext.class), anyList(), anyLong(), any(JobConfig.class), any(Consumer.class))).thenReturn(coordinator);
    TestPlanConfig jobConfig = new TestPlanConfig("/test");
    for (long i = 0; i < TEST_JOB_MASTER_JOB_CAPACITY; i++) {
        mJobMaster.run(jobConfig);
    }
    try {
        mJobMaster.run(jobConfig);
        Assert.fail("should not be able to run more jobs than job master capacity");
    } catch (ResourceExhaustedException e) {
        Assert.assertEquals(ExceptionMessage.JOB_MASTER_FULL_CAPACITY.getMessage(ServerConfiguration.get(PropertyKey.JOB_MASTER_JOB_CAPACITY)), e.getMessage());
    }
}

Example 2 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class JobMasterTest method runNestedNonExistingJobConfig.

@Test
public void runNestedNonExistingJobConfig() throws Exception {
    JobConfig innerJobConfig = new CompositeConfig(Lists.newArrayList(new DummyPlanConfig()), true);
    CompositeConfig jobConfig = new CompositeConfig(Lists.newArrayList(innerJobConfig), true);
    long jobId = mJobMaster.run(jobConfig);
    JobInfo status = mJobMaster.getStatus(jobId);
    Assert.assertEquals(Status.FAILED, status.getStatus());
    List<JobInfo> children = status.getChildren();
    Assert.assertEquals(1, children.size());
    JobInfo child = children.get(0);
    Assert.assertEquals(Status.FAILED, child.getStatus());
    Assert.assertEquals(0, child.getChildren().size());
}

Example 3 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class JobMasterTest method run.

@Test
public void run() throws Exception {
    PlanCoordinator coordinator = PowerMockito.mock(PlanCoordinator.class);
    mockStatic(PlanCoordinator.class);
    when(PlanCoordinator.create(any(CommandManager.class), any(JobServerContext.class), anyList(), anyLong(), any(JobConfig.class), any(Consumer.class))).thenReturn(coordinator);
    TestPlanConfig jobConfig = new TestPlanConfig("/test");
    List<Long> jobIdList = new ArrayList<>();
    for (long i = 0; i < TEST_JOB_MASTER_JOB_CAPACITY; i++) {
        jobIdList.add(mJobMaster.run(jobConfig));
    }
    final List<Long> list = mJobMaster.list(ListAllPOptions.getDefaultInstance());
    Assert.assertEquals(jobIdList, list);
    Assert.assertEquals(TEST_JOB_MASTER_JOB_CAPACITY, mJobMaster.list(ListAllPOptions.getDefaultInstance()).size());
}

Example 4 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class PlanCoordinatorTest method before.

@Before
public void before() throws Exception {
    mCommandManager = new CommandManager();
    // Create mock JobServerContext
    FileSystem fs = mock(FileSystem.class);
    FileSystemContext fsCtx = PowerMockito.mock(FileSystemContext.class);
    UfsManager ufsManager = Mockito.mock(UfsManager.class);
    mJobServerContext = new JobServerContext(fs, fsCtx, ufsManager);
    // Create mock job info.
    mJobconfig = Mockito.mock(JobConfig.class, Mockito.withSettings().serializable());
    Mockito.when(mJobconfig.getName()).thenReturn("mock");
    mJobId = 1;
    // Create mock job definition.
    @SuppressWarnings("unchecked") PlanDefinition<JobConfig, Serializable, Serializable> mockPlanDefinition = Mockito.mock(PlanDefinition.class);
    PlanDefinitionRegistry singleton = PowerMockito.mock(PlanDefinitionRegistry.class);
    Whitebox.setInternalState(PlanDefinitionRegistry.class, "INSTANCE", singleton);
    Mockito.when(singleton.getJobDefinition(mJobconfig)).thenReturn(mockPlanDefinition);
    mPlanDefinition = mockPlanDefinition;
    // Create test worker.
    mWorkerInfo = new WorkerInfo();
    mWorkerInfo.setId(0);
    mWorkerInfoList = Lists.newArrayList(mWorkerInfo);
}

Example 5 with JobConfig

use of alluxio.job.JobConfig in project alluxio by Alluxio.

the class WorkflowTrackerTest method testCleanup.

@Test
public void testCleanup() throws Exception {
    SleepJobConfig jobConfig = new SleepJobConfig(1);
    mPlanTracker.run(jobConfig, mCommandManager, mMockJobServerContext, mWorkers, 1);
    jobConfig = new SleepJobConfig(1);
    mPlanTracker.run(jobConfig, mCommandManager, mMockJobServerContext, mWorkers, 2);
    jobConfig = new SleepJobConfig(1);
    mPlanTracker.run(jobConfig, mCommandManager, mMockJobServerContext, mWorkers, 3);
    doAnswer(invocation -> {
        PlanConfig config = invocation.getArgument(0, PlanConfig.class);
        long jobId = invocation.getArgument(1, Long.class);
        mPlanTracker.run(config, mCommandManager, mMockJobServerContext, mWorkers, jobId);
        return null;
    }).when(mMockJobMaster).run(any(PlanConfig.class), any(Long.class));
    ArrayList<JobConfig> jobs = Lists.newArrayList();
    SleepJobConfig child1 = new SleepJobConfig(1);
    SleepJobConfig child2 = new SleepJobConfig(2);
    jobs.add(child1);
    jobs.add(child2);
    CompositeConfig config = new CompositeConfig(jobs, false);
    mWorkflowTracker.run(config, 0);
    try {
        mPlanTracker.run(new SleepJobConfig(1), mCommandManager, mMockJobServerContext, mWorkers, 4);
        fail();
    } catch (ResourceExhaustedException e) {
    // Should fail
    }
    mPlanTracker.coordinators().stream().filter(coordinator -> coordinator.getJobId() == 100).findFirst().get().setJobAsFailed("TestError", "failed");
    mPlanTracker.run(new SleepJobConfig(1), mCommandManager, mMockJobServerContext, mWorkers, 4);
    assertNotNull(mWorkflowTracker.getStatus(0, true));
    try {
        mPlanTracker.run(new SleepJobConfig(1), mCommandManager, mMockJobServerContext, mWorkers, 5);
        fail();
    } catch (ResourceExhaustedException e) {
    // Should fail
    }
    mPlanTracker.coordinators().stream().filter(coordinator -> coordinator.getJobId() == 101).findFirst().get().setJobAsFailed("TestError", "failed");
    mPlanTracker.run(new SleepJobConfig(1), mCommandManager, mMockJobServerContext, mWorkers, 5);
    assertNull(mWorkflowTracker.getStatus(100, true));
}