Examples with IJobCapacityController org.apache.hyracks.api.job.resource.IJobCapacityController used on opensource projects

Example 1 with IJobCapacityController

use of org.apache.hyracks.api.job.resource.IJobCapacityController in project asterixdb by apache.

the class JobManagerTest method testAdmitThenReject.

@Test
public void testAdmitThenReject() throws HyracksException {
    IJobCapacityController jobCapacityController = mock(IJobCapacityController.class);
    IJobManager jobManager = spy(new JobManager(ccConfig, mockClusterControllerService(), jobCapacityController));
    // A pending job should also be rejected if its requirement exceeds the updated maximum capacity of the cluster.
    // A normal run.
    JobRun run1 = mockJobRun(1);
    JobSpecification job1 = mock(JobSpecification.class);
    when(run1.getJobSpecification()).thenReturn(job1);
    when(jobCapacityController.allocate(job1)).thenReturn(IJobCapacityController.JobSubmissionStatus.EXECUTE);
    jobManager.add(run1);
    // A failure run.
    JobRun run2 = mockJobRun(2);
    JobSpecification job2 = mock(JobSpecification.class);
    when(run2.getJobSpecification()).thenReturn(job2);
    when(jobCapacityController.allocate(job2)).thenReturn(IJobCapacityController.JobSubmissionStatus.QUEUE).thenThrow(HyracksException.create(ErrorCode.JOB_REQUIREMENTS_EXCEED_CAPACITY, "1", "0"));
    jobManager.add(run2);
    // Completes the first run.
    jobManager.prepareComplete(run1, JobStatus.TERMINATED, Collections.emptyList());
    jobManager.finalComplete(run1);
    // Verifies job status of the failed job.
    verify(run2, times(1)).setStatus(eq(JobStatus.PENDING), any());
    verify(run2, times(1)).setPendingStatus(eq(JobStatus.FAILURE), any());
}

Example 2 with IJobCapacityController

use of org.apache.hyracks.api.job.resource.IJobCapacityController in project asterixdb by apache.

the class JobManagerTest method testCancel.

@Test
public void testCancel() throws HyracksException {
    CCConfig ccConfig = new CCConfig();
    IJobCapacityController jobCapacityController = mock(IJobCapacityController.class);
    IJobManager jobManager = spy(new JobManager(ccConfig, mockClusterControllerService(), jobCapacityController));
    // Submits runnable jobs.
    List<JobRun> acceptedRuns = new ArrayList<>();
    for (int id = 0; id < 4096; ++id) {
        // Mocks an immediately executable job.
        JobRun run = mockJobRun(id);
        JobSpecification job = mock(JobSpecification.class);
        when(run.getJobSpecification()).thenReturn(job);
        when(jobCapacityController.allocate(job)).thenReturn(IJobCapacityController.JobSubmissionStatus.EXECUTE);
        // Submits the job.
        acceptedRuns.add(run);
        jobManager.add(run);
        Assert.assertTrue(jobManager.getRunningJobs().size() == id + 1);
        Assert.assertTrue(jobManager.getPendingJobs().isEmpty());
    }
    // Submits jobs that will be deferred due to the capacity limitation.
    List<JobRun> deferredRuns = new ArrayList<>();
    for (int id = 4096; id < 8192; ++id) {
        // Mocks a deferred job.
        JobRun run = mockJobRun(id);
        JobSpecification job = mock(JobSpecification.class);
        when(run.getJobSpecification()).thenReturn(job);
        when(jobCapacityController.allocate(job)).thenReturn(IJobCapacityController.JobSubmissionStatus.QUEUE).thenReturn(IJobCapacityController.JobSubmissionStatus.EXECUTE);
        // Submits the job.
        deferredRuns.add(run);
        jobManager.add(run);
        Assert.assertTrue(jobManager.getRunningJobs().size() == 4096);
        Assert.assertTrue(jobManager.getPendingJobs().size() == id + 1 - 4096);
    }
    // Cancels deferred jobs.
    for (JobRun run : deferredRuns) {
        jobManager.cancel(run.getJobId());
    }
    // Cancels runnable jobs.
    for (JobRun run : acceptedRuns) {
        jobManager.cancel(run.getJobId());
    }
    Assert.assertTrue(jobManager.getPendingJobs().isEmpty());
    Assert.assertTrue(jobManager.getArchivedJobs().size() == ccConfig.getJobHistorySize());
    verify(jobManager, times(0)).prepareComplete(any(), any(), any());
    verify(jobManager, times(0)).finalComplete(any());
}

Example 3 with IJobCapacityController

use of org.apache.hyracks.api.job.resource.IJobCapacityController in project asterixdb by apache.

the class JobManagerTest method test.

@Test
public void test() throws IOException, CmdLineException {
    IJobCapacityController jobCapacityController = mock(IJobCapacityController.class);
    IJobManager jobManager = spy(new JobManager(ccConfig, mockClusterControllerService(), jobCapacityController));
    // Submits runnable jobs.
    List<JobRun> acceptedRuns = new ArrayList<>();
    for (int id = 0; id < 4096; ++id) {
        // Mocks an immediately executable job.
        JobRun run = mockJobRun(id);
        JobSpecification job = mock(JobSpecification.class);
        when(run.getJobSpecification()).thenReturn(job);
        when(jobCapacityController.allocate(job)).thenReturn(IJobCapacityController.JobSubmissionStatus.EXECUTE);
        // Submits the job.
        acceptedRuns.add(run);
        jobManager.add(run);
        Assert.assertTrue(jobManager.getRunningJobs().size() == id + 1);
        Assert.assertTrue(jobManager.getPendingJobs().isEmpty());
    }
    // Submits jobs that will be deferred due to the capacity limitation.
    List<JobRun> deferredRuns = new ArrayList<>();
    for (int id = 4096; id < 8192; ++id) {
        // Mocks a deferred job.
        JobRun run = mockJobRun(id);
        JobSpecification job = mock(JobSpecification.class);
        when(run.getJobSpecification()).thenReturn(job);
        when(jobCapacityController.allocate(job)).thenReturn(IJobCapacityController.JobSubmissionStatus.QUEUE).thenReturn(IJobCapacityController.JobSubmissionStatus.EXECUTE);
        // Submits the job.
        deferredRuns.add(run);
        jobManager.add(run);
        Assert.assertTrue(jobManager.getRunningJobs().size() == 4096);
        Assert.assertTrue(jobManager.getPendingJobs().size() == id + 1 - 4096);
    }
    // Further jobs will be denied because the job queue is full.
    boolean jobQueueFull = false;
    try {
        JobRun run = mockJobRun(8193);
        JobSpecification job = mock(JobSpecification.class);
        when(run.getJobSpecification()).thenReturn(job);
        when(jobCapacityController.allocate(job)).thenReturn(IJobCapacityController.JobSubmissionStatus.QUEUE).thenReturn(IJobCapacityController.JobSubmissionStatus.EXECUTE);
        jobManager.add(run);
    } catch (HyracksException e) {
        // Verifies the error code.
        jobQueueFull = e.getErrorCode() == ErrorCode.JOB_QUEUE_FULL;
    }
    Assert.assertTrue(jobQueueFull);
    // Completes runnable jobs.
    for (JobRun run : acceptedRuns) {
        jobManager.prepareComplete(run, JobStatus.TERMINATED, Collections.emptyList());
        jobManager.finalComplete(run);
    }
    Assert.assertTrue(jobManager.getRunningJobs().size() == 4096);
    Assert.assertTrue(jobManager.getPendingJobs().isEmpty());
    Assert.assertTrue(jobManager.getArchivedJobs().size() == ccConfig.getJobHistorySize());
    // Completes deferred jobs.
    for (JobRun run : deferredRuns) {
        jobManager.prepareComplete(run, JobStatus.TERMINATED, Collections.emptyList());
        jobManager.finalComplete(run);
    }
    Assert.assertTrue(jobManager.getRunningJobs().isEmpty());
    Assert.assertTrue(jobManager.getPendingJobs().isEmpty());
    Assert.assertTrue(jobManager.getArchivedJobs().size() == ccConfig.getJobHistorySize());
    verify(jobManager, times(8192)).prepareComplete(any(), any(), any());
    verify(jobManager, times(8192)).finalComplete(any());
}

Example 4 with IJobCapacityController

use of org.apache.hyracks.api.job.resource.IJobCapacityController in project asterixdb by apache.

the class JobManagerTest method testExceedMax.

@Test
public void testExceedMax() throws HyracksException {
    IJobCapacityController jobCapacityController = mock(IJobCapacityController.class);
    IJobManager jobManager = spy(new JobManager(ccConfig, mockClusterControllerService(), jobCapacityController));
    boolean rejected = false;
    // A job should be rejected immediately if its requirement exceeds the maximum capacity of the cluster.
    try {
        JobRun run = mockJobRun(1);
        JobSpecification job = mock(JobSpecification.class);
        when(run.getJobSpecification()).thenReturn(job);
        when(jobCapacityController.allocate(job)).thenThrow(HyracksException.create(ErrorCode.JOB_REQUIREMENTS_EXCEED_CAPACITY, "1", "0"));
        jobManager.add(run);
    } catch (HyracksException e) {
        // Verifies the error code.
        rejected = e.getErrorCode() == ErrorCode.JOB_REQUIREMENTS_EXCEED_CAPACITY;
    }
    Assert.assertTrue(rejected);
    Assert.assertTrue(jobManager.getRunningJobs().isEmpty());
    Assert.assertTrue(jobManager.getPendingJobs().isEmpty());
    Assert.assertTrue(jobManager.getArchivedJobs().size() == 0);
}

Example 5 with IJobCapacityController

use of org.apache.hyracks.api.job.resource.IJobCapacityController in project asterixdb by apache.

the class ClusterControllerService method startApplication.

private void startApplication() throws Exception {
    serviceCtx = new CCServiceContext(this, serverCtx, ccContext, ccConfig.getAppConfig());
    serviceCtx.addJobLifecycleListener(datasetDirectoryService);
    executor = Executors.newCachedThreadPool(serviceCtx.getThreadFactory());
    application.start(serviceCtx, ccConfig.getAppArgsArray());
    IJobCapacityController jobCapacityController = application.getJobCapacityController();
    // Job manager is in charge of job lifecycle management.
    try {
        Constructor<?> jobManagerConstructor = this.getClass().getClassLoader().loadClass(ccConfig.getJobManagerClass()).getConstructor(CCConfig.class, ClusterControllerService.class, IJobCapacityController.class);
        jobManager = (IJobManager) jobManagerConstructor.newInstance(ccConfig, this, jobCapacityController);
    } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
        if (LOGGER.isLoggable(Level.WARNING)) {
            LOGGER.log(Level.WARNING, "class " + ccConfig.getJobManagerClass() + " could not be used: ", e);
        }
        // Falls back to the default implementation if the user-provided class name is not valid.
        jobManager = new JobManager(ccConfig, this, jobCapacityController);
    }
}