use of org.ow2.proactive.scheduler.descriptor.JobDescriptorImpl in project scheduling by ow2-proactive.
the class SchedulingMethodImpl method selectAndStartTasks.
private int selectAndStartTasks(Policy currentPolicy, Map<JobId, JobDescriptor> jobMap, Set<String> freeResources, LinkedList<EligibleTaskDescriptor> fullListOfTaskRetrievedFromPolicy) {
int numberOfTaskStarted = 0;
VariableBatchSizeIterator progressiveIterator = new VariableBatchSizeIterator(fullListOfTaskRetrievedFromPolicy);
while (progressiveIterator.hasMoreElements() && !freeResources.isEmpty()) {
LinkedList<EligibleTaskDescriptor> taskRetrievedFromPolicy = new LinkedList<>(progressiveIterator.getNextElements(freeResources.size()));
if (logger.isDebugEnabled()) {
loggingEligibleTasksDetails(fullListOfTaskRetrievedFromPolicy, taskRetrievedFromPolicy);
}
updateVariablesForTasksToSchedule(taskRetrievedFromPolicy);
for (EligibleTaskDescriptor etd : taskRetrievedFromPolicy) {
// load and Initialize the executable container
loadAndInit(((EligibleTaskDescriptorImpl) etd).getInternal());
}
while (!taskRetrievedFromPolicy.isEmpty()) {
if (freeResources.isEmpty()) {
break;
}
// get the next compatible tasks from the whole returned policy tasks
LinkedList<EligibleTaskDescriptor> tasksToSchedule = new LinkedList<>();
int neededResourcesNumber = 0;
while (!taskRetrievedFromPolicy.isEmpty() && neededResourcesNumber == 0) {
// the loop will search for next compatible task until it find something
neededResourcesNumber = getNextcompatibleTasks(jobMap, taskRetrievedFromPolicy, freeResources.size(), tasksToSchedule);
}
if (logger.isDebugEnabled()) {
logger.debug("tasksToSchedule : " + tasksToSchedule);
}
logger.debug("required number of nodes : " + neededResourcesNumber);
if (neededResourcesNumber == 0 || tasksToSchedule.isEmpty()) {
break;
}
NodeSet nodeSet = getRMNodes(jobMap, neededResourcesNumber, tasksToSchedule, freeResources);
if (nodeSet != null) {
freeResources.removeAll(nodeSet.getAllNodesUrls());
}
// start selected tasks
Node node = null;
InternalJob currentJob = null;
try {
while (nodeSet != null && !nodeSet.isEmpty()) {
EligibleTaskDescriptor taskDescriptor = tasksToSchedule.removeFirst();
currentJob = ((JobDescriptorImpl) jobMap.get(taskDescriptor.getJobId())).getInternal();
InternalTask internalTask = ((EligibleTaskDescriptorImpl) taskDescriptor).getInternal();
if (currentPolicy.isTaskExecutable(nodeSet, taskDescriptor)) {
// create launcher and try to start the task
node = nodeSet.get(0);
if (createExecution(nodeSet, node, currentJob, internalTask, taskDescriptor)) {
numberOfTaskStarted++;
}
}
// if every task that should be launched have been removed
if (tasksToSchedule.isEmpty()) {
// get back unused nodes to the RManager
if (!nodeSet.isEmpty()) {
releaseNodes(currentJob, nodeSet);
freeResources.addAll(nodeSet.getAllNodesUrls());
}
// and leave the loop
break;
}
}
} catch (ActiveObjectCreationException e1) {
// Something goes wrong with the active object creation (createLauncher)
logger.warn("An exception occured while creating the task launcher.", e1);
// so try to get back every remaining nodes to the resource manager
try {
releaseNodes(currentJob, nodeSet);
freeResources.addAll(nodeSet.getAllNodesUrls());
} catch (Exception e2) {
logger.info("Unable to get back the nodeSet to the RM", e2);
}
if (--activeObjectCreationRetryTimeNumber == 0) {
break;
}
} catch (Exception e1) {
// if we are here, it is that something append while launching the current task.
logger.warn("An exception occured while starting task.", e1);
// so try to get back every remaining nodes to the resource manager
try {
releaseNodes(currentJob, nodeSet);
freeResources.addAll(nodeSet.getAllNodesUrls());
} catch (Exception e2) {
logger.info("Unable to get back the nodeSet to the RM", e2);
}
}
}
if (freeResources.isEmpty()) {
break;
}
if (activeObjectCreationRetryTimeNumber == 0) {
break;
}
}
return numberOfTaskStarted;
}
use of org.ow2.proactive.scheduler.descriptor.JobDescriptorImpl in project scheduling by ow2-proactive.
the class SchedulingMethodImpl method getNextcompatibleTasks.
/**
* Extract the n first compatible tasks from the first argument list,
* and return them according that the extraction is stopped when the maxResource number is reached.<br>
* Two tasks are compatible if and only if they have the same list of selection script and
* the same list of node exclusion.
* The check of compliance is currently done by the {@link SchedulingTaskComparator} class.<br>
* This method has two side effects : extracted tasks are removed from the bagOfTasks and put in the toFill list
*
* @param bagOfTasks the list of tasks form which to extract tasks
* @param maxResource the limit number of resources that the extraction should not exceed
* @param toFill the list that will contains the task to schedule at the end. This list must not be null but must be empty.<br>
* this list will be filled with the n first compatible tasks according that the number of resources needed
* by these tasks does not exceed the given max resource number.
* @return the number of nodes needed to start every task present in the 'toFill' argument at the end of the method.
*/
protected int getNextcompatibleTasks(Map<JobId, JobDescriptor> jobsMap, LinkedList<EligibleTaskDescriptor> bagOfTasks, int maxResource, LinkedList<EligibleTaskDescriptor> toFill) {
if (toFill == null || bagOfTasks == null) {
throw new IllegalArgumentException("The two given lists must not be null !");
}
int neededResource = 0;
if (!PASchedulerProperties.SCHEDULER_REST_URL.isSet()) {
Iterator<EligibleTaskDescriptor> it = bagOfTasks.iterator();
EligibleTaskDescriptor etd;
while (it.hasNext()) {
etd = it.next();
if (checkEligibleTaskDescriptorScript.isTaskContainsAPIBinding(etd)) {
// skip task here
it.remove();
}
}
}
if (maxResource > 0 && !bagOfTasks.isEmpty()) {
EligibleTaskDescriptor etd = bagOfTasks.removeFirst();
((EligibleTaskDescriptorImpl) etd).addAttempt();
InternalJob currentJob = ((JobDescriptorImpl) jobsMap.get(etd.getJobId())).getInternal();
InternalTask internalTask = currentJob.getIHMTasks().get(etd.getTaskId());
int neededNodes = internalTask.getNumberOfNodesNeeded();
SchedulingTaskComparator referent = new SchedulingTaskComparator(internalTask, currentJob);
boolean firstLoop = true;
do {
if (!firstLoop) {
// if bagOfTasks is not empty
if (!bagOfTasks.isEmpty()) {
etd = bagOfTasks.removeFirst();
((EligibleTaskDescriptorImpl) etd).addAttempt();
currentJob = ((JobDescriptorImpl) jobsMap.get(etd.getJobId())).getInternal();
internalTask = currentJob.getIHMTasks().get(etd.getTaskId());
neededNodes = internalTask.getNumberOfNodesNeeded();
}
} else {
firstLoop = false;
}
if (neededNodes > maxResource) {
// no instruction is important :
// in this case, a multi node task leads the search to be stopped and the
// the current task would be retried on the next step
// we continue to start the maximum number of task in a single scheduling loop.
// this case will focus on starting single node task first if lot of resources are busy.
// (multi-nodes starvation may occurs)
} else {
// check if the task is compatible with the other previous one
if (referent.equals(new SchedulingTaskComparator(internalTask, currentJob))) {
tlogger.debug(internalTask.getId(), "scheduling");
neededResource += neededNodes;
maxResource -= neededNodes;
toFill.add(etd);
} else {
bagOfTasks.addFirst(etd);
break;
}
}
} while (maxResource > 0 && !bagOfTasks.isEmpty());
}
return neededResource;
}
use of org.ow2.proactive.scheduler.descriptor.JobDescriptorImpl in project scheduling by ow2-proactive.
the class SchedulingServiceTest2 method testFailedTask.
private void testFailedTask(boolean failNativeTask) throws Exception {
service.submitJob(createJob(createTestJob()));
listener.assertEvents(SchedulerEvent.JOB_SUBMITTED);
Map<JobId, JobDescriptor> jobsMap;
JobDescriptor jobDesc;
jobsMap = service.lockJobsToSchedule();
assertEquals(1, jobsMap.size());
jobDesc = jobsMap.values().iterator().next();
Assert.assertEquals(2, jobDesc.getEligibleTasks().size());
for (TaskDescriptor taskDesc : jobDesc.getEligibleTasks()) {
taskStarted(jobDesc, (EligibleTaskDescriptor) taskDesc);
}
service.unlockJobsToSchedule(jobsMap.values());
if (failNativeTask) {
InternalTask nativeTask = ((JobDescriptorImpl) jobDesc).getInternal().getTask("nativeTask");
// native task terminates with code 1, flag 'cancelJobOnError' was
// set so job should be cancelled
service.taskTerminatedWithResult(nativeTask.getId(), new TaskResultImpl(nativeTask.getId(), new RuntimeException(), null, 0));
} else {
InternalTask javaTask = ((JobDescriptorImpl) jobDesc).getInternal().getTask("javaTask");
// java task terminates with exception, flag 'cancelJobOnError' was
// set so job should be cancelled
service.taskTerminatedWithResult(javaTask.getId(), new TaskResultImpl(javaTask.getId(), new RuntimeException(), null, 0));
}
jobsMap = service.lockJobsToSchedule();
assertEquals(0, jobsMap.size());
listener.assertEvents(SchedulerEvent.JOB_PENDING_TO_RUNNING, SchedulerEvent.JOB_UPDATED, SchedulerEvent.TASK_PENDING_TO_RUNNING, SchedulerEvent.TASK_PENDING_TO_RUNNING, SchedulerEvent.TASK_RUNNING_TO_FINISHED, SchedulerEvent.TASK_RUNNING_TO_FINISHED, SchedulerEvent.JOB_RUNNING_TO_FINISHED, SchedulerEvent.JOB_UPDATED);
infrastructure.assertRequests(2);
}
use of org.ow2.proactive.scheduler.descriptor.JobDescriptorImpl in project scheduling by ow2-proactive.
the class SchedulingServiceTest3 method testTaskKill.
@Test
public void testTaskKill() throws Exception {
service.submitJob(createJob(createTestJob(false)));
listener.assertEvents(SchedulerEvent.JOB_SUBMITTED);
Map<JobId, JobDescriptor> jobsMap;
JobDescriptor jobDesc;
jobsMap = service.lockJobsToSchedule();
assertEquals(1, jobsMap.size());
jobDesc = jobsMap.values().iterator().next();
Assert.assertEquals(2, jobDesc.getEligibleTasks().size());
for (TaskDescriptor taskDesc : jobDesc.getEligibleTasks()) {
taskStarted(jobDesc, (EligibleTaskDescriptor) taskDesc);
}
service.unlockJobsToSchedule(jobsMap.values());
try {
service.killTask(jobDesc.getJobId(), "invalid task name");
Assert.fail();
} catch (UnknownTaskException e) {
}
try {
service.killTask(JobIdImpl.makeJobId("1234567"), "javaTask");
Assert.fail();
} catch (UnknownJobException e) {
}
Assert.assertTrue(service.killTask(jobDesc.getJobId(), "javaTask"));
listener.assertEvents(SchedulerEvent.JOB_PENDING_TO_RUNNING, SchedulerEvent.JOB_UPDATED, SchedulerEvent.TASK_PENDING_TO_RUNNING, SchedulerEvent.TASK_PENDING_TO_RUNNING, SchedulerEvent.TASK_RUNNING_TO_FINISHED);
infrastructure.assertRequests(1);
Assert.assertFalse(service.killTask(jobDesc.getJobId(), "javaTask"));
TaskId nativeTaskId = ((JobDescriptorImpl) jobDesc).getInternal().getTask("nativeTask").getId();
service.taskTerminatedWithResult(nativeTaskId, new TaskResultImpl(nativeTaskId, new Integer(0), null, 0));
listener.assertEvents(SchedulerEvent.TASK_RUNNING_TO_FINISHED, SchedulerEvent.JOB_RUNNING_TO_FINISHED, SchedulerEvent.JOB_UPDATED);
infrastructure.assertRequests(1);
try {
service.killTask(jobDesc.getJobId(), "javaTask");
} catch (UnknownJobException e) {
Assert.fail("The job should still exist in the memory context as the auto Job removal feature isn't enabled");
}
}
use of org.ow2.proactive.scheduler.descriptor.JobDescriptorImpl in project scheduling by ow2-proactive.
the class TaskResultCreatorTest method testPausedTasksAreCheckedForATaskId.
@Test
public void testPausedTasksAreCheckedForATaskId() throws UnknownTaskException {
TaskResultCreator taskResultCreator = new TaskResultCreator();
TaskResultImpl mockedTaskResultImpl = mock(TaskResultImpl.class);
Map<TaskId, TaskResult> loadTaskResultsValue = new HashMap<>();
loadTaskResultsValue.put(this.createTaskID(), mockedTaskResultImpl);
SchedulerDBManager mockedschedulerDbManager = mock(SchedulerDBManager.class);
when(mockedschedulerDbManager.loadTasksResults(any(JobId.class), any(List.class))).thenReturn(loadTaskResultsValue);
JobDescriptorImpl mockedJobDescriptorHasPausedTask = this.getMockedJobDescriptorWithPausedTask();
taskResultCreator.getTaskResult(mockedschedulerDbManager, this.getMockedInternalJob(mockedJobDescriptorHasPausedTask), this.getMockedInternalTask());
verify(mockedJobDescriptorHasPausedTask, atLeastOnce()).getPausedTasks();
}
Aggregations