Examples with EligibleTaskDescriptorImpl org.ow2.proactive.scheduler.descriptor.EligibleTaskDescriptorImpl used on opensource projects

Example 1 with EligibleTaskDescriptorImpl

use of org.ow2.proactive.scheduler.descriptor.EligibleTaskDescriptorImpl in project scheduling by ow2-proactive.

the class SchedulingMethodImpl method selectAndStartTasks.

private int selectAndStartTasks(Policy currentPolicy, Map<JobId, JobDescriptor> jobMap, Set<String> freeResources, LinkedList<EligibleTaskDescriptor> fullListOfTaskRetrievedFromPolicy) {
    int numberOfTaskStarted = 0;
    VariableBatchSizeIterator progressiveIterator = new VariableBatchSizeIterator(fullListOfTaskRetrievedFromPolicy);
    while (progressiveIterator.hasMoreElements() && !freeResources.isEmpty()) {
        LinkedList<EligibleTaskDescriptor> taskRetrievedFromPolicy = new LinkedList<>(progressiveIterator.getNextElements(freeResources.size()));
        if (logger.isDebugEnabled()) {
            loggingEligibleTasksDetails(fullListOfTaskRetrievedFromPolicy, taskRetrievedFromPolicy);
        }
        updateVariablesForTasksToSchedule(taskRetrievedFromPolicy);
        for (EligibleTaskDescriptor etd : taskRetrievedFromPolicy) {
            // load and Initialize the executable container
            loadAndInit(((EligibleTaskDescriptorImpl) etd).getInternal());
        }
        while (!taskRetrievedFromPolicy.isEmpty()) {
            if (freeResources.isEmpty()) {
                break;
            }
            // get the next compatible tasks from the whole returned policy tasks
            LinkedList<EligibleTaskDescriptor> tasksToSchedule = new LinkedList<>();
            int neededResourcesNumber = 0;
            while (!taskRetrievedFromPolicy.isEmpty() && neededResourcesNumber == 0) {
                // the loop will search for next compatible task until it find something
                neededResourcesNumber = getNextcompatibleTasks(jobMap, taskRetrievedFromPolicy, freeResources.size(), tasksToSchedule);
            }
            if (logger.isDebugEnabled()) {
                logger.debug("tasksToSchedule : " + tasksToSchedule);
            }
            logger.debug("required number of nodes : " + neededResourcesNumber);
            if (neededResourcesNumber == 0 || tasksToSchedule.isEmpty()) {
                break;
            }
            NodeSet nodeSet = getRMNodes(jobMap, neededResourcesNumber, tasksToSchedule, freeResources);
            if (nodeSet != null) {
                freeResources.removeAll(nodeSet.getAllNodesUrls());
            }
            // start selected tasks
            Node node = null;
            InternalJob currentJob = null;
            try {
                while (nodeSet != null && !nodeSet.isEmpty()) {
                    EligibleTaskDescriptor taskDescriptor = tasksToSchedule.removeFirst();
                    currentJob = ((JobDescriptorImpl) jobMap.get(taskDescriptor.getJobId())).getInternal();
                    InternalTask internalTask = ((EligibleTaskDescriptorImpl) taskDescriptor).getInternal();
                    if (currentPolicy.isTaskExecutable(nodeSet, taskDescriptor)) {
                        // create launcher and try to start the task
                        node = nodeSet.get(0);
                        if (createExecution(nodeSet, node, currentJob, internalTask, taskDescriptor)) {
                            numberOfTaskStarted++;
                        }
                    }
                    // if every task that should be launched have been removed
                    if (tasksToSchedule.isEmpty()) {
                        // get back unused nodes to the RManager
                        if (!nodeSet.isEmpty()) {
                            releaseNodes(currentJob, nodeSet);
                            freeResources.addAll(nodeSet.getAllNodesUrls());
                        }
                        // and leave the loop
                        break;
                    }
                }
            } catch (ActiveObjectCreationException e1) {
                // Something goes wrong with the active object creation (createLauncher)
                logger.warn("An exception occured while creating the task launcher.", e1);
                // so try to get back every remaining nodes to the resource manager
                try {
                    releaseNodes(currentJob, nodeSet);
                    freeResources.addAll(nodeSet.getAllNodesUrls());
                } catch (Exception e2) {
                    logger.info("Unable to get back the nodeSet to the RM", e2);
                }
                if (--activeObjectCreationRetryTimeNumber == 0) {
                    break;
                }
            } catch (Exception e1) {
                // if we are here, it is that something append while launching the current task.
                logger.warn("An exception occured while starting task.", e1);
                // so try to get back every remaining nodes to the resource manager
                try {
                    releaseNodes(currentJob, nodeSet);
                    freeResources.addAll(nodeSet.getAllNodesUrls());
                } catch (Exception e2) {
                    logger.info("Unable to get back the nodeSet to the RM", e2);
                }
            }
        }
        if (freeResources.isEmpty()) {
            break;
        }
        if (activeObjectCreationRetryTimeNumber == 0) {
            break;
        }
    }
    return numberOfTaskStarted;
}

Example 2 with EligibleTaskDescriptorImpl

use of org.ow2.proactive.scheduler.descriptor.EligibleTaskDescriptorImpl in project scheduling by ow2-proactive.

the class SchedulingMethodImpl method updateVariablesForTasksToSchedule.

/**
 * Update all variables for the given scheduled tasks
 */
private void updateVariablesForTasksToSchedule(LinkedList<EligibleTaskDescriptor> tasksToSchedule) {
    for (EligibleTaskDescriptor taskDescriptor : tasksToSchedule) {
        InternalTask internalTask = ((EligibleTaskDescriptorImpl) taskDescriptor).getInternal();
        internalTask.updateVariables(schedulingService);
    }
}

Example 3 with EligibleTaskDescriptorImpl

use of org.ow2.proactive.scheduler.descriptor.EligibleTaskDescriptorImpl in project scheduling by ow2-proactive.

the class SchedulingMethodImpl method getNextcompatibleTasks.

/**
 * Extract the n first compatible tasks from the first argument list,
 * and return them according that the extraction is stopped when the maxResource number is reached.<br>
 * Two tasks are compatible if and only if they have the same list of selection script and
 * the same list of node exclusion.
 * The check of compliance is currently done by the {@link SchedulingTaskComparator} class.<br>
 * This method has two side effects : extracted tasks are removed from the bagOfTasks and put in the toFill list
 *
 * @param bagOfTasks the list of tasks form which to extract tasks
 * @param maxResource the limit number of resources that the extraction should not exceed
 * @param toFill the list that will contains the task to schedule at the end. This list must not be null but must be empty.<br>
 * 		  this list will be filled with the n first compatible tasks according that the number of resources needed
 * 		  by these tasks does not exceed the given max resource number.
 * @return the number of nodes needed to start every task present in the 'toFill' argument at the end of the method.
 */
protected int getNextcompatibleTasks(Map<JobId, JobDescriptor> jobsMap, LinkedList<EligibleTaskDescriptor> bagOfTasks, int maxResource, LinkedList<EligibleTaskDescriptor> toFill) {
    if (toFill == null || bagOfTasks == null) {
        throw new IllegalArgumentException("The two given lists must not be null !");
    }
    int neededResource = 0;
    if (!PASchedulerProperties.SCHEDULER_REST_URL.isSet()) {
        Iterator<EligibleTaskDescriptor> it = bagOfTasks.iterator();
        EligibleTaskDescriptor etd;
        while (it.hasNext()) {
            etd = it.next();
            if (checkEligibleTaskDescriptorScript.isTaskContainsAPIBinding(etd)) {
                // skip task here
                it.remove();
            }
        }
    }
    if (maxResource > 0 && !bagOfTasks.isEmpty()) {
        EligibleTaskDescriptor etd = bagOfTasks.removeFirst();
        ((EligibleTaskDescriptorImpl) etd).addAttempt();
        InternalJob currentJob = ((JobDescriptorImpl) jobsMap.get(etd.getJobId())).getInternal();
        InternalTask internalTask = currentJob.getIHMTasks().get(etd.getTaskId());
        int neededNodes = internalTask.getNumberOfNodesNeeded();
        SchedulingTaskComparator referent = new SchedulingTaskComparator(internalTask, currentJob);
        boolean firstLoop = true;
        do {
            if (!firstLoop) {
                // if bagOfTasks is not empty
                if (!bagOfTasks.isEmpty()) {
                    etd = bagOfTasks.removeFirst();
                    ((EligibleTaskDescriptorImpl) etd).addAttempt();
                    currentJob = ((JobDescriptorImpl) jobsMap.get(etd.getJobId())).getInternal();
                    internalTask = currentJob.getIHMTasks().get(etd.getTaskId());
                    neededNodes = internalTask.getNumberOfNodesNeeded();
                }
            } else {
                firstLoop = false;
            }
            if (neededNodes > maxResource) {
            // no instruction is important :
            // in this case, a multi node task leads the search to be stopped and the
            // the current task would be retried on the next step
            // we continue to start the maximum number of task in a single scheduling loop.
            // this case will focus on starting single node task first if lot of resources are busy.
            // (multi-nodes starvation may occurs)
            } else {
                // check if the task is compatible with the other previous one
                if (referent.equals(new SchedulingTaskComparator(internalTask, currentJob))) {
                    tlogger.debug(internalTask.getId(), "scheduling");
                    neededResource += neededNodes;
                    maxResource -= neededNodes;
                    toFill.add(etd);
                } else {
                    bagOfTasks.addFirst(etd);
                    break;
                }
            }
        } while (maxResource > 0 && !bagOfTasks.isEmpty());
    }
    return neededResource;
}

Example 4 with EligibleTaskDescriptorImpl

use of org.ow2.proactive.scheduler.descriptor.EligibleTaskDescriptorImpl in project scheduling by ow2-proactive.

the class TimedDoTaskAction method call.

/**
 * {@inheritDoc}
 */
public Void call() throws Exception {
    try {
        // Set to empty array to emulate varargs behavior (i.e. not defined is
        // equivalent to empty array, not null.
        TaskResult[] params = new TaskResult[0];
        // if job is TASKSFLOW, preparing the list of parameters for this task.
        int resultSize = taskDescriptor.getParents().size();
        if ((job.getType() == JobType.TASKSFLOW) && (resultSize > 0) && task.handleResultsArguments()) {
            Set<TaskId> parentIds = new HashSet<>(resultSize);
            for (int i = 0; i < resultSize; i++) {
                parentIds.addAll(internalTaskParentFinder.getFirstNotSkippedParentTaskIds(((EligibleTaskDescriptorImpl) taskDescriptor.getParents().get(i)).getInternal()));
            }
            params = new TaskResult[parentIds.size()];
            // If parentTaskResults is null after a system failure (a very rare case)
            if (task.getParentTasksResults() == null) {
                Map<TaskId, TaskResult> taskResults = new HashMap<>();
                // Batch fetching of parent tasks results
                for (List<TaskId> parentsSubList : ListUtils.partition(new ArrayList<>(parentIds), PASchedulerProperties.SCHEDULER_DB_FETCH_TASK_RESULTS_BATCH_SIZE.getValueAsInt())) {
                    taskResults.putAll(schedulingService.getInfrastructure().getDBManager().loadTasksResults(job.getId(), parentsSubList));
                }
                // store the parent tasks results in InternalTask for future executions.
                task.setParentTasksResults(taskResults);
            }
            int i = 0;
            for (TaskId taskId : parentIds) {
                params[i] = task.getParentTasksResults().get(taskId);
                i++;
            }
        }
        // activate loggers for this task if needed
        schedulingService.getListenJobLogsSupport().activeLogsIfNeeded(job.getId(), launcher);
        fillContainer();
        // try launch the task
        launcher.doTask(task.getExecutableContainer(), params, terminateNotification, taskRecoveryData.getTerminateNotificationNodeURL(), taskRecoveryData.isTaskRecoverable());
    } catch (Throwable e) {
        logger.warn("Failed to start task: " + e.getMessage(), e);
        restartTask();
    }
    return null;
}

Example 5 with EligibleTaskDescriptorImpl

use of org.ow2.proactive.scheduler.descriptor.EligibleTaskDescriptorImpl in project scheduling by ow2-proactive.

the class JobDescriptorImpl method terminate.

/**
 * Update the eligible list of task and dependencies if necessary.
 * This function considered that the taskId is in eligible task list.
 * Visibility is package because user cannot use this method.
 *
 * @param taskId the task to remove from running task.
 */
public void terminate(TaskId taskId, boolean inErrorTask) {
    Map<TaskId, ? extends TaskDescriptor> currentTasks = inErrorTask ? pausedTasks : runningTasks;
    List<TaskId> taskIdsToSkip = new ArrayList<>();
    if (getInternal().getType() == JobType.TASKSFLOW) {
        TaskDescriptor taskToTerminate = currentTasks.get(taskId);
        if (taskToTerminate != null) {
            for (TaskDescriptor childTask : taskToTerminate.getChildren()) {
                decreaseParentCount(childTask);
                if (((EligibleTaskDescriptorImpl) childTask).getCount() == 0) {
                    if (internalJob.getStatus() == JobStatus.PAUSED) {
                        pausedTasks.put(childTask.getTaskId(), (EligibleTaskDescriptor) childTask);
                    } else if (internalJob.getStatus() == JobStatus.IN_ERROR && ((EligibleTaskDescriptorImpl) childTask).getInternal().getStatus() == TaskStatus.PAUSED) {
                        pausedTasks.put(childTask.getTaskId(), (EligibleTaskDescriptor) childTask);
                    } else if (((EligibleTaskDescriptorImpl) childTask).getInternal().getStatus() == TaskStatus.SKIPPED) {
                        runningTasks.put(childTask.getTaskId(), (EligibleTaskDescriptor) childTask);
                        taskIdsToSkip.add(childTask.getTaskId());
                    } else {
                        eligibleTasks.put(childTask.getTaskId(), (EligibleTaskDescriptor) childTask);
                    }
                }
            }
            decreaseChildrenCountForAllParents(taskToTerminate);
        }
    }
    currentTasks.remove(taskId);
    for (TaskId taskIdToSkip : taskIdsToSkip) {
        terminate(taskIdToSkip);
    }
}