Examples with JobDescriptor org.ow2.proactive.scheduler.common.JobDescriptor used on opensource projects

Example 1 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class SchedulingMethodImpl method selectAndStartTasks.

private int selectAndStartTasks(Policy currentPolicy, Map<JobId, JobDescriptor> jobMap, Set<String> freeResources, LinkedList<EligibleTaskDescriptor> fullListOfTaskRetrievedFromPolicy) {
    int numberOfTaskStarted = 0;
    VariableBatchSizeIterator progressiveIterator = new VariableBatchSizeIterator(fullListOfTaskRetrievedFromPolicy);
    while (progressiveIterator.hasMoreElements() && !freeResources.isEmpty()) {
        LinkedList<EligibleTaskDescriptor> taskRetrievedFromPolicy = new LinkedList<>(progressiveIterator.getNextElements(freeResources.size()));
        if (logger.isDebugEnabled()) {
            loggingEligibleTasksDetails(fullListOfTaskRetrievedFromPolicy, taskRetrievedFromPolicy);
        }
        updateVariablesForTasksToSchedule(taskRetrievedFromPolicy);
        for (EligibleTaskDescriptor etd : taskRetrievedFromPolicy) {
            // load and Initialize the executable container
            loadAndInit(((EligibleTaskDescriptorImpl) etd).getInternal());
        }
        while (!taskRetrievedFromPolicy.isEmpty()) {
            if (freeResources.isEmpty()) {
                break;
            }
            // get the next compatible tasks from the whole returned policy tasks
            LinkedList<EligibleTaskDescriptor> tasksToSchedule = new LinkedList<>();
            int neededResourcesNumber = 0;
            while (!taskRetrievedFromPolicy.isEmpty() && neededResourcesNumber == 0) {
                // the loop will search for next compatible task until it find something
                neededResourcesNumber = getNextcompatibleTasks(jobMap, taskRetrievedFromPolicy, freeResources.size(), tasksToSchedule);
            }
            if (logger.isDebugEnabled()) {
                logger.debug("tasksToSchedule : " + tasksToSchedule);
            }
            logger.debug("required number of nodes : " + neededResourcesNumber);
            if (neededResourcesNumber == 0 || tasksToSchedule.isEmpty()) {
                break;
            }
            NodeSet nodeSet = getRMNodes(jobMap, neededResourcesNumber, tasksToSchedule, freeResources);
            if (nodeSet != null) {
                freeResources.removeAll(nodeSet.getAllNodesUrls());
            }
            // start selected tasks
            Node node = null;
            InternalJob currentJob = null;
            try {
                while (nodeSet != null && !nodeSet.isEmpty()) {
                    EligibleTaskDescriptor taskDescriptor = tasksToSchedule.removeFirst();
                    currentJob = ((JobDescriptorImpl) jobMap.get(taskDescriptor.getJobId())).getInternal();
                    InternalTask internalTask = ((EligibleTaskDescriptorImpl) taskDescriptor).getInternal();
                    if (currentPolicy.isTaskExecutable(nodeSet, taskDescriptor)) {
                        // create launcher and try to start the task
                        node = nodeSet.get(0);
                        if (createExecution(nodeSet, node, currentJob, internalTask, taskDescriptor)) {
                            numberOfTaskStarted++;
                        }
                    }
                    // if every task that should be launched have been removed
                    if (tasksToSchedule.isEmpty()) {
                        // get back unused nodes to the RManager
                        if (!nodeSet.isEmpty()) {
                            releaseNodes(currentJob, nodeSet);
                            freeResources.addAll(nodeSet.getAllNodesUrls());
                        }
                        // and leave the loop
                        break;
                    }
                }
            } catch (ActiveObjectCreationException e1) {
                // Something goes wrong with the active object creation (createLauncher)
                logger.warn("An exception occured while creating the task launcher.", e1);
                // so try to get back every remaining nodes to the resource manager
                try {
                    releaseNodes(currentJob, nodeSet);
                    freeResources.addAll(nodeSet.getAllNodesUrls());
                } catch (Exception e2) {
                    logger.info("Unable to get back the nodeSet to the RM", e2);
                }
                if (--activeObjectCreationRetryTimeNumber == 0) {
                    break;
                }
            } catch (Exception e1) {
                // if we are here, it is that something append while launching the current task.
                logger.warn("An exception occured while starting task.", e1);
                // so try to get back every remaining nodes to the resource manager
                try {
                    releaseNodes(currentJob, nodeSet);
                    freeResources.addAll(nodeSet.getAllNodesUrls());
                } catch (Exception e2) {
                    logger.info("Unable to get back the nodeSet to the RM", e2);
                }
            }
        }
        if (freeResources.isEmpty()) {
            break;
        }
        if (activeObjectCreationRetryTimeNumber == 0) {
            break;
        }
    }
    return numberOfTaskStarted;
}

Example 2 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class SchedulingMethodImpl method getNextcompatibleTasks.

/**
 * Extract the n first compatible tasks from the first argument list,
 * and return them according that the extraction is stopped when the maxResource number is reached.<br>
 * Two tasks are compatible if and only if they have the same list of selection script and
 * the same list of node exclusion.
 * The check of compliance is currently done by the {@link SchedulingTaskComparator} class.<br>
 * This method has two side effects : extracted tasks are removed from the bagOfTasks and put in the toFill list
 *
 * @param bagOfTasks the list of tasks form which to extract tasks
 * @param maxResource the limit number of resources that the extraction should not exceed
 * @param toFill the list that will contains the task to schedule at the end. This list must not be null but must be empty.<br>
 * 		  this list will be filled with the n first compatible tasks according that the number of resources needed
 * 		  by these tasks does not exceed the given max resource number.
 * @return the number of nodes needed to start every task present in the 'toFill' argument at the end of the method.
 */
protected int getNextcompatibleTasks(Map<JobId, JobDescriptor> jobsMap, LinkedList<EligibleTaskDescriptor> bagOfTasks, int maxResource, LinkedList<EligibleTaskDescriptor> toFill) {
    if (toFill == null || bagOfTasks == null) {
        throw new IllegalArgumentException("The two given lists must not be null !");
    }
    int neededResource = 0;
    if (!PASchedulerProperties.SCHEDULER_REST_URL.isSet()) {
        Iterator<EligibleTaskDescriptor> it = bagOfTasks.iterator();
        EligibleTaskDescriptor etd;
        while (it.hasNext()) {
            etd = it.next();
            if (checkEligibleTaskDescriptorScript.isTaskContainsAPIBinding(etd)) {
                // skip task here
                it.remove();
            }
        }
    }
    if (maxResource > 0 && !bagOfTasks.isEmpty()) {
        EligibleTaskDescriptor etd = bagOfTasks.removeFirst();
        ((EligibleTaskDescriptorImpl) etd).addAttempt();
        InternalJob currentJob = ((JobDescriptorImpl) jobsMap.get(etd.getJobId())).getInternal();
        InternalTask internalTask = currentJob.getIHMTasks().get(etd.getTaskId());
        int neededNodes = internalTask.getNumberOfNodesNeeded();
        SchedulingTaskComparator referent = new SchedulingTaskComparator(internalTask, currentJob);
        boolean firstLoop = true;
        do {
            if (!firstLoop) {
                // if bagOfTasks is not empty
                if (!bagOfTasks.isEmpty()) {
                    etd = bagOfTasks.removeFirst();
                    ((EligibleTaskDescriptorImpl) etd).addAttempt();
                    currentJob = ((JobDescriptorImpl) jobsMap.get(etd.getJobId())).getInternal();
                    internalTask = currentJob.getIHMTasks().get(etd.getTaskId());
                    neededNodes = internalTask.getNumberOfNodesNeeded();
                }
            } else {
                firstLoop = false;
            }
            if (neededNodes > maxResource) {
            // no instruction is important :
            // in this case, a multi node task leads the search to be stopped and the
            // the current task would be retried on the next step
            // we continue to start the maximum number of task in a single scheduling loop.
            // this case will focus on starting single node task first if lot of resources are busy.
            // (multi-nodes starvation may occurs)
            } else {
                // check if the task is compatible with the other previous one
                if (referent.equals(new SchedulingTaskComparator(internalTask, currentJob))) {
                    tlogger.debug(internalTask.getId(), "scheduling");
                    neededResource += neededNodes;
                    maxResource -= neededNodes;
                    toFill.add(etd);
                } else {
                    bagOfTasks.addFirst(etd);
                    break;
                }
            }
        } while (maxResource > 0 && !bagOfTasks.isEmpty());
    }
    return neededResource;
}

Example 3 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class TaskResultCreator method getTaskResult.

public TaskResultImpl getTaskResult(SchedulerDBManager dbManager, InternalJob job, InternalTask task, Throwable exception, TaskLogs output) throws UnknownTaskException {
    if (task == null) {
        throw new UnknownTaskException();
    }
    JobDescriptor jobDescriptor = job.getJobDescriptor();
    EligibleTaskDescriptor eligibleTaskDescriptor = null;
    if (jobDescriptor.getPausedTasks().get(task.getId()) != null) {
        eligibleTaskDescriptor = (EligibleTaskDescriptor) jobDescriptor.getPausedTasks().get(task.getId());
    } else if (jobDescriptor.getRunningTasks().get(task.getId()) != null) {
        eligibleTaskDescriptor = (EligibleTaskDescriptor) jobDescriptor.getRunningTasks().get(task.getId());
    }
    TaskResultImpl taskResult = getEmptyTaskResult(task, exception, output);
    taskResult.setPropagatedVariables(getPropagatedVariables(dbManager, eligibleTaskDescriptor, job, task));
    return taskResult;
}

Example 4 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class JobDescriptorImpl method doLoop.

/**
 * Complete LOOP action on JobDescriptor side
 *
 * @param initiator Task initiating the LOOP action
 * @param tree InternalTask tree of replicated tasks
 * @param target Target task of the LOOP action
 */
public void doLoop(TaskId initiator, Map<TaskId, InternalTask> tree, InternalTask target, InternalTask newInit) {
    Map<TaskId, EligibleTaskDescriptorImpl> acc = new HashMap<>();
    // create new EligibleTasks and accumulate it
    for (Entry<TaskId, InternalTask> it : tree.entrySet()) {
        TaskId itId = it.getValue().getId();
        EligibleTaskDescriptorImpl td = new EligibleTaskDescriptorImpl(it.getValue());
        acc.put(itId, td);
    }
    EligibleTaskDescriptorImpl oldEnd = (EligibleTaskDescriptorImpl) runningTasks.get(initiator);
    EligibleTaskDescriptorImpl newStart = acc.get(target.getId());
    EligibleTaskDescriptorImpl newEnd = acc.get(newInit.getId());
    // plug the end of the old tree (initiator) to the beginning of the new (target)
    for (TaskDescriptor ot : oldEnd.getChildren()) {
        newEnd.addChild(ot);
        ot.getParents().remove(oldEnd);
        ot.getParents().add(newEnd);
    }
    oldEnd.clearChildren();
    // recreate the dependencies
    for (Entry<TaskId, InternalTask> it : tree.entrySet()) {
        TaskId itId = it.getValue().getTaskInfo().getTaskId();
        EligibleTaskDescriptorImpl down = acc.get(itId);
        List<InternalTask> ideps = new ArrayList<>();
        int deptype = 0;
        if (it.getValue().hasDependences()) {
            ideps.addAll(it.getValue().getIDependences());
        }
        if (it.getValue().getIfBranch() != null) {
            deptype = 1;
            ideps.add(it.getValue().getIfBranch());
        }
        if (it.getValue().getJoinedBranches() != null) {
            deptype = 2;
            ideps.addAll(it.getValue().getJoinedBranches());
        }
        if (ideps.size() > 0 && !target.equals(itId)) {
            for (InternalTask parent : ideps) {
                if (parent == null) {
                    continue;
                }
                EligibleTaskDescriptorImpl up = acc.get(parent.getTaskInfo().getTaskId());
                switch(deptype) {
                    case 0:
                        if (parent.getId().equals(initiator)) {
                            up = (EligibleTaskDescriptorImpl) runningTasks.get(initiator);
                        }
                        up.addChild(down);
                        down.addParent(up);
                        break;
                    case 1:
                    case 2:
                        // 'weak' dependencies from FlowAction#IF are not
                        // represented in TaskDescriptor
                        branchTasks.put(down.getTaskId(), down);
                        break;
                }
            }
        }
    }
    // EligibleTaskDescriptorImpl newTask = (EligibleTaskDescriptorImpl) acc.get(target.getId());
    setNewLoopTaskToPausedIfJobIsPaused(newStart);
    putNewLoopTaskIntoPausedOrEligableList(target.getId(), newStart);
    runningTasks.remove(initiator);
}

Example 5 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class JobDescriptorImpl method doIf.

/**
 * Complete IF action on JobDescriptor side
 *
 * @param initiator Task initiating the IF action
 * @param branchStart START task of the IF branch
 * @param branchEnd END task of the IF branch
 * @param ifJoin JOIN task of the IF action, or null
 * @param elseTarget the START task of the ELSE branch that will not be executed
 * @param elseTasks list of tasks contained in the not executed ELSE branch
 */
public void doIf(TaskId initiator, TaskId branchStart, TaskId branchEnd, TaskId ifJoin, TaskId elseTarget, List<InternalTask> elseTasks) {
    EligibleTaskDescriptorImpl init = (EligibleTaskDescriptorImpl) runningTasks.get(initiator);
    EligibleTaskDescriptorImpl start = (EligibleTaskDescriptorImpl) branchTasks.get(branchStart);
    EligibleTaskDescriptorImpl end = null;
    EligibleTaskDescriptorImpl join = null;
    if (ifJoin != null) {
        join = (EligibleTaskDescriptorImpl) branchTasks.get(ifJoin);
    }
    // plug the initiator with the beginning of the IF block
    init.addChild(start);
    start.addParent(init);
    // the join task is optional
    if (join != null) {
        for (EligibleTaskDescriptor td : branchTasks.values()) {
            LinkedList<EligibleTaskDescriptorImpl> q = new LinkedList<>();
            q.offer((EligibleTaskDescriptorImpl) td);
            // find the matching end block task
            do {
                EligibleTaskDescriptorImpl ptr = q.poll();
                // if (ptr.getChildren() == null || ptr.getChildren().size() == 0) {
                if (ptr.getTaskId().equals(branchEnd)) {
                    end = ptr;
                    break;
                } else {
                    for (TaskDescriptor desc : ptr.getChildren()) {
                        if (!q.contains(desc)) {
                            q.offer((EligibleTaskDescriptorImpl) desc);
                        }
                    }
                }
            } while (q.size() > 0);
            if (end != null) {
                break;
            }
        }
        // plug the join task with the end of the if block
        join.addParent(end);
        end.addChild(join);
    }
    branchTasks.remove(start);
    if (join != null) {
        branchTasks.remove(join);
    }
    for (InternalTask it : elseTasks) {
        EligibleTaskDescriptorImpl td = (EligibleTaskDescriptorImpl) branchTasks.remove(it.getId());
        LinkedList<EligibleTaskDescriptorImpl> q = new LinkedList<>();
        if (td != null) {
            q.clear();
            q.offer(td);
            while (q.size() > 0) {
                EligibleTaskDescriptorImpl ptr = q.poll();
                ptr.setChildrenCount(0);
                ptr.setCount(0);
                if (ptr.getChildren() != null) {
                    for (TaskDescriptor child : ptr.getChildren()) {
                        q.offer((EligibleTaskDescriptorImpl) child);
                    }
                }
            }
        }
    }
}