Examples with JobDescriptor org.ow2.proactive.scheduler.common.JobDescriptor used on opensource projects

Example 26 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class LiveJobs method lockJobsToSchedule.

Map<JobId, JobDescriptor> lockJobsToSchedule() {
    TreeSet<JobPriority> prioritiesScheduled = new TreeSet<>();
    TreeSet<JobPriority> prioritiesNotScheduled = new TreeSet<>();
    Map<JobId, JobDescriptor> result = new HashMap<>();
    for (Map.Entry<JobId, JobData> entry : jobs.entrySet()) {
        JobData value = entry.getValue();
        if (value.jobLock.tryLock()) {
            InternalJob job = entry.getValue().job;
            result.put(job.getId(), job.getJobDescriptor());
            prioritiesScheduled.add(job.getPriority());
            if (unlockIfConflict(prioritiesScheduled, prioritiesNotScheduled, result))
                return new HashMap<>(0);
        } else {
            prioritiesNotScheduled.add(value.job.getPriority());
            if (unlockIfConflict(prioritiesScheduled, prioritiesNotScheduled, result))
                return new HashMap<>(0);
        }
    }
    return result;
}

Example 27 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class ExtendedSchedulerPolicy method getOrderedTasks.

/*
     * Utilize 'startAt' generic info and filter any tasks that should not be scheduled for current
     * execution cycle.
     */
@Override
public LinkedList<EligibleTaskDescriptor> getOrderedTasks(List<JobDescriptor> jobDescList) {
    Date now = new Date();
    LinkedList<EligibleTaskDescriptor> executionCycleTasks = new LinkedList<>();
    Collections.sort(jobDescList, FIFO_BY_PRIORITY_COMPARATOR);
    for (JobDescriptor jobDesc : jobDescList) {
        Collection<TaskDescriptor> tasks = jobDesc.getEligibleTasks();
        Collection<EligibleTaskDescriptor> eligibleTasks = (Collection) tasks;
        for (EligibleTaskDescriptor candidate : eligibleTasks) {
            String startAt = getStartAtValue(jobDesc, candidate);
            if (startAt == null) {
                executionCycleTasks.add(candidate);
            } else {
                try {
                    if (now.after(ISO8601DateUtil.toDate(startAt))) {
                        executionCycleTasks.add(candidate);
                    } else {
                        if (logger.isTraceEnabled()) {
                            logger.trace(String.format("Task [jobId:\"%s\", taskId:\"%s\"] is scheduled to be executed at %s." + " It will not be scheduled for this execution cycle at %s.", jobDesc.getJobId(), candidate.getTaskId(), startAt, ISO8601DateUtil.parse(now)));
                        }
                    }
                } catch (IllegalArgumentException e) {
                    logger.error(String.format("An error occurred while processing 'startAt' generic info.%n" + "Task ([job-id:\"%s\", task-id:\"%s\"]) will be scheduled immediately for execution.", jobDesc.getJobId().toString(), candidate.getTaskId().toString()), e);
                    executionCycleTasks.add(candidate);
                }
            }
        }
    }
    return executionCycleTasks;
}

Example 28 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class SchedulingMethodImpl method schedule.

/**
 * Scheduling process. For this implementation, steps are :<br>
 * <ul>
 * 	<li>Select running and pending jobs to be scheduled
 * 	<li>Get an ordered list of the selected tasks to be scheduled
 * 	<li>While returned tasks list is not empty :
 * 		<ul>
 * 			<li>Get n first compatible tasks (same selection script, same node exclusion)
 * 			<li>Ask nodes to RM according to the previous specification
 * 			<li>Try to start each tasks
 * 			<li>Job started event if needed
 * 			<li>Task started event
 * 		</ul>
 * 	<li>Manage exception while deploying tasks on nodes
 * </ul>
 *
 * @return the number of tasks that have been started
 */
public int schedule() {
    Policy currentPolicy = schedulingService.getPolicy();
    // Number of time to retry an active object creation before leaving scheduling loop
    activeObjectCreationRetryTimeNumber = ACTIVEOBJECT_CREATION_RETRY_TIME_NUMBER;
    // get job Descriptor list with eligible jobs (running and pending)
    Map<JobId, JobDescriptor> jobMap = schedulingService.lockJobsToSchedule();
    Map<JobId, JobDescriptor> toUnlock = jobMap;
    logSelectedJobs(jobMap);
    // we wait for next scheduling loop and don't start any task
    if (jobMap.isEmpty()) {
        return 0;
    }
    return startTasks(currentPolicy, jobMap, toUnlock);
}

Example 29 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class SchedulingMethodImpl method getRMNodes.

/**
 * Ask to the RM the given number of node resources.<br>
 * If there is a problem with these task selection (such as bad selectionScript) this method
 * will terminate the corresponding tasks and jobs. As the selection scripts contain errors, the task
 * and its surrounding jobs must be stopped.
 *
 * @param neededResourcesNumber the number of resources to ask for (must be &gt; 0).
 * @param tasksToSchedule the task to be scheduled
 * @return A nodeSet that contains at most 'neededResourcesNumber' available compatible resources.
 * 		   An empty nodeSet if no nodes could be found
 * 		   null if the their was an exception when asking for the nodes (ie : selection script has failed)
 */
protected NodeSet getRMNodes(Map<JobId, JobDescriptor> jobMap, int neededResourcesNumber, LinkedList<EligibleTaskDescriptor> tasksToSchedule, Set<String> freeResources) {
    NodeSet nodeSet;
    if (neededResourcesNumber <= 0) {
        throw new IllegalArgumentException("'neededResourcesNumber' must be greater than 0");
    }
    EligibleTaskDescriptor etd = tasksToSchedule.getFirst();
    InternalJob currentJob = ((JobDescriptorImpl) jobMap.get(etd.getJobId())).getInternal();
    InternalTask internalTask0 = currentJob.getIHMTasks().get(etd.getTaskId());
    try {
        TopologyDescriptor descriptor = null;
        boolean bestEffort = true;
        if (internalTask0.isParallel()) {
            descriptor = internalTask0.getParallelEnvironment().getTopologyDescriptor();
            bestEffort = false;
            if (descriptor == null) {
                logger.debug("Topology is not defined for the task " + internalTask0.getName());
            }
        }
        if (descriptor == null) {
            // descriptor is not defined, use default
            descriptor = TopologyDescriptor.ARBITRARY;
        }
        try {
            Criteria criteria = new Criteria(neededResourcesNumber);
            criteria.setTopology(descriptor);
            // resolve script variables (if any) in the list of selection
            // scripts and then set it as the selection criteria.
            criteria.setScripts(resolveScriptVariables(internalTask0.getSelectionScripts(), internalTask0.getRuntimeVariables()));
            criteria.setBlackList(internalTask0.getNodeExclusion());
            criteria.setBestEffort(bestEffort);
            criteria.setAcceptableNodesUrls(freeResources);
            criteria.setBindings(createBindingsForSelectionScripts(currentJob, internalTask0));
            if (internalTask0.getRuntimeGenericInformation().containsKey(SchedulerConstants.NODE_ACCESS_TOKEN)) {
                criteria.setNodeAccessToken(internalTask0.getRuntimeGenericInformation().get(SchedulerConstants.NODE_ACCESS_TOKEN));
            }
            Collection<String> computationDescriptors = new ArrayList<>(tasksToSchedule.size());
            for (EligibleTaskDescriptor task : tasksToSchedule) {
                computationDescriptors.add(TaskLogger.getTaskLogRelativePath(task.getTaskId()));
            }
            criteria.setComputationDescriptors(computationDescriptors);
            nodeSet = getRMProxiesManager().getUserRMProxy(currentJob.getOwner(), currentJob.getCredentials()).getNodes(criteria);
        } catch (TopologyDisabledException tde) {
            jlogger.warn(currentJob.getId(), "will be canceled as the topology is disabled");
            schedulingService.simulateJobStartAndCancelIt(tasksToSchedule, "Topology is disabled");
            return null;
        }
        // the following line is used to unwrap the future, warning when moving or removing
        // it may also throw a ScriptException which is a RuntimeException
        PAFuture.waitFor(nodeSet, true);
        logger.debug("provided nodes " + nodeSet.size());
        return nodeSet;
    } catch (IOException | ClassNotFoundException e) {
        logger.warn("Failed to deserialize previous task variables before selection for task " + internalTask0.getId().toString(), e);
        schedulingService.simulateJobStartAndCancelIt(tasksToSchedule, "Failed to deserialize previous task variables before selection for task " + internalTask0.getId().toString());
        return null;
    } catch (RMProxyCreationException e) {
        logger.warn("Failed to create User RM Proxy", e);
        // simulate jobs starts and cancel it
        schedulingService.simulateJobStartAndCancelIt(tasksToSchedule, "Failed to create User RM Proxy : Authentication Failed to Resource Manager for user '" + currentJob.getOwner() + "'");
        // leave the method by ss failure
        return null;
    }
}

Example 30 with JobDescriptor

use of org.ow2.proactive.scheduler.common.JobDescriptor in project scheduling by ow2-proactive.

the class JobDescriptorImpl method doReplicate.

/**
 * Complete REPLICATE action on JobDescriptor side
 *
 * @param initiator Task initiating the REPLICATE action
 * @param tree InternalTask tree of replicated tasks
 * @param target Target task of the REPLICATE action: first task of the block
 * @param oldEnd End task of the replicated block ; original version
 * @param newEnd End task of the replicated block ; dup version
 */
public void doReplicate(TaskId initiator, Map<TaskId, InternalTask> tree, InternalTask target, TaskId oldEnd, TaskId newEnd) {
    Map<TaskId, EligibleTaskDescriptorImpl> acc = new HashMap<>();
    // create new EligibleTasks and accumulate it
    for (Entry<TaskId, InternalTask> it : tree.entrySet()) {
        TaskId itId = it.getValue().getTaskInfo().getTaskId();
        EligibleTaskDescriptorImpl td = new EligibleTaskDescriptorImpl(it.getValue());
        acc.put(itId, td);
    }
    // recreate the dependencies
    for (Entry<TaskId, InternalTask> it : tree.entrySet()) {
        TaskId itId = it.getValue().getTaskInfo().getTaskId();
        EligibleTaskDescriptorImpl down = acc.get(itId);
        List<InternalTask> ideps = new ArrayList<>();
        int deptype = 0;
        if (it.getValue().hasDependences()) {
            ideps.addAll(it.getValue().getIDependences());
        } else if (it.getValue().getIfBranch() != null) {
            deptype = 1;
            ideps.add(it.getValue().getIfBranch());
        } else if (it.getValue().getJoinedBranches() != null) {
            deptype = 2;
            ideps.addAll(it.getValue().getJoinedBranches());
        }
        if (ideps != null && !target.equals(itId)) {
            for (InternalTask parent : ideps) {
                if (parent == null) {
                    continue;
                }
                EligibleTaskDescriptorImpl up = acc.get(parent.getId());
                if (up == null) {
                    continue;
                }
                switch(deptype) {
                    case 0:
                        up.addChild(down);
                        down.addParent(up);
                        break;
                    case 1:
                    case 2:
                        branchTasks.put(down.getTaskId(), down);
                        break;
                }
            }
        }
    }
    EligibleTaskDescriptorImpl oldTask = (EligibleTaskDescriptorImpl) runningTasks.get(initiator);
    EligibleTaskDescriptorImpl newTask = acc.get(target.getId());
    if (oldTask == null) {
        oldTask = (EligibleTaskDescriptorImpl) eligibleTasks.get(initiator);
    }
    HashSet<TaskId> excl = new HashSet<>();
    EligibleTaskDescriptorImpl endTask = (EligibleTaskDescriptorImpl) findTask(oldTask, oldEnd, excl);
    if (endTask == null) {
        // findTask cannot walk weak dependencies (IF/ELSE) down, lets walk these branches ourselves
        for (TaskDescriptor branch : branchTasks.values()) {
            endTask = (EligibleTaskDescriptorImpl) findTask(branch, oldEnd, excl);
            if (endTask != null) {
                break;
            }
        }
    }
    EligibleTaskDescriptorImpl end = acc.get(newEnd);
    for (TaskDescriptor t : endTask.getChildren()) {
        end.addChild(t);
        ((EligibleTaskDescriptorImpl) t).addParent(end);
    }
    newTask.addParent(oldTask);
    oldTask.addChild(newTask);
    eligibleTasks.put(target.getId(), newTask);
}