Examples with AwaitedJob org.ow2.proactive.scheduler.smartproxy.common.AwaitedJob used on opensource projects

Example 6 with AwaitedJob

use of org.ow2.proactive.scheduler.smartproxy.common.AwaitedJob in project scheduling by ow2-proactive.

the class AbstractSmartProxy method updateTask.

/**
 * Check if the task concerned by this notification is awaited. Retrieve
 * corresponding data if needed
 *
 * @param notification
 */
protected void updateTask(NotificationData<TaskInfo> notification) {
    // am I interested in this task?
    TaskInfo taskInfoData = notification.getData();
    JobId id = taskInfoData.getJobId();
    TaskId tid = taskInfoData.getTaskId();
    String tname = tid.getReadableName();
    TaskStatus status = taskInfoData.getStatus();
    AwaitedJob aj = jobTracker.getAwaitedJob(id.toString());
    if (aj == null)
        return;
    AwaitedTask at = aj.getAwaitedTask(tname);
    if (at == null)
        return;
    at.setTaskId(tid.toString());
    jobTracker.putAwaitedJob(id.toString(), aj);
    switch(status) {
        case ABORTED:
        case NOT_RESTARTED:
        case NOT_STARTED:
        case SKIPPED:
            {
                log.debug("The task " + tname + " from job " + id + " couldn't start. No data will be transfered");
                jobTracker.removeAwaitedTask(id.toString(), tname);
                break;
            }
        case FINISHED:
            {
                log.debug("The task " + tname + " from job " + id + " is finished.");
                if (aj.isAutomaticTransfer()) {
                    log.debug("Transferring data for finished task " + tname + " from job " + id);
                    try {
                        downloadTaskOutputFiles(aj, id.toString(), tname, aj.getLocalOutputFolder());
                    } catch (Throwable t) {
                        log.error("Error while handling data for finished task " + tname + " for job " + id + ", task will be removed");
                        jobTracker.removeAwaitedTask(id.toString(), tname);
                    }
                }
                break;
            }
        case FAILED:
        case FAULTY:
            {
                log.debug("The task " + tname + " from job " + id + " is faulty.");
                jobTracker.removeAwaitedTask(id.toString(), tname);
                break;
            }
    }
}

Example 7 with AwaitedJob

use of org.ow2.proactive.scheduler.smartproxy.common.AwaitedJob in project scheduling by ow2-proactive.

the class AbstractSmartProxy method updateJob.

// ********* Awaited Jobs methods ******************************* //
/**
 * @return a new HashSet with the awaited jobs. Modifying the result of this
 *         method will not affect the source HashSet (the awaited jobs)
 */
/**
 * Check if the job concerned by this notification is awaited. Retrieve
 * corresponding data if needed
 *
 * @param notification
 */
protected void updateJob(NotificationData<?> notification) {
    // am I interested in this job?
    JobId id = ((NotificationData<JobInfo>) notification).getData().getJobId();
    AwaitedJob aj = jobTracker.getAwaitedJob(id.toString());
    if (aj == null)
        return;
    JobStatus status = ((NotificationData<JobInfo>) notification).getData().getStatus();
    switch(status) {
        case KILLED:
            {
                log.debug("The job " + id + "has been killed.");
                jobTracker.removeAwaitedJob(id.toString());
                break;
            }
        case FINISHED:
            {
                log.debug("The job " + id + " is finished.");
                // removeAwaitedJob(id.toString());
                break;
            }
        case CANCELED:
            {
                log.debug("The job " + id + " is canceled.");
                jobTracker.removeAwaitedJob(id.toString());
                break;
            }
        case FAILED:
            {
                log.debug("The job " + id + " is failed.");
                // removeAwaitedJob(id.toString());
                break;
            }
    }
}

Example 8 with AwaitedJob

use of org.ow2.proactive.scheduler.smartproxy.common.AwaitedJob in project scheduling by ow2-proactive.

the class JobTrackerImpl method removeAwaitedTask.

/**
 * Removes from the proxy knowledge all info related with the given task.
 * If all tasks of a job have been removed this way, the job itself will be removed.
 *
 * @param id    jobID
 * @param taskName task name
 */
public void removeAwaitedTask(String id, String taskName) {
    AwaitedJob awaitedJob = jobDatabase.getAwaitedJob(id);
    if (awaitedJob == null) {
        logger.warn("Job " + id + " not in the awaited list");
        return;
    }
    AwaitedTask awaitedTask = awaitedJob.getAwaitedTask(taskName);
    if (awaitedTask == null) {
        logger.warn("Task " + taskName + " from Job " + id + " not in the awaited list");
        return;
    }
    logger.debug("Removing knowledge of task " + taskName + " from job " + id);
    if (awaitedJob.isIsolateTaskOutputs() && awaitedTask.getTaskId() != null) {
        // If the output data as been isolated in a dedicated folder we can delete it.
        String pullUrl = awaitedJob.getPullURL();
        pullUrl = pullUrl.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + awaitedTask.getTaskId());
        FileObject remotePullFolder = null;
        try {
            remotePullFolder = resolveFile(pullUrl);
            logger.debug("Deleting directory " + remotePullFolder);
            remotePullFolder.delete(Selectors.SELECT_ALL);
            remotePullFolder.delete();
        } catch (Exception e) {
            logger.warn("Could not remove data for task " + taskName + " of job " + id, e);
        }
    }
    awaitedJob.removeAwaitedTask(taskName);
    if (awaitedJob.getAwaitedTasks().isEmpty()) {
        removeAwaitedJob(id);
        return;
    } else {
        // this is done to ensure persistence of the operation
        jobDatabase.putAwaitedJob(id, awaitedJob);
    }
    try {
        jobDatabase.commit();
    } catch (IOException e) {
        logger.error("Could not save status file after removing task Task " + taskName + " from Job" + id, e);
    }
}

Example 9 with AwaitedJob

use of org.ow2.proactive.scheduler.smartproxy.common.AwaitedJob in project scheduling by ow2-proactive.

the class AbstractSmartProxy method submit.

/**
 * Submits a job to the scheduler and handle data transfer via the SmartProxy
 *
 * @param job                   job to submit
 * @param localInputFolderPath  path to the local directory containing input files
 * @param pushUrl               url of the dataspace server used to push input files to
 * @param localOutputFolderPath path to the local directory which will contain output files
 * @param pullUrl               url of the dataspace server used to pull output files from
 * @param isolateTaskOutputs    if set to true, output files from each tasks will be isolated from each other in the dataspace server (to prevent overlapping)
 * @param automaticTransfer     if set to true, output files will be automatically transferred from the dataspace server to the local machine as soon as the task is finished.
 *                              If set to false, the files will not be automatically transferred and a call to pullData must be done to transfer files
 * @return the new job id
 * @throws Exception
 * @throws SubmissionClosedException
 * @throws JobCreationException
 */
public JobId submit(TaskFlowJob job, String localInputFolderPath, String pushUrl, String localOutputFolderPath, String pullUrl, boolean isolateTaskOutputs, boolean automaticTransfer) throws Exception, SubmissionClosedException, JobCreationException {
    checkInitialized();
    if (isNullOrEmpty(pushUrl)) {
        pushUrl = getLocalUserSpace();
    }
    if (isNullOrEmpty(pullUrl)) {
        pullUrl = getLocalUserSpace();
    }
    String newFolderName = createNewFolderName();
    String pushUrlUpdate = prepareJobInput(job, localInputFolderPath, pushUrl, newFolderName);
    String pullUrlUpdate = prepareJobOutput(job, localOutputFolderPath, pullUrl, newFolderName, isolateTaskOutputs);
    uploadInputfiles(job, localInputFolderPath);
    JobId id = null;
    try {
        id = submit(job);
    } catch (Exception e) {
        log.error("Error while submitting job", e);
        try {
            removeJobIO(job, pushUrl, pullUrl, newFolderName);
        } catch (Exception e2) {
            log.error("Error while removing job IO", e2);
        }
        propagateIfInstanceOf(e, NotConnectedException.class);
        propagateIfInstanceOf(e, PermissionException.class);
        propagateIfInstanceOf(e, SubmissionClosedException.class);
        propagateIfInstanceOf(e, JobCreationException.class);
        propagateIfInstanceOf(e, RuntimeException.class);
        propagate(e);
    }
    HashMap<String, AwaitedTask> awaitedTaskMap = new HashMap<>();
    for (Task t : job.getTasks()) {
        awaitedTaskMap.put(t.getName(), new AwaitedTask(t.getName(), t.getOutputFilesList()));
    }
    AwaitedJob awaitedJob = new AwaitedJob(id.toString(), localInputFolderPath, job.getInputSpace(), pushUrlUpdate, localOutputFolderPath, job.getOutputSpace(), pullUrlUpdate, isolateTaskOutputs, automaticTransfer, awaitedTaskMap);
    jobTracker.putAwaitedJob(id.toString(), awaitedJob);
    return id;
}

Example 10 with AwaitedJob

use of org.ow2.proactive.scheduler.smartproxy.common.AwaitedJob in project scheduling by ow2-proactive.

the class JobTrackerImpl method removeAwaitedJob.

/**
 * Removes from the proxy knowledge all info related with the given job.
 * This will also delete every folder created by the job in the shared input and output spaces
 *
 * @param id jobID
 */
public void removeAwaitedJob(String id) {
    AwaitedJob aj = jobDatabase.getAwaitedJob(id);
    if (aj == null) {
        logger.warn("Job " + id + " not in the awaited list");
        return;
    }
    logger.debug("Removing knowledge of job " + id);
    String pullUrl = aj.getPullURL();
    String pushUrl = aj.getPushURL();
    FileObject remotePullFolder = null;
    FileObject remotePushFolder = null;
    try {
        remotePullFolder = resolveFile(pullUrl);
        remotePushFolder = resolveFile(pushUrl);
    } catch (Exception e) {
        logger.error("Could not remove data for job " + id, e);
        return;
    }
    if (aj.isIsolateTaskOutputs()) {
        try {
            remotePullFolder = remotePullFolder.getParent();
        } catch (FileSystemException e) {
            logger.error("Could not get the parent of folder " + remotePullFolder, e);
        }
    }
    Set<FileObject> foldersToDelete = new HashSet<>();
    try {
        foldersToDelete.add(remotePullFolder.getParent());
        if (!remotePullFolder.getParent().equals(remotePushFolder.getParent()))
            foldersToDelete.add(remotePushFolder.getParent());
    } catch (FileSystemException e) {
        logger.warn("Data in folders " + pullUrl + " and " + pushUrl + " cannot be deleted due to an unexpected error ", e);
    }
    String url = "NOT YET DEFINED";
    for (FileObject fo : foldersToDelete) {
        try {
            url = fo.getURL().toString();
            if (!logger.isTraceEnabled()) {
                logger.debug("Deleting directory " + url);
                fo.delete(Selectors.SELECT_ALL);
                fo.delete();
            }
        } catch (FileSystemException e) {
            logger.warn("Could not delete temporary files at location " + url + " .", e);
        }
    }
    jobDatabase.removeAwaitedJob(id);
    try {
        jobDatabase.commit();
    } catch (IOException e) {
        logger.error("Could not save status file after removing job " + id, e);
    }
}