Examples with Job org.ow2.proactive.scheduler.common.job.Job used on opensource projects

Example 36 with Job

use of org.ow2.proactive.scheduler.common.job.Job in project scheduling by ow2-proactive.

the class NoVncSecuredTargetResolverTest method testMagicStringFoundInLiveLogs_MagicStringOnSeveralLines.

@Test
public void testMagicStringFoundInLiveLogs_MagicStringOnSeveralLines() throws Exception {
    String sessionId = SharedSessionStoreTestUtils.createValidSession(schedulerMock);
    SharedSessionStore.getInstance().get(sessionId).getJobsOutputController().addJobOutputAppender("42", createLiveLogs("PA_REMOTE_CONNECTION\nPA_REMOTE_CONNECTION;42;1;vnc;node.grid.com:5900 "));
    when(schedulerMock.getTaskResult("42", "remoteVisuTask")).thenReturn(new TaskResultImpl(TaskIdImpl.createTaskId(new JobIdImpl(42, "job"), "remoteVisuTask", 1), new byte[0], new byte[0], new SimpleTaskLogs("", ""), true));
    InetSocketAddress targetVncHost = new NoVncSecuredTargetResolver().doResolve(sessionId, "42", "remoteVisuTask");
    assertEquals(5900, targetVncHost.getPort());
    assertEquals("node.grid.com", targetVncHost.getHostName());
}

Example 37 with Job

use of org.ow2.proactive.scheduler.common.job.Job in project scheduling by ow2-proactive.

the class RestJobTrackerImpl method removeAwaitedJob.

@Override
public void removeAwaitedJob(String id) {
    AwaitedJob aj = jobDatabase.getAwaitedJob(id);
    if (aj == null) {
        logger.warn("Job " + id + " not in the awaited list");
        return;
    }
    logger.debug("Removing knowledge of job " + id);
    String pullUrl = aj.getPullURL();
    String pushUrl = aj.getPushURL();
    Path remotePullFolder = null;
    Path remotePushFolder = null;
    try {
        remotePullFolder = Paths.get(new URI(pullUrl));
        remotePushFolder = Paths.get(new URI(pushUrl));
    } catch (Exception e) {
        logger.error("Could not remove data for job " + id, e);
        return;
    }
    if (aj.isIsolateTaskOutputs()) {
        Path tmp = remotePullFolder.getParent();
        if (tmp != null) {
            remotePullFolder = tmp;
        }
    }
    Set<Path> foldersToDelete = new HashSet<>();
    foldersToDelete.add(remotePullFolder.getParent());
    if (!remotePullFolder.getParent().equals(remotePushFolder.getParent())) {
        foldersToDelete.add(remotePushFolder.getParent());
    }
    RemoteSource remoteSource = new RemoteSource(IDataSpaceClient.Dataspace.USER);
    remoteSource.setType(FileType.FOLDER);
    for (Path path : foldersToDelete) {
        String location = path.toUri().toString();
        try {
            if (!logger.isTraceEnabled()) {
                logger.debug("Deleting directory " + location);
                remoteSource.setPath(location);
                restDataSpaceClient.delete(remoteSource);
            }
        } catch (NotConnectedException | PermissionException e) {
            logger.warn("Could not delete temporary files at location " + location + " .", e);
        }
    }
    jobDatabase.removeAwaitedJob(id);
    try {
        jobDatabase.commit();
    } catch (IOException e) {
        logger.error("Could not save status file after removing job " + id, e);
    }
}

Example 38 with Job

use of org.ow2.proactive.scheduler.common.job.Job in project scheduling by ow2-proactive.

the class RestJobTrackerImpl method removeAwaitedTask.

@Override
public void removeAwaitedTask(String id, String taskName) {
    AwaitedJob awaitedJob = jobDatabase.getAwaitedJob(id);
    if (awaitedJob == null) {
        logger.warn("Job " + id + " not in the awaited list");
        return;
    }
    AwaitedTask awaitedTask = awaitedJob.getAwaitedTask(taskName);
    if (awaitedTask == null) {
        logger.warn("Task " + taskName + " from Job " + id + " not in the awaited list");
        return;
    }
    logger.debug("Removing knowledge of task " + taskName + " from job " + id);
    if (awaitedJob.isIsolateTaskOutputs() && awaitedTask.getTaskId() != null) {
        // If the output data as been isolated in a dedicated folder we can delete it.
        String pullUrl = awaitedJob.getPullURL();
        pullUrl = pullUrl.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + awaitedTask.getTaskId());
        try {
            RemoteSource remoteSource = new RemoteSource(IDataSpaceClient.Dataspace.USER, pullUrl + "/");
            remoteSource.setType(FileType.FOLDER);
            restDataSpaceClient.delete(remoteSource);
        } catch (Throwable t) {
            logger.warn("Could not remove data for task " + taskName + " of job " + id, t);
        }
    }
    awaitedJob.removeAwaitedTask(taskName);
    if (awaitedJob.getAwaitedTasks().isEmpty()) {
        removeAwaitedJob(id);
        return;
    } else {
        // this is done to ensure persistence of the operation
        jobDatabase.putAwaitedJob(id, awaitedJob);
    }
    try {
        jobDatabase.commit();
    } catch (IOException e) {
        logger.error("Could not save status file after removing task Task " + taskName + " from Job" + id, e);
    }
}

Example 39 with Job

use of org.ow2.proactive.scheduler.common.job.Job in project scheduling by ow2-proactive.

the class RestSmartProxyImpl method downloadTaskOutputFiles.

@Override
protected void downloadTaskOutputFiles(AwaitedJob awaitedjob, String jobId, String taskName, String localFolder) throws NotConnectedException, PermissionException {
    AwaitedTask atask = awaitedjob.getAwaitedTask(taskName);
    if (atask == null) {
        throw new IllegalArgumentException("The task " + taskName + " does not belong to job " + jobId + " or has already been removed");
    }
    if (atask.isTransferring()) {
        logger.warn("The task " + taskName + " of job " + jobId + " is already transferring its output");
        return;
    }
    String outputSpace = awaitedjob.getOutputSpaceURL();
    String sourceFile;
    try {
        String userSpace = getLocalUserSpace();
        if (!outputSpace.startsWith(userSpace)) {
            logger.warn("RestSmartProxy does not support data transfers outside USERSPACE.");
        }
        sourceFile = outputSpace.substring(userSpace.length() + 1);
    } catch (Throwable error) {
        throw Throwables.propagate(error);
    }
    if (awaitedjob.isIsolateTaskOutputs()) {
        sourceFile = sourceFile.replace(SchedulerConstants.TASKID_DIR_DEFAULT_NAME, SchedulerConstants.TASKID_DIR_DEFAULT_NAME + "/" + atask.getTaskId());
    }
    List<OutputSelector> outputFileSelectors = atask.getOutputSelectors();
    List<String> includes = Lists.newArrayList();
    List<String> excludes = Lists.newArrayList();
    if (outputFileSelectors != null) {
        for (OutputSelector os : outputFileSelectors) {
            addfileSelection(os.getOutputFiles(), includes, excludes);
        }
    }
    jobTracker.setTaskTransferring(jobId, taskName, true);
    if (awaitedjob.isAutomaticTransfer()) {
        threadPool.submit(new DownloadHandler(jobId, taskName, sourceFile, includes, excludes, localFolder));
    } else {
        try {
            RemoteSource source = new RemoteSource(USER, sourceFile);
            source.setIncludes(includes);
            source.setExcludes(excludes);
            File localDir = new File(localFolder);
            LocalDestination dest = new LocalDestination(localDir);
            restDataSpaceClient.download(source, dest);
        } catch (NotConnectedException | PermissionException e) {
            logger.error(String.format("Cannot download files, jobId=%s, taskId=%s, source=%s, destination=%s", jobId, taskName, sourceFile, localFolder), e);
            throw e;
        } finally {
            jobTracker.setTaskTransferring(jobId, taskName, false);
        }
        // task is removed from the job tracker only if the transfer is successful
        jobTracker.removeAwaitedTask(jobId, taskName);
    }
}

Example 40 with Job

use of org.ow2.proactive.scheduler.common.job.Job in project scheduling by ow2-proactive.

the class RestSmartProxyTest method createTestJob.

private TaskFlowJob createTestJob(boolean isolateOutputs) throws Exception {
    TaskFlowJob job = new TaskFlowJob();
    // add a special character to the job name to ensure the job is parsed
    // correctly by the server
    job.setName(this.getClass().getSimpleName() + " é");
    for (int i = 0; i < NB_TASKS; i++) {
        JavaTask testTask = new JavaTask();
        testTask.setName(TASK_NAME + i);
        testTask.setExecutableClassName(SimpleJavaExecutable.class.getName());
        testTask.setForkEnvironment(new ForkEnvironment());
        File inputFile = new File(inputLocalFolder, INPUT_FILE_BASE_NAME + "_" + i + INPUT_FILE_EXT);
        String outputFileName = OUTPUT_FILE_BASE_NAME + "_" + i + OUTPUT_FILE_EXT;
        // delete files after the test is finished
        File outputFile = new File(outputLocalFolder, outputFileName);
        outputFile.deleteOnExit();
        inputFile.deleteOnExit();
        FileWriter fileWriter = new FileWriter(inputFile);
        for (int j = 0; j <= Math.round(Math.random() * 100) + 1; j++) {
            fileWriter.write("Some random input");
        }
        fileWriter.close();
        // Add dummy input files, make sure no error happen
        testTask.addInputFiles("DUMMY", InputAccessMode.TransferFromInputSpace);
        testTask.addInputFiles(inputFile.getName(), InputAccessMode.TransferFromInputSpace);
        if (isolateOutputs) {
            testTask.addOutputFiles("*.out", OutputAccessMode.TransferToOutputSpace);
        } else {
            testTask.addOutputFiles(outputFileName, OutputAccessMode.TransferToOutputSpace);
        }
        job.addTask(testTask);
    }
    job.setInputSpace(userspace);
    job.setOutputSpace(userspace);
    return job;
}