Examples with JobResult org.ow2.proactive.scheduler.common.job.JobResult used on opensource projects

Example 46 with JobResult

use of org.ow2.proactive.scheduler.common.job.JobResult in project scheduling by ow2-proactive.

the class TestUserSpace method testUserSpace.

@Test
public void testUserSpace() throws Throwable {
    File in = tmpFolder.newFolder("input_space");
    String inPath = in.getAbsolutePath();
    File out = tmpFolder.newFolder("output_space");
    String outPath = out.getAbsolutePath();
    FileSystemManager fsManager = VFSFactory.createDefaultFileSystemManager();
    Scheduler sched = schedulerHelper.getSchedulerInterface();
    String userURI = sched.getUserSpaceURIs().get(0);
    assertTrue(userURI.startsWith("file:"));
    log("User URI is " + userURI);
    String userPath = new File(new URI(userURI)).getAbsolutePath();
    FileObject pathReplaceFO = fsManager.resolveFile(userURI + "/" + pathReplaceFile);
    if (pathReplaceFO.exists()) {
        pathReplaceFO.delete();
    }
    /**
     * Writes inFiles in INPUT
     */
    writeFiles(inFiles, inPath);
    File testPathRepl = new File(inPath + File.separator + pathReplaceFile);
    testPathRepl.createNewFile();
    PrintWriter out2 = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(testPathRepl))));
    out2.print(pathReplaceFile);
    out2.close();
    TaskFlowJob job = new TaskFlowJob();
    job.setName(this.getClass().getSimpleName());
    job.setInputSpace(in.toURI().toURL().toString());
    job.setOutputSpace(out.toURI().toURL().toString());
    JavaTask A = new JavaTask();
    A.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    A.setForkEnvironment(new ForkEnvironment());
    A.setName("A");
    for (String[] file : inFiles) {
        A.addInputFiles(file[0], InputAccessMode.TransferFromInputSpace);
        A.addOutputFiles(file[0] + ".glob.A", OutputAccessMode.TransferToUserSpace);
    }
    A.setPreScript(new SimpleScript(scriptA, "groovy"));
    job.addTask(A);
    JavaTask B = new JavaTask();
    B.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    B.setForkEnvironment(new ForkEnvironment());
    B.setName("B");
    B.addDependence(A);
    for (String[] file : inFiles) {
        B.addInputFiles(file[0] + ".glob.A", InputAccessMode.TransferFromUserSpace);
        B.addOutputFiles(file[0] + ".out", OutputAccessMode.TransferToOutputSpace);
    }
    B.setPreScript(new SimpleScript(scriptB, "groovy"));
    job.addTask(B);
    JobId id = sched.submit(job);
    schedulerHelper.waitForEventJobFinished(id);
    JobResult jr = schedulerHelper.getJobResult(id);
    Assert.assertFalse(jr.hadException());
    /**
     * check: inFiles > IN > LOCAL A > GLOBAL > LOCAL B > OUT
     */
    for (String[] inFile : inFiles) {
        File f = new File(outPath + File.separator + inFile[0] + ".out");
        assertTrue("File does not exist: " + f.getAbsolutePath(), f.exists());
        Assert.assertEquals("Original and copied files differ", inFile[1], FileUtils.readFileToString(f));
        File inf = new File(inPath + File.separator + inFile[0]);
    }
    for (String[] file : inFiles) {
        FileObject outFile = fsManager.resolveFile(userURI + "/" + file[0] + ".glob.A");
        log("Checking existence of " + outFile.getURL());
        assertTrue(outFile.getURL() + " exists", outFile.exists());
        File outFile2 = new File(userPath, file[0] + ".glob.A");
        log("Checking existence of " + outFile2);
        assertTrue(outFile2 + " exists", outFile2.exists());
    }
}

Example 47 with JobResult

use of org.ow2.proactive.scheduler.common.job.JobResult in project scheduling by ow2-proactive.

the class TestLoadJobResult method testLoadJobResult.

@Test
public void testLoadJobResult() throws Throwable {
    TaskFlowJob job = new TaskFlowJob();
    ForkEnvironment forkEnvironment = new ForkEnvironment();
    forkEnvironment.addAdditionalClasspath("/path1/path1");
    forkEnvironment.addAdditionalClasspath("/path2/path2");
    JavaTask javaTask1 = createDefaultTask("task1");
    javaTask1.setPreciousResult(true);
    javaTask1.setForkEnvironment(forkEnvironment);
    job.addTask(javaTask1);
    for (int i = 2; i <= 4; i++) {
        JavaTask task = createDefaultTask("task" + i);
        task.setForkEnvironment(forkEnvironment);
        job.addTask(task);
    }
    JavaTask javaTask5 = createDefaultTask("task5");
    javaTask5.setPreciousResult(true);
    javaTask5.setForkEnvironment(forkEnvironment);
    job.addTask(javaTask5);
    InternalJob internalJob = defaultSubmitJobAndLoadInternal(true, job);
    InternalTask task1 = internalJob.getTask("task1");
    InternalTask task2 = internalJob.getTask("task2");
    InternalTask task3 = internalJob.getTask("task3");
    InternalTask task4 = internalJob.getTask("task4");
    InternalTask task5 = internalJob.getTask("task5");
    dbManager.updateAfterTaskFinished(internalJob, task1, new TaskResultImpl(null, new TestResult(0, "1_1"), null, 0));
    dbManager.updateAfterTaskFinished(internalJob, task1, new TaskResultImpl(null, new TestResult(0, "1_2"), null, 0));
    dbManager.updateAfterTaskFinished(internalJob, task2, new TaskResultImpl(null, new TestResult(0, "2_1"), null, 0));
    dbManager.updateAfterTaskFinished(internalJob, task2, new TaskResultImpl(null, new TestResult(0, "2_2"), null, 0));
    dbManager.updateAfterTaskFinished(internalJob, task3, new TaskResultImpl(null, new TestResult(0, "3_1"), null, 0));
    dbManager.updateAfterTaskFinished(internalJob, task4, new TaskResultImpl(null, new TestException("message4_1", "data4_1"), null, 0));
    dbManager.updateAfterTaskFinished(internalJob, task4, new TaskResultImpl(null, new TestException("message4_2", "data4_2"), null, 0));
    dbManager.updateAfterTaskFinished(internalJob, task5, new TaskResultImpl(null, new TestException("message5_1", "data5_1"), null, 0));
    TaskFlowJob job2 = new TaskFlowJob();
    job2.addTask(createDefaultTask("job2 task1"));
    InternalJob internalJob2 = defaultSubmitJobAndLoadInternal(true, job2);
    InternalTask task2_1 = internalJob2.getTask("job2 task1");
    dbManager.updateAfterTaskFinished(internalJob2, task2_1, new TaskResultImpl(null, new TestResult(0, "job2_task1"), null, 0));
    System.out.println("Load job result1");
    JobResult result = dbManager.loadJobResult(internalJob.getId());
    Assert.assertEquals(5, result.getAllResults().size());
    Assert.assertEquals(2, result.getExceptionResults().size());
    Assert.assertEquals(2, result.getPreciousResults().size());
    Assert.assertNotNull(result.getJobInfo());
    Assert.assertEquals(internalJob.getId(), result.getJobId());
    Assert.assertEquals(5, result.getJobInfo().getTotalNumberOfTasks());
    TestResult taskResultValue;
    taskResultValue = (TestResult) result.getResult("task1").value();
    Assert.assertEquals("1_2", taskResultValue.getB());
    taskResultValue = (TestResult) result.getResult("task2").value();
    Assert.assertEquals("2_2", taskResultValue.getB());
    taskResultValue = (TestResult) result.getResult("task3").value();
    Assert.assertEquals("3_1", taskResultValue.getB());
    TestException taskException;
    taskException = (TestException) result.getResult("task4").getException();
    Assert.assertEquals("message4_2", taskException.getMessage());
    taskException = (TestException) result.getResult("task5").getException();
    Assert.assertEquals("message5_1", taskException.getMessage());
    System.out.println("Load job result2");
    result = dbManager.loadJobResult(internalJob2.getId());
    Assert.assertEquals(1, result.getAllResults().size());
    Assert.assertEquals(0, result.getExceptionResults().size());
    Assert.assertEquals(0, result.getPreciousResults().size());
}

Example 48 with JobResult

use of org.ow2.proactive.scheduler.common.job.JobResult in project scheduling by ow2-proactive.

the class TestErrorAndFailure method testErrorAndFailure.

@Test
public void testErrorAndFailure() throws Throwable {
    String javaCmd = System.getProperty("java.home") + "/bin/java";
    log("Test 1 : Creating job...");
    // creating job
    TaskFlowJob submittedJob = new TaskFlowJob();
    submittedJob.setName(this.getClass().getSimpleName() + "_12_tasks");
    submittedJob.setDescription("12 tasks job testing the behavior of error code and normal task ending.");
    submittedJob.setMaxNumberOfExecution(10);
    NativeTask finalTask = new NativeTask();
    finalTask.setName("TestMerge");
    finalTask.setCommandLine(new String[] { javaCmd, "-cp", TestScheduler.testClasspath(), NativeTestWithRandomDefault.class.getName(), "final" });
    for (int i = 1; i < 6; i++) {
        NativeTask task = new NativeTask();
        task.setName("Test" + i);
        task.setCommandLine(new String[] { javaCmd, "-cp", TestScheduler.testClasspath(), NativeTestWithRandomDefault.class.getName(), "0" });
        finalTask.addDependence(task);
        submittedJob.addTask(task);
    }
    submittedJob.addTask(finalTask);
    // test submission and event reception
    JobId id = schedulerHelper.submitJob(submittedJob);
    log("Job submitted, id " + id.toString());
    log("Waiting for jobSubmitted Event");
    Job receivedJob = schedulerHelper.waitForEventJobSubmitted(id);
    assertEquals(receivedJob.getId(), id);
    log("Waiting for job running");
    JobInfo jInfo = schedulerHelper.waitForEventJobRunning(id);
    assertEquals(jInfo.getJobId(), id);
    // TODO how to check that ?
    for (Task t : submittedJob.getTasks()) {
        log("Waiting for task running : " + t.getName());
        schedulerHelper.waitForEventTaskRunning(id, t.getName());
    }
    for (Task t : submittedJob.getTasks()) {
        log("Waiting for task finished : " + t.getName());
        schedulerHelper.waitForEventTaskFinished(id, t.getName());
    }
    log("Waiting for job finished");
    jInfo = schedulerHelper.waitForEventJobFinished(id);
    assertEquals(JobStatus.FINISHED, jInfo.getStatus());
    // check job results
    JobResult res = schedulerHelper.getJobResult(id);
    // Check the results
    Map<String, TaskResult> results = res.getAllResults();
    // check that number of results correspond to number of tasks
    assertEquals(submittedJob.getTasks().size(), results.size());
    // remove jobs and check its event
    schedulerHelper.removeJob(id);
    log("Waiting for job removed");
    jInfo = schedulerHelper.waitForEventJobRemoved(id);
    assertEquals(JobStatus.FINISHED, jInfo.getStatus());
    assertEquals(jInfo.getJobId(), id);
}

Example 49 with JobResult

use of org.ow2.proactive.scheduler.common.job.JobResult in project scheduling by ow2-proactive.

the class TestListenJobLogs method testLogs.

public void testLogs() throws Exception {
    CommunicationObject communicationObject1 = PAActiveObject.newActive(CommunicationObject.class, new Object[] {});
    String communicationObjectUrl1 = PAActiveObject.getUrl(communicationObject1);
    CommunicationObject communicationObject2 = PAActiveObject.newActive(CommunicationObject.class, new Object[] {});
    String communicationObjectUrl2 = PAActiveObject.getUrl(communicationObject2);
    Scheduler scheduler = schedulerHelper.getSchedulerInterface();
    Job job = createJob(communicationObjectUrl1, communicationObjectUrl2);
    JobId jobId = scheduler.submit(job);
    schedulerHelper.waitForEventTaskRunning(jobId, TASK_NAME1);
    communicationObject1.setCommand("output1");
    // listenJobLogs for running task
    TestAppender appender1 = new TestAppender("appender1");
    String loggerName = Log4JTaskLogs.JOB_LOGGER_PREFIX + jobId;
    logForwardingService.addAppender(loggerName, appender1);
    scheduler.listenJobLogs(jobId, logForwardingService.getAppenderProvider());
    appender1.waitForLoggingEvent(LOG_EVENT_TIMEOUT, "output1");
    /*
         * second listenJobLogs for running task (should get output produced before listenJobLogs
         * was called)
         */
    TestAppender appender2 = new TestAppender("appender2");
    logForwardingService.addAppender(loggerName, appender2);
    scheduler.listenJobLogs(jobId, logForwardingService.getAppenderProvider());
    appender2.waitForLoggingEvent(LOG_EVENT_TIMEOUT, "output1");
    // task produces more output, check it is received
    communicationObject1.setCommand("output2");
    /*
         * TODO: at the time of this writing there is no way to remove first log appender,
         * so this JVM receives two identical log events for running job
         */
    appender2.waitForLoggingEvent(LOG_EVENT_TIMEOUT, "output2", "output2");
    // let first task finish
    communicationObject1.setCommand("stop");
    schedulerHelper.waitForEventTaskFinished(jobId, TASK_NAME1);
    schedulerHelper.waitForEventTaskRunning(jobId, TASK_NAME2);
    // now first task had finished, second tasks is still running
    communicationObject2.setCommand("output3");
    /*
         * TODO: at the time of this writing there is no way to remove first log appender,
         * so this JVM receives two identical log events for running job
         */
    appender2.waitForLoggingEvent(LOG_EVENT_TIMEOUT, "output3", "output3");
    // add appender after first task had finished, appender should receive its output
    TestAppender appender3 = new TestAppender("appender3");
    logForwardingService.removeAllAppenders(loggerName);
    logForwardingService.addAppender(loggerName, appender3);
    scheduler.listenJobLogs(jobId, logForwardingService.getAppenderProvider());
    appender3.waitForLoggingEvent(LOG_EVENT_TIMEOUT, "output1", "output2", "output3");
    // let second task finish
    communicationObject2.setCommand("stop");
    schedulerHelper.waitForEventJobFinished(jobId);
    // add appender after job had finished, appender should receive output of all tasks
    TestAppender appender4 = new TestAppender("appender4");
    logForwardingService.removeAllAppenders(loggerName);
    logForwardingService.addAppender(loggerName, appender4);
    scheduler.listenJobLogs(jobId, logForwardingService.getAppenderProvider());
    appender4.waitForLoggingEvent(LOG_EVENT_TIMEOUT, "output1", "output2", "output3");
    System.out.println("Check job result");
    JobResult jobResult = scheduler.getJobResult(jobId);
    for (TaskResult taskResult : jobResult.getAllResults().values()) {
        if (taskResult.getException() != null) {
            taskResult.getException().printStackTrace();
            Assert.fail("Task failed with exception " + taskResult.getException());
        }
        System.out.println("Task output:");
        System.out.println(taskResult.getOutput().getAllLogs(false));
    }
    PAActiveObject.terminateActiveObject(communicationObject1, true);
    PAActiveObject.terminateActiveObject(communicationObject2, true);
    logForwardingService.removeLogger(loggerName);
}

Example 50 with JobResult

use of org.ow2.proactive.scheduler.common.job.JobResult in project scheduling by ow2-proactive.

the class JobRecoverTest method testJobRecover.

@Test
public void testJobRecover() throws Throwable {
    JobId firstJobId = schedulerHelper.submitJob(getWorkflowFile());
    JobId secondJobId = schedulerHelper.submitJob(getWorkflowFile());
    JobId thirdJobId = schedulerHelper.submitJob(getWorkflowFile());
    schedulerHelper.waitForEventJobRunning(firstJobId);
    SchedulerTHelper.log("Waiting for job 1 to finish");
    schedulerHelper.waitForFinishedJob(firstJobId);
    SchedulerTHelper.log("Kill Scheduler");
    schedulerHelper.killSchedulerAndNodesAndRestart(new File(SchedulerTHelper.class.getResource("/functionaltests/config/functionalTSchedulerProperties-updateDB.ini").toURI()).getAbsolutePath());
    SchedulerTHelper.log("Waiting for job 2 to finish");
    schedulerHelper.waitForEventJobFinished(secondJobId);
    SchedulerTHelper.log("Waiting for job 3 to finish");
    schedulerHelper.waitForFinishedJob(thirdJobId);
    SchedulerTHelper.log("Check result job 1");
    JobResult result = schedulerHelper.getJobResult(firstJobId);
    checkJobResults(result);
    SchedulerTHelper.log("Check result job 2");
    result = schedulerHelper.getJobResult(secondJobId);
    checkJobResults(result);
    SchedulerTHelper.log("Check result job 3");
    result = schedulerHelper.getJobResult(thirdJobId);
    checkJobResults(result);
}