Examples with ForkEnvironment org.ow2.proactive.scheduler.common.task.ForkEnvironment used on opensource projects

Example 21 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestGlobalSpace method testGlobalSpace.

@Test
public void testGlobalSpace() throws Throwable {
    File in = tmpFolder.newFolder("input_space");
    String inPath = in.getAbsolutePath();
    File out = tmpFolder.newFolder("output_space");
    String outPath = out.getAbsolutePath();
    writeFiles(inFiles, inPath);
    TaskFlowJob job = new TaskFlowJob();
    job.setName(this.getClass().getSimpleName());
    job.setInputSpace(in.toURI().toURL().toString());
    job.setOutputSpace(out.toURI().toURL().toString());
    JavaTask A = new JavaTask();
    A.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    A.setName("A");
    for (String[] file : inFiles) {
        A.addInputFiles(file[0], InputAccessMode.TransferFromInputSpace);
        A.addOutputFiles(file[0] + ".glob.A", OutputAccessMode.TransferToGlobalSpace);
    }
    A.setPreScript(new SimpleScript(scriptA, "groovy"));
    A.setForkEnvironment(new ForkEnvironment());
    job.addTask(A);
    JavaTask B = new JavaTask();
    B.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    B.setName("B");
    B.addDependence(A);
    for (String[] file : inFiles) {
        B.addInputFiles(file[0] + ".glob.A", InputAccessMode.TransferFromGlobalSpace);
        B.addOutputFiles(file[0] + ".out", OutputAccessMode.TransferToOutputSpace);
    }
    B.setPreScript(new SimpleScript(scriptB, "groovy"));
    B.setForkEnvironment(new ForkEnvironment());
    job.addTask(B);
    Scheduler scheduler = schedulerHelper.getSchedulerInterface();
    JobId id = scheduler.submit(job);
    schedulerHelper.waitForEventJobFinished(id);
    assertFalse(schedulerHelper.getJobResult(id).hadException());
    /**
     * check: inFiles > IN > LOCAL A > GLOBAL > LOCAL B > OUT
     */
    for (String[] inFile : inFiles) {
        File f = new File(outPath + File.separator + inFile[0] + ".out");
        assertTrue("File does not exist: " + f.getAbsolutePath(), f.exists());
        Assert.assertEquals("Original and copied files differ", inFile[1], FileUtils.readFileToString(f));
        f.delete();
        File inf = new File(inPath + File.separator + inFile[0]);
        inf.delete();
    }
    /**
     * check that the file produced is accessible in the global user space via the scheduler API
     */
    String globalURI = scheduler.getGlobalSpaceURIs().get(0);
    assertTrue(globalURI.startsWith("file:"));
    String globalPath = new File(new URI(globalURI)).getAbsolutePath();
    FileSystemManager fsManager = VFSFactory.createDefaultFileSystemManager();
    for (String[] file : inFiles) {
        FileObject outFile = fsManager.resolveFile(globalURI + "/" + file[0] + ".glob.A");
        log("Checking existence of " + outFile.getURL());
        assertTrue(outFile.getURL() + " exists", outFile.exists());
        File outFile2 = new File(globalPath, file[0] + ".glob.A");
        log("Checking existence of " + outFile2);
        assertTrue(outFile2 + " exists", outFile2.exists());
    }
}

Example 22 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestWorkflowDataspace method testJavaTask.

private void testJavaTask() throws Throwable {
    String tmpdir = System.getProperty("java.io.tmpdir");
    File inputSpace = new File(tmpdir + File.separator + "inputSpace." + System.currentTimeMillis());
    File outputSpace = new File(tmpdir + File.separator + "outputSpace." + System.currentTimeMillis());
    inputSpace.mkdir();
    outputSpace.mkdir();
    for (int it = 0; it < 3; it++) {
        for (int dup = 0; dup < 3; dup++) {
            File f = new File(inputSpace.getCanonicalPath() + File.separator + it + "_" + dup + ".in");
            f.createNewFile();
            BufferedWriter out = new BufferedWriter(new FileWriter(f));
            out.write("it " + it + " dup " + dup + "\n");
            out.close();
        }
    }
    TaskFlowJob job = new TaskFlowJob();
    job.setName(TestWorkflowDataspace.class.getSimpleName());
    job.setInputSpace(inputSpace.toURI().toURL().toString());
    job.setOutputSpace(outputSpace.toURI().toURL().toString());
    ForkEnvironment forkEnvironment = new ForkEnvironment();
    JavaTask t = new JavaTask();
    t.setName("T");
    t.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    t.setFlowScript(FlowScript.createReplicateFlowScript("runs = 3;"));
    t.setFlowBlock(FlowBlock.START);
    t.setForkEnvironment(forkEnvironment);
    job.addTask(t);
    JavaTask t1 = new JavaTask();
    t1.setName("T1");
    t1.setForkEnvironment(forkEnvironment);
    t1.setExecutableClassName(JobWorkflowDataspace.class.getCanonicalName());
    t1.setForkEnvironment(new ForkEnvironment());
    t1.addDependence(t);
    t1.addInputFiles("$PA_TASK_ITERATION_$PA_TASK_REPLICATION.in", InputAccessMode.TransferFromInputSpace);
    t1.addOutputFiles("$PA_TASK_ITERATION_$PA_TASK_REPLICATION.out", OutputAccessMode.TransferToOutputSpace);
    job.addTask(t1);
    JavaTask t2 = new JavaTask();
    t2.setName("T2");
    t2.setForkEnvironment(forkEnvironment);
    t2.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    t2.addDependence(t1);
    t2.setFlowScript(// 
    FlowScript.createLoopFlowScript(// 
    "if (variables.get('PA_TASK_ITERATION') < 2) {" + // 
    "loop = true;" + // 
    "} else {" + // 
    "loop = false;" + "}", "T"));
    t2.setFlowBlock(FlowBlock.END);
    job.addTask(t2);
    JobId id = TWorkflowJobs.testJobSubmission(schedulerHelper, job, null);
    Assert.assertFalse(schedulerHelper.getJobResult(id).hadException());
    for (int it = 0; it < 3; it++) {
        for (int dup = 0; dup < 3; dup++) {
            File f = new File(outputSpace.getCanonicalPath() + File.separator + it + "_" + dup + ".out");
            Assert.assertTrue("Missing output file " + f.getName(), f.exists());
            BufferedReader in = new BufferedReader(new FileReader(f));
            String line = in.readLine();
            Assert.assertTrue("Wrong content for " + f.getCanonicalPath(), line.equals("it " + it + " dup " + dup));
        }
    }
}

Example 23 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestReadSchedulerAccount method singleJobScenario.

private void singleJobScenario(AccountData accountData) throws Exception {
    TaskFlowJob jobDef1 = new TaskFlowJob();
    JavaTask javaTask = new JavaTask();
    javaTask.setExecutableClassName(TestDummyExecutable.class.getName());
    javaTask.setName("task1");
    jobDef1.addTask(javaTask);
    JavaTask forkJavaTask = createDefaultTask("task2");
    forkJavaTask.setExecutableClassName(TestDummyExecutable.class.getName());
    forkJavaTask.setForkEnvironment(new ForkEnvironment());
    jobDef1.addTask(forkJavaTask);
    NativeTask nativeTask = new NativeTask();
    nativeTask.setName("task3");
    nativeTask.setCommandLine("command");
    jobDef1.addTask(nativeTask);
    InternalJob job1 = defaultSubmitJobAndLoadInternal(true, jobDef1, accountData.userName);
    // job is submitted
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task1 started
    job1.start();
    startTask(job1, job1.getTask("task1"));
    dbManager.jobTaskStarted(job1, job1.getTask("task1"), true);
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task1 finished
    accountData.taskTime += finishTask(job1, "task1");
    accountData.taskCount++;
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task2 and task3 started
    startTask(job1, job1.getTask("task2"));
    dbManager.jobTaskStarted(job1, job1.getTask("task2"), true);
    startTask(job1, job1.getTask("task3"));
    dbManager.jobTaskStarted(job1, job1.getTask("task3"), true);
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task2 finished
    accountData.taskTime += finishTask(job1, "task2");
    accountData.taskCount++;
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task3 and job finished
    accountData.taskTime += finishTask(job1, "task3");
    accountData.taskCount++;
    accountData.jobCount++;
    accountData.jobTime += job1.getFinishedTime() - job1.getStartTime();
    checkAccount(invalidUser);
    checkAccount(accountData);
}

Example 24 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestReportingQueries method doTest.

@Test
public void doTest() throws Exception {
    checkInvalidIds();
    checkJobAndTasksNumbers(0, 0, 0, 0, 0, 0, 0, 0);
    checkMeanPendingTime();
    checkMeanExecutionTime();
    checkMeanSubmittingPeriod();
    TaskFlowJob jobDef1 = new TaskFlowJob();
    JavaTask javaTask = new JavaTask();
    javaTask.setExecutableClassName(TestDummyExecutable.class.getName());
    javaTask.setName("task1");
    jobDef1.addTask(javaTask);
    JavaTask forkJavaTask = createDefaultTask("task2");
    forkJavaTask.setExecutableClassName(TestDummyExecutable.class.getName());
    forkJavaTask.setForkEnvironment(new ForkEnvironment());
    jobDef1.addTask(forkJavaTask);
    NativeTask nativeTask = new NativeTask();
    nativeTask.setName("task3");
    nativeTask.setCommandLine("command");
    jobDef1.addTask(nativeTask);
    InternalJob job1 = defaultSubmitJobAndLoadInternal(true, jobDef1);
    try {
        dbManager.getJobPendingTime(job1.getJobInfo().getJobId().value());
        Assert.fail();
    } catch (RuntimeException e) {
    }
    try {
        dbManager.getJobRunningTime(job1.getJobInfo().getJobId().value());
        Assert.fail();
    } catch (RuntimeException e) {
    }
    checkJobAndTasksNumbers(1, 0, 0, 1, 3, 0, 0, 3);
    checkMeanSubmittingPeriod(job1);
    checkMeanTaskPendingTime(job1);
    checkMeanTaskRunningTime(job1);
    checkNumberOfHosts(job1, 0);
    Thread.sleep(100);
    InternalJob job2 = defaultSubmitJobAndLoadInternal(true, jobDef1);
    checkMeanSubmittingPeriod(job1, job2);
    checkJobAndTasksNumbers(2, 0, 0, 2, 6, 0, 0, 6);
    // job1: task1 started
    job1.start();
    startTask(job1, job1.getTask("task1"));
    dbManager.jobTaskStarted(job1, job1.getTask("task1"), true);
    checkNumberOfHosts(job1, 1);
    checkJobPendingTime(job1);
    checkMeanPendingTime(job1);
    checkMeanTaskPendingTime(job1);
    checkMeanTaskRunningTime(job1);
    checkJobAndTasksNumbers(1, 1, 0, 2, 5, 1, 0, 6);
    // job1: task1 finished
    finishTask(job1, "task1");
    checkJobAndTasksNumbers(1, 1, 0, 2, 5, 0, 1, 6);
    checkMeanTaskRunningTime(job1);
    checkNumberOfHosts(job1, 1);
    // job2: task1 started
    job2.start();
    startTask(job2, job2.getTask("task1"));
    dbManager.jobTaskStarted(job2, job2.getTask("task1"), true);
    checkMeanPendingTime(job1, job2);
    checkMeanTaskRunningTime(job2);
    checkJobAndTasksNumbers(0, 2, 0, 2, 4, 1, 1, 6);
    // job1: task2 and task3 started
    startTask(job1, job1.getTask("task2"));
    dbManager.jobTaskStarted(job1, job1.getTask("task2"), false);
    startTask(job1, job1.getTask("task3"));
    dbManager.jobTaskStarted(job1, job1.getTask("task3"), false);
    checkMeanTaskPendingTime(job1);
    checkMeanTaskRunningTime(job1);
    checkJobAndTasksNumbers(0, 2, 0, 2, 2, 3, 1, 6);
    // job1: task2 and task3 finished, job1 finished
    finishTask(job1, "task2");
    finishTask(job1, "task3");
    checkJobRunningTime(job1);
    checkMeanExecutionTime(job1);
    checkMeanTaskRunningTime(job1);
    checkJobAndTasksNumbers(0, 1, 1, 2, 2, 1, 3, 6);
    checkNumberOfHosts(job1, 1);
    // job2: task1, task2 and task3 finished, job2 finished
    finishTask(job2, "task1");
    finishTask(job2, "task2");
    finishTask(job2, "task3");
    checkJobRunningTime(job2);
    checkMeanExecutionTime(job1, job2);
    checkMeanTaskRunningTime(job2);
    checkJobAndTasksNumbers(0, 0, 2, 2, 0, 0, 6, 6);
    // remove job2
    dbManager.removeJob(job2.getId(), System.currentTimeMillis(), false);
    checkJobAndTasksNumbers(0, 0, 1, 1, 0, 0, 3, 3);
    checkMeanPendingTime(job1, job2);
    checkMeanExecutionTime(job1, job2);
    checkMeanSubmittingPeriod(job1, job2);
    InternalJob job3 = defaultSubmitJobAndLoadInternal(true, jobDef1);
    checkMeanSubmittingPeriod(job1, job2, job3);
}

Example 25 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestTaskAttributes method testForkEnv.

@Test
public void testForkEnv() throws Exception {
    JavaTask task = createDefaultTask("task");
    ForkEnvironment env = new ForkEnvironment();
    task.setForkEnvironment(env);
    InternalTask taskData = saveSingleTask(task).getTask(task.getName());
    Assert.assertNotNull(taskData.getForkEnvironment());
    env = new ForkEnvironment();
    env.setEnvScript(new SimpleScript("forkenvscript", "js", new String[] { "p1", "p2" }));
    task.setForkEnvironment(env);
    taskData = saveSingleTask(task).getTask(task.getName());
    Assert.assertNotNull(taskData.getForkEnvironment().getEnvScript());
    Assert.assertArrayEquals(new String[] { "p1", "p2" }, task.getForkEnvironment().getEnvScript().getParameters());
    env = new ForkEnvironment();
    env.setJavaHome("javahome");
    env.setWorkingDir("workingdir");
    env.addAdditionalClasspath("classpath");
    env.addJVMArgument("jvmargument");
    env.addSystemEnvironmentVariable("var1", "value1");
    StringBuilder longString = buildLongString();
    env.addSystemEnvironmentVariable("longvar", longString.toString());
    task.setForkEnvironment(env);
    taskData = saveSingleTask(task).getTask(task.getName());
    Assert.assertEquals("javahome", taskData.getForkEnvironment().getJavaHome());
    Assert.assertEquals("workingdir", taskData.getForkEnvironment().getWorkingDir());
    Assert.assertEquals(1, taskData.getForkEnvironment().getAdditionalClasspath().size());
    Assert.assertEquals("classpath", taskData.getForkEnvironment().getAdditionalClasspath().get(0));
    Assert.assertEquals(1, taskData.getForkEnvironment().getJVMArguments().size());
    Assert.assertEquals("jvmargument", taskData.getForkEnvironment().getJVMArguments().get(0));
    Assert.assertEquals(2, taskData.getForkEnvironment().getSystemEnvironment().size());
    Assert.assertEquals("value1", taskData.getForkEnvironment().getSystemEnvironment().get("var1"));
    Assert.assertEquals(longString.toString(), taskData.getForkEnvironment().getSystemEnvironment().get("longvar"));
}