Examples with ForkEnvironment org.ow2.proactive.scheduler.common.task.ForkEnvironment used on opensource projects

Example 51 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestKillTaskWhileExecutingScripts method forkedJavaTaskKillEndlessEnvScript.

public void forkedJavaTaskKillEndlessEnvScript() throws Throwable {
    log("Test Forked Java Task : killing an Endless Java Executable ...");
    String tname = "forkedJavaTaskKillEndlessEnvScript";
    // pre script interruption
    TaskFlowJob job = new TaskFlowJob();
    job.setName(this.getClass().getSimpleName() + "_" + tname);
    JavaTask task1 = new JavaTask();
    task1.setName(tname);
    task1.setExecutableClassName(EmptyExecutable.class.getName());
    ForkEnvironment fe = new ForkEnvironment();
    fe.setEnvScript(endlessScript);
    task1.setForkEnvironment(fe);
    job.addTask(task1);
    submitAndCheckJob(job, tname);
}

Example 52 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestProcessTreeKiller method testProcessTreeKiller.

@Test
public void testProcessTreeKiller() throws Throwable {
    schedulerHelper.addExtraNodes(2);
    Logger.getLogger(ProcessTree.class).setLevel(Level.DEBUG);
    Logger.getLogger(TaskLauncher.class).setLevel(Level.DEBUG);
    for (int i = 0; i < NB_ITERATIONS; i++) {
        log("***************************************************");
        log("************** Iteration " + i + " *************************");
        log("***************************************************");
        log("Creating job...");
        // create job 1 NativeExecutable
        TaskFlowJob job1 = new TaskFlowJob();
        job1.setName(this.getClass().getSimpleName() + "_1");
        job1.setDescription("a command that spawn processes");
        NativeTask task1 = new NativeTask();
        String task1Name = "TestPTK1";
        task1.setName(task1Name);
        String workingDir = new File(TestProcessTreeKiller.launchersDir.toURI()).getParentFile().getCanonicalPath();
        task1.setForkEnvironment(new ForkEnvironment(workingDir));
        JavaSpawnExecutable executable = new JavaSpawnExecutable();
        executable.home = PASchedulerProperties.SCHEDULER_HOME.getValueAsString();
        task1.setCommandLine(executable.getNativeExecLauncher(false));
        job1.addTask(task1);
        String task2Name = "TestTK2";
        TaskFlowJob job2 = createJavaExecutableJob(task2Name, true);
        log("************** Test with Job Killing *************");
        // submit three jobs
        JobId id1 = schedulerHelper.submitJob(job1);
        JobId id2 = schedulerHelper.submitJob(job2);
        schedulerHelper.waitForEventTaskRunning(id1, task1Name);
        schedulerHelper.waitForEventTaskRunning(id2, task2Name);
        log("************** All 2 tasks running *************");
        TestProcessTreeKiller.waitUntilForkedProcessesAreRunning(detachedProcNumber * 2);
        // we should have 2 times (2 jobs) number of detached processes
        // kill the first job
        log("************** Waiting for the first job (NativeExecutable) to be killed *************");
        schedulerHelper.getSchedulerInterface().killJob(id1);
        schedulerHelper.waitForEventJobFinished(id1);
        log("************** First job killed *************");
        TestProcessTreeKiller.waitUntilForkedProcessesAreRunning(detachedProcNumber);
        // kill the second job
        log("************** Waiting for the second job (JavaExecutable) to be killed *************");
        schedulerHelper.getSchedulerInterface().killJob(id2);
        schedulerHelper.waitForEventJobFinished(id2);
        log("************** Second job killed *************");
        TestProcessTreeKiller.waitUntilForkedProcessesAreRunning(0);
        JobResult res = schedulerHelper.getJobResult(id1);
        assertEquals(JobStatus.KILLED, res.getJobInfo().getStatus());
        res = schedulerHelper.getJobResult(id2);
        assertEquals(JobStatus.KILLED, res.getJobInfo().getStatus());
        log("************** Test with Normal Job termination *************");
        // The test for normal termination in case of NativeExecutable is slightly different
        // we don't spawn here a native zombie process that will wait forever, because that would mean that the
        // NativeExecutable task will also wait forever !
        // This is related to the current implementation of NativeExecutable, as long as there are still some IO streamed
        // from the subprocesses of the main process, the task will wait
        // TODO improve the test by finding a way to run a detached process without IO redirection
        TaskFlowJob job4 = new TaskFlowJob();
        job4.setName(this.getClass().getSimpleName() + "_4");
        job4.setDescription("a command that spawn processes");
        NativeTask task4 = new NativeTask();
        String task4Name = "TestPTK4";
        task4.setName(task4Name);
        task4.setForkEnvironment(new ForkEnvironment(workingDir));
        task4.setCommandLine(executable.getNativeExecLauncher(true));
        job4.addTask(task4);
        // submit three jobs
        id1 = schedulerHelper.submitJob(job4);
        id2 = schedulerHelper.submitJob(job2);
        schedulerHelper.waitForEventTaskRunning(id1, task4Name);
        schedulerHelper.waitForEventTaskRunning(id2, task2Name);
        log("************** All 2 tasks running *************");
        TestProcessTreeKiller.waitUntilForkedProcessesAreRunning(detachedProcNumber);
        // we should have 1 time (2 jobs) number of detached processes as the first job won't spawn any process
        log("************** Waiting for first job (NativeExecutable) to finish *************");
        // wait for the first job to finish normally
        schedulerHelper.waitForEventJobFinished(id1);
        log("************** First job finished *************");
        int runningDetachedProcNumber = countProcesses();
        log("************** number of processes : " + runningDetachedProcNumber);
        assertEquals(detachedProcNumber, runningDetachedProcNumber);
        log("************** Waiting for second job (JavaExecutable) to finish *************");
        // wait for the second job to finish normally
        schedulerHelper.waitForEventJobFinished(id2);
        log("************** Second job finished *************");
        runningDetachedProcNumber = countProcesses();
        log("************** number of processes : " + runningDetachedProcNumber);
        assertEquals(0, runningDetachedProcNumber);
        res = schedulerHelper.getJobResult(id1);
        assertEquals(JobStatus.FINISHED, res.getJobInfo().getStatus());
        res = schedulerHelper.getJobResult(id2);
        assertEquals(JobStatus.FINISHED, res.getJobInfo().getStatus());
    }
}

Example 53 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class SchedulerDbManagerRecoveryTest method testLoadNotFinishedForkEnvironment.

/**
 * Regression test related to issue #1849.
 * <p>
 * https://github.com/ow2-proactive/scheduling/issues/1849
 */
@Test
public void testLoadNotFinishedForkEnvironment() throws Exception {
    TaskFlowJob job = new TaskFlowJob();
    JavaTask task1 = new JavaTask();
    task1.setName("task1");
    task1.setExecutableClassName("MyClass");
    JavaTask task2 = new JavaTask();
    task2.setName("task2");
    task2.setExecutableClassName("MyClass");
    task2.addDependence(task1);
    job.addTask(task1);
    job.addTask(task2);
    ForkEnvironment forkEnvironment = new ForkEnvironment();
    forkEnvironment.addAdditionalClasspath("$USERSPACE/test.jar");
    for (Task task : job.getTasks()) {
        task.setForkEnvironment(forkEnvironment);
    }
    dbManager.newJobSubmitted(Jobs.createJob(job));
    closeAndRestartDatabase();
    List<InternalJob> internalJobs = dbManager.loadNotFinishedJobs(true);
    assertThat(internalJobs).hasSize(1);
    for (InternalTask internalTask : internalJobs.get(0).getITasks()) {
        List<String> additionalClasspath = internalTask.getForkEnvironment().getAdditionalClasspath();
        assertThat(additionalClasspath).hasSize(1);
        assertThat(additionalClasspath.get(0)).isEqualTo("$USERSPACE/test.jar");
    }
}

Example 54 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestDataspaceScripts method testDataspaceScripts.

/**
 * Creates a task with a Pre/Post/Flow scripts that copy files from input files to output files
 */
@Test
public void testDataspaceScripts() throws Throwable {
    File input = tmpFolder.newFolder("input");
    File output = tmpFolder.newFolder("output");
    File global = tmpFolder.newFolder("global");
    File user = tmpFolder.newFolder("user");
    /**
     * creates the testfile in both input and output spaces
     */
    BufferedOutputStream inout = new BufferedOutputStream(new FileOutputStream(new File(input.getAbsolutePath() + File.separator + fileName + "_input")));
    BufferedOutputStream outout = new BufferedOutputStream(new FileOutputStream(new File(output.getAbsolutePath() + File.separator + fileName + "_output")));
    BufferedOutputStream globout = new BufferedOutputStream(new FileOutputStream(new File(global.getAbsolutePath() + File.separator + fileName + "_global")));
    BufferedOutputStream userout = new BufferedOutputStream(new FileOutputStream(new File(user.getAbsolutePath() + File.separator + fileName + "_user")));
    for (String line : fileContent) {
        inout.write((line + "\n").getBytes());
        outout.write((line + "\n").getBytes());
        globout.write((line + "\n").getBytes());
        userout.write((line + "\n").getBytes());
    }
    inout.close();
    outout.close();
    globout.close();
    userout.close();
    /**
     * single job with single empty task
     */
    TaskFlowJob job = new TaskFlowJob();
    job.setInputSpace(input.toURI().toString());
    job.setOutputSpace(output.toURI().toString());
    job.setGlobalSpace(global.toURI().toString());
    job.setUserSpace(user.toURI().toString());
    JavaTask t = new JavaTask();
    job.addTask(t);
    job.setName(this.getClass().getSimpleName());
    t.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    t.setName("T");
    t.addInputFiles(fileName + "_input", InputAccessMode.TransferFromInputSpace);
    t.addInputFiles(fileName + "_user", InputAccessMode.TransferFromUserSpace);
    t.addInputFiles(fileName + "_global", InputAccessMode.TransferFromGlobalSpace);
    t.addInputFiles(fileName + "_output", InputAccessMode.TransferFromOutputSpace);
    t.addOutputFiles("res_*", OutputAccessMode.TransferToOutputSpace);
    t.addOutputFiles("res_*", OutputAccessMode.TransferToUserSpace);
    t.addOutputFiles("res_*", OutputAccessMode.TransferToGlobalSpace);
    File results = org.ow2.proactive.utils.FileUtils.createTempDirectory("test", ".results", null);
    // for the JS engine on Windows
    String windowsReadyResultsPath = results.getAbsolutePath().replace("\\", "/");
    String scriptContentFiltered = scriptContent.replaceAll(folderMacro, windowsReadyResultsPath);
    t.setPreScript(new SimpleScript(scriptContentFiltered.replaceAll(typeMacro, "pre"), "groovy"));
    t.setPostScript(new SimpleScript(scriptContentFiltered.replaceAll(typeMacro, "post"), "groovy"));
    t.setFlowScript(FlowScript.createLoopFlowScript(scriptContentFiltered.replaceAll(typeMacro, "flow"), "groovy", "T"));
    t.setForkEnvironment(new ForkEnvironment());
    /**
     * job submission, wait on result, removal
     */
    JobId id = schedulerHelper.testJobSubmission(job);
    assertFalse(schedulerHelper.getJobResult(id).hadException());
    /**
     * check content of the files created by the script
     */
    checkFile(new File(output, "res_" + "pre" + fileName + "_user"));
    checkFile(new File(output, "res_" + "pre" + fileName + "_global"));
    checkFile(new File(output, "res_" + "pre" + fileName + "_input"));
    checkFile(new File(output, "res_" + "pre" + fileName + "_output"));
    checkFile(new File(output, "res_" + "post" + fileName + "_user"));
    checkFile(new File(output, "res_" + "post" + fileName + "_global"));
    checkFile(new File(output, "res_" + "post" + fileName + "_input"));
    checkFile(new File(output, "res_" + "post" + fileName + "_output"));
    checkFile(new File(output, "res_" + "flow" + fileName + "_user"));
    checkFile(new File(output, "res_" + "flow" + fileName + "_global"));
    checkFile(new File(output, "res_" + "flow" + fileName + "_input"));
    checkFile(new File(output, "res_" + "flow" + fileName + "_output"));
}

Example 55 with ForkEnvironment

use of org.ow2.proactive.scheduler.common.task.ForkEnvironment in project scheduling by ow2-proactive.

the class TestUserSpace method testUserSpace.

@Test
public void testUserSpace() throws Throwable {
    File in = tmpFolder.newFolder("input_space");
    String inPath = in.getAbsolutePath();
    File out = tmpFolder.newFolder("output_space");
    String outPath = out.getAbsolutePath();
    FileSystemManager fsManager = VFSFactory.createDefaultFileSystemManager();
    Scheduler sched = schedulerHelper.getSchedulerInterface();
    String userURI = sched.getUserSpaceURIs().get(0);
    assertTrue(userURI.startsWith("file:"));
    log("User URI is " + userURI);
    String userPath = new File(new URI(userURI)).getAbsolutePath();
    FileObject pathReplaceFO = fsManager.resolveFile(userURI + "/" + pathReplaceFile);
    if (pathReplaceFO.exists()) {
        pathReplaceFO.delete();
    }
    /**
     * Writes inFiles in INPUT
     */
    writeFiles(inFiles, inPath);
    File testPathRepl = new File(inPath + File.separator + pathReplaceFile);
    testPathRepl.createNewFile();
    PrintWriter out2 = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(testPathRepl))));
    out2.print(pathReplaceFile);
    out2.close();
    TaskFlowJob job = new TaskFlowJob();
    job.setName(this.getClass().getSimpleName());
    job.setInputSpace(in.toURI().toURL().toString());
    job.setOutputSpace(out.toURI().toURL().toString());
    JavaTask A = new JavaTask();
    A.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    A.setForkEnvironment(new ForkEnvironment());
    A.setName("A");
    for (String[] file : inFiles) {
        A.addInputFiles(file[0], InputAccessMode.TransferFromInputSpace);
        A.addOutputFiles(file[0] + ".glob.A", OutputAccessMode.TransferToUserSpace);
    }
    A.setPreScript(new SimpleScript(scriptA, "groovy"));
    job.addTask(A);
    JavaTask B = new JavaTask();
    B.setExecutableClassName("org.ow2.proactive.scheduler.examples.EmptyTask");
    B.setForkEnvironment(new ForkEnvironment());
    B.setName("B");
    B.addDependence(A);
    for (String[] file : inFiles) {
        B.addInputFiles(file[0] + ".glob.A", InputAccessMode.TransferFromUserSpace);
        B.addOutputFiles(file[0] + ".out", OutputAccessMode.TransferToOutputSpace);
    }
    B.setPreScript(new SimpleScript(scriptB, "groovy"));
    job.addTask(B);
    JobId id = sched.submit(job);
    schedulerHelper.waitForEventJobFinished(id);
    JobResult jr = schedulerHelper.getJobResult(id);
    Assert.assertFalse(jr.hadException());
    /**
     * check: inFiles > IN > LOCAL A > GLOBAL > LOCAL B > OUT
     */
    for (String[] inFile : inFiles) {
        File f = new File(outPath + File.separator + inFile[0] + ".out");
        assertTrue("File does not exist: " + f.getAbsolutePath(), f.exists());
        Assert.assertEquals("Original and copied files differ", inFile[1], FileUtils.readFileToString(f));
        File inf = new File(inPath + File.separator + inFile[0]);
    }
    for (String[] file : inFiles) {
        FileObject outFile = fsManager.resolveFile(userURI + "/" + file[0] + ".glob.A");
        log("Checking existence of " + outFile.getURL());
        assertTrue(outFile.getURL() + " exists", outFile.exists());
        File outFile2 = new File(userPath, file[0] + ".glob.A");
        log("Checking existence of " + outFile2);
        assertTrue(outFile2 + " exists", outFile2.exists());
    }
}