Examples with NativeTask org.ow2.proactive.scheduler.common.task.NativeTask used on opensource projects

Example 6 with NativeTask

use of org.ow2.proactive.scheduler.common.task.NativeTask in project scheduling by ow2-proactive.

the class TestJobDataspaceSubmission method testJobDataspaceSubmission.

/**
 * Tests start here.
 *
 * @throws Throwable any exception that can be thrown during the test.
 */
@org.junit.Test
public void testJobDataspaceSubmission() throws Throwable {
    // create initial directories and files
    setup();
    TaskFlowJob job = new TaskFlowJob();
    FileSystemServerDeployer filesServerIn = new FileSystemServerDeployer(IN, IOSPACE + IN, true, true);
    String url = filesServerIn.getVFSRootURL();
    job.setInputSpace(url);
    FileSystemServerDeployer filesServerOut = new FileSystemServerDeployer(OUT, IOSPACE + OUT, true);
    url = filesServerOut.getVFSRootURL();
    job.setOutputSpace(url);
    FileSystemServerDeployer filesServerGlob = new FileSystemServerDeployer(GLOB, IOSPACE + GLOB, true);
    url = filesServerGlob.getVFSRootURL();
    job.setGlobalSpace(url);
    FileSystemServerDeployer filesServerUser = new FileSystemServerDeployer(USER, IOSPACE + USER, true);
    url = filesServerUser.getVFSRootURL();
    job.setUserSpace(url);
    job.setName(this.getClass().getSimpleName());
    NativeTask t1 = new NativeTask();
    t1.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t1.addOutputFiles(out1, OutputAccessMode.TransferToOutputSpace);
    t1.setName("native_java1");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t1.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " > " + out1 });
            break;
        case unix:
            t1.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " > " + out1 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t1);
    NativeTask t2 = new NativeTask();
    t2.addInputFiles(in2, InputAccessMode.TransferFromOutputSpace);
    t2.addOutputFiles(out2, OutputAccessMode.TransferToOutputSpace);
    t2.setName("native_java2");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t2.setCommandLine(new String[] { "cmd", "/C", "type " + in2 + " > " + out2 });
            break;
        case unix:
            t2.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in2 + " > " + out2 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t2);
    NativeTask t3 = new NativeTask();
    t3.addInputFiles(in3, InputAccessMode.TransferFromGlobalSpace);
    t3.addOutputFiles(out3, OutputAccessMode.TransferToGlobalSpace);
    t3.setName("native_java3");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t3.setCommandLine(new String[] { "cmd", "/C", "type " + in3 + " > " + out3 });
            break;
        case unix:
            t3.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in3 + " > " + out3 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t3);
    NativeTask t4 = new NativeTask();
    t4.addInputFiles(in4, InputAccessMode.TransferFromUserSpace);
    t4.addOutputFiles(out4, OutputAccessMode.TransferToUserSpace);
    t4.setName("native_java4");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t4.setCommandLine(new String[] { "cmd", "/C", "type " + in4 + " > " + out4 });
            break;
        case unix:
            t4.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in4 + " > " + out4 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t4);
    NativeTask t10 = new NativeTask();
    t10.addDependence(t2);
    t10.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t10.addInputFiles(new FileSelector("*b.txt"), InputAccessMode.TransferFromOutputSpace);
    t10.addOutputFiles("*aa.txt", OutputAccessMode.TransferToOutputSpace);
    t10.setName("native_java10");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t10.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out2 + " > " + out10 });
            break;
        case unix:
            t10.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out2 + " > " + out10 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t10);
    NativeTask t11 = new NativeTask();
    t11.addDependence(t3);
    t11.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t11.addInputFiles(new FileSelector("*c.txt"), InputAccessMode.TransferFromGlobalSpace);
    t11.addOutputFiles("*bb.txt", OutputAccessMode.TransferToGlobalSpace);
    t11.setName("native_java11");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t11.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out3 + " > " + out11 });
            break;
        case unix:
            t11.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out3 + " > " + out11 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t11);
    NativeTask t12 = new NativeTask();
    t12.addDependence(t4);
    t12.addInputFiles(in1, InputAccessMode.TransferFromInputSpace);
    t12.addInputFiles(new FileSelector("*d.txt"), InputAccessMode.TransferFromUserSpace);
    t12.addOutputFiles("*cc.txt", OutputAccessMode.TransferToUserSpace);
    t12.setName("native_java7");
    switch(OperatingSystem.getOperatingSystem()) {
        case windows:
            t12.setCommandLine(new String[] { "cmd", "/C", "type " + in1 + " " + out4 + " > " + out12 });
            break;
        case unix:
            t12.setCommandLine(new String[] { "/bin/bash", "-c", "cat " + in1 + " " + out4 + " > " + out12 });
            break;
        default:
            throw new IllegalStateException("Unsupported operating system");
    }
    job.addTask(t12);
    JobId id = schedulerHelper.testJobSubmission(job);
    // check results are 0
    JobResult res = schedulerHelper.getJobResult(id);
    Assert.assertFalse(schedulerHelper.getJobResult(id).hadException());
    for (Map.Entry<String, TaskResult> entry : res.getAllResults().entrySet()) {
        Assert.assertEquals(0, entry.getValue().value());
    }
    // remove job
    schedulerHelper.removeJob(id);
    schedulerHelper.waitForEventJobRemoved(id);
    // check files
    File fout = new File(outputDir.getAbsolutePath() + File.separator + out1);
    Assert.assertEquals(in1, getContent(fout));
    fout = new File(outputDir.getAbsolutePath() + File.separator + out2);
    Assert.assertEquals(in2, getContent(fout));
    fout = new File(globalDir.getAbsolutePath() + File.separator + out3);
    Assert.assertEquals(in3, getContent(fout));
    fout = new File(userDir.getAbsolutePath() + File.separator + out4);
    Assert.assertEquals(in4, getContent(fout));
    fout = new File(outputDir.getAbsolutePath() + File.separator + out10);
    Assert.assertEquals(in1 + in2, getContent(fout));
    fout = new File(globalDir.getAbsolutePath() + File.separator + out11);
    Assert.assertEquals(in1 + in3, getContent(fout));
    fout = new File(userDir.getAbsolutePath() + File.separator + out12);
    Assert.assertEquals(in1 + in4, getContent(fout));
    // fout = new File(outputDir.getAbsolutePath() + File.separator + out20);
    // Assert.assertEquals(in1 + in2, getContent(fout));
    // fout = new File(globalDir.getAbsolutePath() + File.separator + out21);
    // Assert.assertEquals(in1 + in3, getContent(fout));
    // fout = new File(userDir.getAbsolutePath() + File.separator + out22);
    // Assert.assertEquals(in1 + in4, getContent(fout));
    filesServerIn.terminate();
    filesServerOut.terminate();
    filesServerGlob.terminate();
    filesServerUser.terminate();
    // and clean tmp space
    clean();
}

Example 7 with NativeTask

use of org.ow2.proactive.scheduler.common.task.NativeTask in project scheduling by ow2-proactive.

the class TestJobRemove method testRuntimeDataRemoveAfterFinish.

@Test
public void testRuntimeDataRemoveAfterFinish() throws Throwable {
    TaskFlowJob jobDef = createJob(2);
    InternalJob job = defaultSubmitJobAndLoadInternal(false, jobDef);
    dbManager.updateAfterTaskFinished(job, job.getTask("javaTask-0"), new TaskResultImpl(null, "OK1", null, 0));
    dbManager.updateAfterTaskFinished(job, job.getTask("forkedJavaTask-0"), new TaskResultImpl(null, "OK2", null, 0));
    dbManager.updateAfterTaskFinished(job, job.getTask("nativeTask-0"), new TaskResultImpl(null, "OK3", null, 0));
    job.setStatus(JobStatus.FINISHED);
    dbManager.updateAfterTaskFinished(job, null, null);
    checkAllEntitiesDeleted(JobData.class.getName(), JobContent.class.getName(), TaskData.class.getName(), TaskResultData.class.getName());
    // check can still load task results
    Assert.assertEquals("OK1", dbManager.loadTaskResult(job.getTask("javaTask-0").getId(), 0).value());
    Assert.assertEquals("OK2", dbManager.loadTaskResult(job.getTask("forkedJavaTask-0").getId(), 0).value());
    Assert.assertEquals("OK3", dbManager.loadTaskResult(job.getTask("nativeTask-0").getId(), 0).value());
}

Example 8 with NativeTask

use of org.ow2.proactive.scheduler.common.task.NativeTask in project scheduling by ow2-proactive.

the class TestJobRemove method removeScenario.

private void removeScenario(int tasksNumber) throws Exception {
    TaskFlowJob jobDef = createJob(tasksNumber);
    InternalJob job = defaultSubmitJobAndLoadInternal(false, jobDef);
    for (int i = 0; i < tasksNumber; i++) {
        dbManager.updateAfterTaskFinished(job, job.getTask("javaTask-" + i), new TaskResultImpl(null, "OK", null, 0));
        dbManager.updateAfterTaskFinished(job, job.getTask("javaTask-" + i), new TaskResultImpl(null, "OK", null, 0));
        dbManager.updateAfterTaskFinished(job, job.getTask("forkedJavaTask-" + i), new TaskResultImpl(null, "OK", null, 0));
        dbManager.updateAfterTaskFinished(job, job.getTask("forkedJavaTask-" + i), new TaskResultImpl(null, "OK", null, 0));
        dbManager.updateAfterTaskFinished(job, job.getTask("nativeTask-" + i), new TaskResultImpl(null, "OK", null, 0));
        dbManager.updateAfterTaskFinished(job, job.getTask("nativeTask-" + i), new TaskResultImpl(null, "OK", null, 0));
    }
    System.out.println("Remove job");
    long start = System.currentTimeMillis();
    dbManager.removeJob(job.getId(), 0, true);
    System.out.println("Remove time (tasks: " + tasksNumber + ")" + (System.currentTimeMillis() - start));
    checkAllEntitiesDeleted();
}

Example 9 with NativeTask

use of org.ow2.proactive.scheduler.common.task.NativeTask in project scheduling by ow2-proactive.

the class TestJobRemove method testRuntimeDataRemoveAfterFinishParallel.

@Test
public void testRuntimeDataRemoveAfterFinishParallel() throws Throwable {
    int THREAD_COUNT = 4;
    ExecutorService executorService = Executors.newCachedThreadPool();
    List<InternalJob> jobs = new ArrayList<>(THREAD_COUNT);
    TaskFlowJob jobDef;
    for (int i = 0; i < THREAD_COUNT; i++) {
        jobDef = createJob(2);
        jobs.add(defaultSubmitJobAndLoadInternal(false, jobDef));
    }
    for (int i = 0; i < THREAD_COUNT; i++) {
        final InternalJob job = jobs.get(i);
        executorService.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    dbManager.updateAfterTaskFinished(job, job.getTask("javaTask-0"), new TaskResultImpl(null, "OK1", null, 0));
                    dbManager.updateAfterTaskFinished(job, job.getTask("forkedJavaTask-0"), new TaskResultImpl(null, "OK2", null, 0));
                    dbManager.updateAfterTaskFinished(job, job.getTask("nativeTask-0"), new TaskResultImpl(null, "OK3", null, 0));
                    job.setStatus(JobStatus.FINISHED);
                    dbManager.updateAfterTaskFinished(job, null, null);
                } catch (Throwable throwable) {
                    throwable.printStackTrace();
                }
            }
        });
    }
    executorService.shutdown();
    executorService.awaitTermination(30, TimeUnit.SECONDS);
    for (int i = 0; i < THREAD_COUNT; i++) {
        final InternalJob job = jobs.get(i);
        List<InternalJob> jobsNotFinished = dbManager.loadNotFinishedJobs(true);
        Assert.assertEquals("All jobs should be finished", 0, jobsNotFinished.size());
        checkAllEntitiesDeleted(JobData.class.getName(), JobContent.class.getName(), TaskData.class.getName(), TaskResultData.class.getName());
        // check can still load task results
        Assert.assertEquals("OK1", dbManager.loadTaskResult(job.getTask("javaTask-0").getId(), 0).value());
        Assert.assertEquals("OK2", dbManager.loadTaskResult(job.getTask("forkedJavaTask-0").getId(), 0).value());
        Assert.assertEquals("OK3", dbManager.loadTaskResult(job.getTask("nativeTask-0").getId(), 0).value());
    }
}

Example 10 with NativeTask

use of org.ow2.proactive.scheduler.common.task.NativeTask in project scheduling by ow2-proactive.

the class TestReadSchedulerAccount method singleJobScenario.

private void singleJobScenario(AccountData accountData) throws Exception {
    TaskFlowJob jobDef1 = new TaskFlowJob();
    JavaTask javaTask = new JavaTask();
    javaTask.setExecutableClassName(TestDummyExecutable.class.getName());
    javaTask.setName("task1");
    jobDef1.addTask(javaTask);
    JavaTask forkJavaTask = createDefaultTask("task2");
    forkJavaTask.setExecutableClassName(TestDummyExecutable.class.getName());
    forkJavaTask.setForkEnvironment(new ForkEnvironment());
    jobDef1.addTask(forkJavaTask);
    NativeTask nativeTask = new NativeTask();
    nativeTask.setName("task3");
    nativeTask.setCommandLine("command");
    jobDef1.addTask(nativeTask);
    InternalJob job1 = defaultSubmitJobAndLoadInternal(true, jobDef1, accountData.userName);
    // job is submitted
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task1 started
    job1.start();
    startTask(job1, job1.getTask("task1"));
    dbManager.jobTaskStarted(job1, job1.getTask("task1"), true);
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task1 finished
    accountData.taskTime += finishTask(job1, "task1");
    accountData.taskCount++;
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task2 and task3 started
    startTask(job1, job1.getTask("task2"));
    dbManager.jobTaskStarted(job1, job1.getTask("task2"), true);
    startTask(job1, job1.getTask("task3"));
    dbManager.jobTaskStarted(job1, job1.getTask("task3"), true);
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task2 finished
    accountData.taskTime += finishTask(job1, "task2");
    accountData.taskCount++;
    checkAccount(invalidUser);
    checkAccount(accountData);
    // task3 and job finished
    accountData.taskTime += finishTask(job1, "task3");
    accountData.taskCount++;
    accountData.jobCount++;
    accountData.jobTime += job1.getFinishedTime() - job1.getStartTime();
    checkAccount(invalidUser);
    checkAccount(accountData);
}